From d7ece9915b21b4622cd59fd9e46d8aaac7dea7e1 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Sun, 23 Jan 2022 13:44:50 +0800 Subject: [PATCH] pmtiles-convert --gzip flag explicitly controls tile-level compression (default disabled) [#26] --- python/bin/pmtiles-convert | 6 ++++-- python/pmtiles/convert.py | 31 +++++++++++++++++++++++-------- python/pmtiles/writer.py | 4 ---- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/python/bin/pmtiles-convert b/python/bin/pmtiles-convert index c6d3b24..cc9336d 100755 --- a/python/bin/pmtiles-convert +++ b/python/bin/pmtiles-convert @@ -11,7 +11,7 @@ parser = argparse.ArgumentParser(description='Convert between PMTiles and other parser.add_argument('input',help='Input .mbtiles or .pmtiles') parser.add_argument('output',help='Output .mbtiles, .pmtiles, or directory') parser.add_argument('--maxzoom', help='the maximum zoom level to include in the output.') -parser.add_argument('--gzip', help='Add gzip encoding to the output if it is not already gzipped.',action='store_true') +parser.add_argument('--gzip', help='The output should be gzip-compressed.',action='store_true') parser.add_argument('--overwrite', help='Overwrite the existing output.',action='store_true') args = parser.parse_args() @@ -24,8 +24,10 @@ if args.overwrite: elif os.path.isdir(args.output): shutil.rmtree(args.output) +print("compression:", "gzip" if args.gzip else "disabled") + if args.input.endswith('.mbtiles') and args.output.endswith('.pmtiles'): - mbtiles_to_pmtiles(args.input, args.output, args.maxzoom) + mbtiles_to_pmtiles(args.input, args.output, args.maxzoom, args.gzip) elif args.input.endswith('.pmtiles') and args.output.endswith('.mbtiles'): pmtiles_to_mbtiles(args.input, args.output, args.gzip) diff --git a/python/pmtiles/convert.py b/python/pmtiles/convert.py index dc22206..21e99ed 100644 --- a/python/pmtiles/convert.py +++ b/python/pmtiles/convert.py @@ -7,27 +7,40 @@ import sqlite3 from pmtiles.reader import read from pmtiles.writer import write - -def may_compress(data,compress): +# if the tile is GZIP-encoded, it won't work with range queries +# until transfer-encoding: gzip is well supported. +def force_compress(data,compress): if compress and data[0:2] != b'\x1f\x8b': return gzip.compress(data) + if not compress and data[0:2] == b'\x1f\x8b': + return gzip.decompress(data) return data +def set_metadata_compression(metadata,gzip): + if gzip: + metadata['compression'] = 'gzip' + else: + try: + del metadata['compression'] + except: + pass + return metadata -def mbtiles_to_pmtiles(input, output, maxzoom): +def mbtiles_to_pmtiles(input, output, maxzoom, gzip): conn = sqlite3.connect(input) cursor = conn.cursor() with write(output) as writer: for row in cursor.execute('SELECT zoom_level,tile_column,tile_row,tile_data FROM tiles WHERE zoom_level <= ? ORDER BY zoom_level,tile_column,tile_row ASC',(maxzoom or 99,)): flipped = (1 << row[0]) - 1 - row[2] - writer.write_tile(row[0],row[1],flipped,row[3]) + writer.write_tile(row[0],row[1],flipped,force_compress(row[3],gzip)) metadata = {} for row in cursor.execute('SELECT name,value FROM metadata'): metadata[row[0]] = row[1] if maxzoom: metadata['maxzoom'] = str(maxzoom) + metadata = set_metadata_compression(metadata,gzip) result = writer.finalize(metadata) print("Num tiles:",result['num_tiles']) print("Num unique tiles:",result['num_unique_tiles']) @@ -43,11 +56,13 @@ def pmtiles_to_mbtiles(input, output, gzip): cursor.execute('CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);') with read(input) as reader: - for k,v in reader.metadata.items(): + metadata = reader.metadata + metadata = set_metadata_compression(metadata,gzip) + for k,v in metadata.items(): cursor.execute('INSERT INTO metadata VALUES(?,?)',(k,v)) for tile, data in reader.tiles(): flipped = (1 << tile[0]) - 1 - tile[2] - cursor.execute('INSERT INTO tiles VALUES(?,?,?,?)',(tile[0],tile[1],flipped,may_compress(data,gzip))) + cursor.execute('INSERT INTO tiles VALUES(?,?,?,?)',(tile[0],tile[1],flipped,force_compress(data,gzip))) cursor.execute('CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);') conn.commit() @@ -58,7 +73,7 @@ def pmtiles_to_dir(input, output, gzip): with read(input) as reader: metadata = reader.metadata - metadata['format'] + metadata = set_metadata_compression(metadata,gzip) with open(os.path.join(output,'metadata.json'),'w') as f: f.write(json.dumps(metadata)) @@ -67,4 +82,4 @@ def pmtiles_to_dir(input, output, gzip): path = os.path.join(directory,str(tile[2]) + '.' + metadata['format']) os.makedirs(directory,exist_ok=True) with open(path,'wb') as f: - f.write(may_compress(data,gzip)) + f.write(force_compress(data,gzip)) diff --git a/python/pmtiles/writer.py b/python/pmtiles/writer.py index eea3f1f..140e4ad 100644 --- a/python/pmtiles/writer.py +++ b/python/pmtiles/writer.py @@ -21,10 +21,6 @@ class Writer: self.leaves = [] def write_tile(self,z,x,y,data): - # if the tile is GZIP-encoded, it won't work with range queries - # until transfer-encoding: gzip is well supported. - if data[0:2] == b'\x1f\x8b': - data = gzip.decompress(data) hsh = hash(data) if hsh in self.hash_to_offset: self.tiles.append((z,x,y,self.hash_to_offset[hsh],len(data)))