pmtiles-convert --gzip flag explicitly controls tile-level compression (default disabled) [#26]

python/bin/pmtiles-convert

@@ -11,7 +11,7 @@ parser = argparse.ArgumentParser(description='Convert between PMTiles and other
 parser.add_argument('input',help='Input .mbtiles or .pmtiles')
 parser.add_argument('output',help='Output .mbtiles, .pmtiles, or directory')
 parser.add_argument('--maxzoom', help='the maximum zoom level to include in the output.')
-parser.add_argument('--gzip', help='Add gzip encoding to the output if it is not already gzipped.',action='store_true')
+parser.add_argument('--gzip', help='The output should be gzip-compressed.',action='store_true')
 parser.add_argument('--overwrite', help='Overwrite the existing output.',action='store_true')
 args = parser.parse_args()
 
@@ -24,8 +24,10 @@ if args.overwrite:
     elif os.path.isdir(args.output):
         shutil.rmtree(args.output)
 
+print("compression:", "gzip" if args.gzip else "disabled")
+
 if args.input.endswith('.mbtiles') and args.output.endswith('.pmtiles'):
-    mbtiles_to_pmtiles(args.input, args.output, args.maxzoom)
+    mbtiles_to_pmtiles(args.input, args.output, args.maxzoom, args.gzip)
 
 elif args.input.endswith('.pmtiles') and args.output.endswith('.mbtiles'):
     pmtiles_to_mbtiles(args.input, args.output, args.gzip)

python/pmtiles/convert.py

@@ -7,27 +7,40 @@ import sqlite3
 from pmtiles.reader import read
 from pmtiles.writer import write
 
-
+# if the tile is GZIP-encoded, it won't work with range queries
-def may_compress(data,compress):
+# until transfer-encoding: gzip is well supported.
+def force_compress(data,compress):
     if compress and data[0:2] !=  b'\x1f\x8b':
         return gzip.compress(data)
+    if not compress and data[0:2] == b'\x1f\x8b':
+        return gzip.decompress(data)
     return data
 
+def set_metadata_compression(metadata,gzip):
+    if gzip:
+        metadata['compression'] = 'gzip'
+    else:
+        try:
+            del metadata['compression']
+        except:
+            pass
+    return metadata
 
-def mbtiles_to_pmtiles(input, output, maxzoom):
+def mbtiles_to_pmtiles(input, output, maxzoom, gzip):
     conn = sqlite3.connect(input)
     cursor = conn.cursor()
 
     with write(output) as writer:
         for row in cursor.execute('SELECT zoom_level,tile_column,tile_row,tile_data FROM tiles WHERE zoom_level <= ? ORDER BY zoom_level,tile_column,tile_row ASC',(maxzoom or 99,)):
             flipped = (1 << row[0]) - 1 - row[2]
-            writer.write_tile(row[0],row[1],flipped,row[3])
+            writer.write_tile(row[0],row[1],flipped,force_compress(row[3],gzip))
 
         metadata = {}
         for row in cursor.execute('SELECT name,value FROM metadata'):
             metadata[row[0]] = row[1]
         if maxzoom:
             metadata['maxzoom'] = str(maxzoom)
+        metadata = set_metadata_compression(metadata,gzip)
         result = writer.finalize(metadata)
         print("Num tiles:",result['num_tiles'])
         print("Num unique tiles:",result['num_unique_tiles'])
@@ -43,11 +56,13 @@ def pmtiles_to_mbtiles(input, output, gzip):
     cursor.execute('CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);')
 
     with read(input) as reader:
-        for k,v in reader.metadata.items():
+        metadata = reader.metadata
+        metadata = set_metadata_compression(metadata,gzip)
+        for k,v in metadata.items():
             cursor.execute('INSERT INTO metadata VALUES(?,?)',(k,v))
         for tile, data in reader.tiles():
             flipped = (1 << tile[0]) - 1 - tile[2]
-            cursor.execute('INSERT INTO tiles VALUES(?,?,?,?)',(tile[0],tile[1],flipped,may_compress(data,gzip)))
+            cursor.execute('INSERT INTO tiles VALUES(?,?,?,?)',(tile[0],tile[1],flipped,force_compress(data,gzip)))
 
     cursor.execute('CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);')
     conn.commit()
@@ -58,7 +73,7 @@ def pmtiles_to_dir(input, output, gzip):
 
     with read(input) as reader:
         metadata = reader.metadata
-        metadata['format']
+        metadata = set_metadata_compression(metadata,gzip)
         with open(os.path.join(output,'metadata.json'),'w') as f:
             f.write(json.dumps(metadata))
 
@@ -67,4 +82,4 @@ def pmtiles_to_dir(input, output, gzip):
             path = os.path.join(directory,str(tile[2]) + '.' + metadata['format'])
             os.makedirs(directory,exist_ok=True)
             with open(path,'wb') as f:
-                f.write(may_compress(data,gzip))
+                f.write(force_compress(data,gzip))

python/pmtiles/writer.py

@@ -21,10 +21,6 @@ class Writer:
         self.leaves = []
 
     def write_tile(self,z,x,y,data):
-        # if the tile is GZIP-encoded, it won't work with range queries
-        # until transfer-encoding: gzip is well supported.
-        if data[0:2] == b'\x1f\x8b':
-            data = gzip.decompress(data)
         hsh = hash(data)
         if hsh in self.hash_to_offset:
             self.tiles.append((z,x,y,self.hash_to_offset[hsh],len(data)))