replace python reader with v3 reader

python/bin/pmtiles-show

@@ -1,44 +1,21 @@
 #!/usr/bin/env python
 
 import sys
-from pmtiles.reader import Reader, MmapSource, load_directory
+import pprint
+from pmtiles.reader import Reader, MmapSource
 
 if len(sys.argv) <= 1:
     print("Usage: pmtiles-show PMTILES_FILE")
     print("Usage: pmtiles-show PMTILES_FILE Z X Y")
-    print("Usage: pmtiles-show PMTILES_FILE list")
     exit(1)
 
 with open(sys.argv[1], "r+b") as f:
     reader = Reader(MmapSource(f))
-    spec_version = reader.header().version
     if len(sys.argv) == 2:
-        print("spec version: ", spec_version)
-        print("metadata:")
-        for k, v in reader.header().metadata.items():
-            print(k, "=", v)
-        print("root dir tiles:", len(reader.header().root_dir))
-        print("leaf directories:", len(set(reader.header().leaves.values())))
-    elif len(sys.argv) == 3:
-
-        last_val = None
-        for k, v in reader.header().root_dir.items():
-            print(f"{k[0]} {k[1]} {k[2]} {v[0]} {v[1]}")
-            if last_val and k <= last_val:
-                raise Exception("Error: directory entries not sorted")
-            last_val = k
-
-        for val in set(reader.header().leaves.values()):
-            dir_bytes = reader.get_bytes(val[0], val[1])
-            leaf_dir, _ = load_directory(dir_bytes, 0, val[1] // 17)
-            last_val = None
-            for k, v in leaf_dir.items():
-                print(f"{k[0]} {k[1]} {k[2]} {v[0]} {v[1]}")
-                if last_val and k <= last_val:
-                    raise Exception("Error: directory entries not sorted")
-                last_val = k
+        pprint.pprint(reader.header())
+        pprint.pprint(reader.metadata())
     else:
         z = int(sys.argv[2])
         x = int(sys.argv[3])
         y = int(sys.argv[4])
-        print(reader.get(z, x, y))
+        sys.stdout.buffer.write(reader.get(z, x, y))

python/pmtiles/reader.py

@@ -1,7 +1,13 @@
 import json
 import mmap
-from contextlib import contextmanager
-from collections import namedtuple
+from .tile import (
+    deserialize_header,
+    deserialize_directory,
+    zxy_to_tileid,
+    find_tile,
+    Compression,
+)
+import gzip
 
 
 def MmapSource(f):
@@ -20,79 +26,33 @@ def MemorySource(buf):
     return get_bytes
 
 
-def load_directory(data_bytes, offset, num_entries):
-    tile_entries = {}
-    leaves = {}
-    for i in range(offset, offset + num_entries * 17, 17):
-        z = int.from_bytes(data_bytes[i : i + 1], byteorder="little")
-        x = int.from_bytes(data_bytes[i + 1 : i + 4], byteorder="little")
-        y = int.from_bytes(data_bytes[i + 4 : i + 7], byteorder="little")
-        tile_off = int.from_bytes(data_bytes[i + 7 : i + 13], byteorder="little")
-        tile_len = int.from_bytes(data_bytes[i + 13 : i + 17], byteorder="little")
-        if z & 0b10000000:
-            leaves[(z & 0b01111111, x, y)] = (tile_off, tile_len)
-        else:
-            tile_entries[(z, x, y)] = (tile_off, tile_len)
-    return tile_entries, leaves
-
-
-Header = namedtuple("Header", ["version", "metadata", "root_dir", "leaves"])
-
-
 class Reader:
     def __init__(self, get_bytes):
         self.get_bytes = get_bytes
-        self._header = None
 
     def header(self):
-        if self._header:
-            return self._header
-        else:
-            header_bytes = self.get_bytes(0, 512000)
-            assert int.from_bytes(header_bytes[0:2], byteorder="little") == 0x4D50
-            version = int.from_bytes(header_bytes[2:4], byteorder="little")
-            metadata_len = int.from_bytes(header_bytes[4:8], byteorder="little")
-            metadata = json.loads(header_bytes[10 : 10 + metadata_len])
-            num_entries = int.from_bytes(header_bytes[8:10], byteorder="little")
-            root_dir, leaves = load_directory(
-                header_bytes, 10 + metadata_len, num_entries
-            )
-            self._header = Header(version, metadata, root_dir, leaves)
-            return self._header
+        return deserialize_header(self.get_bytes(0, 127))
 
-    def _leaf_level(self):
-        h = self.header()
-        return next(iter(h.leaves))[0]
+    def metadata(self):
+        header = deserialize_header(self.get_bytes(0, 127))
+        metadata = self.get_bytes(header["metadata_offset"], header["metadata_length"])
+        if header["internal_compression"] == Compression.GZIP:
+            metadata = gzip.decompress(metadata)
+        return json.loads(metadata)
 
     def get(self, z, x, y):
-        h = self.header()
-        val = h.root_dir.get((z, x, y))
-        if val:
-            return self.get_bytes(val[0], val[1])
+        tile_id = zxy_to_tileid(z, x, y)
+        header = deserialize_header(self.get_bytes(0, 127))
+        dir_offset = header["root_offset"]
+        dir_length = header["root_length"]
+        for depth in range(0, 3):  # max depth
+            directory = deserialize_directory(self.get_bytes(dir_offset, dir_length))
+            result = find_tile(directory, tile_id)
+            if result:
+                if result.run_length == 0:
+                    dir_offset = header["leaf_directory_offset"] + result.offset
+                    dir_length = result.length
                 else:
-            if len(self.header().leaves) > 0:
-                level_diff = z - self._leaf_level()
-                if level_diff < 0:
-                    return None
-                leaf = (
-                    self._leaf_level(),
-                    x // (1 << level_diff),
-                    y // (1 << level_diff),
+                    return self.get_bytes(
+                        header["tile_data_offset"] + result.offset, result.length
                     )
-                val = h.leaves.get(leaf)
-                if val:
-                    dir_bytes = self.get_bytes(val[0], val[1])
-                    directory, _ = load_directory(dir_bytes, 0, val[1] // 17)
-                    val = directory.get((z, x, y))
-                    if val:
-                        return self.get_bytes(val[0], val[1])
-
-    def tiles(self):
-        h = self.header()
-        for k, v in h.root_dir.items():
-            yield (k, self.get_bytes(v[0], v[1]))
-        for val in set(h.leaves.values()):
-            dir_bytes = self.get_bytes(val[0], val[1])
-            leaf_dir, _ = load_directory(dir_bytes, 0, val[1] // 17)
-            for k, v in leaf_dir.items():
-                yield (k, self.get_bytes(v[0], v[1]))

python/pmtiles/v2.py

@@ -0,0 +1,79 @@
+from collections import namedtuple
+
+
+def load_directory(data_bytes, offset, num_entries):
+    tile_entries = {}
+    leaves = {}
+    for i in range(offset, offset + num_entries * 17, 17):
+        z = int.from_bytes(data_bytes[i : i + 1], byteorder="little")
+        x = int.from_bytes(data_bytes[i + 1 : i + 4], byteorder="little")
+        y = int.from_bytes(data_bytes[i + 4 : i + 7], byteorder="little")
+        tile_off = int.from_bytes(data_bytes[i + 7 : i + 13], byteorder="little")
+        tile_len = int.from_bytes(data_bytes[i + 13 : i + 17], byteorder="little")
+        if z & 0b10000000:
+            leaves[(z & 0b01111111, x, y)] = (tile_off, tile_len)
+        else:
+            tile_entries[(z, x, y)] = (tile_off, tile_len)
+    return tile_entries, leaves
+
+
+Header = namedtuple("Header", ["version", "metadata", "root_dir", "leaves"])
+
+
+class Reader:
+    def __init__(self, get_bytes):
+        self.get_bytes = get_bytes
+        self._header = None
+
+    def header(self):
+        if self._header:
+            return self._header
+        else:
+            header_bytes = self.get_bytes(0, 512000)
+            assert int.from_bytes(header_bytes[0:2], byteorder="little") == 0x4D50
+            version = int.from_bytes(header_bytes[2:4], byteorder="little")
+            metadata_len = int.from_bytes(header_bytes[4:8], byteorder="little")
+            metadata = json.loads(header_bytes[10 : 10 + metadata_len])
+            num_entries = int.from_bytes(header_bytes[8:10], byteorder="little")
+            root_dir, leaves = load_directory(
+                header_bytes, 10 + metadata_len, num_entries
+            )
+            self._header = Header(version, metadata, root_dir, leaves)
+            return self._header
+
+    def _leaf_level(self):
+        h = self.header()
+        return next(iter(h.leaves))[0]
+
+    def get(self, z, x, y):
+        h = self.header()
+        val = h.root_dir.get((z, x, y))
+        if val:
+            return self.get_bytes(val[0], val[1])
+        else:
+            if len(self.header().leaves) > 0:
+                level_diff = z - self._leaf_level()
+                if level_diff < 0:
+                    return None
+                leaf = (
+                    self._leaf_level(),
+                    x // (1 << level_diff),
+                    y // (1 << level_diff),
+                )
+                val = h.leaves.get(leaf)
+                if val:
+                    dir_bytes = self.get_bytes(val[0], val[1])
+                    directory, _ = load_directory(dir_bytes, 0, val[1] // 17)
+                    val = directory.get((z, x, y))
+                    if val:
+                        return self.get_bytes(val[0], val[1])
+
+    def tiles(self):
+        h = self.header()
+        for k, v in h.root_dir.items():
+            yield (k, self.get_bytes(v[0], v[1]))
+        for val in set(h.leaves.values()):
+            dir_bytes = self.get_bytes(val[0], val[1])
+            leaf_dir, _ = load_directory(dir_bytes, 0, val[1] // 17)
+            for k, v in leaf_dir.items():
+                yield (k, self.get_bytes(v[0], v[1]))