mirrors/PMTiles
1
0
Fork 0
mirror of https://github.com/protomaps/PMTiles.git synced 2026-02-04 10:51:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

replace python reader with v3 reader

Browse Source
This commit is contained in:
Brandon Liu
2022-10-13 14:54:33 +08:00
parent 9d0b2ba0c3
commit b89bf951bb
3 changed files with 114 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
python/bin/pmtiles-show
View File

@@ -1,44 +1,21 @@
#!/usr/bin/env python
import sys
from pmtiles.reader import Reader, MmapSource, load_directory
import pprint
from pmtiles.reader import Reader, MmapSource
if len(sys.argv) <= 1:
print("Usage: pmtiles-show PMTILES_FILE")
print("Usage: pmtiles-show PMTILES_FILE Z X Y")
print("Usage: pmtiles-show PMTILES_FILE list")
exit(1)
with open(sys.argv[1], "r+b") as f:
reader = Reader(MmapSource(f))
spec_version = reader.header().version
if len(sys.argv) == 2:
print("spec version: ", spec_version)
print("metadata:")
for k, v in reader.header().metadata.items():
print(k, "=", v)
print("root dir tiles:", len(reader.header().root_dir))
print("leaf directories:", len(set(reader.header().leaves.values())))
elif len(sys.argv) == 3:
last_val = None
for k, v in reader.header().root_dir.items():
print(f"{k[0]} {k[1]} {k[2]} {v[0]} {v[1]}")
if last_val and k <= last_val:
raise Exception("Error: directory entries not sorted")
last_val = k
for val in set(reader.header().leaves.values()):
dir_bytes = reader.get_bytes(val[0], val[1])
leaf_dir, _ = load_directory(dir_bytes, 0, val[1] // 17)
last_val = None
for k, v in leaf_dir.items():
print(f"{k[0]} {k[1]} {k[2]} {v[0]} {v[1]}")
if last_val and k <= last_val:
raise Exception("Error: directory entries not sorted")
last_val = k
pprint.pprint(reader.header())
pprint.pprint(reader.metadata())
else:
z = int(sys.argv[2])
x = int(sys.argv[3])
y = int(sys.argv[4])
print(reader.get(z, x, y))
sys.stdout.buffer.write(reader.get(z, x, y))

96
python/pmtiles/reader.py
View File

@@ -1,7 +1,13 @@
import json
import mmap
from contextlib import contextmanager
from collections import namedtuple
from .tile import (
deserialize_header,
deserialize_directory,
zxy_to_tileid,
find_tile,
Compression,
)
import gzip
def MmapSource(f):
@@ -20,79 +26,33 @@ def MemorySource(buf):
return get_bytes
def load_directory(data_bytes, offset, num_entries):
tile_entries = {}
leaves = {}
for i in range(offset, offset + num_entries * 17, 17):
z = int.from_bytes(data_bytes[i : i + 1], byteorder="little")
x = int.from_bytes(data_bytes[i + 1 : i + 4], byteorder="little")
y = int.from_bytes(data_bytes[i + 4 : i + 7], byteorder="little")
tile_off = int.from_bytes(data_bytes[i + 7 : i + 13], byteorder="little")
tile_len = int.from_bytes(data_bytes[i + 13 : i + 17], byteorder="little")
if z & 0b10000000:
leaves[(z & 0b01111111, x, y)] = (tile_off, tile_len)
else:
tile_entries[(z, x, y)] = (tile_off, tile_len)
return tile_entries, leaves
Header = namedtuple("Header", ["version", "metadata", "root_dir", "leaves"])
class Reader:
def __init__(self, get_bytes):
self.get_bytes = get_bytes
self._header = None
def header(self):
if self._header:
return self._header
else:
header_bytes = self.get_bytes(0, 512000)
assert int.from_bytes(header_bytes[0:2], byteorder="little") == 0x4D50
version = int.from_bytes(header_bytes[2:4], byteorder="little")
metadata_len = int.from_bytes(header_bytes[4:8], byteorder="little")
metadata = json.loads(header_bytes[10 : 10 + metadata_len])
num_entries = int.from_bytes(header_bytes[8:10], byteorder="little")
root_dir, leaves = load_directory(
header_bytes, 10 + metadata_len, num_entries
)
self._header = Header(version, metadata, root_dir, leaves)
return self._header
return deserialize_header(self.get_bytes(0, 127))
def _leaf_level(self):
h = self.header()
return next(iter(h.leaves))[0]
def metadata(self):
header = deserialize_header(self.get_bytes(0, 127))
metadata = self.get_bytes(header["metadata_offset"], header["metadata_length"])
if header["internal_compression"] == Compression.GZIP:
metadata = gzip.decompress(metadata)
return json.loads(metadata)
def get(self, z, x, y):
h = self.header()
val = h.root_dir.get((z, x, y))
if val:
return self.get_bytes(val[0], val[1])
tile_id = zxy_to_tileid(z, x, y)
header = deserialize_header(self.get_bytes(0, 127))
dir_offset = header["root_offset"]
dir_length = header["root_length"]
for depth in range(0, 3): # max depth
directory = deserialize_directory(self.get_bytes(dir_offset, dir_length))
result = find_tile(directory, tile_id)
if result:
if result.run_length == 0:
dir_offset = header["leaf_directory_offset"] + result.offset
dir_length = result.length
else:
if len(self.header().leaves) > 0:
level_diff = z - self._leaf_level()
if level_diff < 0:
return None
leaf = (
self._leaf_level(),
x // (1 << level_diff),
y // (1 << level_diff),
return self.get_bytes(
header["tile_data_offset"] + result.offset, result.length
)
val = h.leaves.get(leaf)
if val:
dir_bytes = self.get_bytes(val[0], val[1])
directory, _ = load_directory(dir_bytes, 0, val[1] // 17)
val = directory.get((z, x, y))
if val:
return self.get_bytes(val[0], val[1])
def tiles(self):
h = self.header()
for k, v in h.root_dir.items():
yield (k, self.get_bytes(v[0], v[1]))
for val in set(h.leaves.values()):
dir_bytes = self.get_bytes(val[0], val[1])
leaf_dir, _ = load_directory(dir_bytes, 0, val[1] // 17)
for k, v in leaf_dir.items():
yield (k, self.get_bytes(v[0], v[1]))

79
python/pmtiles/v2.py Normal file
View File

@@ -0,0 +1,79 @@
from collections import namedtuple
def load_directory(data_bytes, offset, num_entries):
tile_entries = {}
leaves = {}
for i in range(offset, offset + num_entries * 17, 17):
z = int.from_bytes(data_bytes[i : i + 1], byteorder="little")
x = int.from_bytes(data_bytes[i + 1 : i + 4], byteorder="little")
y = int.from_bytes(data_bytes[i + 4 : i + 7], byteorder="little")
tile_off = int.from_bytes(data_bytes[i + 7 : i + 13], byteorder="little")
tile_len = int.from_bytes(data_bytes[i + 13 : i + 17], byteorder="little")
if z & 0b10000000:
leaves[(z & 0b01111111, x, y)] = (tile_off, tile_len)
else:
tile_entries[(z, x, y)] = (tile_off, tile_len)
return tile_entries, leaves
Header = namedtuple("Header", ["version", "metadata", "root_dir", "leaves"])
class Reader:
def __init__(self, get_bytes):
self.get_bytes = get_bytes
self._header = None
def header(self):
if self._header:
return self._header
else:
header_bytes = self.get_bytes(0, 512000)
assert int.from_bytes(header_bytes[0:2], byteorder="little") == 0x4D50
version = int.from_bytes(header_bytes[2:4], byteorder="little")
metadata_len = int.from_bytes(header_bytes[4:8], byteorder="little")
metadata = json.loads(header_bytes[10 : 10 + metadata_len])
num_entries = int.from_bytes(header_bytes[8:10], byteorder="little")
root_dir, leaves = load_directory(
header_bytes, 10 + metadata_len, num_entries
)
self._header = Header(version, metadata, root_dir, leaves)
return self._header
def _leaf_level(self):
h = self.header()
return next(iter(h.leaves))[0]
def get(self, z, x, y):
h = self.header()
val = h.root_dir.get((z, x, y))
if val:
return self.get_bytes(val[0], val[1])
else:
if len(self.header().leaves) > 0:
level_diff = z - self._leaf_level()
if level_diff < 0:
return None
leaf = (
self._leaf_level(),
x // (1 << level_diff),
y // (1 << level_diff),
)
val = h.leaves.get(leaf)
if val:
dir_bytes = self.get_bytes(val[0], val[1])
directory, _ = load_directory(dir_bytes, 0, val[1] // 17)
val = directory.get((z, x, y))
if val:
return self.get_bytes(val[0], val[1])
def tiles(self):
h = self.header()
for k, v in h.root_dir.items():
yield (k, self.get_bytes(v[0], v[1]))
for val in set(h.leaves.values()):
dir_bytes = self.get_bytes(val[0], val[1])
leaf_dir, _ = load_directory(dir_bytes, 0, val[1] // 17)
for k, v in leaf_dir.items():
yield (k, self.get_bytes(v[0], v[1]))