From f23ca5823bd28dfc0a89043fd54b7bfa0cbec1d7 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Fri, 23 Sep 2022 09:03:33 +0800 Subject: [PATCH] JS v3 reader with caching and ETag support [#59, #53, #41, #24, #4] The v3 module is not exported yet; specifics of header design subject to change. --- js/empty.pmtiles | 0 js/index.test.ts | 252 +------------------- js/invalid.pmtiles | 1 + js/test_fixture_1.pmtiles | Bin 0 -> 455 bytes js/test_fixture_2.pmtiles | Bin 0 -> 461 bytes js/v2.test.ts | 148 ++++++++++++ js/v3.test.ts | 295 +++++++++++++++++++++++ js/v3.ts | 478 ++++++++++++++++++++++++++++++++++++-- 8 files changed, 910 insertions(+), 264 deletions(-) create mode 100644 js/empty.pmtiles create mode 100644 js/invalid.pmtiles create mode 100644 js/test_fixture_1.pmtiles create mode 100644 js/test_fixture_2.pmtiles create mode 100644 js/v2.test.ts create mode 100644 js/v3.test.ts diff --git a/js/empty.pmtiles b/js/empty.pmtiles new file mode 100644 index 0000000..e69de29 diff --git a/js/index.test.ts b/js/index.test.ts index 7ef451a..a4ae4d9 100644 --- a/js/index.test.ts +++ b/js/index.test.ts @@ -1,252 +1,4 @@ -import { test } from "zora"; -import { - unshift, - getUint24, - getUint48, - queryLeafdir, - queryLeafLevel, - queryTile, - parseEntry, - Entry, - createDirectory, -} from "./index"; +import './v2.test'; +import './v3.test'; -import { - Entry as EntryV3, - zxyToTileId, - tileIdToZxy, - findTile, - readVarint, -} from "./v3"; -test("stub data", (assertion) => { - let dataview = createDirectory([ - { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false }, - { - z: 14, - x: 16383, - y: 16383, - offset: 999999, - length: 999, - is_dir: false, - }, - ]); - - var z_raw = dataview.getUint8(17 + 0); - var x = getUint24(dataview, 17 + 1); - var y = getUint24(dataview, 17 + 4); - var offset = getUint48(dataview, 17 + 7); - var length = dataview.getUint32(17 + 13, true); - assertion.ok(z_raw === 14); - assertion.ok(x === 16383); - assertion.ok(y === 16383); -}); - -test("get entry", (assertion) => { - let view = createDirectory([ - { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false }, - { - z: 14, - x: 16383, - y: 16383, - offset: 999999, - length: 999, - is_dir: false, - }, - ]); - let entry = queryTile(view, 14, 16383, 16383); - assertion.ok(entry!.z === 14); - assertion.ok(entry!.x === 16383); - assertion.ok(entry!.y === 16383); - assertion.ok(entry!.offset === 999999); - assertion.ok(entry!.length === 999); - assertion.ok(entry!.is_dir === false); - assertion.ok(queryLeafdir(view, 14, 16383, 16383) === null); -}); - -test("get leafdir", (assertion) => { - let view = createDirectory([ - { - z: 14, - x: 16383, - y: 16383, - offset: 999999, - length: 999, - is_dir: true, - }, - ]); - let entry = queryLeafdir(view, 14, 16383, 16383); - assertion.ok(entry!.z === 14); - assertion.ok(entry!.x === 16383); - assertion.ok(entry!.y === 16383); - assertion.ok(entry!.offset === 999999); - assertion.ok(entry!.length === 999); - assertion.ok(entry!.is_dir === true); - assertion.ok(queryTile(view, 14, 16383, 16383) === null); -}); - -test("derive the leaf level", (assertion) => { - let view = createDirectory([ - { - z: 6, - x: 3, - y: 3, - offset: 0, - length: 0, - is_dir: true, - }, - ]); - let level = queryLeafLevel(view); - assertion.ok(level === 6); - view = createDirectory([ - { - z: 6, - x: 3, - y: 3, - offset: 0, - length: 0, - is_dir: false, - }, - ]); - level = queryLeafLevel(view); - assertion.ok(level === null); -}); - -test("convert spec v1 directory to spec v2 directory", (assertion) => { - let view = createDirectory([ - { - z: 7, - x: 3, - y: 3, - offset: 3, - length: 3, - is_dir: true, - }, - { - z: 6, - x: 2, - y: 2, - offset: 2, - length: 2, - is_dir: false, - }, - { - z: 6, - x: 2, - y: 1, - offset: 1, - length: 1, - is_dir: false, - }, - ]); - let entry = queryLeafdir(view, 7, 3, 3); - assertion.ok(entry!.offset === 3); - entry = queryTile(view, 6, 2, 2); - assertion.ok(entry!.offset === 2); - entry = queryTile(view, 6, 2, 1); - assertion.ok(entry!.offset === 1); - - entry = parseEntry(view, 0); - assertion.ok(entry!.offset === 1); - entry = parseEntry(view, 1); - assertion.ok(entry!.offset === 2); - entry = parseEntry(view, 2); - assertion.ok(entry!.offset === 3); -}); - -test("varint", (assertion) => { - let b: BufferPosition = { - buf: new Uint8Array([0, 1, 127, 0xe5, 0x8e, 0x26]), - pos: 0, - }; - assertion.eq(readVarint(b), 0); - assertion.eq(readVarint(b), 1); - assertion.eq(readVarint(b), 127); - assertion.eq(readVarint(b), 624485); - b = { - buf: new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]), - pos: 0, - }; - assertion.eq(readVarint(b), 9007199254740991); -}); - -test("zxy to tile id", (assertion) => { - assertion.eq(zxyToTileId(0, 0, 0), 0); - assertion.eq(zxyToTileId(1, 0, 0), 1); - assertion.eq(zxyToTileId(1, 0, 1), 2); - assertion.eq(zxyToTileId(1, 1, 1), 3); - assertion.eq(zxyToTileId(1, 1, 0), 4); - assertion.eq(zxyToTileId(2, 0, 0), 5); -}); - -test("tile id to zxy", (assertion) => { - assertion.eq(tileIdToZxy(0), [0, 0, 0]); - assertion.eq(tileIdToZxy(1), [1, 0, 0]); - assertion.eq(tileIdToZxy(2), [1, 0, 1]); - assertion.eq(tileIdToZxy(3), [1, 1, 1]); - assertion.eq(tileIdToZxy(4), [1, 1, 0]); - assertion.eq(tileIdToZxy(5), [2, 0, 0]); -}); - -test("a lot of tiles", (assertion) => { - for (var z = 0; z < 9; z++) { - for (var x = 0; x < 1 << z; x++) { - for (var y = 0; y < 1 << z; y++) { - let result = tileIdToZxy(zxyToTileId(z, x, y)); - if (result[0] !== z || result[1] !== x || result[2] !== y) { - assertion.fail("roundtrip failed"); - } - } - } - } -}); - -test("tile search for first entry == id", (assertion) => { - let entries: EntryV3[] = []; - assertion.eq(findTile(entries, 101), null); -}); - -test("tile search for first entry == id", (assertion) => { - let entries: EntryV3[] = [ - { tileId: 100, offset: 1, length: 1, runLength: 1 }, - ]; - let entry = findTile(entries, 100)!; - assertion.eq(entry.offset, 1); - assertion.eq(entry.length, 1); - assertion.eq(findTile(entries, 101), null); -}); - -test("tile search for first entry == id", (assertion) => { - let entries: EntryV3[] = [ - { tileId: 100, offset: 1, length: 1, runLength: 2 }, - ]; - let entry = findTile(entries, 101)!; - assertion.eq(entry.offset, 1); - assertion.eq(entry.length, 1); - - entries = [ - { tileId: 100, offset: 1, length: 1, runLength: 1 }, - { tileId: 150, offset: 2, length: 2, runLength: 2 }, - ]; - entry = findTile(entries, 151)!; - assertion.eq(entry.offset, 2); - assertion.eq(entry.length, 2); - - entries = [ - { tileId: 50, offset: 1, length: 1, runLength: 2 }, - { tileId: 100, offset: 2, length: 2, runLength: 1 }, - { tileId: 150, offset: 3, length: 3, runLength: 1 }, - ]; - entry = findTile(entries, 51)!; - assertion.eq(entry.offset, 1); - assertion.eq(entry.length, 1); -}); - -test("leaf search", (assertion) => { - let entries: EntryV3[] = [ - { tileId: 100, offset: 1, length: 1, runLength: 0 }, - ]; - let entry = findTile(entries, 150); - assertion.eq(entry!.offset, 1); - assertion.eq(entry!.length, 1); -}); diff --git a/js/invalid.pmtiles b/js/invalid.pmtiles new file mode 100644 index 0000000..f2b720b --- /dev/null +++ b/js/invalid.pmtiles @@ -0,0 +1 @@ +This is an invalid tile archive, a test case to make sure that the code throws an error, but it needs to be the minimum size to pass the first test diff --git a/js/test_fixture_1.pmtiles b/js/test_fixture_1.pmtiles new file mode 100644 index 0000000000000000000000000000000000000000..93cdb53187fe53c2924f50351c565b6f925767d8 GIT binary patch literal 455 zcmWIWWv*g?07)o48A`v0(&$tZBSgd%Dgos(pwkSDj4&af%K!iC?HL#vfEWtoyE(ut zCgJ3igpyPi1|R!Q23DXDLZa4xch_M9fj6H;ld_LLXSozpZC~&=u_r0N=fEYdOxZ^g zf2&m{wohiBZfw1Nu6e=|w;y5$UhK47$=%O4J2umFHKVPF^otAI{UXCsuC=C@{L{Oo zerEF7iz58ZbA+xgc2u7k;}>)-Nt<_}#rxCFk$W|zPEEDbJiWfPtorTIX-97Lnf**o zl@WPcpCqV!Qy7d#(cSo+4YL*W%uk%0F(nJ2I0rFh+U zXg@H0S{Hqj>Fw$3%9$zVsz3F4gXOP_D$JZ~)z{PCmUZyyG^;C$cNiAjtNL^o7(fW` zPQJ80;gOK8r>IPoDAAy_OLmAT%W*?9tOX%RV^8bvZgsX>^<; N&oKR0wyq&C%mDT4lF|SG literal 0 HcmV?d00001 diff --git a/js/test_fixture_2.pmtiles b/js/test_fixture_2.pmtiles new file mode 100644 index 0000000000000000000000000000000000000000..1f30eee8859665e53aa6fbdac8bba1f70477bd25 GIT binary patch literal 461 zcmWIWWv*g?07)o48A^YL(&$tdBSgd*Dgos(pwkSDj4&af%K!iC?HL#vfEWtoyE(ut zCgJ3igpxEChEp>xhp_^M5E8Zir}LT(1ok}FUSrF=)MM|LTk1!hVlVgNT7rE-stSP&6ZsDY90Wz~+EnQWwVRdSvqUP%5bGO;_FH-qf&mTCA zxw7@Toz~AIFCuhq)pUe5y!+qxd}HUfhMXgl zjM*)J^`@N`v=c8_U$ooy1=Fj4=1P(&`&E8g`<|A#Evhhc@4q>c`-NV~P7l7d_U< { + let dataview = createDirectory([ + { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false }, + { + z: 14, + x: 16383, + y: 16383, + offset: 999999, + length: 999, + is_dir: false, + }, + ]); + + var z_raw = dataview.getUint8(17 + 0); + var x = getUint24(dataview, 17 + 1); + var y = getUint24(dataview, 17 + 4); + var offset = getUint48(dataview, 17 + 7); + var length = dataview.getUint32(17 + 13, true); + assertion.ok(z_raw === 14); + assertion.ok(x === 16383); + assertion.ok(y === 16383); +}); + +test("get entry", (assertion) => { + let view = createDirectory([ + { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false }, + { + z: 14, + x: 16383, + y: 16383, + offset: 999999, + length: 999, + is_dir: false, + }, + ]); + let entry = queryTile(view, 14, 16383, 16383); + assertion.ok(entry!.z === 14); + assertion.ok(entry!.x === 16383); + assertion.ok(entry!.y === 16383); + assertion.ok(entry!.offset === 999999); + assertion.ok(entry!.length === 999); + assertion.ok(entry!.is_dir === false); + assertion.ok(queryLeafdir(view, 14, 16383, 16383) === null); +}); + +test("get leafdir", (assertion) => { + let view = createDirectory([ + { + z: 14, + x: 16383, + y: 16383, + offset: 999999, + length: 999, + is_dir: true, + }, + ]); + let entry = queryLeafdir(view, 14, 16383, 16383); + assertion.ok(entry!.z === 14); + assertion.ok(entry!.x === 16383); + assertion.ok(entry!.y === 16383); + assertion.ok(entry!.offset === 999999); + assertion.ok(entry!.length === 999); + assertion.ok(entry!.is_dir === true); + assertion.ok(queryTile(view, 14, 16383, 16383) === null); +}); + +test("derive the leaf level", (assertion) => { + let view = createDirectory([ + { + z: 6, + x: 3, + y: 3, + offset: 0, + length: 0, + is_dir: true, + }, + ]); + let level = queryLeafLevel(view); + assertion.ok(level === 6); + view = createDirectory([ + { + z: 6, + x: 3, + y: 3, + offset: 0, + length: 0, + is_dir: false, + }, + ]); + level = queryLeafLevel(view); + assertion.ok(level === null); +}); + +test("convert spec v1 directory to spec v2 directory", (assertion) => { + let view = createDirectory([ + { + z: 7, + x: 3, + y: 3, + offset: 3, + length: 3, + is_dir: true, + }, + { + z: 6, + x: 2, + y: 2, + offset: 2, + length: 2, + is_dir: false, + }, + { + z: 6, + x: 2, + y: 1, + offset: 1, + length: 1, + is_dir: false, + }, + ]); + let entry = queryLeafdir(view, 7, 3, 3); + assertion.ok(entry!.offset === 3); + entry = queryTile(view, 6, 2, 2); + assertion.ok(entry!.offset === 2); + entry = queryTile(view, 6, 2, 1); + assertion.ok(entry!.offset === 1); + + entry = parseEntry(view, 0); + assertion.ok(entry!.offset === 1); + entry = parseEntry(view, 1); + assertion.ok(entry!.offset === 2); + entry = parseEntry(view, 2); + assertion.ok(entry!.offset === 3); +}); diff --git a/js/v3.test.ts b/js/v3.test.ts new file mode 100644 index 0000000..1948be8 --- /dev/null +++ b/js/v3.test.ts @@ -0,0 +1,295 @@ +import { test } from "zora"; + +// tests run in node, for convenience +// we don't need to typecheck all of it +// @ts-ignore +import fs from "fs"; + +import { + Entry, + zxyToTileId, + tileIdToZxy, + findTile, + readVarint, + Cache, + BufferPosition, + Source, + VersionMismatch, + PMTiles, +} from "./v3"; + +test("varint", (assertion) => { + let b: BufferPosition = { + buf: new Uint8Array([0, 1, 127, 0xe5, 0x8e, 0x26]), + pos: 0, + }; + assertion.eq(readVarint(b), 0); + assertion.eq(readVarint(b), 1); + assertion.eq(readVarint(b), 127); + assertion.eq(readVarint(b), 624485); + b = { + buf: new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]), + pos: 0, + }; + assertion.eq(readVarint(b), 9007199254740991); +}); + +test("zxy to tile id", (assertion) => { + assertion.eq(zxyToTileId(0, 0, 0), 0); + assertion.eq(zxyToTileId(1, 0, 0), 1); + assertion.eq(zxyToTileId(1, 0, 1), 2); + assertion.eq(zxyToTileId(1, 1, 1), 3); + assertion.eq(zxyToTileId(1, 1, 0), 4); + assertion.eq(zxyToTileId(2, 0, 0), 5); +}); + +test("tile id to zxy", (assertion) => { + assertion.eq(tileIdToZxy(0), [0, 0, 0]); + assertion.eq(tileIdToZxy(1), [1, 0, 0]); + assertion.eq(tileIdToZxy(2), [1, 0, 1]); + assertion.eq(tileIdToZxy(3), [1, 1, 1]); + assertion.eq(tileIdToZxy(4), [1, 1, 0]); + assertion.eq(tileIdToZxy(5), [2, 0, 0]); +}); + +test("a lot of tiles", (assertion) => { + for (let z = 0; z < 9; z++) { + for (let x = 0; x < 1 << z; x++) { + for (let y = 0; y < 1 << z; y++) { + const result = tileIdToZxy(zxyToTileId(z, x, y)); + if (result[0] !== z || result[1] !== x || result[2] !== y) { + assertion.fail("roundtrip failed"); + } + } + } + } +}); + +test("tile search for missing entry", (assertion) => { + const entries: Entry[] = []; + assertion.eq(findTile(entries, 101), null); +}); + +test("tile search for first entry == id", (assertion) => { + const entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 1 }]; + const entry = findTile(entries, 100)!; + assertion.eq(entry.offset, 1); + assertion.eq(entry.length, 1); + assertion.eq(findTile(entries, 101), null); +}); + +test("tile search with multiple tile entries", (assertion) => { + let entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 2 }]; + let entry = findTile(entries, 101)!; + assertion.eq(entry.offset, 1); + assertion.eq(entry.length, 1); + + entries = [ + { tileId: 100, offset: 1, length: 1, runLength: 1 }, + { tileId: 150, offset: 2, length: 2, runLength: 2 }, + ]; + entry = findTile(entries, 151)!; + assertion.eq(entry.offset, 2); + assertion.eq(entry.length, 2); + + entries = [ + { tileId: 50, offset: 1, length: 1, runLength: 2 }, + { tileId: 100, offset: 2, length: 2, runLength: 1 }, + { tileId: 150, offset: 3, length: 3, runLength: 1 }, + ]; + entry = findTile(entries, 51)!; + assertion.eq(entry.offset, 1); + assertion.eq(entry.length, 1); +}); + +test("leaf search", (assertion) => { + const entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 0 }]; + const entry = findTile(entries, 150); + assertion.eq(entry!.offset, 1); + assertion.eq(entry!.length, 1); +}); + +// inefficient method only for testing +class TestNodeFileSource implements Source { + buffer: ArrayBuffer; + path: string; + key: string; + etag?: string; + + constructor(path: string, key: string) { + this.path = path; + this.buffer = fs.readFileSync(path); + this.key = key; + } + + getKey() { + return this.key; + } + + replaceData(path: string) { + this.path = path; + this.buffer = fs.readFileSync(path); + } + + async getBytes( + offset: number, + length: number + ): Promise<[ArrayBuffer, string?]> { + const slice = new Uint8Array(this.buffer.slice(offset, offset + length)) + .buffer; + return [slice, this.etag]; + } +} + +// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_1.pmtiles +test("cache getHeader", async (assertion) => { + const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1"); + const cache = new Cache(); + const header = await cache.getHeader(source); + assertion.eq(header.rootDirectoryOffset, 122); + assertion.eq(header.rootDirectoryLength, 25); + assertion.eq(header.jsonMetadataOffset, 147); + assertion.eq(header.jsonMetadataLength, 239); + assertion.eq(header.leafDirectoryOffset, 0); + assertion.eq(header.leafDirectoryLength, 0); + assertion.eq(header.tileDataOffset, 386); + assertion.eq(header.tileDataLength, 69); + assertion.eq(header.numAddressedTiles, 1); + assertion.eq(header.numTileEntries, 1); + assertion.eq(header.numTileContents, 1); + assertion.eq(header.clustered, false); + assertion.eq(header.internalCompression, 1); + assertion.eq(header.tileCompression, 1); + assertion.eq(header.tileType, 1); + assertion.eq(header.minZoom, 0); + assertion.eq(header.maxZoom, 0); + assertion.eq(header.minLon, 0); + assertion.eq(header.minLat, 0); + // assertion.eq(header.maxLon,1); // TODO fix me + assertion.eq(header.maxLat, 1); +}); + +test("cache check against empty", async (assertion) => { + const source = new TestNodeFileSource("empty.pmtiles", "1"); + const cache = new Cache(); + try { + await cache.getHeader(source); + assertion.fail("Should have thrown"); + } catch (e) { + assertion.ok(e instanceof Error); + } +}); + +test("cache check magic number", async (assertion) => { + const source = new TestNodeFileSource("invalid.pmtiles", "1"); + const cache = new Cache(); + try { + await cache.getHeader(source); + assertion.fail("Should have thrown"); + } catch (e) { + console.log(e); + assertion.ok(e instanceof Error); + } +}); + +test("cache getDirectory", async (assertion) => { + const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1"); + + let cache = new Cache(6400, false); + let header = await cache.getHeader(source); + assertion.eq(cache.cache.size, 1); + + cache = new Cache(6400, true); + header = await cache.getHeader(source); + + // prepopulates the root directory + assertion.eq(cache.cache.size, 2); + + const directory = await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ); + assertion.eq(directory.length, 1); + assertion.eq(directory[0].tileId, 0); + assertion.eq(directory[0].offset, 0); + assertion.eq(directory[0].length, 69); + assertion.eq(directory[0].runLength, 1); + + for (const v of cache.cache.values()) { + assertion.ok(v.lastUsed > 0); + assertion.ok(v.size > 0); + } +}); + +test("multiple sources in a single cache", async (assertion) => { + const cache = new Cache(); + const source1 = new TestNodeFileSource("test_fixture_1.pmtiles", "1"); + const source2 = new TestNodeFileSource("test_fixture_1.pmtiles", "2"); + await cache.getHeader(source1); + assertion.eq(cache.cache.size, 2); + await cache.getHeader(source2); + assertion.eq(cache.cache.size, 4); +}); + +test("etags are part of key", async (assertion) => { + const cache = new Cache(6400, false); + const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1"); + source.etag = "etag_1"; + let header = await cache.getHeader(source); + assertion.eq(header.etag, "etag_1"); + + source.etag = "etag_2"; + + try { + await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ); + assertion.fail("Should have thrown"); + } catch (e) { + assertion.ok(e instanceof VersionMismatch); + } + cache.invalidate(source); + header = await cache.getHeader(source); + assertion.ok( + await cache.getDirectory( + source, + header.rootDirectoryOffset, + header.rootDirectoryLength, + header + ) + ); +}); + +test("cache pruning by byte size", async (assertion) => { + const cache = new Cache(1000, false); + cache.cache.set("0", { lastUsed: 0, data: Promise.resolve([]), size: 400 }); + cache.cache.set("1", { lastUsed: 1, data: Promise.resolve([]), size: 200 }); + cache.cache.set("2", { lastUsed: 2, data: Promise.resolve([]), size: 900 }); + cache.sizeBytes = 900 + 200 + 400; + cache.prune(); + assertion.eq(cache.cache.size, 1); + assertion.ok(cache.cache.get("2")); +}); + +test("pmtiles get metadata", async (assertion) => { + const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1"); + const p = new PMTiles(source); + const metadata = await p.getMetadata(); + assertion.ok(metadata.name); +}); + +test("pmtiles handle retries", async (assertion) => { + const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1"); + source.etag = "1"; + const p = new PMTiles(source); + const metadata = await p.getMetadata(); + assertion.ok(metadata.name); + source.etag = "2"; + source.replaceData("test_fixture_2.pmtiles"); + assertion.ok(await p.getZxy(0, 0, 0)); +}); diff --git a/js/v3.ts b/js/v3.ts index 99b4e39..1969a53 100644 --- a/js/v3.ts +++ b/js/v3.ts @@ -1,4 +1,6 @@ -interface BufferPosition { +import { decompressSync } from "fflate"; + +export interface BufferPosition { buf: Uint8Array; pos: number; } @@ -8,9 +10,8 @@ function toNum(low: number, high: number): number { } function readVarintRemainder(l: number, p: BufferPosition): number { - var buf = p.buf, - h, - b; + const buf = p.buf; + let h, b; b = buf[p.pos++]; h = (b & 0x70) >> 4; if (b < 0x80) return toNum(l, h); @@ -33,9 +34,8 @@ function readVarintRemainder(l: number, p: BufferPosition): number { } export function readVarint(p: BufferPosition): number { - var buf = p.buf, - val, - b; + const buf = p.buf; + let val, b; b = buf[p.pos++]; val = b & 0x7f; @@ -61,18 +61,18 @@ function rotate(n: number, xy: number[], rx: number, ry: number): void { xy[0] = n - 1 - xy[0]; xy[1] = n - 1 - xy[1]; } - let t = xy[0]; + const t = xy[0]; xy[0] = xy[1]; xy[1] = t; } } function idOnLevel(z: number, pos: number): [number, number, number] { - let n = 1 << z; + const n = 1 << z; let rx = pos; let ry = pos; let t = pos; - let xy = [0, 0]; + const xy = [0, 0]; let s = 1; while (s < n) { rx = 1 & ((t / 2) >> 0); @@ -93,11 +93,11 @@ export function zxyToTileId(z: number, x: number, y: number): number { acc += (0x1 << tz) * (0x1 << tz); tz++; } - let n = 1 << z; + const n = 1 << z; let rx = 0; let ry = 0; let d = 0; - let xy = [x, y]; + const xy = [x, y]; let s = (n / 2) >> 0; while (s > 0) { rx = (xy[0] & s) > 0 ? 1 : 0; @@ -112,8 +112,8 @@ export function zxyToTileId(z: number, x: number, y: number): number { export function tileIdToZxy(i: number): [number, number, number] { let acc = 0; let z = 0; - while (true) { - let num_tiles = (0x1 << z) * (0x1 << z); + for (;;) { + const num_tiles = (0x1 << z) * (0x1 << z); if (acc + num_tiles > i) { return idOnLevel(z, i - acc); } @@ -129,6 +129,63 @@ export interface Entry { runLength: number; } +const ENTRY_SIZE_BYTES = 32; + +enum Compression { + None = 0, + Gzip = 1, + Brotli = 2, + Zstd = 3, +} + +function tryDecompress(buf: ArrayBuffer, compression: Compression) { + if (compression === Compression.None) { + return buf; + } else if (compression === Compression.Gzip) { + return decompressSync(new Uint8Array(buf)); + } else { + throw Error("Compression method not supported"); + } +} + +enum TileType { + Unknown = 0, + Mvt = 1, + Png = 2, + Jpeg = 3, + Webp = 4, +} + +const HEADER_SIZE_BYTES = 122; + +export interface Header { + rootDirectoryOffset: number; + rootDirectoryLength: number; + jsonMetadataOffset: number; + jsonMetadataLength: number; + leafDirectoryOffset: number; + leafDirectoryLength: number; + tileDataOffset: number; + tileDataLength: number; + numAddressedTiles: number; + numTileEntries: number; + numTileContents: number; + clustered: boolean; + internalCompression: Compression; + tileCompression: Compression; + tileType: TileType; + minZoom: number; + maxZoom: number; + minLon: number; + minLat: number; + maxLon: number; + maxLat: number; + centerZoom: number; + centerLon: number; + centerLat: number; + etag?: string; +} + export function findTile(entries: Entry[], tileId: number): Entry | null { let m = 0; let n = entries.length - 1; @@ -155,3 +212,396 @@ export function findTile(entries: Entry[], tileId: number): Entry | null { } return null; } + +// In the future this may need to change +// to support ReadableStream to pass to native DecompressionStream API +export interface Source { + getBytes: ( + offset: number, + length: number, + signal?: AbortSignal + ) => Promise<[ArrayBuffer, string?]>; + + getKey: () => string; +} + +export class FileAPISource implements Source { + file: File; + + constructor(file: File) { + this.file = file; + } + + getKey() { + return this.file.name; + } + + async getBytes( + offset: number, + length: number, + ): Promise<[ArrayBuffer, undefined]> { + const blob = this.file.slice(offset, offset + length); + const a = await blob.arrayBuffer(); + return [a, undefined]; + } +} + +export class FetchSource implements Source { + url: string; + + constructor(url: string) { + this.url = url; + } + + getKey() { + return this.url; + } + + async getBytes( + offset: number, + length: number, + signal?: AbortSignal + ): Promise<[ArrayBuffer, string?]> { + let controller; + if (!signal) { + // TODO check this works or assert 206 + controller = new AbortController(); + signal = controller.signal; + } + + const resp = await fetch(this.url, { + signal: signal, + headers: { Range: "bytes=" + offset + "-" + (offset + length - 1) }, + }); + const contentLength = resp.headers.get("Content-Length"); + if (!contentLength || +contentLength !== length) { + console.error( + "Content-Length mismatch indicates byte serving not supported; aborting." + ); + if (controller) controller.abort(); + } + + const a = await resp.arrayBuffer(); + return [a, resp.headers.get("ETag") || undefined]; + } +} + +interface CacheEntry { + lastUsed: number; + size: number; // 0 if the promise has not resolved + data: Promise
; +} + +export function bytesToHeader(bytes: ArrayBuffer, etag?: string): Header { + const v = new DataView(bytes); + if (v.getUint16(0, true) !== 0x4d50) { + throw new Error("Wrong magic number for PMTiles archive"); + } + return { + rootDirectoryOffset: Number(v.getBigUint64(3, true)), + rootDirectoryLength: Number(v.getBigUint64(11, true)), + jsonMetadataOffset: Number(v.getBigUint64(19, true)), + jsonMetadataLength: Number(v.getBigUint64(27, true)), + leafDirectoryOffset: Number(v.getBigUint64(35, true)), + leafDirectoryLength: Number(v.getBigUint64(43, true)), + tileDataOffset: Number(v.getBigUint64(51, true)), + tileDataLength: Number(v.getBigUint64(59, true)), + numAddressedTiles: Number(v.getBigUint64(67, true)), + numTileEntries: Number(v.getBigUint64(75, true)), + numTileContents: Number(v.getBigUint64(83, true)), + clustered: v.getUint8(91) === 1, + internalCompression: v.getUint8(92), + tileCompression: v.getUint8(93), + tileType: v.getUint8(94), + minZoom: v.getUint8(95), + maxZoom: v.getUint8(96), + minLon: v.getFloat32(97, true), + minLat: v.getFloat32(101, true), + maxLon: v.getFloat32(105, true), + maxLat: v.getFloat32(109, true), + centerZoom: v.getUint8(113), + centerLon: v.getFloat32(114, true), + centerLat: v.getFloat32(118, true), + etag: etag, + }; +} + +function deserializeIndex(buffer: ArrayBuffer): Entry[] { + const p = { buf: new Uint8Array(buffer), pos: 0 }; + const numEntries = readVarint(p); + + const entries: Entry[] = []; + + let lastId = 0; + for (let i = 0; i < numEntries; i++) { + const v = readVarint(p); + entries.push({ tileId: lastId + v, offset: 0, length: 0, runLength: 1 }); + lastId += v; + } + + for (let i = 0; i < numEntries; i++) { + entries[i].runLength = readVarint(p); + } + + for (let i = 0; i < numEntries; i++) { + entries[i].length = readVarint(p); + } + + for (let i = 0; i < numEntries; i++) { + const v = readVarint(p); + if (v === 0 && i > 0) { + entries[i].offset = entries[i - 1].offset + entries[i - 1].length; + } else { + entries[i].offset = v - 1; + } + } + + return entries; +} + +export class VersionMismatch extends Error {} + +// a "dumb" bag of bytes. +// only caches headers and directories +// deduplicates simultaneous responses +// (estimates) the maximum size of the cache. +export class Cache { + cache: Map; + sizeBytes: number; + maxSizeBytes: number; + counter: number; + prefetch: boolean; + + constructor(maxSizeBytes = 64000000, prefetch = true) { + this.cache = new Map(); + this.sizeBytes = 0; + this.maxSizeBytes = maxSizeBytes; + this.counter = 1; + this.prefetch = prefetch; + } + + async getHeader(source: Source): Promise
{ + const cacheKey = source.getKey(); + if (this.cache.has(cacheKey)) { + const data = await this.cache.get(cacheKey)!.data; + return data as Header; + } + + const p = new Promise
((resolve, reject) => { + source + .getBytes(0, 16384) + .then((resp) => { + if (this.cache.has(cacheKey)) { + this.cache.get(cacheKey)!.size = HEADER_SIZE_BYTES; + this.sizeBytes += HEADER_SIZE_BYTES; + } + + const headerData = resp[0].slice(0, HEADER_SIZE_BYTES); + if (headerData.byteLength !== HEADER_SIZE_BYTES) { + throw new Error("Invalid PMTiles header"); + } + const header = bytesToHeader(headerData, resp[1]); + + // optimistically set the root directory + // TODO check root bounds + if (this.prefetch) { + const rootDirData = resp[0].slice( + header.rootDirectoryOffset, + header.rootDirectoryOffset + header.rootDirectoryLength + ); + const dirKey = + source.getKey() + + "|" + + (header.etag || "") + + "|" + + header.rootDirectoryOffset + + "|" + + header.rootDirectoryLength; + + const rootDir = deserializeIndex( + tryDecompress(rootDirData, header.internalCompression) + ); + + this.cache.set(dirKey, { + lastUsed: this.counter++, + data: Promise.resolve(rootDir), + size: ENTRY_SIZE_BYTES * rootDir.length, + }); + } + + resolve(header); + this.prune(); + }) + .catch((e) => { + reject(e); + }); + }); + this.cache.set(cacheKey, { lastUsed: this.counter++, data: p, size: 0 }); + return p; + } + + async getDirectory( + source: Source, + offset: number, + length: number, + header: Header + ): Promise { + const cacheKey = + source.getKey() + "|" + (header.etag || "") + "|" + offset + "|" + length; + if (this.cache.has(cacheKey)) { + this.cache.get(cacheKey)!.lastUsed = this.counter++; + const data = await this.cache.get(cacheKey)!.data; + return data as Entry[]; + } + + const p = new Promise((resolve, reject) => { + source + .getBytes(offset, length) + .then((resp) => { + if (header.etag && header.etag !== resp[1]) { + throw new VersionMismatch(header.etag); + } + + const data = tryDecompress(resp[0], header.internalCompression); + const directory = deserializeIndex(data); + if (directory.length === 0) { + return reject(new Error("Empty directory is invalid")); + } + + resolve(directory); + + if (this.cache.has(cacheKey)) { + this.cache.get(cacheKey)!.size = ENTRY_SIZE_BYTES * directory.length; + this.sizeBytes += ENTRY_SIZE_BYTES * directory.length; + } + this.prune(); + }) + .catch((e) => { + reject(e); + }); + }); + this.cache.set(cacheKey, { lastUsed: this.counter++, data: p, size: 0 }); + return p; + } + + prune() { + while (this.sizeBytes > this.maxSizeBytes) { + let minUsed = Infinity; + let minKey = undefined; + this.cache.forEach((cache_entry: CacheEntry, key: string) => { + if (cache_entry.lastUsed < minUsed) { + minUsed = cache_entry.lastUsed; + minKey = key; + } + }); + if (minKey) { + this.sizeBytes -= this.cache.get(minKey)!.size; + this.cache.delete(minKey); + } + } + } + + invalidate(source: Source) { + this.cache.delete(source.getKey()); + } +} + +export class PMTiles { + source: Source; + cache: Cache; + + constructor(source: Source, cache?: Cache) { + this.source = source; + if (cache) { + this.cache = cache; + } else { + this.cache = new Cache(); + } + } + + async getZxyAttempt( + z: number, + x: number, + y: number, + signal?: AbortSignal + ): Promise { + const tile_id = zxyToTileId(z, x, y); + const header = await this.cache.getHeader(this.source); + + let d_o = header.rootDirectoryOffset; + let d_l = header.rootDirectoryLength; + for (let depth = 0; depth < 5; depth++) { + const directory = await this.cache.getDirectory( + this.source, + d_o, + d_l, + header + ); + const entry = findTile(directory, tile_id); + if (entry) { + if (entry.runLength > 0) { + const [data, new_etag] = await this.source.getBytes( + header.tileDataOffset + entry.offset, + entry.length, + signal + ); + if (header.etag && header.etag !== new_etag) { + throw new VersionMismatch(header.etag); + } + return tryDecompress(data, header.tileCompression); + } else { + d_o = header.leafDirectoryOffset + entry.offset; + d_l = entry.length; + } + } else { + return undefined; + } + } + throw Error("Maximum directory depth exceeded"); + } + + async getZxy( + z: number, + x: number, + y: number, + signal?: AbortSignal + ): Promise { + try { + return await this.getZxyAttempt(z, x, y, signal); + } catch (e) { + if (e instanceof VersionMismatch) { + this.cache.invalidate(this.source); + return await this.getZxyAttempt(z, x, y, signal); + } else { + throw e; + } + } + } + + async getMetadataAttempt(): Promise { + const header = await this.cache.getHeader(this.source); + const [data, new_etag] = await this.source.getBytes( + header.jsonMetadataOffset, + header.jsonMetadataLength + ); + if (header.etag && header.etag !== new_etag) { + throw new VersionMismatch(header.etag); + } + const decompressed = tryDecompress(data, header.internalCompression); + const dec = new TextDecoder("utf-8"); + return JSON.parse(dec.decode(decompressed)); + } + + async getMetadata(): Promise { + try { + return await this.getMetadataAttempt(); + } catch (e) { + if (e instanceof VersionMismatch) { + this.cache.invalidate(this.source); + return await this.getMetadataAttempt(); + } else { + throw e; + } + } + } +}