JS v3 reader with caching and ETag support [#59, #53, #41, #24, #4]

The v3 module is not exported yet; specifics of header design subject to change.

js/index.test.ts

@@ -1,252 +1,4 @@
-import { test } from "zora";
+import './v2.test';
-import {
+import './v3.test';
-  unshift,
-  getUint24,
-  getUint48,
-  queryLeafdir,
-  queryLeafLevel,
-  queryTile,
-  parseEntry,
-  Entry,
-  createDirectory,
-} from "./index";
 
-import {
-  Entry as EntryV3,
-  zxyToTileId,
-  tileIdToZxy,
-  findTile,
-  readVarint,
-} from "./v3";
 
-test("stub data", (assertion) => {
-  let dataview = createDirectory([
-    { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
-    {
-      z: 14,
-      x: 16383,
-      y: 16383,
-      offset: 999999,
-      length: 999,
-      is_dir: false,
-    },
-  ]);
-
-  var z_raw = dataview.getUint8(17 + 0);
-  var x = getUint24(dataview, 17 + 1);
-  var y = getUint24(dataview, 17 + 4);
-  var offset = getUint48(dataview, 17 + 7);
-  var length = dataview.getUint32(17 + 13, true);
-  assertion.ok(z_raw === 14);
-  assertion.ok(x === 16383);
-  assertion.ok(y === 16383);
-});
-
-test("get entry", (assertion) => {
-  let view = createDirectory([
-    { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
-    {
-      z: 14,
-      x: 16383,
-      y: 16383,
-      offset: 999999,
-      length: 999,
-      is_dir: false,
-    },
-  ]);
-  let entry = queryTile(view, 14, 16383, 16383);
-  assertion.ok(entry!.z === 14);
-  assertion.ok(entry!.x === 16383);
-  assertion.ok(entry!.y === 16383);
-  assertion.ok(entry!.offset === 999999);
-  assertion.ok(entry!.length === 999);
-  assertion.ok(entry!.is_dir === false);
-  assertion.ok(queryLeafdir(view, 14, 16383, 16383) === null);
-});
-
-test("get leafdir", (assertion) => {
-  let view = createDirectory([
-    {
-      z: 14,
-      x: 16383,
-      y: 16383,
-      offset: 999999,
-      length: 999,
-      is_dir: true,
-    },
-  ]);
-  let entry = queryLeafdir(view, 14, 16383, 16383);
-  assertion.ok(entry!.z === 14);
-  assertion.ok(entry!.x === 16383);
-  assertion.ok(entry!.y === 16383);
-  assertion.ok(entry!.offset === 999999);
-  assertion.ok(entry!.length === 999);
-  assertion.ok(entry!.is_dir === true);
-  assertion.ok(queryTile(view, 14, 16383, 16383) === null);
-});
-
-test("derive the leaf level", (assertion) => {
-  let view = createDirectory([
-    {
-      z: 6,
-      x: 3,
-      y: 3,
-      offset: 0,
-      length: 0,
-      is_dir: true,
-    },
-  ]);
-  let level = queryLeafLevel(view);
-  assertion.ok(level === 6);
-  view = createDirectory([
-    {
-      z: 6,
-      x: 3,
-      y: 3,
-      offset: 0,
-      length: 0,
-      is_dir: false,
-    },
-  ]);
-  level = queryLeafLevel(view);
-  assertion.ok(level === null);
-});
-
-test("convert spec v1 directory to spec v2 directory", (assertion) => {
-  let view = createDirectory([
-    {
-      z: 7,
-      x: 3,
-      y: 3,
-      offset: 3,
-      length: 3,
-      is_dir: true,
-    },
-    {
-      z: 6,
-      x: 2,
-      y: 2,
-      offset: 2,
-      length: 2,
-      is_dir: false,
-    },
-    {
-      z: 6,
-      x: 2,
-      y: 1,
-      offset: 1,
-      length: 1,
-      is_dir: false,
-    },
-  ]);
-  let entry = queryLeafdir(view, 7, 3, 3);
-  assertion.ok(entry!.offset === 3);
-  entry = queryTile(view, 6, 2, 2);
-  assertion.ok(entry!.offset === 2);
-  entry = queryTile(view, 6, 2, 1);
-  assertion.ok(entry!.offset === 1);
-
-  entry = parseEntry(view, 0);
-  assertion.ok(entry!.offset === 1);
-  entry = parseEntry(view, 1);
-  assertion.ok(entry!.offset === 2);
-  entry = parseEntry(view, 2);
-  assertion.ok(entry!.offset === 3);
-});
-
-test("varint", (assertion) => {
-  let b: BufferPosition = {
-    buf: new Uint8Array([0, 1, 127, 0xe5, 0x8e, 0x26]),
-    pos: 0,
-  };
-  assertion.eq(readVarint(b), 0);
-  assertion.eq(readVarint(b), 1);
-  assertion.eq(readVarint(b), 127);
-  assertion.eq(readVarint(b), 624485);
-  b = {
-    buf: new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]),
-    pos: 0,
-  };
-  assertion.eq(readVarint(b), 9007199254740991);
-});
-
-test("zxy to tile id", (assertion) => {
-  assertion.eq(zxyToTileId(0, 0, 0), 0);
-  assertion.eq(zxyToTileId(1, 0, 0), 1);
-  assertion.eq(zxyToTileId(1, 0, 1), 2);
-  assertion.eq(zxyToTileId(1, 1, 1), 3);
-  assertion.eq(zxyToTileId(1, 1, 0), 4);
-  assertion.eq(zxyToTileId(2, 0, 0), 5);
-});
-
-test("tile id to zxy", (assertion) => {
-  assertion.eq(tileIdToZxy(0), [0, 0, 0]);
-  assertion.eq(tileIdToZxy(1), [1, 0, 0]);
-  assertion.eq(tileIdToZxy(2), [1, 0, 1]);
-  assertion.eq(tileIdToZxy(3), [1, 1, 1]);
-  assertion.eq(tileIdToZxy(4), [1, 1, 0]);
-  assertion.eq(tileIdToZxy(5), [2, 0, 0]);
-});
-
-test("a lot of tiles", (assertion) => {
-  for (var z = 0; z < 9; z++) {
-    for (var x = 0; x < 1 << z; x++) {
-      for (var y = 0; y < 1 << z; y++) {
-        let result = tileIdToZxy(zxyToTileId(z, x, y));
-        if (result[0] !== z || result[1] !== x || result[2] !== y) {
-          assertion.fail("roundtrip failed");
-        }
-      }
-    }
-  }
-});
-
-test("tile search for first entry == id", (assertion) => {
-  let entries: EntryV3[] = [];
-  assertion.eq(findTile(entries, 101), null);
-});
-
-test("tile search for first entry == id", (assertion) => {
-  let entries: EntryV3[] = [
-    { tileId: 100, offset: 1, length: 1, runLength: 1 },
-  ];
-  let entry = findTile(entries, 100)!;
-  assertion.eq(entry.offset, 1);
-  assertion.eq(entry.length, 1);
-  assertion.eq(findTile(entries, 101), null);
-});
-
-test("tile search for first entry == id", (assertion) => {
-  let entries: EntryV3[] = [
-    { tileId: 100, offset: 1, length: 1, runLength: 2 },
-  ];
-  let entry = findTile(entries, 101)!;
-  assertion.eq(entry.offset, 1);
-  assertion.eq(entry.length, 1);
-
-  entries = [
-    { tileId: 100, offset: 1, length: 1, runLength: 1 },
-    { tileId: 150, offset: 2, length: 2, runLength: 2 },
-  ];
-  entry = findTile(entries, 151)!;
-  assertion.eq(entry.offset, 2);
-  assertion.eq(entry.length, 2);
-
-  entries = [
-    { tileId: 50, offset: 1, length: 1, runLength: 2 },
-    { tileId: 100, offset: 2, length: 2, runLength: 1 },
-    { tileId: 150, offset: 3, length: 3, runLength: 1 },
-  ];
-  entry = findTile(entries, 51)!;
-  assertion.eq(entry.offset, 1);
-  assertion.eq(entry.length, 1);
-});
-
-test("leaf search", (assertion) => {
-  let entries: EntryV3[] = [
-    { tileId: 100, offset: 1, length: 1, runLength: 0 },
-  ];
-  let entry = findTile(entries, 150);
-  assertion.eq(entry!.offset, 1);
-  assertion.eq(entry!.length, 1);
-});

js/invalid.pmtiles

@@ -0,0 +1 @@
+This is an invalid tile archive, a test case to make sure that the code throws an error, but it needs to be the minimum size to pass the first test

js/v2.test.ts

@@ -0,0 +1,148 @@
+import { test } from "zora";
+
+import {
+  unshift,
+  getUint24,
+  getUint48,
+  queryLeafdir,
+  queryLeafLevel,
+  queryTile,
+  parseEntry,
+  Entry,
+  createDirectory,
+} from "./index";
+
+test("stub data", (assertion) => {
+  let dataview = createDirectory([
+    { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
+    {
+      z: 14,
+      x: 16383,
+      y: 16383,
+      offset: 999999,
+      length: 999,
+      is_dir: false,
+    },
+  ]);
+
+  var z_raw = dataview.getUint8(17 + 0);
+  var x = getUint24(dataview, 17 + 1);
+  var y = getUint24(dataview, 17 + 4);
+  var offset = getUint48(dataview, 17 + 7);
+  var length = dataview.getUint32(17 + 13, true);
+  assertion.ok(z_raw === 14);
+  assertion.ok(x === 16383);
+  assertion.ok(y === 16383);
+});
+
+test("get entry", (assertion) => {
+  let view = createDirectory([
+    { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
+    {
+      z: 14,
+      x: 16383,
+      y: 16383,
+      offset: 999999,
+      length: 999,
+      is_dir: false,
+    },
+  ]);
+  let entry = queryTile(view, 14, 16383, 16383);
+  assertion.ok(entry!.z === 14);
+  assertion.ok(entry!.x === 16383);
+  assertion.ok(entry!.y === 16383);
+  assertion.ok(entry!.offset === 999999);
+  assertion.ok(entry!.length === 999);
+  assertion.ok(entry!.is_dir === false);
+  assertion.ok(queryLeafdir(view, 14, 16383, 16383) === null);
+});
+
+test("get leafdir", (assertion) => {
+  let view = createDirectory([
+    {
+      z: 14,
+      x: 16383,
+      y: 16383,
+      offset: 999999,
+      length: 999,
+      is_dir: true,
+    },
+  ]);
+  let entry = queryLeafdir(view, 14, 16383, 16383);
+  assertion.ok(entry!.z === 14);
+  assertion.ok(entry!.x === 16383);
+  assertion.ok(entry!.y === 16383);
+  assertion.ok(entry!.offset === 999999);
+  assertion.ok(entry!.length === 999);
+  assertion.ok(entry!.is_dir === true);
+  assertion.ok(queryTile(view, 14, 16383, 16383) === null);
+});
+
+test("derive the leaf level", (assertion) => {
+  let view = createDirectory([
+    {
+      z: 6,
+      x: 3,
+      y: 3,
+      offset: 0,
+      length: 0,
+      is_dir: true,
+    },
+  ]);
+  let level = queryLeafLevel(view);
+  assertion.ok(level === 6);
+  view = createDirectory([
+    {
+      z: 6,
+      x: 3,
+      y: 3,
+      offset: 0,
+      length: 0,
+      is_dir: false,
+    },
+  ]);
+  level = queryLeafLevel(view);
+  assertion.ok(level === null);
+});
+
+test("convert spec v1 directory to spec v2 directory", (assertion) => {
+  let view = createDirectory([
+    {
+      z: 7,
+      x: 3,
+      y: 3,
+      offset: 3,
+      length: 3,
+      is_dir: true,
+    },
+    {
+      z: 6,
+      x: 2,
+      y: 2,
+      offset: 2,
+      length: 2,
+      is_dir: false,
+    },
+    {
+      z: 6,
+      x: 2,
+      y: 1,
+      offset: 1,
+      length: 1,
+      is_dir: false,
+    },
+  ]);
+  let entry = queryLeafdir(view, 7, 3, 3);
+  assertion.ok(entry!.offset === 3);
+  entry = queryTile(view, 6, 2, 2);
+  assertion.ok(entry!.offset === 2);
+  entry = queryTile(view, 6, 2, 1);
+  assertion.ok(entry!.offset === 1);
+
+  entry = parseEntry(view, 0);
+  assertion.ok(entry!.offset === 1);
+  entry = parseEntry(view, 1);
+  assertion.ok(entry!.offset === 2);
+  entry = parseEntry(view, 2);
+  assertion.ok(entry!.offset === 3);
+});

js/v3.test.ts

@@ -0,0 +1,295 @@
+import { test } from "zora";
+
+// tests run in node, for convenience
+// we don't need to typecheck all of it
+// @ts-ignore
+import fs from "fs";
+
+import {
+	Entry,
+	zxyToTileId,
+	tileIdToZxy,
+	findTile,
+	readVarint,
+	Cache,
+	BufferPosition,
+	Source,
+	VersionMismatch,
+	PMTiles,
+} from "./v3";
+
+test("varint", (assertion) => {
+	let b: BufferPosition = {
+		buf: new Uint8Array([0, 1, 127, 0xe5, 0x8e, 0x26]),
+		pos: 0,
+	};
+	assertion.eq(readVarint(b), 0);
+	assertion.eq(readVarint(b), 1);
+	assertion.eq(readVarint(b), 127);
+	assertion.eq(readVarint(b), 624485);
+	b = {
+		buf: new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]),
+		pos: 0,
+	};
+	assertion.eq(readVarint(b), 9007199254740991);
+});
+
+test("zxy to tile id", (assertion) => {
+	assertion.eq(zxyToTileId(0, 0, 0), 0);
+	assertion.eq(zxyToTileId(1, 0, 0), 1);
+	assertion.eq(zxyToTileId(1, 0, 1), 2);
+	assertion.eq(zxyToTileId(1, 1, 1), 3);
+	assertion.eq(zxyToTileId(1, 1, 0), 4);
+	assertion.eq(zxyToTileId(2, 0, 0), 5);
+});
+
+test("tile id to zxy", (assertion) => {
+	assertion.eq(tileIdToZxy(0), [0, 0, 0]);
+	assertion.eq(tileIdToZxy(1), [1, 0, 0]);
+	assertion.eq(tileIdToZxy(2), [1, 0, 1]);
+	assertion.eq(tileIdToZxy(3), [1, 1, 1]);
+	assertion.eq(tileIdToZxy(4), [1, 1, 0]);
+	assertion.eq(tileIdToZxy(5), [2, 0, 0]);
+});
+
+test("a lot of tiles", (assertion) => {
+	for (let z = 0; z < 9; z++) {
+		for (let x = 0; x < 1 << z; x++) {
+			for (let y = 0; y < 1 << z; y++) {
+				const result = tileIdToZxy(zxyToTileId(z, x, y));
+				if (result[0] !== z || result[1] !== x || result[2] !== y) {
+					assertion.fail("roundtrip failed");
+				}
+			}
+		}
+	}
+});
+
+test("tile search for missing entry", (assertion) => {
+	const entries: Entry[] = [];
+	assertion.eq(findTile(entries, 101), null);
+});
+
+test("tile search for first entry == id", (assertion) => {
+	const entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 1 }];
+	const entry = findTile(entries, 100)!;
+	assertion.eq(entry.offset, 1);
+	assertion.eq(entry.length, 1);
+	assertion.eq(findTile(entries, 101), null);
+});
+
+test("tile search with multiple tile entries", (assertion) => {
+	let entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 2 }];
+	let entry = findTile(entries, 101)!;
+	assertion.eq(entry.offset, 1);
+	assertion.eq(entry.length, 1);
+
+	entries = [
+		{ tileId: 100, offset: 1, length: 1, runLength: 1 },
+		{ tileId: 150, offset: 2, length: 2, runLength: 2 },
+	];
+	entry = findTile(entries, 151)!;
+	assertion.eq(entry.offset, 2);
+	assertion.eq(entry.length, 2);
+
+	entries = [
+		{ tileId: 50, offset: 1, length: 1, runLength: 2 },
+		{ tileId: 100, offset: 2, length: 2, runLength: 1 },
+		{ tileId: 150, offset: 3, length: 3, runLength: 1 },
+	];
+	entry = findTile(entries, 51)!;
+	assertion.eq(entry.offset, 1);
+	assertion.eq(entry.length, 1);
+});
+
+test("leaf search", (assertion) => {
+	const entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 0 }];
+	const entry = findTile(entries, 150);
+	assertion.eq(entry!.offset, 1);
+	assertion.eq(entry!.length, 1);
+});
+
+// inefficient method only for testing
+class TestNodeFileSource implements Source {
+	buffer: ArrayBuffer;
+	path: string;
+	key: string;
+	etag?: string;
+
+	constructor(path: string, key: string) {
+		this.path = path;
+		this.buffer = fs.readFileSync(path);
+		this.key = key;
+	}
+
+	getKey() {
+		return this.key;
+	}
+
+	replaceData(path: string) {
+		this.path = path;
+		this.buffer = fs.readFileSync(path);
+	}
+
+	async getBytes(
+		offset: number,
+		length: number
+	): Promise<[ArrayBuffer, string?]> {
+		const slice = new Uint8Array(this.buffer.slice(offset, offset + length))
+			.buffer;
+		return [slice, this.etag];
+	}
+}
+
+// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_1.pmtiles
+test("cache getHeader", async (assertion) => {
+	const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
+	const cache = new Cache();
+	const header = await cache.getHeader(source);
+	assertion.eq(header.rootDirectoryOffset, 122);
+	assertion.eq(header.rootDirectoryLength, 25);
+	assertion.eq(header.jsonMetadataOffset, 147);
+	assertion.eq(header.jsonMetadataLength, 239);
+	assertion.eq(header.leafDirectoryOffset, 0);
+	assertion.eq(header.leafDirectoryLength, 0);
+	assertion.eq(header.tileDataOffset, 386);
+	assertion.eq(header.tileDataLength, 69);
+	assertion.eq(header.numAddressedTiles, 1);
+	assertion.eq(header.numTileEntries, 1);
+	assertion.eq(header.numTileContents, 1);
+	assertion.eq(header.clustered, false);
+	assertion.eq(header.internalCompression, 1);
+	assertion.eq(header.tileCompression, 1);
+	assertion.eq(header.tileType, 1);
+	assertion.eq(header.minZoom, 0);
+	assertion.eq(header.maxZoom, 0);
+	assertion.eq(header.minLon, 0);
+	assertion.eq(header.minLat, 0);
+	// assertion.eq(header.maxLon,1); // TODO fix me
+	assertion.eq(header.maxLat, 1);
+});
+
+test("cache check against empty", async (assertion) => {
+	const source = new TestNodeFileSource("empty.pmtiles", "1");
+	const cache = new Cache();
+	try {
+		await cache.getHeader(source);
+		assertion.fail("Should have thrown");
+	} catch (e) {
+		assertion.ok(e instanceof Error);
+	}
+});
+
+test("cache check magic number", async (assertion) => {
+	const source = new TestNodeFileSource("invalid.pmtiles", "1");
+	const cache = new Cache();
+	try {
+		await cache.getHeader(source);
+		assertion.fail("Should have thrown");
+	} catch (e) {
+		console.log(e);
+		assertion.ok(e instanceof Error);
+	}
+});
+
+test("cache getDirectory", async (assertion) => {
+	const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
+
+	let cache = new Cache(6400, false);
+	let header = await cache.getHeader(source);
+	assertion.eq(cache.cache.size, 1);
+
+	cache = new Cache(6400, true);
+	header = await cache.getHeader(source);
+
+	// prepopulates the root directory
+	assertion.eq(cache.cache.size, 2);
+
+	const directory = await cache.getDirectory(
+		source,
+		header.rootDirectoryOffset,
+		header.rootDirectoryLength,
+		header
+	);
+	assertion.eq(directory.length, 1);
+	assertion.eq(directory[0].tileId, 0);
+	assertion.eq(directory[0].offset, 0);
+	assertion.eq(directory[0].length, 69);
+	assertion.eq(directory[0].runLength, 1);
+
+	for (const v of cache.cache.values()) {
+		assertion.ok(v.lastUsed > 0);
+		assertion.ok(v.size > 0);
+	}
+});
+
+test("multiple sources in a single cache", async (assertion) => {
+	const cache = new Cache();
+	const source1 = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
+	const source2 = new TestNodeFileSource("test_fixture_1.pmtiles", "2");
+	await cache.getHeader(source1);
+	assertion.eq(cache.cache.size, 2);
+	await cache.getHeader(source2);
+	assertion.eq(cache.cache.size, 4);
+});
+
+test("etags are part of key", async (assertion) => {
+	const cache = new Cache(6400, false);
+	const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
+	source.etag = "etag_1";
+	let header = await cache.getHeader(source);
+	assertion.eq(header.etag, "etag_1");
+
+	source.etag = "etag_2";
+
+	try {
+		await cache.getDirectory(
+			source,
+			header.rootDirectoryOffset,
+			header.rootDirectoryLength,
+			header
+		);
+		assertion.fail("Should have thrown");
+	} catch (e) {
+		assertion.ok(e instanceof VersionMismatch);
+	}
+	cache.invalidate(source);
+	header = await cache.getHeader(source);
+	assertion.ok(
+		await cache.getDirectory(
+			source,
+			header.rootDirectoryOffset,
+			header.rootDirectoryLength,
+			header
+		)
+	);
+});
+
+test("cache pruning by byte size", async (assertion) => {
+	const cache = new Cache(1000, false);
+	cache.cache.set("0", { lastUsed: 0, data: Promise.resolve([]), size: 400 });
+	cache.cache.set("1", { lastUsed: 1, data: Promise.resolve([]), size: 200 });
+	cache.cache.set("2", { lastUsed: 2, data: Promise.resolve([]), size: 900 });
+	cache.sizeBytes = 900 + 200 + 400;
+	cache.prune();
+	assertion.eq(cache.cache.size, 1);
+	assertion.ok(cache.cache.get("2"));
+});
+
+test("pmtiles get metadata", async (assertion) => {
+	const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
+	const p = new PMTiles(source);
+	const metadata = await p.getMetadata();
+	assertion.ok(metadata.name);
+});
+
+test("pmtiles handle retries", async (assertion) => {
+	const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
+	source.etag = "1";
+	const p = new PMTiles(source);
+	const metadata = await p.getMetadata();
+	assertion.ok(metadata.name);
+	source.etag = "2";
+	source.replaceData("test_fixture_2.pmtiles");
+	assertion.ok(await p.getZxy(0, 0, 0));
+});

js/v3.ts

@@ -1,4 +1,6 @@
-interface BufferPosition {
+import { decompressSync } from "fflate";
+
+export interface BufferPosition {
 	buf: Uint8Array;
 	pos: number;
 }
@@ -8,9 +10,8 @@ function toNum(low: number, high: number): number {
 }
 
 function readVarintRemainder(l: number, p: BufferPosition): number {
-	var buf = p.buf,
+	const buf = p.buf;
-		h,
+	let h, b;
-		b;
 	b = buf[p.pos++];
 	h = (b & 0x70) >> 4;
 	if (b < 0x80) return toNum(l, h);
@@ -33,9 +34,8 @@ function readVarintRemainder(l: number, p: BufferPosition): number {
 }
 
 export function readVarint(p: BufferPosition): number {
-	var buf = p.buf,
+	const buf = p.buf;
-		val,
+	let val, b;
-		b;
 
 	b = buf[p.pos++];
 	val = b & 0x7f;
@@ -61,18 +61,18 @@ function rotate(n: number, xy: number[], rx: number, ry: number): void {
 			xy[0] = n - 1 - xy[0];
 			xy[1] = n - 1 - xy[1];
 		}
-		let t = xy[0];
+		const t = xy[0];
 		xy[0] = xy[1];
 		xy[1] = t;
 	}
 }
 
 function idOnLevel(z: number, pos: number): [number, number, number] {
-	let n = 1 << z;
+	const n = 1 << z;
 	let rx = pos;
 	let ry = pos;
 	let t = pos;
-	let xy = [0, 0];
+	const xy = [0, 0];
 	let s = 1;
 	while (s < n) {
 		rx = 1 & ((t / 2) >> 0);
@@ -93,11 +93,11 @@ export function zxyToTileId(z: number, x: number, y: number): number {
 		acc += (0x1 << tz) * (0x1 << tz);
 		tz++;
 	}
-	let n = 1 << z;
+	const n = 1 << z;
 	let rx = 0;
 	let ry = 0;
 	let d = 0;
-	let xy = [x, y];
+	const xy = [x, y];
 	let s = (n / 2) >> 0;
 	while (s > 0) {
 		rx = (xy[0] & s) > 0 ? 1 : 0;
@@ -112,8 +112,8 @@ export function zxyToTileId(z: number, x: number, y: number): number {
 export function tileIdToZxy(i: number): [number, number, number] {
 	let acc = 0;
 	let z = 0;
-	while (true) {
+	for (;;) {
-		let num_tiles = (0x1 << z) * (0x1 << z);
+		const num_tiles = (0x1 << z) * (0x1 << z);
 		if (acc + num_tiles > i) {
 			return idOnLevel(z, i - acc);
 		}
@@ -129,6 +129,63 @@ export interface Entry {
 	runLength: number;
 }
 
+const ENTRY_SIZE_BYTES = 32;
+
+enum Compression {
+	None = 0,
+	Gzip = 1,
+	Brotli = 2,
+	Zstd = 3,
+}
+
+function tryDecompress(buf: ArrayBuffer, compression: Compression) {
+	if (compression === Compression.None) {
+		return buf;
+	} else if (compression === Compression.Gzip) {
+		return decompressSync(new Uint8Array(buf));
+	} else {
+		throw Error("Compression method not supported");
+	}
+}
+
+enum TileType {
+	Unknown = 0,
+	Mvt = 1,
+	Png = 2,
+	Jpeg = 3,
+	Webp = 4,
+}
+
+const HEADER_SIZE_BYTES = 122;
+
+export interface Header {
+	rootDirectoryOffset: number;
+	rootDirectoryLength: number;
+	jsonMetadataOffset: number;
+	jsonMetadataLength: number;
+	leafDirectoryOffset: number;
+	leafDirectoryLength: number;
+	tileDataOffset: number;
+	tileDataLength: number;
+	numAddressedTiles: number;
+	numTileEntries: number;
+	numTileContents: number;
+	clustered: boolean;
+	internalCompression: Compression;
+	tileCompression: Compression;
+	tileType: TileType;
+	minZoom: number;
+	maxZoom: number;
+	minLon: number;
+	minLat: number;
+	maxLon: number;
+	maxLat: number;
+	centerZoom: number;
+	centerLon: number;
+	centerLat: number;
+	etag?: string;
+}
+
 export function findTile(entries: Entry[], tileId: number): Entry | null {
 	let m = 0;
 	let n = entries.length - 1;
@@ -155,3 +212,396 @@ export function findTile(entries: Entry[], tileId: number): Entry | null {
 	}
 	return null;
 }
+
+// In the future this may need to change
+// to support ReadableStream to pass to native DecompressionStream API
+export interface Source {
+	getBytes: (
+		offset: number,
+		length: number,
+		signal?: AbortSignal
+	) => Promise<[ArrayBuffer, string?]>;
+
+	getKey: () => string;
+}
+
+export class FileAPISource implements Source {
+	file: File;
+
+	constructor(file: File) {
+		this.file = file;
+	}
+
+	getKey() {
+		return this.file.name;
+	}
+
+	async getBytes(
+		offset: number,
+		length: number,
+	): Promise<[ArrayBuffer, undefined]> {
+		const blob = this.file.slice(offset, offset + length);
+		const a = await blob.arrayBuffer();
+		return [a, undefined];
+	}
+}
+
+export class FetchSource implements Source {
+	url: string;
+
+	constructor(url: string) {
+		this.url = url;
+	}
+
+	getKey() {
+		return this.url;
+	}
+
+	async getBytes(
+		offset: number,
+		length: number,
+		signal?: AbortSignal
+	): Promise<[ArrayBuffer, string?]> {
+		let controller;
+		if (!signal) {
+			// TODO check this works or assert 206
+			controller = new AbortController();
+			signal = controller.signal;
+		}
+
+		const resp = await fetch(this.url, {
+			signal: signal,
+			headers: { Range: "bytes=" + offset + "-" + (offset + length - 1) },
+		});
+		const contentLength = resp.headers.get("Content-Length");
+		if (!contentLength || +contentLength !== length) {
+			console.error(
+				"Content-Length mismatch indicates byte serving not supported; aborting."
+			);
+			if (controller) controller.abort();
+		}
+
+		const a = await resp.arrayBuffer();
+		return [a, resp.headers.get("ETag") || undefined];
+	}
+}
+
+interface CacheEntry {
+	lastUsed: number;
+	size: number; // 0 if the promise has not resolved
+	data: Promise<Header | Entry[] | ArrayBuffer>;
+}
+
+export function bytesToHeader(bytes: ArrayBuffer, etag?: string): Header {
+	const v = new DataView(bytes);
+	if (v.getUint16(0, true) !== 0x4d50) {
+		throw new Error("Wrong magic number for PMTiles archive");
+	}
+	return {
+		rootDirectoryOffset: Number(v.getBigUint64(3, true)),
+		rootDirectoryLength: Number(v.getBigUint64(11, true)),
+		jsonMetadataOffset: Number(v.getBigUint64(19, true)),
+		jsonMetadataLength: Number(v.getBigUint64(27, true)),
+		leafDirectoryOffset: Number(v.getBigUint64(35, true)),
+		leafDirectoryLength: Number(v.getBigUint64(43, true)),
+		tileDataOffset: Number(v.getBigUint64(51, true)),
+		tileDataLength: Number(v.getBigUint64(59, true)),
+		numAddressedTiles: Number(v.getBigUint64(67, true)),
+		numTileEntries: Number(v.getBigUint64(75, true)),
+		numTileContents: Number(v.getBigUint64(83, true)),
+		clustered: v.getUint8(91) === 1,
+		internalCompression: v.getUint8(92),
+		tileCompression: v.getUint8(93),
+		tileType: v.getUint8(94),
+		minZoom: v.getUint8(95),
+		maxZoom: v.getUint8(96),
+		minLon: v.getFloat32(97, true),
+		minLat: v.getFloat32(101, true),
+		maxLon: v.getFloat32(105, true),
+		maxLat: v.getFloat32(109, true),
+		centerZoom: v.getUint8(113),
+		centerLon: v.getFloat32(114, true),
+		centerLat: v.getFloat32(118, true),
+		etag: etag,
+	};
+}
+
+function deserializeIndex(buffer: ArrayBuffer): Entry[] {
+	const p = { buf: new Uint8Array(buffer), pos: 0 };
+	const numEntries = readVarint(p);
+
+	const entries: Entry[] = [];
+
+	let lastId = 0;
+	for (let i = 0; i < numEntries; i++) {
+		const v = readVarint(p);
+		entries.push({ tileId: lastId + v, offset: 0, length: 0, runLength: 1 });
+		lastId += v;
+	}
+
+	for (let i = 0; i < numEntries; i++) {
+		entries[i].runLength = readVarint(p);
+	}
+
+	for (let i = 0; i < numEntries; i++) {
+		entries[i].length = readVarint(p);
+	}
+
+	for (let i = 0; i < numEntries; i++) {
+		const v = readVarint(p);
+		if (v === 0 && i > 0) {
+			entries[i].offset = entries[i - 1].offset + entries[i - 1].length;
+		} else {
+			entries[i].offset = v - 1;
+		}
+	}
+
+	return entries;
+}
+
+export class VersionMismatch extends Error {}
+
+// a "dumb" bag of bytes.
+// only caches headers and directories
+// deduplicates simultaneous responses
+// (estimates) the maximum size of the cache.
+export class Cache {
+	cache: Map<string, CacheEntry>;
+	sizeBytes: number;
+	maxSizeBytes: number;
+	counter: number;
+	prefetch: boolean;
+
+	constructor(maxSizeBytes = 64000000, prefetch = true) {
+		this.cache = new Map<string, CacheEntry>();
+		this.sizeBytes = 0;
+		this.maxSizeBytes = maxSizeBytes;
+		this.counter = 1;
+		this.prefetch = prefetch;
+	}
+
+	async getHeader(source: Source): Promise<Header> {
+		const cacheKey = source.getKey();
+		if (this.cache.has(cacheKey)) {
+			const data = await this.cache.get(cacheKey)!.data;
+			return data as Header;
+		}
+
+		const p = new Promise<Header>((resolve, reject) => {
+			source
+				.getBytes(0, 16384)
+				.then((resp) => {
+					if (this.cache.has(cacheKey)) {
+						this.cache.get(cacheKey)!.size = HEADER_SIZE_BYTES;
+						this.sizeBytes += HEADER_SIZE_BYTES;
+					}
+
+					const headerData = resp[0].slice(0, HEADER_SIZE_BYTES);
+					if (headerData.byteLength !== HEADER_SIZE_BYTES) {
+						throw new Error("Invalid PMTiles header");
+					}
+					const header = bytesToHeader(headerData, resp[1]);
+
+					// optimistically set the root directory
+					// TODO check root bounds
+					if (this.prefetch) {
+						const rootDirData = resp[0].slice(
+							header.rootDirectoryOffset,
+							header.rootDirectoryOffset + header.rootDirectoryLength
+						);
+						const dirKey =
+							source.getKey() +
+							"|" +
+							(header.etag || "") +
+							"|" +
+							header.rootDirectoryOffset +
+							"|" +
+							header.rootDirectoryLength;
+
+						const rootDir = deserializeIndex(
+							tryDecompress(rootDirData, header.internalCompression)
+						);
+
+						this.cache.set(dirKey, {
+							lastUsed: this.counter++,
+							data: Promise.resolve(rootDir),
+							size: ENTRY_SIZE_BYTES * rootDir.length,
+						});
+					}
+
+					resolve(header);
+					this.prune();
+				})
+				.catch((e) => {
+					reject(e);
+				});
+		});
+		this.cache.set(cacheKey, { lastUsed: this.counter++, data: p, size: 0 });
+		return p;
+	}
+
+	async getDirectory(
+		source: Source,
+		offset: number,
+		length: number,
+		header: Header
+	): Promise<Entry[]> {
+		const cacheKey =
+			source.getKey() + "|" + (header.etag || "") + "|" + offset + "|" + length;
+		if (this.cache.has(cacheKey)) {
+			this.cache.get(cacheKey)!.lastUsed = this.counter++;
+			const data = await this.cache.get(cacheKey)!.data;
+			return data as Entry[];
+		}
+
+		const p = new Promise<Entry[]>((resolve, reject) => {
+			source
+				.getBytes(offset, length)
+				.then((resp) => {
+					if (header.etag && header.etag !== resp[1]) {
+						throw new VersionMismatch(header.etag);
+					}
+
+					const data = tryDecompress(resp[0], header.internalCompression);
+					const directory = deserializeIndex(data);
+					if (directory.length === 0) {
+						return reject(new Error("Empty directory is invalid"));
+					}
+
+					resolve(directory);
+
+					if (this.cache.has(cacheKey)) {
+						this.cache.get(cacheKey)!.size = ENTRY_SIZE_BYTES * directory.length;
+						this.sizeBytes += ENTRY_SIZE_BYTES * directory.length;
+					}
+					this.prune();
+				})
+				.catch((e) => {
+					reject(e);
+				});
+		});
+		this.cache.set(cacheKey, { lastUsed: this.counter++, data: p, size: 0 });
+		return p;
+	}
+
+	prune() {
+		while (this.sizeBytes > this.maxSizeBytes) {
+			let minUsed = Infinity;
+			let minKey = undefined;
+			this.cache.forEach((cache_entry: CacheEntry, key: string) => {
+				if (cache_entry.lastUsed < minUsed) {
+					minUsed = cache_entry.lastUsed;
+					minKey = key;
+				}
+			});
+			if (minKey) {
+				this.sizeBytes -= this.cache.get(minKey)!.size;
+				this.cache.delete(minKey);
+			}
+		}
+	}
+
+	invalidate(source: Source) {
+		this.cache.delete(source.getKey());
+	}
+}
+
+export class PMTiles {
+	source: Source;
+	cache: Cache;
+
+	constructor(source: Source, cache?: Cache) {
+		this.source = source;
+		if (cache) {
+			this.cache = cache;
+		} else {
+			this.cache = new Cache();
+		}
+	}
+
+	async getZxyAttempt(
+		z: number,
+		x: number,
+		y: number,
+		signal?: AbortSignal
+	): Promise<ArrayBuffer | undefined> {
+		const tile_id = zxyToTileId(z, x, y);
+		const header = await this.cache.getHeader(this.source);
+
+		let d_o = header.rootDirectoryOffset;
+		let d_l = header.rootDirectoryLength;
+		for (let depth = 0; depth < 5; depth++) {
+			const directory = await this.cache.getDirectory(
+				this.source,
+				d_o,
+				d_l,
+				header
+			);
+			const entry = findTile(directory, tile_id);
+			if (entry) {
+				if (entry.runLength > 0) {
+					const [data, new_etag] = await this.source.getBytes(
+						header.tileDataOffset + entry.offset,
+						entry.length,
+						signal
+					);
+					if (header.etag && header.etag !== new_etag) {
+						throw new VersionMismatch(header.etag);
+					}
+					return tryDecompress(data, header.tileCompression);
+				} else {
+					d_o = header.leafDirectoryOffset + entry.offset;
+					d_l = entry.length;
+				}
+			} else {
+				return undefined;
+			}
+		}
+		throw Error("Maximum directory depth exceeded");
+	}
+
+	async getZxy(
+		z: number,
+		x: number,
+		y: number,
+		signal?: AbortSignal
+	): Promise<ArrayBuffer | undefined> {
+		try {
+			return await this.getZxyAttempt(z, x, y, signal);
+		} catch (e) {
+			if (e instanceof VersionMismatch) {
+				this.cache.invalidate(this.source);
+				return await this.getZxyAttempt(z, x, y, signal);
+			} else {
+				throw e;
+			}
+		}
+	}
+
+	async getMetadataAttempt(): Promise<any> {
+		const header = await this.cache.getHeader(this.source);
+		const [data, new_etag] = await this.source.getBytes(
+			header.jsonMetadataOffset,
+			header.jsonMetadataLength
+		);
+		if (header.etag && header.etag !== new_etag) {
+			throw new VersionMismatch(header.etag);
+		}
+		const decompressed = tryDecompress(data, header.internalCompression);
+		const dec = new TextDecoder("utf-8");
+		return JSON.parse(dec.decode(decompressed));
+	}
+
+	async getMetadata(): Promise<any> {
+		try {
+			return await this.getMetadataAttempt();
+		} catch (e) {
+			if (e instanceof VersionMismatch) {
+				this.cache.invalidate(this.source);
+				return await this.getMetadataAttempt();
+			} else {
+				throw e;
+			}
+		}
+	}
+}