mirrors/PMTiles
1
0
Fork 0
mirror of https://github.com/protomaps/PMTiles.git synced 2026-02-04 02:41:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

JS v3 reader with caching and ETag support [#59, #53, #41, #24, #4]

Browse Source 
The v3 module is not exported yet; specifics of header design subject to change.
This commit is contained in:
Brandon Liu
2022-09-23 09:03:33 +08:00
parent fbb0f4614c
commit f23ca5823b
8 changed files with 910 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
js/empty.pmtiles Normal file
View File

252
js/index.test.ts
View File

@@ -1,252 +1,4 @@
import { test } from "zora";
import {
unshift,
getUint24,
getUint48,
queryLeafdir,
queryLeafLevel,
queryTile,
parseEntry,
Entry,
createDirectory,
} from "./index";
import './v2.test';
import './v3.test';
import {
Entry as EntryV3,
zxyToTileId,
tileIdToZxy,
findTile,
readVarint,
} from "./v3";
test("stub data", (assertion) => {
let dataview = createDirectory([
{ z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
{
z: 14,
x: 16383,
y: 16383,
offset: 999999,
length: 999,
is_dir: false,
},
]);
var z_raw = dataview.getUint8(17 + 0);
var x = getUint24(dataview, 17 + 1);
var y = getUint24(dataview, 17 + 4);
var offset = getUint48(dataview, 17 + 7);
var length = dataview.getUint32(17 + 13, true);
assertion.ok(z_raw === 14);
assertion.ok(x === 16383);
assertion.ok(y === 16383);
});
test("get entry", (assertion) => {
let view = createDirectory([
{ z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
{
z: 14,
x: 16383,
y: 16383,
offset: 999999,
length: 999,
is_dir: false,
},
]);
let entry = queryTile(view, 14, 16383, 16383);
assertion.ok(entry!.z === 14);
assertion.ok(entry!.x === 16383);
assertion.ok(entry!.y === 16383);
assertion.ok(entry!.offset === 999999);
assertion.ok(entry!.length === 999);
assertion.ok(entry!.is_dir === false);
assertion.ok(queryLeafdir(view, 14, 16383, 16383) === null);
});
test("get leafdir", (assertion) => {
let view = createDirectory([
{
z: 14,
x: 16383,
y: 16383,
offset: 999999,
length: 999,
is_dir: true,
},
]);
let entry = queryLeafdir(view, 14, 16383, 16383);
assertion.ok(entry!.z === 14);
assertion.ok(entry!.x === 16383);
assertion.ok(entry!.y === 16383);
assertion.ok(entry!.offset === 999999);
assertion.ok(entry!.length === 999);
assertion.ok(entry!.is_dir === true);
assertion.ok(queryTile(view, 14, 16383, 16383) === null);
});
test("derive the leaf level", (assertion) => {
let view = createDirectory([
{
z: 6,
x: 3,
y: 3,
offset: 0,
length: 0,
is_dir: true,
},
]);
let level = queryLeafLevel(view);
assertion.ok(level === 6);
view = createDirectory([
{
z: 6,
x: 3,
y: 3,
offset: 0,
length: 0,
is_dir: false,
},
]);
level = queryLeafLevel(view);
assertion.ok(level === null);
});
test("convert spec v1 directory to spec v2 directory", (assertion) => {
let view = createDirectory([
{
z: 7,
x: 3,
y: 3,
offset: 3,
length: 3,
is_dir: true,
},
{
z: 6,
x: 2,
y: 2,
offset: 2,
length: 2,
is_dir: false,
},
{
z: 6,
x: 2,
y: 1,
offset: 1,
length: 1,
is_dir: false,
},
]);
let entry = queryLeafdir(view, 7, 3, 3);
assertion.ok(entry!.offset === 3);
entry = queryTile(view, 6, 2, 2);
assertion.ok(entry!.offset === 2);
entry = queryTile(view, 6, 2, 1);
assertion.ok(entry!.offset === 1);
entry = parseEntry(view, 0);
assertion.ok(entry!.offset === 1);
entry = parseEntry(view, 1);
assertion.ok(entry!.offset === 2);
entry = parseEntry(view, 2);
assertion.ok(entry!.offset === 3);
});
test("varint", (assertion) => {
let b: BufferPosition = {
buf: new Uint8Array([0, 1, 127, 0xe5, 0x8e, 0x26]),
pos: 0,
};
assertion.eq(readVarint(b), 0);
assertion.eq(readVarint(b), 1);
assertion.eq(readVarint(b), 127);
assertion.eq(readVarint(b), 624485);
b = {
buf: new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]),
pos: 0,
};
assertion.eq(readVarint(b), 9007199254740991);
});
test("zxy to tile id", (assertion) => {
assertion.eq(zxyToTileId(0, 0, 0), 0);
assertion.eq(zxyToTileId(1, 0, 0), 1);
assertion.eq(zxyToTileId(1, 0, 1), 2);
assertion.eq(zxyToTileId(1, 1, 1), 3);
assertion.eq(zxyToTileId(1, 1, 0), 4);
assertion.eq(zxyToTileId(2, 0, 0), 5);
});
test("tile id to zxy", (assertion) => {
assertion.eq(tileIdToZxy(0), [0, 0, 0]);
assertion.eq(tileIdToZxy(1), [1, 0, 0]);
assertion.eq(tileIdToZxy(2), [1, 0, 1]);
assertion.eq(tileIdToZxy(3), [1, 1, 1]);
assertion.eq(tileIdToZxy(4), [1, 1, 0]);
assertion.eq(tileIdToZxy(5), [2, 0, 0]);
});
test("a lot of tiles", (assertion) => {
for (var z = 0; z < 9; z++) {
for (var x = 0; x < 1 << z; x++) {
for (var y = 0; y < 1 << z; y++) {
let result = tileIdToZxy(zxyToTileId(z, x, y));
if (result[0] !== z || result[1] !== x || result[2] !== y) {
assertion.fail("roundtrip failed");
}
}
}
}
});
test("tile search for first entry == id", (assertion) => {
let entries: EntryV3[] = [];
assertion.eq(findTile(entries, 101), null);
});
test("tile search for first entry == id", (assertion) => {
let entries: EntryV3[] = [
{ tileId: 100, offset: 1, length: 1, runLength: 1 },
];
let entry = findTile(entries, 100)!;
assertion.eq(entry.offset, 1);
assertion.eq(entry.length, 1);
assertion.eq(findTile(entries, 101), null);
});
test("tile search for first entry == id", (assertion) => {
let entries: EntryV3[] = [
{ tileId: 100, offset: 1, length: 1, runLength: 2 },
];
let entry = findTile(entries, 101)!;
assertion.eq(entry.offset, 1);
assertion.eq(entry.length, 1);
entries = [
{ tileId: 100, offset: 1, length: 1, runLength: 1 },
{ tileId: 150, offset: 2, length: 2, runLength: 2 },
];
entry = findTile(entries, 151)!;
assertion.eq(entry.offset, 2);
assertion.eq(entry.length, 2);
entries = [
{ tileId: 50, offset: 1, length: 1, runLength: 2 },
{ tileId: 100, offset: 2, length: 2, runLength: 1 },
{ tileId: 150, offset: 3, length: 3, runLength: 1 },
];
entry = findTile(entries, 51)!;
assertion.eq(entry.offset, 1);
assertion.eq(entry.length, 1);
});
test("leaf search", (assertion) => {
let entries: EntryV3[] = [
{ tileId: 100, offset: 1, length: 1, runLength: 0 },
];
let entry = findTile(entries, 150);
assertion.eq(entry!.offset, 1);
assertion.eq(entry!.length, 1);
});

1
js/invalid.pmtiles Normal file
View File

@@ -0,0 +1 @@
This is an invalid tile archive, a test case to make sure that the code throws an error, but it needs to be the minimum size to pass the first test

BIN
js/test_fixture_1.pmtiles Normal file
View File

Binary file not shown.

BIN
js/test_fixture_2.pmtiles Normal file
View File

Binary file not shown.

148
js/v2.test.ts Normal file
View File

@@ -0,0 +1,148 @@
import { test } from "zora";
import {
unshift,
getUint24,
getUint48,
queryLeafdir,
queryLeafLevel,
queryTile,
parseEntry,
Entry,
createDirectory,
} from "./index";
test("stub data", (assertion) => {
let dataview = createDirectory([
{ z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
{
z: 14,
x: 16383,
y: 16383,
offset: 999999,
length: 999,
is_dir: false,
},
]);
var z_raw = dataview.getUint8(17 + 0);
var x = getUint24(dataview, 17 + 1);
var y = getUint24(dataview, 17 + 4);
var offset = getUint48(dataview, 17 + 7);
var length = dataview.getUint32(17 + 13, true);
assertion.ok(z_raw === 14);
assertion.ok(x === 16383);
assertion.ok(y === 16383);
});
test("get entry", (assertion) => {
let view = createDirectory([
{ z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
{
z: 14,
x: 16383,
y: 16383,
offset: 999999,
length: 999,
is_dir: false,
},
]);
let entry = queryTile(view, 14, 16383, 16383);
assertion.ok(entry!.z === 14);
assertion.ok(entry!.x === 16383);
assertion.ok(entry!.y === 16383);
assertion.ok(entry!.offset === 999999);
assertion.ok(entry!.length === 999);
assertion.ok(entry!.is_dir === false);
assertion.ok(queryLeafdir(view, 14, 16383, 16383) === null);
});
test("get leafdir", (assertion) => {
let view = createDirectory([
{
z: 14,
x: 16383,
y: 16383,
offset: 999999,
length: 999,
is_dir: true,
},
]);
let entry = queryLeafdir(view, 14, 16383, 16383);
assertion.ok(entry!.z === 14);
assertion.ok(entry!.x === 16383);
assertion.ok(entry!.y === 16383);
assertion.ok(entry!.offset === 999999);
assertion.ok(entry!.length === 999);
assertion.ok(entry!.is_dir === true);
assertion.ok(queryTile(view, 14, 16383, 16383) === null);
});
test("derive the leaf level", (assertion) => {
let view = createDirectory([
{
z: 6,
x: 3,
y: 3,
offset: 0,
length: 0,
is_dir: true,
},
]);
let level = queryLeafLevel(view);
assertion.ok(level === 6);
view = createDirectory([
{
z: 6,
x: 3,
y: 3,
offset: 0,
length: 0,
is_dir: false,
},
]);
level = queryLeafLevel(view);
assertion.ok(level === null);
});
test("convert spec v1 directory to spec v2 directory", (assertion) => {
let view = createDirectory([
{
z: 7,
x: 3,
y: 3,
offset: 3,
length: 3,
is_dir: true,
},
{
z: 6,
x: 2,
y: 2,
offset: 2,
length: 2,
is_dir: false,
},
{
z: 6,
x: 2,
y: 1,
offset: 1,
length: 1,
is_dir: false,
},
]);
let entry = queryLeafdir(view, 7, 3, 3);
assertion.ok(entry!.offset === 3);
entry = queryTile(view, 6, 2, 2);
assertion.ok(entry!.offset === 2);
entry = queryTile(view, 6, 2, 1);
assertion.ok(entry!.offset === 1);
entry = parseEntry(view, 0);
assertion.ok(entry!.offset === 1);
entry = parseEntry(view, 1);
assertion.ok(entry!.offset === 2);
entry = parseEntry(view, 2);
assertion.ok(entry!.offset === 3);
});

295
js/v3.test.ts Normal file
View File

@@ -0,0 +1,295 @@
import { test } from "zora";
// tests run in node, for convenience
// we don't need to typecheck all of it
// @ts-ignore
import fs from "fs";
import {
Entry,
zxyToTileId,
tileIdToZxy,
findTile,
readVarint,
Cache,
BufferPosition,
Source,
VersionMismatch,
PMTiles,
} from "./v3";
test("varint", (assertion) => {
let b: BufferPosition = {
buf: new Uint8Array([0, 1, 127, 0xe5, 0x8e, 0x26]),
pos: 0,
};
assertion.eq(readVarint(b), 0);
assertion.eq(readVarint(b), 1);
assertion.eq(readVarint(b), 127);
assertion.eq(readVarint(b), 624485);
b = {
buf: new Uint8Array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f]),
pos: 0,
};
assertion.eq(readVarint(b), 9007199254740991);
});
test("zxy to tile id", (assertion) => {
assertion.eq(zxyToTileId(0, 0, 0), 0);
assertion.eq(zxyToTileId(1, 0, 0), 1);
assertion.eq(zxyToTileId(1, 0, 1), 2);
assertion.eq(zxyToTileId(1, 1, 1), 3);
assertion.eq(zxyToTileId(1, 1, 0), 4);
assertion.eq(zxyToTileId(2, 0, 0), 5);
});
test("tile id to zxy", (assertion) => {
assertion.eq(tileIdToZxy(0), [0, 0, 0]);
assertion.eq(tileIdToZxy(1), [1, 0, 0]);
assertion.eq(tileIdToZxy(2), [1, 0, 1]);
assertion.eq(tileIdToZxy(3), [1, 1, 1]);
assertion.eq(tileIdToZxy(4), [1, 1, 0]);
assertion.eq(tileIdToZxy(5), [2, 0, 0]);
});
test("a lot of tiles", (assertion) => {
for (let z = 0; z < 9; z++) {
for (let x = 0; x < 1 << z; x++) {
for (let y = 0; y < 1 << z; y++) {
const result = tileIdToZxy(zxyToTileId(z, x, y));
if (result[0] !== z || result[1] !== x || result[2] !== y) {
assertion.fail("roundtrip failed");
}
}
}
}
});
test("tile search for missing entry", (assertion) => {
const entries: Entry[] = [];
assertion.eq(findTile(entries, 101), null);
});
test("tile search for first entry == id", (assertion) => {
const entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 1 }];
const entry = findTile(entries, 100)!;
assertion.eq(entry.offset, 1);
assertion.eq(entry.length, 1);
assertion.eq(findTile(entries, 101), null);
});
test("tile search with multiple tile entries", (assertion) => {
let entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 2 }];
let entry = findTile(entries, 101)!;
assertion.eq(entry.offset, 1);
assertion.eq(entry.length, 1);
entries = [
{ tileId: 100, offset: 1, length: 1, runLength: 1 },
{ tileId: 150, offset: 2, length: 2, runLength: 2 },
];
entry = findTile(entries, 151)!;
assertion.eq(entry.offset, 2);
assertion.eq(entry.length, 2);
entries = [
{ tileId: 50, offset: 1, length: 1, runLength: 2 },
{ tileId: 100, offset: 2, length: 2, runLength: 1 },
{ tileId: 150, offset: 3, length: 3, runLength: 1 },
];
entry = findTile(entries, 51)!;
assertion.eq(entry.offset, 1);
assertion.eq(entry.length, 1);
});
test("leaf search", (assertion) => {
const entries: Entry[] = [{ tileId: 100, offset: 1, length: 1, runLength: 0 }];
const entry = findTile(entries, 150);
assertion.eq(entry!.offset, 1);
assertion.eq(entry!.length, 1);
});
// inefficient method only for testing
class TestNodeFileSource implements Source {
buffer: ArrayBuffer;
path: string;
key: string;
etag?: string;
constructor(path: string, key: string) {
this.path = path;
this.buffer = fs.readFileSync(path);
this.key = key;
}
getKey() {
return this.key;
}
replaceData(path: string) {
this.path = path;
this.buffer = fs.readFileSync(path);
}
async getBytes(
offset: number,
length: number
): Promise<[ArrayBuffer, string?]> {
const slice = new Uint8Array(this.buffer.slice(offset, offset + length))
.buffer;
return [slice, this.etag];
}
}
// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_1.pmtiles
test("cache getHeader", async (assertion) => {
const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
const cache = new Cache();
const header = await cache.getHeader(source);
assertion.eq(header.rootDirectoryOffset, 122);
assertion.eq(header.rootDirectoryLength, 25);
assertion.eq(header.jsonMetadataOffset, 147);
assertion.eq(header.jsonMetadataLength, 239);
assertion.eq(header.leafDirectoryOffset, 0);
assertion.eq(header.leafDirectoryLength, 0);
assertion.eq(header.tileDataOffset, 386);
assertion.eq(header.tileDataLength, 69);
assertion.eq(header.numAddressedTiles, 1);
assertion.eq(header.numTileEntries, 1);
assertion.eq(header.numTileContents, 1);
assertion.eq(header.clustered, false);
assertion.eq(header.internalCompression, 1);
assertion.eq(header.tileCompression, 1);
assertion.eq(header.tileType, 1);
assertion.eq(header.minZoom, 0);
assertion.eq(header.maxZoom, 0);
assertion.eq(header.minLon, 0);
assertion.eq(header.minLat, 0);
// assertion.eq(header.maxLon,1); // TODO fix me
assertion.eq(header.maxLat, 1);
});
test("cache check against empty", async (assertion) => {
const source = new TestNodeFileSource("empty.pmtiles", "1");
const cache = new Cache();
try {
await cache.getHeader(source);
assertion.fail("Should have thrown");
} catch (e) {
assertion.ok(e instanceof Error);
}
});
test("cache check magic number", async (assertion) => {
const source = new TestNodeFileSource("invalid.pmtiles", "1");
const cache = new Cache();
try {
await cache.getHeader(source);
assertion.fail("Should have thrown");
} catch (e) {
console.log(e);
assertion.ok(e instanceof Error);
}
});
test("cache getDirectory", async (assertion) => {
const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
let cache = new Cache(6400, false);
let header = await cache.getHeader(source);
assertion.eq(cache.cache.size, 1);
cache = new Cache(6400, true);
header = await cache.getHeader(source);
// prepopulates the root directory
assertion.eq(cache.cache.size, 2);
const directory = await cache.getDirectory(
source,
header.rootDirectoryOffset,
header.rootDirectoryLength,
header
);
assertion.eq(directory.length, 1);
assertion.eq(directory[0].tileId, 0);
assertion.eq(directory[0].offset, 0);
assertion.eq(directory[0].length, 69);
assertion.eq(directory[0].runLength, 1);
for (const v of cache.cache.values()) {
assertion.ok(v.lastUsed > 0);
assertion.ok(v.size > 0);
}
});
test("multiple sources in a single cache", async (assertion) => {
const cache = new Cache();
const source1 = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
const source2 = new TestNodeFileSource("test_fixture_1.pmtiles", "2");
await cache.getHeader(source1);
assertion.eq(cache.cache.size, 2);
await cache.getHeader(source2);
assertion.eq(cache.cache.size, 4);
});
test("etags are part of key", async (assertion) => {
const cache = new Cache(6400, false);
const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
source.etag = "etag_1";
let header = await cache.getHeader(source);
assertion.eq(header.etag, "etag_1");
source.etag = "etag_2";
try {
await cache.getDirectory(
source,
header.rootDirectoryOffset,
header.rootDirectoryLength,
header
);
assertion.fail("Should have thrown");
} catch (e) {
assertion.ok(e instanceof VersionMismatch);
}
cache.invalidate(source);
header = await cache.getHeader(source);
assertion.ok(
await cache.getDirectory(
source,
header.rootDirectoryOffset,
header.rootDirectoryLength,
header
)
);
});
test("cache pruning by byte size", async (assertion) => {
const cache = new Cache(1000, false);
cache.cache.set("0", { lastUsed: 0, data: Promise.resolve([]), size: 400 });
cache.cache.set("1", { lastUsed: 1, data: Promise.resolve([]), size: 200 });
cache.cache.set("2", { lastUsed: 2, data: Promise.resolve([]), size: 900 });
cache.sizeBytes = 900 + 200 + 400;
cache.prune();
assertion.eq(cache.cache.size, 1);
assertion.ok(cache.cache.get("2"));
});
test("pmtiles get metadata", async (assertion) => {
const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
const p = new PMTiles(source);
const metadata = await p.getMetadata();
assertion.ok(metadata.name);
});
test("pmtiles handle retries", async (assertion) => {
const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
source.etag = "1";
const p = new PMTiles(source);
const metadata = await p.getMetadata();
assertion.ok(metadata.name);
source.etag = "2";
source.replaceData("test_fixture_2.pmtiles");
assertion.ok(await p.getZxy(0, 0, 0));
});

478
js/v3.ts
View File

@@ -1,4 +1,6 @@
interface BufferPosition {
import { decompressSync } from "fflate";
export interface BufferPosition {
buf: Uint8Array;
pos: number;
}
@@ -8,9 +10,8 @@ function toNum(low: number, high: number): number {
}
function readVarintRemainder(l: number, p: BufferPosition): number {
var buf = p.buf,
h,
b;
const buf = p.buf;
let h, b;
b = buf[p.pos++];
h = (b & 0x70) >> 4;
if (b < 0x80) return toNum(l, h);
@@ -33,9 +34,8 @@ function readVarintRemainder(l: number, p: BufferPosition): number {
}
export function readVarint(p: BufferPosition): number {
var buf = p.buf,
val,
b;
const buf = p.buf;
let val, b;
b = buf[p.pos++];
val = b & 0x7f;
@@ -61,18 +61,18 @@ function rotate(n: number, xy: number[], rx: number, ry: number): void {
xy[0] = n - 1 - xy[0];
xy[1] = n - 1 - xy[1];
}
let t = xy[0];
const t = xy[0];
xy[0] = xy[1];
xy[1] = t;
}
}
function idOnLevel(z: number, pos: number): [number, number, number] {
let n = 1 << z;
const n = 1 << z;
let rx = pos;
let ry = pos;
let t = pos;
let xy = [0, 0];
const xy = [0, 0];
let s = 1;
while (s < n) {
rx = 1 & ((t / 2) >> 0);
@@ -93,11 +93,11 @@ export function zxyToTileId(z: number, x: number, y: number): number {
acc += (0x1 << tz) * (0x1 << tz);
tz++;
}
let n = 1 << z;
const n = 1 << z;
let rx = 0;
let ry = 0;
let d = 0;
let xy = [x, y];
const xy = [x, y];
let s = (n / 2) >> 0;
while (s > 0) {
rx = (xy[0] & s) > 0 ? 1 : 0;
@@ -112,8 +112,8 @@ export function zxyToTileId(z: number, x: number, y: number): number {
export function tileIdToZxy(i: number): [number, number, number] {
let acc = 0;
let z = 0;
while (true) {
let num_tiles = (0x1 << z) * (0x1 << z);
for (;;) {
const num_tiles = (0x1 << z) * (0x1 << z);
if (acc + num_tiles > i) {
return idOnLevel(z, i - acc);
}
@@ -129,6 +129,63 @@ export interface Entry {
runLength: number;
}
const ENTRY_SIZE_BYTES = 32;
enum Compression {
None = 0,
Gzip = 1,
Brotli = 2,
Zstd = 3,
}
function tryDecompress(buf: ArrayBuffer, compression: Compression) {
if (compression === Compression.None) {
return buf;
} else if (compression === Compression.Gzip) {
return decompressSync(new Uint8Array(buf));
} else {
throw Error("Compression method not supported");
}
}
enum TileType {
Unknown = 0,
Mvt = 1,
Png = 2,
Jpeg = 3,
Webp = 4,
}
const HEADER_SIZE_BYTES = 122;
export interface Header {
rootDirectoryOffset: number;
rootDirectoryLength: number;
jsonMetadataOffset: number;
jsonMetadataLength: number;
leafDirectoryOffset: number;
leafDirectoryLength: number;
tileDataOffset: number;
tileDataLength: number;
numAddressedTiles: number;
numTileEntries: number;
numTileContents: number;
clustered: boolean;
internalCompression: Compression;
tileCompression: Compression;
tileType: TileType;
minZoom: number;
maxZoom: number;
minLon: number;
minLat: number;
maxLon: number;
maxLat: number;
centerZoom: number;
centerLon: number;
centerLat: number;
etag?: string;
}
export function findTile(entries: Entry[], tileId: number): Entry | null {
let m = 0;
let n = entries.length - 1;
@@ -155,3 +212,396 @@ export function findTile(entries: Entry[], tileId: number): Entry | null {
}
return null;
}
// In the future this may need to change
// to support ReadableStream to pass to native DecompressionStream API
export interface Source {
getBytes: (
offset: number,
length: number,
signal?: AbortSignal
) => Promise<[ArrayBuffer, string?]>;
getKey: () => string;
}
export class FileAPISource implements Source {
file: File;
constructor(file: File) {
this.file = file;
}
getKey() {
return this.file.name;
}
async getBytes(
offset: number,
length: number,
): Promise<[ArrayBuffer, undefined]> {
const blob = this.file.slice(offset, offset + length);
const a = await blob.arrayBuffer();
return [a, undefined];
}
}
export class FetchSource implements Source {
url: string;
constructor(url: string) {
this.url = url;
}
getKey() {
return this.url;
}
async getBytes(
offset: number,
length: number,
signal?: AbortSignal
): Promise<[ArrayBuffer, string?]> {
let controller;
if (!signal) {
// TODO check this works or assert 206
controller = new AbortController();
signal = controller.signal;
}
const resp = await fetch(this.url, {
signal: signal,
headers: { Range: "bytes=" + offset + "-" + (offset + length - 1) },
});
const contentLength = resp.headers.get("Content-Length");
if (!contentLength || +contentLength !== length) {
console.error(
"Content-Length mismatch indicates byte serving not supported; aborting."
);
if (controller) controller.abort();
}
const a = await resp.arrayBuffer();
return [a, resp.headers.get("ETag") || undefined];
}
}
interface CacheEntry {
lastUsed: number;
size: number; // 0 if the promise has not resolved
data: Promise<Header | Entry[] | ArrayBuffer>;
}
export function bytesToHeader(bytes: ArrayBuffer, etag?: string): Header {
const v = new DataView(bytes);
if (v.getUint16(0, true) !== 0x4d50) {
throw new Error("Wrong magic number for PMTiles archive");
}
return {
rootDirectoryOffset: Number(v.getBigUint64(3, true)),
rootDirectoryLength: Number(v.getBigUint64(11, true)),
jsonMetadataOffset: Number(v.getBigUint64(19, true)),
jsonMetadataLength: Number(v.getBigUint64(27, true)),
leafDirectoryOffset: Number(v.getBigUint64(35, true)),
leafDirectoryLength: Number(v.getBigUint64(43, true)),
tileDataOffset: Number(v.getBigUint64(51, true)),
tileDataLength: Number(v.getBigUint64(59, true)),
numAddressedTiles: Number(v.getBigUint64(67, true)),
numTileEntries: Number(v.getBigUint64(75, true)),
numTileContents: Number(v.getBigUint64(83, true)),
clustered: v.getUint8(91) === 1,
internalCompression: v.getUint8(92),
tileCompression: v.getUint8(93),
tileType: v.getUint8(94),
minZoom: v.getUint8(95),
maxZoom: v.getUint8(96),
minLon: v.getFloat32(97, true),
minLat: v.getFloat32(101, true),
maxLon: v.getFloat32(105, true),
maxLat: v.getFloat32(109, true),
centerZoom: v.getUint8(113),
centerLon: v.getFloat32(114, true),
centerLat: v.getFloat32(118, true),
etag: etag,
};
}
function deserializeIndex(buffer: ArrayBuffer): Entry[] {
const p = { buf: new Uint8Array(buffer), pos: 0 };
const numEntries = readVarint(p);
const entries: Entry[] = [];
let lastId = 0;
for (let i = 0; i < numEntries; i++) {
const v = readVarint(p);
entries.push({ tileId: lastId + v, offset: 0, length: 0, runLength: 1 });
lastId += v;
}
for (let i = 0; i < numEntries; i++) {
entries[i].runLength = readVarint(p);
}
for (let i = 0; i < numEntries; i++) {
entries[i].length = readVarint(p);
}
for (let i = 0; i < numEntries; i++) {
const v = readVarint(p);
if (v === 0 && i > 0) {
entries[i].offset = entries[i - 1].offset + entries[i - 1].length;
} else {
entries[i].offset = v - 1;
}
}
return entries;
}
export class VersionMismatch extends Error {}
// a "dumb" bag of bytes.
// only caches headers and directories
// deduplicates simultaneous responses
// (estimates) the maximum size of the cache.
export class Cache {
cache: Map<string, CacheEntry>;
sizeBytes: number;
maxSizeBytes: number;
counter: number;
prefetch: boolean;
constructor(maxSizeBytes = 64000000, prefetch = true) {
this.cache = new Map<string, CacheEntry>();
this.sizeBytes = 0;
this.maxSizeBytes = maxSizeBytes;
this.counter = 1;
this.prefetch = prefetch;
}
async getHeader(source: Source): Promise<Header> {
const cacheKey = source.getKey();
if (this.cache.has(cacheKey)) {
const data = await this.cache.get(cacheKey)!.data;
return data as Header;
}
const p = new Promise<Header>((resolve, reject) => {
source
.getBytes(0, 16384)
.then((resp) => {
if (this.cache.has(cacheKey)) {
this.cache.get(cacheKey)!.size = HEADER_SIZE_BYTES;
this.sizeBytes += HEADER_SIZE_BYTES;
}
const headerData = resp[0].slice(0, HEADER_SIZE_BYTES);
if (headerData.byteLength !== HEADER_SIZE_BYTES) {
throw new Error("Invalid PMTiles header");
}
const header = bytesToHeader(headerData, resp[1]);
// optimistically set the root directory
// TODO check root bounds
if (this.prefetch) {
const rootDirData = resp[0].slice(
header.rootDirectoryOffset,
header.rootDirectoryOffset + header.rootDirectoryLength
);
const dirKey =
source.getKey() +
"|" +
(header.etag || "") +
"|" +
header.rootDirectoryOffset +
"|" +
header.rootDirectoryLength;
const rootDir = deserializeIndex(
tryDecompress(rootDirData, header.internalCompression)
);
this.cache.set(dirKey, {
lastUsed: this.counter++,
data: Promise.resolve(rootDir),
size: ENTRY_SIZE_BYTES * rootDir.length,
});
}
resolve(header);
this.prune();
})
.catch((e) => {
reject(e);
});
});
this.cache.set(cacheKey, { lastUsed: this.counter++, data: p, size: 0 });
return p;
}
async getDirectory(
source: Source,
offset: number,
length: number,
header: Header
): Promise<Entry[]> {
const cacheKey =
source.getKey() + "|" + (header.etag || "") + "|" + offset + "|" + length;
if (this.cache.has(cacheKey)) {
this.cache.get(cacheKey)!.lastUsed = this.counter++;
const data = await this.cache.get(cacheKey)!.data;
return data as Entry[];
}
const p = new Promise<Entry[]>((resolve, reject) => {
source
.getBytes(offset, length)
.then((resp) => {
if (header.etag && header.etag !== resp[1]) {
throw new VersionMismatch(header.etag);
}
const data = tryDecompress(resp[0], header.internalCompression);
const directory = deserializeIndex(data);
if (directory.length === 0) {
return reject(new Error("Empty directory is invalid"));
}
resolve(directory);
if (this.cache.has(cacheKey)) {
this.cache.get(cacheKey)!.size = ENTRY_SIZE_BYTES * directory.length;
this.sizeBytes += ENTRY_SIZE_BYTES * directory.length;
}
this.prune();
})
.catch((e) => {
reject(e);
});
});
this.cache.set(cacheKey, { lastUsed: this.counter++, data: p, size: 0 });
return p;
}
prune() {
while (this.sizeBytes > this.maxSizeBytes) {
let minUsed = Infinity;
let minKey = undefined;
this.cache.forEach((cache_entry: CacheEntry, key: string) => {
if (cache_entry.lastUsed < minUsed) {
minUsed = cache_entry.lastUsed;
minKey = key;
}
});
if (minKey) {
this.sizeBytes -= this.cache.get(minKey)!.size;
this.cache.delete(minKey);
}
}
}
invalidate(source: Source) {
this.cache.delete(source.getKey());
}
}
export class PMTiles {
source: Source;
cache: Cache;
constructor(source: Source, cache?: Cache) {
this.source = source;
if (cache) {
this.cache = cache;
} else {
this.cache = new Cache();
}
}
async getZxyAttempt(
z: number,
x: number,
y: number,
signal?: AbortSignal
): Promise<ArrayBuffer | undefined> {
const tile_id = zxyToTileId(z, x, y);
const header = await this.cache.getHeader(this.source);
let d_o = header.rootDirectoryOffset;
let d_l = header.rootDirectoryLength;
for (let depth = 0; depth < 5; depth++) {
const directory = await this.cache.getDirectory(
this.source,
d_o,
d_l,
header
);
const entry = findTile(directory, tile_id);
if (entry) {
if (entry.runLength > 0) {
const [data, new_etag] = await this.source.getBytes(
header.tileDataOffset + entry.offset,
entry.length,
signal
);
if (header.etag && header.etag !== new_etag) {
throw new VersionMismatch(header.etag);
}
return tryDecompress(data, header.tileCompression);
} else {
d_o = header.leafDirectoryOffset + entry.offset;
d_l = entry.length;
}
} else {
return undefined;
}
}
throw Error("Maximum directory depth exceeded");
}
async getZxy(
z: number,
x: number,
y: number,
signal?: AbortSignal
): Promise<ArrayBuffer | undefined> {
try {
return await this.getZxyAttempt(z, x, y, signal);
} catch (e) {
if (e instanceof VersionMismatch) {
this.cache.invalidate(this.source);
return await this.getZxyAttempt(z, x, y, signal);
} else {
throw e;
}
}
}
async getMetadataAttempt(): Promise<any> {
const header = await this.cache.getHeader(this.source);
const [data, new_etag] = await this.source.getBytes(
header.jsonMetadataOffset,
header.jsonMetadataLength
);
if (header.etag && header.etag !== new_etag) {
throw new VersionMismatch(header.etag);
}
const decompressed = tryDecompress(data, header.internalCompression);
const dec = new TextDecoder("utf-8");
return JSON.parse(dec.decode(decompressed));
}
async getMetadata(): Promise<any> {
try {
return await this.getMetadataAttempt();
} catch (e) {
if (e instanceof VersionMismatch) {
this.cache.invalidate(this.source);
return await this.getMetadataAttempt();
} else {
throw e;
}
}
}
}