v3 getZxy response is object with optional cacheControl, expires [#24]

This commit is contained in:
Brandon Liu
2022-10-03 13:57:59 +08:00
parent 87bf3994d0
commit 92c1c8bbdf
2 changed files with 52 additions and 26 deletions

View File

@@ -14,6 +14,7 @@ import {
Cache, Cache,
BufferPosition, BufferPosition,
Source, Source,
Response,
VersionMismatch, VersionMismatch,
PMTiles, PMTiles,
} from "./v3"; } from "./v3";
@@ -134,10 +135,10 @@ class TestNodeFileSource implements Source {
async getBytes( async getBytes(
offset: number, offset: number,
length: number length: number
): Promise<[ArrayBuffer, string?]> { ): Promise<Response> {
const slice = new Uint8Array(this.buffer.slice(offset, offset + length)) const slice = new Uint8Array(this.buffer.slice(offset, offset + length))
.buffer; .buffer;
return [slice, this.etag]; return {data:slice, etag:this.etag};
} }
} }
@@ -283,6 +284,7 @@ test("pmtiles get metadata", async (assertion) => {
assertion.ok(metadata.name); assertion.ok(metadata.name);
}); });
// echo '{"type":"Polygon","coordinates":[[[0,0],[0,1],[1,0],[0,0]]]}' | ./tippecanoe -zg -o test_fixture_2.pmtiles
test("pmtiles handle retries", async (assertion) => { test("pmtiles handle retries", async (assertion) => {
const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1"); const source = new TestNodeFileSource("test_fixture_1.pmtiles", "1");
source.etag = "1"; source.etag = "1";

View File

@@ -213,6 +213,13 @@ export function findTile(entries: Entry[], tileId: number): Entry | null {
return null; return null;
} }
export interface Response {
data: ArrayBuffer;
etag?: string;
expires?: string;
cacheControl?: string;
}
// In the future this may need to change // In the future this may need to change
// to support ReadableStream to pass to native DecompressionStream API // to support ReadableStream to pass to native DecompressionStream API
export interface Source { export interface Source {
@@ -220,7 +227,7 @@ export interface Source {
offset: number, offset: number,
length: number, length: number,
signal?: AbortSignal signal?: AbortSignal
) => Promise<[ArrayBuffer, string?]>; ) => Promise<Response>;
getKey: () => string; getKey: () => string;
} }
@@ -236,13 +243,10 @@ export class FileAPISource implements Source {
return this.file.name; return this.file.name;
} }
async getBytes( async getBytes(offset: number, length: number): Promise<Response> {
offset: number,
length: number
): Promise<[ArrayBuffer, undefined]> {
const blob = this.file.slice(offset, offset + length); const blob = this.file.slice(offset, offset + length);
const a = await blob.arrayBuffer(); const a = await blob.arrayBuffer();
return [a, undefined]; return { data: a };
} }
} }
@@ -261,7 +265,7 @@ export class FetchSource implements Source {
offset: number, offset: number,
length: number, length: number,
signal?: AbortSignal signal?: AbortSignal
): Promise<[ArrayBuffer, string?]> { ): Promise<Response> {
let controller; let controller;
if (!signal) { if (!signal) {
// TODO check this works or assert 206 // TODO check this works or assert 206
@@ -282,7 +286,12 @@ export class FetchSource implements Source {
} }
const a = await resp.arrayBuffer(); const a = await resp.arrayBuffer();
return [a, resp.headers.get("ETag") || undefined]; return {
data: a,
etag: resp.headers.get("ETag") || undefined,
cacheControl: resp.headers.get("Cache-Control") || undefined,
expires: resp.headers.get("Expires") || undefined,
};
} }
} }
@@ -396,16 +405,16 @@ export class Cache {
this.sizeBytes += HEADER_SIZE_BYTES; this.sizeBytes += HEADER_SIZE_BYTES;
} }
const headerData = resp[0].slice(0, HEADER_SIZE_BYTES); const headerData = resp.data.slice(0, HEADER_SIZE_BYTES);
if (headerData.byteLength !== HEADER_SIZE_BYTES) { if (headerData.byteLength !== HEADER_SIZE_BYTES) {
throw new Error("Invalid PMTiles header"); throw new Error("Invalid PMTiles header");
} }
const header = bytesToHeader(headerData, resp[1]); const header = bytesToHeader(headerData, resp.etag);
// optimistically set the root directory // optimistically set the root directory
// TODO check root bounds // TODO check root bounds
if (this.prefetch) { if (this.prefetch) {
const rootDirData = resp[0].slice( const rootDirData = resp.data.slice(
header.rootDirectoryOffset, header.rootDirectoryOffset,
header.rootDirectoryOffset + header.rootDirectoryLength header.rootDirectoryOffset + header.rootDirectoryLength
); );
@@ -458,11 +467,11 @@ export class Cache {
source source
.getBytes(offset, length) .getBytes(offset, length)
.then((resp) => { .then((resp) => {
if (header.etag && header.etag !== resp[1]) { if (header.etag && header.etag !== resp.etag) {
throw new VersionMismatch(header.etag); throw new VersionMismatch("ETag mismatch: " + header.etag);
} }
const data = tryDecompress(resp[0], header.internalCompression); const data = tryDecompress(resp.data, header.internalCompression);
const directory = deserializeIndex(data); const directory = deserializeIndex(data);
if (directory.length === 0) { if (directory.length === 0) {
return reject(new Error("Empty directory is invalid")); return reject(new Error("Empty directory is invalid"));
@@ -520,12 +529,23 @@ export class PMTiles {
} }
} }
async root_entries() {
const header = await this.cache.getHeader(this.source);
let d_o = header.rootDirectoryOffset;
let d_l = header.rootDirectoryLength;
return await this.cache.getDirectory(this.source, d_o, d_l, header);
}
async getHeader() {
return await this.cache.getHeader(this.source);
}
async getZxyAttempt( async getZxyAttempt(
z: number, z: number,
x: number, x: number,
y: number, y: number,
signal?: AbortSignal signal?: AbortSignal
): Promise<ArrayBuffer | undefined> { ): Promise<Response | undefined> {
const tile_id = zxyToTileId(z, x, y); const tile_id = zxyToTileId(z, x, y);
const header = await this.cache.getHeader(this.source); const header = await this.cache.getHeader(this.source);
@@ -545,15 +565,19 @@ export class PMTiles {
const entry = findTile(directory, tile_id); const entry = findTile(directory, tile_id);
if (entry) { if (entry) {
if (entry.runLength > 0) { if (entry.runLength > 0) {
const [data, new_etag] = await this.source.getBytes( const resp = await this.source.getBytes(
header.tileDataOffset + entry.offset, header.tileDataOffset + entry.offset,
entry.length, entry.length,
signal signal
); );
if (header.etag && header.etag !== new_etag) { if (header.etag && header.etag !== resp.etag) {
throw new VersionMismatch(header.etag); throw new VersionMismatch("ETag mismatch: " + header.etag);
} }
return tryDecompress(data, header.tileCompression); return {
data: tryDecompress(resp.data, header.tileCompression),
cacheControl: resp.cacheControl,
expires: resp.expires,
};
} else { } else {
d_o = header.leafDirectoryOffset + entry.offset; d_o = header.leafDirectoryOffset + entry.offset;
d_l = entry.length; d_l = entry.length;
@@ -570,7 +594,7 @@ export class PMTiles {
x: number, x: number,
y: number, y: number,
signal?: AbortSignal signal?: AbortSignal
): Promise<ArrayBuffer | undefined> { ): Promise<Response | undefined> {
try { try {
return await this.getZxyAttempt(z, x, y, signal); return await this.getZxyAttempt(z, x, y, signal);
} catch (e) { } catch (e) {
@@ -585,14 +609,14 @@ export class PMTiles {
async getMetadataAttempt(): Promise<any> { async getMetadataAttempt(): Promise<any> {
const header = await this.cache.getHeader(this.source); const header = await this.cache.getHeader(this.source);
const [data, new_etag] = await this.source.getBytes( const resp = await this.source.getBytes(
header.jsonMetadataOffset, header.jsonMetadataOffset,
header.jsonMetadataLength header.jsonMetadataLength
); );
if (header.etag && header.etag !== new_etag) { if (header.etag && header.etag !== resp.etag) {
throw new VersionMismatch(header.etag); throw new VersionMismatch("Etag mismatch: " + header.etag);
} }
const decompressed = tryDecompress(data, header.internalCompression); const decompressed = tryDecompress(resp.data, header.internalCompression);
const dec = new TextDecoder("utf-8"); const dec = new TextDecoder("utf-8");
return JSON.parse(dec.decode(decompressed)); return JSON.parse(dec.decode(decompressed));
} }