Merge pull request #46 from protomaps/refactor-js-cache

modularize JS client to take non-HTTP sources, caching behavior
2026-02-04 10:51:07 +00:00 · 2022-05-30 12:43:06 +08:00
parent f31b8b2bca 5d7fd9bdf7
commit c186560738
2 changed files with 149 additions and 109 deletions
--- a/js/index.test.ts
+++ b/js/index.test.ts
@@ -12,7 +12,7 @@ import {
 } from "./index";

 test("stub data", (assertion) => {
-  let data = createDirectory([
+  let dataview = createDirectory([
    { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
    {
      z: 14,
@@ -23,7 +23,6 @@ test("stub data", (assertion) => {
      is_dir: false,
    },
  ]);
-  let dataview = new DataView(data);

  var z_raw = dataview.getUint8(17 + 0);
  var x = getUint24(dataview, 17 + 1);
@@ -36,7 +35,7 @@ test("stub data", (assertion) => {
 });

 test("get entry", (assertion) => {
-  let data = createDirectory([
+  let view = createDirectory([
    { z: 5, x: 1000, y: 2000, offset: 1000, length: 2000, is_dir: false },
    {
      z: 14,
@@ -47,7 +46,6 @@ test("get entry", (assertion) => {
      is_dir: false,
    },
  ]);
-  let view = new DataView(data);
  let entry = queryTile(view, 14, 16383, 16383);
  assertion.ok(entry!.z === 14);
  assertion.ok(entry!.x === 16383);
@@ -59,7 +57,7 @@ test("get entry", (assertion) => {
 });

 test("get leafdir", (assertion) => {
-  let data = createDirectory([
+  let view = createDirectory([
    {
      z: 14,
      x: 16383,
@@ -69,7 +67,6 @@ test("get leafdir", (assertion) => {
      is_dir: true,
    },
  ]);
-  let view = new DataView(data);
  let entry = queryLeafdir(view, 14, 16383, 16383);
  assertion.ok(entry!.z === 14);
  assertion.ok(entry!.x === 16383);
@@ -81,7 +78,7 @@ test("get leafdir", (assertion) => {
 });

 test("derive the leaf level", (assertion) => {
-  let data = createDirectory([
+  let view = createDirectory([
    {
      z: 6,
      x: 3,
@@ -91,10 +88,9 @@ test("derive the leaf level", (assertion) => {
      is_dir: true,
    },
  ]);
-  let view = new DataView(data);
  let level = queryLeafLevel(view);
  assertion.ok(level === 6);
-  data = createDirectory([
+  view = createDirectory([
    {
      z: 6,
      x: 3,
@@ -104,13 +100,12 @@ test("derive the leaf level", (assertion) => {
      is_dir: false,
    },
  ]);
-  view = new DataView(data);
  level = queryLeafLevel(view);
  assertion.ok(level === null);
 });

 test("convert spec v1 directory to spec v2 directory", (assertion) => {
-  let data = createDirectory([
+  let view = createDirectory([
    {
      z: 7,
      x: 3,
@@ -136,7 +131,6 @@ test("convert spec v1 directory to spec v2 directory", (assertion) => {
      is_dir: false,
    },
  ]);
-  let view = new DataView(data);
  let entry = queryLeafdir(view, 7, 3, 3);
  assertion.ok(entry!.offset === 3);
  entry = queryTile(view, 6, 2, 2);
--- a/js/index.ts
+++ b/js/index.ts
@@ -30,9 +30,8 @@ interface Header {

 interface Root {
  header: Header;
-  buffer: ArrayBuffer;
  dir: DataView;
-  // etag: string | null;
+  view: DataView;
 }

 export interface Entry {
@@ -44,11 +43,6 @@ export interface Entry {
  is_dir: boolean;
 }

-interface CachedLeaf {
-  lastUsed: number;
-  buffer: Promise<ArrayBuffer>;
-}
-
 const compare = (
  tz: number,
  tx: number,
@@ -158,7 +152,7 @@ export const parseEntry = (dataview: DataView, i: number): Entry => {
  };
 };

-export const sortDir = (dataview: DataView): ArrayBuffer => {
+export const sortDir = (dataview: DataView): DataView => {
  const entries: Entry[] = [];
  for (let i = 0; i < dataview.byteLength / 17; i++) {
    entries.push(parseEntry(dataview, i));
@@ -166,7 +160,7 @@ export const sortDir = (dataview: DataView): ArrayBuffer => {
  return createDirectory(entries);
 };

-export const createDirectory = (entries: Entry[]): ArrayBuffer => {
+export const createDirectory = (entries: Entry[]): DataView => {
  entries.sort(entrySort);

  const buffer = new ArrayBuffer(17 * entries.length);
@@ -197,7 +191,7 @@ export const createDirectory = (entries: Entry[]): ArrayBuffer => {
    arr[i * 17 + 15] = (entry.length >> 16) & 0xff;
    arr[i * 17 + 16] = (entry.length >> 24) & 0xff;
  }
-  return buffer;
+  return new DataView(arr.buffer, arr.byteOffset, arr.length);
 };

 export const deriveLeaf = (root: Root, tile: Zxy): Zxy | null => {
@@ -226,28 +220,49 @@ export const parseHeader = (dataview: DataView): Header => {
  };
 };

-export class PMTiles {
-  root: Promise<Root> | null;
-  url: string;
-  leaves: Map<number, CachedLeaf>;
-  maxLeaves: number;
-
-  constructor(url: string, maxLeaves = 64) {
-    this.root = null;
-    this.url = url;
-    this.leaves = new Map<number, CachedLeaf>();
-    this.maxLeaves = maxLeaves;
+export interface Source {
+  getBytes: (offset: number, length: number) => Promise<DataView>;
+  getKey: () => string;
 }

-  async fetchRoot(url: string): Promise<Root> {
+class FileSource implements Source {
+  file: File;
+
+  constructor(file: File) {
+    this.file = file;
+  }
+
+  getKey() {
+    return this.file.name;
+  }
+
+  async getBytes(offset: number, length: number) {
+    let blob = this.file.slice(offset, offset + length);
+    let a = await blob.arrayBuffer();
+    return new DataView(a);
+  }
+}
+
+class FetchSource implements Source {
+  url: string;
+
+  constructor(url: string) {
+    this.url = url;
+  }
+
+  getKey() {
+    return this.url;
+  }
+
+  async getBytes(offset: number, length: number) {
    const controller = new AbortController();
    const signal = controller.signal;
-    const resp = await fetch(url, {
+    const resp = await fetch(this.url, {
      signal: signal,
-      headers: { Range: "bytes=0-511999" },
+      headers: { Range: "bytes=" + offset + "-" + (offset + length - 1) },
    });
    const contentLength = resp.headers.get("Content-Length");
-    if (!contentLength || +contentLength !== 512000) {
+    if (!contentLength || +contentLength !== length) {
      console.error(
        "Content-Length mismatch indicates byte serving not supported; aborting."
      );
@@ -255,36 +270,103 @@ export class PMTiles {
    }

    const a = await resp.arrayBuffer();
-    const header = parseHeader(new DataView(a, 0, 10));
+    return new DataView(a);
+  }
+}
+
+interface CacheEntry {
+  lastUsed: number;
+  buffer: Promise<DataView>;
+}
+
+class LRUCacheSource implements Source {
+  entries: Map<number, CacheEntry>;
+  maxEntries: number;
+  source: Source;
+
+  constructor(source: Source, maxEntries: number) {
+    this.source = source;
+    this.entries = new Map<number, CacheEntry>();
+    this.maxEntries = maxEntries;
+  }
+
+  getKey = () => {
+    return this.source.getKey();
+  };
+
+  async getBytes(offset: number, length: number) {
+    let val = this.entries.get(offset);
+    if (val) {
+      val.lastUsed = performance.now();
+      return val.buffer;
+    }
+
+    let promise = this.source.getBytes(offset, length);
+
+    this.entries.set(offset, {
+      lastUsed: performance.now(),
+      buffer: promise,
+    });
+
+    if (this.entries.size > this.maxEntries) {
+      let minUsed = Infinity;
+      let minKey = undefined;
+      this.entries.forEach((val, key) => {
+        if (val.lastUsed < minUsed) {
+          minUsed = val.lastUsed;
+          minKey = key;
+        }
+      });
+      if (minKey) this.entries.delete(minKey);
+    }
+
+    return promise;
+  }
+}
+
+export class PMTiles {
+  source: Source;
+
+  constructor(source: any, maxLeaves = 64) {
+    if (typeof source === "string") {
+      this.source = new LRUCacheSource(new FetchSource(source), maxLeaves);
+    } else {
+      this.source = source;
+    }
+  }
+
+  async fetchRoot(): Promise<Root> {
+    const v = await this.source.getBytes(0, 512000);
+    const header = parseHeader(new DataView(v.buffer, v.byteOffset, 10));

    let root_dir = new DataView(
-      a,
+      v.buffer,
      10 + header.json_size,
      17 * header.root_entries
    );
    if (header.version === 1) {
      console.warn("Sorting pmtiles v1 directory");
-      root_dir = new DataView(sortDir(root_dir));
+      root_dir = sortDir(root_dir);
    }

    return {
-      buffer: a,
      header: header,
+      view: v,
      dir: root_dir,
    };
  }

-  async getRoot(): Promise<Root> {
-    if (this.root) return this.root;
-    this.root = this.fetchRoot(this.url);
-    return this.root;
-  }
-
  async metadata(): Promise<any> {
-    const root = await this.getRoot();
+    const root = await this.fetchRoot();
    const dec = new TextDecoder("utf-8");
    const result = JSON.parse(
-      dec.decode(new DataView(root.buffer, 10, root.header.json_size))
+      dec.decode(
+        new DataView(
+          root.view.buffer,
+          root.view.byteOffset + 10,
+          root.header.json_size
+        )
+      )
    );
    if (result.compression) {
      console.warn(
@@ -309,56 +391,35 @@ export class PMTiles {
    return result;
  }

-  async fetchLeafdir(version: number, entry: Entry): Promise<ArrayBuffer> {
-    const resp = await fetch(this.url, {
-      headers: {
-        Range:
-          "bytes=" + entry.offset + "-" + (entry.offset + entry.length - 1),
-      },
-    });
-    let buf = await resp.arrayBuffer();
+  async fetchLeafdir(version: number, entry: Entry): Promise<DataView> {
+    let buf = await this.source.getBytes(entry.offset, entry.length);

    if (version === 1) {
      console.warn("Sorting pmtiles v1 directory.");
-      buf = sortDir(new DataView(buf));
+      buf = sortDir(buf);
    }

    return buf;
  }

-  async getLeafdir(version: number, entry: Entry): Promise<ArrayBuffer> {
-    const leaf = this.leaves.get(entry.offset);
-    if (leaf) return await leaf.buffer;
-
-    const buf = this.fetchLeafdir(version, entry);
-
-    this.leaves.set(entry.offset, {
-      lastUsed: performance.now(),
-      buffer: buf,
-    });
-    if (this.leaves.size > this.maxLeaves) {
-      let minUsed = Infinity;
-      let minKey = undefined;
-      this.leaves.forEach((val, key) => {
-        if (val.lastUsed < minUsed) {
-          minUsed = val.lastUsed;
-          minKey = key;
-        }
-      });
-      if (minKey) this.leaves.delete(minKey);
-    }
-    return await buf;
+  async getLeafdir(version: number, entry: Entry): Promise<DataView> {
+    return this.fetchLeafdir(version, entry);
  }

  async getZxy(z: number, x: number, y: number): Promise<Entry | null> {
-    const root = await this.getRoot();
-    const entry = queryTile(root.dir, z, x, y);
+    const root = await this.fetchRoot();
+    const entry = queryTile(
+      new DataView(root.dir.buffer, root.dir.byteOffset, root.dir.byteLength),
+      z,
+      x,
+      y
+    );
    if (entry) return entry;

    const leafcoords = deriveLeaf(root, { z: z, x: x, y: y });
    if (leafcoords) {
      const leafdir_entry = queryLeafdir(
-        root.dir,
+        new DataView(root.dir.buffer, root.dir.byteOffset, root.dir.byteLength),
        leafcoords.z,
        leafcoords.x,
        leafcoords.y
@@ -368,7 +429,12 @@ export class PMTiles {
          root.header.version,
          leafdir_entry
        );
-        return queryTile(new DataView(leafdir), z, x, y);
+        return queryTile(
+          new DataView(leafdir.buffer, leafdir.byteOffset, leafdir.byteLength),
+          z,
+          x,
+          y
+        );
      }
    }
    return null;
@@ -387,19 +453,9 @@ export const leafletLayer = (source: PMTiles, options: any) => {
        tile.cancel = () => {
          controller.abort();
        };
-        fetch(source.url, {
-          signal: signal,
-          headers: {
-            Range:
-              "bytes=" +
-              result.offset +
-              "-" +
-              (result.offset + result.length - 1),
-          },
-        })
-          .then((resp) => {
-            return resp.arrayBuffer();
-          })
+
+        source.source
+          .getBytes(result.offset, result.length)
          .then((buf) => {
            const blob = new Blob([buf], { type: "image/png" });
            const imageUrl = window.URL.createObjectURL(blob);
@@ -444,7 +500,7 @@ export class ProtocolCache {
  }

  add(p: PMTiles) {
-    this.tiles.set(p.url, p);
+    this.tiles.set(p.source.getKey(), p);
  }

  get(url: string) {
@@ -468,18 +524,8 @@ export class ProtocolCache {

    instance.getZxy(+z, +x, +y).then((val) => {
      if (val) {
-        const headers = {
-          Range: "bytes=" + val.offset + "-" + (val.offset + val.length - 1),
-        };
-        const controller = new AbortController();
-        const signal = controller.signal;
-        cancel = () => {
-          controller.abort();
-        };
-        fetch(pmtiles_url, { signal: signal, headers: headers })
-          .then((resp) => {
-            return resp.arrayBuffer();
-          })
+        instance!.source
+          .getBytes(val.offset, val.length)
          .then((arr) => {
            callback(null, arr, null, null);
          })