Remove prefetch boolean from JS sources - no reason to ever not use i… (#353)

* Remove prefetch boolean from JS sources - no reason to ever not use it

* JS optimization for header fetches on etag invalidation [#90]

* If promises are shared between requests, only the first invalidation makes the header request.
* If promises are not shared, simply delete the key.

* JS 3.0.3 [#90]

* update CHANGELOG

js/CHANGELOG.md

@@ -1,4 +1,14 @@
-3.0.1-alpha.2
+3.0.3
+
+* Deprecate `prefetch`-ing the first 16 kb as an option, always true
+* Optimize invalidation when etag changes when promises are shared between tile requests. [#90]
+
+3.0.2
+
+* Fix name of script includes (IIFE) name from `index.js` to `pmtiles.js`
+* Fix name of ES6 module from `index.mjs` to `index.js`, which fixes bundlers detecting TypeScript types (index.d.ts)
+
+3.0.1
 
 * FileApiSource renamed to FileSource
 * package.json defines **ES6 module only** (no CommonJS), fixing issues related to named imports [#317, #248]

js/examples/leaflet.html

@@ -4,7 +4,7 @@
         <meta charset="utf-8"/>
         <link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css" />
         <script src="https://unpkg.com/leaflet@1.9.0/dist/leaflet.js"></script>
-        <script src="https://unpkg.com/pmtiles@3.0.2/dist/pmtiles.js"></script>
+        <script src="https://unpkg.com/pmtiles@3.0.3/dist/pmtiles.js"></script>
         <style>
             body, #map {
                 height:100vh;

js/examples/maplibre.html

@@ -4,7 +4,7 @@
         <meta charset="utf-8"/>
         <link rel="stylesheet" href="https://unpkg.com/maplibre-gl@3.3.1/dist/maplibre-gl.css" crossorigin="anonymous">
         <script src="https://unpkg.com/maplibre-gl@3.3.1/dist/maplibre-gl.js" crossorigin="anonymous"></script>
-        <script src="https://unpkg.com/pmtiles@3.0.2/dist/pmtiles.js"></script>
+        <script src="https://unpkg.com/pmtiles@3.0.3/dist/pmtiles.js"></script>
         <style>
             body {
                 margin: 0;

js/examples/maplibre_raster_dem.html

@@ -4,7 +4,7 @@
         <meta charset="utf-8"/>
         <link rel="stylesheet" href="https://unpkg.com/maplibre-gl@3.3.1/dist/maplibre-gl.css" crossorigin="anonymous">
         <script src="https://unpkg.com/maplibre-gl@3.3.1/dist/maplibre-gl.js" crossorigin="anonymous"></script>
-        <script src="https://unpkg.com/pmtiles@3.0.2/dist/pmtiles.js"></script>
+        <script src="https://unpkg.com/pmtiles@3.0.3/dist/pmtiles.js"></script>
         <style>
             body {
                 margin: 0;

js/index.ts

@@ -509,8 +509,7 @@ export interface Cache {
 
 async function getHeaderAndRoot(
   source: Source,
-  decompress: DecompressFunc,
+  decompress: DecompressFunc
-  prefetch: boolean
 ): Promise<[Header, [string, number, Entry[] | ArrayBuffer]?]> {
   const resp = await source.getBytes(0, 16384);
 
@@ -530,22 +529,18 @@ async function getHeaderAndRoot(
 
   // optimistically set the root directory
   // TODO check root bounds
-  if (prefetch) {
+  const rootDirData = resp.data.slice(
-    const rootDirData = resp.data.slice(
+    header.rootDirectoryOffset,
-      header.rootDirectoryOffset,
+    header.rootDirectoryOffset + header.rootDirectoryLength
-      header.rootDirectoryOffset + header.rootDirectoryLength
+  );
-    );
+  const dirKey = `${source.getKey()}|${header.etag || ""}|${
-    const dirKey = `${source.getKey()}|${header.etag || ""}|${
+    header.rootDirectoryOffset
-      header.rootDirectoryOffset
+  }|${header.rootDirectoryLength}`;
-    }|${header.rootDirectoryLength}`;
 
-    const rootDir = deserializeIndex(
+  const rootDir = deserializeIndex(
-      await decompress(rootDirData, header.internalCompression)
+    await decompress(rootDirData, header.internalCompression)
-    );
+  );
-    return [header, [dirKey, rootDir.length, rootDir]];
+  return [header, [dirKey, rootDir.length, rootDir]];
-  }
-
-  return [header, undefined];
 }
 
 async function getDirectory(
@@ -574,22 +569,20 @@ export class ResolvedValueCache {
   cache: Map<string, ResolvedValue>;
   maxCacheEntries: number;
   counter: number;
-  prefetch: boolean;
   decompress: DecompressFunc;
 
   constructor(
     maxCacheEntries = 100,
-    prefetch = true,
+    prefetch = true, // deprecated
     decompress: DecompressFunc = defaultDecompress
   ) {
     this.cache = new Map<string, ResolvedValue>();
     this.maxCacheEntries = maxCacheEntries;
     this.counter = 1;
-    this.prefetch = prefetch;
     this.decompress = decompress;
   }
 
-  async getHeader(source: Source, currentEtag?: string): Promise<Header> {
+  async getHeader(source: Source): Promise<Header> {
     const cacheKey = source.getKey();
     const cacheValue = this.cache.get(cacheKey);
     if (cacheValue) {
@@ -598,7 +591,7 @@ export class ResolvedValueCache {
       return data as Header;
     }
 
-    const res = await getHeaderAndRoot(source, this.decompress, this.prefetch);
+    const res = await getHeaderAndRoot(source, this.decompress);
     if (res[1]) {
       this.cache.set(res[1][0], {
         lastUsed: this.counter++,
@@ -690,7 +683,6 @@ export class ResolvedValueCache {
 
   async invalidate(source: Source) {
     this.cache.delete(source.getKey());
-    await this.getHeader(source);
   }
 }
 
@@ -705,24 +697,24 @@ interface SharedPromiseCacheValue {
 // (estimates) the maximum size of the cache.
 export class SharedPromiseCache {
   cache: Map<string, SharedPromiseCacheValue>;
+  invalidations: Map<string, Promise<void>>;
   maxCacheEntries: number;
   counter: number;
-  prefetch: boolean;
   decompress: DecompressFunc;
 
   constructor(
     maxCacheEntries = 100,
-    prefetch = true,
+    prefetch = true, // deprecated
     decompress: DecompressFunc = defaultDecompress
   ) {
     this.cache = new Map<string, SharedPromiseCacheValue>();
+    this.invalidations = new Map<string, Promise<void>>();
     this.maxCacheEntries = maxCacheEntries;
     this.counter = 1;
-    this.prefetch = prefetch;
     this.decompress = decompress;
   }
 
-  async getHeader(source: Source, currentEtag?: string): Promise<Header> {
+  async getHeader(source: Source): Promise<Header> {
     const cacheKey = source.getKey();
     const cacheValue = this.cache.get(cacheKey);
     if (cacheValue) {
@@ -732,7 +724,7 @@ export class SharedPromiseCache {
     }
 
     const p = new Promise<Header>((resolve, reject) => {
-      getHeaderAndRoot(source, this.decompress, this.prefetch)
+      getHeaderAndRoot(source, this.decompress)
         .then((res) => {
           if (res[1]) {
             this.cache.set(res[1][0], {
@@ -832,8 +824,22 @@ export class SharedPromiseCache {
   }
 
   async invalidate(source: Source) {
+    const key = source.getKey();
+    if (this.invalidations.get(key)) {
+      return await this.invalidations.get(key);
+    }
     this.cache.delete(source.getKey());
-    await this.getHeader(source);
+    const p = new Promise<void>((resolve, reject) => {
+      this.getHeader(source)
+        .then((h) => {
+          resolve();
+          this.invalidations.delete(key);
+        })
+        .catch((e) => {
+          reject(e);
+        });
+    });
+    this.invalidations.set(key, p);
   }
 }
 

js/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "pmtiles",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "pmtiles",
-      "version": "3.0.2",
+      "version": "3.0.3",
       "license": "BSD-3-Clause",
       "dependencies": {
         "@types/leaflet": "^1.9.8",

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "description": "PMTiles archive decoder for browsers",
   "type": "module",
   "exports": "./dist/index.js",

js/test/v3.test.ts

@@ -289,12 +289,8 @@ test("cache getDirectory", async () => {
     "1"
   );
 
-  let cache = new SharedPromiseCache(6400, false);
+  const cache = new SharedPromiseCache(6400);
-  let header = await cache.getHeader(source);
+  const header = await cache.getHeader(source);
-  assert.strictEqual(cache.cache.size, 1);
-
-  cache = new SharedPromiseCache(6400, true);
-  header = await cache.getHeader(source);
 
   // prepopulates the root directory
   assert.strictEqual(cache.cache.size, 2);
@@ -358,7 +354,7 @@ test("weak etags", async () => {
 // handle DigitalOcean case returning 200 instead of 206
 
 test("cache pruning by byte size", async () => {
-  const cache = new SharedPromiseCache(2, false);
+  const cache = new SharedPromiseCache(2);
   cache.cache.set("0", { lastUsed: 0, data: Promise.resolve([]) });
   cache.cache.set("1", { lastUsed: 1, data: Promise.resolve([]) });
   cache.cache.set("2", { lastUsed: 2, data: Promise.resolve([]) });