From b2aa3164fbe695b02642683ca864af3cf561ad15 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Mon, 22 May 2023 10:49:16 +0800 Subject: [PATCH 1/2] use compression-stream-polyfill for faster, native gzip decompression [#97, #167] --- js/index.ts | 15 +++++++++------ js/package-lock.json | 43 ++++++++++++++++++++++++++++++++++++------- js/package.json | 3 ++- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/js/index.ts b/js/index.ts index dec1ea4..99cebdf 100644 --- a/js/index.ts +++ b/js/index.ts @@ -1,4 +1,4 @@ -import { decompressSync } from "fflate"; +import { makeDecompressionStream } from "compression-streams-polyfill/ponyfill"; import v2 from "./v2"; export * from "./adapters"; @@ -155,14 +155,17 @@ type DecompressFunc = ( compression: Compression ) => Promise; -async function fflateDecompress( +async function defaultDecompress( buf: ArrayBuffer, compression: Compression ): Promise { if (compression === Compression.None || compression === Compression.Unknown) { return buf; } else if (compression === Compression.Gzip) { - return decompressSync(new Uint8Array(buf)); + let stream = new Response(buf).body!; + const decompressionStream = makeDecompressionStream(TransformStream); + let result = stream.pipeThrough(new decompressionStream("gzip")); + return new Response(result).arrayBuffer(); } else { throw Error("Compression method not supported"); } @@ -547,7 +550,7 @@ export class ResolvedValueCache { constructor( maxCacheEntries = 100, prefetch = true, - decompress: DecompressFunc = fflateDecompress + decompress: DecompressFunc = defaultDecompress ) { this.cache = new Map(); this.maxCacheEntries = maxCacheEntries; @@ -682,7 +685,7 @@ export class SharedPromiseCache { constructor( maxCacheEntries = 100, prefetch = true, - decompress: DecompressFunc = fflateDecompress + decompress: DecompressFunc = defaultDecompress ) { this.cache = new Map(); this.maxCacheEntries = maxCacheEntries; @@ -824,7 +827,7 @@ export class PMTiles { if (decompress) { this.decompress = decompress; } else { - this.decompress = fflateDecompress; + this.decompress = defaultDecompress; } if (cache) { this.cache = cache; diff --git a/js/package-lock.json b/js/package-lock.json index 966302a..08a1a23 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -9,7 +9,8 @@ "version": "2.7.2", "license": "BSD-3-Clause", "dependencies": { - "fflate": "^0.7.3" + "compression-streams-polyfill": "^0.1.3", + "fflate": "^0.8.0" }, "devDependencies": { "@types/node": "^18.11.9", @@ -63,6 +64,19 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/compression-streams-polyfill": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/compression-streams-polyfill/-/compression-streams-polyfill-0.1.3.tgz", + "integrity": "sha512-FOvusJGoWTKRb9JNMtCc42C81SdyJhJws/lbExADOkH1R8KpdzY4yFO6953lW5Vw498pnGXoyosr807KTr8BAA==", + "dependencies": { + "fflate": "^0.7.4" + } + }, + "node_modules/compression-streams-polyfill/node_modules/fflate": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.4.tgz", + "integrity": "sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw==" + }, "node_modules/esbuild": { "version": "0.15.15", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.15.tgz", @@ -437,9 +451,9 @@ } }, "node_modules/fflate": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.3.tgz", - "integrity": "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw==" + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.0.tgz", + "integrity": "sha512-FAdS4qMuFjsJj6XHbBaZeXOgaypXp8iw/Tpyuq/w3XA41jjLHT8NPA+n7czH/DDhdncq0nAyDZmPeWXh2qmdIg==" }, "node_modules/prettier": { "version": "2.8.4", @@ -522,6 +536,21 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "compression-streams-polyfill": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/compression-streams-polyfill/-/compression-streams-polyfill-0.1.3.tgz", + "integrity": "sha512-FOvusJGoWTKRb9JNMtCc42C81SdyJhJws/lbExADOkH1R8KpdzY4yFO6953lW5Vw498pnGXoyosr807KTr8BAA==", + "requires": { + "fflate": "^0.7.4" + }, + "dependencies": { + "fflate": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.4.tgz", + "integrity": "sha512-5u2V/CDW15QM1XbbgS+0DfPxVB+jUKhWEKuuFuHncbk3tEEqzmoXL+2KyOFuKGqOnmdIy0/davWF1CkuwtibCw==" + } + } + }, "esbuild": { "version": "0.15.15", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.15.tgz", @@ -703,9 +732,9 @@ "optional": true }, "fflate": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.3.tgz", - "integrity": "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw==" + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.0.tgz", + "integrity": "sha512-FAdS4qMuFjsJj6XHbBaZeXOgaypXp8iw/Tpyuq/w3XA41jjLHT8NPA+n7czH/DDhdncq0nAyDZmPeWXh2qmdIg==" }, "prettier": { "version": "2.8.4", diff --git a/js/package.json b/js/package.json index 3abdf27..cbdb3fa 100644 --- a/js/package.json +++ b/js/package.json @@ -32,6 +32,7 @@ "typescript": "^4.5.5" }, "dependencies": { - "fflate": "^0.7.3" + "compression-streams-polyfill": "^0.1.3", + "fflate": "^0.8.0" } } From db7ed668bb75c6105868a4f5cabaa7b9fe9ac5d8 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Wed, 24 May 2023 13:05:34 +0800 Subject: [PATCH 2/2] js 2.8.0-beta.0 --- js/package-lock.json | 4 ++-- js/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/js/package-lock.json b/js/package-lock.json index 08a1a23..62e69bb 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "pmtiles", - "version": "2.7.2", + "version": "2.8.0-beta.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "pmtiles", - "version": "2.7.2", + "version": "2.8.0-beta.0", "license": "BSD-3-Clause", "dependencies": { "compression-streams-polyfill": "^0.1.3", diff --git a/js/package.json b/js/package.json index cbdb3fa..c767a71 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "pmtiles", - "version": "2.7.2", + "version": "2.8.0-beta.0", "description": "PMTiles archive decoder for browsers", "main": "dist/index.cjs", "module": "dist/index.mjs",