From dc4cde0d1d3c6a1f85d0589385bcc44b0da6e28f Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Sun, 30 Jan 2022 20:03:08 +0800 Subject: [PATCH] source files for faster directory fetches [#26] --- js/package-lock.json | 143 ++++++++++++++++++++++++++++++++++++++++++- js/package.json | 10 ++- js/pmtiles.test.ts | 83 +++++++++++++++++++++++++ js/pmtiles.ts | 48 +++++++++++++++ js/tsconfig.json | 12 ++++ 5 files changed, 291 insertions(+), 5 deletions(-) create mode 100644 js/pmtiles.test.ts create mode 100644 js/pmtiles.ts create mode 100644 js/tsconfig.json diff --git a/js/package-lock.json b/js/package-lock.json index b73c093..d12840d 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,14 +1,153 @@ { "name": "pmtiles", - "version": "0.3.1", - "lockfileVersion": 1, + "version": "0.5.0", + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "pmtiles", + "version": "0.5.0", + "license": "BSD-3-Clause", + "devDependencies": { + "esbuild": "^0.11.14", + "esbuild-runner": "^2.2.1", + "typescript": "^4.5.5", + "zora": "^5.0.2" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.11.14", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.14.tgz", + "integrity": "sha512-ejheEPkqhq5y0LM9rG9e+3yDihPtqeeE4NZmG7VQiSGJ3CjO4HvPOHmhhttSksfSztjLAGo2+0/zSIvlqj4JOQ==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + } + }, + "node_modules/esbuild-runner": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/esbuild-runner/-/esbuild-runner-2.2.1.tgz", + "integrity": "sha512-VP0VfJJZiZ3cKzdOH59ZceDxx/GzBKra7tiGM8MfFMLv6CR1/cpsvtQ3IsJI3pz7HyeYxtbPyecj3fHwR+3XcQ==", + "dev": true, + "dependencies": { + "source-map-support": "0.5.19", + "tslib": "2.3.1" + }, + "bin": { + "esr": "bin/esr.js" + }, + "peerDependencies": { + "esbuild": "*" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/tslib": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", + "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==", + "dev": true + }, + "node_modules/typescript": { + "version": "4.5.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz", + "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/zora": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/zora/-/zora-5.0.2.tgz", + "integrity": "sha512-HZtoEtYm9iJrym0nvel84NmmmOZyvS9I6q27kO1zAA5W4vY87LnRkokRGd5Qm6nrv0+YhGlr/i5VfVcS+VEK4g==", + "dev": true + } + }, "dependencies": { + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, "esbuild": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.14.tgz", "integrity": "sha512-ejheEPkqhq5y0LM9rG9e+3yDihPtqeeE4NZmG7VQiSGJ3CjO4HvPOHmhhttSksfSztjLAGo2+0/zSIvlqj4JOQ==", "dev": true + }, + "esbuild-runner": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/esbuild-runner/-/esbuild-runner-2.2.1.tgz", + "integrity": "sha512-VP0VfJJZiZ3cKzdOH59ZceDxx/GzBKra7tiGM8MfFMLv6CR1/cpsvtQ3IsJI3pz7HyeYxtbPyecj3fHwR+3XcQ==", + "dev": true, + "requires": { + "source-map-support": "0.5.19", + "tslib": "2.3.1" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "tslib": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", + "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==", + "dev": true + }, + "typescript": { + "version": "4.5.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz", + "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==", + "dev": true + }, + "zora": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/zora/-/zora-5.0.2.tgz", + "integrity": "sha512-HZtoEtYm9iJrym0nvel84NmmmOZyvS9I6q27kO1zAA5W4vY87LnRkokRGd5Qm6nrv0+YhGlr/i5VfVcS+VEK4g==", + "dev": true } } } diff --git a/js/package.json b/js/package.json index a64733c..edb261e 100644 --- a/js/package.json +++ b/js/package.json @@ -11,15 +11,19 @@ "index.js" ], "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", "build-iife": "esbuild index.src.mjs --outfile=index.js --target=es2015 --global-name=pmtiles --format=iife", "build-esm": "esbuild index.src.mjs --outfile=index.mjs --target=es6 --global-name=pmtiles --format=esm", - "build": "npm run build-iife && npm run build-esm" + "build": "npm run build-iife && npm run build-esm", + "test": "node -r esbuild-runner/register pmtiles.test.ts", + "tsc": "tsc --noEmit --watch" }, "homepage": "https://github.com/protomaps/pmtiles", "author": "Brandon Liu", "license": "BSD-3-Clause", "devDependencies": { - "esbuild": "^0.11.14" + "esbuild": "^0.11.14", + "esbuild-runner": "^2.2.1", + "typescript": "^4.5.5", + "zora": "^5.0.2" } } diff --git a/js/pmtiles.test.ts b/js/pmtiles.test.ts new file mode 100644 index 0000000..a19e3ea --- /dev/null +++ b/js/pmtiles.test.ts @@ -0,0 +1,83 @@ +import { test } from "zora"; +import { unshift, getUint24, getUint48, getLeafdir, getTile } from "./pmtiles"; + +interface StubEntry { + z:number; + x:number; + y:number; + offset:number; + length:number; + is_dir:boolean; +} + +const stubData = (entries:StubEntry[]) => { + // sort entries + let buffer = new ArrayBuffer(17*entries.length); + let arr = new Uint8Array(buffer); + for (var i = 0; i < entries.length; i++) { + var entry = entries[i]; + var z = entry.z; + if (entry.is_dir) z = z | 0x80; + arr[i*17] = z; + + arr[i*17+1] = entry.x & 0xFF; + arr[i*17+2] = (entry.x >> 8) & 0xFF; + arr[i*17+3] = (entry.x >> 16) & 0xFF; + `` + arr[i*17+4] = entry.y & 0xFF; + arr[i*17+5] = (entry.y >> 8) & 0xFF; + arr[i*17+6] = (entry.y >> 16) & 0xFF; + + arr[i*17+7] = entry.offset & 0xFF; + arr[i*17+8] = unshift(entry.offset,8) & 0xFF; + arr[i*17+9] = unshift(entry.offset,16) & 0xFF; + arr[i*17+10] = unshift(entry.offset,24) & 0xFF; + arr[i*17+11] = unshift(entry.offset,32) & 0xFF; + arr[i*17+12] = unshift(entry.offset,48) & 0xFF; + + arr[i*17+13] = entry.length & 0xFF; + arr[i*17+14] = (entry.length >> 8) & 0xFF; + arr[i*17+15] = (entry.length >> 16) & 0xFF; + arr[i*17+16] = (entry.length >> 24) & 0xFF; + } + return buffer; +} + +test("stub data", assertion => { + let data = stubData([ + {z:5,x:1000,y:2000,offset:1000,length:2000,is_dir:false}, + {z:14,x:16383,y:16383,offset:999999,length:999,is_dir:false} + ]); + let dataview = new DataView(data); + + var z_raw = dataview.getUint8(17+0); + var x = getUint24(dataview,17+1); + var y = getUint24(dataview,17+4); + var offset = getUint48(dataview,17+7); + var length = dataview.getUint32(17+13,true); + assertion.ok(z_raw === 14); + assertion.ok(x === 16383); + assertion.ok(y === 16383); +}); + +test("get entry", assertion => { + let data = stubData([ + {z:5,x:1000,y:2000,offset:1000,length:2000,is_dir:false}, + {z:14,x:16383,y:16383,offset:999999,length:999,is_dir:false} + ]); + let view = new DataView(data); + let entry = getTile(view,14,16383,16383); + assertion.ok(entry!.offset === 999999); + assertion.ok(entry!.length = 999); +}) + +test("get leafdir", assertion => { + let data = stubData([ + {z:14,x:16383,y:16383,offset:999999,length:999,is_dir:true} + ]); + let view = new DataView(data); + let entry = getLeafdir(view,14,16383,16383); + assertion.ok(entry!.offset === 999999); + assertion.ok(entry!.length = 999); + assertion.ok(getTile(view,14,16383,16383) === null); +}) diff --git a/js/pmtiles.ts b/js/pmtiles.ts new file mode 100644 index 0000000..fbdf925 --- /dev/null +++ b/js/pmtiles.ts @@ -0,0 +1,48 @@ +export const shift = (n:number, shift:number) => { + return n * Math.pow(2, shift); +} + +export const unshift = (n:number, shift:number) => { + return Math.floor(n / Math.pow(2, shift)); +} + +export const getUint24 = (view:DataView, pos:number) => { + return shift(view.getUint16(pos+1,true),8) + view.getUint8(pos) +} + +export const getUint48 = (view:DataView, pos:number) => { + return shift(view.getUint32(pos+2,true),16) + view.getUint16(pos,true) +} + +const compare = (tz:number,tx:number,ty:number,view:DataView,i:number) => { + if (tz != view.getUint8(i)) return tz - view.getUint8(i); + var x = getUint24(view,i+1); + if (tx != x) return tx - x; + var y = getUint24(view,i+4); + if (ty != y) return ty - y; + return 0; +} + +export const getLeafdir = (view:DataView, z:number, x:number, y:number) => { + return getTile(view,z | 0x80,x,y); +} + +export const getTile = (view:DataView, z:number, x:number, y:number) => { + var m = 0; + var n = view.byteLength / 17 - 1; + while (m <= n) { + var k = (n + m) >> 1; + var cmp = compare(z,x,y,view,k*17); + if (cmp > 0) { + m = k + 1; + } else if (cmp < 0) { + n = k - 1; + } else { + return { + offset:getUint48(view,k*17+7), + length:view.getUint32(k*17+13,true) + } + } + } + return null; +} \ No newline at end of file diff --git a/js/tsconfig.json b/js/tsconfig.json new file mode 100644 index 0000000..4b7419c --- /dev/null +++ b/js/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es6", + "lib": ["es2020", "dom"], + "strict": true, + "moduleResolution": "node", + "paths": { + }, + "types": [] + }, + "include": ["*.ts"] +} \ No newline at end of file