source files for faster directory fetches [#26]

js/package-lock.json

@@ -1,14 +1,153 @@
 {
   "name": "pmtiles",
-  "version": "0.3.1",
+  "version": "0.5.0",
-  "lockfileVersion": 1,
+  "lockfileVersion": 2,
   "requires": true,
+  "packages": {
+    "": {
+      "name": "pmtiles",
+      "version": "0.5.0",
+      "license": "BSD-3-Clause",
+      "devDependencies": {
+        "esbuild": "^0.11.14",
+        "esbuild-runner": "^2.2.1",
+        "typescript": "^4.5.5",
+        "zora": "^5.0.2"
+      }
+    },
+    "node_modules/buffer-from": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
+      "dev": true
+    },
+    "node_modules/esbuild": {
+      "version": "0.11.14",
+      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.14.tgz",
+      "integrity": "sha512-ejheEPkqhq5y0LM9rG9e+3yDihPtqeeE4NZmG7VQiSGJ3CjO4HvPOHmhhttSksfSztjLAGo2+0/zSIvlqj4JOQ==",
+      "dev": true,
+      "hasInstallScript": true,
+      "bin": {
+        "esbuild": "bin/esbuild"
+      }
+    },
+    "node_modules/esbuild-runner": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/esbuild-runner/-/esbuild-runner-2.2.1.tgz",
+      "integrity": "sha512-VP0VfJJZiZ3cKzdOH59ZceDxx/GzBKra7tiGM8MfFMLv6CR1/cpsvtQ3IsJI3pz7HyeYxtbPyecj3fHwR+3XcQ==",
+      "dev": true,
+      "dependencies": {
+        "source-map-support": "0.5.19",
+        "tslib": "2.3.1"
+      },
+      "bin": {
+        "esr": "bin/esr.js"
+      },
+      "peerDependencies": {
+        "esbuild": "*"
+      }
+    },
+    "node_modules/source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/source-map-support": {
+      "version": "0.5.19",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz",
+      "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==",
+      "dev": true,
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "source-map": "^0.6.0"
+      }
+    },
+    "node_modules/tslib": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz",
+      "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==",
+      "dev": true
+    },
+    "node_modules/typescript": {
+      "version": "4.5.5",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz",
+      "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==",
+      "dev": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=4.2.0"
+      }
+    },
+    "node_modules/zora": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/zora/-/zora-5.0.2.tgz",
+      "integrity": "sha512-HZtoEtYm9iJrym0nvel84NmmmOZyvS9I6q27kO1zAA5W4vY87LnRkokRGd5Qm6nrv0+YhGlr/i5VfVcS+VEK4g==",
+      "dev": true
+    }
+  },
   "dependencies": {
+    "buffer-from": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
+      "dev": true
+    },
     "esbuild": {
       "version": "0.11.14",
       "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.14.tgz",
       "integrity": "sha512-ejheEPkqhq5y0LM9rG9e+3yDihPtqeeE4NZmG7VQiSGJ3CjO4HvPOHmhhttSksfSztjLAGo2+0/zSIvlqj4JOQ==",
       "dev": true
+    },
+    "esbuild-runner": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/esbuild-runner/-/esbuild-runner-2.2.1.tgz",
+      "integrity": "sha512-VP0VfJJZiZ3cKzdOH59ZceDxx/GzBKra7tiGM8MfFMLv6CR1/cpsvtQ3IsJI3pz7HyeYxtbPyecj3fHwR+3XcQ==",
+      "dev": true,
+      "requires": {
+        "source-map-support": "0.5.19",
+        "tslib": "2.3.1"
+      }
+    },
+    "source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+      "dev": true
+    },
+    "source-map-support": {
+      "version": "0.5.19",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz",
+      "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==",
+      "dev": true,
+      "requires": {
+        "buffer-from": "^1.0.0",
+        "source-map": "^0.6.0"
+      }
+    },
+    "tslib": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz",
+      "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==",
+      "dev": true
+    },
+    "typescript": {
+      "version": "4.5.5",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.5.tgz",
+      "integrity": "sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==",
+      "dev": true
+    },
+    "zora": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/zora/-/zora-5.0.2.tgz",
+      "integrity": "sha512-HZtoEtYm9iJrym0nvel84NmmmOZyvS9I6q27kO1zAA5W4vY87LnRkokRGd5Qm6nrv0+YhGlr/i5VfVcS+VEK4g==",
+      "dev": true
     }
   }
 }

js/package.json

@@ -11,15 +11,19 @@
     "index.js"
   ],
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
     "build-iife": "esbuild index.src.mjs --outfile=index.js --target=es2015 --global-name=pmtiles --format=iife",
     "build-esm": "esbuild index.src.mjs --outfile=index.mjs --target=es6 --global-name=pmtiles --format=esm",
-    "build": "npm run build-iife && npm run build-esm"
+    "build": "npm run build-iife && npm run build-esm",
+    "test": "node -r esbuild-runner/register pmtiles.test.ts",
+    "tsc": "tsc --noEmit --watch"
   },
   "homepage": "https://github.com/protomaps/pmtiles",
   "author": "Brandon Liu",
   "license": "BSD-3-Clause",
   "devDependencies": {
-    "esbuild": "^0.11.14"
+    "esbuild": "^0.11.14",
+    "esbuild-runner": "^2.2.1",
+    "typescript": "^4.5.5",
+    "zora": "^5.0.2"
   }
 }

js/pmtiles.test.ts

@@ -0,0 +1,83 @@
+import { test } from "zora";
+import { unshift, getUint24, getUint48, getLeafdir, getTile } from "./pmtiles";
+
+interface StubEntry {
+	z:number;
+	x:number;
+	y:number;
+	offset:number;
+	length:number;
+	is_dir:boolean;
+}
+
+const stubData = (entries:StubEntry[]) => {
+	// sort entries
+	let buffer = new ArrayBuffer(17*entries.length);
+	let arr = new Uint8Array(buffer);
+	for (var i = 0; i < entries.length; i++) {
+		var entry = entries[i];
+		var z = entry.z;
+		if (entry.is_dir) z = z | 0x80;
+		arr[i*17] = z;
+
+		arr[i*17+1] = entry.x & 0xFF;
+		arr[i*17+2] = (entry.x >> 8) & 0xFF;
+		arr[i*17+3] = (entry.x >> 16) & 0xFF;
+		``
+		arr[i*17+4] = entry.y & 0xFF;
+		arr[i*17+5] = (entry.y >> 8) & 0xFF;
+		arr[i*17+6] = (entry.y >> 16) & 0xFF;
+
+		arr[i*17+7] = entry.offset & 0xFF;
+		arr[i*17+8] = unshift(entry.offset,8) & 0xFF;
+		arr[i*17+9] = unshift(entry.offset,16) & 0xFF;
+		arr[i*17+10] = unshift(entry.offset,24) & 0xFF;
+		arr[i*17+11] = unshift(entry.offset,32) & 0xFF;
+		arr[i*17+12] = unshift(entry.offset,48) & 0xFF;
+
+		arr[i*17+13] = entry.length & 0xFF;
+		arr[i*17+14] = (entry.length >> 8) & 0xFF;
+		arr[i*17+15] = (entry.length >> 16) & 0xFF;
+		arr[i*17+16] = (entry.length >> 24) & 0xFF;
+	}
+	return buffer;
+}
+
+test("stub data", assertion => {
+	let data = stubData([
+		{z:5,x:1000,y:2000,offset:1000,length:2000,is_dir:false},
+		{z:14,x:16383,y:16383,offset:999999,length:999,is_dir:false}
+	]);
+	let dataview = new DataView(data);	
+
+	var z_raw = dataview.getUint8(17+0);
+	var x = getUint24(dataview,17+1);
+	var y = getUint24(dataview,17+4);
+	var offset = getUint48(dataview,17+7);
+	var length = dataview.getUint32(17+13,true);
+ 	assertion.ok(z_raw === 14);
+ 	assertion.ok(x === 16383);
+ 	assertion.ok(y === 16383);
+});
+
+test("get entry", assertion => {
+	let data = stubData([
+		{z:5,x:1000,y:2000,offset:1000,length:2000,is_dir:false},
+		{z:14,x:16383,y:16383,offset:999999,length:999,is_dir:false}
+	]);
+	let view = new DataView(data);
+	let entry = getTile(view,14,16383,16383);
+	assertion.ok(entry!.offset === 999999);
+	assertion.ok(entry!.length = 999);
+})
+
+test("get leafdir", assertion => {
+	let data = stubData([
+		{z:14,x:16383,y:16383,offset:999999,length:999,is_dir:true}
+	]);
+	let view = new DataView(data);
+	let entry = getLeafdir(view,14,16383,16383);
+	assertion.ok(entry!.offset === 999999);
+	assertion.ok(entry!.length = 999);
+	assertion.ok(getTile(view,14,16383,16383) === null);
+})

js/pmtiles.ts

@@ -0,0 +1,48 @@
+export const shift = (n:number, shift:number) => {
+    return n * Math.pow(2, shift);
+}
+
+export const unshift = (n:number, shift:number) => {
+	return Math.floor(n / Math.pow(2, shift));
+}
+
+export const getUint24 = (view:DataView, pos:number) => {
+  return shift(view.getUint16(pos+1,true),8) + view.getUint8(pos)
+}
+
+export const getUint48 = (view:DataView, pos:number) => {
+    return shift(view.getUint32(pos+2,true),16) + view.getUint16(pos,true)
+}
+
+const compare = (tz:number,tx:number,ty:number,view:DataView,i:number) => {
+	if (tz != view.getUint8(i)) return tz - view.getUint8(i);
+	var x = getUint24(view,i+1);
+	if (tx != x) return tx - x;
+	var y = getUint24(view,i+4);
+	if (ty != y) return ty - y;
+	return 0;
+}
+
+export const getLeafdir = (view:DataView, z:number, x:number, y:number) => {
+	return getTile(view,z | 0x80,x,y);
+}
+
+export const getTile = (view:DataView, z:number, x:number, y:number) => {
+	var m = 0;
+	var n = view.byteLength / 17 - 1;	
+	while (m <= n) {
+		var k = (n + m) >> 1;
+		var cmp = compare(z,x,y,view,k*17);
+		if (cmp > 0) {
+			m = k + 1;
+		} else if (cmp < 0) {
+			n = k - 1;
+		} else {
+			return {
+				offset:getUint48(view,k*17+7),
+				length:view.getUint32(k*17+13,true)
+			}
+		}
+	}
+	return null;
+}

js/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "lib": ["es2020", "dom"],
+    "strict": true,
+    "moduleResolution": "node",
+    "paths": {
+    },
+    "types": []
+  },
+  "include": ["*.ts"]
+}