lint cloudflare implementation (#352)

Add linters to AWS and cloudflare implementations [#287]
This commit is contained in:
Brandon Liu
2024-02-05 16:59:06 +08:00
committed by GitHub
parent b229c1a8c7
commit e86dd554be
9 changed files with 598 additions and 126 deletions

View File

@@ -21,8 +21,8 @@ jobs:
- run: cd js && npm install && npm run build - run: cd js && npm install && npm run build
- run: echo "VITE_GIT_SHA=$(git rev-parse --short HEAD)" >> app/.env - run: echo "VITE_GIT_SHA=$(git rev-parse --short HEAD)" >> app/.env
- run: cd app && npm install && ./node_modules/.bin/tsc && npm run prettier-check && ./node_modules/.bin/vite build --base=/PMTiles/ - run: cd app && npm install && ./node_modules/.bin/tsc && npm run prettier-check && ./node_modules/.bin/vite build --base=/PMTiles/
- run: cd serverless/aws && npm install && npx tsc && npm run build && cp dist/lambda_function.zip ../../app/dist - run: cd serverless/aws && npm install && npx tsc && npm run biome-check && npm run build && cp dist/lambda_function.zip ../../app/dist
- run: cd serverless/cloudflare && cp wrangler.toml.example wrangler.toml && npm install && npx tsc && npm run build && cp dist/index.js ../../app/dist - run: cd serverless/cloudflare && cp wrangler.toml.example wrangler.toml && npm install && npx tsc && npm run biome-check && npm run build && cp dist/index.js ../../app/dist
- run: cd spec/v3 && cp *.pmtiles ../../app/dist - run: cd spec/v3 && cp *.pmtiles ../../app/dist
- run: cd js/examples && mkdir ../../app/dist/examples && cp *.html ../../app/dist/examples/ - run: cd js/examples && mkdir ../../app/dist/examples && cp *.html ../../app/dist/examples/
- run: cd openlayers/examples && mkdir ../../app/dist/examples/openlayers && cp *.html ../../app/dist/examples/openlayers - run: cd openlayers/examples && mkdir ../../app/dist/examples/openlayers && cp *.html ../../app/dist/examples/openlayers

View File

@@ -12,6 +12,7 @@
"@aws-sdk/node-http-handler": "^3.360.0" "@aws-sdk/node-http-handler": "^3.360.0"
}, },
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^1.5.3",
"@types/aws-lambda": "^8.10.108", "@types/aws-lambda": "^8.10.108",
"@types/node": "^18.11.2", "@types/node": "^18.11.2",
"esbuild": "^0.20.0", "esbuild": "^0.20.0",
@@ -915,6 +916,161 @@
"node": ">=14.0.0" "node": ">=14.0.0"
} }
}, },
"node_modules/@biomejs/biome": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.5.3.tgz",
"integrity": "sha512-yvZCa/g3akwTaAQ7PCwPWDCkZs3Qa5ONg/fgOUT9e6wAWsPftCjLQFPXBeGxPK30yZSSpgEmRCfpGTmVbUjGgg==",
"dev": true,
"hasInstallScript": true,
"bin": {
"biome": "bin/biome"
},
"engines": {
"node": ">=14.*"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/biome"
},
"optionalDependencies": {
"@biomejs/cli-darwin-arm64": "1.5.3",
"@biomejs/cli-darwin-x64": "1.5.3",
"@biomejs/cli-linux-arm64": "1.5.3",
"@biomejs/cli-linux-arm64-musl": "1.5.3",
"@biomejs/cli-linux-x64": "1.5.3",
"@biomejs/cli-linux-x64-musl": "1.5.3",
"@biomejs/cli-win32-arm64": "1.5.3",
"@biomejs/cli-win32-x64": "1.5.3"
}
},
"node_modules/@biomejs/cli-darwin-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.5.3.tgz",
"integrity": "sha512-ImU7mh1HghEDyqNmxEZBoMPr8SxekkZuYcs+gynKlNW+TALQs7swkERiBLkG9NR0K1B3/2uVzlvYowXrmlW8hw==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-darwin-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.5.3.tgz",
"integrity": "sha512-vCdASqYnlpq/swErH7FD6nrFz0czFtK4k/iLgj0/+VmZVjineFPgevOb+Sr9vz0tk0GfdQO60bSpI74zU8M9Dw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-linux-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.5.3.tgz",
"integrity": "sha512-cupBQv0sNF1OKqBfx7EDWMSsKwRrBUZfjXawT4s6hKV6ALq7p0QzWlxr/sDmbKMLOaLQtw2Qgu/77N9rm+f9Rg==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-linux-arm64-musl": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.5.3.tgz",
"integrity": "sha512-DYuMizUYUBYfS0IHGjDrOP1RGipqWfMGEvNEJ398zdtmCKLXaUvTimiox5dvx4X15mBK5M2m8wgWUgOP1giUpQ==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-linux-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.5.3.tgz",
"integrity": "sha512-YQrSArQvcv4FYsk7Q91Yv4uuu5F8hJyORVcv3zsjCLGkjIjx2RhjYLpTL733SNL7v33GmOlZY0eFR1ko38tuUw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-linux-x64-musl": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.5.3.tgz",
"integrity": "sha512-UUHiAnlDqr2Y/LpvshBFhUYMWkl2/Jn+bi3U6jKuav0qWbbBKU/ByHgR4+NBxpKBYoCtWxhnmatfH1bpPIuZMw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-win32-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.5.3.tgz",
"integrity": "sha512-HxatYH7vf/kX9nrD+pDYuV2GI9GV8EFo6cfKkahAecTuZLPxryHx1WEfJthp5eNsE0+09STGkKIKjirP0ufaZA==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-win32-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.5.3.tgz",
"integrity": "sha512-fMvbSouZEASU7mZH8SIJSANDm5OqsjgtVXlbUqxwed6BP7uuHRSs396Aqwh2+VoW8fwTpp6ybIUoC9FrzB0kyA==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@esbuild/aix-ppc64": { "node_modules/@esbuild/aix-ppc64": {
"version": "0.20.0", "version": "0.20.0",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.0.tgz", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.0.tgz",
@@ -3168,6 +3324,78 @@
"tslib": "^2.5.0" "tslib": "^2.5.0"
} }
}, },
"@biomejs/biome": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.5.3.tgz",
"integrity": "sha512-yvZCa/g3akwTaAQ7PCwPWDCkZs3Qa5ONg/fgOUT9e6wAWsPftCjLQFPXBeGxPK30yZSSpgEmRCfpGTmVbUjGgg==",
"dev": true,
"requires": {
"@biomejs/cli-darwin-arm64": "1.5.3",
"@biomejs/cli-darwin-x64": "1.5.3",
"@biomejs/cli-linux-arm64": "1.5.3",
"@biomejs/cli-linux-arm64-musl": "1.5.3",
"@biomejs/cli-linux-x64": "1.5.3",
"@biomejs/cli-linux-x64-musl": "1.5.3",
"@biomejs/cli-win32-arm64": "1.5.3",
"@biomejs/cli-win32-x64": "1.5.3"
}
},
"@biomejs/cli-darwin-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.5.3.tgz",
"integrity": "sha512-ImU7mh1HghEDyqNmxEZBoMPr8SxekkZuYcs+gynKlNW+TALQs7swkERiBLkG9NR0K1B3/2uVzlvYowXrmlW8hw==",
"dev": true,
"optional": true
},
"@biomejs/cli-darwin-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.5.3.tgz",
"integrity": "sha512-vCdASqYnlpq/swErH7FD6nrFz0czFtK4k/iLgj0/+VmZVjineFPgevOb+Sr9vz0tk0GfdQO60bSpI74zU8M9Dw==",
"dev": true,
"optional": true
},
"@biomejs/cli-linux-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.5.3.tgz",
"integrity": "sha512-cupBQv0sNF1OKqBfx7EDWMSsKwRrBUZfjXawT4s6hKV6ALq7p0QzWlxr/sDmbKMLOaLQtw2Qgu/77N9rm+f9Rg==",
"dev": true,
"optional": true
},
"@biomejs/cli-linux-arm64-musl": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.5.3.tgz",
"integrity": "sha512-DYuMizUYUBYfS0IHGjDrOP1RGipqWfMGEvNEJ398zdtmCKLXaUvTimiox5dvx4X15mBK5M2m8wgWUgOP1giUpQ==",
"dev": true,
"optional": true
},
"@biomejs/cli-linux-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.5.3.tgz",
"integrity": "sha512-YQrSArQvcv4FYsk7Q91Yv4uuu5F8hJyORVcv3zsjCLGkjIjx2RhjYLpTL733SNL7v33GmOlZY0eFR1ko38tuUw==",
"dev": true,
"optional": true
},
"@biomejs/cli-linux-x64-musl": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.5.3.tgz",
"integrity": "sha512-UUHiAnlDqr2Y/LpvshBFhUYMWkl2/Jn+bi3U6jKuav0qWbbBKU/ByHgR4+NBxpKBYoCtWxhnmatfH1bpPIuZMw==",
"dev": true,
"optional": true
},
"@biomejs/cli-win32-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.5.3.tgz",
"integrity": "sha512-HxatYH7vf/kX9nrD+pDYuV2GI9GV8EFo6cfKkahAecTuZLPxryHx1WEfJthp5eNsE0+09STGkKIKjirP0ufaZA==",
"dev": true,
"optional": true
},
"@biomejs/cli-win32-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.5.3.tgz",
"integrity": "sha512-fMvbSouZEASU7mZH8SIJSANDm5OqsjgtVXlbUqxwed6BP7uuHRSs396Aqwh2+VoW8fwTpp6ybIUoC9FrzB0kyA==",
"dev": true,
"optional": true
},
"@esbuild/aix-ppc64": { "@esbuild/aix-ppc64": {
"version": "0.20.0", "version": "0.20.0",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.0.tgz", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.0.tgz",

View File

@@ -2,6 +2,7 @@
"name": "pmtiles-aws", "name": "pmtiles-aws",
"version": "0.0.0", "version": "0.0.0",
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^1.5.3",
"@types/aws-lambda": "^8.10.108", "@types/aws-lambda": "^8.10.108",
"@types/node": "^18.11.2", "@types/node": "^18.11.2",
"esbuild": "^0.20.0", "esbuild": "^0.20.0",
@@ -12,7 +13,9 @@
"scripts": { "scripts": {
"tsc": "tsc --noEmit --watch", "tsc": "tsc --noEmit --watch",
"build": "esbuild src/index.ts --target=es2020 --outfile=dist/index.mjs --format=esm --bundle --platform=node --target=node18 --external:@aws-sdk/client-s3 --external:@aws-sdk/node-http-handler --banner:js=//$(git describe --always) && cd dist && zip lambda_function.zip index.mjs", "build": "esbuild src/index.ts --target=es2020 --outfile=dist/index.mjs --format=esm --bundle --platform=node --target=node18 --external:@aws-sdk/client-s3 --external:@aws-sdk/node-http-handler --banner:js=//$(git describe --always) && cd dist && zip lambda_function.zip index.mjs",
"test": "tsx ../shared/index.test.ts" "test": "tsx ../shared/index.test.ts",
"biome": "biome check --config-path=../../js/ src --apply",
"biome-check": "biome check --config-path=../../js src"
}, },
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.360.0", "@aws-sdk/client-s3": "^3.360.0",

View File

@@ -1,37 +1,37 @@
import { test } from "node:test";
import assert from "node:assert"; import assert from "node:assert";
import { test } from "node:test";
import { get_region } from "./aws_region"; import { getRegion } from "./aws_region";
test("one bucket", () => { test("one bucket", () => {
let result = get_region( const result = getRegion(
"us-west-1", "us-west-1",
{ bucket: "mybucket", region: "us-west-1" }, { bucket: "mybucket", region: "us-west-1" },
[], []
); );
assert.deepEqual(result, { bucket: "mybucket", region: "us-west-1" }); assert.deepEqual(result, { bucket: "mybucket", region: "us-west-1" });
}); });
test("unknown region", () => { test("unknown region", () => {
let result = get_region( const result = getRegion(
"us-nullisland-1", "us-nullisland-1",
{ bucket: "mybucket", region: "us-west-1" }, { bucket: "mybucket", region: "us-west-1" },
[], []
); );
assert.deepEqual(result, { bucket: "mybucket", region: "us-west-1" }); assert.deepEqual(result, { bucket: "mybucket", region: "us-west-1" });
}); });
test("exact region match", () => { test("exact region match", () => {
let result = get_region( let result = getRegion(
"us-west-1", "us-west-1",
{ bucket: "mybucket", region: "us-west-1" }, { bucket: "mybucket", region: "us-west-1" },
[{ bucket: "mybucket-ap-south-1", region: "ap-south-1" }], [{ bucket: "mybucket-ap-south-1", region: "ap-south-1" }]
); );
assert.deepEqual(result, { bucket: "mybucket", region: "us-west-1" }); assert.deepEqual(result, { bucket: "mybucket", region: "us-west-1" });
result = get_region( result = getRegion(
"ap-south-1", "ap-south-1",
{ bucket: "mybucket", region: "us-west-1" }, { bucket: "mybucket", region: "us-west-1" },
[{ bucket: "mybucket-ap-south-1", region: "ap-south-1" }], [{ bucket: "mybucket-ap-south-1", region: "ap-south-1" }]
); );
assert.deepEqual(result, { assert.deepEqual(result, {
bucket: "mybucket-ap-south-1", bucket: "mybucket-ap-south-1",
@@ -40,10 +40,10 @@ test("exact region match", () => {
}); });
test("priority match", () => { test("priority match", () => {
let result = get_region( const result = getRegion(
"us-west-1", "us-west-1",
{ bucket: "mybucket", region: "ap-south-1" }, { bucket: "mybucket", region: "ap-south-1" },
[{ bucket: "mybucket-us-west-2", region: "us-west-2" }], [{ bucket: "mybucket-us-west-2", region: "us-west-2" }]
); );
assert.deepEqual(result, { assert.deepEqual(result, {
bucket: "mybucket-us-west-2", bucket: "mybucket-us-west-2",

View File

@@ -19,24 +19,24 @@ const REGION_MATRIX: Record<string, string[]> = {
"sa-east-1": ["us-east-1", "us-east-2"], // sao paulo "sa-east-1": ["us-east-1", "us-east-2"], // sao paulo
}; };
export let get_region = ( export const getRegion = (
exec_region: string, execRegion: string,
primary: Bucket, primary: Bucket,
replicas: Bucket[], replicas: Bucket[]
): Bucket => { ): Bucket => {
if (primary.region === exec_region) { if (primary.region === execRegion) {
return primary; return primary;
} }
for (let replica of replicas) { for (const replica of replicas) {
if (replica.region === exec_region) { if (replica.region === execRegion) {
return replica; return replica;
} }
} }
if (exec_region in REGION_MATRIX) { if (execRegion in REGION_MATRIX) {
for (let region of REGION_MATRIX[exec_region]) { for (const region of REGION_MATRIX[execRegion]) {
for (let replica of replicas) { for (const replica of replicas) {
if (replica.region === region) { if (replica.region === region) {
return replica; return replica;
} }

View File

@@ -1,22 +1,22 @@
import { import {
Context,
APIGatewayProxyResult,
APIGatewayProxyEventV2, APIGatewayProxyEventV2,
APIGatewayProxyResult,
Context,
} from "aws-lambda"; } from "aws-lambda";
import { import {
PMTiles,
ResolvedValueCache,
RangeResponse,
Source,
Compression, Compression,
PMTiles,
RangeResponse,
ResolvedValueCache,
Source,
TileType, TileType,
} from "../../../js/index"; } from "../../../js/index";
import { pmtiles_path, tile_path, tileJSON } from "../../shared/index"; import { pmtiles_path, tileJSON, tile_path } from "../../shared/index";
import { createHash } from "crypto";
import zlib from "zlib"; import zlib from "zlib";
import { createHash } from "crypto"
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3"; import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
import { NodeHttpHandler } from "@aws-sdk/node-http-handler"; import { NodeHttpHandler } from "@aws-sdk/node-http-handler";
// the region should default to the same one as the function // the region should default to the same one as the function
@@ -33,37 +33,47 @@ async function nativeDecompress(
): Promise<ArrayBuffer> { ): Promise<ArrayBuffer> {
if (compression === Compression.None || compression === Compression.Unknown) { if (compression === Compression.None || compression === Compression.Unknown) {
return buf; return buf;
} else if (compression === Compression.Gzip) {
return zlib.gunzipSync(buf);
} else {
throw Error("Compression method not supported");
} }
if (compression === Compression.Gzip) {
return zlib.gunzipSync(buf);
}
throw Error("Compression method not supported");
} }
// Lambda needs to run with 512MB, empty function takes about 70 // Lambda needs to run with 512MB, empty function takes about 70
const CACHE = new ResolvedValueCache(undefined, undefined, nativeDecompress); const CACHE = new ResolvedValueCache(undefined, undefined, nativeDecompress);
class S3Source implements Source { class S3Source implements Source {
archive_name: string; archiveName: string;
constructor(archive_name: string) { constructor(archiveName: string) {
this.archive_name = archive_name; this.archiveName = archiveName;
} }
getKey() { getKey() {
return this.archive_name; return this.archiveName;
} }
async getBytes(offset: number, length: number, signal?:AbortSignal, etag?: string): Promise<RangeResponse> { async getBytes(
offset: number,
length: number,
signal?: AbortSignal,
etag?: string
): Promise<RangeResponse> {
const resp = await s3client.send( const resp = await s3client.send(
new GetObjectCommand({ new GetObjectCommand({
// biome-ignore lint: aws api
Bucket: process.env.BUCKET!, Bucket: process.env.BUCKET!,
Key: pmtiles_path(this.archive_name, process.env.PMTILES_PATH), // biome-ignore lint: aws api
Key: pmtiles_path(this.archiveName, process.env.PMTILES_PATH),
// biome-ignore lint: aws api
Range: "bytes=" + offset + "-" + (offset + length - 1), Range: "bytes=" + offset + "-" + (offset + length - 1),
}) })
); );
const arr = await resp.Body!.transformToByteArray(); const arr = await resp.Body?.transformToByteArray();
if (!arr) throw Error("Failed to read S3 response body");
return { return {
data: arr.buffer, data: arr.buffer,
@@ -100,12 +110,12 @@ export const handlerRaw = async (
_context: Context, _context: Context,
tilePostprocess?: (a: ArrayBuffer, t: TileType) => ArrayBuffer tilePostprocess?: (a: ArrayBuffer, t: TileType) => ArrayBuffer
): Promise<APIGatewayProxyResult> => { ): Promise<APIGatewayProxyResult> => {
let path; let path: string;
let is_api_gateway; let isApiGateway = false;
if (event.pathParameters) { if (event.pathParameters) {
is_api_gateway = true; isApiGateway = true;
if (event.pathParameters.proxy) { if (event.pathParameters.proxy) {
path = "/" + event.pathParameters.proxy; path = `/${event.pathParameters.proxy}`;
} else { } else {
return apiResp(500, "Proxy integration missing tile_path parameter"); return apiResp(500, "Proxy integration missing tile_path parameter");
} }
@@ -167,7 +177,7 @@ export const handlerRaw = async (
[TileType.Avif, "avif"], [TileType.Avif, "avif"],
]) { ]) {
if (header.tileType === pair[0] && ext !== pair[1]) { if (header.tileType === pair[0] && ext !== pair[1]) {
if (header.tileType == TileType.Mvt && ext === "pbf") { if (header.tileType === TileType.Mvt && ext === "pbf") {
// allow this for now. Eventually we will delete this in favor of .mvt // allow this for now. Eventually we will delete this in favor of .mvt
continue; continue;
} }
@@ -180,8 +190,8 @@ export const handlerRaw = async (
} }
} }
const tile_result = await p.getZxy(tile[0], tile[1], tile[2]); const tileResult = await p.getZxy(tile[0], tile[1], tile[2]);
if (tile_result) { if (tileResult) {
switch (header.tileType) { switch (header.tileType) {
case TileType.Mvt: case TileType.Mvt:
// part of the list of Cloudfront compressible types. // part of the list of Cloudfront compressible types.
@@ -201,38 +211,35 @@ export const handlerRaw = async (
break; break;
} }
let data = tile_result.data; let data = tileResult.data;
if (tilePostprocess) { if (tilePostprocess) {
data = tilePostprocess(data, header.tileType); data = tilePostprocess(data, header.tileType);
} }
headers["Cache-Control"] = `public, max-age=${process.env.CACHE_MAX_AGE || 86400}`; headers["Cache-Control"] = `public, max-age=${
headers["ETag"] = `"${createHash("sha256").update(Buffer.from(data)).digest("hex")}"` process.env.CACHE_MAX_AGE || 86400
}`;
headers.ETag = `"${createHash("sha256")
.update(Buffer.from(data))
.digest("hex")}"`;
if (is_api_gateway) { if (isApiGateway) {
// this is wasted work, but we need to force API Gateway to interpret the Lambda response as binary // this is wasted work, but we need to force API Gateway to interpret the Lambda response as binary
// without depending on clients sending matching Accept: headers in the request. // without depending on clients sending matching Accept: headers in the request.
const recompressed_data = zlib.gzipSync(data); const recompressedData = zlib.gzipSync(data);
headers["Content-Encoding"] = "gzip"; headers["Content-Encoding"] = "gzip";
return apiResp( return apiResp(
200, 200,
Buffer.from(recompressed_data).toString("base64"), Buffer.from(recompressedData).toString("base64"),
true, true,
headers headers
); );
} else { }
// returns uncompressed response // returns uncompressed response
return apiResp( return apiResp(200, Buffer.from(data).toString("base64"), true, headers);
200,
Buffer.from(data).toString("base64"),
true,
headers
);
} }
} else {
return apiResp(204, "", false, headers); return apiResp(204, "", false, headers);
}
} catch (e) { } catch (e) {
if ((e as Error).name === "AccessDenied") { if ((e as Error).name === "AccessDenied") {
return apiResp(403, "Bucket access unauthorized", false, headers); return apiResp(403, "Bucket access unauthorized", false, headers);

View File

@@ -8,12 +8,168 @@
"name": "pmtiles-cloudflare", "name": "pmtiles-cloudflare",
"version": "0.0.1", "version": "0.0.1",
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^1.5.3",
"@cloudflare/workers-types": "^4.20230518.0", "@cloudflare/workers-types": "^4.20230518.0",
"tsx": "^4.7.0", "tsx": "^4.7.0",
"typescript": "^4.8.4", "typescript": "^4.8.4",
"wrangler": "3.19.0" "wrangler": "3.19.0"
} }
}, },
"node_modules/@biomejs/biome": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.5.3.tgz",
"integrity": "sha512-yvZCa/g3akwTaAQ7PCwPWDCkZs3Qa5ONg/fgOUT9e6wAWsPftCjLQFPXBeGxPK30yZSSpgEmRCfpGTmVbUjGgg==",
"dev": true,
"hasInstallScript": true,
"bin": {
"biome": "bin/biome"
},
"engines": {
"node": ">=14.*"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/biome"
},
"optionalDependencies": {
"@biomejs/cli-darwin-arm64": "1.5.3",
"@biomejs/cli-darwin-x64": "1.5.3",
"@biomejs/cli-linux-arm64": "1.5.3",
"@biomejs/cli-linux-arm64-musl": "1.5.3",
"@biomejs/cli-linux-x64": "1.5.3",
"@biomejs/cli-linux-x64-musl": "1.5.3",
"@biomejs/cli-win32-arm64": "1.5.3",
"@biomejs/cli-win32-x64": "1.5.3"
}
},
"node_modules/@biomejs/cli-darwin-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.5.3.tgz",
"integrity": "sha512-ImU7mh1HghEDyqNmxEZBoMPr8SxekkZuYcs+gynKlNW+TALQs7swkERiBLkG9NR0K1B3/2uVzlvYowXrmlW8hw==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-darwin-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.5.3.tgz",
"integrity": "sha512-vCdASqYnlpq/swErH7FD6nrFz0czFtK4k/iLgj0/+VmZVjineFPgevOb+Sr9vz0tk0GfdQO60bSpI74zU8M9Dw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-linux-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.5.3.tgz",
"integrity": "sha512-cupBQv0sNF1OKqBfx7EDWMSsKwRrBUZfjXawT4s6hKV6ALq7p0QzWlxr/sDmbKMLOaLQtw2Qgu/77N9rm+f9Rg==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-linux-arm64-musl": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.5.3.tgz",
"integrity": "sha512-DYuMizUYUBYfS0IHGjDrOP1RGipqWfMGEvNEJ398zdtmCKLXaUvTimiox5dvx4X15mBK5M2m8wgWUgOP1giUpQ==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-linux-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.5.3.tgz",
"integrity": "sha512-YQrSArQvcv4FYsk7Q91Yv4uuu5F8hJyORVcv3zsjCLGkjIjx2RhjYLpTL733SNL7v33GmOlZY0eFR1ko38tuUw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-linux-x64-musl": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.5.3.tgz",
"integrity": "sha512-UUHiAnlDqr2Y/LpvshBFhUYMWkl2/Jn+bi3U6jKuav0qWbbBKU/ByHgR4+NBxpKBYoCtWxhnmatfH1bpPIuZMw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-win32-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.5.3.tgz",
"integrity": "sha512-HxatYH7vf/kX9nrD+pDYuV2GI9GV8EFo6cfKkahAecTuZLPxryHx1WEfJthp5eNsE0+09STGkKIKjirP0ufaZA==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@biomejs/cli-win32-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.5.3.tgz",
"integrity": "sha512-fMvbSouZEASU7mZH8SIJSANDm5OqsjgtVXlbUqxwed6BP7uuHRSs396Aqwh2+VoW8fwTpp6ybIUoC9FrzB0kyA==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=14.*"
}
},
"node_modules/@cloudflare/kv-asset-handler": { "node_modules/@cloudflare/kv-asset-handler": {
"version": "0.2.0", "version": "0.2.0",
"resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.2.0.tgz", "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.2.0.tgz",
@@ -1615,6 +1771,78 @@
} }
}, },
"dependencies": { "dependencies": {
"@biomejs/biome": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.5.3.tgz",
"integrity": "sha512-yvZCa/g3akwTaAQ7PCwPWDCkZs3Qa5ONg/fgOUT9e6wAWsPftCjLQFPXBeGxPK30yZSSpgEmRCfpGTmVbUjGgg==",
"dev": true,
"requires": {
"@biomejs/cli-darwin-arm64": "1.5.3",
"@biomejs/cli-darwin-x64": "1.5.3",
"@biomejs/cli-linux-arm64": "1.5.3",
"@biomejs/cli-linux-arm64-musl": "1.5.3",
"@biomejs/cli-linux-x64": "1.5.3",
"@biomejs/cli-linux-x64-musl": "1.5.3",
"@biomejs/cli-win32-arm64": "1.5.3",
"@biomejs/cli-win32-x64": "1.5.3"
}
},
"@biomejs/cli-darwin-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.5.3.tgz",
"integrity": "sha512-ImU7mh1HghEDyqNmxEZBoMPr8SxekkZuYcs+gynKlNW+TALQs7swkERiBLkG9NR0K1B3/2uVzlvYowXrmlW8hw==",
"dev": true,
"optional": true
},
"@biomejs/cli-darwin-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.5.3.tgz",
"integrity": "sha512-vCdASqYnlpq/swErH7FD6nrFz0czFtK4k/iLgj0/+VmZVjineFPgevOb+Sr9vz0tk0GfdQO60bSpI74zU8M9Dw==",
"dev": true,
"optional": true
},
"@biomejs/cli-linux-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.5.3.tgz",
"integrity": "sha512-cupBQv0sNF1OKqBfx7EDWMSsKwRrBUZfjXawT4s6hKV6ALq7p0QzWlxr/sDmbKMLOaLQtw2Qgu/77N9rm+f9Rg==",
"dev": true,
"optional": true
},
"@biomejs/cli-linux-arm64-musl": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.5.3.tgz",
"integrity": "sha512-DYuMizUYUBYfS0IHGjDrOP1RGipqWfMGEvNEJ398zdtmCKLXaUvTimiox5dvx4X15mBK5M2m8wgWUgOP1giUpQ==",
"dev": true,
"optional": true
},
"@biomejs/cli-linux-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.5.3.tgz",
"integrity": "sha512-YQrSArQvcv4FYsk7Q91Yv4uuu5F8hJyORVcv3zsjCLGkjIjx2RhjYLpTL733SNL7v33GmOlZY0eFR1ko38tuUw==",
"dev": true,
"optional": true
},
"@biomejs/cli-linux-x64-musl": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.5.3.tgz",
"integrity": "sha512-UUHiAnlDqr2Y/LpvshBFhUYMWkl2/Jn+bi3U6jKuav0qWbbBKU/ByHgR4+NBxpKBYoCtWxhnmatfH1bpPIuZMw==",
"dev": true,
"optional": true
},
"@biomejs/cli-win32-arm64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.5.3.tgz",
"integrity": "sha512-HxatYH7vf/kX9nrD+pDYuV2GI9GV8EFo6cfKkahAecTuZLPxryHx1WEfJthp5eNsE0+09STGkKIKjirP0ufaZA==",
"dev": true,
"optional": true
},
"@biomejs/cli-win32-x64": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.5.3.tgz",
"integrity": "sha512-fMvbSouZEASU7mZH8SIJSANDm5OqsjgtVXlbUqxwed6BP7uuHRSs396Aqwh2+VoW8fwTpp6ybIUoC9FrzB0kyA==",
"dev": true,
"optional": true
},
"@cloudflare/kv-asset-handler": { "@cloudflare/kv-asset-handler": {
"version": "0.2.0", "version": "0.2.0",
"resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.2.0.tgz", "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.2.0.tgz",

View File

@@ -2,6 +2,7 @@
"name": "pmtiles-cloudflare", "name": "pmtiles-cloudflare",
"version": "0.0.1", "version": "0.0.1",
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^1.5.3",
"@cloudflare/workers-types": "^4.20230518.0", "@cloudflare/workers-types": "^4.20230518.0",
"tsx": "^4.7.0", "tsx": "^4.7.0",
"typescript": "^4.8.4", "typescript": "^4.8.4",
@@ -13,6 +14,8 @@
"deploy": "wrangler deploy", "deploy": "wrangler deploy",
"test": "tsx ../shared/index.test.ts", "test": "tsx ../shared/index.test.ts",
"tsc": "tsc --watch", "tsc": "tsc --watch",
"build": "wrangler publish --outdir dist --dry-run" "build": "wrangler publish --outdir dist --dry-run",
"biome": "biome check --config-path=../../js/ src/index.ts --apply",
"biome-check": "biome check --config-path=../../js src/index.ts"
} }
} }

View File

@@ -1,26 +1,27 @@
import { import {
Compression,
PMTiles, PMTiles,
Source,
RangeResponse, RangeResponse,
ResolvedValueCache, ResolvedValueCache,
Source,
TileType, TileType,
Compression,
} from "../../../js/index"; } from "../../../js/index";
import { pmtiles_path, tile_path, tileJSON } from "../../shared/index"; import { pmtiles_path, tileJSON, tile_path } from "../../shared/index";
interface Env { interface Env {
// biome-ignore lint: config name
ALLOWED_ORIGINS?: string; ALLOWED_ORIGINS?: string;
// biome-ignore lint: config name
BUCKET: R2Bucket; BUCKET: R2Bucket;
// biome-ignore lint: config name
CACHE_MAX_AGE?: number; CACHE_MAX_AGE?: number;
// biome-ignore lint: config name
PMTILES_PATH?: string; PMTILES_PATH?: string;
// biome-ignore lint: config name
PUBLIC_HOSTNAME?: string; PUBLIC_HOSTNAME?: string;
} }
class KeyNotFoundError extends Error { class KeyNotFoundError extends Error {}
constructor(message: string) {
super(message);
}
}
async function nativeDecompress( async function nativeDecompress(
buf: ArrayBuffer, buf: ArrayBuffer,
@@ -28,33 +29,38 @@ async function nativeDecompress(
): Promise<ArrayBuffer> { ): Promise<ArrayBuffer> {
if (compression === Compression.None || compression === Compression.Unknown) { if (compression === Compression.None || compression === Compression.Unknown) {
return buf; return buf;
} else if (compression === Compression.Gzip) {
let stream = new Response(buf).body!;
let result = stream.pipeThrough(new DecompressionStream("gzip"));
return new Response(result).arrayBuffer();
} else {
throw Error("Compression method not supported");
} }
if (compression === Compression.Gzip) {
const stream = new Response(buf).body;
const result = stream?.pipeThrough(new DecompressionStream("gzip"));
return new Response(result).arrayBuffer();
}
throw Error("Compression method not supported");
} }
const CACHE = new ResolvedValueCache(25, undefined, nativeDecompress); const CACHE = new ResolvedValueCache(25, undefined, nativeDecompress);
class R2Source implements Source { class R2Source implements Source {
env: Env; env: Env;
archive_name: string; archiveName: string;
constructor(env: Env, archive_name: string) { constructor(env: Env, archiveName: string) {
this.env = env; this.env = env;
this.archive_name = archive_name; this.archiveName = archiveName;
} }
getKey() { getKey() {
return this.archive_name; return this.archiveName;
} }
async getBytes(offset: number, length: number, signal?: AbortSignal, etag?: string): Promise<RangeResponse> { async getBytes(
offset: number,
length: number,
signal?: AbortSignal,
etag?: string
): Promise<RangeResponse> {
const resp = await this.env.BUCKET.get( const resp = await this.env.BUCKET.get(
pmtiles_path(this.archive_name, this.env.PMTILES_PATH), pmtiles_path(this.archiveName, this.env.PMTILES_PATH),
{ {
range: { offset: offset, length: length }, range: { offset: offset, length: length },
} }
@@ -88,73 +94,72 @@ export default {
const cache = caches.default; const cache = caches.default;
if (ok) { if (ok) {
let allowed_origin = ""; let allowedOrigin = "";
if (typeof env.ALLOWED_ORIGINS !== "undefined") { if (typeof env.ALLOWED_ORIGINS !== "undefined") {
for (const o of env.ALLOWED_ORIGINS.split(",")) { for (const o of env.ALLOWED_ORIGINS.split(",")) {
if (o === request.headers.get("Origin") || o === "*") { if (o === request.headers.get("Origin") || o === "*") {
allowed_origin = o; allowedOrigin = o;
} }
} }
} }
const cached = await cache.match(request.url); const cached = await cache.match(request.url);
if (cached) { if (cached) {
const resp_headers = new Headers(cached.headers); const respHeaders = new Headers(cached.headers);
if (allowed_origin) if (allowedOrigin)
resp_headers.set("Access-Control-Allow-Origin", allowed_origin); respHeaders.set("Access-Control-Allow-Origin", allowedOrigin);
resp_headers.set("Vary", "Origin"); respHeaders.set("Vary", "Origin");
return new Response(cached.body, { return new Response(cached.body, {
headers: resp_headers, headers: respHeaders,
status: cached.status, status: cached.status,
}); });
} }
const cacheableResponse = ( const cacheableResponse = (
body: ArrayBuffer | string | undefined, body: ArrayBuffer | string | undefined,
cacheable_headers: Headers, cacheableHeaders: Headers,
status: number status: number
) => { ) => {
cacheable_headers.set( cacheableHeaders.set(
"Cache-Control", "Cache-Control",
"max-age=" + (env.CACHE_MAX_AGE || 86400) `max-age=${env.CACHE_MAX_AGE || 86400}`
); );
const cacheable = new Response(body, { const cacheable = new Response(body, {
headers: cacheable_headers, headers: cacheableHeaders,
status: status, status: status,
}); });
// normalize HEAD requests
ctx.waitUntil(cache.put(request.url, cacheable)); ctx.waitUntil(cache.put(request.url, cacheable));
const resp_headers = new Headers(cacheable_headers); const respHeaders = new Headers(cacheableHeaders);
if (allowed_origin) if (allowedOrigin)
resp_headers.set("Access-Control-Allow-Origin", allowed_origin); respHeaders.set("Access-Control-Allow-Origin", allowedOrigin);
resp_headers.set("Vary", "Origin"); respHeaders.set("Vary", "Origin");
return new Response(body, { headers: resp_headers, status: status }); return new Response(body, { headers: respHeaders, status: status });
}; };
const cacheable_headers = new Headers(); const cacheableHeaders = new Headers();
const source = new R2Source(env, name); const source = new R2Source(env, name);
const p = new PMTiles(source, CACHE, nativeDecompress); const p = new PMTiles(source, CACHE, nativeDecompress);
try { try {
const p_header = await p.getHeader(); const pHeader = await p.getHeader();
if (!tile) { if (!tile) {
cacheable_headers.set("Content-Type", "application/json"); cacheableHeaders.set("Content-Type", "application/json");
const t = tileJSON( const t = tileJSON(
p_header, pHeader,
await p.getMetadata(), await p.getMetadata(),
env.PUBLIC_HOSTNAME || url.hostname, env.PUBLIC_HOSTNAME || url.hostname,
name name
); );
return cacheableResponse(JSON.stringify(t), cacheable_headers, 200); return cacheableResponse(JSON.stringify(t), cacheableHeaders, 200);
} }
if (tile[0] < p_header.minZoom || tile[0] > p_header.maxZoom) { if (tile[0] < pHeader.minZoom || tile[0] > pHeader.maxZoom) {
return cacheableResponse(undefined, cacheable_headers, 404); return cacheableResponse(undefined, cacheableHeaders, 404);
} }
for (const pair of [ for (const pair of [
@@ -164,14 +169,14 @@ export default {
[TileType.Webp, "webp"], [TileType.Webp, "webp"],
[TileType.Avif, "avif"], [TileType.Avif, "avif"],
]) { ]) {
if (p_header.tileType === pair[0] && ext !== pair[1]) { if (pHeader.tileType === pair[0] && ext !== pair[1]) {
if (p_header.tileType == TileType.Mvt && ext === "pbf") { if (pHeader.tileType === TileType.Mvt && ext === "pbf") {
// allow this for now. Eventually we will delete this in favor of .mvt // allow this for now. Eventually we will delete this in favor of .mvt
continue; continue;
} }
return cacheableResponse( return cacheableResponse(
`Bad request: requested .${ext} but archive has type .${pair[1]}`, `Bad request: requested .${ext} but archive has type .${pair[1]}`,
cacheable_headers, cacheableHeaders,
400 400
); );
} }
@@ -179,32 +184,30 @@ export default {
const tiledata = await p.getZxy(tile[0], tile[1], tile[2]); const tiledata = await p.getZxy(tile[0], tile[1], tile[2]);
switch (p_header.tileType) { switch (pHeader.tileType) {
case TileType.Mvt: case TileType.Mvt:
cacheable_headers.set("Content-Type", "application/x-protobuf"); cacheableHeaders.set("Content-Type", "application/x-protobuf");
break; break;
case TileType.Png: case TileType.Png:
cacheable_headers.set("Content-Type", "image/png"); cacheableHeaders.set("Content-Type", "image/png");
break; break;
case TileType.Jpeg: case TileType.Jpeg:
cacheable_headers.set("Content-Type", "image/jpeg"); cacheableHeaders.set("Content-Type", "image/jpeg");
break; break;
case TileType.Webp: case TileType.Webp:
cacheable_headers.set("Content-Type", "image/webp"); cacheableHeaders.set("Content-Type", "image/webp");
break; break;
} }
if (tiledata) { if (tiledata) {
return cacheableResponse(tiledata.data, cacheable_headers, 200); return cacheableResponse(tiledata.data, cacheableHeaders, 200);
} else {
return cacheableResponse(undefined, cacheable_headers, 204);
} }
return cacheableResponse(undefined, cacheableHeaders, 204);
} catch (e) { } catch (e) {
if (e instanceof KeyNotFoundError) { if (e instanceof KeyNotFoundError) {
return cacheableResponse("Archive not found", cacheable_headers, 404); return cacheableResponse("Archive not found", cacheableHeaders, 404);
} else {
throw e;
} }
throw e;
} }
} }