migrate Lambda implementation from python -> node [#80]

This commit is contained in:
Brandon Liu
2022-10-19 23:01:53 +08:00
parent 814761e152
commit 6ed943d156
11 changed files with 642 additions and 265 deletions

View File

@@ -1 +1 @@
lambda_function.zip
dist

View File

@@ -1,12 +0,0 @@
import zipfile
import subprocess
sha = subprocess.check_output(["git", "describe", "--always"]).strip()
with zipfile.ZipFile("lambda_function.zip", "w", zipfile.ZIP_DEFLATED) as z:
z.write("lambda_function.py")
z.write("util.py")
z.write("../../python/pmtiles/reader.py", "pmtiles/reader.py")
z.writestr("version",sha)
print(f"created lambda_function.zip")

View File

@@ -1,109 +0,0 @@
import base64
from functools import lru_cache
import gzip
import json
import os
# Exists inside all lambda functions
import boto3
from botocore.exceptions import ClientError
# create_lambda_function.py will vendor the relevant file
from pmtiles.reader import Reader
from util import pmtiles_path, parse_tile_path
s3 = boto3.client("s3")
# Given a 512MB lambda function, use half of the memory for the cache,
# assuming the average root/leaf/tile size is 512 KB
@lru_cache(maxsize=500)
def get_object_bytes(key, offset, length):
end = offset + length - 1
return (
s3.get_object(
Bucket=os.environ["BUCKET"],
Key=key,
Range=f"bytes={offset}-{end}",
)
.get("Body")
.read()
)
# Assumes event is a API Gateway V2 or Lambda Function URL formatted dict
# and returns API Gateway V2 / Lambda Function dict responses
# Does not work with CloudFront events/Lambda@Edge; see README
def lambda_handler(event, context):
path = None
is_api_gateway = False
if event.get("pathParameters"):
is_api_gateway = True
# API Gateway (HTTP or REST)
if "proxy" in event["pathParameters"]:
path = "/" + event["pathParameters"]["proxy"]
else:
return {
"statusCode": 500,
"body": "Proxy integration missing tile_path parameter",
}
else:
# Lambda Function URL
path = event.get("rawPath")
if not path:
return {
"statusCode": 500,
"body": "Invalid event configuration",
}
name, tile = parse_tile_path(os.environ.get("TILE_PATH"), path)
if not tile:
return {"statusCode": 400, "body": "Invalid tile URL"}
def get_bytes(offset, length):
return get_object_bytes(
pmtiles_path(os.environ.get("PMTILES_PATH"), name), offset, length
)
headers = {}
if "CORS" in os.environ:
headers["Access-Control-Allow-Origin"] = os.environ.get("CORS")
reader = Reader(get_bytes)
try:
minzoom = int(reader.header().metadata["minzoom"])
maxzoom = int(reader.header().metadata["maxzoom"])
if tile.z < minzoom or tile.z > maxzoom:
return {"statusCode": 404, "headers": headers, "body": "Tile not found"}
tile_data = reader.get(tile.z, tile.x, tile.y)
if not tile_data:
return {"statusCode": 204, "headers": headers}
except ClientError as e:
error_code = e.response["Error"]["Code"]
if error_code == "AccessDenied":
return {"statusCode": 404, "headers": headers, "body": "Archive not found"}
else:
raise e
headers["Content-Type"] = "application/protobuf"
if reader.header().metadata.get("compression") == "gzip":
if is_api_gateway:
# API Gateway requires a compressed response to correctly return binary data
# instead of base64
headers["Content-Encoding"] = "gzip"
else:
# CloudFront requires decompressed responses from lambda
# in order to implement the Compressed CacheOptimized policy correctly
# as well as Brotli support
tile_data = gzip.decompress(tile_data)
return {
"statusCode": 200,
"body": base64.b64encode(tile_data),
"isBase64Encoded": True,
"headers": headers,
}

636
serverless/aws/package-lock.json generated Normal file
View File

@@ -0,0 +1,636 @@
{
"name": "pmtiles-aws",
"version": "0.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "pmtiles-aws",
"version": "0.0.0",
"devDependencies": {
"@types/aws-lambda": "^8.10.108",
"@types/node": "^18.11.2",
"esbuild": "^0.15.11",
"typescript": "^4.8.4"
}
},
"node_modules/@esbuild/android-arm": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.11.tgz",
"integrity": "sha512-PzMcQLazLBkwDEkrNPi9AbjFt6+3I7HKbiYF2XtWQ7wItrHvEOeO3T8Am434zAozWtVP7lrTue1bEfc2nYWeCA==",
"cpu": [
"arm"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-loong64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.15.11.tgz",
"integrity": "sha512-geWp637tUhNmhL3Xgy4Bj703yXB9dqiLJe05lCUfjSFDrQf9C/8pArusyPUbUbPwlC/EAUjBw32sxuIl/11dZw==",
"cpu": [
"loong64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@types/aws-lambda": {
"version": "8.10.108",
"resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.108.tgz",
"integrity": "sha512-1yh1W1WoqK3lGHy+V/Fi55zobxrDHUUsluCWdMlOXkCvtsCmHPXOG+CQ2STIL4B1g6xi6I6XzxaF8V9+zeIFLA==",
"dev": true
},
"node_modules/@types/node": {
"version": "18.11.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.2.tgz",
"integrity": "sha512-BWN3M23gLO2jVG8g/XHIRFWiiV4/GckeFIqbU/C4V3xpoBBWSMk4OZomouN0wCkfQFPqgZikyLr7DOYDysIkkw==",
"dev": true
},
"node_modules/esbuild": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.11.tgz",
"integrity": "sha512-OgHGuhlfZ//mToxjte1D5iiiQgWfJ2GByVMwEC/IuoXsBGkuyK1+KrjYu0laSpnN/L1UmLUCv0s25vObdc1bVg==",
"dev": true,
"hasInstallScript": true,
"bin": {
"esbuild": "bin/esbuild"
},
"engines": {
"node": ">=12"
},
"optionalDependencies": {
"@esbuild/android-arm": "0.15.11",
"@esbuild/linux-loong64": "0.15.11",
"esbuild-android-64": "0.15.11",
"esbuild-android-arm64": "0.15.11",
"esbuild-darwin-64": "0.15.11",
"esbuild-darwin-arm64": "0.15.11",
"esbuild-freebsd-64": "0.15.11",
"esbuild-freebsd-arm64": "0.15.11",
"esbuild-linux-32": "0.15.11",
"esbuild-linux-64": "0.15.11",
"esbuild-linux-arm": "0.15.11",
"esbuild-linux-arm64": "0.15.11",
"esbuild-linux-mips64le": "0.15.11",
"esbuild-linux-ppc64le": "0.15.11",
"esbuild-linux-riscv64": "0.15.11",
"esbuild-linux-s390x": "0.15.11",
"esbuild-netbsd-64": "0.15.11",
"esbuild-openbsd-64": "0.15.11",
"esbuild-sunos-64": "0.15.11",
"esbuild-windows-32": "0.15.11",
"esbuild-windows-64": "0.15.11",
"esbuild-windows-arm64": "0.15.11"
}
},
"node_modules/esbuild-android-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.15.11.tgz",
"integrity": "sha512-rrwoXEiuI1kaw4k475NJpexs8GfJqQUKcD08VR8sKHmuW9RUuTR2VxcupVvHdiGh9ihxL9m3lpqB1kju92Ialw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-android-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.15.11.tgz",
"integrity": "sha512-/hDubOg7BHOhUUsT8KUIU7GfZm5bihqssvqK5PfO4apag7YuObZRZSzViyEKcFn2tPeHx7RKbSBXvAopSHDZJQ==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-darwin-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.11.tgz",
"integrity": "sha512-1DqHD0ms3AhiwkKnjRUzmiW7JnaJJr5FKrPiR7xuyMwnjDqvNWDdMq4rKSD9OC0piFNK6n0LghsglNMe2MwJtA==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-darwin-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.15.11.tgz",
"integrity": "sha512-OMzhxSbS0lwwrW40HHjRCeVIJTURdXFA8c3GU30MlHKuPCcvWNUIKVucVBtNpJySXmbkQMDJdJNrXzNDyvoqvQ==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-freebsd-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.15.11.tgz",
"integrity": "sha512-8dKP26r0/Qyez8nTCwpq60QbuYKOeBygdgOAWGCRalunyeqWRoSZj9TQjPDnTTI9joxd3QYw3UhVZTKxO9QdRg==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-freebsd-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.15.11.tgz",
"integrity": "sha512-aSGiODiukLGGnSg/O9+cGO2QxEacrdCtCawehkWYTt5VX1ni2b9KoxpHCT9h9Y6wGqNHmXFnB47RRJ8BIqZgmQ==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-linux-32": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.15.11.tgz",
"integrity": "sha512-lsrAfdyJBGx+6aHIQmgqUonEzKYeBnyfJPkT6N2dOf1RoXYYV1BkWB6G02tjsrz1d5wZzaTc3cF+TKmuTo/ZwA==",
"cpu": [
"ia32"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-linux-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.15.11.tgz",
"integrity": "sha512-Y2Rh+PcyVhQqXKBTacPCltINN3uIw2xC+dsvLANJ1SpK5NJUtxv8+rqWpjmBgaNWKQT1/uGpMmA9olALy9PLVA==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-linux-arm": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.15.11.tgz",
"integrity": "sha512-TJllTVk5aSyqPFvvcHTvf6Wu1ZKhWpJ/qNmZO8LL/XeB+LXCclm7HQHNEIz6MT7IX8PmlC1BZYrOiw2sXSB95A==",
"cpu": [
"arm"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-linux-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.15.11.tgz",
"integrity": "sha512-uhcXiTwTmD4OpxJu3xC5TzAAw6Wzf9O1XGWL448EE9bqGjgV1j+oK3lIHAfsHnuIn8K4nDW8yjX0Sv5S++oRuw==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-linux-mips64le": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.15.11.tgz",
"integrity": "sha512-WD61y/R1M4BLe4gxXRypoQ0Ci+Vjf714QYzcPNkiYv5I8K8WDz2ZR8Bm6cqKxd6rD+e/rZgPDbhQ9PCf7TMHmA==",
"cpu": [
"mips64el"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-linux-ppc64le": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.15.11.tgz",
"integrity": "sha512-JVleZS9oPVLTlBhPTWgOwxFWU/wMUdlBwTbGA4GF8c38sLbS13cupj+C8bLq929jU7EMWry4SaL+tKGIaTlqKg==",
"cpu": [
"ppc64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-linux-riscv64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.15.11.tgz",
"integrity": "sha512-9aLIalZ2HFHIOZpmVU11sEAS9F8TnHw49daEjcgMpBXHFF57VuT9f9/9LKJhw781Gda0P9jDkuCWJ0tFbErvJw==",
"cpu": [
"riscv64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-linux-s390x": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.15.11.tgz",
"integrity": "sha512-sZHtiXXOKsLI3XGBGoYO4qKBzJlb8xNsWmvFiwFMHFzA4AXgDP1KDp7Dawe9C2pavTRBDvl+Ok4n/DHQ59oaTg==",
"cpu": [
"s390x"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-netbsd-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.15.11.tgz",
"integrity": "sha512-hUC9yN06K9sg7ju4Vgu9ChAPdsEgtcrcLfyNT5IKwKyfpLvKUwCMZSdF+gRD3WpyZelgTQfJ+pDx5XFbXTlB0A==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-openbsd-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.15.11.tgz",
"integrity": "sha512-0bBo9SQR4t66Wd91LGMAqmWorzO0TTzVjYiifwoFtel8luFeXuPThQnEm5ztN4g0fnvcp7AnUPPzS/Depf17wQ==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-sunos-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.11.tgz",
"integrity": "sha512-EuBdTGlsMTjEl1sQnBX2jfygy7iR6CKfvOzi+gEOfhDqbHXsmY1dcpbVtcwHAg9/2yUZSfMJHMAgf1z8M4yyyw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"sunos"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-windows-32": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.15.11.tgz",
"integrity": "sha512-O0/Wo1Wk6dc0rZSxkvGpmTNIycEznHmkObTFz2VHBhjPsO4ZpCgfGxNkCpz4AdAIeMczpTXt/8d5vdJNKEGC+Q==",
"cpu": [
"ia32"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-windows-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.15.11.tgz",
"integrity": "sha512-x977Q4HhNjnHx00b4XLAnTtj5vfbdEvkxaQwC1Zh5AN8g5EX+izgZ6e5QgqJgpzyRNJqh4hkgIJF1pyy1be0mQ==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=12"
}
},
"node_modules/esbuild-windows-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.15.11.tgz",
"integrity": "sha512-VwUHFACuBahrvntdcMKZteUZ9HaYrBRODoKe4tIWxguQRvvYoYb7iu5LrcRS/FQx8KPZNaa72zuqwVtHeXsITw==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=12"
}
},
"node_modules/typescript": {
"version": "4.8.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz",
"integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=4.2.0"
}
}
},
"dependencies": {
"@esbuild/android-arm": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.11.tgz",
"integrity": "sha512-PzMcQLazLBkwDEkrNPi9AbjFt6+3I7HKbiYF2XtWQ7wItrHvEOeO3T8Am434zAozWtVP7lrTue1bEfc2nYWeCA==",
"dev": true,
"optional": true
},
"@esbuild/linux-loong64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.15.11.tgz",
"integrity": "sha512-geWp637tUhNmhL3Xgy4Bj703yXB9dqiLJe05lCUfjSFDrQf9C/8pArusyPUbUbPwlC/EAUjBw32sxuIl/11dZw==",
"dev": true,
"optional": true
},
"@types/aws-lambda": {
"version": "8.10.108",
"resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.108.tgz",
"integrity": "sha512-1yh1W1WoqK3lGHy+V/Fi55zobxrDHUUsluCWdMlOXkCvtsCmHPXOG+CQ2STIL4B1g6xi6I6XzxaF8V9+zeIFLA==",
"dev": true
},
"@types/node": {
"version": "18.11.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.2.tgz",
"integrity": "sha512-BWN3M23gLO2jVG8g/XHIRFWiiV4/GckeFIqbU/C4V3xpoBBWSMk4OZomouN0wCkfQFPqgZikyLr7DOYDysIkkw==",
"dev": true
},
"esbuild": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.11.tgz",
"integrity": "sha512-OgHGuhlfZ//mToxjte1D5iiiQgWfJ2GByVMwEC/IuoXsBGkuyK1+KrjYu0laSpnN/L1UmLUCv0s25vObdc1bVg==",
"dev": true,
"requires": {
"@esbuild/android-arm": "0.15.11",
"@esbuild/linux-loong64": "0.15.11",
"esbuild-android-64": "0.15.11",
"esbuild-android-arm64": "0.15.11",
"esbuild-darwin-64": "0.15.11",
"esbuild-darwin-arm64": "0.15.11",
"esbuild-freebsd-64": "0.15.11",
"esbuild-freebsd-arm64": "0.15.11",
"esbuild-linux-32": "0.15.11",
"esbuild-linux-64": "0.15.11",
"esbuild-linux-arm": "0.15.11",
"esbuild-linux-arm64": "0.15.11",
"esbuild-linux-mips64le": "0.15.11",
"esbuild-linux-ppc64le": "0.15.11",
"esbuild-linux-riscv64": "0.15.11",
"esbuild-linux-s390x": "0.15.11",
"esbuild-netbsd-64": "0.15.11",
"esbuild-openbsd-64": "0.15.11",
"esbuild-sunos-64": "0.15.11",
"esbuild-windows-32": "0.15.11",
"esbuild-windows-64": "0.15.11",
"esbuild-windows-arm64": "0.15.11"
}
},
"esbuild-android-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.15.11.tgz",
"integrity": "sha512-rrwoXEiuI1kaw4k475NJpexs8GfJqQUKcD08VR8sKHmuW9RUuTR2VxcupVvHdiGh9ihxL9m3lpqB1kju92Ialw==",
"dev": true,
"optional": true
},
"esbuild-android-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.15.11.tgz",
"integrity": "sha512-/hDubOg7BHOhUUsT8KUIU7GfZm5bihqssvqK5PfO4apag7YuObZRZSzViyEKcFn2tPeHx7RKbSBXvAopSHDZJQ==",
"dev": true,
"optional": true
},
"esbuild-darwin-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.11.tgz",
"integrity": "sha512-1DqHD0ms3AhiwkKnjRUzmiW7JnaJJr5FKrPiR7xuyMwnjDqvNWDdMq4rKSD9OC0piFNK6n0LghsglNMe2MwJtA==",
"dev": true,
"optional": true
},
"esbuild-darwin-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.15.11.tgz",
"integrity": "sha512-OMzhxSbS0lwwrW40HHjRCeVIJTURdXFA8c3GU30MlHKuPCcvWNUIKVucVBtNpJySXmbkQMDJdJNrXzNDyvoqvQ==",
"dev": true,
"optional": true
},
"esbuild-freebsd-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.15.11.tgz",
"integrity": "sha512-8dKP26r0/Qyez8nTCwpq60QbuYKOeBygdgOAWGCRalunyeqWRoSZj9TQjPDnTTI9joxd3QYw3UhVZTKxO9QdRg==",
"dev": true,
"optional": true
},
"esbuild-freebsd-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.15.11.tgz",
"integrity": "sha512-aSGiODiukLGGnSg/O9+cGO2QxEacrdCtCawehkWYTt5VX1ni2b9KoxpHCT9h9Y6wGqNHmXFnB47RRJ8BIqZgmQ==",
"dev": true,
"optional": true
},
"esbuild-linux-32": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.15.11.tgz",
"integrity": "sha512-lsrAfdyJBGx+6aHIQmgqUonEzKYeBnyfJPkT6N2dOf1RoXYYV1BkWB6G02tjsrz1d5wZzaTc3cF+TKmuTo/ZwA==",
"dev": true,
"optional": true
},
"esbuild-linux-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.15.11.tgz",
"integrity": "sha512-Y2Rh+PcyVhQqXKBTacPCltINN3uIw2xC+dsvLANJ1SpK5NJUtxv8+rqWpjmBgaNWKQT1/uGpMmA9olALy9PLVA==",
"dev": true,
"optional": true
},
"esbuild-linux-arm": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.15.11.tgz",
"integrity": "sha512-TJllTVk5aSyqPFvvcHTvf6Wu1ZKhWpJ/qNmZO8LL/XeB+LXCclm7HQHNEIz6MT7IX8PmlC1BZYrOiw2sXSB95A==",
"dev": true,
"optional": true
},
"esbuild-linux-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.15.11.tgz",
"integrity": "sha512-uhcXiTwTmD4OpxJu3xC5TzAAw6Wzf9O1XGWL448EE9bqGjgV1j+oK3lIHAfsHnuIn8K4nDW8yjX0Sv5S++oRuw==",
"dev": true,
"optional": true
},
"esbuild-linux-mips64le": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.15.11.tgz",
"integrity": "sha512-WD61y/R1M4BLe4gxXRypoQ0Ci+Vjf714QYzcPNkiYv5I8K8WDz2ZR8Bm6cqKxd6rD+e/rZgPDbhQ9PCf7TMHmA==",
"dev": true,
"optional": true
},
"esbuild-linux-ppc64le": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.15.11.tgz",
"integrity": "sha512-JVleZS9oPVLTlBhPTWgOwxFWU/wMUdlBwTbGA4GF8c38sLbS13cupj+C8bLq929jU7EMWry4SaL+tKGIaTlqKg==",
"dev": true,
"optional": true
},
"esbuild-linux-riscv64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.15.11.tgz",
"integrity": "sha512-9aLIalZ2HFHIOZpmVU11sEAS9F8TnHw49daEjcgMpBXHFF57VuT9f9/9LKJhw781Gda0P9jDkuCWJ0tFbErvJw==",
"dev": true,
"optional": true
},
"esbuild-linux-s390x": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.15.11.tgz",
"integrity": "sha512-sZHtiXXOKsLI3XGBGoYO4qKBzJlb8xNsWmvFiwFMHFzA4AXgDP1KDp7Dawe9C2pavTRBDvl+Ok4n/DHQ59oaTg==",
"dev": true,
"optional": true
},
"esbuild-netbsd-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.15.11.tgz",
"integrity": "sha512-hUC9yN06K9sg7ju4Vgu9ChAPdsEgtcrcLfyNT5IKwKyfpLvKUwCMZSdF+gRD3WpyZelgTQfJ+pDx5XFbXTlB0A==",
"dev": true,
"optional": true
},
"esbuild-openbsd-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.15.11.tgz",
"integrity": "sha512-0bBo9SQR4t66Wd91LGMAqmWorzO0TTzVjYiifwoFtel8luFeXuPThQnEm5ztN4g0fnvcp7AnUPPzS/Depf17wQ==",
"dev": true,
"optional": true
},
"esbuild-sunos-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.11.tgz",
"integrity": "sha512-EuBdTGlsMTjEl1sQnBX2jfygy7iR6CKfvOzi+gEOfhDqbHXsmY1dcpbVtcwHAg9/2yUZSfMJHMAgf1z8M4yyyw==",
"dev": true,
"optional": true
},
"esbuild-windows-32": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.15.11.tgz",
"integrity": "sha512-O0/Wo1Wk6dc0rZSxkvGpmTNIycEznHmkObTFz2VHBhjPsO4ZpCgfGxNkCpz4AdAIeMczpTXt/8d5vdJNKEGC+Q==",
"dev": true,
"optional": true
},
"esbuild-windows-64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.15.11.tgz",
"integrity": "sha512-x977Q4HhNjnHx00b4XLAnTtj5vfbdEvkxaQwC1Zh5AN8g5EX+izgZ6e5QgqJgpzyRNJqh4hkgIJF1pyy1be0mQ==",
"dev": true,
"optional": true
},
"esbuild-windows-arm64": {
"version": "0.15.11",
"resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.15.11.tgz",
"integrity": "sha512-VwUHFACuBahrvntdcMKZteUZ9HaYrBRODoKe4tIWxguQRvvYoYb7iu5LrcRS/FQx8KPZNaa72zuqwVtHeXsITw==",
"dev": true,
"optional": true
},
"typescript": {
"version": "4.8.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz",
"integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==",
"dev": true
}
}
}

View File

@@ -0,0 +1,15 @@
{
"name": "pmtiles-aws",
"version": "0.0.0",
"devDependencies": {
"@types/aws-lambda": "^8.10.108",
"@types/node": "^18.11.2",
"esbuild": "^0.15.11",
"typescript": "^4.8.4"
},
"private": true,
"scripts": {
"tsc": "tsc --noEmit --watch",
"build": "esbuild src/index.ts --target=es2020 --outfile=dist/index.js --format=cjs --bundle --platform=node --target=node16 --external:/var/runtime/node_modules/aws-sdk/clients/s3.js --banner:js=//$(git describe --always) && cd dist && zip lambda_function.zip index.js"
}
}

178
serverless/aws/src/index.ts Normal file
View File

@@ -0,0 +1,178 @@
import { Readable } from "stream";
import {
Context,
APIGatewayProxyResult,
APIGatewayProxyEventV2,
} from "aws-lambda";
import {
PMTiles,
ResolvedValueCache,
RangeResponse,
Source,
Compression,
} from "../../../js";
// @ts-ignore
import https from "https";
// @ts-ignore
import s3client from "/var/runtime/node_modules/aws-sdk/clients/s3.js";
const keepAliveAgent = new https.Agent({ keepAlive: true });
const s3 = new s3client({
region: process.env.BUCKET_REGION!,
httpOptions: { agent: keepAliveAgent },
});
// TODO: figure out how much memory to allocate
const CACHE = new ResolvedValueCache();
// duplicated code below
export const pmtiles_path = (name: string, setting?: string): string => {
if (setting) {
return setting.replace("{name}", name);
}
return name + ".pmtiles";
};
const TILE =
/^\/(?<NAME>[0-9a-zA-Z\/!\-_\.\*\'\(\)]+)\/(?<Z>\d+)\/(?<X>\d+)\/(?<Y>\d+).(?<EXT>[a-z]+)$/;
export const tile_path = (
path: string,
setting?: string
): {
ok: boolean;
name: string;
tile: [number, number, number];
ext: string;
} => {
let pattern = TILE;
if (setting) {
// escape regex
setting = setting.replace(/[.*+?^$()|[\]\\]/g, "\\$&");
setting = setting.replace("{name}", "(?<NAME>[0-9a-zA-Z/!-_.*'()]+)");
setting = setting.replace("{z}", "(?<Z>\\d+)");
setting = setting.replace("{x}", "(?<X>\\d+)");
setting = setting.replace("{y}", "(?<Y>\\d+)");
setting = setting.replace("{ext}", "(?<EXT>[a-z]+)");
pattern = new RegExp(setting);
}
let match = path.match(pattern);
if (match) {
const g = match.groups!;
return { ok: true, name: g.NAME, tile: [+g.Z, +g.X, +g.Y], ext: g.EXT };
}
return { ok: false, name: "", tile: [0, 0, 0], ext: "" };
};
class S3Source implements Source {
archive_name: string;
constructor(archive_name: string) {
this.archive_name = archive_name;
}
getKey() {
return "";
}
async getBytes(offset: number, length: number): Promise<RangeResponse> {
const resp = await s3
.getObject({
Bucket: process.env.BUCKET!,
Key: pmtiles_path(this.archive_name, process.env.PMTILES_PATH),
Range: "bytes=" + offset + "-" + (offset + length - 1),
})
.promise();
return { data: resp!.Body.buffer };
}
}
interface Headers {
[key: string]: string;
}
const apiResp = (
statusCode: number,
body: string,
isBase64Encoded = false,
headers: Headers = {}
): APIGatewayProxyResult => {
return {
statusCode: statusCode,
body: body,
headers: headers,
isBase64Encoded: isBase64Encoded,
};
};
// Assumes event is a API Gateway V2 or Lambda Function URL formatted dict
// and returns API Gateway V2 / Lambda Function dict responses
// Does not work with CloudFront events/Lambda@Edge; see README
export const handler = async (
event: APIGatewayProxyEventV2,
context: Context
): Promise<APIGatewayProxyResult> => {
var path;
var is_api_gateway;
if (event.pathParameters) {
is_api_gateway = true;
if (event.pathParameters.proxy) {
path = "/" + event.pathParameters.proxy;
} else {
return apiResp(500, "Proxy integration missing tile_path parameter");
}
} else {
path = event.rawPath;
}
if (!path) {
return apiResp(500, "Invalid event configuration");
}
const { ok, name, tile, ext } = tile_path(path, process.env.TILE_PATH);
if (!ok) {
return apiResp(400, "Invalid tile URL");
}
var headers: Headers = {};
if (process.env.CORS) {
headers["Access-Control-Allow-Origin"] = process.env.CORS;
}
// TODO: extension enforcement and MIME types
const source = new S3Source(name);
const p = new PMTiles(source, CACHE);
try {
const header = await p.getHeader();
// TODO optimize by checking min/max zoom, return 404
headers["Content-Type"] = "application/vnd.vector-tile";
const tile = await p.getZxy(0, 0, 0);
if (tile) {
// returns uncompressed response
// TODO: may need to special case API gateway to return compressed response with gzip content-encoding header
return apiResp(
200,
Buffer.from(tile.data).toString("base64"),
true,
headers
);
} else {
return apiResp(204, "", false, headers);
}
} catch (e) {
if ((e as Error).name === "AccessDenied") {
return apiResp(403, "Bucket access unauthorized");
}
throw e;
}
return apiResp(404, "Invalid URL");
};

View File

@@ -1,50 +0,0 @@
import unittest
from util import parse_tile_path, pmtiles_path
class TestLambda(unittest.TestCase):
def test_parse_tile_default(self):
name, tile = parse_tile_path(None, "abcd")
self.assertEqual(tile, None)
name, tile = parse_tile_path(None, "/foo/11/22/33.pbf")
self.assertEqual(name, "foo")
self.assertEqual(tile.z, 11)
self.assertEqual(tile.x, 22)
self.assertEqual(tile.y, 33)
def test_parse_tile_path_setting(self):
name, tile = parse_tile_path("/{name}/{z}/{y}/{x}.pbf", "/foo/11/22/33.pbf")
self.assertEqual(tile.x, 33)
self.assertEqual(tile.y, 22)
name, tile = parse_tile_path(
"/tiles/{name}/{z}/{x}/{y}.mvt", "/tiles/foo/4/2/3.mvt"
)
self.assertEqual(name, "foo")
self.assertEqual(tile.z, 4)
self.assertEqual(tile.x, 2)
self.assertEqual(tile.y, 3)
def test_parse_tile_path_setting_special_chars(self):
name, tile = parse_tile_path(
"/folder(new/{name}/{z}/{y}/{x}.pbf", "/folder(new/foo/11/22/33.pbf"
)
self.assertEqual(name, "foo")
def test_parse_tile_path_setting_slash(self):
name, tile = parse_tile_path("/{name}/{z}/{y}/{x}.pbf", "/foo/bar/11/22/33.pbf")
self.assertEqual(name, "foo/bar")
def test_pmtiles_path(self):
self.assertEqual(pmtiles_path(None, "foo"), "foo.pmtiles")
self.assertEqual(
pmtiles_path("folder/{name}/file.pmtiles", "foo"),
"folder/foo/file.pmtiles",
)
def test_pmtiles_path_slash(self):
self.assertEqual(
pmtiles_path("folder/{name}.pmtiles", "foo/bar"),
"folder/foo/bar.pmtiles",
)

View File

@@ -0,0 +1,16 @@
{
"compilerOptions": {
"target": "es2020",
"strict": true,
"preserveConstEnums": true,
"noEmit": true,
"sourceMap": false,
"module":"commonjs",
"moduleResolution":"node",
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
},
"exclude": ["node_modules"]
}

View File

@@ -1,27 +0,0 @@
import collections
import re
Zxy = collections.namedtuple("Zxy", ["z", "x", "y"])
def pmtiles_path(p, name):
if not p:
p = "{name}.pmtiles"
return p.replace("{name}", name)
def parse_tile_path(p, str):
if not p:
p = "/{name}/{z}/{x}/{y}.pbf"
p = re.escape(p)
p = p.replace(r"\{name\}", r"(?P<name>[0-9a-zA-Z/!\-_\.\*'\(\)]+)")
p = p.replace(r"\{z\}", r"(?P<z>\d+)")
p = p.replace(r"\{x\}", r"(?P<x>\d+)")
p = p.replace(r"\{y\}", r"(?P<y>\d+)")
m = re.match(f"^{p}$", str)
if not m:
return None, None
return (
m.group("name"),
Zxy(int(m.group("z")), int(m.group("x")), int(m.group("y"))),
)