From be57f1f61d3708b862f359a53f9c0de0672659aa Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Wed, 20 Jul 2022 16:13:14 +0800 Subject: [PATCH] improvements to CI test suite --- .github/workflows/actions.yml | 4 +- serverless/aws/create_lambda_function.py | 1 + serverless/aws/lambda_function.py | 28 +---- .../{test_lambda_function.py => test_util.py} | 6 +- serverless/aws/util.py | 27 +++++ serverless/cloudflare/package-lock.json | 101 +++++++++++++++++- serverless/cloudflare/package.json | 5 +- serverless/cloudflare/worker.test.ts | 5 + 8 files changed, 142 insertions(+), 35 deletions(-) rename serverless/aws/{test_lambda_function.py => test_util.py} (94%) create mode 100644 serverless/aws/util.py create mode 100644 serverless/cloudflare/worker.test.ts diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 8bbdc95..210e1b9 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -39,4 +39,6 @@ jobs: with: node-version: 16.x - run: cd js && npm install && npm test - - run : cd python && python -m unittest test/test_* + - run: cd python && python -m unittest test/test_* + - run: cd serverless/aws && python -m unittest test_*.py + - run: cd serverless/cloudflare && npm install && npm test diff --git a/serverless/aws/create_lambda_function.py b/serverless/aws/create_lambda_function.py index b3a07c1..6850d7d 100644 --- a/serverless/aws/create_lambda_function.py +++ b/serverless/aws/create_lambda_function.py @@ -5,6 +5,7 @@ sha = subprocess.check_output(["git", "describe", "--always"]).strip() with zipfile.ZipFile("lambda_function.zip", "w", zipfile.ZIP_DEFLATED) as z: z.write("lambda_function.py") + z.write("util.py") z.write("../../python/pmtiles/reader.py", "pmtiles/reader.py") z.writestr("version",sha) diff --git a/serverless/aws/lambda_function.py b/serverless/aws/lambda_function.py index df03eb1..53c312e 100644 --- a/serverless/aws/lambda_function.py +++ b/serverless/aws/lambda_function.py @@ -1,10 +1,8 @@ import base64 -import collections from functools import lru_cache import gzip import json import os -import re # Exists inside all lambda functions import boto3 @@ -12,8 +10,7 @@ from botocore.exceptions import ClientError # create_lambda_function.py will vendor the relevant file from pmtiles.reader import Reader - -Zxy = collections.namedtuple("Zxy", ["z", "x", "y"]) +from util import pmtiles_path, parse_tile_path s3 = boto3.client("s3") @@ -33,29 +30,6 @@ def get_object_bytes(key, offset, length): ) -def pmtiles_path(p, name): - if not p: - p = "{name}.pmtiles" - return p.replace("{name}", name) - - -def parse_tile_path(p, str): - if not p: - p = "/{name}/{z}/{x}/{y}.pbf" - p = re.escape(p) - p = p.replace(r"\{name\}", r"(?P[0-9a-zA-Z/!\-_\.\*'\(\)]+)") - p = p.replace(r"\{z\}", r"(?P\d+)") - p = p.replace(r"\{x\}", r"(?P\d+)") - p = p.replace(r"\{y\}", r"(?P\d+)") - m = re.match(f"^{p}$", str) - if not m: - return None, None - return ( - m.group("name"), - Zxy(int(m.group("z")), int(m.group("x")), int(m.group("y"))), - ) - - # Assumes event is a API Gateway V2 or Lambda Function URL formatted dict # and returns API Gateway V2 / Lambda Function dict responses # Does not work with CloudFront events/Lambda@Edge; see README diff --git a/serverless/aws/test_lambda_function.py b/serverless/aws/test_util.py similarity index 94% rename from serverless/aws/test_lambda_function.py rename to serverless/aws/test_util.py index 6fe2c19..e0b43c5 100644 --- a/serverless/aws/test_lambda_function.py +++ b/serverless/aws/test_util.py @@ -1,5 +1,5 @@ import unittest -from lambda_function import parse_tile_path, pmtiles_path +from util import parse_tile_path, pmtiles_path class TestLambda(unittest.TestCase): @@ -48,7 +48,3 @@ class TestLambda(unittest.TestCase): pmtiles_path("folder/{name}.pmtiles", "foo/bar"), "folder/foo/bar.pmtiles", ) - - -if __name__ == "__main__": - unittest.main() diff --git a/serverless/aws/util.py b/serverless/aws/util.py new file mode 100644 index 0000000..c8ac0f4 --- /dev/null +++ b/serverless/aws/util.py @@ -0,0 +1,27 @@ +import collections +import re + +Zxy = collections.namedtuple("Zxy", ["z", "x", "y"]) + + +def pmtiles_path(p, name): + if not p: + p = "{name}.pmtiles" + return p.replace("{name}", name) + + +def parse_tile_path(p, str): + if not p: + p = "/{name}/{z}/{x}/{y}.pbf" + p = re.escape(p) + p = p.replace(r"\{name\}", r"(?P[0-9a-zA-Z/!\-_\.\*'\(\)]+)") + p = p.replace(r"\{z\}", r"(?P\d+)") + p = p.replace(r"\{x\}", r"(?P\d+)") + p = p.replace(r"\{y\}", r"(?P\d+)") + m = re.match(f"^{p}$", str) + if not m: + return None, None + return ( + m.group("name"), + Zxy(int(m.group("z")), int(m.group("x")), int(m.group("y"))), + ) diff --git a/serverless/cloudflare/package-lock.json b/serverless/cloudflare/package-lock.json index 46870a4..5a32aaa 100644 --- a/serverless/cloudflare/package-lock.json +++ b/serverless/cloudflare/package-lock.json @@ -6,9 +6,17 @@ "": { "devDependencies": { "esbuild": "^0.14.42", - "typescript": "^4.7.2" + "esbuild-runner": "^2.2.1", + "typescript": "^4.7.2", + "zora": "^5.0.2" } }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, "node_modules/esbuild": { "version": "0.14.42", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.14.42.tgz", @@ -300,6 +308,22 @@ "node": ">=12" } }, + "node_modules/esbuild-runner": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/esbuild-runner/-/esbuild-runner-2.2.1.tgz", + "integrity": "sha512-VP0VfJJZiZ3cKzdOH59ZceDxx/GzBKra7tiGM8MfFMLv6CR1/cpsvtQ3IsJI3pz7HyeYxtbPyecj3fHwR+3XcQ==", + "dev": true, + "dependencies": { + "source-map-support": "0.5.19", + "tslib": "2.3.1" + }, + "bin": { + "esr": "bin/esr.js" + }, + "peerDependencies": { + "esbuild": "*" + } + }, "node_modules/esbuild-sunos-64": { "version": "0.14.42", "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.42.tgz", @@ -364,6 +388,31 @@ "node": ">=12" } }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/tslib": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", + "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==", + "dev": true + }, "node_modules/typescript": { "version": "4.7.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.2.tgz", @@ -376,9 +425,21 @@ "engines": { "node": ">=4.2.0" } + }, + "node_modules/zora": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/zora/-/zora-5.0.4.tgz", + "integrity": "sha512-yRd/0VNjEjBoV53zOnIYelo7iyywCbDfF6PqbvTanbQXTdm0XiBagR4H+eVQaoVwgmUa/0A1WRiphkAv2l96lg==", + "dev": true } }, "dependencies": { + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, "esbuild": { "version": "0.14.42", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.14.42.tgz", @@ -519,6 +580,16 @@ "dev": true, "optional": true }, + "esbuild-runner": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/esbuild-runner/-/esbuild-runner-2.2.1.tgz", + "integrity": "sha512-VP0VfJJZiZ3cKzdOH59ZceDxx/GzBKra7tiGM8MfFMLv6CR1/cpsvtQ3IsJI3pz7HyeYxtbPyecj3fHwR+3XcQ==", + "dev": true, + "requires": { + "source-map-support": "0.5.19", + "tslib": "2.3.1" + } + }, "esbuild-sunos-64": { "version": "0.14.42", "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.42.tgz", @@ -547,11 +618,39 @@ "dev": true, "optional": true }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "tslib": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", + "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==", + "dev": true + }, "typescript": { "version": "4.7.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.2.tgz", "integrity": "sha512-Mamb1iX2FDUpcTRzltPxgWMKy3fhg0TN378ylbktPGPK/99KbDtMQ4W1hwgsbPAsG3a0xKa1vmw4VKZQbkvz5A==", "dev": true + }, + "zora": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/zora/-/zora-5.0.4.tgz", + "integrity": "sha512-yRd/0VNjEjBoV53zOnIYelo7iyywCbDfF6PqbvTanbQXTdm0XiBagR4H+eVQaoVwgmUa/0A1WRiphkAv2l96lg==", + "dev": true } } } diff --git a/serverless/cloudflare/package.json b/serverless/cloudflare/package.json index b90ef73..5ea4aef 100644 --- a/serverless/cloudflare/package.json +++ b/serverless/cloudflare/package.json @@ -1,10 +1,13 @@ { "devDependencies": { "esbuild": "^0.14.42", - "typescript": "^4.7.2" + "esbuild-runner": "^2.2.1", + "typescript": "^4.7.2", + "zora": "^5.0.2" }, "scripts": { "build": "esbuild worker.ts --target=es2020 --outfile=dist/worker.js --format=esm --bundle --banner:js=//$(git describe --always)", + "test": "node -r esbuild-runner/register worker.test.ts", "tsc": "tsc --noEmit --watch" } } diff --git a/serverless/cloudflare/worker.test.ts b/serverless/cloudflare/worker.test.ts new file mode 100644 index 0000000..0bdb080 --- /dev/null +++ b/serverless/cloudflare/worker.test.ts @@ -0,0 +1,5 @@ +import { test } from "zora"; + +test("stub test", (assertion) => { + assertion.ok(true); +});