diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..6718dcf --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,21 @@ +--- +name: Docs +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + markdown-link-check: + name: Broken Links + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Run link check + uses: gaurav-nelson/github-action-markdown-link-check@1.0.13 + with: + use-quiet-mode: 'no' + use-verbose-mode: 'yes' + config-file: '.github/workflows/docs_mlc_config.json' diff --git a/.github/workflows/docs_mlc_config.json b/.github/workflows/docs_mlc_config.json new file mode 100644 index 0000000..2020362 --- /dev/null +++ b/.github/workflows/docs_mlc_config.json @@ -0,0 +1,7 @@ +{ + "ignorePatterns": [ + { + "pattern": "^http://localhost.*$" + } + ] +} diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 0000000..f60cb53 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,202 @@ +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: CI + +on: + pull_request: + branches: [ main ] + +jobs: + # When spotless:apply fails, the error message is a bit cryptic, so try to make it obvious that + # is the problem by putting the check into a standalone job + lint: + name: Check formatting + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout parent Planetiler repo + uses: actions/checkout@v3 + with: + repository: lazaa32/planetiler + ref: omt_submodule + path: planetiler + - name: Checkout this PR planetiler-openmaptiles repo + uses: actions/checkout@v3 + with: + path: planetiler-openmaptiles + - name: Move planetiler-openmaptiles into planetiler + run: mv planetiler-openmaptiles planetiler + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - name: Ensure code formatted with mvn spotless:apply + working-directory: planetiler + run: ./mvnw -DskipTests --batch-mode -no-transfer-progress spotless:check + + build: + name: Java ${{ matrix.jdk }} / ${{ matrix.os }} ${{ matrix.args }} + # Wait until after we check that you ran mvn spotless:apply, otherwise will fail with a cryptic error message + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] + jdk: [ 17 ] + include: + - os: ubuntu-latest + jdk: 16 + - os: ubuntu-latest + jdk: 18 + - os: ubuntu-latest + jdk: 17 + args: "-DargLine='-Duser.language=fr -Duser.country=FR'" + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + steps: + - name: Checkout parent Planetiler repo + uses: actions/checkout@v3 + with: + repository: lazaa32/planetiler + ref: omt_submodule + path: planetiler + - name: Checkout this PR planetiler-openmaptiles repo + uses: actions/checkout@v3 + with: + path: planetiler-openmaptiles + - name: Move planetiler-openmaptiles into planetiler (linux/mac) + if: ${{ !contains(matrix.os, 'windows') }} + run: mv planetiler-openmaptiles planetiler + - name: Move planetiler-openmaptiles into planetiler (windows) + if: ${{ contains(matrix.os, 'windows') }} + run: mv -Force planetiler-openmaptiles planetiler + - name: Set up JDK ${{ matrix.jdk }} + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.jdk }} + distribution: 'temurin' + cache: 'maven' + # Skip spotless since that gets checked in a separate task + - name: Build with mvnw (linux/mac) + if: ${{ !contains(matrix.os, 'windows') }} + working-directory: planetiler + run: ./mvnw ${{matrix.args}} -Dspotless.check.skip --batch-mode -no-transfer-progress package verify jib:buildTar --file pom.xml + - name: Build with mvnw.cmd (windows) + if: ${{ contains(matrix.os, 'windows') }} + working-directory: planetiler + run: mvnw.cmd ${{matrix.args}} -Dspotless.check.skip --batch-mode -no-transfer-progress package verify jib:buildTar --file pom.xml + shell: cmd + + regenerate: + name: Regenerate + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout parent Planetiler repo + uses: actions/checkout@v3 + with: + repository: lazaa32/planetiler + ref: omt_submodule + path: planetiler + - name: Checkout this PR planetiler-openmaptiles repo + uses: actions/checkout@v3 + with: + path: planetiler-openmaptiles + - name: Move planetiler-openmaptiles into planetiler + run: mv planetiler-openmaptiles planetiler + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - run: ./scripts/regenerate-openmaptiles.sh + working-directory: planetiler + # Skip spotless since that gets checked in a separate task + - run: ./mvnw -Dspotless.check.skip -DskipTests --batch-mode -no-transfer-progress clean install -pl planetiler-openmaptiles -am + working-directory: planetiler + - run: ./mvnw -Dspotless.check.skip --batch-mode -no-transfer-progress verify -pl planetiler-openmaptiles + working-directory: planetiler + + examples: + name: Example project + runs-on: ubuntu-latest + timeout-minutes: 15 + continue-on-error: true + steps: + - name: Checkout parent Planetiler repo + uses: actions/checkout@v3 + with: + repository: lazaa32/planetiler + ref: omt_submodule + path: planetiler + - name: Checkout this PR planetiler-openmaptiles repo + uses: actions/checkout@v3 + with: + path: planetiler-openmaptiles + - name: Move planetiler-openmaptiles into planetiler + run: mv planetiler-openmaptiles planetiler + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'temurin' + - name: Build and test + run: mvn --batch-mode -no-transfer-progress package --file standalone.pom.xml + working-directory: planetiler/planetiler-examples + - name: Find jar + run: mv target/*with-deps.jar ./run.jar + working-directory: planetiler/planetiler-examples + - name: Run + run: java -jar run.jar --osm-path=../planetiler-core/src/test/resources/monaco-latest.osm.pbf --mbtiles=data/out.mbtiles + working-directory: planetiler/planetiler-examples + - name: Verify + run: java -cp run.jar com.onthegomap.planetiler.mbtiles.Verify data/out.mbtiles + working-directory: planetiler/planetiler-examples + + run: + name: Build / Run + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout parent Planetiler repo + uses: actions/checkout@v3 + with: + repository: lazaa32/planetiler + ref: omt_submodule + path: planetiler + - name: Checkout this PR planetiler-openmaptiles repo + uses: actions/checkout@v3 + with: + path: planetiler-openmaptiles + - name: Move planetiler-openmaptiles into planetiler + run: mv planetiler-openmaptiles planetiler + - name: Cache data/sources + uses: ./planetiler/.github/cache-sources-action + - name: Set up JDK + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - name: Build this branch + working-directory: planetiler + run: ./mvnw -DskipTests -Dimage.version=CI_ONLY --batch-mode -no-transfer-progress package jib:dockerBuild --file pom.xml + - name: 'Upload artifact' + uses: actions/upload-artifact@v3 + with: + name: planetiler-build + path: planetiler/planetiler-dist/target/*with-deps.jar + - name: 'Download data (java)' + working-directory: planetiler + run: java -jar planetiler-dist/target/*with-deps.jar --only-download --area=monaco + - name: 'Download wikidata (java)' + working-directory: planetiler + run: java -jar planetiler-dist/target/*with-deps.jar --only-fetch-wikidata --area=monaco + - name: 'Verify build' + working-directory: planetiler + run: ./scripts/test-release.sh CI_ONLY + env: + SKIP_EXAMPLE_PROJECT: true diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 0000000..33aa6fd --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,111 @@ +# This workflow builds a map using the base and branch commit of a PR and uploads +# the logs as an artifact that update-pr.yml uses to add back as a comment. + +name: Performance + +on: + pull_request: + branches: [ main ] + +env: + # For performance tests, run this branch against main with: + AREA: rhode island + RAM: 4g + # Also pick up a good chunk of the atlantic ocean to catch any regressions in repeated tile performance + # Omit to infer from .osm.pbf bounds + BOUNDS_ARG: "--bounds=-74.07,21.34,-17.84,43.55" + +jobs: + performance: + name: Performance Test + runs-on: ubuntu-latest + timeout-minutes: 20 + continue-on-error: true + steps: + - name: 'Cancel previous runs' + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + - name: 'Checkout parent Planetiler repo for branch' + uses: actions/checkout@v3 + with: + repository: lazaa32/planetiler + ref: omt_submodule + path: branch/planetiler + - name: 'Checkout parent Planetiler repo for base' + uses: actions/checkout@v3 + with: + repository: lazaa32/planetiler + ref: omt_submodule + path: base/planetiler + + - name: 'Checkout PT-OMT PR branch' + uses: actions/checkout@v3 + with: + path: branch/planetiler-openmaptiles + - name: 'Move planetiler-openmaptiles into planetiler for base' + run: mv branch/planetiler-openmaptiles branch/planetiler + + - name: 'Checkout PT-OMT base' + uses: actions/checkout@v3 + with: + path: base/planetiler-openmaptiles + ref: ${{ github.event.pull_request.base.sha }} + - name: 'Move planetiler-openmaptiles into planetiler for base' + run: mv base/planetiler-openmaptiles base/planetiler + + - name: 'Cache data/sources' + uses: ./branch/planetiler/.github/cache-sources-action + with: + basedir: branch + - name: 'Set up JDK' + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - uses: actions/setup-node@v3 + with: + node-version: '14' + - run: npm install -g strip-ansi-cli@3.0.2 + + - name: 'Build branch' + run: ./scripts/build.sh + working-directory: branch/planetiler + - name: 'Build base' + run: ./scripts/build.sh + working-directory: base/planetiler + + - name: 'Download data' + run: | + set -eo pipefail + cp base/planetiler/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + cp branch/planetiler/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + + - name: 'Store build info' + run: | + mkdir build-info + echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha + echo "${{ github.sha }}" > build-info/branch_sha + echo "${{ github.event.number }}" > build-info/pull_request_number + + - name: 'Run branch' + run: | + rm -rf data/out.mbtiles data/tmp + cp branch/planetiler/planetiler-dist/target/*with-deps.jar run.jar + java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log + ls -alh run.jar | tee -a log + cat log | strip-ansi > build-info/branchlogs.txt + - name: 'Run base' + run: | + rm -rf data/out.mbtiles data/tmp + cp base/planetiler/planetiler-dist/target/*with-deps.jar run.jar + java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log + ls -alh run.jar | tee -a log + cat log | strip-ansi > build-info/baselogs.txt + + - name: 'Upload build-info' + uses: actions/upload-artifact@v3 + with: + name: build-info + path: ./build-info diff --git a/.github/workflows/update-pr.yml b/.github/workflows/update-pr.yml new file mode 100644 index 0000000..c21b42a --- /dev/null +++ b/.github/workflows/update-pr.yml @@ -0,0 +1,89 @@ +# This workflow posts the result of a performance test back to the pull request as a comment. +# Needs to be separate from CI because it has elevated privileges so only runs from main branch. + +name: Update PR + +on: + workflow_run: + workflows: [ "Performance" ] + types: + - completed + +jobs: + updatepr: + runs-on: ubuntu-latest + continue-on-error: true + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} + timeout-minutes: 5 + steps: + # report status back to pull request + - uses: haya14busa/action-workflow_run-status@v1 + - uses: actions/checkout@v3 + - name: 'Download branch build info' + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ${{ github.event.workflow_run.workflow_id }} + run_id: ${{ github.event.workflow_run.id }} + name: build-info + path: build-info + repo: openmaptiles/planetiler-openmaptiles + - name: 'Get build info' + id: build_info + run: echo "::set-output name=pr_number::$(cat build-info/pull_request_number)" + - name: 'Build comment-body' + run: | + cat build-info/branchlogs.txt | sed -n '/^.*Tile stats/,$p' > branchsummary.txt + cat build-info/branchlogs.txt | sed -n '/^.*Exception in thread/,$p' >> branchsummary.txt + cat build-info/baselogs.txt | sed -n '/^.*Tile stats:/,$p' > basesummary.txt + cat build-info/baselogs.txt | sed -n '/^.*Exception in thread/,$p' >> basesummary.txt + + cat << EOF > comment-body.txt + + + + + + + + + + + +
Base $(cat build-info/base_sha)This Branch $(cat build-info/branch_sha)
+ + \`\`\` + $(cat basesummary.txt) + \`\`\` + + + \`\`\` + $(cat build-info/branchlogs.txt | sed -n '/^.*Tile stats:/,$p') + \`\`\` +
+ + https://github.com/onthegomap/planetiler/actions/runs/${{ github.event.workflow_run.id }} + +
ℹ️ Base Logs $(cat build-info/base_sha) + + \`\`\` + $(cat build-info/baselogs.txt) + \`\`\` +
+ +
ℹ️ This Branch Logs $(cat build-info/branch_sha) + + \`\`\` + $(cat build-info/branchlogs.txt) + \`\`\` +
+ EOF + + - name: 'Dump comment body' + run: cat comment-body.txt + + - uses: marocchino/sticky-pull-request-comment@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + path: comment-body.txt + header: performance-tests + number: ${{ steps.build_info.outputs.pr_number }}