rio-pmtiles python package [#338] (#542)

Add rio-pmtiles command line tool. [#338]

This is derived from the original mapbox/rio-mbtiles implementation, with changes:

* output PMTiles only instead of MBTiles.
* Python 3.7+ only.
* remove --implementation, --image-dump, --append/--overwrite, --covers features.
* bump dependency versions.
* better progress reporting; add pyroaring. 
* update README and license texts.
* rio-pmtiles v0.0.6 on PyPI

python/rio-pmtiles/tests/conftest.py

@@ -0,0 +1,63 @@
+import functools
+import operator
+import os
+import shutil
+import sys
+
+import py
+import pytest
+import rasterio
+
+from unittest import mock
+
+test_files = [
+    os.path.join(os.path.dirname(__file__), p)
+    for p in [
+        "data/RGB.byte.tif",
+        "data/RGBA.byte.tif",
+        "data/rgb-193f513.vrt",
+        "data/rgb-fa48952.vrt",
+    ]
+]
+
+
+def pytest_cmdline_main(config):
+    # Bail if the test raster data is not present. Test data is not
+    # distributed with sdists since 0.12.
+    if functools.reduce(operator.and_, map(os.path.exists, test_files)):
+        print("Test data present.")
+    else:
+        print("Test data not present. See download directions in tests/README.txt")
+        sys.exit(1)
+
+
+@pytest.fixture(scope="function")
+def data(tmpdir):
+    """A temporary directory containing a copy of the files in data."""
+    datadir = tmpdir.ensure("tests/data", dir=True)
+    for filename in test_files:
+        shutil.copy(filename, str(datadir))
+    return datadir
+
+
+@pytest.fixture(scope="function")
+def empty_data(tmpdir):
+    """A temporary directory containing a folder with an empty data file."""
+    filename = test_files[0]
+    out_filename = os.path.join(str(tmpdir), "empty.tif")
+    with rasterio.open(filename, "r") as src:
+        with rasterio.open(out_filename, "w", **src.meta) as dst:
+            pass
+    return out_filename
+
+
+@pytest.fixture()
+def rgba_cutline_path():
+    """Path to a GeoJSON rhombus within the extents of RGBA.byte.tif"""
+    return os.path.join(os.path.dirname(__file__), "data/rgba_cutline.geojson")
+
+
+@pytest.fixture()
+def rgba_points_path():
+    """Path to a pair of GeoJSON points. This is not a valid cutline."""
+    return os.path.join(os.path.dirname(__file__), "data/rgba_points.geojson")

python/rio-pmtiles/tests/data/rgb-193f513.vrt

@@ -0,0 +1,40 @@
+<VRTDataset rasterXSize="4024" rasterYSize="3457">
+  <SRS>PROJCS["UTM Zone 18, Northern Hemisphere",GEOGCS["Unknown datum based upon the WGS 84 ellipsoid",DATUM["Not_specified_based_on_WGS_84_spheroid",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-75],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]</SRS>
+  <GeoTransform> -1.0618653799999999e+06,  3.0003792667509481e+02,  0.0000000000000000e+00,  3.5412780899999999e+06,  0.0000000000000000e+00, -3.0004178272980499e+02</GeoTransform>
+  <VRTRasterBand dataType="Byte" band="1">
+    <NoDataValue>0</NoDataValue>
+    <ColorInterp>Red</ColorInterp>
+    <ComplexSource>
+      <SourceFilename relativeToVRT="1">RGB.byte.tif</SourceFilename>
+      <SourceBand>1</SourceBand>
+      <SourceProperties RasterXSize="791" RasterYSize="718" DataType="Byte" BlockXSize="791" BlockYSize="3" />
+      <SrcRect xOff="0" yOff="0" xSize="791" ySize="718" />
+      <DstRect xOff="3879.01087338305" yOff="2380.87870129508" xSize="791" ySize="718" />
+      <NODATA>0</NODATA>
+    </ComplexSource>
+  </VRTRasterBand>
+  <VRTRasterBand dataType="Byte" band="2">
+    <NoDataValue>0</NoDataValue>
+    <ColorInterp>Green</ColorInterp>
+    <ComplexSource>
+      <SourceFilename relativeToVRT="1">RGB.byte.tif</SourceFilename>
+      <SourceBand>2</SourceBand>
+      <SourceProperties RasterXSize="791" RasterYSize="718" DataType="Byte" BlockXSize="791" BlockYSize="3" />
+      <SrcRect xOff="0" yOff="0" xSize="791" ySize="718" />
+      <DstRect xOff="3879.01087338305" yOff="2380.87870129508" xSize="791" ySize="718" />
+      <NODATA>0</NODATA>
+    </ComplexSource>
+  </VRTRasterBand>
+  <VRTRasterBand dataType="Byte" band="3">
+    <NoDataValue>0</NoDataValue>
+    <ColorInterp>Blue</ColorInterp>
+    <ComplexSource>
+      <SourceFilename relativeToVRT="1">RGB.byte.tif</SourceFilename>
+      <SourceBand>3</SourceBand>
+      <SourceProperties RasterXSize="791" RasterYSize="718" DataType="Byte" BlockXSize="791" BlockYSize="3" />
+      <SrcRect xOff="0" yOff="0" xSize="791" ySize="718" />
+      <DstRect xOff="3879.01087338305" yOff="2380.87870129508" xSize="791" ySize="718" />
+      <NODATA>0</NODATA>
+    </ComplexSource>
+  </VRTRasterBand>
+</VRTDataset>

python/rio-pmtiles/tests/data/rgb-fa48952.vrt

@@ -0,0 +1,40 @@
+<VRTDataset rasterXSize="3657" rasterYSize="3761">
+  <SRS>PROJCS["UTM Zone 18, Northern Hemisphere",GEOGCS["Unknown datum based upon the WGS 84 ellipsoid",DATUM["Not_specified_based_on_WGS_84_spheroid",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-75],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]</SRS>
+  <GeoTransform>  1.1256674000000001e+05,  3.0003792667509481e+02,  0.0000000000000000e+00,  3.5598010899999999e+06,  0.0000000000000000e+00, -3.0004178272980499e+02</GeoTransform>
+  <VRTRasterBand dataType="Byte" band="1">
+    <NoDataValue>0</NoDataValue>
+    <ColorInterp>Red</ColorInterp>
+    <ComplexSource>
+      <SourceFilename relativeToVRT="1">RGB.byte.tif</SourceFilename>
+      <SourceBand>1</SourceBand>
+      <SourceProperties RasterXSize="791" RasterYSize="718" DataType="Byte" BlockXSize="791" BlockYSize="3" />
+      <SrcRect xOff="35.2680080057304" yOff="0" xSize="791" ySize="718" />
+      <DstRect xOff="0" yOff="2442.61343647589" xSize="791" ySize="718" />
+      <NODATA>0</NODATA>
+    </ComplexSource>
+  </VRTRasterBand>
+  <VRTRasterBand dataType="Byte" band="2">
+    <NoDataValue>0</NoDataValue>
+    <ColorInterp>Green</ColorInterp>
+    <ComplexSource>
+      <SourceFilename relativeToVRT="1">RGB.byte.tif</SourceFilename>
+      <SourceBand>2</SourceBand>
+      <SourceProperties RasterXSize="791" RasterYSize="718" DataType="Byte" BlockXSize="791" BlockYSize="3" />
+      <SrcRect xOff="35.2680080057304" yOff="0" xSize="791" ySize="718" />
+      <DstRect xOff="0" yOff="2442.61343647589" xSize="791" ySize="718" />
+      <NODATA>0</NODATA>
+    </ComplexSource>
+  </VRTRasterBand>
+  <VRTRasterBand dataType="Byte" band="3">
+    <NoDataValue>0</NoDataValue>
+    <ColorInterp>Blue</ColorInterp>
+    <ComplexSource>
+      <SourceFilename relativeToVRT="1">RGB.byte.tif</SourceFilename>
+      <SourceBand>3</SourceBand>
+      <SourceProperties RasterXSize="791" RasterYSize="718" DataType="Byte" BlockXSize="791" BlockYSize="3" />
+      <SrcRect xOff="35.2680080057304" yOff="0" xSize="791" ySize="718" />
+      <DstRect xOff="0" yOff="2442.61343647589" xSize="791" ySize="718" />
+      <NODATA>0</NODATA>
+    </ComplexSource>
+  </VRTRasterBand>
+</VRTDataset>

python/rio-pmtiles/tests/data/rgba_cutline.geojson

@@ -0,0 +1 @@
+{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[-76.47171020507812,37.2198034112712],[-76.47891998291016,37.196834728499866],[-76.46072387695312,37.16770367048253],[-76.40974044799805,37.161000570006095],[-76.39875411987305,37.15224460472995],[-76.34931564331053,37.1284341983056],[-76.31103515625,37.139382442337094],[-76.30794525146484,37.16291580223116],[-76.32253646850586,37.197108206316365],[-76.37094497680664,37.23798199321937],[-76.44441604614258,37.22827818273987],[-76.47171020507812,37.2198034112712]]]}},{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[-78.585205078125,24.67946552658519],[-78.22265625,24.410889551000935],[-77.69119262695312,24.69942955501979],[-77.9754638671875,24.992281691278635],[-78.585205078125,24.67946552658519]]]}}]}

python/rio-pmtiles/tests/data/rgba_points.geojson

@@ -0,0 +1 @@
+{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":{"type":"Point","coordinates":[-77.82028198242188,24.720637830132038]}},{"type":"Feature","properties":{},"geometry":{"type":"Point","coordinates":[-77.607421875,24.165549146828848]}}]}

python/rio-pmtiles/tests/test_cli.py

@@ -0,0 +1,290 @@
+import os
+import sqlite3
+import sys
+import warnings
+
+import click
+from click.testing import CliRunner
+import pytest
+import rasterio
+from rasterio.rio.main import main_group
+
+from pmtiles.reader import Reader, MmapSource, all_tiles
+import rio_pmtiles.scripts.cli
+
+from conftest import mock
+
+class Output:
+    def __init__(self, fname):
+        self.file = open(fname, 'rb')
+
+    def __enter__(self):
+        return Reader(MmapSource(self.file))
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.file.close()
+
+def test_cli_help():
+    runner = CliRunner()
+    result = runner.invoke(main_group, ["pmtiles", "--help"])
+    assert result.exit_code == 0
+    assert "Export a dataset to PMTiles." in result.output
+
+
+@mock.patch("rio_pmtiles.scripts.cli.rasterio")
+def test_dst_nodata_validation(rio):
+    """--dst-nodata requires source nodata in some form"""
+    rio.open.return_value.__enter__.return_value.profile.get.return_value = None
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group, ["pmtiles", "--dst-nodata", "0", "in.tif", "out.pmtiles"]
+    )
+    assert result.exit_code == 2
+
+
+@pytest.mark.parametrize("filename", ["RGB.byte.tif", "RGBA.byte.tif"])
+def test_export_metadata(tmpdir, data, filename):
+    inputfile = str(data.join(filename))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(main_group, ["pmtiles", inputfile, outputfile])
+    assert result.exit_code == 0
+
+    with Output(outputfile) as p:
+        assert p.metadata()['name'] == filename
+
+def test_export_metadata_output_opt(tmpdir, data):
+    inputfile = str(data.join("RGB.byte.tif"))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(main_group, ["pmtiles", inputfile, "-o", outputfile])
+    assert result.exit_code == 0
+
+    with Output(outputfile) as p:
+        assert p.metadata()['name'] == "RGB.byte.tif"
+
+def test_export_tiles(tmpdir, data):
+    inputfile = str(data.join("RGB.byte.tif"))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(main_group, ["pmtiles", inputfile, outputfile])
+    assert result.exit_code == 0
+
+    with open(outputfile, 'rb') as f:
+        src = MmapSource(f)
+        assert len(list(all_tiles(src))) == 6
+
+def test_export_zoom(tmpdir, data):
+    inputfile = str(data.join("RGB.byte.tif"))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group, ["pmtiles", inputfile, outputfile, "--zoom-levels", "6..7"]
+    )
+    assert result.exit_code == 0
+
+    with open(outputfile, 'rb') as f:
+        src = MmapSource(f)
+        assert len(list(all_tiles(src))) == 6
+
+def test_export_jobs(tmpdir, data):
+    inputfile = str(data.join("RGB.byte.tif"))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(main_group, ["pmtiles", inputfile, outputfile, "-j", "4"])
+    assert result.exit_code == 0
+
+    with open(outputfile, 'rb') as f:
+        src = MmapSource(f)
+        assert len(list(all_tiles(src))) == 6
+
+
+def test_export_src_nodata(tmpdir, data):
+    inputfile = str(data.join("RGB.byte.tif"))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        ["pmtiles", inputfile, outputfile, "--src-nodata", "0", "--dst-nodata", "0"],
+    )
+    assert result.exit_code == 0
+
+    with open(outputfile, 'rb') as f:
+        src = MmapSource(f)
+        assert len(list(all_tiles(src))) == 6
+
+
+def test_export_bilinear(tmpdir, data):
+    inputfile = str(data.join("RGB.byte.tif"))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group, ["pmtiles", inputfile, outputfile, "--resampling", "bilinear"]
+    )
+    assert result.exit_code == 0
+
+    with open(outputfile, 'rb') as f:
+        src = MmapSource(f)
+        assert len(list(all_tiles(src))) == 6
+
+
+def test_skip_empty(tmpdir, empty_data):
+    """This file has the same shape as RGB.byte.tif, but no data."""
+    inputfile = empty_data
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(main_group, ["pmtiles", inputfile, outputfile])
+    assert result.exit_code == 0
+
+    with open(outputfile, 'rb') as f:
+        src = MmapSource(f)
+        assert len(list(all_tiles(src))) == 0
+
+
+def test_include_empty(tmpdir, empty_data):
+    """This file has the same shape as RGB.byte.tif, but no data."""
+    inputfile = empty_data
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group, ["pmtiles", "--include-empty-tiles", inputfile, outputfile]
+    )
+    assert result.exit_code == 0
+
+    with open(outputfile, 'rb') as f:
+        src = MmapSource(f)
+        assert len(list(all_tiles(src))) == 6
+
+
+def test_invalid_format_rgba(tmpdir, empty_data):
+    """--format JPEG --rgba is not allowed"""
+    inputfile = empty_data
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group, ["pmtiles", "--format", "JPEG", "--rgba", inputfile, outputfile]
+    )
+    assert result.exit_code == 2
+
+
+@pytest.mark.parametrize("filename", ["RGBA.byte.tif", "RGB.byte.tif"])
+def test_rgba_png(tmpdir, data, filename):
+    inputfile = str(data.join(filename))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group, ["pmtiles", "--rgba", "--format", "PNG", inputfile, outputfile]
+    )
+    with Output(outputfile) as p:
+        assert p.metadata()['name'] == filename
+
+
+@pytest.mark.parametrize(
+    "minzoom,maxzoom,exp_num_tiles,source",
+    [
+        (4, 10, 70, "RGB.byte.tif"),
+        (6, 7, 6, "RGB.byte.tif"),
+        (4, 10, 12, "rgb-193f513.vrt"),
+        (4, 10, 70, "rgb-fa48952.vrt"),
+    ],
+)
+def test_export_count(tmpdir, data, minzoom, maxzoom, exp_num_tiles, source):
+    inputfile = str(data.join(source))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        [
+            "pmtiles",
+            "--zoom-levels",
+            "{}..{}".format(minzoom, maxzoom),
+            inputfile,
+            outputfile,
+        ],
+    )
+    assert result.exit_code == 0
+    with open(outputfile, 'rb') as f:
+        src = MmapSource(f)
+        assert len(list(all_tiles(src))) == exp_num_tiles
+
+
+@pytest.mark.parametrize("filename", ["RGBA.byte.tif"])
+def test_progress_bar(tmpdir, data, filename):
+    inputfile = str(data.join(filename))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        [
+            "pmtiles",
+            "-#",
+            "--zoom-levels",
+            "4..11",
+            "--rgba",
+            "--format",
+            "PNG",
+            inputfile,
+            outputfile,
+        ],
+    )
+    assert result.exit_code == 0
+    assert "100%" in result.output
+
+
+@pytest.mark.parametrize("inputfiles", [[], ["a.tif", "b.tif"]])
+def test_input_required(inputfiles):
+    """We require exactly one input file"""
+    runner = CliRunner()
+    result = runner.invoke(main_group, ["pmtiles"] + inputfiles + ["foo.pmtiles"])
+    assert result.exit_code == 2
+
+
+@pytest.mark.parametrize("filename", ["RGBA.byte.tif"])
+def test_cutline_progress_bar(tmpdir, data, rgba_cutline_path, filename):
+    """rio-pmtiles accepts and uses a cutline"""
+    inputfile = str(data.join(filename))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        [
+            "pmtiles",
+            "-#",
+            "--zoom-levels",
+            "4..11",
+            "--rgba",
+            "--format",
+            "PNG",
+            "--cutline",
+            rgba_cutline_path,
+            inputfile,
+            outputfile,
+        ],
+    )
+    assert result.exit_code == 0
+    assert "100%" in result.output
+
+
+@pytest.mark.parametrize("filename", ["RGBA.byte.tif"])
+def test_invalid_cutline(tmpdir, data, rgba_points_path, filename):
+    """Points cannot serve as a cutline"""
+    inputfile = str(data.join(filename))
+    outputfile = str(tmpdir.join("export.pmtiles"))
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        [
+            "pmtiles",
+            "-#",
+            "--zoom-levels",
+            "4..11",
+            "--rgba",
+            "--format",
+            "PNG",
+            "--cutline",
+            rgba_points_path,
+            inputfile,
+            outputfile,
+        ],
+    )
+    assert result.exit_code == 1

python/rio-pmtiles/tests/test_mod.py

@@ -0,0 +1,31 @@
+"""Module tests"""
+
+from mercantile import Tile
+import pytest
+
+import rio_pmtiles.worker
+
+
+@pytest.mark.parametrize("tile", [Tile(36, 73, 7), Tile(0, 0, 0), Tile(1, 1, 1)])
+@pytest.mark.parametrize("filename", ["RGB.byte.tif", "RGBA.byte.tif"])
+def test_process_tile(data, filename, tile):
+    sourcepath = str(data.join(filename))
+    rio_pmtiles.worker.init_worker(
+        sourcepath,
+        {
+            "driver": "PNG",
+            "dtype": "uint8",
+            "nodata": 0,
+            "height": 256,
+            "width": 256,
+            "count": 3,
+            "crs": "EPSG:3857",
+        },
+        "nearest",
+        {},
+        {},
+    )
+    t, contents = rio_pmtiles.worker.process_tile(tile)
+    assert t.x == tile.x
+    assert t.y == tile.y
+    assert t.z == tile.z