#ifndef PMTILES_HPP #define PMTILES_HPP #include #include #include namespace pmtiles { struct headerv3 { uint64_t root_dir_offset; uint64_t root_dir_bytes; uint64_t json_metadata_offset; uint64_t json_metadata_bytes; uint64_t leaf_dirs_offset; uint64_t leaf_dirs_bytes; uint64_t tile_data_offset; uint64_t tile_data_bytes; uint64_t addressed_tiles_count; uint64_t tile_entries_count; uint64_t tile_contents_count; bool clustered; uint8_t internal_compression; uint8_t tile_compression; uint8_t tile_type; uint8_t min_zoom; uint8_t max_zoom; int32_t min_lon_e7; int32_t min_lat_e7; int32_t max_lon_e7; int32_t max_lat_e7; uint8_t center_zoom; int32_t center_lon_e7; int32_t center_lat_e7; // WARNING: this is limited to little-endian std::string serialize() { std::stringstream ss; ss << "PMTiles"; uint8_t version = 3; ss.write((char *)&version,1); ss.write((char *)&root_dir_offset,8); ss.write((char *)&root_dir_bytes,8); ss.write((char *)&json_metadata_offset,8); ss.write((char *)&json_metadata_bytes,8); ss.write((char *)&leaf_dirs_offset,8); ss.write((char *)&leaf_dirs_bytes,8); ss.write((char *)&tile_data_offset,8); ss.write((char *)&tile_data_bytes,8); ss.write((char *)&addressed_tiles_count,8); ss.write((char *)&tile_entries_count,8); ss.write((char *)&tile_contents_count,8); uint8_t clustered_val = 0x0; if (clustered) { clustered_val = 0x1; } ss.write((char *)&clustered_val,1); ss.write((char *)&internal_compression,1); ss.write((char *)&tile_compression,1); ss.write((char *)&tile_type,1); ss.write((char *)&min_zoom,1); ss.write((char *)&max_zoom,1); ss.write((char *)&min_lon_e7,4); ss.write((char *)&min_lat_e7,4); ss.write((char *)&max_lon_e7,4); ss.write((char *)&max_lat_e7,4); ss.write((char *)¢er_zoom,1); ss.write((char *)¢er_lon_e7,4); ss.write((char *)¢er_lat_e7,4); return ss.str(); } }; struct pmtiles_magic_number_exception : std::exception { const char* what() const noexcept override { return "pmtiles magic number exception"; } }; struct pmtiles_version_exception : std::exception { const char* what() const noexcept override { return "pmtiles version: must be 3"; } }; inline headerv3 deserialize_header(const std::string &s) { if (s.substr(0,7) != "PMTiles") { throw pmtiles_magic_number_exception{}; } if (s.size() != 127 || s[7] != 0x3) { throw pmtiles_version_exception{}; } headerv3 h; s.copy((char *)&h.root_dir_offset,8,8); s.copy((char *)&h.root_dir_bytes,8,16); s.copy((char *)&h.json_metadata_offset,8,24); s.copy((char *)&h.json_metadata_bytes,8,32); s.copy((char *)&h.leaf_dirs_offset,8,40); s.copy((char *)&h.leaf_dirs_bytes,8,48); s.copy((char *)&h.tile_data_offset,8,56); s.copy((char *)&h.tile_data_bytes,8,64); s.copy((char *)&h.addressed_tiles_count,8,72); s.copy((char *)&h.tile_entries_count,8,80); s.copy((char *)&h.tile_contents_count,8,88); if (s[96] == 0x1) { h.clustered = true; } else { h.clustered = false; } h.internal_compression = s[97]; h.tile_compression = s[98]; h.tile_type = s[99]; h.min_zoom = s[100]; h.max_zoom = s[101]; s.copy((char *)&h.min_lon_e7,4,102); s.copy((char *)&h.min_lat_e7,4,106); s.copy((char *)&h.max_lon_e7,4,110); s.copy((char *)&h.max_lat_e7,4,114); h.center_zoom = s[118]; s.copy((char *)&h.center_lon_e7,4,119); s.copy((char *)&h.center_lat_e7,4,123); return h; } struct zxy { uint8_t z; uint32_t x; uint32_t y; zxy(int _z, int _x, int _y) : z(_z), x(_x), y(_y) { } }; struct entryv3 { uint64_t tile_id; uint64_t offset; uint32_t length; uint32_t run_length; entryv3() : tile_id(0), offset(0), length(0), run_length(0) { } entryv3(uint64_t _tile_id, uint64_t _offset, uint32_t _length, uint32_t _run_length) : tile_id(_tile_id), offset(_offset), length(_length), run_length(_run_length) { } }; struct { bool operator()(entryv3 a, entryv3 b) const { return a.tile_id < b.tile_id; } } entryv3_cmp; struct varint_too_long_exception : std::exception { const char* what() const noexcept override { return "varint too long exception"; } }; struct end_of_buffer_exception : std::exception { const char* what() const noexcept override { return "end of buffer exception"; } }; namespace detail { constexpr const int8_t max_varint_length = sizeof(uint64_t) * 8 / 7 + 1; // from https://github.com/mapbox/protozero/blob/master/include/protozero/varint.hpp inline uint64_t decode_varint_impl(const char** data, const char* end) { const auto* begin = reinterpret_cast(*data); const auto* iend = reinterpret_cast(end); const int8_t* p = begin; uint64_t val = 0; if (iend - begin >= max_varint_length) { // fast path do { int64_t b = *p++; val = ((uint64_t(b) & 0x7fU) ); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x7fU) << 7U); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x7fU) << 14U); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x7fU) << 21U); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x7fU) << 28U); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x7fU) << 35U); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x7fU) << 42U); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x7fU) << 49U); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x7fU) << 56U); if (b >= 0) { break; } b = *p++; val |= ((uint64_t(b) & 0x01U) << 63U); if (b >= 0) { break; } throw varint_too_long_exception{}; } while (false); } else { unsigned int shift = 0; while (p != iend && *p < 0) { val |= (uint64_t(*p++) & 0x7fU) << shift; shift += 7; } if (p == iend) { throw end_of_buffer_exception{}; } val |= uint64_t(*p++) << shift; } *data = reinterpret_cast(p); return val; } inline uint64_t decode_varint(const char** data, const char* end) { // If this is a one-byte varint, decode it here. if (end != *data && ((static_cast(**data) & 0x80U) == 0)) { const auto val = static_cast(**data); ++(*data); return val; } // If this varint is more than one byte, defer to complete implementation. return detail::decode_varint_impl(data, end); } inline void rotate(int64_t n, int64_t &x, int64_t &y, int64_t rx, int64_t ry) { if (ry == 0) { if (rx == 1) { x = n-1 - x; y = n-1 - y; } int64_t t = x; x = y; y = t; } } inline zxy t_on_level(uint8_t z, uint64_t pos) { int64_t n = 1 << z; int64_t rx, ry, s, t = pos; int64_t tx = 0; int64_t ty = 0; for (s=1; s data, uint64_t value) { int n = 1; while (value >= 0x80U) { *data++ = char((value & 0x7fU) | 0x80U); value >>= 7U; ++n; } *data = char(value); return n; } inline zxy tileid_to_zxy(uint64_t tileid) { uint64_t acc = 0; uint8_t t_z = 0; while(true) { uint64_t num_tiles = (1 << t_z) * (1 << t_z); if (acc + num_tiles > tileid) { return detail::t_on_level(t_z, tileid - acc); } acc += num_tiles; t_z++; } } inline uint64_t zxy_to_tileid(uint8_t z, uint32_t x, uint32_t y) { uint64_t acc = 0; for (uint8_t t_z = 0; t_z < z; t_z++) acc += (0x1 << t_z) * (0x1 << t_z); int64_t n = 1 << z; int64_t rx, ry, s, d=0; int64_t tx = x; int64_t ty = y; for (s=n/2; s>0; s/=2) { rx = (tx & s) > 0; ry = (ty & s) > 0; d += s * s * ((3 * rx) ^ ry); detail::rotate(s, tx, ty, rx, ry); } return acc + d; } // returns an uncompressed byte buffer inline std::string serialize_directory(const std::vector& entries) { std::string data; write_varint(std::back_inserter(data), entries.size()); uint64_t last_id = 0; for (auto const &entry : entries) { write_varint(std::back_inserter(data), entry.tile_id - last_id); last_id = entry.tile_id; } for (auto const &entry : entries) { write_varint(std::back_inserter(data), entry.run_length); } for (auto const &entry : entries) { write_varint(std::back_inserter(data), entry.length); } for (size_t i = 0; i < entries.size(); i++) { if (i > 0 && entries[i].offset == entries[i-1].offset + entries[i-1].length) { write_varint(std::back_inserter(data), 0); } else { write_varint(std::back_inserter(data), entries[i].offset+1); } } return data; } // takes an uncompressed byte buffer inline std::vector deserialize_directory(const std::string &decompressed) { const char *t = decompressed.data(); const char *end = t + decompressed.size(); uint64_t num_entries = detail::decode_varint(&t,end); std::vector result; result.resize(num_entries); uint64_t last_id = 0; for (size_t i = 0; i < num_entries; i++) { uint64_t tile_id = last_id + detail::decode_varint(&t,end); result[i].tile_id = tile_id; last_id = tile_id; } for (size_t i = 0; i < num_entries; i++) { result[i].run_length = detail::decode_varint(&t,end); } for (size_t i = 0; i < num_entries; i++) { result[i].length = detail::decode_varint(&t,end); } for (size_t i = 0; i < num_entries; i++) { uint64_t tmp = detail::decode_varint(&t,end); if (i > 0 && tmp == 0) { result[i].offset = result[i-1].offset + result[i-1].length; } else { result[i].offset = tmp - 1; } } // assert the directory has been fully consumed if (t != end) { fprintf(stderr, "Error: malformed pmtiles directory\n"); exit(EXIT_FAILURE); } return result; } } #endif