From 47bffa32cf77c53f0c38845544af705b0760ec27 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Wed, 12 Oct 2022 16:17:33 +0800 Subject: [PATCH] c++ v3 utility functions --- cpp/.gitignore | 1 + cpp/Makefile | 3 + cpp/minunit.h | 391 +++++++++++++++++++++++++++++++++++++++++++ cpp/pmtiles.hpp | 437 ++++++++++++++++++++++++++++++++---------------- cpp/test.cpp | 81 +++++++++ 5 files changed, 773 insertions(+), 140 deletions(-) create mode 100644 cpp/.gitignore create mode 100644 cpp/Makefile create mode 100644 cpp/minunit.h create mode 100644 cpp/test.cpp diff --git a/cpp/.gitignore b/cpp/.gitignore new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/cpp/.gitignore @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/cpp/Makefile b/cpp/Makefile new file mode 100644 index 0000000..bdde73d --- /dev/null +++ b/cpp/Makefile @@ -0,0 +1,3 @@ +.PHONY: test +test: + clang test.cpp -std=c++11 -lstdc++ -o test && ./test diff --git a/cpp/minunit.h b/cpp/minunit.h new file mode 100644 index 0000000..b019c34 --- /dev/null +++ b/cpp/minunit.h @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2012 David Siñuela Pastor, siu.4coders@gmail.com + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef MINUNIT_MINUNIT_H +#define MINUNIT_MINUNIT_H + +#ifdef __cplusplus + extern "C" { +#endif + +#if defined(_WIN32) +#include +#if defined(_MSC_VER) && _MSC_VER < 1900 + #define snprintf _snprintf + #define __func__ __FUNCTION__ +#endif + +#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) + +/* Change POSIX C SOURCE version for pure c99 compilers */ +#if !defined(_POSIX_C_SOURCE) || _POSIX_C_SOURCE < 200112L +#undef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200112L +#endif + +#include /* POSIX flags */ +#include /* clock_gettime(), time() */ +#include /* gethrtime(), gettimeofday() */ +#include +#include +#include + +#if defined(__MACH__) && defined(__APPLE__) +#include +#include +#endif + +#if __GNUC__ >= 5 && !defined(__STDC_VERSION__) +#define __func__ __extension__ __FUNCTION__ +#endif + +#else +#error "Unable to define timers for an unknown OS." +#endif + +#include +#include + +/* Maximum length of last message */ +#define MINUNIT_MESSAGE_LEN 1024 +/* Accuracy with which floats are compared */ +#define MINUNIT_EPSILON 1E-12 + +/* Misc. counters */ +static int minunit_run = 0; +static int minunit_assert = 0; +static int minunit_fail = 0; +static int minunit_status = 0; + +/* Timers */ +static double minunit_real_timer = 0; +static double minunit_proc_timer = 0; + +/* Last message */ +static char minunit_last_message[MINUNIT_MESSAGE_LEN]; + +/* Test setup and teardown function pointers */ +static void (*minunit_setup)(void) = NULL; +static void (*minunit_teardown)(void) = NULL; + +/* Definitions */ +#define MU_TEST(method_name) static void method_name(void) +#define MU_TEST_SUITE(suite_name) static void suite_name(void) + +#define MU__SAFE_BLOCK(block) do {\ + block\ +} while(0) + +/* Run test suite and unset setup and teardown functions */ +#define MU_RUN_SUITE(suite_name) MU__SAFE_BLOCK(\ + suite_name();\ + minunit_setup = NULL;\ + minunit_teardown = NULL;\ +) + +/* Configure setup and teardown functions */ +#define MU_SUITE_CONFIGURE(setup_fun, teardown_fun) MU__SAFE_BLOCK(\ + minunit_setup = setup_fun;\ + minunit_teardown = teardown_fun;\ +) + +/* Test runner */ +#define MU_RUN_TEST(test) MU__SAFE_BLOCK(\ + if (minunit_real_timer==0 && minunit_proc_timer==0) {\ + minunit_real_timer = mu_timer_real();\ + minunit_proc_timer = mu_timer_cpu();\ + }\ + if (minunit_setup) (*minunit_setup)();\ + minunit_status = 0;\ + test();\ + minunit_run++;\ + if (minunit_status) {\ + minunit_fail++;\ + printf("F");\ + printf("\n%s\n", minunit_last_message);\ + }\ + fflush(stdout);\ + if (minunit_teardown) (*minunit_teardown)();\ +) + +/* Report */ +#define MU_REPORT() MU__SAFE_BLOCK(\ + double minunit_end_real_timer;\ + double minunit_end_proc_timer;\ + printf("\n\n%d tests, %d assertions, %d failures\n", minunit_run, minunit_assert, minunit_fail);\ + minunit_end_real_timer = mu_timer_real();\ + minunit_end_proc_timer = mu_timer_cpu();\ + printf("\nFinished in %.8f seconds (real) %.8f seconds (proc)\n\n",\ + minunit_end_real_timer - minunit_real_timer,\ + minunit_end_proc_timer - minunit_proc_timer);\ +) +#define MU_EXIT_CODE minunit_fail + +/* Assertions */ +#define mu_check(test) MU__SAFE_BLOCK(\ + minunit_assert++;\ + if (!(test)) {\ + snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %s", __func__, __FILE__, __LINE__, #test);\ + minunit_status = 1;\ + return;\ + } else {\ + printf(".");\ + }\ +) + +#define mu_fail(message) MU__SAFE_BLOCK(\ + minunit_assert++;\ + snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %s", __func__, __FILE__, __LINE__, message);\ + minunit_status = 1;\ + return;\ +) + +#define mu_assert(test, message) MU__SAFE_BLOCK(\ + minunit_assert++;\ + if (!(test)) {\ + snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %s", __func__, __FILE__, __LINE__, message);\ + minunit_status = 1;\ + return;\ + } else {\ + printf(".");\ + }\ +) + +#define mu_assert_int_eq(expected, result) MU__SAFE_BLOCK(\ + int minunit_tmp_e;\ + int minunit_tmp_r;\ + minunit_assert++;\ + minunit_tmp_e = (expected);\ + minunit_tmp_r = (result);\ + if (minunit_tmp_e != minunit_tmp_r) {\ + snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %d expected but was %d", __func__, __FILE__, __LINE__, minunit_tmp_e, minunit_tmp_r);\ + minunit_status = 1;\ + return;\ + } else {\ + printf(".");\ + }\ +) + +#define mu_assert_double_eq(expected, result) MU__SAFE_BLOCK(\ + double minunit_tmp_e;\ + double minunit_tmp_r;\ + minunit_assert++;\ + minunit_tmp_e = (expected);\ + minunit_tmp_r = (result);\ + if (fabs(minunit_tmp_e-minunit_tmp_r) > MINUNIT_EPSILON) {\ + int minunit_significant_figures = 1 - log10(MINUNIT_EPSILON);\ + snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %.*g expected but was %.*g", __func__, __FILE__, __LINE__, minunit_significant_figures, minunit_tmp_e, minunit_significant_figures, minunit_tmp_r);\ + minunit_status = 1;\ + return;\ + } else {\ + printf(".");\ + }\ +) + +#define mu_assert_string_eq(expected, result) MU__SAFE_BLOCK(\ + const char* minunit_tmp_e = expected;\ + const char* minunit_tmp_r = result;\ + minunit_assert++;\ + if (!minunit_tmp_e) {\ + minunit_tmp_e = "";\ + }\ + if (!minunit_tmp_r) {\ + minunit_tmp_r = "";\ + }\ + if(strcmp(minunit_tmp_e, minunit_tmp_r)) {\ + snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: '%s' expected but was '%s'", __func__, __FILE__, __LINE__, minunit_tmp_e, minunit_tmp_r);\ + minunit_status = 1;\ + return;\ + } else {\ + printf(".");\ + }\ +) + +/* + * The following two functions were written by David Robert Nadeau + * from http://NadeauSoftware.com/ and distributed under the + * Creative Commons Attribution 3.0 Unported License + */ + +/** + * Returns the real time, in seconds, or -1.0 if an error occurred. + * + * Time is measured since an arbitrary and OS-dependent start time. + * The returned real time is only useful for computing an elapsed time + * between two calls to this function. + */ +static double mu_timer_real(void) +{ +#if defined(_WIN32) + /* Windows 2000 and later. ---------------------------------- */ + LARGE_INTEGER Time; + LARGE_INTEGER Frequency; + + QueryPerformanceFrequency(&Frequency); + QueryPerformanceCounter(&Time); + + Time.QuadPart *= 1000000; + Time.QuadPart /= Frequency.QuadPart; + + return (double)Time.QuadPart / 1000000.0; + +#elif (defined(__hpux) || defined(hpux)) || ((defined(__sun__) || defined(__sun) || defined(sun)) && (defined(__SVR4) || defined(__svr4__))) + /* HP-UX, Solaris. ------------------------------------------ */ + return (double)gethrtime( ) / 1000000000.0; + +#elif defined(__MACH__) && defined(__APPLE__) + /* OSX. ----------------------------------------------------- */ + static double timeConvert = 0.0; + if ( timeConvert == 0.0 ) + { + mach_timebase_info_data_t timeBase; + (void)mach_timebase_info( &timeBase ); + timeConvert = (double)timeBase.numer / + (double)timeBase.denom / + 1000000000.0; + } + return (double)mach_absolute_time( ) * timeConvert; + +#elif defined(_POSIX_VERSION) + /* POSIX. --------------------------------------------------- */ + struct timeval tm; +#if defined(_POSIX_TIMERS) && (_POSIX_TIMERS > 0) + { + struct timespec ts; +#if defined(CLOCK_MONOTONIC_PRECISE) + /* BSD. --------------------------------------------- */ + const clockid_t id = CLOCK_MONOTONIC_PRECISE; +#elif defined(CLOCK_MONOTONIC_RAW) + /* Linux. ------------------------------------------- */ + const clockid_t id = CLOCK_MONOTONIC_RAW; +#elif defined(CLOCK_HIGHRES) + /* Solaris. ----------------------------------------- */ + const clockid_t id = CLOCK_HIGHRES; +#elif defined(CLOCK_MONOTONIC) + /* AIX, BSD, Linux, POSIX, Solaris. ----------------- */ + const clockid_t id = CLOCK_MONOTONIC; +#elif defined(CLOCK_REALTIME) + /* AIX, BSD, HP-UX, Linux, POSIX. ------------------- */ + const clockid_t id = CLOCK_REALTIME; +#else + const clockid_t id = (clockid_t)-1; /* Unknown. */ +#endif /* CLOCK_* */ + if ( id != (clockid_t)-1 && clock_gettime( id, &ts ) != -1 ) + return (double)ts.tv_sec + + (double)ts.tv_nsec / 1000000000.0; + /* Fall thru. */ + } +#endif /* _POSIX_TIMERS */ + + /* AIX, BSD, Cygwin, HP-UX, Linux, OSX, POSIX, Solaris. ----- */ + gettimeofday( &tm, NULL ); + return (double)tm.tv_sec + (double)tm.tv_usec / 1000000.0; +#else + return -1.0; /* Failed. */ +#endif +} + +/** + * Returns the amount of CPU time used by the current process, + * in seconds, or -1.0 if an error occurred. + */ +static double mu_timer_cpu(void) +{ +#if defined(_WIN32) + /* Windows -------------------------------------------------- */ + FILETIME createTime; + FILETIME exitTime; + FILETIME kernelTime; + FILETIME userTime; + + /* This approach has a resolution of 1/64 second. Unfortunately, Windows' API does not offer better */ + if ( GetProcessTimes( GetCurrentProcess( ), + &createTime, &exitTime, &kernelTime, &userTime ) != 0 ) + { + ULARGE_INTEGER userSystemTime; + memcpy(&userSystemTime, &userTime, sizeof(ULARGE_INTEGER)); + return (double)userSystemTime.QuadPart / 10000000.0; + } + +#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) + /* AIX, BSD, Cygwin, HP-UX, Linux, OSX, and Solaris --------- */ + +#if defined(_POSIX_TIMERS) && (_POSIX_TIMERS > 0) + /* Prefer high-res POSIX timers, when available. */ + { + clockid_t id; + struct timespec ts; +#if _POSIX_CPUTIME > 0 + /* Clock ids vary by OS. Query the id, if possible. */ + if ( clock_getcpuclockid( 0, &id ) == -1 ) +#endif +#if defined(CLOCK_PROCESS_CPUTIME_ID) + /* Use known clock id for AIX, Linux, or Solaris. */ + id = CLOCK_PROCESS_CPUTIME_ID; +#elif defined(CLOCK_VIRTUAL) + /* Use known clock id for BSD or HP-UX. */ + id = CLOCK_VIRTUAL; +#else + id = (clockid_t)-1; +#endif + if ( id != (clockid_t)-1 && clock_gettime( id, &ts ) != -1 ) + return (double)ts.tv_sec + + (double)ts.tv_nsec / 1000000000.0; + } +#endif + +#if defined(RUSAGE_SELF) + { + struct rusage rusage; + if ( getrusage( RUSAGE_SELF, &rusage ) != -1 ) + return (double)rusage.ru_utime.tv_sec + + (double)rusage.ru_utime.tv_usec / 1000000.0; + } +#endif + +#if defined(_SC_CLK_TCK) + { + const double ticks = (double)sysconf( _SC_CLK_TCK ); + struct tms tms; + if ( times( &tms ) != (clock_t)-1 ) + return (double)tms.tms_utime / ticks; + } +#endif + +#if defined(CLOCKS_PER_SEC) + { + clock_t cl = clock( ); + if ( cl != (clock_t)-1 ) + return (double)cl / (double)CLOCKS_PER_SEC; + } +#endif + +#endif + + return -1; /* Failed. */ +} + +#ifdef __cplusplus +} +#endif + +#endif /* MINUNIT_MINUNIT_H */ \ No newline at end of file diff --git a/cpp/pmtiles.hpp b/cpp/pmtiles.hpp index acfc2c9..d7e67ad 100644 --- a/cpp/pmtiles.hpp +++ b/cpp/pmtiles.hpp @@ -1,155 +1,312 @@ -#include -#include -#include +#ifndef PMTILES_HPP +#define PMTILES_HPP + +#include +#include #include -#include "xxhash.h" -#include -void writePmtilesHeader(std::ostream &outfile, const std::string &metadata, uint16_t root_entries_len) { - uint16_t MAGIC = 0x4d50; - outfile.write((char *)&MAGIC,2); - uint16_t version = 2; - outfile.write((char *)&version,2); - uint32_t metadata_size = metadata.size(); - outfile.write((char *)&metadata_size,4); - outfile.write((char *)&root_entries_len,2); - outfile << metadata; -} +namespace pmtiles { -void writeEntry(std::ostream &outfile, const std::tuple &tile, bool is_directory = false) { - uint8_t z_val = std::get<0>(tile); - if (is_directory) z_val |= 0b10000000; - outfile.write((char *)&z_val,1); - outfile.write((char *)&std::get<1>(tile),3); - outfile.write((char *)&std::get<2>(tile),3); - outfile.write((char *)&std::get<3>(tile),6); - outfile.write((char *)&std::get<4>(tile),4); -} +struct headerv3 { + uint64_t root_dir_offset; + uint64_t root_dir_bytes; + uint64_t json_metadata_offset; + uint64_t json_metadata_bytes; + uint64_t leaf_dirs_offset; + uint64_t leaf_dirs_bytes; + uint64_t tile_data_offset; + uint64_t tile_data_bytes; + uint64_t addressed_tiles_count; + uint64_t tile_entries_count; + uint64_t tile_contents_count; + bool clustered; + uint8_t internal_compression; + uint8_t tile_compression; + uint8_t tile_type; + uint8_t min_zoom; + uint8_t max_zoom; + int32_t min_lon_e7; + int32_t min_lat_e7; + int32_t max_lon_e7; + int32_t max_lat_e7; + uint8_t center_zoom; + int32_t center_lon_e7; + int32_t center_lat_e7; -struct pmtiles_v2_writer { - std::vector> entries{}; - std::ofstream ostream; - uint64_t offset = 0; - std::map hash_to_offset; + // WARNING: this is limited to little-endian + std::string serialize() { + std::stringstream ss; + ss << "PMTiles"; + uint8_t version = 3; + ss.write((char *)&version,1); + ss.write((char *)&root_dir_offset,8); + ss.write((char *)&root_dir_bytes,8); + ss.write((char *)&json_metadata_offset,8); + ss.write((char *)&json_metadata_bytes,8); + ss.write((char *)&leaf_dirs_offset,8); + ss.write((char *)&leaf_dirs_bytes,8); + ss.write((char *)&tile_data_offset,8); + ss.write((char *)&tile_data_bytes,8); + ss.write((char *)&addressed_tiles_count,8); + ss.write((char *)&tile_entries_count,8); + ss.write((char *)&tile_contents_count,8); + + uint8_t clustered_val = 0x0; + if (clustered) { + clustered_val = 0x1; + } + + ss.write((char *)&clustered_val,1); + ss.write((char *)&internal_compression,1); + ss.write((char *)&tile_compression,1); + ss.write((char *)&tile_type,1); + ss.write((char *)&min_zoom,1); + ss.write((char *)&max_zoom,1); + ss.write((char *)&min_lon_e7,4); + ss.write((char *)&min_lat_e7,4); + ss.write((char *)&max_lon_e7,4); + ss.write((char *)&max_lat_e7,4); + ss.write((char *)¢er_zoom,1); + ss.write((char *)¢er_lon_e7,4); + ss.write((char *)¢er_lat_e7,4); + + return ss.str(); + } }; -pmtiles_v2_writer *pmtiles_v2_open(const char *filename) { - pmtiles_v2_writer *w = new pmtiles_v2_writer; - w->ostream.open(filename,std::ios::out | std::ios::binary); +struct zxy { + uint8_t z; + uint32_t x; + uint32_t y; - w->offset = 512000; - - for (int i = 0; i < w->offset; ++i) { - char zero = 0; - w->ostream.write(&zero,sizeof(char)); - } - - return w; -} - -void pmtiles_v2_write_tile(pmtiles_v2_writer *w, int z, int x, int y, const std::string &data) { - XXH64_hash_t hash = XXH64(data.data(),data.size(),3857); - if (w->hash_to_offset.count(hash) > 0) { - w->entries.emplace_back(z,x,y,w->hash_to_offset[hash],data.size()); - } else { - w->ostream << data; - w->entries.emplace_back(z,x,y,w->offset,data.size()); - w->hash_to_offset[hash] = w->offset; - w->offset += data.size(); - } -} - -struct TileCompare { - bool operator()(std::tuple const &lhs, std::tuple const &rhs) const - { - uint8_t zl = std::get<0>(lhs); - uint8_t zr = std::get<0>(rhs); - if (zl != zr) return zl < zr; - uint32_t xl = std::get<1>(lhs); - uint32_t xr = std::get<1>(rhs); - if (xl != xr) return xl < xr; - uint32_t yl = std::get<2>(lhs); - uint32_t yr = std::get<2>(rhs); - return yl < yr; - } + zxy(int _z, int _x, int _y) : z(_z), x(_x), y(_y) { + } }; -void pmtiles_v2_finalize(pmtiles_v2_writer *w, const std::string serialized_metadata) { - if (w->entries.size() < 21845) { - w->ostream.seekp(0); - writePmtilesHeader(w->ostream,serialized_metadata,w->entries.size()); - sort(begin(w->entries),end(w->entries),TileCompare()); +struct entryv3 { + uint64_t tile_id; + uint64_t offset; + uint32_t length; + uint32_t run_length; - for (auto const &entry : w->entries) { - writeEntry(w->ostream,entry); - } + entryv3() : tile_id(0), offset(0), length(0), run_length(0) { + } + + entryv3(uint64_t _tile_id, uint64_t _offset, uint32_t _length, uint32_t _run_length) + : tile_id(_tile_id), offset(_offset), length(_length), run_length(_run_length) { + } +}; + +struct varint_too_long_exception : std::exception { + const char* what() const noexcept override { + return "varint too long exception"; + } +}; + +struct end_of_buffer_exception : std::exception { + const char* what() const noexcept override { + return "end of buffer exception"; + } +}; + +namespace detail { + constexpr const int8_t max_varint_length = sizeof(uint64_t) * 8 / 7 + 1; + + // from https://github.com/mapbox/protozero/blob/master/include/protozero/varint.hpp + inline uint64_t decode_varint_impl(const char** data, const char* end) { + const auto* begin = reinterpret_cast(*data); + const auto* iend = reinterpret_cast(end); + const int8_t* p = begin; + uint64_t val = 0; + + if (iend - begin >= max_varint_length) { // fast path + do { + int64_t b = *p++; + val = ((uint64_t(b) & 0x7fU) ); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x7fU) << 7U); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x7fU) << 14U); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x7fU) << 21U); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x7fU) << 28U); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x7fU) << 35U); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x7fU) << 42U); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x7fU) << 49U); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x7fU) << 56U); if (b >= 0) { break; } + b = *p++; val |= ((uint64_t(b) & 0x01U) << 63U); if (b >= 0) { break; } + throw varint_too_long_exception{}; + } while (false); } else { - // this eats too much ram - std::map,std::vector>> by_z7; - for (auto const &entry : w->entries) { - if (std::get<0>(entry) >= 7) { - int level_diff = std::get<0>(entry) - 7; - std::tuple z7_tile{7,std::get<1>(entry)/(1 << level_diff),std::get<2>(entry)/(1 << level_diff)}; - if (by_z7.count(z7_tile) > 0) { - by_z7[z7_tile].push_back(entry); - } else { - by_z7[z7_tile] = {entry}; - } - } - } - - std::vector> leaves; - std::vector> leafdir_z7s; - int leafdir_size = 0; - - for (auto const &group : by_z7) { - auto key = group.first; - if (leafdir_size + group.second.size() <= 21845) { - leafdir_z7s.push_back(key); - leafdir_size += group.second.size(); - } else { - for (auto const &k : leafdir_z7s) { - leaves.emplace_back(std::get<0>(k),std::get<1>(k),std::get<2>(k),w->offset,17*leafdir_size); - auto to_sort = by_z7[k]; - sort(begin(to_sort),end(to_sort),TileCompare()); - for (auto const &entry : to_sort) writeEntry(w->ostream,entry); - } - w->offset += 17 * leafdir_size; - leafdir_z7s = {key}; - leafdir_size = group.second.size(); - } - } - - if (leafdir_size > 0) { - for (auto const &k : leafdir_z7s) { - leaves.emplace_back(std::get<0>(k),std::get<1>(k),std::get<2>(k),w->offset,17*leafdir_size); - auto to_sort = by_z7[k]; - sort(begin(to_sort),end(to_sort),TileCompare()); - for (auto const &entry : to_sort) writeEntry(w->ostream,entry); - } - } - - std::vector> root_entries; - for (auto const &entry : w->entries) { - if (std::get<0>(entry) < 7) root_entries.push_back(entry); - } - - w->ostream.seekp(0); - writePmtilesHeader(w->ostream,serialized_metadata,root_entries.size() + leaves.size()); - - std::sort(begin(root_entries),end(root_entries),TileCompare()); - for (auto const &entry : root_entries) { - writeEntry(w->ostream,entry); - } - std::sort(begin(leaves),end(leaves),TileCompare()); - for (auto const & leaf : leaves) { - writeEntry(w->ostream,leaf,true); - } + unsigned int shift = 0; + while (p != iend && *p < 0) { + val |= (uint64_t(*p++) & 0x7fU) << shift; + shift += 7; + } + if (p == iend) { + throw end_of_buffer_exception{}; + } + val |= uint64_t(*p++) << shift; } - // cout << "Num tiles: " << tiles.size() << endl; - // cout << "Num unique tiles: " << hash_to_offset.size() << endl; + *data = reinterpret_cast(p); + return val; + } - w->ostream.close(); + inline uint64_t decode_varint(const char** data, const char* end) { + // If this is a one-byte varint, decode it here. + if (end != *data && ((static_cast(**data) & 0x80U) == 0)) { + const auto val = static_cast(**data); + ++(*data); + return val; + } + // If this varint is more than one byte, defer to complete implementation. + return detail::decode_varint_impl(data, end); + } -} \ No newline at end of file + inline void rotate(int64_t n, int64_t &x, int64_t &y, int64_t rx, int64_t ry) { + if (ry == 0) { + if (rx == 1) { + x = n-1 - x; + y = n-1 - y; + } + int64_t t = x; + x = y; + y = t; + } + } + + zxy t_on_level(uint8_t z, uint64_t pos) { + int64_t n = 1 << z; + int64_t rx, ry, s, t = pos; + int64_t tx = 0; + int64_t ty = 0; + + for (s=1; s data, uint64_t value) { + int n = 1; + + while (value >= 0x80U) { + *data++ = char((value & 0x7fU) | 0x80U); + value >>= 7U; + ++n; + } + *data = char(value); + + return n; +} + +zxy tileid_to_zxy(uint64_t tileid) { + uint64_t acc = 0; + uint8_t t_z = 0; + while(true) { + uint64_t num_tiles = (1 << t_z) * (1 << t_z); + if (acc + num_tiles > tileid) { + return detail::t_on_level(t_z, tileid - acc); + } + acc += num_tiles; + t_z++; + } +} + +uint64_t zxy_to_tileid(uint8_t z, uint32_t x, uint32_t y) { + uint64_t acc = 0; + for (uint8_t t_z = 0; t_z < z; t_z++) acc += (0x1 << t_z) * (0x1 << t_z); + int64_t n = 1 << z; + int64_t rx, ry, s, d=0; + int64_t tx = x; + int64_t ty = y; + for (s=n/2; s>0; s/=2) { + rx = (tx & s) > 0; + ry = (ty & s) > 0; + d += s * s * ((3 * rx) ^ ry); + detail::rotate(s, tx, ty, rx, ry); + } + return acc + d; +} + +// returns an uncompressed byte buffer +std::string serialize_directory(const std::vector& entries) { + std::string data; + + write_varint(std::back_inserter(data), entries.size()); + + uint64_t last_id = 0; + for (auto const &entry : entries) { + write_varint(std::back_inserter(data), entry.tile_id - last_id); + last_id = entry.tile_id; + } + + for (auto const &entry : entries) { + write_varint(std::back_inserter(data), entry.run_length); + } + + for (auto const &entry : entries) { + write_varint(std::back_inserter(data), entry.length); + } + + for (size_t i = 0; i < entries.size(); i++) { + if (i > 0 && entries[i].offset == entries[i-1].offset + entries[i-1].length) { + write_varint(std::back_inserter(data), 0); + } else { + write_varint(std::back_inserter(data), entries[i].offset+1); + } + } + + return data; +} + +// takes an uncompressed byte buffer +std::vector deserialize_directory(const std::string &decompressed) { + const char *t = decompressed.data(); + const char *end = t + decompressed.size(); + + uint64_t num_entries = detail::decode_varint(&t,end); + + std::vector result; + result.resize(num_entries); + + uint64_t last_id = 0; + for (size_t i = 0; i < num_entries; i++) { + uint64_t tile_id = last_id + detail::decode_varint(&t,end); + result[i].tile_id = tile_id; + last_id = tile_id; + } + + for (size_t i = 0; i < num_entries; i++) { + result[i].run_length = detail::decode_varint(&t,end); + } + + for (size_t i = 0; i < num_entries; i++) { + result[i].length = detail::decode_varint(&t,end); + } + + for (size_t i = 0; i < num_entries; i++) { + uint64_t tmp = detail::decode_varint(&t,end); + + if (i > 0 && tmp == 0) { + result[i].offset = result[i-1].offset + result[i-1].length; + } else { + result[i].offset = tmp - 1; + } + } + + // assert the directory has been fully consumed + if (t != end) { + fprintf(stderr, "Error: malformed pmtiles directory\n"); + exit(EXIT_FAILURE); + } + + return result; +} + +} +#endif \ No newline at end of file diff --git a/cpp/test.cpp b/cpp/test.cpp new file mode 100644 index 0000000..eaa8fa6 --- /dev/null +++ b/cpp/test.cpp @@ -0,0 +1,81 @@ +#include "minunit.h" +#include "pmtiles.hpp" + +using namespace pmtiles; + +MU_TEST(test_tileid_to_zxy) { + auto result = tileid_to_zxy(0); + mu_check(result.z == 0); + mu_check(result.x == 0); + mu_check(result.y == 0); + result = tileid_to_zxy(1); + mu_check(result.z == 1); + mu_check(result.x == 0); + mu_check(result.y == 0); + result = tileid_to_zxy(2); + mu_check(result.z == 1); + mu_check(result.x == 0); + mu_check(result.y == 1); + result = tileid_to_zxy(3); + mu_check(result.z == 1); + mu_check(result.x == 1); + mu_check(result.y == 1); + result = tileid_to_zxy(4); + mu_check(result.z == 1); + mu_check(result.x == 1); + mu_check(result.y == 0); + result = tileid_to_zxy(5); + mu_check(result.z == 2); + mu_check(result.x == 0); + mu_check(result.y == 0); +} + +MU_TEST(test_zxy_to_tileid) { + mu_check(zxy_to_tileid(0,0,0) == 0); + mu_check(zxy_to_tileid(1,0,0) == 1); + mu_check(zxy_to_tileid(1,0,1) == 2); + mu_check(zxy_to_tileid(1,1,1) == 3); + mu_check(zxy_to_tileid(1,1,0) == 4); + mu_check(zxy_to_tileid(2,0,0) == 5); +} + +MU_TEST(test_serialize_directory) { + std::vector entries; + entries.push_back(entryv3(0,0,0,0)); + entries.push_back(entryv3(1,1,1,1)); + entries.push_back(entryv3(2,2,2,2)); + auto serialized = serialize_directory(entries); + auto result = deserialize_directory(serialized); + mu_check(result.size() == 3); + mu_check(result[0].tile_id == 0); + mu_check(result[0].offset == 0); + mu_check(result[0].length == 0); + mu_check(result[0].run_length == 0); + mu_check(result[1].tile_id == 1); + mu_check(result[1].offset == 1); + mu_check(result[1].length == 1); + mu_check(result[1].run_length == 1); + mu_check(result[2].tile_id == 2); + mu_check(result[2].offset == 2); + mu_check(result[2].length == 2); + mu_check(result[2].run_length == 2); +} + +MU_TEST(test_serialize_header) { + headerv3 header; + auto len = header.serialize().size(); + mu_check(len == 127); +} + +MU_TEST_SUITE(test_suite) { + MU_RUN_TEST(test_tileid_to_zxy); + MU_RUN_TEST(test_zxy_to_tileid); + MU_RUN_TEST(test_serialize_directory); + MU_RUN_TEST(test_serialize_header); +} + +int main(int argc, char *argv[]) { + MU_RUN_SUITE(test_suite); + MU_REPORT(); + return MU_EXIT_CODE; +} \ No newline at end of file