mirrors/PMTiles
1
0
Fork 0
mirror of https://github.com/protomaps/PMTiles.git synced 2026-02-04 02:41:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

c++ v3 utility functions

Browse Source
This commit is contained in:
Brandon Liu
2022-10-12 16:17:33 +08:00
parent 0f03dc3312
commit 47bffa32cf
5 changed files with 773 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
test

3
cpp/Makefile Normal file
View File

@@ -0,0 +1,3 @@
.PHONY: test
test:
clang test.cpp -std=c++11 -lstdc++ -o test && ./test

391
cpp/minunit.h Normal file
View File

@@ -0,0 +1,391 @@
/*
* Copyright (c) 2012 David Siñuela Pastor, siu.4coders@gmail.com
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef MINUNIT_MINUNIT_H
#define MINUNIT_MINUNIT_H
#ifdef __cplusplus
extern "C" {
#endif
#if defined(_WIN32)
#include <Windows.h>
#if defined(_MSC_VER) && _MSC_VER < 1900
#define snprintf _snprintf
#define __func__ __FUNCTION__
#endif
#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
/* Change POSIX C SOURCE version for pure c99 compilers */
#if !defined(_POSIX_C_SOURCE) || _POSIX_C_SOURCE < 200112L
#undef _POSIX_C_SOURCE
#define _POSIX_C_SOURCE 200112L
#endif
#include <unistd.h> /* POSIX flags */
#include <time.h> /* clock_gettime(), time() */
#include <sys/time.h> /* gethrtime(), gettimeofday() */
#include <sys/resource.h>
#include <sys/times.h>
#include <string.h>
#if defined(__MACH__) && defined(__APPLE__)
#include <mach/mach.h>
#include <mach/mach_time.h>
#endif
#if __GNUC__ >= 5 && !defined(__STDC_VERSION__)
#define __func__ __extension__ __FUNCTION__
#endif
#else
#error "Unable to define timers for an unknown OS."
#endif
#include <stdio.h>
#include <math.h>
/* Maximum length of last message */
#define MINUNIT_MESSAGE_LEN 1024
/* Accuracy with which floats are compared */
#define MINUNIT_EPSILON 1E-12
/* Misc. counters */
static int minunit_run = 0;
static int minunit_assert = 0;
static int minunit_fail = 0;
static int minunit_status = 0;
/* Timers */
static double minunit_real_timer = 0;
static double minunit_proc_timer = 0;
/* Last message */
static char minunit_last_message[MINUNIT_MESSAGE_LEN];
/* Test setup and teardown function pointers */
static void (*minunit_setup)(void) = NULL;
static void (*minunit_teardown)(void) = NULL;
/* Definitions */
#define MU_TEST(method_name) static void method_name(void)
#define MU_TEST_SUITE(suite_name) static void suite_name(void)
#define MU__SAFE_BLOCK(block) do {\
block\
} while(0)
/* Run test suite and unset setup and teardown functions */
#define MU_RUN_SUITE(suite_name) MU__SAFE_BLOCK(\
suite_name();\
minunit_setup = NULL;\
minunit_teardown = NULL;\
)
/* Configure setup and teardown functions */
#define MU_SUITE_CONFIGURE(setup_fun, teardown_fun) MU__SAFE_BLOCK(\
minunit_setup = setup_fun;\
minunit_teardown = teardown_fun;\
)
/* Test runner */
#define MU_RUN_TEST(test) MU__SAFE_BLOCK(\
if (minunit_real_timer==0 && minunit_proc_timer==0) {\
minunit_real_timer = mu_timer_real();\
minunit_proc_timer = mu_timer_cpu();\
}\
if (minunit_setup) (*minunit_setup)();\
minunit_status = 0;\
test();\
minunit_run++;\
if (minunit_status) {\
minunit_fail++;\
printf("F");\
printf("\n%s\n", minunit_last_message);\
}\
fflush(stdout);\
if (minunit_teardown) (*minunit_teardown)();\
)
/* Report */
#define MU_REPORT() MU__SAFE_BLOCK(\
double minunit_end_real_timer;\
double minunit_end_proc_timer;\
printf("\n\n%d tests, %d assertions, %d failures\n", minunit_run, minunit_assert, minunit_fail);\
minunit_end_real_timer = mu_timer_real();\
minunit_end_proc_timer = mu_timer_cpu();\
printf("\nFinished in %.8f seconds (real) %.8f seconds (proc)\n\n",\
minunit_end_real_timer - minunit_real_timer,\
minunit_end_proc_timer - minunit_proc_timer);\
)
#define MU_EXIT_CODE minunit_fail
/* Assertions */
#define mu_check(test) MU__SAFE_BLOCK(\
minunit_assert++;\
if (!(test)) {\
snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %s", __func__, __FILE__, __LINE__, #test);\
minunit_status = 1;\
return;\
} else {\
printf(".");\
}\
)
#define mu_fail(message) MU__SAFE_BLOCK(\
minunit_assert++;\
snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %s", __func__, __FILE__, __LINE__, message);\
minunit_status = 1;\
return;\
)
#define mu_assert(test, message) MU__SAFE_BLOCK(\
minunit_assert++;\
if (!(test)) {\
snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %s", __func__, __FILE__, __LINE__, message);\
minunit_status = 1;\
return;\
} else {\
printf(".");\
}\
)
#define mu_assert_int_eq(expected, result) MU__SAFE_BLOCK(\
int minunit_tmp_e;\
int minunit_tmp_r;\
minunit_assert++;\
minunit_tmp_e = (expected);\
minunit_tmp_r = (result);\
if (minunit_tmp_e != minunit_tmp_r) {\
snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %d expected but was %d", __func__, __FILE__, __LINE__, minunit_tmp_e, minunit_tmp_r);\
minunit_status = 1;\
return;\
} else {\
printf(".");\
}\
)
#define mu_assert_double_eq(expected, result) MU__SAFE_BLOCK(\
double minunit_tmp_e;\
double minunit_tmp_r;\
minunit_assert++;\
minunit_tmp_e = (expected);\
minunit_tmp_r = (result);\
if (fabs(minunit_tmp_e-minunit_tmp_r) > MINUNIT_EPSILON) {\
int minunit_significant_figures = 1 - log10(MINUNIT_EPSILON);\
snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: %.*g expected but was %.*g", __func__, __FILE__, __LINE__, minunit_significant_figures, minunit_tmp_e, minunit_significant_figures, minunit_tmp_r);\
minunit_status = 1;\
return;\
} else {\
printf(".");\
}\
)
#define mu_assert_string_eq(expected, result) MU__SAFE_BLOCK(\
const char* minunit_tmp_e = expected;\
const char* minunit_tmp_r = result;\
minunit_assert++;\
if (!minunit_tmp_e) {\
minunit_tmp_e = "<null pointer>";\
}\
if (!minunit_tmp_r) {\
minunit_tmp_r = "<null pointer>";\
}\
if(strcmp(minunit_tmp_e, minunit_tmp_r)) {\
snprintf(minunit_last_message, MINUNIT_MESSAGE_LEN, "%s failed:\n\t%s:%d: '%s' expected but was '%s'", __func__, __FILE__, __LINE__, minunit_tmp_e, minunit_tmp_r);\
minunit_status = 1;\
return;\
} else {\
printf(".");\
}\
)
/*
* The following two functions were written by David Robert Nadeau
* from http://NadeauSoftware.com/ and distributed under the
* Creative Commons Attribution 3.0 Unported License
*/
/**
* Returns the real time, in seconds, or -1.0 if an error occurred.
*
* Time is measured since an arbitrary and OS-dependent start time.
* The returned real time is only useful for computing an elapsed time
* between two calls to this function.
*/
static double mu_timer_real(void)
{
#if defined(_WIN32)
/* Windows 2000 and later. ---------------------------------- */
LARGE_INTEGER Time;
LARGE_INTEGER Frequency;
QueryPerformanceFrequency(&Frequency);
QueryPerformanceCounter(&Time);
Time.QuadPart *= 1000000;
Time.QuadPart /= Frequency.QuadPart;
return (double)Time.QuadPart / 1000000.0;
#elif (defined(__hpux) || defined(hpux)) || ((defined(__sun__) || defined(__sun) || defined(sun)) && (defined(__SVR4) || defined(__svr4__)))
/* HP-UX, Solaris. ------------------------------------------ */
return (double)gethrtime( ) / 1000000000.0;
#elif defined(__MACH__) && defined(__APPLE__)
/* OSX. ----------------------------------------------------- */
static double timeConvert = 0.0;
if ( timeConvert == 0.0 )
{
mach_timebase_info_data_t timeBase;
(void)mach_timebase_info( &timeBase );
timeConvert = (double)timeBase.numer /
(double)timeBase.denom /
1000000000.0;
}
return (double)mach_absolute_time( ) * timeConvert;
#elif defined(_POSIX_VERSION)
/* POSIX. --------------------------------------------------- */
struct timeval tm;
#if defined(_POSIX_TIMERS) && (_POSIX_TIMERS > 0)
{
struct timespec ts;
#if defined(CLOCK_MONOTONIC_PRECISE)
/* BSD. --------------------------------------------- */
const clockid_t id = CLOCK_MONOTONIC_PRECISE;
#elif defined(CLOCK_MONOTONIC_RAW)
/* Linux. ------------------------------------------- */
const clockid_t id = CLOCK_MONOTONIC_RAW;
#elif defined(CLOCK_HIGHRES)
/* Solaris. ----------------------------------------- */
const clockid_t id = CLOCK_HIGHRES;
#elif defined(CLOCK_MONOTONIC)
/* AIX, BSD, Linux, POSIX, Solaris. ----------------- */
const clockid_t id = CLOCK_MONOTONIC;
#elif defined(CLOCK_REALTIME)
/* AIX, BSD, HP-UX, Linux, POSIX. ------------------- */
const clockid_t id = CLOCK_REALTIME;
#else
const clockid_t id = (clockid_t)-1; /* Unknown. */
#endif /* CLOCK_* */
if ( id != (clockid_t)-1 && clock_gettime( id, &ts ) != -1 )
return (double)ts.tv_sec +
(double)ts.tv_nsec / 1000000000.0;
/* Fall thru. */
}
#endif /* _POSIX_TIMERS */
/* AIX, BSD, Cygwin, HP-UX, Linux, OSX, POSIX, Solaris. ----- */
gettimeofday( &tm, NULL );
return (double)tm.tv_sec + (double)tm.tv_usec / 1000000.0;
#else
return -1.0; /* Failed. */
#endif
}
/**
* Returns the amount of CPU time used by the current process,
* in seconds, or -1.0 if an error occurred.
*/
static double mu_timer_cpu(void)
{
#if defined(_WIN32)
/* Windows -------------------------------------------------- */
FILETIME createTime;
FILETIME exitTime;
FILETIME kernelTime;
FILETIME userTime;
/* This approach has a resolution of 1/64 second. Unfortunately, Windows' API does not offer better */
if ( GetProcessTimes( GetCurrentProcess( ),
&createTime, &exitTime, &kernelTime, &userTime ) != 0 )
{
ULARGE_INTEGER userSystemTime;
memcpy(&userSystemTime, &userTime, sizeof(ULARGE_INTEGER));
return (double)userSystemTime.QuadPart / 10000000.0;
}
#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
/* AIX, BSD, Cygwin, HP-UX, Linux, OSX, and Solaris --------- */
#if defined(_POSIX_TIMERS) && (_POSIX_TIMERS > 0)
/* Prefer high-res POSIX timers, when available. */
{
clockid_t id;
struct timespec ts;
#if _POSIX_CPUTIME > 0
/* Clock ids vary by OS. Query the id, if possible. */
if ( clock_getcpuclockid( 0, &id ) == -1 )
#endif
#if defined(CLOCK_PROCESS_CPUTIME_ID)
/* Use known clock id for AIX, Linux, or Solaris. */
id = CLOCK_PROCESS_CPUTIME_ID;
#elif defined(CLOCK_VIRTUAL)
/* Use known clock id for BSD or HP-UX. */
id = CLOCK_VIRTUAL;
#else
id = (clockid_t)-1;
#endif
if ( id != (clockid_t)-1 && clock_gettime( id, &ts ) != -1 )
return (double)ts.tv_sec +
(double)ts.tv_nsec / 1000000000.0;
}
#endif
#if defined(RUSAGE_SELF)
{
struct rusage rusage;
if ( getrusage( RUSAGE_SELF, &rusage ) != -1 )
return (double)rusage.ru_utime.tv_sec +
(double)rusage.ru_utime.tv_usec / 1000000.0;
}
#endif
#if defined(_SC_CLK_TCK)
{
const double ticks = (double)sysconf( _SC_CLK_TCK );
struct tms tms;
if ( times( &tms ) != (clock_t)-1 )
return (double)tms.tms_utime / ticks;
}
#endif
#if defined(CLOCKS_PER_SEC)
{
clock_t cl = clock( );
if ( cl != (clock_t)-1 )
return (double)cl / (double)CLOCKS_PER_SEC;
}
#endif
#endif
return -1; /* Failed. */
}
#ifdef __cplusplus
}
#endif
#endif /* MINUNIT_MINUNIT_H */

435
cpp/pmtiles.hpp
View File

@@ -1,155 +1,312 @@
#include <iostream>
#include <fstream>
#include <tuple>
#ifndef PMTILES_HPP
#define PMTILES_HPP
#include <string>
#include <sstream>
#include <vector>
#include "xxhash.h"
#include <map>
void writePmtilesHeader(std::ostream &outfile, const std::string &metadata, uint16_t root_entries_len) {
uint16_t MAGIC = 0x4d50;
outfile.write((char *)&MAGIC,2);
uint16_t version = 2;
outfile.write((char *)&version,2);
uint32_t metadata_size = metadata.size();
outfile.write((char *)&metadata_size,4);
outfile.write((char *)&root_entries_len,2);
outfile << metadata;
}
namespace pmtiles {
void writeEntry(std::ostream &outfile, const std::tuple<uint8_t,uint32_t,uint32_t,uint64_t,uint32_t> &tile, bool is_directory = false) {
uint8_t z_val = std::get<0>(tile);
if (is_directory) z_val |= 0b10000000;
outfile.write((char *)&z_val,1);
outfile.write((char *)&std::get<1>(tile),3);
outfile.write((char *)&std::get<2>(tile),3);
outfile.write((char *)&std::get<3>(tile),6);
outfile.write((char *)&std::get<4>(tile),4);
}
struct headerv3 {
uint64_t root_dir_offset;
uint64_t root_dir_bytes;
uint64_t json_metadata_offset;
uint64_t json_metadata_bytes;
uint64_t leaf_dirs_offset;
uint64_t leaf_dirs_bytes;
uint64_t tile_data_offset;
uint64_t tile_data_bytes;
uint64_t addressed_tiles_count;
uint64_t tile_entries_count;
uint64_t tile_contents_count;
bool clustered;
uint8_t internal_compression;
uint8_t tile_compression;
uint8_t tile_type;
uint8_t min_zoom;
uint8_t max_zoom;
int32_t min_lon_e7;
int32_t min_lat_e7;
int32_t max_lon_e7;
int32_t max_lat_e7;
uint8_t center_zoom;
int32_t center_lon_e7;
int32_t center_lat_e7;
struct pmtiles_v2_writer {
std::vector<std::tuple<uint8_t,uint32_t,uint32_t,uint64_t,uint32_t>> entries{};
std::ofstream ostream;
uint64_t offset = 0;
std::map<XXH64_hash_t,uint64_t> hash_to_offset;
// WARNING: this is limited to little-endian
std::string serialize() {
std::stringstream ss;
ss << "PMTiles";
uint8_t version = 3;
ss.write((char *)&version,1);
ss.write((char *)&root_dir_offset,8);
ss.write((char *)&root_dir_bytes,8);
ss.write((char *)&json_metadata_offset,8);
ss.write((char *)&json_metadata_bytes,8);
ss.write((char *)&leaf_dirs_offset,8);
ss.write((char *)&leaf_dirs_bytes,8);
ss.write((char *)&tile_data_offset,8);
ss.write((char *)&tile_data_bytes,8);
ss.write((char *)&addressed_tiles_count,8);
ss.write((char *)&tile_entries_count,8);
ss.write((char *)&tile_contents_count,8);
uint8_t clustered_val = 0x0;
if (clustered) {
clustered_val = 0x1;
}
ss.write((char *)&clustered_val,1);
ss.write((char *)&internal_compression,1);
ss.write((char *)&tile_compression,1);
ss.write((char *)&tile_type,1);
ss.write((char *)&min_zoom,1);
ss.write((char *)&max_zoom,1);
ss.write((char *)&min_lon_e7,4);
ss.write((char *)&min_lat_e7,4);
ss.write((char *)&max_lon_e7,4);
ss.write((char *)&max_lat_e7,4);
ss.write((char *)&center_zoom,1);
ss.write((char *)&center_lon_e7,4);
ss.write((char *)&center_lat_e7,4);
return ss.str();
}
};
pmtiles_v2_writer *pmtiles_v2_open(const char *filename) {
pmtiles_v2_writer *w = new pmtiles_v2_writer;
w->ostream.open(filename,std::ios::out | std::ios::binary);
struct zxy {
uint8_t z;
uint32_t x;
uint32_t y;
w->offset = 512000;
for (int i = 0; i < w->offset; ++i) {
char zero = 0;
w->ostream.write(&zero,sizeof(char));
}
return w;
}
void pmtiles_v2_write_tile(pmtiles_v2_writer *w, int z, int x, int y, const std::string &data) {
XXH64_hash_t hash = XXH64(data.data(),data.size(),3857);
if (w->hash_to_offset.count(hash) > 0) {
w->entries.emplace_back(z,x,y,w->hash_to_offset[hash],data.size());
} else {
w->ostream << data;
w->entries.emplace_back(z,x,y,w->offset,data.size());
w->hash_to_offset[hash] = w->offset;
w->offset += data.size();
}
}
struct TileCompare {
bool operator()(std::tuple<uint8_t,uint32_t,uint32_t,uint64_t,uint32_t> const &lhs, std::tuple<uint8_t,uint32_t,uint32_t,uint64_t,uint32_t> const &rhs) const
{
uint8_t zl = std::get<0>(lhs);
uint8_t zr = std::get<0>(rhs);
if (zl != zr) return zl < zr;
uint32_t xl = std::get<1>(lhs);
uint32_t xr = std::get<1>(rhs);
if (xl != xr) return xl < xr;
uint32_t yl = std::get<2>(lhs);
uint32_t yr = std::get<2>(rhs);
return yl < yr;
}
zxy(int _z, int _x, int _y) : z(_z), x(_x), y(_y) {
}
};
void pmtiles_v2_finalize(pmtiles_v2_writer *w, const std::string serialized_metadata) {
if (w->entries.size() < 21845) {
w->ostream.seekp(0);
writePmtilesHeader(w->ostream,serialized_metadata,w->entries.size());
sort(begin(w->entries),end(w->entries),TileCompare());
struct entryv3 {
uint64_t tile_id;
uint64_t offset;
uint32_t length;
uint32_t run_length;
for (auto const &entry : w->entries) {
writeEntry(w->ostream,entry);
}
entryv3() : tile_id(0), offset(0), length(0), run_length(0) {
}
entryv3(uint64_t _tile_id, uint64_t _offset, uint32_t _length, uint32_t _run_length)
: tile_id(_tile_id), offset(_offset), length(_length), run_length(_run_length) {
}
};
struct varint_too_long_exception : std::exception {
const char* what() const noexcept override {
return "varint too long exception";
}
};
struct end_of_buffer_exception : std::exception {
const char* what() const noexcept override {
return "end of buffer exception";
}
};
namespace detail {
constexpr const int8_t max_varint_length = sizeof(uint64_t) * 8 / 7 + 1;
// from https://github.com/mapbox/protozero/blob/master/include/protozero/varint.hpp
inline uint64_t decode_varint_impl(const char** data, const char* end) {
const auto* begin = reinterpret_cast<const int8_t*>(*data);
const auto* iend = reinterpret_cast<const int8_t*>(end);
const int8_t* p = begin;
uint64_t val = 0;
if (iend - begin >= max_varint_length) { // fast path
do {
int64_t b = *p++;
val = ((uint64_t(b) & 0x7fU) ); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x7fU) << 7U); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x7fU) << 14U); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x7fU) << 21U); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x7fU) << 28U); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x7fU) << 35U); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x7fU) << 42U); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x7fU) << 49U); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x7fU) << 56U); if (b >= 0) { break; }
b = *p++; val |= ((uint64_t(b) & 0x01U) << 63U); if (b >= 0) { break; }
throw varint_too_long_exception{};
} while (false);
} else {
// this eats too much ram
std::map<std::tuple<uint8_t,uint32_t,uint32_t>,std::vector<std::tuple<uint8_t,uint32_t,uint32_t,uint64_t,uint32_t>>> by_z7;
for (auto const &entry : w->entries) {
if (std::get<0>(entry) >= 7) {
int level_diff = std::get<0>(entry) - 7;
std::tuple<uint8_t,uint32_t,uint32_t> z7_tile{7,std::get<1>(entry)/(1 << level_diff),std::get<2>(entry)/(1 << level_diff)};
if (by_z7.count(z7_tile) > 0) {
by_z7[z7_tile].push_back(entry);
} else {
by_z7[z7_tile] = {entry};
}
}
}
std::vector<std::tuple<uint8_t,uint32_t,uint32_t,uint64_t,uint32_t>> leaves;
std::vector<std::tuple<uint8_t,uint32_t,uint32_t>> leafdir_z7s;
int leafdir_size = 0;
for (auto const &group : by_z7) {
auto key = group.first;
if (leafdir_size + group.second.size() <= 21845) {
leafdir_z7s.push_back(key);
leafdir_size += group.second.size();
} else {
for (auto const &k : leafdir_z7s) {
leaves.emplace_back(std::get<0>(k),std::get<1>(k),std::get<2>(k),w->offset,17*leafdir_size);
auto to_sort = by_z7[k];
sort(begin(to_sort),end(to_sort),TileCompare());
for (auto const &entry : to_sort) writeEntry(w->ostream,entry);
}
w->offset += 17 * leafdir_size;
leafdir_z7s = {key};
leafdir_size = group.second.size();
}
}
if (leafdir_size > 0) {
for (auto const &k : leafdir_z7s) {
leaves.emplace_back(std::get<0>(k),std::get<1>(k),std::get<2>(k),w->offset,17*leafdir_size);
auto to_sort = by_z7[k];
sort(begin(to_sort),end(to_sort),TileCompare());
for (auto const &entry : to_sort) writeEntry(w->ostream,entry);
}
}
std::vector<std::tuple<uint8_t,uint32_t,uint32_t,uint64_t,uint32_t>> root_entries;
for (auto const &entry : w->entries) {
if (std::get<0>(entry) < 7) root_entries.push_back(entry);
}
w->ostream.seekp(0);
writePmtilesHeader(w->ostream,serialized_metadata,root_entries.size() + leaves.size());
std::sort(begin(root_entries),end(root_entries),TileCompare());
for (auto const &entry : root_entries) {
writeEntry(w->ostream,entry);
}
std::sort(begin(leaves),end(leaves),TileCompare());
for (auto const & leaf : leaves) {
writeEntry(w->ostream,leaf,true);
}
unsigned int shift = 0;
while (p != iend && *p < 0) {
val |= (uint64_t(*p++) & 0x7fU) << shift;
shift += 7;
}
if (p == iend) {
throw end_of_buffer_exception{};
}
val |= uint64_t(*p++) << shift;
}
// cout << "Num tiles: " << tiles.size() << endl;
// cout << "Num unique tiles: " << hash_to_offset.size() << endl;
*data = reinterpret_cast<const char*>(p);
return val;
}
w->ostream.close();
inline uint64_t decode_varint(const char** data, const char* end) {
// If this is a one-byte varint, decode it here.
if (end != *data && ((static_cast<uint64_t>(**data) & 0x80U) == 0)) {
const auto val = static_cast<uint64_t>(**data);
++(*data);
return val;
}
// If this varint is more than one byte, defer to complete implementation.
return detail::decode_varint_impl(data, end);
}
inline void rotate(int64_t n, int64_t &x, int64_t &y, int64_t rx, int64_t ry) {
if (ry == 0) {
if (rx == 1) {
x = n-1 - x;
y = n-1 - y;
}
int64_t t = x;
x = y;
y = t;
}
}
zxy t_on_level(uint8_t z, uint64_t pos) {
int64_t n = 1 << z;
int64_t rx, ry, s, t = pos;
int64_t tx = 0;
int64_t ty = 0;
for (s=1; s<n; s*=2) {
rx = 1 & (t/2);
ry = 1 & (t ^ rx);
rotate(s, tx, ty, rx, ry);
tx += s * rx;
ty += s * ry;
t /= 4;
}
return zxy(z,tx,ty);
}
} // end namespace detail
inline int write_varint(std::back_insert_iterator<std::string> data, uint64_t value) {
int n = 1;
while (value >= 0x80U) {
*data++ = char((value & 0x7fU) | 0x80U);
value >>= 7U;
++n;
}
*data = char(value);
return n;
}
zxy tileid_to_zxy(uint64_t tileid) {
uint64_t acc = 0;
uint8_t t_z = 0;
while(true) {
uint64_t num_tiles = (1 << t_z) * (1 << t_z);
if (acc + num_tiles > tileid) {
return detail::t_on_level(t_z, tileid - acc);
}
acc += num_tiles;
t_z++;
}
}
uint64_t zxy_to_tileid(uint8_t z, uint32_t x, uint32_t y) {
uint64_t acc = 0;
for (uint8_t t_z = 0; t_z < z; t_z++) acc += (0x1 << t_z) * (0x1 << t_z);
int64_t n = 1 << z;
int64_t rx, ry, s, d=0;
int64_t tx = x;
int64_t ty = y;
for (s=n/2; s>0; s/=2) {
rx = (tx & s) > 0;
ry = (ty & s) > 0;
d += s * s * ((3 * rx) ^ ry);
detail::rotate(s, tx, ty, rx, ry);
}
return acc + d;
}
// returns an uncompressed byte buffer
std::string serialize_directory(const std::vector<entryv3>& entries) {
std::string data;
write_varint(std::back_inserter(data), entries.size());
uint64_t last_id = 0;
for (auto const &entry : entries) {
write_varint(std::back_inserter(data), entry.tile_id - last_id);
last_id = entry.tile_id;
}
for (auto const &entry : entries) {
write_varint(std::back_inserter(data), entry.run_length);
}
for (auto const &entry : entries) {
write_varint(std::back_inserter(data), entry.length);
}
for (size_t i = 0; i < entries.size(); i++) {
if (i > 0 && entries[i].offset == entries[i-1].offset + entries[i-1].length) {
write_varint(std::back_inserter(data), 0);
} else {
write_varint(std::back_inserter(data), entries[i].offset+1);
}
}
return data;
}
// takes an uncompressed byte buffer
std::vector<entryv3> deserialize_directory(const std::string &decompressed) {
const char *t = decompressed.data();
const char *end = t + decompressed.size();
uint64_t num_entries = detail::decode_varint(&t,end);
std::vector<entryv3> result;
result.resize(num_entries);
uint64_t last_id = 0;
for (size_t i = 0; i < num_entries; i++) {
uint64_t tile_id = last_id + detail::decode_varint(&t,end);
result[i].tile_id = tile_id;
last_id = tile_id;
}
for (size_t i = 0; i < num_entries; i++) {
result[i].run_length = detail::decode_varint(&t,end);
}
for (size_t i = 0; i < num_entries; i++) {
result[i].length = detail::decode_varint(&t,end);
}
for (size_t i = 0; i < num_entries; i++) {
uint64_t tmp = detail::decode_varint(&t,end);
if (i > 0 && tmp == 0) {
result[i].offset = result[i-1].offset + result[i-1].length;
} else {
result[i].offset = tmp - 1;
}
}
// assert the directory has been fully consumed
if (t != end) {
fprintf(stderr, "Error: malformed pmtiles directory\n");
exit(EXIT_FAILURE);
}
return result;
}
}
#endif

81
cpp/test.cpp Normal file
View File

@@ -0,0 +1,81 @@
#include "minunit.h"
#include "pmtiles.hpp"
using namespace pmtiles;
MU_TEST(test_tileid_to_zxy) {
auto result = tileid_to_zxy(0);
mu_check(result.z == 0);
mu_check(result.x == 0);
mu_check(result.y == 0);
result = tileid_to_zxy(1);
mu_check(result.z == 1);
mu_check(result.x == 0);
mu_check(result.y == 0);
result = tileid_to_zxy(2);
mu_check(result.z == 1);
mu_check(result.x == 0);
mu_check(result.y == 1);
result = tileid_to_zxy(3);
mu_check(result.z == 1);
mu_check(result.x == 1);
mu_check(result.y == 1);
result = tileid_to_zxy(4);
mu_check(result.z == 1);
mu_check(result.x == 1);
mu_check(result.y == 0);
result = tileid_to_zxy(5);
mu_check(result.z == 2);
mu_check(result.x == 0);
mu_check(result.y == 0);
}
MU_TEST(test_zxy_to_tileid) {
mu_check(zxy_to_tileid(0,0,0) == 0);
mu_check(zxy_to_tileid(1,0,0) == 1);
mu_check(zxy_to_tileid(1,0,1) == 2);
mu_check(zxy_to_tileid(1,1,1) == 3);
mu_check(zxy_to_tileid(1,1,0) == 4);
mu_check(zxy_to_tileid(2,0,0) == 5);
}
MU_TEST(test_serialize_directory) {
std::vector<entryv3> entries;
entries.push_back(entryv3(0,0,0,0));
entries.push_back(entryv3(1,1,1,1));
entries.push_back(entryv3(2,2,2,2));
auto serialized = serialize_directory(entries);
auto result = deserialize_directory(serialized);
mu_check(result.size() == 3);
mu_check(result[0].tile_id == 0);
mu_check(result[0].offset == 0);
mu_check(result[0].length == 0);
mu_check(result[0].run_length == 0);
mu_check(result[1].tile_id == 1);
mu_check(result[1].offset == 1);
mu_check(result[1].length == 1);
mu_check(result[1].run_length == 1);
mu_check(result[2].tile_id == 2);
mu_check(result[2].offset == 2);
mu_check(result[2].length == 2);
mu_check(result[2].run_length == 2);
}
MU_TEST(test_serialize_header) {
headerv3 header;
auto len = header.serialize().size();
mu_check(len == 127);
}
MU_TEST_SUITE(test_suite) {
MU_RUN_TEST(test_tileid_to_zxy);
MU_RUN_TEST(test_zxy_to_tileid);
MU_RUN_TEST(test_serialize_directory);
MU_RUN_TEST(test_serialize_header);
}
int main(int argc, char *argv[]) {
MU_RUN_SUITE(test_suite);
MU_REPORT();
return MU_EXIT_CODE;
}