Merge pull request #48 from GIScience/feat/tiles-download

feat: tiles download within the tool

.gitignore

@@ -8,7 +8,7 @@
 osm-transform
 test/srtm
 srtm*/*
-cgiardata
+srtmdata
 test/gmted
 gmted*/*
 gmteddata

CMakeLists.txt

@@ -16,16 +16,19 @@ option(BUILD_TESTING "Build the tests" ON)
 # https://cmake.org/cmake/help/git-master/module/FindBoost.html
 find_package(Boost REQUIRED regex program_options)
 find_package(GDAL REQUIRED)
+find_package(ZLIB REQUIRED)
+find_package(CURL REQUIRED)
 
 include_directories(${PROJECT_BINARY_DIR}/src)
 
-set(SOURCE_FILES 
+set(SOURCE_FILES
     location_area_service.cpp
     location_elevation_service.cpp
     rewrite_handler.cpp
 )
 
 add_subdirectory(src)
+target_link_libraries(${PROJECT_NAME} PRIVATE z minizip bz2 expat GDAL::GDAL Boost::regex Boost::program_options CURL::libcurl)
 
 if (BUILD_TESTING)
     enable_testing()

Dockerfile

@@ -12,6 +12,7 @@ RUN set -eux; \
     libboost-regex-dev \
     libboost-program-options-dev \
     libboost-test-dev \
+    ca-certificates \
     ; \
     \
     rm -rf /var/lib/apt/lists/*;
@@ -24,4 +25,4 @@ RUN set -eux; \
    cmake --install /osm-transform/cmake-build
 
 WORKDIR /osm
-ENTRYPOINT ["/usr/local/bin/osm-transform"]:
+ENTRYPOINT ["/usr/local/bin/osm-transform"]

README.md

@@ -67,11 +67,35 @@ Configuration:
   -t [ --threshold ] arg                (=0.5) only used in combination with interpolation, threshold for elevation
   -f [ --config-file ] arg              absolute file path to config file to use
   -d [ --debug_mode ]                   debug_mode
+
+Generic options:
+  -v [ --version ]                      print version string
+  -h [ --help ]                         produce help message
+
+Configuration:
+  -p [ --osm_pbf ] arg                  path to osm pbf file to process
+  -e [ --skip_elevation ]               skip elevation data merge
+  --srtm                                fetch SRTM tiles and exit
+  --gmted                               fetch GMTED tiles and exit
+  -i [ --interpolate ]                  interpolate intermediate nodes
+  -T [ --remove_tag ] arg               (=(.*:)?source(:.*)?|(.*:)?note(:.*)?|url|created_by|fixme|wikipedia) regex to match removable tags
+  -F [ --geo_tiff_folders ] arg         (=tiffs, srtmdata, gmteddata)  paths to geotiff folders
+  -S [ --cache_limit ] arg              (=1073741824) maximum memory used to store tiles in cache
+  -t [ --threshold ] arg                (=0.5) only used in combination with interpolation, threshold for elevation
+  -a [ --area_mapping ] arg             path to area mapping file to use
+  --area_mapping_id_col arg             (=0) column number (zero-based) in area mapping file of area id
+  --area_mapping_geo_col arg            (=1) column number (zero-based) in area mapping file of area geometry
+  --area_mapping_geo_type arg           (=wkt) type of geometry string in area mapping file (possible values: 'wkt' (default), 'geojson')
+  --area_mapping_has_header arg         (=1) area mapping file has header row
+  --area_mapping_processed_file_prefix arg (=mapping_) file prefix for processed mapping files
+  -f [ --config_file ] arg              path to config file to use
+  --index_type arg (=flex_mem)          index type for locations, needed for interpolate. see https://docs.osmcode.org/osmium/latest/osmium-index-types.html
+  -d [ --debug_mode ]                   debug_mode
 ```
 
 [//]: # (TODO update all below)
 
-The `osm-transform.cfg` file is used to set up the tool. The default configuration is as follows:
+A `osm-transform.cfg` file can be used to set up the tool. The default configuration is as follows:
 
 ```
 # number of elevation tiles to keep open in memory simultaneously
@@ -96,24 +120,26 @@ remove_tag = "(.*:)?source(:.*)?|(.*:)?note(:.*)?|url|created_by|fixme|wikipedia
 
 ### Elevation data
 
-For the elevation data merge, also download the elevations tiles for CGIAR SRTM data (
-see [srtm.csi.cgiar.org](http://srtm.csi.cgiar.org/)) and
-GMTED2010 data (as a fallback option,
-see [GMTED2010](https://www.usgs.gov/land-resources/eros/coastal-changes-and-impacts/gmted2010)).
-A python script is provided that can be used to download (and, in the case of CGIAR data, unpack) these resources.
+For the elevation data merge, also download the elevations tiles for CGIAR SRTM data (see [srtm.csi.cgiar.org](http://srtm.csi.cgiar.org/)) and
+GMTED2010 data (as a fallback option, see [GMTED2010](https://www.usgs.gov/land-resources/eros/coastal-changes-and-impacts/gmted2010)).
+The osm-transform tool provides an operation modeto download (and, in the case of CGIAR data, unpack) these resources.
 Be aware those require approx. 63.1 Gb and 26.3 Gb drive space respectively.
 
-You need [python](https://www.python.org/downloads/) and [poetry](https://python-poetry.org/docs/) installed on your
-system.
 In the project directory, run
 
+```shell
+./docker_run.sh --srtm # download CGIAR SRTM tile files
+./docker_run.sh --gmted # download GMTED tile files
 ```
-poetry install 
-poetry run osm-transform srtm-download
-poetry run osm-transform gmted-download
+
+If you have built the tool on your machine, you can run it natively using the following commands: 
+
+```shell
+./osm-transform --srtm # download CGIAR SRTM tile files
+./osm-transform --gmted # download GMTED tile files
 ```
 
-This will create directories `srtmdata` and `gmteddata` respectively and download the geotiff tiles.
+This will create directories `srtmdata` and `gmteddata` respectively and download the geotiff tiles. Note that in those modes the tool does not perform any other operations. 
 
 ## Functionality details
 
@@ -169,7 +195,7 @@ Elevation:                0.04 % failed (660459)
 ```
 
 A log file is written containing all coordinates where no elevation value could be determined.
-The number of cases in the above example is normal (paths on Antarctica etc.).
+The number of cases in the above example is normal (paths on Antarctica and other remote areas etc.).
 
 ### Developers: CLion (Nova) cmake setup
 

src/config.h

@@ -21,6 +21,8 @@ struct Config {
     std::string area_mapping_geo_type;
     bool area_mapping_has_header;
     std::string area_mapping_processed_file_prefix;
+    bool download_srtm = false;
+    bool download_gmted = false;
 
     auto cmd(int argc, char **argv) {
 
@@ -39,6 +41,8 @@ struct Config {
         config.add_options()
                 ("osm_pbf,p", po::value<std::vector<std::string>>(), "path to osm pbf file to process")
                 ("skip_elevation,e", "skip elevation data merge")
+                ("srtm", "fetch SRTM tiles and exit")
+                ("gmted", "fetch GMTED tiles and exit")
                 ("interpolate,i", "interpolate intermediate nodes")
                 ("remove_tag,T", po::value<std::string>(&remove_tag_regex_str)->default_value("(.*:)?source(:.*)?|(.*:)?note(:.*)?|url|created_by|fixme|wikipedia"), "regex to match removable tags")
                 ("geo_tiff_folders,F", po::value<std::vector<std::string>>(&geo_tiff_folders)->multitoken()->default_value(std::vector<std::string>{"tiffs", "srtmdata", "gmteddata"}, "tiffs, srtmdata, gmteddata"), "paths to geotiff folders")
@@ -99,6 +103,17 @@ struct Config {
             exit(0);
         }
 
+        debug_mode = vm.contains("debug_mode");
+        if (debug_mode) {
+            std::cout << "DEBUG MODE" << "\n";
+        }
+
+        download_srtm = vm.contains("srtm");
+        download_gmted = vm.contains("gmted");
+        if (download_srtm || download_gmted) {
+            return;
+        }
+
         if (!vm.contains("osm_pbf")) {
             std::cerr << "no file name" << std::endl;
             exit(1);
@@ -118,11 +133,6 @@ struct Config {
         if (vm.contains("skip_elevation")) {
             add_elevation = false;
         }
-
-        debug_mode = vm.contains("debug_mode");
-        if (debug_mode) {
-            std::cout << "DEBUG MODE" << "\n";
-        }
     }
 };
 

src/osm-transform.cpp

@@ -9,12 +9,15 @@
 
 #include <boost/regex.hpp>
 
-#include <osmium/io/any_input.hpp>
 #include <osmium/io/any_output.hpp>
 #include <osmium/util/file.hpp>
 #include <osmium/util/progress_bar.hpp>
 #include <osmium/util/memory.hpp>
 #include <osmium/visitor.hpp>
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include "minizip/unzip.h"
+#include "minizip/ioapi.h"
 
 using namespace std;
 
@@ -31,13 +34,115 @@ auto show_memory_used() {
     }
 }
 
+void download_tiles(const string type) {
+    string outdir = type + "data";
+    if (mkdir(outdir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == -1) {
+        if (errno == EEXIST) {
+            // alredy exists
+        } else {
+            // something else
+            std::cout << "cannot create tile data folder! Error:" << strerror(errno) << std::endl;
+            return;
+        }
+    }
+    curl_global_init(CURL_GLOBAL_ALL);
+    auto request = 0;
+    auto present = 0;
+    auto loaded = 0;
+    ifstream csv("tiles_"+type+".csv");
+    while(!csv.eof()) {
+        string fname = "";
+        string url = "";
+        getline(csv,fname,',');
+        getline(csv,url);
+        if (fname.empty() || url.empty()) {
+            cout << "Invalid download instruction! " << type << ": " << fname << " - " << url << endl;
+            continue;
+        }
+        request++;
+        string outfile = outdir + "/" + fname;
+        if (filesystem::exists(outfile)) {
+            present++;
+            continue;
+        }
+        CURL *curl;
+        CURLcode res;
+        curl = curl_easy_init();
+        if (curl) {
+            cout << "Download: " << url << endl;
+            auto fp = fopen((outfile + (type == "srtm" ? ".zip" : "")).c_str(),"wb");
+            curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+            curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
+            res = curl_easy_perform(curl);
+            /* always cleanup */
+            curl_easy_cleanup(curl);
+            fclose(fp);
+            if (res > 0) {
+                cout << "ERROR " << res << " downloading from url " << url << endl;
+                continue;
+            }
+        }
+        if (type == "srtm") {
+            cout << "Unpack: " << outfile << endl;
+            unzFile uf = NULL;
+            uf = unzOpen64((outfile + ".zip").c_str());
+            if (uf==NULL) {
+                printf("Cannot open %s.zip\n", outfile.c_str());
+                unzClose(uf);
+                continue;
+            }
+            if (unzLocateFile(uf,fname.c_str(),true)!=UNZ_OK) {
+                printf("file %s not found in the zipfile\n", fname.c_str());
+                unzClose(uf);
+                continue;
+            }
+            if (unzOpenCurrentFile(uf)!=UNZ_OK) {
+                printf("Error within zipfile!\n");
+                unzClose(uf);
+                continue;
+            }
+            ofstream ofile(outfile);
+            if (ofile.is_open()) {
+                int err;
+                int size_buf = 16384;
+                char* buf = (char*)malloc(size_buf);
+                do {
+                    err = unzReadCurrentFile(uf,buf,size_buf);
+                    if (err < 0) {
+                        printf("error %d with zipfile in unzReadCurrentFile\n",err);
+                        break;
+                    }
+                    ofile.write(buf, size_buf);
+                } while (err > 0);
+                ofile.close();
+            }
+            unzCloseCurrentFile(uf);
+            unzClose(uf);
+            filesystem::remove(outfile + ".zip");
+        }
+        loaded++;
+    }
+    cout << "Requested " << request << " tiles. " << present << " files already present, " << loaded << " downloaded." << endl;
+}
+
 void first_pass(Config &config, boost::regex &remove_tag_regex, osmium::nwr_array<osmium::index::IdSetDense<osmium::unsigned_object_id_type>> &valid_ids, osmium::nwr_array<osmium::index::IdSetSmall<osmium::unsigned_object_id_type>> &no_elevation);
 void second_pass(Config &config, boost::regex &remove_tag_regex, osmium::nwr_array<osmium::index::IdSetDense<osmium::unsigned_object_id_type>> &valid_ids, osmium::nwr_array<osmium::index::IdSetSmall<osmium::unsigned_object_id_type>> &no_elevation);
 
 int main(int argc, char **argv) {
     Config config;
     config.cmd(argc, argv);
     try {
+        if (config.download_srtm) {
+            cout << "Downloading SRTM tiles. This might take a while..." << endl;
+            download_tiles("srtm");
+            return 0;
+        }
+        if (config.download_gmted) {
+            cout << "Downloading GMTED tiles. This might take a while..." << endl;
+            download_tiles("gmted");
+            return 0;
+        }
+
         boost::regex remove_tag_regex(config.remove_tag_regex_str, boost::regex::icase);
         osmium::nwr_array<osmium::index::IdSetDense<osmium::unsigned_object_id_type>> valid_ids;
         osmium::nwr_array<osmium::index::IdSetSmall<osmium::unsigned_object_id_type>> no_elevation;
@@ -183,7 +288,7 @@ void second_pass(Config &config, boost::regex &remove_tag_regex,
            reduction, static_cast<float>(reduction) / static_cast<float>(insize) * 100);
     if (config.add_elevation) {
         auto valid_nodes = valid_ids.nodes().size();
-        printf("All Nodes: %19llu Nodes\n", valid_nodes);
+        printf("All Nodes: %19lu Nodes\n", valid_nodes);
         if (config.interpolate) {
             printf("Added Nodes: %17llu Nodes\n",handler.nodes_added_by_interpolation_);
         }