topo_map_processor 0.1.8__tar.gz → 0.1.9__tar.gz

{topo_map_processor-0.1.8 → topo_map_processor-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topo_map_processor
-Version: 0.1.8
+Version: 0.1.9
 Summary: Library for processing topographic maps
 Project-URL: homepage, https://github.com/ramSeraph/topo_map_processor
 Project-URL: repository, https://github.com/ramSeraph/topo_map_processor.git

{topo_map_processor-0.1.8 → topo_map_processor-0.1.9}/topo_map_processor/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.8'
-__version_tuple__ = version_tuple = (0, 1, 8)
+__version__ = version = '0.1.9'
+__version_tuple__ = version_tuple = (0, 1, 9)

{topo_map_processor-0.1.8 → topo_map_processor-0.1.9}/topo_map_processor/bash_scripts/upload_to_release.sh

@@ -81,10 +81,6 @@ for release in $RELEASES_TO_PROCESS; do
   fi
 done
 
-if [ -z "$EXISTING_ASSETS_WITH_RELEASES" ]; then
-    echo "Could not fetch any assets. This could mean the releases do not have assets, or you lack permissions."
-fi
-
 # Determine available releases and their current asset counts
 AVAILABLE_RELEASES=()
 AVAILABLE_ASSET_COUNTS=()
@@ -107,8 +103,11 @@ echo "Starting upload process from folder '$FOLDER'..."
 # Find all files in the folder that are not yet in any release and loop through them.
 find "${FOLDER}" -type f | grep "^.*\.${EXT}$" | while read -r FILE_PATH; do
     FILENAME=$(basename "$FILE_PATH")
+    echo "Processing file: $FILENAME"
 
+    set +o pipefail
     RELEASE_FOR_ASSET=$(echo -n "$EXISTING_ASSETS_WITH_RELEASES" | grep "^${FILENAME} " | head -n 1 | awk '{print $2}')
+    set -o pipefail
 
     if [ -n "$RELEASE_FOR_ASSET" ]; then
         # Asset exists

{topo_map_processor-0.1.8 → topo_map_processor-0.1.9}/topo_map_processor/tools/download_mosaic.py

@@ -191,6 +191,18 @@ def cli():
 
     mosaic_fname = get_filename_from_url(mosaic_url)
 
+    global session, timeout
+    session = requests.session()
+    retries = args.num_http_retries
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+    )
+    session.mount('http://', HTTPAdapter(max_retries=retry))
+    session.mount('https://', HTTPAdapter(max_retries=retry))
+    timeout = args.request_timeout_secs
+
     mosaic_file = Path(mosaic_fname)
     mosaic_data = get_mosaic(mosaic_file, mosaic_url)
     metadata = get_metadata(mosaic_data)
@@ -221,24 +233,14 @@ def cli():
         initialize_tables(cursor, metadata)
         mark_done(tracker_file, 'table_init')
 
-    global session, timeout
-    session = requests.session()
-    retries = args.num_http_retries
-    retry = Retry(
-        total=retries,
-        read=retries,
-        connect=retries,
-    )
-    session.mount('http://', HTTPAdapter(max_retries=retry))
-    session.mount('https://', HTTPAdapter(max_retries=retry))
-    timeout = args.request_timeout_secs
-
     for k in mosaic_data.keys():
         pmtiles_url = get_pmtiles_url(mosaic_url, k)
         if stage_done(tracker_file, k):
             continue
 
         pmtiles_fname = get_filename_from_url(pmtiles_url)
+        if Path(pmtiles_fname).exists():
+            raise Exception(f'{pmtiles_fname} already exists, delete existing file to continue')
         download_file(pmtiles_url, pmtiles_fname)
         add_to_mbtiles(pmtiles_fname, cursor, conn)
         mark_done(tracker_file, k)

{topo_map_processor-0.1.8 → topo_map_processor-0.1.9}/topo_map_processor/tools/partition.py

@@ -298,7 +298,7 @@ def create_pmtiles(partition_info, reader, to_pmtiles_prefix):
             tiles_to_suffix_idx[t] = i
         i += 1
 
-    curr_zs = {}
+    curr_low_zs = {}
     max_lats = {}
     min_lats = {}
     max_lons = {}
@@ -306,7 +306,7 @@ def create_pmtiles(partition_info, reader, to_pmtiles_prefix):
     min_zooms = {}
     max_zooms = {}
     for suffix in suffix_arr:
-        curr_zs[suffix] = None
+        curr_low_zs[suffix] = None
         max_lats[suffix] = min_lats[suffix] = max_lons[suffix] = min_lons[suffix] = None
         min_zooms[suffix] = max_zooms[suffix] = None
 
@@ -319,21 +319,21 @@ def create_pmtiles(partition_info, reader, to_pmtiles_prefix):
         suffix = suffix_arr[tiles_to_suffix_idx[t]]
         writer = writers[suffix]
 
-        if curr_zs[suffix] is None or curr_zs[suffix] < t.z:
+        if curr_low_zs[suffix] is None or curr_low_zs[suffix] > t.z:
             max_lats[suffix] = min_lats[suffix] = max_lons[suffix] = min_lons[suffix] = None
-            curr_zs[suffix] = t.z
+            curr_low_zs[suffix] = t.z
 
         t_bounds = mercantile.bounds(t)
-        if curr_zs[suffix] == t.z and (max_lats[suffix] is None or t_bounds.north > max_lats[suffix]):
+        if curr_low_zs[suffix] == t.z and (max_lats[suffix] is None or t_bounds.north > max_lats[suffix]):
             max_lats[suffix] = t_bounds.north
 
-        if curr_zs[suffix] == t.z and (min_lats[suffix] is None or t_bounds.south < min_lats[suffix]):
+        if curr_low_zs[suffix] == t.z and (min_lats[suffix] is None or t_bounds.south < min_lats[suffix]):
             min_lats[suffix] = t_bounds.south
 
-        if curr_zs[suffix] == t.z and (max_lons[suffix] is None or t_bounds.east > max_lons[suffix]):
+        if curr_low_zs[suffix] == t.z and (max_lons[suffix] is None or t_bounds.east > max_lons[suffix]):
             max_lons[suffix] = t_bounds.east
 
-        if curr_zs[suffix] == t.z and (min_lons[suffix] is None or t_bounds.west < min_lons[suffix]):
+        if curr_low_zs[suffix] == t.z and (min_lons[suffix] is None or t_bounds.west < min_lons[suffix]):
             min_lons[suffix] = t_bounds.west
                         
         if min_zooms[suffix] is None or min_zooms[suffix] > t.z:
@@ -357,10 +357,15 @@ def create_pmtiles(partition_info, reader, to_pmtiles_prefix):
         if format_string == "jpg":
             format_string = "jpeg"
 
-        is_vector_tiles = format_string == "pbf"
+        is_vector_tiles = (format_string == "pbf")
+
+        if is_vector_tiles or src_metadata.get('compression', None) == "gzip":
+            tile_compression = Compression.GZIP
+        else:
+            tile_compression = Compression.NONE
 
         header = {
-            "tile_compression": Compression.GZIP if is_vector_tiles else Compression.NONE,
+            "tile_compression": tile_compression,
             "min_lon_e7": int(min_lons[suffix] * 10000000),
             "min_lat_e7": int(min_lats[suffix] * 10000000),
             "max_lon_e7": int(max_lons[suffix] * 10000000),

{topo_map_processor-0.1.8 → topo_map_processor-0.1.9}/topo_map_processor/tools/tile_sources.py

@@ -199,7 +199,7 @@ class MBTilesSource:
             all_metadata[k] = v
 
         metadata = {}
-        for k in ['type', 'format', 'attribution', 'description', 'name', 'version', 'vector_layers', 'maxzoom', 'minzoom']:
+        for k in ['type', 'format', 'attribution', 'description', 'name', 'version', 'vector_layers', 'maxzoom', 'minzoom', 'compression']:
             if k not in all_metadata:
                 continue
             metadata[k] = all_metadata[k]
@@ -320,7 +320,7 @@ class StackedTileSource:
     def for_all_z(self, z):
         seen = set()
         for i, src in enumerate(self.srcs):
-            print(f'iterating over source {i} for {z}')
+            #print(f'iterating over source {i} for {z}')
             for (tile, size) in src.for_all_z(z):
                 if tile in seen:
                     continue
@@ -330,7 +330,7 @@ class StackedTileSource:
     def all(self):
         seen = set()
         for i, src in enumerate(self.srcs):
-            print(f'iterating over source {i} for all levels')
+            #print(f'iterating over source {i} for all levels')
             for (tile, data) in src.all():
                 if tile in seen:
                     continue
@@ -340,7 +340,7 @@ class StackedTileSource:
     def all_sizes(self):
         seen = set()
         for i, src in enumerate(self.srcs):
-            print(f'iterating over source {i} for all levels')
+            #print(f'iterating over source {i} for all levels')
             for (tile, size) in src.all_sizes():
                 if tile in seen:
                     continue