topo_map_processor 0.1.7__tar.gz → 0.1.8__tar.gz

{topo_map_processor-0.1.7 → topo_map_processor-0.1.8}/.gitignore

@@ -1,5 +1,6 @@
 # Python-generated files
 __pycache__/
+topo_map_processor/_version.py
 *.py[oc]
 build/
 dist/

{topo_map_processor-0.1.7 → topo_map_processor-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topo_map_processor
-Version: 0.1.7
+Version: 0.1.8
 Summary: Library for processing topographic maps
 Project-URL: homepage, https://github.com/ramSeraph/topo_map_processor
 Project-URL: repository, https://github.com/ramSeraph/topo_map_processor.git
@@ -33,6 +33,7 @@ License-File: LICENSE
 Requires-Python: >=3.12
 Requires-Dist: mercantile>=1.2.1
 Requires-Dist: pmtiles>=3.4.1
+Requires-Dist: requests>=2.32.4
 Requires-Dist: shapely>=2.1.1
 Provides-Extra: easyocr
 Requires-Dist: easyocr>=1.7.2; extra == 'easyocr'
@@ -211,9 +212,10 @@ Similar to corner detection, the `locate_grid_lines` method can be implemented u
 
 The `topo_map_processor.tools.tile_sources` module provides classes for reading tiles from different sources.
 
--   `DiskSource`: Reads tiles from a directory on the local disk.
--   `PartitionedPMTilesSource`: Reads tiles from a set of partitioned PMTiles files.
--   `DiskAndPartitionedPMTilesSource`: Reads tiles from both a local directory and a set of partitioned PMTiles files, useful for retiling.
+-   `DiskTilesSource`: Reads tiles from a directory on the local disk.
+-   `MBTilesSource`: Reads tiles from an MBTiles file.
+-   `PMTilesSource`: Reads tiles from a PMTiles file.
+-   `StackedTileSource`: Combines multiple tile sources (Disk, MBTiles, PMTiles) into a single logical source, allowing for seamless access across different storage types.
 
 ### Command-Line Tools
 
@@ -231,24 +233,37 @@ collect-bounds --bounds-dir <directory> --output-file <file> [--preexisting-file
 -   `--output-file`: Output GeoJSON file path. (Required)
 -   `--preexisting-file`: Pre existing GeoJSON file path. (Optional)
 
+#### `download-mosaic`
+
+Downloads a PMTiles mosaic and converts it to a single MBTiles file. A mosaic is a JSON file that lists multiple PMTiles files.
+
+```bash
+download-mosaic --mosaic_url <url> [--output_file <file>] [--request-timeout-secs <seconds>] [--num-http-retries <retries>]
+```
+
+-   `--mosaic_url`, `-u`: URL of the mosaic JSON file. (Required)
+-   `--output_file`, `-o`: Output MBTiles file name. If not specified, it is derived from the mosaic URL. (Optional)
+-   `--request-timeout-secs`, `-t`: Timeout for HTTP requests in seconds (default: 60).
+-   `--num-http-retries`, `-r`: Number of retries for HTTP requests (default: 3).
+
 #### `partition`
 
-Partitions a large set of tiles into smaller PMTiles files. This is useful for managing large datasets, for example, to stay within file size limits for services like GitHub Releases. When run with `--only-disk`, it creates a new partitioned `pmtiles` fileset from a directory of tiles. When run without, it can be used to re-partition an existing `pmtiles` fileset, for example after having retiled some of the sheets.
+Partitions a large set of tiles into smaller PMTiles files. This is useful for managing large datasets, for example, to stay within file size limits for services like GitHub Releases. It creates a mosaic file if the output is partitioned into multiple files.
 
 ```bash
-partition --to-pmtiles-prefix <prefix> --from-tiles-dir <dir> [--from-pmtiles-prefix <prefix>] [--name <name>] [--description <desc>] [--max-zoom <zoom>] [--min-zoom <zoom>] [--only-disk] (--attribution <text> | --attribution-file <file>)
+partition --from-source <source> [--from-source <source> ...] --to-pmtiles <file> [--size-limit <limit>]
 ```
 
--   `--to-pmtiles-prefix`: Prefix for the output PMTiles files. (Required)
--   `--from-tiles-dir`: Directory containing the input tiles. (Required)
--   `--from-pmtiles-prefix`: Prefix for input PMTiles files (required if not using `--only-disk`).
--   `--name`: Name of the mosaic. (Required if using `--only-disk`)
--   `--description`: Description of the mosaic. (Required if using `--only-disk`)
--   `--max-zoom`: Maximum zoom level for the mosaic. (Defaults to the max zoom found in the source)
--   `--min-zoom`: Minimum zoom level for the mosaic (default: 0).
--   `--only-disk`: Only read tiles from the local disk.
--   `--attribution`: Attribution text for the mosaic. (Required if using `--only-disk`)
--   `--attribution-file`: File containing attribution text. (Required if using `--only-disk`)
+-   `--from-source`: Path to a source file or directory. Can be repeated. Supported sources:
+    -   `.mbtiles` file: e.g., `path/to/file.mbtiles`
+    -   `.pmtiles` file (supports glob patterns): e.g., `path/to/*.pmtiles`
+    -   Directory containing tiles: e.g., `path/to/tiles_dir`
+-   `--to-pmtiles`: Output PMTiles file. The prefix for partitioned files will be derived from this argument.
+-   `--size-limit`: Maximum size of each partition. Can be a number in bytes (e.g., `100000000`), or with units (e.g., `100M`, `2G`), or a preset:
+    -   `github_release`: 2GB limit
+    -   `github_file`: 100MB limit
+    -   `cloudflare_object`: 512MB limit
+    The overhead (delta) is scaled proportionally based on the `github_release` preset (2GB with 5MB overhead). Defaults to `github_release`.
 
 #### `retile`
 
@@ -259,31 +274,37 @@ The tool works in two stages. First, it calculates the full set of sheets that n
 In the second stage, it retiles the affected area. It uses the pull list to create a temporary virtual raster (`.vrt`), generates new base tiles for the affected region, and then reconstructs the upper-level tiles by pulling existing, unaffected tiles from the PMTiles source.
 
 ```bash
-retile --retile-list-file <file> --bounds-file <file> [--max-zoom <zoom>] [--min-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-pmtiles-prefix <prefix>] [--sheets-to-pull-list-outfile <file>] [--num-parallel <processes>]
+retile --retile-list-file <file> --bounds-file <file> [--max-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-source <source>] [--sheets-to-pull-list-outfile <file>] [--num-parallel <processes>] [--tile-quality <quality>]
 ```
 
 -   `--retile-list-file`: File containing the list of sheets to retile. (Required)
 -   `--bounds-file`: GeoJSON file containing the list of available sheets and their geographic bounds. (Required)
--   `--max-zoom`: Maximum zoom level to create tiles for. (Defaults to the max zoom from the PMTiles source)
--   `--min-zoom`: Minimum zoom level to create tiles for. (Defaults to the min zoom from the PMTiles source)
+-   `--max-zoom`: Maximum zoom level to create tiles for. Defaults to the max zoom from the source if `--from-source` is provided, otherwise it is required.
 -   `--tiffs-dir`: Directory where the GeoTIFFs are located. (Required if not in pull-list generation mode)
 -   `--tiles-dir`: Directory where the tiles will be created. (Required if not in pull-list generation mode)
--   `--from-pmtiles-prefix`: Prefix for the PMTiles source from which to pull existing tiles. (Required if not in pull-list generation mode)
+-   `--from-source`: Location of the source from which to pull existing tiles. Can be a glob pattern to match a group of pmtiles. Required if not in pull-list generation mode.
 -   `--sheets-to-pull-list-outfile`: If provided, the script will only calculate the full list of sheets that need to be processed (including adjacent ones) and write it to this file, then exit.
 -   `--num-parallel`: Number of parallel processes to use for tiling (default: number of CPU cores).
+-   `--tile-quality`: Quality of compression for webp and jpg (default: 75).
 
 #### `tile`
 
 Creates web map tiles from a directory of GeoTIFF files. It combines them into a virtual raster (`.vrt`) for efficient processing.
 
 ```bash
-tile --tiles-dir <dir> --tiffs-dir <dir> --max-zoom <zoom> [--min-zoom <zoom>] [--num-parallel <processes>]
+tile --tiles-dir <dir> --tiffs-dir <dir> --max-zoom <zoom> --name <name> --description <desc> (--attribution <text> | --attribution-file <file>) [--min-zoom <zoom>] [--tile-extension <ext>] [--tile-quality <quality>] [--num-parallel <processes>]
 ```
 
--   `<tiles-dir>`: Directory to store the output tiles. (Required)
--   `<tiffs-dir>`: Directory containing the input GeoTIFF files. (Required)
--   `<max-zoom>`: Maximum zoom level for tiling. (Required)
--   `<min-zoom>`: Minimum zoom level for tiling (default: 0).
+-   `--tiles-dir`: Directory to store the output tiles. (Required)
+-   `--tiffs-dir`: Directory containing the input GeoTIFF files. (Required)
+-   `--max-zoom`: Maximum zoom level for tiling. (Required)
+-   `--name`: Name of the mosaic. (Required)
+-   `--description`: Description of the mosaic. (Required)
+-   `--attribution`: Attribution text for the mosaic. Required if `--attribution-file` is not used.
+-   `--attribution-file`: File containing attribution text for the mosaic. Required if `--attribution` is not used.
+-   `--min-zoom`: Minimum zoom level for tiling (default: 0).
+-   `--tile-extension`: Tile file extension (default: webp). Choices: `webp`, `jpg`, `jpeg`, `png`.
+-   `--tile-quality`: Compression quality of the tiles for `webp` and `jpg` (default: 75).
 -   `--num-parallel`: Number of parallel processes to use for tiling (default: number of CPU cores).
 
 ### Running Tools Directly with `uvx`

{topo_map_processor-0.1.7 → topo_map_processor-0.1.8}/README.md

@@ -163,9 +163,10 @@ Similar to corner detection, the `locate_grid_lines` method can be implemented u
 
 The `topo_map_processor.tools.tile_sources` module provides classes for reading tiles from different sources.
 
--   `DiskSource`: Reads tiles from a directory on the local disk.
--   `PartitionedPMTilesSource`: Reads tiles from a set of partitioned PMTiles files.
--   `DiskAndPartitionedPMTilesSource`: Reads tiles from both a local directory and a set of partitioned PMTiles files, useful for retiling.
+-   `DiskTilesSource`: Reads tiles from a directory on the local disk.
+-   `MBTilesSource`: Reads tiles from an MBTiles file.
+-   `PMTilesSource`: Reads tiles from a PMTiles file.
+-   `StackedTileSource`: Combines multiple tile sources (Disk, MBTiles, PMTiles) into a single logical source, allowing for seamless access across different storage types.
 
 ### Command-Line Tools
 
@@ -183,24 +184,37 @@ collect-bounds --bounds-dir <directory> --output-file <file> [--preexisting-file
 -   `--output-file`: Output GeoJSON file path. (Required)
 -   `--preexisting-file`: Pre existing GeoJSON file path. (Optional)
 
+#### `download-mosaic`
+
+Downloads a PMTiles mosaic and converts it to a single MBTiles file. A mosaic is a JSON file that lists multiple PMTiles files.
+
+```bash
+download-mosaic --mosaic_url <url> [--output_file <file>] [--request-timeout-secs <seconds>] [--num-http-retries <retries>]
+```
+
+-   `--mosaic_url`, `-u`: URL of the mosaic JSON file. (Required)
+-   `--output_file`, `-o`: Output MBTiles file name. If not specified, it is derived from the mosaic URL. (Optional)
+-   `--request-timeout-secs`, `-t`: Timeout for HTTP requests in seconds (default: 60).
+-   `--num-http-retries`, `-r`: Number of retries for HTTP requests (default: 3).
+
 #### `partition`
 
-Partitions a large set of tiles into smaller PMTiles files. This is useful for managing large datasets, for example, to stay within file size limits for services like GitHub Releases. When run with `--only-disk`, it creates a new partitioned `pmtiles` fileset from a directory of tiles. When run without, it can be used to re-partition an existing `pmtiles` fileset, for example after having retiled some of the sheets.
+Partitions a large set of tiles into smaller PMTiles files. This is useful for managing large datasets, for example, to stay within file size limits for services like GitHub Releases. It creates a mosaic file if the output is partitioned into multiple files.
 
 ```bash
-partition --to-pmtiles-prefix <prefix> --from-tiles-dir <dir> [--from-pmtiles-prefix <prefix>] [--name <name>] [--description <desc>] [--max-zoom <zoom>] [--min-zoom <zoom>] [--only-disk] (--attribution <text> | --attribution-file <file>)
+partition --from-source <source> [--from-source <source> ...] --to-pmtiles <file> [--size-limit <limit>]
 ```
 
--   `--to-pmtiles-prefix`: Prefix for the output PMTiles files. (Required)
--   `--from-tiles-dir`: Directory containing the input tiles. (Required)
--   `--from-pmtiles-prefix`: Prefix for input PMTiles files (required if not using `--only-disk`).
--   `--name`: Name of the mosaic. (Required if using `--only-disk`)
--   `--description`: Description of the mosaic. (Required if using `--only-disk`)
--   `--max-zoom`: Maximum zoom level for the mosaic. (Defaults to the max zoom found in the source)
--   `--min-zoom`: Minimum zoom level for the mosaic (default: 0).
--   `--only-disk`: Only read tiles from the local disk.
--   `--attribution`: Attribution text for the mosaic. (Required if using `--only-disk`)
--   `--attribution-file`: File containing attribution text. (Required if using `--only-disk`)
+-   `--from-source`: Path to a source file or directory. Can be repeated. Supported sources:
+    -   `.mbtiles` file: e.g., `path/to/file.mbtiles`
+    -   `.pmtiles` file (supports glob patterns): e.g., `path/to/*.pmtiles`
+    -   Directory containing tiles: e.g., `path/to/tiles_dir`
+-   `--to-pmtiles`: Output PMTiles file. The prefix for partitioned files will be derived from this argument.
+-   `--size-limit`: Maximum size of each partition. Can be a number in bytes (e.g., `100000000`), or with units (e.g., `100M`, `2G`), or a preset:
+    -   `github_release`: 2GB limit
+    -   `github_file`: 100MB limit
+    -   `cloudflare_object`: 512MB limit
+    The overhead (delta) is scaled proportionally based on the `github_release` preset (2GB with 5MB overhead). Defaults to `github_release`.
 
 #### `retile`
 
@@ -211,31 +225,37 @@ The tool works in two stages. First, it calculates the full set of sheets that n
 In the second stage, it retiles the affected area. It uses the pull list to create a temporary virtual raster (`.vrt`), generates new base tiles for the affected region, and then reconstructs the upper-level tiles by pulling existing, unaffected tiles from the PMTiles source.
 
 ```bash
-retile --retile-list-file <file> --bounds-file <file> [--max-zoom <zoom>] [--min-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-pmtiles-prefix <prefix>] [--sheets-to-pull-list-outfile <file>] [--num-parallel <processes>]
+retile --retile-list-file <file> --bounds-file <file> [--max-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-source <source>] [--sheets-to-pull-list-outfile <file>] [--num-parallel <processes>] [--tile-quality <quality>]
 ```
 
 -   `--retile-list-file`: File containing the list of sheets to retile. (Required)
 -   `--bounds-file`: GeoJSON file containing the list of available sheets and their geographic bounds. (Required)
--   `--max-zoom`: Maximum zoom level to create tiles for. (Defaults to the max zoom from the PMTiles source)
--   `--min-zoom`: Minimum zoom level to create tiles for. (Defaults to the min zoom from the PMTiles source)
+-   `--max-zoom`: Maximum zoom level to create tiles for. Defaults to the max zoom from the source if `--from-source` is provided, otherwise it is required.
 -   `--tiffs-dir`: Directory where the GeoTIFFs are located. (Required if not in pull-list generation mode)
 -   `--tiles-dir`: Directory where the tiles will be created. (Required if not in pull-list generation mode)
--   `--from-pmtiles-prefix`: Prefix for the PMTiles source from which to pull existing tiles. (Required if not in pull-list generation mode)
+-   `--from-source`: Location of the source from which to pull existing tiles. Can be a glob pattern to match a group of pmtiles. Required if not in pull-list generation mode.
 -   `--sheets-to-pull-list-outfile`: If provided, the script will only calculate the full list of sheets that need to be processed (including adjacent ones) and write it to this file, then exit.
 -   `--num-parallel`: Number of parallel processes to use for tiling (default: number of CPU cores).
+-   `--tile-quality`: Quality of compression for webp and jpg (default: 75).
 
 #### `tile`
 
 Creates web map tiles from a directory of GeoTIFF files. It combines them into a virtual raster (`.vrt`) for efficient processing.
 
 ```bash
-tile --tiles-dir <dir> --tiffs-dir <dir> --max-zoom <zoom> [--min-zoom <zoom>] [--num-parallel <processes>]
+tile --tiles-dir <dir> --tiffs-dir <dir> --max-zoom <zoom> --name <name> --description <desc> (--attribution <text> | --attribution-file <file>) [--min-zoom <zoom>] [--tile-extension <ext>] [--tile-quality <quality>] [--num-parallel <processes>]
 ```
 
--   `<tiles-dir>`: Directory to store the output tiles. (Required)
--   `<tiffs-dir>`: Directory containing the input GeoTIFF files. (Required)
--   `<max-zoom>`: Maximum zoom level for tiling. (Required)
--   `<min-zoom>`: Minimum zoom level for tiling (default: 0).
+-   `--tiles-dir`: Directory to store the output tiles. (Required)
+-   `--tiffs-dir`: Directory containing the input GeoTIFF files. (Required)
+-   `--max-zoom`: Maximum zoom level for tiling. (Required)
+-   `--name`: Name of the mosaic. (Required)
+-   `--description`: Description of the mosaic. (Required)
+-   `--attribution`: Attribution text for the mosaic. Required if `--attribution-file` is not used.
+-   `--attribution-file`: File containing attribution text for the mosaic. Required if `--attribution` is not used.
+-   `--min-zoom`: Minimum zoom level for tiling (default: 0).
+-   `--tile-extension`: Tile file extension (default: webp). Choices: `webp`, `jpg`, `jpeg`, `png`.
+-   `--tile-quality`: Compression quality of the tiles for `webp` and `jpg` (default: 75).
 -   `--num-parallel`: Number of parallel processes to use for tiling (default: number of CPU cores).
 
 ### Running Tools Directly with `uvx`

{topo_map_processor-0.1.7 → topo_map_processor-0.1.8}/pyproject.toml

@@ -10,6 +10,7 @@ authors = [
 dependencies = [
     "mercantile>=1.2.1",
     "pmtiles>=3.4.1",
+    "requests>=2.32.4",
     "shapely>=2.1.1",
 ]
 dynamic = [
@@ -37,6 +38,7 @@ repository = "https://github.com/ramSeraph/topo_map_processor.git"
 
 [project.scripts]
 collect-bounds = "topo_map_processor.tools.collect_bounds:cli"
+download-mosaic = "topo_map_processor.tools.download_mosaic:cli"
 partition = "topo_map_processor.tools.partition:cli"
 retile = "topo_map_processor.tools.retile:cli"
 tile = "topo_map_processor.tools.tile:cli"

{topo_map_processor-0.1.7 → topo_map_processor-0.1.8}/topo_map_processor/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.7'
-__version_tuple__ = version_tuple = (0, 1, 7)
+__version__ = version = '0.1.8'
+__version_tuple__ = version_tuple = (0, 1, 8)

{topo_map_processor-0.1.7 → topo_map_processor-0.1.8}/topo_map_processor/bash_scripts/generate_lists.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 
+prog_name=${COMMNAND_NAME:-$(basename "$0")}
 release_base=$1
 ext=$2
 
 if [ -z "$release_base" ] || [ -z "$ext" ]; then
-  echo "Usage: $0 <release_base> <extension>"
+  echo "Usage: $prog_name <release_base> <extension>"
   exit 1
 fi
 

{topo_map_processor-0.1.7 → topo_map_processor-0.1.8}/topo_map_processor/bash_scripts/upload_to_release.sh

@@ -11,9 +11,10 @@ set -o pipefail
 # TAG: The git tag of the release to upload to.
 # FOLDER: The local folder containing the files to upload.
 # OVERWRITE: Optional. Set to 'yes' to allow overwriting existing assets.
+prog_name=${COMMNAND_NAME:-$(basename "$0")}
 
 if [ "$#" -lt 3 ] || [ "$#" -gt 4 ]; then
-    echo "Usage: $0 <tag> <folder> <extension_without_the_leading_dot> [yes_to_overwrite]"
+    echo "Usage: $prog_name <tag> <folder> <extension_without_the_leading_dot> [yes_to_overwrite]"
     exit 1
 fi
 

topo_map_processor-0.1.8/topo_map_processor/tools/download_mosaic.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env -S uv run 
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "mercantile",
+#     "pmtiles",
+#     "requests",
+# ]
+# ///
+
+import json
+import sqlite3
+
+from pathlib import Path, PurePosixPath
+from urllib.parse import urlparse, urljoin, unquote
+
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+from pmtiles.reader import Reader, MmapSource, all_tiles
+
+# heavily copied from https://github.com/protomaps/PMTiles/blob/main/python/pmtiles/convert.py
+# and https://github.com/mapbox/mbutil/blob/master/mbutil/util.py
+
+session = None
+timeout = None
+
+def download_file(url, fname):
+    global session, timeout
+    print(f'downloading {url} to {fname}')
+    resp = session.get(url, stream=True, timeout=timeout)
+    with open(fname, 'wb') as f:
+        for data in resp.iter_content(chunk_size=4096):
+            f.write(data)
+
+
+def get_mosaic(mosaic_file, mosaic_url):
+    if not mosaic_file.exists():
+        download_file(mosaic_url, mosaic_file)
+
+    return json.loads(mosaic_file.read_text())
+
+
+def get_metadata(mosaic_data):
+    ch = {}
+
+    cm = list(mosaic_data.values())[0]['metadata']
+    for k, item in mosaic_data.items():
+        h = item['header']
+        if 'max_zoom' not in ch or ch['max_zoom'] < h['max_zoom']:
+            ch['max_zoom'] = h['max_zoom']
+        if 'min_zoom' not in ch or ch['min_zoom'] > h['min_zoom']:
+            ch['min_zoom'] = h['min_zoom']
+        if 'max_lat_e7' not in ch or ch['max_lat_e7'] < h['max_lat_e7']:
+            ch['max_lat_e7'] = h['max_lat_e7']
+        if 'min_lat_e7' not in ch or ch['min_lat_e7'] > h['min_lat_e7']:
+            ch['min_lat_e7'] = h['min_lat_e7']
+        if 'max_lon_e7' not in ch or ch['max_lon_e7'] < h['max_lon_e7']:
+            ch['max_lon_e7'] = h['max_lon_e7']
+        if 'min_lon_e7' not in ch or ch['min_lon_e7'] > h['min_lon_e7']:
+            ch['min_lon_e7'] = h['min_lon_e7']
+
+    cm['maxzoom'] = ch['max_zoom']
+    cm['minzoom'] = ch['min_zoom']
+
+    max_lat = ch['max_lat_e7'] / 10000000 
+    min_lat = ch['min_lat_e7'] / 10000000
+    max_lon = ch['max_lon_e7'] / 10000000
+    min_lon = ch['min_lon_e7'] / 10000000
+    cm['bounds'] = f"{min_lon},{min_lat},{max_lon},{max_lat}"
+
+    center_lat = (max_lat + min_lat) / 2
+    center_lon = (max_lon + min_lon) / 2 
+    center_zoom = (ch['max_zoom'] + ch['min_zoom']) // 2
+    cm['center'] = f"{center_lon},{center_lat},{center_zoom}"
+
+    vector_layers = cm.get('vector_layers', [])
+    for layer in vector_layers:
+        layer['minzoom'] = cm['minzoom']
+        layer['maxzoom'] = cm['maxzoom']
+
+    return cm
+
+
+def finalize_mbtiles(conn, cursor):
+    cursor.execute(
+        "CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);"
+    )
+    conn.commit()
+    cursor.execute("""ANALYZE;""")
+
+    conn.close()
+
+
+def add_to_mbtiles(pmtiles_fname, cursor, conn):
+    print(f'adding {pmtiles_fname} to mbtiles')
+    with open(pmtiles_fname, "r+b") as f:
+        source = MmapSource(f)
+        reader = Reader(source)
+
+        for zxy, tile_data in all_tiles(reader.get_bytes):
+            flipped_y = (1 << zxy[0]) - 1 - zxy[2]
+            cursor.execute(
+                "INSERT INTO tiles VALUES(?,?,?,?)",
+                (zxy[0], zxy[1], flipped_y, tile_data),
+            )
+        conn.commit()
+
+
+def optimize_cursor(cursor):
+    cursor.execute("""PRAGMA synchronous=0""")
+    cursor.execute("""PRAGMA locking_mode=EXCLUSIVE""")
+    cursor.execute("""PRAGMA journal_mode=DELETE""")
+
+
+def initialize_tables(cursor, metadata):
+    cursor.execute("CREATE TABLE metadata (name text, value text);")
+    cursor.execute(
+        "CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);"
+    )
+
+    json_metadata = {}
+    for k, v in metadata.items():
+        if k == "vector_layers":
+            json_metadata["vector_layers"] = v
+            continue
+        elif k == "tilestats":
+            json_metadata["tilestats"] = v
+            continue
+        elif not isinstance(v, str):
+            v = json.dumps(v, ensure_ascii=False)
+        cursor.execute("INSERT INTO metadata VALUES(?,?)", (k, v))
+
+    if len(json_metadata) > 0:
+        cursor.execute(
+            "INSERT INTO metadata VALUES(?,?)",
+            ("json", json.dumps(json_metadata, ensure_ascii=False)),
+        )
+
+
+
+def get_mbtiles_conn(output_file):
+    conn = sqlite3.connect(output_file)
+    cursor = conn.cursor()
+    optimize_cursor(cursor)
+    return conn, cursor
+
+
+def get_pmtiles_url(mosaic_url, k):
+    # undo prioir ididocy
+    if k.startswith('../') and mosaic_url.startswith('https://github.com/ramSeraph/'):
+        k = k[3:]
+    return urljoin(mosaic_url, k)
+
+
+def init_tracker(tracker_file):
+    if not tracker_file.exists():
+        tracker_file.write_text("")
+
+def stage_done(tracker_file, stage):
+    txt = tracker_file.read_text()
+
+    done_stages = txt.split('\n')
+
+    if stage in done_stages:
+        return True
+
+    return False
+
+
+def mark_done(tracker_file, stage):
+    with open(tracker_file, 'a') as f:
+        f.write(stage)
+        f.write('\n')
+
+def get_filename_from_url(url):
+    url_parsed = urlparse(url)
+    return unquote(PurePosixPath(url_parsed.path).name)
+
+def cli():
+    import argparse
+    parser = argparse.ArgumentParser(description='Download a mosaic and convert it to MBTiles format.')
+    parser.add_argument('--mosaic_url',  '-u', required=True, type=str, help='URL of the mosaic JSON file')
+    parser.add_argument('--output_file', '-o', type=str, help='Output MBTiles file name, if not specified is derived from the mosaic URL')
+    parser.add_argument('--request-timeout-secs', '-t', type=int, default=60, help='Timeout for HTTP requests in seconds')
+    parser.add_argument('--num-http-retries', '-r', type=int, default=3, help='Number of retries for HTTP requests')
+    args = parser.parse_args()
+
+    mosaic_url = args.mosaic_url
+
+    mosaic_fname = get_filename_from_url(mosaic_url)
+
+    mosaic_file = Path(mosaic_fname)
+    mosaic_data = get_mosaic(mosaic_file, mosaic_url)
+    metadata = get_metadata(mosaic_data)
+
+    if args.output_file:
+        out_mbtiles_file = Path(args.output_file)
+    else:
+        if not mosaic_fname.endswith('.mosaic.json'):
+            raise ValueError('Output file must end with .mosaic.json, if the output_file is not specified.')
+
+        out_mbtiles_fname = mosaic_fname[:-len('.mosaic.json')] + '.mbtiles'
+        out_mbtiles_file = Path(out_mbtiles_fname)
+
+    tracker_file = Path('tracker.txt')
+
+    if out_mbtiles_file.exists() and not tracker_file.exists():
+        print(f'Output file {out_mbtiles_file} already exists, and no tracker file found. Exiting to avoid overwriting.')
+        return
+
+    conn, cursor = get_mbtiles_conn(out_mbtiles_file)
+
+    init_tracker(tracker_file)
+
+    print(f'Output MBTiles file: {out_mbtiles_file}')
+
+    if not stage_done(tracker_file, 'table_init'):
+        print('Initializing tables...')
+        initialize_tables(cursor, metadata)
+        mark_done(tracker_file, 'table_init')
+
+    global session, timeout
+    session = requests.session()
+    retries = args.num_http_retries
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+    )
+    session.mount('http://', HTTPAdapter(max_retries=retry))
+    session.mount('https://', HTTPAdapter(max_retries=retry))
+    timeout = args.request_timeout_secs
+
+    for k in mosaic_data.keys():
+        pmtiles_url = get_pmtiles_url(mosaic_url, k)
+        if stage_done(tracker_file, k):
+            continue
+
+        pmtiles_fname = get_filename_from_url(pmtiles_url)
+        download_file(pmtiles_url, pmtiles_fname)
+        add_to_mbtiles(pmtiles_fname, cursor, conn)
+        mark_done(tracker_file, k)
+        Path(pmtiles_fname).unlink()
+
+    finalize_mbtiles(conn, cursor)
+    tracker_file.unlink()
+    mosaic_file.unlink()
+    print('Done!!!')
+    
+
+if __name__ == '__main__':
+    cli()

{topo_map_processor-0.1.7 → topo_map_processor-0.1.8}/topo_map_processor/tools/execute_bash.py

@@ -1,6 +1,7 @@
-import subprocess
+import os
 import sys
 import shutil
+import subprocess
 import importlib.resources as resources
 
 def execute_bash_script(script_name):
@@ -18,6 +19,8 @@ def execute_bash_script(script_name):
     except FileNotFoundError:
         raise FileNotFoundError(f"The script {script_name} does not exist or is not a file. Searched in package data.")
 
+    command_name = sys.argv[0]
+    os.environ["COMMAND_NAME"] = command_name
     # Execute the bash script and stream output, merging stderr into stdout
     process = subprocess.Popen(
         [shutil.which("bash"), str(script_path)] + sys.argv[1:],