topo_map_processor 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/PKG-INFO +35 -36
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/README.md +34 -35
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/pyproject.toml +0 -1
- topo_map_processor-0.1.6/samples/corner1.png +0 -0
- topo_map_processor-0.1.6/samples/corner2.png +0 -0
- topo_map_processor-0.1.6/samples/corner3.png +0 -0
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/topo_map_processor/__init__.py +10 -9
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/topo_map_processor/_version.py +2 -2
- topo_map_processor-0.1.6/topo_map_processor/tools/collect_bounds.py +51 -0
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/topo_map_processor/tools/partition.py +39 -6
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/topo_map_processor/tools/retile.py +35 -21
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/topo_map_processor/tools/tile_sources.py +144 -28
- topo_map_processor-0.1.4/topo_map_processor/tools/collect_bounds.py +0 -25
- topo_map_processor-0.1.4/topo_map_processor/tools/update_bounds.py +0 -54
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/.github/workflows/publish.yml +0 -0
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/.gitignore +0 -0
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/.python-version +0 -0
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/LICENSE +0 -0
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/topo_map_processor/tools/__init__.py +0 -0
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/topo_map_processor/tools/tile.py +0 -0
- {topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: topo_map_processor
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: Library for processing topographic maps
|
|
5
5
|
Project-URL: homepage, https://github.com/ramSeraph/topo_map_processor
|
|
6
6
|
Project-URL: repository, https://github.com/ramSeraph/topo_map_processor.git
|
|
@@ -51,7 +51,7 @@ A Python utility for processing topographic maps.
|
|
|
51
51
|
|
|
52
52
|
## Description
|
|
53
53
|
|
|
54
|
-
This project provides a collection of command-line tools and a Python library to process,
|
|
54
|
+
This project provides a collection of command-line tools and a Python library to process, georeference, and export topographic map data in raster form, specifically for creating web-mappable tiles from GeoTIFF files. The tools allow you to create tiles, partition large tile sets, and update existing tile sets efficiently.
|
|
55
55
|
|
|
56
56
|
## Installation
|
|
57
57
|
|
|
@@ -106,7 +106,7 @@ The main steps, executed by the `processor.process()` method, are as follows:
|
|
|
106
106
|
1. **Image Preparation (`rotate`):** To minimize computation, the processor first creates a smaller, shrunken version of the full-resolution map. On this small image, it locates the main contour of the map's frame (the border area). By calculating the angle of this contour's minimum bounding box, it determines the skew of the map. The full-resolution image is then rotated using this angle to make the frame perfectly horizontal, correcting for any tilt introduced during scanning.
|
|
107
107
|
2. **Corner Detection (`get_corners`):** Once the image is straightened, the processor uses the same map frame contour to identify the four corner regions. By narrowing the search to these smaller areas, it can apply more precise logic to find the exact pixel coordinates of the map's neatline corners. This is a critical step that often requires custom implementation in a subclass (`get_intersection_point`) to handle the specific visual features of the map's corners (e.g., crosses, specific colors, patterns).
|
|
108
108
|
3. **Grid Line Removal (`remove_grid_lines`):** If configured, the processor can detect and remove grid lines (graticules) from the map image. This is achieved by implementing the `locate_grid_lines` method. The removal helps create a cleaner final map by inpainting the areas where the lines were.
|
|
109
|
-
4. **Georeferencing (`georeference`):** Using the detected corner coordinates and the corresponding real-world geographic coordinates (provided via the `
|
|
109
|
+
4. **Georeferencing (`georeference`):** Using the detected corner coordinates and the corresponding real-world geographic coordinates (provided via the `index_box`), the tool creates Ground Control Points (GCPs). It then uses `gdal_translate` to create an initial georeferenced GeoTIFF.
|
|
110
110
|
5. **Warping and Cutting (`warp`):** The georeferenced image is then warped into the standard `EPSG:3857` projection used by most web maps. During this step, it's also precisely cropped to the map's boundary (neatline) using a cutline derived from the corner points. The output is a clean, map-only image, correctly projected.
|
|
111
111
|
6. **Exporting (`export`):** The final warped image is converted into a Cloud-Optimized GeoTIFF (COG), which is highly efficient for web-based tiling. A corresponding bounds file (`.geojsonl`) is also created, defining the geographic extent of the map sheet.
|
|
112
112
|
|
|
@@ -116,15 +116,15 @@ The output of this library workflow—a directory of COG `.tif` files and a dire
|
|
|
116
116
|
|
|
117
117
|
The `TopoMapProcessor` class provides a framework for processing individual map sheets. To use it, you need to create a subclass and implement the required methods.
|
|
118
118
|
|
|
119
|
-
Here's an example of how to use the `TopoMapProcessor` class
|
|
119
|
+
Here's an example of how to use the `TopoMapProcessor` class.
|
|
120
120
|
|
|
121
121
|
```python
|
|
122
122
|
from topo_map_processor import TopoMapProcessor, LineRemovalParams
|
|
123
123
|
|
|
124
124
|
class SampleProcessor(TopoMapProcessor):
|
|
125
125
|
|
|
126
|
-
def __init__(self, filepath, extra,
|
|
127
|
-
super().__init__(filepath, extra,
|
|
126
|
+
def __init__(self, filepath, extra, index_box, index_props):
|
|
127
|
+
super().__init__(filepath, extra, index_box, index_props)
|
|
128
128
|
# ... (additional initialization)
|
|
129
129
|
|
|
130
130
|
def get_inter_dir(self):
|
|
@@ -155,14 +155,16 @@ def process_files():
|
|
|
155
155
|
# ... (load image files and special cases)
|
|
156
156
|
for filepath in image_files:
|
|
157
157
|
extra = special_cases.get(filepath.name, {})
|
|
158
|
-
|
|
159
|
-
processor = SampleProcessor(filepath, extra,
|
|
158
|
+
index_box, index_props = get_index_data(filepath.name) # This should return the corresponding index box and properties for the file
|
|
159
|
+
processor = SampleProcessor(filepath, extra, index_box, index_props)
|
|
160
160
|
processor.process()
|
|
161
161
|
```
|
|
162
162
|
|
|
163
163
|
The library is configurable and most parameters can be overridden at a per sheet level using the `extra` dictionary.
|
|
164
164
|
|
|
165
|
-
The `
|
|
165
|
+
The `index_box` is used to provide the coordinates of the corners of the sheet. This information is used to create the bounds file and georeference the sheet. This is expected to be a list of coordinates in counter clockwise order starting at the top left corner of the map sheet.
|
|
166
|
+
|
|
167
|
+
The optional `index_properties` dictionary can contain additional metadata about the sheet, such as its name, description, and other properties which end up in the generated bounds files.
|
|
166
168
|
|
|
167
169
|
The end result is a Cloud optimized GeoTIFF file, a bounds file in GeoJSON format from which the tiles can be generated, using the `tile` command-line tool, and a set of tiles in the specified directory.
|
|
168
170
|
|
|
@@ -172,18 +174,22 @@ I would recommend looking at the [nepal_survey_maps](https://github.com/ramSerap
|
|
|
172
174
|
|
|
173
175
|
The `get_intersection_point` method is the most critical part to customize for a new map series. The library provides several helper methods that can be used to implement this logic, each suited for different types of corner features:
|
|
174
176
|
|
|
177
|
+

|
|
175
178
|
- **`get_nearest_intersection_point`**: This method is useful when the map corners are defined by the intersection of two perpendicular lines (like a simple crosshair). It works by:
|
|
176
179
|
1. Detecting all horizontal and vertical lines in the corner region.
|
|
177
180
|
2. Finding all intersection points between these lines.
|
|
178
181
|
3. Identifying the point closest to the expected corner position, while also matching an expected angle and distance from an anchor point.
|
|
179
182
|
|
|
183
|
+

|
|
180
184
|
- **`get_4way_intersection_point`**: This is a more robust version for crosshair-style corners. It specifically looks for intersections where four lines meet, making it less prone to errors from stray lines in the map collar.
|
|
181
185
|
|
|
186
|
+

|
|
182
187
|
- **`get_biggest_contour_corner`**: This strategy is designed for corners that are marked by a solid shape or a filled area (e.g., a colored square or circle). It works by:
|
|
183
188
|
1. Isolating the specific color of the corner feature.
|
|
184
189
|
2. Finding the largest contour (shape) of that color in the corner region.
|
|
185
190
|
3. Returning the outermost point of that contour as the corner.
|
|
186
191
|
|
|
192
|
+

|
|
187
193
|
- **`get_nearest_intersection_point_from_biggest_corner_contour`**: This is a hybrid approach. It first uses the "biggest contour" method to locate a general corner area (e.g., a colored box) and then searches for a precise line intersection *within* that area. This is effective for maps that have both a colored corner block and a fine crosshair inside it.
|
|
188
194
|
|
|
189
195
|
By combining these methods in your subclass's implementation of `get_intersection_point`, you can build a reliable corner detection system for your specific map type.
|
|
@@ -208,46 +214,51 @@ This project provides several command-line tools to work with topographic maps.
|
|
|
208
214
|
Collects individual GeoJSONL bound files into a single GeoJSON FeatureCollection.
|
|
209
215
|
|
|
210
216
|
```bash
|
|
211
|
-
collect-bounds --bounds-dir <directory> --output-file <file>
|
|
217
|
+
collect-bounds --bounds-dir <directory> --output-file <file> [--preexisting-file <file>]
|
|
212
218
|
```
|
|
213
219
|
|
|
214
220
|
- `--bounds-dir`: Directory containing GeoJSONL files. (Required)
|
|
215
221
|
- `--output-file`: Output GeoJSON file path. (Required)
|
|
222
|
+
- `--preexisting-file`: Pre existing GeoJSON file path. (Optional)
|
|
216
223
|
|
|
217
224
|
#### `partition`
|
|
218
225
|
|
|
219
|
-
Partitions a large set of tiles into smaller PMTiles files. This is useful for managing large datasets, for example, to stay within file size limits for services like GitHub Releases.
|
|
226
|
+
Partitions a large set of tiles into smaller PMTiles files. This is useful for managing large datasets, for example, to stay within file size limits for services like GitHub Releases. When run with `--only-disk`, it creates a new partitioned `pmtiles` fileset from a directory of tiles. When run without, it can be used to re-partition an existing `pmtiles` fileset, for example after having retiled some of the sheets.
|
|
220
227
|
|
|
221
228
|
```bash
|
|
222
|
-
partition --to-pmtiles-prefix <prefix> --from-tiles-dir <dir> --name <name> --description <desc> --max-zoom <zoom> [--min-zoom <zoom>] [--
|
|
229
|
+
partition --to-pmtiles-prefix <prefix> --from-tiles-dir <dir> [--from-pmtiles-prefix <prefix>] [--name <name>] [--description <desc>] [--max-zoom <zoom>] [--min-zoom <zoom>] [--only-disk] (--attribution <text> | --attribution-file <file>)
|
|
223
230
|
```
|
|
224
231
|
|
|
225
232
|
- `--to-pmtiles-prefix`: Prefix for the output PMTiles files. (Required)
|
|
226
233
|
- `--from-tiles-dir`: Directory containing the input tiles. (Required)
|
|
227
|
-
- `--name`: Name of the mosaic. (Required)
|
|
228
|
-
- `--description`: Description of the mosaic. (Required)
|
|
229
|
-
- `--max-zoom`: Maximum zoom level for the mosaic. (Required)
|
|
230
|
-
- `--min-zoom`: Minimum zoom level for the mosaic (default: 0).
|
|
231
234
|
- `--from-pmtiles-prefix`: Prefix for input PMTiles files (required if not using `--only-disk`).
|
|
235
|
+
- `--name`: Name of the mosaic. (Required if using `--only-disk`)
|
|
236
|
+
- `--description`: Description of the mosaic. (Required if using `--only-disk`)
|
|
237
|
+
- `--max-zoom`: Maximum zoom level for the mosaic. (Defaults to the max zoom found in the source)
|
|
238
|
+
- `--min-zoom`: Minimum zoom level for the mosaic (default: 0).
|
|
232
239
|
- `--only-disk`: Only read tiles from the local disk.
|
|
233
|
-
- `--attribution`: Attribution text for the mosaic. (Required
|
|
234
|
-
- `--attribution-file`: File containing attribution text. (Required
|
|
240
|
+
- `--attribution`: Attribution text for the mosaic. (Required if using `--only-disk`)
|
|
241
|
+
- `--attribution-file`: File containing attribution text. (Required if using `--only-disk`)
|
|
235
242
|
|
|
236
243
|
#### `retile`
|
|
237
244
|
|
|
238
245
|
Retiles a specific list of sheets (GeoTIFFs) and updates the corresponding tiles in an existing tile set. It can determine which other sheets are affected and need to be pulled in to correctly retile the area.
|
|
239
246
|
|
|
247
|
+
The tool works in two stages. First, it calculates the full set of sheets that need to be processed (the "pull list"), which includes the sheets from the input list plus any adjacent sheets that share tiles. You can run the tool with `--sheets-to-pull-list-outfile` to generate this list and then exit. This is useful for fetching the required GeoTIFFs before proceeding.
|
|
248
|
+
|
|
249
|
+
In the second stage, it retiles the affected area. It uses the pull list to create a temporary virtual raster (`.vrt`), generates new base tiles for the affected region, and then reconstructs the upper-level tiles by pulling existing, unaffected tiles from the PMTiles source.
|
|
250
|
+
|
|
240
251
|
```bash
|
|
241
|
-
retile --retile-list-file <file> --bounds-file <file> --max-zoom <zoom> [--min-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-pmtiles-prefix <prefix>] [--sheets-to-pull-list-outfile <file>]
|
|
252
|
+
retile --retile-list-file <file> --bounds-file <file> [--max-zoom <zoom>] [--min-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-pmtiles-prefix <prefix>] [--sheets-to-pull-list-outfile <file>]
|
|
242
253
|
```
|
|
243
254
|
|
|
244
255
|
- `--retile-list-file`: File containing the list of sheets to retile. (Required)
|
|
245
256
|
- `--bounds-file`: GeoJSON file containing the list of available sheets and their geographic bounds. (Required)
|
|
246
|
-
- `--max-zoom`: Maximum zoom level to create tiles for. (
|
|
247
|
-
- `--min-zoom`: Minimum zoom level to create tiles for (
|
|
248
|
-
- `--tiffs-dir`: Directory where the GeoTIFFs are located.
|
|
249
|
-
- `--tiles-dir`: Directory where the tiles will be created.
|
|
250
|
-
- `--from-pmtiles-prefix`: Prefix for the PMTiles source from which to pull existing tiles.
|
|
257
|
+
- `--max-zoom`: Maximum zoom level to create tiles for. (Defaults to the max zoom from the PMTiles source)
|
|
258
|
+
- `--min-zoom`: Minimum zoom level to create tiles for. (Defaults to the min zoom from the PMTiles source)
|
|
259
|
+
- `--tiffs-dir`: Directory where the GeoTIFFs are located. (Required if not in pull-list generation mode)
|
|
260
|
+
- `--tiles-dir`: Directory where the tiles will be created. (Required if not in pull-list generation mode)
|
|
261
|
+
- `--from-pmtiles-prefix`: Prefix for the PMTiles source from which to pull existing tiles. (Required if not in pull-list generation mode)
|
|
251
262
|
- `--sheets-to-pull-list-outfile`: If provided, the script will only calculate the full list of sheets that need to be processed (including adjacent ones) and write it to this file, then exit.
|
|
252
263
|
|
|
253
264
|
#### `tile`
|
|
@@ -263,18 +274,6 @@ tile <tiles-dir> <tiffs-dir> <max-zoom> [<min-zoom>]
|
|
|
263
274
|
- `<max-zoom>`: Maximum zoom level for tiling. (Required)
|
|
264
275
|
- `<min-zoom>`: Minimum zoom level for tiling (default: 0).
|
|
265
276
|
|
|
266
|
-
#### `update-bounds`
|
|
267
|
-
|
|
268
|
-
Updates a main GeoJSON bounds file with new or modified bounds from individual `.geojsonl` files.
|
|
269
|
-
|
|
270
|
-
```bash
|
|
271
|
-
update-bounds --bounds-file <file> --bounds-dir <dir> --retile-list-file <file>
|
|
272
|
-
```
|
|
273
|
-
|
|
274
|
-
- `--bounds-file`: The main GeoJSON bounds file to update. (Required)
|
|
275
|
-
- `--bounds-dir`: Directory containing the new or updated individual `.geojsonl` files. (Required)
|
|
276
|
-
- `--retile-list-file`: A file containing the list of sheet names that have been updated (corresponding to the files in `--bounds-dir`). (Required)
|
|
277
|
-
|
|
278
277
|
### Running Tools Directly with `uvx`
|
|
279
278
|
|
|
280
279
|
You can run the command-line tools directly from PyPI without a persistent installation using `uvx`. This is convenient for one-off tasks.
|
|
@@ -5,7 +5,7 @@ A Python utility for processing topographic maps.
|
|
|
5
5
|
|
|
6
6
|
## Description
|
|
7
7
|
|
|
8
|
-
This project provides a collection of command-line tools and a Python library to process,
|
|
8
|
+
This project provides a collection of command-line tools and a Python library to process, georeference, and export topographic map data in raster form, specifically for creating web-mappable tiles from GeoTIFF files. The tools allow you to create tiles, partition large tile sets, and update existing tile sets efficiently.
|
|
9
9
|
|
|
10
10
|
## Installation
|
|
11
11
|
|
|
@@ -60,7 +60,7 @@ The main steps, executed by the `processor.process()` method, are as follows:
|
|
|
60
60
|
1. **Image Preparation (`rotate`):** To minimize computation, the processor first creates a smaller, shrunken version of the full-resolution map. On this small image, it locates the main contour of the map's frame (the border area). By calculating the angle of this contour's minimum bounding box, it determines the skew of the map. The full-resolution image is then rotated using this angle to make the frame perfectly horizontal, correcting for any tilt introduced during scanning.
|
|
61
61
|
2. **Corner Detection (`get_corners`):** Once the image is straightened, the processor uses the same map frame contour to identify the four corner regions. By narrowing the search to these smaller areas, it can apply more precise logic to find the exact pixel coordinates of the map's neatline corners. This is a critical step that often requires custom implementation in a subclass (`get_intersection_point`) to handle the specific visual features of the map's corners (e.g., crosses, specific colors, patterns).
|
|
62
62
|
3. **Grid Line Removal (`remove_grid_lines`):** If configured, the processor can detect and remove grid lines (graticules) from the map image. This is achieved by implementing the `locate_grid_lines` method. The removal helps create a cleaner final map by inpainting the areas where the lines were.
|
|
63
|
-
4. **Georeferencing (`georeference`):** Using the detected corner coordinates and the corresponding real-world geographic coordinates (provided via the `
|
|
63
|
+
4. **Georeferencing (`georeference`):** Using the detected corner coordinates and the corresponding real-world geographic coordinates (provided via the `index_box`), the tool creates Ground Control Points (GCPs). It then uses `gdal_translate` to create an initial georeferenced GeoTIFF.
|
|
64
64
|
5. **Warping and Cutting (`warp`):** The georeferenced image is then warped into the standard `EPSG:3857` projection used by most web maps. During this step, it's also precisely cropped to the map's boundary (neatline) using a cutline derived from the corner points. The output is a clean, map-only image, correctly projected.
|
|
65
65
|
6. **Exporting (`export`):** The final warped image is converted into a Cloud-Optimized GeoTIFF (COG), which is highly efficient for web-based tiling. A corresponding bounds file (`.geojsonl`) is also created, defining the geographic extent of the map sheet.
|
|
66
66
|
|
|
@@ -70,15 +70,15 @@ The output of this library workflow—a directory of COG `.tif` files and a dire
|
|
|
70
70
|
|
|
71
71
|
The `TopoMapProcessor` class provides a framework for processing individual map sheets. To use it, you need to create a subclass and implement the required methods.
|
|
72
72
|
|
|
73
|
-
Here's an example of how to use the `TopoMapProcessor` class
|
|
73
|
+
Here's an example of how to use the `TopoMapProcessor` class.
|
|
74
74
|
|
|
75
75
|
```python
|
|
76
76
|
from topo_map_processor import TopoMapProcessor, LineRemovalParams
|
|
77
77
|
|
|
78
78
|
class SampleProcessor(TopoMapProcessor):
|
|
79
79
|
|
|
80
|
-
def __init__(self, filepath, extra,
|
|
81
|
-
super().__init__(filepath, extra,
|
|
80
|
+
def __init__(self, filepath, extra, index_box, index_props):
|
|
81
|
+
super().__init__(filepath, extra, index_box, index_props)
|
|
82
82
|
# ... (additional initialization)
|
|
83
83
|
|
|
84
84
|
def get_inter_dir(self):
|
|
@@ -109,14 +109,16 @@ def process_files():
|
|
|
109
109
|
# ... (load image files and special cases)
|
|
110
110
|
for filepath in image_files:
|
|
111
111
|
extra = special_cases.get(filepath.name, {})
|
|
112
|
-
|
|
113
|
-
processor = SampleProcessor(filepath, extra,
|
|
112
|
+
index_box, index_props = get_index_data(filepath.name) # This should return the corresponding index box and properties for the file
|
|
113
|
+
processor = SampleProcessor(filepath, extra, index_box, index_props)
|
|
114
114
|
processor.process()
|
|
115
115
|
```
|
|
116
116
|
|
|
117
117
|
The library is configurable and most parameters can be overridden at a per sheet level using the `extra` dictionary.
|
|
118
118
|
|
|
119
|
-
The `
|
|
119
|
+
The `index_box` is used to provide the coordinates of the corners of the sheet. This information is used to create the bounds file and georeference the sheet. This is expected to be a list of coordinates in counter clockwise order starting at the top left corner of the map sheet.
|
|
120
|
+
|
|
121
|
+
The optional `index_properties` dictionary can contain additional metadata about the sheet, such as its name, description, and other properties which end up in the generated bounds files.
|
|
120
122
|
|
|
121
123
|
The end result is a Cloud optimized GeoTIFF file, a bounds file in GeoJSON format from which the tiles can be generated, using the `tile` command-line tool, and a set of tiles in the specified directory.
|
|
122
124
|
|
|
@@ -126,18 +128,22 @@ I would recommend looking at the [nepal_survey_maps](https://github.com/ramSerap
|
|
|
126
128
|
|
|
127
129
|
The `get_intersection_point` method is the most critical part to customize for a new map series. The library provides several helper methods that can be used to implement this logic, each suited for different types of corner features:
|
|
128
130
|
|
|
131
|
+

|
|
129
132
|
- **`get_nearest_intersection_point`**: This method is useful when the map corners are defined by the intersection of two perpendicular lines (like a simple crosshair). It works by:
|
|
130
133
|
1. Detecting all horizontal and vertical lines in the corner region.
|
|
131
134
|
2. Finding all intersection points between these lines.
|
|
132
135
|
3. Identifying the point closest to the expected corner position, while also matching an expected angle and distance from an anchor point.
|
|
133
136
|
|
|
137
|
+

|
|
134
138
|
- **`get_4way_intersection_point`**: This is a more robust version for crosshair-style corners. It specifically looks for intersections where four lines meet, making it less prone to errors from stray lines in the map collar.
|
|
135
139
|
|
|
140
|
+

|
|
136
141
|
- **`get_biggest_contour_corner`**: This strategy is designed for corners that are marked by a solid shape or a filled area (e.g., a colored square or circle). It works by:
|
|
137
142
|
1. Isolating the specific color of the corner feature.
|
|
138
143
|
2. Finding the largest contour (shape) of that color in the corner region.
|
|
139
144
|
3. Returning the outermost point of that contour as the corner.
|
|
140
145
|
|
|
146
|
+

|
|
141
147
|
- **`get_nearest_intersection_point_from_biggest_corner_contour`**: This is a hybrid approach. It first uses the "biggest contour" method to locate a general corner area (e.g., a colored box) and then searches for a precise line intersection *within* that area. This is effective for maps that have both a colored corner block and a fine crosshair inside it.
|
|
142
148
|
|
|
143
149
|
By combining these methods in your subclass's implementation of `get_intersection_point`, you can build a reliable corner detection system for your specific map type.
|
|
@@ -162,46 +168,51 @@ This project provides several command-line tools to work with topographic maps.
|
|
|
162
168
|
Collects individual GeoJSONL bound files into a single GeoJSON FeatureCollection.
|
|
163
169
|
|
|
164
170
|
```bash
|
|
165
|
-
collect-bounds --bounds-dir <directory> --output-file <file>
|
|
171
|
+
collect-bounds --bounds-dir <directory> --output-file <file> [--preexisting-file <file>]
|
|
166
172
|
```
|
|
167
173
|
|
|
168
174
|
- `--bounds-dir`: Directory containing GeoJSONL files. (Required)
|
|
169
175
|
- `--output-file`: Output GeoJSON file path. (Required)
|
|
176
|
+
- `--preexisting-file`: Pre existing GeoJSON file path. (Optional)
|
|
170
177
|
|
|
171
178
|
#### `partition`
|
|
172
179
|
|
|
173
|
-
Partitions a large set of tiles into smaller PMTiles files. This is useful for managing large datasets, for example, to stay within file size limits for services like GitHub Releases.
|
|
180
|
+
Partitions a large set of tiles into smaller PMTiles files. This is useful for managing large datasets, for example, to stay within file size limits for services like GitHub Releases. When run with `--only-disk`, it creates a new partitioned `pmtiles` fileset from a directory of tiles. When run without, it can be used to re-partition an existing `pmtiles` fileset, for example after having retiled some of the sheets.
|
|
174
181
|
|
|
175
182
|
```bash
|
|
176
|
-
partition --to-pmtiles-prefix <prefix> --from-tiles-dir <dir> --name <name> --description <desc> --max-zoom <zoom> [--min-zoom <zoom>] [--
|
|
183
|
+
partition --to-pmtiles-prefix <prefix> --from-tiles-dir <dir> [--from-pmtiles-prefix <prefix>] [--name <name>] [--description <desc>] [--max-zoom <zoom>] [--min-zoom <zoom>] [--only-disk] (--attribution <text> | --attribution-file <file>)
|
|
177
184
|
```
|
|
178
185
|
|
|
179
186
|
- `--to-pmtiles-prefix`: Prefix for the output PMTiles files. (Required)
|
|
180
187
|
- `--from-tiles-dir`: Directory containing the input tiles. (Required)
|
|
181
|
-
- `--name`: Name of the mosaic. (Required)
|
|
182
|
-
- `--description`: Description of the mosaic. (Required)
|
|
183
|
-
- `--max-zoom`: Maximum zoom level for the mosaic. (Required)
|
|
184
|
-
- `--min-zoom`: Minimum zoom level for the mosaic (default: 0).
|
|
185
188
|
- `--from-pmtiles-prefix`: Prefix for input PMTiles files (required if not using `--only-disk`).
|
|
189
|
+
- `--name`: Name of the mosaic. (Required if using `--only-disk`)
|
|
190
|
+
- `--description`: Description of the mosaic. (Required if using `--only-disk`)
|
|
191
|
+
- `--max-zoom`: Maximum zoom level for the mosaic. (Defaults to the max zoom found in the source)
|
|
192
|
+
- `--min-zoom`: Minimum zoom level for the mosaic (default: 0).
|
|
186
193
|
- `--only-disk`: Only read tiles from the local disk.
|
|
187
|
-
- `--attribution`: Attribution text for the mosaic. (Required
|
|
188
|
-
- `--attribution-file`: File containing attribution text. (Required
|
|
194
|
+
- `--attribution`: Attribution text for the mosaic. (Required if using `--only-disk`)
|
|
195
|
+
- `--attribution-file`: File containing attribution text. (Required if using `--only-disk`)
|
|
189
196
|
|
|
190
197
|
#### `retile`
|
|
191
198
|
|
|
192
199
|
Retiles a specific list of sheets (GeoTIFFs) and updates the corresponding tiles in an existing tile set. It can determine which other sheets are affected and need to be pulled in to correctly retile the area.
|
|
193
200
|
|
|
201
|
+
The tool works in two stages. First, it calculates the full set of sheets that need to be processed (the "pull list"), which includes the sheets from the input list plus any adjacent sheets that share tiles. You can run the tool with `--sheets-to-pull-list-outfile` to generate this list and then exit. This is useful for fetching the required GeoTIFFs before proceeding.
|
|
202
|
+
|
|
203
|
+
In the second stage, it retiles the affected area. It uses the pull list to create a temporary virtual raster (`.vrt`), generates new base tiles for the affected region, and then reconstructs the upper-level tiles by pulling existing, unaffected tiles from the PMTiles source.
|
|
204
|
+
|
|
194
205
|
```bash
|
|
195
|
-
retile --retile-list-file <file> --bounds-file <file> --max-zoom <zoom> [--min-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-pmtiles-prefix <prefix>] [--sheets-to-pull-list-outfile <file>]
|
|
206
|
+
retile --retile-list-file <file> --bounds-file <file> [--max-zoom <zoom>] [--min-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-pmtiles-prefix <prefix>] [--sheets-to-pull-list-outfile <file>]
|
|
196
207
|
```
|
|
197
208
|
|
|
198
209
|
- `--retile-list-file`: File containing the list of sheets to retile. (Required)
|
|
199
210
|
- `--bounds-file`: GeoJSON file containing the list of available sheets and their geographic bounds. (Required)
|
|
200
|
-
- `--max-zoom`: Maximum zoom level to create tiles for. (
|
|
201
|
-
- `--min-zoom`: Minimum zoom level to create tiles for (
|
|
202
|
-
- `--tiffs-dir`: Directory where the GeoTIFFs are located.
|
|
203
|
-
- `--tiles-dir`: Directory where the tiles will be created.
|
|
204
|
-
- `--from-pmtiles-prefix`: Prefix for the PMTiles source from which to pull existing tiles.
|
|
211
|
+
- `--max-zoom`: Maximum zoom level to create tiles for. (Defaults to the max zoom from the PMTiles source)
|
|
212
|
+
- `--min-zoom`: Minimum zoom level to create tiles for. (Defaults to the min zoom from the PMTiles source)
|
|
213
|
+
- `--tiffs-dir`: Directory where the GeoTIFFs are located. (Required if not in pull-list generation mode)
|
|
214
|
+
- `--tiles-dir`: Directory where the tiles will be created. (Required if not in pull-list generation mode)
|
|
215
|
+
- `--from-pmtiles-prefix`: Prefix for the PMTiles source from which to pull existing tiles. (Required if not in pull-list generation mode)
|
|
205
216
|
- `--sheets-to-pull-list-outfile`: If provided, the script will only calculate the full list of sheets that need to be processed (including adjacent ones) and write it to this file, then exit.
|
|
206
217
|
|
|
207
218
|
#### `tile`
|
|
@@ -217,18 +228,6 @@ tile <tiles-dir> <tiffs-dir> <max-zoom> [<min-zoom>]
|
|
|
217
228
|
- `<max-zoom>`: Maximum zoom level for tiling. (Required)
|
|
218
229
|
- `<min-zoom>`: Minimum zoom level for tiling (default: 0).
|
|
219
230
|
|
|
220
|
-
#### `update-bounds`
|
|
221
|
-
|
|
222
|
-
Updates a main GeoJSON bounds file with new or modified bounds from individual `.geojsonl` files.
|
|
223
|
-
|
|
224
|
-
```bash
|
|
225
|
-
update-bounds --bounds-file <file> --bounds-dir <dir> --retile-list-file <file>
|
|
226
|
-
```
|
|
227
|
-
|
|
228
|
-
- `--bounds-file`: The main GeoJSON bounds file to update. (Required)
|
|
229
|
-
- `--bounds-dir`: Directory containing the new or updated individual `.geojsonl` files. (Required)
|
|
230
|
-
- `--retile-list-file`: A file containing the list of sheet names that have been updated (corresponding to the files in `--bounds-dir`). (Required)
|
|
231
|
-
|
|
232
231
|
### Running Tools Directly with `uvx`
|
|
233
232
|
|
|
234
233
|
You can run the command-line tools directly from PyPI without a persistent installation using `uvx`. This is convenient for one-off tasks.
|
|
@@ -31,7 +31,6 @@ repository = "https://github.com/ramSeraph/topo_map_processor.git"
|
|
|
31
31
|
|
|
32
32
|
[project.scripts]
|
|
33
33
|
collect-bounds = "topo_map_processor.tools.collect_bounds:cli"
|
|
34
|
-
update-bounds = "topo_map_processor.tools.update_bounds:cli"
|
|
35
34
|
partition = "topo_map_processor.tools.partition:cli"
|
|
36
35
|
retile = "topo_map_processor.tools.retile:cli"
|
|
37
36
|
tile = "topo_map_processor.tools.tile:cli"
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -70,9 +70,10 @@ class GCPBasedTransformer:
|
|
|
70
70
|
|
|
71
71
|
|
|
72
72
|
class TopoMapProcessor:
|
|
73
|
-
def __init__(self, filepath, extra,
|
|
73
|
+
def __init__(self, filepath, extra, index_box, index_properties={}):
|
|
74
74
|
self.filepath = filepath
|
|
75
|
-
self.
|
|
75
|
+
self.index_box = index_box
|
|
76
|
+
self.index_properties = index_properties
|
|
76
77
|
|
|
77
78
|
self.SHOW_IMG = os.getenv('SHOW_IMG', '0') == '1'
|
|
78
79
|
self.INSPECT = os.getenv('INSPECT', '0') == '1'
|
|
@@ -141,11 +142,8 @@ class TopoMapProcessor:
|
|
|
141
142
|
ext = self.filepath.suffix
|
|
142
143
|
return self.filepath.name.replace(ext, '')
|
|
143
144
|
|
|
144
|
-
def get_sheet_ibox(self
|
|
145
|
-
|
|
146
|
-
sheet_id = self.get_id()
|
|
147
|
-
|
|
148
|
-
return self.index_map[sheet_id]
|
|
145
|
+
def get_sheet_ibox(self):
|
|
146
|
+
return self.index_box
|
|
149
147
|
|
|
150
148
|
def get_data_dir(self):
|
|
151
149
|
return Path('data')
|
|
@@ -1607,12 +1605,15 @@ class TopoMapProcessor:
|
|
|
1607
1605
|
def get_cutline_props(self):
|
|
1608
1606
|
crs_proj = self.get_crs_proj()
|
|
1609
1607
|
|
|
1610
|
-
props = {
|
|
1608
|
+
props = {}
|
|
1609
|
+
props.update(self.index_properties)
|
|
1610
|
+
|
|
1611
|
+
props.update({
|
|
1611
1612
|
'id': self.get_id(),
|
|
1612
1613
|
'crs': crs_proj,
|
|
1613
1614
|
'gcps': self.get_gcps(pre_rotated=True),
|
|
1614
1615
|
'pixel_cutline': self.get_full_pixel_cutline(pre_rotated=True),
|
|
1615
|
-
}
|
|
1616
|
+
})
|
|
1616
1617
|
return props
|
|
1617
1618
|
|
|
1618
1619
|
def create_cutline(self, ibox, file):
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
|
|
2
|
+
import json
|
|
3
|
+
import argparse
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
def cli():
|
|
7
|
+
parser = argparse.ArgumentParser(description='Collect GeoJSONL files into a single GeoJSON file.')
|
|
8
|
+
parser.add_argument('--bounds-dir', required=True, help='Directory containing GeoJSONL files')
|
|
9
|
+
parser.add_argument('--output-file', required=True, help='Output GeoJSON file path')
|
|
10
|
+
parser.add_argument('--preexisting-file', help='preexisting GeoJSON file to base things on', default=None)
|
|
11
|
+
|
|
12
|
+
args = parser.parse_args()
|
|
13
|
+
|
|
14
|
+
feat_map = {}
|
|
15
|
+
if args.preexisting_file:
|
|
16
|
+
preexisting_file = Path(args.preexisting_file)
|
|
17
|
+
if not preexisting_file.exists():
|
|
18
|
+
raise FileNotFoundError(f"Preexisting file does not exist: {preexisting_file}")
|
|
19
|
+
|
|
20
|
+
existing_data = json.loads(preexisting_file.read_text())
|
|
21
|
+
for feature in existing_data.get('features', []):
|
|
22
|
+
if 'properties' in feature and 'id' in feature['properties']:
|
|
23
|
+
sheet_name = feature['properties']['id']
|
|
24
|
+
feat_map[sheet_name] = feature
|
|
25
|
+
|
|
26
|
+
bounds_dir = Path(args.bounds_dir)
|
|
27
|
+
|
|
28
|
+
paths = list(bounds_dir.glob('*.geojsonl'))
|
|
29
|
+
for p in paths:
|
|
30
|
+
feat = json.loads(p.read_text())
|
|
31
|
+
if 'properties' in feat and 'id' in feat['properties']:
|
|
32
|
+
sheet_name = feat['properties']['id']
|
|
33
|
+
feat_map[sheet_name] = feat
|
|
34
|
+
|
|
35
|
+
with open(args.output_file, 'w') as f:
|
|
36
|
+
f.write('{ "type": "FeatureCollection", "features": [\n')
|
|
37
|
+
|
|
38
|
+
feats = list(feat_map.values())
|
|
39
|
+
for i,feat in enumerate(feats):
|
|
40
|
+
line = json.dumps(feat)
|
|
41
|
+
|
|
42
|
+
if i != len(feats) - 1:
|
|
43
|
+
line += ','
|
|
44
|
+
line += '\n'
|
|
45
|
+
f.write(line)
|
|
46
|
+
|
|
47
|
+
f.write(']}\n')
|
|
48
|
+
|
|
49
|
+
if __name__ == "__main__":
|
|
50
|
+
cli()
|
|
51
|
+
|
|
@@ -319,11 +319,11 @@ def cli():
|
|
|
319
319
|
parser.add_argument('--to-pmtiles-prefix', required=True, help='Prefix for the output PMTiles files.')
|
|
320
320
|
parser.add_argument('--from-pmtiles-prefix', help='Prefix for the input PMTiles files. Required if not using --only-disk.')
|
|
321
321
|
parser.add_argument('--from-tiles-dir', required=True, help='Directory containing the input tiles')
|
|
322
|
-
parser.add_argument('--name', required=
|
|
323
|
-
parser.add_argument('--description', required=
|
|
322
|
+
parser.add_argument('--name', required=False, help='Name of the mosaic.')
|
|
323
|
+
parser.add_argument('--description', required=False, help='Description of the mosaic.')
|
|
324
324
|
parser.add_argument('--min-zoom', default=0, type=int, help='Minimum zoom level for the mosaic.')
|
|
325
|
-
parser.add_argument('--max-zoom', required=
|
|
326
|
-
group = parser.add_mutually_exclusive_group(required=
|
|
325
|
+
parser.add_argument('--max-zoom', required=False, type=int, help='Maximum zoom level for the mosaic.')
|
|
326
|
+
group = parser.add_mutually_exclusive_group(required=False)
|
|
327
327
|
group.add_argument('--attribution', help='Attribution text for the mosaic.')
|
|
328
328
|
group.add_argument('--attribution-file', help='File containing attribution text for the mosaic.')
|
|
329
329
|
args = parser.parse_args()
|
|
@@ -337,19 +337,52 @@ def cli():
|
|
|
337
337
|
if not args.only_disk and not args.from_pmtiles_prefix:
|
|
338
338
|
parser.error("--from-pmtiles-prefix is required unless --only-disk is set.")
|
|
339
339
|
|
|
340
|
+
if args.only_disk:
|
|
341
|
+
missing_args = []
|
|
342
|
+
if args.name is None:
|
|
343
|
+
missing_args.append('--name')
|
|
344
|
+
if args.description is None:
|
|
345
|
+
missing_args.append('--description')
|
|
346
|
+
if attribution_text is None:
|
|
347
|
+
missing_args.append('--attribution or --attribution-file')
|
|
348
|
+
if missing_args:
|
|
349
|
+
parser.error(f'Missing required arguments for --only-disk: {", ".join(missing_args)}')
|
|
340
350
|
|
|
341
351
|
if args.only_disk:
|
|
342
352
|
reader = DiskSource(args.from_tiles_dir)
|
|
343
353
|
else:
|
|
344
354
|
reader = DiskAndPartitionedPMTilesSource(args.from_tiles_dir, args.from_pmtiles_prefix)
|
|
345
355
|
|
|
356
|
+
if args.max_zoom is None:
|
|
357
|
+
max_zoom = reader.max_zoom
|
|
358
|
+
else:
|
|
359
|
+
max_zoom = args.max_zoom
|
|
360
|
+
|
|
361
|
+
if args.min_zoom is None:
|
|
362
|
+
min_zoom = reader.min_zoom
|
|
363
|
+
else:
|
|
364
|
+
min_zoom = args.min_zoom
|
|
365
|
+
|
|
366
|
+
if args.only_disk:
|
|
367
|
+
name = args.name
|
|
368
|
+
else:
|
|
369
|
+
name = args.name or reader.name
|
|
370
|
+
|
|
371
|
+
if args.only_disk:
|
|
372
|
+
description = args.description
|
|
373
|
+
else:
|
|
374
|
+
description = args.description or reader.description
|
|
375
|
+
|
|
376
|
+
if not args.only_disk:
|
|
377
|
+
attribution_text = attribution_text or reader.attribution
|
|
378
|
+
|
|
346
379
|
print('getting partition info')
|
|
347
380
|
to_partition_file = Path(f'{args.to_pmtiles_prefix}.partition_info.json')
|
|
348
|
-
partition_info = get_partition_info(reader,
|
|
381
|
+
partition_info = get_partition_info(reader, min_zoom, max_zoom, to_partition_file)
|
|
349
382
|
|
|
350
383
|
print('creating pmtiles')
|
|
351
384
|
mosaic_data = create_pmtiles(partition_info, reader, args.to_pmtiles_prefix,
|
|
352
|
-
|
|
385
|
+
name, description, attribution_text)
|
|
353
386
|
pprint(mosaic_data)
|
|
354
387
|
if len(mosaic_data) > 1:
|
|
355
388
|
Path(f'{args.to_pmtiles_prefix}.mosaic.json').write_text(json.dumps(mosaic_data))
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
# dependencies = [
|
|
4
4
|
# "mercantile",
|
|
5
5
|
# "pmtiles",
|
|
6
|
+
# "shapely",
|
|
6
7
|
# ]
|
|
7
8
|
# ///
|
|
8
9
|
|
|
@@ -18,18 +19,15 @@ from pathlib import Path
|
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
import mercantile
|
|
21
|
-
from shapely.geometry import
|
|
22
|
+
from shapely.geometry import shape
|
|
22
23
|
|
|
23
24
|
from osgeo_utils.gdal2tiles import main as gdal2tiles_main
|
|
24
25
|
from osgeo_utils.gdal2tiles import create_overview_tile, TileJobInfo, GDAL2Tiles
|
|
25
26
|
|
|
26
|
-
|
|
27
27
|
from tile_sources import PartitionedPMTilesSource, MissingTileError
|
|
28
28
|
|
|
29
29
|
WEBP_QUALITY = 75
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
31
|
class AttrDict(dict):
|
|
34
32
|
def __init__(self, *args, **kwargs):
|
|
35
33
|
super(AttrDict, self).__init__(*args, **kwargs)
|
|
@@ -110,7 +108,7 @@ def get_tile_file(tile, tiles_dir):
|
|
|
110
108
|
|
|
111
109
|
def check_sheets(sheets_to_pull, tiffs_dir):
|
|
112
110
|
for sheet_no in sheets_to_pull:
|
|
113
|
-
to = tiffs_dir.joinpath(f'{sheet_no}
|
|
111
|
+
to = tiffs_dir.joinpath(f'{sheet_no}')
|
|
114
112
|
if not to.exists():
|
|
115
113
|
raise Exception(f'missing file {to}')
|
|
116
114
|
|
|
@@ -122,7 +120,7 @@ def copy_tiles_over(tiles_to_pull, tiles_dir, from_pmtiles_prefix):
|
|
|
122
120
|
pull_from_pmtiles(to, from_pmtiles_prefix)
|
|
123
121
|
|
|
124
122
|
|
|
125
|
-
def create_upper_tiles(tiles_to_create, tiles_dir):
|
|
123
|
+
def create_upper_tiles(z, tiles_to_create, tiles_dir):
|
|
126
124
|
options = AttrDict({
|
|
127
125
|
'resume': True,
|
|
128
126
|
'verbose': False,
|
|
@@ -162,7 +160,7 @@ def get_sheet_data(bounds_fname):
|
|
|
162
160
|
sheets_to_box = {}
|
|
163
161
|
for f in index_data['features']:
|
|
164
162
|
sheet_no = f['properties']['id']
|
|
165
|
-
geom =
|
|
163
|
+
geom = shape(f['geometry'])
|
|
166
164
|
xmin, ymin, xmax, ymax = geom.bounds
|
|
167
165
|
box = mercantile.LngLatBbox(xmin, ymin, xmax, ymax)
|
|
168
166
|
sheets_to_box[sheet_no] = box
|
|
@@ -203,7 +201,7 @@ def create_vrt_file(sheets, tiffs_dir):
|
|
|
203
201
|
vrt_file = Path(f'{tiffs_dir}/combined.vrt')
|
|
204
202
|
if vrt_file.exists():
|
|
205
203
|
return vrt_file
|
|
206
|
-
tiff_list = [ tiffs_dir.joinpath(f'{p_sheet}
|
|
204
|
+
tiff_list = [ tiffs_dir.joinpath(f'{p_sheet}').resolve() for p_sheet in sheets ]
|
|
207
205
|
tiff_list = [ str(f) for f in tiff_list if f.exists() ]
|
|
208
206
|
|
|
209
207
|
tiff_list_str = ' '.join(tiff_list)
|
|
@@ -217,8 +215,8 @@ def cli():
|
|
|
217
215
|
parser = argparse.ArgumentParser()
|
|
218
216
|
parser.add_argument('--retile-list-file', required=True, help='File containing list of sheets to retile')
|
|
219
217
|
parser.add_argument('--bounds-file', required=True, help='Geojson file containing list of available sheets and their georaphic bounds')
|
|
220
|
-
parser.add_argument('--min-zoom',
|
|
221
|
-
parser.add_argument('--max-zoom',
|
|
218
|
+
parser.add_argument('--min-zoom', type=int, help='Minimum zoom level to create tiles for')
|
|
219
|
+
parser.add_argument('--max-zoom', type=int, help='Maximum zoom level to create tiles for')
|
|
222
220
|
parser.add_argument('--sheets-to-pull-list-outfile', default=None,
|
|
223
221
|
help='Output into which we write the list of sheet that need to be pulled, if set, the script ends after it created the list file')
|
|
224
222
|
parser.add_argument('--from-pmtiles-prefix', help='Prefix to the PMTiles source from which we pull tiles, if needs to be set when --sheets-to-pull-list-outfile is not set')
|
|
@@ -239,11 +237,25 @@ def cli():
|
|
|
239
237
|
parser.error(f"The following arguments are required when --sheets-to-pull-list-outfile is not provided: {', '.join(missing)}")
|
|
240
238
|
|
|
241
239
|
retile_sheets = Path(args.retile_list_file).read_text().split('\n')
|
|
242
|
-
retile_sheets = set([ r.strip() for r in retile_sheets if r.strip() != '' ])
|
|
240
|
+
retile_sheets = set([ r.strip().replace('.tif', '') for r in retile_sheets if r.strip() != '' ])
|
|
241
|
+
|
|
242
|
+
if args.from_pmtiles_prefix is None:
|
|
243
|
+
if not args.max_zoom:
|
|
244
|
+
parser.error('--max-zoom is required when --from-pmtiles-prefix is not set')
|
|
245
|
+
max_zoom = args.max_zoom
|
|
246
|
+
|
|
247
|
+
if not args.min_zoom:
|
|
248
|
+
parser.error('--min-zoom is required when --from-pmtiles-prefix is not set')
|
|
249
|
+
min_zoom = args.min_zoom
|
|
250
|
+
else:
|
|
251
|
+
reader = get_pmtiles_reader(args.from_pmtiles_prefix)
|
|
252
|
+
max_zoom = reader.max_zoom
|
|
253
|
+
min_zoom = reader.min_zoom
|
|
254
|
+
|
|
243
255
|
|
|
244
256
|
print('getting base tiles to sheet mapping')
|
|
245
257
|
sheets_to_box = get_sheet_data(args.bounds_file)
|
|
246
|
-
sheets_to_base_tiles, base_tiles_to_sheets = get_base_tile_sheet_mappings(sheets_to_box,
|
|
258
|
+
sheets_to_base_tiles, base_tiles_to_sheets = get_base_tile_sheet_mappings(sheets_to_box, max_zoom)
|
|
247
259
|
|
|
248
260
|
all_affected_tiles = set()
|
|
249
261
|
|
|
@@ -257,7 +269,7 @@ def cli():
|
|
|
257
269
|
for tile in affected_base_tiles:
|
|
258
270
|
to_add = base_tiles_to_sheets[tile]
|
|
259
271
|
for sheet in to_add:
|
|
260
|
-
sheets_to_pull.add(sheet)
|
|
272
|
+
sheets_to_pull.add(sheet + '.tif')
|
|
261
273
|
|
|
262
274
|
print(f'{sheets_to_pull=}')
|
|
263
275
|
|
|
@@ -268,23 +280,25 @@ def cli():
|
|
|
268
280
|
exit(0)
|
|
269
281
|
|
|
270
282
|
|
|
283
|
+
tiles_dir = Path(args.tiles_dir)
|
|
284
|
+
tiffs_dir = Path(args.tiffs_dir)
|
|
271
285
|
|
|
272
286
|
print('check the sheets availability')
|
|
273
|
-
check_sheets(sheets_to_pull,
|
|
287
|
+
check_sheets(sheets_to_pull, tiffs_dir)
|
|
274
288
|
|
|
275
289
|
print('creating vrt file from sheets involved')
|
|
276
|
-
vrt_file = create_vrt_file(sheets_to_pull,
|
|
290
|
+
vrt_file = create_vrt_file(sheets_to_pull, tiffs_dir)
|
|
277
291
|
|
|
278
292
|
Path(args.tiles_dir).mkdir(exist_ok=True, parents=True)
|
|
279
293
|
|
|
280
294
|
print('creating tiles for base zoom with a vrt')
|
|
281
|
-
create_base_tiles(
|
|
295
|
+
create_base_tiles(str(vrt_file), tiles_dir, f'{max_zoom}')
|
|
282
296
|
|
|
283
297
|
print('deleting unwanted base tiles')
|
|
284
|
-
delete_unwanted_tiles(affected_base_tiles,
|
|
298
|
+
delete_unwanted_tiles(affected_base_tiles, max_zoom, tiles_dir)
|
|
285
299
|
|
|
286
300
|
prev_affected_tiles = affected_base_tiles
|
|
287
|
-
for z in range(
|
|
301
|
+
for z in range(max_zoom-1, min_zoom-1, -1):
|
|
288
302
|
print(f'handling level {z}')
|
|
289
303
|
|
|
290
304
|
curr_affected_tiles = set()
|
|
@@ -299,13 +313,13 @@ def cli():
|
|
|
299
313
|
child_tiles_to_pull.add(ctile)
|
|
300
314
|
|
|
301
315
|
print('copying additional child tiles required for curr level')
|
|
302
|
-
copy_tiles_over(child_tiles_to_pull,
|
|
316
|
+
copy_tiles_over(child_tiles_to_pull, tiles_dir, args.from_pmtiles_prefix)
|
|
303
317
|
|
|
304
318
|
print('creating tiles for current level')
|
|
305
|
-
create_upper_tiles(curr_affected_tiles,
|
|
319
|
+
create_upper_tiles(z, curr_affected_tiles, tiles_dir)
|
|
306
320
|
|
|
307
321
|
print('removing unwanted child tiles')
|
|
308
|
-
delete_unwanted_tiles(prev_affected_tiles, z+1,
|
|
322
|
+
delete_unwanted_tiles(prev_affected_tiles, z+1, tiles_dir)
|
|
309
323
|
|
|
310
324
|
prev_affected_tiles = curr_affected_tiles
|
|
311
325
|
|
{topo_map_processor-0.1.4 → topo_map_processor-0.1.6}/topo_map_processor/tools/tile_sources.py
RENAMED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import os
|
|
2
1
|
import glob
|
|
3
|
-
import json
|
|
4
2
|
|
|
5
3
|
from pathlib import Path
|
|
6
4
|
|
|
@@ -14,52 +12,87 @@ class MissingTileError(Exception):
|
|
|
14
12
|
|
|
15
13
|
class DiskSource:
|
|
16
14
|
def __init__(self, directory):
|
|
17
|
-
self.dir = directory
|
|
15
|
+
self.dir = Path(directory)
|
|
18
16
|
|
|
19
|
-
def get_tile_from_file(self,
|
|
20
|
-
parts =
|
|
17
|
+
def get_tile_from_file(self, file):
|
|
18
|
+
parts = file.parts
|
|
21
19
|
tile = mercantile.Tile(z=int(parts[-3]),
|
|
22
20
|
x=int(parts[-2]),
|
|
23
21
|
y=int(parts[-1].replace('.webp', '')))
|
|
24
22
|
return tile
|
|
25
23
|
|
|
26
24
|
def file_from_tile(self, tile):
|
|
27
|
-
return
|
|
25
|
+
return self.dir / f'{tile.z}' / f'{tile.x}' / f'{tile.y}.webp'
|
|
28
26
|
|
|
29
27
|
def get_tile_data(self, tile):
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
os.stat(fname)
|
|
33
|
-
except FileNotFoundError:
|
|
28
|
+
file = self.file_from_tile(tile)
|
|
29
|
+
if not file.exists():
|
|
34
30
|
raise MissingTileError()
|
|
35
31
|
|
|
36
|
-
|
|
37
|
-
return f.read()
|
|
32
|
+
return file.read_bytes()
|
|
38
33
|
|
|
39
34
|
def get_tile_size(self, tile):
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
fstats = os.stat(fname)
|
|
43
|
-
except FileNotFoundError:
|
|
35
|
+
file = self.file_from_tile(tile)
|
|
36
|
+
if not file.exists():
|
|
44
37
|
raise MissingTileError()
|
|
45
|
-
|
|
38
|
+
|
|
39
|
+
return file.stat().st_size
|
|
46
40
|
|
|
47
41
|
def for_all_z(self, z):
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
fstats = os.stat(fname)
|
|
42
|
+
for file in self.dir.glob(f'{z}/*/*.webp'):
|
|
43
|
+
tile = self.get_tile_from_file(file)
|
|
44
|
+
fstats = file.stat()
|
|
52
45
|
yield (tile, fstats.st_size)
|
|
53
46
|
|
|
54
47
|
def all(self):
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
t_data = Path(fname).read_bytes()
|
|
48
|
+
for file in self.dir.glob('*/*/*.webp'):
|
|
49
|
+
tile = self.get_tile_from_file(file)
|
|
50
|
+
t_data = file.read_bytes()
|
|
59
51
|
yield (tile, t_data)
|
|
60
52
|
|
|
61
53
|
def cleanup(self):
|
|
62
54
|
pass
|
|
55
|
+
|
|
56
|
+
def get_zomm_levels(self):
|
|
57
|
+
zoom_levels = []
|
|
58
|
+
for dir in self.dir.glob('*'):
|
|
59
|
+
if not dir.is_dir():
|
|
60
|
+
continue
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
zoom = int(dir.name)
|
|
64
|
+
except ValueError:
|
|
65
|
+
continue
|
|
66
|
+
zoom_levels.append(zoom)
|
|
67
|
+
return zoom_levels
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def min_zoom(self):
|
|
71
|
+
zoom_levels = self.get_zomm_levels()
|
|
72
|
+
if len(zoom_levels) == 0:
|
|
73
|
+
raise ValueError("No zoom directories found in the disk source.")
|
|
74
|
+
|
|
75
|
+
return min(zoom_levels)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def max_zoom(self):
|
|
79
|
+
zoom_levels = self.get_zomm_levels()
|
|
80
|
+
if len(zoom_levels) == 0:
|
|
81
|
+
raise ValueError("No zoom directories found in the disk source.")
|
|
82
|
+
|
|
83
|
+
return max(zoom_levels)
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def name(self):
|
|
87
|
+
raise ValueError("DiskSource does not have a name property.")
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def description(self):
|
|
91
|
+
raise ValueError("DiskSource does not have a description property.")
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def attribution(self):
|
|
95
|
+
raise ValueError("DiskSource does not have a attribution property.")
|
|
63
96
|
|
|
64
97
|
class PartitionedPMTilesSource:
|
|
65
98
|
def __init__(self, pmtiles_prefix):
|
|
@@ -69,13 +102,13 @@ class PartitionedPMTilesSource:
|
|
|
69
102
|
for fname in pmtiles_files:
|
|
70
103
|
file = open(fname, 'rb')
|
|
71
104
|
self.files.append(file)
|
|
72
|
-
src = MmapSource(
|
|
105
|
+
src = MmapSource(file)
|
|
73
106
|
reader = PMTilesReader(src)
|
|
74
107
|
self.readers[fname] = reader
|
|
75
108
|
|
|
76
109
|
self.fnames = list(pmtiles_files)
|
|
77
110
|
fname_to_index = { fname:i for i,fname in enumerate(self.fnames) }
|
|
78
|
-
self.
|
|
111
|
+
self.tiles_to_findex = {}
|
|
79
112
|
self.tiles_to_size = {}
|
|
80
113
|
for fname in self.fnames:
|
|
81
114
|
print(f'collecting tile sizes from {fname}')
|
|
@@ -86,7 +119,7 @@ class PartitionedPMTilesSource:
|
|
|
86
119
|
self.tiles_to_size[tile] = len(t_data)
|
|
87
120
|
|
|
88
121
|
def get_reader_from_tile(self, tile):
|
|
89
|
-
if tile not in self.
|
|
122
|
+
if tile not in self.tiles_to_findex:
|
|
90
123
|
raise MissingTileError()
|
|
91
124
|
s = self.tiles_to_findex[tile]
|
|
92
125
|
fname = self.fnames[s]
|
|
@@ -122,6 +155,34 @@ class PartitionedPMTilesSource:
|
|
|
122
155
|
for f in self.files:
|
|
123
156
|
f.close()
|
|
124
157
|
|
|
158
|
+
@property
|
|
159
|
+
def min_zoom(self):
|
|
160
|
+
return min([int(reader.header()['min_zoom']) for reader in self.readers.values()])
|
|
161
|
+
|
|
162
|
+
@property
|
|
163
|
+
def max_zoom(self):
|
|
164
|
+
return max([int(reader.header()['max_zoom']) for reader in self.readers.values()])
|
|
165
|
+
|
|
166
|
+
def get_meta_prop(self, prop_name):
|
|
167
|
+
for reader in self.readers.values():
|
|
168
|
+
metadata = reader.metadata()
|
|
169
|
+
if prop_name in metadata:
|
|
170
|
+
return metadata[prop_name]
|
|
171
|
+
raise ValueError(f"Source does not have a {prop_name} property.")
|
|
172
|
+
|
|
173
|
+
@property
|
|
174
|
+
def name(self):
|
|
175
|
+
return self.get_meta_prop('name')
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def description(self):
|
|
179
|
+
return self.get_meta_prop('description')
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def attribution(self):
|
|
183
|
+
return self.get_meta_prop('attribution')
|
|
184
|
+
|
|
185
|
+
|
|
125
186
|
|
|
126
187
|
|
|
127
188
|
class DiskAndPartitionedPMTilesSource:
|
|
@@ -167,4 +228,59 @@ class DiskAndPartitionedPMTilesSource:
|
|
|
167
228
|
self.dsrc.cleanup()
|
|
168
229
|
self.psrc.cleanup()
|
|
169
230
|
|
|
231
|
+
@property
|
|
232
|
+
def min_zoom(self):
|
|
233
|
+
min_zooms = []
|
|
234
|
+
|
|
235
|
+
try:
|
|
236
|
+
min_disk_zoom = self.dsrc.min_zoom
|
|
237
|
+
min_zooms.append(min_disk_zoom)
|
|
238
|
+
except ValueError:
|
|
239
|
+
pass
|
|
240
|
+
|
|
241
|
+
try:
|
|
242
|
+
min_pmtiles_zoom = self.psrc.min_zoom
|
|
243
|
+
min_zooms.append(min_pmtiles_zoom)
|
|
244
|
+
except ValueError:
|
|
245
|
+
pass
|
|
246
|
+
|
|
247
|
+
if len(min_zooms) == 0:
|
|
248
|
+
raise ValueError("No zoom directories found in the disk source or PMTiles source.")
|
|
249
|
+
|
|
250
|
+
return min(min_zooms)
|
|
251
|
+
|
|
252
|
+
@property
|
|
253
|
+
def max_zoom(self):
|
|
254
|
+
max_zooms = []
|
|
255
|
+
|
|
256
|
+
try:
|
|
257
|
+
max_disk_zoom = self.dsrc.max_zoom
|
|
258
|
+
max_zooms.append(max_disk_zoom)
|
|
259
|
+
except ValueError:
|
|
260
|
+
pass
|
|
261
|
+
|
|
262
|
+
try:
|
|
263
|
+
max_pmtiles_zoom = self.psrc.max_zoom
|
|
264
|
+
max_zooms.append(max_pmtiles_zoom)
|
|
265
|
+
except ValueError:
|
|
266
|
+
pass
|
|
267
|
+
|
|
268
|
+
if len(max_zooms) == 0:
|
|
269
|
+
raise ValueError("No zoom directories found in the disk source or PMTiles source.")
|
|
270
|
+
|
|
271
|
+
return max(max_zooms)
|
|
272
|
+
|
|
273
|
+
@property
|
|
274
|
+
def name(self):
|
|
275
|
+
return self.psrc.name
|
|
276
|
+
|
|
277
|
+
@property
|
|
278
|
+
def description(self):
|
|
279
|
+
return self.psrc.description
|
|
280
|
+
|
|
281
|
+
@property
|
|
282
|
+
def attribution(self):
|
|
283
|
+
return self.psrc.attribution
|
|
284
|
+
|
|
285
|
+
|
|
170
286
|
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import argparse
|
|
3
|
-
|
|
4
|
-
def cli():
|
|
5
|
-
parser = argparse.ArgumentParser(description='Collect GeoJSONL files into a single GeoJSON file.')
|
|
6
|
-
parser.add_argument('--bounds-dir', required=True, help='Directory containing GeoJSONL files')
|
|
7
|
-
parser.add_argument('--output-file', required=True, help='Output GeoJSON file path')
|
|
8
|
-
|
|
9
|
-
args = parser.parse_args()
|
|
10
|
-
|
|
11
|
-
paths = list(args.bounds_dir.glob('*.geojsonl'))
|
|
12
|
-
with open(args.output_file, 'w') as f:
|
|
13
|
-
f.write('{ "type": "FeatureCollection", "features": [\n')
|
|
14
|
-
for i,p in enumerate(paths):
|
|
15
|
-
line = p.read_text().strip('\n')
|
|
16
|
-
|
|
17
|
-
if i != len(paths) - 1:
|
|
18
|
-
line += ','
|
|
19
|
-
line += '\n'
|
|
20
|
-
f.write(line)
|
|
21
|
-
f.write(']}\n')
|
|
22
|
-
|
|
23
|
-
if __name__ == "__main__":
|
|
24
|
-
cli()
|
|
25
|
-
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import json
|
|
3
|
-
import argparse
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
def cli():
|
|
7
|
-
parser = argparse.ArgumentParser(description="Update bounds in a GeoJSON file.")
|
|
8
|
-
parser.add_argument("--bounds-file", required=True, help="Input GeoJSON file with bounds.")
|
|
9
|
-
parser.add_argument("--bounds-dir", required=True, help="Directory containing the individual bounds files")
|
|
10
|
-
parser.add_argument("--retile-list-file", required=True, help="File containing the list of sheets to update.. .tif extension is expected")
|
|
11
|
-
|
|
12
|
-
args = parser.parse_args()
|
|
13
|
-
|
|
14
|
-
bounds_file = Path(args.bounds_file)
|
|
15
|
-
bounds_dir = Path(args.bounds_dir)
|
|
16
|
-
retile_list_file = Path(args.retile_list_file)
|
|
17
|
-
|
|
18
|
-
existing_map = {}
|
|
19
|
-
if not bounds_file.exists():
|
|
20
|
-
raise FileNotFoundError(f"Bounds file does not exist: {bounds_file}")
|
|
21
|
-
|
|
22
|
-
# Read the existing bounds file
|
|
23
|
-
existing_data = json.loads(bounds_file.read_text())
|
|
24
|
-
for feature in existing_data.get('features', []):
|
|
25
|
-
if 'properties' in feature and 'id' in feature['properties']:
|
|
26
|
-
sheet_name = feature['properties']['id']
|
|
27
|
-
existing_map[sheet_name] = feature
|
|
28
|
-
|
|
29
|
-
sheets = retile_list_file.read_text().strip().splitlines()
|
|
30
|
-
sheets = [s.strip() for s in sheets if s.strip()]
|
|
31
|
-
for s in sheets:
|
|
32
|
-
s = s.replace('.tif', '')
|
|
33
|
-
bfile = bounds_dir / f'{s}.geojsonl'
|
|
34
|
-
if not bfile.exists():
|
|
35
|
-
raise FileNotFoundError(f"Bounds file for sheet '{s}' does not exist: {bfile}")
|
|
36
|
-
|
|
37
|
-
feature = json.loads(bfile.read_text())
|
|
38
|
-
existing_data[s] = feature
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
all_feats = list(existing_data.values())
|
|
42
|
-
|
|
43
|
-
with open(bounds_file, 'w') as f:
|
|
44
|
-
f.write('{ "type": "FeatureCollection", "features": [\n')
|
|
45
|
-
for i,feat in enumerate(all_feats):
|
|
46
|
-
line = json.dumps(feat)
|
|
47
|
-
if i != len(all_feats) - 1:
|
|
48
|
-
line += ','
|
|
49
|
-
line += '\n'
|
|
50
|
-
f.write(line)
|
|
51
|
-
f.write(']}\n')
|
|
52
|
-
|
|
53
|
-
if __name__ == "__main__":
|
|
54
|
-
cli()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|