topo_map_processor 0.1.12__tar.gz → 0.1.14__tar.gz

{topo_map_processor-0.1.12 → topo_map_processor-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topo_map_processor
-Version: 0.1.12
+Version: 0.1.14
 Summary: Library for processing topographic maps
 Project-URL: homepage, https://github.com/ramSeraph/topo_map_processor
 Project-URL: repository, https://github.com/ramSeraph/topo_map_processor.git
@@ -239,11 +239,12 @@ The tool works in two stages. First, it calculates the full set of sheets that n
 In the second stage, it retiles the affected area. It uses the pull list to create a temporary virtual raster (`.vrt`), generates new base tiles for the affected region, and then reconstructs the upper-level tiles by pulling existing, unaffected tiles from the PMTiles source.
 
 ```bash
-retile --retile-list-file <file> --bounds-file <file> [--max-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-source <source>] [--sheets-to-pull-list-outfile <file>] [--num-parallel <processes>] [--tile-quality <quality>]
+retile --retile-list-file <file> --bounds-file <file> [--dummy-bounds-file <file>] [--max-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-source <source>] [--sheets-to-pull-list-outfile <file>] [--num-parallel <processes>] [--tile-quality <quality>]
 ```
 
 -   `--retile-list-file`: File containing the list of sheets to retile. (Required)
 -   `--bounds-file`: GeoJSON file containing the list of available sheets and their geographic bounds. (Required)
+-   `--dummy-bounds-file`: GeoJSON file containing geographic bounds of areas that need to be updated. These areas might not have any new sheets. (Optional)
 -   `--max-zoom`: Maximum zoom level to create tiles for. Defaults to the max zoom from the source if `--from-source` is provided, otherwise it is required.
 -   `--tiffs-dir`: Directory where the GeoTIFFs are located. (Required if not in pull-list generation mode)
 -   `--tiles-dir`: Directory where the tiles will be created. (Required if not in pull-list generation mode)

{topo_map_processor-0.1.12 → topo_map_processor-0.1.14}/README.md

@@ -190,11 +190,12 @@ The tool works in two stages. First, it calculates the full set of sheets that n
 In the second stage, it retiles the affected area. It uses the pull list to create a temporary virtual raster (`.vrt`), generates new base tiles for the affected region, and then reconstructs the upper-level tiles by pulling existing, unaffected tiles from the PMTiles source.
 
 ```bash
-retile --retile-list-file <file> --bounds-file <file> [--max-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-source <source>] [--sheets-to-pull-list-outfile <file>] [--num-parallel <processes>] [--tile-quality <quality>]
+retile --retile-list-file <file> --bounds-file <file> [--dummy-bounds-file <file>] [--max-zoom <zoom>] [--tiffs-dir <dir>] [--tiles-dir <dir>] [--from-source <source>] [--sheets-to-pull-list-outfile <file>] [--num-parallel <processes>] [--tile-quality <quality>]
 ```
 
 -   `--retile-list-file`: File containing the list of sheets to retile. (Required)
 -   `--bounds-file`: GeoJSON file containing the list of available sheets and their geographic bounds. (Required)
+-   `--dummy-bounds-file`: GeoJSON file containing geographic bounds of areas that need to be updated. These areas might not have any new sheets. (Optional)
 -   `--max-zoom`: Maximum zoom level to create tiles for. Defaults to the max zoom from the source if `--from-source` is provided, otherwise it is required.
 -   `--tiffs-dir`: Directory where the GeoTIFFs are located. (Required if not in pull-list generation mode)
 -   `--tiles-dir`: Directory where the tiles will be created. (Required if not in pull-list generation mode)

{topo_map_processor-0.1.12 → topo_map_processor-0.1.14}/topo_map_processor/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.12'
-__version_tuple__ = version_tuple = (0, 1, 12)
+__version__ = version = '0.1.14'
+__version_tuple__ = version_tuple = (0, 1, 14)
 
 __commit_id__ = commit_id = None

{topo_map_processor-0.1.12 → topo_map_processor-0.1.14}/topo_map_processor/processor.py

@@ -85,6 +85,8 @@ class TopoMapProcessor:
         self.extents = extra.get('extents', None)
         self.pixel_cutlines = extra.get('pixel_cutlines', [])
 
+        self.inset_pixel_cutlines = extra.get('inset_pixel_cutlines', None)
+        self.inset_inpaint_color = extra.get('inset_inpaint_color', (255, 255, 255))
         # rotation related
         self.auto_rotate_thresh = extra.get('auto_rotate_thresh', 0.0)
 
@@ -490,6 +492,10 @@ class TopoMapProcessor:
         #cv2.imwrite('temp.jpg', img_strip)
 
         line_poly_t = translate(line_poly, xoff=-bb[0], yoff=-bb[1])
+        # if empty then nothing to do
+        if line_poly_t.is_empty:
+            print('line poly empty, nothing to do')
+            return
         mask = np.zeros(img_strip.shape[:2], dtype=np.uint8)
         poly_coords = np.array([ [int(x[0]), int(x[1])] for x in line_poly_t.exterior.coords ])
         cv2.fillPoly(mask, pts=[poly_coords], color=1)
@@ -767,7 +773,7 @@ class TopoMapProcessor:
             map_contour = ctuples[map_inner_contour_idx][0]
 
         return map_contour
- 
+
     def get_maparea(self):
         workdir = self.get_workdir()
 
@@ -827,7 +833,6 @@ class TopoMapProcessor:
             print('already rotated.. skipping rotation')
             return
 
-
         map_bbox, map_min_rect, _ = self.get_maparea()
         _, _, angle = map_min_rect
         if angle > 45:
@@ -1588,9 +1593,6 @@ class TopoMapProcessor:
             self.remove_grid_lines()
             from_file = workdir.joinpath('nogrid.jpg')
 
-        ibox = self.get_sheet_ibox()
-        print(ibox)
-
         crs_proj = self.get_crs_proj()
 
         gcps = self.get_gcps()
@@ -1604,6 +1606,7 @@ class TopoMapProcessor:
         creation_options = '-co TILED=YES -co COMPRESS=DEFLATE -co PREDICTOR=2' 
         perf_options = '--config GDAL_CACHEMAX 128 --config GDAL_NUM_THREADS ALL_CPUS'
 
+        self.ensure_dir(workdir)
         translate_cmd = f'gdal_translate {creation_options} {perf_options} {gcp_str} -a_srs "{crs_proj}" -of GTiff {str(from_file)} {str(georef_file)}' 
         self.run_external(translate_cmd)
 
@@ -1749,7 +1752,13 @@ class TopoMapProcessor:
             reproj_options = f'-tps -ts 0 {self.warp_output_height} -r bilinear -t_srs "EPSG:3857"' 
         else:
             res = self.get_resolution()
-            reproj_options = f'-tps -tr {res} {res} -r bilinear -t_srs "EPSG:3857"' 
+            if res == 'auto':
+                reproj_options = '-tps -r bilinear -t_srs "EPSG:3857"'
+            else:
+                if type(res) is not list:
+                    res = [res, res]
+                reproj_options = f'-tps -tr {res[0]} {res[1]} -r bilinear -t_srs "EPSG:3857"' 
+
         #nodata_options = '-dstnodata 0'
         nodata_options = '-dstalpha'
         perf_options = '-multi -wo NUM_THREADS=ALL_CPUS --config GDAL_CACHEMAX 1024 -wm 1024' 
@@ -1772,6 +1781,22 @@ class TopoMapProcessor:
 
         return sheet_ibox
 
+    def inpaint_insets(self, img, inset_pixel_cutlines, inpaint_color):
+        cv_img = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+        for inset_pixel_cutline in inset_pixel_cutlines:
+            print(f'filling in inset with pixel cutline: {inset_pixel_cutline}')
+            # Create a mask for the inset area
+            mask = np.zeros(cv_img.shape[:2], dtype=np.uint8)
+            pts = np.array([inset_pixel_cutline], dtype=np.int32)
+            cv2.fillPoly(mask, pts, 255)
+            # fill the area in cv_img with inpaint_color
+            cv_img[mask == 255] = inpaint_color
+
+        inpainted_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2RGB)
+        return Image.fromarray(inpainted_img)
+
+
     def export_bounds_file(self):
         bounds_dir = self.get_bounds_dir()
 
@@ -1800,6 +1825,7 @@ class TopoMapProcessor:
 
         sheet_ibox = self.get_updated_sheet_ibox()
         
+        self.ensure_dir(workdir)
         self.warp_file(sheet_ibox, cutline_file, georef_file, final_file, self.warp_jpeg_export_quality)
 
     def export_gtiff(self, filename, out_filename, jpeg_export_quality):
@@ -1825,12 +1851,24 @@ class TopoMapProcessor:
 
         self.export_gtiff(str(final_file), str(export_file), self.jpeg_export_quality)
 
+    def remove_insets(self):
+        if self.inset_pixel_cutlines is None or len(self.inset_pixel_cutlines) == 0:
+            return
+        full_img = self.get_full_img()
+        inpainted_img = self.inpaint_insets(full_img, self.inset_pixel_cutlines, self.inset_inpaint_color)
+        workdir = self.get_workdir()
+        inpainted_file = workdir.joinpath('full.jpg')
+        self.ensure_dir(workdir)
+        inpainted_img.save(inpainted_file)
+        self.full_img = None
+
 
     def process(self):
         export_file = self.get_export_file()
         if export_file.exists():
             return True
 
+        self.remove_insets()
         self.rotate()
 
         # pause to debug 

{topo_map_processor-0.1.12 → topo_map_processor-0.1.14}/topo_map_processor/tools/retile.py

@@ -4,6 +4,7 @@ import re
 import sys
 import json
 import argparse
+import tempfile
 
 import time
 import subprocess
@@ -24,7 +25,8 @@ from osgeo_utils.gdal2tiles import (
     DividedCache,
 )
 
-from pmtiles_mosaic.tile_sources import create_source_from_paths, MissingTileError
+from pmtiles_mosaic.tile_sources import create_source_from_paths
+from pmtiles_mosaic.tiles_common import MissingTileError
 
 from .utils import relax_open_file_limit
 
@@ -84,7 +86,7 @@ class DiskTilesHandler:
         print(f'deleted {delete_count} files at level {z}')
 
     def save_metadata(self, metadata):
-        metadata_file = self.dir.joinpath('tiles.json')
+        metadata_file = self.dir.joinpath('metadata.json')
         metadata_file.write_text(json.dumps(metadata, indent=2))
 
  
@@ -318,7 +320,7 @@ def check_sheets(sheets_to_pull, tiffs_dir):
         raise Exception(f'missing files {missing}')
 
 
-def get_sheet_data(bounds_fname):
+def get_sheet_data(bounds_fname, dummy=False):
 
     index_file = Path(bounds_fname)
     if not index_file.exists():
@@ -326,9 +328,14 @@ def get_sheet_data(bounds_fname):
 
     index_data = json.loads(index_file.read_text())
 
+    dummy_count = 0
     sheets_to_box = {}
     for f in index_data['features']:
-        sheet_no = f['properties']['id']
+        if not dummy:
+            sheet_no = f['properties']['id']
+        else:
+            sheet_no = f'dummy_{dummy_count}'
+            dummy_count += 1
         geom = shape(f['geometry'])
         xmin, ymin, xmax, ymax = geom.bounds
         box = mercantile.LngLatBbox(xmin, ymin, xmax, ymax)
@@ -353,33 +360,61 @@ def get_base_tile_sheet_mappings(sheets_to_box, base_zoom):
 
    
 
-def assess_sheet_requirements(retile_list_file, bounds_file, max_zoom):
+def assess_sheet_requirements(retile_list_file, bounds_file, dummy_bounds_file, max_zoom):
 
     retile_sheets = retile_list_file.read_text().split('\n')
     retile_sheets = set([ r.strip().replace('.tif', '') for r in retile_sheets if r.strip() != '' ])
 
     print('getting base tiles to sheet mapping')
     sheets_to_box = get_sheet_data(bounds_file)
+    dummy_sheets_to_box = {}
+    if dummy_bounds_file:
+        dummy_sheets_to_box = get_sheet_data(dummy_bounds_file, dummy=True)
+        sheets_to_box.update(dummy_sheets_to_box)
+
     sheets_to_base_tiles, base_tiles_to_sheets = get_base_tile_sheet_mappings(sheets_to_box, max_zoom)
 
     print('calculating sheets to pull')
     affected_base_tiles = set()
     for sheet_no in retile_sheets:
         affected_base_tiles.update(sheets_to_base_tiles[sheet_no])
+    for sheet_no in dummy_sheets_to_box.keys():
+        affected_base_tiles.update(sheets_to_base_tiles[sheet_no])
 
     sheets_to_pull = set()
     for tile in affected_base_tiles:
         to_add = base_tiles_to_sheets[tile]
         for sheet in to_add:
-            sheets_to_pull.add(sheet + '.tif')
+            if sheet not in dummy_sheets_to_box:
+                sheets_to_pull.add(sheet + '.tif')
 
     return sheets_to_pull, affected_base_tiles
 
+def create_dummy_sheets(dummy_bounds_file, tiffs_dir):
+    print('creating dummy sheets')
+    dummy_sheets_to_box = get_sheet_data(dummy_bounds_file, dummy=True)
+
+    for sheet_no, box in dummy_sheets_to_box.items():
+        file = tiffs_dir.joinpath(f'{sheet_no}.tif')
+        if file.exists():
+            continue
+
+        west, south, east, north = box.west, box.south, box.east, box.north
+        temp_file = Path(tempfile.mktemp(suffix='.tif'))
+        cmd = f'gdal_create -of GTiff -a_srs EPSG:4326 -a_ullr {west} {north} {east} {south} -a_nodata 0 -ot Byte -co COMPRESS=LZW -bands 3 -burn 0 0 0 -outsize 256 256 {temp_file}'
+        run_external(cmd)
+        temp_file_1 = Path(tempfile.mktemp(suffix='.tif'))
+        cmd = f'gdalwarp -dstalpha -t_srs EPSG:3857 -co COMPRESS=LZW {temp_file} {temp_file_1}'
+        run_external(cmd)
+        cmd = f'gdal_translate -co TILING_SCHEME=GoogleMapsCompatible -co COMPRESS=JPEG -co QUALITY=100 --config GDAL_TIFF_INTERNAL_MASK YES  -b 1 -b 2 -b 3 -mask 4 --config GDAL_CACHEMAX 512 -of COG {temp_file_1} {file}'
+        run_external(cmd)
+
 def retile_main(args):
 
     parser = argparse.ArgumentParser()
     parser.add_argument('--retile-list-file', required=True, help='File containing list of sheets to retile')
     parser.add_argument('--bounds-file', required=True, help='Geojson file containing list of available sheets and their georaphic bounds')
+    parser.add_argument('--dummy-bounds-file', required=False, help='Geojson file containing georaphic bounds of areas that need to be updated, these areas might not have any new sheets')
     parser.add_argument('--max-zoom', type=int, help='Maximum zoom level to create tiles for, needed only when --from-source is not provided')
     parser.add_argument('--sheets-to-pull-list-outfile', default=None,
                         help='Output into which we write the list of sheet that need to be pulled, if set, the script ends after it created the list file')
@@ -399,6 +434,12 @@ def retile_main(args):
     if not bounds_file.exists():
         parser.error(f'Bounds file {args.bounds_file} does not exist')
 
+    dummy_bounds_file = None
+    if args.dummy_bounds_file:
+        dummy_bounds_file = Path(args.dummy_bounds_file) 
+        if not dummy_bounds_file.exists():
+            parser.error(f'Dummy bounds file {args.dummy_bounds_file} does not exist')
+
 
     if not args.sheets_to_pull_list_outfile:
         missing = []
@@ -429,7 +470,7 @@ def retile_main(args):
             print(f'--max-zoom argument is ignored when --from-source is set, using max zoom from original source: {max_zoom}')
 
     # calculate the sheets to pull and affected base tiles
-    sheets_to_pull, affected_base_tiles = assess_sheet_requirements(retile_list_file, bounds_file, max_zoom)
+    sheets_to_pull, affected_base_tiles = assess_sheet_requirements(retile_list_file, bounds_file, dummy_bounds_file, max_zoom)
 
     if args.sheets_to_pull_list_outfile is not None:
         Path(args.sheets_to_pull_list_outfile).write_text('\n'.join(sheets_to_pull) + '\n')
@@ -442,6 +483,9 @@ def retile_main(args):
     print('check the sheets availability')
     check_sheets(sheets_to_pull, tiffs_dir)
 
+    if dummy_bounds_file is not None:
+        create_dummy_sheets(dummy_bounds_file, tiffs_dir)
+
     metadata = orig_source.get_metadata()
     tile_extension = metadata['format']
     if tile_extension == 'jpg':

{topo_map_processor-0.1.12 → topo_map_processor-0.1.14}/topo_map_processor/tools/tile.py

@@ -141,7 +141,7 @@ def cli():
 
     vrt_file.unlink()
 
-    metadata_file = tiles_dir / 'tiles.json'
+    metadata_file = tiles_dir / 'metadata.json'
 
     fmt = get_format(args.tile_extension)