topo_map_processor 0.1.3__py3-none-any.whl

topo_map_processor/tools/retile.py

@@ -0,0 +1,311 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "mercantile",
+#     "pmtiles",
+# ]
+# ///
+
+import os
+import re
+import json
+import argparse
+
+import time
+import subprocess
+
+from pathlib import Path
+
+
+import mercantile
+from shapely.geometry import mapping
+
+from osgeo_utils.gdal2tiles import main as gdal2tiles_main
+from osgeo_utils.gdal2tiles import create_overview_tile, TileJobInfo, GDAL2Tiles
+
+
+from tile_sources import PartitionedPMTilesSource, MissingTileError
+
+WEBP_QUALITY = 75
+
+
+
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+def run_external(cmd):
+    print(f'running cmd - {cmd}')
+    start = time.time()
+    res = subprocess.run(cmd, shell=True, capture_output=True, text=True)
+    end = time.time()
+    print(f'STDOUT: {res.stdout}')
+    print(f'STDERR: {res.stderr}')
+    print(f'command took {end - start} secs to run')
+    if res.returncode != 0:
+        raise Exception(f'command {cmd} failed')
+
+
+def convert_paths_in_vrt(vrt_file):
+    # <SourceFilename relativeToVRT="1">40M_15.tif</SourceFilename>
+    vrt_dirname = str(vrt_file.resolve().parent)
+    vrt_text = vrt_file.read_text()
+    replaced = re.sub(
+        r'<SourceFilename relativeToVRT="1">(.*)</SourceFilename>',
+        rf'<SourceFilename relativeToVRT="0">{vrt_dirname}/\1</SourceFilename>',
+        vrt_text
+    )
+    vrt_file.write_text(replaced)
+
+    
+def create_base_tiles(inp_file, output_dir, zoom_levels):
+    print('start tiling')
+    os.environ['GDAL_CACHEMAX'] = '2048'
+    os.environ['GDAL_MAX_DATASET_POOL_SIZE'] = '5000'
+    os.environ['GDAL_DISABLE_READDIR_ON_OPEN'] = 'TRUE'
+    #os.environ['VRT_SHARED_SOURCE'] = '1'
+    #os.environ['GTIFF_VIRTUAL_MEM_IO'] = 'TRUE'
+    gdal2tiles_main(['gdal2tiles.py',
+                     '-r', 'antialias',
+                     '--verbose',
+                     '-w', 'none',
+                     '--exclude', 
+                     '--resume', 
+                     '--xyz', 
+                     '--processes=8', 
+                     '-z', zoom_levels,
+                     '--tiledriver', 'WEBP',
+                     '--webp-quality', f'{WEBP_QUALITY}',
+                     inp_file, output_dir])
+
+
+
+pmtiles_reader = None
+def get_pmtiles_reader(from_pmtiles_prefix):
+    global pmtiles_reader
+    if pmtiles_reader is None:
+        pmtiles_reader = PartitionedPMTilesSource(from_pmtiles_prefix)
+    return pmtiles_reader
+
+
+def pull_from_pmtiles(file, from_pmtiles_prefix):
+    reader = get_pmtiles_reader(from_pmtiles_prefix)
+    fname = str(file)
+    pieces = fname.split('/')
+    tile = mercantile.Tile(x=int(pieces[-2]),
+                           y=int(pieces[-1].replace('.webp', '')),
+                           z=int(pieces[-3]))
+    try:
+        t_data = reader.get_tile_data(tile)
+    except MissingTileError:
+        return
+    file.parent.mkdir(exist_ok=True, parents=True)
+    file.write_bytes(t_data)
+
+
+def get_tile_file(tile, tiles_dir):
+    return f'{tiles_dir}/{tile.z}/{tile.x}/{tile.y}.webp'
+
+def check_sheets(sheets_to_pull, tiffs_dir):
+    for sheet_no in sheets_to_pull:
+        to = tiffs_dir.joinpath(f'{sheet_no}.tif')
+        if not to.exists():
+            raise Exception(f'missing file {to}')
+
+def copy_tiles_over(tiles_to_pull, tiles_dir, from_pmtiles_prefix):
+    for tile in tiles_to_pull:
+        to = Path(get_tile_file(tile, tiles_dir))
+        if to.exists():
+            continue
+        pull_from_pmtiles(to, from_pmtiles_prefix)
+
+
+def create_upper_tiles(tiles_to_create, tiles_dir):
+    options = AttrDict({
+        'resume': True,
+        'verbose': False,
+        'quiet': False,
+        'xyz': True,
+        'exclude_transparent': True,
+        'profile': 'mercator',
+        'resampling': 'antialias',
+        'tiledriver': 'WEBP',
+        'webp_quality': WEBP_QUALITY,
+        'webp_lossless': False
+    })
+    tile_job_info = TileJobInfo(
+        tile_driver='WEBP',
+        nb_data_bands=3,
+        tile_size=256,
+        tile_extension='webp',
+        kml=False
+    )
+
+    for tile in tiles_to_create:
+        ctiles = mercantile.children(tile)
+        tiles_dir.joinpath(str(tile.z), str(tile.x)).mkdir(parents=True, exist_ok=True)
+        base_tile_group = [ (t.x, GDAL2Tiles.getYTile(t.y, t.z, options)) for t in ctiles ]
+        #print(f'{tile=}, {base_tile_group=}')
+        create_overview_tile(z + 1, output_folder=str(tiles_dir), tile_job_info=tile_job_info, options=options, base_tiles=base_tile_group)
+ 
+
+def get_sheet_data(bounds_fname):
+
+    index_file = Path(bounds_fname)
+    if not index_file.exists():
+        raise Exception(F'missing index file at {bounds_fname}')
+
+    index_data = json.loads(index_file.read_text())
+
+    sheets_to_box = {}
+    for f in index_data['features']:
+        sheet_no = f['properties']['id']
+        geom = mapping(f['geometry'])
+        xmin, ymin, xmax, ymax = geom.bounds
+        box = mercantile.LngLatBbox(xmin, ymin, xmax, ymax)
+        sheets_to_box[sheet_no] = box
+
+    return sheets_to_box
+
+
+def get_base_tile_sheet_mappings(sheets_to_box, base_zoom):
+    sheets_to_base_tiles = {}
+    base_tiles_to_sheets = {}
+
+    for sheet_no, box in sheets_to_box.items():
+        tiles = set(mercantile.tiles(box.west, box.south, box.east, box.north, [base_zoom])) 
+        sheets_to_base_tiles[sheet_no] = tiles
+        for tile in tiles:
+            if tile not in base_tiles_to_sheets:
+                base_tiles_to_sheets[tile] = set()
+            base_tiles_to_sheets[tile].add(sheet_no)
+
+    return sheets_to_base_tiles, base_tiles_to_sheets
+
+
+def delete_unwanted_tiles(tiles_to_keep, z, tiles_dir):
+    delete_count = 0
+    for file in tiles_dir.glob(f'{z}/*/*.webp'):
+        y = int(file.name[:-5])
+        x = int(file.parent.name)
+            
+        disk_tile = mercantile.Tile(z=z, x=x, y=y)
+        if disk_tile not in tiles_to_keep:
+            #print(f'deleting {disk_tile}')
+            file.unlink()
+            delete_count += 1
+    print(f'deleted {delete_count} files at level {z}')
+
+
+def create_vrt_file(sheets, tiffs_dir):
+    vrt_file = Path(f'{tiffs_dir}/combined.vrt')
+    if vrt_file.exists():
+        return vrt_file
+    tiff_list = [ tiffs_dir.joinpath(f'{p_sheet}.tif').resolve() for p_sheet in sheets ]
+    tiff_list = [ str(f) for f in tiff_list if f.exists() ]
+
+    tiff_list_str = ' '.join(tiff_list)
+
+    run_external(f'gdalbuildvrt {vrt_file} {tiff_list_str}')
+    convert_paths_in_vrt(vrt_file)
+
+    return vrt_file
+
+def cli():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--retile-list-file', required=True, help='File containing list of sheets to retile')
+    parser.add_argument('--bounds-file', required=True, help='Geojson file containing list of available sheets and their georaphic bounds')
+    parser.add_argument('--min-zoom', default=0, type=int)
+    parser.add_argument('--max-zoom', required=True, type=int)
+    parser.add_argument('--sheets-to-pull-list-outfile', default=None,
+                        help='Output into which we write the list of sheet that need to be pulled, if set the script ends after created the list file')
+    parser.add_argument('--from-pmtiles-prefix')
+    parser.add_argument('--tiles-dir', required=True)
+    parser.add_argument('--tiffs-dir', required=True)
+
+    args = parser.parse_args()
+    if not args.sheets_to_pull_list_outfile:
+        missing = []
+        if not args.from_pmtiles_prefix:
+            missing.append('--from-pmtiles-prefix')
+        
+        if missing:
+            parser.error(f"The following arguments are required when --sheets-to-pull-list-outfile is not provided: {', '.join(missing)}")
+
+    retile_sheets = Path(args.retile_list_file).read_text().split('\n')
+    retile_sheets = set([ r.strip() for r in retile_sheets if r.strip() != '' ])
+
+    print('getting base tiles to sheet mapping')
+    sheets_to_box = get_sheet_data(args.bounds_file)
+    sheets_to_base_tiles, base_tiles_to_sheets = get_base_tile_sheet_mappings(sheets_to_box, args.max_zoom)
+
+    all_affected_tiles = set()
+
+    print('calculating sheets to pull')
+    affected_base_tiles = set()
+    for sheet_no in retile_sheets:
+        affected_base_tiles.update(sheets_to_base_tiles[sheet_no])
+    all_affected_tiles.update(affected_base_tiles)
+
+    sheets_to_pull = set()
+    for tile in affected_base_tiles:
+        to_add = base_tiles_to_sheets[tile]
+        for sheet in to_add:
+            sheets_to_pull.add(sheet)
+
+    Path(args.tiffs_dir).mkdir(exist_ok=True, parents=True)
+    Path(args.tiles_dir).mkdir(exist_ok=True, parents=True)
+    print(f'{sheets_to_pull=}')
+    if args.sheets_to_pull_list_outfile is not None:
+        Path(args.sheets_to_pull_list_outfile).write_text('\n'.join(sheets_to_pull) + '\n')
+        exit(0)
+
+
+    print('check the sheets availability')
+    check_sheets(sheets_to_pull, args.tiffs_dir)
+
+    print('creating vrt file from sheets involved')
+    vrt_file = create_vrt_file(sheets_to_pull, args.tiffs_dir)
+
+    print('creating tiles for base zoom with a vrt')
+    create_base_tiles(f'{vrt_file}', str(args.tiles_dir), f'{args.max_zoom}')
+
+    print('deleting unwanted base tiles')
+    delete_unwanted_tiles(affected_base_tiles, args.max_zoom, args.tiles_dir)
+
+    prev_affected_tiles = affected_base_tiles
+    for z in range(args.max_zoom-1, args.min_zoom-1, -1):
+        print(f'handling level {z}')
+
+        curr_affected_tiles = set()
+        for tile in prev_affected_tiles:
+            curr_affected_tiles.add(mercantile.parent(tile))
+        all_affected_tiles.update(curr_affected_tiles)
+
+        child_tiles_to_pull = set()
+        for ptile in curr_affected_tiles:
+            for ctile in mercantile.children(ptile):
+                if ctile not in prev_affected_tiles:
+                    child_tiles_to_pull.add(ctile)
+
+        print('copying additional child tiles required for curr level')
+        copy_tiles_over(child_tiles_to_pull, args.tiles_dir, args.from_pmtiles_prefix)
+
+        print('creating tiles for current level')
+        create_upper_tiles(curr_affected_tiles, args.tiles_dir)
+
+        print('removing unwanted child tiles')
+        delete_unwanted_tiles(prev_affected_tiles, z+1, args.tiles_dir)
+
+        prev_affected_tiles = curr_affected_tiles
+
+    print('All Done!!!')
+
+
+if __name__ == '__main__':
+    cli()
+
+
+

topo_map_processor/tools/tile.py

@@ -0,0 +1,86 @@
+import re
+import os
+import glob
+import argparse
+
+import time
+import subprocess
+
+from pathlib import Path
+
+from osgeo_utils.gdal2tiles import main as gdal2tiles_main
+
+def run_external(cmd):
+    print(f'running cmd - {cmd}')
+    start = time.time()
+    res = subprocess.run(cmd, shell=True, capture_output=True, text=True)
+    end = time.time()
+    print(f'STDOUT: {res.stdout}')
+    print(f'STDERR: {res.stderr}')
+    print(f'command took {end - start} secs to run')
+    if res.returncode != 0:
+        raise Exception(f'command {cmd} failed')
+
+def convert_paths_in_vrt(vrt_file):
+    vrt_dirname = str(vrt_file.resolve().parent)
+    vrt_text = vrt_file.read_text()
+    replaced = re.sub(
+        r'<SourceFilename relativeToVRT="1">(.*)</SourceFilename>',
+        rf'<SourceFilename relativeToVRT="0">{vrt_dirname}/\1</SourceFilename>',
+        vrt_text
+    )
+    vrt_file.write_text(replaced)
+
+def cli():
+    parser = argparse.ArgumentParser(description='Tile the GTiffs')
+    parser.add_argument('tiles-dir', required=True, help='Directory to store the tiles')
+    parser.add_argument('tiffs-dir', required=True, help='Directory with GTiffs to tile')
+    parser.add_argument('max-zoom', type=int, required=True, help='Maximum zoom level for tiling')
+    parser.add_argument('min-zoom', type=int, default=0, help='Minimum zoom level for tiling')
+
+    args = parser.parse_args()
+
+    tiles_dir = Path(args.tiles_dir)
+    tiles_dir.mkdir(parents=True, exist_ok=True)
+
+
+    file_names = list(glob.glob(f'{args.tiffs_dir}/*.tif'))
+    file_names = [ str(Path(f).resolve()) for f in file_names ]
+    print(f'total files: {len(file_names)}')
+
+    file_list_file = Path('files_to_tile.txt')
+    file_list_file.write_text('\n'.join(file_names))
+
+    vrt_file = Path('files_to_tile.vrt')
+
+    # create vrt file
+    run_external(f'gdalbuildvrt -input_file_list {str(file_list_file)} {str(vrt_file)}')
+    convert_paths_in_vrt(vrt_file)
+    file_list_file.unlink()
+
+    print('start tiling')
+    os.environ['GDAL_CACHEMAX'] = '2048'
+    os.environ['GDAL_MAX_DATASET_POOL_SIZE'] = '5000'
+    os.environ['GDAL_DISABLE_READDIR_ON_OPEN'] = 'TRUE'
+    #os.environ['VRT_SHARED_SOURCE'] = '1'
+    #os.environ['GTIFF_VIRTUAL_MEM_IO'] = 'TRUE'
+    gdal2tiles_main(['gdal2tiles.py',
+                     '-r', 'antialias',
+                     '--verbose',
+                     '--exclude', 
+                     '--resume', 
+                     '--xyz', 
+                     '--processes=8', 
+                     '-z', f'{args.min_zoom}-{args.max_zoom}',
+                     '--tiledriver', 'WEBP',
+                     '--webp-quality', '75',
+                     str(vrt_file), str(tiles_dir)])
+
+    vrt_file.unlink()
+
+    print('All Done!!')
+
+if __name__ == '__main__':
+    cli()
+
+

topo_map_processor/tools/tile_sources.py

@@ -0,0 +1,170 @@
+import os
+import glob
+import json
+
+from pathlib import Path
+
+import mercantile
+
+from pmtiles.reader import MmapSource, Reader as PMTilesReader, all_tiles
+
+class MissingTileError(Exception):
+    pass
+
+
+class DiskSource:
+    def __init__(self, directory):
+        self.dir = directory
+
+    def get_tile_from_file(self, fname):
+        parts = fname.split('/')
+        tile = mercantile.Tile(z=int(parts[-3]),
+                               x=int(parts[-2]),
+                               y=int(parts[-1].replace('.webp', '')))
+        return tile
+
+    def file_from_tile(self, tile):
+        return f'{self.dir}/{tile.z}/{tile.x}/{tile.y}.webp'
+
+    def get_tile_data(self, tile):
+        fname = self.file_from_tile(tile)
+        try:
+            os.stat(fname)
+        except FileNotFoundError:
+            raise MissingTileError()
+
+        with open(fname, 'rb') as f:
+            return f.read()
+
+    def get_tile_size(self, tile):
+        fname = self.file_from_tile(tile)
+        try:
+            fstats = os.stat(fname)
+        except FileNotFoundError:
+            raise MissingTileError()
+        return fstats.st_size
+        
+    def for_all_z(self, z):
+        fnames = glob.glob(f'{self.dir}/{z}/*/*.webp')
+        for fname in fnames:
+            tile = self.get_tile_from_file(fname)
+            fstats = os.stat(fname)
+            yield (tile, fstats.st_size)
+
+    def all(self):
+        fnames = glob.glob(f'{self.dir}/*/*/*.webp')
+        for fname in fnames:
+            tile = self.get_tile_from_file(fname)
+            t_data = Path(fname).read_bytes()
+            yield (tile, t_data)
+
+    def cleanup(self):
+        pass
+        
+class PartitionedPMTilesSource:
+    def __init__(self, pmtiles_prefix):
+        pmtiles_files = glob.glob(f'{pmtiles_prefix}*.pmtiles')
+        self.files = []
+        self.readers = {}
+        for fname in pmtiles_files:
+            file = open(fname, 'rb')
+            self.files.append(file)
+            src = MmapSource(open(fname, 'rb'))
+            reader = PMTilesReader(src)
+            self.readers[fname] = reader
+
+        self.fnames = list(pmtiles_files)
+        fname_to_index = { fname:i for i,fname in enumerate(self.fnames) }
+        self.tiles_to_suffix = {}
+        self.tiles_to_size = {}
+        for fname in self.fnames:
+            print(f'collecting tile sizes from {fname}')
+            reader = self.readers[fname]
+            for t, t_data in all_tiles(reader.get_bytes):
+                tile = mercantile.Tile(x=t[1], y=t[2], z=t[0])
+                self.tiles_to_findex[tile] = fname_to_index[fname]
+                self.tiles_to_size[tile] = len(t_data)
+
+    def get_reader_from_tile(self, tile):
+        if tile not in self.tiles_to_suffix:
+            raise MissingTileError()
+        s = self.tiles_to_findex[tile]
+        fname = self.fnames[s]
+        reader = self.readers[fname]
+        return reader
+
+
+    def get_tile_data(self, tile):
+        reader = self.get_reader_from_tile(tile)
+        data = reader.get(tile.z, tile.x, tile.y)
+        if data is None:
+            raise MissingTileError()
+        return data
+
+    def get_tile_size(self, tile):
+        if tile not in self.tiles_to_size:
+            raise MissingTileError()
+        return self.tiles_to_size[tile]
+
+    def for_all_z(self, z):
+        for tile, size in self.tiles_to_size.items():
+            if tile.z == z:
+                yield (tile, size)
+
+    def all(self):
+        for suffix, reader in self.readers.items():
+            print(f'yielding from {suffix}')
+            for t, t_data in all_tiles(reader.get_bytes):
+                tile = mercantile.Tile(x=t[1], y=t[2], z=t[0])
+                yield tile, t_data
+
+    def cleanup(self):
+        for f in self.files:
+            f.close()
+
+ 
+
+class DiskAndPartitionedPMTilesSource:
+    def __init__(self, directory, pmtiles_prefix):
+        self.dsrc = DiskSource(directory)
+        self.psrc = PartitionedPMTilesSource(pmtiles_prefix)
+        
+    def get_tile_data(self, tile):
+        try:
+            return self.dsrc.get_tile_data(tile)
+        except MissingTileError:
+            return self.psrc.get_tile_data(tile)
+
+    def get_tile_size(self, tile):
+        try:
+            return self.dsrc.get_tile_size(tile)
+        except MissingTileError:
+            return self.psrc.get_tile_size(tile)
+
+    def for_all_z(self, z):
+        print(f'iterating over all {z}')
+        seen = set()
+        for (tile, size) in self.dsrc.for_all_z(z):
+            seen.add(tile)
+            yield (tile, size)
+
+        print(f'iterated over all {z} disk')
+        for (tile, size) in self.psrc.for_all_z(z):
+            if tile in seen:
+                continue
+            yield (tile, size)
+        print(f'iterated over all {z} pmtiles')
+
+    def all(self):
+        print('yielding from disk')
+        for res in self.dsrc.all():
+            yield res
+        print('yielding from pmtiles files')
+        for res in self.psrc.all():
+            yield res
+
+    def cleanup(self):
+        self.dsrc.cleanup()
+        self.psrc.cleanup()
+
+

topo_map_processor/tools/update_bounds.py

@@ -0,0 +1,53 @@
+
+import json
+import argparse
+from pathlib import Path
+
+def cli():
+    parser = argparse.ArgumentParser(description="Update bounds in a GeoJSON file.")
+    parser.add_argument("bounds-file", help="Input GeoJSON file with bounds.")
+    parser.add_argument("bounds-dir", help="Directory containing the individual bounds files")
+    parser.add_argument("retile-list-file", help="File containing the list of sheets to update.")
+
+    args = parser.parse_args()
+
+    bounds_file = Path(args.bounds_file)
+    bounds_dir = Path(args.bounds_dir)
+    retile_list_file = Path(args.retile_list_file)
+
+    existing_map = {}
+    if not bounds_file.exists():
+        raise FileNotFoundError(f"Bounds file does not exist: {bounds_file}")
+
+    # Read the existing bounds file
+    existing_data = json.loads(bounds_file.read_text())
+    for feature in existing_data.get('features', []):
+        if 'properties' in feature and 'id' in feature['properties']:
+            sheet_name = feature['properties']['id']
+            existing_map[sheet_name] = feature
+
+    sheets = retile_list_file.read_text().strip().splitlines()
+    sheets = [s.strip() for s in sheets if s.strip()]
+    for s in sheets:
+        bfile = bounds_dir / f'{s}.geojsonl'
+        if not bfile.exists():
+            raise FileNotFoundError(f"Bounds file for sheet '{s}' does not exist: {bfile}")
+
+        feature = json.loads(bfile.read_text())
+        existing_data[s] = feature
+
+
+    all_feats = list(existing_data.values())
+
+    with open(bounds_file, 'w') as f:
+        f.write('{ "type": "FeatureCollection", "features": [\n')
+        for i,feat in enumerate(all_feats):
+            line = json.dumps(feat)
+            if i != len(all_feats) - 1:
+                line += ','
+            line += '\n'
+            f.write(line)
+        f.write(']}\n')
+
+if __name__ == "__main__":
+    cli()