pingmapper 5.5.4__tar.gz → 5.5.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {pingmapper-5.5.4 → pingmapper-5.5.6}/PKG-INFO +1 -1
  2. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_rectObj.py +7 -13
  3. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_sonObj.py +10 -0
  4. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/main_rectify.py +17 -9
  5. pingmapper-5.5.6/pingmapper/utils/kml2mosaic.py +203 -0
  6. pingmapper-5.5.6/pingmapper/utils/tile2mosaic.py +335 -0
  7. pingmapper-5.5.6/pingmapper/version.py +1 -0
  8. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/PKG-INFO +1 -1
  9. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/SOURCES.txt +2 -0
  10. pingmapper-5.5.4/pingmapper/version.py +0 -1
  11. {pingmapper-5.5.4 → pingmapper-5.5.6}/LICENSE +0 -0
  12. {pingmapper-5.5.4 → pingmapper-5.5.6}/README.md +0 -0
  13. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/__init__.py +0 -0
  14. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/__main__.py +0 -0
  15. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_mapSubstrateObj.py +0 -0
  16. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_portstarObj.py +0 -0
  17. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_sonObj_nadirgaptest.py +0 -0
  18. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/default_params.json +0 -0
  19. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/doWork.py +0 -0
  20. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/funcs_common.py +0 -0
  21. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/funcs_model.py +0 -0
  22. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/funcs_rectify.py +0 -0
  23. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/gui_main.py +0 -0
  24. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/main_mapSubstrate.py +0 -0
  25. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/main_readFiles.py +0 -0
  26. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/nonGUI_batch_main.py +0 -0
  27. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/nonGui_main.py +0 -0
  28. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/processing_scripts/main_batchDirectory_2024-01-18_0926.py +0 -0
  29. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/processing_scripts/main_batchDirectory_2024-01-18_0929.py +0 -0
  30. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/scratch/funcs_pyhum_correct.py +0 -0
  31. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/scratch/main.py +0 -0
  32. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/scratch/main_batchDirectory.py +0 -0
  33. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/self_check.py +0 -0
  34. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/test_PINGMapper.py +0 -0
  35. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/test_cli_self_check.py +0 -0
  36. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/test_dq_filter.py +0 -0
  37. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/test_time.py +0 -0
  38. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/avg_predictions_Mussel_WBL.py +0 -0
  39. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/gen_centerline.py +0 -0
  40. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/gen_centerline_from_bankline.py +0 -0
  41. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/gen_centerline_trkpnts_fitspline_DRAFT.py +0 -0
  42. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/testEXAMPLE_mosaic_logit.py +0 -0
  43. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/RawEGN_avg_predictions.py +0 -0
  44. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/00_substrate_logits_mosaic_transects.py +0 -0
  45. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/00_substrate_shps_mosaic_transects.py +0 -0
  46. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/01_gen_centerline_from_coverage.py +0 -0
  47. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/02_gen_summary_stamp_shps.py +0 -0
  48. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/03_gen_summary_shp.py +0 -0
  49. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/04_combine_summary_shp_csv.py +0 -0
  50. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/05_gen_summary_shp_plots.py +0 -0
  51. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/06_compare_raw-egn_volume.py +0 -0
  52. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/08_raw-egn_hardReacheFreq_hist.py +0 -0
  53. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/09_raw-egn_PatchSize_density.py +0 -0
  54. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/summarize_project_substrate.py +0 -0
  55. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/export_coverage.py +0 -0
  56. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/main_mosaic_transects.py +0 -0
  57. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/dependency_links.txt +0 -0
  58. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/requires.txt +0 -0
  59. {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/top_level.txt +0 -0
  60. {pingmapper-5.5.4 → pingmapper-5.5.6}/pyproject.toml +0 -0
  61. {pingmapper-5.5.4 → pingmapper-5.5.6}/setup.cfg +0 -0
  62. {pingmapper-5.5.4 → pingmapper-5.5.6}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pingmapper
3
- Version: 5.5.4
3
+ Version: 5.5.6
4
4
  Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
5
5
  Author: Cameron Bodine, Daniel Buscombe, Iury T Simoes-Sousa
6
6
  Author-email: bodine.cs@gmail.email
@@ -2185,13 +2185,10 @@ class rectObj(sonObj):
2185
2185
  )
2186
2186
  del trkMeta
2187
2187
 
2188
- # Save to shp
2189
- trkMetaShp = os.path.basename(trkMetaFile).replace('csv', 'shp')
2190
- trkMetaShp = os.path.join(outDir, trkMetaShp)
2191
- gdf.to_file(trkMetaShp)
2192
- del trkMetaFile
2188
+ gdf['project'] = os.path.basename(self.projDir)
2189
+ gdf['beam'] = self.beamName
2193
2190
 
2194
- return
2191
+ return gdf
2195
2192
 
2196
2193
  #===========================================================================
2197
2194
  def _sanitizeProjectedTrackMeta(self, trkMeta, wgs=False):
@@ -2356,7 +2353,7 @@ class rectObj(sonObj):
2356
2353
  del chunk_geom
2357
2354
 
2358
2355
  if 'gdf' not in locals() or len(gdf) == 0:
2359
- return
2356
+ return None
2360
2357
 
2361
2358
  gdf['chunk_id'] = gdf.index
2362
2359
 
@@ -2371,13 +2368,10 @@ class rectObj(sonObj):
2371
2368
 
2372
2369
  # gdf['geometry'] = gdf.buffer(-10, join_style=2)
2373
2370
 
2374
- # Save to shapefile
2375
- projName = os.path.basename(self.projDir)
2376
- outFile = os.path.join(self.metaDir, 'shapefiles', projName+"_"+beam+"_coverage.shp")
2377
- gdf.to_file(outFile)
2378
- del gdf
2371
+ gdf['project'] = os.path.basename(self.projDir)
2372
+ gdf['side'] = 'port' if beam.startswith('ss_port') else ('star' if beam.startswith('ss_star') else beam)
2379
2373
 
2380
- return
2374
+ return gdf
2381
2375
 
2382
2376
  ############################################################################
2383
2377
  # Rectify sonar imagery - Rubbersheeting #
@@ -1356,6 +1356,16 @@ class sonObj(object):
1356
1356
  arr[~valid] = 0.0
1357
1357
  return arr
1358
1358
 
1359
+ def _normalize_with_bounds(self, data, lo, hi):
1360
+ arr = np.asarray(data, dtype=np.float32)
1361
+ arr[~np.isfinite(arr)] = 0.0
1362
+
1363
+ if (not np.isfinite(lo)) or (not np.isfinite(hi)) or hi <= lo:
1364
+ return np.clip(arr, 0.0, 1.0).astype(np.float32)
1365
+
1366
+ norm = (arr - float(lo)) / (float(hi) - float(lo))
1367
+ return np.clip(norm, 0.0, 1.0).astype(np.float32)
1368
+
1359
1369
  def _apply_display_db_transform(self, sonDat):
1360
1370
  arr = np.asarray(sonDat, dtype=np.float32)
1361
1371
  if arr.size == 0:
@@ -346,17 +346,25 @@ def rectify_master_func(logfilename='',
346
346
  if coverage:
347
347
  start_time = time.time()
348
348
  print("\nExporting coverage and trackline shapefiles:\n")
349
- portstar[0]._exportTrkShp()
349
+ projName = os.path.basename(projDir)
350
+ shpDir = os.path.join(projDir, 'meta', 'shapefiles')
351
+ os.makedirs(shpDir, exist_ok=True)
350
352
 
351
- trk_files = []
352
- for son in portstar:
353
- # trk_files.append(son.smthTrkFile)
354
-
355
- son._exportCovShp()
353
+ trk_gdf = portstar[0]._exportTrkShp()
354
+ trk_gdf.to_file(os.path.join(shpDir, projName+'_trackline.shp'))
355
+ del trk_gdf
356
356
 
357
- # print(trk_files)
358
-
359
- # portstar[0]._exportCovShp(trk_files)
357
+ cov_gdfs = []
358
+ for son in portstar:
359
+ cov = son._exportCovShp()
360
+ if cov is not None:
361
+ cov_gdfs.append(cov)
362
+
363
+ if len(cov_gdfs) > 0:
364
+ cov_gdf = pd.concat(cov_gdfs, ignore_index=True)
365
+ cov_gdf.to_file(os.path.join(shpDir, projName+'_coverage.shp'))
366
+ del cov_gdf
367
+ del cov_gdfs
360
368
 
361
369
  print("Done!")
362
370
  print("Time (s):", round(time.time() - start_time, ndigits=1))
@@ -0,0 +1,203 @@
1
+ '''
2
+ Create georeferenced GeoTIFF mosaics from SonarTRX-style KML files.
3
+
4
+ The KML supplies each image tile's WGS84 LatLonBox. A VRT is generated
5
+ directly from that information, referencing the original image tiles, and
6
+ GDAL translates the VRT once to the final GeoTIFF. No per-tile conversion or
7
+ Python pixel compositing is performed.
8
+ '''
9
+
10
+ import os
11
+ import xml.etree.ElementTree as ET
12
+ from glob import glob
13
+
14
+ from osgeo import gdal
15
+ from joblib import Parallel, delayed
16
+ from tqdm import tqdm
17
+
18
+
19
+ inKmlDir = r"D:\scratch\USGS_CERC_CarpSchools_All_Tiles\All_Tiles"
20
+ inTileDir = inKmlDir
21
+ outMosaicDir = r"D:\scratch\USGS_CERC_CarpSchools_All_Mosaics"
22
+ outputFormat = 'tif'
23
+ threadCnt = 16
24
+
25
+ _KML_NS = '{http://www.opengis.net/kml/2.2}'
26
+
27
+
28
+ def _findKmls(inKmlDir: str):
29
+ return sorted(glob(os.path.join(inKmlDir, '**', '*.kml'), recursive=True))
30
+
31
+
32
+ def _parseKmlTiles(kmlPath: str, tileDir: str):
33
+ tree = ET.parse(kmlPath)
34
+ root = tree.getroot()
35
+ tiles = []
36
+
37
+ for overlay in root.iter('{}GroundOverlay'.format(_KML_NS)):
38
+ icon = overlay.find('{}Icon'.format(_KML_NS))
39
+ box = overlay.find('{}LatLonBox'.format(_KML_NS))
40
+ if icon is None or box is None:
41
+ continue
42
+
43
+ href = icon.find('{}href'.format(_KML_NS))
44
+ if href is None or not href.text:
45
+ continue
46
+
47
+ fileName = os.path.basename(href.text.replace('\\', '/'))
48
+ tilePath = os.path.join(tileDir, fileName)
49
+ if not os.path.isfile(tilePath):
50
+ continue
51
+
52
+ try:
53
+ north = float(box.find('{}north'.format(_KML_NS)).text)
54
+ south = float(box.find('{}south'.format(_KML_NS)).text)
55
+ east = float(box.find('{}east'.format(_KML_NS)).text)
56
+ west = float(box.find('{}west'.format(_KML_NS)).text)
57
+ except (AttributeError, TypeError, ValueError):
58
+ continue
59
+
60
+ if east <= west or north <= south:
61
+ continue
62
+
63
+ tiles.append({
64
+ 'path': os.path.abspath(tilePath),
65
+ 'north': north,
66
+ 'south': south,
67
+ 'east': east,
68
+ 'west': west,
69
+ })
70
+
71
+ return tiles
72
+
73
+
74
+ def _sourceInfo(tiles):
75
+ sources = []
76
+ for tile in tiles:
77
+ source = gdal.Open(tile['path'], gdal.GA_ReadOnly)
78
+ if source is None:
79
+ raise RuntimeError('GDAL could not open {}'.format(tile['path']))
80
+
81
+ sources.append({
82
+ **tile,
83
+ 'width': source.RasterXSize,
84
+ 'height': source.RasterYSize,
85
+ 'bands': source.RasterCount,
86
+ 'dataType': gdal.GetDataTypeName(source.GetRasterBand(1).DataType),
87
+ })
88
+ source = None
89
+
90
+ return sources
91
+
92
+
93
+ def _sourceElement(source, bandNumber, colOff, rowOff):
94
+ sourceElement = ET.Element('ComplexSource')
95
+ ET.SubElement(sourceElement, 'SourceFilename', {'relativeToVRT': '0'}).text = source['path']
96
+ ET.SubElement(sourceElement, 'SourceBand').text = str(bandNumber)
97
+ ET.SubElement(sourceElement, 'SourceProperties', {
98
+ 'RasterXSize': str(source['width']),
99
+ 'RasterYSize': str(source['height']),
100
+ 'DataType': source['dataType'],
101
+ 'BlockXSize': str(source['width']),
102
+ 'BlockYSize': '1',
103
+ })
104
+ ET.SubElement(sourceElement, 'SrcRect', {
105
+ 'xOff': '0',
106
+ 'yOff': '0',
107
+ 'xSize': str(source['width']),
108
+ 'ySize': str(source['height']),
109
+ })
110
+ ET.SubElement(sourceElement, 'DstRect', {
111
+ 'xOff': str(colOff),
112
+ 'yOff': str(rowOff),
113
+ 'xSize': str(source['width']),
114
+ 'ySize': str(source['height']),
115
+ })
116
+ ET.SubElement(sourceElement, 'NODATA').text = '0'
117
+ return sourceElement
118
+
119
+
120
+ def _writeVrt(vrtPath: str, sources):
121
+ minx = min(source['west'] for source in sources)
122
+ maxx = max(source['east'] for source in sources)
123
+ miny = min(source['south'] for source in sources)
124
+ maxy = max(source['north'] for source in sources)
125
+
126
+ resX = (sources[0]['east'] - sources[0]['west']) / sources[0]['width']
127
+ resY = (sources[0]['north'] - sources[0]['south']) / sources[0]['height']
128
+ width = max(1, int(round((maxx - minx) / resX)))
129
+ height = max(1, int(round((maxy - miny) / resY)))
130
+ bandCount = max(source['bands'] for source in sources)
131
+
132
+ vrt = ET.Element('VRTDataset', {
133
+ 'rasterXSize': str(width),
134
+ 'rasterYSize': str(height),
135
+ })
136
+ ET.SubElement(vrt, 'SRS').text = 'EPSG:4326'
137
+ ET.SubElement(vrt, 'GeoTransform').text = '{}, {}, 0, {}, 0, -{}'.format(minx, resX, maxy, resY)
138
+
139
+ for bandNumber in range(1, bandCount + 1):
140
+ band = ET.SubElement(vrt, 'VRTRasterBand', {
141
+ 'dataType': sources[0]['dataType'],
142
+ 'band': str(bandNumber),
143
+ })
144
+ ET.SubElement(band, 'NoDataValue').text = '0'
145
+ if bandCount == 4 and bandNumber == 4:
146
+ ET.SubElement(band, 'ColorInterp').text = 'Alpha'
147
+
148
+ for source in sources:
149
+ if bandNumber > source['bands']:
150
+ continue
151
+ colOff = int(round((source['west'] - minx) / resX))
152
+ rowOff = int(round((maxy - source['north']) / resY))
153
+ band.append(_sourceElement(source, bandNumber, colOff, rowOff))
154
+
155
+ ET.ElementTree(vrt).write(vrtPath, encoding='UTF-8', xml_declaration=True)
156
+
157
+
158
+ def buildMosaicFromKml(kmlPath: str, tileDir: str, outDir: str, outputFormat: str = 'tif'):
159
+ if outputFormat.lower().lstrip('.') not in ('tif', 'tiff'):
160
+ raise ValueError('Only GeoTIFF output is supported')
161
+
162
+ tiles = _parseKmlTiles(kmlPath, tileDir)
163
+ if not tiles:
164
+ return None
165
+
166
+ sources = _sourceInfo(tiles)
167
+ os.makedirs(outDir, exist_ok=True)
168
+ outPath = os.path.join(outDir, os.path.splitext(os.path.basename(kmlPath))[0] + '.tif')
169
+ vrtPath = os.path.splitext(outPath)[0] + '.vrt'
170
+
171
+ gdal.UseExceptions()
172
+ try:
173
+ _writeVrt(vrtPath, sources)
174
+ translated = gdal.Translate(
175
+ outPath,
176
+ vrtPath,
177
+ format='GTiff',
178
+ creationOptions=['TILED=YES', 'COMPRESS=LZW', 'BIGTIFF=IF_SAFER', 'NUM_THREADS=ALL_CPUS'],
179
+ )
180
+ if translated is None:
181
+ raise RuntimeError('GDAL failed to translate {}'.format(vrtPath))
182
+ translated = None
183
+ return outPath
184
+ finally:
185
+ if os.path.exists(vrtPath):
186
+ os.remove(vrtPath)
187
+
188
+
189
+ def doWork():
190
+ os.makedirs(outMosaicDir, exist_ok=True)
191
+ kmlFiles = _findKmls(inKmlDir)
192
+ print('Found {} KML(s) under {}'.format(len(kmlFiles), inKmlDir))
193
+ results = list(Parallel(n_jobs=threadCnt)(
194
+ delayed(buildMosaicFromKml)(kmlPath, inTileDir, outMosaicDir, outputFormat)
195
+ for kmlPath in tqdm(kmlFiles)
196
+ ))
197
+ results = [result for result in results if result]
198
+ print('Wrote {} mosaic(s) to {}'.format(len(results), outMosaicDir))
199
+ return results
200
+
201
+
202
+ if __name__ == '__main__':
203
+ doWork()
@@ -0,0 +1,335 @@
1
+ '''
2
+ Group tiles (recursively found under inTileDir) into mosaics by matching
3
+ components of their filenames, and combine overlapping pixels using
4
+ overlap_method ('average', 'max', or 'min').
5
+
6
+ Mosaicking uses rasterio + numpy in fixed-size blocks instead of GDAL's
7
+ BuildVRT + Python VRT pixel-function approach. VRT pixel functions invoke a
8
+ Python callback per raster block during gdal.Translate (slow), and naively
9
+ accumulating whole-mosaic numpy arrays in memory can OOM on large mosaics.
10
+ Processing/writing one block at a time keeps peak memory bounded by
11
+ blockSize regardless of mosaic extent.
12
+ '''
13
+
14
+ #########
15
+ # Imports
16
+
17
+ import os
18
+ import re
19
+ import warnings
20
+ from glob import glob
21
+
22
+ import numpy as np
23
+ import rasterio
24
+ from rasterio.windows import Window
25
+ from rasterio.transform import from_origin
26
+ from joblib import Parallel, delayed, cpu_count
27
+ from tqdm import tqdm
28
+
29
+
30
+ ############
31
+ # Parameters
32
+
33
+ inTileDir = r"D:\scratch\USGS_CERC_CarpSchools_All_Tiles\All_Tiles"
34
+
35
+ # File pattern with delimiter
36
+ delimiter = "-"
37
+ filePattern = delimiter.join(["{recording}", "{tile}"]) + ".png"
38
+ mosaicMatchPattern = delimiter.join(["{recording}"]) + ".png"
39
+
40
+ outMosaicDir = r"D:\scratch\USGS_CERC_CarpSchools_All_Mosaics"
41
+ overlap_method = "average" # Options: 'average', 'max', 'min'
42
+
43
+ outputFormat = "tif" # Output mosaic file extension/format; independent of input tile extension
44
+
45
+ force_greyscale = True # Force output to be greyscale (single band) if True, else keep original bands
46
+ tiles_overlap = True # Set False when tile footprints are guaranteed not to overlap
47
+
48
+ threadCnt = cpu_count() # Number of mosaics to build in parallel
49
+
50
+ blockSize = 2048 # Block edge (px) processed/written at a time; bounds peak memory use
51
+
52
+
53
+ ###########
54
+ # Functions
55
+
56
+ # GDAL driver for each supported output extension; GTiff is the only one
57
+ # validated for windowed/blockwise writes used by buildMosaic.
58
+ _DRIVER_BY_EXT = {
59
+ 'tif': 'GTiff',
60
+ 'tiff': 'GTiff',
61
+ }
62
+
63
+
64
+ def _driverForFormat(outputFormat: str):
65
+ ext = outputFormat.lower().lstrip('.')
66
+ if ext not in _DRIVER_BY_EXT:
67
+ raise ValueError("Unsupported outputFormat '{}'; supported: {}".format(outputFormat, sorted(_DRIVER_BY_EXT)))
68
+ return ext, _DRIVER_BY_EXT[ext]
69
+
70
+
71
+ def _build_pattern_regex(filePattern: str, delimiter: str):
72
+ '''
73
+ Convert a delimiter-based file pattern (e.g. "{recording}_{tile}.tif")
74
+ into a compiled regex with named capture groups.
75
+ '''
76
+
77
+ fieldNames = re.findall(r'\{(\w+)\}', filePattern)
78
+
79
+ escaped = re.escape(filePattern)
80
+ for name in fieldNames:
81
+ placeholder = re.escape('{' + name + '}')
82
+ group = '(?P<{}>[^{}]+)'.format(name, re.escape(delimiter))
83
+ escaped = escaped.replace(placeholder, group)
84
+
85
+ return re.compile('^' + escaped + '$'), fieldNames
86
+
87
+
88
+ def _findTiles(inTileDir: str, ext: str):
89
+ '''Recursively find all tile files under inTileDir with extension ext.'''
90
+
91
+ return sorted(glob(os.path.join(inTileDir, '**', '*' + ext), recursive=True))
92
+
93
+
94
+ def _groupTiles(tileFiles: list, regex, mosaicMatchPattern: str):
95
+ '''
96
+ Group tile file paths by the mosaic key derived from mosaicMatchPattern,
97
+ using the named fields extracted from each filename via regex.
98
+ '''
99
+
100
+ groups = {}
101
+ skipped = []
102
+
103
+ for f in tileFiles:
104
+ m = regex.match(os.path.basename(f))
105
+ if not m:
106
+ skipped.append(f)
107
+ continue
108
+
109
+ fields = m.groupdict()
110
+ mosaicName = mosaicMatchPattern.format(**fields)
111
+ groups.setdefault(mosaicName, []).append(f)
112
+
113
+ return groups, skipped
114
+
115
+
116
+ def _toGrey(arr: np.ndarray) -> np.ndarray:
117
+ '''Convert a (bands, H, W) array to single-band greyscale using luma weights.'''
118
+
119
+ if arr.shape[0] == 1:
120
+ return arr[0:1]
121
+
122
+ if arr.shape[0] >= 3:
123
+ r = arr[0].astype(np.float64)
124
+ g = arr[1].astype(np.float64)
125
+ b = arr[2].astype(np.float64)
126
+ grey = 0.299 * r + 0.587 * g + 0.114 * b
127
+ return grey[np.newaxis, ...]
128
+
129
+ # 2-band fallback
130
+ return arr.mean(axis=0, keepdims=True)
131
+
132
+
133
+ def buildMosaic(mosaicName: str,
134
+ tileList: list,
135
+ outDir: str,
136
+ overlap_method: str,
137
+ force_greyscale: bool,
138
+ outputFormat: str = 'tif',
139
+ blockSize: int = 2048,
140
+ tiles_overlap: bool = True):
141
+ '''
142
+ Merge tileList into a single mosaic GeoTIFF, combining overlapping
143
+ pixels per overlap_method. Processed and written one block at a time
144
+ (rasterio/numpy only, no GDAL VRT/pixel functions) so peak memory is
145
+ bounded by blockSize instead of the full mosaic extent.
146
+ '''
147
+
148
+ with warnings.catch_warnings():
149
+ warnings.filterwarnings('ignore', category=rasterio.errors.NotGeoreferencedWarning)
150
+ srcs = [rasterio.open(f) for f in tileList]
151
+
152
+ try:
153
+ ref = srcs[0]
154
+ resX, resY = ref.res
155
+ crs = ref.crs
156
+ dtype = ref.dtypes[0]
157
+ nodata = ref.nodata if ref.nodata is not None else 0
158
+ bandCnt = 1 if force_greyscale else ref.count
159
+
160
+ lefts, bottoms, rights, tops = zip(*(s.bounds for s in srcs))
161
+ minx, miny, maxx, maxy = min(lefts), min(bottoms), max(rights), max(tops)
162
+
163
+ width = max(1, int(round((maxx - minx) / resX)))
164
+ height = max(1, int(round((maxy - miny) / resY)))
165
+ outTransform = from_origin(minx, maxy, resX, resY)
166
+
167
+ if overlap_method not in ('average', 'max', 'min'):
168
+ raise ValueError("overlap_method must be 'average', 'max', or 'min'; got '{}'".format(overlap_method))
169
+
170
+ # Precompute each src's pixel offset/size within the full output grid
171
+ srcInfo = []
172
+ for s in srcs:
173
+ colOff = int(round((s.bounds.left - minx) / resX))
174
+ rowOff = int(round((maxy - s.bounds.top) / resY))
175
+ srcInfo.append((s, colOff, rowOff, s.width, s.height))
176
+
177
+ blockCols = (width + blockSize - 1) // blockSize
178
+ blockRows = (height + blockSize - 1) // blockSize
179
+ blockSources = [[] for _ in range(blockRows * blockCols)]
180
+ for sourceIndex, (_, colOff, rowOff, sWidth, sHeight) in enumerate(srcInfo):
181
+ firstBlockRow = max(0, rowOff // blockSize)
182
+ lastBlockRow = min(blockRows - 1, (rowOff + sHeight - 1) // blockSize)
183
+ firstBlockCol = max(0, colOff // blockSize)
184
+ lastBlockCol = min(blockCols - 1, (colOff + sWidth - 1) // blockSize)
185
+ for blockRow in range(firstBlockRow, lastBlockRow + 1):
186
+ for blockCol in range(firstBlockCol, lastBlockCol + 1):
187
+ blockSources[blockRow * blockCols + blockCol].append(sourceIndex)
188
+
189
+ # Internal GTiff block size must be a multiple of 16 and not exceed raster dims
190
+ def _tiffBlockDim(dimSize):
191
+ dim = min(blockSize, dimSize)
192
+ dim = max(16, (dim // 16) * 16)
193
+ return dim
194
+
195
+ outExt, outDriver = _driverForFormat(outputFormat)
196
+ outName = os.path.splitext(mosaicName)[0] + '.' + outExt
197
+ outPath = os.path.join(outDir, outName)
198
+ profile = {
199
+ 'driver': outDriver,
200
+ 'height': height,
201
+ 'width': width,
202
+ 'count': bandCnt,
203
+ 'dtype': dtype,
204
+ 'crs': crs,
205
+ 'transform': outTransform,
206
+ 'nodata': nodata,
207
+ 'compress': 'lzw',
208
+ 'tiled': True,
209
+ 'blockxsize': _tiffBlockDim(width),
210
+ 'blockysize': _tiffBlockDim(height),
211
+ 'BIGTIFF': 'IF_SAFER',
212
+ }
213
+
214
+ with rasterio.open(outPath, 'w', **profile) as dst:
215
+ if not tiles_overlap:
216
+ for s, colOff, rowOff, sWidth, sHeight in srcInfo:
217
+ data = s.read()
218
+ if force_greyscale:
219
+ data = _toGrey(data.astype(np.float64))
220
+ dst.write(
221
+ data.astype(dtype),
222
+ window=Window(col_off=colOff, row_off=rowOff, width=sWidth, height=sHeight),
223
+ )
224
+ return outPath
225
+
226
+ for rowStart in range(0, height, blockSize):
227
+ bh = min(blockSize, height - rowStart)
228
+ for colStart in range(0, width, blockSize):
229
+ bw = min(blockSize, width - colStart)
230
+
231
+ if overlap_method == 'average':
232
+ sumBlock = np.zeros((bandCnt, bh, bw), dtype=np.float64)
233
+ cntBlock = np.zeros((bandCnt, bh, bw), dtype=np.uint32)
234
+ else:
235
+ outBlock = np.full((bandCnt, bh, bw), nodata, dtype=np.float64)
236
+ hasBlock = np.zeros((bandCnt, bh, bw), dtype=bool)
237
+
238
+ blockIndex = (rowStart // blockSize) * blockCols + (colStart // blockSize)
239
+ for sourceIndex in blockSources[blockIndex]:
240
+ s, colOff, rowOff, sWidth, sHeight = srcInfo[sourceIndex]
241
+ # Intersection of this output block with the src's footprint
242
+ interRow0 = max(rowStart, rowOff)
243
+ interRow1 = min(rowStart + bh, rowOff + sHeight)
244
+ interCol0 = max(colStart, colOff)
245
+ interCol1 = min(colStart + bw, colOff + sWidth)
246
+
247
+ if interRow0 >= interRow1 or interCol0 >= interCol1:
248
+ continue
249
+
250
+ srcWin = Window(
251
+ col_off=interCol0 - colOff,
252
+ row_off=interRow0 - rowOff,
253
+ width=interCol1 - interCol0,
254
+ height=interRow1 - interRow0,
255
+ )
256
+ data = s.read(window=srcWin, masked=True)
257
+
258
+ if force_greyscale:
259
+ filled = data.filled(nodata).astype(np.float64)
260
+ grey = _toGrey(filled)
261
+ validMask = np.any(~np.ma.getmaskarray(data), axis=0, keepdims=True)
262
+ validMask &= np.any(data.data != nodata, axis=0, keepdims=True)
263
+ filled = grey
264
+ valid = np.broadcast_to(validMask, grey.shape)
265
+ else:
266
+ filled = data.filled(nodata).astype(np.float64)
267
+ valid = (~np.ma.getmaskarray(data) & (data.data != nodata))
268
+
269
+ bRow0 = interRow0 - rowStart
270
+ bCol0 = interCol0 - colStart
271
+ h, w = filled.shape[1], filled.shape[2]
272
+
273
+ if overlap_method == 'average':
274
+ sumBlock[:, bRow0:bRow0 + h, bCol0:bCol0 + w] += np.where(valid, filled, 0)
275
+ cntBlock[:, bRow0:bRow0 + h, bCol0:bCol0 + w] += valid.astype(np.uint32)
276
+ else:
277
+ region = outBlock[:, bRow0:bRow0 + h, bCol0:bCol0 + w]
278
+ regionHas = hasBlock[:, bRow0:bRow0 + h, bCol0:bCol0 + w]
279
+
280
+ if overlap_method == 'max':
281
+ take = valid & (~regionHas | (filled > region))
282
+ else: # min
283
+ take = valid & (~regionHas | (filled < region))
284
+
285
+ region[take] = filled[take]
286
+ regionHas |= valid
287
+
288
+ if overlap_method == 'average':
289
+ with np.errstate(invalid='ignore', divide='ignore'):
290
+ outBlock = np.where(cntBlock > 0, sumBlock / np.maximum(cntBlock, 1), nodata)
291
+
292
+ dst.write(
293
+ outBlock.astype(dtype),
294
+ window=Window(col_off=colStart, row_off=rowStart, width=bw, height=bh),
295
+ )
296
+
297
+ return outPath
298
+
299
+ finally:
300
+ for s in srcs:
301
+ if not s.closed:
302
+ s.close()
303
+
304
+
305
+ def doWork():
306
+
307
+ os.makedirs(outMosaicDir, exist_ok=True)
308
+
309
+ ext = os.path.splitext(filePattern)[-1]
310
+ regex, _ = _build_pattern_regex(filePattern, delimiter)
311
+
312
+ tileFiles = _findTiles(inTileDir, ext)
313
+ print('Found {} tile(s) under {}'.format(len(tileFiles), inTileDir))
314
+
315
+ groups, skipped = _groupTiles(tileFiles, regex, mosaicMatchPattern)
316
+ if skipped:
317
+ print('Skipped {} file(s) not matching filePattern.'.format(len(skipped)))
318
+
319
+ print('Grouped into {} mosaic(s).'.format(len(groups)))
320
+
321
+ results = list(Parallel(n_jobs=threadCnt)(
322
+ delayed(buildMosaic)(mosaicName, tileList, outMosaicDir, overlap_method, force_greyscale, outputFormat, blockSize, tiles_overlap)
323
+ for mosaicName, tileList in tqdm(groups.items(), total=len(groups))
324
+ ))
325
+
326
+ print('Wrote {} mosaic(s) to {}'.format(len(results), outMosaicDir))
327
+
328
+ return results
329
+
330
+
331
+ #########
332
+ # Do Work
333
+
334
+ if __name__ == '__main__':
335
+ doWork()
@@ -0,0 +1 @@
1
+ __version__ = '5.5.6'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pingmapper
3
- Version: 5.5.4
3
+ Version: 5.5.6
4
4
  Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
5
5
  Author: Cameron Bodine, Daniel Buscombe, Iury T Simoes-Sousa
6
6
  Author-email: bodine.cs@gmail.email
@@ -38,7 +38,9 @@ pingmapper/scratch/main.py
38
38
  pingmapper/scratch/main_batchDirectory.py
39
39
  pingmapper/utils/RawEGN_avg_predictions.py
40
40
  pingmapper/utils/export_coverage.py
41
+ pingmapper/utils/kml2mosaic.py
41
42
  pingmapper/utils/main_mosaic_transects.py
43
+ pingmapper/utils/tile2mosaic.py
42
44
  pingmapper/utils/DRAFT_Workflows/avg_predictions_Mussel_WBL.py
43
45
  pingmapper/utils/DRAFT_Workflows/gen_centerline.py
44
46
  pingmapper/utils/DRAFT_Workflows/gen_centerline_from_bankline.py
@@ -1 +0,0 @@
1
- __version__ = '5.5.4'
File without changes
File without changes
File without changes
File without changes
File without changes