pingmapper 5.5.4__tar.gz → 5.5.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pingmapper-5.5.4 → pingmapper-5.5.6}/PKG-INFO +1 -1
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_rectObj.py +7 -13
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_sonObj.py +10 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/main_rectify.py +17 -9
- pingmapper-5.5.6/pingmapper/utils/kml2mosaic.py +203 -0
- pingmapper-5.5.6/pingmapper/utils/tile2mosaic.py +335 -0
- pingmapper-5.5.6/pingmapper/version.py +1 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/PKG-INFO +1 -1
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/SOURCES.txt +2 -0
- pingmapper-5.5.4/pingmapper/version.py +0 -1
- {pingmapper-5.5.4 → pingmapper-5.5.6}/LICENSE +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/README.md +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/__init__.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/__main__.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_mapSubstrateObj.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_portstarObj.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/class_sonObj_nadirgaptest.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/default_params.json +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/doWork.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/funcs_common.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/funcs_model.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/funcs_rectify.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/gui_main.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/main_mapSubstrate.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/main_readFiles.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/nonGUI_batch_main.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/nonGui_main.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/processing_scripts/main_batchDirectory_2024-01-18_0926.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/processing_scripts/main_batchDirectory_2024-01-18_0929.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/scratch/funcs_pyhum_correct.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/scratch/main.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/scratch/main_batchDirectory.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/self_check.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/test_PINGMapper.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/test_cli_self_check.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/test_dq_filter.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/test_time.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/avg_predictions_Mussel_WBL.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/gen_centerline.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/gen_centerline_from_bankline.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/gen_centerline_trkpnts_fitspline_DRAFT.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/testEXAMPLE_mosaic_logit.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/RawEGN_avg_predictions.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/00_substrate_logits_mosaic_transects.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/00_substrate_shps_mosaic_transects.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/01_gen_centerline_from_coverage.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/02_gen_summary_stamp_shps.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/03_gen_summary_shp.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/04_combine_summary_shp_csv.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/05_gen_summary_shp_plots.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/06_compare_raw-egn_volume.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/08_raw-egn_hardReacheFreq_hist.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/09_raw-egn_PatchSize_density.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/summarize_project_substrate.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/export_coverage.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/main_mosaic_transects.py +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/dependency_links.txt +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/requires.txt +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper.egg-info/top_level.txt +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/pyproject.toml +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/setup.cfg +0 -0
- {pingmapper-5.5.4 → pingmapper-5.5.6}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pingmapper
|
|
3
|
-
Version: 5.5.
|
|
3
|
+
Version: 5.5.6
|
|
4
4
|
Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
|
|
5
5
|
Author: Cameron Bodine, Daniel Buscombe, Iury T Simoes-Sousa
|
|
6
6
|
Author-email: bodine.cs@gmail.email
|
|
@@ -2185,13 +2185,10 @@ class rectObj(sonObj):
|
|
|
2185
2185
|
)
|
|
2186
2186
|
del trkMeta
|
|
2187
2187
|
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
trkMetaShp = os.path.join(outDir, trkMetaShp)
|
|
2191
|
-
gdf.to_file(trkMetaShp)
|
|
2192
|
-
del trkMetaFile
|
|
2188
|
+
gdf['project'] = os.path.basename(self.projDir)
|
|
2189
|
+
gdf['beam'] = self.beamName
|
|
2193
2190
|
|
|
2194
|
-
return
|
|
2191
|
+
return gdf
|
|
2195
2192
|
|
|
2196
2193
|
#===========================================================================
|
|
2197
2194
|
def _sanitizeProjectedTrackMeta(self, trkMeta, wgs=False):
|
|
@@ -2356,7 +2353,7 @@ class rectObj(sonObj):
|
|
|
2356
2353
|
del chunk_geom
|
|
2357
2354
|
|
|
2358
2355
|
if 'gdf' not in locals() or len(gdf) == 0:
|
|
2359
|
-
return
|
|
2356
|
+
return None
|
|
2360
2357
|
|
|
2361
2358
|
gdf['chunk_id'] = gdf.index
|
|
2362
2359
|
|
|
@@ -2371,13 +2368,10 @@ class rectObj(sonObj):
|
|
|
2371
2368
|
|
|
2372
2369
|
# gdf['geometry'] = gdf.buffer(-10, join_style=2)
|
|
2373
2370
|
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
outFile = os.path.join(self.metaDir, 'shapefiles', projName+"_"+beam+"_coverage.shp")
|
|
2377
|
-
gdf.to_file(outFile)
|
|
2378
|
-
del gdf
|
|
2371
|
+
gdf['project'] = os.path.basename(self.projDir)
|
|
2372
|
+
gdf['side'] = 'port' if beam.startswith('ss_port') else ('star' if beam.startswith('ss_star') else beam)
|
|
2379
2373
|
|
|
2380
|
-
return
|
|
2374
|
+
return gdf
|
|
2381
2375
|
|
|
2382
2376
|
############################################################################
|
|
2383
2377
|
# Rectify sonar imagery - Rubbersheeting #
|
|
@@ -1356,6 +1356,16 @@ class sonObj(object):
|
|
|
1356
1356
|
arr[~valid] = 0.0
|
|
1357
1357
|
return arr
|
|
1358
1358
|
|
|
1359
|
+
def _normalize_with_bounds(self, data, lo, hi):
|
|
1360
|
+
arr = np.asarray(data, dtype=np.float32)
|
|
1361
|
+
arr[~np.isfinite(arr)] = 0.0
|
|
1362
|
+
|
|
1363
|
+
if (not np.isfinite(lo)) or (not np.isfinite(hi)) or hi <= lo:
|
|
1364
|
+
return np.clip(arr, 0.0, 1.0).astype(np.float32)
|
|
1365
|
+
|
|
1366
|
+
norm = (arr - float(lo)) / (float(hi) - float(lo))
|
|
1367
|
+
return np.clip(norm, 0.0, 1.0).astype(np.float32)
|
|
1368
|
+
|
|
1359
1369
|
def _apply_display_db_transform(self, sonDat):
|
|
1360
1370
|
arr = np.asarray(sonDat, dtype=np.float32)
|
|
1361
1371
|
if arr.size == 0:
|
|
@@ -346,17 +346,25 @@ def rectify_master_func(logfilename='',
|
|
|
346
346
|
if coverage:
|
|
347
347
|
start_time = time.time()
|
|
348
348
|
print("\nExporting coverage and trackline shapefiles:\n")
|
|
349
|
-
|
|
349
|
+
projName = os.path.basename(projDir)
|
|
350
|
+
shpDir = os.path.join(projDir, 'meta', 'shapefiles')
|
|
351
|
+
os.makedirs(shpDir, exist_ok=True)
|
|
350
352
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
son._exportCovShp()
|
|
353
|
+
trk_gdf = portstar[0]._exportTrkShp()
|
|
354
|
+
trk_gdf.to_file(os.path.join(shpDir, projName+'_trackline.shp'))
|
|
355
|
+
del trk_gdf
|
|
356
356
|
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
357
|
+
cov_gdfs = []
|
|
358
|
+
for son in portstar:
|
|
359
|
+
cov = son._exportCovShp()
|
|
360
|
+
if cov is not None:
|
|
361
|
+
cov_gdfs.append(cov)
|
|
362
|
+
|
|
363
|
+
if len(cov_gdfs) > 0:
|
|
364
|
+
cov_gdf = pd.concat(cov_gdfs, ignore_index=True)
|
|
365
|
+
cov_gdf.to_file(os.path.join(shpDir, projName+'_coverage.shp'))
|
|
366
|
+
del cov_gdf
|
|
367
|
+
del cov_gdfs
|
|
360
368
|
|
|
361
369
|
print("Done!")
|
|
362
370
|
print("Time (s):", round(time.time() - start_time, ndigits=1))
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Create georeferenced GeoTIFF mosaics from SonarTRX-style KML files.
|
|
3
|
+
|
|
4
|
+
The KML supplies each image tile's WGS84 LatLonBox. A VRT is generated
|
|
5
|
+
directly from that information, referencing the original image tiles, and
|
|
6
|
+
GDAL translates the VRT once to the final GeoTIFF. No per-tile conversion or
|
|
7
|
+
Python pixel compositing is performed.
|
|
8
|
+
'''
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import xml.etree.ElementTree as ET
|
|
12
|
+
from glob import glob
|
|
13
|
+
|
|
14
|
+
from osgeo import gdal
|
|
15
|
+
from joblib import Parallel, delayed
|
|
16
|
+
from tqdm import tqdm
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
inKmlDir = r"D:\scratch\USGS_CERC_CarpSchools_All_Tiles\All_Tiles"
|
|
20
|
+
inTileDir = inKmlDir
|
|
21
|
+
outMosaicDir = r"D:\scratch\USGS_CERC_CarpSchools_All_Mosaics"
|
|
22
|
+
outputFormat = 'tif'
|
|
23
|
+
threadCnt = 16
|
|
24
|
+
|
|
25
|
+
_KML_NS = '{http://www.opengis.net/kml/2.2}'
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _findKmls(inKmlDir: str):
|
|
29
|
+
return sorted(glob(os.path.join(inKmlDir, '**', '*.kml'), recursive=True))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _parseKmlTiles(kmlPath: str, tileDir: str):
|
|
33
|
+
tree = ET.parse(kmlPath)
|
|
34
|
+
root = tree.getroot()
|
|
35
|
+
tiles = []
|
|
36
|
+
|
|
37
|
+
for overlay in root.iter('{}GroundOverlay'.format(_KML_NS)):
|
|
38
|
+
icon = overlay.find('{}Icon'.format(_KML_NS))
|
|
39
|
+
box = overlay.find('{}LatLonBox'.format(_KML_NS))
|
|
40
|
+
if icon is None or box is None:
|
|
41
|
+
continue
|
|
42
|
+
|
|
43
|
+
href = icon.find('{}href'.format(_KML_NS))
|
|
44
|
+
if href is None or not href.text:
|
|
45
|
+
continue
|
|
46
|
+
|
|
47
|
+
fileName = os.path.basename(href.text.replace('\\', '/'))
|
|
48
|
+
tilePath = os.path.join(tileDir, fileName)
|
|
49
|
+
if not os.path.isfile(tilePath):
|
|
50
|
+
continue
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
north = float(box.find('{}north'.format(_KML_NS)).text)
|
|
54
|
+
south = float(box.find('{}south'.format(_KML_NS)).text)
|
|
55
|
+
east = float(box.find('{}east'.format(_KML_NS)).text)
|
|
56
|
+
west = float(box.find('{}west'.format(_KML_NS)).text)
|
|
57
|
+
except (AttributeError, TypeError, ValueError):
|
|
58
|
+
continue
|
|
59
|
+
|
|
60
|
+
if east <= west or north <= south:
|
|
61
|
+
continue
|
|
62
|
+
|
|
63
|
+
tiles.append({
|
|
64
|
+
'path': os.path.abspath(tilePath),
|
|
65
|
+
'north': north,
|
|
66
|
+
'south': south,
|
|
67
|
+
'east': east,
|
|
68
|
+
'west': west,
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
return tiles
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _sourceInfo(tiles):
|
|
75
|
+
sources = []
|
|
76
|
+
for tile in tiles:
|
|
77
|
+
source = gdal.Open(tile['path'], gdal.GA_ReadOnly)
|
|
78
|
+
if source is None:
|
|
79
|
+
raise RuntimeError('GDAL could not open {}'.format(tile['path']))
|
|
80
|
+
|
|
81
|
+
sources.append({
|
|
82
|
+
**tile,
|
|
83
|
+
'width': source.RasterXSize,
|
|
84
|
+
'height': source.RasterYSize,
|
|
85
|
+
'bands': source.RasterCount,
|
|
86
|
+
'dataType': gdal.GetDataTypeName(source.GetRasterBand(1).DataType),
|
|
87
|
+
})
|
|
88
|
+
source = None
|
|
89
|
+
|
|
90
|
+
return sources
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _sourceElement(source, bandNumber, colOff, rowOff):
|
|
94
|
+
sourceElement = ET.Element('ComplexSource')
|
|
95
|
+
ET.SubElement(sourceElement, 'SourceFilename', {'relativeToVRT': '0'}).text = source['path']
|
|
96
|
+
ET.SubElement(sourceElement, 'SourceBand').text = str(bandNumber)
|
|
97
|
+
ET.SubElement(sourceElement, 'SourceProperties', {
|
|
98
|
+
'RasterXSize': str(source['width']),
|
|
99
|
+
'RasterYSize': str(source['height']),
|
|
100
|
+
'DataType': source['dataType'],
|
|
101
|
+
'BlockXSize': str(source['width']),
|
|
102
|
+
'BlockYSize': '1',
|
|
103
|
+
})
|
|
104
|
+
ET.SubElement(sourceElement, 'SrcRect', {
|
|
105
|
+
'xOff': '0',
|
|
106
|
+
'yOff': '0',
|
|
107
|
+
'xSize': str(source['width']),
|
|
108
|
+
'ySize': str(source['height']),
|
|
109
|
+
})
|
|
110
|
+
ET.SubElement(sourceElement, 'DstRect', {
|
|
111
|
+
'xOff': str(colOff),
|
|
112
|
+
'yOff': str(rowOff),
|
|
113
|
+
'xSize': str(source['width']),
|
|
114
|
+
'ySize': str(source['height']),
|
|
115
|
+
})
|
|
116
|
+
ET.SubElement(sourceElement, 'NODATA').text = '0'
|
|
117
|
+
return sourceElement
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _writeVrt(vrtPath: str, sources):
|
|
121
|
+
minx = min(source['west'] for source in sources)
|
|
122
|
+
maxx = max(source['east'] for source in sources)
|
|
123
|
+
miny = min(source['south'] for source in sources)
|
|
124
|
+
maxy = max(source['north'] for source in sources)
|
|
125
|
+
|
|
126
|
+
resX = (sources[0]['east'] - sources[0]['west']) / sources[0]['width']
|
|
127
|
+
resY = (sources[0]['north'] - sources[0]['south']) / sources[0]['height']
|
|
128
|
+
width = max(1, int(round((maxx - minx) / resX)))
|
|
129
|
+
height = max(1, int(round((maxy - miny) / resY)))
|
|
130
|
+
bandCount = max(source['bands'] for source in sources)
|
|
131
|
+
|
|
132
|
+
vrt = ET.Element('VRTDataset', {
|
|
133
|
+
'rasterXSize': str(width),
|
|
134
|
+
'rasterYSize': str(height),
|
|
135
|
+
})
|
|
136
|
+
ET.SubElement(vrt, 'SRS').text = 'EPSG:4326'
|
|
137
|
+
ET.SubElement(vrt, 'GeoTransform').text = '{}, {}, 0, {}, 0, -{}'.format(minx, resX, maxy, resY)
|
|
138
|
+
|
|
139
|
+
for bandNumber in range(1, bandCount + 1):
|
|
140
|
+
band = ET.SubElement(vrt, 'VRTRasterBand', {
|
|
141
|
+
'dataType': sources[0]['dataType'],
|
|
142
|
+
'band': str(bandNumber),
|
|
143
|
+
})
|
|
144
|
+
ET.SubElement(band, 'NoDataValue').text = '0'
|
|
145
|
+
if bandCount == 4 and bandNumber == 4:
|
|
146
|
+
ET.SubElement(band, 'ColorInterp').text = 'Alpha'
|
|
147
|
+
|
|
148
|
+
for source in sources:
|
|
149
|
+
if bandNumber > source['bands']:
|
|
150
|
+
continue
|
|
151
|
+
colOff = int(round((source['west'] - minx) / resX))
|
|
152
|
+
rowOff = int(round((maxy - source['north']) / resY))
|
|
153
|
+
band.append(_sourceElement(source, bandNumber, colOff, rowOff))
|
|
154
|
+
|
|
155
|
+
ET.ElementTree(vrt).write(vrtPath, encoding='UTF-8', xml_declaration=True)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def buildMosaicFromKml(kmlPath: str, tileDir: str, outDir: str, outputFormat: str = 'tif'):
|
|
159
|
+
if outputFormat.lower().lstrip('.') not in ('tif', 'tiff'):
|
|
160
|
+
raise ValueError('Only GeoTIFF output is supported')
|
|
161
|
+
|
|
162
|
+
tiles = _parseKmlTiles(kmlPath, tileDir)
|
|
163
|
+
if not tiles:
|
|
164
|
+
return None
|
|
165
|
+
|
|
166
|
+
sources = _sourceInfo(tiles)
|
|
167
|
+
os.makedirs(outDir, exist_ok=True)
|
|
168
|
+
outPath = os.path.join(outDir, os.path.splitext(os.path.basename(kmlPath))[0] + '.tif')
|
|
169
|
+
vrtPath = os.path.splitext(outPath)[0] + '.vrt'
|
|
170
|
+
|
|
171
|
+
gdal.UseExceptions()
|
|
172
|
+
try:
|
|
173
|
+
_writeVrt(vrtPath, sources)
|
|
174
|
+
translated = gdal.Translate(
|
|
175
|
+
outPath,
|
|
176
|
+
vrtPath,
|
|
177
|
+
format='GTiff',
|
|
178
|
+
creationOptions=['TILED=YES', 'COMPRESS=LZW', 'BIGTIFF=IF_SAFER', 'NUM_THREADS=ALL_CPUS'],
|
|
179
|
+
)
|
|
180
|
+
if translated is None:
|
|
181
|
+
raise RuntimeError('GDAL failed to translate {}'.format(vrtPath))
|
|
182
|
+
translated = None
|
|
183
|
+
return outPath
|
|
184
|
+
finally:
|
|
185
|
+
if os.path.exists(vrtPath):
|
|
186
|
+
os.remove(vrtPath)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def doWork():
|
|
190
|
+
os.makedirs(outMosaicDir, exist_ok=True)
|
|
191
|
+
kmlFiles = _findKmls(inKmlDir)
|
|
192
|
+
print('Found {} KML(s) under {}'.format(len(kmlFiles), inKmlDir))
|
|
193
|
+
results = list(Parallel(n_jobs=threadCnt)(
|
|
194
|
+
delayed(buildMosaicFromKml)(kmlPath, inTileDir, outMosaicDir, outputFormat)
|
|
195
|
+
for kmlPath in tqdm(kmlFiles)
|
|
196
|
+
))
|
|
197
|
+
results = [result for result in results if result]
|
|
198
|
+
print('Wrote {} mosaic(s) to {}'.format(len(results), outMosaicDir))
|
|
199
|
+
return results
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
if __name__ == '__main__':
|
|
203
|
+
doWork()
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Group tiles (recursively found under inTileDir) into mosaics by matching
|
|
3
|
+
components of their filenames, and combine overlapping pixels using
|
|
4
|
+
overlap_method ('average', 'max', or 'min').
|
|
5
|
+
|
|
6
|
+
Mosaicking uses rasterio + numpy in fixed-size blocks instead of GDAL's
|
|
7
|
+
BuildVRT + Python VRT pixel-function approach. VRT pixel functions invoke a
|
|
8
|
+
Python callback per raster block during gdal.Translate (slow), and naively
|
|
9
|
+
accumulating whole-mosaic numpy arrays in memory can OOM on large mosaics.
|
|
10
|
+
Processing/writing one block at a time keeps peak memory bounded by
|
|
11
|
+
blockSize regardless of mosaic extent.
|
|
12
|
+
'''
|
|
13
|
+
|
|
14
|
+
#########
|
|
15
|
+
# Imports
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import warnings
|
|
20
|
+
from glob import glob
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
import rasterio
|
|
24
|
+
from rasterio.windows import Window
|
|
25
|
+
from rasterio.transform import from_origin
|
|
26
|
+
from joblib import Parallel, delayed, cpu_count
|
|
27
|
+
from tqdm import tqdm
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
############
|
|
31
|
+
# Parameters
|
|
32
|
+
|
|
33
|
+
inTileDir = r"D:\scratch\USGS_CERC_CarpSchools_All_Tiles\All_Tiles"
|
|
34
|
+
|
|
35
|
+
# File pattern with delimiter
|
|
36
|
+
delimiter = "-"
|
|
37
|
+
filePattern = delimiter.join(["{recording}", "{tile}"]) + ".png"
|
|
38
|
+
mosaicMatchPattern = delimiter.join(["{recording}"]) + ".png"
|
|
39
|
+
|
|
40
|
+
outMosaicDir = r"D:\scratch\USGS_CERC_CarpSchools_All_Mosaics"
|
|
41
|
+
overlap_method = "average" # Options: 'average', 'max', 'min'
|
|
42
|
+
|
|
43
|
+
outputFormat = "tif" # Output mosaic file extension/format; independent of input tile extension
|
|
44
|
+
|
|
45
|
+
force_greyscale = True # Force output to be greyscale (single band) if True, else keep original bands
|
|
46
|
+
tiles_overlap = True # Set False when tile footprints are guaranteed not to overlap
|
|
47
|
+
|
|
48
|
+
threadCnt = cpu_count() # Number of mosaics to build in parallel
|
|
49
|
+
|
|
50
|
+
blockSize = 2048 # Block edge (px) processed/written at a time; bounds peak memory use
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
###########
|
|
54
|
+
# Functions
|
|
55
|
+
|
|
56
|
+
# GDAL driver for each supported output extension; GTiff is the only one
|
|
57
|
+
# validated for windowed/blockwise writes used by buildMosaic.
|
|
58
|
+
_DRIVER_BY_EXT = {
|
|
59
|
+
'tif': 'GTiff',
|
|
60
|
+
'tiff': 'GTiff',
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _driverForFormat(outputFormat: str):
|
|
65
|
+
ext = outputFormat.lower().lstrip('.')
|
|
66
|
+
if ext not in _DRIVER_BY_EXT:
|
|
67
|
+
raise ValueError("Unsupported outputFormat '{}'; supported: {}".format(outputFormat, sorted(_DRIVER_BY_EXT)))
|
|
68
|
+
return ext, _DRIVER_BY_EXT[ext]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _build_pattern_regex(filePattern: str, delimiter: str):
|
|
72
|
+
'''
|
|
73
|
+
Convert a delimiter-based file pattern (e.g. "{recording}_{tile}.tif")
|
|
74
|
+
into a compiled regex with named capture groups.
|
|
75
|
+
'''
|
|
76
|
+
|
|
77
|
+
fieldNames = re.findall(r'\{(\w+)\}', filePattern)
|
|
78
|
+
|
|
79
|
+
escaped = re.escape(filePattern)
|
|
80
|
+
for name in fieldNames:
|
|
81
|
+
placeholder = re.escape('{' + name + '}')
|
|
82
|
+
group = '(?P<{}>[^{}]+)'.format(name, re.escape(delimiter))
|
|
83
|
+
escaped = escaped.replace(placeholder, group)
|
|
84
|
+
|
|
85
|
+
return re.compile('^' + escaped + '$'), fieldNames
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _findTiles(inTileDir: str, ext: str):
|
|
89
|
+
'''Recursively find all tile files under inTileDir with extension ext.'''
|
|
90
|
+
|
|
91
|
+
return sorted(glob(os.path.join(inTileDir, '**', '*' + ext), recursive=True))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _groupTiles(tileFiles: list, regex, mosaicMatchPattern: str):
|
|
95
|
+
'''
|
|
96
|
+
Group tile file paths by the mosaic key derived from mosaicMatchPattern,
|
|
97
|
+
using the named fields extracted from each filename via regex.
|
|
98
|
+
'''
|
|
99
|
+
|
|
100
|
+
groups = {}
|
|
101
|
+
skipped = []
|
|
102
|
+
|
|
103
|
+
for f in tileFiles:
|
|
104
|
+
m = regex.match(os.path.basename(f))
|
|
105
|
+
if not m:
|
|
106
|
+
skipped.append(f)
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
fields = m.groupdict()
|
|
110
|
+
mosaicName = mosaicMatchPattern.format(**fields)
|
|
111
|
+
groups.setdefault(mosaicName, []).append(f)
|
|
112
|
+
|
|
113
|
+
return groups, skipped
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _toGrey(arr: np.ndarray) -> np.ndarray:
|
|
117
|
+
'''Convert a (bands, H, W) array to single-band greyscale using luma weights.'''
|
|
118
|
+
|
|
119
|
+
if arr.shape[0] == 1:
|
|
120
|
+
return arr[0:1]
|
|
121
|
+
|
|
122
|
+
if arr.shape[0] >= 3:
|
|
123
|
+
r = arr[0].astype(np.float64)
|
|
124
|
+
g = arr[1].astype(np.float64)
|
|
125
|
+
b = arr[2].astype(np.float64)
|
|
126
|
+
grey = 0.299 * r + 0.587 * g + 0.114 * b
|
|
127
|
+
return grey[np.newaxis, ...]
|
|
128
|
+
|
|
129
|
+
# 2-band fallback
|
|
130
|
+
return arr.mean(axis=0, keepdims=True)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def buildMosaic(mosaicName: str,
|
|
134
|
+
tileList: list,
|
|
135
|
+
outDir: str,
|
|
136
|
+
overlap_method: str,
|
|
137
|
+
force_greyscale: bool,
|
|
138
|
+
outputFormat: str = 'tif',
|
|
139
|
+
blockSize: int = 2048,
|
|
140
|
+
tiles_overlap: bool = True):
|
|
141
|
+
'''
|
|
142
|
+
Merge tileList into a single mosaic GeoTIFF, combining overlapping
|
|
143
|
+
pixels per overlap_method. Processed and written one block at a time
|
|
144
|
+
(rasterio/numpy only, no GDAL VRT/pixel functions) so peak memory is
|
|
145
|
+
bounded by blockSize instead of the full mosaic extent.
|
|
146
|
+
'''
|
|
147
|
+
|
|
148
|
+
with warnings.catch_warnings():
|
|
149
|
+
warnings.filterwarnings('ignore', category=rasterio.errors.NotGeoreferencedWarning)
|
|
150
|
+
srcs = [rasterio.open(f) for f in tileList]
|
|
151
|
+
|
|
152
|
+
try:
|
|
153
|
+
ref = srcs[0]
|
|
154
|
+
resX, resY = ref.res
|
|
155
|
+
crs = ref.crs
|
|
156
|
+
dtype = ref.dtypes[0]
|
|
157
|
+
nodata = ref.nodata if ref.nodata is not None else 0
|
|
158
|
+
bandCnt = 1 if force_greyscale else ref.count
|
|
159
|
+
|
|
160
|
+
lefts, bottoms, rights, tops = zip(*(s.bounds for s in srcs))
|
|
161
|
+
minx, miny, maxx, maxy = min(lefts), min(bottoms), max(rights), max(tops)
|
|
162
|
+
|
|
163
|
+
width = max(1, int(round((maxx - minx) / resX)))
|
|
164
|
+
height = max(1, int(round((maxy - miny) / resY)))
|
|
165
|
+
outTransform = from_origin(minx, maxy, resX, resY)
|
|
166
|
+
|
|
167
|
+
if overlap_method not in ('average', 'max', 'min'):
|
|
168
|
+
raise ValueError("overlap_method must be 'average', 'max', or 'min'; got '{}'".format(overlap_method))
|
|
169
|
+
|
|
170
|
+
# Precompute each src's pixel offset/size within the full output grid
|
|
171
|
+
srcInfo = []
|
|
172
|
+
for s in srcs:
|
|
173
|
+
colOff = int(round((s.bounds.left - minx) / resX))
|
|
174
|
+
rowOff = int(round((maxy - s.bounds.top) / resY))
|
|
175
|
+
srcInfo.append((s, colOff, rowOff, s.width, s.height))
|
|
176
|
+
|
|
177
|
+
blockCols = (width + blockSize - 1) // blockSize
|
|
178
|
+
blockRows = (height + blockSize - 1) // blockSize
|
|
179
|
+
blockSources = [[] for _ in range(blockRows * blockCols)]
|
|
180
|
+
for sourceIndex, (_, colOff, rowOff, sWidth, sHeight) in enumerate(srcInfo):
|
|
181
|
+
firstBlockRow = max(0, rowOff // blockSize)
|
|
182
|
+
lastBlockRow = min(blockRows - 1, (rowOff + sHeight - 1) // blockSize)
|
|
183
|
+
firstBlockCol = max(0, colOff // blockSize)
|
|
184
|
+
lastBlockCol = min(blockCols - 1, (colOff + sWidth - 1) // blockSize)
|
|
185
|
+
for blockRow in range(firstBlockRow, lastBlockRow + 1):
|
|
186
|
+
for blockCol in range(firstBlockCol, lastBlockCol + 1):
|
|
187
|
+
blockSources[blockRow * blockCols + blockCol].append(sourceIndex)
|
|
188
|
+
|
|
189
|
+
# Internal GTiff block size must be a multiple of 16 and not exceed raster dims
|
|
190
|
+
def _tiffBlockDim(dimSize):
|
|
191
|
+
dim = min(blockSize, dimSize)
|
|
192
|
+
dim = max(16, (dim // 16) * 16)
|
|
193
|
+
return dim
|
|
194
|
+
|
|
195
|
+
outExt, outDriver = _driverForFormat(outputFormat)
|
|
196
|
+
outName = os.path.splitext(mosaicName)[0] + '.' + outExt
|
|
197
|
+
outPath = os.path.join(outDir, outName)
|
|
198
|
+
profile = {
|
|
199
|
+
'driver': outDriver,
|
|
200
|
+
'height': height,
|
|
201
|
+
'width': width,
|
|
202
|
+
'count': bandCnt,
|
|
203
|
+
'dtype': dtype,
|
|
204
|
+
'crs': crs,
|
|
205
|
+
'transform': outTransform,
|
|
206
|
+
'nodata': nodata,
|
|
207
|
+
'compress': 'lzw',
|
|
208
|
+
'tiled': True,
|
|
209
|
+
'blockxsize': _tiffBlockDim(width),
|
|
210
|
+
'blockysize': _tiffBlockDim(height),
|
|
211
|
+
'BIGTIFF': 'IF_SAFER',
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
with rasterio.open(outPath, 'w', **profile) as dst:
|
|
215
|
+
if not tiles_overlap:
|
|
216
|
+
for s, colOff, rowOff, sWidth, sHeight in srcInfo:
|
|
217
|
+
data = s.read()
|
|
218
|
+
if force_greyscale:
|
|
219
|
+
data = _toGrey(data.astype(np.float64))
|
|
220
|
+
dst.write(
|
|
221
|
+
data.astype(dtype),
|
|
222
|
+
window=Window(col_off=colOff, row_off=rowOff, width=sWidth, height=sHeight),
|
|
223
|
+
)
|
|
224
|
+
return outPath
|
|
225
|
+
|
|
226
|
+
for rowStart in range(0, height, blockSize):
|
|
227
|
+
bh = min(blockSize, height - rowStart)
|
|
228
|
+
for colStart in range(0, width, blockSize):
|
|
229
|
+
bw = min(blockSize, width - colStart)
|
|
230
|
+
|
|
231
|
+
if overlap_method == 'average':
|
|
232
|
+
sumBlock = np.zeros((bandCnt, bh, bw), dtype=np.float64)
|
|
233
|
+
cntBlock = np.zeros((bandCnt, bh, bw), dtype=np.uint32)
|
|
234
|
+
else:
|
|
235
|
+
outBlock = np.full((bandCnt, bh, bw), nodata, dtype=np.float64)
|
|
236
|
+
hasBlock = np.zeros((bandCnt, bh, bw), dtype=bool)
|
|
237
|
+
|
|
238
|
+
blockIndex = (rowStart // blockSize) * blockCols + (colStart // blockSize)
|
|
239
|
+
for sourceIndex in blockSources[blockIndex]:
|
|
240
|
+
s, colOff, rowOff, sWidth, sHeight = srcInfo[sourceIndex]
|
|
241
|
+
# Intersection of this output block with the src's footprint
|
|
242
|
+
interRow0 = max(rowStart, rowOff)
|
|
243
|
+
interRow1 = min(rowStart + bh, rowOff + sHeight)
|
|
244
|
+
interCol0 = max(colStart, colOff)
|
|
245
|
+
interCol1 = min(colStart + bw, colOff + sWidth)
|
|
246
|
+
|
|
247
|
+
if interRow0 >= interRow1 or interCol0 >= interCol1:
|
|
248
|
+
continue
|
|
249
|
+
|
|
250
|
+
srcWin = Window(
|
|
251
|
+
col_off=interCol0 - colOff,
|
|
252
|
+
row_off=interRow0 - rowOff,
|
|
253
|
+
width=interCol1 - interCol0,
|
|
254
|
+
height=interRow1 - interRow0,
|
|
255
|
+
)
|
|
256
|
+
data = s.read(window=srcWin, masked=True)
|
|
257
|
+
|
|
258
|
+
if force_greyscale:
|
|
259
|
+
filled = data.filled(nodata).astype(np.float64)
|
|
260
|
+
grey = _toGrey(filled)
|
|
261
|
+
validMask = np.any(~np.ma.getmaskarray(data), axis=0, keepdims=True)
|
|
262
|
+
validMask &= np.any(data.data != nodata, axis=0, keepdims=True)
|
|
263
|
+
filled = grey
|
|
264
|
+
valid = np.broadcast_to(validMask, grey.shape)
|
|
265
|
+
else:
|
|
266
|
+
filled = data.filled(nodata).astype(np.float64)
|
|
267
|
+
valid = (~np.ma.getmaskarray(data) & (data.data != nodata))
|
|
268
|
+
|
|
269
|
+
bRow0 = interRow0 - rowStart
|
|
270
|
+
bCol0 = interCol0 - colStart
|
|
271
|
+
h, w = filled.shape[1], filled.shape[2]
|
|
272
|
+
|
|
273
|
+
if overlap_method == 'average':
|
|
274
|
+
sumBlock[:, bRow0:bRow0 + h, bCol0:bCol0 + w] += np.where(valid, filled, 0)
|
|
275
|
+
cntBlock[:, bRow0:bRow0 + h, bCol0:bCol0 + w] += valid.astype(np.uint32)
|
|
276
|
+
else:
|
|
277
|
+
region = outBlock[:, bRow0:bRow0 + h, bCol0:bCol0 + w]
|
|
278
|
+
regionHas = hasBlock[:, bRow0:bRow0 + h, bCol0:bCol0 + w]
|
|
279
|
+
|
|
280
|
+
if overlap_method == 'max':
|
|
281
|
+
take = valid & (~regionHas | (filled > region))
|
|
282
|
+
else: # min
|
|
283
|
+
take = valid & (~regionHas | (filled < region))
|
|
284
|
+
|
|
285
|
+
region[take] = filled[take]
|
|
286
|
+
regionHas |= valid
|
|
287
|
+
|
|
288
|
+
if overlap_method == 'average':
|
|
289
|
+
with np.errstate(invalid='ignore', divide='ignore'):
|
|
290
|
+
outBlock = np.where(cntBlock > 0, sumBlock / np.maximum(cntBlock, 1), nodata)
|
|
291
|
+
|
|
292
|
+
dst.write(
|
|
293
|
+
outBlock.astype(dtype),
|
|
294
|
+
window=Window(col_off=colStart, row_off=rowStart, width=bw, height=bh),
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
return outPath
|
|
298
|
+
|
|
299
|
+
finally:
|
|
300
|
+
for s in srcs:
|
|
301
|
+
if not s.closed:
|
|
302
|
+
s.close()
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def doWork():
|
|
306
|
+
|
|
307
|
+
os.makedirs(outMosaicDir, exist_ok=True)
|
|
308
|
+
|
|
309
|
+
ext = os.path.splitext(filePattern)[-1]
|
|
310
|
+
regex, _ = _build_pattern_regex(filePattern, delimiter)
|
|
311
|
+
|
|
312
|
+
tileFiles = _findTiles(inTileDir, ext)
|
|
313
|
+
print('Found {} tile(s) under {}'.format(len(tileFiles), inTileDir))
|
|
314
|
+
|
|
315
|
+
groups, skipped = _groupTiles(tileFiles, regex, mosaicMatchPattern)
|
|
316
|
+
if skipped:
|
|
317
|
+
print('Skipped {} file(s) not matching filePattern.'.format(len(skipped)))
|
|
318
|
+
|
|
319
|
+
print('Grouped into {} mosaic(s).'.format(len(groups)))
|
|
320
|
+
|
|
321
|
+
results = list(Parallel(n_jobs=threadCnt)(
|
|
322
|
+
delayed(buildMosaic)(mosaicName, tileList, outMosaicDir, overlap_method, force_greyscale, outputFormat, blockSize, tiles_overlap)
|
|
323
|
+
for mosaicName, tileList in tqdm(groups.items(), total=len(groups))
|
|
324
|
+
))
|
|
325
|
+
|
|
326
|
+
print('Wrote {} mosaic(s) to {}'.format(len(results), outMosaicDir))
|
|
327
|
+
|
|
328
|
+
return results
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
#########
|
|
332
|
+
# Do Work
|
|
333
|
+
|
|
334
|
+
if __name__ == '__main__':
|
|
335
|
+
doWork()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '5.5.6'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pingmapper
|
|
3
|
-
Version: 5.5.
|
|
3
|
+
Version: 5.5.6
|
|
4
4
|
Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
|
|
5
5
|
Author: Cameron Bodine, Daniel Buscombe, Iury T Simoes-Sousa
|
|
6
6
|
Author-email: bodine.cs@gmail.email
|
|
@@ -38,7 +38,9 @@ pingmapper/scratch/main.py
|
|
|
38
38
|
pingmapper/scratch/main_batchDirectory.py
|
|
39
39
|
pingmapper/utils/RawEGN_avg_predictions.py
|
|
40
40
|
pingmapper/utils/export_coverage.py
|
|
41
|
+
pingmapper/utils/kml2mosaic.py
|
|
41
42
|
pingmapper/utils/main_mosaic_transects.py
|
|
43
|
+
pingmapper/utils/tile2mosaic.py
|
|
42
44
|
pingmapper/utils/DRAFT_Workflows/avg_predictions_Mussel_WBL.py
|
|
43
45
|
pingmapper/utils/DRAFT_Workflows/gen_centerline.py
|
|
44
46
|
pingmapper/utils/DRAFT_Workflows/gen_centerline_from_bankline.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '5.5.4'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/avg_predictions_Mussel_WBL.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/DRAFT_Workflows/testEXAMPLE_mosaic_logit.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pingmapper-5.5.4 → pingmapper-5.5.6}/pingmapper/utils/Substrate_Summaries/03_gen_summary_shp.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|