pingmapper 5.5.2__tar.gz → 5.5.4__tar.gz

{pingmapper-5.5.2 → pingmapper-5.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 5.5.2
+Version: 5.5.4
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Cameron Bodine, Daniel Buscombe, Iury T Simoes-Sousa
 Author-email: bodine.cs@gmail.email

{pingmapper-5.5.2 → pingmapper-5.5.4}/pingmapper/class_rectObj.py

@@ -554,18 +554,22 @@ class rectObj(sonObj):
                 fb[lats] = fb[ylat]
 
             if es not in fb.columns:
-                if xutm in fb.columns:
-                    fb[es] = fb[xutm]
-                elif lons in fb.columns and lats in fb.columns:
+                if lons in fb.columns and lats in fb.columns:
+                    # Prefer projecting lon/lat through self.trans (the correct
+                    # target EPSG) rather than reusing a raw xutm column, which
+                    # may hold a different (e.g. Humminbird-internal Web Mercator)
+                    # coordinate system and silently corrupt downstream geometry.
                     e_fb, _ = self.trans(fb[lons].to_numpy(), fb[lats].to_numpy())
                     fb[es] = e_fb
+                elif xutm in fb.columns:
+                    fb[es] = fb[xutm]
 
             if ns not in fb.columns:
-                if yutm in fb.columns:
-                    fb[ns] = fb[yutm]
-                elif lons in fb.columns and lats in fb.columns:
+                if lons in fb.columns and lats in fb.columns:
                     _, n_fb = self.trans(fb[lons].to_numpy(), fb[lats].to_numpy())
                     fb[ns] = n_fb
+                elif yutm in fb.columns:
+                    fb[ns] = fb[yutm]
 
             if 'cog' not in fb.columns:
                 if 'instr_heading' in fb.columns:
@@ -1099,6 +1103,26 @@ class rectObj(sonObj):
                     else:
                         continue
 
+                # Guard against spline overshoot: the linear spline fit to range
+                # extent coordinates can extrapolate far beyond the sonar's actual
+                # range near gaps left by upstream filtering (time/speed/coord
+                # outlier filters). Fall back to the geometrically-derived
+                # (un-smoothed) range extent point for any ping whose smoothed
+                # point lands unreasonably far from its trackline origin.
+                if {'record_num', 'range_lon', 'range_lat', 'utm_es', 'utm_ns', 'range'}.issubset(schunkDF.columns):
+                    e_smth_chk, n_smth_chk = self.trans(smthChunk['range_lons'].to_numpy(), smthChunk['range_lats'].to_numpy())
+                    chk = schunkDF[['record_num', 'utm_es', 'utm_ns', 'range', 'range_lon', 'range_lat']].set_index('record_num')
+                    chk = chk.reindex(smthChunk['record_num'].to_numpy())
+
+                    dist = np.hypot(e_smth_chk - chk['utm_es'].to_numpy(), n_smth_chk - chk['utm_ns'].to_numpy())
+                    max_expected = np.maximum(chk['range'].to_numpy(), 1.0) * 3.0 + 50.0
+                    bad = np.isfinite(dist) & (dist > max_expected)
+                    n_bad = int(np.sum(bad))
+                    if n_bad > 0:
+                        print(f"\n  _interpRangeCoords: corrected {n_bad} spline-overshoot range extent point(s) in chunk {chunk}.")
+                        smthChunk.loc[bad, 'range_lons'] = chk['range_lon'].to_numpy()[bad]
+                        smthChunk.loc[bad, 'range_lats'] = chk['range_lat'].to_numpy()[bad]
+
                 rs_chunks.append(smthChunk.copy())
 
         ##################################################
@@ -2724,6 +2748,8 @@ class rectObj(sonObj):
             img = self.sonDat.copy()
             if son:
                 img = self._reserve_zero_for_nodata(img, preserve_zeros=True)
+                if bool(getattr(self, 'sonar_db_transform', False)) or bool(getattr(self, 'sonar_clahe', False)):
+                    img = self._apply_display_enhancements(img)
             apply_post_rect_colormap = use_16bit and self._rect_colormap_selected(son=son)
             source_scale_bounds = None
             if apply_post_rect_colormap:
@@ -2747,6 +2773,8 @@ class rectObj(sonObj):
             out = np.flip(np.flip(np.flip(out,1),0),1)
             if use_16bit:
                 out = out.astype(np.float32)
+            elif bool(getattr(self, 'sonar_db_transform', False)) or bool(getattr(self, 'sonar_clahe', False)):
+                out = out.astype(np.float32)
             else:
                 out = out.astype('uint8')
 
@@ -2780,7 +2808,10 @@ class rectObj(sonObj):
                     out16 = out16_raw
                 self._write_rect_geotiff(gtiff, out16, epsg, transform, colormap=None)
             else:
-                out8 = np.clip(out, 0, 255).astype(np.uint8)
+                if bool(getattr(self, 'sonar_db_transform', False)) or bool(getattr(self, 'sonar_clahe', False)):
+                    out8 = self._convert_son_dat_to_uint8(out)
+                else:
+                    out8 = np.clip(out, 0, 255).astype(np.uint8)
                 self._write_rect_geotiff(gtiff, out8, epsg, transform, colormap=self.son_colorMap)
 
             del out, img
@@ -2815,6 +2846,8 @@ class rectObj(sonObj):
             img = self.sonDat
             if son:
                 img = self._reserve_zero_for_nodata(img, preserve_zeros=True)
+                if bool(getattr(self, 'sonar_db_transform', False)) or bool(getattr(self, 'sonar_clahe', False)):
+                    img = self._apply_display_enhancements(img)
             apply_post_rect_colormap = son and use_16bit and self._rect_colormap_selected(son=son)
             source_scale_bounds = None
             if apply_post_rect_colormap:
@@ -2887,6 +2920,8 @@ class rectObj(sonObj):
             out = np.flip(np.flip(np.flip(out,1),0),1)
             if son and use_16bit:
                 out = out.astype(np.float32)
+            elif son and (bool(getattr(self, 'sonar_db_transform', False)) or bool(getattr(self, 'sonar_clahe', False))):
+                out = out.astype(np.float32)
             else:
                 out = out.astype('uint8')
 
@@ -2926,7 +2961,10 @@ class rectObj(sonObj):
                 self._write_rect_geotiff(gtiff, out16, epsg, transform, colormap=None)
             else:
                 colormap = self.son_colorMap if son else class_colormap
-                out8 = np.clip(out, 0, 255).astype(np.uint8)
+                if son and (bool(getattr(self, 'sonar_db_transform', False)) or bool(getattr(self, 'sonar_clahe', False))):
+                    out8 = self._convert_son_dat_to_uint8(out)
+                else:
+                    out8 = np.clip(out, 0, 255).astype(np.uint8)
                 self._write_rect_geotiff(gtiff, out8, epsg, transform, colormap=colormap)
 
             del out
@@ -2986,6 +3024,34 @@ class rectObj(sonObj):
         # Set output name
         f_out = f.replace('temp.tif', '.tif')
 
+        # Sanity check the georeferenced extent before warping. A corrupt/outlier
+        # coordinate upstream can produce a geotransform spanning thousands of km,
+        # which causes gdal.Warp to attempt allocating a raster with an absurd
+        # pixel count (surfacing as a misleading "free disk space" GDAL error).
+        src = gdal.Open(f)
+        if src is not None:
+            gt = src.GetGeoTransform()
+            width_m = abs(gt[1]) * src.RasterXSize
+            height_m = abs(gt[5]) * src.RasterYSize
+            src = None
+
+            est_cols = width_m / pix_res
+            est_rows = height_m / pix_res
+
+            _MAX_PIXELS = 5e8  # ~500 million pixels safety cap
+            if est_cols * est_rows > _MAX_PIXELS:
+                print(
+                    f"\nWARNING: Skipping pixel resolution resize for {os.path.basename(f)}. "
+                    f"Georeferenced extent ({width_m:.1f} x {height_m:.1f} m) at {pix_res} m resolution "
+                    f"would require ~{est_cols*est_rows:,.0f} pixels. This indicates a corrupt or outlier "
+                    f"ping coordinate in this chunk. Keeping the un-resized GeoTiff instead."
+                )
+                try:
+                    os.rename(f, f_out)
+                except OSError:
+                    pass
+                return
+
         # Reopen f and warp to pix_res
         t = gdal.Warp(f_out, f, xRes = pix_res, yRes = pix_res, targetAlignedPixels=True)
 

{pingmapper-5.5.2 → pingmapper-5.5.4}/pingmapper/class_sonObj.py

@@ -1319,6 +1319,44 @@ class sonObj(object):
         if not use_db and not use_clahe:
             return sonDat
 
+        arr = self._apply_display_db_transform(sonDat)
+        if arr.size == 0:
+            return arr
+
+        valid = arr > 0
+        if not np.any(valid):
+            return arr
+
+        if use_clahe:
+            clip_limit = float(getattr(self, 'sonar_clahe_clip_limit', 0.01))
+            if (not np.isfinite(clip_limit)) or clip_limit <= 0:
+                clip_limit = 0.01
+
+            global_bounds = None
+            if bool(getattr(self, 'sonar_clahe_global', True)):
+                global_bounds = self._ensure_clahe_global_bounds()
+
+            if global_bounds is not None:
+                arr01 = self._normalize_with_bounds(arr, global_bounds[0], global_bounds[1])
+            else:
+                max_val = float(np.nanmax(arr))
+                if not (np.isfinite(max_val) and max_val > 0):
+                    return arr
+                arr01 = np.clip(arr / max_val, 0.0, 1.0)
+
+            try:
+                arr01 = exposure.equalize_adapthist(arr01, clip_limit=clip_limit)
+                arr = np.asarray(arr01, dtype=np.float32) * 255.0
+            except MemoryError:
+                arr = self._apply_clahe_chunked(arr01, clip_limit)
+            except np.core._exceptions._ArrayMemoryError:
+                arr = self._apply_clahe_chunked(arr01, clip_limit)
+
+        # Preserve no-data convention for empty samples.
+        arr[~valid] = 0.0
+        return arr
+
+    def _apply_display_db_transform(self, sonDat):
         arr = np.asarray(sonDat, dtype=np.float32)
         if arr.size == 0:
             return arr
@@ -1330,7 +1368,7 @@ class sonObj(object):
         if not np.any(valid):
             return arr
 
-        if use_db:
+        if bool(getattr(self, 'sonar_db_transform', False)):
             # Use the smallest positive intensity as the log floor to avoid
             # collapsing zeros to -inf and to preserve relative low amplitudes.
             floor = float(np.nanmin(arr[valid]))
@@ -1340,25 +1378,73 @@ class sonObj(object):
             arr = 20.0 * np.log10(arr)
             arr = arr - np.nanmin(arr)
 
-        if use_clahe:
-            max_val = float(np.nanmax(arr))
-            if np.isfinite(max_val) and max_val > 0:
-                arr01 = np.clip(arr / max_val, 0.0, 1.0)
-                clip_limit = float(getattr(self, 'sonar_clahe_clip_limit', 0.01))
-                if (not np.isfinite(clip_limit)) or clip_limit <= 0:
-                    clip_limit = 0.01
-                try:
-                    arr01 = exposure.equalize_adapthist(arr01, clip_limit=clip_limit)
-                    arr = np.asarray(arr01, dtype=np.float32)
-                except MemoryError:
-                    arr = self._apply_clahe_chunked(arr01, clip_limit)
-                except np.core._exceptions._ArrayMemoryError:
-                    arr = self._apply_clahe_chunked(arr01, clip_limit)
-
-        # Preserve no-data convention for empty samples.
         arr[~valid] = 0.0
         return arr
 
+    def _ensure_clahe_global_bounds(self):
+        if not bool(getattr(self, 'sonar_clahe_global', True)):
+            return None
+
+        cache = getattr(self, '_sonar_clahe_global_bounds', None)
+        if isinstance(cache, tuple) and len(cache) == 2:
+            lo, hi = cache
+            if np.isfinite(lo) and np.isfinite(hi) and hi > lo:
+                return cache
+
+        if not hasattr(self, 'sonMetaDF'):
+            self._loadSonMeta()
+
+        if not hasattr(self, 'sonMetaDF') or self.sonMetaDF is None or len(self.sonMetaDF) == 0:
+            return None
+
+        saved_sonDat = getattr(self, 'sonDat', None)
+        saved_sonDat16 = getattr(self, 'sonDat16', None)
+        saved_pingMax = getattr(self, 'pingMax', None)
+
+        chunk_ids = pd.unique(self.sonMetaDF['chunk_id'])
+        sample_cap = int(getattr(self, 'sonar_clahe_global_sample_cap', 200000))
+        if sample_cap <= 0:
+            sample_cap = 200000
+
+        sampled_vals = []
+        for chunk in chunk_ids:
+            self._getScanChunkSingle(int(chunk))
+            arr = self._apply_display_db_transform(self.sonDat)
+            valid = arr > 0
+            if not np.any(valid):
+                continue
+
+            vals = arr[valid].astype(np.float32, copy=False)
+            if vals.size > sample_cap:
+                idx = np.linspace(0, vals.size - 1, sample_cap, dtype=np.int64)
+                vals = vals[idx]
+            sampled_vals.append(vals)
+
+        if len(sampled_vals) == 0:
+            self.sonDat = saved_sonDat
+            self.sonDat16 = saved_sonDat16
+            self.pingMax = saved_pingMax
+            return None
+
+        pooled = np.concatenate(sampled_vals)
+        lo = float(np.nanpercentile(pooled, 1.0))
+        hi = float(np.nanpercentile(pooled, 99.5))
+        if (not np.isfinite(lo)) or (not np.isfinite(hi)) or hi <= lo:
+            lo = float(np.nanmin(pooled))
+            hi = float(np.nanmax(pooled))
+
+        if (not np.isfinite(lo)) or (not np.isfinite(hi)) or hi <= lo:
+            self.sonDat = saved_sonDat
+            self.sonDat16 = saved_sonDat16
+            self.pingMax = saved_pingMax
+            return None
+
+        self._sonar_clahe_global_bounds = (lo, hi)
+        self.sonDat = saved_sonDat
+        self.sonDat16 = saved_sonDat16
+        self.pingMax = saved_pingMax
+        return self._sonar_clahe_global_bounds
+
     def _apply_clahe_chunked(self, arr01, clip_limit):
         """Apply CLAHE in row-wise stripes to reduce peak memory usage."""
         src = np.asarray(arr01, dtype=np.float32)
@@ -1378,7 +1464,7 @@ class sonObj(object):
             end = min(start + stripe_rows, h)
             stripe = src[start:end, :]
             stripe_eq = exposure.equalize_adapthist(stripe, clip_limit=clip_limit)
-            out[start:end, :] = np.asarray(stripe_eq, dtype=np.float32)
+            out[start:end, :] = np.asarray(stripe_eq, dtype=np.float32) * 255.0
 
         return out
 

{pingmapper-5.5.2 → pingmapper-5.5.4}/pingmapper/default_params.json

@@ -27,6 +27,7 @@
     "tone_gain":1.0,
     "sonar_db_transform":false,
     "sonar_clahe":false,
+    "sonar_clahe_global":true,
     "sonar_clahe_clip_limit":0.01,
     "wcp":false,
     "wcm":true,

{pingmapper-5.5.2 → pingmapper-5.5.4}/pingmapper/funcs_rectify.py

@@ -149,11 +149,44 @@ def smoothTrackline(projDir='', x_offset='', y_offset='', nchunk ='', cog=True,
             #     filter = int(nchunk*0.1)
 
             # print('\n\n\ntransect:', name, 'filter:', filter)
+            # Snapshot raw coordinates before _interpTrack() may drop duplicates
+            # in-place, so we can validate/repair the smoothed output below.
+            raw_lon = group['lon'].copy()
+            raw_lat = group['lat'].copy()
+
             smoothed = son._interpTrack(df=group, dropDup=True, filt=filter, deg=3)
 
             # smooth trackline fit
             # if smoothed is not None:
             if len(smoothed.columns) > 4:
+                # Guard against cubic spline overshoot (Runge's phenomenon) across
+                # long or gappy transects (e.g. gaps left by time/speed/coord
+                # outlier filters). Fall back to the raw GPS fix for any ping
+                # whose smoothed position was displaced further than the
+                # transect's own spatial extent would allow.
+                raw_e, raw_n = son.trans(raw_lon.to_numpy(), raw_lat.to_numpy())
+                raw_e = pd.Series(raw_e, index=raw_lon.index).reindex(smoothed.index).to_numpy()
+                raw_n = pd.Series(raw_n, index=raw_lat.index).reindex(smoothed.index).to_numpy()
+
+                smth_e, smth_n = son.trans(smoothed['lons'].to_numpy(), smoothed['lats'].to_numpy())
+
+                dist = np.hypot(smth_e - raw_e, smth_n - raw_n)
+                span = np.hypot(np.nanmax(raw_e) - np.nanmin(raw_e), np.nanmax(raw_n) - np.nanmin(raw_n))
+                max_allowed = max(span, 50.0) * 2.0
+
+                bad = np.isfinite(dist) & (dist > max_allowed)
+                n_bad = int(np.sum(bad))
+                if n_bad > 0:
+                    print(f"\n  smoothTrackline: corrected {n_bad} spline-overshoot trackline point(s) in transect {name}.")
+                    fix_lon = pd.Series(raw_lon.to_numpy(), index=raw_lon.index).reindex(smoothed.index).to_numpy()
+                    fix_lat = pd.Series(raw_lat.to_numpy(), index=raw_lat.index).reindex(smoothed.index).to_numpy()
+                    smoothed.loc[bad, 'lons'] = fix_lon[bad]
+                    smoothed.loc[bad, 'lats'] = fix_lat[bad]
+                    if 'utm_es' in smoothed.columns and 'utm_ns' in smoothed.columns:
+                        e_fix, n_fix = son.trans(smoothed.loc[bad, 'lons'].to_numpy(), smoothed.loc[bad, 'lats'].to_numpy())
+                        smoothed.loc[bad, 'utm_es'] = e_fix
+                        smoothed.loc[bad, 'utm_ns'] = n_fix
+
                 smoothed['transect'] = int(name)
                 sDF = pd.concat([sDF, smoothed], ignore_index=False)
 
@@ -166,6 +199,7 @@ def smoothTrackline(projDir='', x_offset='', y_offset='', nchunk ='', cog=True,
 
             del smoothed
 
+
         # Save sonDF
         if len(transect_dropped) > 0:
 

{pingmapper-5.5.2 → pingmapper-5.5.4}/pingmapper/gui_main.py

@@ -321,6 +321,12 @@ def gui(batch: bool):
         default=default_params.get('sonar_clahe', False),
         tooltip=tip_clahe,
     )
+    check_sonar_clahe_global = sg.Checkbox(
+        'Global CLAHE reference',
+        key='sonar_clahe_global',
+        default=default_params.get('sonar_clahe_global', True),
+        tooltip='Use one project-wide intensity reference before CLAHE so chunk brightness stays consistent. This adds a prepass, but reduces chunk-to-chunk contrast shifts.',
+    )
     text_clahe_clip = sg.Text('CLAHE Clip Limit', size=(20,1))
     in_clahe_clip = sg.Input(
         key='sonar_clahe_clip_limit',
@@ -335,6 +341,7 @@ def gui(batch: bool):
             [text_egn_gain, slide_egn_gain],
             [check_sonar_db],
             [check_sonar_clahe],
+            [check_sonar_clahe_global],
             [text_clahe_clip, in_clahe_clip],
         ],
         pad=0,
@@ -837,6 +844,7 @@ def gui(batch: bool):
             'tone_gain':float(values['tone_gain']),
             'sonar_db_transform':values['sonar_db_transform'],
             'sonar_clahe':values['sonar_clahe'],
+            'sonar_clahe_global':values['sonar_clahe_global'],
             'sonar_clahe_clip_limit':float(values['sonar_clahe_clip_limit']),
             'wcp':values['wcp'],
             'wcm':values['wcm'],

{pingmapper-5.5.2 → pingmapper-5.5.4}/pingmapper/main_readFiles.py

@@ -720,6 +720,7 @@ def read_master_func(logfilename='',
                      tone_gain=1.0,
                      sonar_db_transform=False,
                      sonar_clahe=False,
+                     sonar_clahe_global=True,
                      sonar_clahe_clip_limit=0.01,
                      wcp=False,
                      wcm=False,
@@ -1209,7 +1210,10 @@ def read_master_func(logfilename='',
             son.tone_gain = tone_gain
             son.sonar_db_transform = bool(sonar_db_transform)
             son.sonar_clahe = bool(sonar_clahe)
+            son.sonar_clahe_global = bool(sonar_clahe_global)
             son.sonar_clahe_clip_limit = float(sonar_clahe_clip_limit)
+            if son.sonar_clahe:
+                son._ensure_clahe_global_bounds()
             # Do range crop, if necessary
             if cropRange > 0.0:
                 if file_type == '.xtf' and _is_sidescan_beam(getattr(son, 'beamName', '')):
@@ -1405,7 +1409,10 @@ def read_master_func(logfilename='',
             son.tone_gain = tone_gain
             son.sonar_db_transform = bool(sonar_db_transform)
             son.sonar_clahe = bool(sonar_clahe)
+            son.sonar_clahe_global = bool(sonar_clahe_global)
             son.sonar_clahe_clip_limit = float(sonar_clahe_clip_limit)
+            if son.sonar_clahe:
+                son._ensure_clahe_global_bounds()
 
             beam = son.beamName
             if wcr:

{pingmapper-5.5.2 → pingmapper-5.5.4}/pingmapper/main_rectify.py

@@ -126,6 +126,10 @@ def rectify_master_func(logfilename='',
                         egn_stretch_factor=1,
                         tone_gamma=1.0,
                         tone_gain=1.0,
+                        sonar_db_transform=False,
+                        sonar_clahe=False,
+                        sonar_clahe_global=True,
+                        sonar_clahe_clip_limit=0.01,
                         wcp=False,
                         wcm=False,
                         wcr=False,
@@ -368,6 +372,12 @@ def rectify_master_func(logfilename='',
         son.rect_wcr = rect_wcr
         son.export_16bit = bool(export_16bit) and (not bool(getattr(son, 'son8bit', True)))
         son.export_colormap_uint8 = bool(export_colormap_uint8)
+        son.sonar_db_transform = bool(sonar_db_transform)
+        son.sonar_clahe = bool(sonar_clahe)
+        son.sonar_clahe_global = bool(sonar_clahe_global)
+        son.sonar_clahe_clip_limit = float(sonar_clahe_clip_limit)
+        if son.sonar_clahe:
+            son._ensure_clahe_global_bounds()
 
     # ############################################################################
     # # Rectify Heading sonar imagery - Pingwise, not Rubbersheeting             #
@@ -462,6 +472,12 @@ def rectify_master_func(logfilename='',
     for son in portstar:
         son.rect_wcp = rect_wcp
         son.rect_wcr = rect_wcr
+        son.sonar_db_transform = bool(sonar_db_transform)
+        son.sonar_clahe = bool(sonar_clahe)
+        son.sonar_clahe_global = bool(sonar_clahe_global)
+        son.sonar_clahe_clip_limit = float(sonar_clahe_clip_limit)
+        if son.sonar_clahe:
+            son._ensure_clahe_global_bounds()
 
     if (rect_wcp and rubberSheeting) or (rect_wcr and rubberSheeting):
         # Always use COG for rubber sheeting

pingmapper-5.5.4/pingmapper/version.py

@@ -0,0 +1 @@
+__version__ = '5.5.4'

{pingmapper-5.5.2 → pingmapper-5.5.4}/pingmapper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 5.5.2
+Version: 5.5.4
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Cameron Bodine, Daniel Buscombe, Iury T Simoes-Sousa
 Author-email: bodine.cs@gmail.email

pingmapper-5.5.2/pingmapper/version.py

@@ -1 +0,0 @@
-__version__ = '5.5.2'