pingmapper 5.4.0__tar.gz → 5.4.2__tar.gz

{pingmapper-5.4.0 → pingmapper-5.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 5.4.0
+Version: 5.4.2
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Daniel Buscombe
 Author-email: Cameron Bodine <bodine.cs@gmail.email>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering :: Visualization
 Classifier: Topic :: Scientific/Engineering :: Oceanography
 Classifier: Topic :: Scientific/Engineering :: GIS
 Classifier: Topic :: Scientific/Engineering :: Hydrology
-Requires-Python: >=3.10
+Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pinginstaller<3,>=2
@@ -33,6 +33,7 @@ Requires-Dist: tensorflow<3,>=2.20; extra == "ml"
 Requires-Dist: tf-keras<3,>=2.20; extra == "ml"
 Requires-Dist: transformers<5,>=4.57; extra == "ml"
 Dynamic: license-file
+Dynamic: requires-python
 
 # PING-Mapper
 [![PyPI - Version](https://img.shields.io/pypi/v/pingmapper?style=flat-square&label=Latest%20Version%20(PyPi))](https://pypi.org/project/pingmapper/)

{pingmapper-5.4.0 → pingmapper-5.4.2}/pingmapper/class_portstarObj.py

@@ -31,6 +31,7 @@
 
 
 import os, sys
+import time
 
 # Add 'pingmapper' to the path, may not need after pypi package...
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -1199,13 +1200,45 @@ class portstarObj(object):
         # Find ping-wise water column width from min and max depth prediction
         Wp = maxDepths+minDepths
 
-        # Try cropping so water column ~1/3 of target size area
-        WCProp = 1/3
+        # Try cropping so water column ~1/4 of target size area.
+        # Keeping less water column generally improves bed segmentation stability.
+        WCProp = 1/4
 
         # Buffers so we don't crop too much
         WwcBuf = 150
         WsBuf = 150
 
+        # Use instrument depth to constrain max bed search depth when available.
+        # This helps avoid far-range false positives when returns are only valid
+        # near the center (e.g., shallow channels with large configured range).
+        # inst_depth_mult=3.0 means max search depth = 3x instrument depth, which
+        # keeps the water column at roughly 25% of each side's range.
+        inst_depth_mult = 3.0
+
+        if not hasattr(self.port, 'sonMetaDF'):
+            self.port._loadSonMeta()
+        if not hasattr(self.star, 'sonMetaDF'):
+            self.star._loadSonMeta()
+
+        portChunk = self.port.sonMetaDF[self.port.sonMetaDF['chunk_id'] == i]
+        starChunk = self.star.sonMetaDF[self.star.sonMetaDF['chunk_id'] == i]
+
+        portInstM = pd.to_numeric(portChunk['inst_dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
+        starInstM = pd.to_numeric(starChunk['inst_dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
+        portPixM = pd.to_numeric(portChunk['pixM'], errors='coerce').to_numpy(dtype=float, copy=True)
+        starPixM = pd.to_numeric(starChunk['pixM'], errors='coerce').to_numpy(dtype=float, copy=True)
+
+        portInstPix = np.where((portInstM > 0) & (portPixM > 0), portInstM / portPixM, np.nan)
+        starInstPix = np.where((starInstM > 0) & (starPixM > 0), starInstM / starPixM, np.nan)
+
+        instPix = np.concatenate((portInstPix, starInstPix))
+        validInstPix = instPix[np.isfinite(instPix) & (instPix > 0)]
+        if validInstPix.size > 0:
+            instMedPix = np.nanmedian(validInstPix)
+            maxDepByInst = instMedPix * inst_depth_mult
+            if np.isfinite(maxDepByInst) and maxDepByInst > 0:
+                maxDep = min(maxDep, maxDepByInst)
+
         # Sum Wp to determine area of water column
         WpArea = np.nansum(Wp)
 
@@ -1235,16 +1268,31 @@ class portstarObj(object):
         if Ws > (C-(Wwc/2)):
             Ws = int( C - (Wwc/2) - (W/2) - WsBuf)
 
+        # If Ws is negative, inst_depth * mult exceeds the recorded range.
+        # Pad the far-range side with zeros so the crop proportions are correct.
+        # Zero columns = no acoustic return, which the model interprets as bed,
+        # anchoring the pick at the data boundary when depth approaches range.
+        pad_far = 0
+        if Ws < 0:
+            pad_far = int(-Ws)
+            Ws = 0
+
         # Crop the original sonogram
         ## Port Crop
         lC = Ws # left side crop
         rC = int(C - (Wwc/2)) # right side crop
         portCrop = son3bnd[:, lC:rC,:]
+        if pad_far > 0:
+            _pad = np.zeros((portCrop.shape[0], pad_far, portCrop.shape[2]), dtype=np.uint8)
+            portCrop = np.concatenate((_pad, portCrop), axis=1)  # extend far range (left)
 
         ## Star Crop
         lC = int(C + (Wwc/2)) # left side crop
         rC = int(N - Ws) # right side crop
         starCrop = son3bnd[:, lC:rC, :]
+        if pad_far > 0:
+            _pad = np.zeros((starCrop.shape[0], pad_far, starCrop.shape[2]), dtype=np.uint8)
+            starCrop = np.concatenate((starCrop, _pad), axis=1)  # extend far range (right)
 
 
         ## Concatenate port & star crop
@@ -1277,8 +1325,9 @@ class portstarObj(object):
         # Calculate depth from prediction
         portDepPixCrop, starDepPixCrop = self._findBed(crop_label) # get pixel location of bed
 
-        # add Wwc/2 to get final estimate at original sonogram dimensions
-        portDepPixFinal = np.flip( np.asarray(portDepPixCrop) + int(Wwc/2) )
+        # add Wwc/2 to get final estimate at original sonogram dimensions.
+        # Subtract pad_far from port since padding shifted its columns left.
+        portDepPixFinal = np.flip( np.asarray(portDepPixCrop) + int(Wwc/2) - pad_far )
         starDepPixFinal = np.flip( np.asarray(starDepPixCrop) + int(Wwc/2) )
 
         #############
@@ -1400,6 +1449,34 @@ class portstarObj(object):
             isChunk = son.sonMetaDF['chunk_id'] == chunk
             sonMeta = son.sonMetaDF[isChunk].reset_index()
             acoustic_depth = pd.to_numeric(sonMeta['inst_dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
+            acoustic_med = pd.Series(acoustic_depth).rolling(window=31, center=True, min_periods=1).median().to_numpy()
+            acoustic_resid = np.abs(acoustic_depth - acoustic_med)
+            acoustic_valid = np.isfinite(acoustic_depth) & (acoustic_depth > 0)
+            acoustic_bad = np.zeros(acoustic_depth.shape, dtype=bool)
+            if acoustic_valid.any():
+                resid_valid = acoustic_resid[acoustic_valid]
+                resid_center = np.nanmedian(resid_valid)
+                resid_mad = np.nanmedian(np.abs(resid_valid - resid_center))
+                resid_thr = max(0.5, 6.0 * resid_mad if np.isfinite(resid_mad) else 0.5)
+                acoustic_bad |= acoustic_valid & (acoustic_resid > resid_thr)
+
+            if acoustic_depth.size >= 3:
+                prev_step = acoustic_depth[1:-1] - acoustic_depth[:-2]
+                next_step = acoustic_depth[2:] - acoustic_depth[1:-1]
+                step_mag = np.abs(np.concatenate((prev_step, next_step)))
+                step_mag = step_mag[np.isfinite(step_mag)]
+                if step_mag.size > 0:
+                    step_center = np.nanmedian(step_mag)
+                    step_mad = np.nanmedian(np.abs(step_mag - step_center))
+                    jump_thr = max(0.5, 6.0 * step_mad if np.isfinite(step_mad) else 0.5)
+                    acoustic_bad[1:-1] |= (
+                        acoustic_valid[1:-1] & acoustic_valid[:-2] & acoustic_valid[2:] &
+                        (np.abs(prev_step) > jump_thr) &
+                        (np.abs(next_step) > jump_thr) &
+                        ((prev_step * next_step) < 0)
+                    )
+
+            acoustic_depth = np.where(acoustic_bad, np.nan, acoustic_depth)
             acousticBed = np.round(acoustic_depth / sonMeta['pixM'].to_numpy(dtype=float, copy=True), 0)
             acousticBed = acousticBed[np.isfinite(acousticBed) & (acousticBed > 0)]
 
@@ -1407,8 +1484,8 @@ class portstarObj(object):
             # Step 1 : Acoustic Bedpick Filter
             # Use acoustic bed pick to crop image
             if acousticBed.size > 0:
-                bedMin = max(int(np.nanmin(acousticBed)) - 50, 0)
-                bedMax = int(np.nanmax(acousticBed)) + pix_buf
+                bedMin = max(int(np.nanpercentile(acousticBed, 5)) - 50, 0)
+                bedMax = int(np.nanpercentile(acousticBed, 95)) + pix_buf
             else:
                 bedMin = 0
                 bedMax = H
@@ -1570,6 +1647,87 @@ class portstarObj(object):
                 return '0 pixels'
             return str(float(adjDep) / float(valid_pix.iloc[0])) + ' pixels'
 
+        def _rolling_median(vals, window=31):
+            return pd.Series(vals).rolling(window=window, center=True, min_periods=1).median().to_numpy()
+
+        def _flag_depth_outliers(depth, inst_depth=None, inst_depth_mult=None,
+             resid_floor_m=0.5, jump_floor_m=0.5,
+             iterative_jump=False, iterative_max_iter=64,
+             iterative_jump_floor_m=None):
+            depth = np.asarray(depth, dtype=float)
+            flags = np.zeros(depth.shape, dtype=bool)
+
+            valid = np.isfinite(depth) & (depth > 0)
+            if not valid.any():
+                return flags
+
+            med = _rolling_median(depth)
+            resid = np.abs(depth - med)
+            resid_valid = resid[valid]
+            resid_center = np.nanmedian(resid_valid)
+            resid_mad = np.nanmedian(np.abs(resid_valid - resid_center))
+            resid_thr = max(resid_floor_m, 6.0 * resid_mad if np.isfinite(resid_mad) else resid_floor_m)
+            flags |= valid & (resid > resid_thr)
+
+            if depth.size >= 3:
+                prev_step = depth[1:-1] - depth[:-2]
+                next_step = depth[2:] - depth[1:-1]
+                step_mag = np.abs(np.concatenate((prev_step, next_step)))
+                step_mag = step_mag[np.isfinite(step_mag)]
+                if step_mag.size > 0:
+                    step_center = np.nanmedian(step_mag)
+                    step_mad = np.nanmedian(np.abs(step_mag - step_center))
+                    jump_thr = max(jump_floor_m, 6.0 * step_mad if np.isfinite(step_mad) else jump_floor_m)
+                    spike_mid = (
+                        valid[1:-1] & valid[:-2] & valid[2:] &
+                        (np.abs(prev_step) > jump_thr) &
+                        (np.abs(next_step) > jump_thr) &
+                        ((prev_step * next_step) < 0)
+                    )
+                    flags[1:-1] |= spike_mid
+
+            # Acoustic depth failures often persist as a step change rather than
+            # a single-ping spike (e.g., 2.4 m -> 6.3 m for many consecutive
+            # records). Iteratively flagging the later side of large jumps peels
+            # back those runs until continuity is restored.
+            if iterative_jump:
+                work = depth.copy()
+                if iterative_jump_floor_m is None:
+                    iterative_jump_floor_m = jump_floor_m
+                # Cap iterations so very large tracks do not degrade to near O(n^2)
+                # behavior when many jump-like artifacts are present.
+                max_iter = max(1, int(iterative_max_iter))
+                for _ in range(max_iter):
+                    valid_work = np.isfinite(work) & (work > 0)
+                    valid_idx = np.flatnonzero(valid_work)
+                    if valid_idx.size < 2:
+                        break
+
+                    step_vals = np.abs(np.diff(work[valid_idx]))
+                    step_vals = step_vals[np.isfinite(step_vals)]
+                    if step_vals.size == 0:
+                        break
+
+                    step_center = np.nanmedian(step_vals)
+                    step_mad = np.nanmedian(np.abs(step_vals - step_center))
+                    jump_thr = max(iterative_jump_floor_m, 6.0 * step_mad if np.isfinite(step_mad) else iterative_jump_floor_m)
+
+                    diffs = np.abs(np.diff(work[valid_idx]))
+                    bad_step_pos = np.flatnonzero(np.isfinite(diffs) & (diffs > jump_thr))
+                    if bad_step_pos.size == 0:
+                        break
+
+                    new_bad_idx = valid_idx[bad_step_pos + 1]
+                    flags[new_bad_idx] = True
+                    work[new_bad_idx] = np.nan
+
+            if inst_depth is not None and inst_depth_mult is not None:
+                inst_depth = np.asarray(inst_depth, dtype=float)
+                inst_valid = np.isfinite(inst_depth) & (inst_depth > 0)
+                flags |= valid & inst_valid & (depth > (inst_depth * inst_depth_mult))
+
+            return flags
+
         def _sync_trackline_depth(beam_obj, beam_df):
             trk_file = os.path.join(beam_obj.metaDir, 'Trackline_Smth_' + beam_obj.beamName + '.csv')
             if not os.path.exists(trk_file):
@@ -1586,6 +1744,8 @@ class portstarObj(object):
                 depth_cols.append('dep_m_smth')
             if 'dep_m_adjBy' in beam_df.columns:
                 depth_cols.append('dep_m_adjBy')
+            if 'dep_m_interp' in beam_df.columns:
+                depth_cols.append('dep_m_interp')
 
             depth_df = beam_df[depth_cols].drop_duplicates(subset=['record_num'], keep='last').set_index('record_num')
             trk_df = trk_df.set_index('record_num')
@@ -1597,18 +1757,36 @@ class portstarObj(object):
                 trk_df['dep_m_smth'] = depth_df['dep_m_smth']
             if 'dep_m_adjBy' in depth_df.columns:
                 trk_df['dep_m_adjBy'] = depth_df['dep_m_adjBy']
+            if 'dep_m_interp' in depth_df.columns:
+                trk_df['dep_m_interp'] = depth_df['dep_m_interp']
 
             trk_df.reset_index().to_csv(trk_file, index=False, float_format='%.14f')
 
+        depth_timer_start = time.perf_counter()
+        depth_timer_last = depth_timer_start
+        beam_pair = '{} / {}'.format(self.port.beamName, self.star.beamName)
+
+        def _depth_timing(label):
+            nonlocal depth_timer_last
+            now = time.perf_counter()
+            print('\tDepth timing [{}] {}: {:.2f}s'.format(beam_pair, label, now - depth_timer_last))
+            depth_timer_last = now
+
         # Load sonar metadata file
         self.port._loadSonMeta()
         portDF = self.port.sonMetaDF
         self.star._loadSonMeta()
         starDF = self.star.sonMetaDF
+        _depth_timing('load metadata')
 
         # Get all chunks
         chunks = pd.unique(portDF['chunk_id'])
 
+        # Track points invalidated during pre-smoothing QC so output metadata
+        # reflects all flagged depth values, not only post-smoothing flags.
+        portPreFlags = np.zeros(len(portDF), dtype=bool)
+        starPreFlags = np.zeros(len(starDF), dtype=bool)
+
         if detectDep == 0:
             def _depth_series(df):
                 # Some formats (e.g., Garmin RSD) may not include dep_m in metadata.
@@ -1630,6 +1808,36 @@ class portstarObj(object):
             portInstDepth = np.where(portValid, portInstDepth, portMetaDepth)
             starInstDepth = np.where(starValid, starInstDepth, starMetaDepth)
 
+            # Flag outliers on raw instrument depth BEFORE smoothing so sharp
+            # jumps (e.g. sonar lock-on to a false deep target) are caught on
+            # the un-blurred signal rather than on the smoothed ramp they become
+            # after savgol. NaNs are linearly filled so savgol has no gaps.
+            def _fill_nans_linear(arr):
+                nans = np.isnan(arr) | (arr <= 0)
+                x = np.arange(len(arr))
+                if nans.any() and (~nans).any():
+                    arr[nans] = np.interp(x[nans], x[~nans], arr[~nans])
+                return arr
+
+            # Keep one-ping spike sensitivity, but require a much larger jump for
+            # iterative run peeling so true depth regime changes are not over-flagged.
+            portPreFlags = _flag_depth_outliers(
+                portInstDepth,
+                iterative_jump=True,
+                iterative_max_iter=512,
+                iterative_jump_floor_m=4.0,
+            )
+            starPreFlags = _flag_depth_outliers(
+                starInstDepth,
+                iterative_jump=True,
+                iterative_max_iter=512,
+                iterative_jump_floor_m=4.0,
+            )
+            portInstDepth[portPreFlags] = np.nan
+            starInstDepth[starPreFlags] = np.nan
+            portInstDepth = _fill_nans_linear(portInstDepth)
+            starInstDepth = _fill_nans_linear(starInstDepth)
+
             if smthDep:
                 # print("\nSmoothing depth values...")
                 portInstDepth = savgol_filter(portInstDepth, 51, 3)
@@ -1662,6 +1870,7 @@ class portstarObj(object):
 
             portDF['dep_m_adjBy'] = _format_depth_adjustment(portDF['pixM'])
             starDF['dep_m_adjBy'] = _format_depth_adjustment(starDF['pixM'])
+            _depth_timing('prepare instrument depth')
 
         elif detectDep > 0:
             # Prepare depth detection dictionaries
@@ -1748,11 +1957,66 @@ class portstarObj(object):
 
             portDF['dep_m_adjBy'] = _format_depth_adjustment(portDF['pixM'])
             starDF['dep_m_adjBy'] = _format_depth_adjustment(starDF['pixM'])
+            _depth_timing('prepare detected depth')
+
+        # Outlier and jump filtering before interpolation.
+        # detectDep=0: continuity-only acoustic QC.
+        # detectDep=1/2: continuity QC plus instrument-depth proportional cap.
+        portArr = pd.to_numeric(portDF['dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
+        starArr = pd.to_numeric(starDF['dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
+        portRawArr = portArr.copy()
+        starRawArr = starArr.copy()
+        portInst = pd.to_numeric(portDF['inst_dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
+        starInst = pd.to_numeric(starDF['inst_dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
+
+        portFlags = np.zeros(portArr.shape, dtype=bool)
+        starFlags = np.zeros(starArr.shape, dtype=bool)
+
+        if detectDep == 0:
+            portFlags |= _flag_depth_outliers(portArr, iterative_jump=True, iterative_jump_floor_m=4.0)
+            starFlags |= _flag_depth_outliers(starArr, iterative_jump=True, iterative_jump_floor_m=4.0)
+        elif detectDep in (1, 2):
+            portFlags |= _flag_depth_outliers(portArr, portInst, inst_depth_mult=3.0)
+            starFlags |= _flag_depth_outliers(starArr, starInst, inst_depth_mult=3.0)
+
+            # If sides diverge strongly, invalidate the side farther from
+            # instrument depth so interpolation can recover continuity.
+            pair_valid = np.isfinite(portArr) & np.isfinite(starArr) & (portArr > 0) & (starArr > 0)
+            diverge = pair_valid & (np.abs(portArr - starArr) > 5.0)
+            if diverge.any():
+                portErr = np.abs(portArr - portInst)
+                starErr = np.abs(starArr - starInst)
+
+                inst_pair_valid = np.isfinite(portInst) & (portInst > 0) & np.isfinite(starInst) & (starInst > 0)
+                choose_by_inst = diverge & inst_pair_valid
+                portFlags |= choose_by_inst & (portErr >= starErr)
+                starFlags |= choose_by_inst & (starErr > portErr)
+
+                # Fallback for rows without valid instrument depth on one/both sides.
+                fallback = diverge & (~inst_pair_valid)
+                if fallback.any():
+                    pmed = _rolling_median(portArr)
+                    smed = _rolling_median(starArr)
+                    presid = np.abs(portArr - pmed)
+                    sresid = np.abs(starArr - smed)
+                    portFlags |= fallback & (presid >= sresid)
+                    starFlags |= fallback & (sresid > presid)
+
+        portArr[portFlags] = np.nan
+        starArr[starFlags] = np.nan
+        portDF['dep_m'] = portArr
+        portDF['dep_m_raw'] = portRawArr
+        starDF['dep_m'] = starArr
+        starDF['dep_m_raw'] = starRawArr
+        _depth_timing('outlier and jump filtering')
 
         # Interpolate over nan's (and set zeros to nan)
         portDep = portDF['dep_m'].to_numpy(copy=True)
         starDep = starDF['dep_m'].to_numpy(copy=True)
 
+        portInterp = np.isnan(portDep) | (portDep == 0) | portPreFlags
+        starInterp = np.isnan(starDep) | (starDep == 0) | starPreFlags
+
         portDep[portDep == 0] = np.nan
         starDep[starDep == 0] = np.nan
 
@@ -1762,6 +2026,7 @@ class portstarObj(object):
         else:
             portDep[nans] = 0
         portDF['dep_m'] = portDep
+        portDF['dep_m_interp'] = portInterp.astype(np.uint8)
 
         nans, x = np.isnan(starDep), lambda z: z.nonzero()[0]
         if (~nans).any():
@@ -1769,27 +2034,38 @@ class portstarObj(object):
         else:
             starDep[nans] = 0
         starDF['dep_m'] = starDep
+        starDF['dep_m_interp'] = starInterp.astype(np.uint8)
+        _depth_timing('interpolate depth')
 
         # Export to csv
         portDF.to_csv(self.port.sonMetaFile, index=False, float_format='%.14f')
         starDF.to_csv(self.star.sonMetaFile, index=False, float_format='%.14f')
         _sync_trackline_depth(self.port, portDF)
         _sync_trackline_depth(self.star, starDF)
+        _depth_timing('write metadata and trackline')
 
         try:
             # Take average of both estimates to store with downlooking sonar csv
-            depDF = pd.DataFrame(columns=['dep_m', 'dep_m_Method', 'dep_m_smth', 'dep_m_adjBy'])
+            depDF = pd.DataFrame(columns=['dep_m', 'dep_m_Method', 'dep_m_smth', 'dep_m_adjBy', 'dep_m_interp'])
             depDF['dep_m'] = np.nanmean([portDF['dep_m'].to_numpy(), starDF['dep_m'].to_numpy()], axis=0)
             depDF['dep_m_Method'] = portDF['dep_m_Method']
             depDF['dep_m_smth'] = portDF['dep_m_smth']
             depDF['dep_m_adjBy'] = portDF['dep_m_adjBy']
+            depDF['dep_m_interp'] = np.maximum(
+                pd.to_numeric(portDF['dep_m_interp'], errors='coerce').fillna(0).to_numpy(dtype=np.uint8, copy=True),
+                pd.to_numeric(starDF['dep_m_interp'], errors='coerce').fillna(0).to_numpy(dtype=np.uint8, copy=True)
+            )
         except:
             # In case port and star are not same length
-            depDF = pd.DataFrame(columns=['dep_m', 'dep_m_Method', 'dep_m_smth', 'dep_m_adjBy'])
+            depDF = pd.DataFrame(columns=['dep_m', 'dep_m_Method', 'dep_m_smth', 'dep_m_adjBy', 'dep_m_interp'])
             depDF['dep_m'] = portDF['dep_m']
             depDF['dep_m_Method'] = portDF['dep_m_Method']
             depDF['dep_m_smth'] = portDF['dep_m_smth']
             depDF['dep_m_adjBy'] = portDF['dep_m_adjBy']
+            depDF['dep_m_interp'] = portDF['dep_m_interp']
+
+        _depth_timing('build return depth dataframe')
+        print('\tDepth timing [{}] total: {:.2f}s'.format(beam_pair, time.perf_counter() - depth_timer_start))
 
         del portDF, starDF
         gc.collect()
@@ -1847,20 +2123,65 @@ class portstarObj(object):
         self.star._loadSonMeta()
         starDF = self.star.sonMetaDF
 
-        portDF = portDF.loc[portDF['chunk_id'] == i, ['inst_dep_m', 'dep_m', 'pixM']]
-        starDF = starDF.loc[starDF['chunk_id'] == i, ['inst_dep_m', 'dep_m', 'pixM']]
+        def _depth_to_pixels(df, depth_col):
+            depth = pd.to_numeric(df[depth_col], errors='coerce').to_numpy(dtype=float, copy=True)
+            pix = pd.to_numeric(df['pixM'], errors='coerce').to_numpy(dtype=float, copy=True)
+            return np.divide(
+                depth,
+                pix,
+                out=np.full(depth.shape, np.nan, dtype=float),
+                where=np.isfinite(pix) & (pix != 0)
+            )
+
+        def _pad_to_match(values, reference):
+            diff = reference.shape[0] - values.shape[0]
+            if diff > 0:
+                values = np.append(values, reference[-diff:])
+            return values
+
+        portCols = ['inst_dep_m', 'dep_m', 'pixM']
+        starCols = ['inst_dep_m', 'dep_m', 'pixM']
+        if 'dep_m_interp' in portDF.columns:
+            portCols.append('dep_m_interp')
+        if 'dep_m_raw' in portDF.columns:
+            portCols.append('dep_m_raw')
+        if 'dep_m_interp' in starDF.columns:
+            starCols.append('dep_m_interp')
+        if 'dep_m_raw' in starDF.columns:
+            starCols.append('dep_m_raw')
+
+        portDF = portDF.loc[portDF['chunk_id'] == i, portCols]
+        starDF = starDF.loc[starDF['chunk_id'] == i, starCols]
+
+        detectDepMode = int(getattr(self.port, 'detectDep', getattr(self.star, 'detectDep', -1)))
 
         portInstDepth = pd.to_numeric(portDF['inst_dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
         portMetaDepth = pd.to_numeric(portDF['dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
-        portInstDepth = np.where(np.isfinite(portInstDepth) & (portInstDepth > 0), portInstDepth, portMetaDepth)
-        portInst = np.nan_to_num(portInstDepth / portDF['pixM'].to_numpy(dtype=float), nan=0.0).astype(int)
-        portAuto = (portDF['dep_m'] / portDF['pixM']).to_numpy(dtype=int, copy=True)
+        if not np.isfinite(portInstDepth).any():
+            portInstDepth = portMetaDepth.copy()
+        portDF = portDF.copy()
+        portDF['inst_depth_plot'] = portInstDepth
+        portInst = np.nan_to_num(_depth_to_pixels(portDF, 'inst_depth_plot'), nan=0.0)
+        portAuto = _depth_to_pixels(portDF, 'dep_m')
+        portRaw = _depth_to_pixels(portDF, 'dep_m_raw') if 'dep_m_raw' in portDF.columns else portAuto.copy()
+        if 'dep_m_interp' in portDF.columns:
+            portInterp = pd.to_numeric(portDF['dep_m_interp'], errors='coerce').fillna(0).to_numpy(dtype=np.uint8, copy=True).astype(bool)
+        else:
+            portInterp = np.zeros(portAuto.shape, dtype=bool)
 
         starInstDepth = pd.to_numeric(starDF['inst_dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
         starMetaDepth = pd.to_numeric(starDF['dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
-        starInstDepth = np.where(np.isfinite(starInstDepth) & (starInstDepth > 0), starInstDepth, starMetaDepth)
-        starInst = np.nan_to_num(starInstDepth / starDF['pixM'].to_numpy(dtype=float), nan=0.0).astype(int)
-        starAuto = (starDF['dep_m'] / starDF['pixM']).to_numpy(dtype=int, copy=True)
+        if not np.isfinite(starInstDepth).any():
+            starInstDepth = starMetaDepth.copy()
+        starDF = starDF.copy()
+        starDF['inst_depth_plot'] = starInstDepth
+        starInst = np.nan_to_num(_depth_to_pixels(starDF, 'inst_depth_plot'), nan=0.0)
+        starAuto = _depth_to_pixels(starDF, 'dep_m')
+        starRaw = _depth_to_pixels(starDF, 'dep_m_raw') if 'dep_m_raw' in starDF.columns else starAuto.copy()
+        if 'dep_m_interp' in starDF.columns:
+            starInterp = pd.to_numeric(starDF['dep_m_interp'], errors='coerce').fillna(0).to_numpy(dtype=np.uint8, copy=True).astype(bool)
+        else:
+            starInterp = np.zeros(starAuto.shape, dtype=bool)
 
         # Ensure port/star same length
         if (portAuto.shape[0] != starAuto.shape[0]):
@@ -1868,27 +2189,37 @@ class portstarObj(object):
             sL = starAuto.shape[0]
             # Add rows to shortest array from longest array
             if (pL > sL):
-                starAuto = np.append(starAuto, portAuto[(sL-pL):])
-                starInst = np.append(starInst, portInst[(sL-pL):])
+                starAuto = _pad_to_match(starAuto, portAuto)
+                starInst = _pad_to_match(starInst, portInst)
+                starRaw = _pad_to_match(starRaw, portRaw)
+                starInterp = _pad_to_match(starInterp, portInterp)
             else:
-                portAuto = np.append(portAuto, starAuto[(pL-sL):])
-                portInst = np.append(portInst, starInst[(pL-sL):])
+                portAuto = _pad_to_match(portAuto, starAuto)
+                portInst = _pad_to_match(portInst, starInst)
+                portRaw = _pad_to_match(portRaw, starRaw)
+                portInterp = _pad_to_match(portInterp, starInterp)
 
         # Relocate depths relative to horizontal center of image
         c = int(mergeSon.shape[1]/2)
 
         portInst = c - portInst
         portAuto = c - portAuto
+        portRaw = c - portRaw
 
         starInst = c + starInst
         starAuto = c + starAuto
+        starRaw = c + starRaw
 
         # maybe flip???
         portInst = np.flip(portInst)
         portAuto = np.flip(portAuto)
+        portRaw = np.flip(portRaw)
+        portInterp = np.flip(portInterp)
 
         starInst = np.flip(starInst)
         starAuto = np.flip(starAuto)
+        starRaw = np.flip(starRaw)
+        starInterp = np.flip(starInterp)
 
         #############
         # Export Plot
@@ -1915,14 +2246,31 @@ class portstarObj(object):
         outFile = os.path.join(outDir, projName+'_Bedpick_'+addZero+str(i)+tileFile)
 
         plt.imshow(mergeSon, cmap='gray')
-        if acousticBed:
-            plt.plot(portInst, y, 'r-.', lw=1, label='Acoustic Depth')
-            plt.plot(starInst, y, 'r-.', lw=1)
-            del portInst, starInst
-        if autoBed:
-            plt.plot(portAuto, y, 'b-.', lw=1, label='Auto Depth')
-            plt.plot(starAuto, y, 'b-.', lw=1)
-            del portAuto, starAuto
+        if detectDepMode == 0:
+            if acousticBed:
+                portInstGood = np.where(~portInterp, portInst, np.nan)
+                starInstGood = np.where(~starInterp, starInst, np.nan)
+                portInstBad = np.where(portInterp, portInst, np.nan)
+                starInstBad = np.where(starInterp, starInst, np.nan)
+                portInterpDepth = np.where(portInterp, portAuto, np.nan)
+                starInterpDepth = np.where(starInterp, starAuto, np.nan)
+
+                plt.plot(portInstGood, y, '-.', color='lime', lw=1, label='Instrument Depth (Good)')
+                plt.plot(starInstGood, y, '-.', color='lime', lw=1)
+                plt.plot(portInstBad, y, '-.', color='red', lw=1, label='Instrument Depth (Flagged)')
+                plt.plot(starInstBad, y, '-.', color='red', lw=1)
+                plt.plot(portInterpDepth, y, '-.', color='yellow', lw=1, label='Interpolated Depth')
+                plt.plot(starInterpDepth, y, '-.', color='yellow', lw=1)
+                del portInstGood, starInstGood, portInstBad, starInstBad, portInterpDepth, starInterpDepth
+        else:
+            if acousticBed:
+                plt.plot(portInst, y, 'r-.', lw=1, label='Instrument Depth')
+                plt.plot(starInst, y, 'r-.', lw=1)
+            if autoBed:
+                plt.plot(portAuto, y, 'b-.', lw=1, label='Auto Depth')
+                plt.plot(starAuto, y, 'b-.', lw=1)
+
+        del portInst, starInst, portAuto, starAuto, portRaw, starRaw, portInterp, starInterp
 
         plt.legend(loc = 'lower right', prop={'size':4}) # create the plot legend
         plt.savefig(outFile, dpi=300, bbox_inches='tight')

{pingmapper-5.4.0 → pingmapper-5.4.2}/pingmapper/class_rectObj.py

@@ -2151,6 +2151,7 @@ class rectObj(sonObj):
 
         # Open smoothed trackline/range extent file
         trkMeta = pd.read_csv(trkMetaFile)
+        trkMeta = self._sanitizeProjectedTrackMeta(trkMeta, wgs=wgs)
 
         # Create geodataframe
         gdf = gpd.GeoDataFrame(
@@ -2165,6 +2166,59 @@ class rectObj(sonObj):
         del trkMetaFile        
 
         return
+
+    #===========================================================================
+    def _sanitizeProjectedTrackMeta(self, trkMeta, wgs=False):
+
+        if wgs:
+            return trkMeta
+
+        trkMeta = trkMeta.copy()
+
+        def _reproject_from_lonlat(df, lon_col, lat_col, x_col, y_col):
+            if lon_col not in df.columns or lat_col not in df.columns:
+                return df
+
+            lon = pd.to_numeric(df[lon_col], errors='coerce')
+            lat = pd.to_numeric(df[lat_col], errors='coerce')
+            valid_lonlat = lon.notna() & lat.notna()
+            if not valid_lonlat.any():
+                return df
+
+            need_xy = pd.Series(False, index=df.index)
+            if x_col not in df.columns or y_col not in df.columns:
+                need_xy[:] = True
+            else:
+                x = pd.to_numeric(df[x_col], errors='coerce')
+                y = pd.to_numeric(df[y_col], errors='coerce')
+
+                # EPSG:326xx eastings should be positive and reasonably bounded.
+                # Rebuild projected XY when stored values are missing or obviously invalid.
+                need_xy = (
+                    x.isna() |
+                    y.isna() |
+                    (x <= 0) |
+                    (x > 1000000) |
+                    (y <= 0)
+                ) & valid_lonlat
+
+            if not need_xy.any():
+                return df
+
+            geo = gpd.GeoDataFrame(
+                df.loc[need_xy, [lon_col, lat_col]].copy(),
+                geometry=gpd.points_from_xy(lon[need_xy], lat[need_xy]),
+                crs='EPSG:4326',
+            ).to_crs(self.humDat['epsg'])
+
+            df.loc[need_xy, x_col] = geo.geometry.x.to_numpy()
+            df.loc[need_xy, y_col] = geo.geometry.y.to_numpy()
+            return df
+
+        trkMeta = _reproject_from_lonlat(trkMeta, 'trk_lons', 'trk_lats', 'trk_utm_es', 'trk_utm_ns')
+        trkMeta = _reproject_from_lonlat(trkMeta, 'range_lons', 'range_lats', 'range_es', 'range_ns')
+
+        return trkMeta
     
     #===========================================================================
     def _exportCovShp(self,
@@ -2212,6 +2266,7 @@ class rectObj(sonObj):
         # dfs = [df1, df2]
 
         df1 = pd.read_csv(self.smthTrkFile)
+        df1 = self._sanitizeProjectedTrackMeta(df1, wgs=wgs)
         dfs = [df1]
 
         filt = 0

{pingmapper-5.4.0 → pingmapper-5.4.2}/pingmapper/class_sonObj.py

@@ -260,6 +260,8 @@ class sonObj(object):
                           dq_src_utc_offset=0.0,
                           dq_target_utc_offset=0.0,
                           dq_time_offset=0.0,
+                          filter_coord_outliers=True,
+                          coord_iqr_scale=3.0,
                           ):
         '''
         '''
@@ -271,6 +273,11 @@ class sonObj(object):
         # print('len', len(sonDF))
         # print(sonDF)
 
+        ##############################
+        # GPS Coordinate Outlier Filter
+        if filter_coord_outliers:
+            sonDF = self._filterCoordOutliers(sonDF, iqr_scale=coord_iqr_scale)
+
         #############################
         # Do Heading Deviation Filter
         if max_heading_dev > 0:
@@ -690,7 +697,16 @@ class sonObj(object):
         if not filtCol in sonDF.columns:
             sonDF[filtCol] = True
 
-        time_table = pd.read_csv(time_table)
+        # Guard against bool/test toggles; only load when a real table is provided.
+        if isinstance(time_table, (bool, np.bool_)) or time_table is None:
+            return sonDF
+        if isinstance(time_table, str) and time_table.strip() == '':
+            return sonDF
+
+        if isinstance(time_table, pd.DataFrame):
+            time_table = time_table.copy()
+        else:
+            time_table = pd.read_csv(time_table)
 
         for i, row in time_table.iterrows():
 
@@ -733,6 +749,57 @@ class sonObj(object):
 
         return sonDF
 
+    # ======================================================================
+    def _filterCoordOutliers(self,
+                              sonDF,
+                              iqr_scale=3.0):
+        '''
+        Flag pings with extreme GPS coordinate outliers using the IQR method
+        on the lon and lat columns.  For each coordinate field, pings that fall
+        outside  Q1 - iqr_scale*IQR .. Q3 + iqr_scale*IQR  are marked False
+        in the 'filter' column.
+
+        ----------
+        Parameters
+        ----------
+        sonDF : DataFrame
+            Ping metadata dataframe.
+        iqr_scale : float
+            Multiplier applied to the IQR to set the outlier fence.
+            Default 3.0 (flags only extreme outliers).
+
+        -------
+        Returns
+        -------
+        sonDF with 'filter' column updated.
+        '''
+
+        filtCol = 'filter'
+
+        if filtCol not in sonDF.columns:
+            sonDF[filtCol] = True
+
+        for coord_col in ['lon', 'lat']:
+            if coord_col not in sonDF.columns:
+                continue
+
+            vals = sonDF[coord_col]
+            q1 = vals.quantile(0.25)
+            q3 = vals.quantile(0.75)
+            iqr = q3 - q1
+
+            lower = q1 - iqr_scale * iqr
+            upper = q3 + iqr_scale * iqr
+
+            outlier_mask = (vals < lower) | (vals > upper)
+            n_flagged = outlier_mask.sum()
+            if n_flagged > 0:
+                print(f"\n  _filterCoordOutliers: flagged {n_flagged} pings with {coord_col} outside "
+                      f"[{lower:.6f}, {upper:.6f}]")
+            sonDF.loc[outlier_mask, filtCol] = False
+
+        return sonDF
+
     # ======================================================================
     def _filterAOI(self,
                    sonDF,

{pingmapper-5.4.0 → pingmapper-5.4.2}/pingmapper/class_sonObj_nadirgaptest.py

@@ -675,7 +675,16 @@ class sonObj(object):
         if not filtCol in sonDF.columns:
             sonDF[filtCol] = True
 
-        time_table = pd.read_csv(time_table)
+        # Guard against bool/test toggles; only load when a real table is provided.
+        if isinstance(time_table, (bool, np.bool_)) or time_table is None:
+            return sonDF
+        if isinstance(time_table, str) and time_table.strip() == '':
+            return sonDF
+
+        if isinstance(time_table, pd.DataFrame):
+            time_table = time_table.copy()
+        else:
+            time_table = pd.read_csv(time_table)
 
         for i, row in time_table.iterrows():
 

{pingmapper-5.4.0 → pingmapper-5.4.2}/pingmapper/main_readFiles.py

@@ -126,6 +126,8 @@ def read_master_func(logfilename='',
                      dq_src_utc_offset = 0.0,
                      dq_target_utc_offset = 0.0,
                      dq_time_offset = 0.0,
+                     filter_coord_outliers = True,
+                     coord_iqr_scale = 3.0,
                      tempC=10,
                      nchunk=500,
                      cropRange=0,
@@ -1025,7 +1027,7 @@ def read_master_func(logfilename='',
     # For Filtering                                                            #
     ############################################################################
 
-    if dq_table or max_heading_deviation > 0 or min_speed > 0 or max_speed > 0 or aoi or time_table:
+    if dq_table or max_heading_deviation > 0 or min_speed > 0 or max_speed > 0 or aoi or time_table or filter_coord_outliers:
 
         start_time = time.time()
 
@@ -1060,7 +1062,8 @@ def read_master_func(logfilename='',
         son0 = portstar[maxRec]
         df0 = son0._doSonarFiltering(max_heading_deviation, max_heading_distance, min_speed, max_speed, aoi, time_table,
                                       dq_table, dq_time_field, dq_flag_field, dq_keep_values,
-                                      dq_src_utc_offset, dq_target_utc_offset, dq_time_offset)
+                                      dq_src_utc_offset, dq_target_utc_offset, dq_time_offset,
+                                      filter_coord_outliers=filter_coord_outliers, coord_iqr_scale=coord_iqr_scale)
 
         # Add filter to other beam
         son1 = portstar[minRec]
@@ -1110,7 +1113,8 @@ def read_master_func(logfilename='',
         for son in downbeams:
             df = son._doSonarFiltering(max_heading_deviation, max_heading_distance, min_speed, max_speed, aoi, time_table,
                                         dq_table, dq_time_field, dq_flag_field, dq_keep_values,
-                                        dq_src_utc_offset, dq_target_utc_offset, dq_time_offset)
+                                        dq_src_utc_offset, dq_target_utc_offset, dq_time_offset,
+                                        filter_coord_outliers=filter_coord_outliers, coord_iqr_scale=coord_iqr_scale)
 
             df = df[df['filter'] == True]
 

pingmapper-5.4.2/pingmapper/version.py

@@ -0,0 +1 @@
+__version__ = '5.4.2'

{pingmapper-5.4.0 → pingmapper-5.4.2}/pingmapper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 5.4.0
+Version: 5.4.2
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Daniel Buscombe
 Author-email: Cameron Bodine <bodine.cs@gmail.email>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering :: Visualization
 Classifier: Topic :: Scientific/Engineering :: Oceanography
 Classifier: Topic :: Scientific/Engineering :: GIS
 Classifier: Topic :: Scientific/Engineering :: Hydrology
-Requires-Python: >=3.10
+Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pinginstaller<3,>=2
@@ -33,6 +33,7 @@ Requires-Dist: tensorflow<3,>=2.20; extra == "ml"
 Requires-Dist: tf-keras<3,>=2.20; extra == "ml"
 Requires-Dist: transformers<5,>=4.57; extra == "ml"
 Dynamic: license-file
+Dynamic: requires-python
 
 # PING-Mapper
 [![PyPI - Version](https://img.shields.io/pypi/v/pingmapper?style=flat-square&label=Latest%20Version%20(PyPi))](https://pypi.org/project/pingmapper/)

{pingmapper-5.4.0 → pingmapper-5.4.2}/pingmapper.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 LICENSE
 README.md
 pyproject.toml
+setup.py
 pingmapper/__init__.py
 pingmapper/__main__.py
 pingmapper/class_mapSubstrateObj.py

pingmapper-5.4.2/setup.py

@@ -0,0 +1,50 @@
+from setuptools import setup, find_packages
+from pathlib import Path
+
+DESCRIPTION = 'Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat'
+LONG_DESCRIPTION = Path('README.md').read_text()
+
+exec(open('pingmapper/version.py').read())
+
+setup(
+    name="pingmapper",
+    version=__version__,
+    author="Cameron Bodine, Daniel Buscombe",
+    author_email="bodine.cs@gmail.email",
+    description=DESCRIPTION,
+    long_description=LONG_DESCRIPTION,
+    long_description_content_type='text/markdown',
+    packages=find_packages(),
+    data_files=[("pingmapper_config", ["pingmapper/default_params.json"])],
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Scientific/Engineering :: Visualization",
+        "Topic :: Scientific/Engineering :: Oceanography",
+        "Topic :: Scientific/Engineering :: GIS",
+        "Topic :: Scientific/Engineering :: Hydrology"
+        ],
+    keywords=[
+        "pingmapper",
+        "sonar",
+        "ecology",
+        "remotesensing",
+        "sidescan",
+        "sidescan-sonar",
+        "aquatic",
+        "humminbird",
+        "lowrance",
+        "gis",
+        "oceanography",
+        "limnology",],
+    python_requires=">=3.6",
+    install_requires=['pinginstaller', 'pingwizard', 'pingverter'],
+    project_urls={
+        "Issues": "https://github.com/CameronBodine/PINGMapper/issues",
+        "GitHub":"https://github.com/CameronBodine/PINGMapper",
+        "Homepage":"https://cameronbodine.github.io/PINGMapper/",
+    },
+)

pingmapper-5.4.0/pingmapper/version.py

@@ -1 +0,0 @@
-__version__ = '5.4.0'