pingmapper 5.3.7__tar.gz → 5.4.0__tar.gz

{pingmapper-5.3.7 → pingmapper-5.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 5.3.7
+Version: 5.4.0
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Daniel Buscombe
 Author-email: Cameron Bodine <bodine.cs@gmail.email>

{pingmapper-5.3.7 → pingmapper-5.4.0}/pingmapper/class_sonObj.py

@@ -253,6 +253,13 @@ class sonObj(object):
                           max_speed,
                           aoi,
                           time_table,
+                          dq_table=False,
+                          dq_time_field=False,
+                          dq_flag_field=False,
+                          dq_keep_values=False,
+                          dq_src_utc_offset=0.0,
+                          dq_target_utc_offset=0.0,
+                          dq_time_offset=0.0,
                           ):
         '''
         '''
@@ -284,6 +291,20 @@ class sonObj(object):
         if time_table:
             sonDF = self._filterTime(sonDF, time_table)
 
+        ####################
+        # Data Quality Filter
+        if dq_table:
+            sonDF = self._filterDQ(
+                sonDF,
+                dq_table,
+                dq_time_field,
+                dq_flag_field,
+                dq_keep_values,
+                dq_src_utc_offset,
+                dq_target_utc_offset,
+                dq_time_offset,
+            )
+
         return sonDF
     
     # ======================================================================
@@ -304,6 +325,232 @@ class sonObj(object):
         return df
 
 
+    # ======================================================================
+    def _filterDQ(self,
+                  sonDF,
+                  dq_table,
+                  dq_time_field,
+                  dq_flag_field,
+                  dq_keep_values,
+                  dq_src_utc_offset,
+                  dq_target_utc_offset,
+                  dq_time_offset,
+                  ):
+        '''
+        Filter sonar pings using a data-quality (DQ) log CSV.
+
+        Each row in the DQ log is treated as a state-change event: the flag
+        recorded at time T applies to every sonar ping from T until the next
+        event row. Pings that occur before the first DQ event are removed.
+        '''
+
+        filtDQCol = 'filter_dq'
+        filtCol = 'filter'
+        dqTimeCol = '_dq_ts'
+
+        if not dq_time_field:
+            raise ValueError('dq_time_field is required when dq_table is provided.')
+        if not dq_flag_field:
+            raise ValueError('dq_flag_field is required when dq_table is provided.')
+
+        keep_vals = self._normalizeDQKeepValues(dq_keep_values)
+        if len(keep_vals) == 0:
+            raise ValueError('dq_keep_values must contain at least one value to keep.')
+
+        sonDF = sonDF.copy()
+        sonDF[filtDQCol] = False
+
+        if filtCol not in sonDF.columns:
+            sonDF[filtCol] = True
+
+        dqDF = pd.read_csv(dq_table)
+        missing_cols = [c for c in [dq_time_field, dq_flag_field] if c not in dqDF.columns]
+        if missing_cols:
+            raise ValueError('dqLog missing required column(s): {}'.format(', '.join(missing_cols)))
+
+        dqTimes, dqKind = self._coerceDQTimestampSeries(dqDF[dq_time_field], dq_time_field)
+        sonTimes, sonKind = self._getSonarFilterTimestamp(sonDF)
+
+        if dqKind != sonKind:
+            raise ValueError(
+                'dqLog timestamp type ({}) does not match sonar timestamp type ({}).'.format(dqKind, sonKind)
+            )
+
+        if dqKind == 'datetime':
+            dqTimes = self._shiftDQDatetimeToTargetOffset(
+                dqTimes,
+                dq_src_utc_offset,
+                dq_target_utc_offset,
+            )
+
+        dqDF = dqDF.copy()
+        dqDF[dqTimeCol] = dqTimes
+        dqDF = dqDF[dqDF[dqTimeCol].notna()].copy()
+        if dqDF.empty:
+            raise ValueError('dqLog contained no valid timestamps after parsing {}.'.format(dq_time_field))
+
+        offset = float(dq_time_offset)
+        if sonKind == 'datetime':
+            sonTimes = sonTimes + pd.to_timedelta(offset, unit='s')
+        else:
+            sonTimes = sonTimes + offset
+
+        sonMerge = pd.DataFrame({
+            '_son_idx': sonDF.index,
+            '_son_ts': sonTimes,
+        })
+        sonMerge = sonMerge[sonMerge['_son_ts'].notna()].copy()
+
+        dqDF['_dq_keep'] = dqDF[dq_flag_field].map(self._normalizeDQValue).isin(keep_vals)
+
+        event_state = dqDF[[dqTimeCol, '_dq_keep']].copy()
+        event_state.sort_values(dqTimeCol, inplace=True)
+        event_state = event_state.groupby(dqTimeCol, as_index=False)['_dq_keep'].last()
+
+        keep_idx = self._applyDQEventState(sonMerge, event_state, dqTimeCol)
+
+        sonDF.loc[keep_idx, filtDQCol] = True
+        sonDF[filtCol] = sonDF[filtCol] & sonDF[filtDQCol]
+
+        return sonDF
+
+    # ======================================================================
+    def _normalizeDQKeepValues(self, dq_keep_values):
+
+        if dq_keep_values is False or dq_keep_values is None:
+            return set()
+
+        if isinstance(dq_keep_values, str):
+            dq_keep_values = dq_keep_values.split(',')
+
+        keep_vals = set()
+        for value in dq_keep_values:
+            norm = self._normalizeDQValue(value)
+            if norm:
+                keep_vals.add(norm)
+
+        return keep_vals
+
+    # ======================================================================
+    def _normalizeDQValue(self, value):
+
+        if pd.isna(value):
+            return ''
+        return str(value).strip().lower()
+
+    # ======================================================================
+    def _coerceDQTimestampSeries(self, series, field_name):
+
+        non_na = series.dropna()
+        numeric = pd.to_numeric(series, errors='coerce')
+        if len(non_na) > 0 and numeric.notna().sum() == len(non_na):
+            return numeric, 'numeric'
+
+        dt = pd.to_datetime(series, errors='coerce')
+        if dt.notna().any():
+            try:
+                if dt.dt.tz is not None:
+                    dt = dt.dt.tz_localize(None)
+            except AttributeError:
+                pass
+            return dt, 'datetime'
+
+        if numeric.notna().any():
+            return numeric, 'numeric'
+
+        raise ValueError('Unable to parse dqLog timestamps from column: {}'.format(field_name))
+
+    # ======================================================================
+    def _shiftDQDatetimeToTargetOffset(self,
+                                       dq_times,
+                                       dq_src_utc_offset=False,
+                                       dq_target_utc_offset=False):
+
+        src_offset = self._coerceDQUtcOffset(dq_src_utc_offset, 'dq_src_utc_offset')
+        target_offset = self._coerceDQUtcOffset(dq_target_utc_offset, 'dq_target_utc_offset')
+
+        if src_offset is None and target_offset is None:
+            return dq_times
+
+        if src_offset is None or target_offset is None:
+            raise ValueError(
+                'dq_src_utc_offset and dq_target_utc_offset must both be provided when either is set.'
+            )
+
+        return dq_times + pd.to_timedelta(target_offset - src_offset, unit='h')
+
+    # ======================================================================
+    def _coerceDQUtcOffset(self, value, field_name):
+
+        if value is False or value is None or value == '':
+            return None
+
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            raise ValueError('{} must be a numeric UTC offset in hours.'.format(field_name))
+
+    # ======================================================================
+    def _getSonarFilterTimestamp(self, sonDF):
+
+        if 'date' in sonDF.columns and 'time' in sonDF.columns:
+            dt = pd.to_datetime(
+                sonDF['date'].astype(str).str.strip() + ' ' + sonDF['time'].astype(str).str.strip(),
+                errors='coerce',
+                format='mixed',
+            )
+            if dt.notna().any():
+                try:
+                    if dt.dt.tz is not None:
+                        dt = dt.dt.tz_localize(None)
+                except AttributeError:
+                    pass
+                return dt, 'datetime'
+
+        if 'time' in sonDF.columns:
+            dt = pd.to_datetime(sonDF['time'], errors='coerce', format='mixed')
+            if dt.notna().any():
+                try:
+                    if dt.dt.tz is not None:
+                        dt = dt.dt.tz_localize(None)
+                except AttributeError:
+                    pass
+                return dt, 'datetime'
+
+        if 'time_s' in sonDF.columns:
+            numeric = pd.to_numeric(sonDF['time_s'], errors='coerce')
+            if numeric.notna().any():
+                return numeric, 'numeric'
+
+        raise ValueError('Unable to determine sonar timestamps for dqLog filtering.')
+
+
+    # ======================================================================
+    def _applyDQEventState(self, son, event_state, dqTimeCol):
+        '''
+        Return the subset of sonar indices whose ping timestamp falls within a
+        "keep" state block as defined by the DQ event log.
+
+        Uses np.searchsorted so each ping inherits the state of the most-recent
+        event that preceded it.  Pings before the first event are excluded.
+        '''
+
+        event_times = event_state[dqTimeCol].to_numpy()
+        event_keep  = event_state['_dq_keep'].to_numpy(dtype=bool)
+        son_times   = son['_son_ts'].to_numpy()
+
+        # searchsorted(side='right') - 1 gives index of last event <= ping time
+        event_idx = np.searchsorted(event_times, son_times, side='right') - 1
+
+        # Pings before the first event get event_idx == -1 → exclude
+        valid_idx = event_idx >= 0
+
+        keep_mask = np.zeros(len(son), dtype=bool)
+        keep_mask[valid_idx] = event_keep[event_idx[valid_idx]]
+
+        return son.loc[keep_mask, '_son_idx']
+
+
     # ======================================================================
     def _filterHeading(self,
                        df,

{pingmapper-5.3.7 → pingmapper-5.4.0}/pingmapper/class_sonObj_nadirgaptest.py

@@ -253,6 +253,13 @@ class sonObj(object):
                           max_speed,
                           aoi,
                           time_table,
+                          dq_table=False,
+                          dq_time_field=False,
+                          dq_flag_field=False,
+                          dq_keep_values=False,
+                          dq_src_utc_offset=0.0,
+                          dq_target_utc_offset=0.0,
+                          dq_time_offset=0.0,
                           ):
         '''
         '''
@@ -284,6 +291,20 @@ class sonObj(object):
         if time_table:
             sonDF = self._filterTime(sonDF, time_table)
 
+        ####################
+        # Data Quality Filter
+        if dq_table:
+            sonDF = self._filterDQ(
+                sonDF,
+                dq_table,
+                dq_time_field,
+                dq_flag_field,
+                dq_keep_values,
+                dq_src_utc_offset,
+                dq_target_utc_offset,
+                dq_time_offset,
+            )
+
         return sonDF
     
     # ======================================================================
@@ -304,6 +325,232 @@ class sonObj(object):
         return df
 
 
+    # ======================================================================
+    def _filterDQ(self,
+                  sonDF,
+                  dq_table,
+                  dq_time_field,
+                  dq_flag_field,
+                  dq_keep_values,
+                  dq_src_utc_offset,
+                  dq_target_utc_offset,
+                  dq_time_offset,
+                  ):
+        '''
+        Filter sonar pings using a data-quality (DQ) log CSV.
+
+        Each row in the DQ log is treated as a state-change event: the flag
+        recorded at time T applies to every sonar ping from T until the next
+        event row. Pings that occur before the first DQ event are removed.
+        '''
+
+        filtDQCol = 'filter_dq'
+        filtCol = 'filter'
+        dqTimeCol = '_dq_ts'
+
+        if not dq_time_field:
+            raise ValueError('dq_time_field is required when dq_table is provided.')
+        if not dq_flag_field:
+            raise ValueError('dq_flag_field is required when dq_table is provided.')
+
+        keep_vals = self._normalizeDQKeepValues(dq_keep_values)
+        if len(keep_vals) == 0:
+            raise ValueError('dq_keep_values must contain at least one value to keep.')
+
+        sonDF = sonDF.copy()
+        sonDF[filtDQCol] = False
+
+        if filtCol not in sonDF.columns:
+            sonDF[filtCol] = True
+
+        dqDF = pd.read_csv(dq_table)
+        missing_cols = [c for c in [dq_time_field, dq_flag_field] if c not in dqDF.columns]
+        if missing_cols:
+            raise ValueError('dqLog missing required column(s): {}'.format(', '.join(missing_cols)))
+
+        dqTimes, dqKind = self._coerceDQTimestampSeries(dqDF[dq_time_field], dq_time_field)
+        sonTimes, sonKind = self._getSonarFilterTimestamp(sonDF)
+
+        if dqKind != sonKind:
+            raise ValueError(
+                'dqLog timestamp type ({}) does not match sonar timestamp type ({}).'.format(dqKind, sonKind)
+            )
+
+        if dqKind == 'datetime':
+            dqTimes = self._shiftDQDatetimeToTargetOffset(
+                dqTimes,
+                dq_src_utc_offset,
+                dq_target_utc_offset,
+            )
+
+        dqDF = dqDF.copy()
+        dqDF[dqTimeCol] = dqTimes
+        dqDF = dqDF[dqDF[dqTimeCol].notna()].copy()
+        if dqDF.empty:
+            raise ValueError('dqLog contained no valid timestamps after parsing {}.'.format(dq_time_field))
+
+        offset = float(dq_time_offset)
+        if sonKind == 'datetime':
+            sonTimes = sonTimes + pd.to_timedelta(offset, unit='s')
+        else:
+            sonTimes = sonTimes + offset
+
+        sonMerge = pd.DataFrame({
+            '_son_idx': sonDF.index,
+            '_son_ts': sonTimes,
+        })
+        sonMerge = sonMerge[sonMerge['_son_ts'].notna()].copy()
+
+        dqDF['_dq_keep'] = dqDF[dq_flag_field].map(self._normalizeDQValue).isin(keep_vals)
+
+        event_state = dqDF[[dqTimeCol, '_dq_keep']].copy()
+        event_state.sort_values(dqTimeCol, inplace=True)
+        event_state = event_state.groupby(dqTimeCol, as_index=False)['_dq_keep'].last()
+
+        keep_idx = self._applyDQEventState(sonMerge, event_state, dqTimeCol)
+
+        sonDF.loc[keep_idx, filtDQCol] = True
+        sonDF[filtCol] = sonDF[filtCol] & sonDF[filtDQCol]
+
+        return sonDF
+
+    # ======================================================================
+    def _normalizeDQKeepValues(self, dq_keep_values):
+
+        if dq_keep_values is False or dq_keep_values is None:
+            return set()
+
+        if isinstance(dq_keep_values, str):
+            dq_keep_values = dq_keep_values.split(',')
+
+        keep_vals = set()
+        for value in dq_keep_values:
+            norm = self._normalizeDQValue(value)
+            if norm:
+                keep_vals.add(norm)
+
+        return keep_vals
+
+    # ======================================================================
+    def _normalizeDQValue(self, value):
+
+        if pd.isna(value):
+            return ''
+        return str(value).strip().lower()
+
+    # ======================================================================
+    def _coerceDQTimestampSeries(self, series, field_name):
+
+        non_na = series.dropna()
+        numeric = pd.to_numeric(series, errors='coerce')
+        if len(non_na) > 0 and numeric.notna().sum() == len(non_na):
+            return numeric, 'numeric'
+
+        dt = pd.to_datetime(series, errors='coerce')
+        if dt.notna().any():
+            try:
+                if dt.dt.tz is not None:
+                    dt = dt.dt.tz_localize(None)
+            except AttributeError:
+                pass
+            return dt, 'datetime'
+
+        if numeric.notna().any():
+            return numeric, 'numeric'
+
+        raise ValueError('Unable to parse dqLog timestamps from column: {}'.format(field_name))
+
+    # ======================================================================
+    def _shiftDQDatetimeToTargetOffset(self,
+                                       dq_times,
+                                       dq_src_utc_offset=False,
+                                       dq_target_utc_offset=False):
+
+        src_offset = self._coerceDQUtcOffset(dq_src_utc_offset, 'dq_src_utc_offset')
+        target_offset = self._coerceDQUtcOffset(dq_target_utc_offset, 'dq_target_utc_offset')
+
+        if src_offset is None and target_offset is None:
+            return dq_times
+
+        if src_offset is None or target_offset is None:
+            raise ValueError(
+                'dq_src_utc_offset and dq_target_utc_offset must both be provided when either is set.'
+            )
+
+        return dq_times + pd.to_timedelta(target_offset - src_offset, unit='h')
+
+    # ======================================================================
+    def _coerceDQUtcOffset(self, value, field_name):
+
+        if value is False or value is None or value == '':
+            return None
+
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            raise ValueError('{} must be a numeric UTC offset in hours.'.format(field_name))
+
+    # ======================================================================
+    def _getSonarFilterTimestamp(self, sonDF):
+
+        if 'date' in sonDF.columns and 'time' in sonDF.columns:
+            dt = pd.to_datetime(
+                sonDF['date'].astype(str).str.strip() + ' ' + sonDF['time'].astype(str).str.strip(),
+                errors='coerce',
+                format='mixed',
+            )
+            if dt.notna().any():
+                try:
+                    if dt.dt.tz is not None:
+                        dt = dt.dt.tz_localize(None)
+                except AttributeError:
+                    pass
+                return dt, 'datetime'
+
+        if 'time' in sonDF.columns:
+            dt = pd.to_datetime(sonDF['time'], errors='coerce', format='mixed')
+            if dt.notna().any():
+                try:
+                    if dt.dt.tz is not None:
+                        dt = dt.dt.tz_localize(None)
+                except AttributeError:
+                    pass
+                return dt, 'datetime'
+
+        if 'time_s' in sonDF.columns:
+            numeric = pd.to_numeric(sonDF['time_s'], errors='coerce')
+            if numeric.notna().any():
+                return numeric, 'numeric'
+
+        raise ValueError('Unable to determine sonar timestamps for dqLog filtering.')
+
+
+    # ======================================================================
+    def _applyDQEventState(self, son, event_state, dqTimeCol):
+        '''
+        Return the subset of sonar indices whose ping timestamp falls within a
+        "keep" state block as defined by the DQ event log.
+
+        Uses np.searchsorted so each ping inherits the state of the most-recent
+        event that preceded it.  Pings before the first event are excluded.
+        '''
+
+        event_times = event_state[dqTimeCol].to_numpy()
+        event_keep  = event_state['_dq_keep'].to_numpy(dtype=bool)
+        son_times   = son['_son_ts'].to_numpy()
+
+        # searchsorted(side='right') - 1 gives index of last event <= ping time
+        event_idx = np.searchsorted(event_times, son_times, side='right') - 1
+
+        # Pings before the first event get event_idx == -1 → exclude
+        valid_idx = event_idx >= 0
+
+        keep_mask = np.zeros(len(son), dtype=bool)
+        keep_mask[valid_idx] = event_keep[event_idx[valid_idx]]
+
+        return son.loc[keep_mask, '_son_idx']
+
+
     # ======================================================================
     def _filterHeading(self,
                        df,

{pingmapper-5.3.7 → pingmapper-5.4.0}/pingmapper/doWork.py

@@ -98,6 +98,9 @@ def doWork(
         exportUnknown (bool), fixNoDat (bool), threadCnt (int | float)
         aoi (str | False), max_heading_deviation (float), max_heading_distance (float)
         min_speed (float), max_speed (float), time_table (str | False)
+        dq_table (str | False), dq_time_field (str | False), dq_flag_field (str | False)
+        dq_keep_values (list[str] | str | False), dq_src_utc_offset (float)
+        dq_target_utc_offset (float), dq_time_offset (float)
         pix_res_son (float), pix_res_map (float)
         x_offset (float), y_offset (float)
         egn (bool), egn_stretch (int: 0/1/2), egn_stretch_factor (float)

{pingmapper-5.3.7 → pingmapper-5.4.0}/pingmapper/main_readFiles.py

@@ -119,6 +119,13 @@ def read_master_func(logfilename='',
                      min_speed = False,
                      max_speed = False,
                      time_table = False,
+                     dq_table = False,
+                     dq_time_field = False,
+                     dq_flag_field = False,
+                     dq_keep_values = False,
+                     dq_src_utc_offset = 0.0,
+                     dq_target_utc_offset = 0.0,
+                     dq_time_offset = 0.0,
                      tempC=10,
                      nchunk=500,
                      cropRange=0,
@@ -1018,7 +1025,7 @@ def read_master_func(logfilename='',
     # For Filtering                                                            #
     ############################################################################
 
-    if max_heading_deviation > 0 or min_speed > 0 or max_speed > 0 or aoi or time_table:
+    if dq_table or max_heading_deviation > 0 or min_speed > 0 or max_speed > 0 or aoi or time_table:
 
         start_time = time.time()
 
@@ -1051,7 +1058,9 @@ def read_master_func(logfilename='',
 
         # Do filtering on longest recording
         son0 = portstar[maxRec]
-        df0 = son0._doSonarFiltering(max_heading_deviation, max_heading_distance, min_speed, max_speed, aoi, time_table)
+        df0 = son0._doSonarFiltering(max_heading_deviation, max_heading_distance, min_speed, max_speed, aoi, time_table,
+                                      dq_table, dq_time_field, dq_flag_field, dq_keep_values,
+                                      dq_src_utc_offset, dq_target_utc_offset, dq_time_offset)
 
         # Add filter to other beam
         son1 = portstar[minRec]
@@ -1074,7 +1083,7 @@ def read_master_func(logfilename='',
         if df0.empty or df1.empty:
             raise ValueError(
                 '\n\nFiltering removed all side-scan pings. No metadata remains to process. '\
-                'Adjust filtering parameters (max_heading_deviation, min_speed, max_speed, aoi, time_table) '\
+                'Adjust filtering parameters (dq_table, max_heading_deviation, min_speed, max_speed, aoi, time_table) '\
                 'or reduce nchunk.'
             )
 
@@ -1099,7 +1108,9 @@ def read_master_func(logfilename='',
 
         # Do filtering on downbeams
         for son in downbeams:
-            df = son._doSonarFiltering(max_heading_deviation, max_heading_distance, min_speed, max_speed, aoi, time_table)
+            df = son._doSonarFiltering(max_heading_deviation, max_heading_distance, min_speed, max_speed, aoi, time_table,
+                                        dq_table, dq_time_field, dq_flag_field, dq_keep_values,
+                                        dq_src_utc_offset, dq_target_utc_offset, dq_time_offset)
 
             df = df[df['filter'] == True]
 
@@ -1452,9 +1463,12 @@ def read_master_func(logfilename='',
     # Cleanup
     try:
         psObj._cleanup()
-        del psObj, portstar
-    except:
+    except Exception:
         pass
+    if 'psObj' in locals():
+        del psObj
+    if 'portstar' in locals():
+        del portstar
 
 
     ############################################################################

pingmapper-5.4.0/pingmapper/nonGUI_batch_main.py

@@ -0,0 +1,129 @@
+import sys
+from pathlib import Path
+
+# Ensure local repo package is imported when running this script directly.
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from pingmapper.doWork import doWork
+
+params = {
+    # ------------------------------------------------------------------
+    # Project / Runtime
+    # ------------------------------------------------------------------
+    "project_mode": 1,
+    "threadCnt": 0,
+
+    # ------------------------------------------------------------------
+    # Survey / Sonar Basics
+    # ------------------------------------------------------------------
+    "tempC": 12.0,
+    "nchunk": 500,
+    "cropRange": 0,
+
+    # ------------------------------------------------------------------
+    # Navigation + Filtering
+    # ------------------------------------------------------------------
+    "aoi": False,
+    "max_heading_deviation": 0,
+    "max_heading_distance": 0,
+    "min_speed": 0,
+    "max_speed": 0,
+    "time_table": False,
+
+    # dqLog event-state filtering
+    # Set dq_table and field names to enable; keep values can be list or csv string.
+    "dq_table": False,
+    "dq_time_field": False,
+    "dq_flag_field": False,
+    "dq_keep_values": False,
+    "dq_src_utc_offset": 0.0,
+    "dq_target_utc_offset": 0.0,
+    "dq_time_offset": 0.0,
+
+    # ------------------------------------------------------------------
+    # Input Handling
+    # ------------------------------------------------------------------
+    "exportUnknown": False,
+    "fixNoDat": False,
+
+    # ------------------------------------------------------------------
+    # Georeferencing / Resolution
+    # ------------------------------------------------------------------
+    "pix_res_son": 0.1,
+    "pix_res_map": 0,
+    "x_offset": 0.0,
+    "y_offset": 0.0,
+
+    # ------------------------------------------------------------------
+    # Intensity / Tone
+    # ------------------------------------------------------------------
+    "egn": False,
+    "egn_stretch": 0,
+    "egn_stretch_factor": 1.0,
+
+    # ------------------------------------------------------------------
+    # Sonogram Exports
+    # ------------------------------------------------------------------
+    "wcp": False,
+    "wcm": False,
+    "wcr": False,
+    "wco": False,
+    "sonogram_colorMap": "Greys_r",
+    "mask_shdw": False,
+    "tileFile": ".png",
+    "spdCor": False,
+    "maxCrop": False,
+
+    # ------------------------------------------------------------------
+    # Depth / Shadows
+    # ------------------------------------------------------------------
+    "remShadow": 0,
+    "detectDep": 0,
+    "smthDep": False,
+    "adjDep": 0.0,
+    "pltBedPick": False,
+
+    # ------------------------------------------------------------------
+    # Rectification / Mosaics
+    # ------------------------------------------------------------------
+    "rect_wcp": False,
+    "rect_wcr": False,
+    "rubberSheeting": False,
+    "rectMethod": "Heading",
+    "rectInterpDist": 50,
+    "son_colorMap": "Greys",
+    "mosaic_nchunk": 0,
+
+    # ------------------------------------------------------------------
+    # Substrate Mapping
+    # ------------------------------------------------------------------
+    "pred_sub": False,
+    "pltSubClass": False,
+    "map_sub": False,
+    "export_poly": False,
+    "map_class_method": "max",
+    "map_predict": 0,
+
+    # ------------------------------------------------------------------
+    # Final Exports
+    # ------------------------------------------------------------------
+    "mosaic": 0,
+    "map_mosaic": 0,
+    "banklines": False,
+    "coverage": False,
+}
+
+results = doWork(
+    in_dir=r"C:\Users\cbodine\Downloads\NewRiver\SonarRecording",
+    out_dir=r"C:\Users\cbodine\Downloads\NewRiver",
+    proj_name=None,
+    batch=True,
+    preserve_subdirs=False,
+    prefix="",
+    suffix="",
+    params=params,
+)
+
+print(results)

pingmapper-5.4.0/pingmapper/nonGui_main.py

@@ -0,0 +1,126 @@
+import sys
+from pathlib import Path
+
+# Ensure local repo package is imported when running this script directly.
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+	sys.path.insert(0, str(REPO_ROOT))
+
+from pingmapper.doWork import doWork
+
+params = {
+	# ------------------------------------------------------------------
+	# Project / Runtime
+	# ------------------------------------------------------------------
+	"project_mode": 1,
+	"threadCnt": 0,
+
+	# ------------------------------------------------------------------
+	# Survey / Sonar Basics
+	# ------------------------------------------------------------------
+	"tempC": 12.0,
+	"nchunk": 500,
+	"cropRange": 0,
+
+	# ------------------------------------------------------------------
+	# Navigation + Filtering
+	# ------------------------------------------------------------------
+	"aoi": False,
+	"max_heading_deviation": 0,
+	"max_heading_distance": 0,
+	"min_speed": 0,
+	"max_speed": 0,
+	"time_table": False,
+
+	# dqLog event-state filtering
+	# Set dq_table and field names to enable; keep values can be list or csv string.
+	"dq_table": r"C:\Users\cbodine\Downloads\NewRiver\NewRvr_DataLogger_Cameron_R00066.csv",
+	"dq_time_field": "Timestamp..UTC.",
+	"dq_flag_field": "DataQuality",
+	"dq_keep_values": ["Use"],
+	"dq_src_utc_offset": 0.0,
+	"dq_target_utc_offset": -4.0,
+	"dq_time_offset": 5.0,
+
+	# ------------------------------------------------------------------
+	# Input Handling
+	# ------------------------------------------------------------------
+	"exportUnknown": False,
+	"fixNoDat": False,
+
+	# ------------------------------------------------------------------
+	# Georeferencing / Resolution
+	# ------------------------------------------------------------------
+	"pix_res_son": 0.1,
+	"pix_res_map": 0,
+	"x_offset": 0.0,
+	"y_offset": 0.0,
+
+	# ------------------------------------------------------------------
+	# Intensity / Tone
+	# ------------------------------------------------------------------
+	"egn": False,
+	"egn_stretch": 0,
+	"egn_stretch_factor": 1.0,
+
+	# ------------------------------------------------------------------
+	# Sonogram Exports
+	# ------------------------------------------------------------------
+	"wcp": False,
+	"wcm": False,
+	"wcr": False,
+	"wco": False,
+	"sonogram_colorMap": "Greys_r",
+	"mask_shdw": False,
+	"tileFile": ".png",
+	"spdCor": False,
+	"maxCrop": False,
+
+	# ------------------------------------------------------------------
+	# Depth / Shadows
+	# ------------------------------------------------------------------
+	"remShadow": 0,
+	"detectDep": 0,
+	"smthDep": False,
+	"adjDep": 0.0,
+	"pltBedPick": False,
+
+	# ------------------------------------------------------------------
+	# Rectification / Mosaics
+	# ------------------------------------------------------------------
+	"rect_wcp": False,
+	"rect_wcr": False,
+	"rubberSheeting": False,
+	"rectMethod": "Heading",
+	"rectInterpDist": 50,
+	"son_colorMap": "Greys",
+	"mosaic_nchunk": 0,
+
+	# ------------------------------------------------------------------
+	# Substrate Mapping
+	# ------------------------------------------------------------------
+	"pred_sub": False,
+	"pltSubClass": False,
+	"map_sub": False,
+	"export_poly": False,
+	"map_class_method": "max",
+	"map_predict": 0,
+
+	# ------------------------------------------------------------------
+	# Final Exports
+	# ------------------------------------------------------------------
+	"mosaic": 0,
+	"map_mosaic": 0,
+	"banklines": False,
+	"coverage": False,
+}
+
+results = doWork(
+	in_file=r"C:\Users\cbodine\Downloads\NewRiver\SonarRecording\R00066.DAT",
+	out_dir=r"C:\Users\cbodine\Downloads\NewRiver",
+	proj_name="FilterTest_take2",
+	batch=False,
+	params=params,
+)
+
+print(results)

pingmapper-5.4.0/pingmapper/test_dq_filter.py

@@ -0,0 +1,203 @@
+"""Unit tests for _filterDQ / _applyDQEventState on sonObj."""
+
+import io
+import unittest
+import types
+import numpy as np
+import pandas as pd
+
+# ---------------------------------------------------------------------------
+# Minimal stub that satisfies _filterDQ without loading real sonar files
+# ---------------------------------------------------------------------------
+from pingmapper.class_sonObj import sonObj
+
+
+def _make_stub(son_rows: list[dict]) -> sonObj:
+    """Return a bare sonObj instance with sonMetaDF pre-populated."""
+    obj = sonObj.__new__(sonObj)
+    obj.sonMetaDF = pd.DataFrame(son_rows)
+    return obj
+
+
+# ---------------------------------------------------------------------------
+# Helper to write a temporary CSV for dq_table
+# ---------------------------------------------------------------------------
+def _dq_csv(rows: list[dict]) -> str:
+    """Serialise rows to a CSV string and return a path via tmp file."""
+    import tempfile, os
+    df = pd.DataFrame(rows)
+    fd, path = tempfile.mkstemp(suffix='.csv')
+    df.to_csv(path, index=False)
+    os.close(fd)
+    return path
+
+
+# ===========================================================================
+class TestDQFilterDatetimeOffset(unittest.TestCase):
+    """dq_time_offset shifts sonar timestamps before matching."""
+
+    def _run(self, son_ts_list, dq_rows, dq_time_offset, expected_kept_indices):
+        stub = _make_stub([
+            {'date': ts.split()[0], 'time': ts.split()[1], 'time_s': i + 1}
+            for i, ts in enumerate(son_ts_list)
+        ])
+        stub.sonMetaDF.index = range(len(son_ts_list))
+
+        path = _dq_csv(dq_rows)
+        result = stub._filterDQ(
+            stub.sonMetaDF,
+            dq_table=path,
+            dq_time_field='ts',
+            dq_flag_field='flag',
+            dq_keep_values=['Use'],
+            dq_src_utc_offset=0.0,
+            dq_target_utc_offset=0.0,
+            dq_time_offset=dq_time_offset,
+        )
+        kept = list(result[result['filter_dq'] == True].index)
+        self.assertEqual(kept, expected_kept_indices)
+
+    def test_dq_filter_uses_datetime_and_offset(self):
+        """Positive offset shifts sonar timestamps forward into a 'Use' block."""
+        son_ts = [
+            '2024-01-01 00:00:01',
+            '2024-01-01 00:00:02',
+            '2024-01-01 00:00:03',
+        ]
+        dq_rows = [
+            {'ts': '2024-01-01T00:00:10+00:00', 'flag': 'Use'},
+        ]
+        # offset +10 → sonar times become 11, 12, 13 — all after the Use event at t=10
+        self._run(son_ts, dq_rows, dq_time_offset=10.0, expected_kept_indices=[0, 1, 2])
+
+    def test_dq_filter_uses_datetime_timezone_offsets(self):
+        """UTC offset difference is applied to dq timestamps."""
+        son_ts = [
+            '2024-01-01 00:00:01',
+            '2024-01-01 00:00:02',
+            '2024-01-01 00:00:03',
+        ]
+        # DQ stored in UTC+1 (3600 s ahead), no sonar offset
+        dq_rows = [
+            {'ts': '2024-01-01T01:00:00+01:00', 'flag': 'Use'},
+        ]
+        # After tz conversion dq ts == 2024-01-01 00:00:00 UTC == 0 epoch offset
+        # sonar pings at 1,2,3 s — all after the Use event
+        from pingmapper.class_sonObj import sonObj as SO
+        stub = _make_stub([
+            {'date': ts.split()[0], 'time': ts.split()[1], 'time_s': i + 1}
+            for i, ts in enumerate(son_ts)
+        ])
+        path = _dq_csv(dq_rows)
+        result = stub._filterDQ(
+            stub.sonMetaDF,
+            dq_table=path,
+            dq_time_field='ts',
+            dq_flag_field='flag',
+            dq_keep_values=['Use'],
+            dq_src_utc_offset=1.0,   # DQ was recorded in UTC+1
+            dq_target_utc_offset=0.0,
+            dq_time_offset=0.0,
+        )
+        kept = list(result[result['filter_dq'] == True].index)
+        self.assertEqual(kept, [0, 1, 2])
+
+    def test_dq_filter_handles_fractional_second_sonar_times(self):
+        stub = _make_stub([
+            {'date': '2025-08-10', 'time': '08:04:07', 'time_s': 0.0},
+            {'date': '2025-08-10', 'time': '08:04:07.064000', 'time_s': 0.064},
+            {'date': '2025-08-10', 'time': '08:04:07.130000', 'time_s': 0.130},
+            {'date': '2025-08-10', 'time': '08:04:07.195000', 'time_s': 0.195},
+        ])
+
+        path = _dq_csv([
+            {'ts': '2025/08/10 12:04:07', 'flag': 'Use'},
+        ])
+
+        result = stub._filterDQ(
+            stub.sonMetaDF,
+            dq_table=path,
+            dq_time_field='ts',
+            dq_flag_field='flag',
+            dq_keep_values=['Use'],
+            dq_src_utc_offset=0.0,
+            dq_target_utc_offset=-4.0,
+            dq_time_offset=0.0,
+        )
+
+        kept = list(result[result['filter_dq'] == True].index)
+        self.assertEqual(kept, [0, 1, 2, 3])
+
+
+class TestDQFilterNumericTime(unittest.TestCase):
+    """Falls back to numeric time_s when datetime parse fails."""
+
+    def test_dq_filter_uses_numeric_time(self):
+        rows = [{'time_s': float(i)} for i in range(1, 6)]
+        stub = _make_stub(rows)
+
+        dq_rows = [{'ts': 2.5, 'flag': 'Use'}]
+        path = _dq_csv(dq_rows)
+
+        result = stub._filterDQ(
+            stub.sonMetaDF,
+            dq_table=path,
+            dq_time_field='ts',
+            dq_flag_field='flag',
+            dq_keep_values=['Use'],
+            dq_src_utc_offset=0.0,
+            dq_target_utc_offset=0.0,
+            dq_time_offset=0.0,
+        )
+        # pings at time_s 1,2 are before the Use event at 2.5 → excluded
+        # pings at time_s 3,4,5 are in the Use block
+        kept = list(result[result['filter_dq'] == True].index)
+        self.assertEqual(kept, [2, 3, 4])
+
+
+class TestDQFilterEventStateBlocks(unittest.TestCase):
+    """State-block semantics: alternating Use/NoUse transitions."""
+
+    def test_dq_filter_uses_event_state_blocks(self):
+        """
+        DQ transitions:
+            t=2.2  → Use    (good)
+            t=4.1  → NoUse  (bad)
+            t=6.4  → Use    (good)
+
+        Sonar pings at t=1..7 (numeric).
+        Expected kept: pings whose (adjusted) time falls in a Use block
+            ping 0 (t=1): before first event  → excluded
+            ping 1 (t=2): before first event  → excluded
+            ping 2 (t=3): in [2.2, 4.1) Use  → kept
+            ping 3 (t=4): in [2.2, 4.1) Use  → kept
+            ping 4 (t=5): in [4.1, 6.4) NoUse → excluded
+            ping 5 (t=6): in [4.1, 6.4) NoUse → excluded
+            ping 6 (t=7): in [6.4, ∞) Use    → kept
+        """
+        rows = [{'time_s': float(t)} for t in range(1, 8)]
+        stub = _make_stub(rows)
+
+        dq_rows = [
+            {'ts': 2.2, 'flag': 'Use'},
+            {'ts': 4.1, 'flag': 'NoUse'},
+            {'ts': 6.4, 'flag': 'Use'},
+        ]
+        path = _dq_csv(dq_rows)
+
+        result = stub._filterDQ(
+            stub.sonMetaDF,
+            dq_table=path,
+            dq_time_field='ts',
+            dq_flag_field='flag',
+            dq_keep_values=['Use'],
+            dq_src_utc_offset=0.0,
+            dq_target_utc_offset=0.0,
+            dq_time_offset=0.0,
+        )
+        kept = list(result[result['filter_dq'] == True].index)
+        self.assertEqual(kept, [2, 3, 6])
+
+
+if __name__ == '__main__':
+    unittest.main()

pingmapper-5.4.0/pingmapper/version.py

@@ -0,0 +1 @@
+__version__ = '5.4.0'

{pingmapper-5.3.7 → pingmapper-5.4.0}/pingmapper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 5.3.7
+Version: 5.4.0
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Daniel Buscombe
 Author-email: Cameron Bodine <bodine.cs@gmail.email>

{pingmapper-5.3.7 → pingmapper-5.4.0}/pingmapper.egg-info/SOURCES.txt

@@ -17,7 +17,10 @@ pingmapper/gui_main.py
 pingmapper/main_mapSubstrate.py
 pingmapper/main_readFiles.py
 pingmapper/main_rectify.py
+pingmapper/nonGUI_batch_main.py
+pingmapper/nonGui_main.py
 pingmapper/test_PINGMapper.py
+pingmapper/test_dq_filter.py
 pingmapper/test_time.py
 pingmapper/version.py
 pingmapper.egg-info/PKG-INFO

pingmapper-5.3.7/pingmapper/version.py

@@ -1 +0,0 @@
-__version__ = '5.3.7'