pingmapper 5.3.5__tar.gz → 5.3.7__tar.gz

{pingmapper-5.3.5 → pingmapper-5.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 5.3.5
+Version: 5.3.7
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Daniel Buscombe
 Author-email: Cameron Bodine <bodine.cs@gmail.email>

{pingmapper-5.3.5 → pingmapper-5.3.7}/pingmapper/class_portstarObj.py

@@ -979,20 +979,20 @@ class portstarObj(object):
         self._depthZheng() or self._depthThreshold()
         '''
         
-        # Check if depth detection dependencies are available
-        if not DEPTH_DETECTION_AVAILABLE:
-            raise ImportError(
-                "TensorFlow, Transformers, and/or Doodleverse Utils are not installed. "
-                "These packages are required for automatic depth detection. "
-                "Please install them using: pip install tensorflow transformers doodleverse-utils"
-            )
-
-        # Open model configuration file
-        with open(self.configfile) as f:
-            config = json.load(f)
-        globals().update(config)
-
         if method == 1:
+            # Check if depth detection dependencies are available
+            if not DEPTH_DETECTION_AVAILABLE:
+                raise ImportError(
+                    "TensorFlow, Transformers, and/or Doodleverse Utils are not installed. "
+                    "These packages are required for ML depth detection. "
+                    "Use method=2 for binary-threshold depth picking instead."
+                )
+
+            # Open model configuration file
+            with open(self.configfile) as f:
+                config = json.load(f)
+            globals().update(config)
+
             if not hasattr(self, 'bedpickModel'):
                 # model = self._initModel(USE_GPU)
                 model = initModel(self.weights, self.configfile, USE_GPU)
@@ -1003,7 +1003,7 @@ class portstarObj(object):
         elif method == 2:
             self.port._loadSonMeta()
             self.star._loadSonMeta()
-            portDepPixCrop, starDepPixCrop, i = self._depthThreshold(i, tileFile)
+            portDepPixCrop, starDepPixCrop, i = self._depthThreshold(i)
 
         gc.collect()
         return portDepPixCrop, starDepPixCrop, i
@@ -1391,25 +1391,31 @@ class portstarObj(object):
         for son in portstar:
             # Load sonar intensity, standardize & rescale
             son._getScanChunkSingle(chunk)
-            img = son.sonDat
-            img = standardize(img, 0, 1, True)[:,:,-1].squeeze()
+            img = standardize(np.asarray(son.sonDat)).squeeze()
+            if img.ndim == 3:
+                img = img[:, :, -1]
             W, H = img.shape[1], img.shape[0]
 
             # Get chunks sonar metadata and instrument depth
-            isChunk = son.sonMetaDF['chunk_id']==1
+            isChunk = son.sonMetaDF['chunk_id'] == chunk
             sonMeta = son.sonMetaDF[isChunk].reset_index()
-            # acousticBed = round(sonMeta['inst_dep_m'] / sonMeta['pix_m'], 0).astype(int)
-            acousticBed = round(sonMeta['inst_dep_m'] / sonMeta['pixM'], 0).astype(int)
+            acoustic_depth = pd.to_numeric(sonMeta['inst_dep_m'], errors='coerce').to_numpy(dtype=float, copy=True)
+            acousticBed = np.round(acoustic_depth / sonMeta['pixM'].to_numpy(dtype=float, copy=True), 0)
+            acousticBed = acousticBed[np.isfinite(acousticBed) & (acousticBed > 0)]
 
             ##################################
             # Step 1 : Acoustic Bedpick Filter
             # Use acoustic bed pick to crop image
-            bedMin = max(min(acousticBed)-50, 0)
-            bedMax = max(acousticBed)+pix_buf
+            if acousticBed.size > 0:
+                bedMin = max(int(np.nanmin(acousticBed)) - 50, 0)
+                bedMax = int(np.nanmax(acousticBed)) + pix_buf
+            else:
+                bedMin = 0
+                bedMax = H
 
             cropMask = np.ones((H, W)).astype(int)
             cropMask[:bedMin,:] = 0
-            cropMask[bedMax:,:] = 0
+            cropMask[min(bedMax, H):,:] = 0
 
             # Mask the image with bed_mask
             imgMasked = img*cropMask
@@ -1464,7 +1470,7 @@ class portstarObj(object):
             imgBinaryMask[imgBinaryMask>0] = 1 # Now set all val's greater than 0 to 1 to create the mask
 
             # Now fill in above last row filled to make sure no gaps in bed pixels
-            lastRow = bedMax
+            lastRow = min(bedMax, H - 1)
             imgBinaryMask[lastRow] = True
             for i in range(W):
                 if imgBinaryMask[lastRow-1,i] == 0:
@@ -1509,10 +1515,10 @@ class portstarObj(object):
             bed[nans] = np.interp(x(nans), x(~nans), bed[~nans])
             bed = bed.astype(int)
 
-            if son.beamName == 'ss_port':
+            if str(son.beamName).startswith('ss_port'):
                 # self.portDepDetect[chunk] = bed
                 portDepPixCrop = bed
-            elif son.beamName == 'ss_star':
+            elif str(son.beamName).startswith('ss_star'):
                 # self.starDepDetect[chunk] = bed
                 starDepPixCrop = bed
 
@@ -1557,6 +1563,43 @@ class portstarObj(object):
                           Integer < 0 = decrease depth estimate by x pixels.
                           0 = use depth estimate with no adjustment.
         '''
+        def _format_depth_adjustment(pix_m_series):
+            pix_m = pd.to_numeric(pix_m_series, errors='coerce')
+            valid_pix = pix_m[np.isfinite(pix_m) & (pix_m > 0)]
+            if len(valid_pix) == 0:
+                return '0 pixels'
+            return str(float(adjDep) / float(valid_pix.iloc[0])) + ' pixels'
+
+        def _sync_trackline_depth(beam_obj, beam_df):
+            trk_file = os.path.join(beam_obj.metaDir, 'Trackline_Smth_' + beam_obj.beamName + '.csv')
+            if not os.path.exists(trk_file):
+                return
+
+            trk_df = pd.read_csv(trk_file)
+            if 'record_num' not in trk_df.columns or 'record_num' not in beam_df.columns:
+                return
+
+            depth_cols = ['record_num', 'dep_m']
+            if 'dep_m_Method' in beam_df.columns:
+                depth_cols.append('dep_m_Method')
+            if 'dep_m_smth' in beam_df.columns:
+                depth_cols.append('dep_m_smth')
+            if 'dep_m_adjBy' in beam_df.columns:
+                depth_cols.append('dep_m_adjBy')
+
+            depth_df = beam_df[depth_cols].drop_duplicates(subset=['record_num'], keep='last').set_index('record_num')
+            trk_df = trk_df.set_index('record_num')
+
+            trk_df['dep_m'] = depth_df['dep_m']
+            if 'dep_m_Method' in depth_df.columns:
+                trk_df['dep_m_Method'] = depth_df['dep_m_Method']
+            if 'dep_m_smth' in depth_df.columns:
+                trk_df['dep_m_smth'] = depth_df['dep_m_smth']
+            if 'dep_m_adjBy' in depth_df.columns:
+                trk_df['dep_m_adjBy'] = depth_df['dep_m_adjBy']
+
+            trk_df.reset_index().to_csv(trk_file, index=False, float_format='%.14f')
+
         # Load sonar metadata file
         self.port._loadSonMeta()
         portDF = self.port.sonMetaDF
@@ -1617,8 +1660,8 @@ class portstarObj(object):
             portDF['dep_m_smth'] = smthDep
             starDF['dep_m_smth'] = smthDep
 
-            portDF['dep_m_adjBy'] = str(adjDep / portDF['pixM']) + ' pixels'
-            starDF['dep_m_adjBy'] = str(adjDep / starDF['pixM']) + ' pixels'
+            portDF['dep_m_adjBy'] = _format_depth_adjustment(portDF['pixM'])
+            starDF['dep_m_adjBy'] = _format_depth_adjustment(starDF['pixM'])
 
         elif detectDep > 0:
             # Prepare depth detection dictionaries
@@ -1703,8 +1746,8 @@ class portstarObj(object):
             portDF['dep_m_smth'] = smthDep
             starDF['dep_m_smth'] = smthDep
 
-            portDF['dep_m_adjBy'] = str(adjDep / portDF['pixM']) + ' pixels'
-            starDF['dep_m_adjBy'] = str(adjDep / starDF['pixM']) + ' pixels'
+            portDF['dep_m_adjBy'] = _format_depth_adjustment(portDF['pixM'])
+            starDF['dep_m_adjBy'] = _format_depth_adjustment(starDF['pixM'])
 
         # Interpolate over nan's (and set zeros to nan)
         portDep = portDF['dep_m'].to_numpy(copy=True)
@@ -1730,6 +1773,8 @@ class portstarObj(object):
         # Export to csv
         portDF.to_csv(self.port.sonMetaFile, index=False, float_format='%.14f')
         starDF.to_csv(self.star.sonMetaFile, index=False, float_format='%.14f')
+        _sync_trackline_depth(self.port, portDF)
+        _sync_trackline_depth(self.star, starDF)
 
         try:
             # Take average of both estimates to store with downlooking sonar csv

{pingmapper-5.3.5 → pingmapper-5.3.7}/pingmapper/doWork.py

@@ -293,6 +293,7 @@ def doWork(
             if ss_chan_avail:
                 rect_wcp = run_params.get('rect_wcp', False)
                 rect_wcr = run_params.get('rect_wcr', False)
+                force_rectify = run_params.get('force_rectify', False)
                 banklines = run_params.get('banklines', False)
                 coverage = run_params.get('coverage', False)
                 pred_sub = run_params.get('pred_sub', False)
@@ -301,11 +302,12 @@ def doWork(
                 plt_subclass = run_params.get('pltSubClass', False)
 
                 if not nav_available:
-                    if rect_wcp or rect_wcr or banklines or coverage or pred_sub or map_sub or export_poly or plt_subclass:
+                    if rect_wcp or rect_wcr or force_rectify or banklines or coverage or pred_sub or map_sub or export_poly or plt_subclass:
                         print('\nWARNING: Navigation info is unavailable for this recording.')
                         print('Skipping rectification and substrate mapping workflows (non-georeferenced sonogram-only processing).')
                     rect_wcp = False
                     rect_wcr = False
+                    force_rectify = False
                     banklines = False
                     coverage = False
                     pred_sub = False
@@ -313,7 +315,7 @@ def doWork(
                     export_poly = False
                     plt_subclass = False
 
-                if rect_wcp or rect_wcr or banklines or coverage or pred_sub or map_sub or export_poly:
+                if rect_wcp or rect_wcr or force_rectify or banklines or coverage or pred_sub or map_sub or export_poly:
                     print('\n===========================================')
                     print('===========================================')
                     print('***** RECTIFYING *****')

{pingmapper-5.3.5 → pingmapper-5.3.7}/pingmapper/funcs_common.py

@@ -31,6 +31,12 @@
 
 import os, sys, struct, gc, io, contextlib
 
+# Configure TensorFlow runtime defaults as early as possible so informational
+# startup messages do not leak to stderr before logging is initialized.
+os.environ.setdefault('TF_CPP_MIN_LOG_LEVEL', '3')
+os.environ.setdefault('TF_ENABLE_ONEDNN_OPTS', '0')
+os.environ.setdefault('AUTOGRAPH_VERBOSITY', '0')
+
 # Add 'pingmapper' to the path, may not need after pypi package...
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 PACKAGE_DIR = os.path.dirname(SCRIPT_DIR)
@@ -97,13 +103,27 @@ import json
 
 import logging
 
-from tqdm import tqdm
+from tqdm import tqdm as _tqdm
 
 import subprocess
 
 # from funcs_pyhum_correct import doPyhumCorrections
 
 
+# =========================================================
+def tqdm(*args, **kwargs):
+    '''
+    Shared tqdm wrapper so CLI progress bars stay readable in the GUI console.
+    Use PINGMAPPER_TQDM_NCOLS to override the default width when needed.
+    '''
+    if 'ncols' not in kwargs:
+        try:
+            kwargs['ncols'] = int(os.environ.get('PINGMAPPER_TQDM_NCOLS', '80'))
+        except (TypeError, ValueError):
+            kwargs['ncols'] = 80
+    return _tqdm(*args, **kwargs)
+
+
 # =========================================================
 def safe_n_jobs(task_count, thread_count=0):
     '''
@@ -139,6 +159,7 @@ def quiet_tensorflow_warnings():
     Safe to call even if TensorFlow is not installed.
     '''
     os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+    os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
     os.environ['AUTOGRAPH_VERBOSITY'] = '0'
 
     logging.getLogger('tensorflow').setLevel(logging.ERROR)

{pingmapper-5.3.5 → pingmapper-5.3.7}/pingmapper/main_readFiles.py

@@ -94,6 +94,17 @@ def _is_sidescan_beam(beam_name):
     return beam_name.startswith('ss_port') or beam_name.startswith('ss_star')
 
 
+def _sidescan_group_key(beam_name):
+    beam_name = str(beam_name)
+    if beam_name.startswith('ss_port_'):
+        return beam_name[len('ss_port_'):]
+    if beam_name.startswith('ss_star_'):
+        return beam_name[len('ss_star_'):]
+    if beam_name in {'ss_port', 'ss_star'}:
+        return 'default'
+    return beam_name
+
+
 #===========================================
 def read_master_func(logfilename='',
                      project_mode=0,
@@ -158,6 +169,7 @@ def read_master_func(logfilename='',
                      mosaic=False,
                      map_mosaic=0,
                      banklines=False,
+                     side_scan_only=False,
                      return_context=False,
                      **kwargs):
 
@@ -354,7 +366,7 @@ def read_master_func(logfilename='',
     # Prepare Cerulean file for PINGMapper
     elif file_type == '.svlog':
         sonar_obj = cerul2pingmapper(inFile, projDir, nchunk, tempC, exportUnknown)
-        detectDep = 1 # No depth in cerulean files, so set to Zheng et al. 2021
+        detectDep = 1 if DEPTH_DETECTION_AVAILABLE else 2
         instDepAvail = False
 
     # Prepare JSF file for PINGMapper
@@ -375,6 +387,7 @@ def read_master_func(logfilename='',
         sys.exit()
 
     nav_available = bool(getattr(sonar_obj, 'has_position', True))
+    side_scan_only = bool(side_scan_only)
 
     # Sonar-only fallback for sources without navigation fields (e.g., some Cerulean logs).
     # Disable filters that rely on geospatial motion/position so processing can continue.
@@ -547,6 +560,19 @@ def read_master_func(logfilename='',
     else:
         pass
 
+    if side_scan_only:
+        if len(ss_chan_avail) == 0:
+            print('\n\nSide-scan only mode enabled, but no side-scan channels were recognized. Skipping processing.')
+            if return_context:
+                return {
+                    'has_sidescan': False,
+                    'has_nav': nav_available,
+                }
+            return False
+
+        print('\n\nSide-scan only mode enabled. Skipping non-side-scan channels.')
+        sonObjs = [son for son in sonObjs if _is_sidescan_beam(getattr(son, 'beamName', ''))]
+
     print(sonObjs)
     ####
     # OLD    
@@ -949,10 +975,15 @@ def read_master_func(logfilename='',
                     valid=True
                 elif (attMax != 0) or ("unknown" in att) or (att =="beam"):
                     valid=True
-                elif (att == "inst_dep_m") and (attAvg == 0): # Automatically detect depth if no instrument depth
-                    valid=False
-                    invalid[son.beam+"."+att] = False
-                    detectDep=1
+                elif att == "inst_dep_m":
+                    depth_values = pd.to_numeric(df[att], errors='coerce').to_numpy(dtype=float, copy=False)
+                    has_valid_instrument_depth = np.isfinite(depth_values).any() and np.nanmax(depth_values) > 0
+                    if not has_valid_instrument_depth: # Automatically detect depth if instrument depth is missing/empty
+                        valid=False
+                        invalid[son.beam+"."+att] = False
+                        detectDep = 1 if DEPTH_DETECTION_AVAILABLE else 2
+                    else:
+                        valid=True
                 else:
                     valid=False
                     invalid[son.beam+"."+att] = False
@@ -1097,17 +1128,23 @@ def read_master_func(logfilename='',
 
     start_time = time.time()
 
-    # Determine which sonObj is port/star
-    portstar = []
+    # Determine which sonObj pairs should be depth processed together.
+    sidescan_groups = {}
     for son in sonObjs:
-        beam = son.beamName
-        if _is_sidescan_beam(beam):
-            portstar.append(son)
+        beam = str(son.beamName)
+        if not _is_sidescan_beam(beam):
+            continue
+
+        group_key = _sidescan_group_key(beam)
+        group = sidescan_groups.setdefault(group_key, {})
+        if beam.startswith('ss_port'):
+            group['port'] = son
+        elif beam.startswith('ss_star'):
+            group['star'] = son
 
-    psObj = None
-    chunks = np.array([], dtype=int)
+    ps_depth_jobs = []
 
-    if len(portstar) == 0:
+    if len(sidescan_groups) == 0:
         print(
             '\n\nNo recognized side-scan channels available for depth processing. '\
             'Continuing with down-looking beams only.'
@@ -1117,20 +1154,28 @@ def read_master_func(logfilename='',
         pltBedPick = False
         remShadow = 0
     else:
-        # Create portstarObj
-        psObj = portstarObj(portstar)
+        for group_key, group in sorted(sidescan_groups.items()):
+            if 'port' not in group or 'star' not in group:
+                print(
+                    '\nSkipping side-scan depth group {} because a matching port/star pair was not found.'.format(group_key)
+                )
+                continue
+
+            psObj = portstarObj([group['port'], group['star']])
 
-        chunks = []
-        for son in portstar:
-            # Get chunk id's, ignoring those with nodata
-            c = son._getChunkID()
+            chunks = []
+            for son in [group['port'], group['star']]:
+                c = son._getChunkID()
+                chunks.extend(c)
+                del c
 
-            chunks.extend(c)
-            del c
+            chunks = np.unique(chunks).astype(int)
+            if len(chunks) == 0:
+                continue
 
-        chunks = np.unique(chunks).astype(int)
+            ps_depth_jobs.append((group_key, psObj, chunks))
 
-        if len(chunks) == 0:
+        if len(ps_depth_jobs) == 0:
             print(
                 '\n\nNo valid side-scan chunks available for depth processing. '\
                 'Continuing with down-looking beams only.'
@@ -1143,7 +1188,12 @@ def read_master_func(logfilename='',
     # # Automatically estimate depth
     if detectDep > 0:
         # Check if depth detection dependencies are available
-        if not DEPTH_DETECTION_AVAILABLE:
+        if detectDep == 1 and not DEPTH_DETECTION_AVAILABLE:
+            print('\n\nML depth detection dependencies are unavailable.')
+            print('Falling back to binary-threshold depth detection...\n')
+            detectDep = 2
+
+        if detectDep == 1 and not DEPTH_DETECTION_AVAILABLE:
             print('\n\nCannot estimate depth automatically:')
             print('TensorFlow, Transformers, and/or Doodleverse Utils are not installed.')
             print('These packages are required for automatic depth detection.')
@@ -1153,37 +1203,28 @@ def read_master_func(logfilename='',
             autoBed = False
             saveDepth = True
         else:
-            print('\n\nAutomatically estimating depth for', len(chunks), 'chunks:')
-
-            #Dictionary to store chunk : np.array(depth estimate)
-            psObj.portDepDetect = {}
-            psObj.starDepDetect = {}
-
-            # Estimate depth using:
-            # Zheng et al. 2021
-            # Load model weights and configuration file
-            if detectDep == 1:
-
-                # Store configuration file and model weights
-                # These were downloaded at the beginning of the script
-                depthModelVer = 'Bedpick_Zheng2021_Segmentation_unet_v1.0'
-                psObj.configfile = os.path.join(modelDir, depthModelVer, 'config', depthModelVer+'.json')
-                psObj.weights = os.path.join(modelDir, depthModelVer, 'weights', depthModelVer+'_fullmodel.h5')
-                print('\n\tUsing Zheng et al. 2021 method. Loading model:', os.path.basename(psObj.weights))
-
-            # With binary thresholding
-            elif detectDep == 2:
-                print('\n\tUsing binary thresholding...')
-
-            # Parallel estimate depth for each chunk using appropriate method
-            r = Parallel(n_jobs=safe_n_jobs(len(chunks), threadCnt))(delayed(psObj._detectDepth)(detectDep, int(chunk), USE_GPU, tileFile) for chunk in tqdm(chunks))
-
-            # store the depth predictions in the class
-            for ret in r:
-                psObj.portDepDetect[ret[2]] = ret[0]
-                psObj.starDepDetect[ret[2]] = ret[1]
-                del ret
-            del r
+            total_chunks = sum(len(chunks) for _, _, chunks in ps_depth_jobs)
+            print('\n\nAutomatically estimating depth for', total_chunks, 'chunks across', len(ps_depth_jobs), 'side-scan group(s):')
+
+            for group_key, psObj, chunks in ps_depth_jobs:
+                psObj.portDepDetect = {}
+                psObj.starDepDetect = {}
+
+                if detectDep == 1:
+                    depthModelVer = 'Bedpick_Zheng2021_Segmentation_unet_v1.0'
+                    psObj.configfile = os.path.join(modelDir, depthModelVer, 'config', depthModelVer+'.json')
+                    psObj.weights = os.path.join(modelDir, depthModelVer, 'weights', depthModelVer+'_fullmodel.h5')
+                    print('\n\tGroup {}: Using Zheng et al. 2021 method. Loading model: {}'.format(group_key, os.path.basename(psObj.weights)))
+                elif detectDep == 2:
+                    print('\n\tGroup {}: Using binary thresholding...'.format(group_key))
+
+                r = Parallel(n_jobs=safe_n_jobs(len(chunks), threadCnt))(delayed(psObj._detectDepth)(detectDep, int(chunk), USE_GPU, tileFile) for chunk in tqdm(chunks))
+
+                for ret in r:
+                    psObj.portDepDetect[ret[2]] = ret[0]
+                    psObj.starDepDetect[ret[2]] = ret[1]
+                    del ret
+                del r
 
             # Flag indicating depth autmatically estimated
             autoBed = True
@@ -1201,9 +1242,10 @@ def read_master_func(logfilename='',
 
     if saveDepth:
 
-        if ss_chan_avail and psObj is not None:
-            # Save detected depth to csv
-            depDF = psObj._saveDepth(chunks, detectDep, smthDep, adjDep, instDepAvail)
+        if ss_chan_avail and len(ps_depth_jobs) > 0:
+            depDF = []
+            for _, psObj, chunks in ps_depth_jobs:
+                depDF.append(psObj._saveDepth(chunks, detectDep, smthDep, adjDep, instDepAvail))
         else:
             depDF = []
 
@@ -1260,7 +1302,7 @@ def read_master_func(logfilename='',
         del depDF
 
         # Cleanup
-        if psObj is not None:
+        for _, psObj, _ in ps_depth_jobs:
             psObj._cleanup()
 
         print("\nDone!")
@@ -1281,7 +1323,10 @@ def read_master_func(logfilename='',
     # Cleanup
     if psObj is not None:
         psObj._cleanup()
-    del psObj, portstar
+    if 'psObj' in locals():
+        del psObj
+    if 'portstar' in locals():
+        del portstar
 
     for son in sonObjs:
         son._cleanup()

pingmapper-5.3.7/pingmapper/version.py

@@ -0,0 +1 @@
+__version__ = '5.3.7'

{pingmapper-5.3.5 → pingmapper-5.3.7}/pingmapper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 5.3.5
+Version: 5.3.7
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Daniel Buscombe
 Author-email: Cameron Bodine <bodine.cs@gmail.email>

pingmapper-5.3.5/pingmapper/version.py

@@ -1 +0,0 @@
-__version__ = '5.3.5'