pingmapper 4.2.12__tar.gz → 5.0.0__tar.gz

{pingmapper-4.2.12 → pingmapper-5.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 4.2.12
+Version: 5.0.0
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Cameron Bodine
 Author-email: bodine.cs@gmail.email

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/class_mapSubstrateObj.py

@@ -36,9 +36,15 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 PACKAGE_DIR = os.path.dirname(SCRIPT_DIR)
 sys.path.append(PACKAGE_DIR)
 
+# # For Debug
+# from funcs_common import *
+# from funcs_model import *
+# from class_rectObj import rectObj
+
 from pingmapper.funcs_common import *
 from pingmapper.funcs_model import *
 from pingmapper.class_rectObj import rectObj
+
 from mpl_toolkits.axes_grid1 import make_axes_locatable
 
 import matplotlib
@@ -313,7 +319,7 @@ class mapSubObj(rectObj):
         lOffL = self.sonDat.shape[1]
 
         # Get sonMetaDF
-        lMetaDF = df.loc[df['chunk_id'] == l, ['dep_m']].copy().reset_index()
+        lMetaDF = df.loc[df['chunk_id'] == l, ['dep_m', 'pixM']].copy().reset_index()
 
         # Remove shadows
         if self.remShadow:
@@ -357,7 +363,7 @@ class mapSubObj(rectObj):
         lOffR = lOffL + self.sonDat.shape[1]
 
         # Get sonMetaDF
-        cMetaDF = df.loc[df['chunk_id'] == c, ['dep_m']].copy().reset_index()
+        cMetaDF = df.loc[df['chunk_id'] == c, ['dep_m', 'pixM']].copy().reset_index()
 
         # Remove shadows
         if self.remShadow:
@@ -391,7 +397,7 @@ class mapSubObj(rectObj):
         self._getScanChunkSingle(r)
 
         # Get sonMetaDF
-        rMetaDF = df.loc[df['chunk_id'] == r, ['dep_m']].copy().reset_index()
+        rMetaDF = df.loc[df['chunk_id'] == r, ['dep_m', 'pixM']].copy().reset_index()
 
         # Remove shadows
         if self.remShadow:
@@ -688,7 +694,7 @@ class mapSubObj(rectObj):
         df = self.sonMetaDF
 
         # Get sonMetaDF
-        df = df.loc[df['chunk_id'] == chunk, ['dep_m']].copy().reset_index()
+        df = df.loc[df['chunk_id'] == chunk, ['dep_m', 'pixM']].copy().reset_index()
 
         # Load sonDat
         self._getScanChunkSingle(chunk)
@@ -718,7 +724,7 @@ class mapSubObj(rectObj):
         # Plot Classification
 
         # Get final classification
-        label = self._classifySoftmax(chunk, softmax, map_class_method, mask_wc=True, mask_shw=True)
+        label = self._classifySoftmax(chunk, softmax, map_class_method, df=df, mask_wc=True, mask_shw=True)
 
         # Do speed correction
         if spdCor>0:
@@ -904,7 +910,7 @@ class mapSubObj(rectObj):
     ############################################################################
 
     #=======================================================================
-    def _classifySoftmax(self, i, arr, map_class_method='max', mask_wc=True, mask_shw=True, do_filt=True):
+    def _classifySoftmax(self, i, arr, map_class_method='max', df=None, mask_wc=True, mask_shw=True, do_filt=True):
         '''
         Classify pixels from softmax values.
 
@@ -1005,7 +1011,7 @@ class mapSubObj(rectObj):
             min_size = 28
 
             # Filter small regions and holes
-            label = self._filterLabel(label, min_size)
+            label = self._filterLabel(label, min_size, df=df)
 
         return label
 
@@ -1058,7 +1064,7 @@ class mapSubObj(rectObj):
 
 
     #=======================================================================
-    def _filterLabel(self, l, min_size):
+    def _filterLabel(self, l, min_size, df=None):
         '''
         For a classified substrate label, small holes/objects are removed,
         and pixels classified as NoData are removed and adjecent class is
@@ -1080,8 +1086,10 @@ class mapSubObj(rectObj):
         Next Processing Step
         --------------------
         '''
-        # Get pixel size (in meters)
-        pix_m = self.pixM
+        # # Get pixel size (in meters)
+        # pix_m = self.pixM
+        pix_m = df['pixM'].values[0] if df is not None else 0.02
+
 
         # Convert min size to pixels
         min_size = int(min_size/pix_m)

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/class_portstarObj.py

@@ -37,6 +37,10 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 PACKAGE_DIR = os.path.dirname(SCRIPT_DIR)
 sys.path.append(PACKAGE_DIR)
 
+# # For Debug
+# from funcs_common import *
+# from funcs_model import *
+
 from pingmapper.funcs_common import *
 from pingmapper.funcs_model import *
 
@@ -1278,7 +1282,7 @@ class portstarObj(object):
             isChunk = son.sonMetaDF['chunk_id']==1
             sonMeta = son.sonMetaDF[isChunk].reset_index()
             # acousticBed = round(sonMeta['inst_dep_m'] / sonMeta['pix_m'], 0).astype(int)
-            acousticBed = round(sonMeta['inst_dep_m'] / self.pixM, 0).astype(int)
+            acousticBed = round(sonMeta['inst_dep_m'] / sonMeta['pixM'], 0).astype(int)
 
             ##################################
             # Step 1 : Acoustic Bedpick Filter
@@ -1403,7 +1407,8 @@ class portstarObj(object):
                    chunksPred,
                    detectDep=0,
                    smthDep=False,
-                   adjDep=False):
+                   adjDep=False,
+                   instDepAvail=True):
         '''
         Converts bedpick location (in pixels) to a depth in meters and additionally
         smooth and adjust depth estimate.
@@ -1478,8 +1483,8 @@ class portstarObj(object):
             portDF['dep_m_smth'] = smthDep
             starDF['dep_m_smth'] = smthDep
 
-            portDF['dep_m_adjBy'] = str(adjDep / self.port.pixM) + ' pixels'
-            starDF['dep_m_adjBy'] = str(adjDep / self.port.pixM) + ' pixels'
+            portDF['dep_m_adjBy'] = str(adjDep / portDF['pixM']) + ' pixels'
+            starDF['dep_m_adjBy'] = str(adjDep / starDF['pixM']) + ' pixels'
 
         elif detectDep > 0:
             # Prepare depth detection dictionaries
@@ -1532,8 +1537,8 @@ class portstarObj(object):
                 starFinal = savgol_filter(starFinal, 51, 3)
 
             # Convert pix to depth [m]
-            portFinal = np.asarray(portFinal) * self.port.pixM
-            starFinal = np.asarray(starFinal) * self.star.pixM
+            portFinal = np.asarray(portFinal) * portDF['pixM']
+            starFinal = np.asarray(starFinal) * starDF['pixM']
 
             # Set negatives to 0
             portFinal = np.where(portFinal<0, 0, portFinal)
@@ -1545,6 +1550,10 @@ class portstarObj(object):
             portDF['dep_m'] = portFinal
             starDF['dep_m'] = starFinal
 
+            if not instDepAvail:
+                portDF['inst_dep_m'] = 0
+                starDF['inst_dep_m'] = 0
+
             if adjDep != 0:
                 adjBy = adjDep
                 portDF['dep_m'] += adjBy
@@ -1560,8 +1569,23 @@ class portstarObj(object):
             portDF['dep_m_smth'] = smthDep
             starDF['dep_m_smth'] = smthDep
 
-            portDF['dep_m_adjBy'] = str(adjDep / self.port.pixM) + ' pixels'
-            starDF['dep_m_adjBy'] = str(adjDep / self.port.pixM) + ' pixels'
+            portDF['dep_m_adjBy'] = str(adjDep / portDF['pixM']) + ' pixels'
+            starDF['dep_m_adjBy'] = str(adjDep / starDF['pixM']) + ' pixels'
+
+        # Interpolate over nan's (and set zeros to nan)
+        portDep = portDF['dep_m'].to_numpy()
+        starDep = starDF['dep_m'].to_numpy()
+
+        portDep[portDep == 0] = np.nan
+        starDep[starDep == 0] = np.nan
+
+        nans, x = np.isnan(portDep), lambda z: z.nonzero()[0]
+        portDep[nans] = np.interp(x(nans), x(~nans), portDep[~nans])
+        portDF['dep_m'] = portDep
+
+        nans, x = np.isnan(starDep), lambda z: z.nonzero()[0]
+        starDep[nans] = np.interp(x(nans), x(~nans), starDep[~nans])
+        starDF['dep_m'] = starDep
 
         # Export to csv
         portDF.to_csv(self.port.sonMetaFile, index=False, float_format='%.14f')
@@ -1638,14 +1662,14 @@ class portstarObj(object):
         self.star._loadSonMeta()
         starDF = self.star.sonMetaDF
 
-        portDF = portDF.loc[portDF['chunk_id'] == i, ['inst_dep_m', 'dep_m']]
-        starDF = starDF.loc[starDF['chunk_id'] == i, ['inst_dep_m', 'dep_m']]
+        portDF = portDF.loc[portDF['chunk_id'] == i, ['inst_dep_m', 'dep_m', 'pixM']]
+        starDF = starDF.loc[starDF['chunk_id'] == i, ['inst_dep_m', 'dep_m', 'pixM']]
 
-        portInst = (portDF['inst_dep_m'] / self.port.pixM).to_numpy(dtype=int, copy=True)
-        portAuto = (portDF['dep_m'] / self.port.pixM).to_numpy(dtype=int, copy=True)
+        portInst = (portDF['inst_dep_m'] / portDF['pixM']).to_numpy(dtype=int, copy=True)
+        portAuto = (portDF['dep_m'] / portDF['pixM']).to_numpy(dtype=int, copy=True)
 
-        starInst = (starDF['inst_dep_m'] / self.star.pixM).to_numpy(dtype=int, copy=True)
-        starAuto = (starDF['dep_m'] / self.star.pixM).to_numpy(dtype=int, copy=True)
+        starInst = (starDF['inst_dep_m'] / starDF['pixM']).to_numpy(dtype=int, copy=True)
+        starAuto = (starDF['dep_m'] / starDF['pixM']).to_numpy(dtype=int, copy=True)
 
         # Ensure port/star same length
         if (portAuto.shape[0] != starAuto.shape[0]):
@@ -1807,8 +1831,8 @@ class portstarObj(object):
         starDF = self.star.sonMetaDF
 
         # Get depth/ pix scaler for given chunk
-        portDF = portDF.loc[portDF['chunk_id'] == i, ['dep_m']].reset_index()
-        starDF = starDF.loc[starDF['chunk_id'] == i, ['dep_m']].reset_index()
+        portDF = portDF.loc[portDF['chunk_id'] == i, ['dep_m', 'pixM']].reset_index()
+        starDF = starDF.loc[starDF['chunk_id'] == i, ['dep_m', 'pixM']].reset_index()
 
         # Load sonar
         self.port._getScanChunkSingle(i)
@@ -1842,8 +1866,8 @@ class portstarObj(object):
 
         ###########################################
         # Remove shadow predictions in water column
-        bedpickPort = round(portDF['dep_m'] / self.port.pixM, 0).astype(int)
-        bedpickStar = round(starDF['dep_m'] / self.star.pixM, 0).astype(int)
+        bedpickPort = round(portDF['dep_m'] / portDF['pixM'], 0).astype(int)
+        bedpickStar = round(starDF['dep_m'] / starDF['pixM'], 0).astype(int)
 
         for j in range(pMask.shape[1]):
             depth = bedpickPort[j]
@@ -2336,7 +2360,17 @@ class portstarObj(object):
         ## top-left coordinate a value of (0,0)
 
         # Get pixel size
-        pix_m = self.port.pixM
+        # pix_m = self.port.pixM
+        self.port._loadSonMeta()
+        isChunk = self.port.sonMetaDF['chunk_id']==chunk
+        sonMeta = self.port.sonMetaDF[isChunk].reset_index()
+
+        pixM = sonMeta['pixM']
+        # Find most common pixel size
+        if len(pixM.unique()) > 1:
+            pixM = pixM.mode()[0]
+        else:
+            pixM = pixM.iloc[0]
 
         # Determine min/max for rescaling
         xMin, xMax = dst[:,0].min(), dst[:,0].max() # Min/Max of x coordinates
@@ -2346,7 +2380,7 @@ class portstarObj(object):
         outShapeM = [xMax-xMin, yMax-yMin] # Calculate range of x,y coordinates
         outShape=[0,0]
         # Divide by pixel size to arrive at output shape of warped image
-        outShape[0], outShape[1] = round(outShapeM[0]/pix_m,0), round(outShapeM[1]/pix_m,0)
+        outShape[0], outShape[1] = round(outShapeM[0]/pixM,0), round(outShapeM[1]/pixM,0)
 
         # Rescale destination coordinates
         # X values

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/class_rectObj.py

@@ -36,8 +36,13 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 PACKAGE_DIR = os.path.dirname(SCRIPT_DIR)
 sys.path.append(PACKAGE_DIR)
 
+# # For depug
+# from funcs_common import *
+# from class_sonObj import sonObj
+
 from pingmapper.funcs_common import *
 from pingmapper.class_sonObj import sonObj
+
 from osgeo import gdal, ogr, osr
 from osgeo_utils.gdal_sieve import gdal_sieve
 from scipy.interpolate import splprep, splev
@@ -245,9 +250,15 @@ class rectObj(sonObj):
         # Attempt to fix error
         # https://stackoverflow.com/questions/47948453/scipy-interpolate-splprep-error-invalid-inputs
         okay = np.where(np.abs(np.diff(x))+np.abs(np.diff(y))>0)
-        x = np.r_[x[okay], x[-1]]
-        y = np.r_[y[okay], y[-1]]
-        t = np.r_[t[okay], t[-1]]
+        x = np.r_[x[okay], x[-1]].astype('float64')
+        y = np.r_[y[okay], y[-1]].astype('float64')
+        t = np.r_[t[okay], t[-1]].astype('float64')
+
+        # Remove any non-finite values (NaN or inf)
+        mask = np.isfinite(x) & np.isfinite(y) & np.isfinite(t)
+        x = x[mask]
+        y = y[mask]
+        t = t[mask]
 
         # Check if enough points to interpolate
         # If not, too many overlapping pings
@@ -258,14 +269,29 @@ class rectObj(sonObj):
         # Fit a spline to filtered coordinates and parameterize with time ellapsed
         try:
             tck, _ = splprep([x,y], u=t, k=deg, s=0)
-        except:
+        # except:
+        #     # Time is messed up (negative time offset)
+        #     # Parameterize with record num instead
+        #     zU = 'record_num'
+        #     t = dfFilt[zU].to_numpy()
+        #     t = np.r_[t[okay], t[-1]]
+        #     tck, _ = splprep([x,y], u=t, k=deg, s=0)
+        #     u_interp = dfOrig[zU].to_numpy()
+        except Exception as e:
+            print("splprep failed with error:", e)
             # Time is messed up (negative time offset)
             # Parameterize with record num instead
             zU = 'record_num'
-            t = dfFilt[zU].to_numpy()
+            t = dfFilt[zU].to_numpy(dtype='float64')
             t = np.r_[t[okay], t[-1]]
-            tck, _ = splprep([x,y], u=t, k=deg, s=0)
-            u_interp = dfOrig[zU].to_numpy()
+            # Ensure float and finite
+            t = np.asarray(t, dtype='float64')
+            mask = np.isfinite(x) & np.isfinite(y) & np.isfinite(t)
+            x = x[mask]
+            y = y[mask]
+            t = t[mask]
+            tck, _ = splprep([x, y], u=t, k=deg, s=0)
+            u_interp = dfOrig[zU].to_numpy(dtype='float64')
 
         x_interp = splev(u_interp, tck) # Use u_interp to get smoothed x/y coordinates from spline
 
@@ -279,7 +305,7 @@ class rectObj(sonObj):
                   'record_num': dfOrig['record_num'],
                   'ping_cnt': dfOrig['ping_cnt'],
                   'time_s': dfOrig['time_s'],
-                  'pix_m': self.pixM,
+                  'pixM': dfOrig['pixM'],
                   lons: x_interp[0],
                   lats: x_interp[1],
                   'dep_m': dfOrig['dep_m'],
@@ -563,17 +589,29 @@ class rectObj(sonObj):
         # maxPing = chunk[ping_cnt].max() # Find max ping count for each chunk
         # New method to find maxPing based on most numerous ping count
         maxPing = []
+        pixM_all = []
         for name, group in sDF.groupby(chunk_id):
             rangeCnt = np.unique(group[ping_cnt], return_counts=True)
             pingMaxi = np.argmax(rangeCnt[1])
             maxPing.append(int(rangeCnt[0][pingMaxi]))
+
+            # Get pixM from sonMetaDF
+            pixM = sonMetaDF.loc[sonMetaDF['chunk_id']==name, 'pixM'] # Get pixel size for each chunk
+            # Find most common pixel size
+            if len(pixM.unique()) > 1:
+                pixM = pixM.mode()[0]
+            else:
+                pixM = pixM.iloc[0]
+            pixM_all.append(pixM)
+
+
         # Convert maxPing i to pd series
         maxPing = pd.Series(maxPing)
 
         # pix_m = chunk['pix_m'].min() # Get pixel size for each chunk
-        pix_m = self.pixM # Get pixel size for each chunk
+        # pix_m = self.pixM # Get pixel size for each chunk
         for i in maxPing.index: # Calculate range (in meters) for each chunk
-            sDF.loc[sDF[chunk_id]==i, range_] = maxPing[i]*pix_m
+            sDF.loc[sDF[chunk_id]==i, range_] = maxPing[i]* pixM_all[i] # Calculate range in meters for each chunk
 
         ##################################################
         # Calculate range extent coordinates for each ping
@@ -617,7 +655,7 @@ class rectObj(sonObj):
         if cog:
             self._interpRangeCoords(filt)
         else:
-            sDF = sDF[['record_num', 'chunk_id', 'ping_cnt', 'time_s', 'lons', 'lats', 'utm_es', 'utm_ns', 'instr_heading', 'cog', 'dep_m', 'range', 'range_lon', 'range_lat', 'range_e', 'range_n', ping_bearing, 'transect']].copy()
+            sDF = sDF[['record_num', 'chunk_id', 'ping_cnt', 'time_s', 'lons', 'lats', 'utm_es', 'utm_ns', 'instr_heading', 'cog', 'dep_m', 'range', 'range_lon', 'range_lat', 'range_e', 'range_n', ping_bearing, 'transect', 'pixM']].copy()
             sDF.rename(columns={'lons': 'trk_lons', 'lats': 'trk_lats', 'utm_es': 'trk_utm_es', 'utm_ns': 'trk_utm_ns', 'cog': 'trk_cog', 'range_lat':'range_lats', 'range_lon':'range_lons', 'range_e':'range_es', 'range_n':'range_ns'}, inplace=True)
             sDF['chunk_id_2'] = sDF.index.astype(int)
 
@@ -736,7 +774,7 @@ class rectObj(sonObj):
         ##################################################
         # Join smoothed trackline to smoothed range extent
         # sDF = sDF[['record_num', 'chunk_id', 'ping_cnt', 'time_s', 'pix_m', 'lons', 'lats', 'utm_es', 'utm_ns', 'cog', 'dep_m']].copy()
-        sDF = sDF[['record_num', 'chunk_id', 'ping_cnt', 'time_s', 'lons', 'lats', 'utm_es', 'utm_ns', 'instr_heading', 'cog', 'dep_m', 'transect']].copy()
+        sDF = sDF[['record_num', 'chunk_id', 'ping_cnt', 'time_s', 'lons', 'lats', 'utm_es', 'utm_ns', 'instr_heading', 'cog', 'dep_m', 'transect', 'pixM']].copy()
         sDF.rename(columns={'lons': 'trk_lons', 'lats': 'trk_lats', 'utm_es': 'trk_utm_es', 'utm_ns': 'trk_utm_ns', 'cog': 'trk_cog'}, inplace=True)
         rsDF.rename(columns={'cog': 'range_cog'}, inplace=True)
         rsDF = rsDF[['record_num', 'range_lons', 'range_lats', 'range_cog']]
@@ -1037,6 +1075,7 @@ class rectObj(sonObj):
         n = 'n'
         record_num = 'record_num'
         chunk_id = 'chunk_id'
+        pixM = 'pixM'
 
         flip = False
 
@@ -1066,10 +1105,12 @@ class rectObj(sonObj):
         # Calculate ping bearing and normalize to range 0-360
         pingDF[ping_bearing] = (row[heading]+rotate) % 360
 
-        pix_m = self.pixM # Get pixel size for each chunk
+        # pix_m = self.pixM # Get pixel size for each chunk
+        pix_m = row['pixM'] # Get pixel size for each chunk
 
         # Calculate pixel size
         pingDF[son_range] = pingDF[son_idx] * pix_m
+        pingDF[pixM] = pix_m # Store pixel size in dataframe
 
         ##################################################
         # Calculate range extent coordinates for each ping
@@ -1123,7 +1164,7 @@ class rectObj(sonObj):
         # Calculate easting and northing
         pingDF[e], pingDF[n] = self.trans(pingDF[lons].to_numpy(), pingDF[lats].to_numpy())
 
-        pingDF = pingDF[[chunk_id, record_num, son_idx, lons, lats, e, n, son_range]]
+        pingDF = pingDF[[chunk_id, record_num, son_idx, lons, lats, e, n, son_range, pixM]]
 
         # Set index to help speed concatenation
         pingDF.set_index([record_num, son_idx], inplace=True)
@@ -1157,7 +1198,14 @@ class rectObj(sonObj):
         ## Destination coordinates describe the geographic location in lat/lon
         ## or easting/northing that directly map to the pix coordinates.
 
-        pix_m = self.pixM # Get pixel size
+        # pix_m = self.pixM # Get pixel size
+        pixM = df['pixM']
+        # Find most common pixel size
+        if len(pixM.unique()) > 1:
+            pixM = pixM.mode()[0]
+        else:
+            pixM = pixM.iloc[0]
+        pix_m = pixM
 
         # Get extent of chunk
         xMin, xMax = df[xCoord].min(), df[xCoord].max()
@@ -1324,10 +1372,17 @@ class rectObj(sonObj):
         
         '''
 
+        pixM = df['pixM']
+        # Find most common pixel size
+        if len(pixM.unique()) > 1:
+            pixM = pixM.mode()[0]
+        else:
+            pixM = pixM.iloc[0]
+
         pix_res = self.pix_res_son
         do_resize = True
         if pix_res == 0:
-            pix_res = self.pixM
+            pix_res = pixM
             do_resize = False
 
         if son:
@@ -1364,13 +1419,13 @@ class rectObj(sonObj):
         ##################
         # Do Rectification
 
-        pix_m = self.pixM # Get pixel size
+        pix_m = pixM # Get pixel size
 
-        xPixMax, yPixMax = df[xPix].max().astype(int), df[yPix].max().astype(int)
+        xPixMax, yPixMax = int(df[xPix].max()), int(df[yPix].max())
 
         # Get extent of chunk
-        xMin, xMax = df[xCoord].min().astype(int), df[xCoord].max().astype(int)
-        yMin, yMax = df[yCoord].min().astype(int), df[yCoord].max().astype(int)
+        xMin, xMax = int(df[xCoord].min()), int(df[xCoord].max())
+        yMin, yMax = int(df[yCoord].min()), int(df[yCoord].max())
 
         # Setup outupt array
         # Determine output shape dimensions
@@ -1876,9 +1931,20 @@ class rectObj(sonObj):
         filterIntensity = False
         pix_res = self.pix_res_son
         do_resize = True
-        if pix_res == 0:
-            pix_res = self.pixM
-            do_resize = False
+
+        # # Set pixel resolution
+        # self._loadSonMeta()
+        # sonMeta = self.sonMetaDF['chunk_id']==chunk
+        # pixM = sonMeta['pixM']
+        # # Find most common pixel size
+        # if len(pixM.unique()) > 1:
+        #     pixM = pixM.mode()[0]
+        # else:
+        #     pixM = pixM.iloc[0]
+
+        # if pix_res == 0:
+        #     pix_res = pixM
+        #     do_resize = False
 
         if son:
             # Create output directory if it doesn't exist
@@ -1909,10 +1975,18 @@ class rectObj(sonObj):
         # # Determine leading zeros to match naming convention
         addZero = self._addZero(chunk)
 
-        #################################
-        # Prepare pixel (pix) coordinates
-        ## Pix coordinates describe the size of the coordinates in pixel
-        ## coordinates (top left of image == (0,0); top right == (0,nchunk)...)
+        #############################################################
+        # Open smoothed trackline/range extent file
+        trkMeta = pd.read_csv(trkMetaFile)
+        if cog:
+            trkMeta = trkMeta[trkMeta['chunk_id']==chunk].reset_index(drop=False) # Filter df by chunk_id
+        else:
+            # trkMeta = trkMeta[trkMeta['chunk_id_2']==chunk].reset_index(drop=False)
+            # next = trkMeta[trkMeta['chunk_id_2']==chunk+1].reset_index(drop=False)
+            # trkMeta = pd.concat([trkMeta, next], ignore_index=True)
+            isChunk = trkMeta['chunk_id_2']==chunk
+            isChunk.iloc[chunk+1] = True
+            trkMeta = trkMeta[isChunk].reset_index(drop=False)
 
         # Filter sonMetaDF by chunk
         if not hasattr(self, 'sonMetaDF'):
@@ -1929,10 +2003,56 @@ class rectObj(sonObj):
 
         sonMeta = sonMetaAll[isChunk].reset_index()
 
+        filtSon = False
+        if len(sonMeta) != len(trkMeta):
+            if len(sonMeta) > len(trkMeta):
+                filtSon = True
+
+            # Filter df's to make sure they both have the same record_num
+            # Get the intersection of record_num values
+            common_record_nums = np.intersect1d(sonMeta['record_num'], trkMeta['record_num'])
+
+            # Filter both DataFrames to only include these record_num values
+            sonMeta_filtered = sonMeta[sonMeta['record_num'].isin(common_record_nums)]#.reset_index(drop=True)
+            trkMeta_filtered = trkMeta[trkMeta['record_num'].isin(common_record_nums)]#.reset_index(drop=True)
+
+            # Store the index's that were dropped
+            # Get dropped indexes for each DataFrame
+            dropped_sonMeta_idx = sonMeta.index.difference(sonMeta_filtered.index)
+            dropped_trkMeta_idx = trkMeta.index.difference(trkMeta_filtered.index)
+
+            sonMeta = sonMeta_filtered
+            trkMeta = trkMeta_filtered
+            
+            if filtSon:
+                idx_to_filt = dropped_sonMeta_idx.tolist()
+        
+
+        #################################
+        # Prepare pixel (pix) coordinates
+        ## Pix coordinates describe the size of the coordinates in pixel
+        ## coordinates (top left of image == (0,0); top right == (0,nchunk)...)
+
+        # # Filter sonMetaDF by chunk
+        # if not hasattr(self, 'sonMetaDF'):
+        #     self._loadSonMeta()
+
+        # sonMetaAll = self.sonMetaDF
+        # if cog:
+        #     isChunk = sonMetaAll['chunk_id']==chunk
+        # else:
+        #     isChunk = sonMetaAll['chunk_id_2']==chunk
+        #     # next = sonMetaAll['chunk_id_2']==(chunk+1)
+        #     # isChunk = pd.concat([isChunk, next], ignore_index=True)
+        #     isChunk.iloc[chunk+1] = True
+
+        # sonMeta = sonMetaAll[isChunk].reset_index()
+
         # Update class attributes based on current chunk
         self.pingMax = np.nanmax(sonMeta['ping_cnt']) # store to determine max range per chunk
         self.headIdx = sonMeta['index'] # store byte offset per ping
         self.pingCnt = sonMeta['ping_cnt'] # store ping count per ping
+        self.pixM = sonMeta['pixM'] # store pixel size per ping
 
         if son:
             # Open image to rectify
@@ -1951,6 +2071,13 @@ class rectObj(sonObj):
             del self.shadowMask
 
         img = self.sonDat
+
+        # Drop image columns if needed
+        if filtSon:
+            img = np.delete(img, dropped_sonMeta_idx, axis=1)
+            self.sonDat = img.copy()
+
+
         # if not cog:
         #     # Zero out second ping
         #     img[:,1] = 0
@@ -1983,19 +2110,26 @@ class rectObj(sonObj):
         ## Destination coordinates describe the geographic location in lat/lon
         ## or easting/northing that directly map to the pix coordinates.
 
-        # Open smoothed trackline/range extent file
-        trkMeta = pd.read_csv(trkMetaFile)
-        if cog:
-            trkMeta = trkMeta[trkMeta['chunk_id']==chunk].reset_index(drop=False) # Filter df by chunk_id
+        # # Open smoothed trackline/range extent file
+        # trkMeta = pd.read_csv(trkMetaFile)
+        # if cog:
+        #     trkMeta = trkMeta[trkMeta['chunk_id']==chunk].reset_index(drop=False) # Filter df by chunk_id
+        # else:
+        #     # trkMeta = trkMeta[trkMeta['chunk_id_2']==chunk].reset_index(drop=False)
+        #     # next = trkMeta[trkMeta['chunk_id_2']==chunk+1].reset_index(drop=False)
+        #     # trkMeta = pd.concat([trkMeta, next], ignore_index=True)
+        #     isChunk = trkMeta['chunk_id_2']==chunk
+        #     isChunk.iloc[chunk+1] = True
+        #     trkMeta = trkMeta[isChunk].reset_index(drop=False)
+
+        pixM = self.pixM
+        # Find most common pixel size
+        if len(pixM.unique()) > 1:
+            pixM = pixM.mode()[0]
         else:
-            # trkMeta = trkMeta[trkMeta['chunk_id_2']==chunk].reset_index(drop=False)
-            # next = trkMeta[trkMeta['chunk_id_2']==chunk+1].reset_index(drop=False)
-            # trkMeta = pd.concat([trkMeta, next], ignore_index=True)
-            isChunk = trkMeta['chunk_id_2']==chunk
-            isChunk.iloc[chunk+1] = True
-            trkMeta = trkMeta[isChunk].reset_index(drop=False)
+            pixM = pixM.iloc[0]
 
-        pix_m = self.pixM # Get pixel size
+        pix_m = pixM # Get pixel size
 
         # Get range (outer extent) coordinates [xR, yR] to transposed numpy arrays
         xR, yR = trkMeta[xRange].to_numpy().T, trkMeta[yRange].to_numpy().T
@@ -2031,7 +2165,9 @@ class rectObj(sonObj):
         outShapeM = [xMax-xMin, yMax-yMin] # Calculate range of x,y coordinates
         outShape=[0,0]
         # Divide by pixel size to arrive at output shape of warped image
-        outShape[0], outShape[1] = round(outShapeM[0]/pix_m,0), round(outShapeM[1]/pix_m,0)
+        # outShape[0], outShape[1] = round(outShapeM[0]/pix_m,0), round(outShapeM[1]/pix_m,0)
+        outShape[0], outShape[1] = round(outShapeM[0]/pix_res,0), round(outShapeM[1]/pix_res,0)
+        outShape = np.array(outShape).astype(int) # Convert to int
 
         # Rescale destination coordinates
         # X values

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/class_sonObj.py

@@ -285,6 +285,23 @@ class sonObj(object):
             sonDF = self._filterTime(sonDF, time_table)
 
         return sonDF
+    
+    # ======================================================================
+    def _filterShortTran(self, df):
+
+        '''
+        '''
+
+        # Make transects from consective pings using dataframe index
+        idx = df.index.values
+        transect_groups = np.split(idx, np.where(np.diff(idx) != 1)[0]+1)
+
+        for t in transect_groups:
+            if len(t) < self.nchunk:
+                # False means remove
+                df.loc[t, 'filter'] = False
+
+        return df
 
 
     # ======================================================================
@@ -515,7 +532,6 @@ class sonObj(object):
         idx = sonDF.index.values
         transect_groups = np.split(idx, np.where(np.diff(idx) != 1)[0]+1)
 
-        # print(transect_groups)
 
         # Assign transect
         transect = 0
@@ -741,6 +757,14 @@ class sonObj(object):
         for i in range(len(self.headIdx)):
             if ~np.isnan(self.headIdx[i]):
                 ping_len = min(self.pingCnt[i].astype(int), self.pingMax)
+
+
+                # #### Do not commit!!!!
+                # # if self.beamName == 'ss_star' or self.beamName == 'ss_port':
+                # #     ping_len *= 2
+                if not self.son8bit:
+                    ping_len *= 2
+
                 headIDX = self.headIdx[i].astype(int)
                 son_offset = self.son_offset[i].astype(int)
                 # pingIdx = headIDX + self.headBytes # Determine byte offset to sonar returns
@@ -755,7 +779,13 @@ class sonObj(object):
                     buffer = buffer[::-1]
 
                 # Read the data
-                dat = np.frombuffer(buffer, dtype='>u1')
+                if self.son8bit:# and self.beamName != 'ss_star' and self.beamName != 'ss_port':
+                    dat = np.frombuffer(buffer, dtype='>u1')
+                else:
+                    try:
+                        dat = np.frombuffer(buffer, dtype='>u2')
+                    except:
+                        dat = np.frombuffer(buffer[:-1], dtype='>u2')
 
                 try:
                     sonDat[:ping_len, i] = dat
@@ -764,7 +794,7 @@ class sonObj(object):
                     sonDat[:ping_len, i] = dat
         
         file.close()
-        self.sonDat = sonDat
+        self.sonDat = sonDat.astype(np.uint8)
         return
 
     # ======================================================================
@@ -786,7 +816,7 @@ class sonObj(object):
 
         # Load depth (in real units) and convert to pixels
         # bedPick = round(sonMeta['dep_m'] / sonMeta['pix_m'], 0).astype(int)
-        bedPick = round(sonMeta['dep_m'] / self.pixM, 0).astype(int)
+        bedPick = round(sonMeta['dep_m'] / sonMeta['pixM'], 0).astype(int)
         minDep = min(bedPick)
 
         del sonMeta, self.sonMetaDF
@@ -841,7 +871,7 @@ class sonObj(object):
         '''
         # Load depth (in real units) and convert to pixels
         # bedPick = round(sonMeta['dep_m'] / sonMeta['pix_m'], 0).astype(int)
-        bedPick = round(sonMeta['dep_m'] / self.pixM, 0).astype(int)
+        bedPick = round(sonMeta['dep_m'] / sonMeta['pixM'], 0).astype(int).reset_index(drop=True)
 
         # Initialize 2d array to store relocated sonar records
         srcDat = np.zeros((self.sonDat.shape[0], self.sonDat.shape[1])).astype(np.float32)#.astype(int)
@@ -890,7 +920,7 @@ class sonObj(object):
                   sonMeta,
                   crop=True):
         # Load depth (in real units) and convert to pixels
-        bedPick = round(sonMeta['dep_m'] / self.pixM, 0).astype(int)
+        bedPick = round(sonMeta['dep_m'] / sonMeta['pixM'], 0).astype(int)
         minDep = min(bedPick)
 
         sonDat = self.sonDat
@@ -910,7 +940,7 @@ class sonObj(object):
     def _WCO(self,
                   sonMeta):
         # Load depth (in real units) and convert to pixels
-        bedPick = round(sonMeta['dep_m'] / self.pixM, 0).astype(int)
+        bedPick = round(sonMeta['dep_m'] / sonMeta['pixM'], 0).astype(int)
         maxDep = max(bedPick)
 
         sonDat = self.sonDat
@@ -1273,8 +1303,15 @@ class sonObj(object):
                 d = sonMeta['trk_dist'].to_numpy()
                 d = np.max(d) - np.min(d)
 
+                pixM = sonMeta['pixM']
+                # Find most common pixel size
+                if len(pixM.unique()) > 1:
+                    pixM = pixM.mode()[0]
+                else:
+                    pixM = pixM.iloc[0]
+
                 # Distance in pix
-                d = round(d / self.pixM, 0).astype(int)
+                d = round(d / pixM, 0).astype(int)
 
                 sonDat = resize(sonDat,
                                 (sonDat.shape[0], d),
@@ -1988,7 +2025,7 @@ class sonObj(object):
         egn_means = self.egn_bed_means.copy() # Don't want to overwrite
 
         # Get bedpicks, in pixel units
-        bedPick = round(sonMeta['dep_m'] / self.pixM, 0).astype(int)
+        bedPick = round(sonMeta['dep_m'] / sonMeta['pixM'], 0).astype(int).to_numpy()
 
         # Iterate each ping
         for j in range(sonDat.shape[1]):
@@ -2057,7 +2094,7 @@ class sonObj(object):
             del t, l
 
         # Get bedpicks, in pixel units
-        bedPick = round(sonMeta['dep_m'] / self.pixM, 0).astype(int)
+        bedPick = round(sonMeta['dep_m'] / sonMeta['pixM'], 0).astype(int).to_numpy()
 
         # Iterate each ping
         for j in range(sonDat.shape[1]):

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/funcs_rectify.py

@@ -36,6 +36,10 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 PACKAGE_DIR = os.path.dirname(SCRIPT_DIR)
 sys.path.append(PACKAGE_DIR)
 
+# # For Debug
+# from funcs_common import *
+# from class_rectObj import rectObj
+
 from pingmapper.funcs_common import *
 from pingmapper.class_rectObj import rectObj
 
@@ -171,6 +175,7 @@ def smoothTrackline(projDir='', x_offset='', y_offset='', nchunk ='', cog=True,
 
             sDF['chunk_id'] = sonDF['chunk_id']
             sDF['transect'] = sonDF['transect']
+            # sDF['pixM'] = sonDF['pixM'] # Add pixel size to smoothed trackline coordinates
 
             sDF.reset_index(inplace=True)
 
@@ -250,14 +255,17 @@ def smoothTrackline(projDir='', x_offset='', y_offset='', nchunk ='', cog=True,
                 sDF.at[curRow, "utm_ns"] = lastRow["utm_ns"]
                 sDF.at[curRow, "cog"] = lastRow["cog"]
                 sDF.at[curRow, "instr_heading"] = lastRow["instr_heading"]
+                # sDF.at[curRow, 'pixM'] = lastRow['pixM']
+
+                del lastRow
             else:
                 t += 1
 
             i+=1
-        del lastRow, curRow, i
+        del curRow, i
 
         son0.smthTrk = sDF # Store smoothed trackline coordinates in rectObj.
-
+        
         # Do positional correction
         if x_offset != 0.0 or y_offset != 0.0:
             son0._applyPosOffset(x_offset, y_offset)
@@ -274,6 +282,7 @@ def smoothTrackline(projDir='', x_offset='', y_offset='', nchunk ='', cog=True,
         df = son1.sonMetaDF
         sDF['chunk_id'] = df['chunk_id'] # Update chunk_id for smoothed coordinates
         sDF['record_num'] = df['record_num'] # Update record_num for smoothed coordinates
+        # sDF['pixM'] = df['pixM']
         son1.smthTrk = sDF # Store smoothed trackline coordinates in rectObj
 
         del sDF, df, son0, son1

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/gui_main.py

@@ -80,7 +80,8 @@ def gui(batch: bool):
         text_input = sg.Text('Recording to Process')
         # in_input = sg.In(key='inFile', size=(80,1))
         in_input = sg.In(key='inFile', size=(80,1), default_text=default_params['inFile'])
-        browse_input = sg.FileBrowse(file_types=(("Sonar File", "*.DAT *.sl2 *.sl3 *.svlog") ), initial_folder=os.path.dirname(default_params['inFile']))
+        browse_input = sg.FileBrowse(file_types=(("Sonar File", "*.DAT *.sl2 *.sl3 *.RSD *.svlog") ), initial_folder=os.path.dirname(default_params['inFile']))
+        # browse_input = sg.FileBrowse(file_types=(("Sonar File", "*.DAT *.sl2 *.sl3 *.svlog") ), initial_folder=os.path.dirname(default_params['inFile']))
 
     # Add to layout
     layout.append([text_io])
@@ -591,9 +592,10 @@ def gui(batch: bool):
             # Find all DAT and SON files in all subdirectories of inDir
             inFiles=[]
             for root, dirs, files in os.walk(inDir):
-                for file in files:
-                    if file.endswith('.DAT') or file.endswith('.sl2') or file.endswith('.sl3'):
-                        inFiles.append(os.path.join(root, file))
+                if '__MACOSX' not in root:
+                    for file in files:
+                        if file.endswith('.DAT') or file.endswith('.sl2') or file.endswith('.sl3') or file.endswith('.RSD') or file.endswith('.svlog'):
+                            inFiles.append(os.path.join(root, file))
 
             inFiles = sorted(inFiles)
 

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/main_mapSubstrate.py

@@ -35,8 +35,13 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 PACKAGE_DIR = os.path.dirname(SCRIPT_DIR)
 sys.path.append(PACKAGE_DIR)
 
-from pingmapper.funcs_common import *
+# # For debug
+# from funcs_common import *
+# from class_mapSubstrateObj import mapSubObj
+# from class_portstarObj import portstarObj
+# from funcs_model import *
 
+from pingmapper.funcs_common import *
 from pingmapper.class_mapSubstrateObj import mapSubObj
 from pingmapper.class_portstarObj import portstarObj
 from pingmapper.funcs_model import *
@@ -275,7 +280,7 @@ def map_master_func(logfilename='',
             # Do prediction (make parallel later)
             print('\n\tPredicting substrate for', len(chunks), son.beamName, 'chunks')
 
-            Parallel(n_jobs=np.min([len(chunks), threadCnt]), verbose=10)(delayed(son._detectSubstrate)(i, USE_GPU) for i in chunks)
+            Parallel(n_jobs=np.min([len(chunks), threadCnt]))(delayed(son._detectSubstrate)(i, USE_GPU) for i in tqdm(chunks))
 
             son._cleanup()
             son._pickleSon()
@@ -324,7 +329,7 @@ def map_master_func(logfilename='',
 
             # Plot substrate classification()
             # sys.exit()
-            Parallel(n_jobs=np.min([len(toMap), threadCnt]), verbose=10)(delayed(son._pltSubClass)(map_class_method, c, f, spdCor=spdCor, maxCrop=maxCrop, probs=probs) for c, f in toMap.items())
+            Parallel(n_jobs=np.min([len(toMap), threadCnt]))(delayed(son._pltSubClass)(map_class_method, c, f, spdCor=spdCor, maxCrop=maxCrop, probs=probs) for c, f in tqdm((toMap.items())))
             son._pickleSon()
             del toMap
 
@@ -384,7 +389,7 @@ def map_master_func(logfilename='',
         # Create portstarObj
         psObj = portstarObj(mapObjs)
 
-        Parallel(n_jobs=np.min([len(toMap), threadCnt]), verbose=10)(delayed(psObj._mapSubstrate)(map_class_method, c, f) for c, f in toMap.items())
+        Parallel(n_jobs=np.min([len(toMap), threadCnt]))(delayed(psObj._mapSubstrate)(map_class_method, c, f) for c, f in tqdm(toMap.items()))
 
         del toMap
         print("\nDone!")
@@ -524,7 +529,7 @@ def map_master_func(logfilename='',
         # Create portstarObj
         psObj = portstarObj(mapObjs)
 
-        Parallel(n_jobs=np.min([len(toMap), threadCnt]), verbose=10)(delayed(psObj._mapPredictions)(map_predict, 'map_'+a, c, f) for c, f in toMap.items())
+        Parallel(n_jobs=np.min([len(toMap), threadCnt]))(delayed(psObj._mapPredictions)(map_predict, 'map_'+a, c, f) for c, f in tqdm(toMap.items()))
 
         del toMap, psObj
         print("\nDone!")

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/main_readFiles.py

@@ -50,9 +50,9 @@ from doodleverse_utils.imports import *
 
 from scipy.signal import savgol_filter
 
-# sys.path.insert(0, r'C:\Users\cbodine\PythonRepos\PINGVerter')
+sys.path.insert(0, r'Z:\UDEL\PythonRepos\PINGVerter')
 
-from pingverter import hum2pingmapper, low2pingmapper, cerul2pingmapper
+from pingverter import hum2pingmapper, low2pingmapper, cerul2pingmapper, gar2pingmapper
 
 import cv2
 
@@ -314,6 +314,7 @@ def read_master_func(logfilename='',
     # Use PINGVerter to read the sonar file
     #######################################
 
+    instDepAvail = True
     start_time = time.time()
     # Determine sonar recording type
     _, file_type = os.path.splitext(inFile)
@@ -326,15 +327,27 @@ def read_master_func(logfilename='',
     elif file_type == '.sl2' or file_type == '.sl3':
         sonar_obj = low2pingmapper(inFile, projDir, nchunk, tempC, exportUnknown)
 
+    # Prepare Garmin file for PINGMapper
+    elif file_type == '.RSD':
+        sonar_obj = gar2pingmapper(inFile, projDir, nchunk, tempC, exportUnknown)
+
     # Prepare Cerulean file for PINGMapper
     elif file_type == '.svlog':
         sonar_obj = cerul2pingmapper(inFile, projDir, nchunk, tempC, exportUnknown)
         detectDep = 1 # No depth in cerulean files, so set to Zheng et al. 2021
+        instDepAvail = False
+
+    # Unknown
+    else:
+        print('\n\nERROR!\n\nFile type {} not supported at this time.'.format(file_type))
+        sys.exit()
 
     ####################
     # Create son objects
     ####################
 
+    # print(sonar_obj)
+
     # Get available beams and metadata
     beamMeta = sonar_obj.beamMeta
 
@@ -356,10 +369,17 @@ def read_master_func(logfilename='',
         son.beamName = meta['beamName']
         son.beam = beam
         son.headBytes = sonar_obj.headBytes
-        son.pixM = sonar_obj.pixM
+        # son.pixM = sonar_obj.pixM
         son.isOnix = sonar_obj.isOnix
         son.trans = sonar_obj.trans
         son.humDat = sonar_obj.humDat
+        # if son.beamName == 'ss_port' or son.beamName == 'ss_star':
+        #     son.son8bit = sonar_obj.son8bit
+        # else:
+        son.son8bit = sonar_obj.son8bit
+
+        # print(son.beamName, son.son8bit)
+
 
         if pix_res_son == 0:
             son.pix_res_son = 0
@@ -408,13 +428,24 @@ def read_master_func(logfilename='',
             son.cropRange = cropRange
             # Do range crop, if necessary
             if cropRange > 0.0:
-                # Convert to distance in pix
-                d = round(cropRange / son.pixM, 0).astype(int)
+                # # Convert to distance in pix
+                # d = round(cropRange / son.pixM, 0).astype(int)
+
+                # # Get sonMetaDF
+                # son._loadSonMeta()
+                # son.sonMetaDF.loc[son.sonMetaDF['ping_cnt'] > d, 'ping_cnt'] = d
+                # son._saveSonMetaCSV(son.sonMetaDF)
 
                 # Get sonMetaDF
                 son._loadSonMeta()
-                son.sonMetaDF.loc[son.sonMetaDF['ping_cnt'] > d, 'ping_cnt'] = d
-                son._saveSonMetaCSV(son.sonMetaDF)
+                df = son.sonMetaDF
+
+                # Convert to distance in pixels
+                d = round(cropRange / df['pixM'], 0).astype(int)
+
+                # Filter df
+                df.loc[df['ping_cnt'] > d, 'ping_cnt'] = d
+                son._saveSonMetaCSV(df)
 
         # Store flag to export un-rectified sonar tiles in each sonObj.
         for son in sonObjs:
@@ -632,7 +663,7 @@ def read_master_func(logfilename='',
         del c, r, n, startB, rowCnt
 
         # Fix no data in parallel
-        r = Parallel(n_jobs=threadCnt)(delayed(son._fixNoDat)(dfAll[r[0]:r[1]].copy().reset_index(drop=True), beams) for r in tqdm(range(len(rowsToProc))))
+        r = Parallel(n_jobs=threadCnt)(delayed(son._fixNoDat)(dfAll[r[0]:r[1]].copy().reset_index(drop=True), beams) for r in tqdm(rowsToProc))
         gc.collect()
 
         # Concatenate results from parallel processing
@@ -820,6 +851,14 @@ def read_master_func(logfilename='',
         df0 = df0[df0['filter'] == True]
         df1 = df1[df1['filter'] == True]
 
+        # Remove transect shorter then nchunk
+        df0=son0._filterShortTran(df0)
+        df1['filter'] = df0['filter']
+
+        # Apply the filter
+        df0 = df0[df0['filter'] == True]
+        df1 = df1[df1['filter'] == True]
+
         # Reasign the chunks
         df0 = son0._reassignChunks(df0)
         df1['chunk_id'] = df0['chunk_id']
@@ -916,7 +955,7 @@ def read_master_func(logfilename='',
             print('\n\tUsing binary thresholding...')
 
         # Parallel estimate depth for each chunk using appropriate method
-        r = Parallel(n_jobs=np.min([len(chunks), threadCnt]))(delayed(psObj._detectDepth)(detectDep, int(chunk), USE_GPU, tileFile) for chunk in tqdm(range(len(chunks))))
+        r = Parallel(n_jobs=np.min([len(chunks), threadCnt]))(delayed(psObj._detectDepth)(detectDep, int(chunk), USE_GPU, tileFile) for chunk in tqdm(chunks))
 
         # store the depth predictions in the class
         for ret in r:
@@ -941,7 +980,7 @@ def read_master_func(logfilename='',
 
     if saveDepth:
         # Save detected depth to csv
-        depDF = psObj._saveDepth(chunks, detectDep, smthDep, adjDep)
+        depDF = psObj._saveDepth(chunks, detectDep, smthDep, adjDep, instDepAvail)
 
         # Store depths in downlooking sonar files also
         for son in sonObjs:
@@ -961,6 +1000,12 @@ def read_master_func(logfilename='',
                 dep = sonDF['inst_dep_m']
                 if smthDep:
                     dep = savgol_filter(dep, 51, 3)
+
+                # Interpolate over nan's (and set zero's to nan)
+                dep = dep.to_numpy()
+                dep[dep==0] = np.nan
+                nans, x = np.isnan(dep), lambda z: z.nonzero()[0]
+                dep[nans] = np.interp(x(nans), x(~nans), dep[~nans])
                 
                 sonDF['dep_m'] = dep + adjDep
 
@@ -982,7 +1027,7 @@ def read_master_func(logfilename='',
         start_time = time.time()
 
         print("\n\nExporting bedpick plots to {}...".format(tileFile))
-        Parallel(n_jobs=np.min([len(chunks), threadCnt]))(delayed(psObj._plotBedPick)(int(chunk), True, autoBed, tileFile) for chunk in tqdm(range(len(chunks))))
+        Parallel(n_jobs=np.min([len(chunks), threadCnt]))(delayed(psObj._plotBedPick)(int(chunk), True, autoBed, tileFile) for chunk in tqdm(chunks))
 
         print("\nDone!")
         print("Time (s):", round(time.time() - start_time, ndigits=1))
@@ -1068,7 +1113,7 @@ def read_master_func(logfilename='',
         psObj.port.shadow = defaultdict()
         psObj.star.shadow = defaultdict()
 
-        r = Parallel(n_jobs=np.min([len(chunks), threadCnt]))(delayed(psObj._detectShadow)(remShadow, int(chunk), USE_GPU, False, tileFile) for chunk in tqdm(range(len(chunks))))
+        r = Parallel(n_jobs=np.min([len(chunks), threadCnt]))(delayed(psObj._detectShadow)(remShadow, int(chunk), USE_GPU, False, tileFile) for chunk in tqdm(chunks))
 
         for ret in r:
             psObj.port.shadow[ret[0]] = ret[1]
@@ -1123,7 +1168,7 @@ def read_master_func(logfilename='',
 
                 # Calculate range-wise mean intensity for each chunk
                 print('\n\tCalculating range-wise mean intensity for each chunk...')
-                chunk_means = Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._egnCalcChunkMeans)(i) for i in tqdm(range(len(chunks))))
+                chunk_means = Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._egnCalcChunkMeans)(i) for i in tqdm(chunks))
 
                 # Calculate global means
                 print('\n\tCalculating range-wise global means...')
@@ -1132,7 +1177,7 @@ def read_master_func(logfilename='',
 
                 # Calculate egn min and max for each chunk
                 print('\n\tCalculating EGN min and max values for each chunk...')
-                min_max = Parallel(n_jobs= np.min([len(chunks)]))(delayed(son._egnCalcMinMax)(i) for i in tqdm(range(len(chunks))))
+                min_max = Parallel(n_jobs= np.min([len(chunks)]))(delayed(son._egnCalcMinMax)(i) for i in tqdm(chunks))
 
                 # Calculate global min max for each channel
                 son._egnCalcGlobalMinMax(min_max)
@@ -1184,7 +1229,7 @@ def read_master_func(logfilename='',
                     chunks = chunks[:-1] # remove last chunk
 
                     print('\n\tCalculating EGN corrected histogram for', son.beamName)
-                    hist = Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._egnCalcHist)(i) for i in tqdm(range(len(chunks))))
+                    hist = Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._egnCalcHist)(i) for i in tqdm(chunks))
 
                     print('\n\tCalculating global EGN corrected histogram')
                     son._egnCalcGlobalHist(hist)
@@ -1311,8 +1356,10 @@ def read_master_func(logfilename='',
                 # Load sonMetaDF
                 son._loadSonMeta()
 
-                # Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._exportTiles)(i, tileFile) for i in tqdm(range(len(chunks))))
-                Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._exportTilesSpd)(i, tileFile=imgType, spdCor=spdCor, mask_shdw=mask_shdw, maxCrop=maxCrop) for i in tqdm(range(len(chunks))))
+                Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._exportTilesSpd)(i, tileFile=imgType, spdCor=spdCor, mask_shdw=mask_shdw, maxCrop=maxCrop) for i in tqdm(chunks))
+                # for i in tqdm(chunks):
+                #     son._exportTilesSpd(i, tileFile=imgType, spdCor=spdCor, mask_shdw=mask_shdw, maxCrop=maxCrop)
+                #     sys.exit()
 
                 if moving_window and not spdCor:
 

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/main_rectify.py

@@ -44,10 +44,10 @@ sys.path.append(PACKAGE_DIR)
 # from class_portstarObj import portstarObj
 # from funcs_rectify import smoothTrackline
 
-# from pingmapper.funcs_common import *
-# from pingmapper.class_rectObj import rectObj
-# from pingmapper.class_portstarObj import portstarObj
-# from pingmapper.funcs_rectify import smoothTrackline
+from pingmapper.funcs_common import *
+from pingmapper.class_rectObj import rectObj
+from pingmapper.class_portstarObj import portstarObj
+from pingmapper.funcs_rectify import smoothTrackline
 
 import inspect
 
@@ -310,7 +310,7 @@ def rectify_master_func(logfilename='',
     # COG Pre-processing                                                       #
     # ##########################################################################
 
-    for son in portstar:
+    # for son in portstar:
         son.rect_wcp = rect_wcp
         son.rect_wcr = rect_wcr
 
@@ -364,7 +364,7 @@ def rectify_master_func(logfilename='',
             print('\n\tExporting', len(chunks), 'GeoTiffs for', son.beamName)
 
             # Parallel(n_jobs= np.min([len(sDF), threadCnt]))(delayed(son._rectSonHeadingMain)(sonarCoordsDF[sonarCoordsDF['chunk_id']==chunk], chunk) for chunk in tqdm(range(len(chunks))))
-            Parallel(n_jobs= np.min([len(sDF), threadCnt]))(delayed(son._rectSonHeadingMain)(sDF[sDF['chunk_id']==chunk], chunk, heading=heading, interp_dist=rectInterpDist) for chunk in tqdm(range(len(chunks))))
+            Parallel(n_jobs= np.min([len(sDF), threadCnt]))(delayed(son._rectSonHeadingMain)(sDF[sDF['chunk_id']==chunk], chunk, heading=heading, interp_dist=rectInterpDist) for chunk in tqdm(chunks))
             # for i in chunks:
             #     # son._rectSonHeading(sonarCoordsDF[sonarCoordsDF['chunk_id']==i], i)
             #     r = son._rectSonHeadingMain(sDF[sDF['chunk_id']==i], i, heading=heading, interp_dist=rectInterpDist)
@@ -402,6 +402,8 @@ def rectify_master_func(logfilename='',
         son.rect_wcr = rect_wcr
 
     if (rect_wcp and rubberSheeting) or (rect_wcr and rubberSheeting):
+        # Always use COG for rubber sheeting
+        cog = True
         for son in portstar:
             # Set output directory
             son.outDir = os.path.join(son.projDir, son.beamName)
@@ -418,8 +420,8 @@ def rectify_master_func(logfilename='',
             print('\n\tExporting', len(chunks), 'GeoTiffs for', son.beamName)
             # for i in chunks:
             #     son._rectSonRubber(i, filter, cog, wgs=False)
-            #     sys.exit()
-            Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._rectSonRubber)(i, filter, cog, wgs=False) for i in tqdm(range(len(chunks))))
+                # sys.exit()
+            Parallel(n_jobs= np.min([len(chunks), threadCnt]))(delayed(son._rectSonRubber)(i, filter, cog, wgs=False) for i in tqdm(chunks))
             son._cleanup()
             gc.collect()
             printUsage()

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper/test_time.py

@@ -17,9 +17,5 @@ converted_date = start_date + timedelta(seconds=custom_unix_time)
 eastern = pytz.timezone("US/Eastern")
 converted_date_eastern = converted_date.astimezone(eastern)
 
-print(start_date)
-print(unix_timestamp)
 print("Custom Unix Start Time:", unix_timestamp)
-print(converted_date)
-print("Converted Date and Time:", converted_date_eastern)
-print(timedelta(seconds=custom_unix_time))
+print("Converted Date and Time:", converted_date_eastern)

pingmapper-5.0.0/pingmapper/version.py

@@ -0,0 +1 @@
+__version__ = '5.0.0'

{pingmapper-4.2.12 → pingmapper-5.0.0}/pingmapper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pingmapper
-Version: 4.2.12
+Version: 5.0.0
 Summary: Open-source interface for processing recreation-grade side scan sonar datasets and reproducibly mapping benthic habitat
 Author: Cameron Bodine
 Author-email: bodine.cs@gmail.email

pingmapper-4.2.12/pingmapper/version.py

@@ -1 +0,0 @@
-__version__ = '4.2.12'