pymast 1.0.1__tar.gz → 1.0.2__tar.gz

{pymast-1.0.1 → pymast-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pymast
-Version: 1.0.1
+Version: 1.0.2
 Summary: Movement Analysis Software for Telemetry (MAST) - False positive removal and movement analysis for radio telemetry data
 Author: Theodore Castro-Santos
 Author-email: "Kevin P. Nebiolo" <kevin.nebiolo@kleinschmidtgroup.com>

{pymast-1.0.1 → pymast-1.0.2}/pymast/__init__.py

@@ -28,7 +28,7 @@ from .parsers import *
 from .radio_project import *
 
 # Version
-__version__ = '1.0.0'
+__version__ = '1.0.2'
 
 # Define what's available when using "from pymast import *"
 __all__ = [

{pymast-1.0.1 → pymast-1.0.2}/pymast/parsers.py

@@ -1840,19 +1840,19 @@ def PIT(file_name,
         else:
             # try to find an antenna column in the fixed-width frame
             antenna_col = None
-            for col in telem_dat.columns:
-                col_lower = str(col).lower().strip()
-                if col_lower in (
-                    'antenna id',
-                    'antenna',
-                    'ant',
-                    'antennae',
-                    'antennae id',
-                    'reader id',
-                    'readerid',
-                ):
-                    antenna_col = col
-                    break
+            for col in telem_dat.columns:
+                col_lower = str(col).lower().strip()
+                if col_lower in (
+                    'antenna id',
+                    'antenna',
+                    'ant',
+                    'antennae',
+                    'antennae id',
+                    'reader id',
+                    'readerid',
+                ):
+                    antenna_col = col
+                    break
 
             if antenna_col is not None:
                 # extract numeric antenna identifier and map using provided dictionary
@@ -1884,11 +1884,11 @@ def PIT(file_name,
 
                 # drop detections that do not map to a known receiver
                 telem_dat = telem_dat.dropna(subset=['rec_id'])
-            else:
-                raise ValueError(
-                    'Multi-antenna fixed-width PIT file requires an antenna/reader column '
-                    '(e.g., "Antenna ID" or "Reader ID"), but none was found'
-                )
+            else:
+                raise ValueError(
+                    'Multi-antenna fixed-width PIT file requires an antenna/reader column '
+                    '(e.g., "Antenna ID" or "Reader ID"), but none was found'
+                )
 
     # Data cleaning - remove invalid entries
     print(f"\nCleaning data - original records: {len(telem_dat)}")

{pymast-1.0.1 → pymast-1.0.2}/pymast/radio_project.py

@@ -95,21 +95,21 @@ import pymast.predictors as predictors
 import matplotlib.pyplot as plt
 from matplotlib import rcParams
 from scipy import interpolate
-try:
-    from tqdm import tqdm
-except ImportError:
-    def tqdm(iterable, **kwargs):
-        return iterable
+try:
+    from tqdm import tqdm
+except ImportError:
+    def tqdm(iterable, **kwargs):
+        return iterable
 import shutil
 import warnings
 import dask.dataframe as dd
 import dask.array as da
-try:
-    from dask_ml.cluster import KMeans
-    _KMEANS_IMPL = 'dask'
-except ImportError:
-    from sklearn.cluster import KMeans
-    _KMEANS_IMPL = 'sklearn'
+try:
+    from dask_ml.cluster import KMeans
+    _KMEANS_IMPL = 'dask'
+except ImportError:
+    from sklearn.cluster import KMeans
+    _KMEANS_IMPL = 'sklearn'
 
 # Initialize logger
 logger = logging.getLogger('pymast.radio_project')
@@ -415,12 +415,12 @@ class radio_project():
         if self.non_interactive:
             logger.debug(f"Non-interactive mode: auto-answering '{prompt_text}' with '{default}'")
             return default
-        try:
-            return input(prompt_text)
-        except (EOFError, OSError) as exc:
-            raise RuntimeError(
-                "Input prompt failed. Set project.non_interactive = True to use defaults."
-            ) from exc
+        try:
+            return input(prompt_text)
+        except (EOFError, OSError) as exc:
+            raise RuntimeError(
+                "Input prompt failed. Set project.non_interactive = True to use defaults."
+            ) from exc
             
     def telem_data_import(self,
                           rec_id,
@@ -496,9 +496,19 @@ class radio_project():
         
         logger.info(f"  Found {len(tFiles)} file(s) to import")
         
+        # Track detections per file for statistics
+        detections_per_file = []
+        
         # for every file call the correct text parser and import
         for i, f in enumerate(tqdm(tFiles, desc=f"Importing {rec_id}", unit="file"), 1):
             logger.debug(f"  Processing file {i}/{len(tFiles)}: {f}")
+            
+            # Count detections before import
+            try:
+                pre_count = len(pd.read_hdf(self.db, key='raw_data', where=f'rec_id = "{rec_id}"'))
+            except (KeyError, FileNotFoundError):
+                pre_count = 0
+            
             # get the complete file directory
             f_dir = os.path.join(file_dir,f)
             
@@ -533,8 +543,91 @@ class radio_project():
             else:
                 logger.error(f"No import routine for receiver type: {rec_type}")
                 raise ValueError(f"No import routine available for receiver type: {rec_type}")
+            
+            # Count detections after import
+            try:
+                post_count = len(pd.read_hdf(self.db, key='raw_data', where=f'rec_id = "{rec_id}"'))
+                detections_this_file = post_count - pre_count
+                detections_per_file.append(detections_this_file)
+            except (KeyError, FileNotFoundError):
+                detections_per_file.append(0)
         
         logger.info(f"✓ Import complete for receiver {rec_id}: {len(tFiles)} file(s) processed")
+        
+        # Calculate and display import statistics
+        try:
+            raw_data = pd.read_hdf(self.db, key='raw_data', where=f'rec_id = "{rec_id}"')
+            
+            # Total Detection Count
+            total_detections = len(raw_data)
+            logger.info(f"\n{'='*60}")
+            logger.info(f"IMPORT STATISTICS FOR {rec_id}")
+            logger.info(f"{'='*60}")
+            logger.info(f"Total Detection Count: {total_detections:,}")
+            
+            if total_detections > 0:
+                # Detection count summary statistics
+                logger.info(f"\nDetection Summary Statistics:")
+                logger.info(f"  Mean detections per file: {total_detections / len(tFiles):.1f}")
+                logger.info(f"  Files processed: {len(tFiles)}")
+                
+                # 5-number summary for detections per file
+                if len(detections_per_file) > 0:
+                    det_array = np.array(detections_per_file)
+                    logger.info(f"\nDetections Per File (5-number summary):")
+                    logger.info(f"  Min:    {np.min(det_array):,.0f}")
+                    logger.info(f"  Q1:     {np.percentile(det_array, 25):,.0f}")
+                    logger.info(f"  Median: {np.median(det_array):,.0f}")
+                    logger.info(f"  Q3:     {np.percentile(det_array, 75):,.0f}")
+                    logger.info(f"  Max:    {np.max(det_array):,.0f}")
+                
+                # Unique Tag Count
+                unique_tags = raw_data['freq_code'].nunique()
+                logger.info(f"\nUnique Tag Count: {unique_tags}")
+                
+                # Duplicate Tag Count and IDs
+                # Check for detections at the exact same timestamp (true duplicates)
+                if 'time_stamp' in raw_data.columns:
+                    dup_mask = raw_data.duplicated(subset=['freq_code', 'time_stamp'], keep=False)
+                    duplicate_count = dup_mask.sum()
+                    
+                    if duplicate_count > 0:
+                        duplicate_tags = raw_data.loc[dup_mask, 'freq_code'].unique()
+                        logger.info(f"\nDuplicate Detection Count (same timestamp): {duplicate_count:,}")
+                        logger.info(f"Duplicate Tag IDs ({len(duplicate_tags)} tags):")
+                        for tag in sorted(duplicate_tags)[:10]:  # Show first 10
+                            tag_dups = dup_mask & (raw_data['freq_code'] == tag)
+                            logger.info(f"  {tag}: {tag_dups.sum()} duplicate(s)")
+                        if len(duplicate_tags) > 10:
+                            logger.info(f"  ... and {len(duplicate_tags) - 10} more")
+                    else:
+                        logger.info(f"\nDuplicate Detection Count: 0 (no exact timestamp duplicates)")
+                
+                # Time Coverage
+                if 'time_stamp' in raw_data.columns:
+                    raw_data['time_stamp'] = pd.to_datetime(raw_data['time_stamp'])
+                    start_time = raw_data['time_stamp'].min()
+                    end_time = raw_data['time_stamp'].max()
+                    duration = end_time - start_time
+                    
+                    logger.info(f"\nTime Coverage:")
+                    logger.info(f"  Start: {start_time}")
+                    logger.info(f"  End:   {end_time}")
+                    logger.info(f"  Duration: {duration.days} days, {duration.seconds // 3600} hours")
+                    
+                    # Detection rate
+                    if duration.total_seconds() > 0:
+                        det_per_hour = total_detections / (duration.total_seconds() / 3600)
+                        logger.info(f"  Detection rate: {det_per_hour:.1f} detections/hour")
+                
+                logger.info(f"{'='*60}\n")
+            else:
+                logger.warning(f"No detections found for receiver {rec_id}")
+                
+        except KeyError:
+            logger.warning(f"Could not retrieve statistics - raw_data table not found in database")
+        except Exception as e:
+            logger.warning(f"Error calculating import statistics: {e}")
     
     def get_fish(self, rec_id, train = True, reclass_iter = None):
         logger.info(f"Getting fish for receiver {rec_id}")
@@ -1576,16 +1669,16 @@ class radio_project():
                     node_path = node._v_pathname
                     print(f"  Copying {node_path}...")
                     
-                    try:
-                        # Use recursive=True to copy entire subtree (Groups, Tables, Arrays, etc.)
-                        h5in.copy_node(
-                            where=node_path,
-                            newparent=h5out.root,
-                            recursive=True,
-                            filters=filters
-                        )
-                    except (tables.NodeError, tables.HDF5ExtError, OSError, ValueError) as e:
-                        raise RuntimeError(f"Failed to copy HDF5 node {node_path}: {e}") from e
+                    try:
+                        # Use recursive=True to copy entire subtree (Groups, Tables, Arrays, etc.)
+                        h5in.copy_node(
+                            where=node_path,
+                            newparent=h5out.root,
+                            recursive=True,
+                            filters=filters
+                        )
+                    except (tables.NodeError, tables.HDF5ExtError, OSError, ValueError) as e:
+                        raise RuntimeError(f"Failed to copy HDF5 node {node_path}: {e}") from e
         
         # Get new size
         new_size = os.path.getsize(output_path)
@@ -1603,26 +1696,29 @@ class radio_project():
     def make_recaptures_table(self, export=True, pit_study=False):
         '''Creates a recaptures key in the HDF5 file, iterating over receivers to manage memory.'''
         logger.info("Creating recaptures table")
+        logger.info(f"  PIT study mode: {pit_study}")
         logger.info(f"  Processing {len(self.receivers)} receiver(s)")
         # prepare a heartbeat log so long runs can be monitored (one-line per receiver)
         heartbeat_dir = os.path.join(self.project_dir, 'build')
-        try:
-            os.makedirs(heartbeat_dir, exist_ok=True)
-        except OSError as e:
-            raise RuntimeError(
-                f"Failed to create heartbeat directory '{heartbeat_dir}': {e}"
-            ) from e
+        try:
+            os.makedirs(heartbeat_dir, exist_ok=True)
+        except OSError as e:
+            raise RuntimeError(
+                f"Failed to create heartbeat directory '{heartbeat_dir}': {e}"
+            ) from e
         heartbeat_path = os.path.join(heartbeat_dir, 'recaptures_heartbeat.log')
         print(f"Starting recaptures: {len(self.receivers)} receivers. Heartbeat -> {heartbeat_path}")
-        try:
-            with open(heartbeat_path, 'a') as _hb:
-                _hb.write(f"START {datetime.datetime.now().isoformat()} receivers={len(self.receivers)}\n")
-        except OSError as e:
-            raise RuntimeError(
-                f"Failed to write heartbeat start to '{heartbeat_path}': {e}"
-            ) from e
-        
-        if pit_study==False:
+        try:
+            with open(heartbeat_path, 'a') as _hb:
+                _hb.write(f"START {datetime.datetime.now().isoformat()} receivers={len(self.receivers)}\n")
+        except OSError as e:
+            raise RuntimeError(
+                f"Failed to write heartbeat start to '{heartbeat_path}': {e}"
+            ) from e
+        
+        if not pit_study:
+            # RADIO STUDY PATH
+            logger.info("  Using RADIO study processing path")
             # Convert release dates to datetime if not already done
             self.tags['rel_date'] = pd.to_datetime(self.tags['rel_date'])
             tags_copy = self.tags.copy()
@@ -1787,15 +1883,17 @@ class radio_project():
                 logger.info(f"  ✓ Recaps for {rec} compiled and written to HDF5")
                 print(f"[recaptures] ✓ {rec} written to database", flush=True)
                 # append heartbeat line
-                try:
-                    with open(heartbeat_path, 'a') as _hb:
-                        _hb.write(f"{datetime.datetime.now().isoformat()} rec={rec} rows={len(rec_dat)}\n")
-                except OSError as e:
-                    raise RuntimeError(
-                        f"Failed to write heartbeat for receiver {rec} to '{heartbeat_path}': {e}"
-                    ) from e
+                try:
+                    with open(heartbeat_path, 'a') as _hb:
+                        _hb.write(f"{datetime.datetime.now().isoformat()} rec={rec} rows={len(rec_dat)}\n")
+                except OSError as e:
+                    raise RuntimeError(
+                        f"Failed to write heartbeat for receiver {rec} to '{heartbeat_path}': {e}"
+                    ) from e
                 
         else:
+            # PIT STUDY PATH
+            logger.info("  Using PIT study processing path")
             # Loop over each receiver in self.receivers
             for rec in tqdm(self.receivers.index, desc="Processing PIT receivers", unit="receiver"):
                 logger.info(f"  Processing {rec} (PIT study)...")
@@ -1917,13 +2015,13 @@ class radio_project():
         
                 logger.info(f"  ✓ PIT recaps for {rec} compiled and written to HDF5")
                 print(f"[recaptures] ✓ {rec} PIT data written to database", flush=True)
-                try:
-                    with open(heartbeat_path, 'a') as _hb:
-                        _hb.write(f"{datetime.datetime.now().isoformat()} pit_rec={rec} rows={len(pit_data)}\n")
-                except OSError as e:
-                    raise RuntimeError(
-                        f"Failed to write PIT heartbeat for receiver {rec} to '{heartbeat_path}': {e}"
-                    ) from e
+                try:
+                    with open(heartbeat_path, 'a') as _hb:
+                        _hb.write(f"{datetime.datetime.now().isoformat()} pit_rec={rec} rows={len(pit_data)}\n")
+                except OSError as e:
+                    raise RuntimeError(
+                        f"Failed to write PIT heartbeat for receiver {rec} to '{heartbeat_path}': {e}"
+                    ) from e
 
 
         if export:
@@ -1933,16 +2031,16 @@ class radio_project():
             rec_data.to_csv(os.path.join(self.output_dir,'recaptures.csv'), index=False)
             logger.info(f"  ✓ Export complete: {os.path.join(self.output_dir,'recaptures.csv')}")
             print(f"[recaptures] ✓ Export complete: {os.path.join(self.output_dir,'recaptures.csv')}", flush=True)
-            try:
-                with open(heartbeat_path, 'a') as _hb:
-                    _hb.write(
-                        f"DONE {datetime.datetime.now().isoformat()} export="
-                        f"{os.path.join(self.output_dir, 'recaptures.csv')}\n"
-                    )
-            except OSError as e:
-                raise RuntimeError(
-                    f"Failed to write heartbeat completion to '{heartbeat_path}': {e}"
-                ) from e
+            try:
+                with open(heartbeat_path, 'a') as _hb:
+                    _hb.write(
+                        f"DONE {datetime.datetime.now().isoformat()} export="
+                        f"{os.path.join(self.output_dir, 'recaptures.csv')}\n"
+                    )
+            except OSError as e:
+                raise RuntimeError(
+                    f"Failed to write heartbeat completion to '{heartbeat_path}': {e}"
+                ) from e
 
                 
     def undo_recaptures(self):

{pymast-1.0.1 → pymast-1.0.2}/pymast.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pymast
-Version: 1.0.1
+Version: 1.0.2
 Summary: Movement Analysis Software for Telemetry (MAST) - False positive removal and movement analysis for radio telemetry data
 Author: Theodore Castro-Santos
 Author-email: "Kevin P. Nebiolo" <kevin.nebiolo@kleinschmidtgroup.com>

{pymast-1.0.1 → pymast-1.0.2}/pyproject.toml

@@ -1,23 +1,23 @@
-[build-system]
-requires = ["setuptools>=61", "wheel"]
-build-backend = "setuptools.build_meta"
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
 
 [project]
 name = "pymast"
-version = "1.0.1"
+version = "1.0.2"
 description = "Movement Analysis Software for Telemetry (MAST) - False positive removal and movement analysis for radio telemetry data"
 readme = "README.md"
 authors = [
     {name = "Kevin P. Nebiolo", email = "kevin.nebiolo@kleinschmidtgroup.com"},
     {name = "Theodore Castro-Santos"}
 ]
-license = "MIT"
-classifiers = [
-    "Development Status :: 4 - Beta",
-    "Intended Audience :: Science/Research",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
+license = "MIT"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Topic :: Scientific/Engineering :: Bio-Informatics",
@@ -30,14 +30,14 @@ dependencies = [
     "matplotlib>=3.4.0",
     "statsmodels>=0.12.0",
     "networkx>=2.5",
-    "scipy>=1.7.1",
-    "scikit-learn>=0.24.0",
-    "h5py>=3.0.0",
-    "dask>=2021.3.0",
-    "dask-ml>=1.9.0",
-    "distributed>=2021.3.0",
-    "numba>=0.53.0",
-    "tables>=3.8.0",
+    "scipy>=1.7.1",
+    "scikit-learn>=0.24.0",
+    "h5py>=3.0.0",
+    "dask>=2021.3.0",
+    "dask-ml>=1.9.0",
+    "distributed>=2021.3.0",
+    "numba>=0.53.0",
+    "tables>=3.8.0",
     "intervaltree>=3.1.0",
 ]