simba-uw-tf-dev 4.6.2__py3-none-any.whl → 4.7.1__py3-none-any.whl

simba/assets/.recent_projects.txt

@@ -1,2 +1,3 @@
+E:/troubleshooting/mitra_emergence/project_folder/project_config.ini
 C:/troubleshooting/meberled/project_folder/project_config.ini
 C:/troubleshooting/mitra/project_folder/project_config.ini

simba/assets/lookups/tooptips.json

@@ -38,5 +38,10 @@
   "EGOCENTRIC_ANCHOR": "This body-part will be placed in the center of the video",
   "EGOCENTRIC_DIRECTION_ANCHOR": "This body-part will be placed at N degrees relative to the anchor",
   "EGOCENTRIC_DIRECTION": "The anchor body-part will always be placed at these degrees relative to the center anchor",
-  "CORE_COUNT": "Higher core counts speeds up processing but may require more RAM memory"
+  "CORE_COUNT": "Higher core counts speeds up processing but may require more RAM memory",
+  "KLEINBERG_SIGMA": "Higher values (e.g., 2-3) produce fewer but longer bursts; lower values (e.g., 1.1-1.5) detect more frequent, shorter bursts. Must be > 1.01",
+  "KLEINBERG_GAMMA": "Higher values (e.g., 0.5-1.0) reduce total burst count by making downward transitions costly; lower values (e.g., 0.1-0.3) allow more flexible state changes",
+  "KLEINBERG_HIERARCHY": "Hierarchy level to extract bursts from (0=lowest, higher=more selective).\n Level 0 captures all bursts; level 1-2 typically filters noise; level 3+ selects only the most prominent, sustained bursts.\nHigher levels yield fewer but more confident detections",
+  "KLEINBERG_HIERARCHY_SEARCH": "If True, searches for target hierarchy level within detected burst periods,\n falling back to lower levels if target not found. If False, extracts only bursts at the exact specified hierarchy level.\n Recommended when target hierarchy may be sparse.",
+  "KLEINBERG_SAVE_ORIGINALS": "If True, saves the original data in a new sub-directory of \nthe project_folder/csv/machine_results directory"
 }

simba/data_processors/agg_clf_counter_mp.py

@@ -20,7 +20,7 @@ from simba.utils.checks import (
     check_all_file_names_are_represented_in_video_log,
     check_file_exist_and_readable, check_if_dir_exists, check_int,
     check_valid_boolean, check_valid_dataframe, check_valid_lst)
-from simba.utils.data import detect_bouts
+from simba.utils.data import detect_bouts, terminate_cpu_pool
 from simba.utils.enums import TagNames
 from simba.utils.errors import NoChoosenMeasurementError
 from simba.utils.printing import SimbaTimer, log_event, stdout_success
@@ -210,8 +210,7 @@ class AggregateClfCalculatorMultiprocess(ConfigReader):
                 self.bouts_df_lst.append(batch_bouts_df_lst)
                 print(f"Data batch core {batch_id+1} / {self.core_cnt} complete...")
         self.bouts_df_lst = [df for sub in self.bouts_df_lst for df in sub]
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
 
     def save(self) -> None:
         """
@@ -242,56 +241,56 @@ class AggregateClfCalculatorMultiprocess(ConfigReader):
         self.timer.stop_timer()
         stdout_success(msg=f"Data aggregate log saved at {self.save_path}", elapsed_time=self.timer.elapsed_time_str, source=self.__class__.__name__)
 
-if __name__ == "__main__" and not hasattr(sys, 'ps1'):
-    parser = argparse.ArgumentParser(description='Compute aggregate descriptive statistics from classification data.')
-    parser.add_argument('--config_path', type=str, required=True, help='Path to SimBA project config file')
-    parser.add_argument('--classifiers', type=str, nargs='+', required=True, help='List of classifier names to analyze')
-    parser.add_argument('--data_dir', type=str, default=None, help='Directory containing machine results CSV files (default: project machine_results directory)')
-    parser.add_argument('--detailed_bout_data', action='store_true', help='Save detailed bout data for each bout')
-    parser.add_argument('--transpose', action='store_true', help='Create output with one video per row')
-    parser.add_argument('--no_first_occurrence', action='store_true', help='Disable first occurrence calculation')
-    parser.add_argument('--no_event_count', action='store_true', help='Disable event count calculation')
-    parser.add_argument('--no_total_event_duration', action='store_true', help='Disable total event duration calculation')
-    parser.add_argument('--no_mean_event_duration', action='store_true', help='Disable mean event duration calculation')
-    parser.add_argument('--no_median_event_duration', action='store_true', help='Disable median event duration calculation')
-    parser.add_argument('--no_mean_interval_duration', action='store_true', help='Disable mean interval duration calculation')
-    parser.add_argument('--no_median_interval_duration', action='store_true', help='Disable median interval duration calculation')
-    parser.add_argument('--frame_count', action='store_true', help='Include frame count in output')
-    parser.add_argument('--video_length', action='store_true', help='Include video length in output')
-
-    args = parser.parse_args()
-
-    clf_calculator = AggregateClfCalculatorMultiprocess(
-        config_path=args.config_path,
-        classifiers=args.classifiers,
-        data_dir=args.data_dir,
-        detailed_bout_data=args.detailed_bout_data,
-        transpose=args.transpose,
-        first_occurrence=not args.no_first_occurrence,
-        event_count=not args.no_event_count,
-        total_event_duration=not args.no_total_event_duration,
-        mean_event_duration=not args.no_mean_event_duration,
-        median_event_duration=not args.no_median_event_duration,
-        mean_interval_duration=not args.no_mean_interval_duration,
-        median_interval_duration=not args.no_median_interval_duration,
-        frame_count=args.frame_count,
-        video_length=args.video_length
-    )
-    clf_calculator.run()
-    clf_calculator.save()
-
-# if __name__ == "__main__":
-#     test = AggregateClfCalculatorMultiprocess(config_path=r"D:\troubleshooting\maplight_ri\project_folder\project_config.ini",
-#                                   classifiers=['attack'],
-#                                   transpose=True,
-#                                   mean_event_duration = True,
-#                                   median_event_duration = True,
-#                                   mean_interval_duration = True,
-#                                   median_interval_duration = True,
-#                                   detailed_bout_data=True,
-#                                   core_cnt=12)
-#     test.run()
-#     test.save()
+# if __name__ == "__main__" and not hasattr(sys, 'ps1'):
+#     parser = argparse.ArgumentParser(description='Compute aggregate descriptive statistics from classification data.')
+#     parser.add_argument('--config_path', type=str, required=True, help='Path to SimBA project config file')
+#     parser.add_argument('--classifiers', type=str, nargs='+', required=True, help='List of classifier names to analyze')
+#     parser.add_argument('--data_dir', type=str, default=None, help='Directory containing machine results CSV files (default: project machine_results directory)')
+#     parser.add_argument('--detailed_bout_data', action='store_true', help='Save detailed bout data for each bout')
+#     parser.add_argument('--transpose', action='store_true', help='Create output with one video per row')
+#     parser.add_argument('--no_first_occurrence', action='store_true', help='Disable first occurrence calculation')
+#     parser.add_argument('--no_event_count', action='store_true', help='Disable event count calculation')
+#     parser.add_argument('--no_total_event_duration', action='store_true', help='Disable total event duration calculation')
+#     parser.add_argument('--no_mean_event_duration', action='store_true', help='Disable mean event duration calculation')
+#     parser.add_argument('--no_median_event_duration', action='store_true', help='Disable median event duration calculation')
+#     parser.add_argument('--no_mean_interval_duration', action='store_true', help='Disable mean interval duration calculation')
+#     parser.add_argument('--no_median_interval_duration', action='store_true', help='Disable median interval duration calculation')
+#     parser.add_argument('--frame_count', action='store_true', help='Include frame count in output')
+#     parser.add_argument('--video_length', action='store_true', help='Include video length in output')
+#
+#     args = parser.parse_args()
+#
+#     clf_calculator = AggregateClfCalculatorMultiprocess(
+#         config_path=args.config_path,
+#         classifiers=args.classifiers,
+#         data_dir=args.data_dir,
+#         detailed_bout_data=args.detailed_bout_data,
+#         transpose=args.transpose,
+#         first_occurrence=not args.no_first_occurrence,
+#         event_count=not args.no_event_count,
+#         total_event_duration=not args.no_total_event_duration,
+#         mean_event_duration=not args.no_mean_event_duration,
+#         median_event_duration=not args.no_median_event_duration,
+#         mean_interval_duration=not args.no_mean_interval_duration,
+#         median_interval_duration=not args.no_median_interval_duration,
+#         frame_count=args.frame_count,
+#         video_length=args.video_length
+#     )
+#     clf_calculator.run()
+#     clf_calculator.save()
+
+if __name__ == "__main__":
+    test = AggregateClfCalculatorMultiprocess(config_path=r"D:\troubleshooting\maplight_ri\project_folder\project_config.ini",
+                                  classifiers=['attack'],
+                                  transpose=True,
+                                  mean_event_duration = True,
+                                  median_event_duration = True,
+                                  mean_interval_duration = True,
+                                  median_interval_duration = True,
+                                  detailed_bout_data=True,
+                                  core_cnt=12)
+    test.run()
+    test.save()
 
 
 # test = AggregateClfCalculator(config_path=r"C:\troubleshooting\mitra\project_folder\project_config.ini",

simba/data_processors/blob_location_computer.py

@@ -51,7 +51,7 @@ class BlobLocationComputer(object):
     :param Optional[bool] multiprocessing: If True, video background subtraction will be done using  multiprocessing. Default is False.
 
     :example:
-    >>> x = BlobLocationComputer(data_path=r"C:\troubleshooting\RAT_NOR\project_folder\videos\2022-06-20_NOB_DOT_4_downsampled_bg_subtracted.mp4", multiprocessing=True, gpu=True, batch_size=2000, save_dir=r"C:\blob_positions")
+    >>> x = BlobLocationComputer(data_path=r"C:/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_DOT_4_downsampled_bg_subtracted.mp4", multiprocessing=True, gpu=True, batch_size=2000, save_dir=r"C:/blob_positions")
     >>> x.run()
     """
     def __init__(self,

simba/data_processors/circling_detector.py

@@ -11,12 +11,13 @@ from simba.mixins.feature_extraction_mixin import FeatureExtractionMixin
 from simba.mixins.timeseries_features_mixin import TimeseriesFeatureMixin
 from simba.utils.checks import (
     check_all_file_names_are_represented_in_video_log, check_if_dir_exists,
-    check_int, check_str, check_valid_dataframe)
+    check_str, check_valid_dataframe)
 from simba.utils.data import detect_bouts, plug_holes_shortest_bout
 from simba.utils.enums import Formats
 from simba.utils.printing import stdout_success
 from simba.utils.read_write import (find_files_of_filetypes_in_directory,
-                                    get_fn_ext, read_df, read_video_info)
+                                    get_current_time, get_fn_ext, read_df,
+                                    read_video_info)
 
 CIRCLING = 'CIRCLING'
 
@@ -58,30 +59,34 @@ class CirclingDetector(ConfigReader):
     """
 
     def __init__(self,
-                 data_dir: Union[str, os.PathLike],
                  config_path: Union[str, os.PathLike],
                  nose_name: Optional[str] = 'nose',
+                 data_dir: Optional[Union[str, os.PathLike]] = None,
                  left_ear_name: Optional[str] = 'left_ear',
                  right_ear_name: Optional[str] = 'right_ear',
                  tail_base_name: Optional[str] = 'tail_base',
                  center_name: Optional[str] = 'center',
-                 time_threshold: Optional[int] = 10,
-                 circular_range_threshold: Optional[int] = 320,
+                 time_threshold: Optional[int] = 7,
+                 circular_range_threshold: Optional[int] = 350,
+                 shortest_bout: int = 100,
                  movement_threshold: Optional[int] = 60,
                  save_dir: Optional[Union[str, os.PathLike]] = None):
 
-        check_if_dir_exists(in_dir=data_dir)
         for bp_name in [nose_name, left_ear_name, right_ear_name, tail_base_name]: check_str(name='body part name', value=bp_name, allow_blank=False)
-        self.data_paths = find_files_of_filetypes_in_directory(directory=data_dir, extensions=['.csv'])
         ConfigReader.__init__(self, config_path=config_path, read_video_info=True, create_logger=False)
+        if data_dir is not None:
+            check_if_dir_exists(in_dir=data_dir)
+        else:
+            data_dir = self.outlier_corrected_dir
+        self.data_paths = find_files_of_filetypes_in_directory(directory=data_dir, extensions=['.csv'])
         self.nose_heads = [f'{nose_name}_x'.lower(), f'{nose_name}_y'.lower()]
         self.left_ear_heads = [f'{left_ear_name}_x'.lower(), f'{left_ear_name}_y'.lower()]
         self.right_ear_heads = [f'{right_ear_name}_x'.lower(), f'{right_ear_name}_y'.lower()]
         self.center_heads = [f'{center_name}_x'.lower(), f'{center_name}_y'.lower()]
         self.required_field = self.nose_heads + self.left_ear_heads + self.right_ear_heads
-        self.save_dir = save_dir
+        self.save_dir, self.shortest_bout = save_dir, shortest_bout
         if self.save_dir is None:
-            self.save_dir = os.path.join(self.logs_path, f'circling_data_{self.datetime}')
+            self.save_dir = os.path.join(self.logs_path, f'circling_data_{time_threshold}s_{circular_range_threshold}d_{movement_threshold}mm_{self.datetime}')
             os.makedirs(self.save_dir)
         else:
             check_if_dir_exists(in_dir=self.save_dir)
@@ -93,7 +98,7 @@ class CirclingDetector(ConfigReader):
         check_all_file_names_are_represented_in_video_log(video_info_df=self.video_info_df, data_paths=self.data_paths)
         for file_cnt, file_path in enumerate(self.data_paths):
             video_name = get_fn_ext(filepath=file_path)[1]
-            print(f'Analyzing {video_name} ({file_cnt+1}/{len(self.data_paths)})...')
+            print(f'[{get_current_time()}] Analyzing circling {video_name}... (video {file_cnt+1}/{len(self.data_paths)})')
             save_file_path = os.path.join(self.save_dir, f'{video_name}.csv')
             df = read_df(file_path=file_path, file_type='csv').reset_index(drop=True)
             _, px_per_mm, fps = read_video_info(video_info_df=self.video_info_df, video_name=video_name)
@@ -115,11 +120,24 @@ class CirclingDetector(ConfigReader):
             circling_idx = np.argwhere(sliding_circular_range >= self.circular_range_threshold).astype(np.int32).flatten()
             movement_idx = np.argwhere(movement_sum >= self.movement_threshold).astype(np.int32).flatten()
             circling_idx = [x for x in movement_idx if x in circling_idx]
+            df[f'Probability_{CIRCLING}'] = 0
             df[CIRCLING] = 0
             df.loc[circling_idx, CIRCLING] = 1
+            df.loc[circling_idx, f'Probability_{CIRCLING}'] = 1
+            df = plug_holes_shortest_bout(data_df=df, clf_name=CIRCLING, fps=fps, shortest_bout=self.shortest_bout)
             bouts = detect_bouts(data_df=df, target_lst=[CIRCLING], fps=fps)
-            df = plug_holes_shortest_bout(data_df=df, clf_name=CIRCLING, fps=fps, shortest_bout=100)
+            if len(bouts) > 0:
+                df[CIRCLING] = 0
+                circling_idx = list(bouts.apply(lambda x: list(range(int(x["Start_frame"]), int(x["End_frame"]) + 1)), 1))
+                circling_idx = [x for xs in circling_idx for x in xs]
+                df.loc[circling_idx, CIRCLING] = 1
+                df.loc[circling_idx, f'Probability_{CIRCLING}'] = 1
+            else:
+                df[CIRCLING] = 0
+                circling_idx = []
+
             df.to_csv(save_file_path)
+            #print(video_name, len(circling_idx), round(len(circling_idx) / fps, 4), df[CIRCLING].sum())
             agg_results.loc[len(agg_results)] = [video_name, len(circling_idx), round(len(circling_idx) / fps, 4), len(bouts), round((len(circling_idx) / len(df)) * 100, 4), len(df), round(len(df)/fps, 2) ]
 
         agg_results.to_csv(agg_results_path)
@@ -127,7 +145,6 @@ class CirclingDetector(ConfigReader):
 
 #
 #
-# detector = CirclingDetector(data_dir=r'C:\troubleshooting\mitra\project_folder\csv\outlier_corrected_movement_location',
-#                             config_path=r"C:\troubleshooting\mitra\project_folder\project_config.ini")
+# detector = CirclingDetector(config_path=r"E:\troubleshooting\mitra_emergence\project_folder\project_config.ini")
 # detector.run()
 

simba/data_processors/cuda/geometry.py

@@ -8,6 +8,7 @@ from numba import cuda, njit
 
 from simba.utils.checks import check_float, check_valid_array
 from simba.utils.enums import Formats
+from simba.utils.printing import SimbaTimer
 
 try:
     import cupy as cp
@@ -401,20 +402,21 @@ def find_midpoints(x: np.ndarray,
 
 
 @cuda.jit()
-def _directionality_to_static_targets_kernel(left_ear, right_ear, nose, target, results):
+def _directionality_to_static_targets_kernel(left_ear, right_ear, nose, target_x, target_y, results):
     i = cuda.grid(1)
-    if i > left_ear.shape[0]:
+    if i >= left_ear.shape[0]:
         return
     else:
-        LE, RE = left_ear[i], right_ear[i]
-        N, Tx, Ty = nose[i], target[0], target[1]
-
-        Px = abs(LE[0] - Tx)
-        Py = abs(LE[1] - Ty)
-        Qx = abs(RE[0] - Tx)
-        Qy = abs(RE[1] - Ty)
-        Nx = abs(N[0] - Tx)
-        Ny = abs(N[1] - Ty)
+        LE = left_ear[i]
+        RE = right_ear[i]
+        N = nose[i]
+
+        Px = abs(LE[0] - target_x)
+        Py = abs(LE[1] - target_y)
+        Qx = abs(RE[0] - target_x)
+        Qy = abs(RE[1] - target_y)
+        Nx = abs(N[0] - target_x)
+        Ny = abs(N[1] - target_y)
         Ph = math.sqrt(Px * Px + Py * Py)
         Qh = math.sqrt(Qx * Qx + Qy * Qy)
         Nh = math.sqrt(Nx * Nx + Ny * Ny)
@@ -438,7 +440,8 @@ def _directionality_to_static_targets_kernel(left_ear, right_ear, nose, target,
 def directionality_to_static_targets(left_ear: np.ndarray,
                                      right_ear: np.ndarray,
                                      nose: np.ndarray,
-                                     target: np.ndarray) -> np.ndarray:
+                                     target: np.ndarray,
+                                     verbose: bool = False) -> np.ndarray:
     """
     GPU helper to calculate if an animal is directing towards a static location (e.g., ROI centroid), given the target location and the left ear, right ear, and nose coordinates of the observer.
 
@@ -487,32 +490,38 @@ def directionality_to_static_targets(left_ear: np.ndarray,
     >>> directionality_to_static_targets(left_ear=left_ear, right_ear=right_ear, nose=nose, target=target)
 
     """
-
+    timer = SimbaTimer(start=True)
     left_ear = np.ascontiguousarray(left_ear).astype(np.int32)
     right_ear = np.ascontiguousarray(right_ear).astype(np.int32)
     nose = np.ascontiguousarray(nose).astype(np.int32)
     target = np.ascontiguousarray(target).astype(np.int32)
 
+    target_x = int(target[0])
+    target_y = int(target[1])
+
     left_ear_dev = cuda.to_device(left_ear)
     right_ear_dev = cuda.to_device(right_ear)
     nose_dev = cuda.to_device(nose)
-    target_dev = cuda.to_device(target)
     results = cuda.device_array((left_ear.shape[0], 4), dtype=np.int32)
     bpg = (left_ear.shape[0] + (THREADS_PER_BLOCK - 1)) // THREADS_PER_BLOCK
-    _directionality_to_static_targets_kernel[bpg, THREADS_PER_BLOCK](left_ear_dev, right_ear_dev, nose_dev, target_dev, results)
+    _directionality_to_static_targets_kernel[bpg, THREADS_PER_BLOCK](left_ear_dev, right_ear_dev, nose_dev, target_x, target_y, results)
 
     results = results.copy_to_host()
+    timer.stop_timer()
+    if verbose: print(f'Directionality to static target computed in for {left_ear.shape[0]} observations (elapsed time: {timer.elapsed_time_str}s)')
     return results
 
 
 @cuda.jit()
 def _directionality_to_nonstatic_targets_kernel(left_ear, right_ear, nose, target, results):
     i = cuda.grid(1)
-    if i > left_ear.shape[0]:
+    if i >= left_ear.shape[0]:
         return
     else:
-        LE, RE = left_ear[i], right_ear[i]
-        N, T = nose[i], target[i]
+        LE = left_ear[i]
+        RE = right_ear[i]
+        N = nose[i]
+        T = target[i]
 
         Px = abs(LE[0] - T[0])
         Py = abs(LE[1] - T[1])
@@ -543,11 +552,19 @@ def _directionality_to_nonstatic_targets_kernel(left_ear, right_ear, nose, targe
 def directionality_to_nonstatic_target(left_ear: np.ndarray,
                                        right_ear: np.ndarray,
                                        nose: np.ndarray,
-                                       target: np.ndarray) -> np.ndarray:
+                                       target: np.ndarray,
+                                       verbose: bool = False) -> np.ndarray:
 
     """
     GPU method to calculate if an animal is directing towards a moving point location given the target location and the left ear, right ear, and nose coordinates of the observer.
 
+    .. csv-table::
+       :header: EXPECTED RUNTIMES
+       :file: ../../../docs/tables/directionality_to_nonstatic_target_cuda.csv
+       :widths: 30, 30, 20, 10, 10
+       :align: center
+       :class: simba-table
+       :header-rows: 1
 
     .. image:: _static/img/directing_moving_targets.png
        :width: 400
@@ -573,27 +590,28 @@ def directionality_to_nonstatic_target(left_ear: np.ndarray,
     >>> right_ear = np.random.randint(0, 500, (100, 2))
     >>> nose = np.random.randint(0, 500, (100, 2))
     >>> target = np.random.randint(0, 500, (100, 2))
-    >>> directionality_to_static_targets(left_ear=left_ear, right_ear=right_ear, nose=nose, target=target)
+    >>> directionality_to_nonstatic_target(left_ear=left_ear, right_ear=right_ear, nose=nose, target=target)
     """
 
-    left_ear = np.ascontiguousarray(left_ear).astype(np.int32)
-    right_ear = np.ascontiguousarray(right_ear).astype(np.int32)
-    nose = np.ascontiguousarray(nose).astype(np.int32)
-    target = np.ascontiguousarray(target).astype(np.int32)
+    timer = SimbaTimer(start=True)
+    left_ear = np.ascontiguousarray(left_ear).astype(np.int64)
+    right_ear = np.ascontiguousarray(right_ear).astype(np.int64)
+    nose = np.ascontiguousarray(nose).astype(np.int64)
+    target = np.ascontiguousarray(target).astype(np.int64)
 
     left_ear_dev = cuda.to_device(left_ear)
     right_ear_dev = cuda.to_device(right_ear)
     nose_dev = cuda.to_device(nose)
     target_dev = cuda.to_device(target)
-    results = cuda.device_array((left_ear.shape[0], 4), dtype=np.int32)
+    results = cuda.device_array((left_ear.shape[0], 4), dtype=np.int64)
     bpg = (left_ear.shape[0] + (THREADS_PER_BLOCK - 1)) // THREADS_PER_BLOCK
     _directionality_to_nonstatic_targets_kernel[bpg, THREADS_PER_BLOCK](left_ear_dev, right_ear_dev, nose_dev, target_dev, results)
 
     results = results.copy_to_host()
+    timer.stop_timer()
+    if verbose: print(f'Directionality to moving target computed in for {left_ear.shape[0]} observations (elapsed time: {timer.elapsed_time_str}s)')
     return results
 
 
 
 
-
-