simba-uw-tf-dev 4.6.2__py3-none-any.whl → 4.6.4__py3-none-any.whl

simba/assets/lookups/tooptips.json

@@ -38,5 +38,10 @@
   "EGOCENTRIC_ANCHOR": "This body-part will be placed in the center of the video",
   "EGOCENTRIC_DIRECTION_ANCHOR": "This body-part will be placed at N degrees relative to the anchor",
   "EGOCENTRIC_DIRECTION": "The anchor body-part will always be placed at these degrees relative to the center anchor",
-  "CORE_COUNT": "Higher core counts speeds up processing but may require more RAM memory"
+  "CORE_COUNT": "Higher core counts speeds up processing but may require more RAM memory",
+  "KLEINBERG_SIGMA": "Higher values (e.g., 2-3) produce fewer but longer bursts; lower values (e.g., 1.1-1.5) detect more frequent, shorter bursts. Must be > 1.01",
+  "KLEINBERG_GAMMA": "Higher values (e.g., 0.5-1.0) reduce total burst count by making downward transitions costly; lower values (e.g., 0.1-0.3) allow more flexible state changes",
+  "KLEINBERG_HIERARCHY": "Hierarchy level to extract bursts from (0=lowest, higher=more selective).\n Level 0 captures all bursts; level 1-2 typically filters noise; level 3+ selects only the most prominent, sustained bursts.\nHigher levels yield fewer but more confident detections",
+  "KLEINBERG_HIERARCHY_SEARCH": "If True, searches for target hierarchy level within detected burst periods,\n falling back to lower levels if target not found. If False, extracts only bursts at the exact specified hierarchy level.\n Recommended when target hierarchy may be sparse.",
+  "KLEINBERG_SAVE_ORIGINALS": "If True, saves the original data in a new sub-directory of \nthe project_folder/csv/machine_results directory"
 }

simba/data_processors/agg_clf_counter_mp.py

@@ -20,7 +20,7 @@ from simba.utils.checks import (
     check_all_file_names_are_represented_in_video_log,
     check_file_exist_and_readable, check_if_dir_exists, check_int,
     check_valid_boolean, check_valid_dataframe, check_valid_lst)
-from simba.utils.data import detect_bouts
+from simba.utils.data import detect_bouts, terminate_cpu_pool
 from simba.utils.enums import TagNames
 from simba.utils.errors import NoChoosenMeasurementError
 from simba.utils.printing import SimbaTimer, log_event, stdout_success
@@ -210,8 +210,7 @@ class AggregateClfCalculatorMultiprocess(ConfigReader):
                 self.bouts_df_lst.append(batch_bouts_df_lst)
                 print(f"Data batch core {batch_id+1} / {self.core_cnt} complete...")
         self.bouts_df_lst = [df for sub in self.bouts_df_lst for df in sub]
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
 
     def save(self) -> None:
         """
@@ -242,56 +241,56 @@ class AggregateClfCalculatorMultiprocess(ConfigReader):
         self.timer.stop_timer()
         stdout_success(msg=f"Data aggregate log saved at {self.save_path}", elapsed_time=self.timer.elapsed_time_str, source=self.__class__.__name__)
 
-if __name__ == "__main__" and not hasattr(sys, 'ps1'):
-    parser = argparse.ArgumentParser(description='Compute aggregate descriptive statistics from classification data.')
-    parser.add_argument('--config_path', type=str, required=True, help='Path to SimBA project config file')
-    parser.add_argument('--classifiers', type=str, nargs='+', required=True, help='List of classifier names to analyze')
-    parser.add_argument('--data_dir', type=str, default=None, help='Directory containing machine results CSV files (default: project machine_results directory)')
-    parser.add_argument('--detailed_bout_data', action='store_true', help='Save detailed bout data for each bout')
-    parser.add_argument('--transpose', action='store_true', help='Create output with one video per row')
-    parser.add_argument('--no_first_occurrence', action='store_true', help='Disable first occurrence calculation')
-    parser.add_argument('--no_event_count', action='store_true', help='Disable event count calculation')
-    parser.add_argument('--no_total_event_duration', action='store_true', help='Disable total event duration calculation')
-    parser.add_argument('--no_mean_event_duration', action='store_true', help='Disable mean event duration calculation')
-    parser.add_argument('--no_median_event_duration', action='store_true', help='Disable median event duration calculation')
-    parser.add_argument('--no_mean_interval_duration', action='store_true', help='Disable mean interval duration calculation')
-    parser.add_argument('--no_median_interval_duration', action='store_true', help='Disable median interval duration calculation')
-    parser.add_argument('--frame_count', action='store_true', help='Include frame count in output')
-    parser.add_argument('--video_length', action='store_true', help='Include video length in output')
-
-    args = parser.parse_args()
-
-    clf_calculator = AggregateClfCalculatorMultiprocess(
-        config_path=args.config_path,
-        classifiers=args.classifiers,
-        data_dir=args.data_dir,
-        detailed_bout_data=args.detailed_bout_data,
-        transpose=args.transpose,
-        first_occurrence=not args.no_first_occurrence,
-        event_count=not args.no_event_count,
-        total_event_duration=not args.no_total_event_duration,
-        mean_event_duration=not args.no_mean_event_duration,
-        median_event_duration=not args.no_median_event_duration,
-        mean_interval_duration=not args.no_mean_interval_duration,
-        median_interval_duration=not args.no_median_interval_duration,
-        frame_count=args.frame_count,
-        video_length=args.video_length
-    )
-    clf_calculator.run()
-    clf_calculator.save()
-
-# if __name__ == "__main__":
-#     test = AggregateClfCalculatorMultiprocess(config_path=r"D:\troubleshooting\maplight_ri\project_folder\project_config.ini",
-#                                   classifiers=['attack'],
-#                                   transpose=True,
-#                                   mean_event_duration = True,
-#                                   median_event_duration = True,
-#                                   mean_interval_duration = True,
-#                                   median_interval_duration = True,
-#                                   detailed_bout_data=True,
-#                                   core_cnt=12)
-#     test.run()
-#     test.save()
+# if __name__ == "__main__" and not hasattr(sys, 'ps1'):
+#     parser = argparse.ArgumentParser(description='Compute aggregate descriptive statistics from classification data.')
+#     parser.add_argument('--config_path', type=str, required=True, help='Path to SimBA project config file')
+#     parser.add_argument('--classifiers', type=str, nargs='+', required=True, help='List of classifier names to analyze')
+#     parser.add_argument('--data_dir', type=str, default=None, help='Directory containing machine results CSV files (default: project machine_results directory)')
+#     parser.add_argument('--detailed_bout_data', action='store_true', help='Save detailed bout data for each bout')
+#     parser.add_argument('--transpose', action='store_true', help='Create output with one video per row')
+#     parser.add_argument('--no_first_occurrence', action='store_true', help='Disable first occurrence calculation')
+#     parser.add_argument('--no_event_count', action='store_true', help='Disable event count calculation')
+#     parser.add_argument('--no_total_event_duration', action='store_true', help='Disable total event duration calculation')
+#     parser.add_argument('--no_mean_event_duration', action='store_true', help='Disable mean event duration calculation')
+#     parser.add_argument('--no_median_event_duration', action='store_true', help='Disable median event duration calculation')
+#     parser.add_argument('--no_mean_interval_duration', action='store_true', help='Disable mean interval duration calculation')
+#     parser.add_argument('--no_median_interval_duration', action='store_true', help='Disable median interval duration calculation')
+#     parser.add_argument('--frame_count', action='store_true', help='Include frame count in output')
+#     parser.add_argument('--video_length', action='store_true', help='Include video length in output')
+#
+#     args = parser.parse_args()
+#
+#     clf_calculator = AggregateClfCalculatorMultiprocess(
+#         config_path=args.config_path,
+#         classifiers=args.classifiers,
+#         data_dir=args.data_dir,
+#         detailed_bout_data=args.detailed_bout_data,
+#         transpose=args.transpose,
+#         first_occurrence=not args.no_first_occurrence,
+#         event_count=not args.no_event_count,
+#         total_event_duration=not args.no_total_event_duration,
+#         mean_event_duration=not args.no_mean_event_duration,
+#         median_event_duration=not args.no_median_event_duration,
+#         mean_interval_duration=not args.no_mean_interval_duration,
+#         median_interval_duration=not args.no_median_interval_duration,
+#         frame_count=args.frame_count,
+#         video_length=args.video_length
+#     )
+#     clf_calculator.run()
+#     clf_calculator.save()
+
+if __name__ == "__main__":
+    test = AggregateClfCalculatorMultiprocess(config_path=r"D:\troubleshooting\maplight_ri\project_folder\project_config.ini",
+                                  classifiers=['attack'],
+                                  transpose=True,
+                                  mean_event_duration = True,
+                                  median_event_duration = True,
+                                  mean_interval_duration = True,
+                                  median_interval_duration = True,
+                                  detailed_bout_data=True,
+                                  core_cnt=12)
+    test.run()
+    test.save()
 
 
 # test = AggregateClfCalculator(config_path=r"C:\troubleshooting\mitra\project_folder\project_config.ini",

simba/data_processors/cuda/image.py

@@ -213,7 +213,6 @@ def _average_3d_stack_cuda(image_stack: np.ndarray) -> np.ndarray:
     return results
 
 
-
 def create_average_frm_cuda(video_path: Union[str, os.PathLike],
                             start_frm: Optional[int] = None,
                             end_frm: Optional[int] = None,
@@ -1512,6 +1511,9 @@ def pose_plotter(data: Union[str, os.PathLike, np.ndarray],
         stdout_success(msg=f'Pose-estimation video saved at {save_path}.', elapsed_time=total_timer.elapsed_time_str)
 
 
+
+#x = create_average_frm_cuda(video_path=r"D:\troubleshooting\mitra\project_folder\videos\average_cpu_test\20min.mp4", verbose=True, batch_size=500, async_frame_read=False)
+
 # VIDEO_PATH = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/videos/Trial_1_C24_D1_1.mp4"
 # #
 #

simba/data_processors/cue_light_analyzer.py

@@ -1,13 +1,9 @@
 __author__ = "Simon Nilsson; sronilsson@gmail.com"
 
 import functools
-import glob
-import itertools
 import multiprocessing
 import os
-import platform
-import time
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Union
 
 import cv2
 import numpy as np
@@ -17,9 +13,9 @@ from simba.mixins.config_reader import ConfigReader
 from simba.mixins.statistics_mixin import Statistics
 from simba.utils.checks import (
     check_all_file_names_are_represented_in_video_log, check_if_dir_exists,
-    check_if_valid_img, check_int, check_nvidea_gpu_available,
-    check_valid_boolean, check_valid_lst)
-from simba.utils.data import detect_bouts, slice_roi_dict_from_attribute
+    check_if_valid_img, check_int, check_valid_boolean, check_valid_lst)
+from simba.utils.data import (detect_bouts, slice_roi_dict_from_attribute,
+                              terminate_cpu_pool)
 from simba.utils.enums import Defaults, Keys
 from simba.utils.errors import NoROIDataError
 from simba.utils.printing import SimbaTimer, stdout_success
@@ -220,7 +216,7 @@ class CueLightAnalyzer(ConfigReader):
                         else: self.intensities[key] = subdict
                         if self.verbose:
                             print(f'Batch {int(np.ceil(cnt + 1 / self.core_cnt))} complete...')
-            pool.terminate(); pool.join()
+            terminate_cpu_pool(pool=pool, force=False)
             kmeans = self._get_kmeans(intensities=self.intensities)
             self.data_df = self._append_light_data(data_df=self.data_df, kmeans_data=kmeans)
             self.data_df = self._remove_outlier_events(data_df=self.data_df)

simba/data_processors/kleinberg_calculator.py

@@ -13,10 +13,10 @@ from simba.data_processors.pybursts_calculator import kleinberg_burst_detection
 from simba.mixins.config_reader import ConfigReader
 from simba.utils.checks import (check_float, check_if_dir_exists,
                                 check_if_filepath_list_is_empty, check_int,
-                                check_that_column_exist, check_valid_lst)
+                                check_that_column_exist, check_valid_lst, check_valid_boolean)
 from simba.utils.enums import Paths, TagNames
 from simba.utils.printing import SimbaTimer, log_event, stdout_success
-from simba.utils.read_write import get_fn_ext, read_df, write_df
+from simba.utils.read_write import get_fn_ext, read_df, write_df, get_current_time, find_files_of_filetypes_in_directory, remove_a_folder, copy_files_to_directory
 from simba.utils.warnings import KleinbergWarning
 
 
@@ -38,12 +38,13 @@ class KleinbergCalculator(ConfigReader):
 
     :param str config_path: path to SimBA project config file in Configparser format
     :param List[str] classifier_names: Classifier names to apply Kleinberg smoothing to.
-    :param float sigma: Burst detection sigma value. Higher sigma values and fewer, longer, behavioural bursts will be recognised. Default: 2.
-    :param float gamma: Burst detection gamma value. Higher gamma values and fewer behavioural bursts will be recognised. Default: 0.3.
-    :param int hierarchy: Burst detection hierarchy level. Higher hierarchy values and fewer behavioural bursts will to be recognised. Default: 1.
-    :param bool hierarchical_search: See `Tutorial <https://github.com/sgoldenlab/simba/blob/master/docs/kleinberg_filter.md#hierarchical-search-example>`_ Default: False.
+    :param float sigma: State transition cost for moving to higher burst levels. Higher values (e.g., 2-3) produce fewer but longer bursts; lower values (e.g., 1.1-1.5) detect more frequent, shorter bursts. Must be > 1.01. Default: 2.
+    :param float gamma: State transition cost for moving to lower burst levels. Higher values (e.g., 0.5-1.0) reduce total burst count by making downward transitions costly; lower values (e.g., 0.1-0.3) allow more flexible state changes. Must be >= 0. Default: 0.3.
+    :param int hierarchy: Hierarchy level to extract bursts from (0=lowest, higher=more selective). Level 0 captures all bursts; level 1-2 typically filters noise; level 3+ selects only the most prominent, sustained bursts. Higher levels yield fewer but more confident detections. Must be >= 0. Default: 1.
+    :param bool hierarchical_search: If True, searches for target hierarchy level within detected burst periods, falling back to lower levels if target not found. If False, extracts only bursts at the exact specified hierarchy level. Recommended when target hierarchy may be sparse. Default: False.
     :param Optional[Union[str, os.PathLike]] input_dir: The directory with files to perform kleinberg smoothing on. If None, defaults to `project_folder/csv/machine_results`
     :param Optional[Union[str, os.PathLike]] output_dir: Location to save smoothened data in. If None, defaults to `project_folder/csv/machine_results`
+    :param Optional[bool] save_originals: If True, saves the original data in sub-directory of the ouput directory.`
 
     :example I:
     >>> kleinberg_calculator = KleinbergCalculator(config_path='MySimBAConfigPath', classifier_names=['Attack'], sigma=2, gamma=0.3, hierarchy=2, hierarchical_search=False)
@@ -68,10 +69,12 @@ class KleinbergCalculator(ConfigReader):
 
     def __init__(self,
                  config_path: Union[str, os.PathLike],
-                 classifier_names: List[str],
-                 sigma: Optional[int] = 2,
-                 gamma: Optional[float] = 0.3,
+                 classifier_names: Optional[List[str]] = None,
+                 sigma: float = 2,
+                 gamma: float = 0.3,
                  hierarchy: Optional[int] = 1,
+                 verbose: bool = True,
+                 save_originals: bool = True,
                  hierarchical_search: Optional[bool] = False,
                  input_dir: Optional[Union[str, os.PathLike]] = None,
                  output_dir: Optional[Union[str, os.PathLike]] = None):
@@ -81,25 +84,31 @@ class KleinbergCalculator(ConfigReader):
         check_float(value=sigma, name=f'{self.__class__.__name__} sigma', min_value=1.01)
         check_float(value=gamma, name=f'{self.__class__.__name__} gamma', min_value=0)
         check_int(value=hierarchy, name=f'{self.__class__.__name__} hierarchy', min_value=0)
-        check_valid_lst(data=classifier_names, source=f'{self.__class__.__name__} classifier_names', valid_dtypes=(str,), min_len=1)
+        if isinstance(classifier_names, list):
+            check_valid_lst(data=classifier_names, source=f'{self.__class__.__name__} classifier_names', valid_dtypes=(str,), min_len=1)
+        else:
+            classifier_names = deepcopy(self.clf_names)
+        check_valid_boolean(value=verbose, source=f'{self.__class__.__name__} verbose', raise_error=True)
+        check_valid_boolean(value=save_originals, source=f'{self.__class__.__name__} save_originals', raise_error=True)
         self.hierarchical_search, sigma, gamma, hierarchy, self.output_dir = (hierarchical_search, float(sigma), float(gamma), int(hierarchy), output_dir)
-        self.sigma, self.gamma, self.hierarchy, self.clfs = ( float(sigma), float(gamma), float(hierarchy), classifier_names)
+        self.sigma, self.gamma, self.hierarchy, self.clfs = ( float(sigma), float(gamma), int(hierarchy), classifier_names)
+        self.verbose, self.save_originals = verbose, save_originals
         if input_dir is None:
-            self.data_paths, self.output_dir = self.machine_results_paths, self.machine_results_dir
-            check_if_filepath_list_is_empty(filepaths=self.machine_results_paths, error_msg=f"SIMBA ERROR: No data files found in {self.machine_results_dir}. Cannot perform Kleinberg smoothing")
-            original_data_files_folder = os.path.join(self.project_path, Paths.MACHINE_RESULTS_DIR.value, f"Pre_Kleinberg_{self.datetime}")
-            if not os.path.exists(original_data_files_folder):
-                os.makedirs(original_data_files_folder)
-            for file_path in self.machine_results_paths:
-                _, file_name, ext = get_fn_ext(file_path)
-                shutil.copyfile(file_path, os.path.join(original_data_files_folder, file_name + ext))
+            self.input_dir = os.path.join(self.project_path, Paths.MACHINE_RESULTS_DIR.value)
         else:
             check_if_dir_exists(in_dir=input_dir)
-            self.data_paths = glob.glob(input_dir + f"/*.{self.file_type}")
-            check_if_filepath_list_is_empty(filepaths=self.data_paths, error_msg=f"SIMBA ERROR: No data files found in {input_dir}. Cannot perform Kleinberg smoothing")
-            if not os.path.isdir(output_dir):
-                os.makedirs(output_dir)
-        print(f"Processing Kleinberg burst detection for {len(self.data_paths)} file(s) and {len(classifier_names)} classifier(s)...")
+            self.input_dir = deepcopy(input_dir)
+        self.data_paths = find_files_of_filetypes_in_directory(directory=self.input_dir, extensions=[f'.{self.file_type}'], sort_alphabetically=True, raise_error=True)
+        if output_dir is None:
+            self.output_dir = deepcopy(self.input_dir)
+        else:
+            check_if_dir_exists(in_dir=output_dir)
+            self.output_dir = deepcopy(output_dir)
+        self.original_data_files_folder = os.path.join(self.output_dir, f"Pre_Kleinberg_{self.datetime}")
+        remove_a_folder(folder_dir=self.original_data_files_folder, ignore_errors=True)
+        os.makedirs(self.original_data_files_folder)
+        copy_files_to_directory(file_paths=self.data_paths, dir=self.original_data_files_folder, verbose=False, integer_save_names=False)
+        if self.verbose: print(f"Processing Kleinberg burst detection for {len(self.data_paths)} file(s) and {len(classifier_names)} classifier(s)...")
 
     def hierarchical_searcher(self):
         if (len(self.kleinberg_bouts["Hierarchy"]) == 1) and (int(self.kleinberg_bouts.at[0, "Hierarchy"]) == 0):
@@ -135,7 +144,7 @@ class KleinbergCalculator(ConfigReader):
         for file_cnt, file_path in enumerate(self.data_paths):
             _, video_name, _ = get_fn_ext(file_path)
             video_timer = SimbaTimer(start=True)
-            print(f"Performing Kleinberg burst detection for video {video_name}  (Video {file_cnt+1}/{len(self.data_paths)})...")
+            if self.verbose: print(f"[{get_current_time()}] Performing Kleinberg burst detection for video {video_name} (Video {file_cnt+1}/{len(self.data_paths)})...")
             data_df = read_df(file_path, self.file_type).reset_index(drop=True)
             video_out_df = deepcopy(data_df)
             check_that_column_exist(df=data_df, column_name=self.clfs, file_name=video_name)
@@ -150,7 +159,7 @@ class KleinbergCalculator(ConfigReader):
                     self.kleinberg_bouts.insert(loc=0, column="Video", value=video_name)
                     detailed_df_lst.append(self.kleinberg_bouts)
                     if self.hierarchical_search:
-                        print(f"Applying hierarchical search for video {video_name}...")
+                        if self.verbose: print(f"[{get_current_time()}] Applying hierarchical search for video {video_name}...")
                         self.hierarchical_searcher()
                     else:
                         self.clf_bouts_in_hierarchy = self.kleinberg_bouts[self.kleinberg_bouts["Hierarchy"] == self.hierarchy]
@@ -160,19 +169,38 @@ class KleinbergCalculator(ConfigReader):
                     video_out_df.loc[hierarchy_idx, clf] = 1
             write_df(video_out_df, self.file_type, save_path)
             video_timer.stop_timer()
-            print(f'Kleinberg analysis complete for video {video_name} (saved at {save_path}), elapsed time: {video_timer.elapsed_time_str}s.')
+            if self.verbose: print(f'[{get_current_time()}] Kleinberg analysis complete for video {video_name} (saved at {save_path}), elapsed time: {video_timer.elapsed_time_str}s.')
 
         self.timer.stop_timer()
+        if not self.save_originals:
+            remove_a_folder(folder_dir=self.original_data_files_folder, ignore_errors=False)
+        else:
+            if self.verbose: stdout_success(msg=f"Original, un-smoothened data, saved in {self.original_data_files_folder} directory", elapsed_time=self.timer.elapsed_time_str, source=self.__class__.__name__)
         if len(detailed_df_lst) > 0:
             self.detailed_df = pd.concat(detailed_df_lst, axis=0)
             detailed_save_path = os.path.join(self.logs_path, f"Kleinberg_detailed_log_{self.datetime}.csv")
             self.detailed_df.to_csv(detailed_save_path)
-            stdout_success(msg=f"Kleinberg analysis complete. See {detailed_save_path} for details of detected bouts of all classifiers in all hierarchies", elapsed_time=self.timer.elapsed_time_str, source=self.__class__.__name__)
+            if self.verbose:  stdout_success(msg=f"Kleinberg analysis complete for {len(self.data_paths)} files. Results stored in {self.output_dir} directory. See {detailed_save_path} for details of detected bouts of all classifiers in all hierarchies", elapsed_time=self.timer.elapsed_time_str, source=self.__class__.__name__)
         else:
-            print("Kleinberg analysis complete.")
+            if self.verbose: print(f"[{get_current_time()}] Kleinberg analysis complete for {len(self.data_paths)} files. Results stored in {self.output_dir} directory.")
             KleinbergWarning(msg="All behavior bouts removed following kleinberg smoothing", source=self.__class__.__name__)
 
 
+
+
+# test = KleinbergCalculator(config_path=r"C:\troubleshooting\mitra\project_folder\project_config.ini",
+#                            classifier_names=['straub_tail'],
+#                            sigma=1.1,
+#                            gamma=0.1,
+#                            hierarchy=1,
+#                            save_originals=False,
+#                            hierarchical_search=False)
+#
+# test.run()
+#
+
+
+
 # test = KleinbergCalculator(config_path='/Users/simon/Desktop/envs/simba/troubleshooting/levi/project_folder/project_config.ini',
 #                            classifier_names=['No_Seizure_(0)'],
 #                            sigma=1.1,

simba/mixins/geometry_mixin.py

@@ -1339,8 +1339,7 @@ class GeometryMixin(object):
             )
             for cnt, result in enumerate(pool.imap(constants, data, chunksize=1)):
                 results.append(result)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         if data_ndim == 2:
             return [i for s in results for i in s]
         else:
@@ -1370,8 +1369,7 @@ class GeometryMixin(object):
                                           cap_style=cap_style)
             for cnt, mp_return in enumerate(pool.imap(constants, geomety_lst, chunksize=1)):
                 results.append(mp_return)
-            pool.join()
-            pool.terminate()
+            terminate_cpu_pool(pool=pool, force=False)
         return [l for ll in results for l in ll]
 
     def multiframe_bodyparts_to_circle(self,
@@ -1524,8 +1522,7 @@ class GeometryMixin(object):
             )
             for cnt, result in enumerate(pool.imap(constants, data, chunksize=1)):
                 results.append(result)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     def multiframe_compute_pct_shape_overlap(self,
@@ -1798,8 +1795,7 @@ class GeometryMixin(object):
         timer.stop_timer()
         if verbose:
             stdout_success(msg="Rotated rectangles complete.", elapsed_time=timer.elapsed_time_str)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     @staticmethod
@@ -2003,8 +1999,7 @@ class GeometryMixin(object):
             )
             for cnt, result in enumerate(pool.imap(constants, shapes, chunksize=1)):
                 results.append(result)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     def multiframe_union(self, shapes: Iterable[Union[LineString, MultiLineString, Polygon]], core_cnt: int = -1) -> \
@@ -2043,8 +2038,7 @@ class GeometryMixin(object):
         with multiprocessing.Pool(core_cnt, maxtasksperchild=Defaults.LARGE_MAX_TASK_PER_CHILD.value) as pool:
             for cnt, result in enumerate(pool.imap(GeometryMixin().union, shapes, chunksize=1)):
                 results.append(result)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     def multiframe_symmetric_difference(self, shapes: Iterable[Union[LineString, MultiLineString, Polygon]],
@@ -2084,8 +2078,7 @@ class GeometryMixin(object):
                     pool.imap(GeometryMixin().symmetric_difference, shapes, chunksize=1)
             ):
                 results.append(result)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     def multiframe_delaunay_triangulate_keypoints(self, data: np.ndarray, core_cnt: int = -1) -> List[List[Polygon]]:
@@ -2132,8 +2125,7 @@ class GeometryMixin(object):
             ):
                 results.append(result)
 
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     def multiframe_difference(
@@ -2221,8 +2213,7 @@ class GeometryMixin(object):
             msg="Multi-frame difference compute complete",
             elapsed_time=timer.elapsed_time_str,
         )
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     def multiframe_area(self,
@@ -2276,8 +2267,7 @@ class GeometryMixin(object):
 
         timer.stop_timer()
         stdout_success(msg="Multi-frame area compute complete", elapsed_time=timer.elapsed_time_str)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     def multiframe_bodyparts_to_multistring_skeleton(
@@ -2619,8 +2609,7 @@ class GeometryMixin(object):
                     pool.imap(GeometryMixin.is_shape_covered, shapes, chunksize=1)
             ):
                 results.append(mp_return)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     @staticmethod
@@ -3321,8 +3310,7 @@ class GeometryMixin(object):
             for cnt, result in enumerate(pool.imap(constants, data, chunksize=1)):
                 if result[1] != -1:
                     img_arr[result[0], result[2] - 1, result[1] - 1] = 1
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         timer.stop_timer()
         stdout_success(
             msg="Cumulative coordinates in geometries complete",
@@ -3415,8 +3403,7 @@ class GeometryMixin(object):
             for cnt, result in enumerate(pool.imap(constants, data, chunksize=1)):
                 if result[1] != -1:
                     img_arr[result[0], result[2] - 1, result[1] - 1] = 1
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         if fps is None:
             return np.cumsum(img_arr, axis=0)
         else:
@@ -3559,8 +3546,7 @@ class GeometryMixin(object):
             constants = functools.partial(GeometryMixin._compute_framewise_geometry_idx, grid=grid, verbose=verbose)
             for cnt, result in enumerate(pool.imap(constants, data, chunksize=1)):
                 results.append(result)
-        pool.join();
-        pool.terminate();
+        terminate_cpu_pool(pool=pool, force=False)
         del data
 
         results = np.vstack(results)[:, 1:].astype(np.int32)

simba/mixins/image_mixin.py

@@ -18,7 +18,7 @@ from collections import ChainMap
 import cv2
 import pandas as pd
 from numba import float64, int64, jit, njit, prange, uint8
-from shapely.geometry import MultiPolygon, Polygon
+from shapely.geometry import Polygon
 from skimage.metrics import structural_similarity
 
 from simba.utils.checks import (check_file_exist_and_readable, check_float,
@@ -27,16 +27,14 @@ from simba.utils.checks import (check_file_exist_and_readable, check_float,
                                 check_int, check_str, check_valid_array,
                                 check_valid_boolean, check_valid_lst,
                                 check_valid_tuple, is_img_bw, is_img_greyscale)
+from simba.utils.data import terminate_cpu_pool
 from simba.utils.enums import Defaults, Formats, GeometryEnum, Options
-from simba.utils.errors import (ArrayError, FFMPEGCodecGPUError,
-                                FrameRangeError, InvalidInputError,
-                                NotDirectoryError)
+from simba.utils.errors import ArrayError, FrameRangeError, InvalidInputError
 from simba.utils.printing import SimbaTimer, stdout_success
 from simba.utils.read_write import (find_core_cnt,
                                     find_files_of_filetypes_in_directory,
                                     get_fn_ext, get_video_meta_data,
-                                    read_frm_of_video,
-                                    read_img_batch_from_video_gpu, write_df)
+                                    read_frm_of_video)
 
 
 class ImageMixin(object):
@@ -546,8 +544,8 @@ class ImageMixin(object):
                 pool.imap(constants, split_frm_idx, chunksize=1)
             ):
                 results.append(result)
-        pool.terminate()
-        pool.join()
+
+        terminate_cpu_pool(pool=pool, force=False)
         results = dict(ChainMap(*results))
 
         max_value, max_frm = -np.inf, None
@@ -876,8 +874,7 @@ class ImageMixin(object):
                 pool.imap(ImageMixin()._image_reader_helper, file_paths, chunksize=1)
             ):
                 imgs.update(result)
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return imgs
 
     @staticmethod
@@ -1027,8 +1024,7 @@ class ImageMixin(object):
             for cnt, result in enumerate(pool.imap(constants, frm_lst, chunksize=1)):
                 results.update(result)
 
-        pool.join()
-        pool.terminate()
+        terminate_cpu_pool(pool=pool, force=False)
         return results
 
     @staticmethod
@@ -1509,8 +1505,8 @@ class ImageMixin(object):
                 for cnt, result in enumerate(pool.imap(constants, shapes, chunksize=1)):
                     results.append(result)
                 results = dict(ChainMap(*results))
-        pool.join()
-        pool.terminate()
+
+        terminate_cpu_pool(pool=pool, force=False)
         results = dict(sorted(results.items(), key=lambda item: int(item[0])))
         timer.stop_timer()
         stdout_success(msg="Geometry image slicing complete.", elapsed_time=timer.elapsed_time_str, source=self.__class__.__name__)

simba/mixins/train_model_mixin.py

@@ -67,7 +67,7 @@ from simba.utils.checks import (check_all_dfs_in_list_has_same_cols,
                                 check_valid_boolean, check_valid_dataframe,
                                 check_valid_lst, is_lxc_container)
 from simba.utils.data import (detect_bouts, detect_bouts_multiclass,
-                              get_library_version)
+                              get_library_version, terminate_cpu_pool)
 from simba.utils.enums import (OS, ConfigKey, Defaults, Dtypes, Formats, Links,
                                Methods, MLParamKeys, Options)
 from simba.utils.errors import (ClassifierInferenceError, CorruptedFileError,
@@ -1859,7 +1859,7 @@ class TrainModelMixin(object):
                     shap_raw.append(shap_data[result[1]][1].drop(clf_name, axis=1))
                     if verbose: print(f"Completed SHAP care batch (Batch {result[1] + 1}/{len(shap_data)}).")
 
-            pool.terminate(); pool.join()
+            terminate_cpu_pool(pool=pool, force=False)
             shap_df = pd.DataFrame(data=np.row_stack(shap_results), columns=list(x_names) + ["Expected_value", "Sum", "Prediction_probability", clf_name])
             raw_df = pd.DataFrame(data=np.row_stack(shap_raw), columns=list(x_names))
             out_shap_path, out_raw_path, img_save_path, df_save_paths, summary_dfs, img = None, None, None, None, None, None

simba/plotting/ROI_feature_visualizer_mp.py

@@ -24,10 +24,9 @@ from simba.utils.checks import (check_file_exist_and_readable,
                                 check_if_valid_rgb_tuple, check_int, check_str,
                                 check_valid_boolean, check_valid_lst,
                                 check_video_and_data_frm_count_align)
-from simba.utils.data import slice_roi_dict_for_video
+from simba.utils.data import slice_roi_dict_for_video, terminate_cpu_pool
 from simba.utils.enums import Formats, TextOptions
-from simba.utils.errors import (BodypartColumnNotFoundError, NoFilesFoundError,
-                                ROICoordinatesNotFoundError)
+from simba.utils.errors import BodypartColumnNotFoundError, NoFilesFoundError
 from simba.utils.printing import stdout_success
 from simba.utils.read_write import (concatenate_videos_in_folder,
                                     find_core_cnt, get_fn_ext,
@@ -315,8 +314,7 @@ class ROIfeatureVisualizerMultiprocess(ConfigReader):
             print(f"Joining {self.video_name} multi-processed video...")
             concatenate_videos_in_folder(in_folder=self.save_temp_dir, save_path=self.save_path, video_format="mp4", remove_splits=True, gpu=self.gpu)
             self.timer.stop_timer()
-            pool.terminate()
-            pool.join()
+            terminate_cpu_pool(pool=pool, force=False)
             stdout_success(msg=f"Video {self.video_name} complete. Video saved in directory {self.roi_features_save_dir}.", elapsed_time=self.timer.elapsed_time_str)
 
 

simba/plotting/clf_validator_mp.py

@@ -14,9 +14,9 @@ from simba.mixins.plotting_mixin import PlottingMixin
 from simba.utils.checks import (check_float, check_if_valid_rgb_tuple,
                                 check_int, check_str, check_that_column_exist,
                                 check_valid_lst)
-from simba.utils.data import detect_bouts
-from simba.utils.enums import Formats, TagNames, TextOptions
-from simba.utils.errors import NoFilesFoundError, NoSpecifiedOutputError
+from simba.utils.data import detect_bouts, terminate_cpu_pool
+from simba.utils.enums import Formats, TextOptions
+from simba.utils.errors import NoSpecifiedOutputError
 from simba.utils.printing import SimbaTimer, log_event, stdout_success
 from simba.utils.read_write import (concatenate_videos_in_folder,
                                     find_core_cnt, get_fn_ext,
@@ -218,8 +218,7 @@ class ClassifierValidationClipsMultiprocess(ConfigReader):
                 for cnt, result in enumerate(
                     pool.imap(constants, clip_data, chunksize=self.multiprocess_chunksize)):
                     print(f"Bout {cnt+1} complete...")
-                pool.terminate()
-                pool.join()
+                terminate_cpu_pool(pool=pool, force=False)
 
             if self.concat_video:
                 print(f"Joining {file_name} multiprocessed video...")