simba-uw-tf-dev 4.6.2__py3-none-any.whl → 4.6.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of simba-uw-tf-dev might be problematic. Click here for more details.

Files changed (46) hide show
  1. simba/assets/lookups/tooptips.json +6 -1
  2. simba/data_processors/agg_clf_counter_mp.py +52 -53
  3. simba/data_processors/cuda/image.py +3 -1
  4. simba/data_processors/cue_light_analyzer.py +5 -9
  5. simba/data_processors/kleinberg_calculator.py +57 -29
  6. simba/mixins/geometry_mixin.py +14 -28
  7. simba/mixins/image_mixin.py +10 -14
  8. simba/mixins/train_model_mixin.py +2 -2
  9. simba/plotting/ROI_feature_visualizer_mp.py +3 -5
  10. simba/plotting/clf_validator_mp.py +4 -5
  11. simba/plotting/cue_light_visualizer.py +6 -7
  12. simba/plotting/directing_animals_visualizer_mp.py +2 -3
  13. simba/plotting/distance_plotter_mp.py +378 -378
  14. simba/plotting/gantt_creator_mp.py +61 -31
  15. simba/plotting/geometry_plotter.py +270 -272
  16. simba/plotting/heat_mapper_clf_mp.py +2 -4
  17. simba/plotting/heat_mapper_location_mp.py +2 -2
  18. simba/plotting/light_dark_box_plotter.py +2 -2
  19. simba/plotting/path_plotter_mp.py +26 -29
  20. simba/plotting/plot_clf_results_mp.py +455 -454
  21. simba/plotting/pose_plotter_mp.py +28 -29
  22. simba/plotting/probability_plot_creator_mp.py +288 -288
  23. simba/plotting/roi_plotter_mp.py +31 -31
  24. simba/plotting/single_run_model_validation_video_mp.py +427 -427
  25. simba/plotting/spontaneous_alternation_plotter.py +2 -3
  26. simba/plotting/yolo_pose_track_visualizer.py +32 -27
  27. simba/plotting/yolo_pose_visualizer.py +35 -36
  28. simba/plotting/yolo_seg_visualizer.py +2 -3
  29. simba/roi_tools/roi_aggregate_stats_mp.py +4 -3
  30. simba/roi_tools/roi_clf_calculator_mp.py +3 -3
  31. simba/sandbox/get_cpu_pool.py +5 -0
  32. simba/ui/pop_ups/kleinberg_pop_up.py +39 -41
  33. simba/ui/tkinter_functions.py +3 -0
  34. simba/utils/data.py +89 -12
  35. simba/utils/enums.py +1 -0
  36. simba/utils/printing.py +124 -124
  37. simba/utils/read_write.py +3730 -3721
  38. simba/video_processors/egocentric_video_rotator.py +2 -4
  39. simba/video_processors/video_processing.py +19 -8
  40. simba/video_processors/videos_to_frames.py +1 -1
  41. {simba_uw_tf_dev-4.6.2.dist-info → simba_uw_tf_dev-4.6.4.dist-info}/METADATA +1 -1
  42. {simba_uw_tf_dev-4.6.2.dist-info → simba_uw_tf_dev-4.6.4.dist-info}/RECORD +46 -45
  43. {simba_uw_tf_dev-4.6.2.dist-info → simba_uw_tf_dev-4.6.4.dist-info}/LICENSE +0 -0
  44. {simba_uw_tf_dev-4.6.2.dist-info → simba_uw_tf_dev-4.6.4.dist-info}/WHEEL +0 -0
  45. {simba_uw_tf_dev-4.6.2.dist-info → simba_uw_tf_dev-4.6.4.dist-info}/entry_points.txt +0 -0
  46. {simba_uw_tf_dev-4.6.2.dist-info → simba_uw_tf_dev-4.6.4.dist-info}/top_level.txt +0 -0
@@ -14,7 +14,7 @@ from simba.data_processors.spontaneous_alternation_calculator import \
14
14
  from simba.mixins.config_reader import ConfigReader
15
15
  from simba.utils.checks import (check_file_exist_and_readable, check_float,
16
16
  check_int, check_str, check_valid_lst)
17
- from simba.utils.data import detect_bouts
17
+ from simba.utils.data import detect_bouts, terminate_cpu_pool
18
18
  from simba.utils.enums import Formats, Paths, TextOptions
19
19
  from simba.utils.errors import AnimalNumberError, InvalidInputError
20
20
  from simba.utils.printing import stdout_success
@@ -296,8 +296,7 @@ class SpontaneousAlternationsPlotter(ConfigReader):
296
296
  pool.imap(constants, frm_index, chunksize=self.multiprocess_chunksize)
297
297
  ):
298
298
  print(f"Section {cnt} complete...")
299
- pool.terminate()
300
- pool.join()
299
+ terminate_cpu_pool(pool=pool, force=False)
301
300
  print(f"Joining {sa_computer.video_name} multiprocessed video...")
302
301
  concatenate_videos_in_folder(in_folder=self.temp_folder, save_path=save_path)
303
302
  self.timer.stop_timer()
@@ -13,9 +13,11 @@ from simba.mixins.plotting_mixin import PlottingMixin
13
13
  from simba.utils.checks import (check_file_exist_and_readable, check_float,
14
14
  check_if_dir_exists, check_int,
15
15
  check_valid_boolean, check_valid_dataframe)
16
+ from simba.utils.data import get_cpu_pool, terminate_cpu_pool
16
17
  from simba.utils.enums import Defaults, Formats
17
18
  from simba.utils.errors import InvalidFilepathError, NoFilesFoundError
18
- from simba.utils.lookups import get_random_color_palette, intermittent_palette
19
+ from simba.utils.lookups import (get_current_time, get_random_color_palette,
20
+ intermittent_palette)
19
21
  from simba.utils.printing import SimbaTimer, stdout_success
20
22
  from simba.utils.read_write import (concatenate_videos_in_folder,
21
23
  create_directory,
@@ -53,13 +55,12 @@ def _yolo_keypoint_track_visualizer(frm_ids: np.ndarray,
53
55
  video_save_path = os.path.join(save_dir, f'{batch_id}.mp4')
54
56
  video_writer = cv2.VideoWriter(video_save_path, fourcc, video_meta_data["fps"], (video_meta_data["width"], video_meta_data["height"]))
55
57
  while current_frm <= end_frm:
56
- print(f'Processing frame {current_frm}/{video_meta_data["frame_count"]} (batch: {batch_id})...')
58
+ print(f'[{get_current_time()}] Processing frame {current_frm}/{video_meta_data["frame_count"]} (batch: {batch_id}, video name: {video_meta_data["video_name"]})...')
57
59
  img = read_frm_of_video(video_path=video_path, frame_index=current_frm, raise_error=False)
58
60
  if img is not None:
59
61
  frm_data = data.loc[data[FRAME] == current_frm]
60
62
  frm_data = frm_data[frm_data[CONFIDENCE] > threshold]
61
63
  for cnt, (row, row_data) in enumerate(frm_data.iterrows()):
62
-
63
64
  clrs = np.array(palettes[int(row_data[TRACK])]).astype(np.int32)
64
65
  bbox_cords = row_data[BOX_CORD_FIELDS].values.astype(np.int32).reshape(-1, 2)
65
66
  kp_coords = row_data.drop(EXPECTED_COLS).values.astype(np.int32).reshape(-1, 3)[:, :-1]
@@ -159,6 +160,8 @@ class YOLOPoseTrackVisualizer():
159
160
  self.threshold, self.circle_size, self.thickness, self.show_bbox, self.overwrite = threshold, circle_size, thickness, bbox, overwrite
160
161
 
161
162
  def run(self):
163
+ self.pool = get_cpu_pool(core_cnt=self.core_cnt, maxtasksperchild=Defaults.MAXIMUM_MAX_TASK_PER_CHILD.value, source=self.__class__.__name__)
164
+ self.timer = SimbaTimer(start=True)
162
165
  for video_cnt, (video_name, data_path) in enumerate(self.data_paths.items()):
163
166
  print(f'Visualizing YOLO pose tracks in video {video_name} ({video_cnt+1}/{len(self.data_paths.keys())}) ...')
164
167
  video_timer = SimbaTimer(start=True)
@@ -189,23 +192,25 @@ class YOLOPoseTrackVisualizer():
189
192
 
190
193
  frm_batches = np.array_split(np.array(list(range(0, df_frm_cnt))), self.core_cnt)
191
194
  frm_batches = [(i, j) for i, j in enumerate(frm_batches)]
192
- with multiprocessing.Pool(self.core_cnt, maxtasksperchild=Defaults.MAXIMUM_MAX_TASK_PER_CHILD.value) as pool:
193
- constants = functools.partial(_yolo_keypoint_track_visualizer,
194
- data=self.data_df,
195
- threshold=self.threshold,
196
- video_path=self.video_paths[video_name],
197
- save_dir=video_temp_dir,
198
- circle_size=video_circle_size,
199
- thickness=video_thickness,
200
- palettes=video_palettes,
201
- show_bbox=self.show_bbox)
202
- for cnt, result in enumerate(pool.imap(constants, frm_batches, chunksize=1)):
203
- print(f'Video batch {result+1}/{self.core_cnt} complete...')
204
- pool.terminate()
205
- pool.join()
195
+ constants = functools.partial(_yolo_keypoint_track_visualizer,
196
+ data=self.data_df,
197
+ threshold=self.threshold,
198
+ video_path=self.video_paths[video_name],
199
+ save_dir=video_temp_dir,
200
+ circle_size=video_circle_size,
201
+ thickness=video_thickness,
202
+ palettes=video_palettes,
203
+ show_bbox=self.show_bbox)
204
+ for cnt, result in enumerate(self.pool.imap(constants, frm_batches, chunksize=1)):
205
+ print(f'[{get_current_time()}] Video batch {result+1}/{self.core_cnt} complete...')
206
206
  video_timer.stop_timer()
207
207
  concatenate_videos_in_folder(in_folder=video_temp_dir, save_path=save_path, gpu=True)
208
208
  stdout_success(msg=f'YOLO track pose video saved at {save_path}', source=self.__class__.__name__, elapsed_time=video_timer.elapsed_time_str)
209
+ terminate_cpu_pool(pool=self.pool, force=False, source=self.__class__.__name__)
210
+ self.timer.stop_timer()
211
+ stdout_success(msg=f'YOLO track pose video data for {len(self.data_paths.keys())} videos saved in {self.save_dir}', source=self.__class__.__name__, elapsed_time=self.timer.elapsed_time_str)
212
+
213
+
209
214
  #
210
215
  # if __name__ == "__main__" and not hasattr(sys, 'ps1'):
211
216
  # parser = argparse.ArgumentParser(description="Visualize YOLO pose tracking CSV outputs on their source videos.")
@@ -247,13 +252,13 @@ class YOLOPoseTrackVisualizer():
247
252
  # #kp_vis.run()
248
253
 
249
254
 
250
- if __name__ == "__main__":
251
- VIDEO_PATH = r"E:\netholabs_videos\primeintellect_100_videos"
252
- DATA_PATH = r"E:\netholabs_videos\primeintellect_100_largest"
253
- SAVE_DIR = r"E:\netholabs_videos\primeintellect_100_videos\out"
254
- kp_vis = YOLOPoseTrackVisualizer(data_path=DATA_PATH,
255
- video_path=VIDEO_PATH,
256
- save_dir=SAVE_DIR,
257
- core_cnt=8,
258
- bbox=True)
259
- kp_vis.run()
255
+ # if __name__ == "__main__":
256
+ # VIDEO_PATH = r"E:\netholabs_videos\primeintellect_100_videos"
257
+ # DATA_PATH = r"E:\netholabs_videos\primeintellect_100_largest"
258
+ # SAVE_DIR = r"E:\netholabs_videos\primeintellect_100_videos\out"
259
+ # kp_vis = YOLOPoseTrackVisualizer(data_path=DATA_PATH,
260
+ # video_path=VIDEO_PATH,
261
+ # save_dir=SAVE_DIR,
262
+ # core_cnt=8,
263
+ # bbox=True)
264
+ # kp_vis.run()
@@ -14,7 +14,8 @@ from simba.utils.checks import (check_file_exist_and_readable, check_float,
14
14
  check_if_dir_exists, check_int,
15
15
  check_valid_boolean, check_valid_dataframe,
16
16
  check_valid_lst, check_valid_tuple)
17
- from simba.utils.data import create_color_palette
17
+ from simba.utils.data import (create_color_palette, get_cpu_pool,
18
+ terminate_cpu_pool)
18
19
  from simba.utils.enums import Defaults, Options
19
20
  from simba.utils.errors import (CountError, DataHeaderError, FrameRangeError,
20
21
  InvalidInputError, NoDataError)
@@ -22,10 +23,9 @@ from simba.utils.printing import SimbaTimer, stdout_success
22
23
  from simba.utils.read_write import (concatenate_videos_in_folder,
23
24
  create_directory, find_core_cnt,
24
25
  find_files_of_filetypes_in_directory,
25
- get_fn_ext, get_video_meta_data,
26
- read_frm_of_video, recursive_file_search,
27
- remove_a_folder)
28
- from simba.utils.warnings import InvalidValueWarning
26
+ get_current_time, get_fn_ext,
27
+ get_video_meta_data, read_frm_of_video,
28
+ recursive_file_search, remove_a_folder)
29
29
 
30
30
  FRAME = 'FRAME'
31
31
  CLASS_ID = 'CLASS_ID'
@@ -58,7 +58,7 @@ def _yolo_keypoint_visualizer(frm_ids: np.ndarray,
58
58
  if TRACK in data.columns:
59
59
  data = data.drop([TRACK], axis=1)
60
60
  while current_frm <= end_frm:
61
- print(f'Processing frame {current_frm}/{video_meta_data["frame_count"]} (batch: {batch_id}, video: {video_meta_data["video_name"]})...')
61
+ print(f'[{get_current_time()}] Processing frame {current_frm}/{video_meta_data["frame_count"]} (batch: {batch_id}, video: {video_meta_data["video_name"]})...')
62
62
  img = read_frm_of_video(video_path=video_path, frame_index=current_frm)
63
63
  frm_data = data.loc[data[FRAME] == current_frm]
64
64
  frm_data = frm_data[frm_data[CONFIDENCE] > threshold]
@@ -206,6 +206,7 @@ class YOLOPoseVisualizer():
206
206
  self.timer = SimbaTimer(start=True)
207
207
 
208
208
  def run(self):
209
+ self.pool = get_cpu_pool(core_cnt=self.core_cnt, maxtasksperchild=Defaults.MAXIMUM_MAX_TASK_PER_CHILD.value, verbose=True, source=self.__class__.__name__)
209
210
  for video_cnt, (video_name, data_path) in enumerate(self.data_paths.items()):
210
211
  video_timer = SimbaTimer(start=True)
211
212
  self.video_temp_dir = os.path.join(self.save_dir, video_name, "temp")
@@ -248,26 +249,24 @@ class YOLOPoseVisualizer():
248
249
  thickness = deepcopy(self.thickness)
249
250
  frm_batches = np.array_split(np.array(list(range(0, self.df_frm_cnt))), self.core_cnt)
250
251
  frm_batches = [(i, j) for i, j in enumerate(frm_batches)]
251
- if self.verbose: print(f'Visualizing video {self.video_meta_data["video_name"]} (frame count: {self.video_meta_data["frame_count"]})...')
252
- with multiprocessing.Pool(self.core_cnt, maxtasksperchild=Defaults.MAXIMUM_MAX_TASK_PER_CHILD.value) as pool:
253
- constants = functools.partial(_yolo_keypoint_visualizer,
254
- data=self.data_df,
255
- threshold=self.threshold,
256
- video_path=self.video_paths[video_name],
257
- save_dir=self.video_temp_dir,
258
- circle_size=circle_size,
259
- thickness=thickness,
260
- palettes=self.clrs,
261
- bbox=self.bbox,
262
- skeleton=self.skeleton)
263
- for cnt, result in enumerate(pool.imap(constants, frm_batches, chunksize=1)):
264
- print(f'Video batch {result+1}/{self.core_cnt} complete...')
265
- pool.terminate()
266
- pool.join()
252
+ if self.verbose: print(f'[{get_current_time()}] Visualizing video {self.video_meta_data["video_name"]} (frame count: {self.video_meta_data["frame_count"]})...')
253
+ constants = functools.partial(_yolo_keypoint_visualizer,
254
+ data=self.data_df,
255
+ threshold=self.threshold,
256
+ video_path=self.video_paths[video_name],
257
+ save_dir=self.video_temp_dir,
258
+ circle_size=circle_size,
259
+ thickness=thickness,
260
+ palettes=self.clrs,
261
+ bbox=self.bbox,
262
+ skeleton=self.skeleton)
263
+ for cnt, result in enumerate(self.pool.imap(constants, frm_batches, chunksize=1)):
264
+ print(f'[{get_current_time()}] Video batch {result+1}/{self.core_cnt} complete...')
267
265
  video_timer.stop_timer()
268
266
  concatenate_videos_in_folder(in_folder=self.video_temp_dir, save_path=self.save_path, gpu=True)
269
267
  stdout_success(msg=f'YOLO pose video saved at {self.save_path} (Video {video_cnt+1}/{len(list(self.data_paths.keys()))})', source=self.__class__.__name__, elapsed_time=video_timer.elapsed_time_str)
270
268
 
269
+ terminate_cpu_pool(pool=self.pool, force=False, source=self.__class__.__name__)
271
270
  self.timer.stop_timer()
272
271
  stdout_success(msg=f'{len(list(self.data_paths.keys()))} YOLO pose video saved in directory {self.save_dir}', source=self.__class__.__name__, elapsed_time=self.timer.elapsed_time_str)
273
272
 
@@ -413,18 +412,18 @@ class YOLOPoseVisualizer():
413
412
  # kp_vis.run()
414
413
 
415
414
 
416
- # if __name__ == "__main__":
417
- # video_path = r"E:\netholabs_videos\primeintellect_100_videos\cage_1_date_2025_08_28_hour_20_minute_21.avi"
418
- # data_path = r"E:\netholabs_videos\primeintellect_100_largest\cage_1_date_2025_08_28_hour_20_minute_21.csv"
419
- # save_dir = r'E:\netholabs_videos\test_order'
420
- # kp_vis = YOLOPoseVisualizer(data_path=data_path,
421
- # video_path=video_path,
422
- # save_dir=save_dir,
423
- # core_cnt=14,
424
- # palettes=('tab20',),
425
- # recursive=True,
426
- # sample_n=None)
427
- #
428
- #
429
- # kp_vis.run()
415
+ if __name__ == "__main__":
416
+ video_path = r"E:\netholabs_videos\primeintellect_100_videos\cage_1_date_2025_08_28_hour_20_minute_21.avi"
417
+ data_path = r"E:\netholabs_videos\primeintellect_100_largest\cage_1_date_2025_08_28_hour_20_minute_21.csv"
418
+ save_dir = r'E:\netholabs_videos\test_order'
419
+ kp_vis = YOLOPoseVisualizer(data_path=data_path,
420
+ video_path=video_path,
421
+ save_dir=save_dir,
422
+ core_cnt=14,
423
+ palettes=('tab20',),
424
+ recursive=True,
425
+ sample_n=None)
426
+
427
+
428
+ kp_vis.run()
430
429
 
@@ -12,7 +12,7 @@ from simba.utils.checks import (check_file_exist_and_readable, check_float,
12
12
  check_int, check_valid_boolean,
13
13
  check_valid_dataframe, check_valid_lst,
14
14
  check_valid_tuple)
15
- from simba.utils.data import create_color_palette
15
+ from simba.utils.data import create_color_palette, terminate_cpu_pool
16
16
  from simba.utils.enums import Defaults, Options
17
17
  from simba.utils.errors import CountError, DataHeaderError, FrameRangeError
18
18
  from simba.utils.printing import SimbaTimer, stdout_success
@@ -140,8 +140,7 @@ class YOLOSegmentationVisualizer():
140
140
  shape_opacity=self.shape_opacity)
141
141
  for cnt, result in enumerate(pool.imap(constants, frm_batches, chunksize=1)):
142
142
  print(f'Video batch {result+1}/{self.core_cnt} complete...')
143
- pool.terminate()
144
- pool.join()
143
+ terminate_cpu_pool(pool=pool, force=False)
145
144
  video_timer.stop_timer()
146
145
  concatenate_videos_in_folder(in_folder=self.video_temp_dir, save_path=self.save_path, gpu=True)
147
146
  stdout_success(msg=f'YOLO pose video saved at {self.save_path}', source=self.__class__.__name__, elapsed_time=video_timer.elapsed_time_str)
@@ -17,7 +17,8 @@ from simba.utils.checks import (
17
17
  check_all_file_names_are_represented_in_video_log,
18
18
  check_file_exist_and_readable, check_float, check_if_dir_exists, check_int,
19
19
  check_that_column_exist, check_valid_boolean, check_valid_lst)
20
- from simba.utils.data import detect_bouts, slice_roi_dict_for_video
20
+ from simba.utils.data import (detect_bouts, slice_roi_dict_for_video,
21
+ terminate_cpu_pool)
21
22
  from simba.utils.enums import ROI_SETTINGS, Formats, Keys
22
23
  from simba.utils.errors import CountError, ROICoordinatesNotFoundError
23
24
  from simba.utils.printing import SimbaTimer, stdout_success
@@ -297,8 +298,8 @@ class ROIAggregateStatisticsAnalyzerMultiprocess(ConfigReader, FeatureExtraction
297
298
  self.results.append(result); self.detailed_dfs.append(detailed_dfs)
298
299
  print(f"Data batch core {batch_id} / {self.core_cnt} complete...")
299
300
  self.results = pd.concat(self.results, axis=0).reset_index(drop=True)
300
- pool.join()
301
- pool.terminate()
301
+ terminate_cpu_pool(pool=pool, force=False)
302
+
302
303
 
303
304
  def save(self):
304
305
  self.__clean_results()
@@ -16,7 +16,8 @@ from simba.utils.checks import (
16
16
  check_all_file_names_are_represented_in_video_log,
17
17
  check_file_exist_and_readable, check_if_dir_exists, check_int,
18
18
  check_valid_boolean, check_valid_dataframe, check_valid_lst)
19
- from simba.utils.data import detect_bouts, slice_roi_dict_for_video
19
+ from simba.utils.data import (detect_bouts, slice_roi_dict_for_video,
20
+ terminate_cpu_pool)
20
21
  from simba.utils.enums import ROI_SETTINGS, Keys
21
22
  from simba.utils.errors import InvalidInputError, NoROIDataError
22
23
  from simba.utils.lookups import get_current_time
@@ -237,8 +238,7 @@ class ROIClfCalculatorMultiprocess(ConfigReader):
237
238
  self.bouts_results.append(batch_bout_results)
238
239
  print(f"Data batch core {batch_id + 1} / {self.core_cnt} complete...")
239
240
  self.bouts_results = pd.concat(self.bouts_results, axis=0).reset_index(drop=True) if len(self.bouts_results) > 0 else None
240
- pool.join()
241
- pool.terminate()
241
+ terminate_cpu_pool(pool=pool, force=False)
242
242
 
243
243
  def save(self):
244
244
  self.timer.stop_timer()
@@ -0,0 +1,5 @@
1
+ from simba.utils.data import get_cpu_pool, terminate_cpu_pool
2
+
3
+
4
+ pool = get_cpu_pool()
5
+ terminate_cpu_pool(pool=pool)
@@ -7,13 +7,13 @@ from typing import Union
7
7
  from simba.data_processors.kleinberg_calculator import KleinbergCalculator
8
8
  from simba.mixins.config_reader import ConfigReader
9
9
  from simba.mixins.pop_up_mixin import PopUpMixin
10
- from simba.ui.tkinter_functions import (CreateLabelFrameWithIcon, Entry_Box,
11
- SimbaButton, SimBADropDown)
10
+ from simba.ui.tkinter_functions import (CreateLabelFrameWithIcon, Entry_Box, SimbaButton, SimBADropDown, SimBALabel)
12
11
  from simba.utils.checks import check_float, check_int
13
12
  from simba.utils.enums import Formats, Keys, Links
14
13
  from simba.utils.errors import NoChoosenClassifierError, NoDataError
15
- from simba.utils.read_write import str_2_bool
14
+ from simba.utils.read_write import str_2_bool, get_current_time
16
15
 
16
+ INSTRUCTIONS_TXT = 'Results in the project_folder/csv/machine_results folder are overwritten.\n If saving the originals, the original un-smoothened data is saved in a subdirectory of \nthe project_folder/csv/machine_results folder'
17
17
 
18
18
  class KleinbergPopUp(PopUpMixin, ConfigReader):
19
19
  def __init__(self,
@@ -24,61 +24,59 @@ class KleinbergPopUp(PopUpMixin, ConfigReader):
24
24
  raise NoDataError(msg=f'Cannot perform Kleinberg smoothing: No data files found in {self.machine_results_dir} directory', source=self.__class__.__name__)
25
25
  PopUpMixin.__init__(self, title="APPLY KLEINBERG BEHAVIOR CLASSIFICATION SMOOTHING", icon='smooth')
26
26
  kleinberg_settings_frm = CreateLabelFrameWithIcon(parent=self.main_frm, header="KLEINBERG SETTINGS", icon_name=Keys.DOCUMENTATION.value, icon_link=Links.KLEINBERG.value)
27
- self.k_sigma = Entry_Box(kleinberg_settings_frm, fileDescription="SIGMA", img='sigma', value='2', justify='center', labelwidth=35, entry_box_width=35)
28
- self.k_gamma = Entry_Box(kleinberg_settings_frm, fileDescription="GAMMA", img='gamma', value='0.3', justify='center', labelwidth=35, entry_box_width=35)
29
- self.k_hierarchy = Entry_Box(kleinberg_settings_frm, fileDescription="HIERARCHY", value=1, img='hierarchy_2', justify='center', labelwidth=35, entry_box_width=35)
30
- self.h_search_dropdown = SimBADropDown(parent=kleinberg_settings_frm, dropdown_options=['TRUE', 'FALSE'], label="HIERACHICAL SEARCH", value='FALSE', img='hierarchy', label_width=35, dropdown_width=35)
27
+ self.k_sigma = Entry_Box(kleinberg_settings_frm, fileDescription="SIGMA", img='sigma', value='2', justify='center', labelwidth=35, entry_box_width=35, tooltip_key='KLEINBERG_SIGMA')
28
+ self.k_gamma = Entry_Box(kleinberg_settings_frm, fileDescription="GAMMA", img='gamma', value='0.3', justify='center', labelwidth=35, entry_box_width=35, tooltip_key='KLEINBERG_GAMMA')
29
+ self.k_hierarchy = Entry_Box(kleinberg_settings_frm, fileDescription="HIERARCHY", value=1, img='hierarchy_2', justify='center', labelwidth=35, entry_box_width=35, validation='numeric', tooltip_key='KLEINBERG_HIERARCHY')
30
+ self.h_search_dropdown = SimBADropDown(parent=kleinberg_settings_frm, dropdown_options=['TRUE', 'FALSE'], label="HIERARCHICAL SEARCH", value='FALSE', img='hierarchy', label_width=35, dropdown_width=35, tooltip_key='KLEINBERG_HIERARCHY_SEARCH')
31
+ self.save_originals_dropdown = SimBADropDown(parent=kleinberg_settings_frm, dropdown_options=['TRUE', 'FALSE'], label="SAVE ORIGINAL DATA:", value='TRUE', img='save', label_width=35, dropdown_width=35, tooltip_key='KLEINBERG_SAVE_ORIGINALS')
32
+ self.instructions_lbl = SimBALabel(parent=kleinberg_settings_frm, txt=INSTRUCTIONS_TXT, justify='center', txt_clr='blue', font=Formats.FONT_REGULAR_ITALICS.value)
31
33
 
32
34
 
33
- kleinberg_table_frame = LabelFrame(self.main_frm, text="CHOOSE CLASSIFIER(S) FOR KLEINBERG SMOOTHING", font=Formats.FONT_HEADER.value)
35
+ kleinberg_table_frame = CreateLabelFrameWithIcon(parent=self.main_frm, header="CHOOSE CLASSIFIER(S) FOR KLEINBERG SMOOTHING", icon_name=Keys.DOCUMENTATION.value, icon_link=Links.KLEINBERG.value)
34
36
  clf_var_dict, clf_cb_dict = {}, {}
35
37
  for clf_cnt, clf in enumerate(self.clf_names):
36
38
  clf_var_dict[clf] = BooleanVar()
37
39
  clf_cb_dict[clf] = Checkbutton(kleinberg_table_frame, text=clf, font=Formats.FONT_REGULAR.value, variable=clf_var_dict[clf])
38
- clf_cb_dict[clf].grid(row=clf_cnt, sticky=NW)
40
+ clf_cb_dict[clf].grid(row=clf_cnt, column=0, sticky=NW)
39
41
 
40
42
  run_kleinberg_btn = SimbaButton(parent=self.main_frm, txt="APPLY KLEINBERG SMOOTHER", img='rocket', txt_clr="blue", font=Formats.FONT_REGULAR.value, cmd=self.run_kleinberg, cmd_kwargs={'behaviors_dict': lambda: clf_var_dict, 'hierarchical_search': lambda: str_2_bool(self.h_search_dropdown.get_value())})
41
- kleinberg_settings_frm.grid(row=0, sticky=W, padx=10)
42
- self.k_sigma.grid(row=0, sticky=W)
43
- self.k_gamma.grid(row=1, sticky=W)
44
- self.k_hierarchy.grid(row=2, sticky=W)
45
- self.h_search_dropdown.grid(row=3, column=0, sticky=W)
46
- kleinberg_table_frame.grid(row=1, pady=10, padx=10)
47
- run_kleinberg_btn.grid(row=2)
43
+ kleinberg_settings_frm.grid(row=0, sticky=W, pady=(15, 0))
44
+ self.instructions_lbl.grid(row=0, sticky=W)
45
+ self.k_sigma.grid(row=1, sticky=W)
46
+ self.k_gamma.grid(row=2, sticky=W)
47
+ self.k_hierarchy.grid(row=3, sticky=W)
48
+ self.h_search_dropdown.grid(row=4, column=0, sticky=W)
49
+ self.save_originals_dropdown.grid(row=5, column=0, sticky=W)
50
+ kleinberg_table_frame.grid(row=1, column=0, sticky=NW, pady=(15, 0))
51
+ run_kleinberg_btn.grid(row=2, column=0, sticky=NW, pady=(15, 0))
48
52
  self.main_frm.mainloop()
49
53
 
50
54
  def run_kleinberg(self, behaviors_dict: dict, hierarchical_search: bool):
51
55
  targets = []
52
56
  for behaviour, behavior_val in behaviors_dict.items():
53
- if behavior_val.get():
54
- targets.append(behaviour)
57
+ if behavior_val.get(): targets.append(behaviour)
55
58
 
56
59
  if len(targets) == 0:
57
60
  raise NoChoosenClassifierError(source=self.__class__.__name__)
58
61
 
59
- check_int(name="Hierarchy", value=self.k_hierarchy.entry_get)
60
- check_float(name="Sigma", value=self.k_sigma.entry_get)
61
- check_float(name="Gamma", value=self.k_gamma.entry_get)
62
-
63
- try:
64
- print(
65
- "Applying kleinberg hyperparameter Setting: Sigma: {}, Gamma: {}, Hierarchy: {}".format(
66
- str(self.k_sigma.entry_get),
67
- str(self.k_gamma.entry_get),
68
- str(self.k_hierarchy.entry_get),
69
- )
70
- )
71
- except:
72
- print("Please insert accurate values for all hyperparameters.")
73
-
74
- kleinberg_analyzer = KleinbergCalculator(
75
- config_path=self.config_path,
76
- classifier_names=targets,
77
- sigma=self.k_sigma.entry_get,
78
- gamma=self.k_gamma.entry_get,
79
- hierarchy=self.k_hierarchy.entry_get,
80
- hierarchical_search=hierarchical_search,
81
- )
62
+ k_hierarchy = self.k_hierarchy.entry_get
63
+ k_sigma = self.k_sigma.entry_get
64
+ k_gamma = self.k_gamma.entry_get
65
+ save_originals = str_2_bool(self.save_originals_dropdown.get_value())
66
+
67
+ check_int(name="Hierarchy", value=k_hierarchy, min_value=1, allow_negative=False, allow_zero=False, raise_error=True)
68
+ check_float(name="Sigma", value=k_sigma, allow_negative=False, allow_zero=False, raise_error=True)
69
+ check_float(name="Gamma", value=k_gamma, allow_negative=False, allow_zero=False, raise_error=True)
70
+
71
+ print(f"[{get_current_time()}] Applying kleinberg hyperparameter Setting: Sigma: {k_sigma}, Gamma: {k_gamma}, Hierarchy: {k_hierarchy}")
72
+
73
+ kleinberg_analyzer = KleinbergCalculator(config_path=self.config_path,
74
+ classifier_names=targets,
75
+ sigma=float(k_sigma),
76
+ gamma=float(k_gamma),
77
+ hierarchy=int(k_hierarchy),
78
+ hierarchical_search=hierarchical_search,
79
+ save_originals=save_originals)
82
80
  kleinberg_analyzer.run()
83
81
 
84
82
 
@@ -229,6 +229,7 @@ class Entry_Box(Frame):
229
229
  value: Optional[Any] = None,
230
230
  label_font: tuple = Formats.FONT_REGULAR.value,
231
231
  entry_font: tuple = Formats.FONT_REGULAR.value,
232
+ tooltip_key: Optional[str] = None,
232
233
  justify: Literal["left", "center", "right"] = 'left',
233
234
  cmd: Optional[Callable] = None,
234
235
  **kw):
@@ -249,6 +250,8 @@ class Entry_Box(Frame):
249
250
  self.filePath = StringVar()
250
251
  self.lblName = Label(self, text=fileDescription, width=labelwidth, anchor=W, font=label_font, bg=label_bg_clr)
251
252
  self.lblName.grid(row=0, column=1)
253
+ if tooltip_key in TOOLTIPS.keys():
254
+ CreateToolTip(widget=self.lblName, text=TOOLTIPS[tooltip_key])
252
255
  if not entry_box_width:
253
256
  self.entPath = Entry(self, textvariable=self.filePath, state=self.status, validate="key", validatecommand=self.validation_methods.get(validation, None), font=entry_font, justify=justify, bg=entry_box_clr)
254
257
  else:
simba/utils/data.py CHANGED
@@ -5,6 +5,7 @@ import configparser
5
5
  import gc
6
6
  import io
7
7
  import os
8
+ import platform
8
9
  import subprocess
9
10
  from copy import deepcopy
10
11
  from datetime import datetime
@@ -38,15 +39,19 @@ from simba.utils.checks import (check_file_exist_and_readable, check_float,
38
39
  check_if_valid_rgb_tuple, check_instance,
39
40
  check_int, check_str, check_that_column_exist,
40
41
  check_that_hhmmss_start_is_before_end,
41
- check_valid_array, check_valid_cpu_pool,
42
- check_valid_dataframe, check_valid_lst)
43
- from simba.utils.enums import ConfigKey, Dtypes, Formats, Keys, Options
42
+ check_valid_array, check_valid_boolean,
43
+ check_valid_cpu_pool, check_valid_dataframe,
44
+ check_valid_lst)
45
+ from simba.utils.enums import (OS, ConfigKey, Defaults, Dtypes, Formats, Keys,
46
+ Options)
44
47
  from simba.utils.errors import (BodypartColumnNotFoundError, CountError,
45
48
  InvalidFileTypeError, InvalidInputError,
46
49
  NoFilesFoundError, NoROIDataError,
47
50
  SimBAModuleNotFoundError)
51
+ from simba.utils.lookups import get_current_time
48
52
  from simba.utils.printing import stdout_success, stdout_warning
49
- from simba.utils.read_write import (find_video_of_file, get_fn_ext,
53
+ from simba.utils.read_write import (find_core_cnt, find_video_of_file,
54
+ get_current_time, get_fn_ext,
50
55
  get_video_meta_data, read_config_entry,
51
56
  read_config_file, read_df,
52
57
  read_project_path_and_file_type,
@@ -1813,33 +1818,105 @@ def fft_lowpass_filter(data: np.ndarray, cut_off: float = 0.1) -> np.ndarray:
1813
1818
  return results.astype(data.dtype)
1814
1819
 
1815
1820
 
1816
- def terminate_cpu_pool(pool: Optional[multiprocessing.pool.Pool],
1817
- force: bool = False) -> None:
1821
+ def terminate_cpu_pool(pool: multiprocessing.pool.Pool,
1822
+ force: bool = False,
1823
+ verbose: bool = True,
1824
+ source: Optional[str] = None) -> None:
1818
1825
  """
1819
- Safely terminates a multiprocessing.Pool instance.
1826
+ Safely terminates a multiprocessing.Pool instance with optional graceful shutdown.
1820
1827
 
1821
- :param Optional[multiprocessing.pool.Pool] pool: The pool to terminate. If None, function returns without action.
1822
- :param bool force: If True, skips join() and immediately terminates. Default: False.
1823
- :raises InvalidInputError: If pool is not a valid Pool instance.
1828
+ .. note::
1829
+ If pool is None or invalid, function returns without action. Exceptions during termination are silently caught.
1830
+
1831
+ :param multiprocessing.pool.Pool pool: The multiprocessing pool to terminate. If None, function returns without action.
1832
+ :param bool force: If True, skips graceful shutdown (close/join) and immediately terminates. Default: False.
1833
+ :param bool verbose: If True, prints termination message with timestamp. Default: True.
1834
+ :param Optional[str] source: Optional identifier string for logging purposes (e.g., 'VideoProcessor'). Default: None.
1824
1835
 
1825
1836
  :example:
1826
1837
  >>> import multiprocessing
1827
1838
  >>> pool = multiprocessing.Pool(4)
1828
- >>> terminate_cpu_pool(pool)
1839
+ >>> terminate_cpu_pool(pool=pool, force=False, verbose=True, source='FeatureExtractor')
1829
1840
  """
1830
1841
  if pool is None:
1831
1842
  return
1832
- check_valid_cpu_pool(value=pool, source=terminate_cpu_pool.__name__, raise_error=True)
1843
+ if not check_valid_cpu_pool(value=pool, source=terminate_cpu_pool.__name__, raise_error=False):
1844
+ return
1833
1845
  try:
1846
+ core_cnt = pool._processes if hasattr(pool, '_processes') else None
1834
1847
  if not force:
1835
1848
  pool.close()
1836
1849
  pool.join()
1837
1850
  pool.terminate()
1851
+ if verbose: print(f'[{get_current_time()}] {"" if source is None else f"{core_cnt} core"} SimBA CPU pool {"" if source is None else source} terminated.')
1838
1852
  except (ValueError, AssertionError, AttributeError):
1839
1853
  pass
1840
1854
  gc.collect()
1841
1855
 
1842
1856
 
1857
+
1858
+ def get_cpu_pool(core_cnt: int = -1,
1859
+ maxtasksperchild: int = Defaults.MAXIMUM_MAX_TASK_PER_CHILD.value,
1860
+ context: Literal['fork', 'spawn', 'forkserver'] = None,
1861
+ verbose: bool = True,
1862
+ source: Optional[str] = None) -> multiprocessing.Pool:
1863
+ """
1864
+ Creates and returns a multiprocessing.Pool instance with platform-appropriate defaults and validation.
1865
+
1866
+ :param int core_cnt: Number of worker processes. -1 uses all available cores. Default: -1.
1867
+ :param int maxtasksperchild: Maximum number of tasks a worker process can complete before being replaced. Default: From Defaults.MAXIMUM_MAX_TASK_PER_CHILD.
1868
+ :param Optional[Literal['fork', 'spawn', 'forkserver']] context: Multiprocessing start method. None uses platform default. Default: None.
1869
+ :param bool verbose: If True, prints pool creation message with timestamp. Default: True.
1870
+ :param Optional[str] source: Optional identifier string for logging purposes (e.g., 'VideoProcessor'). Default: None.
1871
+ :return: Configured multiprocessing.Pool instance.
1872
+ :rtype: multiprocessing.Pool
1873
+
1874
+ :example:
1875
+ >>> pool = get_cpu_pool(core_cnt=4, source='FeatureExtractor')
1876
+ >>> pool = get_cpu_pool(core_cnt=-1, context='spawn', verbose=True)
1877
+ >>> pool = get_cpu_pool(core_cnt=8, maxtasksperchild=100, source='VideoProcessor')
1878
+ """
1879
+
1880
+ check_int(name=f'{get_cpu_pool.__name__} core_cnt', min_value=-1, unaccepted_vals=[0], value=core_cnt, raise_error=True)
1881
+ check_int(name=f'{get_cpu_pool.__name__} maxtasksperchild', min_value=1, value=maxtasksperchild, raise_error=True)
1882
+ check_valid_boolean(value=verbose, source=f'{get_cpu_pool.__name__} verbose', raise_error=True)
1883
+ if source is not None: check_str(name=f'{get_cpu_pool.__name__} source', value=source, raise_error=True, allow_blank=True)
1884
+ current_process = multiprocessing.current_process()
1885
+ if current_process.name != 'MainProcess': core_cnt = 1
1886
+ core_cnt = find_core_cnt()[0] if core_cnt == -1 or core_cnt > find_core_cnt()[0] else core_cnt
1887
+ if verbose: print(f'[{get_current_time()}] {core_cnt} core SimBA CPU pool {"" if source is None else source} started.')
1888
+ if context is not None:
1889
+ check_str(name=f'{get_cpu_pool.__name__} context', value=context, options=('fork', 'spawn', 'forkserver'), raise_error=True)
1890
+ else:
1891
+ existing_method = multiprocessing.get_start_method(allow_none=True)
1892
+ if existing_method is not None:
1893
+ context = existing_method
1894
+ else:
1895
+ system = platform.system()
1896
+ if system == OS.WINDOWS.value: context = OS.SPAWN.value
1897
+ elif system == OS.MAC.value: context = OS.SPAWN.value
1898
+ else: context = OS.FORK.value
1899
+
1900
+ if context is not None:
1901
+ try:
1902
+ ctx = multiprocessing.get_context(context)
1903
+ except ValueError:
1904
+ system = platform.system()
1905
+ if system == OS.WINDOWS.value: fallback_context = OS.SPAWN.value
1906
+ elif system == OS.MAC.value: fallback_context = OS.SPAWN.value
1907
+ else: fallback_context = OS.FORK.value
1908
+ try:
1909
+ ctx = multiprocessing.get_context(fallback_context)
1910
+ except ValueError:
1911
+ pool = multiprocessing.Pool(processes=core_cnt, maxtasksperchild=maxtasksperchild)
1912
+ return pool
1913
+ pool = ctx.Pool(processes=core_cnt, maxtasksperchild=maxtasksperchild)
1914
+ else:
1915
+ pool = multiprocessing.Pool(processes=core_cnt, maxtasksperchild=maxtasksperchild)
1916
+ return pool
1917
+
1918
+
1919
+ #get_cpu_pool()
1843
1920
  # run_user_defined_feature_extraction_class(config_path='/Users/simon/Desktop/envs/troubleshooting/circular_features_zebrafish/project_folder/project_config.ini', file_path='/Users/simon/Desktop/fish_feature_extractor_2023_version_5.py')
1844
1921
 
1845
1922
 
simba/utils/enums.py CHANGED
@@ -127,6 +127,7 @@ class OS(Enum):
127
127
  LINUX = "Linux"
128
128
  MAC = "Darwin"
129
129
  SPAWN = 'spawn'
130
+ FORK = 'fork'
130
131
  PYTHON_VER = str(f"{sys.version_info.major}.{sys.version_info.minor}")
131
132
  try:
132
133
  SIMBA_VERSION = pkg_resources.get_distribution("simba-uw-tf-dev").version