simba-uw-tf-dev 4.6.4__py3-none-any.whl → 4.6.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of simba-uw-tf-dev might be problematic. Click here for more details.

Files changed (33) hide show
  1. simba/data_processors/cuda/geometry.py +45 -27
  2. simba/data_processors/cuda/image.py +1620 -1600
  3. simba/data_processors/cuda/statistics.py +17 -9
  4. simba/data_processors/egocentric_aligner.py +24 -6
  5. simba/data_processors/kleinberg_calculator.py +6 -2
  6. simba/feature_extractors/feature_subsets.py +12 -5
  7. simba/feature_extractors/straub_tail_analyzer.py +0 -2
  8. simba/mixins/statistics_mixin.py +9 -2
  9. simba/sandbox/analyze_runtimes.py +30 -0
  10. simba/sandbox/cuda/egocentric_rotator.py +374 -374
  11. simba/sandbox/proboscis_to_tip.py +28 -0
  12. simba/sandbox/test_directionality.py +47 -0
  13. simba/sandbox/test_nonstatic_directionality.py +27 -0
  14. simba/sandbox/test_pycharm_cuda.py +51 -0
  15. simba/sandbox/test_simba_install.py +41 -0
  16. simba/sandbox/test_static_directionality.py +26 -0
  17. simba/sandbox/test_static_directionality_2d.py +26 -0
  18. simba/sandbox/verify_env.py +42 -0
  19. simba/ui/pop_ups/fsttc_pop_up.py +27 -25
  20. simba/ui/pop_ups/kleinberg_pop_up.py +3 -2
  21. simba/utils/data.py +0 -1
  22. simba/utils/errors.py +441 -440
  23. simba/utils/lookups.py +1203 -1203
  24. simba/utils/read_write.py +38 -13
  25. simba/video_processors/egocentric_video_rotator.py +41 -36
  26. simba/video_processors/video_processing.py +5247 -5233
  27. simba/video_processors/videos_to_frames.py +41 -31
  28. {simba_uw_tf_dev-4.6.4.dist-info → simba_uw_tf_dev-4.6.6.dist-info}/METADATA +2 -2
  29. {simba_uw_tf_dev-4.6.4.dist-info → simba_uw_tf_dev-4.6.6.dist-info}/RECORD +33 -24
  30. {simba_uw_tf_dev-4.6.4.dist-info → simba_uw_tf_dev-4.6.6.dist-info}/LICENSE +0 -0
  31. {simba_uw_tf_dev-4.6.4.dist-info → simba_uw_tf_dev-4.6.6.dist-info}/WHEEL +0 -0
  32. {simba_uw_tf_dev-4.6.4.dist-info → simba_uw_tf_dev-4.6.6.dist-info}/entry_points.txt +0 -0
  33. {simba_uw_tf_dev-4.6.4.dist-info → simba_uw_tf_dev-4.6.6.dist-info}/top_level.txt +0 -0
@@ -1,1600 +1,1620 @@
1
- __author__ = "Simon Nilsson; sronilsson@gmail.com"
2
-
3
-
4
- import math
5
- import multiprocessing as mp
6
- import os
7
- import time
8
- from typing import Optional, Tuple, Union
9
-
10
- try:
11
- from typing import Literal
12
- except:
13
- from typing_extensions import Literal
14
- try:
15
- import cupy as cp
16
- from cupyx.scipy.ndimage import rotate
17
- except:
18
- import numpy as cp
19
- from scipy.ndimage import rotate
20
-
21
- import platform
22
- import warnings
23
- from copy import deepcopy
24
-
25
- import cv2
26
- import numpy as np
27
- from numba import cuda
28
- from numba.core.errors import NumbaPerformanceWarning
29
-
30
- from simba.data_processors.cuda.utils import (_cuda_luminance_pixel_to_grey,
31
- _cuda_mse, _is_cuda_available)
32
- from simba.mixins.image_mixin import ImageMixin
33
- from simba.mixins.plotting_mixin import PlottingMixin
34
- from simba.utils.checks import (check_file_exist_and_readable, check_float,
35
- check_if_dir_exists,
36
- check_if_string_value_is_valid_video_timestamp,
37
- check_if_valid_img, check_if_valid_rgb_tuple,
38
- check_instance, check_int,
39
- check_nvidea_gpu_available,
40
- check_that_hhmmss_start_is_before_end,
41
- check_valid_array, check_valid_boolean,
42
- is_video_color)
43
- from simba.utils.data import (create_color_palette,
44
- find_frame_numbers_from_time_stamp)
45
- from simba.utils.enums import OS, Formats
46
- from simba.utils.errors import (FFMPEGCodecGPUError, FrameRangeError,
47
- InvalidInputError, SimBAGPUError)
48
- from simba.utils.lookups import get_current_time
49
- from simba.utils.printing import SimbaTimer, stdout_success
50
- from simba.utils.read_write import (
51
- check_if_hhmmss_timestamp_is_valid_part_of_video,
52
- concatenate_videos_in_folder, create_directory, get_fn_ext,
53
- get_memory_usage_array, get_video_meta_data, read_df, read_img,
54
- read_img_batch_from_video, read_img_batch_from_video_gpu)
55
- from simba.video_processors.async_frame_reader import (AsyncVideoFrameReader,
56
- get_async_frame_batch)
57
-
58
- warnings.simplefilter('ignore', category=NumbaPerformanceWarning)
59
-
60
-
61
- PHOTOMETRIC = 'photometric'
62
- DIGITAL = 'digital'
63
- THREADS_PER_BLOCK = 2024
64
- if platform.system() != OS.WINDOWS.value: mp.set_start_method("spawn", force=True)
65
-
66
- def create_average_frm_cupy(video_path: Union[str, os.PathLike],
67
- start_frm: Optional[int] = None,
68
- end_frm: Optional[int] = None,
69
- start_time: Optional[str] = None,
70
- end_time: Optional[str] = None,
71
- save_path: Optional[Union[str, os.PathLike]] = None,
72
- batch_size: Optional[int] = 3000,
73
- verbose: Optional[bool] = False,
74
- async_frame_read: bool = False) -> Union[None, np.ndarray]:
75
-
76
- """
77
- Computes the average frame using GPU acceleration from a specified range of frames or time interval in a video file.
78
- This average frame is typically used for background subtraction.
79
-
80
- The function reads frames from the video, calculates their average, and optionally saves the result
81
- to a specified file. If `save_path` is provided, the average frame is saved as an image file;
82
- otherwise, the average frame is returned as a NumPy array.
83
-
84
- .. seealso::
85
- For CPU function see :func:`~simba.video_processors.video_processing.create_average_frm`.
86
- For CUDA function see :func:`~simba.data_processors.cuda.image.create_average_frm_cuda`
87
-
88
-
89
- .. csv-table::
90
- :header: EXPECTED RUNTIMES
91
- :file: ../../../docs/tables/create_average_frm_cupy.csv
92
- :widths: 10, 45, 45
93
- :align: center
94
- :class: simba-table
95
- :header-rows: 1
96
-
97
- :param Union[str, os.PathLike] video_path: The path to the video file from which to extract frames.
98
- :param Optional[int] start_frm: The starting frame number (inclusive). Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both. If both `start_frm` and `end_frm` are `None`, processes all frames in the video.
99
- :param Optional[int] end_frm: The ending frame number (exclusive). Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both.
100
- :param Optional[str] start_time: The start time in the format 'HH:MM:SS' from which to begin extracting frames. Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both.
101
- :param Optional[str] end_time: The end time in the format 'HH:MM:SS' up to which frames should be extracted. Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both.
102
- :param Optional[Union[str, os.PathLike]] save_path: The path where the average frame image will be saved. If `None`, the average frame is returned as a NumPy array.
103
- :param Optional[int] batch_size: The number of frames to process in each batch. Default is 3000. Increase if your RAM allows it.
104
- :param Optional[bool] verbose: If `True`, prints progress and informational messages during execution. Default: False.
105
- :param bool async_frame_read: If `True`, uses asynchronous frame reading for improved performance. Default: False.
106
- :return: Returns `None` if the result is saved to `save_path`. Otherwise, returns the average frame as a NumPy array.
107
-
108
- :example:
109
- >>> create_average_frm_cupy(video_path=r"C:/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_DOT_4_downsampled.mp4", verbose=True, start_frm=0, end_frm=9000)
110
- >>> create_average_frm_cupy(video_path=r"C:/videos/my_video.mp4", start_time="00:00:00", end_time="00:01:00", async_frame_read=True, save_path=r"C:/output/avg_frame.png")
111
-
112
- """
113
-
114
- def average_3d_stack(image_stack: np.ndarray) -> np.ndarray:
115
- num_frames, height, width, _ = image_stack.shape
116
- image_stack = cp.array(image_stack).astype(cp.float32)
117
- img = cp.clip(cp.sum(image_stack, axis=0) / num_frames, 0, 255).astype(cp.uint8)
118
- return img.get()
119
-
120
- if not check_nvidea_gpu_available():
121
- raise FFMPEGCodecGPUError(msg="No GPU found (as evaluated by nvidea-smi returning None)", source=create_average_frm_cupy.__name__)
122
-
123
- timer = SimbaTimer(start=True)
124
- if ((start_frm is not None) or (end_frm is not None)) and ((start_time is not None) or (end_time is not None)):
125
- raise InvalidInputError(msg=f'Pass start_frm and end_frm OR start_time and end_time', source=create_average_frm_cupy.__name__)
126
- elif type(start_frm) != type(end_frm):
127
- raise InvalidInputError(msg=f'Pass start frame and end frame', source=create_average_frm_cupy.__name__)
128
- elif type(start_time) != type(end_time):
129
- raise InvalidInputError(msg=f'Pass start time and end time', source=create_average_frm_cupy.__name__)
130
- if save_path is not None:
131
- check_if_dir_exists(in_dir=os.path.dirname(save_path), source=create_average_frm_cupy.__name__)
132
- check_file_exist_and_readable(file_path=video_path)
133
- video_meta_data = get_video_meta_data(video_path=video_path)
134
- video_name = get_fn_ext(filepath=video_path)[1]
135
- if verbose:
136
- print(f'Getting average frame from {video_name}...')
137
- if (start_frm is not None) and (end_frm is not None):
138
- check_int(name='start_frm', value=start_frm, min_value=0, max_value=video_meta_data['frame_count'])
139
- check_int(name='end_frm', value=end_frm, min_value=0, max_value=video_meta_data['frame_count'])
140
- if start_frm > end_frm:
141
- raise InvalidInputError(msg=f'Start frame ({start_frm}) has to be before end frame ({end_frm}).', source=create_average_frm_cupy.__name__)
142
- frame_ids_lst = list(range(start_frm, end_frm))
143
- elif (start_time is not None) and (end_time is not None):
144
- check_if_string_value_is_valid_video_timestamp(value=start_time, name=create_average_frm_cupy.__name__)
145
- check_if_string_value_is_valid_video_timestamp(value=end_time, name=create_average_frm_cupy.__name__)
146
- check_that_hhmmss_start_is_before_end(start_time=start_time, end_time=end_time, name=create_average_frm_cupy.__name__)
147
- check_if_hhmmss_timestamp_is_valid_part_of_video(timestamp=start_time, video_path=video_path)
148
- frame_ids_lst = find_frame_numbers_from_time_stamp(start_time=start_time, end_time=end_time, fps=video_meta_data['fps'])
149
- else:
150
- frame_ids_lst = list(range(0, video_meta_data['frame_count']))
151
- frame_ids = [frame_ids_lst[i:i+batch_size] for i in range(0,len(frame_ids_lst),batch_size)]
152
- avg_imgs = []
153
- if async_frame_read:
154
- async_frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=5, start_idx=int(min(frame_ids_lst)), end_idx=int(max(frame_ids_lst))+1, verbose=True, gpu=True)
155
- async_frm_reader.start()
156
- else:
157
- async_frm_reader = None
158
- for batch_cnt in range(len(frame_ids)):
159
- start_idx, end_idx = frame_ids[batch_cnt][0], frame_ids[batch_cnt][-1]
160
- if start_idx == end_idx:
161
- continue
162
- if not async_frm_reader:
163
- imgs = read_img_batch_from_video_gpu(video_path=video_path, start_frm=start_idx, end_frm=end_idx, verbose=verbose)
164
- imgs = np.stack(list(imgs.values()), axis=0)
165
- else:
166
- imgs = get_async_frame_batch(batch_reader=async_frm_reader, timeout=15)[2]
167
- avg_imgs.append(average_3d_stack(image_stack=imgs))
168
- avg_img = average_3d_stack(image_stack=np.stack(avg_imgs, axis=0))
169
- timer.stop_timer()
170
- if async_frm_reader is not None: async_frm_reader.kill()
171
- if save_path is not None:
172
- cv2.imwrite(save_path, avg_img)
173
- if verbose:
174
- stdout_success(msg=f'Saved average frame at {save_path}', source=create_average_frm_cupy.__name__, elapsed_time=timer.elapsed_time_str)
175
- else:
176
- if verbose: stdout_success(msg=f'Average frame compute complete', source=create_average_frm_cupy.__name__, elapsed_time=timer.elapsed_time_str)
177
- return avg_img
178
-
179
- def average_3d_stack_cupy(image_stack: np.ndarray) -> np.ndarray:
180
- num_frames, height, width, _ = image_stack.shape
181
- image_stack = cp.array(image_stack).astype(cp.float32)
182
- img = cp.clip(cp.sum(image_stack, axis=0) / num_frames, 0, 255).astype(cp.uint8)
183
- return img.get()
184
-
185
- @cuda.jit()
186
- def _average_3d_stack_cuda_kernel(data, results):
187
- x, y, i = cuda.grid(3)
188
- if i < 0 or x < 0 or y < 0:
189
- return
190
- if i > data.shape[0] - 1 or y > data.shape[1] - 1 or x > data.shape[2] - 1:
191
- return
192
- else:
193
- sum_value = 0.0
194
- for n in range(data.shape[0]):
195
- sum_value += data[n, y, x, i]
196
- results[y, x, i] = sum_value / data.shape[0]
197
-
198
- def _average_3d_stack_cuda(image_stack: np.ndarray) -> np.ndarray:
199
- check_instance(source=_average_3d_stack_cuda.__name__, instance=image_stack, accepted_types=(np.ndarray,))
200
- check_if_valid_img(data=image_stack[0], source=_average_3d_stack_cuda.__name__)
201
- if image_stack.ndim != 4:
202
- return image_stack
203
- x = np.ascontiguousarray(image_stack)
204
- x_dev = cuda.to_device(x)
205
- results = cuda.device_array((x.shape[1], x.shape[2], x.shape[3]), dtype=np.float32)
206
- grid_x = (x.shape[1] + 16 - 1) // 16
207
- grid_y = (x.shape[2] + 16 - 1) // 16
208
- grid_z = 3
209
- threads_per_block = (16, 16, 1)
210
- blocks_per_grid = (grid_y, grid_x, grid_z)
211
- _average_3d_stack_cuda_kernel[blocks_per_grid, threads_per_block](x_dev, results)
212
- results = results.copy_to_host()
213
- return results
214
-
215
-
216
- def create_average_frm_cuda(video_path: Union[str, os.PathLike],
217
- start_frm: Optional[int] = None,
218
- end_frm: Optional[int] = None,
219
- start_time: Optional[str] = None,
220
- end_time: Optional[str] = None,
221
- save_path: Optional[Union[str, os.PathLike]] = None,
222
- batch_size: Optional[int] = 6000,
223
- verbose: Optional[bool] = False,
224
- async_frame_read: bool = False) -> Union[None, np.ndarray]:
225
- """
226
- Computes the average frame using GPU acceleration from a specified range of frames or time interval in a video file.
227
- This average frame typically used for background substraction.
228
-
229
-
230
- The function reads frames from the video, calculates their average, and optionally saves the result
231
- to a specified file. If `save_path` is provided, the average frame is saved as an image file;
232
- otherwise, the average frame is returned as a NumPy array.
233
-
234
- .. seealso::
235
- For CuPy function see :func:`~simba.data_processors.cuda.image.create_average_frm_cupy`.
236
- For CPU function see :func:`~simba.video_processors.video_processing.create_average_frm`.
237
-
238
- :param Union[str, os.PathLike] video_path: The path to the video file from which to extract frames.
239
- :param Optional[int] start_frm: The starting frame number (inclusive). Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both.
240
- :param Optional[int] end_frm: The ending frame number (exclusive).
241
- :param Optional[str] start_time: The start time in the format 'HH:MM:SS' from which to begin extracting frames.
242
- :param Optional[str] end_time: The end time in the format 'HH:MM:SS' up to which frames should be extracted.
243
- :param Optional[Union[str, os.PathLike]] save_path: The path where the average frame image will be saved. If `None`, the average frame is returned as a NumPy array.
244
- :param Optional[int] batch_size: The number of frames to process in each batch. Default is 3000. Increase if your RAM allows it.
245
- :param Optional[bool] verbose: If `True`, prints progress and informational messages during execution.
246
- :return: Returns `None` if the result is saved to `save_path`. Otherwise, returns the average frame as a NumPy array.
247
-
248
- :example:
249
- >>> create_average_frm_cuda(video_path=r"C:/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_DOT_4_downsampled.mp4", verbose=True, start_frm=0, end_frm=9000)
250
-
251
- """
252
-
253
- if not check_nvidea_gpu_available():
254
- raise FFMPEGCodecGPUError(msg="No GPU found (as evaluated by nvidea-smi returning None)", source=create_average_frm_cuda.__name__)
255
-
256
- if ((start_frm is not None) or (end_frm is not None)) and ((start_time is not None) or (end_time is not None)):
257
- raise InvalidInputError(msg=f'Pass start_frm and end_frm OR start_time and end_time', source=create_average_frm_cuda.__name__)
258
- elif type(start_frm) != type(end_frm):
259
- raise InvalidInputError(msg=f'Pass start frame and end frame', source=create_average_frm_cuda.__name__)
260
- elif type(start_time) != type(end_time):
261
- raise InvalidInputError(msg=f'Pass start time and end time', source=create_average_frm_cuda.__name__)
262
- if save_path is not None:
263
- check_if_dir_exists(in_dir=os.path.dirname(save_path), source=create_average_frm_cuda.__name__)
264
- check_file_exist_and_readable(file_path=video_path)
265
- video_meta_data = get_video_meta_data(video_path=video_path)
266
- video_name = get_fn_ext(filepath=video_path)[1]
267
- if verbose:
268
- print(f'Getting average frame from {video_name}...')
269
- if (start_frm is not None) and (end_frm is not None):
270
- check_int(name='start_frm', value=start_frm, min_value=0, max_value=video_meta_data['frame_count'])
271
- check_int(name='end_frm', value=end_frm, min_value=0, max_value=video_meta_data['frame_count'])
272
- if start_frm > end_frm:
273
- raise InvalidInputError(msg=f'Start frame ({start_frm}) has to be before end frame ({end_frm}).', source=create_average_frm_cuda.__name__)
274
- frame_ids_lst = list(range(start_frm, end_frm))
275
- elif (start_time is not None) and (end_time is not None):
276
- check_if_string_value_is_valid_video_timestamp(value=start_time, name=create_average_frm_cuda.__name__)
277
- check_if_string_value_is_valid_video_timestamp(value=end_time, name=create_average_frm_cuda.__name__)
278
- check_that_hhmmss_start_is_before_end(start_time=start_time, end_time=end_time, name=create_average_frm_cuda.__name__)
279
- check_if_hhmmss_timestamp_is_valid_part_of_video(timestamp=start_time, video_path=video_path)
280
- frame_ids_lst = find_frame_numbers_from_time_stamp(start_time=start_time, end_time=end_time, fps=video_meta_data['fps'])
281
- else:
282
- frame_ids_lst = list(range(0, video_meta_data['frame_count']))
283
- frame_ids = [frame_ids_lst[i:i + batch_size] for i in range(0, len(frame_ids_lst), batch_size)]
284
- avg_imgs = []
285
- if async_frame_read:
286
- async_frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=5, start_idx=int(min(frame_ids_lst)), end_idx=int(max(frame_ids_lst))+1, verbose=True, gpu=True)
287
- async_frm_reader.start()
288
- else:
289
- async_frm_reader = None
290
- for batch_cnt in range(len(frame_ids)):
291
- start_idx, end_idx = frame_ids[batch_cnt][0], frame_ids[batch_cnt][-1]
292
- if start_idx == end_idx:
293
- continue
294
- if not async_frm_reader:
295
- imgs = read_img_batch_from_video_gpu(video_path=video_path, start_frm=start_idx, end_frm=end_idx, verbose=verbose)
296
- avg_imgs.append(_average_3d_stack_cuda(image_stack=np.stack(list(imgs.values()), axis=0)))
297
- else:
298
- imgs = get_async_frame_batch(batch_reader=async_frm_reader, timeout=15)[2]
299
- avg_imgs.append(_average_3d_stack_cuda(image_stack=imgs))
300
- avg_img = average_3d_stack_cupy(image_stack=np.stack(avg_imgs, axis=0))
301
- if save_path is not None:
302
- cv2.imwrite(save_path, avg_img)
303
- if verbose:
304
- stdout_success(msg=f'Saved average frame at {save_path}', source=create_average_frm_cuda.__name__)
305
- else:
306
- return avg_img
307
-
308
-
309
-
310
- @cuda.jit()
311
- def _photometric(data, results):
312
- y, x, i = cuda.grid(3)
313
- if i < 0 or x < 0 or y < 0:
314
- return
315
- if i > results.shape[0] - 1 or x > results.shape[1] - 1 or y > results.shape[2] - 1:
316
- return
317
- else:
318
- r, g, b = data[i][x][y][0], data[i][x][y][1], data[i][x][y][2]
319
- results[i][x][y] = (0.2126 * r) + (0.7152 * g) + (0.0722 * b)
320
-
321
- @cuda.jit()
322
- def _digital(data, results):
323
- y, x, i = cuda.grid(3)
324
- if i < 0 or x < 0 or y < 0:
325
- return
326
- if i > results.shape[0] - 1 or x > results.shape[1] - 1 or y > results.shape[2] - 1:
327
- return
328
- else:
329
- r, g, b = data[i][x][y][0], data[i][x][y][1], data[i][x][y][2]
330
- results[i][x][y] = (0.299 * r) + (0.587 * g) + (0.114 * b)
331
-
332
- def img_stack_brightness(x: np.ndarray,
333
- method: Optional[Literal['photometric', 'digital']] = 'digital',
334
- ignore_black: Optional[bool] = True) -> np.ndarray:
335
- """
336
- Calculate the average brightness of a stack of images using a specified method.
337
-
338
-
339
- - **Photometric Method**: The brightness is calculated using the formula:
340
-
341
- .. math::
342
- \text{brightness} = 0.2126 \cdot R + 0.7152 \cdot G + 0.0722 \cdot B
343
-
344
- - **Digital Method**: The brightness is calculated using the formula:
345
-
346
- .. math::
347
- \text{brightness} = 0.299 \cdot R + 0.587 \cdot G + 0.114 \cdot B
348
-
349
- .. selalso::
350
- For CPU function see :func:`~simba.mixins.image_mixin.ImageMixin.brightness_intensity`.
351
-
352
- :param np.ndarray x: A 4D array of images with dimensions (N, H, W, C), where N is the number of images, H and W are the height and width, and C is the number of channels (RGB).
353
- :param Optional[Literal['photometric', 'digital']] method: The method to use for calculating brightness. It can be 'photometric' for the standard luminance calculation or 'digital' for an alternative set of coefficients. Default is 'digital'.
354
- :param Optional[bool] ignore_black: If True, black pixels (i.e., pixels with brightness value 0) will be ignored in the calculation of the average brightness. Default is True.
355
- :return np.ndarray: A 1D array of average brightness values for each image in the stack. If `ignore_black` is True, black pixels are ignored in the averaging process.
356
-
357
-
358
- :example:
359
- >>> imgs = read_img_batch_from_video_gpu(video_path=r"/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_DOT_4_downsampled.mp4", start_frm=0, end_frm=5000)
360
- >>> imgs = np.stack(list(imgs.values()), axis=0)
361
- >>> x = img_stack_brightness(x=imgs)
362
- """
363
-
364
- check_instance(source=img_stack_brightness.__name__, instance=x, accepted_types=(np.ndarray,))
365
- check_if_valid_img(data=x[0], source=img_stack_brightness.__name__)
366
- x = np.ascontiguousarray(x).astype(np.uint8)
367
- if x.ndim == 4:
368
- grid_x = (x.shape[1] + 16 - 1) // 16
369
- grid_y = (x.shape[2] + 16 - 1) // 16
370
- grid_z = x.shape[0]
371
- threads_per_block = (16, 16, 1)
372
- blocks_per_grid = (grid_y, grid_x, grid_z)
373
- x_dev = cuda.to_device(x)
374
- results = cuda.device_array((x.shape[0], x.shape[1], x.shape[2]), dtype=np.uint8)
375
- if method == PHOTOMETRIC:
376
- _photometric[blocks_per_grid, threads_per_block](x_dev, results)
377
- else:
378
- _digital[blocks_per_grid, threads_per_block](x_dev, results)
379
- results = results.copy_to_host()
380
- if ignore_black:
381
- masked_array = np.ma.masked_equal(results, 0)
382
- results = np.mean(masked_array, axis=(1, 2)).filled(0)
383
- else:
384
- results = deepcopy(x)
385
- results = np.mean(results, axis=(1, 2))
386
-
387
- return results
388
-
389
-
390
-
391
- @cuda.jit()
392
- def _grey_mse(data, ref_img, stride, batch_cnt, mse_arr):
393
- y, x, i = cuda.grid(3)
394
- stride = stride[0]
395
- batch_cnt = batch_cnt[0]
396
- if batch_cnt == 0:
397
- if (i - stride) < 0 or x < 0 or y < 0:
398
- return
399
- else:
400
- if i < 0 or x < 0 or y < 0:
401
- return
402
- if i > mse_arr.shape[0] - 1 or x > mse_arr.shape[1] - 1 or y > mse_arr.shape[2] - 1:
403
- return
404
- else:
405
- img_val = data[i][x][y]
406
- if i == 0:
407
- prev_val = ref_img[x][y]
408
- else:
409
- img_val = data[i][x][y]
410
- prev_val = data[i - stride][x][y]
411
- mse_arr[i][x][y] = (img_val - prev_val) ** 2
412
-
413
-
414
- @cuda.jit()
415
- def _rgb_mse(data, ref_img, stride, batch_cnt, mse_arr):
416
- y, x, i = cuda.grid(3)
417
- stride = stride[0]
418
- batch_cnt = batch_cnt[0]
419
- if batch_cnt == 0:
420
- if (i - stride) < 0 or x < 0 or y < 0:
421
- return
422
- else:
423
- if i < 0 or x < 0 or y < 0:
424
- return
425
- if i > mse_arr.shape[0] - 1 or x > mse_arr.shape[1] - 1 or y > mse_arr.shape[2] - 1:
426
- return
427
- else:
428
- img_val = data[i][x][y]
429
- if i != 0:
430
- prev_val = data[i - stride][x][y]
431
- else:
432
- prev_val = ref_img[x][y]
433
- r_diff = (img_val[0] - prev_val[0]) ** 2
434
- g_diff = (img_val[1] - prev_val[1]) ** 2
435
- b_diff = (img_val[2] - prev_val[2]) ** 2
436
- mse_arr[i][x][y] = r_diff + g_diff + b_diff
437
-
438
- def stack_sliding_mse(x: np.ndarray,
439
- stride: Optional[int] = 1,
440
- batch_size: Optional[int] = 1000) -> np.ndarray:
441
- r"""
442
- Computes the Mean Squared Error (MSE) between each image in a stack and a reference image,
443
- where the reference image is determined by a sliding window approach with a specified stride.
444
- The function is optimized for large image stacks by processing them in batches.
445
-
446
- .. seealso::
447
- For CPU function see :func:`~simba.mixins.image_mixin.ImageMixin.img_stack_mse` and
448
- :func:`~simba.mixins.image_mixin.ImageMixin.img_sliding_mse`.
449
-
450
- .. math::
451
-
452
- \text{MSE} = \frac{1}{n} \sum_{i=1}^{n} (y_i - \hat{y}_i)^2
453
-
454
- :param np.ndarray x: Input array of images, where the first dimension corresponds to the stack of images. The array should be either 3D (height, width, channels) or 4D (batch, height, width, channels).
455
- :param Optional[int] stride: The stride or step size for the sliding window that determines the reference image. Defaults to 1, meaning the previous image in the stack is used as the reference.
456
- :param Optional[int] batch_size: The number of images to process in a single batch. Larger batch sizes may improve performance but require more GPU memory. Defaults to 1000.
457
- :return: A 1D NumPy array containing the MSE for each image in the stack compared to its corresponding reference image. The length of the array is equal to the number of images in the input stack.
458
- :rtype: np.ndarray
459
-
460
- """
461
-
462
- check_instance(source=stack_sliding_mse.__name__, instance=x, accepted_types=(np.ndarray,))
463
- check_if_valid_img(data=x[0], source=stack_sliding_mse.__name__)
464
- check_valid_array(data=x, source=stack_sliding_mse.__name__, accepted_ndims=[3, 4])
465
- stride = np.array([stride], dtype=np.int32)
466
- stride_dev = cuda.to_device(stride)
467
- out = np.full((x.shape[0]), fill_value=0.0, dtype=np.float32)
468
- for batch_cnt, l in enumerate(range(0, x.shape[0], batch_size)):
469
- r = l + batch_size
470
- batch_x = x[l:r]
471
- if batch_cnt != 0:
472
- if x.ndim == 3:
473
- ref_img = x[l-stride].astype(np.uint8).reshape(x.shape[1], x.shape[2])
474
- else:
475
- ref_img = x[l-stride].astype(np.uint8).reshape(x.shape[1], x.shape[2], 3)
476
- else:
477
- ref_img = np.full_like(x[l], dtype=np.uint8, fill_value=0)
478
- ref_img = ref_img.astype(np.uint8)
479
- grid_x = (batch_x.shape[1] + 16 - 1) // 16
480
- grid_y = (batch_x.shape[2] + 16 - 1) // 16
481
- grid_z = batch_x.shape[0]
482
- threads_per_block = (16, 16, 1)
483
- blocks_per_grid = (grid_y, grid_x, grid_z)
484
- ref_img_dev = cuda.to_device(ref_img)
485
- x_dev = cuda.to_device(batch_x)
486
- results = cuda.device_array((batch_x.shape[0], batch_x.shape[1], batch_x.shape[2]), dtype=np.uint8)
487
- batch_cnt_dev = np.array([batch_cnt], dtype=np.int32)
488
- if x.ndim == 3:
489
- _grey_mse[blocks_per_grid, threads_per_block](x_dev, ref_img_dev, stride_dev, batch_cnt_dev, results)
490
- else:
491
- _rgb_mse[blocks_per_grid, threads_per_block](x_dev, ref_img_dev, stride_dev, batch_cnt_dev, results)
492
- results = results.copy_to_host()
493
- results = np.mean(results, axis=(1, 2))
494
- out[l:r] = results
495
- return out
496
-
497
-
498
- def img_stack_to_grayscale_cupy(imgs: Union[np.ndarray, cp.ndarray],
499
- batch_size: Optional[int] = 250) -> np.ndarray:
500
- """
501
- Converts a stack of color images to grayscale using GPU acceleration with CuPy.
502
-
503
- .. seealso::
504
- For CPU function single images :func:`~simba.mixins.image_mixin.ImageMixin.img_to_greyscale` and
505
- :func:`~simba.mixins.image_mixin.ImageMixin.img_stack_to_greyscale` for stack. For CUDA JIT, see
506
- :func:`~simba.data_processors.cuda.image.img_stack_to_grayscale_cuda`.
507
-
508
- .. csv-table::
509
- :header: EXPECTED RUNTIMES
510
- :file: ../../../docs/tables/img_stack_to_grayscale_cupy.csv
511
- :widths: 10, 90
512
- :align: center
513
- :class: simba-table
514
- :header-rows: 1
515
-
516
- :param np.ndarray imgs: A 4D NumPy or CuPy array representing a stack of images with shape (num_images, height, width, channels). The images are expected to have 3 channels (RGB).
517
- :param Optional[int] batch_size: The number of images to process in each batch. Defaults to 250. Adjust this parameter to fit your GPU's memory capacity.
518
- :return np.ndarray: m A 3D NumPy or CuPy array of shape (num_images, height, width) containing the grayscale images. If the input array is not 4D, the function returns the input as is.
519
-
520
- :example:
521
- >>> imgs = read_img_batch_from_video_gpu(video_path=r"/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_IOT_1_cropped.mp4", verbose=False, start_frm=0, end_frm=i)
522
- >>> imgs = np.stack(list(imgs.values()), axis=0).astype(np.uint8)
523
- >>> gray_imgs = img_stack_to_grayscale_cupy(imgs=imgs)
524
- """
525
-
526
-
527
- check_instance(source=img_stack_to_grayscale_cupy.__name__, instance=imgs, accepted_types=(np.ndarray, cp.ndarray))
528
- check_if_valid_img(data=imgs[0], source=img_stack_to_grayscale_cupy.__name__)
529
- if imgs.ndim != 4:
530
- return imgs
531
- results = cp.zeros((imgs.shape[0], imgs.shape[1], imgs.shape[2]), dtype=np.uint8)
532
- n = int(np.ceil((imgs.shape[0] / batch_size)))
533
- imgs = np.array_split(imgs, n)
534
- start = 0
535
- for i in range(len(imgs)):
536
- img_batch = cp.array(imgs[i])
537
- batch_cnt = img_batch.shape[0]
538
- end = start + batch_cnt
539
- vals = (0.07 * img_batch[:, :, :, 2] + 0.72 * img_batch[:, :, :, 1] + 0.21 * img_batch[:, :, :, 0])
540
- results[start:end] = vals.astype(cp.uint8)
541
- start = end
542
- if isinstance(imgs, np.ndarray):
543
- return results.get()
544
- else:
545
- return results
546
-
547
-
548
-
549
- @cuda.jit()
550
- def _img_stack_to_grayscale(data, results):
551
- y, x, i = cuda.grid(3)
552
- if i < 0 or x < 0 or y < 0:
553
- return
554
- if i > results.shape[0] - 1 or x > results.shape[1] - 1 or y > results.shape[2] - 1:
555
- return
556
- else:
557
- b = 0.07 * data[i][x][y][2]
558
- g = 0.72 * data[i][x][y][1]
559
- r = 0.21 * data[i][x][y][0]
560
- val = b + g + r
561
- results[i][x][y] = val
562
-
563
- def img_stack_to_grayscale_cuda(x: np.ndarray) -> np.ndarray:
564
- """
565
- Convert image stack to grayscale using CUDA.
566
-
567
- .. seealso::
568
- For CPU function single images :func:`~simba.mixins.image_mixin.ImageMixin.img_to_greyscale` and
569
- :func:`~simba.mixins.image_mixin.ImageMixin.img_stack_to_greyscale` for stack. For CuPy, see
570
- :func:`~simba.data_processors.cuda.image.img_stack_to_grayscale_cupy`.
571
-
572
- .. csv-table::
573
- :header: EXPECTED RUNTIMES
574
- :file: ../../../docs/tables/img_stack_to_grayscale_cuda.csv
575
- :widths: 10, 45, 45
576
- :align: center
577
- :class: simba-table
578
- :header-rows: 1
579
-
580
- :param np.ndarray x: 4d array of color images in numpy format.
581
- :return np.ndarray: 3D array of greyscaled images.
582
-
583
- :example:
584
- >>> imgs = read_img_batch_from_video_gpu(video_path=r"/mnt/c/troubleshooting/mitra/project_folder/videos/temp_2/592_MA147_Gq_Saline_0516_downsampled.mp4", verbose=False, start_frm=0, end_frm=i)
585
- >>> imgs = np.stack(list(imgs.values()), axis=0).astype(np.uint8)
586
- >>> grey_images = img_stack_to_grayscale_cuda(x=imgs)
587
- """
588
- check_instance(source=img_stack_to_grayscale_cuda.__name__, instance=x, accepted_types=(np.ndarray,))
589
- check_if_valid_img(data=x[0], source=img_stack_to_grayscale_cuda.__name__)
590
- if x.ndim != 4:
591
- return x
592
- x = np.ascontiguousarray(x).astype(np.uint8)
593
- x_dev = cuda.to_device(x)
594
- results = cuda.device_array((x.shape[0], x.shape[1], x.shape[2]), dtype=np.uint8)
595
- grid_x = (x.shape[1] + 16 - 1) // 16
596
- grid_y = (x.shape[2] + 16 - 1) // 16
597
- grid_z = x.shape[0]
598
- threads_per_block = (16, 16, 1)
599
- blocks_per_grid = (grid_y, grid_x, grid_z)
600
- _img_stack_to_grayscale[blocks_per_grid, threads_per_block](x_dev, results)
601
- results = results.copy_to_host()
602
- return results
603
-
604
-
605
- def img_stack_to_bw(imgs: np.ndarray,
606
- lower_thresh: Optional[int] = 100,
607
- upper_thresh: Optional[int] = 100,
608
- invert: Optional[bool] = True,
609
- batch_size: Optional[int] = 1000) -> np.ndarray:
610
- """
611
-
612
- Converts a stack of RGB images to binary (black and white) images based on given threshold values using GPU acceleration.
613
-
614
- This function processes a 4D stack of images, converting each RGB image to a binary image using
615
- specified lower and upper threshold values. The conversion can be inverted if desired, and the
616
- processing is done in batches for efficiency.
617
-
618
- .. csv-table::
619
- :header: EXPECTED RUNTIMES
620
- :file: ../../../docs/tables/img_stack_to_bw.csv
621
- :widths: 10, 90
622
- :align: center
623
- :header-rows: 1
624
-
625
- .. seealso::
626
- :func:`simba.mixins.image_mixin.ImageMixin.img_to_bw`
627
- :func:`simba.mixins.image_mixin.ImageMixin.img_stack_to_bw`
628
-
629
- :param np.ndarray imgs: A 4D NumPy array representing a stack of RGB images, with shape (N, H, W, C).
630
- :param Optional[int] lower_thresh: The lower threshold value. Pixel values below this threshold are set to 0 (or 1 if `invert` is True). Default is 100.
631
- :param Optional[int] upper_thresh: The upper threshold value. Pixel values above this threshold are set to 1 (or 0 if `invert` is True). Default is 100.
632
- :param Optional[bool] invert: If True, the binary conversion is inverted, meaning that values below `lower_thresh` become 1, and values above `upper_thresh` become 0. Default is True.
633
- :param Optional[int] batch_size: The number of images to process in a single batch. This helps manage memory usage for large stacks of images. Default is 1000.
634
- :return: A 3D NumPy array of shape (N, H, W), where each image has been converted to a binary format with pixel values of either 0 or 1.
635
- :rtype: np.ndarray
636
- """
637
-
638
- check_valid_array(data=imgs, source=img_stack_to_bw.__name__, accepted_ndims=(4,))
639
- check_int(name='lower_thresh', value=lower_thresh, max_value=255, min_value=0)
640
- check_int(name='upper_thresh', value=upper_thresh, max_value=255, min_value=0)
641
- check_int(name='batch_size', value=batch_size, min_value=1)
642
- results = cp.full((imgs.shape[0], imgs.shape[1], imgs.shape[2]), fill_value=cp.nan, dtype=cp.uint8)
643
-
644
- for l in range(0, imgs.shape[0], batch_size):
645
- r = l + batch_size
646
- batch_imgs = cp.array(imgs[l:r]).astype(cp.uint8)
647
- img_mean = cp.sum(batch_imgs, axis=3) / 3
648
- if not invert:
649
- batch_imgs = cp.where(img_mean < lower_thresh, 0, img_mean)
650
- batch_imgs = cp.where(batch_imgs > upper_thresh, 1, batch_imgs).astype(cp.uint8)
651
- else:
652
- batch_imgs = cp.where(img_mean < lower_thresh, 1, img_mean)
653
- batch_imgs = cp.where(batch_imgs > upper_thresh, 0, batch_imgs).astype(cp.uint8)
654
-
655
- results[l:r] = batch_imgs
656
-
657
- return results.get()
658
-
659
- def segment_img_stack_vertical(imgs: np.ndarray,
660
- pct: float,
661
- left: bool,
662
- right: bool) -> np.ndarray:
663
- """
664
- Segment a stack of images vertically based on a given percentage using GPU acceleration. For example, return the left half, right half, or senter half of each image in the stack.
665
-
666
- .. note::
667
- If both left and right are true, the center portion is returned.
668
-
669
- .. seealso::
670
- :func:`simba.mixins.image_mixin.ImageMixin.segment_img_vertical`
671
-
672
- :param np.ndarray imgs: A 3D or 4D NumPy array representing a stack of images. The array should have shape (N, H, W) for grayscale images or (N, H, W, C) for color images.
673
- :param float pct: The percentage of the image width to be used for segmentation. This value should be between a small positive value (e.g., 10e-6) and 0.99.
674
- :param bool left: If True, the left side of the image stack will be segmented.
675
- :param bool right: If True, the right side of the image stack will be segmented.
676
- :return: A NumPy array containing the segmented images, with the same number of dimensions as the input.
677
- :rtype: np.ndarray
678
- """
679
-
680
- check_valid_boolean(value=[left, right], source=segment_img_stack_vertical.__name__)
681
- check_float(name=f'{segment_img_stack_vertical.__name__} pct', value=pct, min_value=10e-6, max_value=0.99)
682
- check_valid_array(data=imgs, source=f'{segment_img_stack_vertical.__name__} imgs', accepted_ndims=(3, 4,), accepted_dtypes=Formats.NUMERIC_DTYPES.value)
683
- if not left and not right:
684
- raise InvalidInputError(msg='left are right argument are both False. Set one or both to True.', source=segment_img_stack_vertical.__name__)
685
- imgs = cp.array(imgs).astype(cp.uint8)
686
- h, w = imgs[0].shape[0], imgs[0].shape[1]
687
- px_crop = int(w * pct)
688
- if left and not right:
689
- imgs = imgs[:, :, :px_crop]
690
- elif right and not left:
691
- imgs = imgs[:, :, imgs.shape[2] - px_crop:]
692
- else:
693
- imgs = imgs[:, :, int(px_crop/2):int(imgs.shape[2] - (px_crop/2))]
694
- return imgs.get()
695
-
696
-
697
- def segment_img_stack_horizontal(imgs: np.ndarray,
698
- pct: float,
699
- upper: Optional[bool] = False,
700
- lower: Optional[bool] = False) -> np.ndarray:
701
-
702
- """
703
- Segment a stack of images horizontally based on a given percentage using GPU acceleration. For example, return the top half, bottom half, or center half of each image in the stack.
704
-
705
- .. note::
706
- If both top and bottom are true, the center portion is returned.
707
-
708
- .. seealso::
709
- :func:`simba.mixins.image_mixin.ImageMixin.segment_img_stack_horizontal`
710
-
711
- :param np.ndarray imgs: A 3D or 4D NumPy array representing a stack of images. The array should have shape (N, H, W) for grayscale images or (N, H, W, C) for color images.
712
- :param float pct: The percentage of the image width to be used for segmentation. This value should be between a small positive value (e.g., 10e-6) and 0.99.
713
- :param bool upper: If True, the top part of the image stack will be segmented.
714
- :param bool lower: If True, the bottom part of the image stack will be segmented.
715
- :return: A NumPy array containing the segmented images, with the same number of dimensions as the input.
716
- :rtype: np.ndarray
717
- """
718
-
719
- check_valid_boolean(value=[upper, lower], source=segment_img_stack_horizontal.__name__)
720
- check_float(name=f'{segment_img_stack_horizontal.__name__} pct', value=pct, min_value=10e-6, max_value=0.99)
721
- check_valid_array(data=imgs, source=f'{segment_img_stack_vertical.__name__} imgs', accepted_ndims=(3, 4,), accepted_dtypes=Formats.NUMERIC_DTYPES.value)
722
- if not upper and not lower:
723
- raise InvalidInputError(msg='upper and lower argument are both False. Set one or both to True.', source=segment_img_stack_horizontal.__name__)
724
- imgs = cp.array(imgs).astype(cp.uint8)
725
- h, w = imgs[0].shape[0], imgs[0].shape[1]
726
- px_crop = int(h * pct)
727
- if upper and not lower:
728
- imgs = imgs[: , :px_crop, :]
729
- elif not upper and lower:
730
- imgs = imgs[:, imgs.shape[0] - px_crop :, :]
731
- else:
732
- imgs = imgs[:, int(px_crop/2):int((imgs.shape[0] - px_crop) / 2), :]
733
-
734
- return imgs.get()
735
-
736
-
737
-
738
- @cuda.jit(device=True)
739
- def _cuda_is_inside_polygon(x, y, polygon_vertices):
740
- """
741
- Checks if the pixel location is inside the polygon.
742
-
743
- :param int x: Pixel x location.
744
- :param int y: Pixel y location.
745
- :param np.ndarray polygon_vertices: 2-dimensional array representing the x and y coordinates of the polygon vertices.
746
- :return: Boolean representing if the x and y are located in the polygon.
747
- """
748
-
749
- n = len(polygon_vertices)
750
- p2x, p2y, xints, inside = 0.0, 0.0, 0.0, False
751
- p1x, p1y = polygon_vertices[0]
752
- for j in range(n + 1):
753
- p2x, p2y = polygon_vertices[j % n]
754
- if ((y > min(p1y, p2y)) and (y <= max(p1y, p2y)) and (x <= max(p1x, p2x))):
755
- if p1y != p2y:
756
- xints = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
757
- if p1x == p2x or x <= xints:
758
- inside = not inside
759
- p1x, p1y = p2x, p2y
760
- return inside
761
-
762
-
763
-
764
- @cuda.jit(device=True)
765
- def _cuda_is_inside_circle(x, y, circle_x, circle_y, circle_r):
766
- """
767
- Device func to check if the pixel location is inside a circle.
768
-
769
- :param int x: Pixel x location.
770
- :param int y: Pixel y location.
771
- :param int circle_x: Center of circle x coordinate.
772
- :param int circle_y: Center of circle y coordinate.
773
- :param int y: Circle radius.
774
- :return: Boolean representing if the x and y are located in the circle.
775
- """
776
-
777
- p = (math.sqrt((x - circle_x) ** 2 + (y - circle_y) ** 2))
778
- if p <= circle_r:
779
- return True
780
- else:
781
- return False
782
- @cuda.jit()
783
- def _cuda_create_rectangle_masks(shapes, imgs, results, bboxes):
784
- """
785
- CUDA kernel to apply rectangular masks to a batch of images.
786
- """
787
- n, y, x = cuda.grid(3)
788
- if n >= imgs.shape[0]:
789
- return
790
-
791
- x_min = bboxes[n, 0]
792
- y_min = bboxes[n, 1]
793
- x_max = bboxes[n, 2]
794
- y_max = bboxes[n, 3]
795
-
796
- max_w = x_max - x_min
797
- max_h = y_max - y_min
798
-
799
- if x >= max_w or y >= max_h:
800
- return
801
-
802
- x_input = x + x_min
803
- y_input = y + y_min
804
-
805
- polygon = shapes[n]
806
-
807
- if _cuda_is_inside_polygon(x_input, y_input, polygon):
808
- if imgs.ndim == 4:
809
- for c in range(imgs.shape[3]):
810
- results[n, y, x, c] = imgs[n, y_input, x_input, c]
811
- else:
812
- results[n, y, x] = imgs[n, y_input, x_input]
813
-
814
- @cuda.jit()
815
- def _cuda_create_circle_masks(shapes, imgs, results, bboxes):
816
- """
817
- CUDA kernel to apply circular masks to a batch of images.
818
- """
819
- n, y, x = cuda.grid(3)
820
- if n >= imgs.shape[0]:
821
- return
822
-
823
- x_min = bboxes[n, 0]
824
- y_min = bboxes[n, 1]
825
- x_max = bboxes[n, 2]
826
- y_max = bboxes[n, 3]
827
-
828
- max_w = x_max - x_min
829
- max_h = y_max - y_min
830
-
831
- if x >= max_w or y >= max_h:
832
- return
833
-
834
- x_input = x + x_min
835
- y_input = y + y_min
836
-
837
- circle_x = shapes[n, 0]
838
- circle_y = shapes[n, 1]
839
- circle_r = shapes[n, 2]
840
-
841
-
842
- if _cuda_is_inside_circle(x_input, y_input, circle_x, circle_y, circle_r):
843
- if imgs.ndim == 4:
844
- for c in range(imgs.shape[3]):
845
- results[n, y, x, c] = imgs[n, y_input, x_input, c]
846
- else:
847
- results[n, y, x] = imgs[n, y_input, x_input]
848
-
849
-
850
- def _get_bboxes(shapes):
851
- """
852
- Helper to get geometries in :func:`simba.data_processors.cuda.image.slice_imgs`.
853
- """
854
- bboxes = []
855
- for shape in shapes:
856
- if shape.shape[0] == 3: # circle: [cx, cy, r]
857
- cx, cy, r = shape
858
- x_min = int(np.floor(cx - r))
859
- y_min = int(np.floor(cy - r))
860
- x_max = int(np.ceil(cx + r))
861
- y_max = int(np.ceil(cy + r))
862
- else:
863
- xs = shape[:, 0]
864
- ys = shape[:, 1]
865
- x_min = int(np.floor(xs.min()))
866
- y_min = int(np.floor(ys.min()))
867
- x_max = int(np.ceil(xs.max()))
868
- y_max = int(np.ceil(ys.max()))
869
- bboxes.append([x_min, y_min, x_max, y_max])
870
- return np.array(bboxes, dtype=np.int32)
871
-
872
- def slice_imgs(video_path: Union[str, os.PathLike],
873
- shapes: np.ndarray,
874
- batch_size: int = 1000,
875
- verbose: bool = True,
876
- save_dir: Optional[Union[str, os.PathLike]] = None):
877
- """
878
- Slice frames from a video based on given polygon or circle coordinates, and return or save masked/cropped frame regions using GPU acceleration.
879
-
880
- This function supports two types of shapes:
881
- - Polygon: array of shape (N, M, 2), where N = number of frames, M = number of polygon vertices.
882
- - Circle: array of shape (N, 3), where each row represents [center_x, center_y, radius].
883
-
884
- :param Union[str, os.PathLike] video_path: Path to the input video file.
885
- :param np.ndarray shapes: Array of polygon coordinates or circle parameters for each frame. - Polygon: shape = (n_frames, n_vertices, 2) - Circle: shape = (n_frames, 3)
886
- :param int batch_size: Number of frames to process per batch during GPU processing. Default 1000.
887
- :param bool verbose: Whether to print progress and status messages. Default True.
888
- :param Optional[Union[str, os.PathLike]] save_dir: If provided, the masked/cropped video will be saved in this directory. Otherwise, the cropped image stack will be returned.
889
-
890
- .. video:: _static/img/simba.sandbox.cuda_slice_w_crop.slice_imgs.webm
891
- :width: 900
892
- :loop:
893
-
894
- .. video:: _static/img/slice_imgs_gpu.webm
895
- :width: 800
896
- :autoplay:
897
- :loop:
898
-
899
- .. csv-table::
900
- :header: EXPECTED RUNTIMES
901
- :file: ../../../docs/tables/slice_imgs.csv
902
- :widths: 10, 90
903
- :align: center
904
- :class: simba-table
905
- :header-rows: 1
906
-
907
- .. note::
908
- For CPU multicore implementation, see :func:`simba.mixins.image_mixin.ImageMixin.slice_shapes_in_imgs`.
909
- For single core process, see :func:`simba.mixins.image_mixin.ImageMixin.slice_shapes_in_img`
910
-
911
- :example I:
912
- Example 1: Mask video using circular regions derived from body part center positions
913
- >>> video_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4"
914
- >>> data_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/csv/outlier_corrected_movement_location/03152021_NOB_IOT_8.csv"
915
- >>> save_dir = '/mnt/d/netholabs/yolo_videos/input/mp4_20250606083508'
916
- >>> nose_arr = read_df(file_path=data_path, file_type='csv', usecols=['Nose_x', 'Nose_y']).values.reshape(-1, 2).astype(np.int32)
917
- >>> polygons = GeometryMixin().multiframe_bodyparts_to_circle(data=nose_arr, parallel_offset=60)
918
- >>> polygon_lst = []
919
- >>> center = GeometryMixin.get_center(polygons)
920
- >>> polygons = np.hstack([center, np.full(shape=(len(center), 1), fill_value=60)])
921
- >>> slice_imgs(video_path=video_path, shapes=polygons, batch_size=500, save_dir=save_dir)
922
-
923
- :example II:
924
- Example 2: Mask video using minimum rotated rectangles from polygon hulls
925
- >>> video_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4"
926
- >>> data_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/csv/outlier_corrected_movement_location/03152021_NOB_IOT_8.csv"
927
- >>> save_dir = '/mnt/d/netholabs/yolo_videos/input/mp4_20250606083508'
928
- >>> nose_arr = read_df(file_path=data_path, file_type='csv', usecols=['Nose_x', 'Nose_y', 'Tail_base_x', 'Tail_base_y', 'Lat_left_x', 'Lat_left_y', 'Lat_right_x', 'Lat_right_y']).values.reshape(-1, 4, 2).astype(np.int32) ## READ THE BODY-PART THAT DEFINES THE HULL AND CONVERT TO ARRAY
929
- >>> polygons = GeometryMixin().multiframe_bodyparts_to_polygon(data=nose_arr, parallel_offset=60)
930
- >>> polygons = GeometryMixin().multiframe_minimum_rotated_rectangle(shapes=polygons)
931
- >>> polygon_lst = []
932
- >>> for i in polygons:
933
- >>> polygon_lst.append(np.array(i.exterior.coords).astype(np.int32))
934
- >>> polygons = np.stack(polygon_lst, axis=0)
935
- >>> sliced_imgs = slice_imgs(video_path=video_path, shapes=polygons, batch_size=500, save_dir=save_dir)
936
- """
937
-
938
- THREADS_PER_BLOCK = (16, 8, 8)
939
- video_meta_data = get_video_meta_data(video_path=video_path, fps_as_int=False)
940
- video_meta_data['frame_count'] = shapes.shape[0]
941
- n, w, h = video_meta_data['frame_count'], video_meta_data['width'], video_meta_data['height']
942
- is_color = ImageMixin.is_video_color(video=video_path)
943
- timer, save_temp_dir, results, video_out_path = SimbaTimer(start=True), None, None, None
944
- bboxes = _get_bboxes(shapes)
945
- crop_heights = bboxes[:, 3] - bboxes[:, 1]
946
- crop_widths = bboxes[:, 2] - bboxes[:, 0]
947
-
948
- max_h = int(np.max(crop_heights))
949
- max_w = int(np.max(crop_widths))
950
-
951
- if save_dir is None:
952
- if not is_color:
953
- results = np.zeros((n, max_h, max_w), dtype=np.uint8)
954
- else:
955
- results = np.zeros((n, max_h, max_w, 3), dtype=np.uint8)
956
- else:
957
- save_temp_dir = os.path.join(save_dir, f'temp_{video_meta_data["video_name"]}')
958
- create_directory(paths=save_temp_dir, overwrite=True)
959
- video_out_path = os.path.join(save_dir, f'{video_meta_data["video_name"]}.mp4')
960
-
961
- frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, verbose=True, max_que_size=2)
962
- frm_reader.start()
963
-
964
- for batch_cnt in range(frm_reader.batch_cnt):
965
- start_img_idx, end_img_idx, batch_imgs = get_async_frame_batch(batch_reader=frm_reader, timeout=10)
966
- if verbose:
967
- print(f'Processing images {start_img_idx} - {end_img_idx} (of {n}; batch count: {batch_cnt+1}/{frm_reader.batch_cnt})...')
968
-
969
- batch_save_path = os.path.join(save_temp_dir, f'{batch_cnt}.mp4') if save_dir is not None else None
970
-
971
- batch_shapes = shapes[start_img_idx:end_img_idx].astype(np.int32)
972
- batch_bboxes = bboxes[start_img_idx:end_img_idx]
973
-
974
- x_dev = cuda.to_device(batch_shapes)
975
- bboxes_dev = cuda.to_device(batch_bboxes)
976
- batch_img_dev = cuda.to_device(batch_imgs)
977
-
978
- if not is_color:
979
- batch_results = np.zeros((batch_imgs.shape[0], max_h, max_w), dtype=np.uint8)
980
- else:
981
- batch_results = np.zeros((batch_imgs.shape[0], max_h, max_w, 3), dtype=np.uint8)
982
- batch_results_dev = cuda.to_device(batch_results)
983
- grid_n = math.ceil(batch_imgs.shape[0] / THREADS_PER_BLOCK[0])
984
- grid_y = math.ceil(max_h / THREADS_PER_BLOCK[1])
985
- grid_x = math.ceil(max_w / THREADS_PER_BLOCK[2])
986
- bpg = (grid_n, grid_y, grid_x)
987
- if batch_shapes.shape[1] == 3:
988
- _cuda_create_circle_masks[bpg, THREADS_PER_BLOCK](x_dev, batch_img_dev, batch_results_dev, bboxes_dev)
989
- else:
990
- _cuda_create_rectangle_masks[bpg, THREADS_PER_BLOCK](x_dev, batch_img_dev, batch_results_dev, bboxes_dev)
991
- if save_dir is None:
992
- results[start_img_idx:end_img_idx] = batch_results_dev.copy_to_host()
993
- else:
994
- frame_results = batch_results_dev.copy_to_host()
995
- results = {k: v for k, v in enumerate(frame_results)}
996
- ImageMixin().img_stack_to_video(imgs=results, fps=video_meta_data['fps'], save_path=batch_save_path, verbose=False)
997
-
998
- frm_reader.kill()
999
- timer.stop_timer()
1000
-
1001
- if save_dir:
1002
- concatenate_videos_in_folder(in_folder=save_temp_dir, save_path=video_out_path, remove_splits=True, gpu=True)
1003
- if verbose:
1004
- stdout_success(msg=f'Shapes sliced in video saved at {video_out_path}.', elapsed_time=timer.elapsed_time_str)
1005
- return None
1006
- else:
1007
- if verbose:
1008
- stdout_success(msg='Shapes sliced in video.', elapsed_time=timer.elapsed_time_str)
1009
- return results
1010
-
1011
-
1012
- @cuda.jit()
1013
- def _sliding_psnr(data, stride, results):
1014
- r = cuda.grid(1)
1015
- l = int(r - stride[0])
1016
- if (r < 0) or (r > data.shape[0] -1):
1017
- return
1018
- if l < 0:
1019
- return
1020
- else:
1021
- img_1, img_2 = data[r], data[l]
1022
- mse = _cuda_mse(img_1, img_2)
1023
- if mse == 0:
1024
- results[r] = 0.0
1025
- else:
1026
- results[r] = 20 * math.log10(255 / math.sqrt(mse))
1027
-
1028
- def sliding_psnr(data: np.ndarray,
1029
- stride_s: int,
1030
- sample_rate: float) -> np.ndarray:
1031
- r"""
1032
- Computes the Peak Signal-to-Noise Ratio (PSNR) between pairs of images in a stack using a sliding window approach.
1033
-
1034
- This function calculates PSNR for each image in a stack compared to another image in the stack that is separated by a specified stride.
1035
- The sliding window approach allows for the comparison of image quality over a sequence of images.
1036
-
1037
- .. note::
1038
- - PSNR values are measured in decibels (dB).
1039
- - Higher PSNR values indicate better quality with minimal differences from the reference image.
1040
- - Lower PSNR values indicate higher distortion or noise.
1041
-
1042
- .. math::
1043
-
1044
- \text{PSNR} = 20 \log_{10} \left( \frac{\text{MAX}}{\sqrt{\text{MSE}}} \right)
1045
-
1046
- where:
1047
- - :math:`\text{MAX}` is the maximum possible pixel value (255 for 8-bit images)
1048
- - :math:`\text{MSE}` is the Mean Squared Error between the two images
1049
-
1050
- :param data: A 4D NumPy array of shape (N, H, W, C) representing a stack of images, where N is the number of images, H is the height, W is the width, and C is the number of color channels.
1051
- :param stride_s: The base stride length in terms of the number of images between the images being compared. Determines the separation between images for comparison in the stack.
1052
- :param sample_rate: The sample rate to scale the stride length. This allows for adjusting the stride dynamically based on the sample rate.
1053
- :return: A 1D NumPy array of PSNR values, where each element represents the PSNR between the image at index `r` and the image at index `l = r - stride`, for all valid indices `r`.
1054
- :rtype: np.ndarray
1055
-
1056
- :example:
1057
- >>> data = ImageMixin().read_img_batch_from_video(video_path =r"/mnt/c/troubleshooting/mitra/project_folder/videos/clipped/501_MA142_Gi_CNO_0514_clipped.mp4", start_frm=0, end_frm=299)
1058
- >>> data = np.stack(list(data.values()), axis=0).astype(np.uint8)
1059
- >>> data = ImageMixin.img_stack_to_greyscale(imgs=data)
1060
- >>> p = sliding_psnr(data=data, stride_s=1, sample_rate=1)
1061
- """
1062
-
1063
- results = np.full(data.shape[0], fill_value=255.0, dtype=np.float32)
1064
- stride = np.array([stride_s * sample_rate], dtype=np.int32)
1065
- if stride[0] < 1: stride[0] = 1
1066
- stride_dev = cuda.to_device(stride)
1067
- results_dev = cuda.to_device(results)
1068
- data_dev = cuda.to_device(data)
1069
- bpg = (data.shape[0] + (THREADS_PER_BLOCK - 1)) // THREADS_PER_BLOCK
1070
- _sliding_psnr[bpg, THREADS_PER_BLOCK](data_dev, stride_dev, results_dev)
1071
- return results_dev.copy_to_host()
1072
-
1073
- def rotate_img_stack_cupy(imgs: np.ndarray,
1074
- rotation_degrees: Optional[float] = 180,
1075
- batch_size: Optional[int] = 500) -> np.ndarray:
1076
- """
1077
- Rotates a stack of images by a specified number of degrees using GPU acceleration with CuPy.
1078
-
1079
- Accepts a 3D (single-channel images) or 4D (multichannel images) NumPy array, rotates each image in the stack by the specified degree around the center, and returns the result as a NumPy array.
1080
-
1081
- :param np.ndarray imgs: The input stack of images to be rotated. Expected to be a NumPy array with 3 or 4 dimensions. 3D shape: (num_images, height, width) - 4D shape: (num_images, height, width, channels)
1082
- :param Optional[float] rotation_degrees: The angle by which the images should be rotated, in degrees. Must be between 1 and 359 degrees. Defaults to 180 degrees.
1083
- :param Optional[int] batch_size: Number of images to process on GPU in each batch. Decrease if data can't fit on GPU RAM.
1084
- :returns: A NumPy array containing the rotated images with the same shape as the input.
1085
- :rtype: np.ndarray
1086
-
1087
- :example:
1088
- >>> video_path = r"/mnt/c/troubleshooting/mitra/project_folder/videos/F0_gq_Saline_0626_clipped.mp4"
1089
- >>> imgs = read_img_batch_from_video_gpu(video_path=video_path)
1090
- >>> imgs = np.stack(np.array(list(imgs.values())), axis=0)
1091
- >>> imgs = rotate_img_stack_cupy(imgs=imgs, rotation=50)
1092
- """
1093
-
1094
- check_valid_array(data=imgs, source=f'{rotate_img_stack_cupy.__name__} imgs', accepted_ndims=(3, 4))
1095
- check_int(name=f'{rotate_img_stack_cupy.__name__} rotation', value=rotation_degrees, min_value=1, max_value=359)
1096
- results = cp.full_like(imgs, fill_value=np.nan, dtype=np.uint8)
1097
- for l in range(0, imgs.shape[0], batch_size):
1098
- r = l + batch_size
1099
- batch_imgs = cp.array(imgs[l:r])
1100
- results[l:r] = rotate(input=batch_imgs, angle=rotation_degrees, axes=(2, 1), reshape=True)
1101
- return results.get()
1102
-
1103
- def rotate_video_cupy(video_path: Union[str, os.PathLike],
1104
- save_path: Optional[Union[str, os.PathLike]] = None,
1105
- rotation_degrees: Optional[float] = 180,
1106
- batch_size: Optional[int] = None,
1107
- verbose: Optional[bool] = True) -> None:
1108
- """
1109
- Rotates a video by a specified angle using GPU acceleration and CuPy for image processing.
1110
-
1111
- :param Union[str, os.PathLike] video_path: Path to the input video file.
1112
- :param Optional[Union[str, os.PathLike]] save_path: Path to save the rotated video. If None, saves the video in the same directory as the input with '_rotated_<rotation_degrees>' appended to the filename.
1113
- :param nptional[float] rotation_degrees: Degrees to rotate the video. Must be between 1 and 359 degrees. Default is 180.
1114
- :param Optional[int] batch_size: The number of frames to process in each batch. Deafults to None meaning all images will be processed in a single batch.
1115
- :returns: None.
1116
-
1117
- :example:
1118
- >>> video_path = r"/mnt/c/troubleshooting/mitra/project_folder/videos/F0_gq_Saline_0626_clipped.mp4"
1119
- >>> rotate_video_cupy(video_path=video_path, rotation_degrees=45)
1120
- """
1121
-
1122
- timer = SimbaTimer(start=True)
1123
- check_int(name=f'{rotate_img_stack_cupy.__name__} rotation', value=rotation_degrees, min_value=1, max_value=359)
1124
- check_valid_boolean(source=f'{rotate_img_stack_cupy.__name__} verbose', value=verbose)
1125
- if save_path is None:
1126
- video_dir, video_name, _ = get_fn_ext(filepath=video_path)
1127
- save_path = os.path.join(video_dir, f'{video_name}_rotated_{rotation_degrees}.mp4')
1128
- video_meta_data = get_video_meta_data(video_path=video_path)
1129
- if batch_size is not None:
1130
- check_int(name=f'{rotate_img_stack_cupy.__name__} batch_size', value=batch_size, min_value=1)
1131
- else:
1132
- batch_size = video_meta_data['frame_count']
1133
- fourcc = cv2.VideoWriter_fourcc(*Formats.MP4_CODEC.value)
1134
- is_clr = ImageMixin.is_video_color(video=video_path)
1135
- frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=3, verbose=False)
1136
- frm_reader.start()
1137
- for batch_cnt in range(frm_reader.batch_cnt):
1138
- start_idx, end_idx, imgs = get_async_frame_batch(batch_reader=frm_reader, timeout=10)
1139
- if verbose:
1140
- print(f'Rotating frames {start_idx}-{end_idx}... (of {video_meta_data["frame_count"]}, video: {video_meta_data["video_name"]})')
1141
- imgs = rotate_img_stack_cupy(imgs=imgs, rotation_degrees=rotation_degrees, batch_size=batch_size)
1142
- if batch_cnt == 0:
1143
- writer = cv2.VideoWriter(save_path, fourcc, video_meta_data['fps'], (imgs.shape[2], imgs.shape[1]), isColor=is_clr)
1144
- for img in imgs: writer.write(img)
1145
- writer.release()
1146
- timer.stop_timer()
1147
- frm_reader.kill()
1148
- if verbose:
1149
- stdout_success(f'Rotated video saved at {save_path}', source=rotate_video_cupy.__name__)
1150
-
1151
-
1152
- @cuda.jit()
1153
- def _bg_subtraction_cuda_kernel(imgs, avg_img, results, is_clr, fg_clr, threshold):
1154
- x, y, n = cuda.grid(3)
1155
- if n < 0 or n > (imgs.shape[0] -1):
1156
- return
1157
- if y < 0 or y > (imgs.shape[1] -1):
1158
- return
1159
- if x < 0 or x > (imgs.shape[2] -1):
1160
- return
1161
- if is_clr[0] == 1:
1162
- r1, g1, b1 = imgs[n][y][x][0],imgs[n][y][x][1], imgs[n][y][x][2]
1163
- r2, g2, b2 = avg_img[y][x][0], avg_img[y][x][1], avg_img[y][x][2]
1164
- r_diff, g_diff, b_diff = abs(r1-r2), abs(g1-g2), abs(b1-b2)
1165
- grey_diff = _cuda_luminance_pixel_to_grey(r_diff, g_diff, b_diff)
1166
- if grey_diff > threshold[0]:
1167
- if fg_clr[0] != -1:
1168
- r_out, g_out, b_out = fg_clr[0], fg_clr[1], fg_clr[2]
1169
- else:
1170
- r_out, g_out, b_out = r1, g1, b1
1171
- else:
1172
- r_out, g_out, b_out = results[n][y][x][0], results[n][y][x][1], results[n][y][x][2]
1173
- results[n][y][x][0], results[n][y][x][1], results[n][y][x][2] = r_out, g_out, b_out
1174
-
1175
- else:
1176
- val_1, val_2 = imgs[n][y][x][0], avg_img[y][x][0]
1177
- grey_diff = abs(val_1-val_2)
1178
- if grey_diff > threshold[0]:
1179
- if fg_clr[0] != -1:
1180
- val_out = val_1
1181
- else:
1182
- val_out = 255
1183
- else:
1184
- val_out = 0
1185
- results[n][y][x] = val_out
1186
-
1187
-
1188
- def bg_subtraction_cuda(video_path: Union[str, os.PathLike],
1189
- avg_frm: np.ndarray,
1190
- save_path: Optional[Union[str, os.PathLike]] = None,
1191
- bg_clr: Optional[Tuple[int, int, int]] = (0, 0, 0),
1192
- fg_clr: Optional[Tuple[int, int, int]] = None,
1193
- batch_size: Optional[int] = 500,
1194
- threshold: Optional[int] = 50):
1195
- """
1196
- Remove background from videos using GPU acceleration.
1197
-
1198
- .. video:: _static/img/video_bg_subtraction.webm
1199
- :width: 800
1200
- :autoplay:
1201
- :loop:
1202
-
1203
- .. note::
1204
- To create an `avg_frm`, use :func:`simba.video_processors.video_processing.create_average_frm`, :func:`simba.data_processors.cuda.image.create_average_frm_cupy`, or :func:`~simba.data_processors.cuda.image.create_average_frm_cuda`
1205
-
1206
- .. seealso::
1207
- For CPU-based alternative, see :func:`simba.video_processors.video_processing.video_bg_subtraction` or :func:`~simba.video_processors.video_processing.video_bg_subtraction_mp`
1208
- For GPU-based alternative, see :func:`~simba.data_processors.cuda.image.bg_subtraction_cupy`. Needs work, CPU/multicore appears faster.
1209
-
1210
- .. seealso::
1211
- To create average frame on the CPU, see :func:`simba.video_processors.video_processing.create_average_frm`. CPU/multicore appears faster.
1212
-
1213
- .. csv-table::
1214
- :header: EXPECTED RUNTIMES
1215
- :file: ../../../docs/tables/bg_subtraction_cuda.csv
1216
- :widths: 10, 45, 45
1217
- :align: center
1218
- :class: simba-table
1219
- :header-rows: 1
1220
-
1221
- :param Union[str, os.PathLike] video_path: The path to the video to remove the background from.
1222
- :param np.ndarray avg_frm: Average frame of the video. Can be created with e.g., :func:`simba.video_processors.video_processing.create_average_frm`.
1223
- :param Optional[Union[str, os.PathLike]] save_path: Optional location to store the background removed video. If None, then saved in the same directory as the input video with the `_bg_removed` suffix.
1224
- :param Optional[Tuple[int, int, int]] bg_clr: Tuple representing the background color of the video.
1225
- :param Optional[Tuple[int, int, int]] fg_clr: Tuple representing the foreground color of the video (e.g., the animal). If None, then the original pixel colors will be used. Default: 50.
1226
- :param Optional[int] batch_size: Number of frames to process concurrently. Use higher values of RAM memory allows. Default: 500.
1227
- :param Optional[int] threshold: Value between 0-255 representing the difference threshold between the average frame subtracted from each frame. Higher values and more pixels will be considered background. Default: 50.
1228
-
1229
- :example:
1230
- >>> video_path = "/mnt/c/troubleshooting/mitra/project_folder/videos/clipped/592_MA147_Gq_CNO_0515.mp4"
1231
- >>> avg_frm = create_average_frm(video_path=video_path)
1232
- >>> bg_subtraction_cuda(video_path=video_path, avg_frm=avg_frm, fg_clr=(255, 255, 255))
1233
- """
1234
-
1235
- check_if_valid_img(data=avg_frm, source=f'{bg_subtraction_cuda}')
1236
- check_if_valid_rgb_tuple(data=bg_clr)
1237
- check_int(name=f'{bg_subtraction_cuda.__name__} batch_size', value=batch_size, min_value=1)
1238
- check_int(name=f'{bg_subtraction_cuda.__name__} threshold', value=threshold, min_value=0, max_value=255)
1239
- THREADS_PER_BLOCK = (32, 32, 1)
1240
- timer = SimbaTimer(start=True)
1241
- video_meta = get_video_meta_data(video_path=video_path)
1242
- batch_cnt = int(max(1, np.ceil(video_meta['frame_count'] / batch_size)))
1243
- frm_batches = np.array_split(np.arange(0, video_meta['frame_count']), batch_cnt)
1244
- n, w, h = video_meta['frame_count'], video_meta['width'], video_meta['height']
1245
- avg_frm = cv2.resize(avg_frm, (w, h))
1246
- if is_video_color(video_path): is_color = np.array([1])
1247
- else: is_color = np.array([0])
1248
- fourcc = cv2.VideoWriter_fourcc(*Formats.MP4_CODEC.value)
1249
- if save_path is None:
1250
- in_dir, video_name, _ = get_fn_ext(filepath=video_path)
1251
- save_path = os.path.join(in_dir, f'{video_name}_bg_removed.mp4')
1252
- if fg_clr is not None:
1253
- check_if_valid_rgb_tuple(data=fg_clr)
1254
- fg_clr = np.array(fg_clr)
1255
- else:
1256
- fg_clr = np.array([-1])
1257
- threshold = np.array([threshold]).astype(np.int32)
1258
- writer = cv2.VideoWriter(save_path, fourcc, video_meta['fps'], (w, h))
1259
- y_dev = cuda.to_device(avg_frm.astype(np.float32))
1260
- fg_clr_dev = cuda.to_device(fg_clr)
1261
- is_color_dev = cuda.to_device(is_color)
1262
- for frm_batch_cnt, frm_batch in enumerate(frm_batches):
1263
- print(f'Processing frame batch {frm_batch_cnt+1} / {len(frm_batches)} (complete: {round((frm_batch_cnt / len(frm_batches)) * 100, 2)}%)')
1264
- batch_imgs = read_img_batch_from_video_gpu(video_path=video_path, start_frm=frm_batch[0], end_frm=frm_batch[-1])
1265
- batch_imgs = np.stack(list(batch_imgs.values()), axis=0).astype(np.float32)
1266
- batch_n = batch_imgs.shape[0]
1267
- results = np.zeros_like(batch_imgs).astype(np.uint8)
1268
- results[:] = bg_clr
1269
- results = cuda.to_device(results)
1270
- grid_x = math.ceil(w / THREADS_PER_BLOCK[0])
1271
- grid_y = math.ceil(h / THREADS_PER_BLOCK[1])
1272
- grid_z = math.ceil(batch_n / THREADS_PER_BLOCK[2])
1273
- bpg = (grid_x, grid_y, grid_z)
1274
- x_dev = cuda.to_device(batch_imgs)
1275
- _bg_subtraction_cuda_kernel[bpg, THREADS_PER_BLOCK](x_dev, y_dev, results, is_color_dev, fg_clr_dev, threshold)
1276
- results = results.copy_to_host()
1277
- for img_cnt, img in enumerate(results):
1278
- writer.write(img)
1279
- writer.release()
1280
- timer.stop_timer()
1281
- stdout_success(msg=f'Video saved at {save_path}', elapsed_time=timer.elapsed_time_str)
1282
-
1283
-
1284
- def bg_subtraction_cupy(video_path: Union[str, os.PathLike],
1285
- avg_frm: Union[np.ndarray, str, os.PathLike],
1286
- save_path: Optional[Union[str, os.PathLike]] = None,
1287
- bg_clr: Optional[Tuple[int, int, int]] = (0, 0, 0),
1288
- fg_clr: Optional[Tuple[int, int, int]] = None,
1289
- batch_size: Optional[int] = 500,
1290
- threshold: Optional[int] = 50,
1291
- verbose: bool = True,
1292
- async_frame_read: bool = True):
1293
- """
1294
- Remove background from videos using GPU acceleration through CuPY.
1295
-
1296
- .. video:: _static/img/bg_remover_example_1.webm
1297
- :width: 800
1298
- :autoplay:
1299
- :loop:
1300
-
1301
- .. seealso::
1302
- For CPU-based alternative, see :func:`simba.video_processors.video_processing.video_bg_subtraction` or :func:`~simba.video_processors.video_processing.video_bg_subtraction_mp`
1303
- For GPU-based alternative, see :func:`~simba.data_processors.cuda.image.bg_subtraction_cuda`.
1304
- Needs work, CPU/multicore appears faster.
1305
-
1306
- :param Union[str, os.PathLike] video_path: The path to the video to remove the background from.
1307
- :param np.ndarray avg_frm: Average frame of the video. Can be created with e.g., :func:`simba.video_processors.video_processing.create_average_frm`.
1308
- :param Optional[Union[str, os.PathLike]] save_path: Optional location to store the background removed video. If None, then saved in the same directory as the input video with the `_bg_removed` suffix.
1309
- :param Optional[Tuple[int, int, int]] bg_clr: Tuple representing the background color of the video.
1310
- :param Optional[Tuple[int, int, int]] fg_clr: Tuple representing the foreground color of the video (e.g., the animal). If None, then the original pixel colors will be used. Default: 50.
1311
- :param Optional[int] batch_size: Number of frames to process concurrently. Use higher values of RAM memory allows. Default: 500.
1312
- :param Optional[int] threshold: Value between 0-255 representing the difference threshold between the average frame subtracted from each frame. Higher values and more pixels will be considered background. Default: 50.
1313
-
1314
-
1315
- :example:
1316
- >>> avg_frm = create_average_frm(video_path="/mnt/c/troubleshooting/mitra/project_folder/videos/temp/temp_ex_bg_subtraction/original/844_MA131_gq_CNO_0624.mp4")
1317
- >>> video_path = "/mnt/c/troubleshooting/mitra/project_folder/videos/temp/temp_ex_bg_subtraction/844_MA131_gq_CNO_0624_7.mp4"
1318
- >>> bg_subtraction_cupy(video_path=video_path, avg_frm=avg_frm, batch_size=500)
1319
- """
1320
-
1321
- if not _is_cuda_available()[0]:
1322
- raise SimBAGPUError('NP GPU detected using numba.cuda', source=bg_subtraction_cupy.__name__)
1323
- if isinstance(avg_frm, (str, os.PathLike)):
1324
- check_file_exist_and_readable(file_path=avg_frm, raise_error=True)
1325
- avg_frm = read_img(img_path=avg_frm, greyscale=False, clahe=False)
1326
- check_if_valid_img(data=avg_frm, source=f'{bg_subtraction_cupy}')
1327
- check_if_valid_rgb_tuple(data=bg_clr)
1328
- check_int(name=f'{bg_subtraction_cupy.__name__} batch_size', value=batch_size, min_value=1)
1329
- check_int(name=f'{bg_subtraction_cupy.__name__} threshold', value=threshold, min_value=0, max_value=255)
1330
- timer = SimbaTimer(start=True)
1331
- video_meta = get_video_meta_data(video_path=video_path)
1332
- n, w, h = video_meta['frame_count'], video_meta['width'], video_meta['height']
1333
- is_video_color_bool = is_video_color(video_path)
1334
- is_avg_frm_color = avg_frm.ndim == 3 and avg_frm.shape[2] == 3
1335
- if avg_frm.shape[0] != h or avg_frm.shape[1] != w:
1336
- raise InvalidInputError(msg=f'The avg_frm and video must have the same resolution: avg_frm is {avg_frm.shape[1]}x{avg_frm.shape[0]}, video is {w}x{h}', source=bg_subtraction_cupy.__name__)
1337
- if is_video_color_bool != is_avg_frm_color:
1338
- video_type = 'color' if is_video_color_bool else 'grayscale'
1339
- avg_frm_type = 'color' if is_avg_frm_color else 'grayscale'
1340
- raise InvalidInputError(msg=f'Color/grayscale mismatch: video is {video_type} but avg_frm is {avg_frm_type}', source=bg_subtraction_cupy.__name__)
1341
-
1342
- avg_frm = cp.array(avg_frm)
1343
- is_color = is_video_color_bool
1344
- batch_cnt = int(max(1, np.ceil(video_meta['frame_count'] / batch_size)))
1345
- frm_batches = np.array_split(np.arange(0, video_meta['frame_count']), batch_cnt)
1346
- fourcc = cv2.VideoWriter_fourcc(*Formats.MP4_CODEC.value)
1347
- if save_path is None:
1348
- in_dir, video_name, _ = get_fn_ext(filepath=video_path)
1349
- save_path = os.path.join(in_dir, f'{video_name}_bg_removed_ppp.mp4')
1350
- if fg_clr is not None:
1351
- check_if_valid_rgb_tuple(data=fg_clr)
1352
- fg_clr = np.array(fg_clr)
1353
- else:
1354
- fg_clr = np.array([-1])
1355
- writer = cv2.VideoWriter(save_path, fourcc, video_meta['fps'], (w, h), isColor=is_color)
1356
- if async_frame_read:
1357
- async_frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=3, verbose=True, gpu=True)
1358
- async_frm_reader.start()
1359
- else:
1360
- async_frm_reader = None
1361
- for frm_batch_cnt, frm_batch in enumerate(frm_batches):
1362
- if verbose: print(f'Processing frame batch {frm_batch_cnt + 1} / {len(frm_batches)} (complete: {round((frm_batch_cnt / len(frm_batches)) * 100, 2)}%, {get_current_time()})')
1363
- if not async_frame_read:
1364
- batch_imgs = read_img_batch_from_video_gpu(video_path=video_path, start_frm=frm_batch[0], end_frm=frm_batch[-1], verbose=verbose)
1365
- batch_imgs = cp.array(np.stack(list(batch_imgs.values()), axis=0).astype(np.float32))
1366
- else:
1367
- batch_imgs = cp.array(get_async_frame_batch(batch_reader=async_frm_reader, timeout=15)[2])
1368
- img_diff = cp.abs(batch_imgs - avg_frm)
1369
- if is_color:
1370
- img_diff = img_stack_to_grayscale_cupy(imgs=img_diff, batch_size=img_diff.shape[0])
1371
- threshold_cp = cp.array([threshold], dtype=cp.float32)
1372
- mask = cp.where(img_diff > threshold_cp, 1, 0).astype(cp.uint8)
1373
- if is_color:
1374
- batch_imgs[mask == 0] = bg_clr
1375
- if fg_clr[0] != -1:
1376
- batch_imgs[mask == 1] = fg_clr
1377
- else:
1378
- bg_clr_gray = int(0.07 * bg_clr[2] + 0.72 * bg_clr[1] + 0.21 * bg_clr[0])
1379
- batch_imgs[mask == 0] = bg_clr_gray
1380
- if fg_clr[0] != -1:
1381
- fg_clr_gray = int(0.07 * fg_clr[2] + 0.72 * fg_clr[1] + 0.21 * fg_clr[0])
1382
- batch_imgs[mask == 1] = fg_clr_gray
1383
- batch_imgs = batch_imgs.astype(cp.uint8).get()
1384
- for img_cnt, img in enumerate(batch_imgs):
1385
- writer.write(img)
1386
- if async_frm_reader is not None:
1387
- async_frm_reader.kill()
1388
-
1389
- writer.release()
1390
- timer.stop_timer()
1391
- stdout_success(msg=f'Video saved at {save_path}', elapsed_time=timer.elapsed_time_str)
1392
-
1393
-
1394
- @cuda.jit(max_registers=None)
1395
- def _pose_plot_kernel(imgs, data, circle_size, resolution, colors):
1396
- bp_n, img_n = cuda.grid(2)
1397
- if img_n < 0 or img_n > (imgs.shape[0] -1):
1398
- return
1399
- if bp_n < 0 or bp_n > (data[0].shape[0] -1):
1400
- return
1401
-
1402
- img, bp_loc, color = imgs[img_n], data[img_n][bp_n], colors[bp_n]
1403
- for x1 in range(bp_loc[0]-circle_size[0], bp_loc[0]+circle_size[0]):
1404
- for y1 in range(bp_loc[1]-circle_size[0], bp_loc[1]+circle_size[0]):
1405
- if (x1 > 0) and (x1 < resolution[0]):
1406
- if (y1 > 0) and (y1 < resolution[1]):
1407
- b = (x1 - bp_loc[0]) ** 2
1408
- c = (y1 - bp_loc[1]) ** 2
1409
- if (b + c) < (circle_size[0] ** 2):
1410
- imgs[img_n][y1][x1][0] = int(color[0])
1411
- imgs[img_n][y1][x1][1] = int(color[1])
1412
- imgs[img_n][y1][x1][2] = int(color[2])
1413
-
1414
-
1415
- def pose_plotter(data: Union[str, os.PathLike, np.ndarray],
1416
- video_path: Union[str, os.PathLike],
1417
- save_path: Union[str, os.PathLike],
1418
- circle_size: Optional[int] = None,
1419
- colors: Optional[str] = 'Set1',
1420
- batch_size: int = 750,
1421
- verbose: bool = True) -> None:
1422
-
1423
- """
1424
- Creates a video overlaying pose-estimation data on frames from a given video using GPU acceleration.
1425
-
1426
- .. video:: _static/img/pose_plotter_cuda.mp4
1427
- :width: 800
1428
- :autoplay:
1429
- :loop:
1430
-
1431
- .. seealso::
1432
- For CPU based methods, see :func:`~simba.plotting.path_plotter.PathPlotterSingleCore` and :func:`~simba.plotting.path_plotter_mp.PathPlotterMulticore`.
1433
-
1434
- .. csv-table::
1435
- :header: EXPECTED RUNTIMES
1436
- :file: ../../../docs/tables/pose_plotter.csv
1437
- :widths: 10, 90
1438
- :align: center
1439
- :class: simba-table
1440
- :header-rows: 1
1441
-
1442
- :param Union[str, os.PathLike, np.ndarray] data: Path to a CSV file with pose-estimation data or a 3d numpy array (n_images, n_bodyparts, 2) with pose-estimated locations.
1443
- :param Union[str, os.PathLike] video_path: Path to a video file where the ``data`` has been pose-estimated.
1444
- :param Union[str, os.PathLike] save_path: Location where to store the output visualization.
1445
- :param Optional[int] circle_size: The size of the circles representing the location of the pose-estimated locations. If None, the optimal size will be inferred as a 100th of the max(resultion_w, h).
1446
- :param int batch_size: The number of frames to process concurrently on the GPU. Default: 750. Increase of host and device RAM allows it to improve runtime. Decrease if you hit memory errors.
1447
-
1448
- :example:
1449
- >>> DATA_PATH = "/mnt/c/troubleshooting/mitra/project_folder/csv/outlier_corrected_movement_location/501_MA142_Gi_CNO_0521.csv"
1450
- >>> VIDEO_PATH = "/mnt/c/troubleshooting/mitra/project_folder/videos/501_MA142_Gi_CNO_0521.mp4"
1451
- >>> SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1452
- >>> pose_plotter(data=DATA_PATH, video_path=VIDEO_PATH, save_path=SAVE_PATH, circle_size=10, batch_size=1000)
1453
- """
1454
-
1455
- THREADS_PER_BLOCK = (32, 32, 1)
1456
- if isinstance(data, str):
1457
- check_file_exist_and_readable(file_path=data)
1458
- df = read_df(file_path=data, file_type='csv')
1459
- cols = [x for x in df.columns if not x.lower().endswith('_p')]
1460
- data = df[cols].values
1461
- data = np.ascontiguousarray(data.reshape(data.shape[0], int(data.shape[1] / 2), 2).astype(np.int32))
1462
- elif isinstance(data, np.ndarray):
1463
- check_valid_array(data=data, source=pose_plotter.__name__, accepted_ndims=(3,), accepted_dtypes=Formats.NUMERIC_DTYPES.value)
1464
-
1465
- check_int(name=f'{pose_plotter.__name__} batch_size', value=batch_size, min_value=1)
1466
- check_valid_boolean(value=[verbose], source=f'{pose_plotter.__name__} verbose')
1467
- video_meta_data = get_video_meta_data(video_path=video_path)
1468
- n, w, h = video_meta_data['frame_count'], video_meta_data['width'], video_meta_data['height']
1469
- check_if_dir_exists(in_dir=os.path.dirname(save_path))
1470
- if data.shape[0] != video_meta_data['frame_count']:
1471
- raise FrameRangeError(msg=f'The data contains {data.shape[0]} frames while the video contains {video_meta_data["frame_count"]} frames')
1472
- if circle_size is None:
1473
- circle_size = np.array([PlottingMixin().get_optimal_circle_size(frame_size=(w, h))]).astype(np.int32)
1474
- else:
1475
- check_int(name=f'{pose_plotter.__name__} circle_size', value=circle_size, min_value=1)
1476
- circle_size = np.array([circle_size]).astype(np.int32)
1477
- fourcc = cv2.VideoWriter_fourcc(*Formats.MP4_CODEC.value)
1478
- video_writer = cv2.VideoWriter(save_path, fourcc, video_meta_data['fps'], (w, h))
1479
- colors = np.array(create_color_palette(pallete_name=colors, increments=data[0].shape[0])).astype(np.int32)
1480
- circle_size_dev = cuda.to_device(circle_size)
1481
- colors_dev = cuda.to_device(colors)
1482
- resolution_dev = cuda.to_device(np.array([video_meta_data['width'], video_meta_data['height']]))
1483
- data = np.ascontiguousarray(data, dtype=np.int32)
1484
- img_dev = cuda.device_array((batch_size, h, w, 3), dtype=np.int32)
1485
- data_dev = cuda.device_array((batch_size, data.shape[1], 2), dtype=np.int32)
1486
- total_timer, video_start_time = SimbaTimer(start=True), time.time()
1487
- frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=3, verbose=False)
1488
- frm_reader.start()
1489
- for batch_cnt in range(frm_reader.batch_cnt):
1490
- start_img_idx, end_img_idx, batch_frms = get_async_frame_batch(batch_reader=frm_reader, timeout=10)
1491
- video_elapsed_time = str(round(time.time() - video_start_time, 4)) + 's'
1492
- if verbose: print(f'Processing images {start_img_idx} - {end_img_idx} (of {n}; batch count: {batch_cnt+1}/{frm_reader.batch_cnt}, video: {video_meta_data["video_name"]}, elapsed video processing time: {video_elapsed_time})...')
1493
- batch_data = data[start_img_idx:end_img_idx + 1]
1494
- batch_n = batch_frms.shape[0]
1495
- if verbose: print(f'Moving frames {start_img_idx}-{end_img_idx} to device...')
1496
- img_dev[:batch_n].copy_to_device(batch_frms[:batch_n].astype(np.int32))
1497
- data_dev[:batch_n] = cuda.to_device(batch_data[:batch_n])
1498
- del batch_frms; del batch_data
1499
- bpg = (math.ceil(batch_n / THREADS_PER_BLOCK[0]), math.ceil(batch_n / THREADS_PER_BLOCK[2]))
1500
- if verbose: print(f'Creating frames {start_img_idx}-{end_img_idx} ...')
1501
- _pose_plot_kernel[bpg, THREADS_PER_BLOCK](img_dev, data_dev, circle_size_dev, resolution_dev, colors_dev)
1502
- if verbose: print(f'Moving frames to host {start_img_idx}-{end_img_idx} ...')
1503
- batch_frms = img_dev.copy_to_host()
1504
- if verbose: print(f'Writing frames to host {start_img_idx}-{end_img_idx} ...')
1505
- for img_idx in range(0, batch_n):
1506
- video_writer.write(batch_frms[img_idx].astype(np.uint8))
1507
- video_writer.release()
1508
- total_timer.stop_timer()
1509
- frm_reader.kill()
1510
- if verbose:
1511
- stdout_success(msg=f'Pose-estimation video saved at {save_path}.', elapsed_time=total_timer.elapsed_time_str)
1512
-
1513
-
1514
-
1515
- #x = create_average_frm_cuda(video_path=r"D:\troubleshooting\mitra\project_folder\videos\average_cpu_test\20min.mp4", verbose=True, batch_size=500, async_frame_read=False)
1516
-
1517
- # VIDEO_PATH = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/videos/Trial_1_C24_D1_1.mp4"
1518
- # #
1519
- #
1520
- #
1521
- #
1522
- # avg_frm = create_average_frm_cuda(video_path=VIDEO_PATH, verbose=True, batch_size=100, start_frm=0, end_frm=100, async_frame_read=True, save_path=SAVE_PATH)
1523
- # if _
1524
- # VIDEO_PATH = r"D:\troubleshooting\maplight_ri\project_folder\blob\videos\111.mp4"
1525
- # AVG_FRM = r"D:\troubleshooting\maplight_ri\project_folder\blob\Trial_1_C24_D1_1_bg_removed.png"
1526
- # SAVE_PATH = r"D:\troubleshooting\maplight_ri\project_folder\blob\Trial_1_C24_D1_1_bg_removed.mp4"
1527
- #
1528
-
1529
-
1530
- # VIDEO_PATH = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/videos/111.mp4"
1531
- # AVG_FRM = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/Trial_1_C24_D1_1_bg_removed.png"
1532
- # SAVE_PATH = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/Trial_1_C24_D1_1_bg_removed.mp4"
1533
- # bg_subtraction_cupy(video_path=VIDEO_PATH, avg_frm=AVG_FRM, save_path=SAVE_PATH, batch_size=100, verbose=True, async_frame_read=True, threshold=240, fg_clr=(255, 0,0), bg_clr=(0, 0, 255))
1534
-
1535
-
1536
-
1537
- # DATA_PATH = "/mnt/c/troubleshooting/mitra/project_folder/csv/outlier_corrected_movement_location/501_MA142_Gi_CNO_0521.csv"
1538
- # VIDEO_PATH = "/mnt/c/troubleshooting/mitra/project_folder/videos/501_MA142_Gi_CNO_0521.mp4"
1539
- # SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1540
- # pose_plotter(data=DATA_PATH, video_path=VIDEO_PATH, save_path=SAVE_PATH, circle_size=10, batch_size=1000)
1541
- # # VIDEO_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1542
- # # SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test_ROTATED.mp4"
1543
- # #
1544
- # # rotate_video_cupy(video_path=VIDEO_PATH, save_path=SAVE_PATH, batch_size=1000)
1545
- #
1546
- # #"C:\troubleshooting\mitra\project_folder\csv\outlier_corrected_movement_location\501_MA142_Gi_CNO_0521.csv"
1547
- # pose_plotter(data=DATA_PATH, video_path=VIDEO_PATH, save_path=SAVE_PATH, circle_size=10, batch_size=1000)
1548
-
1549
-
1550
-
1551
-
1552
-
1553
-
1554
-
1555
- # from simba.mixins.geometry_mixin import GeometryMixin
1556
- #
1557
- # video_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4"
1558
- # data_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/csv/outlier_corrected_movement_location/03152021_NOB_IOT_8.csv"
1559
- # save_dir = '/mnt/d/netholabs/yolo_videos/input/mp4_20250606083508'
1560
- #
1561
- # get_video_meta_data(video_path)
1562
- #
1563
- # nose_arr = read_df(file_path=data_path, file_type='csv', usecols=['Nose_x', 'Nose_y', 'Tail_base_x', 'Tail_base_y', 'Lat_left_x', 'Lat_left_y', 'Lat_right_x', 'Lat_right_y']).values.reshape(-1, 4, 2).astype(np.int32) ## READ THE BODY-PART THAT DEFINES THE HULL AND CONVERT TO ARRAY
1564
- #
1565
- # polygons = GeometryMixin().multiframe_bodyparts_to_polygon(data=nose_arr, parallel_offset=60) ## CONVERT THE BODY-PART TO POLYGONS WITH A LITTLE BUFFER
1566
- # polygons = GeometryMixin().multiframe_minimum_rotated_rectangle(shapes=polygons) # CONVERT THE POLYGONS TO RECTANGLES (I.E., WITH 4 UNIQUE POINTS).
1567
- # polygon_lst = [] # GET THE POINTS OF THE RECTANGLES
1568
- # for i in polygons: polygon_lst.append(np.array(i.exterior.coords))
1569
- # polygons = np.stack(polygon_lst, axis=0)
1570
- # sliced_imgs = slice_imgs(video_path=video_path, shapes=polygons, batch_size=500, save_dir=save_dir) #SLICE THE RECTANGLES IN THE VIDEO.
1571
-
1572
- #sliced_imgs = {k: v for k, v in enumerate(sliced_imgs)}
1573
-
1574
- #ImageMixin().img_stack_to_video(imgs=sliced_imgs, fps=29.97, save_path=r'/mnt/d/netholabs/yolo_videos/input/mp4_20250606083508/stacked.mp4')
1575
-
1576
- #get_video_meta_data("/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4")
1577
- # cv2.imshow('asdasdas', sliced_imgs[500])
1578
- # cv2.waitKey(0)
1579
-
1580
- # DATA_PATH = "/mnt/c/troubleshooting/RAT_NOR/project_folder/csv/outlier_corrected_movement_location/03152021_NOB_IOT_8.csv"
1581
- # VIDEO_PATH = "/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4"
1582
- # SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1583
- #
1584
- #
1585
- # # DATA_PATH = "/mnt/c/troubleshooting/mitra/project_folder/csv/outlier_corrected_movement_location/501_MA142_Gi_CNO_0514.csv"
1586
- # # VIDEO_PATH = "/mnt/c/troubleshooting/mitra/project_folder/videos/501_MA142_Gi_CNO_0514.mp4"
1587
- # # SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1588
- # pose_plotter(data=DATA_PATH, video_path=VIDEO_PATH, save_path=SAVE_PATH, circle_size=10, batch_size=100)
1589
-
1590
-
1591
-
1592
- #
1593
- # #from simba.data_processors.cuda.image import create_average_frm_cupy
1594
- # SAVE_PATH = "/mnt/c/Users/sroni/Downloads/bg_remove_nb/bg_removed_ex_7.mp4"
1595
- # VIDEO_PATH = "/mnt/c/Users/sroni/Downloads/bg_remove_nb/open_field.mp4"
1596
- # avg_frm = create_average_frm_cuda(video_path=VIDEO_PATH)
1597
- # #
1598
- # get_video_meta_data(VIDEO_PATH)
1599
- # #
1600
- # bg_subtraction_cuda(video_path=VIDEO_PATH, avg_frm=avg_frm, save_path=SAVE_PATH, threshold=70)
1
+ __author__ = "Simon Nilsson; sronilsson@gmail.com"
2
+
3
+
4
+ import math
5
+ import multiprocessing as mp
6
+ import os
7
+ import time
8
+ from typing import Optional, Tuple, Union
9
+
10
+ try:
11
+ from typing import Literal
12
+ except:
13
+ from typing_extensions import Literal
14
+ try:
15
+ import cupy as cp
16
+ from cupyx.scipy.ndimage import rotate
17
+ except:
18
+ import numpy as cp
19
+ from scipy.ndimage import rotate
20
+
21
+ import platform
22
+ import warnings
23
+ from copy import deepcopy
24
+
25
+ import cv2
26
+ import numpy as np
27
+ from numba import cuda
28
+ from numba.core.errors import NumbaPerformanceWarning
29
+
30
+ from simba.data_processors.cuda.utils import (_cuda_luminance_pixel_to_grey,
31
+ _cuda_mse, _is_cuda_available)
32
+ from simba.mixins.image_mixin import ImageMixin
33
+ from simba.mixins.plotting_mixin import PlottingMixin
34
+ from simba.utils.checks import (check_file_exist_and_readable, check_float,
35
+ check_if_dir_exists,
36
+ check_if_string_value_is_valid_video_timestamp,
37
+ check_if_valid_img, check_if_valid_rgb_tuple,
38
+ check_instance, check_int,
39
+ check_nvidea_gpu_available,
40
+ check_that_hhmmss_start_is_before_end,
41
+ check_valid_array, check_valid_boolean,
42
+ is_video_color)
43
+ from simba.utils.data import (create_color_palette,
44
+ find_frame_numbers_from_time_stamp)
45
+ from simba.utils.enums import OS, Formats
46
+ from simba.utils.errors import (FFMPEGCodecGPUError, FrameRangeError,
47
+ InvalidInputError, SimBAGPUError)
48
+ from simba.utils.lookups import get_current_time
49
+ from simba.utils.printing import SimbaTimer, stdout_success
50
+ from simba.utils.read_write import (
51
+ check_if_hhmmss_timestamp_is_valid_part_of_video,
52
+ concatenate_videos_in_folder, create_directory, get_fn_ext,
53
+ get_memory_usage_array, get_video_meta_data, read_df, read_img,
54
+ read_img_batch_from_video, read_img_batch_from_video_gpu)
55
+ from simba.video_processors.async_frame_reader import (AsyncVideoFrameReader,
56
+ get_async_frame_batch)
57
+
58
+ warnings.simplefilter('ignore', category=NumbaPerformanceWarning)
59
+
60
+
61
+ PHOTOMETRIC = 'photometric'
62
+ DIGITAL = 'digital'
63
+ THREADS_PER_BLOCK = 2024
64
+ if platform.system() != OS.WINDOWS.value: mp.set_start_method("spawn", force=True)
65
+
66
+ def create_average_frm_cupy(video_path: Union[str, os.PathLike],
67
+ start_frm: Optional[int] = None,
68
+ end_frm: Optional[int] = None,
69
+ start_time: Optional[str] = None,
70
+ end_time: Optional[str] = None,
71
+ save_path: Optional[Union[str, os.PathLike]] = None,
72
+ batch_size: Optional[int] = 3000,
73
+ verbose: Optional[bool] = False,
74
+ async_frame_read: bool = False) -> Union[None, np.ndarray]:
75
+
76
+ """
77
+ Computes the average frame using GPU acceleration from a specified range of frames or time interval in a video file.
78
+ This average frame is typically used for background subtraction.
79
+
80
+ The function reads frames from the video, calculates their average, and optionally saves the result
81
+ to a specified file. If `save_path` is provided, the average frame is saved as an image file;
82
+ otherwise, the average frame is returned as a NumPy array.
83
+
84
+ .. seealso::
85
+ For CPU function see :func:`~simba.video_processors.video_processing.create_average_frm`.
86
+ For CUDA function see :func:`~simba.data_processors.cuda.image.create_average_frm_cuda`
87
+
88
+
89
+ .. csv-table::
90
+ :header: EXPECTED RUNTIMES
91
+ :file: ../../../docs/tables/create_average_frm_cupy.csv
92
+ :widths: 10, 45, 45
93
+ :align: center
94
+ :class: simba-table
95
+ :header-rows: 1
96
+
97
+ :param Union[str, os.PathLike] video_path: The path to the video file from which to extract frames.
98
+ :param Optional[int] start_frm: The starting frame number (inclusive). Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both. If both `start_frm` and `end_frm` are `None`, processes all frames in the video.
99
+ :param Optional[int] end_frm: The ending frame number (exclusive). Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both.
100
+ :param Optional[str] start_time: The start time in the format 'HH:MM:SS' from which to begin extracting frames. Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both.
101
+ :param Optional[str] end_time: The end time in the format 'HH:MM:SS' up to which frames should be extracted. Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both.
102
+ :param Optional[Union[str, os.PathLike]] save_path: The path where the average frame image will be saved. If `None`, the average frame is returned as a NumPy array.
103
+ :param Optional[int] batch_size: The number of frames to process in each batch. Default is 3000. Increase if your RAM allows it.
104
+ :param Optional[bool] verbose: If `True`, prints progress and informational messages during execution. Default: False.
105
+ :param bool async_frame_read: If `True`, uses asynchronous frame reading for improved performance. Default: False.
106
+ :return: Returns `None` if the result is saved to `save_path`. Otherwise, returns the average frame as a NumPy array.
107
+
108
+ :example:
109
+ >>> create_average_frm_cupy(video_path=r"C:/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_DOT_4_downsampled.mp4", verbose=True, start_frm=0, end_frm=9000)
110
+ >>> create_average_frm_cupy(video_path=r"C:/videos/my_video.mp4", start_time="00:00:00", end_time="00:01:00", async_frame_read=True, save_path=r"C:/output/avg_frame.png")
111
+
112
+ """
113
+
114
+ def average_3d_stack(image_stack: np.ndarray) -> np.ndarray:
115
+ num_frames, height, width, _ = image_stack.shape
116
+ image_stack = cp.array(image_stack).astype(cp.float32)
117
+ img = cp.clip(cp.sum(image_stack, axis=0) / num_frames, 0, 255).astype(cp.uint8)
118
+ return img.get()
119
+
120
+ if not check_nvidea_gpu_available():
121
+ raise FFMPEGCodecGPUError(msg="No GPU found (as evaluated by nvidea-smi returning None)", source=create_average_frm_cupy.__name__)
122
+
123
+ timer = SimbaTimer(start=True)
124
+ if ((start_frm is not None) or (end_frm is not None)) and ((start_time is not None) or (end_time is not None)):
125
+ raise InvalidInputError(msg=f'Pass start_frm and end_frm OR start_time and end_time', source=create_average_frm_cupy.__name__)
126
+ elif type(start_frm) != type(end_frm):
127
+ raise InvalidInputError(msg=f'Pass start frame and end frame', source=create_average_frm_cupy.__name__)
128
+ elif type(start_time) != type(end_time):
129
+ raise InvalidInputError(msg=f'Pass start time and end time', source=create_average_frm_cupy.__name__)
130
+ if save_path is not None:
131
+ check_if_dir_exists(in_dir=os.path.dirname(save_path), source=create_average_frm_cupy.__name__)
132
+ check_file_exist_and_readable(file_path=video_path)
133
+ video_meta_data = get_video_meta_data(video_path=video_path)
134
+ video_name = get_fn_ext(filepath=video_path)[1]
135
+ if verbose:
136
+ print(f'Getting average frame from {video_name}...')
137
+ if (start_frm is not None) and (end_frm is not None):
138
+ check_int(name='start_frm', value=start_frm, min_value=0, max_value=video_meta_data['frame_count'])
139
+ check_int(name='end_frm', value=end_frm, min_value=0, max_value=video_meta_data['frame_count'])
140
+ if start_frm > end_frm:
141
+ raise InvalidInputError(msg=f'Start frame ({start_frm}) has to be before end frame ({end_frm}).', source=create_average_frm_cupy.__name__)
142
+ frame_ids_lst = list(range(start_frm, end_frm))
143
+ elif (start_time is not None) and (end_time is not None):
144
+ check_if_string_value_is_valid_video_timestamp(value=start_time, name=create_average_frm_cupy.__name__)
145
+ check_if_string_value_is_valid_video_timestamp(value=end_time, name=create_average_frm_cupy.__name__)
146
+ check_that_hhmmss_start_is_before_end(start_time=start_time, end_time=end_time, name=create_average_frm_cupy.__name__)
147
+ check_if_hhmmss_timestamp_is_valid_part_of_video(timestamp=start_time, video_path=video_path)
148
+ frame_ids_lst = find_frame_numbers_from_time_stamp(start_time=start_time, end_time=end_time, fps=video_meta_data['fps'])
149
+ else:
150
+ frame_ids_lst = list(range(0, video_meta_data['frame_count']))
151
+ frame_ids = [frame_ids_lst[i:i+batch_size] for i in range(0,len(frame_ids_lst),batch_size)]
152
+ avg_imgs = []
153
+ if async_frame_read:
154
+ async_frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=5, start_idx=int(min(frame_ids_lst)), end_idx=int(max(frame_ids_lst))+1, verbose=True, gpu=True)
155
+ async_frm_reader.start()
156
+ else:
157
+ async_frm_reader = None
158
+ for batch_cnt in range(len(frame_ids)):
159
+ start_idx, end_idx = frame_ids[batch_cnt][0], frame_ids[batch_cnt][-1]
160
+ if start_idx == end_idx:
161
+ continue
162
+ if not async_frm_reader:
163
+ imgs = read_img_batch_from_video_gpu(video_path=video_path, start_frm=start_idx, end_frm=end_idx, verbose=verbose)
164
+ imgs = np.stack(list(imgs.values()), axis=0)
165
+ else:
166
+ imgs = get_async_frame_batch(batch_reader=async_frm_reader, timeout=15)[2]
167
+ avg_imgs.append(average_3d_stack(image_stack=imgs))
168
+ avg_img = average_3d_stack(image_stack=np.stack(avg_imgs, axis=0))
169
+ timer.stop_timer()
170
+ if async_frm_reader is not None: async_frm_reader.kill()
171
+ if save_path is not None:
172
+ cv2.imwrite(save_path, avg_img)
173
+ if verbose:
174
+ stdout_success(msg=f'Saved average frame at {save_path}', source=create_average_frm_cupy.__name__, elapsed_time=timer.elapsed_time_str)
175
+ else:
176
+ if verbose: stdout_success(msg=f'Average frame compute complete', source=create_average_frm_cupy.__name__, elapsed_time=timer.elapsed_time_str)
177
+ return avg_img
178
+
179
+ def average_3d_stack_cupy(image_stack: np.ndarray) -> np.ndarray:
180
+ num_frames, height, width, _ = image_stack.shape
181
+ image_stack = cp.array(image_stack).astype(cp.float32)
182
+ img = cp.clip(cp.sum(image_stack, axis=0) / num_frames, 0, 255).astype(cp.uint8)
183
+ return img.get()
184
+
185
+ @cuda.jit()
186
+ def _average_3d_stack_cuda_kernel(data, results):
187
+ x, y, i = cuda.grid(3)
188
+ if i < 0 or x < 0 or y < 0:
189
+ return
190
+ if i > data.shape[0] - 1 or y > data.shape[1] - 1 or x > data.shape[2] - 1:
191
+ return
192
+ else:
193
+ sum_value = 0.0
194
+ for n in range(data.shape[0]):
195
+ sum_value += data[n, y, x, i]
196
+ results[y, x, i] = sum_value / data.shape[0]
197
+
198
+ def _average_3d_stack_cuda(image_stack: np.ndarray) -> np.ndarray:
199
+ check_instance(source=_average_3d_stack_cuda.__name__, instance=image_stack, accepted_types=(np.ndarray,))
200
+ check_if_valid_img(data=image_stack[0], source=_average_3d_stack_cuda.__name__)
201
+ if image_stack.ndim != 4:
202
+ return image_stack
203
+ x = np.ascontiguousarray(image_stack)
204
+ x_dev = cuda.to_device(x)
205
+ results = cuda.device_array((x.shape[1], x.shape[2], x.shape[3]), dtype=np.float32)
206
+ grid_x = (x.shape[1] + 16 - 1) // 16
207
+ grid_y = (x.shape[2] + 16 - 1) // 16
208
+ grid_z = 3
209
+ threads_per_block = (16, 16, 1)
210
+ blocks_per_grid = (grid_y, grid_x, grid_z)
211
+ _average_3d_stack_cuda_kernel[blocks_per_grid, threads_per_block](x_dev, results)
212
+ results = results.copy_to_host()
213
+ return results
214
+
215
+
216
+ def create_average_frm_cuda(video_path: Union[str, os.PathLike],
217
+ start_frm: Optional[int] = None,
218
+ end_frm: Optional[int] = None,
219
+ start_time: Optional[str] = None,
220
+ end_time: Optional[str] = None,
221
+ save_path: Optional[Union[str, os.PathLike]] = None,
222
+ batch_size: Optional[int] = 6000,
223
+ verbose: Optional[bool] = False,
224
+ async_frame_read: bool = False) -> Union[None, np.ndarray]:
225
+ """
226
+ Computes the average frame using GPU acceleration from a specified range of frames or time interval in a video file.
227
+ This average frame typically used for background substraction.
228
+
229
+
230
+ The function reads frames from the video, calculates their average, and optionally saves the result
231
+ to a specified file. If `save_path` is provided, the average frame is saved as an image file;
232
+ otherwise, the average frame is returned as a NumPy array.
233
+
234
+ .. seealso::
235
+ For CuPy function see :func:`~simba.data_processors.cuda.image.create_average_frm_cupy`.
236
+ For CPU function see :func:`~simba.video_processors.video_processing.create_average_frm`.
237
+
238
+ :param Union[str, os.PathLike] video_path: The path to the video file from which to extract frames.
239
+ :param Optional[int] start_frm: The starting frame number (inclusive). Either `start_frm`/`end_frm` or `start_time`/`end_time` must be provided, but not both.
240
+ :param Optional[int] end_frm: The ending frame number (exclusive).
241
+ :param Optional[str] start_time: The start time in the format 'HH:MM:SS' from which to begin extracting frames.
242
+ :param Optional[str] end_time: The end time in the format 'HH:MM:SS' up to which frames should be extracted.
243
+ :param Optional[Union[str, os.PathLike]] save_path: The path where the average frame image will be saved. If `None`, the average frame is returned as a NumPy array.
244
+ :param Optional[int] batch_size: The number of frames to process in each batch. Default is 3000. Increase if your RAM allows it.
245
+ :param Optional[bool] verbose: If `True`, prints progress and informational messages during execution.
246
+ :return: Returns `None` if the result is saved to `save_path`. Otherwise, returns the average frame as a NumPy array.
247
+
248
+ :example:
249
+ >>> create_average_frm_cuda(video_path=r"C:/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_DOT_4_downsampled.mp4", verbose=True, start_frm=0, end_frm=9000)
250
+
251
+ """
252
+
253
+ if not check_nvidea_gpu_available():
254
+ raise FFMPEGCodecGPUError(msg="No GPU found (as evaluated by nvidea-smi returning None)", source=create_average_frm_cuda.__name__)
255
+
256
+ if ((start_frm is not None) or (end_frm is not None)) and ((start_time is not None) or (end_time is not None)):
257
+ raise InvalidInputError(msg=f'Pass start_frm and end_frm OR start_time and end_time', source=create_average_frm_cuda.__name__)
258
+ elif type(start_frm) != type(end_frm):
259
+ raise InvalidInputError(msg=f'Pass start frame and end frame', source=create_average_frm_cuda.__name__)
260
+ elif type(start_time) != type(end_time):
261
+ raise InvalidInputError(msg=f'Pass start time and end time', source=create_average_frm_cuda.__name__)
262
+ if save_path is not None:
263
+ check_if_dir_exists(in_dir=os.path.dirname(save_path), source=create_average_frm_cuda.__name__)
264
+ check_file_exist_and_readable(file_path=video_path)
265
+ video_meta_data = get_video_meta_data(video_path=video_path)
266
+ video_name = get_fn_ext(filepath=video_path)[1]
267
+ if verbose:
268
+ print(f'Getting average frame from {video_name}...')
269
+ if (start_frm is not None) and (end_frm is not None):
270
+ check_int(name='start_frm', value=start_frm, min_value=0, max_value=video_meta_data['frame_count'])
271
+ check_int(name='end_frm', value=end_frm, min_value=0, max_value=video_meta_data['frame_count'])
272
+ if start_frm > end_frm:
273
+ raise InvalidInputError(msg=f'Start frame ({start_frm}) has to be before end frame ({end_frm}).', source=create_average_frm_cuda.__name__)
274
+ frame_ids_lst = list(range(start_frm, end_frm))
275
+ elif (start_time is not None) and (end_time is not None):
276
+ check_if_string_value_is_valid_video_timestamp(value=start_time, name=create_average_frm_cuda.__name__)
277
+ check_if_string_value_is_valid_video_timestamp(value=end_time, name=create_average_frm_cuda.__name__)
278
+ check_that_hhmmss_start_is_before_end(start_time=start_time, end_time=end_time, name=create_average_frm_cuda.__name__)
279
+ check_if_hhmmss_timestamp_is_valid_part_of_video(timestamp=start_time, video_path=video_path)
280
+ frame_ids_lst = find_frame_numbers_from_time_stamp(start_time=start_time, end_time=end_time, fps=video_meta_data['fps'])
281
+ else:
282
+ frame_ids_lst = list(range(0, video_meta_data['frame_count']))
283
+ frame_ids = [frame_ids_lst[i:i + batch_size] for i in range(0, len(frame_ids_lst), batch_size)]
284
+ avg_imgs = []
285
+ if async_frame_read:
286
+ async_frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=5, start_idx=int(min(frame_ids_lst)), end_idx=int(max(frame_ids_lst))+1, verbose=True, gpu=True)
287
+ async_frm_reader.start()
288
+ else:
289
+ async_frm_reader = None
290
+ for batch_cnt in range(len(frame_ids)):
291
+ start_idx, end_idx = frame_ids[batch_cnt][0], frame_ids[batch_cnt][-1]
292
+ if start_idx == end_idx:
293
+ continue
294
+ if not async_frm_reader:
295
+ imgs = read_img_batch_from_video_gpu(video_path=video_path, start_frm=start_idx, end_frm=end_idx, verbose=verbose)
296
+ avg_imgs.append(_average_3d_stack_cuda(image_stack=np.stack(list(imgs.values()), axis=0)))
297
+ else:
298
+ imgs = get_async_frame_batch(batch_reader=async_frm_reader, timeout=15)[2]
299
+ avg_imgs.append(_average_3d_stack_cuda(image_stack=imgs))
300
+ avg_img = average_3d_stack_cupy(image_stack=np.stack(avg_imgs, axis=0))
301
+ if save_path is not None:
302
+ cv2.imwrite(save_path, avg_img)
303
+ if verbose:
304
+ stdout_success(msg=f'Saved average frame at {save_path}', source=create_average_frm_cuda.__name__)
305
+ else:
306
+ return avg_img
307
+
308
+
309
+
310
+ @cuda.jit()
311
+ def _photometric(data, results):
312
+ y, x, i = cuda.grid(3)
313
+ if i < 0 or x < 0 or y < 0:
314
+ return
315
+ if i > results.shape[0] - 1 or x > results.shape[1] - 1 or y > results.shape[2] - 1:
316
+ return
317
+ else:
318
+ r, g, b = data[i][x][y][0], data[i][x][y][1], data[i][x][y][2]
319
+ results[i][x][y] = (0.2126 * r) + (0.7152 * g) + (0.0722 * b)
320
+
321
+ @cuda.jit()
322
+ def _digital(data, results):
323
+ y, x, i = cuda.grid(3)
324
+ if i < 0 or x < 0 or y < 0:
325
+ return
326
+ if i > results.shape[0] - 1 or x > results.shape[1] - 1 or y > results.shape[2] - 1:
327
+ return
328
+ else:
329
+ r, g, b = data[i][x][y][0], data[i][x][y][1], data[i][x][y][2]
330
+ results[i][x][y] = (0.299 * r) + (0.587 * g) + (0.114 * b)
331
+
332
+ def img_stack_brightness(x: np.ndarray,
333
+ method: Optional[Literal['photometric', 'digital']] = 'digital',
334
+ ignore_black: Optional[bool] = True) -> np.ndarray:
335
+ """
336
+ Calculate the average brightness of a stack of images using a specified method.
337
+
338
+
339
+ - **Photometric Method**: The brightness is calculated using the formula:
340
+
341
+ .. math::
342
+ \text{brightness} = 0.2126 \cdot R + 0.7152 \cdot G + 0.0722 \cdot B
343
+
344
+ - **Digital Method**: The brightness is calculated using the formula:
345
+
346
+ .. math::
347
+ \text{brightness} = 0.299 \cdot R + 0.587 \cdot G + 0.114 \cdot B
348
+
349
+ .. selalso::
350
+ For CPU function see :func:`~simba.mixins.image_mixin.ImageMixin.brightness_intensity`.
351
+
352
+ :param np.ndarray x: A 4D array of images with dimensions (N, H, W, C), where N is the number of images, H and W are the height and width, and C is the number of channels (RGB).
353
+ :param Optional[Literal['photometric', 'digital']] method: The method to use for calculating brightness. It can be 'photometric' for the standard luminance calculation or 'digital' for an alternative set of coefficients. Default is 'digital'.
354
+ :param Optional[bool] ignore_black: If True, black pixels (i.e., pixels with brightness value 0) will be ignored in the calculation of the average brightness. Default is True.
355
+ :return np.ndarray: A 1D array of average brightness values for each image in the stack. If `ignore_black` is True, black pixels are ignored in the averaging process.
356
+
357
+
358
+ :example:
359
+ >>> imgs = read_img_batch_from_video_gpu(video_path=r"/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_DOT_4_downsampled.mp4", start_frm=0, end_frm=5000)
360
+ >>> imgs = np.stack(list(imgs.values()), axis=0)
361
+ >>> x = img_stack_brightness(x=imgs)
362
+ """
363
+
364
+ check_instance(source=img_stack_brightness.__name__, instance=x, accepted_types=(np.ndarray,))
365
+ check_if_valid_img(data=x[0], source=img_stack_brightness.__name__)
366
+ x = np.ascontiguousarray(x).astype(np.uint8)
367
+ if x.ndim == 4:
368
+ grid_x = (x.shape[1] + 16 - 1) // 16
369
+ grid_y = (x.shape[2] + 16 - 1) // 16
370
+ grid_z = x.shape[0]
371
+ threads_per_block = (16, 16, 1)
372
+ blocks_per_grid = (grid_y, grid_x, grid_z)
373
+ x_dev = cuda.to_device(x)
374
+ results = cuda.device_array((x.shape[0], x.shape[1], x.shape[2]), dtype=np.uint8)
375
+ if method == PHOTOMETRIC:
376
+ _photometric[blocks_per_grid, threads_per_block](x_dev, results)
377
+ else:
378
+ _digital[blocks_per_grid, threads_per_block](x_dev, results)
379
+ results = results.copy_to_host()
380
+ if ignore_black:
381
+ masked_array = np.ma.masked_equal(results, 0)
382
+ results = np.mean(masked_array, axis=(1, 2)).filled(0)
383
+ else:
384
+ results = deepcopy(x)
385
+ results = np.mean(results, axis=(1, 2))
386
+
387
+ return results
388
+
389
+
390
+
391
+ @cuda.jit()
392
+ def _grey_mse(data, ref_img, stride, batch_cnt, mse_arr):
393
+ y, x, i = cuda.grid(3)
394
+ stride = stride[0]
395
+ batch_cnt = batch_cnt[0]
396
+ if batch_cnt == 0:
397
+ if (i - stride) < 0 or x < 0 or y < 0:
398
+ return
399
+ else:
400
+ if i < 0 or x < 0 or y < 0:
401
+ return
402
+ if i > mse_arr.shape[0] - 1 or x > mse_arr.shape[1] - 1 or y > mse_arr.shape[2] - 1:
403
+ return
404
+ else:
405
+ img_val = data[i][x][y]
406
+ if i == 0:
407
+ prev_val = ref_img[x][y]
408
+ else:
409
+ img_val = data[i][x][y]
410
+ prev_val = data[i - stride][x][y]
411
+ mse_arr[i][x][y] = (img_val - prev_val) ** 2
412
+
413
+
414
+ @cuda.jit()
415
+ def _rgb_mse(data, ref_img, stride, batch_cnt, mse_arr):
416
+ y, x, i = cuda.grid(3)
417
+ stride = stride[0]
418
+ batch_cnt = batch_cnt[0]
419
+ if batch_cnt == 0:
420
+ if (i - stride) < 0 or x < 0 or y < 0:
421
+ return
422
+ else:
423
+ if i < 0 or x < 0 or y < 0:
424
+ return
425
+ if i > mse_arr.shape[0] - 1 or x > mse_arr.shape[1] - 1 or y > mse_arr.shape[2] - 1:
426
+ return
427
+ else:
428
+ img_val = data[i][x][y]
429
+ if i != 0:
430
+ prev_val = data[i - stride][x][y]
431
+ else:
432
+ prev_val = ref_img[x][y]
433
+ r_diff = (img_val[0] - prev_val[0]) ** 2
434
+ g_diff = (img_val[1] - prev_val[1]) ** 2
435
+ b_diff = (img_val[2] - prev_val[2]) ** 2
436
+ mse_arr[i][x][y] = r_diff + g_diff + b_diff
437
+
438
+ def stack_sliding_mse(x: np.ndarray,
439
+ stride: Optional[int] = 1,
440
+ batch_size: Optional[int] = 1000) -> np.ndarray:
441
+ r"""
442
+ Computes the Mean Squared Error (MSE) between each image in a stack and a reference image,
443
+ where the reference image is determined by a sliding window approach with a specified stride.
444
+ The function is optimized for large image stacks by processing them in batches.
445
+
446
+ .. seealso::
447
+ For CPU function see :func:`~simba.mixins.image_mixin.ImageMixin.img_stack_mse` and
448
+ :func:`~simba.mixins.image_mixin.ImageMixin.img_sliding_mse`.
449
+
450
+ .. math::
451
+
452
+ \text{MSE} = \frac{1}{n} \sum_{i=1}^{n} (y_i - \hat{y}_i)^2
453
+
454
+ :param np.ndarray x: Input array of images, where the first dimension corresponds to the stack of images. The array should be either 3D (height, width, channels) or 4D (batch, height, width, channels).
455
+ :param Optional[int] stride: The stride or step size for the sliding window that determines the reference image. Defaults to 1, meaning the previous image in the stack is used as the reference.
456
+ :param Optional[int] batch_size: The number of images to process in a single batch. Larger batch sizes may improve performance but require more GPU memory. Defaults to 1000.
457
+ :return: A 1D NumPy array containing the MSE for each image in the stack compared to its corresponding reference image. The length of the array is equal to the number of images in the input stack.
458
+ :rtype: np.ndarray
459
+
460
+ """
461
+
462
+ check_instance(source=stack_sliding_mse.__name__, instance=x, accepted_types=(np.ndarray,))
463
+ check_if_valid_img(data=x[0], source=stack_sliding_mse.__name__)
464
+ check_valid_array(data=x, source=stack_sliding_mse.__name__, accepted_ndims=[3, 4])
465
+ stride = np.array([stride], dtype=np.int32)
466
+ stride_dev = cuda.to_device(stride)
467
+ out = np.full((x.shape[0]), fill_value=0.0, dtype=np.float32)
468
+ for batch_cnt, l in enumerate(range(0, x.shape[0], batch_size)):
469
+ r = l + batch_size
470
+ batch_x = x[l:r]
471
+ if batch_cnt != 0:
472
+ if x.ndim == 3:
473
+ ref_img = x[l-stride].astype(np.uint8).reshape(x.shape[1], x.shape[2])
474
+ else:
475
+ ref_img = x[l-stride].astype(np.uint8).reshape(x.shape[1], x.shape[2], 3)
476
+ else:
477
+ ref_img = np.full_like(x[l], dtype=np.uint8, fill_value=0)
478
+ ref_img = ref_img.astype(np.uint8)
479
+ grid_x = (batch_x.shape[1] + 16 - 1) // 16
480
+ grid_y = (batch_x.shape[2] + 16 - 1) // 16
481
+ grid_z = batch_x.shape[0]
482
+ threads_per_block = (16, 16, 1)
483
+ blocks_per_grid = (grid_y, grid_x, grid_z)
484
+ ref_img_dev = cuda.to_device(ref_img)
485
+ x_dev = cuda.to_device(batch_x)
486
+ results = cuda.device_array((batch_x.shape[0], batch_x.shape[1], batch_x.shape[2]), dtype=np.uint8)
487
+ batch_cnt_dev = np.array([batch_cnt], dtype=np.int32)
488
+ if x.ndim == 3:
489
+ _grey_mse[blocks_per_grid, threads_per_block](x_dev, ref_img_dev, stride_dev, batch_cnt_dev, results)
490
+ else:
491
+ _rgb_mse[blocks_per_grid, threads_per_block](x_dev, ref_img_dev, stride_dev, batch_cnt_dev, results)
492
+ results = results.copy_to_host()
493
+ results = np.mean(results, axis=(1, 2))
494
+ out[l:r] = results
495
+ return out
496
+
497
+
498
+ def img_stack_to_grayscale_cupy(imgs: Union[np.ndarray, cp.ndarray],
499
+ batch_size: Optional[int] = 250) -> np.ndarray:
500
+ """
501
+ Converts a stack of color images to grayscale using GPU acceleration with CuPy.
502
+
503
+ .. seealso::
504
+ For CPU function single images :func:`~simba.mixins.image_mixin.ImageMixin.img_to_greyscale` and
505
+ :func:`~simba.mixins.image_mixin.ImageMixin.img_stack_to_greyscale` for stack. For CUDA JIT, see
506
+ :func:`~simba.data_processors.cuda.image.img_stack_to_grayscale_cuda`.
507
+
508
+ .. csv-table::
509
+ :header: EXPECTED RUNTIMES
510
+ :file: ../../../docs/tables/img_stack_to_grayscale_cupy.csv
511
+ :widths: 10, 90
512
+ :align: center
513
+ :class: simba-table
514
+ :header-rows: 1
515
+
516
+ :param np.ndarray imgs: A 4D NumPy or CuPy array representing a stack of images with shape (num_images, height, width, channels). The images are expected to have 3 channels (RGB).
517
+ :param Optional[int] batch_size: The number of images to process in each batch. Defaults to 250. Adjust this parameter to fit your GPU's memory capacity.
518
+ :return np.ndarray: m A 3D NumPy or CuPy array of shape (num_images, height, width) containing the grayscale images. If the input array is not 4D, the function returns the input as is.
519
+
520
+ :example:
521
+ >>> imgs = read_img_batch_from_video_gpu(video_path=r"/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/2022-06-20_NOB_IOT_1_cropped.mp4", verbose=False, start_frm=0, end_frm=i)
522
+ >>> imgs = np.stack(list(imgs.values()), axis=0).astype(np.uint8)
523
+ >>> gray_imgs = img_stack_to_grayscale_cupy(imgs=imgs)
524
+ """
525
+
526
+
527
+ check_instance(source=img_stack_to_grayscale_cupy.__name__, instance=imgs, accepted_types=(np.ndarray, cp.ndarray))
528
+ check_if_valid_img(data=imgs[0], source=img_stack_to_grayscale_cupy.__name__)
529
+ if imgs.ndim != 4:
530
+ return imgs
531
+ results = cp.zeros((imgs.shape[0], imgs.shape[1], imgs.shape[2]), dtype=np.uint8)
532
+ n = int(np.ceil((imgs.shape[0] / batch_size)))
533
+ imgs = np.array_split(imgs, n)
534
+ start = 0
535
+ for i in range(len(imgs)):
536
+ img_batch = cp.array(imgs[i])
537
+ batch_cnt = img_batch.shape[0]
538
+ end = start + batch_cnt
539
+ vals = (0.07 * img_batch[:, :, :, 2] + 0.72 * img_batch[:, :, :, 1] + 0.21 * img_batch[:, :, :, 0])
540
+ results[start:end] = vals.astype(cp.uint8)
541
+ start = end
542
+ if isinstance(imgs, np.ndarray):
543
+ return results.get()
544
+ else:
545
+ return results
546
+
547
+
548
+
549
+ @cuda.jit()
550
+ def _img_stack_to_grayscale(data, results):
551
+ y, x, i = cuda.grid(3)
552
+ if i < 0 or x < 0 or y < 0:
553
+ return
554
+ if i > results.shape[0] - 1 or x > results.shape[1] - 1 or y > results.shape[2] - 1:
555
+ return
556
+ else:
557
+ b = 0.07 * data[i][x][y][2]
558
+ g = 0.72 * data[i][x][y][1]
559
+ r = 0.21 * data[i][x][y][0]
560
+ val = b + g + r
561
+ results[i][x][y] = val
562
+
563
+ def img_stack_to_grayscale_cuda(x: np.ndarray) -> np.ndarray:
564
+ """
565
+ Convert image stack to grayscale using CUDA.
566
+
567
+ .. seealso::
568
+ For CPU function single images :func:`~simba.mixins.image_mixin.ImageMixin.img_to_greyscale` and
569
+ :func:`~simba.mixins.image_mixin.ImageMixin.img_stack_to_greyscale` for stack. For CuPy, see
570
+ :func:`~simba.data_processors.cuda.image.img_stack_to_grayscale_cupy`.
571
+
572
+ .. csv-table::
573
+ :header: EXPECTED RUNTIMES
574
+ :file: ../../../docs/tables/img_stack_to_grayscale_cuda.csv
575
+ :widths: 10, 45, 45
576
+ :align: center
577
+ :class: simba-table
578
+ :header-rows: 1
579
+
580
+ :param np.ndarray x: 4d array of color images in numpy format.
581
+ :return np.ndarray: 3D array of greyscaled images.
582
+
583
+ :example:
584
+ >>> imgs = read_img_batch_from_video_gpu(video_path=r"/mnt/c/troubleshooting/mitra/project_folder/videos/temp_2/592_MA147_Gq_Saline_0516_downsampled.mp4", verbose=False, start_frm=0, end_frm=i)
585
+ >>> imgs = np.stack(list(imgs.values()), axis=0).astype(np.uint8)
586
+ >>> grey_images = img_stack_to_grayscale_cuda(x=imgs)
587
+ """
588
+ check_instance(source=img_stack_to_grayscale_cuda.__name__, instance=x, accepted_types=(np.ndarray,))
589
+ check_if_valid_img(data=x[0], source=img_stack_to_grayscale_cuda.__name__)
590
+ if x.ndim != 4:
591
+ return x
592
+ x = np.ascontiguousarray(x).astype(np.uint8)
593
+ x_dev = cuda.to_device(x)
594
+ results = cuda.device_array((x.shape[0], x.shape[1], x.shape[2]), dtype=np.uint8)
595
+ grid_x = (x.shape[1] + 16 - 1) // 16
596
+ grid_y = (x.shape[2] + 16 - 1) // 16
597
+ grid_z = x.shape[0]
598
+ threads_per_block = (16, 16, 1)
599
+ blocks_per_grid = (grid_y, grid_x, grid_z)
600
+ _img_stack_to_grayscale[blocks_per_grid, threads_per_block](x_dev, results)
601
+ results = results.copy_to_host()
602
+ return results
603
+
604
+
605
+ def img_stack_to_bw(imgs: np.ndarray,
606
+ lower_thresh: Optional[int] = 100,
607
+ upper_thresh: Optional[int] = 100,
608
+ invert: Optional[bool] = True,
609
+ batch_size: Optional[int] = 1000) -> np.ndarray:
610
+ """
611
+
612
+ Converts a stack of RGB images to binary (black and white) images based on given threshold values using GPU acceleration.
613
+
614
+ This function processes a 4D stack of images, converting each RGB image to a binary image using
615
+ specified lower and upper threshold values. The conversion can be inverted if desired, and the
616
+ processing is done in batches for efficiency.
617
+
618
+ .. csv-table::
619
+ :header: EXPECTED RUNTIMES
620
+ :file: ../../../docs/tables/img_stack_to_bw.csv
621
+ :widths: 10, 90
622
+ :align: center
623
+ :header-rows: 1
624
+
625
+ .. seealso::
626
+ :func:`simba.mixins.image_mixin.ImageMixin.img_to_bw`
627
+ :func:`simba.mixins.image_mixin.ImageMixin.img_stack_to_bw`
628
+
629
+ :param np.ndarray imgs: A 4D NumPy array representing a stack of RGB images, with shape (N, H, W, C).
630
+ :param Optional[int] lower_thresh: The lower threshold value. Pixel values below this threshold are set to 0 (or 1 if `invert` is True). Default is 100.
631
+ :param Optional[int] upper_thresh: The upper threshold value. Pixel values above this threshold are set to 1 (or 0 if `invert` is True). Default is 100.
632
+ :param Optional[bool] invert: If True, the binary conversion is inverted, meaning that values below `lower_thresh` become 1, and values above `upper_thresh` become 0. Default is True.
633
+ :param Optional[int] batch_size: The number of images to process in a single batch. This helps manage memory usage for large stacks of images. Default is 1000.
634
+ :return: A 3D NumPy array of shape (N, H, W), where each image has been converted to a binary format with pixel values of either 0 or 1.
635
+ :rtype: np.ndarray
636
+ """
637
+
638
+ check_valid_array(data=imgs, source=img_stack_to_bw.__name__, accepted_ndims=(4,))
639
+ check_int(name='lower_thresh', value=lower_thresh, max_value=255, min_value=0)
640
+ check_int(name='upper_thresh', value=upper_thresh, max_value=255, min_value=0)
641
+ check_int(name='batch_size', value=batch_size, min_value=1)
642
+ results = cp.full((imgs.shape[0], imgs.shape[1], imgs.shape[2]), fill_value=cp.nan, dtype=cp.uint8)
643
+
644
+ for l in range(0, imgs.shape[0], batch_size):
645
+ r = l + batch_size
646
+ batch_imgs = cp.array(imgs[l:r]).astype(cp.uint8)
647
+ img_mean = cp.sum(batch_imgs, axis=3) / 3
648
+ if not invert:
649
+ batch_imgs = cp.where(img_mean < lower_thresh, 0, img_mean)
650
+ batch_imgs = cp.where(batch_imgs > upper_thresh, 1, batch_imgs).astype(cp.uint8)
651
+ else:
652
+ batch_imgs = cp.where(img_mean < lower_thresh, 1, img_mean)
653
+ batch_imgs = cp.where(batch_imgs > upper_thresh, 0, batch_imgs).astype(cp.uint8)
654
+
655
+ results[l:r] = batch_imgs
656
+
657
+ return results.get()
658
+
659
+ def segment_img_stack_vertical(imgs: np.ndarray,
660
+ pct: float,
661
+ left: bool,
662
+ right: bool) -> np.ndarray:
663
+ """
664
+ Segment a stack of images vertically based on a given percentage using GPU acceleration. For example, return the left half, right half, or senter half of each image in the stack.
665
+
666
+ .. note::
667
+ If both left and right are true, the center portion is returned.
668
+
669
+ .. seealso::
670
+ :func:`simba.mixins.image_mixin.ImageMixin.segment_img_vertical`
671
+
672
+ :param np.ndarray imgs: A 3D or 4D NumPy array representing a stack of images. The array should have shape (N, H, W) for grayscale images or (N, H, W, C) for color images.
673
+ :param float pct: The percentage of the image width to be used for segmentation. This value should be between a small positive value (e.g., 10e-6) and 0.99.
674
+ :param bool left: If True, the left side of the image stack will be segmented.
675
+ :param bool right: If True, the right side of the image stack will be segmented.
676
+ :return: A NumPy array containing the segmented images, with the same number of dimensions as the input.
677
+ :rtype: np.ndarray
678
+ """
679
+
680
+ check_valid_boolean(value=[left, right], source=segment_img_stack_vertical.__name__)
681
+ check_float(name=f'{segment_img_stack_vertical.__name__} pct', value=pct, min_value=10e-6, max_value=0.99)
682
+ check_valid_array(data=imgs, source=f'{segment_img_stack_vertical.__name__} imgs', accepted_ndims=(3, 4,), accepted_dtypes=Formats.NUMERIC_DTYPES.value)
683
+ if not left and not right:
684
+ raise InvalidInputError(msg='left are right argument are both False. Set one or both to True.', source=segment_img_stack_vertical.__name__)
685
+ imgs = cp.array(imgs).astype(cp.uint8)
686
+ h, w = imgs[0].shape[0], imgs[0].shape[1]
687
+ px_crop = int(w * pct)
688
+ if left and not right:
689
+ imgs = imgs[:, :, :px_crop]
690
+ elif right and not left:
691
+ imgs = imgs[:, :, imgs.shape[2] - px_crop:]
692
+ else:
693
+ imgs = imgs[:, :, int(px_crop/2):int(imgs.shape[2] - (px_crop/2))]
694
+ return imgs.get()
695
+
696
+
697
+ def segment_img_stack_horizontal(imgs: np.ndarray,
698
+ pct: float,
699
+ upper: Optional[bool] = False,
700
+ lower: Optional[bool] = False) -> np.ndarray:
701
+
702
+ """
703
+ Segment a stack of images horizontally based on a given percentage using GPU acceleration. For example, return the top half, bottom half, or center half of each image in the stack.
704
+
705
+ .. note::
706
+ If both top and bottom are true, the center portion is returned.
707
+
708
+ .. seealso::
709
+ :func:`simba.mixins.image_mixin.ImageMixin.segment_img_stack_horizontal`
710
+
711
+ :param np.ndarray imgs: A 3D or 4D NumPy array representing a stack of images. The array should have shape (N, H, W) for grayscale images or (N, H, W, C) for color images.
712
+ :param float pct: The percentage of the image width to be used for segmentation. This value should be between a small positive value (e.g., 10e-6) and 0.99.
713
+ :param bool upper: If True, the top part of the image stack will be segmented.
714
+ :param bool lower: If True, the bottom part of the image stack will be segmented.
715
+ :return: A NumPy array containing the segmented images, with the same number of dimensions as the input.
716
+ :rtype: np.ndarray
717
+ """
718
+
719
+ check_valid_boolean(value=[upper, lower], source=segment_img_stack_horizontal.__name__)
720
+ check_float(name=f'{segment_img_stack_horizontal.__name__} pct', value=pct, min_value=10e-6, max_value=0.99)
721
+ check_valid_array(data=imgs, source=f'{segment_img_stack_vertical.__name__} imgs', accepted_ndims=(3, 4,), accepted_dtypes=Formats.NUMERIC_DTYPES.value)
722
+ if not upper and not lower:
723
+ raise InvalidInputError(msg='upper and lower argument are both False. Set one or both to True.', source=segment_img_stack_horizontal.__name__)
724
+ imgs = cp.array(imgs).astype(cp.uint8)
725
+ h, w = imgs[0].shape[0], imgs[0].shape[1]
726
+ px_crop = int(h * pct)
727
+ if upper and not lower:
728
+ imgs = imgs[: , :px_crop, :]
729
+ elif not upper and lower:
730
+ imgs = imgs[:, imgs.shape[0] - px_crop :, :]
731
+ else:
732
+ imgs = imgs[:, int(px_crop/2):int((imgs.shape[0] - px_crop) / 2), :]
733
+
734
+ return imgs.get()
735
+
736
+
737
+
738
+ @cuda.jit(device=True)
739
+ def _cuda_is_inside_polygon(x, y, polygon_vertices):
740
+ """
741
+ Checks if the pixel location is inside the polygon.
742
+
743
+ :param int x: Pixel x location.
744
+ :param int y: Pixel y location.
745
+ :param np.ndarray polygon_vertices: 2-dimensional array representing the x and y coordinates of the polygon vertices.
746
+ :return: Boolean representing if the x and y are located in the polygon.
747
+ """
748
+
749
+ n = len(polygon_vertices)
750
+ p2x, p2y, xints, inside = 0.0, 0.0, 0.0, False
751
+ p1x, p1y = polygon_vertices[0]
752
+ for j in range(n + 1):
753
+ p2x, p2y = polygon_vertices[j % n]
754
+ if ((y > min(p1y, p2y)) and (y <= max(p1y, p2y)) and (x <= max(p1x, p2x))):
755
+ if p1y != p2y:
756
+ xints = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
757
+ if p1x == p2x or x <= xints:
758
+ inside = not inside
759
+ p1x, p1y = p2x, p2y
760
+ return inside
761
+
762
+
763
+
764
+ @cuda.jit(device=True)
765
+ def _cuda_is_inside_circle(x, y, circle_x, circle_y, circle_r):
766
+ """
767
+ Device func to check if the pixel location is inside a circle.
768
+
769
+ :param int x: Pixel x location.
770
+ :param int y: Pixel y location.
771
+ :param int circle_x: Center of circle x coordinate.
772
+ :param int circle_y: Center of circle y coordinate.
773
+ :param int y: Circle radius.
774
+ :return: Boolean representing if the x and y are located in the circle.
775
+ """
776
+
777
+ p = (math.sqrt((x - circle_x) ** 2 + (y - circle_y) ** 2))
778
+ if p <= circle_r:
779
+ return True
780
+ else:
781
+ return False
782
+ @cuda.jit()
783
+ def _cuda_create_rectangle_masks(shapes, imgs, results, bboxes):
784
+ """
785
+ CUDA kernel to apply rectangular masks to a batch of images.
786
+ """
787
+ n, y, x = cuda.grid(3)
788
+ if n >= imgs.shape[0]:
789
+ return
790
+
791
+ x_min = bboxes[n, 0]
792
+ y_min = bboxes[n, 1]
793
+ x_max = bboxes[n, 2]
794
+ y_max = bboxes[n, 3]
795
+
796
+ max_w = x_max - x_min
797
+ max_h = y_max - y_min
798
+
799
+ if x >= max_w or y >= max_h:
800
+ return
801
+
802
+ x_input = x + x_min
803
+ y_input = y + y_min
804
+
805
+ polygon = shapes[n]
806
+
807
+ if _cuda_is_inside_polygon(x_input, y_input, polygon):
808
+ if imgs.ndim == 4:
809
+ for c in range(imgs.shape[3]):
810
+ results[n, y, x, c] = imgs[n, y_input, x_input, c]
811
+ else:
812
+ results[n, y, x] = imgs[n, y_input, x_input]
813
+
814
+ @cuda.jit()
815
+ def _cuda_create_circle_masks(shapes, imgs, results, bboxes):
816
+ """
817
+ CUDA kernel to apply circular masks to a batch of images.
818
+ """
819
+ n, y, x = cuda.grid(3)
820
+ if n >= imgs.shape[0]:
821
+ return
822
+
823
+ x_min = bboxes[n, 0]
824
+ y_min = bboxes[n, 1]
825
+ x_max = bboxes[n, 2]
826
+ y_max = bboxes[n, 3]
827
+
828
+ max_w = x_max - x_min
829
+ max_h = y_max - y_min
830
+
831
+ if x >= max_w or y >= max_h:
832
+ return
833
+
834
+ x_input = x + x_min
835
+ y_input = y + y_min
836
+
837
+ circle_x = shapes[n, 0]
838
+ circle_y = shapes[n, 1]
839
+ circle_r = shapes[n, 2]
840
+
841
+
842
+ if _cuda_is_inside_circle(x_input, y_input, circle_x, circle_y, circle_r):
843
+ if imgs.ndim == 4:
844
+ for c in range(imgs.shape[3]):
845
+ results[n, y, x, c] = imgs[n, y_input, x_input, c]
846
+ else:
847
+ results[n, y, x] = imgs[n, y_input, x_input]
848
+
849
+
850
+ def _get_bboxes(shapes):
851
+ """
852
+ Helper to get geometries in :func:`simba.data_processors.cuda.image.slice_imgs`.
853
+ """
854
+ bboxes = []
855
+ for shape in shapes:
856
+ if shape.shape[0] == 3: # circle: [cx, cy, r]
857
+ cx, cy, r = shape
858
+ x_min = int(np.floor(cx - r))
859
+ y_min = int(np.floor(cy - r))
860
+ x_max = int(np.ceil(cx + r))
861
+ y_max = int(np.ceil(cy + r))
862
+ else:
863
+ xs = shape[:, 0]
864
+ ys = shape[:, 1]
865
+ x_min = int(np.floor(xs.min()))
866
+ y_min = int(np.floor(ys.min()))
867
+ x_max = int(np.ceil(xs.max()))
868
+ y_max = int(np.ceil(ys.max()))
869
+ bboxes.append([x_min, y_min, x_max, y_max])
870
+ return np.array(bboxes, dtype=np.int32)
871
+
872
+ def slice_imgs(video_path: Union[str, os.PathLike],
873
+ shapes: np.ndarray,
874
+ batch_size: int = 1000,
875
+ verbose: bool = True,
876
+ save_dir: Optional[Union[str, os.PathLike]] = None):
877
+ """
878
+ Slice frames from a video based on given polygon or circle coordinates, and return or save masked/cropped frame regions using GPU acceleration.
879
+
880
+ This function supports two types of shapes:
881
+ - Polygon: array of shape (N, M, 2), where N = number of frames, M = number of polygon vertices.
882
+ - Circle: array of shape (N, 3), where each row represents [center_x, center_y, radius].
883
+
884
+ :param Union[str, os.PathLike] video_path: Path to the input video file.
885
+ :param np.ndarray shapes: Array of polygon coordinates or circle parameters for each frame. - Polygon: shape = (n_frames, n_vertices, 2) - Circle: shape = (n_frames, 3)
886
+ :param int batch_size: Number of frames to process per batch during GPU processing. Default 1000.
887
+ :param bool verbose: Whether to print progress and status messages. Default True.
888
+ :param Optional[Union[str, os.PathLike]] save_dir: If provided, the masked/cropped video will be saved in this directory. Otherwise, the cropped image stack will be returned.
889
+
890
+ .. video:: _static/img/simba.sandbox.cuda_slice_w_crop.slice_imgs.webm
891
+ :width: 900
892
+ :loop:
893
+
894
+ .. video:: _static/img/slice_imgs_gpu.webm
895
+ :width: 800
896
+ :autoplay:
897
+ :loop:
898
+
899
+ .. csv-table::
900
+ :header: EXPECTED RUNTIMES
901
+ :file: ../../../docs/tables/slice_imgs.csv
902
+ :widths: 10, 90
903
+ :align: center
904
+ :class: simba-table
905
+ :header-rows: 1
906
+
907
+ .. note::
908
+ For CPU multicore implementation, see :func:`simba.mixins.image_mixin.ImageMixin.slice_shapes_in_imgs`.
909
+ For single core process, see :func:`simba.mixins.image_mixin.ImageMixin.slice_shapes_in_img`
910
+
911
+ :example I:
912
+ Example 1: Mask video using circular regions derived from body part center positions
913
+ >>> video_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4"
914
+ >>> data_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/csv/outlier_corrected_movement_location/03152021_NOB_IOT_8.csv"
915
+ >>> save_dir = '/mnt/d/netholabs/yolo_videos/input/mp4_20250606083508'
916
+ >>> nose_arr = read_df(file_path=data_path, file_type='csv', usecols=['Nose_x', 'Nose_y']).values.reshape(-1, 2).astype(np.int32)
917
+ >>> polygons = GeometryMixin().multiframe_bodyparts_to_circle(data=nose_arr, parallel_offset=60)
918
+ >>> polygon_lst = []
919
+ >>> center = GeometryMixin.get_center(polygons)
920
+ >>> polygons = np.hstack([center, np.full(shape=(len(center), 1), fill_value=60)])
921
+ >>> slice_imgs(video_path=video_path, shapes=polygons, batch_size=500, save_dir=save_dir)
922
+
923
+ :example II:
924
+ Example 2: Mask video using minimum rotated rectangles from polygon hulls
925
+ >>> video_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4"
926
+ >>> data_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/csv/outlier_corrected_movement_location/03152021_NOB_IOT_8.csv"
927
+ >>> save_dir = '/mnt/d/netholabs/yolo_videos/input/mp4_20250606083508'
928
+ >>> nose_arr = read_df(file_path=data_path, file_type='csv', usecols=['Nose_x', 'Nose_y', 'Tail_base_x', 'Tail_base_y', 'Lat_left_x', 'Lat_left_y', 'Lat_right_x', 'Lat_right_y']).values.reshape(-1, 4, 2).astype(np.int32) ## READ THE BODY-PART THAT DEFINES THE HULL AND CONVERT TO ARRAY
929
+ >>> polygons = GeometryMixin().multiframe_bodyparts_to_polygon(data=nose_arr, parallel_offset=60)
930
+ >>> polygons = GeometryMixin().multiframe_minimum_rotated_rectangle(shapes=polygons)
931
+ >>> polygon_lst = []
932
+ >>> for i in polygons:
933
+ >>> polygon_lst.append(np.array(i.exterior.coords).astype(np.int32))
934
+ >>> polygons = np.stack(polygon_lst, axis=0)
935
+ >>> sliced_imgs = slice_imgs(video_path=video_path, shapes=polygons, batch_size=500, save_dir=save_dir)
936
+ """
937
+
938
+ THREADS_PER_BLOCK = (16, 8, 8)
939
+ video_meta_data = get_video_meta_data(video_path=video_path, fps_as_int=False)
940
+ video_meta_data['frame_count'] = shapes.shape[0]
941
+ n, w, h = video_meta_data['frame_count'], video_meta_data['width'], video_meta_data['height']
942
+ is_color = ImageMixin.is_video_color(video=video_path)
943
+ timer, save_temp_dir, results, video_out_path = SimbaTimer(start=True), None, None, None
944
+ bboxes = _get_bboxes(shapes)
945
+ crop_heights = bboxes[:, 3] - bboxes[:, 1]
946
+ crop_widths = bboxes[:, 2] - bboxes[:, 0]
947
+
948
+ max_h = int(np.max(crop_heights))
949
+ max_w = int(np.max(crop_widths))
950
+
951
+ if save_dir is None:
952
+ if not is_color:
953
+ results = np.zeros((n, max_h, max_w), dtype=np.uint8)
954
+ else:
955
+ results = np.zeros((n, max_h, max_w, 3), dtype=np.uint8)
956
+ else:
957
+ save_temp_dir = os.path.join(save_dir, f'temp_{video_meta_data["video_name"]}')
958
+ create_directory(paths=save_temp_dir, overwrite=True)
959
+ video_out_path = os.path.join(save_dir, f'{video_meta_data["video_name"]}.mp4')
960
+
961
+ frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, verbose=True, max_que_size=2)
962
+ frm_reader.start()
963
+
964
+ for batch_cnt in range(frm_reader.batch_cnt):
965
+ start_img_idx, end_img_idx, batch_imgs = get_async_frame_batch(batch_reader=frm_reader, timeout=10)
966
+ if verbose:
967
+ print(f'Processing images {start_img_idx} - {end_img_idx} (of {n}; batch count: {batch_cnt+1}/{frm_reader.batch_cnt})...')
968
+
969
+ batch_save_path = os.path.join(save_temp_dir, f'{batch_cnt}.mp4') if save_dir is not None else None
970
+
971
+ batch_shapes = shapes[start_img_idx:end_img_idx].astype(np.int32)
972
+ batch_bboxes = bboxes[start_img_idx:end_img_idx]
973
+
974
+ x_dev = cuda.to_device(batch_shapes)
975
+ bboxes_dev = cuda.to_device(batch_bboxes)
976
+ batch_img_dev = cuda.to_device(batch_imgs)
977
+
978
+ if not is_color:
979
+ batch_results = np.zeros((batch_imgs.shape[0], max_h, max_w), dtype=np.uint8)
980
+ else:
981
+ batch_results = np.zeros((batch_imgs.shape[0], max_h, max_w, 3), dtype=np.uint8)
982
+ batch_results_dev = cuda.to_device(batch_results)
983
+ grid_n = math.ceil(batch_imgs.shape[0] / THREADS_PER_BLOCK[0])
984
+ grid_y = math.ceil(max_h / THREADS_PER_BLOCK[1])
985
+ grid_x = math.ceil(max_w / THREADS_PER_BLOCK[2])
986
+ bpg = (grid_n, grid_y, grid_x)
987
+ if batch_shapes.shape[1] == 3:
988
+ _cuda_create_circle_masks[bpg, THREADS_PER_BLOCK](x_dev, batch_img_dev, batch_results_dev, bboxes_dev)
989
+ else:
990
+ _cuda_create_rectangle_masks[bpg, THREADS_PER_BLOCK](x_dev, batch_img_dev, batch_results_dev, bboxes_dev)
991
+ if save_dir is None:
992
+ results[start_img_idx:end_img_idx] = batch_results_dev.copy_to_host()
993
+ else:
994
+ frame_results = batch_results_dev.copy_to_host()
995
+ results = {k: v for k, v in enumerate(frame_results)}
996
+ ImageMixin().img_stack_to_video(imgs=results, fps=video_meta_data['fps'], save_path=batch_save_path, verbose=False)
997
+
998
+ frm_reader.kill()
999
+ timer.stop_timer()
1000
+
1001
+ if save_dir:
1002
+ concatenate_videos_in_folder(in_folder=save_temp_dir, save_path=video_out_path, remove_splits=True, gpu=True)
1003
+ if verbose:
1004
+ stdout_success(msg=f'Shapes sliced in video saved at {video_out_path}.', elapsed_time=timer.elapsed_time_str)
1005
+ return None
1006
+ else:
1007
+ if verbose:
1008
+ stdout_success(msg='Shapes sliced in video.', elapsed_time=timer.elapsed_time_str)
1009
+ return results
1010
+
1011
+
1012
+ @cuda.jit()
1013
+ def _sliding_psnr(data, stride, results):
1014
+ r = cuda.grid(1)
1015
+ l = int(r - stride[0])
1016
+ if (r < 0) or (r > data.shape[0] -1):
1017
+ return
1018
+ if l < 0:
1019
+ return
1020
+ else:
1021
+ img_1, img_2 = data[r], data[l]
1022
+ mse = _cuda_mse(img_1, img_2)
1023
+ if mse == 0:
1024
+ results[r] = 0.0
1025
+ else:
1026
+ results[r] = 20 * math.log10(255 / math.sqrt(mse))
1027
+
1028
+ def sliding_psnr(data: np.ndarray,
1029
+ stride_s: int,
1030
+ sample_rate: float) -> np.ndarray:
1031
+ r"""
1032
+ Computes the Peak Signal-to-Noise Ratio (PSNR) between pairs of images in a stack using a sliding window approach.
1033
+
1034
+ This function calculates PSNR for each image in a stack compared to another image in the stack that is separated by a specified stride.
1035
+ The sliding window approach allows for the comparison of image quality over a sequence of images.
1036
+
1037
+ .. note::
1038
+ - PSNR values are measured in decibels (dB).
1039
+ - Higher PSNR values indicate better quality with minimal differences from the reference image.
1040
+ - Lower PSNR values indicate higher distortion or noise.
1041
+
1042
+ .. math::
1043
+
1044
+ \text{PSNR} = 20 \log_{10} \left( \frac{\text{MAX}}{\sqrt{\text{MSE}}} \right)
1045
+
1046
+ where:
1047
+ - :math:`\text{MAX}` is the maximum possible pixel value (255 for 8-bit images)
1048
+ - :math:`\text{MSE}` is the Mean Squared Error between the two images
1049
+
1050
+ :param data: A 4D NumPy array of shape (N, H, W, C) representing a stack of images, where N is the number of images, H is the height, W is the width, and C is the number of color channels.
1051
+ :param stride_s: The base stride length in terms of the number of images between the images being compared. Determines the separation between images for comparison in the stack.
1052
+ :param sample_rate: The sample rate to scale the stride length. This allows for adjusting the stride dynamically based on the sample rate.
1053
+ :return: A 1D NumPy array of PSNR values, where each element represents the PSNR between the image at index `r` and the image at index `l = r - stride`, for all valid indices `r`.
1054
+ :rtype: np.ndarray
1055
+
1056
+ :example:
1057
+ >>> data = ImageMixin().read_img_batch_from_video(video_path =r"/mnt/c/troubleshooting/mitra/project_folder/videos/clipped/501_MA142_Gi_CNO_0514_clipped.mp4", start_frm=0, end_frm=299)
1058
+ >>> data = np.stack(list(data.values()), axis=0).astype(np.uint8)
1059
+ >>> data = ImageMixin.img_stack_to_greyscale(imgs=data)
1060
+ >>> p = sliding_psnr(data=data, stride_s=1, sample_rate=1)
1061
+ """
1062
+
1063
+ results = np.full(data.shape[0], fill_value=255.0, dtype=np.float32)
1064
+ stride = np.array([stride_s * sample_rate], dtype=np.int32)
1065
+ if stride[0] < 1: stride[0] = 1
1066
+ stride_dev = cuda.to_device(stride)
1067
+ results_dev = cuda.to_device(results)
1068
+ data_dev = cuda.to_device(data)
1069
+ bpg = (data.shape[0] + (THREADS_PER_BLOCK - 1)) // THREADS_PER_BLOCK
1070
+ _sliding_psnr[bpg, THREADS_PER_BLOCK](data_dev, stride_dev, results_dev)
1071
+ return results_dev.copy_to_host()
1072
+
1073
+ def rotate_img_stack_cupy(imgs: np.ndarray,
1074
+ rotation_degrees: Optional[float] = 180,
1075
+ batch_size: Optional[int] = 500,
1076
+ verbose: bool = True) -> np.ndarray:
1077
+ """
1078
+ Rotates a stack of images by a specified number of degrees using GPU acceleration with CuPy.
1079
+
1080
+ Accepts a 3D (single-channel images) or 4D (multichannel images) NumPy array, rotates each image in the stack by the specified degree around the center, and returns the result as a NumPy array.
1081
+
1082
+ :param np.ndarray imgs: The input stack of images to be rotated. Expected to be a NumPy array with 3 or 4 dimensions. 3D shape: (num_images, height, width) - 4D shape: (num_images, height, width, channels)
1083
+ :param Optional[float] rotation_degrees: The angle by which the images should be rotated, in degrees. Must be between 1 and 359 degrees. Defaults to 180 degrees.
1084
+ :param Optional[int] batch_size: Number of images to process on GPU in each batch. Decrease if data can't fit on GPU RAM.
1085
+ :returns: A NumPy array containing the rotated images with the same shape as the input.
1086
+ :rtype: np.ndarray
1087
+
1088
+ :example:
1089
+ >>> video_path = r"/mnt/c/troubleshooting/mitra/project_folder/videos/F0_gq_Saline_0626_clipped.mp4"
1090
+ >>> imgs = read_img_batch_from_video_gpu(video_path=video_path)
1091
+ >>> imgs = np.stack(np.array(list(imgs.values())), axis=0)
1092
+ >>> imgs = rotate_img_stack_cupy(imgs=imgs, rotation=50)
1093
+ """
1094
+
1095
+ timer = SimbaTimer(start=True)
1096
+ check_valid_array(data=imgs, source=f'{rotate_img_stack_cupy.__name__} imgs', accepted_ndims=(3, 4))
1097
+ check_int(name=f'{rotate_img_stack_cupy.__name__} rotation', value=rotation_degrees, min_value=1, max_value=359)
1098
+ check_valid_boolean(value=verbose, source=f'{rotate_img_stack_cupy.__name__} verbose', raise_error=True)
1099
+
1100
+ first_img = cp.array(imgs[0:1])
1101
+ rotated_first = rotate(input=first_img, angle=rotation_degrees, axes=(2, 1), reshape=True)
1102
+ output_shape = (imgs.shape[0],) + rotated_first.shape[1:]
1103
+
1104
+ results = cp.zeros(output_shape, dtype=np.uint8)
1105
+
1106
+ for l in range(0, imgs.shape[0], batch_size):
1107
+ r = min(l + batch_size, imgs.shape[0])
1108
+ if verbose:
1109
+ print(f'Rotating image {l}-{r}...')
1110
+ batch_imgs = cp.array(imgs[l:r])
1111
+ rotated_batch = rotate(input=batch_imgs, angle=rotation_degrees, axes=(2, 1), reshape=True)
1112
+ results[l:r] = rotated_batch
1113
+
1114
+ if hasattr(results, 'get'):
1115
+ final_results = results.get()
1116
+ else:
1117
+ final_results = results
1118
+
1119
+ timer.stop_timer()
1120
+ if verbose: print(f'[{get_current_time()}] Image rotation complete (elapsed time: {timer.elapsed_time_str}s)')
1121
+ return final_results
1122
+
1123
+ def rotate_video_cupy(video_path: Union[str, os.PathLike],
1124
+ save_path: Optional[Union[str, os.PathLike]] = None,
1125
+ rotation_degrees: Optional[float] = 180,
1126
+ batch_size: Optional[int] = None,
1127
+ verbose: Optional[bool] = True) -> None:
1128
+ """
1129
+ Rotates a video by a specified angle using GPU acceleration and CuPy for image processing.
1130
+
1131
+ :param Union[str, os.PathLike] video_path: Path to the input video file.
1132
+ :param Optional[Union[str, os.PathLike]] save_path: Path to save the rotated video. If None, saves the video in the same directory as the input with '_rotated_<rotation_degrees>' appended to the filename.
1133
+ :param nptional[float] rotation_degrees: Degrees to rotate the video. Must be between 1 and 359 degrees. Default is 180.
1134
+ :param Optional[int] batch_size: The number of frames to process in each batch. Deafults to None meaning all images will be processed in a single batch.
1135
+ :returns: None.
1136
+
1137
+ :example:
1138
+ >>> video_path = r"/mnt/c/troubleshooting/mitra/project_folder/videos/F0_gq_Saline_0626_clipped.mp4"
1139
+ >>> rotate_video_cupy(video_path=video_path, rotation_degrees=45)
1140
+ """
1141
+
1142
+ timer = SimbaTimer(start=True)
1143
+ check_int(name=f'{rotate_img_stack_cupy.__name__} rotation', value=rotation_degrees, min_value=1, max_value=359)
1144
+ check_valid_boolean(source=f'{rotate_img_stack_cupy.__name__} verbose', value=verbose)
1145
+ if save_path is None:
1146
+ video_dir, video_name, _ = get_fn_ext(filepath=video_path)
1147
+ save_path = os.path.join(video_dir, f'{video_name}_rotated_{rotation_degrees}.mp4')
1148
+ video_meta_data = get_video_meta_data(video_path=video_path)
1149
+ if batch_size is not None:
1150
+ check_int(name=f'{rotate_img_stack_cupy.__name__} batch_size', value=batch_size, min_value=1)
1151
+ else:
1152
+ batch_size = video_meta_data['frame_count']
1153
+ fourcc = cv2.VideoWriter_fourcc(*Formats.MP4_CODEC.value)
1154
+ is_clr = ImageMixin.is_video_color(video=video_path)
1155
+ frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=3, verbose=False)
1156
+ frm_reader.start()
1157
+ for batch_cnt in range(frm_reader.batch_cnt):
1158
+ start_idx, end_idx, imgs = get_async_frame_batch(batch_reader=frm_reader, timeout=10)
1159
+ if verbose:
1160
+ print(f'Rotating frames {start_idx}-{end_idx}... (of {video_meta_data["frame_count"]}, video: {video_meta_data["video_name"]})')
1161
+ imgs = rotate_img_stack_cupy(imgs=imgs, rotation_degrees=rotation_degrees, batch_size=batch_size)
1162
+ if batch_cnt == 0:
1163
+ writer = cv2.VideoWriter(save_path, fourcc, video_meta_data['fps'], (imgs.shape[2], imgs.shape[1]), isColor=is_clr)
1164
+ for img in imgs: writer.write(img)
1165
+ writer.release()
1166
+ timer.stop_timer()
1167
+ frm_reader.kill()
1168
+ if verbose:
1169
+ stdout_success(f'Rotated video saved at {save_path}', source=rotate_video_cupy.__name__)
1170
+
1171
+
1172
+ @cuda.jit()
1173
+ def _bg_subtraction_cuda_kernel(imgs, avg_img, results, is_clr, fg_clr, threshold):
1174
+ x, y, n = cuda.grid(3)
1175
+ if n < 0 or n > (imgs.shape[0] -1):
1176
+ return
1177
+ if y < 0 or y > (imgs.shape[1] -1):
1178
+ return
1179
+ if x < 0 or x > (imgs.shape[2] -1):
1180
+ return
1181
+ if is_clr[0] == 1:
1182
+ r1, g1, b1 = imgs[n][y][x][0],imgs[n][y][x][1], imgs[n][y][x][2]
1183
+ r2, g2, b2 = avg_img[y][x][0], avg_img[y][x][1], avg_img[y][x][2]
1184
+ r_diff, g_diff, b_diff = abs(r1-r2), abs(g1-g2), abs(b1-b2)
1185
+ grey_diff = _cuda_luminance_pixel_to_grey(r_diff, g_diff, b_diff)
1186
+ if grey_diff > threshold[0]:
1187
+ if fg_clr[0] != -1:
1188
+ r_out, g_out, b_out = fg_clr[0], fg_clr[1], fg_clr[2]
1189
+ else:
1190
+ r_out, g_out, b_out = r1, g1, b1
1191
+ else:
1192
+ r_out, g_out, b_out = results[n][y][x][0], results[n][y][x][1], results[n][y][x][2]
1193
+ results[n][y][x][0], results[n][y][x][1], results[n][y][x][2] = r_out, g_out, b_out
1194
+
1195
+ else:
1196
+ val_1, val_2 = imgs[n][y][x][0], avg_img[y][x][0]
1197
+ grey_diff = abs(val_1-val_2)
1198
+ if grey_diff > threshold[0]:
1199
+ if fg_clr[0] != -1:
1200
+ val_out = val_1
1201
+ else:
1202
+ val_out = 255
1203
+ else:
1204
+ val_out = 0
1205
+ results[n][y][x] = val_out
1206
+
1207
+
1208
+ def bg_subtraction_cuda(video_path: Union[str, os.PathLike],
1209
+ avg_frm: np.ndarray,
1210
+ save_path: Optional[Union[str, os.PathLike]] = None,
1211
+ bg_clr: Optional[Tuple[int, int, int]] = (0, 0, 0),
1212
+ fg_clr: Optional[Tuple[int, int, int]] = None,
1213
+ batch_size: Optional[int] = 500,
1214
+ threshold: Optional[int] = 50):
1215
+ """
1216
+ Remove background from videos using GPU acceleration.
1217
+
1218
+ .. video:: _static/img/video_bg_subtraction.webm
1219
+ :width: 800
1220
+ :autoplay:
1221
+ :loop:
1222
+
1223
+ .. note::
1224
+ To create an `avg_frm`, use :func:`simba.video_processors.video_processing.create_average_frm`, :func:`simba.data_processors.cuda.image.create_average_frm_cupy`, or :func:`~simba.data_processors.cuda.image.create_average_frm_cuda`
1225
+
1226
+ .. seealso::
1227
+ For CPU-based alternative, see :func:`simba.video_processors.video_processing.video_bg_subtraction` or :func:`~simba.video_processors.video_processing.video_bg_subtraction_mp`
1228
+ For GPU-based alternative, see :func:`~simba.data_processors.cuda.image.bg_subtraction_cupy`. Needs work, CPU/multicore appears faster.
1229
+
1230
+ .. seealso::
1231
+ To create average frame on the CPU, see :func:`simba.video_processors.video_processing.create_average_frm`. CPU/multicore appears faster.
1232
+
1233
+ .. csv-table::
1234
+ :header: EXPECTED RUNTIMES
1235
+ :file: ../../../docs/tables/bg_subtraction_cuda.csv
1236
+ :widths: 10, 45, 45
1237
+ :align: center
1238
+ :class: simba-table
1239
+ :header-rows: 1
1240
+
1241
+ :param Union[str, os.PathLike] video_path: The path to the video to remove the background from.
1242
+ :param np.ndarray avg_frm: Average frame of the video. Can be created with e.g., :func:`simba.video_processors.video_processing.create_average_frm`.
1243
+ :param Optional[Union[str, os.PathLike]] save_path: Optional location to store the background removed video. If None, then saved in the same directory as the input video with the `_bg_removed` suffix.
1244
+ :param Optional[Tuple[int, int, int]] bg_clr: Tuple representing the background color of the video.
1245
+ :param Optional[Tuple[int, int, int]] fg_clr: Tuple representing the foreground color of the video (e.g., the animal). If None, then the original pixel colors will be used. Default: 50.
1246
+ :param Optional[int] batch_size: Number of frames to process concurrently. Use higher values of RAM memory allows. Default: 500.
1247
+ :param Optional[int] threshold: Value between 0-255 representing the difference threshold between the average frame subtracted from each frame. Higher values and more pixels will be considered background. Default: 50.
1248
+
1249
+ :example:
1250
+ >>> video_path = "/mnt/c/troubleshooting/mitra/project_folder/videos/clipped/592_MA147_Gq_CNO_0515.mp4"
1251
+ >>> avg_frm = create_average_frm(video_path=video_path)
1252
+ >>> bg_subtraction_cuda(video_path=video_path, avg_frm=avg_frm, fg_clr=(255, 255, 255))
1253
+ """
1254
+
1255
+ check_if_valid_img(data=avg_frm, source=f'{bg_subtraction_cuda}')
1256
+ check_if_valid_rgb_tuple(data=bg_clr)
1257
+ check_int(name=f'{bg_subtraction_cuda.__name__} batch_size', value=batch_size, min_value=1)
1258
+ check_int(name=f'{bg_subtraction_cuda.__name__} threshold', value=threshold, min_value=0, max_value=255)
1259
+ THREADS_PER_BLOCK = (32, 32, 1)
1260
+ timer = SimbaTimer(start=True)
1261
+ video_meta = get_video_meta_data(video_path=video_path)
1262
+ batch_cnt = int(max(1, np.ceil(video_meta['frame_count'] / batch_size)))
1263
+ frm_batches = np.array_split(np.arange(0, video_meta['frame_count']), batch_cnt)
1264
+ n, w, h = video_meta['frame_count'], video_meta['width'], video_meta['height']
1265
+ avg_frm = cv2.resize(avg_frm, (w, h))
1266
+ if is_video_color(video_path): is_color = np.array([1])
1267
+ else: is_color = np.array([0])
1268
+ fourcc = cv2.VideoWriter_fourcc(*Formats.MP4_CODEC.value)
1269
+ if save_path is None:
1270
+ in_dir, video_name, _ = get_fn_ext(filepath=video_path)
1271
+ save_path = os.path.join(in_dir, f'{video_name}_bg_removed.mp4')
1272
+ if fg_clr is not None:
1273
+ check_if_valid_rgb_tuple(data=fg_clr)
1274
+ fg_clr = np.array(fg_clr)
1275
+ else:
1276
+ fg_clr = np.array([-1])
1277
+ threshold = np.array([threshold]).astype(np.int32)
1278
+ writer = cv2.VideoWriter(save_path, fourcc, video_meta['fps'], (w, h))
1279
+ y_dev = cuda.to_device(avg_frm.astype(np.float32))
1280
+ fg_clr_dev = cuda.to_device(fg_clr)
1281
+ is_color_dev = cuda.to_device(is_color)
1282
+ for frm_batch_cnt, frm_batch in enumerate(frm_batches):
1283
+ print(f'Processing frame batch {frm_batch_cnt+1} / {len(frm_batches)} (complete: {round((frm_batch_cnt / len(frm_batches)) * 100, 2)}%)')
1284
+ batch_imgs = read_img_batch_from_video_gpu(video_path=video_path, start_frm=frm_batch[0], end_frm=frm_batch[-1])
1285
+ batch_imgs = np.stack(list(batch_imgs.values()), axis=0).astype(np.float32)
1286
+ batch_n = batch_imgs.shape[0]
1287
+ results = np.zeros_like(batch_imgs).astype(np.uint8)
1288
+ results[:] = bg_clr
1289
+ results = cuda.to_device(results)
1290
+ grid_x = math.ceil(w / THREADS_PER_BLOCK[0])
1291
+ grid_y = math.ceil(h / THREADS_PER_BLOCK[1])
1292
+ grid_z = math.ceil(batch_n / THREADS_PER_BLOCK[2])
1293
+ bpg = (grid_x, grid_y, grid_z)
1294
+ x_dev = cuda.to_device(batch_imgs)
1295
+ _bg_subtraction_cuda_kernel[bpg, THREADS_PER_BLOCK](x_dev, y_dev, results, is_color_dev, fg_clr_dev, threshold)
1296
+ results = results.copy_to_host()
1297
+ for img_cnt, img in enumerate(results):
1298
+ writer.write(img)
1299
+ writer.release()
1300
+ timer.stop_timer()
1301
+ stdout_success(msg=f'Video saved at {save_path}', elapsed_time=timer.elapsed_time_str)
1302
+
1303
+
1304
+ def bg_subtraction_cupy(video_path: Union[str, os.PathLike],
1305
+ avg_frm: Union[np.ndarray, str, os.PathLike],
1306
+ save_path: Optional[Union[str, os.PathLike]] = None,
1307
+ bg_clr: Optional[Tuple[int, int, int]] = (0, 0, 0),
1308
+ fg_clr: Optional[Tuple[int, int, int]] = None,
1309
+ batch_size: Optional[int] = 500,
1310
+ threshold: Optional[int] = 50,
1311
+ verbose: bool = True,
1312
+ async_frame_read: bool = True):
1313
+ """
1314
+ Remove background from videos using GPU acceleration through CuPY.
1315
+
1316
+ .. video:: _static/img/bg_remover_example_1.webm
1317
+ :width: 800
1318
+ :autoplay:
1319
+ :loop:
1320
+
1321
+ .. seealso::
1322
+ For CPU-based alternative, see :func:`simba.video_processors.video_processing.video_bg_subtraction` or :func:`~simba.video_processors.video_processing.video_bg_subtraction_mp`
1323
+ For GPU-based alternative, see :func:`~simba.data_processors.cuda.image.bg_subtraction_cuda`.
1324
+ Needs work, CPU/multicore appears faster.
1325
+
1326
+ :param Union[str, os.PathLike] video_path: The path to the video to remove the background from.
1327
+ :param np.ndarray avg_frm: Average frame of the video. Can be created with e.g., :func:`simba.video_processors.video_processing.create_average_frm`.
1328
+ :param Optional[Union[str, os.PathLike]] save_path: Optional location to store the background removed video. If None, then saved in the same directory as the input video with the `_bg_removed` suffix.
1329
+ :param Optional[Tuple[int, int, int]] bg_clr: Tuple representing the background color of the video.
1330
+ :param Optional[Tuple[int, int, int]] fg_clr: Tuple representing the foreground color of the video (e.g., the animal). If None, then the original pixel colors will be used. Default: 50.
1331
+ :param Optional[int] batch_size: Number of frames to process concurrently. Use higher values of RAM memory allows. Default: 500.
1332
+ :param Optional[int] threshold: Value between 0-255 representing the difference threshold between the average frame subtracted from each frame. Higher values and more pixels will be considered background. Default: 50.
1333
+
1334
+
1335
+ :example:
1336
+ >>> avg_frm = create_average_frm(video_path="/mnt/c/troubleshooting/mitra/project_folder/videos/temp/temp_ex_bg_subtraction/original/844_MA131_gq_CNO_0624.mp4")
1337
+ >>> video_path = "/mnt/c/troubleshooting/mitra/project_folder/videos/temp/temp_ex_bg_subtraction/844_MA131_gq_CNO_0624_7.mp4"
1338
+ >>> bg_subtraction_cupy(video_path=video_path, avg_frm=avg_frm, batch_size=500)
1339
+ """
1340
+
1341
+ if not _is_cuda_available()[0]:
1342
+ raise SimBAGPUError('NP GPU detected using numba.cuda', source=bg_subtraction_cupy.__name__)
1343
+ if isinstance(avg_frm, (str, os.PathLike)):
1344
+ check_file_exist_and_readable(file_path=avg_frm, raise_error=True)
1345
+ avg_frm = read_img(img_path=avg_frm, greyscale=False, clahe=False)
1346
+ check_if_valid_img(data=avg_frm, source=f'{bg_subtraction_cupy}')
1347
+ check_if_valid_rgb_tuple(data=bg_clr)
1348
+ check_int(name=f'{bg_subtraction_cupy.__name__} batch_size', value=batch_size, min_value=1)
1349
+ check_int(name=f'{bg_subtraction_cupy.__name__} threshold', value=threshold, min_value=0, max_value=255)
1350
+ timer = SimbaTimer(start=True)
1351
+ video_meta = get_video_meta_data(video_path=video_path)
1352
+ n, w, h = video_meta['frame_count'], video_meta['width'], video_meta['height']
1353
+ is_video_color_bool = is_video_color(video_path)
1354
+ is_avg_frm_color = avg_frm.ndim == 3 and avg_frm.shape[2] == 3
1355
+ if avg_frm.shape[0] != h or avg_frm.shape[1] != w:
1356
+ raise InvalidInputError(msg=f'The avg_frm and video must have the same resolution: avg_frm is {avg_frm.shape[1]}x{avg_frm.shape[0]}, video is {w}x{h}', source=bg_subtraction_cupy.__name__)
1357
+ if is_video_color_bool != is_avg_frm_color:
1358
+ video_type = 'color' if is_video_color_bool else 'grayscale'
1359
+ avg_frm_type = 'color' if is_avg_frm_color else 'grayscale'
1360
+ raise InvalidInputError(msg=f'Color/grayscale mismatch: video is {video_type} but avg_frm is {avg_frm_type}', source=bg_subtraction_cupy.__name__)
1361
+
1362
+ avg_frm = cp.array(avg_frm)
1363
+ is_color = is_video_color_bool
1364
+ batch_cnt = int(max(1, np.ceil(video_meta['frame_count'] / batch_size)))
1365
+ frm_batches = np.array_split(np.arange(0, video_meta['frame_count']), batch_cnt)
1366
+ fourcc = cv2.VideoWriter_fourcc(*Formats.MP4_CODEC.value)
1367
+ if save_path is None:
1368
+ in_dir, video_name, _ = get_fn_ext(filepath=video_path)
1369
+ save_path = os.path.join(in_dir, f'{video_name}_bg_removed_ppp.mp4')
1370
+ if fg_clr is not None:
1371
+ check_if_valid_rgb_tuple(data=fg_clr)
1372
+ fg_clr = np.array(fg_clr)
1373
+ else:
1374
+ fg_clr = np.array([-1])
1375
+ writer = cv2.VideoWriter(save_path, fourcc, video_meta['fps'], (w, h), isColor=is_color)
1376
+ if async_frame_read:
1377
+ async_frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=3, verbose=True, gpu=True)
1378
+ async_frm_reader.start()
1379
+ else:
1380
+ async_frm_reader = None
1381
+ for frm_batch_cnt, frm_batch in enumerate(frm_batches):
1382
+ if verbose: print(f'Processing frame batch {frm_batch_cnt + 1} / {len(frm_batches)} (complete: {round((frm_batch_cnt / len(frm_batches)) * 100, 2)}%, {get_current_time()})')
1383
+ if not async_frame_read:
1384
+ batch_imgs = read_img_batch_from_video_gpu(video_path=video_path, start_frm=frm_batch[0], end_frm=frm_batch[-1], verbose=verbose)
1385
+ batch_imgs = cp.array(np.stack(list(batch_imgs.values()), axis=0).astype(np.float32))
1386
+ else:
1387
+ batch_imgs = cp.array(get_async_frame_batch(batch_reader=async_frm_reader, timeout=15)[2])
1388
+ img_diff = cp.abs(batch_imgs - avg_frm)
1389
+ if is_color:
1390
+ img_diff = img_stack_to_grayscale_cupy(imgs=img_diff, batch_size=img_diff.shape[0])
1391
+ threshold_cp = cp.array([threshold], dtype=cp.float32)
1392
+ mask = cp.where(img_diff > threshold_cp, 1, 0).astype(cp.uint8)
1393
+ if is_color:
1394
+ batch_imgs[mask == 0] = bg_clr
1395
+ if fg_clr[0] != -1:
1396
+ batch_imgs[mask == 1] = fg_clr
1397
+ else:
1398
+ bg_clr_gray = int(0.07 * bg_clr[2] + 0.72 * bg_clr[1] + 0.21 * bg_clr[0])
1399
+ batch_imgs[mask == 0] = bg_clr_gray
1400
+ if fg_clr[0] != -1:
1401
+ fg_clr_gray = int(0.07 * fg_clr[2] + 0.72 * fg_clr[1] + 0.21 * fg_clr[0])
1402
+ batch_imgs[mask == 1] = fg_clr_gray
1403
+ batch_imgs = batch_imgs.astype(cp.uint8).get()
1404
+ for img_cnt, img in enumerate(batch_imgs):
1405
+ writer.write(img)
1406
+ if async_frm_reader is not None:
1407
+ async_frm_reader.kill()
1408
+
1409
+ writer.release()
1410
+ timer.stop_timer()
1411
+ stdout_success(msg=f'Video saved at {save_path}', elapsed_time=timer.elapsed_time_str)
1412
+
1413
+
1414
+ @cuda.jit(max_registers=None)
1415
+ def _pose_plot_kernel(imgs, data, circle_size, resolution, colors):
1416
+ bp_n, img_n = cuda.grid(2)
1417
+ if img_n < 0 or img_n > (imgs.shape[0] -1):
1418
+ return
1419
+ if bp_n < 0 or bp_n > (data[0].shape[0] -1):
1420
+ return
1421
+
1422
+ img, bp_loc, color = imgs[img_n], data[img_n][bp_n], colors[bp_n]
1423
+ for x1 in range(bp_loc[0]-circle_size[0], bp_loc[0]+circle_size[0]):
1424
+ for y1 in range(bp_loc[1]-circle_size[0], bp_loc[1]+circle_size[0]):
1425
+ if (x1 > 0) and (x1 < resolution[0]):
1426
+ if (y1 > 0) and (y1 < resolution[1]):
1427
+ b = (x1 - bp_loc[0]) ** 2
1428
+ c = (y1 - bp_loc[1]) ** 2
1429
+ if (b + c) < (circle_size[0] ** 2):
1430
+ imgs[img_n][y1][x1][0] = int(color[0])
1431
+ imgs[img_n][y1][x1][1] = int(color[1])
1432
+ imgs[img_n][y1][x1][2] = int(color[2])
1433
+
1434
+
1435
+ def pose_plotter(data: Union[str, os.PathLike, np.ndarray],
1436
+ video_path: Union[str, os.PathLike],
1437
+ save_path: Union[str, os.PathLike],
1438
+ circle_size: Optional[int] = None,
1439
+ colors: Optional[str] = 'Set1',
1440
+ batch_size: int = 750,
1441
+ verbose: bool = True) -> None:
1442
+
1443
+ """
1444
+ Creates a video overlaying pose-estimation data on frames from a given video using GPU acceleration.
1445
+
1446
+ .. video:: _static/img/pose_plotter_cuda.mp4
1447
+ :width: 800
1448
+ :autoplay:
1449
+ :loop:
1450
+
1451
+ .. seealso::
1452
+ For CPU based methods, see :func:`~simba.plotting.path_plotter.PathPlotterSingleCore` and :func:`~simba.plotting.path_plotter_mp.PathPlotterMulticore`.
1453
+
1454
+ .. csv-table::
1455
+ :header: EXPECTED RUNTIMES
1456
+ :file: ../../../docs/tables/pose_plotter.csv
1457
+ :widths: 10, 90
1458
+ :align: center
1459
+ :class: simba-table
1460
+ :header-rows: 1
1461
+
1462
+ :param Union[str, os.PathLike, np.ndarray] data: Path to a CSV file with pose-estimation data or a 3d numpy array (n_images, n_bodyparts, 2) with pose-estimated locations.
1463
+ :param Union[str, os.PathLike] video_path: Path to a video file where the ``data`` has been pose-estimated.
1464
+ :param Union[str, os.PathLike] save_path: Location where to store the output visualization.
1465
+ :param Optional[int] circle_size: The size of the circles representing the location of the pose-estimated locations. If None, the optimal size will be inferred as a 100th of the max(resultion_w, h).
1466
+ :param int batch_size: The number of frames to process concurrently on the GPU. Default: 750. Increase of host and device RAM allows it to improve runtime. Decrease if you hit memory errors.
1467
+
1468
+ :example:
1469
+ >>> DATA_PATH = "/mnt/c/troubleshooting/mitra/project_folder/csv/outlier_corrected_movement_location/501_MA142_Gi_CNO_0521.csv"
1470
+ >>> VIDEO_PATH = "/mnt/c/troubleshooting/mitra/project_folder/videos/501_MA142_Gi_CNO_0521.mp4"
1471
+ >>> SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1472
+ >>> pose_plotter(data=DATA_PATH, video_path=VIDEO_PATH, save_path=SAVE_PATH, circle_size=10, batch_size=1000)
1473
+ """
1474
+
1475
+ THREADS_PER_BLOCK = (32, 32, 1)
1476
+ if isinstance(data, str):
1477
+ check_file_exist_and_readable(file_path=data)
1478
+ df = read_df(file_path=data, file_type='csv')
1479
+ cols = [x for x in df.columns if not x.lower().endswith('_p')]
1480
+ data = df[cols].values
1481
+ data = np.ascontiguousarray(data.reshape(data.shape[0], int(data.shape[1] / 2), 2).astype(np.int32))
1482
+ elif isinstance(data, np.ndarray):
1483
+ check_valid_array(data=data, source=pose_plotter.__name__, accepted_ndims=(3,), accepted_dtypes=Formats.NUMERIC_DTYPES.value)
1484
+
1485
+ check_int(name=f'{pose_plotter.__name__} batch_size', value=batch_size, min_value=1)
1486
+ check_valid_boolean(value=[verbose], source=f'{pose_plotter.__name__} verbose')
1487
+ video_meta_data = get_video_meta_data(video_path=video_path)
1488
+ n, w, h = video_meta_data['frame_count'], video_meta_data['width'], video_meta_data['height']
1489
+ check_if_dir_exists(in_dir=os.path.dirname(save_path))
1490
+ if data.shape[0] != video_meta_data['frame_count']:
1491
+ raise FrameRangeError(msg=f'The data contains {data.shape[0]} frames while the video contains {video_meta_data["frame_count"]} frames')
1492
+ if circle_size is None:
1493
+ circle_size = np.array([PlottingMixin().get_optimal_circle_size(frame_size=(w, h))]).astype(np.int32)
1494
+ else:
1495
+ check_int(name=f'{pose_plotter.__name__} circle_size', value=circle_size, min_value=1)
1496
+ circle_size = np.array([circle_size]).astype(np.int32)
1497
+ fourcc = cv2.VideoWriter_fourcc(*Formats.MP4_CODEC.value)
1498
+ video_writer = cv2.VideoWriter(save_path, fourcc, video_meta_data['fps'], (w, h))
1499
+ colors = np.array(create_color_palette(pallete_name=colors, increments=data[0].shape[0])).astype(np.int32)
1500
+ circle_size_dev = cuda.to_device(circle_size)
1501
+ # colors_dev = cuda.to_device(colors)
1502
+ # resolution_dev = cuda.to_device(np.array([video_meta_data['width'], video_meta_data['height']]))
1503
+ # data = np.ascontiguousarray(data, dtype=np.int32)
1504
+ # img_dev = cuda.device_array((batch_size, h, w, 3), dtype=np.int32)
1505
+ # data_dev = cuda.device_array((batch_size, data.shape[1], 2), dtype=np.int32)
1506
+ # total_timer, video_start_time = SimbaTimer(start=True), time.time()
1507
+ # frm_reader = AsyncVideoFrameReader(video_path=video_path, batch_size=batch_size, max_que_size=3, verbose=False)
1508
+ # frm_reader.start()
1509
+ # for batch_cnt in range(frm_reader.batch_cnt):
1510
+ # start_img_idx, end_img_idx, batch_frms = get_async_frame_batch(batch_reader=frm_reader, timeout=10)
1511
+ # video_elapsed_time = str(round(time.time() - video_start_time, 4)) + 's'
1512
+ # if verbose: print(f'Processing images {start_img_idx} - {end_img_idx} (of {n}; batch count: {batch_cnt+1}/{frm_reader.batch_cnt}, video: {video_meta_data["video_name"]}, elapsed video processing time: {video_elapsed_time})...')
1513
+ # batch_data = data[start_img_idx:end_img_idx + 1]
1514
+ # batch_n = batch_frms.shape[0]
1515
+ # if verbose: print(f'Moving frames {start_img_idx}-{end_img_idx} to device...')
1516
+ # img_dev[:batch_n].copy_to_device(batch_frms[:batch_n].astype(np.int32))
1517
+ # data_dev[:batch_n] = cuda.to_device(batch_data[:batch_n])
1518
+ # del batch_frms; del batch_data
1519
+ # bpg = (math.ceil(batch_n / THREADS_PER_BLOCK[0]), math.ceil(batch_n / THREADS_PER_BLOCK[2]))
1520
+ # if verbose: print(f'Creating frames {start_img_idx}-{end_img_idx} ...')
1521
+ # _pose_plot_kernel[bpg, THREADS_PER_BLOCK](img_dev, data_dev, circle_size_dev, resolution_dev, colors_dev)
1522
+ # if verbose: print(f'Moving frames to host {start_img_idx}-{end_img_idx} ...')
1523
+ # batch_frms = img_dev.copy_to_host()
1524
+ # if verbose: print(f'Writing frames to host {start_img_idx}-{end_img_idx} ...')
1525
+ # for img_idx in range(0, batch_n):
1526
+ # video_writer.write(batch_frms[img_idx].astype(np.uint8))
1527
+ # video_writer.release()
1528
+ # total_timer.stop_timer()
1529
+ # frm_reader.kill()
1530
+ # if verbose:
1531
+ # stdout_success(msg=f'Pose-estimation video saved at {save_path}.', elapsed_time=total_timer.elapsed_time_str)
1532
+
1533
+
1534
+
1535
+ #x = create_average_frm_cuda(video_path=r"D:\troubleshooting\mitra\project_folder\videos\average_cpu_test\20min.mp4", verbose=True, batch_size=500, async_frame_read=False)
1536
+
1537
+ # VIDEO_PATH = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/videos/Trial_1_C24_D1_1.mp4"
1538
+ # #
1539
+ #
1540
+ #
1541
+ #
1542
+ # avg_frm = create_average_frm_cuda(video_path=VIDEO_PATH, verbose=True, batch_size=100, start_frm=0, end_frm=100, async_frame_read=True, save_path=SAVE_PATH)
1543
+ # if _
1544
+ # VIDEO_PATH = r"D:\troubleshooting\maplight_ri\project_folder\blob\videos\111.mp4"
1545
+ # AVG_FRM = r"D:\troubleshooting\maplight_ri\project_folder\blob\Trial_1_C24_D1_1_bg_removed.png"
1546
+ # SAVE_PATH = r"D:\troubleshooting\maplight_ri\project_folder\blob\Trial_1_C24_D1_1_bg_removed.mp4"
1547
+ #
1548
+
1549
+
1550
+ # VIDEO_PATH = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/videos/111.mp4"
1551
+ # AVG_FRM = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/Trial_1_C24_D1_1_bg_removed.png"
1552
+ # SAVE_PATH = "/mnt/d/troubleshooting/maplight_ri/project_folder/blob/Trial_1_C24_D1_1_bg_removed.mp4"
1553
+ # bg_subtraction_cupy(video_path=VIDEO_PATH, avg_frm=AVG_FRM, save_path=SAVE_PATH, batch_size=100, verbose=True, async_frame_read=True, threshold=240, fg_clr=(255, 0,0), bg_clr=(0, 0, 255))
1554
+
1555
+
1556
+
1557
+ # DATA_PATH = "/mnt/c/troubleshooting/mitra/project_folder/csv/outlier_corrected_movement_location/501_MA142_Gi_CNO_0521.csv"
1558
+ # VIDEO_PATH = "/mnt/c/troubleshooting/mitra/project_folder/videos/501_MA142_Gi_CNO_0521.mp4"
1559
+ # SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1560
+ # pose_plotter(data=DATA_PATH, video_path=VIDEO_PATH, save_path=SAVE_PATH, circle_size=10, batch_size=1000)
1561
+ # # VIDEO_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1562
+ # # SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test_ROTATED.mp4"
1563
+ # #
1564
+ # # rotate_video_cupy(video_path=VIDEO_PATH, save_path=SAVE_PATH, batch_size=1000)
1565
+ #
1566
+ # #"C:\troubleshooting\mitra\project_folder\csv\outlier_corrected_movement_location\501_MA142_Gi_CNO_0521.csv"
1567
+ # pose_plotter(data=DATA_PATH, video_path=VIDEO_PATH, save_path=SAVE_PATH, circle_size=10, batch_size=1000)
1568
+
1569
+
1570
+
1571
+
1572
+
1573
+
1574
+
1575
+ # from simba.mixins.geometry_mixin import GeometryMixin
1576
+ #
1577
+ # video_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4"
1578
+ # data_path = "/mnt/c/troubleshooting/RAT_NOR/project_folder/csv/outlier_corrected_movement_location/03152021_NOB_IOT_8.csv"
1579
+ # save_dir = '/mnt/d/netholabs/yolo_videos/input/mp4_20250606083508'
1580
+ #
1581
+ # get_video_meta_data(video_path)
1582
+ #
1583
+ # nose_arr = read_df(file_path=data_path, file_type='csv', usecols=['Nose_x', 'Nose_y', 'Tail_base_x', 'Tail_base_y', 'Lat_left_x', 'Lat_left_y', 'Lat_right_x', 'Lat_right_y']).values.reshape(-1, 4, 2).astype(np.int32) ## READ THE BODY-PART THAT DEFINES THE HULL AND CONVERT TO ARRAY
1584
+ #
1585
+ # polygons = GeometryMixin().multiframe_bodyparts_to_polygon(data=nose_arr, parallel_offset=60) ## CONVERT THE BODY-PART TO POLYGONS WITH A LITTLE BUFFER
1586
+ # polygons = GeometryMixin().multiframe_minimum_rotated_rectangle(shapes=polygons) # CONVERT THE POLYGONS TO RECTANGLES (I.E., WITH 4 UNIQUE POINTS).
1587
+ # polygon_lst = [] # GET THE POINTS OF THE RECTANGLES
1588
+ # for i in polygons: polygon_lst.append(np.array(i.exterior.coords))
1589
+ # polygons = np.stack(polygon_lst, axis=0)
1590
+ # sliced_imgs = slice_imgs(video_path=video_path, shapes=polygons, batch_size=500, save_dir=save_dir) #SLICE THE RECTANGLES IN THE VIDEO.
1591
+
1592
+ #sliced_imgs = {k: v for k, v in enumerate(sliced_imgs)}
1593
+
1594
+ #ImageMixin().img_stack_to_video(imgs=sliced_imgs, fps=29.97, save_path=r'/mnt/d/netholabs/yolo_videos/input/mp4_20250606083508/stacked.mp4')
1595
+
1596
+ #get_video_meta_data("/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4")
1597
+ # cv2.imshow('asdasdas', sliced_imgs[500])
1598
+ # cv2.waitKey(0)
1599
+
1600
+ # DATA_PATH = "/mnt/c/troubleshooting/RAT_NOR/project_folder/csv/outlier_corrected_movement_location/03152021_NOB_IOT_8.csv"
1601
+ # VIDEO_PATH = "/mnt/c/troubleshooting/RAT_NOR/project_folder/videos/03152021_NOB_IOT_8.mp4"
1602
+ # SAVE_PATH = "/mnt/c/troubleshooting/mitra/project_folder/frames/output/pose_ex/test.mp4"
1603
+ #
1604
+ #
1605
+ DATA_PATH = "/mnt/d/troubleshooting/mitra/project_folder/csv/outlier_corrected_movement_location/592_MA147_CNO1_0515.csv"
1606
+ VIDEO_PATH = "/mnt/d/troubleshooting/mitra/project_folder/videos/592_MA147_CNO1_0515.mp4"
1607
+ SAVE_PATH = "/mnt/d/troubleshooting/mitra/project_folder/videos/test_cuda.mp4"
1608
+ pose_plotter(data=DATA_PATH, video_path=VIDEO_PATH, save_path=SAVE_PATH, circle_size=10, batch_size=100)
1609
+
1610
+
1611
+
1612
+ #
1613
+ # #from simba.data_processors.cuda.image import create_average_frm_cupy
1614
+ # SAVE_PATH = "/mnt/c/Users/sroni/Downloads/bg_remove_nb/bg_removed_ex_7.mp4"
1615
+ # VIDEO_PATH = "/mnt/c/Users/sroni/Downloads/bg_remove_nb/open_field.mp4"
1616
+ # avg_frm = create_average_frm_cuda(video_path=VIDEO_PATH)
1617
+ # #
1618
+ # get_video_meta_data(VIDEO_PATH)
1619
+ # #
1620
+ # bg_subtraction_cuda(video_path=VIDEO_PATH, avg_frm=avg_frm, save_path=SAVE_PATH, threshold=70)