dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
@@ -1,470 +1,470 @@
1
- import asyncio
2
- import time
3
- import types
4
-
5
- import numpy as np
6
- import os
7
- import logging
8
- import dtlpy as dl
9
- import shutil
10
-
11
- logger = logging.getLogger(name='dtlpy')
12
-
13
-
14
- ##########
15
- # Videos #
16
- ##########
17
- class Videos:
18
- def __init__(self):
19
- pass
20
-
21
- @staticmethod
22
- def get_info(filepath):
23
- try:
24
- import ffmpeg
25
- except ImportError:
26
- logger.error(
27
- 'Import Error! Cant import ffmpeg. '
28
- 'Annotations operations will be limited. import manually and fix errors')
29
- raise
30
- probe = ffmpeg.probe(filepath)
31
- return probe
32
-
33
- @staticmethod
34
- def get_max_object_id(item):
35
- max_object_id = 1
36
- annotations_list = item.annotations.list().annotations
37
- if len(annotations_list) < 1:
38
- return 1
39
- for annotation in annotations_list:
40
- if annotation.object_id is not None:
41
- current_object_id = int(annotation.object_id)
42
- if current_object_id > max_object_id:
43
- max_object_id = current_object_id
44
- return max_object_id
45
-
46
- @staticmethod
47
- def video_snapshots_generator(item_id=None, item=None, frame_interval=30, image_ext="png"):
48
- futures = Videos._async_video_snapshots_generator(item_id=item_id,
49
- item=item,
50
- frame_interval=frame_interval,
51
- image_ext=image_ext)
52
- loop = asyncio.new_event_loop()
53
- try:
54
- asyncio.set_event_loop(loop)
55
- return loop.run_until_complete(futures)
56
- finally:
57
- try:
58
- loop.run_until_complete(loop.shutdown_asyncgens())
59
- finally:
60
- asyncio.set_event_loop(None)
61
- loop.close()
62
-
63
- @staticmethod
64
- async def _async_video_snapshots_generator(item_id=None, item=None, frame_interval=30, image_ext="png"):
65
- """
66
- Create video-snapshots
67
-
68
- :param item_id: item id for the video
69
- :param item: item id for the video
70
- :param frame_interval: number of frames to take next snapshot
71
- :param image_ext: png/jpg
72
- :return: the uploaded items
73
- """
74
- if item_id is not None:
75
- item = dl.items.get(item_id=item_id)
76
-
77
- if item is None:
78
- raise ValueError('Missing input item (or item_id)')
79
-
80
- if not isinstance(frame_interval, int):
81
- raise AttributeError('frame_interval is mast to be integer')
82
-
83
- if "video" not in item.mimetype:
84
- raise AttributeError("Got {} file type but only video files are supported".format(item.mimetype))
85
-
86
- video_path = item.download()
87
-
88
- # Get the time for single frame from metadata (duration/# of frames)
89
- if 'system' in item.metadata and \
90
- 'ffmpeg' in item.metadata['system'] and \
91
- 'duration' in item.metadata['system']['ffmpeg'] and \
92
- 'nb_frames' in item.metadata['system']['ffmpeg']:
93
- nb_frames = int(item.metadata["system"]["ffmpeg"]["nb_frames"])
94
- duration = float(item.metadata["system"]["ffmpeg"]["duration"])
95
- video_fps = duration / nb_frames
96
- else:
97
- try:
98
- import cv2
99
- except (ImportError, ModuleNotFoundError):
100
- logger.error(
101
- 'Import Error! Cant import cv2. '
102
- 'Annotations operations will be limited. import manually and fix errors')
103
- raise
104
-
105
- video = cv2.VideoCapture(video_path)
106
- video_fps = video.get(cv2.CAP_PROP_FPS)
107
- nb_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
108
- duration = video_fps * nb_frames
109
-
110
- images_path = Videos.disassemble(filepath=video_path, frame_interval=frame_interval, image_ext=image_ext)
111
- snapshots_items = list()
112
- try:
113
- # rename files
114
- images = []
115
- video_basename = os.path.basename(video_path)
116
- for f in os.listdir(images_path):
117
- images.append(f)
118
- for image in images:
119
- image_split_name, ext = os.path.splitext(image)
120
- try:
121
- frame = int(image_split_name) * frame_interval
122
- file_frame_name = "{}.frame.{}{}".format(video_basename, frame, ext)
123
- full_path = os.path.join(images_path, file_frame_name)
124
- os.rename(os.path.join(images_path, image),
125
- full_path)
126
- except Exception as e:
127
- logger.debug("Rename {} has been failed: {}".format(os.path.join(images_path, image), e))
128
-
129
- remote_path = os.path.join(os.path.split(item.filename)[0], "snapshots")
130
- remote_url = '/items/{}/snapshots'.format(item.id)
131
- to_upload = open(full_path, 'rb')
132
- try:
133
- response = await item._client_api.upload_file_async(to_upload=to_upload,
134
- item_type='file',
135
- item_size=os.stat(full_path).st_size,
136
- remote_url=remote_url,
137
- uploaded_filename=file_frame_name,
138
- remote_path=remote_path)
139
- except Exception:
140
- raise
141
- finally:
142
- to_upload.close()
143
- if response.ok:
144
- snapshots_items.append(item.from_json(response.json(), item._client_api))
145
- else:
146
- raise dl.PlatformException(response)
147
-
148
- # classification tpe annotation creation for each file
149
- builder = item.annotations.builder()
150
- annotation_itemlinks = []
151
-
152
- max_object_id = Videos().get_max_object_id(item=item)
153
- for snapshot_item in snapshots_items:
154
- max_object_id += 1
155
- item_frame = snapshot_item.name.rsplit("frame.", 1)[1].split(".")[0]
156
- if item_frame.isnumeric():
157
- item_time = int(item_frame) * video_fps
158
- else:
159
- item_frame = item_time = 0
160
-
161
- snapshot_item.metadata["system"]["itemLinks"] = [{"type": "snapshotFrom",
162
- "itemId": item.id,
163
- "frame": item_frame,
164
- "time": item_time}]
165
-
166
- annotation_itemlinks.append({"type": "snapshotTo",
167
- "itemId": snapshot_item.id,
168
- "frame": item_frame,
169
- "time": item_time})
170
-
171
- snapshot_item.update(system_metadata=True)
172
- annotation_definition = dl.Classification(label="Snapshot")
173
- builder.add(annotation_definition=annotation_definition,
174
- frame_num=int(item_frame),
175
- end_frame_num=nb_frames if int(item_frame) + int(video_fps) > nb_frames else int(
176
- item_frame) + int(video_fps),
177
- start_time=item_time,
178
- end_time=duration if item_time + 1 > duration else item_time + 1,
179
- object_id=max_object_id)
180
-
181
- annotations = item.annotations.upload(annotations=builder)
182
-
183
- # update system metadata for annotations
184
- count = 0
185
- for annotation in annotations:
186
- annotation.metadata["system"]["itemLinks"] = [annotation_itemlinks[count]]
187
- count += 1
188
-
189
- annotations.update(system_metadata=True)
190
- except Exception as err:
191
- logger.exception(err)
192
- finally:
193
- if os.path.isdir(images_path):
194
- shutil.rmtree(images_path)
195
- return snapshots_items
196
-
197
- @staticmethod
198
- def disassemble(filepath, fps=None, frame_interval=None, loglevel='panic', image_ext='jpg'):
199
- """
200
- Disassemble video to images
201
-
202
- :param filepath: input video filepath
203
- :param fps: rate of disassemble. e.g if 1 frame per second fps is 1. if None all frames will be extracted
204
- :param frame_interval: take image every frame # (if exists function ignore fps)
205
- :param image_ext: png/jpg
206
- :param loglevel: ffmpeg loglevel
207
- :return:
208
- """
209
- try:
210
- import ffmpeg
211
- except ImportError:
212
- logger.error(
213
- 'Import Error! Cant import ffmpeg. '
214
- 'Annotations operations will be limited. import manually and fix errors')
215
- raise
216
- # get video information
217
- video_props = Videos.get_info(filepath)
218
- if 'system' in video_props and \
219
- 'nb_frames' in video_props['system'][0]:
220
- nb_frames = video_props['streams'][0]['nb_frames']
221
- else:
222
- try:
223
- import cv2
224
- except (ImportError, ModuleNotFoundError):
225
- logger.error(
226
- 'Import Error! Cant import cv2. '
227
- 'Annotations operations will be limited. import manually and fix errors')
228
- raise
229
- nb_frames = int(cv2.VideoCapture(filepath).get(cv2.CAP_PROP_FRAME_COUNT))
230
-
231
- if not os.path.isfile(filepath):
232
- raise IOError('File doesnt exists: {}'.format(filepath))
233
- basename, ext = os.path.splitext(filepath)
234
- # create folder for the frames
235
- if os.path.exists(basename):
236
- shutil.rmtree(basename)
237
-
238
- os.makedirs(basename, exist_ok=True)
239
-
240
- if fps is None:
241
- try:
242
- fps = eval(video_props['streams'][0]['avg_frame_rate'])
243
- except ZeroDivisionError:
244
- fps = 0
245
- num_of_zeros = len(str(nb_frames))
246
- # format the output filename
247
- output_regex = os.path.join(basename, '%0{}d.{}'.format(num_of_zeros, image_ext))
248
-
249
- try:
250
- if frame_interval is not None:
251
- frame_number = 0
252
- select = ""
253
- while frame_number < nb_frames:
254
- if select != "":
255
- select += '+'
256
- select += 'eq(n\\,{})'.format(frame_number)
257
- frame_number += frame_interval
258
- stream = ffmpeg.input(filepath, **{'loglevel': loglevel}).output(output_regex,
259
- **{'start_number': '0',
260
- 'vf': 'select=\'{}'.format(select),
261
- 'vsync': 'vfr'})
262
- else:
263
- stream = ffmpeg.input(filepath, **{'loglevel': loglevel}).output(output_regex,
264
- **{'start_number': '0',
265
- 'r': str(fps)})
266
-
267
- ffmpeg.overwrite_output(stream).run()
268
- except Exception:
269
- logger.error('ffmpeg error in disassemble:')
270
- raise
271
- return basename
272
-
273
- @staticmethod
274
- def reencode(filepath, loglevel='panic'):
275
- """
276
- Re-encode video as mp4, remove start offset and set bframes to 0
277
-
278
- :param filepath: input video file
279
- :param loglevel: ffmpeg loglevel
280
- :return:
281
- """
282
- try:
283
- import ffmpeg
284
- except ImportError:
285
- logger.error(
286
- 'Import Error! Cant import ffmpeg. '
287
- 'Annotations operations will be limited. import manually and fix errors')
288
- raise
289
- if not os.path.isfile(filepath):
290
- raise IOError('File doesnt exists: {}'.format(filepath))
291
- # re encode video without b frame and as mp4
292
- basename, ext = os.path.splitext(filepath)
293
- output_filepath = os.path.join(basename, os.path.basename(filepath).replace(ext, '.mp4'))
294
- if not os.path.isdir(os.path.dirname(output_filepath)):
295
- os.makedirs(os.path.dirname(output_filepath))
296
- try:
297
- stream = ffmpeg.input(filepath, **{'loglevel': loglevel}).output(output_filepath,
298
- **{'x264opts': 'bframes=0',
299
- 'f': 'mp4'})
300
- ffmpeg.overwrite_output(stream).run()
301
- except Exception as e:
302
- logger.exception('ffmpeg error in disassemble:')
303
- raise
304
-
305
- output_probe = Videos.get_info(output_filepath)
306
- start_time = eval(output_probe['streams'][0]['start_time'])
307
- fps = eval(output_probe['streams'][0]['avg_frame_rate'])
308
- has_b_frames = output_probe['streams'][0]['has_b_frames']
309
- start_frame = fps * start_time
310
- if start_time != 0:
311
- logger.warning('Video start_time is not 0!')
312
- if has_b_frames != 0:
313
- logger.warning('Video still has b frames!')
314
- return output_filepath
315
-
316
- @staticmethod
317
- def split_and_upload(filepath,
318
- # upload parameters
319
- project_name=None, project_id=None, dataset_name=None, dataset_id=None, remote_path=None,
320
- # split parameters
321
- split_seconds=None, split_chunks=None, split_pairs=None,
322
- loglevel='panic'):
323
- """
324
- Split video to chunks and upload to platform
325
-
326
- :param filepath: input video file
327
- :param project_name:
328
- :param project_id:
329
- :param dataset_name:
330
- :param dataset_id:
331
- :param remote_path:
332
- :param split_seconds: split by seconds per chunk. each chunk's length will be this in seconds
333
- :param split_chunks: split by number of chunks.
334
- :param split_pairs: a list od (start, stop) segments to split in seconds . e.g [(0,400), (600,800)]
335
- :param loglevel: ffmpeg loglevel
336
- :return:
337
- """
338
- try:
339
- import ffmpeg
340
- except ImportError:
341
- logger.error(
342
- 'Import Error! Cant import ffmpeg. '
343
- 'Annotations operations will be limited. import manually and fix errors')
344
- raise
345
- # https://www.ffmpeg.org/ffmpeg-formats.html#Examples-9
346
-
347
- if not os.path.isfile(filepath):
348
- raise IOError('File doesnt exists: {}'.format(filepath))
349
- logger.info('Extracting video information...')
350
- # call to ffmpeg to get frame rate
351
- probe = Videos.get_info(filepath)
352
- fps = eval(probe['streams'][0]['avg_frame_rate'])
353
- n_frames = eval(probe['streams'][0]['nb_frames'])
354
- video_length = eval(probe['streams'][0]['duration'])
355
- logger.info('Video frame rate: {}[fps]'.format(fps))
356
- logger.info('Video number of frames: {}'.format(n_frames))
357
- logger.info('Video length in seconds: {}[s]'.format(video_length))
358
-
359
- # check split params and calc split params for ffmpeg
360
- if split_seconds is not None:
361
- # split by seconds
362
- split_length = split_seconds
363
- if split_length <= 0:
364
- raise ValueError('"split_length" can\'t be 0')
365
- split_count = int(np.ceil(video_length / split_length))
366
- list_frames_to_split = [fps * split_length * n for n in range(1, split_count)]
367
- elif split_chunks is not None:
368
- # split for known number of chunks
369
- split_count = split_chunks
370
- if split_chunks <= 0:
371
- raise ValueError('"split_chunks" size can\'t be 0')
372
- split_length = int(np.ceil(video_length / split_chunks))
373
- list_frames_to_split = [fps * split_length * n for n in range(1, split_count)]
374
- elif split_pairs is not None:
375
- if not isinstance(split_pairs, list):
376
- raise ValueError('"split_times" must be a list of tuples to split at.')
377
- if not (isinstance(split_pairs[0], list) or isinstance(split_pairs[0], tuple)):
378
- raise ValueError('"split_times" must be a list of tuples to split at.')
379
- list_frames_to_split = [fps * split_second for segment in split_pairs for split_second in segment]
380
- split_count = len(list_frames_to_split)
381
- else:
382
- raise ValueError('Must input one split option ("split_chunks", "split_time" or "split_pairs")')
383
- if split_count == 1:
384
- raise ValueError('Video length is less than the target split length.')
385
- # to integers
386
- list_frames_to_split = [int(i) for i in list_frames_to_split]
387
- # remove 0 if in the first segmetn
388
- if list_frames_to_split[0] == 0:
389
- list_frames_to_split.pop(0)
390
- # add last frames if not exists
391
- if list_frames_to_split[-1] != n_frames:
392
- list_frames_to_split = list_frames_to_split + [n_frames]
393
- logger.info('Splitting to %d chunks' % split_count)
394
-
395
- basename, ext = os.path.splitext(filepath)
396
- output_regex = os.path.join(basename, '%%03d.mp4')
397
- # create folder
398
- if not os.path.exists(basename):
399
- os.makedirs(basename, exist_ok=True)
400
- # run ffmpeg
401
- try:
402
- stream = ffmpeg.input(filepath, **{'loglevel': loglevel}).output(output_regex,
403
- **{'x264opts': 'bframes=0',
404
- 'f': 'segment',
405
- 'reset_timestamps': '1',
406
- 'map': '0',
407
- 'segment_frames': ','.join(
408
- [str(i) for i in
409
- list_frames_to_split])
410
- })
411
- ffmpeg.overwrite_output(stream).run(capture_stdout=True)
412
- except Exception:
413
- logger.exception('ffmpeg error in disassemble:')
414
- raise
415
-
416
- # split_cmd = 'ffmpeg -y -i "%s" -b 0 -f mp4 -reset_timestamps 1 -map 0 -f segment -segment_frames %s "%s"' % (
417
- # filepath, ','.join([str(int(i)) for i in list_frames_to_split]), output_regex)
418
- # logger.info('About to run: %s' % split_cmd)
419
- # subprocess.check_call(shlex.split(split_cmd), universal_newlines=True)
420
-
421
- # rename
422
- list_frames_to_split = [0] + list_frames_to_split
423
- filenames = list()
424
- for n in range(split_count):
425
- old_filename = output_regex.replace('%03d', '%03d' % n)
426
- new_filename = output_regex.replace('%03d', '%s__%s' %
427
- (time.strftime('%H_%M_%S', time.gmtime(list_frames_to_split[n] / fps)),
428
- time.strftime('%H_%M_%S',
429
- time.gmtime(list_frames_to_split[n + 1] / fps))))
430
- filenames.append(new_filename)
431
- # rename to informative name
432
- if os.path.isfile(new_filename):
433
- logger.warning('File already exists. Overwriting!: {}'.format(new_filename))
434
- os.remove(new_filename)
435
- os.rename(old_filename, new_filename)
436
- # check if in pairs, if not - delete
437
- if split_pairs is not None:
438
- start_frames = [pair[0] for pair in split_pairs]
439
- end_frames = [pair[1] for pair in split_pairs]
440
- if (list_frames_to_split[n] // fps) in start_frames and (
441
- list_frames_to_split[n + 1] // fps) in end_frames:
442
- # keep video
443
- pass
444
- else:
445
- os.remove(new_filename)
446
- Videos.upload_to_platform(project_name=project_name,
447
- project_id=project_id,
448
- dataset_name=dataset_name,
449
- dataset_id=dataset_id,
450
- remote_path=remote_path,
451
- local_path=basename)
452
-
453
- @staticmethod
454
- def upload_to_platform(project_name=None, project_id=None, dataset_name=None, dataset_id=None,
455
- local_path=None, remote_path=None):
456
-
457
- import dtlpy as dlp
458
- if project_id is not None or project_name is not None:
459
- project = dlp.projects.get(project_name=project_name, project_id=project_id)
460
- dataset = project.get(dataset_name=dataset_name, dataset_id=dataset_id)
461
- dataset.items.upload(dataset_name=dataset_name,
462
- dataset_id=dataset_id,
463
- local_path=local_path,
464
- remote_path=remote_path,
465
- file_types=['.mp4'])
466
- else:
467
- dataset = dlp.datasets.get(dataset_name=dataset_name, dataset_id=dataset_id)
468
- dataset.items.upload(local_path=local_path,
469
- remote_path=remote_path,
470
- file_types=['.mp4'])
1
+ import asyncio
2
+ import time
3
+ import types
4
+
5
+ import numpy as np
6
+ import os
7
+ import logging
8
+ import dtlpy as dl
9
+ import shutil
10
+
11
+ logger = logging.getLogger(name='dtlpy')
12
+
13
+
14
+ ##########
15
+ # Videos #
16
+ ##########
17
+ class Videos:
18
+ def __init__(self):
19
+ pass
20
+
21
+ @staticmethod
22
+ def get_info(filepath):
23
+ try:
24
+ import ffmpeg
25
+ except ImportError:
26
+ logger.error(
27
+ 'Import Error! Cant import ffmpeg. '
28
+ 'Annotations operations will be limited. import manually and fix errors')
29
+ raise
30
+ probe = ffmpeg.probe(filepath)
31
+ return probe
32
+
33
+ @staticmethod
34
+ def get_max_object_id(item):
35
+ max_object_id = 1
36
+ annotations_list = item.annotations.list().annotations
37
+ if len(annotations_list) < 1:
38
+ return 1
39
+ for annotation in annotations_list:
40
+ if annotation.object_id is not None:
41
+ current_object_id = int(annotation.object_id)
42
+ if current_object_id > max_object_id:
43
+ max_object_id = current_object_id
44
+ return max_object_id
45
+
46
+ @staticmethod
47
+ def video_snapshots_generator(item_id=None, item=None, frame_interval=30, image_ext="png"):
48
+ futures = Videos._async_video_snapshots_generator(item_id=item_id,
49
+ item=item,
50
+ frame_interval=frame_interval,
51
+ image_ext=image_ext)
52
+ loop = asyncio.new_event_loop()
53
+ try:
54
+ asyncio.set_event_loop(loop)
55
+ return loop.run_until_complete(futures)
56
+ finally:
57
+ try:
58
+ loop.run_until_complete(loop.shutdown_asyncgens())
59
+ finally:
60
+ asyncio.set_event_loop(None)
61
+ loop.close()
62
+
63
+ @staticmethod
64
+ async def _async_video_snapshots_generator(item_id=None, item=None, frame_interval=30, image_ext="png"):
65
+ """
66
+ Create video-snapshots
67
+
68
+ :param item_id: item id for the video
69
+ :param item: item id for the video
70
+ :param frame_interval: number of frames to take next snapshot
71
+ :param image_ext: png/jpg
72
+ :return: the uploaded items
73
+ """
74
+ if item_id is not None:
75
+ item = dl.items.get(item_id=item_id)
76
+
77
+ if item is None:
78
+ raise ValueError('Missing input item (or item_id)')
79
+
80
+ if not isinstance(frame_interval, int):
81
+ raise AttributeError('frame_interval is mast to be integer')
82
+
83
+ if "video" not in item.mimetype:
84
+ raise AttributeError("Got {} file type but only video files are supported".format(item.mimetype))
85
+
86
+ video_path = item.download()
87
+
88
+ # Get the time for single frame from metadata (duration/# of frames)
89
+ if 'system' in item.metadata and \
90
+ 'ffmpeg' in item.metadata['system'] and \
91
+ 'duration' in item.metadata['system']['ffmpeg'] and \
92
+ 'nb_frames' in item.metadata['system']['ffmpeg']:
93
+ nb_frames = int(item.metadata["system"]["ffmpeg"]["nb_frames"])
94
+ duration = float(item.metadata["system"]["ffmpeg"]["duration"])
95
+ video_fps = duration / nb_frames
96
+ else:
97
+ try:
98
+ import cv2
99
+ except (ImportError, ModuleNotFoundError):
100
+ logger.error(
101
+ 'Import Error! Cant import cv2. '
102
+ 'Annotations operations will be limited. import manually and fix errors')
103
+ raise
104
+
105
+ video = cv2.VideoCapture(video_path)
106
+ video_fps = video.get(cv2.CAP_PROP_FPS)
107
+ nb_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
108
+ duration = video_fps * nb_frames
109
+
110
+ images_path = Videos.disassemble(filepath=video_path, frame_interval=frame_interval, image_ext=image_ext)
111
+ snapshots_items = list()
112
+ try:
113
+ # rename files
114
+ images = []
115
+ video_basename = os.path.basename(video_path)
116
+ for f in os.listdir(images_path):
117
+ images.append(f)
118
+ for image in images:
119
+ image_split_name, ext = os.path.splitext(image)
120
+ try:
121
+ frame = int(image_split_name) * frame_interval
122
+ file_frame_name = "{}.frame.{}{}".format(video_basename, frame, ext)
123
+ full_path = os.path.join(images_path, file_frame_name)
124
+ os.rename(os.path.join(images_path, image),
125
+ full_path)
126
+ except Exception as e:
127
+ logger.debug("Rename {} has been failed: {}".format(os.path.join(images_path, image), e))
128
+
129
+ remote_path = os.path.join(os.path.split(item.filename)[0], "snapshots")
130
+ remote_url = '/items/{}/snapshots'.format(item.id)
131
+ to_upload = open(full_path, 'rb')
132
+ try:
133
+ response = await item._client_api.upload_file_async(to_upload=to_upload,
134
+ item_type='file',
135
+ item_size=os.stat(full_path).st_size,
136
+ remote_url=remote_url,
137
+ uploaded_filename=file_frame_name,
138
+ remote_path=remote_path)
139
+ except Exception:
140
+ raise
141
+ finally:
142
+ to_upload.close()
143
+ if response.ok:
144
+ snapshots_items.append(item.from_json(response.json(), item._client_api))
145
+ else:
146
+ raise dl.PlatformException(response)
147
+
148
+ # classification tpe annotation creation for each file
149
+ builder = item.annotations.builder()
150
+ annotation_itemlinks = []
151
+
152
+ max_object_id = Videos().get_max_object_id(item=item)
153
+ for snapshot_item in snapshots_items:
154
+ max_object_id += 1
155
+ item_frame = snapshot_item.name.rsplit("frame.", 1)[1].split(".")[0]
156
+ if item_frame.isnumeric():
157
+ item_time = int(item_frame) * video_fps
158
+ else:
159
+ item_frame = item_time = 0
160
+
161
+ snapshot_item.metadata["system"]["itemLinks"] = [{"type": "snapshotFrom",
162
+ "itemId": item.id,
163
+ "frame": item_frame,
164
+ "time": item_time}]
165
+
166
+ annotation_itemlinks.append({"type": "snapshotTo",
167
+ "itemId": snapshot_item.id,
168
+ "frame": item_frame,
169
+ "time": item_time})
170
+
171
+ snapshot_item.update(system_metadata=True)
172
+ annotation_definition = dl.Classification(label="Snapshot")
173
+ builder.add(annotation_definition=annotation_definition,
174
+ frame_num=int(item_frame),
175
+ end_frame_num=nb_frames if int(item_frame) + int(video_fps) > nb_frames else int(
176
+ item_frame) + int(video_fps),
177
+ start_time=item_time,
178
+ end_time=duration if item_time + 1 > duration else item_time + 1,
179
+ object_id=max_object_id)
180
+
181
+ annotations = item.annotations.upload(annotations=builder)
182
+
183
+ # update system metadata for annotations
184
+ count = 0
185
+ for annotation in annotations:
186
+ annotation.metadata["system"]["itemLinks"] = [annotation_itemlinks[count]]
187
+ count += 1
188
+
189
+ annotations.update(system_metadata=True)
190
+ except Exception as err:
191
+ logger.exception(err)
192
+ finally:
193
+ if os.path.isdir(images_path):
194
+ shutil.rmtree(images_path)
195
+ return snapshots_items
196
+
197
+ @staticmethod
198
+ def disassemble(filepath, fps=None, frame_interval=None, loglevel='panic', image_ext='jpg'):
199
+ """
200
+ Disassemble video to images
201
+
202
+ :param filepath: input video filepath
203
+ :param fps: rate of disassemble. e.g if 1 frame per second fps is 1. if None all frames will be extracted
204
+ :param frame_interval: take image every frame # (if exists function ignore fps)
205
+ :param image_ext: png/jpg
206
+ :param loglevel: ffmpeg loglevel
207
+ :return:
208
+ """
209
+ try:
210
+ import ffmpeg
211
+ except ImportError:
212
+ logger.error(
213
+ 'Import Error! Cant import ffmpeg. '
214
+ 'Annotations operations will be limited. import manually and fix errors')
215
+ raise
216
+ # get video information
217
+ video_props = Videos.get_info(filepath)
218
+ if 'system' in video_props and \
219
+ 'nb_frames' in video_props['system'][0]:
220
+ nb_frames = video_props['streams'][0]['nb_frames']
221
+ else:
222
+ try:
223
+ import cv2
224
+ except (ImportError, ModuleNotFoundError):
225
+ logger.error(
226
+ 'Import Error! Cant import cv2. '
227
+ 'Annotations operations will be limited. import manually and fix errors')
228
+ raise
229
+ nb_frames = int(cv2.VideoCapture(filepath).get(cv2.CAP_PROP_FRAME_COUNT))
230
+
231
+ if not os.path.isfile(filepath):
232
+ raise IOError('File doesnt exists: {}'.format(filepath))
233
+ basename, ext = os.path.splitext(filepath)
234
+ # create folder for the frames
235
+ if os.path.exists(basename):
236
+ shutil.rmtree(basename)
237
+
238
+ os.makedirs(basename, exist_ok=True)
239
+
240
+ if fps is None:
241
+ try:
242
+ fps = eval(video_props['streams'][0]['avg_frame_rate'])
243
+ except ZeroDivisionError:
244
+ fps = 0
245
+ num_of_zeros = len(str(nb_frames))
246
+ # format the output filename
247
+ output_regex = os.path.join(basename, '%0{}d.{}'.format(num_of_zeros, image_ext))
248
+
249
+ try:
250
+ if frame_interval is not None:
251
+ frame_number = 0
252
+ select = ""
253
+ while frame_number < nb_frames:
254
+ if select != "":
255
+ select += '+'
256
+ select += 'eq(n\\,{})'.format(frame_number)
257
+ frame_number += frame_interval
258
+ stream = ffmpeg.input(filepath, **{'loglevel': loglevel}).output(output_regex,
259
+ **{'start_number': '0',
260
+ 'vf': 'select=\'{}'.format(select),
261
+ 'vsync': 'vfr'})
262
+ else:
263
+ stream = ffmpeg.input(filepath, **{'loglevel': loglevel}).output(output_regex,
264
+ **{'start_number': '0',
265
+ 'r': str(fps)})
266
+
267
+ ffmpeg.overwrite_output(stream).run()
268
+ except Exception:
269
+ logger.error('ffmpeg error in disassemble:')
270
+ raise
271
+ return basename
272
+
273
+ @staticmethod
274
+ def reencode(filepath, loglevel='panic'):
275
+ """
276
+ Re-encode video as mp4, remove start offset and set bframes to 0
277
+
278
+ :param filepath: input video file
279
+ :param loglevel: ffmpeg loglevel
280
+ :return:
281
+ """
282
+ try:
283
+ import ffmpeg
284
+ except ImportError:
285
+ logger.error(
286
+ 'Import Error! Cant import ffmpeg. '
287
+ 'Annotations operations will be limited. import manually and fix errors')
288
+ raise
289
+ if not os.path.isfile(filepath):
290
+ raise IOError('File doesnt exists: {}'.format(filepath))
291
+ # re encode video without b frame and as mp4
292
+ basename, ext = os.path.splitext(filepath)
293
+ output_filepath = os.path.join(basename, os.path.basename(filepath).replace(ext, '.mp4'))
294
+ if not os.path.isdir(os.path.dirname(output_filepath)):
295
+ os.makedirs(os.path.dirname(output_filepath))
296
+ try:
297
+ stream = ffmpeg.input(filepath, **{'loglevel': loglevel}).output(output_filepath,
298
+ **{'x264opts': 'bframes=0',
299
+ 'f': 'mp4'})
300
+ ffmpeg.overwrite_output(stream).run()
301
+ except Exception as e:
302
+ logger.exception('ffmpeg error in disassemble:')
303
+ raise
304
+
305
+ output_probe = Videos.get_info(output_filepath)
306
+ start_time = eval(output_probe['streams'][0]['start_time'])
307
+ fps = eval(output_probe['streams'][0]['avg_frame_rate'])
308
+ has_b_frames = output_probe['streams'][0]['has_b_frames']
309
+ start_frame = fps * start_time
310
+ if start_time != 0:
311
+ logger.warning('Video start_time is not 0!')
312
+ if has_b_frames != 0:
313
+ logger.warning('Video still has b frames!')
314
+ return output_filepath
315
+
316
+ @staticmethod
317
+ def split_and_upload(filepath,
318
+ # upload parameters
319
+ project_name=None, project_id=None, dataset_name=None, dataset_id=None, remote_path=None,
320
+ # split parameters
321
+ split_seconds=None, split_chunks=None, split_pairs=None,
322
+ loglevel='panic'):
323
+ """
324
+ Split video to chunks and upload to platform
325
+
326
+ :param filepath: input video file
327
+ :param project_name:
328
+ :param project_id:
329
+ :param dataset_name:
330
+ :param dataset_id:
331
+ :param remote_path:
332
+ :param split_seconds: split by seconds per chunk. each chunk's length will be this in seconds
333
+ :param split_chunks: split by number of chunks.
334
+ :param split_pairs: a list od (start, stop) segments to split in seconds . e.g [(0,400), (600,800)]
335
+ :param loglevel: ffmpeg loglevel
336
+ :return:
337
+ """
338
+ try:
339
+ import ffmpeg
340
+ except ImportError:
341
+ logger.error(
342
+ 'Import Error! Cant import ffmpeg. '
343
+ 'Annotations operations will be limited. import manually and fix errors')
344
+ raise
345
+ # https://www.ffmpeg.org/ffmpeg-formats.html#Examples-9
346
+
347
+ if not os.path.isfile(filepath):
348
+ raise IOError('File doesnt exists: {}'.format(filepath))
349
+ logger.info('Extracting video information...')
350
+ # call to ffmpeg to get frame rate
351
+ probe = Videos.get_info(filepath)
352
+ fps = eval(probe['streams'][0]['avg_frame_rate'])
353
+ n_frames = eval(probe['streams'][0]['nb_frames'])
354
+ video_length = eval(probe['streams'][0]['duration'])
355
+ logger.info('Video frame rate: {}[fps]'.format(fps))
356
+ logger.info('Video number of frames: {}'.format(n_frames))
357
+ logger.info('Video length in seconds: {}[s]'.format(video_length))
358
+
359
+ # check split params and calc split params for ffmpeg
360
+ if split_seconds is not None:
361
+ # split by seconds
362
+ split_length = split_seconds
363
+ if split_length <= 0:
364
+ raise ValueError('"split_length" can\'t be 0')
365
+ split_count = int(np.ceil(video_length / split_length))
366
+ list_frames_to_split = [fps * split_length * n for n in range(1, split_count)]
367
+ elif split_chunks is not None:
368
+ # split for known number of chunks
369
+ split_count = split_chunks
370
+ if split_chunks <= 0:
371
+ raise ValueError('"split_chunks" size can\'t be 0')
372
+ split_length = int(np.ceil(video_length / split_chunks))
373
+ list_frames_to_split = [fps * split_length * n for n in range(1, split_count)]
374
+ elif split_pairs is not None:
375
+ if not isinstance(split_pairs, list):
376
+ raise ValueError('"split_times" must be a list of tuples to split at.')
377
+ if not (isinstance(split_pairs[0], list) or isinstance(split_pairs[0], tuple)):
378
+ raise ValueError('"split_times" must be a list of tuples to split at.')
379
+ list_frames_to_split = [fps * split_second for segment in split_pairs for split_second in segment]
380
+ split_count = len(list_frames_to_split)
381
+ else:
382
+ raise ValueError('Must input one split option ("split_chunks", "split_time" or "split_pairs")')
383
+ if split_count == 1:
384
+ raise ValueError('Video length is less than the target split length.')
385
+ # to integers
386
+ list_frames_to_split = [int(i) for i in list_frames_to_split]
387
+ # remove 0 if in the first segmetn
388
+ if list_frames_to_split[0] == 0:
389
+ list_frames_to_split.pop(0)
390
+ # add last frames if not exists
391
+ if list_frames_to_split[-1] != n_frames:
392
+ list_frames_to_split = list_frames_to_split + [n_frames]
393
+ logger.info('Splitting to %d chunks' % split_count)
394
+
395
+ basename, ext = os.path.splitext(filepath)
396
+ output_regex = os.path.join(basename, '%%03d.mp4')
397
+ # create folder
398
+ if not os.path.exists(basename):
399
+ os.makedirs(basename, exist_ok=True)
400
+ # run ffmpeg
401
+ try:
402
+ stream = ffmpeg.input(filepath, **{'loglevel': loglevel}).output(output_regex,
403
+ **{'x264opts': 'bframes=0',
404
+ 'f': 'segment',
405
+ 'reset_timestamps': '1',
406
+ 'map': '0',
407
+ 'segment_frames': ','.join(
408
+ [str(i) for i in
409
+ list_frames_to_split])
410
+ })
411
+ ffmpeg.overwrite_output(stream).run(capture_stdout=True)
412
+ except Exception:
413
+ logger.exception('ffmpeg error in disassemble:')
414
+ raise
415
+
416
+ # split_cmd = 'ffmpeg -y -i "%s" -b 0 -f mp4 -reset_timestamps 1 -map 0 -f segment -segment_frames %s "%s"' % (
417
+ # filepath, ','.join([str(int(i)) for i in list_frames_to_split]), output_regex)
418
+ # logger.info('About to run: %s' % split_cmd)
419
+ # subprocess.check_call(shlex.split(split_cmd), universal_newlines=True)
420
+
421
+ # rename
422
+ list_frames_to_split = [0] + list_frames_to_split
423
+ filenames = list()
424
+ for n in range(split_count):
425
+ old_filename = output_regex.replace('%03d', '%03d' % n)
426
+ new_filename = output_regex.replace('%03d', '%s__%s' %
427
+ (time.strftime('%H_%M_%S', time.gmtime(list_frames_to_split[n] / fps)),
428
+ time.strftime('%H_%M_%S',
429
+ time.gmtime(list_frames_to_split[n + 1] / fps))))
430
+ filenames.append(new_filename)
431
+ # rename to informative name
432
+ if os.path.isfile(new_filename):
433
+ logger.warning('File already exists. Overwriting!: {}'.format(new_filename))
434
+ os.remove(new_filename)
435
+ os.rename(old_filename, new_filename)
436
+ # check if in pairs, if not - delete
437
+ if split_pairs is not None:
438
+ start_frames = [pair[0] for pair in split_pairs]
439
+ end_frames = [pair[1] for pair in split_pairs]
440
+ if (list_frames_to_split[n] // fps) in start_frames and (
441
+ list_frames_to_split[n + 1] // fps) in end_frames:
442
+ # keep video
443
+ pass
444
+ else:
445
+ os.remove(new_filename)
446
+ Videos.upload_to_platform(project_name=project_name,
447
+ project_id=project_id,
448
+ dataset_name=dataset_name,
449
+ dataset_id=dataset_id,
450
+ remote_path=remote_path,
451
+ local_path=basename)
452
+
453
+ @staticmethod
454
+ def upload_to_platform(project_name=None, project_id=None, dataset_name=None, dataset_id=None,
455
+ local_path=None, remote_path=None):
456
+
457
+ import dtlpy as dlp
458
+ if project_id is not None or project_name is not None:
459
+ project = dlp.projects.get(project_name=project_name, project_id=project_id)
460
+ dataset = project.get(dataset_name=dataset_name, dataset_id=dataset_id)
461
+ dataset.items.upload(dataset_name=dataset_name,
462
+ dataset_id=dataset_id,
463
+ local_path=local_path,
464
+ remote_path=remote_path,
465
+ file_types=['.mp4'])
466
+ else:
467
+ dataset = dlp.datasets.get(dataset_name=dataset_name, dataset_id=dataset_id)
468
+ dataset.items.upload(local_path=local_path,
469
+ remote_path=remote_path,
470
+ file_types=['.mp4'])