dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
@@ -1,452 +1,452 @@
1
- import tempfile
2
- import requests
3
- import logging
4
- import shutil
5
- import os
6
-
7
- from .. import entities, miscellaneous, PlatformException, exceptions, repositories
8
- from ..services.api_client import ApiClient
9
-
10
- logger = logging.getLogger(name='dtlpy')
11
-
12
-
13
- class Artifacts:
14
- """
15
- Artifacts repository
16
- """
17
-
18
- def __init__(self,
19
- client_api: ApiClient,
20
- project: entities.Project = None,
21
- dataset: entities.Dataset = None,
22
- project_id: str = None,
23
- model: entities.Model = None,
24
- package: entities.Package = None,
25
- dataset_name=None):
26
- self._client_api = client_api
27
- self._project = project
28
- self._dataset = dataset
29
- self._items_repository = None
30
- self.dataset_name = dataset_name
31
- self.project_id = project_id
32
- self.model = model
33
- self.package = package
34
-
35
- ############
36
- # entities #
37
- ############
38
- @property
39
- def dataset(self) -> entities.Dataset:
40
- if self._dataset is None:
41
- # get dataset from project
42
- try:
43
- if self.dataset_name is None:
44
- self.dataset_name = 'Binaries'
45
- self._dataset = self.project.datasets._get_binaries_dataset()
46
- else:
47
- self._dataset = self.project.datasets.get(dataset_name=self.dataset_name)
48
- except exceptions.NotFound:
49
- raise ValueError(
50
- f'Missing "{self.dataset_name}" dataset in the project. Please contact support for help')
51
-
52
- return self._dataset
53
-
54
- @property
55
- def project(self) -> entities.Project:
56
- if self._project is None:
57
- if self._dataset is not None:
58
- self._project = self._dataset.project
59
- elif self.project_id is not None:
60
- self._project = repositories.Projects(client_api=self._client_api).get(project_id=self.project_id)
61
- else:
62
- raise PlatformException(error='400',
63
- message='Artifacts doesnt have a project, at least one must not be None: dataset, project or project_id')
64
- return self._project
65
-
66
- ################
67
- # repositories #
68
- ################
69
- @property
70
- def items_repository(self):
71
- if self._items_repository is None:
72
- # load Binaries dataset
73
- # load items repository
74
- self._items_repository = self.dataset.items
75
- self._items_repository.set_items_entity(entities.Artifact)
76
- return self._items_repository
77
-
78
- ###########
79
- # methods #
80
- ###########
81
- @staticmethod
82
- def _build_path_header(
83
- package_name=None,
84
- package=None,
85
- execution_id=None,
86
- execution=None,
87
- model_name=None,
88
- ):
89
- remote_path = '/artifacts'
90
- if package_name is not None or package is not None:
91
- if package is not None:
92
- package_name = package.name
93
- remote_path += '/packages/{}'.format(package_name)
94
- if execution_id is not None or execution is not None:
95
- if execution is not None:
96
- execution_id = execution.id
97
- remote_path += '/executions/{}'.format(execution_id)
98
- if model_name is not None:
99
- remote_path += '/models/{}'.format(model_name)
100
-
101
- return remote_path
102
-
103
- def list(self,
104
- execution_id: str = None,
105
- package_name: str = None,
106
- model_name: str = None) -> miscellaneous.List[entities.Artifact]:
107
- """
108
- List of artifacts
109
-
110
- :param str execution_id: list by execution id
111
- :param str package_name: list by package name
112
- :param str model_name: list by model name
113
- :return: list of artifacts
114
- :rtype: miscellaneous.List[dtlpy.entities.artifact.Artifact]
115
-
116
- **Example**:
117
-
118
- .. code-block:: python
119
-
120
- project.artifacts.list(package_name='package_name')
121
- """
122
- if self.model is not None:
123
- model_name = self.model.name
124
- if self.package is not None:
125
- package_name = self.package.name
126
-
127
- filters = entities.Filters()
128
- remote_path = self._build_path_header(
129
- package_name=package_name,
130
- execution_id=execution_id,
131
- model_name=model_name
132
- )
133
-
134
- remote_path += '/*'
135
- filters.add(field='filename', values=remote_path)
136
- pages = self.items_repository.list(filters=filters)
137
- items = [entities.Artifact.from_json(_json=item.to_json(),
138
- client_api=self._client_api,
139
- dataset=item._dataset,
140
- project=self.project)
141
- for page in pages for item in page]
142
- return miscellaneous.List(items)
143
-
144
- def get(self,
145
- artifact_id: str = None,
146
- artifact_name: str = None,
147
- model_name: str = None,
148
- execution_id: str = None,
149
- package_name: str = None) -> entities.ItemArtifact:
150
- """
151
-
152
- Get an artifact object by name, id or type
153
- If by name or type - need to input also execution/task id for the artifact folder
154
-
155
- :param str artifact_id: search by artifact id
156
- :param str artifact_name: search by artifact id
157
- :param str model_name: search by model name
158
- :param str execution_id: search by execution id
159
- :param str package_name: search by package name
160
- :return: Artifact object
161
- :rtype: dtlpy.entities.artifact.Artifact
162
-
163
- **Example**:
164
-
165
- .. code-block:: python
166
-
167
- project.artifacts.get(artifact_id='artifact_id')
168
- """
169
- if self.model is not None:
170
- model_name = self.model.name
171
- if self.package is not None:
172
- package_name = self.package.name
173
-
174
- if artifact_id is not None:
175
- artifact = self.items_repository.get(item_id=artifact_id)
176
- # verify input artifact name is same as the given id
177
- if artifact_name is not None and artifact.name != artifact_name:
178
- logger.warning(
179
- "Mismatch found in artifacts.get: artifact_name is different then artifact.name:"
180
- " {!r} != {!r}".format(
181
- artifact_name,
182
- artifact.name))
183
- elif artifact_name is not None:
184
- artifacts = self.list(
185
- execution_id=execution_id,
186
- package_name=package_name,
187
- model_name=model_name
188
- )
189
- artifact = [artifact for artifact in artifacts if artifact.name == artifact_name]
190
- if len(artifact) == 1:
191
- artifact = artifact[0]
192
- elif len(artifact) > 1:
193
- raise PlatformException('404', 'More Than one Artifact found')
194
- else:
195
- raise PlatformException('404', 'Artifact not found')
196
- else:
197
- msg = 'one input must be not None: artifact_id or artifact_name'
198
- raise ValueError(msg)
199
- return artifact
200
-
201
- def download(
202
- self,
203
- artifact_id: str = None,
204
- artifact_name: str = None,
205
- execution_id: str = None,
206
- package_name: str = None,
207
- model_name: str = None,
208
- local_path: str = None,
209
- overwrite: bool = False,
210
- save_locally: bool = True
211
- ):
212
- """
213
-
214
- Download artifact binary.
215
- Get artifact by name, id or type
216
-
217
- :param str artifact_id: search by artifact id
218
- :param str artifact_name: search by artifact id
219
- :param str execution_id: search by execution id
220
- :param str package_name: search by package name
221
- :param str model_name: search by model name
222
- :param str local_path: artifact will be saved to this filepath
223
- :param bool overwrite: optional - default = False
224
- :param bool save_locally: to save the file local
225
- :return: file path
226
- :rtype: str
227
-
228
- **Example**:
229
-
230
- .. code-block:: python
231
-
232
- project.artifacts.download(artifact_id='artifact_id',
233
- local_path='your_path',
234
- overwrite=True,
235
- save_locally=True)
236
- """
237
- if self.model is not None:
238
- model_name = self.model.name
239
- if self.package is not None:
240
- package_name = self.package.name
241
-
242
- if artifact_id is not None:
243
- # download
244
- artifact = self.items_repository.download(items=artifact_id,
245
- save_locally=save_locally,
246
- local_path=local_path,
247
- overwrite=overwrite)
248
- elif artifact_name is not None:
249
- artifact_obj: entities.ItemArtifact = self.get(artifact_id=artifact_id,
250
- execution_id=execution_id,
251
- package_name=package_name,
252
- artifact_name=artifact_name)
253
-
254
- artifact = artifact_obj.download(save_locally=save_locally,
255
- local_path=local_path,
256
- overwrite=overwrite)
257
-
258
- else:
259
- if self.model is not None:
260
- artifact = list()
261
- for m_artifact in self.model.model_artifacts:
262
- if isinstance(m_artifact, entities.ItemArtifact):
263
- if not m_artifact.is_fetched:
264
- m_artifact = self.items_repository.get(item_id=m_artifact.id)
265
- model_remote_root = m_artifact.filename.split('/')
266
- model_remote_root = '/'.join(model_remote_root[:4])
267
- # remove the prefix with relpath
268
- local_dst = os.path.join(local_path,
269
- os.path.relpath(m_artifact.filename, model_remote_root))
270
- if not os.path.isfile(local_dst) or overwrite:
271
- # need_to_download
272
- # 1. download to temp folder
273
- temp_dir = tempfile.mkdtemp()
274
- local_temp_file = m_artifact.download(
275
- local_path=temp_dir,
276
- overwrite=overwrite,
277
- to_items_folder=False,
278
- )
279
- src = local_temp_file
280
- # remove the prefix with relpath
281
- dst = local_dst
282
- os.makedirs(os.path.dirname(dst), exist_ok=True)
283
- shutil.move(src=src, dst=dst)
284
- # clean temp dir
285
- if os.path.isdir(temp_dir):
286
- shutil.rmtree(temp_dir)
287
- artifact.append(local_path)
288
- elif isinstance(m_artifact, entities.LinkArtifact):
289
- # remove the prefix with relpath
290
- local_dst = os.path.join(local_path, m_artifact.filename)
291
- if not os.path.isfile(local_dst) or overwrite:
292
- # need_to_download
293
- # 1. download to temp folder
294
- temp_dir = tempfile.mkdtemp()
295
- response = requests.get(m_artifact.url, stream=True)
296
- local_temp_file = os.path.join(temp_dir, m_artifact.filename)
297
- with open(local_temp_file, "wb") as handle:
298
- for data in response.iter_content(chunk_size=8192):
299
- handle.write(data)
300
- src = local_temp_file
301
- # remove the prefix with relpath
302
- dst = local_dst
303
- os.makedirs(os.path.dirname(dst), exist_ok=True)
304
- shutil.move(src=src, dst=dst)
305
- # clean temp dir
306
- if os.path.isdir(temp_dir):
307
- shutil.rmtree(temp_dir)
308
- artifact.append(local_path)
309
- elif isinstance(m_artifact, entities.LocalArtifact):
310
- pass
311
- else:
312
- raise ValueError('unsupported artifact type: {}'.format(type(m_artifact)))
313
- else:
314
- # for package artifacts - download using filter on the package directory
315
- if all(elem is None for elem in [package_name, execution_id]):
316
- raise PlatformException(error='400', message='Must input package or execution (id or entity)')
317
- remote_path = self._build_path_header(
318
- package_name=package_name,
319
- execution_id=execution_id,
320
- model_name=model_name,
321
- )
322
- without_relative_path = remote_path
323
- remote_path += '/*'
324
- filters = entities.Filters()
325
- filters.add(field='filename', values=remote_path)
326
- artifact = self.items_repository.download(filters=filters,
327
- save_locally=save_locally,
328
- local_path=local_path,
329
- to_items_folder=False,
330
- overwrite=overwrite,
331
- without_relative_path=without_relative_path)
332
- return artifact
333
-
334
- def upload(self,
335
- # what to upload
336
- filepath: str,
337
- # where to upload
338
- package_name: str = None,
339
- package: entities.Package = None,
340
- execution_id: str = None,
341
- execution: entities.Execution = None,
342
- model_name: str = None,
343
- # add information
344
- overwrite: bool = False):
345
- """
346
-
347
- Upload binary file to artifact. get by name, id or type.
348
- If artifact exists - overwriting binary
349
- Else and if create==True a new artifact will be created and uploaded
350
-
351
- :param str filepath: local binary file
352
- :param str package_name: package name that include the artifact
353
- :param dtlpy.entities.package.Package package: package object
354
- :param str execution_id: execution id that include the artifact
355
- :param dtlpy.entities.execution.Execution execution: execution object
356
- :param str model_name: model name that include the artifact
357
- :param bool overwrite: optional - default = False to overwrite an existing object
358
- :return: Artifact Object
359
- :rtype: dtlpy.entities.artifact.Artifact
360
-
361
- **Example**:
362
-
363
- .. code-block:: python
364
-
365
- project.artifacts.upload(filepath='filepath',
366
- package_name='package_name')
367
- """
368
- if self.model is not None:
369
- model_name = self.model.name
370
- if self.package is not None:
371
- package_name = self.package.name
372
-
373
- remote_path = self._build_path_header(package_name=package_name,
374
- package=package,
375
- execution=execution,
376
- execution_id=execution_id,
377
- model_name=model_name)
378
-
379
- if all(elem is None for elem in [package_name, package, execution_id, execution, model_name]):
380
- raise ValueError('Must input package or execution (id or entity)')
381
-
382
- artifact = self.items_repository.upload(local_path=filepath,
383
- remote_path=remote_path,
384
- overwrite=overwrite,
385
- output_entity=entities.Artifact)
386
- if self.model is not None:
387
- # list and update model
388
- filters = entities.Filters()
389
- filters.add(field='dir', values=remote_path + '*')
390
- pages = self.items_repository.list(filters=filters)
391
- model_artifacts = list()
392
- for item in pages.all():
393
- model_artifacts.append(entities.Artifact.from_json(_json=item.to_json(),
394
- client_api=self._client_api,
395
- dataset=item._dataset))
396
- self.model.model_artifacts = model_artifacts
397
- self.model.update()
398
- logger.debug('Artifact uploaded successfully')
399
- return artifact
400
-
401
- def delete(self,
402
- artifact_id=None,
403
- artifact_name=None,
404
- execution_id=None,
405
- model_name=None,
406
- package_name=None):
407
- """
408
- Delete artifacts
409
-
410
- :param str artifact_id: search by artifact id
411
- :param str artifact_name: search by artifact id
412
- :param str execution_id: search by execution id
413
- :param str model_name: search by model name
414
- :param str package_name: search by package name
415
- :return: True if success
416
- :rtype: bool
417
-
418
- **Example**:
419
-
420
- .. code-block:: python
421
-
422
- project.artifacts.delete(artifact_id='artifact_id',
423
- package_name='package_name')
424
- """
425
- if self.model is not None:
426
- model_name = self.model.name
427
- if self.package is not None:
428
- package_name = self.package.name
429
-
430
- if artifact_id is not None or artifact_name is not None:
431
- artifacts = [
432
- self.get(
433
- artifact_id=artifact_id,
434
- artifact_name=artifact_name,
435
- model_name=model_name,
436
- )
437
- ]
438
- elif execution_id is not None or package_name is not None:
439
- artifacts = self.list(
440
- execution_id=execution_id,
441
- package_name=package_name,
442
- model_name=model_name,
443
- )
444
- else:
445
- raise PlatformException('400',
446
- 'Must provide one of: artifact_id, artifact_name, execution_id, package_name')
447
-
448
- values = [artifact.id for artifact in artifacts]
449
- self.items_repository.delete(filters=entities.Filters(field='id', values=values,
450
- operator=entities.FiltersOperations.IN))
451
-
452
- return True
1
+ import tempfile
2
+ import requests
3
+ import logging
4
+ import shutil
5
+ import os
6
+
7
+ from .. import entities, miscellaneous, PlatformException, exceptions, repositories
8
+ from ..services.api_client import ApiClient
9
+
10
+ logger = logging.getLogger(name='dtlpy')
11
+
12
+
13
+ class Artifacts:
14
+ """
15
+ Artifacts repository
16
+ """
17
+
18
+ def __init__(self,
19
+ client_api: ApiClient,
20
+ project: entities.Project = None,
21
+ dataset: entities.Dataset = None,
22
+ project_id: str = None,
23
+ model: entities.Model = None,
24
+ package: entities.Package = None,
25
+ dataset_name=None):
26
+ self._client_api = client_api
27
+ self._project = project
28
+ self._dataset = dataset
29
+ self._items_repository = None
30
+ self.dataset_name = dataset_name
31
+ self.project_id = project_id
32
+ self.model = model
33
+ self.package = package
34
+
35
+ ############
36
+ # entities #
37
+ ############
38
+ @property
39
+ def dataset(self) -> entities.Dataset:
40
+ if self._dataset is None:
41
+ # get dataset from project
42
+ try:
43
+ if self.dataset_name is None:
44
+ self.dataset_name = 'Binaries'
45
+ self._dataset = self.project.datasets._get_binaries_dataset()
46
+ else:
47
+ self._dataset = self.project.datasets.get(dataset_name=self.dataset_name)
48
+ except exceptions.NotFound:
49
+ raise ValueError(
50
+ f'Missing "{self.dataset_name}" dataset in the project. Please contact support for help')
51
+
52
+ return self._dataset
53
+
54
+ @property
55
+ def project(self) -> entities.Project:
56
+ if self._project is None:
57
+ if self._dataset is not None:
58
+ self._project = self._dataset.project
59
+ elif self.project_id is not None:
60
+ self._project = repositories.Projects(client_api=self._client_api).get(project_id=self.project_id)
61
+ else:
62
+ raise PlatformException(error='400',
63
+ message='Artifacts doesnt have a project, at least one must not be None: dataset, project or project_id')
64
+ return self._project
65
+
66
+ ################
67
+ # repositories #
68
+ ################
69
+ @property
70
+ def items_repository(self):
71
+ if self._items_repository is None:
72
+ # load Binaries dataset
73
+ # load items repository
74
+ self._items_repository = self.dataset.items
75
+ self._items_repository.set_items_entity(entities.Artifact)
76
+ return self._items_repository
77
+
78
+ ###########
79
+ # methods #
80
+ ###########
81
+ @staticmethod
82
+ def _build_path_header(
83
+ package_name=None,
84
+ package=None,
85
+ execution_id=None,
86
+ execution=None,
87
+ model_name=None,
88
+ ):
89
+ remote_path = '/artifacts'
90
+ if package_name is not None or package is not None:
91
+ if package is not None:
92
+ package_name = package.name
93
+ remote_path += '/packages/{}'.format(package_name)
94
+ if execution_id is not None or execution is not None:
95
+ if execution is not None:
96
+ execution_id = execution.id
97
+ remote_path += '/executions/{}'.format(execution_id)
98
+ if model_name is not None:
99
+ remote_path += '/models/{}'.format(model_name)
100
+
101
+ return remote_path
102
+
103
+ def list(self,
104
+ execution_id: str = None,
105
+ package_name: str = None,
106
+ model_name: str = None) -> miscellaneous.List[entities.Artifact]:
107
+ """
108
+ List of artifacts
109
+
110
+ :param str execution_id: list by execution id
111
+ :param str package_name: list by package name
112
+ :param str model_name: list by model name
113
+ :return: list of artifacts
114
+ :rtype: miscellaneous.List[dtlpy.entities.artifact.Artifact]
115
+
116
+ **Example**:
117
+
118
+ .. code-block:: python
119
+
120
+ project.artifacts.list(package_name='package_name')
121
+ """
122
+ if self.model is not None:
123
+ model_name = self.model.name
124
+ if self.package is not None:
125
+ package_name = self.package.name
126
+
127
+ filters = entities.Filters()
128
+ remote_path = self._build_path_header(
129
+ package_name=package_name,
130
+ execution_id=execution_id,
131
+ model_name=model_name
132
+ )
133
+
134
+ remote_path += '/*'
135
+ filters.add(field='filename', values=remote_path)
136
+ pages = self.items_repository.list(filters=filters)
137
+ items = [entities.Artifact.from_json(_json=item.to_json(),
138
+ client_api=self._client_api,
139
+ dataset=item._dataset,
140
+ project=self.project)
141
+ for page in pages for item in page]
142
+ return miscellaneous.List(items)
143
+
144
+ def get(self,
145
+ artifact_id: str = None,
146
+ artifact_name: str = None,
147
+ model_name: str = None,
148
+ execution_id: str = None,
149
+ package_name: str = None) -> entities.ItemArtifact:
150
+ """
151
+
152
+ Get an artifact object by name, id or type
153
+ If by name or type - need to input also execution/task id for the artifact folder
154
+
155
+ :param str artifact_id: search by artifact id
156
+ :param str artifact_name: search by artifact id
157
+ :param str model_name: search by model name
158
+ :param str execution_id: search by execution id
159
+ :param str package_name: search by package name
160
+ :return: Artifact object
161
+ :rtype: dtlpy.entities.artifact.Artifact
162
+
163
+ **Example**:
164
+
165
+ .. code-block:: python
166
+
167
+ project.artifacts.get(artifact_id='artifact_id')
168
+ """
169
+ if self.model is not None:
170
+ model_name = self.model.name
171
+ if self.package is not None:
172
+ package_name = self.package.name
173
+
174
+ if artifact_id is not None:
175
+ artifact = self.items_repository.get(item_id=artifact_id)
176
+ # verify input artifact name is same as the given id
177
+ if artifact_name is not None and artifact.name != artifact_name:
178
+ logger.warning(
179
+ "Mismatch found in artifacts.get: artifact_name is different then artifact.name:"
180
+ " {!r} != {!r}".format(
181
+ artifact_name,
182
+ artifact.name))
183
+ elif artifact_name is not None:
184
+ artifacts = self.list(
185
+ execution_id=execution_id,
186
+ package_name=package_name,
187
+ model_name=model_name
188
+ )
189
+ artifact = [artifact for artifact in artifacts if artifact.name == artifact_name]
190
+ if len(artifact) == 1:
191
+ artifact = artifact[0]
192
+ elif len(artifact) > 1:
193
+ raise PlatformException('404', 'More Than one Artifact found')
194
+ else:
195
+ raise PlatformException('404', 'Artifact not found')
196
+ else:
197
+ msg = 'one input must be not None: artifact_id or artifact_name'
198
+ raise ValueError(msg)
199
+ return artifact
200
+
201
+ def download(
202
+ self,
203
+ artifact_id: str = None,
204
+ artifact_name: str = None,
205
+ execution_id: str = None,
206
+ package_name: str = None,
207
+ model_name: str = None,
208
+ local_path: str = None,
209
+ overwrite: bool = False,
210
+ save_locally: bool = True
211
+ ):
212
+ """
213
+
214
+ Download artifact binary.
215
+ Get artifact by name, id or type
216
+
217
+ :param str artifact_id: search by artifact id
218
+ :param str artifact_name: search by artifact id
219
+ :param str execution_id: search by execution id
220
+ :param str package_name: search by package name
221
+ :param str model_name: search by model name
222
+ :param str local_path: artifact will be saved to this filepath
223
+ :param bool overwrite: optional - default = False
224
+ :param bool save_locally: to save the file local
225
+ :return: file path
226
+ :rtype: str
227
+
228
+ **Example**:
229
+
230
+ .. code-block:: python
231
+
232
+ project.artifacts.download(artifact_id='artifact_id',
233
+ local_path='your_path',
234
+ overwrite=True,
235
+ save_locally=True)
236
+ """
237
+ if self.model is not None:
238
+ model_name = self.model.name
239
+ if self.package is not None:
240
+ package_name = self.package.name
241
+
242
+ if artifact_id is not None:
243
+ # download
244
+ artifact = self.items_repository.download(items=artifact_id,
245
+ save_locally=save_locally,
246
+ local_path=local_path,
247
+ overwrite=overwrite)
248
+ elif artifact_name is not None:
249
+ artifact_obj: entities.ItemArtifact = self.get(artifact_id=artifact_id,
250
+ execution_id=execution_id,
251
+ package_name=package_name,
252
+ artifact_name=artifact_name)
253
+
254
+ artifact = artifact_obj.download(save_locally=save_locally,
255
+ local_path=local_path,
256
+ overwrite=overwrite)
257
+
258
+ else:
259
+ if self.model is not None:
260
+ artifact = list()
261
+ for m_artifact in self.model.model_artifacts:
262
+ if isinstance(m_artifact, entities.ItemArtifact):
263
+ if not m_artifact.is_fetched:
264
+ m_artifact = self.items_repository.get(item_id=m_artifact.id)
265
+ model_remote_root = m_artifact.filename.split('/')
266
+ model_remote_root = '/'.join(model_remote_root[:4])
267
+ # remove the prefix with relpath
268
+ local_dst = os.path.join(local_path,
269
+ os.path.relpath(m_artifact.filename, model_remote_root))
270
+ if not os.path.isfile(local_dst) or overwrite:
271
+ # need_to_download
272
+ # 1. download to temp folder
273
+ temp_dir = tempfile.mkdtemp()
274
+ local_temp_file = m_artifact.download(
275
+ local_path=temp_dir,
276
+ overwrite=overwrite,
277
+ to_items_folder=False,
278
+ )
279
+ src = local_temp_file
280
+ # remove the prefix with relpath
281
+ dst = local_dst
282
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
283
+ shutil.move(src=src, dst=dst)
284
+ # clean temp dir
285
+ if os.path.isdir(temp_dir):
286
+ shutil.rmtree(temp_dir)
287
+ artifact.append(local_path)
288
+ elif isinstance(m_artifact, entities.LinkArtifact):
289
+ # remove the prefix with relpath
290
+ local_dst = os.path.join(local_path, m_artifact.filename)
291
+ if not os.path.isfile(local_dst) or overwrite:
292
+ # need_to_download
293
+ # 1. download to temp folder
294
+ temp_dir = tempfile.mkdtemp()
295
+ response = requests.get(m_artifact.url, stream=True)
296
+ local_temp_file = os.path.join(temp_dir, m_artifact.filename)
297
+ with open(local_temp_file, "wb") as handle:
298
+ for data in response.iter_content(chunk_size=8192):
299
+ handle.write(data)
300
+ src = local_temp_file
301
+ # remove the prefix with relpath
302
+ dst = local_dst
303
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
304
+ shutil.move(src=src, dst=dst)
305
+ # clean temp dir
306
+ if os.path.isdir(temp_dir):
307
+ shutil.rmtree(temp_dir)
308
+ artifact.append(local_path)
309
+ elif isinstance(m_artifact, entities.LocalArtifact):
310
+ pass
311
+ else:
312
+ raise ValueError('unsupported artifact type: {}'.format(type(m_artifact)))
313
+ else:
314
+ # for package artifacts - download using filter on the package directory
315
+ if all(elem is None for elem in [package_name, execution_id]):
316
+ raise PlatformException(error='400', message='Must input package or execution (id or entity)')
317
+ remote_path = self._build_path_header(
318
+ package_name=package_name,
319
+ execution_id=execution_id,
320
+ model_name=model_name,
321
+ )
322
+ without_relative_path = remote_path
323
+ remote_path += '/*'
324
+ filters = entities.Filters()
325
+ filters.add(field='filename', values=remote_path)
326
+ artifact = self.items_repository.download(filters=filters,
327
+ save_locally=save_locally,
328
+ local_path=local_path,
329
+ to_items_folder=False,
330
+ overwrite=overwrite,
331
+ without_relative_path=without_relative_path)
332
+ return artifact
333
+
334
+ def upload(self,
335
+ # what to upload
336
+ filepath: str,
337
+ # where to upload
338
+ package_name: str = None,
339
+ package: entities.Package = None,
340
+ execution_id: str = None,
341
+ execution: entities.Execution = None,
342
+ model_name: str = None,
343
+ # add information
344
+ overwrite: bool = False):
345
+ """
346
+
347
+ Upload binary file to artifact. get by name, id or type.
348
+ If artifact exists - overwriting binary
349
+ Else and if create==True a new artifact will be created and uploaded
350
+
351
+ :param str filepath: local binary file
352
+ :param str package_name: package name that include the artifact
353
+ :param dtlpy.entities.package.Package package: package object
354
+ :param str execution_id: execution id that include the artifact
355
+ :param dtlpy.entities.execution.Execution execution: execution object
356
+ :param str model_name: model name that include the artifact
357
+ :param bool overwrite: optional - default = False to overwrite an existing object
358
+ :return: Artifact Object
359
+ :rtype: dtlpy.entities.artifact.Artifact
360
+
361
+ **Example**:
362
+
363
+ .. code-block:: python
364
+
365
+ project.artifacts.upload(filepath='filepath',
366
+ package_name='package_name')
367
+ """
368
+ if self.model is not None:
369
+ model_name = self.model.name
370
+ if self.package is not None:
371
+ package_name = self.package.name
372
+
373
+ remote_path = self._build_path_header(package_name=package_name,
374
+ package=package,
375
+ execution=execution,
376
+ execution_id=execution_id,
377
+ model_name=model_name)
378
+
379
+ if all(elem is None for elem in [package_name, package, execution_id, execution, model_name]):
380
+ raise ValueError('Must input package or execution (id or entity)')
381
+
382
+ artifact = self.items_repository.upload(local_path=filepath,
383
+ remote_path=remote_path,
384
+ overwrite=overwrite,
385
+ output_entity=entities.Artifact)
386
+ if self.model is not None:
387
+ # list and update model
388
+ filters = entities.Filters()
389
+ filters.add(field='dir', values=remote_path + '*')
390
+ pages = self.items_repository.list(filters=filters)
391
+ model_artifacts = list()
392
+ for item in pages.all():
393
+ model_artifacts.append(entities.Artifact.from_json(_json=item.to_json(),
394
+ client_api=self._client_api,
395
+ dataset=item._dataset))
396
+ self.model.model_artifacts = model_artifacts
397
+ self.model.update()
398
+ logger.debug('Artifact uploaded successfully')
399
+ return artifact
400
+
401
+ def delete(self,
402
+ artifact_id=None,
403
+ artifact_name=None,
404
+ execution_id=None,
405
+ model_name=None,
406
+ package_name=None):
407
+ """
408
+ Delete artifacts
409
+
410
+ :param str artifact_id: search by artifact id
411
+ :param str artifact_name: search by artifact id
412
+ :param str execution_id: search by execution id
413
+ :param str model_name: search by model name
414
+ :param str package_name: search by package name
415
+ :return: True if success
416
+ :rtype: bool
417
+
418
+ **Example**:
419
+
420
+ .. code-block:: python
421
+
422
+ project.artifacts.delete(artifact_id='artifact_id',
423
+ package_name='package_name')
424
+ """
425
+ if self.model is not None:
426
+ model_name = self.model.name
427
+ if self.package is not None:
428
+ package_name = self.package.name
429
+
430
+ if artifact_id is not None or artifact_name is not None:
431
+ artifacts = [
432
+ self.get(
433
+ artifact_id=artifact_id,
434
+ artifact_name=artifact_name,
435
+ model_name=model_name,
436
+ )
437
+ ]
438
+ elif execution_id is not None or package_name is not None:
439
+ artifacts = self.list(
440
+ execution_id=execution_id,
441
+ package_name=package_name,
442
+ model_name=model_name,
443
+ )
444
+ else:
445
+ raise PlatformException('400',
446
+ 'Must provide one of: artifact_id, artifact_name, execution_id, package_name')
447
+
448
+ values = [artifact.id for artifact in artifacts]
449
+ self.items_repository.delete(filters=entities.Filters(field='id', values=values,
450
+ operator=entities.FiltersOperations.IN))
451
+
452
+ return True