dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtlpy/__init__.py +488 -488
- dtlpy/__version__.py +1 -1
- dtlpy/assets/__init__.py +26 -26
- dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
- dtlpy/assets/code_server/config.yaml +2 -2
- dtlpy/assets/code_server/installation.sh +24 -24
- dtlpy/assets/code_server/launch.json +13 -13
- dtlpy/assets/code_server/settings.json +2 -2
- dtlpy/assets/main.py +53 -53
- dtlpy/assets/main_partial.py +18 -18
- dtlpy/assets/mock.json +11 -11
- dtlpy/assets/model_adapter.py +83 -83
- dtlpy/assets/package.json +61 -61
- dtlpy/assets/package_catalog.json +29 -29
- dtlpy/assets/package_gitignore +307 -307
- dtlpy/assets/service_runners/__init__.py +33 -33
- dtlpy/assets/service_runners/converter.py +96 -96
- dtlpy/assets/service_runners/multi_method.py +49 -49
- dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
- dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
- dtlpy/assets/service_runners/multi_method_item.py +52 -52
- dtlpy/assets/service_runners/multi_method_json.py +52 -52
- dtlpy/assets/service_runners/single_method.py +37 -37
- dtlpy/assets/service_runners/single_method_annotation.py +43 -43
- dtlpy/assets/service_runners/single_method_dataset.py +43 -43
- dtlpy/assets/service_runners/single_method_item.py +41 -41
- dtlpy/assets/service_runners/single_method_json.py +42 -42
- dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
- dtlpy/assets/voc_annotation_template.xml +23 -23
- dtlpy/caches/base_cache.py +32 -32
- dtlpy/caches/cache.py +473 -473
- dtlpy/caches/dl_cache.py +201 -201
- dtlpy/caches/filesystem_cache.py +89 -89
- dtlpy/caches/redis_cache.py +84 -84
- dtlpy/dlp/__init__.py +20 -20
- dtlpy/dlp/cli_utilities.py +367 -367
- dtlpy/dlp/command_executor.py +764 -764
- dtlpy/dlp/dlp +1 -1
- dtlpy/dlp/dlp.bat +1 -1
- dtlpy/dlp/dlp.py +128 -128
- dtlpy/dlp/parser.py +651 -651
- dtlpy/entities/__init__.py +83 -83
- dtlpy/entities/analytic.py +311 -311
- dtlpy/entities/annotation.py +1879 -1879
- dtlpy/entities/annotation_collection.py +699 -699
- dtlpy/entities/annotation_definitions/__init__.py +20 -20
- dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
- dtlpy/entities/annotation_definitions/box.py +195 -195
- dtlpy/entities/annotation_definitions/classification.py +67 -67
- dtlpy/entities/annotation_definitions/comparison.py +72 -72
- dtlpy/entities/annotation_definitions/cube.py +204 -204
- dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
- dtlpy/entities/annotation_definitions/description.py +32 -32
- dtlpy/entities/annotation_definitions/ellipse.py +124 -124
- dtlpy/entities/annotation_definitions/free_text.py +62 -62
- dtlpy/entities/annotation_definitions/gis.py +69 -69
- dtlpy/entities/annotation_definitions/note.py +139 -139
- dtlpy/entities/annotation_definitions/point.py +117 -117
- dtlpy/entities/annotation_definitions/polygon.py +182 -182
- dtlpy/entities/annotation_definitions/polyline.py +111 -111
- dtlpy/entities/annotation_definitions/pose.py +92 -92
- dtlpy/entities/annotation_definitions/ref_image.py +86 -86
- dtlpy/entities/annotation_definitions/segmentation.py +240 -240
- dtlpy/entities/annotation_definitions/subtitle.py +34 -34
- dtlpy/entities/annotation_definitions/text.py +85 -85
- dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
- dtlpy/entities/app.py +220 -220
- dtlpy/entities/app_module.py +107 -107
- dtlpy/entities/artifact.py +174 -174
- dtlpy/entities/assignment.py +399 -399
- dtlpy/entities/base_entity.py +214 -214
- dtlpy/entities/bot.py +113 -113
- dtlpy/entities/codebase.py +296 -296
- dtlpy/entities/collection.py +38 -38
- dtlpy/entities/command.py +169 -169
- dtlpy/entities/compute.py +442 -442
- dtlpy/entities/dataset.py +1285 -1285
- dtlpy/entities/directory_tree.py +44 -44
- dtlpy/entities/dpk.py +470 -470
- dtlpy/entities/driver.py +222 -222
- dtlpy/entities/execution.py +397 -397
- dtlpy/entities/feature.py +124 -124
- dtlpy/entities/feature_set.py +145 -145
- dtlpy/entities/filters.py +641 -641
- dtlpy/entities/gis_item.py +107 -107
- dtlpy/entities/integration.py +184 -184
- dtlpy/entities/item.py +953 -953
- dtlpy/entities/label.py +123 -123
- dtlpy/entities/links.py +85 -85
- dtlpy/entities/message.py +175 -175
- dtlpy/entities/model.py +694 -691
- dtlpy/entities/node.py +1005 -1005
- dtlpy/entities/ontology.py +803 -803
- dtlpy/entities/organization.py +287 -287
- dtlpy/entities/package.py +657 -657
- dtlpy/entities/package_defaults.py +5 -5
- dtlpy/entities/package_function.py +185 -185
- dtlpy/entities/package_module.py +113 -113
- dtlpy/entities/package_slot.py +118 -118
- dtlpy/entities/paged_entities.py +290 -267
- dtlpy/entities/pipeline.py +593 -593
- dtlpy/entities/pipeline_execution.py +279 -279
- dtlpy/entities/project.py +394 -394
- dtlpy/entities/prompt_item.py +499 -499
- dtlpy/entities/recipe.py +301 -301
- dtlpy/entities/reflect_dict.py +102 -102
- dtlpy/entities/resource_execution.py +138 -138
- dtlpy/entities/service.py +958 -958
- dtlpy/entities/service_driver.py +117 -117
- dtlpy/entities/setting.py +294 -294
- dtlpy/entities/task.py +491 -491
- dtlpy/entities/time_series.py +143 -143
- dtlpy/entities/trigger.py +426 -426
- dtlpy/entities/user.py +118 -118
- dtlpy/entities/webhook.py +124 -124
- dtlpy/examples/__init__.py +19 -19
- dtlpy/examples/add_labels.py +135 -135
- dtlpy/examples/add_metadata_to_item.py +21 -21
- dtlpy/examples/annotate_items_using_model.py +65 -65
- dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
- dtlpy/examples/annotations_convert_to_voc.py +9 -9
- dtlpy/examples/annotations_convert_to_yolo.py +9 -9
- dtlpy/examples/convert_annotation_types.py +51 -51
- dtlpy/examples/converter.py +143 -143
- dtlpy/examples/copy_annotations.py +22 -22
- dtlpy/examples/copy_folder.py +31 -31
- dtlpy/examples/create_annotations.py +51 -51
- dtlpy/examples/create_video_annotations.py +83 -83
- dtlpy/examples/delete_annotations.py +26 -26
- dtlpy/examples/filters.py +113 -113
- dtlpy/examples/move_item.py +23 -23
- dtlpy/examples/play_video_annotation.py +13 -13
- dtlpy/examples/show_item_and_mask.py +53 -53
- dtlpy/examples/triggers.py +49 -49
- dtlpy/examples/upload_batch_of_items.py +20 -20
- dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
- dtlpy/examples/upload_items_with_modalities.py +43 -43
- dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
- dtlpy/examples/upload_yolo_format_annotations.py +70 -70
- dtlpy/exceptions.py +125 -125
- dtlpy/miscellaneous/__init__.py +20 -20
- dtlpy/miscellaneous/dict_differ.py +95 -95
- dtlpy/miscellaneous/git_utils.py +217 -217
- dtlpy/miscellaneous/json_utils.py +14 -14
- dtlpy/miscellaneous/list_print.py +105 -105
- dtlpy/miscellaneous/zipping.py +130 -130
- dtlpy/ml/__init__.py +20 -20
- dtlpy/ml/base_feature_extractor_adapter.py +27 -27
- dtlpy/ml/base_model_adapter.py +945 -940
- dtlpy/ml/metrics.py +461 -461
- dtlpy/ml/predictions_utils.py +274 -274
- dtlpy/ml/summary_writer.py +57 -57
- dtlpy/ml/train_utils.py +60 -60
- dtlpy/new_instance.py +252 -252
- dtlpy/repositories/__init__.py +56 -56
- dtlpy/repositories/analytics.py +85 -85
- dtlpy/repositories/annotations.py +916 -916
- dtlpy/repositories/apps.py +383 -383
- dtlpy/repositories/artifacts.py +452 -452
- dtlpy/repositories/assignments.py +599 -599
- dtlpy/repositories/bots.py +213 -213
- dtlpy/repositories/codebases.py +559 -559
- dtlpy/repositories/collections.py +332 -348
- dtlpy/repositories/commands.py +158 -158
- dtlpy/repositories/compositions.py +61 -61
- dtlpy/repositories/computes.py +434 -406
- dtlpy/repositories/datasets.py +1291 -1291
- dtlpy/repositories/downloader.py +895 -895
- dtlpy/repositories/dpks.py +433 -433
- dtlpy/repositories/drivers.py +266 -266
- dtlpy/repositories/executions.py +817 -817
- dtlpy/repositories/feature_sets.py +226 -226
- dtlpy/repositories/features.py +238 -238
- dtlpy/repositories/integrations.py +484 -484
- dtlpy/repositories/items.py +909 -915
- dtlpy/repositories/messages.py +94 -94
- dtlpy/repositories/models.py +877 -867
- dtlpy/repositories/nodes.py +80 -80
- dtlpy/repositories/ontologies.py +511 -511
- dtlpy/repositories/organizations.py +525 -525
- dtlpy/repositories/packages.py +1941 -1941
- dtlpy/repositories/pipeline_executions.py +448 -448
- dtlpy/repositories/pipelines.py +642 -642
- dtlpy/repositories/projects.py +539 -539
- dtlpy/repositories/recipes.py +399 -399
- dtlpy/repositories/resource_executions.py +137 -137
- dtlpy/repositories/schema.py +120 -120
- dtlpy/repositories/service_drivers.py +213 -213
- dtlpy/repositories/services.py +1704 -1704
- dtlpy/repositories/settings.py +339 -339
- dtlpy/repositories/tasks.py +1124 -1124
- dtlpy/repositories/times_series.py +278 -278
- dtlpy/repositories/triggers.py +536 -536
- dtlpy/repositories/upload_element.py +257 -257
- dtlpy/repositories/uploader.py +651 -651
- dtlpy/repositories/webhooks.py +249 -249
- dtlpy/services/__init__.py +22 -22
- dtlpy/services/aihttp_retry.py +131 -131
- dtlpy/services/api_client.py +1782 -1782
- dtlpy/services/api_reference.py +40 -40
- dtlpy/services/async_utils.py +133 -133
- dtlpy/services/calls_counter.py +44 -44
- dtlpy/services/check_sdk.py +68 -68
- dtlpy/services/cookie.py +115 -115
- dtlpy/services/create_logger.py +156 -156
- dtlpy/services/events.py +84 -84
- dtlpy/services/logins.py +235 -235
- dtlpy/services/reporter.py +256 -256
- dtlpy/services/service_defaults.py +91 -91
- dtlpy/utilities/__init__.py +20 -20
- dtlpy/utilities/annotations/__init__.py +16 -16
- dtlpy/utilities/annotations/annotation_converters.py +269 -269
- dtlpy/utilities/base_package_runner.py +264 -264
- dtlpy/utilities/converter.py +1650 -1650
- dtlpy/utilities/dataset_generators/__init__.py +1 -1
- dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
- dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
- dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
- dtlpy/utilities/local_development/__init__.py +1 -1
- dtlpy/utilities/local_development/local_session.py +179 -179
- dtlpy/utilities/reports/__init__.py +2 -2
- dtlpy/utilities/reports/figures.py +343 -343
- dtlpy/utilities/reports/report.py +71 -71
- dtlpy/utilities/videos/__init__.py +17 -17
- dtlpy/utilities/videos/video_player.py +598 -598
- dtlpy/utilities/videos/videos.py +470 -470
- {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
- dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
- {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
- {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
- {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
- dtlpy-1.114.13.dist-info/RECORD +240 -0
- {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
- tests/features/environment.py +551 -550
- dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
- dtlpy-1.113.10.dist-info/RECORD +0 -244
- tests/assets/__init__.py +0 -0
- tests/assets/models_flow/__init__.py +0 -0
- tests/assets/models_flow/failedmain.py +0 -52
- tests/assets/models_flow/main.py +0 -62
- tests/assets/models_flow/main_model.py +0 -54
- {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
- {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
dtlpy/repositories/artifacts.py
CHANGED
|
@@ -1,452 +1,452 @@
|
|
|
1
|
-
import tempfile
|
|
2
|
-
import requests
|
|
3
|
-
import logging
|
|
4
|
-
import shutil
|
|
5
|
-
import os
|
|
6
|
-
|
|
7
|
-
from .. import entities, miscellaneous, PlatformException, exceptions, repositories
|
|
8
|
-
from ..services.api_client import ApiClient
|
|
9
|
-
|
|
10
|
-
logger = logging.getLogger(name='dtlpy')
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
class Artifacts:
|
|
14
|
-
"""
|
|
15
|
-
Artifacts repository
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
def __init__(self,
|
|
19
|
-
client_api: ApiClient,
|
|
20
|
-
project: entities.Project = None,
|
|
21
|
-
dataset: entities.Dataset = None,
|
|
22
|
-
project_id: str = None,
|
|
23
|
-
model: entities.Model = None,
|
|
24
|
-
package: entities.Package = None,
|
|
25
|
-
dataset_name=None):
|
|
26
|
-
self._client_api = client_api
|
|
27
|
-
self._project = project
|
|
28
|
-
self._dataset = dataset
|
|
29
|
-
self._items_repository = None
|
|
30
|
-
self.dataset_name = dataset_name
|
|
31
|
-
self.project_id = project_id
|
|
32
|
-
self.model = model
|
|
33
|
-
self.package = package
|
|
34
|
-
|
|
35
|
-
############
|
|
36
|
-
# entities #
|
|
37
|
-
############
|
|
38
|
-
@property
|
|
39
|
-
def dataset(self) -> entities.Dataset:
|
|
40
|
-
if self._dataset is None:
|
|
41
|
-
# get dataset from project
|
|
42
|
-
try:
|
|
43
|
-
if self.dataset_name is None:
|
|
44
|
-
self.dataset_name = 'Binaries'
|
|
45
|
-
self._dataset = self.project.datasets._get_binaries_dataset()
|
|
46
|
-
else:
|
|
47
|
-
self._dataset = self.project.datasets.get(dataset_name=self.dataset_name)
|
|
48
|
-
except exceptions.NotFound:
|
|
49
|
-
raise ValueError(
|
|
50
|
-
f'Missing "{self.dataset_name}" dataset in the project. Please contact support for help')
|
|
51
|
-
|
|
52
|
-
return self._dataset
|
|
53
|
-
|
|
54
|
-
@property
|
|
55
|
-
def project(self) -> entities.Project:
|
|
56
|
-
if self._project is None:
|
|
57
|
-
if self._dataset is not None:
|
|
58
|
-
self._project = self._dataset.project
|
|
59
|
-
elif self.project_id is not None:
|
|
60
|
-
self._project = repositories.Projects(client_api=self._client_api).get(project_id=self.project_id)
|
|
61
|
-
else:
|
|
62
|
-
raise PlatformException(error='400',
|
|
63
|
-
message='Artifacts doesnt have a project, at least one must not be None: dataset, project or project_id')
|
|
64
|
-
return self._project
|
|
65
|
-
|
|
66
|
-
################
|
|
67
|
-
# repositories #
|
|
68
|
-
################
|
|
69
|
-
@property
|
|
70
|
-
def items_repository(self):
|
|
71
|
-
if self._items_repository is None:
|
|
72
|
-
# load Binaries dataset
|
|
73
|
-
# load items repository
|
|
74
|
-
self._items_repository = self.dataset.items
|
|
75
|
-
self._items_repository.set_items_entity(entities.Artifact)
|
|
76
|
-
return self._items_repository
|
|
77
|
-
|
|
78
|
-
###########
|
|
79
|
-
# methods #
|
|
80
|
-
###########
|
|
81
|
-
@staticmethod
|
|
82
|
-
def _build_path_header(
|
|
83
|
-
package_name=None,
|
|
84
|
-
package=None,
|
|
85
|
-
execution_id=None,
|
|
86
|
-
execution=None,
|
|
87
|
-
model_name=None,
|
|
88
|
-
):
|
|
89
|
-
remote_path = '/artifacts'
|
|
90
|
-
if package_name is not None or package is not None:
|
|
91
|
-
if package is not None:
|
|
92
|
-
package_name = package.name
|
|
93
|
-
remote_path += '/packages/{}'.format(package_name)
|
|
94
|
-
if execution_id is not None or execution is not None:
|
|
95
|
-
if execution is not None:
|
|
96
|
-
execution_id = execution.id
|
|
97
|
-
remote_path += '/executions/{}'.format(execution_id)
|
|
98
|
-
if model_name is not None:
|
|
99
|
-
remote_path += '/models/{}'.format(model_name)
|
|
100
|
-
|
|
101
|
-
return remote_path
|
|
102
|
-
|
|
103
|
-
def list(self,
|
|
104
|
-
execution_id: str = None,
|
|
105
|
-
package_name: str = None,
|
|
106
|
-
model_name: str = None) -> miscellaneous.List[entities.Artifact]:
|
|
107
|
-
"""
|
|
108
|
-
List of artifacts
|
|
109
|
-
|
|
110
|
-
:param str execution_id: list by execution id
|
|
111
|
-
:param str package_name: list by package name
|
|
112
|
-
:param str model_name: list by model name
|
|
113
|
-
:return: list of artifacts
|
|
114
|
-
:rtype: miscellaneous.List[dtlpy.entities.artifact.Artifact]
|
|
115
|
-
|
|
116
|
-
**Example**:
|
|
117
|
-
|
|
118
|
-
.. code-block:: python
|
|
119
|
-
|
|
120
|
-
project.artifacts.list(package_name='package_name')
|
|
121
|
-
"""
|
|
122
|
-
if self.model is not None:
|
|
123
|
-
model_name = self.model.name
|
|
124
|
-
if self.package is not None:
|
|
125
|
-
package_name = self.package.name
|
|
126
|
-
|
|
127
|
-
filters = entities.Filters()
|
|
128
|
-
remote_path = self._build_path_header(
|
|
129
|
-
package_name=package_name,
|
|
130
|
-
execution_id=execution_id,
|
|
131
|
-
model_name=model_name
|
|
132
|
-
)
|
|
133
|
-
|
|
134
|
-
remote_path += '/*'
|
|
135
|
-
filters.add(field='filename', values=remote_path)
|
|
136
|
-
pages = self.items_repository.list(filters=filters)
|
|
137
|
-
items = [entities.Artifact.from_json(_json=item.to_json(),
|
|
138
|
-
client_api=self._client_api,
|
|
139
|
-
dataset=item._dataset,
|
|
140
|
-
project=self.project)
|
|
141
|
-
for page in pages for item in page]
|
|
142
|
-
return miscellaneous.List(items)
|
|
143
|
-
|
|
144
|
-
def get(self,
|
|
145
|
-
artifact_id: str = None,
|
|
146
|
-
artifact_name: str = None,
|
|
147
|
-
model_name: str = None,
|
|
148
|
-
execution_id: str = None,
|
|
149
|
-
package_name: str = None) -> entities.ItemArtifact:
|
|
150
|
-
"""
|
|
151
|
-
|
|
152
|
-
Get an artifact object by name, id or type
|
|
153
|
-
If by name or type - need to input also execution/task id for the artifact folder
|
|
154
|
-
|
|
155
|
-
:param str artifact_id: search by artifact id
|
|
156
|
-
:param str artifact_name: search by artifact id
|
|
157
|
-
:param str model_name: search by model name
|
|
158
|
-
:param str execution_id: search by execution id
|
|
159
|
-
:param str package_name: search by package name
|
|
160
|
-
:return: Artifact object
|
|
161
|
-
:rtype: dtlpy.entities.artifact.Artifact
|
|
162
|
-
|
|
163
|
-
**Example**:
|
|
164
|
-
|
|
165
|
-
.. code-block:: python
|
|
166
|
-
|
|
167
|
-
project.artifacts.get(artifact_id='artifact_id')
|
|
168
|
-
"""
|
|
169
|
-
if self.model is not None:
|
|
170
|
-
model_name = self.model.name
|
|
171
|
-
if self.package is not None:
|
|
172
|
-
package_name = self.package.name
|
|
173
|
-
|
|
174
|
-
if artifact_id is not None:
|
|
175
|
-
artifact = self.items_repository.get(item_id=artifact_id)
|
|
176
|
-
# verify input artifact name is same as the given id
|
|
177
|
-
if artifact_name is not None and artifact.name != artifact_name:
|
|
178
|
-
logger.warning(
|
|
179
|
-
"Mismatch found in artifacts.get: artifact_name is different then artifact.name:"
|
|
180
|
-
" {!r} != {!r}".format(
|
|
181
|
-
artifact_name,
|
|
182
|
-
artifact.name))
|
|
183
|
-
elif artifact_name is not None:
|
|
184
|
-
artifacts = self.list(
|
|
185
|
-
execution_id=execution_id,
|
|
186
|
-
package_name=package_name,
|
|
187
|
-
model_name=model_name
|
|
188
|
-
)
|
|
189
|
-
artifact = [artifact for artifact in artifacts if artifact.name == artifact_name]
|
|
190
|
-
if len(artifact) == 1:
|
|
191
|
-
artifact = artifact[0]
|
|
192
|
-
elif len(artifact) > 1:
|
|
193
|
-
raise PlatformException('404', 'More Than one Artifact found')
|
|
194
|
-
else:
|
|
195
|
-
raise PlatformException('404', 'Artifact not found')
|
|
196
|
-
else:
|
|
197
|
-
msg = 'one input must be not None: artifact_id or artifact_name'
|
|
198
|
-
raise ValueError(msg)
|
|
199
|
-
return artifact
|
|
200
|
-
|
|
201
|
-
def download(
|
|
202
|
-
self,
|
|
203
|
-
artifact_id: str = None,
|
|
204
|
-
artifact_name: str = None,
|
|
205
|
-
execution_id: str = None,
|
|
206
|
-
package_name: str = None,
|
|
207
|
-
model_name: str = None,
|
|
208
|
-
local_path: str = None,
|
|
209
|
-
overwrite: bool = False,
|
|
210
|
-
save_locally: bool = True
|
|
211
|
-
):
|
|
212
|
-
"""
|
|
213
|
-
|
|
214
|
-
Download artifact binary.
|
|
215
|
-
Get artifact by name, id or type
|
|
216
|
-
|
|
217
|
-
:param str artifact_id: search by artifact id
|
|
218
|
-
:param str artifact_name: search by artifact id
|
|
219
|
-
:param str execution_id: search by execution id
|
|
220
|
-
:param str package_name: search by package name
|
|
221
|
-
:param str model_name: search by model name
|
|
222
|
-
:param str local_path: artifact will be saved to this filepath
|
|
223
|
-
:param bool overwrite: optional - default = False
|
|
224
|
-
:param bool save_locally: to save the file local
|
|
225
|
-
:return: file path
|
|
226
|
-
:rtype: str
|
|
227
|
-
|
|
228
|
-
**Example**:
|
|
229
|
-
|
|
230
|
-
.. code-block:: python
|
|
231
|
-
|
|
232
|
-
project.artifacts.download(artifact_id='artifact_id',
|
|
233
|
-
local_path='your_path',
|
|
234
|
-
overwrite=True,
|
|
235
|
-
save_locally=True)
|
|
236
|
-
"""
|
|
237
|
-
if self.model is not None:
|
|
238
|
-
model_name = self.model.name
|
|
239
|
-
if self.package is not None:
|
|
240
|
-
package_name = self.package.name
|
|
241
|
-
|
|
242
|
-
if artifact_id is not None:
|
|
243
|
-
# download
|
|
244
|
-
artifact = self.items_repository.download(items=artifact_id,
|
|
245
|
-
save_locally=save_locally,
|
|
246
|
-
local_path=local_path,
|
|
247
|
-
overwrite=overwrite)
|
|
248
|
-
elif artifact_name is not None:
|
|
249
|
-
artifact_obj: entities.ItemArtifact = self.get(artifact_id=artifact_id,
|
|
250
|
-
execution_id=execution_id,
|
|
251
|
-
package_name=package_name,
|
|
252
|
-
artifact_name=artifact_name)
|
|
253
|
-
|
|
254
|
-
artifact = artifact_obj.download(save_locally=save_locally,
|
|
255
|
-
local_path=local_path,
|
|
256
|
-
overwrite=overwrite)
|
|
257
|
-
|
|
258
|
-
else:
|
|
259
|
-
if self.model is not None:
|
|
260
|
-
artifact = list()
|
|
261
|
-
for m_artifact in self.model.model_artifacts:
|
|
262
|
-
if isinstance(m_artifact, entities.ItemArtifact):
|
|
263
|
-
if not m_artifact.is_fetched:
|
|
264
|
-
m_artifact = self.items_repository.get(item_id=m_artifact.id)
|
|
265
|
-
model_remote_root = m_artifact.filename.split('/')
|
|
266
|
-
model_remote_root = '/'.join(model_remote_root[:4])
|
|
267
|
-
# remove the prefix with relpath
|
|
268
|
-
local_dst = os.path.join(local_path,
|
|
269
|
-
os.path.relpath(m_artifact.filename, model_remote_root))
|
|
270
|
-
if not os.path.isfile(local_dst) or overwrite:
|
|
271
|
-
# need_to_download
|
|
272
|
-
# 1. download to temp folder
|
|
273
|
-
temp_dir = tempfile.mkdtemp()
|
|
274
|
-
local_temp_file = m_artifact.download(
|
|
275
|
-
local_path=temp_dir,
|
|
276
|
-
overwrite=overwrite,
|
|
277
|
-
to_items_folder=False,
|
|
278
|
-
)
|
|
279
|
-
src = local_temp_file
|
|
280
|
-
# remove the prefix with relpath
|
|
281
|
-
dst = local_dst
|
|
282
|
-
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
283
|
-
shutil.move(src=src, dst=dst)
|
|
284
|
-
# clean temp dir
|
|
285
|
-
if os.path.isdir(temp_dir):
|
|
286
|
-
shutil.rmtree(temp_dir)
|
|
287
|
-
artifact.append(local_path)
|
|
288
|
-
elif isinstance(m_artifact, entities.LinkArtifact):
|
|
289
|
-
# remove the prefix with relpath
|
|
290
|
-
local_dst = os.path.join(local_path, m_artifact.filename)
|
|
291
|
-
if not os.path.isfile(local_dst) or overwrite:
|
|
292
|
-
# need_to_download
|
|
293
|
-
# 1. download to temp folder
|
|
294
|
-
temp_dir = tempfile.mkdtemp()
|
|
295
|
-
response = requests.get(m_artifact.url, stream=True)
|
|
296
|
-
local_temp_file = os.path.join(temp_dir, m_artifact.filename)
|
|
297
|
-
with open(local_temp_file, "wb") as handle:
|
|
298
|
-
for data in response.iter_content(chunk_size=8192):
|
|
299
|
-
handle.write(data)
|
|
300
|
-
src = local_temp_file
|
|
301
|
-
# remove the prefix with relpath
|
|
302
|
-
dst = local_dst
|
|
303
|
-
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
304
|
-
shutil.move(src=src, dst=dst)
|
|
305
|
-
# clean temp dir
|
|
306
|
-
if os.path.isdir(temp_dir):
|
|
307
|
-
shutil.rmtree(temp_dir)
|
|
308
|
-
artifact.append(local_path)
|
|
309
|
-
elif isinstance(m_artifact, entities.LocalArtifact):
|
|
310
|
-
pass
|
|
311
|
-
else:
|
|
312
|
-
raise ValueError('unsupported artifact type: {}'.format(type(m_artifact)))
|
|
313
|
-
else:
|
|
314
|
-
# for package artifacts - download using filter on the package directory
|
|
315
|
-
if all(elem is None for elem in [package_name, execution_id]):
|
|
316
|
-
raise PlatformException(error='400', message='Must input package or execution (id or entity)')
|
|
317
|
-
remote_path = self._build_path_header(
|
|
318
|
-
package_name=package_name,
|
|
319
|
-
execution_id=execution_id,
|
|
320
|
-
model_name=model_name,
|
|
321
|
-
)
|
|
322
|
-
without_relative_path = remote_path
|
|
323
|
-
remote_path += '/*'
|
|
324
|
-
filters = entities.Filters()
|
|
325
|
-
filters.add(field='filename', values=remote_path)
|
|
326
|
-
artifact = self.items_repository.download(filters=filters,
|
|
327
|
-
save_locally=save_locally,
|
|
328
|
-
local_path=local_path,
|
|
329
|
-
to_items_folder=False,
|
|
330
|
-
overwrite=overwrite,
|
|
331
|
-
without_relative_path=without_relative_path)
|
|
332
|
-
return artifact
|
|
333
|
-
|
|
334
|
-
def upload(self,
|
|
335
|
-
# what to upload
|
|
336
|
-
filepath: str,
|
|
337
|
-
# where to upload
|
|
338
|
-
package_name: str = None,
|
|
339
|
-
package: entities.Package = None,
|
|
340
|
-
execution_id: str = None,
|
|
341
|
-
execution: entities.Execution = None,
|
|
342
|
-
model_name: str = None,
|
|
343
|
-
# add information
|
|
344
|
-
overwrite: bool = False):
|
|
345
|
-
"""
|
|
346
|
-
|
|
347
|
-
Upload binary file to artifact. get by name, id or type.
|
|
348
|
-
If artifact exists - overwriting binary
|
|
349
|
-
Else and if create==True a new artifact will be created and uploaded
|
|
350
|
-
|
|
351
|
-
:param str filepath: local binary file
|
|
352
|
-
:param str package_name: package name that include the artifact
|
|
353
|
-
:param dtlpy.entities.package.Package package: package object
|
|
354
|
-
:param str execution_id: execution id that include the artifact
|
|
355
|
-
:param dtlpy.entities.execution.Execution execution: execution object
|
|
356
|
-
:param str model_name: model name that include the artifact
|
|
357
|
-
:param bool overwrite: optional - default = False to overwrite an existing object
|
|
358
|
-
:return: Artifact Object
|
|
359
|
-
:rtype: dtlpy.entities.artifact.Artifact
|
|
360
|
-
|
|
361
|
-
**Example**:
|
|
362
|
-
|
|
363
|
-
.. code-block:: python
|
|
364
|
-
|
|
365
|
-
project.artifacts.upload(filepath='filepath',
|
|
366
|
-
package_name='package_name')
|
|
367
|
-
"""
|
|
368
|
-
if self.model is not None:
|
|
369
|
-
model_name = self.model.name
|
|
370
|
-
if self.package is not None:
|
|
371
|
-
package_name = self.package.name
|
|
372
|
-
|
|
373
|
-
remote_path = self._build_path_header(package_name=package_name,
|
|
374
|
-
package=package,
|
|
375
|
-
execution=execution,
|
|
376
|
-
execution_id=execution_id,
|
|
377
|
-
model_name=model_name)
|
|
378
|
-
|
|
379
|
-
if all(elem is None for elem in [package_name, package, execution_id, execution, model_name]):
|
|
380
|
-
raise ValueError('Must input package or execution (id or entity)')
|
|
381
|
-
|
|
382
|
-
artifact = self.items_repository.upload(local_path=filepath,
|
|
383
|
-
remote_path=remote_path,
|
|
384
|
-
overwrite=overwrite,
|
|
385
|
-
output_entity=entities.Artifact)
|
|
386
|
-
if self.model is not None:
|
|
387
|
-
# list and update model
|
|
388
|
-
filters = entities.Filters()
|
|
389
|
-
filters.add(field='dir', values=remote_path + '*')
|
|
390
|
-
pages = self.items_repository.list(filters=filters)
|
|
391
|
-
model_artifacts = list()
|
|
392
|
-
for item in pages.all():
|
|
393
|
-
model_artifacts.append(entities.Artifact.from_json(_json=item.to_json(),
|
|
394
|
-
client_api=self._client_api,
|
|
395
|
-
dataset=item._dataset))
|
|
396
|
-
self.model.model_artifacts = model_artifacts
|
|
397
|
-
self.model.update()
|
|
398
|
-
logger.debug('Artifact uploaded successfully')
|
|
399
|
-
return artifact
|
|
400
|
-
|
|
401
|
-
def delete(self,
|
|
402
|
-
artifact_id=None,
|
|
403
|
-
artifact_name=None,
|
|
404
|
-
execution_id=None,
|
|
405
|
-
model_name=None,
|
|
406
|
-
package_name=None):
|
|
407
|
-
"""
|
|
408
|
-
Delete artifacts
|
|
409
|
-
|
|
410
|
-
:param str artifact_id: search by artifact id
|
|
411
|
-
:param str artifact_name: search by artifact id
|
|
412
|
-
:param str execution_id: search by execution id
|
|
413
|
-
:param str model_name: search by model name
|
|
414
|
-
:param str package_name: search by package name
|
|
415
|
-
:return: True if success
|
|
416
|
-
:rtype: bool
|
|
417
|
-
|
|
418
|
-
**Example**:
|
|
419
|
-
|
|
420
|
-
.. code-block:: python
|
|
421
|
-
|
|
422
|
-
project.artifacts.delete(artifact_id='artifact_id',
|
|
423
|
-
package_name='package_name')
|
|
424
|
-
"""
|
|
425
|
-
if self.model is not None:
|
|
426
|
-
model_name = self.model.name
|
|
427
|
-
if self.package is not None:
|
|
428
|
-
package_name = self.package.name
|
|
429
|
-
|
|
430
|
-
if artifact_id is not None or artifact_name is not None:
|
|
431
|
-
artifacts = [
|
|
432
|
-
self.get(
|
|
433
|
-
artifact_id=artifact_id,
|
|
434
|
-
artifact_name=artifact_name,
|
|
435
|
-
model_name=model_name,
|
|
436
|
-
)
|
|
437
|
-
]
|
|
438
|
-
elif execution_id is not None or package_name is not None:
|
|
439
|
-
artifacts = self.list(
|
|
440
|
-
execution_id=execution_id,
|
|
441
|
-
package_name=package_name,
|
|
442
|
-
model_name=model_name,
|
|
443
|
-
)
|
|
444
|
-
else:
|
|
445
|
-
raise PlatformException('400',
|
|
446
|
-
'Must provide one of: artifact_id, artifact_name, execution_id, package_name')
|
|
447
|
-
|
|
448
|
-
values = [artifact.id for artifact in artifacts]
|
|
449
|
-
self.items_repository.delete(filters=entities.Filters(field='id', values=values,
|
|
450
|
-
operator=entities.FiltersOperations.IN))
|
|
451
|
-
|
|
452
|
-
return True
|
|
1
|
+
import tempfile
|
|
2
|
+
import requests
|
|
3
|
+
import logging
|
|
4
|
+
import shutil
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
from .. import entities, miscellaneous, PlatformException, exceptions, repositories
|
|
8
|
+
from ..services.api_client import ApiClient
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(name='dtlpy')
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Artifacts:
|
|
14
|
+
"""
|
|
15
|
+
Artifacts repository
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self,
|
|
19
|
+
client_api: ApiClient,
|
|
20
|
+
project: entities.Project = None,
|
|
21
|
+
dataset: entities.Dataset = None,
|
|
22
|
+
project_id: str = None,
|
|
23
|
+
model: entities.Model = None,
|
|
24
|
+
package: entities.Package = None,
|
|
25
|
+
dataset_name=None):
|
|
26
|
+
self._client_api = client_api
|
|
27
|
+
self._project = project
|
|
28
|
+
self._dataset = dataset
|
|
29
|
+
self._items_repository = None
|
|
30
|
+
self.dataset_name = dataset_name
|
|
31
|
+
self.project_id = project_id
|
|
32
|
+
self.model = model
|
|
33
|
+
self.package = package
|
|
34
|
+
|
|
35
|
+
############
|
|
36
|
+
# entities #
|
|
37
|
+
############
|
|
38
|
+
@property
|
|
39
|
+
def dataset(self) -> entities.Dataset:
|
|
40
|
+
if self._dataset is None:
|
|
41
|
+
# get dataset from project
|
|
42
|
+
try:
|
|
43
|
+
if self.dataset_name is None:
|
|
44
|
+
self.dataset_name = 'Binaries'
|
|
45
|
+
self._dataset = self.project.datasets._get_binaries_dataset()
|
|
46
|
+
else:
|
|
47
|
+
self._dataset = self.project.datasets.get(dataset_name=self.dataset_name)
|
|
48
|
+
except exceptions.NotFound:
|
|
49
|
+
raise ValueError(
|
|
50
|
+
f'Missing "{self.dataset_name}" dataset in the project. Please contact support for help')
|
|
51
|
+
|
|
52
|
+
return self._dataset
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def project(self) -> entities.Project:
|
|
56
|
+
if self._project is None:
|
|
57
|
+
if self._dataset is not None:
|
|
58
|
+
self._project = self._dataset.project
|
|
59
|
+
elif self.project_id is not None:
|
|
60
|
+
self._project = repositories.Projects(client_api=self._client_api).get(project_id=self.project_id)
|
|
61
|
+
else:
|
|
62
|
+
raise PlatformException(error='400',
|
|
63
|
+
message='Artifacts doesnt have a project, at least one must not be None: dataset, project or project_id')
|
|
64
|
+
return self._project
|
|
65
|
+
|
|
66
|
+
################
|
|
67
|
+
# repositories #
|
|
68
|
+
################
|
|
69
|
+
@property
|
|
70
|
+
def items_repository(self):
|
|
71
|
+
if self._items_repository is None:
|
|
72
|
+
# load Binaries dataset
|
|
73
|
+
# load items repository
|
|
74
|
+
self._items_repository = self.dataset.items
|
|
75
|
+
self._items_repository.set_items_entity(entities.Artifact)
|
|
76
|
+
return self._items_repository
|
|
77
|
+
|
|
78
|
+
###########
|
|
79
|
+
# methods #
|
|
80
|
+
###########
|
|
81
|
+
@staticmethod
|
|
82
|
+
def _build_path_header(
|
|
83
|
+
package_name=None,
|
|
84
|
+
package=None,
|
|
85
|
+
execution_id=None,
|
|
86
|
+
execution=None,
|
|
87
|
+
model_name=None,
|
|
88
|
+
):
|
|
89
|
+
remote_path = '/artifacts'
|
|
90
|
+
if package_name is not None or package is not None:
|
|
91
|
+
if package is not None:
|
|
92
|
+
package_name = package.name
|
|
93
|
+
remote_path += '/packages/{}'.format(package_name)
|
|
94
|
+
if execution_id is not None or execution is not None:
|
|
95
|
+
if execution is not None:
|
|
96
|
+
execution_id = execution.id
|
|
97
|
+
remote_path += '/executions/{}'.format(execution_id)
|
|
98
|
+
if model_name is not None:
|
|
99
|
+
remote_path += '/models/{}'.format(model_name)
|
|
100
|
+
|
|
101
|
+
return remote_path
|
|
102
|
+
|
|
103
|
+
def list(self,
|
|
104
|
+
execution_id: str = None,
|
|
105
|
+
package_name: str = None,
|
|
106
|
+
model_name: str = None) -> miscellaneous.List[entities.Artifact]:
|
|
107
|
+
"""
|
|
108
|
+
List of artifacts
|
|
109
|
+
|
|
110
|
+
:param str execution_id: list by execution id
|
|
111
|
+
:param str package_name: list by package name
|
|
112
|
+
:param str model_name: list by model name
|
|
113
|
+
:return: list of artifacts
|
|
114
|
+
:rtype: miscellaneous.List[dtlpy.entities.artifact.Artifact]
|
|
115
|
+
|
|
116
|
+
**Example**:
|
|
117
|
+
|
|
118
|
+
.. code-block:: python
|
|
119
|
+
|
|
120
|
+
project.artifacts.list(package_name='package_name')
|
|
121
|
+
"""
|
|
122
|
+
if self.model is not None:
|
|
123
|
+
model_name = self.model.name
|
|
124
|
+
if self.package is not None:
|
|
125
|
+
package_name = self.package.name
|
|
126
|
+
|
|
127
|
+
filters = entities.Filters()
|
|
128
|
+
remote_path = self._build_path_header(
|
|
129
|
+
package_name=package_name,
|
|
130
|
+
execution_id=execution_id,
|
|
131
|
+
model_name=model_name
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
remote_path += '/*'
|
|
135
|
+
filters.add(field='filename', values=remote_path)
|
|
136
|
+
pages = self.items_repository.list(filters=filters)
|
|
137
|
+
items = [entities.Artifact.from_json(_json=item.to_json(),
|
|
138
|
+
client_api=self._client_api,
|
|
139
|
+
dataset=item._dataset,
|
|
140
|
+
project=self.project)
|
|
141
|
+
for page in pages for item in page]
|
|
142
|
+
return miscellaneous.List(items)
|
|
143
|
+
|
|
144
|
+
def get(self,
|
|
145
|
+
artifact_id: str = None,
|
|
146
|
+
artifact_name: str = None,
|
|
147
|
+
model_name: str = None,
|
|
148
|
+
execution_id: str = None,
|
|
149
|
+
package_name: str = None) -> entities.ItemArtifact:
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
Get an artifact object by name, id or type
|
|
153
|
+
If by name or type - need to input also execution/task id for the artifact folder
|
|
154
|
+
|
|
155
|
+
:param str artifact_id: search by artifact id
|
|
156
|
+
:param str artifact_name: search by artifact id
|
|
157
|
+
:param str model_name: search by model name
|
|
158
|
+
:param str execution_id: search by execution id
|
|
159
|
+
:param str package_name: search by package name
|
|
160
|
+
:return: Artifact object
|
|
161
|
+
:rtype: dtlpy.entities.artifact.Artifact
|
|
162
|
+
|
|
163
|
+
**Example**:
|
|
164
|
+
|
|
165
|
+
.. code-block:: python
|
|
166
|
+
|
|
167
|
+
project.artifacts.get(artifact_id='artifact_id')
|
|
168
|
+
"""
|
|
169
|
+
if self.model is not None:
|
|
170
|
+
model_name = self.model.name
|
|
171
|
+
if self.package is not None:
|
|
172
|
+
package_name = self.package.name
|
|
173
|
+
|
|
174
|
+
if artifact_id is not None:
|
|
175
|
+
artifact = self.items_repository.get(item_id=artifact_id)
|
|
176
|
+
# verify input artifact name is same as the given id
|
|
177
|
+
if artifact_name is not None and artifact.name != artifact_name:
|
|
178
|
+
logger.warning(
|
|
179
|
+
"Mismatch found in artifacts.get: artifact_name is different then artifact.name:"
|
|
180
|
+
" {!r} != {!r}".format(
|
|
181
|
+
artifact_name,
|
|
182
|
+
artifact.name))
|
|
183
|
+
elif artifact_name is not None:
|
|
184
|
+
artifacts = self.list(
|
|
185
|
+
execution_id=execution_id,
|
|
186
|
+
package_name=package_name,
|
|
187
|
+
model_name=model_name
|
|
188
|
+
)
|
|
189
|
+
artifact = [artifact for artifact in artifacts if artifact.name == artifact_name]
|
|
190
|
+
if len(artifact) == 1:
|
|
191
|
+
artifact = artifact[0]
|
|
192
|
+
elif len(artifact) > 1:
|
|
193
|
+
raise PlatformException('404', 'More Than one Artifact found')
|
|
194
|
+
else:
|
|
195
|
+
raise PlatformException('404', 'Artifact not found')
|
|
196
|
+
else:
|
|
197
|
+
msg = 'one input must be not None: artifact_id or artifact_name'
|
|
198
|
+
raise ValueError(msg)
|
|
199
|
+
return artifact
|
|
200
|
+
|
|
201
|
+
def download(
|
|
202
|
+
self,
|
|
203
|
+
artifact_id: str = None,
|
|
204
|
+
artifact_name: str = None,
|
|
205
|
+
execution_id: str = None,
|
|
206
|
+
package_name: str = None,
|
|
207
|
+
model_name: str = None,
|
|
208
|
+
local_path: str = None,
|
|
209
|
+
overwrite: bool = False,
|
|
210
|
+
save_locally: bool = True
|
|
211
|
+
):
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
Download artifact binary.
|
|
215
|
+
Get artifact by name, id or type
|
|
216
|
+
|
|
217
|
+
:param str artifact_id: search by artifact id
|
|
218
|
+
:param str artifact_name: search by artifact id
|
|
219
|
+
:param str execution_id: search by execution id
|
|
220
|
+
:param str package_name: search by package name
|
|
221
|
+
:param str model_name: search by model name
|
|
222
|
+
:param str local_path: artifact will be saved to this filepath
|
|
223
|
+
:param bool overwrite: optional - default = False
|
|
224
|
+
:param bool save_locally: to save the file local
|
|
225
|
+
:return: file path
|
|
226
|
+
:rtype: str
|
|
227
|
+
|
|
228
|
+
**Example**:
|
|
229
|
+
|
|
230
|
+
.. code-block:: python
|
|
231
|
+
|
|
232
|
+
project.artifacts.download(artifact_id='artifact_id',
|
|
233
|
+
local_path='your_path',
|
|
234
|
+
overwrite=True,
|
|
235
|
+
save_locally=True)
|
|
236
|
+
"""
|
|
237
|
+
if self.model is not None:
|
|
238
|
+
model_name = self.model.name
|
|
239
|
+
if self.package is not None:
|
|
240
|
+
package_name = self.package.name
|
|
241
|
+
|
|
242
|
+
if artifact_id is not None:
|
|
243
|
+
# download
|
|
244
|
+
artifact = self.items_repository.download(items=artifact_id,
|
|
245
|
+
save_locally=save_locally,
|
|
246
|
+
local_path=local_path,
|
|
247
|
+
overwrite=overwrite)
|
|
248
|
+
elif artifact_name is not None:
|
|
249
|
+
artifact_obj: entities.ItemArtifact = self.get(artifact_id=artifact_id,
|
|
250
|
+
execution_id=execution_id,
|
|
251
|
+
package_name=package_name,
|
|
252
|
+
artifact_name=artifact_name)
|
|
253
|
+
|
|
254
|
+
artifact = artifact_obj.download(save_locally=save_locally,
|
|
255
|
+
local_path=local_path,
|
|
256
|
+
overwrite=overwrite)
|
|
257
|
+
|
|
258
|
+
else:
|
|
259
|
+
if self.model is not None:
|
|
260
|
+
artifact = list()
|
|
261
|
+
for m_artifact in self.model.model_artifacts:
|
|
262
|
+
if isinstance(m_artifact, entities.ItemArtifact):
|
|
263
|
+
if not m_artifact.is_fetched:
|
|
264
|
+
m_artifact = self.items_repository.get(item_id=m_artifact.id)
|
|
265
|
+
model_remote_root = m_artifact.filename.split('/')
|
|
266
|
+
model_remote_root = '/'.join(model_remote_root[:4])
|
|
267
|
+
# remove the prefix with relpath
|
|
268
|
+
local_dst = os.path.join(local_path,
|
|
269
|
+
os.path.relpath(m_artifact.filename, model_remote_root))
|
|
270
|
+
if not os.path.isfile(local_dst) or overwrite:
|
|
271
|
+
# need_to_download
|
|
272
|
+
# 1. download to temp folder
|
|
273
|
+
temp_dir = tempfile.mkdtemp()
|
|
274
|
+
local_temp_file = m_artifact.download(
|
|
275
|
+
local_path=temp_dir,
|
|
276
|
+
overwrite=overwrite,
|
|
277
|
+
to_items_folder=False,
|
|
278
|
+
)
|
|
279
|
+
src = local_temp_file
|
|
280
|
+
# remove the prefix with relpath
|
|
281
|
+
dst = local_dst
|
|
282
|
+
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
283
|
+
shutil.move(src=src, dst=dst)
|
|
284
|
+
# clean temp dir
|
|
285
|
+
if os.path.isdir(temp_dir):
|
|
286
|
+
shutil.rmtree(temp_dir)
|
|
287
|
+
artifact.append(local_path)
|
|
288
|
+
elif isinstance(m_artifact, entities.LinkArtifact):
|
|
289
|
+
# remove the prefix with relpath
|
|
290
|
+
local_dst = os.path.join(local_path, m_artifact.filename)
|
|
291
|
+
if not os.path.isfile(local_dst) or overwrite:
|
|
292
|
+
# need_to_download
|
|
293
|
+
# 1. download to temp folder
|
|
294
|
+
temp_dir = tempfile.mkdtemp()
|
|
295
|
+
response = requests.get(m_artifact.url, stream=True)
|
|
296
|
+
local_temp_file = os.path.join(temp_dir, m_artifact.filename)
|
|
297
|
+
with open(local_temp_file, "wb") as handle:
|
|
298
|
+
for data in response.iter_content(chunk_size=8192):
|
|
299
|
+
handle.write(data)
|
|
300
|
+
src = local_temp_file
|
|
301
|
+
# remove the prefix with relpath
|
|
302
|
+
dst = local_dst
|
|
303
|
+
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
304
|
+
shutil.move(src=src, dst=dst)
|
|
305
|
+
# clean temp dir
|
|
306
|
+
if os.path.isdir(temp_dir):
|
|
307
|
+
shutil.rmtree(temp_dir)
|
|
308
|
+
artifact.append(local_path)
|
|
309
|
+
elif isinstance(m_artifact, entities.LocalArtifact):
|
|
310
|
+
pass
|
|
311
|
+
else:
|
|
312
|
+
raise ValueError('unsupported artifact type: {}'.format(type(m_artifact)))
|
|
313
|
+
else:
|
|
314
|
+
# for package artifacts - download using filter on the package directory
|
|
315
|
+
if all(elem is None for elem in [package_name, execution_id]):
|
|
316
|
+
raise PlatformException(error='400', message='Must input package or execution (id or entity)')
|
|
317
|
+
remote_path = self._build_path_header(
|
|
318
|
+
package_name=package_name,
|
|
319
|
+
execution_id=execution_id,
|
|
320
|
+
model_name=model_name,
|
|
321
|
+
)
|
|
322
|
+
without_relative_path = remote_path
|
|
323
|
+
remote_path += '/*'
|
|
324
|
+
filters = entities.Filters()
|
|
325
|
+
filters.add(field='filename', values=remote_path)
|
|
326
|
+
artifact = self.items_repository.download(filters=filters,
|
|
327
|
+
save_locally=save_locally,
|
|
328
|
+
local_path=local_path,
|
|
329
|
+
to_items_folder=False,
|
|
330
|
+
overwrite=overwrite,
|
|
331
|
+
without_relative_path=without_relative_path)
|
|
332
|
+
return artifact
|
|
333
|
+
|
|
334
|
+
def upload(self,
|
|
335
|
+
# what to upload
|
|
336
|
+
filepath: str,
|
|
337
|
+
# where to upload
|
|
338
|
+
package_name: str = None,
|
|
339
|
+
package: entities.Package = None,
|
|
340
|
+
execution_id: str = None,
|
|
341
|
+
execution: entities.Execution = None,
|
|
342
|
+
model_name: str = None,
|
|
343
|
+
# add information
|
|
344
|
+
overwrite: bool = False):
|
|
345
|
+
"""
|
|
346
|
+
|
|
347
|
+
Upload binary file to artifact. get by name, id or type.
|
|
348
|
+
If artifact exists - overwriting binary
|
|
349
|
+
Else and if create==True a new artifact will be created and uploaded
|
|
350
|
+
|
|
351
|
+
:param str filepath: local binary file
|
|
352
|
+
:param str package_name: package name that include the artifact
|
|
353
|
+
:param dtlpy.entities.package.Package package: package object
|
|
354
|
+
:param str execution_id: execution id that include the artifact
|
|
355
|
+
:param dtlpy.entities.execution.Execution execution: execution object
|
|
356
|
+
:param str model_name: model name that include the artifact
|
|
357
|
+
:param bool overwrite: optional - default = False to overwrite an existing object
|
|
358
|
+
:return: Artifact Object
|
|
359
|
+
:rtype: dtlpy.entities.artifact.Artifact
|
|
360
|
+
|
|
361
|
+
**Example**:
|
|
362
|
+
|
|
363
|
+
.. code-block:: python
|
|
364
|
+
|
|
365
|
+
project.artifacts.upload(filepath='filepath',
|
|
366
|
+
package_name='package_name')
|
|
367
|
+
"""
|
|
368
|
+
if self.model is not None:
|
|
369
|
+
model_name = self.model.name
|
|
370
|
+
if self.package is not None:
|
|
371
|
+
package_name = self.package.name
|
|
372
|
+
|
|
373
|
+
remote_path = self._build_path_header(package_name=package_name,
|
|
374
|
+
package=package,
|
|
375
|
+
execution=execution,
|
|
376
|
+
execution_id=execution_id,
|
|
377
|
+
model_name=model_name)
|
|
378
|
+
|
|
379
|
+
if all(elem is None for elem in [package_name, package, execution_id, execution, model_name]):
|
|
380
|
+
raise ValueError('Must input package or execution (id or entity)')
|
|
381
|
+
|
|
382
|
+
artifact = self.items_repository.upload(local_path=filepath,
|
|
383
|
+
remote_path=remote_path,
|
|
384
|
+
overwrite=overwrite,
|
|
385
|
+
output_entity=entities.Artifact)
|
|
386
|
+
if self.model is not None:
|
|
387
|
+
# list and update model
|
|
388
|
+
filters = entities.Filters()
|
|
389
|
+
filters.add(field='dir', values=remote_path + '*')
|
|
390
|
+
pages = self.items_repository.list(filters=filters)
|
|
391
|
+
model_artifacts = list()
|
|
392
|
+
for item in pages.all():
|
|
393
|
+
model_artifacts.append(entities.Artifact.from_json(_json=item.to_json(),
|
|
394
|
+
client_api=self._client_api,
|
|
395
|
+
dataset=item._dataset))
|
|
396
|
+
self.model.model_artifacts = model_artifacts
|
|
397
|
+
self.model.update()
|
|
398
|
+
logger.debug('Artifact uploaded successfully')
|
|
399
|
+
return artifact
|
|
400
|
+
|
|
401
|
+
def delete(self,
|
|
402
|
+
artifact_id=None,
|
|
403
|
+
artifact_name=None,
|
|
404
|
+
execution_id=None,
|
|
405
|
+
model_name=None,
|
|
406
|
+
package_name=None):
|
|
407
|
+
"""
|
|
408
|
+
Delete artifacts
|
|
409
|
+
|
|
410
|
+
:param str artifact_id: search by artifact id
|
|
411
|
+
:param str artifact_name: search by artifact id
|
|
412
|
+
:param str execution_id: search by execution id
|
|
413
|
+
:param str model_name: search by model name
|
|
414
|
+
:param str package_name: search by package name
|
|
415
|
+
:return: True if success
|
|
416
|
+
:rtype: bool
|
|
417
|
+
|
|
418
|
+
**Example**:
|
|
419
|
+
|
|
420
|
+
.. code-block:: python
|
|
421
|
+
|
|
422
|
+
project.artifacts.delete(artifact_id='artifact_id',
|
|
423
|
+
package_name='package_name')
|
|
424
|
+
"""
|
|
425
|
+
if self.model is not None:
|
|
426
|
+
model_name = self.model.name
|
|
427
|
+
if self.package is not None:
|
|
428
|
+
package_name = self.package.name
|
|
429
|
+
|
|
430
|
+
if artifact_id is not None or artifact_name is not None:
|
|
431
|
+
artifacts = [
|
|
432
|
+
self.get(
|
|
433
|
+
artifact_id=artifact_id,
|
|
434
|
+
artifact_name=artifact_name,
|
|
435
|
+
model_name=model_name,
|
|
436
|
+
)
|
|
437
|
+
]
|
|
438
|
+
elif execution_id is not None or package_name is not None:
|
|
439
|
+
artifacts = self.list(
|
|
440
|
+
execution_id=execution_id,
|
|
441
|
+
package_name=package_name,
|
|
442
|
+
model_name=model_name,
|
|
443
|
+
)
|
|
444
|
+
else:
|
|
445
|
+
raise PlatformException('400',
|
|
446
|
+
'Must provide one of: artifact_id, artifact_name, execution_id, package_name')
|
|
447
|
+
|
|
448
|
+
values = [artifact.id for artifact in artifacts]
|
|
449
|
+
self.items_repository.delete(filters=entities.Filters(field='id', values=values,
|
|
450
|
+
operator=entities.FiltersOperations.IN))
|
|
451
|
+
|
|
452
|
+
return True
|