dtlpy 1.102.14__py3-none-any.whl → 1.103.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtlpy/__version__.py +1 -1
- dtlpy/entities/annotation.py +6 -3
- dtlpy/entities/dataset.py +1 -3
- dtlpy/entities/dpk.py +9 -9
- dtlpy/entities/filters.py +2 -2
- dtlpy/entities/integration.py +0 -1
- dtlpy/entities/organization.py +5 -5
- dtlpy/ml/base_model_adapter.py +8 -8
- dtlpy/repositories/features.py +9 -5
- dtlpy/repositories/integrations.py +52 -1
- dtlpy/repositories/items.py +9 -2
- dtlpy/repositories/pipelines.py +3 -9
- dtlpy/repositories/uploader.py +16 -4
- {dtlpy-1.102.14.dist-info → dtlpy-1.103.12.dist-info}/METADATA +2 -2
- {dtlpy-1.102.14.dist-info → dtlpy-1.103.12.dist-info}/RECORD +22 -22
- {dtlpy-1.102.14.data → dtlpy-1.103.12.data}/scripts/dlp +0 -0
- {dtlpy-1.102.14.data → dtlpy-1.103.12.data}/scripts/dlp.bat +0 -0
- {dtlpy-1.102.14.data → dtlpy-1.103.12.data}/scripts/dlp.py +0 -0
- {dtlpy-1.102.14.dist-info → dtlpy-1.103.12.dist-info}/LICENSE +0 -0
- {dtlpy-1.102.14.dist-info → dtlpy-1.103.12.dist-info}/WHEEL +0 -0
- {dtlpy-1.102.14.dist-info → dtlpy-1.103.12.dist-info}/entry_points.txt +0 -0
- {dtlpy-1.102.14.dist-info → dtlpy-1.103.12.dist-info}/top_level.txt +0 -0
dtlpy/__version__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
version = '1.
|
|
1
|
+
version = '1.103.12'
|
dtlpy/entities/annotation.py
CHANGED
|
@@ -421,7 +421,7 @@ class Annotation(entities.BaseEntity):
|
|
|
421
421
|
|
|
422
422
|
@property
|
|
423
423
|
def attributes(self):
|
|
424
|
-
if self._recipe_2_attributes or
|
|
424
|
+
if self._recipe_2_attributes or self.annotation_definition.attributes == []:
|
|
425
425
|
return self._recipe_2_attributes
|
|
426
426
|
return self.annotation_definition.attributes
|
|
427
427
|
|
|
@@ -1095,7 +1095,7 @@ class Annotation(entities.BaseEntity):
|
|
|
1095
1095
|
if annotation_definition:
|
|
1096
1096
|
res.annotation_definition = annotation_definition
|
|
1097
1097
|
|
|
1098
|
-
if annotation_definition and annotation_definition.attributes:
|
|
1098
|
+
if annotation_definition and annotation_definition.attributes is not None:
|
|
1099
1099
|
res.attributes = annotation_definition.attributes
|
|
1100
1100
|
|
|
1101
1101
|
return res
|
|
@@ -1679,7 +1679,7 @@ class FrameAnnotation(entities.BaseEntity):
|
|
|
1679
1679
|
|
|
1680
1680
|
@property
|
|
1681
1681
|
def attributes(self):
|
|
1682
|
-
if self._recipe_2_attributes or
|
|
1682
|
+
if self._recipe_2_attributes or self.annotation_definition.attributes == []:
|
|
1683
1683
|
return self._recipe_2_attributes
|
|
1684
1684
|
return self.annotation_definition.attributes
|
|
1685
1685
|
|
|
@@ -1873,6 +1873,9 @@ class FrameAnnotation(entities.BaseEntity):
|
|
|
1873
1873
|
'data': self.coordinates
|
|
1874
1874
|
}
|
|
1875
1875
|
|
|
1876
|
+
if self.annotation_definition.description is not None:
|
|
1877
|
+
snapshot_dict['description'] = self.annotation_definition.description
|
|
1878
|
+
|
|
1876
1879
|
if self.annotation._recipe_2_attributes:
|
|
1877
1880
|
snapshot_dict['namedAttributes'] = self._recipe_2_attributes
|
|
1878
1881
|
else:
|
dtlpy/entities/dataset.py
CHANGED
|
@@ -287,10 +287,8 @@ class Dataset(entities.BaseEntity):
|
|
|
287
287
|
'ontologies', 'features', 'settings', 'schema'])
|
|
288
288
|
if self._project is None:
|
|
289
289
|
datasets = repositories.Datasets(client_api=self._client_api, project=self._project)
|
|
290
|
-
features = repositories.Features(client_api=self._client_api, project=self._project)
|
|
291
290
|
else:
|
|
292
291
|
datasets = self._project.datasets
|
|
293
|
-
features = self._project.features
|
|
294
292
|
|
|
295
293
|
return reps(
|
|
296
294
|
items=repositories.Items(client_api=self._client_api, dataset=self, datasets=datasets),
|
|
@@ -300,7 +298,7 @@ class Dataset(entities.BaseEntity):
|
|
|
300
298
|
annotations=repositories.Annotations(client_api=self._client_api, dataset=self),
|
|
301
299
|
datasets=datasets,
|
|
302
300
|
ontologies=repositories.Ontologies(client_api=self._client_api, dataset=self),
|
|
303
|
-
features=
|
|
301
|
+
features=repositories.Features(client_api=self._client_api, project=self._project, dataset=self),
|
|
304
302
|
settings=repositories.Settings(client_api=self._client_api, dataset=self),
|
|
305
303
|
schema=repositories.Schema(client_api=self._client_api, dataset=self)
|
|
306
304
|
)
|
dtlpy/entities/dpk.py
CHANGED
|
@@ -31,8 +31,8 @@ DEFAULT_STOPS = {SlotType.ITEM_VIEWER: {"type": "itemViewer",
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class Slot(entities.DlEntity):
|
|
34
|
-
type = entities.DlProperty(location=['type'], _type=str)
|
|
35
|
-
configuration = entities.DlProperty(location=['configuration'], _type=dict)
|
|
34
|
+
type: str = entities.DlProperty(location=['type'], _type=str)
|
|
35
|
+
configuration: dict = entities.DlProperty(location=['configuration'], _type=dict)
|
|
36
36
|
|
|
37
37
|
def to_json(self) -> dict:
|
|
38
38
|
return self._dict.copy()
|
|
@@ -59,10 +59,10 @@ class Toolbar(entities.DlEntity):
|
|
|
59
59
|
|
|
60
60
|
|
|
61
61
|
class Panel(entities.DlEntity):
|
|
62
|
-
name = entities.DlProperty(location=['name'], _type=str)
|
|
63
|
-
path = entities.DlProperty(location=['path'], _type=str, default=None)
|
|
64
|
-
min_role = entities.DlProperty(location=['minRole'], _type=list)
|
|
65
|
-
supported_slots = entities.DlProperty(location=['supportedSlots'], _type=list)
|
|
62
|
+
name: str = entities.DlProperty(location=['name'], _type=str)
|
|
63
|
+
path: str = entities.DlProperty(location=['path'], _type=str, default=None)
|
|
64
|
+
min_role: list = entities.DlProperty(location=['minRole'], _type=list)
|
|
65
|
+
supported_slots: list = entities.DlProperty(location=['supportedSlots'], _type=list)
|
|
66
66
|
|
|
67
67
|
metadata = entities.DlProperty(location=['metadata'], _type=list)
|
|
68
68
|
default_settings = entities.DlProperty(location=['defaultSettings'], _type=list)
|
|
@@ -233,15 +233,15 @@ class Dpk(entities.DlEntity):
|
|
|
233
233
|
base_id: str = entities.DlProperty(location=['baseId'], _type=str)
|
|
234
234
|
name: str = entities.DlProperty(location=['name'], _type=str)
|
|
235
235
|
version: str = entities.DlProperty(location=['version'], _type=str)
|
|
236
|
-
attributes:
|
|
236
|
+
attributes: dict = entities.DlProperty(location=['attributes'], _type=dict)
|
|
237
237
|
created_at: str = entities.DlProperty(location=['createdAt'], _type=str)
|
|
238
238
|
updated_at: str = entities.DlProperty(location=['updatedAt'], _type=str)
|
|
239
239
|
creator: str = entities.DlProperty(location=['creator'], _type=str)
|
|
240
240
|
display_name: str = entities.DlProperty(location=['displayName'], _type=str)
|
|
241
241
|
icon: str = entities.DlProperty(location=['icon'], _type=str)
|
|
242
242
|
tags: list = entities.DlProperty(location=['tags'], _type=list)
|
|
243
|
-
codebase:
|
|
244
|
-
scope:
|
|
243
|
+
codebase: Union[entities.Codebase, None] = entities.DlProperty(location=['codebase'], _kls="Codebase")
|
|
244
|
+
scope: str = entities.DlProperty(location=['scope'], _type=str)
|
|
245
245
|
context: dict = entities.DlProperty(location=['context'], _type=dict)
|
|
246
246
|
metadata: dict = entities.DlProperty(location=['metadata'], _type=dict)
|
|
247
247
|
dependencies: dict = entities.DlProperty(location=['dependencies'], _type=List[dict])
|
dtlpy/entities/filters.py
CHANGED
|
@@ -497,8 +497,8 @@ class Filters:
|
|
|
497
497
|
_json["resource"] = f'{_json["resource"]}'
|
|
498
498
|
# convert the dictionary to a json string
|
|
499
499
|
_json['dqlFilter'] = json.dumps({'filter': _json.pop('filter'),
|
|
500
|
-
'join': _json.pop('join'),
|
|
501
|
-
'sort': _json.get('sort')})
|
|
500
|
+
'join': _json.pop('join', None),
|
|
501
|
+
'sort': _json.get('sort', None)})
|
|
502
502
|
# set the page size as the UI default
|
|
503
503
|
_json['pageSize'] = 100
|
|
504
504
|
_json['page'] = _json['page']
|
dtlpy/entities/integration.py
CHANGED
dtlpy/entities/organization.py
CHANGED
|
@@ -49,7 +49,7 @@ class Organization(entities.BaseEntity):
|
|
|
49
49
|
logo_url = attr.ib(repr=False)
|
|
50
50
|
plan = attr.ib(repr=False)
|
|
51
51
|
owner = attr.ib(repr=False)
|
|
52
|
-
|
|
52
|
+
creator = attr.ib(repr=False)
|
|
53
53
|
|
|
54
54
|
# api
|
|
55
55
|
_client_api = attr.ib(type=ApiClient, repr=False)
|
|
@@ -67,7 +67,7 @@ class Organization(entities.BaseEntity):
|
|
|
67
67
|
|
|
68
68
|
@property
|
|
69
69
|
def createdBy(self):
|
|
70
|
-
return self.
|
|
70
|
+
return self.creator
|
|
71
71
|
|
|
72
72
|
@_repositories.default
|
|
73
73
|
def set_repositories(self):
|
|
@@ -158,7 +158,7 @@ class Organization(entities.BaseEntity):
|
|
|
158
158
|
logo_url=_json.get('logoUrl', None),
|
|
159
159
|
plan=_json.get('plan', None),
|
|
160
160
|
owner=_json.get('owner', None),
|
|
161
|
-
|
|
161
|
+
creator=_json.get('creator', None),
|
|
162
162
|
client_api=client_api)
|
|
163
163
|
inst.is_fetched = is_fetched
|
|
164
164
|
return inst
|
|
@@ -175,7 +175,7 @@ class Organization(entities.BaseEntity):
|
|
|
175
175
|
attr.fields(Organization)._repositories,
|
|
176
176
|
attr.fields(Organization).created_at,
|
|
177
177
|
attr.fields(Organization).updated_at,
|
|
178
|
-
attr.fields(Organization).
|
|
178
|
+
attr.fields(Organization).creator,
|
|
179
179
|
))
|
|
180
180
|
output_dict['members'] = self.members
|
|
181
181
|
output_dict['groups'] = self.groups
|
|
@@ -188,7 +188,7 @@ class Organization(entities.BaseEntity):
|
|
|
188
188
|
output_dict['logo_url'] = self.logo_url
|
|
189
189
|
output_dict['plan'] = self.plan
|
|
190
190
|
output_dict['owner'] = self.owner
|
|
191
|
-
output_dict['
|
|
191
|
+
output_dict['creator'] = self.creator
|
|
192
192
|
|
|
193
193
|
return output_dict
|
|
194
194
|
|
dtlpy/ml/base_model_adapter.py
CHANGED
|
@@ -488,7 +488,8 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
|
|
|
488
488
|
try:
|
|
489
489
|
self.model_entity.project.feature_sets.get(feature_set_name=self.model_entity.name)
|
|
490
490
|
feature_set_name = f"{self.model_entity.name}-{''.join(random.choices(string.ascii_letters + string.digits, k=5))}"
|
|
491
|
-
logger.warning(
|
|
491
|
+
logger.warning(
|
|
492
|
+
f"Feature set with the model name already exists. Creating new feature set with name {feature_set_name}")
|
|
492
493
|
except exceptions.NotFound:
|
|
493
494
|
feature_set_name = self.model_entity.name
|
|
494
495
|
feature_set = self.model_entity.project.feature_sets.create(name=feature_set_name,
|
|
@@ -518,7 +519,7 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
|
|
|
518
519
|
self.logger.debug(
|
|
519
520
|
"Uploading items' feature vectors for model {!r}.".format(self.model_entity.name))
|
|
520
521
|
try:
|
|
521
|
-
|
|
522
|
+
list(pool.map(partial(self._upload_model_features,
|
|
522
523
|
feature_set.id,
|
|
523
524
|
self.model_entity.project_id),
|
|
524
525
|
batch_items,
|
|
@@ -762,14 +763,13 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
|
|
|
762
763
|
@staticmethod
|
|
763
764
|
def _upload_model_features(feature_set_id, project_id, item: entities.Item, vector):
|
|
764
765
|
try:
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
766
|
+
if vector is not None:
|
|
767
|
+
item.features.create(value=vector,
|
|
768
|
+
project_id=project_id,
|
|
769
|
+
feature_set_id=feature_set_id,
|
|
770
|
+
entity=item)
|
|
770
771
|
except Exception as e:
|
|
771
772
|
logger.error(f'Failed to upload feature vector of length {len(vector)} to item {item.id}, Error: {e}')
|
|
772
|
-
return []
|
|
773
773
|
|
|
774
774
|
def _upload_model_annotations(self, item: entities.Item, predictions, clean_annotations):
|
|
775
775
|
"""
|
dtlpy/repositories/features.py
CHANGED
|
@@ -17,9 +17,11 @@ class Features:
|
|
|
17
17
|
project_id: str = None,
|
|
18
18
|
item: entities.Item = None,
|
|
19
19
|
annotation: entities.Annotation = None,
|
|
20
|
-
feature_set: entities.FeatureSet = None
|
|
20
|
+
feature_set: entities.FeatureSet = None,
|
|
21
|
+
dataset: entities.Dataset = None):
|
|
21
22
|
if project is not None and project_id is None:
|
|
22
23
|
project_id = project.id
|
|
24
|
+
self._dataset = dataset
|
|
23
25
|
self._project = project
|
|
24
26
|
self._project_id = project_id
|
|
25
27
|
self._item = item
|
|
@@ -50,7 +52,7 @@ class Features:
|
|
|
50
52
|
if self._project is None:
|
|
51
53
|
raise exceptions.PlatformException(
|
|
52
54
|
error='2001',
|
|
53
|
-
message='Cannot perform action WITHOUT Project entity in
|
|
55
|
+
message='Cannot perform action WITHOUT Project entity in Features repository.'
|
|
54
56
|
' Please checkout or set a project')
|
|
55
57
|
assert isinstance(self._project, entities.Project)
|
|
56
58
|
return self._project
|
|
@@ -60,8 +62,8 @@ class Features:
|
|
|
60
62
|
###########
|
|
61
63
|
def _list(self, filters: entities.Filters):
|
|
62
64
|
"""
|
|
63
|
-
Get dataset
|
|
64
|
-
user is expected to perform another request then for every folder item to actually get the
|
|
65
|
+
Get dataset feature vectors list. This is a browsing endpoint, for any given path feature count will be returned,
|
|
66
|
+
user is expected to perform another request then for every folder item to actually get the item list.
|
|
65
67
|
|
|
66
68
|
:param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
|
|
67
69
|
:return: json response
|
|
@@ -104,6 +106,8 @@ class Features:
|
|
|
104
106
|
filters.add(field='featureSetId', values=self._feature_set.id)
|
|
105
107
|
if self._item is not None:
|
|
106
108
|
filters.add(field='entityId', values=self._item.id)
|
|
109
|
+
if self._dataset is not None:
|
|
110
|
+
filters.add(field='datasetId', values=self._dataset.id)
|
|
107
111
|
if self._project_id is None:
|
|
108
112
|
self._project_id = self.project.id
|
|
109
113
|
filters.context = {"projects": [self._project_id]}
|
|
@@ -169,7 +173,7 @@ class Features:
|
|
|
169
173
|
if feature_set_id is None:
|
|
170
174
|
if self._feature_set is None:
|
|
171
175
|
raise ValueError(
|
|
172
|
-
'Missing feature_set_id. Must insert the variable or create from context
|
|
176
|
+
'Missing feature_set_id. Must insert the variable or create from context, e.g. feature_set.features.create()')
|
|
173
177
|
feature_set_id = self._feature_set.id
|
|
174
178
|
|
|
175
179
|
payload = {'project': project_id,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Integrations Repository
|
|
3
3
|
"""
|
|
4
|
-
|
|
4
|
+
import base64
|
|
5
|
+
import json
|
|
5
6
|
import logging
|
|
6
7
|
from .. import entities, exceptions, miscellaneous, _api_reference
|
|
7
8
|
from ..services.api_client import ApiClient
|
|
@@ -298,3 +299,53 @@ class Integrations:
|
|
|
298
299
|
|
|
299
300
|
available_integrations = miscellaneous.List(response.json())
|
|
300
301
|
return available_integrations
|
|
302
|
+
|
|
303
|
+
def _create_private_registry_gar(self, service_account: str, location: str):
|
|
304
|
+
password = self.__create_gar_password(service_account, location)
|
|
305
|
+
return self.create(
|
|
306
|
+
integrations_type='private-registry',
|
|
307
|
+
name='gar-1',
|
|
308
|
+
metadata={"provider": "gcp"},
|
|
309
|
+
options={
|
|
310
|
+
"name": "_json_key",
|
|
311
|
+
"spec": {
|
|
312
|
+
"password": password
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
def __create_gar_password(self, service_account: str, location: str) -> str:
|
|
318
|
+
"""
|
|
319
|
+
Generates a Google Artifact Registry JSON configuration and returns it as a base64-encoded string.
|
|
320
|
+
|
|
321
|
+
Parameters:
|
|
322
|
+
location (str): The region where the repository will be created (e.g., 'us-central1').
|
|
323
|
+
service_account (str): The service_account parameter represents the Google Cloud service account credentials
|
|
324
|
+
in the form of a JSON key file. This JSON contains the private key and other metadata
|
|
325
|
+
required for authenticating with Google Artifact Registry. It is used to generate a Kubernetes secret
|
|
326
|
+
that stores the credentials for pulling container images from the registry.
|
|
327
|
+
The JSON key must include fields such as client_email, private_key, and project_id,
|
|
328
|
+
and it is typically downloaded from the Google Cloud Console when creating the service account
|
|
329
|
+
|
|
330
|
+
Returns:
|
|
331
|
+
str: A base64-encoded string representation of the repository JSON configuration.
|
|
332
|
+
"""
|
|
333
|
+
if not service_account:
|
|
334
|
+
raise ValueError('Missing Service Account')
|
|
335
|
+
if not location:
|
|
336
|
+
raise ValueError('Missing Location')
|
|
337
|
+
user_name = "_json_key"
|
|
338
|
+
cred = f"{user_name}:{service_account}"
|
|
339
|
+
auth = str(base64.b64encode(bytes(cred, 'utf-8')))[2:-1]
|
|
340
|
+
|
|
341
|
+
encoded_pass = {
|
|
342
|
+
"auths": {
|
|
343
|
+
f"{location}": {
|
|
344
|
+
"username": user_name,
|
|
345
|
+
"password": service_account,
|
|
346
|
+
"auth": auth
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
return str(base64.b64encode(bytes(json.dumps(encoded_pass), 'utf-8')))[2:-1]
|
dtlpy/repositories/items.py
CHANGED
|
@@ -610,7 +610,9 @@ class Items:
|
|
|
610
610
|
output_entity=entities.Item,
|
|
611
611
|
no_output: bool = False,
|
|
612
612
|
export_version: str = entities.ExportVersion.V1,
|
|
613
|
-
item_description: str = None
|
|
613
|
+
item_description: str = None,
|
|
614
|
+
raise_on_error: bool = False,
|
|
615
|
+
return_as_list: bool = False
|
|
614
616
|
):
|
|
615
617
|
"""
|
|
616
618
|
Upload local file to dataset.
|
|
@@ -630,6 +632,9 @@ class Items:
|
|
|
630
632
|
:param bool no_output: do not return the items after upload
|
|
631
633
|
:param str export_version: exported items will have original extension in filename, `V1` - no original extension in filenames
|
|
632
634
|
:param str item_description: add a string description to the uploaded item
|
|
635
|
+
:param bool raise_on_error: raise an exception if an error occurs
|
|
636
|
+
:param bool return_as_list: return a list of items instead of a generator
|
|
637
|
+
|
|
633
638
|
:return: Output (generator/single item)
|
|
634
639
|
:rtype: generator or single item
|
|
635
640
|
|
|
@@ -657,7 +662,9 @@ class Items:
|
|
|
657
662
|
# metadata to upload with items
|
|
658
663
|
item_metadata=item_metadata,
|
|
659
664
|
export_version=export_version,
|
|
660
|
-
item_description=item_description
|
|
665
|
+
item_description=item_description,
|
|
666
|
+
raise_on_error=raise_on_error,
|
|
667
|
+
return_as_list=return_as_list
|
|
661
668
|
)
|
|
662
669
|
|
|
663
670
|
@property
|
dtlpy/repositories/pipelines.py
CHANGED
|
@@ -376,24 +376,18 @@ class Pipelines:
|
|
|
376
376
|
|
|
377
377
|
pipeline = project.pipelines.update(pipeline='pipeline_entity')
|
|
378
378
|
"""
|
|
379
|
-
if pipeline.status == entities.CompositionStatus.INSTALLED and not pipeline.variables_changed():
|
|
380
|
-
raise exceptions.PlatformException(
|
|
381
|
-
error='400',
|
|
382
|
-
message='Cannot update pipeline while it is installed'
|
|
383
|
-
)
|
|
384
379
|
# payload
|
|
385
380
|
payload = pipeline.to_json()
|
|
386
381
|
|
|
387
382
|
# update settings
|
|
388
383
|
if pipeline.settings_changed():
|
|
389
|
-
self.update_settings(pipeline=pipeline, settings=pipeline.settings)
|
|
384
|
+
new_pipeline = self.update_settings(pipeline=pipeline, settings=pipeline.settings)
|
|
385
|
+
payload['settings'] = new_pipeline.to_json().get('settings', payload.get('settings'))
|
|
390
386
|
|
|
391
387
|
# update variables
|
|
392
388
|
if pipeline.variables_changed():
|
|
393
389
|
new_pipeline = self.__update_variables(pipeline=pipeline)
|
|
394
|
-
|
|
395
|
-
logger.warning('Pipeline is installed, updating pipeline variables only')
|
|
396
|
-
return new_pipeline
|
|
390
|
+
payload['variables'] = new_pipeline.to_json().get('variables', payload.get('variables'))
|
|
397
391
|
|
|
398
392
|
success, response = self._client_api.gen_request(
|
|
399
393
|
req_type='patch',
|
dtlpy/repositories/uploader.py
CHANGED
|
@@ -59,7 +59,9 @@ class Uploader:
|
|
|
59
59
|
overwrite=False,
|
|
60
60
|
item_metadata=None,
|
|
61
61
|
export_version: str = entities.ExportVersion.V1,
|
|
62
|
-
item_description=None
|
|
62
|
+
item_description=None,
|
|
63
|
+
raise_on_error=False,
|
|
64
|
+
return_as_list=False
|
|
63
65
|
):
|
|
64
66
|
"""
|
|
65
67
|
Upload local file to dataset.
|
|
@@ -67,7 +69,7 @@ class Uploader:
|
|
|
67
69
|
If `*` at the end of local_path (e.g. '/images/*') items will be uploaded without head directory
|
|
68
70
|
|
|
69
71
|
:param local_path: local file or folder to upload
|
|
70
|
-
:param local_annotations_path: path to
|
|
72
|
+
:param local_annotations_path: path to Dataloop format annotations json files.
|
|
71
73
|
:param remote_path: remote path to save.
|
|
72
74
|
:param remote_name: remote base name to save.
|
|
73
75
|
:param file_types: list of file type to upload. e.g ['.jpg', '.png']. default is all
|
|
@@ -75,6 +77,8 @@ class Uploader:
|
|
|
75
77
|
:param item_metadata: upload the items with the metadata dictionary
|
|
76
78
|
:param str export_version: exported items will have original extension in filename, `V1` - no original extension in filenames
|
|
77
79
|
:param str item_description: add a string description to the uploaded item
|
|
80
|
+
:param bool raise_on_error: raise an exception if an error occurs
|
|
81
|
+
:param bool return_as_list: always return a list of items
|
|
78
82
|
|
|
79
83
|
:return: Output (list)
|
|
80
84
|
"""
|
|
@@ -114,13 +118,21 @@ class Uploader:
|
|
|
114
118
|
if log_filepath is not None:
|
|
115
119
|
logger.warning("Errors in {n_error} files. See {log_filepath} for full log".format(
|
|
116
120
|
n_error=errors_count, log_filepath=log_filepath))
|
|
117
|
-
|
|
118
|
-
|
|
121
|
+
if raise_on_error is True:
|
|
122
|
+
raise PlatformException(error="400",
|
|
123
|
+
message=f"Errors in {errors_count} files. See above trace for more information")
|
|
124
|
+
|
|
125
|
+
if return_as_list is True:
|
|
126
|
+
# return list of items
|
|
127
|
+
return list(self.reporter.output)
|
|
119
128
|
if len(status_list) == 1:
|
|
129
|
+
# if there is only one item, return it
|
|
120
130
|
try:
|
|
121
131
|
return next(self.reporter.output)
|
|
122
132
|
except StopIteration:
|
|
133
|
+
# if there is no items, return None
|
|
123
134
|
return None
|
|
135
|
+
# if there are multiple items, return the generator
|
|
124
136
|
return self.reporter.output
|
|
125
137
|
|
|
126
138
|
def _build_elements_from_inputs(self,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dtlpy
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.103.12
|
|
4
4
|
Summary: SDK and CLI for Dataloop platform
|
|
5
5
|
Home-page: https://github.com/dataloop-ai/dtlpy
|
|
6
6
|
Author: Dataloop Team
|
|
@@ -26,7 +26,7 @@ Requires-Dist: requests-toolbelt (>=1.0.0)
|
|
|
26
26
|
Requires-Dist: requests (>=2.25.0)
|
|
27
27
|
Requires-Dist: numpy (>=1.16.2)
|
|
28
28
|
Requires-Dist: pandas (>=0.24.2)
|
|
29
|
-
Requires-Dist: tabulate (
|
|
29
|
+
Requires-Dist: tabulate (>=0.8.9)
|
|
30
30
|
Requires-Dist: Pillow (>=7.2)
|
|
31
31
|
Requires-Dist: PyJWT (>=2.4)
|
|
32
32
|
Requires-Dist: jinja2 (>=2.11.3)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
dtlpy/__init__.py,sha256=1Zngp5ftTgWb3r-sc8z98TYpEs6fOB_7snFhsXYQLVg,20899
|
|
2
|
-
dtlpy/__version__.py,sha256=
|
|
2
|
+
dtlpy/__version__.py,sha256=E-gqd4JZ0KTONMWd2TCpHjlUW8UxCInRGQnUkSrQeBY,21
|
|
3
3
|
dtlpy/exceptions.py,sha256=EQCKs3pwhwZhgMByQN3D3LpWpdxwcKPEEt-bIaDwURM,2871
|
|
4
4
|
dtlpy/new_instance.py,sha256=u_c6JtgqsKCr7TU24-g7_CaST9ghqamMhM4Z0Zxt50w,10121
|
|
5
5
|
dtlpy/assets/__init__.py,sha256=D_hAa6NM8Zoy32sF_9b7m0b7I-BQEyBFg8-9Tg2WOeo,976
|
|
@@ -46,7 +46,7 @@ dtlpy/dlp/dlp.py,sha256=YjNBjeCDTXJ7tj8qdiGZ8lFb8DtPZl-FvViyjxt9xF8,4278
|
|
|
46
46
|
dtlpy/dlp/parser.py,sha256=p-TFaiAU2c3QkI97TXzL2LDR3Eq0hGDFrTc9J2jWLh4,30551
|
|
47
47
|
dtlpy/entities/__init__.py,sha256=Sihb652vYCyCWSQfiYTEGQW0EJsHe7BTk6-S5DOsAb0,4908
|
|
48
48
|
dtlpy/entities/analytic.py,sha256=5MpYDKPVsZ1MIy20Ju515RWed6P667j4TLxsan2gyNM,11925
|
|
49
|
-
dtlpy/entities/annotation.py,sha256=
|
|
49
|
+
dtlpy/entities/annotation.py,sha256=QdxPwKyupQ_omqpxtL8qizUcZkFlLfcbuMhiBY3mYTk,67182
|
|
50
50
|
dtlpy/entities/annotation_collection.py,sha256=CEYSBHhhDkC0VJdHsBSrA6TgdKGMcKeI3tFM40UJwS8,29838
|
|
51
51
|
dtlpy/entities/app.py,sha256=dVd87-mP22NWvec5nqA5VjZ8Qk3aJlgUcloIAAOAPUw,6968
|
|
52
52
|
dtlpy/entities/app_module.py,sha256=0UiAbBX1q8iEImi3nY7ySWZZHoRRwu0qUXmyXmgVAc4,3645
|
|
@@ -57,16 +57,16 @@ dtlpy/entities/bot.py,sha256=is3NUCnPg56HSjsHIvFcVkymValMqDV0uHRDC1Ib-ds,3819
|
|
|
57
57
|
dtlpy/entities/codebase.py,sha256=pwRkAq2GV0wvmzshg89IAmE-0I2Wsy_-QNOu8OV8uqc,8999
|
|
58
58
|
dtlpy/entities/command.py,sha256=ARu8ttk-C7_Ice7chRyTtyOtakBTF09FC04mEk73SO8,5010
|
|
59
59
|
dtlpy/entities/compute.py,sha256=4FEpahPFFGHxye_fLh_p_kP6iEQ3QJK7S5hAdd6Afos,12744
|
|
60
|
-
dtlpy/entities/dataset.py,sha256=
|
|
60
|
+
dtlpy/entities/dataset.py,sha256=In9cOfc5COX9I532TDza9V5VmTOXMsA9TKgcfLPArpE,50392
|
|
61
61
|
dtlpy/entities/directory_tree.py,sha256=Rni6pLSWytR6yeUPgEdCCRfTg_cqLOdUc9uCqz9KT-Q,1186
|
|
62
|
-
dtlpy/entities/dpk.py,sha256=
|
|
62
|
+
dtlpy/entities/dpk.py,sha256=FJVhQKk2fj1cO_4rcE_bIF6QmIQZQWUkBnwTNQNMrfE,17857
|
|
63
63
|
dtlpy/entities/driver.py,sha256=O_QdK1EaLjQyQkmvKsmkNgmvmMb1mPjKnJGxK43KrOA,7197
|
|
64
64
|
dtlpy/entities/execution.py,sha256=WBiAws-6wZnQQ3y9wyvOeexA3OjxfaRdwDu5dSFYL1g,13420
|
|
65
65
|
dtlpy/entities/feature.py,sha256=9fFjD0W57anOVSAVU55ypxN_WTCsWTG03Wkc3cAAj78,3732
|
|
66
66
|
dtlpy/entities/feature_set.py,sha256=niw4MkmrDbD_LWQu1X30uE6U4DCzmFhPTaYeZ6VZDB0,4443
|
|
67
|
-
dtlpy/entities/filters.py,sha256=
|
|
67
|
+
dtlpy/entities/filters.py,sha256=PUmgor77m3CWeUgvCdWMg3Bt5SxHXPVBbN5VmD_dglQ,22683
|
|
68
68
|
dtlpy/entities/gis_item.py,sha256=Uk-wMBxwcHsImjz4qOjP-EyZAohbRzN43kMpCaVjCXU,3982
|
|
69
|
-
dtlpy/entities/integration.py,sha256=
|
|
69
|
+
dtlpy/entities/integration.py,sha256=Kdy1j6-cJLW8qNmnqCmdg36phi843YDrlMqcMyMfvYk,5875
|
|
70
70
|
dtlpy/entities/item.py,sha256=UnAZ7yLaTu2vkbD2sxysJQNn-ceC9zS3Uf304OvcC4E,31746
|
|
71
71
|
dtlpy/entities/label.py,sha256=ycDYavIgKhz806plIX-64c07_TeHpDa-V7LnfFVe4Rg,3869
|
|
72
72
|
dtlpy/entities/links.py,sha256=FAmEwHtsrqKet3c0UHH9u_gHgG6_OwF1-rl4xK7guME,2516
|
|
@@ -74,7 +74,7 @@ dtlpy/entities/message.py,sha256=ApJuaKEqxATpXjNYUjGdYPu3ibQzEMo8-LtJ_4xAcPI,586
|
|
|
74
74
|
dtlpy/entities/model.py,sha256=YwjIi3MxAZoyartTvqx_qhtDKQe6zVsQuwZbYLygMxU,26898
|
|
75
75
|
dtlpy/entities/node.py,sha256=yPPYDLtNMc6vZbbf4FIffY86y7tkaTvYm42Jb7k3Ofk,39617
|
|
76
76
|
dtlpy/entities/ontology.py,sha256=N5bsnNUfWNN49WV8lGQL8QRxbKfCZyV-rDyTlGByUrw,31987
|
|
77
|
-
dtlpy/entities/organization.py,sha256=
|
|
77
|
+
dtlpy/entities/organization.py,sha256=Zm-tTHV82PvYyTNetRRXqvmvzBCbXEwS-gAENf7Zny4,9874
|
|
78
78
|
dtlpy/entities/package.py,sha256=QSDePHlp4ik19aUE3dAUC7edh0oUUVjzSmMG867avc4,26363
|
|
79
79
|
dtlpy/entities/package_defaults.py,sha256=wTD7Z7rGYjVy8AcUxTFEnkOkviiJaLVZYvduiUBKNZo,211
|
|
80
80
|
dtlpy/entities/package_function.py,sha256=M42Kvw9A8b6msAkv-wRNAQg_-UC2bejniCjeKDugudc,6314
|
|
@@ -148,7 +148,7 @@ dtlpy/miscellaneous/list_print.py,sha256=leEg3RodgYfH5t_0JG8VuM8NiesR8sJLK_mRStt
|
|
|
148
148
|
dtlpy/miscellaneous/zipping.py,sha256=GMdPhAeHQXeMS5ClaiKWMJWVYQLBLAaJUWxvdYrL4Ro,5337
|
|
149
149
|
dtlpy/ml/__init__.py,sha256=vPkyXpc9kcWWZ_PxyPEOsjKBJdEbowLkZr8FZIb_OBM,799
|
|
150
150
|
dtlpy/ml/base_feature_extractor_adapter.py,sha256=iiEGYAx0Rdn4K46H_FlKrAv3ebTXHSxNVAmio0BxhaI,1178
|
|
151
|
-
dtlpy/ml/base_model_adapter.py,sha256=
|
|
151
|
+
dtlpy/ml/base_model_adapter.py,sha256=QG6-WOH24NBlAGDHBg9t2gzsnRXjujCBeirlTPUiiGo,50601
|
|
152
152
|
dtlpy/ml/metrics.py,sha256=BG2E-1Mvjv2e2No9mIJKVmvzqBvLqytKcw3hA7wVUNc,20037
|
|
153
153
|
dtlpy/ml/predictions_utils.py,sha256=He_84U14oS2Ss7T_-Zj5GDiBZwS-GjMPURUh7u7DjF8,12484
|
|
154
154
|
dtlpy/ml/summary_writer.py,sha256=dehDi8zmGC1sAGyy_3cpSWGXoGQSiQd7bL_Thoo8yIs,2784
|
|
@@ -170,9 +170,9 @@ dtlpy/repositories/dpks.py,sha256=dglvaiSFBvEithhlQ0RAXwzTxoZaICONs-owx3e2nfU,17
|
|
|
170
170
|
dtlpy/repositories/drivers.py,sha256=fF0UuHCyBzop8pHfryex23mf0kVFAkqzNdOmwBbaWxY,10204
|
|
171
171
|
dtlpy/repositories/executions.py,sha256=4UoU6bnB3kl5cMuF1eJvDecfZCaB06gKWxPfv6_g1_k,32598
|
|
172
172
|
dtlpy/repositories/feature_sets.py,sha256=UowMDAl_CRefRB5oZzubnsjU_OFgiPPdQXn8q2j4Kuw,9666
|
|
173
|
-
dtlpy/repositories/features.py,sha256=
|
|
174
|
-
dtlpy/repositories/integrations.py,sha256=
|
|
175
|
-
dtlpy/repositories/items.py,sha256=
|
|
173
|
+
dtlpy/repositories/features.py,sha256=A_RqTJxzjTh-Wbm0uXaoTNyHSfCLbeiH38iB11p2ifY,9915
|
|
174
|
+
dtlpy/repositories/integrations.py,sha256=dBTgvT8WgzNx3f7RuXxk1Or3jwj8XaBXfAVCSYUw-hI,14122
|
|
175
|
+
dtlpy/repositories/items.py,sha256=9_gCSV1GuVcya8NZg-79owktpuEpC1cUOtgUB-7aK94,38427
|
|
176
176
|
dtlpy/repositories/messages.py,sha256=QU0Psckg6CA_Tlw9AVxqa-Ay1fRM4n269sSIJkH9o7E,3066
|
|
177
177
|
dtlpy/repositories/models.py,sha256=IekNMcnuKVaAVTJf2AJv6YvX5qCd9kkSl4ETPMWP4Zc,38213
|
|
178
178
|
dtlpy/repositories/nodes.py,sha256=xXJm_YA0vDUn0dVvaGeq6ORM0vI3YXvfjuylvGRtkxo,3061
|
|
@@ -180,7 +180,7 @@ dtlpy/repositories/ontologies.py,sha256=unnMhD2isR9DVE5S8Fg6fSDf1ZZ5Xemxxufx4LEU
|
|
|
180
180
|
dtlpy/repositories/organizations.py,sha256=6ijUDFbsogfRul1g_vUB5AZOb41MRmV5NhNU7WLHt3A,22825
|
|
181
181
|
dtlpy/repositories/packages.py,sha256=QhkXMZkpseCt0pDropJuqoHJL0RMa5plk8AN0V3w6Nk,86807
|
|
182
182
|
dtlpy/repositories/pipeline_executions.py,sha256=hJX2I939c-bWxveVdikZ_9LWMNCQusTRkkdEa5j3Yvo,17007
|
|
183
|
-
dtlpy/repositories/pipelines.py,sha256=
|
|
183
|
+
dtlpy/repositories/pipelines.py,sha256=Mbe1x-7iX2kec6_f83Ji1xiNktz2jy1dRCvBMJTn7Po,23722
|
|
184
184
|
dtlpy/repositories/projects.py,sha256=tZyFLqVs-8ggTIi5echlX7XdGOJGW4LzKuXke7jkRnw,22140
|
|
185
185
|
dtlpy/repositories/recipes.py,sha256=ZZDhHn9g28C99bsf0nFaIpVYn6f6Jisz9upkHEkeaYY,15843
|
|
186
186
|
dtlpy/repositories/resource_executions.py,sha256=PyzsbdJxz6jf17Gx13GZmqdu6tZo3TTVv-DypnJ_sY0,5374
|
|
@@ -191,7 +191,7 @@ dtlpy/repositories/tasks.py,sha256=v09S2pYGkKx_vBG7SWigJeuMhp0GsefKo3Td7ImrWb0,4
|
|
|
191
191
|
dtlpy/repositories/times_series.py,sha256=m-bKFEgiZ13yQNelDjBfeXMUy_HgsPD_JAHj1GVx9fU,11420
|
|
192
192
|
dtlpy/repositories/triggers.py,sha256=izdNyCN1gDc5uo7AXntso0HSMTDIzGFUp-dSEz8cn_U,21990
|
|
193
193
|
dtlpy/repositories/upload_element.py,sha256=R2KWIXmkp_dMAIr81tu3Y_VRfldj0ju8__V28ombkcg,10677
|
|
194
|
-
dtlpy/repositories/uploader.py,sha256=
|
|
194
|
+
dtlpy/repositories/uploader.py,sha256=5qQbsg701HrL8x0wWCRLPBP_dztqXEb31QfeZnh0SQk,31988
|
|
195
195
|
dtlpy/repositories/webhooks.py,sha256=IIpxOJ-7KeQp1TY9aJZz-FuycSjAoYx0TDk8z86KAK8,9033
|
|
196
196
|
dtlpy/services/__init__.py,sha256=VfVJy2otIrDra6i7Sepjyez2ujiE6171ChQZp-YgxsM,904
|
|
197
197
|
dtlpy/services/aihttp_retry.py,sha256=tgntZsAY0dW9v08rkjX1T5BLNDdDd8svtgn7nH8DSGU,5022
|
|
@@ -223,9 +223,9 @@ dtlpy/utilities/reports/report.py,sha256=3nEsNnIWmdPEsd21nN8vMMgaZVcPKn9iawKTTeO
|
|
|
223
223
|
dtlpy/utilities/videos/__init__.py,sha256=SV3w51vfPuGBxaMeNemx6qEMHw_C4lLpWNGXMvdsKSY,734
|
|
224
224
|
dtlpy/utilities/videos/video_player.py,sha256=LCxg0EZ_DeuwcT7U_r7MRC6Q19s0xdFb7x5Gk39PRms,24072
|
|
225
225
|
dtlpy/utilities/videos/videos.py,sha256=Dj916B4TQRIhI7HZVevl3foFrCsPp0eeWwvGbgX3-_A,21875
|
|
226
|
-
dtlpy-1.
|
|
227
|
-
dtlpy-1.
|
|
228
|
-
dtlpy-1.
|
|
226
|
+
dtlpy-1.103.12.data/scripts/dlp,sha256=-F0vSCWuSOOtgERAtsPMPyMmzitjhB7Yeftg_PDlDjw,10
|
|
227
|
+
dtlpy-1.103.12.data/scripts/dlp.bat,sha256=QOvx8Dlx5dUbCTMpwbhOcAIXL1IWmgVRSboQqDhIn3A,37
|
|
228
|
+
dtlpy-1.103.12.data/scripts/dlp.py,sha256=tEokRaDINISXnq8yNx_CBw1qM5uwjYiZoJOYGqWB3RU,4267
|
|
229
229
|
tests/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
230
230
|
tests/assets/models_flow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
231
231
|
tests/assets/models_flow/failedmain.py,sha256=n8F4eu_u7JPrJ1zedbJPvv9e3lHb3ihoErqrBIcseEc,1847
|
|
@@ -233,9 +233,9 @@ tests/assets/models_flow/main.py,sha256=vnDKyVZaae2RFpvwS22Hzi6Dt2LJerH4yQrmKtaT
|
|
|
233
233
|
tests/assets/models_flow/main_model.py,sha256=Hl_tv7Q6KaRL3yLkpUoLMRqu5-ab1QsUYPL6RPEoamw,2042
|
|
234
234
|
tests/features/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
235
235
|
tests/features/environment.py,sha256=oAO7H7j7Y7czW0t25Gv1KwI2-ofqhZVkbCw9LbZCp4Y,17506
|
|
236
|
-
dtlpy-1.
|
|
237
|
-
dtlpy-1.
|
|
238
|
-
dtlpy-1.
|
|
239
|
-
dtlpy-1.
|
|
240
|
-
dtlpy-1.
|
|
241
|
-
dtlpy-1.
|
|
236
|
+
dtlpy-1.103.12.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
|
|
237
|
+
dtlpy-1.103.12.dist-info/METADATA,sha256=sz-9jRSMf85hTPy6sH1i5vjEgkISude0fUmwR5BxSGA,3020
|
|
238
|
+
dtlpy-1.103.12.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
|
239
|
+
dtlpy-1.103.12.dist-info/entry_points.txt,sha256=C4PyKthCs_no88HU39eioO68oei64STYXC2ooGZTc4Y,43
|
|
240
|
+
dtlpy-1.103.12.dist-info/top_level.txt,sha256=ZWuLmQGUOtWAdgTf4Fbx884w1o0vBYq9dEc1zLv9Mig,12
|
|
241
|
+
dtlpy-1.103.12.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|