dtlpy 1.88.15__py3-none-any.whl → 1.90.37__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtlpy/__init__.py +3 -2
- dtlpy/__version__.py +1 -1
- dtlpy/assets/lock_open.png +0 -0
- dtlpy/dlp/command_executor.py +23 -10
- dtlpy/dlp/parser.py +2 -2
- dtlpy/entities/__init__.py +2 -2
- dtlpy/entities/app.py +56 -4
- dtlpy/entities/dataset.py +7 -1
- dtlpy/entities/dpk.py +29 -34
- dtlpy/entities/filters.py +7 -1
- dtlpy/entities/integration.py +8 -3
- dtlpy/entities/model.py +30 -2
- dtlpy/entities/package_function.py +1 -0
- dtlpy/entities/package_module.py +1 -0
- dtlpy/entities/pipeline.py +1 -1
- dtlpy/entities/service.py +17 -3
- dtlpy/examples/upload_items_with_modalities.py +1 -1
- dtlpy/ml/base_feature_extractor_adapter.py +28 -0
- dtlpy/ml/base_model_adapter.py +37 -10
- dtlpy/repositories/__init__.py +1 -0
- dtlpy/repositories/apps.py +82 -4
- dtlpy/repositories/dpks.py +37 -9
- dtlpy/repositories/executions.py +10 -6
- dtlpy/repositories/features.py +8 -2
- dtlpy/repositories/integrations.py +1 -0
- dtlpy/repositories/models.py +57 -10
- dtlpy/repositories/packages.py +5 -2
- dtlpy/repositories/pipeline_executions.py +8 -5
- dtlpy/repositories/recipes.py +3 -2
- dtlpy/repositories/schema.py +120 -0
- dtlpy/repositories/services.py +5 -2
- dtlpy/services/api_client.py +9 -0
- dtlpy/services/logins.py +49 -18
- dtlpy/utilities/converter.py +37 -20
- dtlpy/utilities/dataset_generators/dataset_generator.py +2 -1
- {dtlpy-1.88.15.dist-info → dtlpy-1.90.37.dist-info}/METADATA +2 -2
- {dtlpy-1.88.15.dist-info → dtlpy-1.90.37.dist-info}/RECORD +45 -43
- tests/features/environment.py +5 -1
- {dtlpy-1.88.15.data → dtlpy-1.90.37.data}/scripts/dlp +0 -0
- {dtlpy-1.88.15.data → dtlpy-1.90.37.data}/scripts/dlp.bat +0 -0
- {dtlpy-1.88.15.data → dtlpy-1.90.37.data}/scripts/dlp.py +0 -0
- {dtlpy-1.88.15.dist-info → dtlpy-1.90.37.dist-info}/LICENSE +0 -0
- {dtlpy-1.88.15.dist-info → dtlpy-1.90.37.dist-info}/WHEEL +0 -0
- {dtlpy-1.88.15.dist-info → dtlpy-1.90.37.dist-info}/entry_points.txt +0 -0
- {dtlpy-1.88.15.dist-info → dtlpy-1.90.37.dist-info}/top_level.txt +0 -0
dtlpy/__init__.py
CHANGED
|
@@ -63,7 +63,7 @@ from . import repositories, exceptions, entities, examples
|
|
|
63
63
|
from .entities import (
|
|
64
64
|
# main entities
|
|
65
65
|
Project, Dataset, ExpirationOptions, ExportVersion, Trigger, Item, Execution, AnnotationCollection, Annotation,
|
|
66
|
-
Recipe, IndexDriver, AttributesTypes, AttributesRange, Dpk, App, AppModule,
|
|
66
|
+
Recipe, IndexDriver, AttributesTypes, AttributesRange, Dpk, App, AppModule, AppScope,
|
|
67
67
|
Ontology, Label, Task, TaskPriority, ConsensusTaskType, Assignment, Service, Package, Codebase, Model,
|
|
68
68
|
PackageModule, PackageFunction,
|
|
69
69
|
# annotations
|
|
@@ -80,7 +80,7 @@ from .entities import (
|
|
|
80
80
|
InstanceCatalog, PackageInputType, ServiceType, ServiceModeType,
|
|
81
81
|
PackageSlot, SlotPostAction, SlotPostActionType, SlotDisplayScope, SlotDisplayScopeResource, UiBindingPanel,
|
|
82
82
|
# roberto
|
|
83
|
-
DatasetSubsetType, PlotSample, ArtifactType, Artifact, ItemArtifact, LinkArtifact, LocalArtifact, EntityScopeLevel,
|
|
83
|
+
DatasetSubsetType, ModelStatus, PlotSample, ArtifactType, Artifact, ItemArtifact, LinkArtifact, LocalArtifact, EntityScopeLevel,
|
|
84
84
|
# features
|
|
85
85
|
FeatureEntityType, Feature, FeatureSet,
|
|
86
86
|
#
|
|
@@ -177,6 +177,7 @@ login = client_api.login
|
|
|
177
177
|
logout = client_api.logout
|
|
178
178
|
login_token = client_api.login_token
|
|
179
179
|
login_secret = client_api.login_secret
|
|
180
|
+
login_api_key = client_api.login_api_key
|
|
180
181
|
login_m2m = client_api.login_m2m
|
|
181
182
|
add_environment = client_api.add_environment
|
|
182
183
|
setenv = client_api.setenv
|
dtlpy/__version__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
version = '1.
|
|
1
|
+
version = '1.90.37'
|
dtlpy/assets/lock_open.png
CHANGED
|
Binary file
|
dtlpy/dlp/command_executor.py
CHANGED
|
@@ -59,6 +59,9 @@ class CommandExecutor:
|
|
|
59
59
|
self.dl.login_token(args.token)
|
|
60
60
|
self.dl.info(with_token=False)
|
|
61
61
|
|
|
62
|
+
def login_api_key(self, args):
|
|
63
|
+
self.dl.login_api_key(api_key=args.api_key)
|
|
64
|
+
|
|
62
65
|
def login_secret(self, args):
|
|
63
66
|
self.login_m2m(args=args)
|
|
64
67
|
|
|
@@ -512,10 +515,10 @@ class CommandExecutor:
|
|
|
512
515
|
return
|
|
513
516
|
name = args.name
|
|
514
517
|
description = args.description
|
|
515
|
-
|
|
518
|
+
attributes = args.attributes
|
|
516
519
|
icon = args.icon
|
|
517
520
|
scope = args.scope
|
|
518
|
-
as_array = [name, description,
|
|
521
|
+
as_array = [name, description, attributes, icon, scope]
|
|
519
522
|
if as_array.count(None) == len(as_array): # No one value is initialized
|
|
520
523
|
dir_name = os.path.basename(os.getcwd())
|
|
521
524
|
questions = [
|
|
@@ -524,8 +527,8 @@ class CommandExecutor:
|
|
|
524
527
|
default=dir_name),
|
|
525
528
|
inquirer.Text(name='description',
|
|
526
529
|
message="Enter the description (or enter for empty): "),
|
|
527
|
-
inquirer.Text(name='
|
|
528
|
-
message="Enter the
|
|
530
|
+
inquirer.Text(name='attributes',
|
|
531
|
+
message="Enter the attributes (an object, or enter for empty): ",
|
|
529
532
|
default=None),
|
|
530
533
|
inquirer.Text(name='icon',
|
|
531
534
|
message="Enter the path to the icon (or enter for empty): "),
|
|
@@ -536,18 +539,16 @@ class CommandExecutor:
|
|
|
536
539
|
answers = inquirer.prompt(questions)
|
|
537
540
|
name = answers.get('name')
|
|
538
541
|
description = answers.get('description')
|
|
539
|
-
|
|
542
|
+
attributes = answers.get('attributes')
|
|
540
543
|
icon = answers.get('icon')
|
|
541
544
|
scope = answers.get('scope')
|
|
542
545
|
|
|
543
|
-
if
|
|
544
|
-
|
|
545
|
-
else:
|
|
546
|
-
categories = [c.strip() for c in categories.split(',')]
|
|
546
|
+
if attributes is not None:
|
|
547
|
+
attributes = json.loads(attributes)
|
|
547
548
|
|
|
548
549
|
self.dl.dpks.init(name=name,
|
|
549
550
|
description=description,
|
|
550
|
-
|
|
551
|
+
attributes=attributes,
|
|
551
552
|
icon=icon,
|
|
552
553
|
scope=scope)
|
|
553
554
|
elif args.app == 'add':
|
|
@@ -589,6 +590,18 @@ class CommandExecutor:
|
|
|
589
590
|
elif args.app == 'update':
|
|
590
591
|
# TODO: I think it changed, not implemented
|
|
591
592
|
logger.info('App updated successfully')
|
|
593
|
+
elif args.app == 'resume':
|
|
594
|
+
succeed = self.utils.get_apps_repo(args).resume(app_id=args.app_id)
|
|
595
|
+
if succeed is True:
|
|
596
|
+
logger.info('Resumed application successfully')
|
|
597
|
+
else:
|
|
598
|
+
logger.info('Application resume failed')
|
|
599
|
+
elif args.app == 'pause':
|
|
600
|
+
succeed = self.utils.get_apps_repo(args).pause(app_id=args.app_id)
|
|
601
|
+
if succeed is True:
|
|
602
|
+
logger.info('Paused application successfully')
|
|
603
|
+
else:
|
|
604
|
+
logger.info('Application pause failed')
|
|
592
605
|
elif args.app == 'install':
|
|
593
606
|
app = self.utils.get_apps_repo(args).install(
|
|
594
607
|
dpk=self.utils.get_dpks_repo(args).get(dpk_id=args.dpk_id),
|
dtlpy/dlp/parser.py
CHANGED
|
@@ -368,8 +368,8 @@ def get_parser():
|
|
|
368
368
|
optional = a.add_argument_group("Optional named arguments")
|
|
369
369
|
optional.add_argument('--name', required=False, dest='name', help="the name of the app")
|
|
370
370
|
optional.add_argument('--description', required=False, dest='description', help="the description of the app")
|
|
371
|
-
optional.add_argument('--
|
|
372
|
-
help="the
|
|
371
|
+
optional.add_argument('--attributes', required=False, dest='attributes',
|
|
372
|
+
help="the attributes of the app (comma separated)")
|
|
373
373
|
optional.add_argument('--icon', required=False, dest='icon', help="the icon of the app")
|
|
374
374
|
optional.add_argument('--scope', required=False, dest='scope',
|
|
375
375
|
help="the scope of the app (default is organization)")
|
dtlpy/entities/__init__.py
CHANGED
|
@@ -53,7 +53,7 @@ from .directory_tree import DirectoryTree
|
|
|
53
53
|
from .user import User
|
|
54
54
|
from .bot import Bot
|
|
55
55
|
from .webhook import Webhook, HttpMethod
|
|
56
|
-
from .model import Model, DatasetSubsetType, PlotSample
|
|
56
|
+
from .model import Model, DatasetSubsetType, PlotSample, ModelStatus
|
|
57
57
|
from .driver import Driver, S3Driver, GcsDriver, AzureBlobDriver
|
|
58
58
|
from .pipeline import Pipeline, PipelineStats, PipelineResumeOption, PipelineSettings, Variable, CompositionStatus
|
|
59
59
|
from .node import PipelineConnection, PipelineNode, PipelineConnectionPort, PipelineNodeIO, TaskNode, \
|
|
@@ -70,7 +70,7 @@ from .setting import Role, PlatformEntityType, SettingsValueTypes, SettingsTypes
|
|
|
70
70
|
from .reflect_dict import ReflectDict
|
|
71
71
|
from .dpk import Dpk, Panel, Toolbar, Components, DpkComputeConfig, DpkComponentChannel, PipelineNode, \
|
|
72
72
|
CustomNodeScope, ToolbarInvoke
|
|
73
|
-
from .app import App
|
|
73
|
+
from .app import App, AppScope
|
|
74
74
|
from .app_module import AppModule
|
|
75
75
|
from .resource_execution import ResourceExecution
|
|
76
76
|
from .message import Message, NotificationEventContext
|
dtlpy/entities/app.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
from collections import namedtuple
|
|
2
2
|
import traceback
|
|
3
3
|
import logging
|
|
4
|
+
from enum import Enum
|
|
5
|
+
|
|
4
6
|
import attr
|
|
5
7
|
|
|
6
8
|
from .. import entities, repositories
|
|
@@ -9,6 +11,24 @@ from ..services.api_client import ApiClient
|
|
|
9
11
|
logger = logging.getLogger(name='dtlpy')
|
|
10
12
|
|
|
11
13
|
|
|
14
|
+
class AppScope(str, Enum):
|
|
15
|
+
""" The scope of the app.
|
|
16
|
+
|
|
17
|
+
.. list-table::
|
|
18
|
+
:widths: 15 150
|
|
19
|
+
:header-rows: 1
|
|
20
|
+
|
|
21
|
+
* - State
|
|
22
|
+
- Description
|
|
23
|
+
* - SYSTEM
|
|
24
|
+
- Dataloop internal app
|
|
25
|
+
* - PROJECT
|
|
26
|
+
- Project app
|
|
27
|
+
"""
|
|
28
|
+
SYSTEM = 'system'
|
|
29
|
+
PROJECT = 'project'
|
|
30
|
+
|
|
31
|
+
|
|
12
32
|
@attr.s
|
|
13
33
|
class App(entities.BaseEntity):
|
|
14
34
|
id = attr.ib(type=str)
|
|
@@ -24,7 +44,9 @@ class App(entities.BaseEntity):
|
|
|
24
44
|
composition_id = attr.ib(type=str)
|
|
25
45
|
scope = attr.ib(type=str)
|
|
26
46
|
routes = attr.ib(type=dict)
|
|
27
|
-
|
|
47
|
+
custom_installation = attr.ib(type=dict)
|
|
48
|
+
metadata = attr.ib(type=dict)
|
|
49
|
+
status = attr.ib(type=entities.CompositionStatus)
|
|
28
50
|
|
|
29
51
|
# sdk
|
|
30
52
|
_project = attr.ib(type=entities.Project, repr=False)
|
|
@@ -77,6 +99,30 @@ class App(entities.BaseEntity):
|
|
|
77
99
|
"""
|
|
78
100
|
return self.apps.update(self)
|
|
79
101
|
|
|
102
|
+
def resume(self):
|
|
103
|
+
"""
|
|
104
|
+
Resume the current app
|
|
105
|
+
|
|
106
|
+
:return bool whether the operation ran successfully or not
|
|
107
|
+
|
|
108
|
+
**Example**
|
|
109
|
+
.. code-block:: python
|
|
110
|
+
succeed = app.resume()
|
|
111
|
+
"""
|
|
112
|
+
return self.apps.resume(self)
|
|
113
|
+
|
|
114
|
+
def pause(self):
|
|
115
|
+
"""
|
|
116
|
+
Pause the current app
|
|
117
|
+
|
|
118
|
+
:return bool whether the operation ran successfully or not
|
|
119
|
+
|
|
120
|
+
**Example**
|
|
121
|
+
.. code-block:: python
|
|
122
|
+
succeed = app.pause()
|
|
123
|
+
"""
|
|
124
|
+
return self.apps.pause(self)
|
|
125
|
+
|
|
80
126
|
@staticmethod
|
|
81
127
|
def _protected_from_json(_json, client_api, project, is_fetched=True):
|
|
82
128
|
"""
|
|
@@ -125,8 +171,12 @@ class App(entities.BaseEntity):
|
|
|
125
171
|
_json['scope'] = self.scope
|
|
126
172
|
if self.routes != {}:
|
|
127
173
|
_json['routes'] = self.routes
|
|
128
|
-
if self.
|
|
129
|
-
_json['
|
|
174
|
+
if self.custom_installation != {}:
|
|
175
|
+
_json['customInstallation'] = self.custom_installation
|
|
176
|
+
if self.metadata is not None:
|
|
177
|
+
_json['metadata'] = self.metadata
|
|
178
|
+
if self.status is not None:
|
|
179
|
+
_json['status'] = self.status
|
|
130
180
|
|
|
131
181
|
return _json
|
|
132
182
|
|
|
@@ -146,9 +196,11 @@ class App(entities.BaseEntity):
|
|
|
146
196
|
composition_id=_json.get('compositionId', None),
|
|
147
197
|
scope=_json.get('scope', None),
|
|
148
198
|
routes=_json.get('routes', {}),
|
|
149
|
-
|
|
199
|
+
custom_installation=_json.get('customInstallation', {}),
|
|
150
200
|
client_api=client_api,
|
|
151
201
|
project=project,
|
|
202
|
+
metadata=_json.get('metadata', None),
|
|
203
|
+
status=_json.get('status', None)
|
|
152
204
|
)
|
|
153
205
|
app.is_fetched = is_fetched
|
|
154
206
|
return app
|
dtlpy/entities/dataset.py
CHANGED
|
@@ -281,7 +281,7 @@ class Dataset(entities.BaseEntity):
|
|
|
281
281
|
def set_repositories(self):
|
|
282
282
|
reps = namedtuple('repositories',
|
|
283
283
|
field_names=['items', 'recipes', 'datasets', 'assignments', 'tasks', 'annotations',
|
|
284
|
-
'ontologies', 'features', 'settings'])
|
|
284
|
+
'ontologies', 'features', 'settings', 'schema'])
|
|
285
285
|
if self._project is None:
|
|
286
286
|
datasets = repositories.Datasets(client_api=self._client_api, project=self._project)
|
|
287
287
|
features = repositories.Features(client_api=self._client_api, project=self._project)
|
|
@@ -299,6 +299,7 @@ class Dataset(entities.BaseEntity):
|
|
|
299
299
|
ontologies=repositories.Ontologies(client_api=self._client_api, dataset=self),
|
|
300
300
|
features=features,
|
|
301
301
|
settings=repositories.Settings(client_api=self._client_api, dataset=self),
|
|
302
|
+
schema=repositories.Schema(client_api=self._client_api, dataset=self)
|
|
302
303
|
)
|
|
303
304
|
|
|
304
305
|
@property
|
|
@@ -346,6 +347,11 @@ class Dataset(entities.BaseEntity):
|
|
|
346
347
|
assert isinstance(self._repositories.features, repositories.Features)
|
|
347
348
|
return self._repositories.features
|
|
348
349
|
|
|
350
|
+
@property
|
|
351
|
+
def schema(self):
|
|
352
|
+
assert isinstance(self._repositories.schema, repositories.Schema)
|
|
353
|
+
return self._repositories.schema
|
|
354
|
+
|
|
349
355
|
@property
|
|
350
356
|
def project(self):
|
|
351
357
|
if self._project is None:
|
dtlpy/entities/dpk.py
CHANGED
|
@@ -3,7 +3,6 @@ from typing import List, Union
|
|
|
3
3
|
import traceback
|
|
4
4
|
import enum
|
|
5
5
|
|
|
6
|
-
|
|
7
6
|
from .. import entities, repositories, exceptions
|
|
8
7
|
from ..services.api_client import ApiClient
|
|
9
8
|
|
|
@@ -105,6 +104,7 @@ class PipelineNode(entities.DlEntity):
|
|
|
105
104
|
description: str = entities.DlProperty(location=['description'], _type=str)
|
|
106
105
|
configuration: dict = entities.DlProperty(location=['configuration'], _type=dict)
|
|
107
106
|
scope: CustomNodeScope = entities.DlProperty(location=['scope'], _type=CustomNodeScope)
|
|
107
|
+
compute_config: str = entities.DlProperty(location=['computeConfig'], _type=str, default=None)
|
|
108
108
|
|
|
109
109
|
def to_json(self) -> dict:
|
|
110
110
|
return self._dict.copy()
|
|
@@ -214,7 +214,6 @@ class Components(entities.DlEntity):
|
|
|
214
214
|
self._dict['pipelineTemplates'] = list()
|
|
215
215
|
return self._dict['pipelineTemplates']
|
|
216
216
|
|
|
217
|
-
|
|
218
217
|
@classmethod
|
|
219
218
|
def from_json(cls, _json):
|
|
220
219
|
inst = cls(_dict=_json)
|
|
@@ -229,7 +228,7 @@ class Dpk(entities.DlEntity):
|
|
|
229
228
|
id: str = entities.DlProperty(location=['id'], _type=str)
|
|
230
229
|
name: str = entities.DlProperty(location=['name'], _type=str)
|
|
231
230
|
version: str = entities.DlProperty(location=['version'], _type=str)
|
|
232
|
-
|
|
231
|
+
attributes: list = entities.DlProperty(location=['attributes'], _type=dict)
|
|
233
232
|
created_at: str = entities.DlProperty(location=['createdAt'], _type=str)
|
|
234
233
|
updated_at: str = entities.DlProperty(location=['updatedAt'], _type=str)
|
|
235
234
|
creator: str = entities.DlProperty(location=['creator'], _type=str)
|
|
@@ -239,6 +238,8 @@ class Dpk(entities.DlEntity):
|
|
|
239
238
|
codebase: str = entities.DlProperty(location=['codebase'], _kls="Codebase")
|
|
240
239
|
scope: dict = entities.DlProperty(location=['scope'], _type=str)
|
|
241
240
|
context: dict = entities.DlProperty(location=['context'], _type=dict)
|
|
241
|
+
metadata: dict = entities.DlProperty(location=['metadata'], _type=dict)
|
|
242
|
+
dependencies: dict = entities.DlProperty(location=['dependencies'], _type=List[dict])
|
|
242
243
|
|
|
243
244
|
# defaults
|
|
244
245
|
components: Components = entities.DlProperty(location=['components'], _kls='Components')
|
|
@@ -251,6 +252,11 @@ class Dpk(entities.DlEntity):
|
|
|
251
252
|
_revisions = None
|
|
252
253
|
__repositories = None
|
|
253
254
|
|
|
255
|
+
@components.default
|
|
256
|
+
def default_components(self):
|
|
257
|
+
self._dict['components'] = dict()
|
|
258
|
+
return self._dict['components']
|
|
259
|
+
|
|
254
260
|
################
|
|
255
261
|
# repositories #
|
|
256
262
|
################
|
|
@@ -258,12 +264,14 @@ class Dpk(entities.DlEntity):
|
|
|
258
264
|
def _repositories(self):
|
|
259
265
|
if self.__repositories is None:
|
|
260
266
|
reps = namedtuple('repositories',
|
|
261
|
-
field_names=['dpks', 'codebases', 'organizations'])
|
|
267
|
+
field_names=['dpks', 'codebases', 'organizations', 'services'])
|
|
262
268
|
|
|
263
|
-
self.__repositories = reps(
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
269
|
+
self.__repositories = reps(
|
|
270
|
+
dpks=repositories.Dpks(client_api=self.client_api, project=self.project),
|
|
271
|
+
codebases=repositories.Codebases(client_api=self.client_api),
|
|
272
|
+
organizations=repositories.Organizations(client_api=self.client_api),
|
|
273
|
+
services=repositories.Services(client_api=self.client_api, project=self.project, package=self),
|
|
274
|
+
)
|
|
267
275
|
|
|
268
276
|
return self.__repositories
|
|
269
277
|
|
|
@@ -282,6 +290,11 @@ class Dpk(entities.DlEntity):
|
|
|
282
290
|
assert isinstance(self._repositories.dpks, repositories.Dpks)
|
|
283
291
|
return self._repositories.dpks
|
|
284
292
|
|
|
293
|
+
@property
|
|
294
|
+
def services(self):
|
|
295
|
+
assert isinstance(self._repositories.services, repositories.Services)
|
|
296
|
+
return self._repositories.services
|
|
297
|
+
|
|
285
298
|
###########
|
|
286
299
|
# methods #
|
|
287
300
|
###########
|
|
@@ -418,29 +431,11 @@ class Dpk(entities.DlEntity):
|
|
|
418
431
|
:return: App entity
|
|
419
432
|
:rtype: dtlpy.entities.App
|
|
420
433
|
"""
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
if components_version == 'v1':
|
|
430
|
-
return cls(
|
|
431
|
-
_dict=_json,
|
|
432
|
-
client_api=client_api,
|
|
433
|
-
project=project,
|
|
434
|
-
is_fetched=is_fetched
|
|
435
|
-
)
|
|
436
|
-
else:
|
|
437
|
-
return DpkV2(
|
|
438
|
-
_dict=_json,
|
|
439
|
-
client_api=client_api,
|
|
440
|
-
project=project,
|
|
441
|
-
is_fetched=is_fetched
|
|
442
|
-
)
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
class DpkV2(Dpk):
|
|
446
|
-
components: dict = entities.DlProperty(location=['components'])
|
|
434
|
+
res = cls(
|
|
435
|
+
_dict=_json,
|
|
436
|
+
client_api=client_api,
|
|
437
|
+
project=project,
|
|
438
|
+
is_fetched=is_fetched
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
return res
|
dtlpy/entities/filters.py
CHANGED
|
@@ -324,7 +324,7 @@ class Filters:
|
|
|
324
324
|
self._unique_fields = ['type']
|
|
325
325
|
self.add(field='type',
|
|
326
326
|
values=['box', 'class', 'comparison', 'ellipse', 'point', 'segment', 'polyline', 'binary',
|
|
327
|
-
'subtitle', 'cube', 'cube_3d', 'pose', 'text_mark'],
|
|
327
|
+
'subtitle', 'cube', 'cube_3d', 'pose', 'text_mark', 'text', 'ref_image'],
|
|
328
328
|
operator=FiltersOperations.IN,
|
|
329
329
|
method=FiltersMethod.AND)
|
|
330
330
|
|
|
@@ -456,6 +456,9 @@ class Filters:
|
|
|
456
456
|
_json['systemSpace'] = self._system_space
|
|
457
457
|
return _json
|
|
458
458
|
|
|
459
|
+
def print(self, indent=2):
|
|
460
|
+
print(json.dumps(self.prepare(), indent=indent))
|
|
461
|
+
|
|
459
462
|
def sort_by(self, field, value: FiltersOrderByDirection = FiltersOrderByDirection.ASCENDING):
|
|
460
463
|
"""
|
|
461
464
|
sort the filter
|
|
@@ -549,3 +552,6 @@ class SingleFilter:
|
|
|
549
552
|
_json[self.field] = value
|
|
550
553
|
|
|
551
554
|
return _json
|
|
555
|
+
|
|
556
|
+
def print(self, indent=2):
|
|
557
|
+
print(json.dumps(self.prepare(), indent=indent))
|
dtlpy/entities/integration.py
CHANGED
|
@@ -32,6 +32,8 @@ class IntegrationType(str, Enum):
|
|
|
32
32
|
- AZURE BLOB Integration - for S3 AZUREBLOB and AZURE_DATALAKE_GEN2 drivers
|
|
33
33
|
* - KEY_VALUE
|
|
34
34
|
- KEY VALUE Integration - for save secrets in the platform
|
|
35
|
+
* - GCP_WORKLOAD_IDENTITY_FEDERATION
|
|
36
|
+
- GCP Workload Identity Federation Integration - for GCP drivers
|
|
35
37
|
"""
|
|
36
38
|
S3 = "s3"
|
|
37
39
|
AWS_CROSS_ACCOUNT = 'aws-cross'
|
|
@@ -40,6 +42,7 @@ class IntegrationType(str, Enum):
|
|
|
40
42
|
GCS_CROSS = "gcp-cross"
|
|
41
43
|
AZUREBLOB = "azureblob"
|
|
42
44
|
KEY_VALUE = "key_value"
|
|
45
|
+
GCP_WORKLOAD_IDENTITY_FEDERATION = "gcp-workload-identity-federation"
|
|
43
46
|
|
|
44
47
|
|
|
45
48
|
@attr.s
|
|
@@ -52,8 +55,9 @@ class Integration(entities.BaseEntity):
|
|
|
52
55
|
type = attr.ib()
|
|
53
56
|
org = attr.ib()
|
|
54
57
|
created_at = attr.ib()
|
|
55
|
-
|
|
58
|
+
creator = attr.ib()
|
|
56
59
|
update_at = attr.ib()
|
|
60
|
+
url = attr.ib()
|
|
57
61
|
_client_api = attr.ib(type=ApiClient, repr=False)
|
|
58
62
|
metadata = attr.ib(default=None, repr=False)
|
|
59
63
|
_project = attr.ib(default=None, repr=False)
|
|
@@ -73,13 +77,14 @@ class Integration(entities.BaseEntity):
|
|
|
73
77
|
"""
|
|
74
78
|
inst = cls(id=_json.get('id', None),
|
|
75
79
|
name=_json.get('name', None),
|
|
76
|
-
|
|
80
|
+
creator=_json.get('creator', None),
|
|
77
81
|
created_at=_json.get('createdAt', None),
|
|
78
82
|
update_at=_json.get('updateAt', None),
|
|
79
83
|
type=_json.get('type', None),
|
|
80
84
|
org=_json.get('org', None),
|
|
81
85
|
client_api=client_api,
|
|
82
|
-
metadata=_json.get('metadata', None)
|
|
86
|
+
metadata=_json.get('metadata', None),
|
|
87
|
+
url=_json.get('url', None))
|
|
83
88
|
inst.is_fetched = is_fetched
|
|
84
89
|
return inst
|
|
85
90
|
|
dtlpy/entities/model.py
CHANGED
|
@@ -19,6 +19,18 @@ class DatasetSubsetType(str, Enum):
|
|
|
19
19
|
TEST = 'test'
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
class ModelStatus(str, Enum):
|
|
23
|
+
"""Available types for model status"""
|
|
24
|
+
CREATED = "created",
|
|
25
|
+
PRE_TRAINED = "pre-trained",
|
|
26
|
+
PENDING = "pending",
|
|
27
|
+
TRAINING = "training",
|
|
28
|
+
TRAINED = "trained",
|
|
29
|
+
DEPLOYED = "deployed",
|
|
30
|
+
FAILED = "failed",
|
|
31
|
+
CLONING = "cloning"
|
|
32
|
+
|
|
33
|
+
|
|
22
34
|
class PlotSample:
|
|
23
35
|
def __init__(self, figure, legend, x, y):
|
|
24
36
|
"""
|
|
@@ -106,6 +118,7 @@ class Model(entities.BaseEntity):
|
|
|
106
118
|
_repositories = attr.ib(repr=False)
|
|
107
119
|
_ontology = attr.ib(repr=False, default=None)
|
|
108
120
|
updated_by = attr.ib(default=None)
|
|
121
|
+
app = attr.ib(default=None)
|
|
109
122
|
|
|
110
123
|
@staticmethod
|
|
111
124
|
def _protected_from_json(_json, client_api, project, package, is_fetched=True):
|
|
@@ -186,7 +199,8 @@ class Model(entities.BaseEntity):
|
|
|
186
199
|
input_type=_json.get('inputType', None),
|
|
187
200
|
output_type=_json.get('outputType', None),
|
|
188
201
|
module_name=_json.get('moduleName', None),
|
|
189
|
-
updated_by=_json.get('updatedBy', None)
|
|
202
|
+
updated_by=_json.get('updatedBy', None),
|
|
203
|
+
app=_json.get('app', None)
|
|
190
204
|
)
|
|
191
205
|
inst.is_fetched = is_fetched
|
|
192
206
|
return inst
|
|
@@ -214,7 +228,8 @@ class Model(entities.BaseEntity):
|
|
|
214
228
|
attr.fields(Model).updated_at,
|
|
215
229
|
attr.fields(Model).input_type,
|
|
216
230
|
attr.fields(Model).output_type,
|
|
217
|
-
attr.fields(Model).updated_by
|
|
231
|
+
attr.fields(Model).updated_by,
|
|
232
|
+
attr.fields(Model).app
|
|
218
233
|
))
|
|
219
234
|
_json['packageId'] = self.package_id
|
|
220
235
|
_json['datasetId'] = self.dataset_id
|
|
@@ -236,6 +251,8 @@ class Model(entities.BaseEntity):
|
|
|
236
251
|
|
|
237
252
|
if self.updated_by:
|
|
238
253
|
_json['updatedBy'] = self.updated_by
|
|
254
|
+
if self.app:
|
|
255
|
+
_json['app'] = self.app
|
|
239
256
|
|
|
240
257
|
return _json
|
|
241
258
|
|
|
@@ -478,6 +495,7 @@ class Model(entities.BaseEntity):
|
|
|
478
495
|
tags: list = None,
|
|
479
496
|
train_filter: entities.Filters = None,
|
|
480
497
|
validation_filter: entities.Filters = None,
|
|
498
|
+
wait=True
|
|
481
499
|
):
|
|
482
500
|
"""
|
|
483
501
|
Clones and creates a new model out of existing one
|
|
@@ -493,6 +511,7 @@ class Model(entities.BaseEntity):
|
|
|
493
511
|
:param list tags: `list` of `str` - label of the model
|
|
494
512
|
:param dtlpy.entities.filters.Filters train_filter: Filters entity or a dictionary to define the items' scope in the specified dataset_id for the model train
|
|
495
513
|
:param dtlpy.entities.filters.Filters validation_filter: Filters entity or a dictionary to define the items' scope in the specified dataset_id for the model validation
|
|
514
|
+
:param bool wait: `bool` wait for the model to be ready before returning
|
|
496
515
|
|
|
497
516
|
:return: dl.Model which is a clone version of the existing model
|
|
498
517
|
"""
|
|
@@ -508,6 +527,7 @@ class Model(entities.BaseEntity):
|
|
|
508
527
|
tags=tags,
|
|
509
528
|
train_filter=train_filter,
|
|
510
529
|
validation_filter=validation_filter,
|
|
530
|
+
wait=wait
|
|
511
531
|
)
|
|
512
532
|
|
|
513
533
|
def train(self, service_config=None):
|
|
@@ -552,6 +572,14 @@ class Model(entities.BaseEntity):
|
|
|
552
572
|
"""
|
|
553
573
|
return self.models.deploy(model_id=self.id, service_config=service_config)
|
|
554
574
|
|
|
575
|
+
def wait_for_model_ready(self):
|
|
576
|
+
"""
|
|
577
|
+
Wait for model to be ready
|
|
578
|
+
|
|
579
|
+
:return:
|
|
580
|
+
"""
|
|
581
|
+
return self.models.wait_for_model_ready(model=self)
|
|
582
|
+
|
|
555
583
|
def log(self,
|
|
556
584
|
service=None,
|
|
557
585
|
size=None,
|
|
@@ -64,6 +64,7 @@ class PackageFunction(entities.DlEntity):
|
|
|
64
64
|
_kls='FunctionIO')
|
|
65
65
|
inputs: typing.Union[typing.List['entities.FunctionIO'], None] = entities.DlProperty(location=['input'],
|
|
66
66
|
_kls='FunctionIO')
|
|
67
|
+
compute_config: str = entities.DlProperty(location=['computeConfig'], _type=str, default=None)
|
|
67
68
|
|
|
68
69
|
def __repr__(self):
|
|
69
70
|
# TODO need to move to DlEntity
|
dtlpy/entities/package_module.py
CHANGED
dtlpy/entities/pipeline.py
CHANGED
|
@@ -506,7 +506,7 @@ class Pipeline(entities.BaseEntity):
|
|
|
506
506
|
"""
|
|
507
507
|
execute a pipeline and return to execute
|
|
508
508
|
|
|
509
|
-
:param execution_inputs: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}
|
|
509
|
+
:param execution_inputs: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}, that represent the extra inputs of the function
|
|
510
510
|
:param filters: Filters entity for a filtering before execute
|
|
511
511
|
:param bool wait: wait until create task finish
|
|
512
512
|
:return: entities.PipelineExecution object
|
dtlpy/entities/service.py
CHANGED
|
@@ -224,6 +224,8 @@ class Service(entities.BaseEntity):
|
|
|
224
224
|
mode = attr.ib(repr=False)
|
|
225
225
|
metadata = attr.ib()
|
|
226
226
|
archive = attr.ib(repr=False)
|
|
227
|
+
config = attr.ib(repr=False)
|
|
228
|
+
settings = attr.ib(repr=False)
|
|
227
229
|
|
|
228
230
|
# SDK
|
|
229
231
|
_package = attr.ib(repr=False)
|
|
@@ -329,7 +331,9 @@ class Service(entities.BaseEntity):
|
|
|
329
331
|
mode=_json.get('mode', dict()),
|
|
330
332
|
metadata=_json.get('metadata', None),
|
|
331
333
|
archive=_json.get('archive', None),
|
|
332
|
-
updated_by=_json.get('updatedBy', None)
|
|
334
|
+
updated_by=_json.get('updatedBy', None),
|
|
335
|
+
config=_json.get('config', None),
|
|
336
|
+
settings=_json.get('settings', None),
|
|
333
337
|
)
|
|
334
338
|
inst.is_fetched = is_fetched
|
|
335
339
|
return inst
|
|
@@ -360,7 +364,8 @@ class Service(entities.BaseEntity):
|
|
|
360
364
|
if self._package is None:
|
|
361
365
|
try:
|
|
362
366
|
self._package = repositories.Packages(client_api=self._client_api).get(package_id=self.package_id,
|
|
363
|
-
fetch=None
|
|
367
|
+
fetch=None,
|
|
368
|
+
log_error=False)
|
|
364
369
|
assert isinstance(self._package, entities.Package)
|
|
365
370
|
except:
|
|
366
371
|
self._package = repositories.Dpks(client_api=self._client_api).get(dpk_id=self.package_id)
|
|
@@ -456,6 +461,8 @@ class Service(entities.BaseEntity):
|
|
|
456
461
|
attr.fields(Service).metadata,
|
|
457
462
|
attr.fields(Service).archive,
|
|
458
463
|
attr.fields(Service).updated_by,
|
|
464
|
+
attr.fields(Service).config,
|
|
465
|
+
attr.fields(Service).settings
|
|
459
466
|
)
|
|
460
467
|
)
|
|
461
468
|
|
|
@@ -504,6 +511,13 @@ class Service(entities.BaseEntity):
|
|
|
504
511
|
|
|
505
512
|
if self.archive is not None:
|
|
506
513
|
_json['archive'] = self.archive
|
|
514
|
+
|
|
515
|
+
if self.config is not None:
|
|
516
|
+
_json['config'] = self.config
|
|
517
|
+
|
|
518
|
+
if self.settings is not None:
|
|
519
|
+
_json['settings'] = self.settings
|
|
520
|
+
|
|
507
521
|
return _json
|
|
508
522
|
|
|
509
523
|
def update(self, force=False):
|
|
@@ -687,7 +701,7 @@ class Service(entities.BaseEntity):
|
|
|
687
701
|
|
|
688
702
|
:param filters: Filters entity for a filtering before execute
|
|
689
703
|
:param str function_name: function name to run
|
|
690
|
-
:param List[FunctionIO] or dict execution_inputs: input dictionary or list of FunctionIO entities
|
|
704
|
+
:param List[FunctionIO] or dict execution_inputs: input dictionary or list of FunctionIO entities, that represent the extra inputs of the function
|
|
691
705
|
:param bool wait: wait until create task finish
|
|
692
706
|
:return: execution object
|
|
693
707
|
:rtype: dtlpy.entities.execution.Execution
|
|
@@ -10,7 +10,7 @@ def main(project_name, dataset_name):
|
|
|
10
10
|
item = dataset.items.upload(local_path=dl.UrlLink(
|
|
11
11
|
ref=w_url,
|
|
12
12
|
name='.{}.json'.format(
|
|
13
|
-
''.join(random.choice(string.
|
|
13
|
+
''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(10)))),
|
|
14
14
|
remote_path='/cats',
|
|
15
15
|
item_metadata=w_metadata)
|
|
16
16
|
return item.id
|