dtlpy 1.107.8__py3-none-any.whl → 1.109.19__py3-none-any.whl

dtlpy/__init__.py

@@ -77,7 +77,7 @@ from .entities import (
     # triggers
     TriggerResource, TriggerAction, TriggerExecutionMode, TriggerType,
     # faas
-    FunctionIO, KubernetesAutoscalerType, KubernetesAutuscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRuntime,
+    FunctionIO, KubernetesAutoscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRuntime,
     InstanceCatalog, PackageInputType, ServiceType, ServiceModeType, KubernetesRPSAutoscaler,
     PackageSlot, SlotPostAction, SlotPostActionType, SlotDisplayScope, SlotDisplayScopeResource, UiBindingPanel,
     # roberto
@@ -232,12 +232,6 @@ def checkout_state():
     state = client_api.state_io.read_json()
     return state
 
-
-def use_attributes_2(state: bool = True):
-    warnings.warn("Function 'use_attributes_2()' is deprecated as of version 1.99.12 and has been non-functional since version 1.90.39.  To work with attributes 2.0, simply use 'update_attributes()'.", DeprecationWarning)
-    client_api.attributes_mode.use_attributes_2 = state
-
-
 class LoggingLevel:
     DEBUG = "debug"
     WARNING = "warning"

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.107.8'
+version = '1.109.19'

dtlpy/entities/__init__.py

@@ -43,7 +43,7 @@ from .package_slot import PackageSlot, SlotPostAction, SlotPostActionType, SlotD
     UiBindingPanel
 from .package_function import PackageFunction, FunctionIO, PackageInputType
 from .time_series import TimeSeries
-from .service import Service, KubernetesAutoscalerType, KubernetesAutuscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRPSAutoscaler, \
+from .service import Service, KubernetesAutoscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRPSAutoscaler, \
     InstanceCatalog, KubernetesRuntime, ServiceType, ServiceModeType
 from .execution import Execution, ExecutionStatus
 from .command import Command, CommandsStatus
@@ -75,8 +75,9 @@ from .app_module import AppModule
 from .resource_execution import ResourceExecution
 from .message import Message, NotificationEventContext
 from .prompt_item import Prompt, PromptItem, PromptType
-from .compute import ClusterProvider, ComputeType, ComputeStatus, Toleration, DeploymentResource, DeploymentResources, \
-    NodePool, AuthenticationIntegration, Authentication, ComputeCluster, ComputeContext, Compute, KubernetesCompute, \
-    ServiceDriver
+from .compute import ClusterProvider, ComputeType, ComputeStatus, Toleration, DeploymentResource, DeploymentResources, ComputeSettings, ComputeConsumptionMethod, \
+    NodePool, AuthenticationIntegration, Authentication, ComputeCluster, ComputeContext, Compute, KubernetesCompute
+from .service_driver import ServiceDriver
 from .gis_item import  ItemGis, Layer
 from .collection import Collection
+

dtlpy/entities/annotation.py

@@ -45,6 +45,8 @@ class AnnotationType(str, Enum):
     SUBTITLE = "subtitle"
     TEXT = "text_mark"
     GIS = "gis"
+    SEMANTIC_3D = "ref_semantic_3d"
+    POLYLINE_3D = "polyline_3d"
 
 
 class ViewAnnotationOptions(str, Enum):
@@ -130,9 +132,7 @@ class Annotation(entities.BaseEntity):
     _annotations = attr.ib(repr=False, default=None)
     __client_api = attr.ib(default=None, repr=False)
     _items = attr.ib(repr=False, default=None)
-
-    # temp
-    _recipe_2_attributes = attr.ib(repr=False, default=None)
+    _recipe_1_attributes = attr.ib(repr=False, default=None)
 
     ############
     # Platform #
@@ -422,28 +422,11 @@ class Annotation(entities.BaseEntity):
 
     @property
     def attributes(self):
-        if self._recipe_2_attributes is not None or self.annotation_definition.attributes == []:
-            return self._recipe_2_attributes
         return self.annotation_definition.attributes
 
     @attributes.setter
     def attributes(self, attributes):
-        if isinstance(attributes, dict):
-            self._recipe_2_attributes = attributes
-        elif isinstance(attributes, list):
-            warnings.warn("List attributes are deprecated and will be removed in version 1.109. Use Attribute 2.0 (Dictionary) instead. "
-                "For more details, refer to the documentation: "
-                "https://developers.dataloop.ai/tutorials/data_management/upload_and_manage_annotations/chapter/#set-attributes-on-annotations",
-                DeprecationWarning,
-            )
-            self.annotation_definition.attributes = attributes
-        elif attributes is None:
-            if self._recipe_2_attributes:
-                self._recipe_2_attributes = {}
-            if self.annotation_definition.attributes:
-                self.annotation_definition.attributes = []
-        else:
-            raise ValueError('Attributes must be a dictionary or a list')
+        self.annotation_definition.attributes = attributes
 
     @property
     def color(self):
@@ -1367,9 +1350,9 @@ class Annotation(entities.BaseEntity):
             status = _json['metadata']['system'].get('status', status)
 
         named_attributes = metadata.get('system', dict()).get('attributes', None)
-        attributes = named_attributes if named_attributes else _json.get('attributes', None)
+        recipe_1_attributes = _json.get('attributes', None)
 
-        first_frame_attributes = attributes
+        first_frame_attributes = recipe_1_attributes
         first_frame_coordinates = list()
         first_frame_number = 0
         first_frame_start_time = 0
@@ -1425,7 +1408,7 @@ class Annotation(entities.BaseEntity):
             def_dict = {'type': _json['type'],
                         'coordinates': coordinates,
                         'label': _json['label'],
-                        'attributes': attributes}
+                        'attributes': named_attributes}
             annotation_definition = FrameAnnotation.json_to_annotation_definition(def_dict)
 
         frames = entities.ReflectDict(
@@ -1470,9 +1453,9 @@ class Annotation(entities.BaseEntity):
             start_frame=start_frame,
             annotations=annotations,
             start_time=start_time,
-            recipe_2_attributes=named_attributes,
             label_suggestions=_json.get('labelSuggestions', None),
-            source=_json.get('source', None)
+            source=_json.get('source', None),
+            recipe_1_attributes=recipe_1_attributes,
         )
         annotation.annotation_definition = annotation_definition
         annotation.__client_api = client_api
@@ -1598,15 +1581,9 @@ class Annotation(entities.BaseEntity):
         if isinstance(self.annotation_definition, entities.Description):
             _json['metadata']['system']['system'] = True
 
-        if self._recipe_2_attributes is not None:
-            _json['metadata']['system']['attributes'] = self._recipe_2_attributes
-            if 'attributes' in self._platform_dict:
-                _json['attributes'] = self._platform_dict['attributes']
-        else:
-            _json['attributes'] = self.attributes
-            orig_metadata_system = self._platform_dict.get('metadata', {}).get('system', {})
-            if 'attributes' in orig_metadata_system:
-                _json['metadata']['system']['attributes'] = orig_metadata_system['attributes']
+        _json['metadata']['system']['attributes'] = self.attributes if self.attributes is not None else dict()
+        _json['attributes'] = self._recipe_1_attributes
+
 
         # add frame info
         if self.is_video or (self.end_time and self.end_time > 0) or (self.end_frame and self.end_frame > 0):
@@ -1643,7 +1620,7 @@ class Annotation(entities.BaseEntity):
         :return: page of scores
         """
         return self.annotations.task_scores(annotation_id=self.id ,task_id=task_id, page_offset=page_offset, page_size=page_size)
-        
+
 
 
 @attr.s
@@ -1663,8 +1640,8 @@ class FrameAnnotation(entities.BaseEntity):
     object_visible = attr.ib()
 
     # temp
-    _recipe_2_attributes = attr.ib(repr=False, default=None)
     _interpolation = attr.ib(repr=False, default=False)
+    _recipe_1_attributes = attr.ib(repr=False, default=None)
 
     ################################
     # parent annotation attributes #
@@ -1696,23 +1673,11 @@ class FrameAnnotation(entities.BaseEntity):
 
     @property
     def attributes(self):
-        if self._recipe_2_attributes or self.annotation_definition.attributes == []:
-            return self._recipe_2_attributes
         return self.annotation_definition.attributes
 
     @attributes.setter
     def attributes(self, attributes):
-        if isinstance(attributes, dict):
-            self._recipe_2_attributes = attributes
-        elif isinstance(attributes, list):
-            warnings.warn("List attributes are deprecated and will be removed in version 1.109. Use Attribute 2.0 (Dictionary) instead. "
-                "For more details, refer to the documentation: "
-                "https://developers.dataloop.ai/tutorials/data_management/upload_and_manage_annotations/chapter/#set-attributes-on-annotations",
-                DeprecationWarning,
-            )
-            self.annotation_definition.attributes = attributes
-        else:
-            raise ValueError('Attributes must be a dictionary or a list')
+        self.annotation_definition.attributes = attributes
 
     @property
     def geo(self):
@@ -1793,6 +1758,11 @@ class FrameAnnotation(entities.BaseEntity):
 
     @staticmethod
     def json_to_annotation_definition(_json):
+        if 'namedAttributes' in _json:
+            _json['attributes'] = _json['namedAttributes']
+        else:
+            if not isinstance(_json.get('attributes'), dict):
+                _json['attributes'] = None
         if _json['type'] == 'segment':
             annotation = entities.Polygon.from_json(_json)
         elif _json['type'] == 'polyline':
@@ -1866,6 +1836,7 @@ class FrameAnnotation(entities.BaseEntity):
         """
         # get annotation class
         _json['type'] = annotation.type
+        attrs = _json.get('attributes', None)
         annotation_definition = cls.json_to_annotation_definition(_json=_json)
 
         frame_num = _json.get('frame', annotation.last_frame + 1)
@@ -1879,9 +1850,7 @@ class FrameAnnotation(entities.BaseEntity):
             frame_num=frame_num,
             fixed=_json.get('fixed', False),
             object_visible=_json.get('objectVisible', True),
-
-            # temp
-            recipe_2_attributes=_json.get('namedAttributes', None)
+            recipe_1_attributes=attrs,
         )
 
     def to_snapshot(self):
@@ -1898,9 +1867,11 @@ class FrameAnnotation(entities.BaseEntity):
         if self.annotation_definition.description is not None:
             snapshot_dict['description'] = self.annotation_definition.description
 
-        if self.annotation._recipe_2_attributes:
-            snapshot_dict['namedAttributes'] = self._recipe_2_attributes
-        else:
-            snapshot_dict['attributes'] = self.attributes
+        if self.attributes is not None:
+            snapshot_dict['namedAttributes'] = self.attributes
+
+        if self._recipe_1_attributes is not None:
+            snapshot_dict['attributes'] = self._recipe_1_attributes
+
 
         return snapshot_dict

dtlpy/entities/annotation_definitions/base_annotation_definition.py

@@ -13,14 +13,9 @@ class BaseAnnotationDefinition:
         self._bottom = 0
         self._right = 0
         self._annotation = None
-
-        if isinstance(attributes, list) and len(attributes) > 0:
-            warnings.warn("List attributes are deprecated and will be removed in version 1.109. Use Attribute 2.0 (Dictionary) instead."
-                "For more details, refer to the documentation: "
-                "https://developers.dataloop.ai/tutorials/data_management/upload_and_manage_annotations/chapter/#set-attributes-on-annotations",
-                DeprecationWarning,
-            )
-        self._attributes = attributes
+        if attributes and not isinstance(attributes, dict):
+            raise TypeError('attributes should be a dictionary')
+        self._attributes = attributes or {}
 
     @property
     def attributes(self):
@@ -28,13 +23,10 @@ class BaseAnnotationDefinition:
 
     @attributes.setter
     def attributes(self, v):
-        if isinstance(v, list):
-            warnings.warn("List attributes are deprecated and will be removed in version 1.109. Use Attribute 2.0 (Dictionary) instead. "
-                "For more details, refer to the documentation: "
-                "https://developers.dataloop.ai/tutorials/data_management/upload_and_manage_annotations/chapter/#set-attributes-on-annotations",
-                DeprecationWarning,
-            )
+        if v and not isinstance(v, dict):
+            raise TypeError('attributes should be a dictionary')
         self._attributes = v
+
     @property
     def top(self):
         return self._top

dtlpy/entities/app.py

@@ -60,7 +60,7 @@ class App(entities.BaseEntity):
         reps = namedtuple('repositories', field_names=['projects', 'apps', 'compositions'])
         return reps(
             projects=repositories.Projects(client_api=self._client_api),
-            apps=repositories.Apps(client_api=self._client_api, project=self._project),
+            apps=repositories.Apps(client_api=self._client_api, project=self._project, project_id=self.project_id),
             compositions=repositories.Compositions(client_api=self._client_api, project=self._project)
         )
 

dtlpy/entities/command.py

@@ -146,21 +146,24 @@ class Command(entities.BaseEntity):
                                    entities.CommandsStatus.ABORTED
                                    ]
 
-    def wait(self, timeout=0, step=None, backoff_factor=1):
+    def wait(self, timeout=0, step=None, backoff_factor=1, iteration_callback=None):
         """
         Wait for Command to finish
 
         :param int timeout: int, seconds to wait until TimeoutError is raised. if 0 - wait until done
         :param int step: int, seconds between polling
         :param float backoff_factor: A backoff factor to apply between attempts after the second try
+        :param function iteration_callback: function to call on each iteration
         :return: Command  object
         """
         if not self.in_progress():
             return self
 
-        return self.commands.wait(command_id=self.id,
-                                  timeout=timeout,
-                                  step=step,
-                                  url=self.url,
-                                  backoff_factor=backoff_factor
-                                  )
+        return self.commands.wait(
+            command_id=self.id,
+            timeout=timeout,
+            step=step,
+            url=self.url,
+            backoff_factor=backoff_factor,
+            iteration_callback=iteration_callback
+        )

dtlpy/entities/compute.py

@@ -1,3 +1,4 @@
+import traceback
 from enum import Enum
 from typing import List, Optional, Dict
 from ..services.api_client import ApiClient
@@ -10,6 +11,7 @@ class ClusterProvider(str, Enum):
     AZURE = 'azure'
     HPC = 'hpc'
     LOCAL = 'local'
+    RANCHER_K3S = 'rancher-k3s'
 
 
 class ComputeType(str, Enum):
@@ -21,6 +23,31 @@ class ComputeStatus(str, Enum):
     INITIALIZING = "initializing"
     PAUSE = "pause"
     FAILED = "failed"
+    VALIDATING = "validating"
+
+
+class ComputeConsumptionMethod(str, Enum):
+    MQ = "MQ",
+    API = "API"
+
+
+class ComputeSettings:
+    def __init__(self, default_namespace: str, consumption_method: ComputeConsumptionMethod):
+        self.consumption_method = consumption_method
+        self.default_namespace = default_namespace
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(
+            default_namespace=_json.get('defaultNamespace'),
+            consumption_method=_json.get('consumptionMethod')
+        )
+
+    def to_json(self):
+        return {
+            'defaultNamespace': self.default_namespace,
+            'consumptionMethod': self.consumption_method
+        }
 
 
 class Toleration:
@@ -80,8 +107,8 @@ class DeploymentResources:
     @classmethod
     def from_json(cls, _json):
         return cls(
-            request=DeploymentResource.from_json(_json.get('request', dict())),
-            limit=DeploymentResource.from_json(_json.get('limit', dict()))
+            request=DeploymentResource.from_json(_json.get('request') or dict()),
+            limit=DeploymentResource.from_json(_json.get('limit') or dict())
         )
 
     def to_json(self):
@@ -100,7 +127,7 @@ class NodePool:
             tolerations: Optional[List[Toleration]] = None,
             description: str = "",
             node_selector: str = "",
-            preemtible: bool = False,
+            preemptible: bool = False,
             deployment_resources: DeploymentResources = None
     ):
         self.name = name
@@ -109,7 +136,7 @@ class NodePool:
         self.tolerations = tolerations if tolerations is not None else []
         self.description = description
         self.node_selector = node_selector
-        self.preemtible = preemtible
+        self.preemptible = preemptible
         self.deployment_resources = deployment_resources
 
     @classmethod
@@ -120,7 +147,7 @@ class NodePool:
             dl_types=_json.get('dlTypes'),
             description=_json.get('description'),
             node_selector=_json.get('nodeSelector'),
-            preemtible=_json.get('preemtible'),
+            preemptible=_json.get('preemptible'),
             deployment_resources=DeploymentResources.from_json(_json.get('deploymentResources', dict())),
             tolerations=[Toleration.from_json(t) for t in _json.get('tolerations', list())]
         )
@@ -133,7 +160,7 @@ class NodePool:
             'isDlTypeDefault': self.is_dl_type_default,
             'description': self.description,
             'nodeSelector': self.node_selector,
-            'preemtible': self.preemtible,
+            'preemptible': self.preemptible,
             'deploymentResources': self.deployment_resources.to_json(),
             'tolerations': [t.to_json() for t in self.tolerations]
         }
@@ -271,6 +298,8 @@ class Compute:
             type: ComputeType = ComputeType.KUBERNETES,
             features: Optional[Dict] = None,
             metadata: Optional[Dict] = None,
+            settings: Optional[ComputeSettings] = None,
+            url: Optional[str] = None
     ):
         self.id = id
         self.name = name
@@ -284,6 +313,8 @@ class Compute:
         self._client_api = client_api
         self._computes = None
         self._serviceDrivers = None
+        self.settings = settings
+        self.url = url
 
     @property
     def computes(self):
@@ -291,18 +322,30 @@ class Compute:
             self._computes = repositories.Computes(client_api=self._client_api)
         return self._computes
 
-    @property
-    def service_drivers(self):
-        if self._serviceDrivers is None:
-            self._serviceDrivers = repositories.ServiceDrivers(client_api=self._client_api)
-        return self._serviceDrivers
-
     def delete(self):
         return self.computes.delete(compute_id=self.id)
 
     def update(self):
         return self.computes.update(compute=self)
 
+    @staticmethod
+    def _protected_from_json(_json: dict, client_api: ApiClient):
+        """
+        Same as from_json but with try-except to catch if error
+
+        :param _json: platform json
+        :param client_api: ApiClient entity
+        :return:
+        """
+        try:
+            compute = Compute.from_json(_json=_json,
+                                        client_api=client_api)
+            status = True
+        except Exception:
+            compute = traceback.format_exc()
+            status = False
+        return status, compute
+
     @classmethod
     def from_json(cls, _json, client_api: ApiClient):
         return cls(
@@ -315,19 +358,24 @@ class Compute:
             type=ComputeType(_json.get('type')),
             features=_json.get('features'),
             client_api=client_api,
-            metadata=_json.get('metadata')
+            metadata=_json.get('metadata'),
+            settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
+            url=_json.get('url'),
         )
 
     def to_json(self):
         return {
             'id': self.id,
+            'name': self.name,
             'context': self.context.to_json(),
             'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
             'global': self.global_,
             'status': self.status.value,
             'type': self.type.value,
             'features': self.features,
-            'metadata': self.metadata
+            'metadata': self.metadata,
+            'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
+            'url': self.url
         }
 
 
@@ -335,6 +383,7 @@ class KubernetesCompute(Compute):
     def __init__(
             self,
             id: str,
+            name: str,
             context: ComputeContext,
             cluster: ComputeCluster,
             shared_contexts: Optional[List[ComputeContext]] = None,
@@ -343,16 +392,20 @@ class KubernetesCompute(Compute):
             type: ComputeType = ComputeType.KUBERNETES,
             features: Optional[Dict] = None,
             metadata: Optional[Dict] = None,
-            client_api: ApiClient = None
+            client_api: ApiClient = None,
+            settings: Optional[ComputeSettings] = None,
+            url: Optional[str] = None
     ):
         super().__init__(id=id, context=context, shared_contexts=shared_contexts, global_=global_, status=status,
-                         type=type, features=features, metadata=metadata, client_api=client_api)
+                         type=type, features=features, metadata=metadata, client_api=client_api, settings=settings,
+                         name=name, url=url)
         self.cluster = cluster
 
     @classmethod
     def from_json(cls, _json, client_api: ApiClient):
         return cls(
             id=_json.get('id'),
+            name=_json.get('name'),
             context=ComputeContext.from_json(_json.get('context', dict())),
             cluster=ComputeCluster.from_json(_json.get('cluster', dict())),
             shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
@@ -361,93 +414,23 @@ class KubernetesCompute(Compute):
             type=ComputeType(_json.get('type')),
             features=_json.get('features'),
             metadata=_json.get('metadata'),
-            client_api=client_api
+            client_api=client_api,
+            settings=ComputeSettings.from_json(_json.get('settings', dict())) if _json.get('settings') else None,
+            url=_json.get('url'),
         )
 
     def to_json(self):
         return {
             'id': self.id,
+            'name': self.name,
             'context': self.context.to_json(),
             'cluster': self.cluster.to_json(),
             'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
             'global': self.global_,
             'status': self.status.value,
             'type': self.type.value,
-            'features': self.features
-        }
-
-
-class ServiceDriver:
-    def __init__(
-            self,
-            name: str,
-            context: ComputeContext,
-            compute_id: str,
-            client_api: ApiClient,
-            type: ComputeType = None,
-            created_at: str = None,
-            updated_at: str = None,
-            namespace: str = None,
-            metadata: Dict = None,
-            url: str = None,
-            archived: bool = None,
-            id: str = None,
-            is_cache_available: bool = None
-    ):
-        self.name = name
-        self.context = context
-        self.compute_id = compute_id
-        self.client_api = client_api
-        self.type = type or ComputeType.KUBERNETES
-        self.created_at = created_at
-        self.updated_at = updated_at
-        self.namespace = namespace
-        self.metadata = metadata
-        self.url = url
-        self.archived = archived
-        self.id = id
-        self.is_cache_available = is_cache_available
-
-    @classmethod
-    def from_json(cls, _json, client_api: ApiClient):
-        return cls(
-            name=_json.get('name'),
-            context=ComputeContext.from_json(_json.get('context', dict())),
-            compute_id=_json.get('computeId'),
-            client_api=client_api,
-            type=_json.get('type', None),
-            created_at=_json.get('createdAt', None),
-            updated_at=_json.get('updatedAt', None),
-            namespace=_json.get('namespace', None),
-            metadata=_json.get('metadata', None),
-            url=_json.get('url', None),
-            archived=_json.get('archived', None),
-            id=_json.get('id', None),
-            is_cache_available=_json.get('isCacheAvailable', None)
-        )
-
-    def to_json(self):
-        _json = {
-            'name': self.name,
-            'context': self.context.to_json(),
-            'computeId': self.compute_id,
-            'type': self.type,
+            'features': self.features,
+            'metadata': self.metadata,
+            'settings': self.settings.to_json() if isinstance(self.settings, ComputeSettings) else self.settings,
+            'url': self.url
         }
-        if self.created_at is not None:
-            _json['createdAt'] = self.namespace
-        if self.updated_at is not None:
-            _json['updatedAt'] = self.updated_at
-        if self.namespace is not None:
-            _json['namespace'] = self.namespace
-        if self.metadata is not None:
-            _json['metadata'] = self.metadata
-        if self.url is not None:
-            _json['url'] = self.url
-        if self.archived is not None:
-            _json['archived'] = self.archived
-        if self.id is not None:
-            _json['id'] = self.id
-        if self.is_cache_available is not None:
-            _json['isCacheAvailable'] = self.is_cache_available
-
-        return _json