dtlpy 1.102.14__py3-none-any.whl → 1.104.14__py3-none-any.whl

dtlpy/__init__.py

@@ -169,6 +169,7 @@ messages = repositories.Messages(client_api=client_api)
 compositions = repositories.Compositions(client_api=client_api)
 computes = repositories.Computes(client_api=client_api)
 service_drivers = repositories.ServiceDrivers(client_api=client_api)
+collections = repositories.Collections(client_api=client_api)
 
 try:
     check_sdk.check(version=__version__, client_api=client_api)

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.102.14'
+version = '1.104.14'

dtlpy/entities/__init__.py

@@ -79,3 +79,4 @@ from .compute import ClusterProvider, ComputeType, ComputeStatus, Toleration, De
     NodePool, AuthenticationIntegration, Authentication, ComputeCluster, ComputeContext, Compute, KubernetesCompute, \
     ServiceDriver
 from .gis_item import  ItemGis, Layer
+from .collection import Collection

dtlpy/entities/annotation.py

@@ -7,6 +7,7 @@ import copy
 import attr
 import json
 import os
+import warnings
 
 from PIL import Image
 from enum import Enum
@@ -421,7 +422,7 @@ class Annotation(entities.BaseEntity):
 
     @property
     def attributes(self):
-        if self._recipe_2_attributes or not self.annotation_definition.attributes:
+        if self._recipe_2_attributes is not None or self.annotation_definition.attributes == []:
             return self._recipe_2_attributes
         return self.annotation_definition.attributes
 
@@ -430,6 +431,11 @@ class Annotation(entities.BaseEntity):
         if isinstance(attributes, dict):
             self._recipe_2_attributes = attributes
         elif isinstance(attributes, list):
+            warnings.warn("List attributes are deprecated and will be removed in version 1.109. Use Attribute 2.0 (Dictionary) instead. "
+                "For more details, refer to the documentation: "
+                "https://developers.dataloop.ai/tutorials/data_management/upload_and_manage_annotations/chapter/#set-attributes-on-annotations",
+                DeprecationWarning,
+            )
             self.annotation_definition.attributes = attributes
         elif attributes is None:
             if self._recipe_2_attributes:
@@ -1095,7 +1101,7 @@ class Annotation(entities.BaseEntity):
         if annotation_definition:
             res.annotation_definition = annotation_definition
 
-        if annotation_definition and annotation_definition.attributes:
+        if annotation_definition and annotation_definition.attributes is not None:
             res.attributes = annotation_definition.attributes
 
         return res
@@ -1679,7 +1685,7 @@ class FrameAnnotation(entities.BaseEntity):
 
     @property
     def attributes(self):
-        if self._recipe_2_attributes or not self.annotation_definition.attributes:
+        if self._recipe_2_attributes or self.annotation_definition.attributes == []:
             return self._recipe_2_attributes
         return self.annotation_definition.attributes
 
@@ -1688,6 +1694,11 @@ class FrameAnnotation(entities.BaseEntity):
         if isinstance(attributes, dict):
             self._recipe_2_attributes = attributes
         elif isinstance(attributes, list):
+            warnings.warn("List attributes are deprecated and will be removed in version 1.109. Use Attribute 2.0 (Dictionary) instead. "
+                "For more details, refer to the documentation: "
+                "https://developers.dataloop.ai/tutorials/data_management/upload_and_manage_annotations/chapter/#set-attributes-on-annotations",
+                DeprecationWarning,
+            )
             self.annotation_definition.attributes = attributes
         else:
             raise ValueError('Attributes must be a dictionary or a list')
@@ -1873,6 +1884,9 @@ class FrameAnnotation(entities.BaseEntity):
             'data': self.coordinates
         }
 
+        if self.annotation_definition.description is not None:
+            snapshot_dict['description'] = self.annotation_definition.description
+
         if self.annotation._recipe_2_attributes:
             snapshot_dict['namedAttributes'] = self._recipe_2_attributes
         else:

dtlpy/entities/annotation_definitions/base_annotation_definition.py

@@ -1,5 +1,6 @@
 import logging
 import numpy as np
+import warnings
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -13,8 +14,12 @@ class BaseAnnotationDefinition:
         self._right = 0
         self._annotation = None
 
-        if attributes is None:
-            attributes = list()
+        if isinstance(attributes, list) and len(attributes) > 0:
+            warnings.warn("List attributes are deprecated and will be removed in version 1.109. Use Attribute 2.0 (Dictionary) instead."
+                "For more details, refer to the documentation: "
+                "https://developers.dataloop.ai/tutorials/data_management/upload_and_manage_annotations/chapter/#set-attributes-on-annotations",
+                DeprecationWarning,
+            )
         self._attributes = attributes
 
     @property
@@ -23,8 +28,12 @@ class BaseAnnotationDefinition:
 
     @attributes.setter
     def attributes(self, v):
-        if not isinstance(v, list):
-            raise ValueError('Failed to update annotation attributes. Please use annotation.attrubeute to set the required values')
+        if isinstance(v, list):
+            warnings.warn("List attributes are deprecated and will be removed in version 1.109. Use Attribute 2.0 (Dictionary) instead. "
+                "For more details, refer to the documentation: "
+                "https://developers.dataloop.ai/tutorials/data_management/upload_and_manage_annotations/chapter/#set-attributes-on-annotations",
+                DeprecationWarning,
+            )
         self._attributes = v
     @property
     def top(self):

dtlpy/entities/collection.py

@@ -0,0 +1,39 @@
+from .. import entities
+from ..services.api_client import ApiClient
+import attr
+
+@attr.s
+class Collection(entities.BaseEntity):
+    """
+    Represents a collection in the dataset.
+    """
+
+    # sdk
+    _client_api = attr.ib(type=ApiClient, repr=False)
+
+    key = attr.ib(type=str)
+    name = attr.ib(type=str)
+
+    @classmethod
+    def from_json(cls, _json, client_api, is_fetched=True):
+        """
+        Create a single Collection entity from the dataset JSON.
+
+        :param _json: A dictionary containing collection data in the format:
+                    { "metadata.system.collections.c0": {"name": "Justice League"} }
+        :param client_api: The client API instance.
+        :param is_fetched: Whether the entity was fetched from the platform.
+        :return: A single Collection entity.
+        """
+        full_key, value = next(iter(_json.items()))
+        # Strip the prefix
+        key = full_key.replace("metadata.system.collections.", "")
+        name = value.get("name")
+
+        inst = cls(
+            key=key,
+            name=name,
+            client_api=client_api,
+        )
+        inst.is_fetched = is_fetched
+        return inst

dtlpy/entities/command.py

@@ -18,6 +18,8 @@ class CommandsStatus(str, Enum):
     SUCCESS = 'success'
     FAILED = 'failed'
     TIMEOUT = 'timeout'
+    CLEANING_UP = 'cleaning-up'
+    ON_ERROR = 'on-error'
 
 
 @attr.s
@@ -135,11 +137,14 @@ class Command(entities.BaseEntity):
         :return: True if command still in progress
         :rtype: bool
         """
-        return self.status in [entities.CommandsStatus.CREATED,
-                               entities.CommandsStatus.MAKING_CHILDREN,
-                               entities.CommandsStatus.WAITING_CHILDREN,
-                               entities.CommandsStatus.FINALIZING,
-                               entities.CommandsStatus.IN_PROGRESS]
+        if self.status not in {status for status in entities.CommandsStatus}:
+            raise ValueError('status is not a valid CommandsStatus')
+        return self.status not in [entities.CommandsStatus.SUCCESS,
+                                   entities.CommandsStatus.FAILED,
+                                   entities.CommandsStatus.TIMEOUT,
+                                   entities.CommandsStatus.CANCELED,
+                                   entities.CommandsStatus.ABORTED
+                                   ]
 
     def wait(self, timeout=0, step=None, backoff_factor=1):
         """

dtlpy/entities/compute.py

@@ -230,10 +230,11 @@ class ComputeCluster:
             devops_output['config']['kubernetesVersion'],
             ClusterProvider(devops_output['config']['provider']),
             node_pools,
-   {},
-            Authentication(AuthenticationIntegration(integration.id,integration.type))
+            {},
+            Authentication(AuthenticationIntegration(integration.id, integration.type))
         )
 
+
 class ComputeContext:
     def __init__(self, labels: List[str], org: str, project: Optional[str] = None):
         self.labels = labels
@@ -376,11 +377,35 @@ class KubernetesCompute(Compute):
 
 
 class ServiceDriver:
-    def __init__(self, name: str, context: ComputeContext, compute_id: str, client_api: ApiClient):
+    def __init__(
+            self,
+            name: str,
+            context: ComputeContext,
+            compute_id: str,
+            client_api: ApiClient,
+            type: ComputeType = None,
+            created_at: str = None,
+            updated_at: str = None,
+            namespace: str = None,
+            metadata: Dict = None,
+            url: str = None,
+            archived: bool = None,
+            id: str = None,
+            is_cache_available: bool = None
+    ):
         self.name = name
         self.context = context
         self.compute_id = compute_id
         self.client_api = client_api
+        self.type = type or ComputeType.KUBERNETES
+        self.created_at = created_at
+        self.updated_at = updated_at
+        self.namespace = namespace
+        self.metadata = metadata
+        self.url = url
+        self.archived = archived
+        self.id = id
+        self.is_cache_available = is_cache_available
 
     @classmethod
     def from_json(cls, _json, client_api: ApiClient):
@@ -388,12 +413,40 @@ class ServiceDriver:
             name=_json.get('name'),
             context=ComputeContext.from_json(_json.get('context', dict())),
             compute_id=_json.get('computeId'),
-            client_api=client_api
+            client_api=client_api,
+            type=_json.get('type', None),
+            created_at=_json.get('createdAt', None),
+            updated_at=_json.get('updatedAt', None),
+            namespace=_json.get('namespace', None),
+            metadata=_json.get('metadata', None),
+            url=_json.get('url', None),
+            archived=_json.get('archived', None),
+            id=_json.get('id', None),
+            is_cache_available=_json.get('isCacheAvailable', None)
         )
 
     def to_json(self):
-        return {
+        _json = {
             'name': self.name,
             'context': self.context.to_json(),
-            'computeId': self.compute_id
+            'computeId': self.compute_id,
+            'type': self.type,
         }
+        if self.created_at is not None:
+            _json['createdAt'] = self.namespace
+        if self.updated_at is not None:
+            _json['updatedAt'] = self.updated_at
+        if self.namespace is not None:
+            _json['namespace'] = self.namespace
+        if self.metadata is not None:
+            _json['metadata'] = self.metadata
+        if self.url is not None:
+            _json['url'] = self.url
+        if self.archived is not None:
+            _json['archived'] = self.archived
+        if self.id is not None:
+            _json['id'] = self.id
+        if self.is_cache_available is not None:
+            _json['isCacheAvailable'] = self.is_cache_available
+
+        return _json

dtlpy/entities/dataset.py

@@ -284,13 +284,11 @@ class Dataset(entities.BaseEntity):
     def set_repositories(self):
         reps = namedtuple('repositories',
                           field_names=['items', 'recipes', 'datasets', 'assignments', 'tasks', 'annotations',
-                                       'ontologies', 'features', 'settings', 'schema'])
+                                       'ontologies', 'features', 'settings', 'schema', 'collections'])
         if self._project is None:
             datasets = repositories.Datasets(client_api=self._client_api, project=self._project)
-            features = repositories.Features(client_api=self._client_api, project=self._project)
         else:
             datasets = self._project.datasets
-            features = self._project.features
 
         return reps(
             items=repositories.Items(client_api=self._client_api, dataset=self, datasets=datasets),
@@ -300,9 +298,10 @@ class Dataset(entities.BaseEntity):
             annotations=repositories.Annotations(client_api=self._client_api, dataset=self),
             datasets=datasets,
             ontologies=repositories.Ontologies(client_api=self._client_api, dataset=self),
-            features=features,
+            features=repositories.Features(client_api=self._client_api, project=self._project, dataset=self),
             settings=repositories.Settings(client_api=self._client_api, dataset=self),
-            schema=repositories.Schema(client_api=self._client_api, dataset=self)
+            schema=repositories.Schema(client_api=self._client_api, dataset=self),
+            collections=repositories.Collections(client_api=self._client_api, dataset=self)
         )
 
     @property
@@ -350,6 +349,11 @@ class Dataset(entities.BaseEntity):
         assert isinstance(self._repositories.features, repositories.Features)
         return self._repositories.features
 
+    @property
+    def collections(self):
+        assert isinstance(self._repositories.collections, repositories.Collections)
+        return self._repositories.collections
+
     @property
     def schema(self):
         assert isinstance(self._repositories.schema, repositories.Schema)

dtlpy/entities/dpk.py

@@ -31,8 +31,8 @@ DEFAULT_STOPS = {SlotType.ITEM_VIEWER: {"type": "itemViewer",
 
 
 class Slot(entities.DlEntity):
-    type = entities.DlProperty(location=['type'], _type=str)
-    configuration = entities.DlProperty(location=['configuration'], _type=dict)
+    type: str = entities.DlProperty(location=['type'], _type=str)
+    configuration: dict = entities.DlProperty(location=['configuration'], _type=dict)
 
     def to_json(self) -> dict:
         return self._dict.copy()
@@ -59,10 +59,10 @@ class Toolbar(entities.DlEntity):
 
 
 class Panel(entities.DlEntity):
-    name = entities.DlProperty(location=['name'], _type=str)
-    path = entities.DlProperty(location=['path'], _type=str, default=None)
-    min_role = entities.DlProperty(location=['minRole'], _type=list)
-    supported_slots = entities.DlProperty(location=['supportedSlots'], _type=list)
+    name: str = entities.DlProperty(location=['name'], _type=str)
+    path: str = entities.DlProperty(location=['path'], _type=str, default=None)
+    min_role: list = entities.DlProperty(location=['minRole'], _type=list)
+    supported_slots: list = entities.DlProperty(location=['supportedSlots'], _type=list)
 
     metadata = entities.DlProperty(location=['metadata'], _type=list)
     default_settings = entities.DlProperty(location=['defaultSettings'], _type=list)
@@ -233,15 +233,15 @@ class Dpk(entities.DlEntity):
     base_id: str = entities.DlProperty(location=['baseId'], _type=str)
     name: str = entities.DlProperty(location=['name'], _type=str)
     version: str = entities.DlProperty(location=['version'], _type=str)
-    attributes: list = entities.DlProperty(location=['attributes'], _type=dict)
+    attributes: dict = entities.DlProperty(location=['attributes'], _type=dict)
     created_at: str = entities.DlProperty(location=['createdAt'], _type=str)
     updated_at: str = entities.DlProperty(location=['updatedAt'], _type=str)
     creator: str = entities.DlProperty(location=['creator'], _type=str)
     display_name: str = entities.DlProperty(location=['displayName'], _type=str)
     icon: str = entities.DlProperty(location=['icon'], _type=str)
     tags: list = entities.DlProperty(location=['tags'], _type=list)
-    codebase: str = entities.DlProperty(location=['codebase'], _kls="Codebase")
-    scope: dict = entities.DlProperty(location=['scope'], _type=str)
+    codebase: Union[entities.Codebase, None] = entities.DlProperty(location=['codebase'], _kls="Codebase")
+    scope: str = entities.DlProperty(location=['scope'], _type=str)
     context: dict = entities.DlProperty(location=['context'], _type=dict)
     metadata: dict = entities.DlProperty(location=['metadata'], _type=dict)
     dependencies: dict = entities.DlProperty(location=['dependencies'], _type=List[dict])

dtlpy/entities/execution.py

@@ -63,6 +63,8 @@ class Execution(entities.BaseEntity):
     # optional
     pipeline = attr.ib(type=dict, default=None, repr=False)
     model = attr.ib(type=dict, default=None, repr=False)
+    app = attr.ib(default=None)
+    driver_id = attr.ib(default=None)
 
     ################
     # repositories #
@@ -189,6 +191,8 @@ class Execution(entities.BaseEntity):
             pipeline=_json.get('pipeline', None),
             model=_json.get('model', None),
             package_revision=_json.get('packageRevision', None),
+            app=_json.get('app', None),
+            driver_id=_json.get('driverId', None)
         )
         inst.is_fetched = is_fetched
         return inst
@@ -226,6 +230,7 @@ class Execution(entities.BaseEntity):
                 attr.fields(Execution).pipeline,
                 attr.fields(Execution).model,
                 attr.fields(Execution).package_revision,
+                attr.fields(Execution).driver_id,
             )
         )
 
@@ -247,6 +252,7 @@ class Execution(entities.BaseEntity):
         _json['feedbackQueue'] = self.feedback_queue
         _json['syncReplyTo '] = self.sync_reply_to
         _json['packageRevision'] = self.package_revision
+        _json['driverId'] = self.driver_id
 
         if self.pipeline:
             _json['pipeline'] = self.pipeline

dtlpy/entities/filters.py

@@ -497,8 +497,8 @@ class Filters:
         _json["resource"] = f'{_json["resource"]}'
         # convert the dictionary to a json string
         _json['dqlFilter'] = json.dumps({'filter': _json.pop('filter'),
-                                         'join': _json.pop('join'),
-                                         'sort': _json.get('sort')})
+                                         'join': _json.pop('join', None),
+                                         'sort': _json.get('sort', None)})
         # set the page size as the UI default
         _json['pageSize'] = 100
         _json['page'] = _json['page']

dtlpy/entities/integration.py

@@ -1,5 +1,4 @@
 from enum import Enum
-
 import logging
 import attr
 

dtlpy/entities/item.py

@@ -12,6 +12,7 @@ from .annotation import ViewAnnotationOptions, ExportVersion
 from ..services.api_client import ApiClient
 from ..services.api_client import client as client_api
 import json
+from typing import List
 import requests
 
 logger = logging.getLogger(name='dtlpy')
@@ -223,7 +224,7 @@ class Item(entities.BaseEntity):
     def set_repositories(self):
         reps = namedtuple('repositories',
                           field_names=['annotations', 'datasets', 'items', 'codebases', 'artifacts', 'modalities',
-                                       'features', 'assignments', 'tasks', 'resource_executions'])
+                                       'features', 'assignments', 'tasks', 'resource_executions', 'collections'])
         reps.__new__.__defaults__ = (None, None, None, None, None, None, None, None, None)
 
         if self._dataset is None:
@@ -270,7 +271,8 @@ class Item(entities.BaseEntity):
                 client_api=self._client_api,
                 project=self._project,
                 resource=self
-            )
+            ),
+            collections=repositories.Collections(client_api=self._client_api, item=self, dataset=self._dataset)
         )
         return r
 
@@ -313,6 +315,11 @@ class Item(entities.BaseEntity):
     def features(self):
         assert isinstance(self._repositories.features, repositories.Features)
         return self._repositories.features
+    
+    @property
+    def collections(self):
+        assert isinstance(self._repositories.collections, repositories.Collections)
+        return self._repositories.collections
 
     ##############
     # Properties #
@@ -770,6 +777,53 @@ class Item(entities.BaseEntity):
             if tags.get(subset) is True:
                 return subset
         return None
+    
+    def assign_collection(self, collections: List[str]) -> bool:
+        """
+        Assign this item to one or more collections.
+
+        :param collections: List of collection names to assign the item to.
+        :return: True if the assignment was successful, otherwise False.
+        """
+        return self.collections.assign(dataset_id=self.dataset_id, collections=collections, item_id=self.id,)
+
+    def unassign_collection(self, collections: List[str]) -> bool:
+        """
+        Unassign this item from one or more collections.
+
+        :param collections: List of collection names to unassign the item from.
+        :return: True if the unassignment was successful, otherwise False.
+        """
+        return self.collections.unassign(dataset_id=self.dataset_id, item_id=self.id, collections=collections)
+
+    def list_collections(self) -> List[dict]:
+        """
+        List all collections associated with this item.
+
+        :return: A list of dictionaries containing collection keys and their respective names.
+                Each dictionary has the structure: {"key": <collection_key>, "name": <collection_name>}.
+        """
+        collections = self.metadata.get("system", {}).get("collections", {})
+        if not isinstance(collections, dict):
+            # Ensure collections is a dictionary
+            return []
+    
+        # Retrieve collection names by their keys
+        return [
+            {"key": key, "name": self.collections.get_name_by_key(key)}
+            for key in collections.keys()
+        ]
+
+    def list_missing_collections(self) -> List[str]:
+        """
+        List all items in the dataset that are not assigned to any collection.
+
+        :return: A list of item IDs that are not part of any collection.
+        """
+        filters = entities.Filters()
+        filters.add(field='metadata.system.collections', values=None)
+        filters.add(field='datasetId', values=self._dataset.id)
+        return self._dataset.items.list(filters=filters)
 
 class ModalityTypeEnum(str, Enum):
     """

dtlpy/entities/organization.py

@@ -49,7 +49,7 @@ class Organization(entities.BaseEntity):
     logo_url = attr.ib(repr=False)
     plan = attr.ib(repr=False)
     owner = attr.ib(repr=False)
-    created_by = attr.ib(repr=False)
+    creator = attr.ib(repr=False)
 
     # api
     _client_api = attr.ib(type=ApiClient, repr=False)
@@ -67,7 +67,7 @@ class Organization(entities.BaseEntity):
 
     @property
     def createdBy(self):
-        return self.created_by
+        return self.creator
 
     @_repositories.default
     def set_repositories(self):
@@ -158,7 +158,7 @@ class Organization(entities.BaseEntity):
                    logo_url=_json.get('logoUrl', None),
                    plan=_json.get('plan', None),
                    owner=_json.get('owner', None),
-                   created_by=_json.get('createdBy', None),
+                   creator=_json.get('creator', None),
                    client_api=client_api)
         inst.is_fetched = is_fetched
         return inst
@@ -175,7 +175,7 @@ class Organization(entities.BaseEntity):
                                                               attr.fields(Organization)._repositories,
                                                               attr.fields(Organization).created_at,
                                                               attr.fields(Organization).updated_at,
-                                                              attr.fields(Organization).created_by,
+                                                              attr.fields(Organization).creator,
                                                               ))
         output_dict['members'] = self.members
         output_dict['groups'] = self.groups
@@ -188,7 +188,7 @@ class Organization(entities.BaseEntity):
         output_dict['logo_url'] = self.logo_url
         output_dict['plan'] = self.plan
         output_dict['owner'] = self.owner
-        output_dict['createdBy'] = self.created_by
+        output_dict['creator'] = self.creator
 
         return output_dict
 

dtlpy/ml/base_model_adapter.py

@@ -488,7 +488,8 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
             try:
                 self.model_entity.project.feature_sets.get(feature_set_name=self.model_entity.name)
                 feature_set_name = f"{self.model_entity.name}-{''.join(random.choices(string.ascii_letters + string.digits, k=5))}"
-                logger.warning(f"Feature set with the model name already exists. Creating new feature set with name {feature_set_name}")
+                logger.warning(
+                    f"Feature set with the model name already exists. Creating new feature set with name {feature_set_name}")
             except exceptions.NotFound:
                 feature_set_name = self.model_entity.name
             feature_set = self.model_entity.project.feature_sets.create(name=feature_set_name,
@@ -518,7 +519,7 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
                 self.logger.debug(
                     "Uploading items' feature vectors for model {!r}.".format(self.model_entity.name))
                 try:
-                    _ = list(pool.map(partial(self._upload_model_features,
+                    list(pool.map(partial(self._upload_model_features,
                                               feature_set.id,
                                               self.model_entity.project_id),
                                       batch_items,
@@ -762,14 +763,13 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
     @staticmethod
     def _upload_model_features(feature_set_id, project_id, item: entities.Item, vector):
         try:
-            feature = item.features.create(value=vector,
-                                           project_id=project_id,
-                                           feature_set_id=feature_set_id,
-                                           entity=item)
-            return feature
+            if vector is not None:
+                item.features.create(value=vector,
+                                     project_id=project_id,
+                                     feature_set_id=feature_set_id,
+                                     entity=item)
         except Exception as e:
             logger.error(f'Failed to upload feature vector of length {len(vector)} to item {item.id}, Error: {e}')
-            return []
 
     def _upload_model_annotations(self, item: entities.Item, predictions, clean_annotations):
         """

dtlpy/repositories/__init__.py

@@ -52,3 +52,4 @@ from .messages import Messages
 from .compositions import Compositions
 from .schema import Schema
 from .computes import Computes, ServiceDrivers
+from .collections import Collections