dtlpy 1.108.7__py3-none-any.whl → 1.109.19__py3-none-any.whl

dtlpy/entities/filters.py

@@ -46,7 +46,9 @@ class FiltersResource(str, Enum):
     DRIVERS = 'drivers'
     SETTINGS = 'setting'
     RESOURCE_EXECUTION = 'resourceExecution'
-    METRICS = 'metrics'
+    METRICS = 'metrics',
+    SERVICE_DRIVER = 'serviceDrivers',
+    COMPUTE = 'compute'
 
 
 class FiltersOperations(str, Enum):

dtlpy/entities/item.py

@@ -455,7 +455,8 @@ class Item(entities.BaseEntity):
             alpha=1,
             export_version=ExportVersion.V1,
             dataset_lock=False,
-            lock_timeout_sec=None
+            lock_timeout_sec=None,
+            export_summary=False,
     ):
         """
         Download dataset by filters.
@@ -470,6 +471,7 @@ class Item(entities.BaseEntity):
         :param dtlpy.entities.filters.Filters annotation_filters: Filters entity to filter annotations for download
         :param bool overwrite: optional - default = False
         :param bool dataset_lock: optional - default = False
+        :param bool export_summary: optional - default = False
         :param int lock_timeout_sec: optional
         :param bool to_items_folder: Create 'items' folder and download items to it
         :param int thickness: optional - line thickness, if -1 annotation will be filled, default =1
@@ -491,7 +493,8 @@ class Item(entities.BaseEntity):
                          alpha=1,
                          save_locally=True,
                          dataset_lock=False
-                         lock_timeout_sec=300
+                         lock_timeout_sec=300,
+                         export_summary=False
                          )
         """
         # if dir - concatenate local path and item name
@@ -527,7 +530,8 @@ class Item(entities.BaseEntity):
                                    export_version=export_version,
                                    filters=filters,
                                    dataset_lock=dataset_lock,
-                                   lock_timeout_sec=lock_timeout_sec)
+                                   lock_timeout_sec=lock_timeout_sec,
+                                   export_summary=export_summary)
 
     def delete(self):
         """
@@ -821,17 +825,6 @@ class Item(entities.BaseEntity):
             {"key": key, "name": self.collections.get_name_by_key(key)}
             for key in collections.keys()
         ]
-
-    def list_missing_collections(self) -> List[str]:
-        """
-        List all items in the dataset that are not assigned to any collection.
-
-        :return: A list of item IDs that are not part of any collection.
-        """
-        filters = entities.Filters()
-        filters.add(field='metadata.system.collections', values=None)
-        filters.add(field='datasetId', values=self._dataset.id)
-        return self._dataset.items.list(filters=filters)
     
     def task_scores(self, task_id: str, page_offset: int = None, page_size: int = None):
         """

dtlpy/entities/node.py

@@ -91,7 +91,6 @@ class PipelineNodeIO:
                  port_id: str = None,
                  color: tuple = None,
                  port_percentage: int = None,
-                 action: str = None,
                  default_value=None,
                  variable_name: str = None,
                  actions: list = None,
@@ -118,19 +117,8 @@ class PipelineNodeIO:
         self.default_value = default_value
         self.variable_name = variable_name
         self.description = description
-
-        if action is not None:
-            warnings.warn('action param has been deprecated in version 1.95', DeprecationWarning)
-            if actions is None:
-                actions = []
-            actions.append(action)
         self.actions = actions
 
-    @property
-    def action(self):
-        warnings.warn('action attribute has been deprecated in version 1.95', DeprecationWarning)
-        return None
-
     @staticmethod
     def from_json(_json: dict):
         return PipelineNodeIO(

dtlpy/entities/service.py

@@ -869,20 +869,11 @@ class KubernetesAutoscalerType(str, Enum):
 class KubernetesAutuscalerTypeMeta(type):
     def __getattribute__(cls, item):
         if hasattr(KubernetesAutoscalerType, item):
-            warnings.warn(
-                'KubernetesAutuscalerType is deprecated and will be removed in version 1.97.0, '
-                'use KubernetesAutoscalerType instead',
-                DeprecationWarning
-            )
             return getattr(KubernetesAutoscalerType, item)
         else:
             raise AttributeError(f"KubernetesAutuscalerType has no attribute '{item}'")
 
 
-class KubernetesAutuscalerType(metaclass=KubernetesAutuscalerTypeMeta):
-    pass
-
-
 class KubernetesAutoscaler(entities.BaseEntity):
     MIN_REPLICA_DEFAULT = 0
     MAX_REPLICA_DEFAULT = 1

dtlpy/entities/service_driver.py

@@ -0,0 +1,118 @@
+import traceback
+from typing import Dict
+from ..services.api_client import ApiClient
+from .. import repositories
+from .compute import ComputeContext, ComputeType
+
+class ServiceDriver:
+    def __init__(
+            self,
+            name: str,
+            context: ComputeContext,
+            compute_id: str,
+            client_api: ApiClient,
+            type: ComputeType = None,
+            created_at: str = None,
+            updated_at: str = None,
+            namespace: str = None,
+            metadata: Dict = None,
+            url: str = None,
+            archived: bool = None,
+            id: str = None,
+            is_cache_available: bool = None
+    ):
+        self.name = name
+        self.context = context
+        self.compute_id = compute_id
+        self.client_api = client_api
+        self.type = type or ComputeType.KUBERNETES
+        self.created_at = created_at
+        self.updated_at = updated_at
+        self.namespace = namespace
+        self.metadata = metadata
+        self.url = url
+        self.archived = archived
+        self.id = id
+        self.is_cache_available = is_cache_available
+        self._service_drivers = None
+        self._client_api = client_api
+
+    @property
+    def service_drivers(self):
+        if self._service_drivers is None:
+            self._service_drivers = repositories.ServiceDrivers(client_api=self._client_api)
+        return self._service_drivers
+
+    @staticmethod
+    def _protected_from_json(_json: dict, client_api: ApiClient):
+        """
+        Same as from_json but with try-except to catch if error
+
+        :param _json: platform json
+        :param client_api: ApiClient entity
+        :return:
+        """
+        try:
+            service = ServiceDriver.from_json(_json=_json,
+                                              client_api=client_api)
+            status = True
+        except Exception:
+            service = traceback.format_exc()
+            status = False
+        return status, service
+
+    @classmethod
+    def from_json(cls, _json, client_api: ApiClient):
+        return cls(
+            name=_json.get('name'),
+            context=ComputeContext.from_json(_json.get('context', dict())),
+            compute_id=_json.get('computeId'),
+            client_api=client_api,
+            type=_json.get('type', None),
+            created_at=_json.get('createdAt', None),
+            updated_at=_json.get('updatedAt', None),
+            namespace=_json.get('namespace', None),
+            metadata=_json.get('metadata', None),
+            url=_json.get('url', None),
+            archived=_json.get('archived', None),
+            id=_json.get('id', None),
+            is_cache_available=_json.get('isCacheAvailable', None)
+        )
+
+    def to_json(self):
+        _json = {
+            'name': self.name,
+            'context': self.context.to_json(),
+            'computeId': self.compute_id,
+            'type': self.type,
+        }
+        if self.created_at is not None:
+            _json['createdAt'] = self.created_at
+        if self.updated_at is not None:
+            _json['updatedAt'] = self.updated_at
+        if self.namespace is not None:
+            _json['namespace'] = self.namespace
+        if self.metadata is not None:
+            _json['metadata'] = self.metadata
+        if self.url is not None:
+            _json['url'] = self.url
+        if self.archived is not None:
+            _json['archived'] = self.archived
+        if self.id is not None:
+            _json['id'] = self.id
+        if self.is_cache_available is not None:
+            _json['isCacheAvailable'] = self.is_cache_available
+
+        return _json
+
+    def delete(self):
+        """
+        Delete a service driver
+        """
+        return self.service_drivers.delete(service_driver_id=self.id)
+
+    def update(self):
+        """
+        Update a service driver
+        """
+        return self.service_drivers.update(service_driver=self)

dtlpy/entities/trigger.py

@@ -343,7 +343,7 @@ class Trigger(BaseTrigger):
             filters=spec.get('filter', dict()),
             project=project,
             service=service,
-            id=_json['id'],
+            id=_json.get('id', None),
             op_type=operation.get('type', None),
             spec=spec,
             pipeline_id=pipeline_id,

dtlpy/new_instance.py

@@ -22,7 +22,7 @@ class Dtlpy:
         # triggers
         TriggerResource, TriggerAction, TriggerExecutionMode, TriggerType,
         # faas
-        FunctionIO, KubernetesAutoscalerType, KubernetesAutuscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRuntime,
+        FunctionIO, KubernetesAutoscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRuntime,
         InstanceCatalog, PackageInputType, ServiceType, ServiceModeType,
         PackageSlot, SlotPostAction, SlotPostActionType, SlotDisplayScope, SlotDisplayScopeResource, UiBindingPanel,
         # roberto

dtlpy/repositories/__init__.py

@@ -51,5 +51,6 @@ from .dpks import Dpks
 from .messages import Messages
 from .compositions import Compositions
 from .schema import Schema
-from .computes import Computes, ServiceDrivers
+from .computes import Computes
+from .service_drivers import ServiceDrivers
 from .collections import Collections

dtlpy/repositories/collections.py

@@ -1,8 +1,5 @@
-from venv import logger
-from dtlpy import entities, exceptions, repositories
-from dtlpy.entities.dataset import Dataset
-from dtlpy.entities.filters import FiltersMethod
-from dtlpy.services.api_client import ApiClient
+from .. import entities, exceptions
+from ..services.api_client import ApiClient
 from typing import List
 
 class Collections:
@@ -15,6 +12,26 @@ class Collections:
         self._dataset = dataset
         self._item = item
 
+    @property
+    def dataset(self) -> entities.Dataset:
+        if self._dataset is None:
+            raise ValueError("Must set dataset for this action.")
+        return self._dataset
+    
+    @dataset.setter
+    def dataset(self, dataset: entities.Dataset):
+        self._dataset = dataset
+
+    @property
+    def item(self) -> entities.Item:
+        if self._item is None:
+            raise ValueError("Must set item for this action.")
+        return self._item
+    
+    @item.setter
+    def item(self, item: entities.Item):
+        self._item = item
+    
     def create(self, name: str) -> entities.Collection:
         """
         Creates a new collection in the dataset.
@@ -22,12 +39,11 @@ class Collections:
         :param name: The name of the new collection.
         :return: The created collection details.
         """
-        dataset_id = self._dataset.id
         self.validate_max_collections()
         self.validate_collection_name(name)
         payload = {"name": name}
         success, response = self._client_api.gen_request(
-            req_type="post", path=f"/datasets/{dataset_id}/items/collections", json_req=payload
+            req_type="post", path=f"/datasets/{self.dataset.id}/items/collections", json_req=payload
         )
         if success:
             collection_json = self._single_collection(data=response.json(), name=name)
@@ -43,11 +59,10 @@ class Collections:
         :param new_name: The new name for the collection.
         :return: The updated collection details.
         """
-        dataset_id = self._dataset.id
         self.validate_collection_name(new_name)
         payload = {"name": new_name}
         success, response = self._client_api.gen_request(
-            req_type="patch", path=f"/datasets/{dataset_id}/items/collections/{collection_name}", json_req=payload
+            req_type="patch", path=f"/datasets/{self.dataset.id}/items/collections/{collection_name}", json_req=payload
         )
         if success:
             collection_json = self._single_collection(data=response.json(), name=new_name)
@@ -61,9 +76,8 @@ class Collections:
 
         :param collection_name: The name of the collection to delete.
         """
-        dataset_id = self._dataset.id
         success, response = self._client_api.gen_request(
-            req_type="delete", path=f"/datasets/{dataset_id}/items/collections/{collection_name}"
+            req_type="delete", path=f"/datasets/{self.dataset.id}/items/collections/{collection_name}"
         )
         if success:
             # Wait for the split operation to complete
@@ -74,7 +88,7 @@ class Collections:
         else:
             raise exceptions.PlatformException(response)
 
-    def clone(self, collection_name: str) -> dict:
+    def clone(self, collection_name: str) -> entities.Collection:
         """
         Clones an existing collection, creating a new one with a unique name.
 
@@ -99,7 +113,10 @@ class Collections:
 
         # Create the cloned collection
         cloned_collection = self.create(name=clone_name)
-        self.assign(dataset_id=self._dataset.id, collections=[cloned_collection.name], collection_key=original_collection['key'])
+        filters = entities.Filters()
+        filters.add(field=f'metadata.system.collections.{original_collection["key"]}', values=True)
+        self.assign(collections=[cloned_collection.name], 
+                    filters=filters)
         return cloned_collection
 
 
@@ -109,9 +126,8 @@ class Collections:
 
         :return: A list of collections in the dataset.
         """
-        dataset_id = self._dataset.id
         success, response = self._client_api.gen_request(
-            req_type="GET", path=f"/datasets/{dataset_id}/items/collections"
+            req_type="GET", path=f"/datasets/{self.dataset.id}/items/collections"
         )
         if success:
             data = response.json()
@@ -140,6 +156,17 @@ class Collections:
         if len(collections) >= 10:
             raise ValueError("The dataset already has the maximum number of collections (10).")
         
+    def list_missing_collections(self) -> List[str]:
+        """
+        List all items in the dataset that are not assigned to any collection.
+
+        :return: A list of item IDs that are not part of any collection.
+        """
+        filters = entities.Filters()
+        filters.add(field='metadata.system.collections', values=None)
+        filters.add(field='datasetId', values=self._dataset.id)
+        return self._dataset.items.list(filters=filters)
+    
     def list_unassigned_items(self) -> list:
         """
         List unassigned items in a dataset (items where all collection fields are false).
@@ -147,7 +174,7 @@ class Collections:
         :return: List of unassigned item IDs
         :rtype: list
         """
-        filters = entities.Filters(method=FiltersMethod.AND)  # Use AND method for all conditions
+        filters = entities.Filters(method=entities.FiltersMethod.AND)  # Use AND method for all conditions
         collection_fields = [
             "collections0",
             "collections1",
@@ -163,7 +190,7 @@ class Collections:
 
         # Add each field to the filter with a value of False
         for field in collection_fields:
-            filters.add(field=field, values=False, method=FiltersMethod.AND)
+            filters.add(field=field, values=False, method=entities.FiltersMethod.AND)
 
         missing_ids = []
         pages = self._dataset.items.list(filters=filters)
@@ -176,31 +203,33 @@ class Collections:
 
     def assign(
         self, 
-        dataset_id: str, 
         collections: List[str],
+        dataset_id: str = None, 
         item_id: str = None, 
-        collection_key: str = None
+        filters: entities.Filters = None
     ) -> bool:
         """
         Assign an item to a collection. Creates the collection if it does not exist.
 
-        :param dataset_id: ID of the dataset.
         :param collections: List of the collections to assign the item to.
+        :param dataset_id: ID of the dataset.
         :param item_id: (Optional) ID of the item to assign. If not provided, all items in the dataset will be updated.
-        :param collection_key: (Optional) Key for the bulk assignment. If not provided, no specific metadata will be updated.
+        :param filters: (Optional) Filters of items to assign to the collections.
         :return: True if the assignment was successful, otherwise raises an exception.
         """
+        if not isinstance(collections, list):
+            raise ValueError("collections must be a list.")
+        if dataset_id is None and self._dataset is not None:
+            dataset_id = self.dataset.id
+        if item_id is None and self._item is not None:
+            item_id = self.item.id 
         # Build the query structure
-        if collection_key:
-            query = {
-                "filter": {
-                    f"metadata.system.collections.{collection_key}": True
-                }
-            }
-        elif item_id:
+        if item_id is not None:
             query = {
                 "id": {"$eq": item_id}
             }
+        elif filters is not None:
+            query = filters.prepare().get("filter")
         else:
             raise ValueError("Either collection_key or item_id must be provided.")
 
@@ -226,16 +255,39 @@ class Collections:
             raise exceptions.PlatformException(f"Failed to assign item to collections: {response}")
 
         
-    def unassign(self, dataset_id: str, item_id: str, collections: List[str]) -> bool:
+    def unassign(self, 
+                 collections: List[str], 
+                 dataset_id: str = None,
+                 item_id: str = None, 
+                 filters: entities.Filters = None) -> bool:
         """
         Unassign an item from a collection.
-        :param item_id: ID of the item.
         :param collections: List of collection names to unassign.
+        :param dataset_id: ID of the dataset.
+        :param item_id: ID of the item.
+        :param filters: (Optional) Filters of items to unassign from the collections.
         """
-        payload = {
-            "query": {"id": {"$eq": item_id}},
-            "collections": collections,
-        }
+        if not isinstance(collections, list):
+            raise ValueError("collections must be a list.")
+        # build the context
+        if dataset_id is None and self._dataset is not None:
+            dataset_id = self._dataset.id
+        if item_id is None and self._item is not None:
+            item_id = self._item.id
+            
+        # build the payload
+        if item_id is not None and filters is None:
+            payload = {
+                "query": {"id": {"$eq": item_id}},
+                "collections": collections,
+            }
+        elif filters is not None and item_id is None:
+            payload = {
+                "query": filters.prepare().get("filter"),
+                "collections": collections,
+            }
+        else:
+            raise ValueError("Either item_id or filters must be provided but not both.")
         success, response = self._client_api.gen_request(
             req_type="post",
             path=f"/datasets/{dataset_id}/items/collections/bulk-remove",

dtlpy/repositories/commands.py

@@ -70,7 +70,7 @@ class Commands:
         return entities.Command.from_json(client_api=self._client_api,
                                           _json=response.json())
 
-    def wait(self, command_id, timeout=0, step=None, url=None, backoff_factor=1):
+    def wait(self, command_id, timeout=0, step=None, url=None, backoff_factor=1, iteration_callback=None):
         """
         Wait for command to finish
 
@@ -84,6 +84,7 @@ class Commands:
         :param int timeout: int, seconds to wait until TimeoutError is raised. if 0 - wait until done
         :param int step: int, seconds between polling
         :param str url: url to the command
+        :param function iteration_callback: function to call on each iteration
         :param float backoff_factor: A backoff factor to apply between attempts after the second try
         :return: Command  object
         """
@@ -112,9 +113,18 @@ class Commands:
             elapsed = time.time() - start
             sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
             num_tries += 1
-            logger.debug("Command {!r} is running for {:.2f}[s] and now Going to sleep {:.2f}[s]".format(command.id,
-                                                                                                         elapsed,
-                                                                                                         sleep_time))
+            logger.debug(
+                "Command {!r} is running for {:.2f}[s] and now Going to sleep {:.2f}[s]".format(
+                    command.id,
+                    elapsed,
+                    sleep_time
+                )
+            )
+            if iteration_callback is not None:
+                try:
+                    iteration_callback()
+                except Exception as e:
+                    logger.warning('iteration_callback failed: {}'.format(e.__str__()))
             time.sleep(sleep_time)
         pbar.close()
         if command is None: