dtlpy 1.102.14__py3-none-any.whl → 1.104.14__py3-none-any.whl

dtlpy/repositories/collections.py

@@ -0,0 +1,296 @@
+from venv import logger
+from dtlpy import entities, exceptions, repositories
+from dtlpy.entities.dataset import Dataset
+from dtlpy.entities.filters import FiltersMethod
+from dtlpy.services.api_client import ApiClient
+from typing import List
+
+class Collections:
+    def __init__(self, 
+                client_api: ApiClient,
+                item: entities.Item = None, 
+                dataset: entities.Dataset = None
+            ):
+        self._client_api = client_api
+        self._dataset = dataset
+        self._item = item
+
+    def create(self, name: str) -> entities.Collection:
+        """
+        Creates a new collection in the dataset.
+
+        :param name: The name of the new collection.
+        :return: The created collection details.
+        """
+        dataset_id = self._dataset.id
+        self.validate_max_collections()
+        self.validate_collection_name(name)
+        payload = {"name": name}
+        success, response = self._client_api.gen_request(
+            req_type="post", path=f"/datasets/{dataset_id}/items/collections", json_req=payload
+        )
+        if success:
+            collection_json = self._single_collection(data=response.json(), name=name)
+            return entities.Collection.from_json(client_api=self._client_api, _json=collection_json)
+        else:
+            raise exceptions.PlatformException(response) 
+
+    def update(self, collection_name: str, new_name: str) -> entities.Collection:
+        """
+        Updates the name of an existing collection.
+
+        :param collection_id: The ID of the collection to update.
+        :param new_name: The new name for the collection.
+        :return: The updated collection details.
+        """
+        dataset_id = self._dataset.id
+        self.validate_collection_name(new_name)
+        payload = {"name": new_name}
+        success, response = self._client_api.gen_request(
+            req_type="patch", path=f"/datasets/{dataset_id}/items/collections/{collection_name}", json_req=payload
+        )
+        if success:
+            collection_json = self._single_collection(data=response.json(), name=new_name)
+            return entities.Collection.from_json(client_api=self._client_api, _json=collection_json)
+        else:
+            raise exceptions.PlatformException(response)
+
+    def delete(self, collection_name: str) -> bool:
+        """
+        Deletes a collection from the dataset.
+
+        :param collection_name: The name of the collection to delete.
+        """
+        dataset_id = self._dataset.id
+        success, response = self._client_api.gen_request(
+            req_type="delete", path=f"/datasets/{dataset_id}/items/collections/{collection_name}"
+        )
+        if success:
+            # Wait for the split operation to complete
+            command = entities.Command.from_json(_json=response.json(),
+                                                client_api=self._client_api)
+            command.wait()
+            return True
+        else:
+            raise exceptions.PlatformException(response)
+
+    def clone(self, collection_name: str) -> dict:
+        """
+        Clones an existing collection, creating a new one with a unique name.
+
+        :param collection_name: The name of the collection to clone.
+        :return: The cloned collection details as a dictionary.
+        """
+        self.validate_max_collections()
+        collections = self.list_all_collections()
+        original_collection = next((c for c in collections if c["name"] == collection_name), None)
+
+        if not original_collection:
+            raise ValueError(f"Collection with name '{collection_name}' not found.")
+
+        source_name = original_collection["name"]
+        num = 0
+        clone_name = ""
+        while True:
+            num += 1
+            clone_name = f"{source_name}-clone-{num}"
+            if not any(c["name"] == clone_name for c in collections):  # Use c["name"] for comparison
+                break
+
+        # Create the cloned collection
+        cloned_collection = self.create(name=clone_name)
+        self.assign(dataset_id=self._dataset.id, collections=[cloned_collection.name], collection_key=original_collection['key'])
+        return cloned_collection
+
+
+    def list_all_collections(self) -> entities.Collection:
+        """
+        Retrieves all collections in the dataset.
+
+        :return: A list of collections in the dataset.
+        """
+        dataset_id = self._dataset.id
+        success, response = self._client_api.gen_request(
+            req_type="GET", path=f"/datasets/{dataset_id}/items/collections"
+        )
+        if success:
+            data = response.json()
+            return self._list_collections(data)
+        else:
+            raise exceptions.PlatformException(response)
+        
+    def validate_collection_name(self, name: str):
+        """
+        Validate that the collection name is unique.
+
+        :param name: The name of the collection to validate.
+        :raises ValueError: If a collection with the same name already exists.
+        """
+        collections = self.list_all_collections()
+        if any(c["name"] == name for c in collections): 
+            raise ValueError(f"A collection with the name '{name}' already exists.")
+
+    def validate_max_collections(self) -> None:
+        """
+        Validates that the dataset has not exceeded the maximum allowed collections.
+
+        :raises ValueError: If the dataset has 10 or more collections.
+        """
+        collections = self.list_all_collections()
+        if len(collections) >= 10:
+            raise ValueError("The dataset already has the maximum number of collections (10).")
+        
+    def list_unassigned_items(self) -> list:
+        """
+        List unassigned items in a dataset (items where all collection fields are false).
+
+        :return: List of unassigned item IDs
+        :rtype: list
+        """
+        filters = entities.Filters(method=FiltersMethod.AND)  # Use AND method for all conditions
+        collection_fields = [
+            "collections0",
+            "collections1",
+            "collections2",
+            "collections3",
+            "collections4",
+            "collections5",
+            "collections6",
+            "collections7",
+            "collections8",
+            "collections9",
+        ]
+
+        # Add each field to the filter with a value of False
+        for field in collection_fields:
+            filters.add(field=field, values=False, method=FiltersMethod.AND)
+
+        missing_ids = []
+        pages = self._dataset.items.list(filters=filters)
+        for page in pages:
+            for item in page:
+                # Items that pass filters mean all collections are false
+                missing_ids.append(item.id)
+
+        return missing_ids
+
+    def assign(
+        self, 
+        dataset_id: str, 
+        collections: List[str],
+        item_id: str = None, 
+        collection_key: str = None
+    ) -> bool:
+        """
+        Assign an item to a collection. Creates the collection if it does not exist.
+
+        :param dataset_id: ID of the dataset.
+        :param collections: List of the collections to assign the item to.
+        :param item_id: (Optional) ID of the item to assign. If not provided, all items in the dataset will be updated.
+        :param collection_key: (Optional) Key for the bulk assignment. If not provided, no specific metadata will be updated.
+        :return: True if the assignment was successful, otherwise raises an exception.
+        """
+        # Build the query structure
+        if collection_key:
+            query = {
+                "filter": {
+                    f"metadata.system.collections.{collection_key}": True
+                }
+            }
+        elif item_id:
+            query = {
+                "id": {"$eq": item_id}
+            }
+        else:
+            raise ValueError("Either collection_key or item_id must be provided.")
+
+        # Create the payload
+        payload = {
+            "query": query,
+            "collections": collections,
+        }
+
+        # Make the API request to assign the item
+        success, response = self._client_api.gen_request(
+            req_type="post",
+            path=f"/datasets/{dataset_id}/items/collections/bulk-add",
+            json_req=payload,
+        )
+
+        if success:
+            # Wait for the operation to complete
+            command = entities.Command.from_json(_json=response.json(), client_api=self._client_api)
+            command.wait()
+            return True
+        else:
+            raise exceptions.PlatformException(f"Failed to assign item to collections: {response}")
+
+        
+    def unassign(self, dataset_id: str, item_id: str, collections: List[str]) -> bool:
+        """
+        Unassign an item from a collection.
+        :param item_id: ID of the item.
+        :param collections: List of collection names to unassign.
+        """
+        payload = {
+            "query": {"id": {"$eq": item_id}},
+            "collections": collections,
+        }
+        success, response = self._client_api.gen_request(
+            req_type="post",
+            path=f"/datasets/{dataset_id}/items/collections/bulk-remove",
+            json_req=payload,
+        )
+        if success:
+            # Wait for the split operation to complete
+            command = entities.Command.from_json(_json=response.json(),
+                                                client_api=self._client_api)
+            command.wait()
+            return True
+        else:
+            raise exceptions.PlatformException(response)
+
+    def _single_collection(sef, data: dict, name: str):
+        """
+        Retrieves the key-value pair from the dictionary where the collection's name matches the given name.
+
+        :param data: A dictionary containing collection data in the format:
+                    { "metadata.system.collections.c0": {"name": "Justice League"}, ... }
+        :param name: The name of the collection to find.
+        :return: The key-value pair where the name matches, or None if not found.
+        """
+        for key, value in data.items():
+            if value.get("name") == name:
+                return {key: value}
+        return None
+    
+    def _list_collections(self, data: dict):
+        """
+        Create a list of Collection entities from the dataset JSON.
+
+        :param data: The flat JSON containing collection data in the format:
+                    { "metadata.system.collections.c0": {"name": "Justice League"}, ... }
+        :return: A list of Collection entities.
+        """
+        collections = []
+        for full_key, value in data.items():
+            if "metadata.system.collections" in full_key:
+                # Strip the prefix
+                key = full_key.replace("metadata.system.collections.", "")
+                collection_name = value.get("name")
+                collections.append({"key": key, "name": collection_name})
+        return collections
+    
+    def get_name_by_key(self, key: str) -> str:
+        """
+        Get the name of a collection by its key.
+
+        :param key: The key of the collection (e.g., 'c0', 'c1').
+        :return: The name of the collection if it exists; otherwise, an empty string.
+        """
+        # Assuming collections is a list of dictionaries
+        collections = self.list_all_collections()
+        for collection in collections:
+            if collection.get("key") == key:
+                return collection.get("name", "")
+        return ""

dtlpy/repositories/downloader.py

@@ -674,6 +674,8 @@ class Downloader:
                                                                                      stream=True,
                                                                                      dataset_id=item.dataset_id)
                     if not result:
+                        if os.path.isfile(local_filepath + '.download'):
+                            os.remove(local_filepath + '.download')
                         raise PlatformException(response)
                 else:
                     _, ext = os.path.splitext(item.metadata['system']['shebang']['linkInfo']['ref'].split('?')[0])

dtlpy/repositories/features.py

@@ -17,9 +17,11 @@ class Features:
                  project_id: str = None,
                  item: entities.Item = None,
                  annotation: entities.Annotation = None,
-                 feature_set: entities.FeatureSet = None):
+                 feature_set: entities.FeatureSet = None,
+                 dataset: entities.Dataset = None):
         if project is not None and project_id is None:
             project_id = project.id
+        self._dataset = dataset
         self._project = project
         self._project_id = project_id
         self._item = item
@@ -50,7 +52,7 @@ class Features:
         if self._project is None:
             raise exceptions.PlatformException(
                 error='2001',
-                message='Cannot perform action WITHOUT Project entity in Datasets repository.'
+                message='Cannot perform action WITHOUT Project entity in Features repository.'
                         ' Please checkout or set a project')
         assert isinstance(self._project, entities.Project)
         return self._project
@@ -60,8 +62,8 @@ class Features:
     ###########
     def _list(self, filters: entities.Filters):
         """
-        Get dataset items list This is a browsing endpoint, for any given path item count will be returned,
-        user is expected to perform another request then for every folder item to actually get the its item list.
+        Get dataset feature vectors list. This is a browsing endpoint, for any given path feature count will be returned,
+        user is expected to perform another request then for every folder item to actually get the item list.
 
         :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
         :return: json response
@@ -104,6 +106,8 @@ class Features:
             filters.add(field='featureSetId', values=self._feature_set.id)
         if self._item is not None:
             filters.add(field='entityId', values=self._item.id)
+        if self._dataset is not None:
+            filters.add(field='datasetId', values=self._dataset.id)
         if self._project_id is None:
             self._project_id = self.project.id
         filters.context = {"projects": [self._project_id]}
@@ -169,7 +173,7 @@ class Features:
         if feature_set_id is None:
             if self._feature_set is None:
                 raise ValueError(
-                    'Missing feature_set_id. Must insert the variable or create from context - feature_set.features.create()')
+                    'Missing feature_set_id. Must insert the variable or create from context, e.g. feature_set.features.create()')
             feature_set_id = self._feature_set.id
 
         payload = {'project': project_id,

dtlpy/repositories/integrations.py

@@ -1,7 +1,8 @@
 """
 Integrations Repository
 """
-
+import base64
+import json
 import logging
 from .. import entities, exceptions, miscellaneous, _api_reference
 from ..services.api_client import ApiClient
@@ -298,3 +299,53 @@ class Integrations:
 
         available_integrations = miscellaneous.List(response.json())
         return available_integrations
+
+    def _create_private_registry_gar(self, service_account: str, location: str):
+        password = self.__create_gar_password(service_account, location)
+        return self.create(
+            integrations_type='private-registry',
+            name='gar-1',
+            metadata={"provider": "gcp"},
+            options={
+                "name": "_json_key",
+                "spec": {
+                    "password": password
+                }
+            }
+        )
+
+    def __create_gar_password(self, service_account: str, location: str) -> str:
+        """
+        Generates a Google Artifact Registry JSON configuration and returns it as a base64-encoded string.
+
+        Parameters:
+            location (str): The region where the repository will be created (e.g., 'us-central1').
+            service_account (str): The service_account parameter represents the Google Cloud service account credentials
+                                    in the form of a JSON key file. This JSON contains the private key and other metadata
+                                    required for authenticating with Google Artifact Registry. It is used to generate a Kubernetes secret
+                                    that stores the credentials for pulling container images from the registry.
+                                    The JSON key must include fields such as client_email, private_key, and project_id,
+                                    and it is typically downloaded from the Google Cloud Console when creating the service account
+
+        Returns:
+            str: A base64-encoded string representation of the repository JSON configuration.
+        """
+        if not service_account:
+            raise ValueError('Missing Service Account')
+        if not location:
+            raise ValueError('Missing Location')
+        user_name = "_json_key"
+        cred = f"{user_name}:{service_account}"
+        auth = str(base64.b64encode(bytes(cred, 'utf-8')))[2:-1]
+
+        encoded_pass = {
+            "auths": {
+                f"{location}": {
+                    "username": user_name,
+                    "password": service_account,
+                    "auth": auth
+                }
+            }
+        }
+
+        return str(base64.b64encode(bytes(json.dumps(encoded_pass), 'utf-8')))[2:-1]

dtlpy/repositories/items.py

@@ -271,7 +271,7 @@ class Items:
                 filters.pop(field='hidden')
                 if is_dir:
                     filters.add(field='type', values='dir')
-                    filters.recursive = False
+                filters.recursive = False
                 filters.add(field='filename', values=filepath)
                 paged_entity = self.list(filters=filters)
                 if len(paged_entity.items) == 0:
@@ -610,7 +610,9 @@ class Items:
             output_entity=entities.Item,
             no_output: bool = False,
             export_version: str = entities.ExportVersion.V1,
-            item_description: str = None
+            item_description: str = None,
+            raise_on_error: bool = False,
+            return_as_list: bool = False
     ):
         """
         Upload local file to dataset.
@@ -630,6 +632,9 @@ class Items:
         :param bool no_output: do not return the items after upload
         :param str export_version:  exported items will have original extension in filename, `V1` - no original extension in filenames
         :param str item_description: add a string description to the uploaded item
+        :param bool raise_on_error: raise an exception if an error occurs
+        :param bool return_as_list: return a list of items instead of a generator
+        
         :return: Output (generator/single item)
         :rtype: generator or single item
 
@@ -657,7 +662,9 @@ class Items:
             # metadata to upload with items
             item_metadata=item_metadata,
             export_version=export_version,
-            item_description=item_description
+            item_description=item_description,
+            raise_on_error=raise_on_error,
+            return_as_list=return_as_list
         )
 
     @property

dtlpy/repositories/pipelines.py

@@ -376,24 +376,18 @@ class Pipelines:
 
             pipeline = project.pipelines.update(pipeline='pipeline_entity')
         """
-        if pipeline.status == entities.CompositionStatus.INSTALLED and not pipeline.variables_changed():
-            raise exceptions.PlatformException(
-                error='400',
-                message='Cannot update pipeline while it is installed'
-            )
         # payload
         payload = pipeline.to_json()
 
         # update settings
         if pipeline.settings_changed():
-            self.update_settings(pipeline=pipeline, settings=pipeline.settings)
+            new_pipeline = self.update_settings(pipeline=pipeline, settings=pipeline.settings)
+            payload['settings'] = new_pipeline.to_json().get('settings', payload.get('settings'))
 
         # update variables
         if pipeline.variables_changed():
             new_pipeline = self.__update_variables(pipeline=pipeline)
-            if new_pipeline.status == entities.CompositionStatus.INSTALLED:
-                logger.warning('Pipeline is installed, updating pipeline variables only')
-                return new_pipeline
+            payload['variables'] = new_pipeline.to_json().get('variables', payload.get('variables'))
 
         success, response = self._client_api.gen_request(
             req_type='patch',

dtlpy/repositories/uploader.py

@@ -59,7 +59,9 @@ class Uploader:
             overwrite=False,
             item_metadata=None,
             export_version: str = entities.ExportVersion.V1,
-            item_description=None
+            item_description=None,
+            raise_on_error=False,
+            return_as_list=False
     ):
         """
         Upload local file to dataset.
@@ -67,7 +69,7 @@ class Uploader:
         If `*` at the end of local_path (e.g. '/images/*') items will be uploaded without head directory
 
         :param local_path: local file or folder to upload
-        :param local_annotations_path: path to dataloop format annotations json files.
+        :param local_annotations_path: path to Dataloop format annotations json files.
         :param remote_path: remote path to save.
         :param remote_name: remote base name to save.
         :param file_types: list of file type to upload. e.g ['.jpg', '.png']. default is all
@@ -75,6 +77,8 @@ class Uploader:
         :param item_metadata: upload the items with the metadata dictionary
         :param str export_version:  exported items will have original extension in filename, `V1` - no original extension in filenames
         :param str item_description: add a string description to the uploaded item
+        :param bool raise_on_error: raise an exception if an error occurs
+        :param bool return_as_list: always return a list of items 
 
         :return: Output (list)
         """
@@ -114,13 +118,21 @@ class Uploader:
             if log_filepath is not None:
                 logger.warning("Errors in {n_error} files. See {log_filepath} for full log".format(
                     n_error=errors_count, log_filepath=log_filepath))
-
-        # TODO 2.0 always return a list
+            if raise_on_error is True:
+                raise PlatformException(error="400",
+                                        message=f"Errors in {errors_count} files. See above trace for more information")
+        
+        if return_as_list is True:
+            # return list of items
+            return list(self.reporter.output)
         if len(status_list) == 1:
+            # if there is only one item, return it
             try:
                 return next(self.reporter.output)
             except StopIteration:
+                # if there is no items, return None
                 return None
+        # if there are multiple items, return the generator
         return self.reporter.output
 
     def _build_elements_from_inputs(self,

dtlpy/services/api_client.py

@@ -45,7 +45,7 @@ threadLock = threading.Lock()
 
 
 def format_message(message):
-    if message:
+    if message and isinstance(message, str):
         return message.replace('\\n', '\n')
     return message
 
@@ -1476,7 +1476,7 @@ class ApiClient:
             msg += '[Response <{val}>]'.format(val=resp.status_code)
         if hasattr(resp, 'reason'):
             msg += '[Reason: {val}]'.format(val=resp.reason)
-        if hasattr(resp, 'text'):
+        if hasattr(resp, 'text') and isinstance(resp.text, str):
             msg += '[Text: {val}]'.format(val=format_message(resp.text))
 
         request_id = resp.headers.get('x-request-id', 'na')

{dtlpy-1.102.14.dist-info → dtlpy-1.104.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dtlpy
-Version: 1.102.14
+Version: 1.104.14
 Summary: SDK and CLI for Dataloop platform
 Home-page: https://github.com/dataloop-ai/dtlpy
 Author: Dataloop Team
@@ -26,7 +26,7 @@ Requires-Dist: requests-toolbelt (>=1.0.0)
 Requires-Dist: requests (>=2.25.0)
 Requires-Dist: numpy (>=1.16.2)
 Requires-Dist: pandas (>=0.24.2)
-Requires-Dist: tabulate (==0.8.9)
+Requires-Dist: tabulate (>=0.8.9)
 Requires-Dist: Pillow (>=7.2)
 Requires-Dist: PyJWT (>=2.4)
 Requires-Dist: jinja2 (>=2.11.3)