dtlpy 1.90.39__py3-none-any.whl → 1.92.18__py3-none-any.whl

dtlpy/repositories/datasets.py

@@ -4,10 +4,12 @@ Datasets Repository
 
 import os
 import sys
+import time
 import copy
 import tqdm
 import logging
-from urllib.parse import urlencode
+import json
+from typing import Union
 
 from .. import entities, repositories, miscellaneous, exceptions, services, PlatformException, _api_reference
 from ..services.api_client import ApiClient
@@ -93,7 +95,11 @@ class Datasets:
             filters._user_query = 'false'
         if not folder_path.startswith('/'):
             folder_path = '/' + folder_path
-        filters.add(field='dir', values=folder_path + '*')
+        filters.add(field='dir', values=folder_path, method=entities.FiltersMethod.OR)
+        if not folder_path.endswith('*'):
+            if not folder_path.endswith('/'):
+                folder_path += '/'
+            filters.add(field='dir', values=folder_path + '*', method=entities.FiltersMethod.OR)
         return filters
 
     def _get_binaries_dataset(self):
@@ -111,6 +117,70 @@ class Datasets:
             dataset = datasets[0]
         return dataset
 
+    def _resolve_dataset_id(self, dataset, dataset_name, dataset_id):
+        if dataset is None and dataset_name is None and dataset_id is None:
+            raise ValueError('Must provide dataset, dataset name or dataset id')
+        if dataset_id is None:
+            if dataset is None:
+                dataset = self.get(dataset_name=dataset_name)
+            dataset_id = dataset.id
+        return dataset_id
+
+    @staticmethod
+    def _build_payload(filters, include_feature_vectors, include_annotations, export_type, annotation_filters,
+                       feature_vector_filters):
+        valid_list = [e.value for e in entities.ExportType]
+        valid_types = ', '.join(valid_list)
+        if export_type not in ['json', 'zip']:
+            raise ValueError('export_type must be one of the following: {}'.format(valid_types))
+        payload = {'exportType': export_type}
+        if filters is None:
+            filters = entities.Filters()
+
+        if isinstance(filters, entities.Filters):
+            payload['itemsQuery'] = {'filter': filters.prepare()['filter'], 'join': filters.prepare().get("join", {})}
+        elif isinstance(filters, dict):
+            payload['itemsQuery'] = filters
+        else:
+            raise exceptions.BadRequest(message='filters must be of type dict or Filters', status_code=500)
+
+        payload['itemsVectorQuery'] = {}
+        if include_feature_vectors:
+            payload['includeItemVectors'] = True
+            payload['itemsVectorQuery']['select'] = {"datasetId": 1, 'featureSetId': 1, 'value': 1}
+
+        if feature_vector_filters is not None:
+            payload['itemsVectorQuery']['filter'] = feature_vector_filters.prepare()['filter']
+
+        payload['annotations'] = {"include": include_annotations, "convertSemantic": False}
+
+        if annotation_filters is not None:
+            payload['annotationsQuery'] = annotation_filters.prepare()['filter']
+            payload['annotations']['filter'] = True
+
+        return payload
+
+    def _download_exported_item(self, item_id, export_type, local_path=None):
+        export_item = repositories.Items(client_api=self._client_api).get(item_id=item_id)
+        export_item_path = export_item.download(local_path=local_path)
+
+        if export_type == entities.ExportType.ZIP:
+            # unzipping annotations to directory
+            if isinstance(export_item_path, list) or not os.path.isfile(export_item_path):
+                raise exceptions.PlatformException(
+                    error='404',
+                    message='error downloading annotation zip file. see above for more information. item id: {!r}'.format(
+                        export_item.id))
+            try:
+                miscellaneous.Zipping.unzip_directory(zip_filename=export_item_path,
+                                                      to_directory=local_path)
+            except Exception as e:
+                logger.warning("Failed to extract zip file error: {}".format(e))
+            finally:
+                # cleanup
+                if isinstance(export_item_path, str) and os.path.isfile(export_item_path):
+                    os.remove(export_item_path)
+
     @property
     def platform_url(self):
         return self._client_api._get_resource_url("projects/{}/datasets".format(self.project.id))
@@ -139,7 +209,7 @@ class Datasets:
         if dataset is not None:
             dataset.open_in_web()
         elif dataset_id is not None:
-            self._client_api._open_in_web(url=self.platform_url + '/' + str(dataset_id))
+            self._client_api._open_in_web(url=f'{self.platform_url}/{dataset_id}/items')
         else:
             self._client_api._open_in_web(url=self.platform_url)
 
@@ -425,12 +495,7 @@ class Datasets:
             directory_tree = dataset.directory_tree
             directory_tree = project.datasets.directory_tree(dataset='dataset_entity')
         """
-        if dataset is None and dataset_name is None and dataset_id is None:
-            raise exceptions.PlatformException('400', 'Must provide dataset, dataset name or dataset id')
-        if dataset_id is None:
-            if dataset is None:
-                dataset = self.get(dataset_name=dataset_name)
-            dataset_id = dataset.id
+        dataset_id = self._resolve_dataset_id(dataset, dataset_name, dataset_id)
 
         url_path = '/datasets/{}/directoryTree'.format(dataset_id)
 
@@ -519,6 +584,72 @@ class Datasets:
                                                .format(response))
         return self.get(dataset_id=command.spec['returnedModelId'])
 
+    @_api_reference.add(path='/datasets/{id}/export', method='post')
+    def export(self,
+               dataset: entities.Dataset = None,
+               dataset_name: str = None,
+               dataset_id: str = None,
+               local_path: str = None,
+               filters: Union[dict, entities.Filters] = None,
+               annotation_filters: entities.Filters = None,
+               feature_vector_filters: entities.Filters = None,
+               include_feature_vectors: bool = False,
+               include_annotations: bool = False,
+               export_type: entities.ExportType = entities.ExportType.JSON,
+               timeout: int = 0):
+        """
+        Export dataset items and annotations.
+
+        **Prerequisites**: You must be an *owner* or *developer* to use this method.
+
+        You must provide at least ONE of the following params: dataset, dataset_name, dataset_id.
+
+        :param dtlpy.entities.dataset.Dataset dataset: Dataset object
+        :param str dataset_name: The name of the dataset
+        :param str dataset_id: The ID of the dataset
+        :param str local_path: Local path to save the exported dataset 
+        :param Union[dict, dtlpy.entities.filters.Filters] filters: Filters entity or a query dictionary
+        :param dtlpy.entities.filters.Filters annotation_filters: Filters entity to filter annotations for export       
+        :param dtlpy.entities.filters.Filters feature_vector_filters: Filters entity to filter feature vectors for export
+        :param bool include_feature_vectors: Include item feature vectors in the export
+        :param bool include_annotations: Include item annotations in the export
+        :param entities.ExportType export_type: Type of export ('json' or 'zip')
+        :param int timeout: Maximum time in seconds to wait for the export to complete
+        :return: Exported item
+        :rtype: dtlpy.entities.item.Item
+
+        **Example**:
+
+        .. code-block:: python
+
+            export_item = project.datasets.export(dataset_id='dataset_id',
+                                                  filters=filters,
+                                                  include_feature_vectors=True,
+                                                  include_annotations=True,
+                                                  export_type=dl.ExportType.JSON)
+        """
+        dataset_id = self._resolve_dataset_id(dataset, dataset_name, dataset_id)
+        payload = self._build_payload(filters, include_feature_vectors, include_annotations, export_type,
+                                      annotation_filters, feature_vector_filters)
+
+        success, response = self._client_api.gen_request(req_type='post', path=f'/datasets/{dataset_id}/export',
+                                                         json_req=payload)
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        command = entities.Command.from_json(_json=response.json(),
+                                             client_api=self._client_api)
+
+        time.sleep(2)  # as the command have wrong progress in the beginning
+        command = command.wait(timeout=timeout)
+        if 'outputItemId' not in command.spec:
+            raise exceptions.PlatformException(
+                error='400',
+                message="outputItemId key is missing in command response: {}".format(response))
+        item_id = command.spec['outputItemId']
+        self._download_exported_item(item_id=item_id, export_type=export_type, local_path=local_path)
+        return local_path
+
     @_api_reference.add(path='/datasets/merge', method='post')
     def merge(self,
               merge_name: str,
@@ -769,7 +900,7 @@ class Datasets:
         Download dataset's annotations by filters.
 
         You may filter the dataset both for items and for annotations and download annotations.
-        
+
         Optional -- download annotations as: mask, instance, image mask of the item.
 
         **Prerequisites**: You must be in the role of an *owner* or *developer*.
@@ -917,9 +1048,8 @@ class Datasets:
                            ):
         """
         Upload annotations to dataset. 
-        
-        Example for remote_root_path: If the item filepath is a/b/item and
-        remote_root_path is /a the start folder will be b instead of a
+
+        Example for remote_root_path: If the item filepath is "/a/b/item" and remote_root_path is "/a" - the start folder will be b instead of a
 
         **Prerequisites**: You must have a dataset with items that are related to the annotations. The relationship between the dataset and annotations is shown in the name. You must be in the role of an *owner* or *developer*.
 

dtlpy/repositories/dpks.py

@@ -196,7 +196,8 @@ class Dpks:
             dpk = dpk_v.items[0]
         return dpk
 
-    def publish(self, dpk: entities.Dpk = None, ignore_max_file_size: bool = False, manifest_filepath='dataloop.json') -> entities.Dpk:
+    def publish(self, dpk: entities.Dpk = None, ignore_max_file_size: bool = False,
+                manifest_filepath='dataloop.json') -> entities.Dpk:
         """
         Upload a dpk entity to the dataloop platform.
 
@@ -290,6 +291,9 @@ class Dpks:
         elif filters.resource != entities.FiltersResource.DPK:
             raise ValueError('Filters resource must to be FiltersResource.DPK. Got: {!r}'.format(filters.resource))
 
+        if self._project is not None:
+            filters.add(field='context.project', values=self._project.id)
+
         paged = entities.PagedEntities(items_repository=self,
                                        filters=filters,
                                        page_offset=filters.page,
@@ -309,6 +313,35 @@ class Dpks:
             raise exceptions.PlatformException(response)
         return response.json()
 
+    def get_revisions(self, dpk_id: str, version: str):
+        """
+        Get the revision of a specific dpk.
+
+        :param str dpk_id: the id of the dpk.
+        :param str version: the version of the dpk.
+        :return the entity of the dpk
+        :rtype entities.Dpk
+
+        ** Example **
+        ..coed-block:: python
+            dpk = dl.dpks.get_revisions(dpk_id='id', version='1.0.0')
+        """
+        if dpk_id is None or version is None:
+            raise ValueError('You must provide both dpk_id and version')
+        url = '/app-registry/{}/revisions/{}'.format(dpk_id, version)
+
+        # request
+        success, response = self._client_api.gen_request(req_type='get',
+                                                         path=url)
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        dpk = entities.Dpk.from_json(_json=response.json(),
+                                     client_api=self._client_api,
+                                     project=self._project,
+                                     is_fetched=False)
+        return dpk
+
     def get(self, dpk_name: str = None, dpk_version: str = None, dpk_id: str = None) -> entities.Dpk:
         """
         Get a specific dpk from the platform.

dtlpy/repositories/executions.py

@@ -7,6 +7,7 @@ from .. import exceptions, entities, repositories, miscellaneous, _api_reference
 from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
+MAX_SLEEP_TIME = 30
 
 
 class Executions:
@@ -562,15 +563,19 @@ class Executions:
         return execution
 
     def wait(self,
-             execution_id: str,
-             timeout: int = None):
+             execution_id: str = None,
+             execution: entities.Execution = None,
+             timeout: int = None,
+             backoff_factor=1):
         """
         Get Service execution object.
 
         **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
 
         :param str execution_id: execution id
+        :param str execution: dl.Execution, optional. must input one of execution or execution_id
         :param int timeout: seconds to wait until TimeoutError is raised. if <=0 - wait until done - by default wait take the service timeout
+        :param float backoff_factor: A backoff factor to apply between attempts after the second try
         :return: Service execution object
         :rtype: dtlpy.entities.execution.Execution
 
@@ -580,40 +585,32 @@ class Executions:
 
             service.executions.wait(execution_id='execution_id')
         """
-        url_path = "/executions/{}".format(execution_id)
+        if execution is None:
+            if execution_id is None:
+                raise ValueError('Must input at least one: [execution, execution_id]')
+            else:
+                execution = self.get(execution_id=execution_id)
         elapsed = 0
         start = int(time.time())
-        if timeout is not None and timeout <= 0:
+        if timeout is None or timeout <= 0:
             timeout = np.inf
 
-        i = 1
-        while True:
-            success, response = self._client_api.gen_request(req_type="get",
-                                                             path=url_path,
-                                                             log_error=False)
-            if not success:
-                raise exceptions.PlatformException(response)
-            # return entity
-            execution = entities.Execution.from_json(client_api=self._client_api,
-                                                     _json=response.json(),
-                                                     project=self._project,
-                                                     service=self._service)
-            if timeout is None:
-                timeout = execution.service.execution_timeout + 60
-            if execution.latest_status['status'] in ['failed', 'success', 'terminated', 'aborted', 'canceled',
-                                                     'system-failure']:
+        num_tries = 1
+        while elapsed < timeout:
+            execution = self.get(execution_id=execution.id)
+            if not execution.in_progress():
                 break
-            elapsed = int(time.time()) - start
-            i += 1
-            if i > 18 or elapsed > timeout:
-                break
-            sleep_time = np.minimum(timeout - elapsed, 2 ** i)
+            elapsed = time.time() - start
+            if elapsed >= timeout:
+                raise TimeoutError(
+                    f"execution wait() got timeout. id: {execution.id!r}, status: {execution.latest_status}")
+            sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
+            num_tries += 1
+            logger.debug("Execution {!r} is running for {:.2f}[s] and now Going to sleep {:.2f}[s]".format(execution.id,
+                                                                                                           elapsed,
+                                                                                                           sleep_time))
             time.sleep(sleep_time)
-        if execution is None:
-            raise ValueError('Nothing to wait for')
-        if elapsed >= timeout:
-            raise TimeoutError("execution wait() got timeout. id: {!r}, status: {}".format(
-                execution.id, execution.latest_status))
+
         return execution
 
     @_api_reference.add(path='/executions/{id}/terminate', method='post')

dtlpy/repositories/feature_sets.py

@@ -11,11 +11,16 @@ class FeatureSets:
     """
     URL = '/features/sets'
 
-    def __init__(self, client_api: ApiClient,
+    def __init__(self,
+                 client_api: ApiClient,
                  project_id: str = None,
-                 project: entities.Project = None):
+                 project: entities.Project = None,
+                 model_id: str = None,
+                 model: entities.Model = None):
         self._project = project
         self._project_id = project_id
+        self._model = model
+        self._model_id = model_id
         self._client_api = client_api
 
     ############
@@ -34,11 +39,23 @@ class FeatureSets:
         if self._project is None:
             raise exceptions.PlatformException(
                 error='2001',
-                message='Cannot perform action WITHOUT Project entity in Datasets repository.'
+                message='Cannot perform action WITHOUT Project entity in FeatureSets repository.'
                         ' Please checkout or set a project')
         assert isinstance(self._project, entities.Project)
         return self._project
 
+    @property
+    def model(self) -> entities.Model:
+        if self._model is None and self._model_id is not None:
+            # get from id
+            self._model = repositories.Models(client_api=self._client_api).get(model_id=self._model_id)
+        if self._model is None:
+            raise exceptions.PlatformException(
+                error='2001',
+                message='Cannot perform action WITHOUT Model entity in FeatureSets repository.')
+        assert isinstance(self._model, entities.Model)
+        return self._model
+
     ###########
     # methods #
     ###########
@@ -132,6 +149,7 @@ class FeatureSets:
                set_type: str,
                entity_type: entities.FeatureEntityType,
                project_id: str = None,
+               model_id: set = None,
                org_id: str = None):
         """
         Create a new Feature Set
@@ -141,6 +159,7 @@ class FeatureSets:
         :param str set_type: string of the feature type: 2d, 3d, modelFC, TSNE,PCA,FFT
         :param entity_type: the entity that feature vector is linked to. Use the enum dl.FeatureEntityType
         :param str project_id: the ID of the project where feature set will be created
+        :param str model_id: the ID of the model that creates the vectors
         :param str org_id: the ID of the org where feature set will be created
         :return: Feature Set object
         """
@@ -154,6 +173,7 @@ class FeatureSets:
                    'size': size,
                    'type': set_type,
                    'project': project_id,
+                   'modelId': model_id,
                    'entityType': entity_type}
         if org_id is not None:
             payload['org'] = org_id

dtlpy/repositories/features.py

@@ -89,6 +89,9 @@ class Features:
         if filters is None:
             filters = entities.Filters(resource=entities.FiltersResource.FEATURE)
             filters._user_query = 'false'
+        # default sorting
+        if filters.sort == dict():
+            filters.sort_by(field='id')
         # assert type filters
         if not isinstance(filters, entities.Filters):
             raise exceptions.PlatformException(error='400',
@@ -149,7 +152,7 @@ class Features:
         :param immutable value: actual vector - immutable (list of floats [1,2,3])
         :param str project_id: the id of the project where feature will be created
         :param str feature_set_id: ref to a featureSet this vector is a part of
-        :param entity: the entity the featureVector is linked to (item.id, annotation.id etc)
+        :param entity: the entity the featureVector is linked to (item, annotation, etc)
         :param str version: version of the featureSet generator
         :param str parent_id: optional: parent FeatureSet id - used when FeatureVector is a subFeature
         :param str org_id: the id of the org where featureVector will be created

dtlpy/repositories/models.py

@@ -668,7 +668,7 @@ class Models:
         :param item_ids: a list of item id to run the prediction.
         :return:
         """
-        if len(model.metadata['system']['deploy']['services']) == 0:
+        if len(model.metadata['system'].get('deploy', {}).get('services', [])) == 0:
             # no services for model
             raise ValueError("Model doesnt have any associated services. Need to deploy before predicting")
         payload = {'input': {'itemIds': item_ids},

dtlpy/repositories/packages.py

@@ -664,9 +664,12 @@ class Packages:
         :return: Package object
         :rtype: dtlpy.entities.package.Package
         """
-        # if is dtlpy entity convert to dict
-        if modules and isinstance(modules[0], entities.PackageModule):
-            modules = [module.to_json() for module in modules]
+        if modules is not None:
+            if not isinstance(modules, list):
+                modules = [modules]
+
+            if isinstance(modules[0], entities.PackageModule):
+                modules = [module.to_json() for module in modules]
 
         if slots and isinstance(slots[0], entities.PackageSlot):
             slots = [slot.to_json() for slot in slots]

dtlpy/repositories/pipeline_executions.py

@@ -1,8 +1,12 @@
 import logging
+import time
+import numpy as np
+
 from .. import entities, repositories, exceptions, miscellaneous, services, _api_reference
 from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
+MAX_SLEEP_TIME = 30
 
 
 class PipelineExecutions:
@@ -81,7 +85,7 @@ class PipelineExecutions:
         if pipeline_id is None and self._pipeline is None:
             raise exceptions.PlatformException('400', 'Must provide param pipeline_id')
         elif pipeline_id is None:
-            pipeline_id = self.pipeline.id
+            pipeline_id = self._pipeline.id
 
         success, response = self._client_api.gen_request(
             req_type="get",
@@ -93,13 +97,13 @@ class PipelineExecutions:
         if not success:
             raise exceptions.PlatformException(response)
 
-        pipeline = entities.PipelineExecution.from_json(
+        pipeline_execution = entities.PipelineExecution.from_json(
             client_api=self._client_api,
             _json=response.json(),
             pipeline=self._pipeline
         )
 
-        return pipeline
+        return pipeline_execution
 
     def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.PipelineExecution]:
         pool = self._client_api.thread_pools(pool_name='entity.create')
@@ -175,7 +179,7 @@ class PipelineExecutions:
         #     filters.add(field='projectId', values=self.project.id)
 
         if self._pipeline is not None:
-            filters.add(field='pipelineId', values=self.pipeline.id)
+            filters.add(field='pipelineId', values=self._pipeline.id)
 
         paged = entities.PagedEntities(
             items_repository=self,
@@ -344,7 +348,7 @@ class PipelineExecutions:
         if pipeline_id is None and self._pipeline is None:
             raise exceptions.PlatformException('400', 'Must provide param pipeline_id')
         elif pipeline_id is None:
-            pipeline_id = self.pipeline.id
+            pipeline_id = self._pipeline.id
 
         if filters is None:
             filters = entities.Filters(resource=entities.FiltersResource.PIPELINE_EXECUTION)
@@ -379,3 +383,52 @@ class PipelineExecutions:
                                                message="cycleOptions key is missing in command response: {!r}"
                                                .format(response))
         return True
+
+    def wait(self,
+             pipeline_execution_id: str = None,
+             pipeline_execution: entities.PipelineExecution = None,
+             timeout: int = None,
+             backoff_factor=1):
+        """
+        Get Service execution object.
+
+        **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
+
+        :param str pipeline_execution_id: pipeline execution id
+        :param str pipeline_execution: dl.PipelineExecution, optional. must input one of pipeline execution or pipeline_execution_id
+        :param int timeout: seconds to wait until TimeoutError is raised. if <=0 - wait until done - by default wait take the service timeout
+        :param float backoff_factor: A backoff factor to apply between attempts after the second try
+        :return: Service execution object
+        :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
+
+        **Example**:
+
+        .. code-block:: python
+
+            pipeline.pipeline_executions.wait(pipeline_execution_id='pipeline_execution_id')
+        """
+        if pipeline_execution is None:
+            if pipeline_execution_id is None:
+                raise ValueError('Must input at least one: [pipeline_execution, pipeline_execution_id]')
+        else:
+            pipeline_execution_id = pipeline_execution.id
+        elapsed = 0
+        start = time.time()
+        if timeout is None or timeout <= 0:
+            timeout = np.inf
+
+        num_tries = 1
+        while elapsed < timeout:
+            pipeline_execution = self.get(pipeline_execution_id=pipeline_execution_id)
+            if not pipeline_execution.in_progress():
+                break
+            elapsed = time.time() - start
+            if elapsed >= timeout:
+                raise TimeoutError(
+                    f"Pipeline execution wait() function timed out. id: {pipeline_execution.id!r}, status: {pipeline_execution.status}.")
+            sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
+            num_tries += 1
+            logger.debug(
+                f"Pipeline execution {pipeline_execution.id!r} has been running for {elapsed:.2f}[s]. Sleeping for {sleep_time:.2f}[s]")
+            time.sleep(sleep_time)
+        return pipeline_execution