dtlpy 1.91.37__py3-none-any.whl → 1.92.18__py3-none-any.whl

dtlpy/new_instance.py

@@ -1,14 +1,61 @@
 class Dtlpy:
+    from .services.api_client import client as client_api
+    from .services.api_client import VerboseLoggingLevel, ApiClient
+    from .services import DataloopLogger, DtlpyFilter, check_sdk, Reporter, service_defaults
+    from .services.api_reference import api_reference as _api_reference
+    from .caches.cache import CacheConfig, CacheType
     from .exceptions import PlatformException
     from . import repositories, exceptions, entities, examples
-    from .__version__ import version as __version__
-    from .entities import Box, Text, Point, Segmentation, Polygon, Ellipse, Classification, Subtitle, Polyline, \
-        Filters, Trigger, Description, \
-        AnnotationCollection, Annotation, Item, Codebase, Filters, Execution, Recipe, Ontology, Label, \
-        ItemLink, UrlLink, PackageModule, PackageFunction, FunctionIO, Modality, Workload, WorkloadUnit
-    from .utilities import Converter, BaseServiceRunner, Progress
-    from .services.api_client import ApiClient
-    from .services import check_sdk
+    from .entities import (
+        # main entities
+        Project, Dataset, ExpirationOptions, ExportVersion, Trigger, Item, Execution, AnnotationCollection, Annotation,
+        Recipe, IndexDriver, AttributesTypes, AttributesRange, Dpk, App, AppModule, AppScope,
+        Ontology, Label, Task, TaskPriority, ConsensusTaskType, Assignment, Service, Package, Codebase, Model,
+        PackageModule, PackageFunction,
+        # annotations
+        Box, Cube, Cube3d, Point, Note, Message, Segmentation, Ellipse, Classification, Subtitle, Polyline, Pose,
+        Description,
+        Polygon, Text, FreeText, RefImage,
+        # filters
+        Filters, FiltersKnownFields, FiltersResource, FiltersOperations, FiltersMethod, FiltersOrderByDirection,
+        FiltersKnownFields as KnownFields,
+        # triggers
+        TriggerResource, TriggerAction, TriggerExecutionMode, TriggerType,
+        # faas
+        FunctionIO, KubernetesAutuscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRuntime,
+        InstanceCatalog, PackageInputType, ServiceType, ServiceModeType,
+        PackageSlot, SlotPostAction, SlotPostActionType, SlotDisplayScope, SlotDisplayScopeResource, UiBindingPanel,
+        # roberto
+        DatasetSubsetType, ModelStatus, PlotSample, ArtifactType, Artifact, ItemArtifact, LinkArtifact, LocalArtifact,
+        EntityScopeLevel,
+        # features
+        FeatureEntityType, Feature, FeatureSet,
+        #
+        RequirementOperator, PackageRequirement,
+        Command, CommandsStatus,
+        LocalCodebase, GitCodebase, ItemCodebase, FilesystemCodebase, PackageCodebaseType,
+        MemberRole, MemberOrgRole,
+        Webhook, HttpMethod,
+        ViewAnnotationOptions, AnnotationStatus, AnnotationType,
+        ItemStatus, ExecutionStatus, ExportMetadata,
+        PromptItem, Prompt, PromptType,
+        ItemLink, UrlLink, LinkTypeEnum,
+        Modality, ModalityTypeEnum, ModalityRefTypeEnum,
+        Workload, WorkloadUnit, ItemAction,
+        PipelineExecution, CycleRerunMethod, PipelineExecutionNode, Pipeline, PipelineConnection,
+        PipelineNode, TaskNode, CodeNode, PipelineStats, PipelineSettings,
+        PipelineNodeType, PipelineNameSpace, PipelineResumeOption, Variable, CompositionStatus,
+        FunctionNode, DatasetNode, PipelineConnectionPort, PipelineNodeIO, Organization, OrganizationsPlans,
+        Integration,
+        Driver, S3Driver, GcsDriver, AzureBlobDriver, CacheAction, PodType,
+        ExternalStorage, IntegrationType, Role, PlatformEntityType, SettingsValueTypes, SettingsTypes,
+        SettingsSectionNames,
+        SettingScope, BaseSetting, UserSetting, Setting, ServiceSample, ExecutionSample, PipelineExecutionSample,
+        ResourceExecution, Message, NotificationEventContext
+    )
+    from .ml import BaseModelAdapter
+    from .utilities import Converter, BaseServiceRunner, Progress, Context, AnnotationFormat
+    from .repositories import FUNCTION_END_LINE, PackageCatalog
 
     def __init__(self, cookie_filepath=None):
         self.client_api = self.ApiClient(cookie_filepath=cookie_filepath)
@@ -24,15 +71,40 @@ class Dtlpy:
         self.triggers = self.repositories.Triggers(client_api=self.client_api)
         self.assignments = self.repositories.Assignments(client_api=self.client_api)
         self.tasks = self.repositories.Tasks(client_api=self.client_api)
+        self.dpks = self.repositories.Dpks(client_api=self.client_api)
         self.annotations = self.repositories.Annotations(client_api=self.client_api)
+        self.models = self.repositories.Models(client_api=self.client_api)
+        self.ontologies = self.repositories.Ontologies(client_api=self.client_api)
+        self.recipes = self.repositories.Recipes(client_api=self.client_api)
+        self.pipelines = self.repositories.Pipelines(client_api=self.client_api)
+        self.pipeline_executions = self.repositories.PipelineExecutions(client_api=self.client_api)
+        self.feature_sets = self.repositories.FeatureSets(client_api=self.client_api)
+        self.features = self.repositories.Features(client_api=self.client_api)
+        self.organizations = self.repositories.Organizations(client_api=self.client_api)
+        self.analytics = self.repositories.Analytics(client_api=self.client_api)
+        self.integrations = self.repositories.Integrations(client_api=self.client_api)
+        self.drivers = self.repositories.Drivers(client_api=self.client_api)
+        self.settings = self.repositories.Settings(client_api=self.client_api)
+        self.apps = self.repositories.Apps(client_api=self.client_api)
+        self.dpks = self.repositories.Dpks(client_api=self.client_api)
+        self.messages = self.repositories.Messages(client_api=self.client_api)
+        self.compositions = self.repositories.Compositions(client_api=self.client_api)
+
         self.verbose = self.client_api.verbose
         self.login = self.client_api.login
+        self.logout = self.client_api.logout
         self.login_token = self.client_api.login_token
         self.login_secret = self.client_api.login_secret
+        self.login_api_key = self.client_api.login_api_key
+        self.login_m2m = self.client_api.login_m2m
         self.add_environment = self.client_api.add_environment
         self.setenv = self.client_api.setenv
         self.token_expired = self.client_api.token_expired
         self.info = self.client_api.info
+        self.cache_state = self.client_api.cache_state
+        self.attributes_mode = self.client_api.attributes_mode
+        self.sdk_cache = self.client_api.sdk_cache
+        self.platform_settings = self.client_api.platform_settings
 
     def __del__(self):
         for name, pool in self.client_api._thread_pools.items():
@@ -176,7 +248,6 @@ class Dtlpy:
         GPU_T4_S = "gpu-t4-s"
         GPU_T4_M = "gpu-t4-m"
 
-
     class LoggingLevel:
         DEBUG = 'debug'
         WARNING = 'warning'

dtlpy/repositories/apps.py

@@ -1,6 +1,6 @@
 import logging
 
-from .. import entities, exceptions, miscellaneous
+from .. import entities, exceptions, miscellaneous, repositories
 from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
@@ -11,6 +11,7 @@ class Apps:
     def __init__(self, client_api: ApiClient, project: entities.Project = None):
         self._client_api = client_api
         self._project = project
+        self._commands = None
 
     @property
     def project(self) -> entities.Project:
@@ -21,6 +22,12 @@ class Apps:
         assert isinstance(self._project, entities.Project)
         return self._project
 
+    @property
+    def commands(self) -> repositories.Commands:
+        if self._commands is None:
+            self._commands = repositories.Commands(client_api=self._client_api)
+        return self._commands
+
     @project.setter
     def project(self, project: entities.Project):
         if not isinstance(project, entities.Project):
@@ -160,12 +167,13 @@ class Apps:
         paged.get_page()
         return paged
 
-    def update(self, app: entities.App = None, app_id: str = None) -> bool:
+    def update(self, app: entities.App = None, app_id: str = None, wait: bool = True) -> bool:
         """
         Update the current app to the new configuration
 
         :param entities.App app: The app to update.
         :param str app_id: The app id to update.
+        :param bool wait: wait for the operation to finish.
         :return bool whether the operation ran successfully or not
 
         **Example**
@@ -179,16 +187,30 @@ class Apps:
         success, response = self._client_api.gen_request(req_type='put',
                                                          path=f"/apps/{app.id}",
                                                          json_req=app.to_json())
-        if success:
-            return success
-        raise exceptions.PlatformException(response)
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        app = entities.App.from_json(
+            _json=response.json(),
+            client_api=self._client_api,
+            project=self.project
+        )
+        if app.metadata:
+            command_id = app.metadata.get('system', {}).get('commands', {}).get('update', None)
+            if wait and app.status == entities.CompositionStatus.UPDATING and command_id is not None:
+                command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
+                command.wait()
+                app = self.get(app_id=app.id)
+
+        return success
 
     def install(self,
                 dpk: entities.Dpk,
                 app_name: str = None,
                 organization_id: str = None,
                 custom_installation: dict = None,
-                scope: entities.AppScope = None
+                scope: entities.AppScope = None,
+                wait: bool = True
                 ) -> entities.App:
         """
         Install the specified app in the project.
@@ -199,6 +221,7 @@ class Apps:
         :param str organization_id: the organization which you want to apply on the filter.
         :param dict custom_installation: partial installation.
         :param str scope: the scope of the app. default is project.
+        :param bool wait: wait for the operation to finish.
 
         :return the installed app.
         :rtype entities.App
@@ -229,11 +252,20 @@ class Apps:
                                                          json_req=app.to_json())
         if not success:
             raise exceptions.PlatformException(response)
-        return entities.App.from_json(_json=response.json(),
-                                      client_api=self._client_api,
-                                      project=self.project)
+        app = entities.App.from_json(_json=response.json(),
+                                     client_api=self._client_api,
+                                     project=self.project)
+
+        if app.metadata:
+            command_id = app.metadata.get('system', {}).get('commands', {}).get('install', None)
+            if wait and app.status == entities.CompositionStatus.INITIALIZING and command_id is not None:
+                command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
+                command.wait()
+                app = self.get(app_id=app.id)
+
+        return app
 
-    def uninstall(self, app_id: str = None, app_name: str = None) -> bool:
+    def uninstall(self, app_id: str = None, app_name: str = None, wait: bool = True) -> bool:
         """
         Delete an app entity.
 
@@ -241,6 +273,7 @@ class Apps:
 
         :param str app_id: optional - the id of the app.
         :param str app_name: optional - the name of the app.
+        :param bool wait: optional - wait for the operation to finish.
         :return whether we succeed uninstalling the specified app.
         :rtype bool
 
@@ -260,6 +293,19 @@ class Apps:
         if not success:
             raise exceptions.PlatformException(response)
 
+        try:
+            app = self.get(app_id=app_id)
+        except Exception as e:
+            if e.status_code == '404':
+                return success
+            else:
+                raise e
+        if app.metadata:
+            command_id = app.metadata.get('system', {}).get('commands', {}).get('uninstall', None)
+            if wait and app.status == entities.CompositionStatus.TERMINATING and command_id is not None:
+                command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
+                command.wait()
+
         logger.debug(f"App deleted successfully (id: {app_id}, name: {app_name}")
         return success
 

dtlpy/repositories/commands.py

@@ -87,17 +87,25 @@ class Commands:
         :param float backoff_factor: A backoff factor to apply between attempts after the second try
         :return: Command  object
         """
+
         elapsed = 0
         start = time.time()
         if timeout is None or timeout <= 0:
             timeout = np.inf
 
         command = None
-        pbar = tqdm.tqdm(total=100, disable=self._client_api.verbose.disable_progress_bar,
-                         file=sys.stdout, desc='Command Progress')
+        pbar = tqdm.tqdm(total=100,
+                         disable=self._client_api.verbose.disable_progress_bar,
+                         file=sys.stdout,
+                         desc='Command Progress')
         num_tries = 1
         while elapsed < timeout:
             command = self.get(command_id=command_id, url=url)
+            if command.type == 'ExportDatasetAsJson':
+                self._client_api.callbacks.run_on_event(event=self._client_api.callbacks.CallbackEvent.DATASET_EXPORT,
+                                                        context=command.spec,
+                                                        progress=command.progress)
+
             pbar.update(command.progress - pbar.n)
             if not command.in_progress():
                 break

dtlpy/repositories/datasets.py

@@ -4,10 +4,12 @@ Datasets Repository
 
 import os
 import sys
+import time
 import copy
 import tqdm
 import logging
-from urllib.parse import urlencode
+import json
+from typing import Union
 
 from .. import entities, repositories, miscellaneous, exceptions, services, PlatformException, _api_reference
 from ..services.api_client import ApiClient
@@ -93,7 +95,11 @@ class Datasets:
             filters._user_query = 'false'
         if not folder_path.startswith('/'):
             folder_path = '/' + folder_path
-        filters.add(field='dir', values=folder_path + '*')
+        filters.add(field='dir', values=folder_path, method=entities.FiltersMethod.OR)
+        if not folder_path.endswith('*'):
+            if not folder_path.endswith('/'):
+                folder_path += '/'
+            filters.add(field='dir', values=folder_path + '*', method=entities.FiltersMethod.OR)
         return filters
 
     def _get_binaries_dataset(self):
@@ -111,6 +117,70 @@ class Datasets:
             dataset = datasets[0]
         return dataset
 
+    def _resolve_dataset_id(self, dataset, dataset_name, dataset_id):
+        if dataset is None and dataset_name is None and dataset_id is None:
+            raise ValueError('Must provide dataset, dataset name or dataset id')
+        if dataset_id is None:
+            if dataset is None:
+                dataset = self.get(dataset_name=dataset_name)
+            dataset_id = dataset.id
+        return dataset_id
+
+    @staticmethod
+    def _build_payload(filters, include_feature_vectors, include_annotations, export_type, annotation_filters,
+                       feature_vector_filters):
+        valid_list = [e.value for e in entities.ExportType]
+        valid_types = ', '.join(valid_list)
+        if export_type not in ['json', 'zip']:
+            raise ValueError('export_type must be one of the following: {}'.format(valid_types))
+        payload = {'exportType': export_type}
+        if filters is None:
+            filters = entities.Filters()
+
+        if isinstance(filters, entities.Filters):
+            payload['itemsQuery'] = {'filter': filters.prepare()['filter'], 'join': filters.prepare().get("join", {})}
+        elif isinstance(filters, dict):
+            payload['itemsQuery'] = filters
+        else:
+            raise exceptions.BadRequest(message='filters must be of type dict or Filters', status_code=500)
+
+        payload['itemsVectorQuery'] = {}
+        if include_feature_vectors:
+            payload['includeItemVectors'] = True
+            payload['itemsVectorQuery']['select'] = {"datasetId": 1, 'featureSetId': 1, 'value': 1}
+
+        if feature_vector_filters is not None:
+            payload['itemsVectorQuery']['filter'] = feature_vector_filters.prepare()['filter']
+
+        payload['annotations'] = {"include": include_annotations, "convertSemantic": False}
+
+        if annotation_filters is not None:
+            payload['annotationsQuery'] = annotation_filters.prepare()['filter']
+            payload['annotations']['filter'] = True
+
+        return payload
+
+    def _download_exported_item(self, item_id, export_type, local_path=None):
+        export_item = repositories.Items(client_api=self._client_api).get(item_id=item_id)
+        export_item_path = export_item.download(local_path=local_path)
+
+        if export_type == entities.ExportType.ZIP:
+            # unzipping annotations to directory
+            if isinstance(export_item_path, list) or not os.path.isfile(export_item_path):
+                raise exceptions.PlatformException(
+                    error='404',
+                    message='error downloading annotation zip file. see above for more information. item id: {!r}'.format(
+                        export_item.id))
+            try:
+                miscellaneous.Zipping.unzip_directory(zip_filename=export_item_path,
+                                                      to_directory=local_path)
+            except Exception as e:
+                logger.warning("Failed to extract zip file error: {}".format(e))
+            finally:
+                # cleanup
+                if isinstance(export_item_path, str) and os.path.isfile(export_item_path):
+                    os.remove(export_item_path)
+
     @property
     def platform_url(self):
         return self._client_api._get_resource_url("projects/{}/datasets".format(self.project.id))
@@ -425,12 +495,7 @@ class Datasets:
             directory_tree = dataset.directory_tree
             directory_tree = project.datasets.directory_tree(dataset='dataset_entity')
         """
-        if dataset is None and dataset_name is None and dataset_id is None:
-            raise exceptions.PlatformException('400', 'Must provide dataset, dataset name or dataset id')
-        if dataset_id is None:
-            if dataset is None:
-                dataset = self.get(dataset_name=dataset_name)
-            dataset_id = dataset.id
+        dataset_id = self._resolve_dataset_id(dataset, dataset_name, dataset_id)
 
         url_path = '/datasets/{}/directoryTree'.format(dataset_id)
 
@@ -519,6 +584,72 @@ class Datasets:
                                                .format(response))
         return self.get(dataset_id=command.spec['returnedModelId'])
 
+    @_api_reference.add(path='/datasets/{id}/export', method='post')
+    def export(self,
+               dataset: entities.Dataset = None,
+               dataset_name: str = None,
+               dataset_id: str = None,
+               local_path: str = None,
+               filters: Union[dict, entities.Filters] = None,
+               annotation_filters: entities.Filters = None,
+               feature_vector_filters: entities.Filters = None,
+               include_feature_vectors: bool = False,
+               include_annotations: bool = False,
+               export_type: entities.ExportType = entities.ExportType.JSON,
+               timeout: int = 0):
+        """
+        Export dataset items and annotations.
+
+        **Prerequisites**: You must be an *owner* or *developer* to use this method.
+
+        You must provide at least ONE of the following params: dataset, dataset_name, dataset_id.
+
+        :param dtlpy.entities.dataset.Dataset dataset: Dataset object
+        :param str dataset_name: The name of the dataset
+        :param str dataset_id: The ID of the dataset
+        :param str local_path: Local path to save the exported dataset 
+        :param Union[dict, dtlpy.entities.filters.Filters] filters: Filters entity or a query dictionary
+        :param dtlpy.entities.filters.Filters annotation_filters: Filters entity to filter annotations for export       
+        :param dtlpy.entities.filters.Filters feature_vector_filters: Filters entity to filter feature vectors for export
+        :param bool include_feature_vectors: Include item feature vectors in the export
+        :param bool include_annotations: Include item annotations in the export
+        :param entities.ExportType export_type: Type of export ('json' or 'zip')
+        :param int timeout: Maximum time in seconds to wait for the export to complete
+        :return: Exported item
+        :rtype: dtlpy.entities.item.Item
+
+        **Example**:
+
+        .. code-block:: python
+
+            export_item = project.datasets.export(dataset_id='dataset_id',
+                                                  filters=filters,
+                                                  include_feature_vectors=True,
+                                                  include_annotations=True,
+                                                  export_type=dl.ExportType.JSON)
+        """
+        dataset_id = self._resolve_dataset_id(dataset, dataset_name, dataset_id)
+        payload = self._build_payload(filters, include_feature_vectors, include_annotations, export_type,
+                                      annotation_filters, feature_vector_filters)
+
+        success, response = self._client_api.gen_request(req_type='post', path=f'/datasets/{dataset_id}/export',
+                                                         json_req=payload)
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        command = entities.Command.from_json(_json=response.json(),
+                                             client_api=self._client_api)
+
+        time.sleep(2)  # as the command have wrong progress in the beginning
+        command = command.wait(timeout=timeout)
+        if 'outputItemId' not in command.spec:
+            raise exceptions.PlatformException(
+                error='400',
+                message="outputItemId key is missing in command response: {}".format(response))
+        item_id = command.spec['outputItemId']
+        self._download_exported_item(item_id=item_id, export_type=export_type, local_path=local_path)
+        return local_path
+
     @_api_reference.add(path='/datasets/merge', method='post')
     def merge(self,
               merge_name: str,
@@ -769,7 +900,7 @@ class Datasets:
         Download dataset's annotations by filters.
 
         You may filter the dataset both for items and for annotations and download annotations.
-        
+
         Optional -- download annotations as: mask, instance, image mask of the item.
 
         **Prerequisites**: You must be in the role of an *owner* or *developer*.
@@ -917,9 +1048,8 @@ class Datasets:
                            ):
         """
         Upload annotations to dataset. 
-        
-        Example for remote_root_path: If the item filepath is a/b/item and
-        remote_root_path is /a the start folder will be b instead of a
+
+        Example for remote_root_path: If the item filepath is "/a/b/item" and remote_root_path is "/a" - the start folder will be b instead of a
 
         **Prerequisites**: You must have a dataset with items that are related to the annotations. The relationship between the dataset and annotations is shown in the name. You must be in the role of an *owner* or *developer*.
 

dtlpy/repositories/dpks.py

@@ -196,7 +196,8 @@ class Dpks:
             dpk = dpk_v.items[0]
         return dpk
 
-    def publish(self, dpk: entities.Dpk = None, ignore_max_file_size: bool = False, manifest_filepath='dataloop.json') -> entities.Dpk:
+    def publish(self, dpk: entities.Dpk = None, ignore_max_file_size: bool = False,
+                manifest_filepath='dataloop.json') -> entities.Dpk:
         """
         Upload a dpk entity to the dataloop platform.
 
@@ -290,6 +291,9 @@ class Dpks:
         elif filters.resource != entities.FiltersResource.DPK:
             raise ValueError('Filters resource must to be FiltersResource.DPK. Got: {!r}'.format(filters.resource))
 
+        if self._project is not None:
+            filters.add(field='context.project', values=self._project.id)
+
         paged = entities.PagedEntities(items_repository=self,
                                        filters=filters,
                                        page_offset=filters.page,

dtlpy/repositories/feature_sets.py

@@ -11,11 +11,16 @@ class FeatureSets:
     """
     URL = '/features/sets'
 
-    def __init__(self, client_api: ApiClient,
+    def __init__(self,
+                 client_api: ApiClient,
                  project_id: str = None,
-                 project: entities.Project = None):
+                 project: entities.Project = None,
+                 model_id: str = None,
+                 model: entities.Model = None):
         self._project = project
         self._project_id = project_id
+        self._model = model
+        self._model_id = model_id
         self._client_api = client_api
 
     ############
@@ -34,11 +39,23 @@ class FeatureSets:
         if self._project is None:
             raise exceptions.PlatformException(
                 error='2001',
-                message='Cannot perform action WITHOUT Project entity in Datasets repository.'
+                message='Cannot perform action WITHOUT Project entity in FeatureSets repository.'
                         ' Please checkout or set a project')
         assert isinstance(self._project, entities.Project)
         return self._project
 
+    @property
+    def model(self) -> entities.Model:
+        if self._model is None and self._model_id is not None:
+            # get from id
+            self._model = repositories.Models(client_api=self._client_api).get(model_id=self._model_id)
+        if self._model is None:
+            raise exceptions.PlatformException(
+                error='2001',
+                message='Cannot perform action WITHOUT Model entity in FeatureSets repository.')
+        assert isinstance(self._model, entities.Model)
+        return self._model
+
     ###########
     # methods #
     ###########
@@ -132,6 +149,7 @@ class FeatureSets:
                set_type: str,
                entity_type: entities.FeatureEntityType,
                project_id: str = None,
+               model_id: set = None,
                org_id: str = None):
         """
         Create a new Feature Set
@@ -141,6 +159,7 @@ class FeatureSets:
         :param str set_type: string of the feature type: 2d, 3d, modelFC, TSNE,PCA,FFT
         :param entity_type: the entity that feature vector is linked to. Use the enum dl.FeatureEntityType
         :param str project_id: the ID of the project where feature set will be created
+        :param str model_id: the ID of the model that creates the vectors
         :param str org_id: the ID of the org where feature set will be created
         :return: Feature Set object
         """
@@ -154,6 +173,7 @@ class FeatureSets:
                    'size': size,
                    'type': set_type,
                    'project': project_id,
+                   'modelId': model_id,
                    'entityType': entity_type}
         if org_id is not None:
             payload['org'] = org_id

dtlpy/repositories/models.py

@@ -668,7 +668,7 @@ class Models:
         :param item_ids: a list of item id to run the prediction.
         :return:
         """
-        if len(model.metadata['system']['deploy']['services']) == 0:
+        if len(model.metadata['system'].get('deploy', {}).get('services', [])) == 0:
             # no services for model
             raise ValueError("Model doesnt have any associated services. Need to deploy before predicting")
         payload = {'input': {'itemIds': item_ids},

dtlpy/repositories/pipeline_executions.py

@@ -1,8 +1,12 @@
 import logging
+import time
+import numpy as np
+
 from .. import entities, repositories, exceptions, miscellaneous, services, _api_reference
 from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
+MAX_SLEEP_TIME = 30
 
 
 class PipelineExecutions:
@@ -379,3 +383,52 @@ class PipelineExecutions:
                                                message="cycleOptions key is missing in command response: {!r}"
                                                .format(response))
         return True
+
+    def wait(self,
+             pipeline_execution_id: str = None,
+             pipeline_execution: entities.PipelineExecution = None,
+             timeout: int = None,
+             backoff_factor=1):
+        """
+        Get Service execution object.
+
+        **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
+
+        :param str pipeline_execution_id: pipeline execution id
+        :param str pipeline_execution: dl.PipelineExecution, optional. must input one of pipeline execution or pipeline_execution_id
+        :param int timeout: seconds to wait until TimeoutError is raised. if <=0 - wait until done - by default wait take the service timeout
+        :param float backoff_factor: A backoff factor to apply between attempts after the second try
+        :return: Service execution object
+        :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
+
+        **Example**:
+
+        .. code-block:: python
+
+            pipeline.pipeline_executions.wait(pipeline_execution_id='pipeline_execution_id')
+        """
+        if pipeline_execution is None:
+            if pipeline_execution_id is None:
+                raise ValueError('Must input at least one: [pipeline_execution, pipeline_execution_id]')
+        else:
+            pipeline_execution_id = pipeline_execution.id
+        elapsed = 0
+        start = time.time()
+        if timeout is None or timeout <= 0:
+            timeout = np.inf
+
+        num_tries = 1
+        while elapsed < timeout:
+            pipeline_execution = self.get(pipeline_execution_id=pipeline_execution_id)
+            if not pipeline_execution.in_progress():
+                break
+            elapsed = time.time() - start
+            if elapsed >= timeout:
+                raise TimeoutError(
+                    f"Pipeline execution wait() function timed out. id: {pipeline_execution.id!r}, status: {pipeline_execution.status}.")
+            sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
+            num_tries += 1
+            logger.debug(
+                f"Pipeline execution {pipeline_execution.id!r} has been running for {elapsed:.2f}[s]. Sleeping for {sleep_time:.2f}[s]")
+            time.sleep(sleep_time)
+        return pipeline_execution

dtlpy/repositories/uploader.py

@@ -522,6 +522,9 @@ class Uploader:
                                    ref=item.id)
                 if pbar is not None:
                     pbar.update()
+                    self.items_repository._client_api.callbacks.run_on_event(event=self.items_repository._client_api.callbacks.CallbackEvent.ITEMS_UPLOAD,
+                                                                             context={'item_id': item.id, 'dataset_id': item.dataset_id},
+                                                                             progress=round(pbar.n / pbar.total * 100, 0))
             else:
                 if isinstance(element.buffer, str):
                     ref = element.buffer