dtlpy 1.73.8__py3-none-any.whl → 1.74.14__py3-none-any.whl

dtlpy/entities/pipeline_execution.py

@@ -3,7 +3,8 @@ import logging
 import traceback
 import attr
 
-from .. import repositories, entities, services
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -63,10 +64,11 @@ class PipelineExecution(entities.BaseEntity):
     updated_at = attr.ib(repr=False)
     pipeline_id = attr.ib()
     max_attempts = attr.ib()
+    creator = attr.ib()
 
     # sdk
     _pipeline = attr.ib(repr=False)
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
 
     @staticmethod
@@ -126,6 +128,7 @@ class PipelineExecution(entities.BaseEntity):
             pipeline_id=_json.get('pipelineId', None),
             status=_json.get('status', None),
             max_attempts=_json.get('maxAttempts', None),
+            creator=_json.get('creator', None),
             nodes=nodes,
             executions=executions,
             pipeline=pipeline,
@@ -150,7 +153,7 @@ class PipelineExecution(entities.BaseEntity):
                                                         attr.fields(PipelineExecution).updated_at,
                                                         attr.fields(PipelineExecution).pipeline_id,
                                                         attr.fields(PipelineExecution).executions,
-                                                        attr.fields(PipelineExecution).max_attempts,
+                                                        attr.fields(PipelineExecution).max_attempts
                                                         ))
         executions = dict()
         for node_id, executions_list in self.executions.items():

dtlpy/entities/project.py

@@ -4,7 +4,8 @@ import traceback
 import logging
 import attr
 
-from .. import repositories, miscellaneous, services, entities
+from .. import repositories, miscellaneous, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -37,7 +38,7 @@ class Project(entities.BaseEntity):
     feature_constraints = attr.ib()
 
     # api
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
 
     # repositories
     _repositories = attr.ib(repr=False)

dtlpy/entities/recipe.py

@@ -1,11 +1,12 @@
-import os
-import traceback
-import uuid
 from collections import namedtuple
+import traceback
 import logging
+import uuid
 import attr
+import os
 
-from .. import repositories, entities, services, exceptions
+from .. import repositories, entities, exceptions
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -31,7 +32,7 @@ class Recipe(entities.BaseEntity):
     ui_settings = attr.ib()
 
     # platform
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     # entities
     _dataset = attr.ib(repr=False, default=None)
     _project = attr.ib(repr=False, default=None)

dtlpy/entities/resource_execution.py

@@ -2,7 +2,8 @@ import attr
 import logging
 import traceback
 
-from .. import repositories, entities, services
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -30,7 +31,7 @@ class ResourceExecution(entities.BaseEntity):
     project_id = attr.ib()
 
     # sdk
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _project = attr.ib()
     resource = attr.ib()
 
@@ -135,4 +136,3 @@ class ResourceExecution(entities.BaseEntity):
                                                                                    fetch=None)
         assert isinstance(self._project, entities.Project)
         return self._project
-

dtlpy/entities/service.py

@@ -5,7 +5,8 @@ import logging
 from typing import List
 from urllib.parse import urlsplit
 import attr
-from .. import services, repositories, entities
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -26,6 +27,24 @@ class ServiceType(str, Enum):
     REGULAR = 'regular'
 
 
+class ServiceModeType(str, Enum):
+    """ The type of the service mode.
+
+    .. list-table::
+       :widths: 15 150
+       :header-rows: 1
+
+       * - State
+         - Description
+       * - REGULAR
+         - Service regular mode type
+       * - DEBUG
+         - Service debug mode type
+    """
+    REGULAR = 'regular'
+    DEBUG = 'debug'
+
+
 class OnResetAction(str, Enum):
     """ The Execution action when the service reset (RERUN, FAILED).
 
@@ -200,10 +219,11 @@ class Service(entities.BaseEntity):
     project_id = attr.ib()
     is_global = attr.ib()
     max_attempts = attr.ib()
+    mode = attr.ib(repr=False)
 
     # SDK
     _package = attr.ib(repr=False)
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _revisions = attr.ib(default=None, repr=False)
     # repositories
     _project = attr.ib(default=None, repr=False)
@@ -218,7 +238,7 @@ class Service(entities.BaseEntity):
         return self.updated_at
 
     @staticmethod
-    def _protected_from_json(_json: dict, client_api: services.ApiClient, package=None, project=None, is_fetched=True):
+    def _protected_from_json(_json: dict, client_api: ApiClient, package=None, project=None, is_fetched=True):
         """
         Same as from_json but with try-except to catch if error
 
@@ -242,7 +262,7 @@ class Service(entities.BaseEntity):
         return status, service
 
     @classmethod
-    def from_json(cls, _json: dict, client_api: services.ApiClient = None, package=None, project=None, is_fetched=True):
+    def from_json(cls, _json: dict, client_api: ApiClient=None, package=None, project=None, is_fetched=True):
         """
         Build a service entity object from a json
 
@@ -300,7 +320,8 @@ class Service(entities.BaseEntity):
             package=package,
             project=project,
             secrets=_json.get("secrets", None),
-            type=_json.get("type", None)
+            type=_json.get("type", None),
+            mode=_json.get('mode', dict()),
         )
         inst.is_fetched = is_fetched
         return inst
@@ -419,6 +440,7 @@ class Service(entities.BaseEntity):
                 attr.fields(Service).updated_at,
                 attr.fields(Service).secrets,
                 attr.fields(Service)._type,
+                attr.fields(Service).mode,
             )
         )
 
@@ -456,6 +478,9 @@ class Service(entities.BaseEntity):
         if self._type is not None:
             _json['type'] = self._type
 
+        if self.mode:
+            _json['mode'] = self.mode
+
         return _json
 
     def update(self, force=False):
@@ -633,7 +658,7 @@ class Service(entities.BaseEntity):
 
             command = service.execute_batch(
                         execution_inputs=dl.FunctionIO(type=dl.PackageInputType.STRING, value='test', name='string'),
-                        filters=dl.Filters(field='dir', values='/test'),
+                        filters=dl.Filters(field='dir', values='/test', context={"datasets": [dataset.id]),
                         function_name='run')
         """
         execution = self.executions.create_batch(service_id=self.id,

dtlpy/entities/trigger.py

@@ -4,7 +4,8 @@ import logging
 from enum import Enum
 from collections import namedtuple
 
-from .. import entities, services, exceptions, repositories
+from .. import entities, exceptions, repositories
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -79,7 +80,7 @@ class BaseTrigger(entities.BaseEntity):
     ##################
     _service = attr.ib(repr=False)
     _project = attr.ib(repr=False)
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _op_type = attr.ib(default='service')
     _repositories = attr.ib(repr=False)
 

dtlpy/entities/webhook.py

@@ -2,7 +2,8 @@ import logging
 import attr
 from enum import Enum
 
-from .. import services, repositories, entities
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -33,7 +34,7 @@ class Webhook(entities.BaseEntity):
     hook_url = attr.ib()
 
     # SDK
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _project = attr.ib()
 
     # repos
@@ -48,7 +49,7 @@ class Webhook(entities.BaseEntity):
         return self.updated_at
 
     @classmethod
-    def from_json(cls, _json: dict, client_api: services.ApiClient, project=None):
+    def from_json(cls, _json: dict, client_api: ApiClient, project=None):
         """
         :param _json: platform json
         :param client_api: ApiClient entity

dtlpy/ml/base_model_adapter.py

@@ -315,21 +315,23 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
             else:
                 raise ValueError('Unknown inputType: {} (from model_entity.input_type'.format(input_type))
             batch_collections = self.predict(batch, **kwargs)
-            for collection in batch_collections:
-                # convert annotation collection to a list aof annotations jsons
-                if isinstance(collection, entities.AnnotationCollection):
-                    annotations.append(collection.to_json()['annotations'])
-                else:
-                    annotations.append(collection)
             if upload_annotations is True:
                 self.logger.debug(
                     "Uploading items' annotation for model {!r}.".format(self.model_entity.name))
                 try:
-                    annotations = list(pool.map(partial(self._upload_model_annotations),
-                                                batch_items, batch_collections))
+                    batch_collections = list(pool.map(partial(self._upload_model_annotations),
+                                                      batch_items, batch_collections))
                 except Exception as err:
                     self.logger.exception("Failed to upload annotations items.")
 
+            for collection in batch_collections:
+                # function needs to return `List[List[dl.Annotation]]`
+                # convert annotation collection to a list of dl.Annotation for each batch
+                if isinstance(collection, entities.AnnotationCollection):
+                    annotations.extend([annotation for annotation in collection.annotations])
+                else:
+                    logger.warning(f'RETURN TYPE MAY BE INVALID: {type(collection)}')
+                    annotations.extend(collection)
         pool.shutdown()
         return items, annotations
 

dtlpy/new_instance.py

@@ -7,7 +7,8 @@ class Dtlpy:
         AnnotationCollection, Annotation, Item, Codebase, Filters, Execution, Recipe, Ontology, Label, Similarity, \
         ItemLink, UrlLink, PackageModule, PackageFunction, FunctionIO, Modality, Workload, WorkloadUnit
     from .utilities import Converter, BaseServiceRunner, Progress
-    from .services import ApiClient, check_sdk
+    from .services.api_client import ApiClient
+    from .services import check_sdk
 
     def __init__(self, cookie_filepath=None):
         self.client_api = self.ApiClient(cookie_filepath=cookie_filepath)

dtlpy/repositories/analytics.py

@@ -1,7 +1,7 @@
 import logging
 import pandas as pd
-
-from dtlpy import entities, exceptions, services
+from dtlpy import entities, exceptions
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -11,7 +11,7 @@ class Analytics:
     Time series Repository
     """
 
-    def __init__(self, client_api: services.ApiClient, project: entities.Project = None):
+    def __init__(self, client_api: ApiClient, project: entities.Project = None):
         self._client_api = client_api
         self._project = project
 

dtlpy/repositories/annotations.py

@@ -5,7 +5,8 @@ import json
 import jwt
 import os
 
-from .. import entities, exceptions, miscellaneous, services, _api_reference
+from .. import entities, exceptions, miscellaneous, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -21,7 +22,7 @@ class Annotations:
     `Show Video with Annotations <https://dataloop.ai/docs/sdk-show-videos>`_.
     """
 
-    def __init__(self, client_api: services.ApiClient, item=None, dataset=None, dataset_id=None):
+    def __init__(self, client_api: ApiClient, item=None, dataset=None, dataset_id=None):
         self._client_api = client_api
         self._item = item
         self._dataset = dataset

dtlpy/repositories/apps.py

@@ -1,13 +1,14 @@
 import logging
 
-from .. import entities, services, exceptions, miscellaneous
+from .. import entities, exceptions, miscellaneous
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
 
 class Apps:
 
-    def __init__(self, client_api: services.ApiClient, project: entities.Project = None):
+    def __init__(self, client_api: ApiClient, project: entities.Project = None):
         self._client_api = client_api
         self._project = project
 

dtlpy/repositories/artifacts.py

@@ -4,7 +4,8 @@ import logging
 import shutil
 import os
 
-from .. import entities, miscellaneous, PlatformException, exceptions, services, repositories
+from .. import entities, miscellaneous, PlatformException, exceptions, repositories
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -15,7 +16,7 @@ class Artifacts:
     """
 
     def __init__(self,
-                 client_api: services.ApiClient,
+                 client_api: ApiClient,
                  project: entities.Project = None,
                  dataset: entities.Dataset = None,
                  project_id: str = None,

dtlpy/repositories/assignments.py

@@ -1,6 +1,7 @@
 import logging
 
-from .. import exceptions, miscellaneous, entities, repositories, services, _api_reference
+from .. import exceptions, miscellaneous, entities, repositories, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -13,7 +14,7 @@ class Assignments:
     """
 
     def __init__(self,
-                 client_api: services.ApiClient,
+                 client_api: ApiClient,
                  project: entities.Project = None,
                  task: entities.Task = None,
                  dataset: entities.Dataset = None,

dtlpy/repositories/bots.py

@@ -1,5 +1,6 @@
 import logging
-from .. import entities, miscellaneous, exceptions, services
+from .. import entities, miscellaneous, exceptions
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -11,7 +12,7 @@ class Bots:
     The Bots class allows the user to manage bots and their properties. See our documentation for more information on `bots <https://dataloop.ai/docs/faas-bot>`_.
     """
 
-    def __init__(self, client_api: services.ApiClient, project: entities.Project):
+    def __init__(self, client_api: ApiClient, project: entities.Project):
         self._client_api = client_api
         self._project = project
 

dtlpy/repositories/codebases.py

@@ -5,7 +5,9 @@ import io
 import random
 from typing import List
 
-from .. import entities, PlatformException, exceptions, repositories, miscellaneous, services
+from .. import entities, PlatformException, exceptions, repositories, miscellaneous
+from ..services.api_client import ApiClient
+from ..services.api_client import client as client_api
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -18,7 +20,7 @@ class Codebases:
     """
 
     def __init__(self,
-                 client_api: services.ApiClient,
+                 client_api: ApiClient,
                  project: entities.Project = None,
                  dataset: entities.Dataset = None,
                  project_id: str = None):
@@ -35,7 +37,7 @@ class Codebases:
             if self._dataset is not None or self._project is not None:
                 self._items_repository = self.dataset.items
             else:
-                self._items_repository = repositories.Items(client_api=services.ApiClient())
+                self._items_repository = repositories.Items(client_api=client_api)
         assert isinstance(self._items_repository, repositories.Items)
         return self._items_repository
 

dtlpy/repositories/commands.py

@@ -1,11 +1,12 @@
+import numpy as np
+import warnings
 import logging
-import sys
 import time
-import numpy as np
 import tqdm
-import warnings
+import sys
 
-from .. import exceptions, entities, services, miscellaneous
+from .. import exceptions, entities, miscellaneous
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -17,7 +18,7 @@ class Commands:
     Service Commands repository
     """
 
-    def __init__(self, client_api: services.ApiClient):
+    def __init__(self, client_api: ApiClient):
         self._client_api = client_api
 
     ############

dtlpy/repositories/datasets.py

@@ -8,7 +8,9 @@ import copy
 import tqdm
 import logging
 from urllib.parse import urlencode
+
 from .. import entities, repositories, miscellaneous, exceptions, services, PlatformException, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -20,7 +22,7 @@ class Datasets:
     The Datasets class allows the user to manage datasets. Read more about datasets in our `documentation <https://dataloop.ai/docs/dataset>`_ and `SDK documentation <https://dataloop.ai/docs/sdk-create-dataset>`_.
     """
 
-    def __init__(self, client_api: services.ApiClient, project: entities.Project = None):
+    def __init__(self, client_api: ApiClient, project: entities.Project = None):
         self._client_api = client_api
         self._project = project
 

dtlpy/repositories/dpks.py

@@ -4,12 +4,13 @@ import os
 from typing import List, Optional
 
 from .. import exceptions, entities, services, miscellaneous, assets
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
 
 class Dpks:
-    def __init__(self, client_api: services.ApiClient, project: entities.Project = None):
+    def __init__(self, client_api: ApiClient, project: entities.Project = None):
         self._client_api = client_api
         self._project = project
         self._revisions = None

dtlpy/repositories/drivers.py

@@ -1,6 +1,7 @@
 import logging
 
-from .. import entities, miscellaneous, exceptions, services, _api_reference
+from .. import entities, miscellaneous, exceptions, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -12,7 +13,7 @@ class Drivers:
     The Drivers class allows users to manage drivers that are used to connect with external storage. Read more about external storage in our `documentation <https://dataloop.ai/docs/storage>`_ and `SDK documentation <https://dataloop.ai/docs/sdk-sync-storage>`_.
     """
 
-    def __init__(self, client_api: services.ApiClient, project: entities.Project = None):
+    def __init__(self, client_api: ApiClient, project: entities.Project = None):
         self._client_api = client_api
         self._project = project
 
@@ -87,7 +88,7 @@ class Drivers:
             driver_class = entities.S3Driver
         elif driver_type == entities.ExternalStorage.GCS:
             driver_class = entities.GcsDriver
-        elif driver_type == entities.ExternalStorage.AZUREBLOB:
+        elif driver_type in [entities.ExternalStorage.AZUREBLOB, entities.ExternalStorage.AZURE_DATALAKE_GEN2]:
             driver_class = entities.AzureBlobDriver
         else:
             driver_class = entities.Driver

dtlpy/repositories/executions.py

@@ -3,7 +3,8 @@ import logging
 import time
 import numpy as np
 
-from .. import exceptions, entities, repositories, miscellaneous, services, _api_reference
+from .. import exceptions, entities, repositories, miscellaneous, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -16,7 +17,7 @@ class Executions:
     """
 
     def __init__(self,
-                 client_api: services.ApiClient,
+                 client_api: ApiClient,
                  service: entities.Service = None,
                  project: entities.Project = None):
         self._client_api = client_api

dtlpy/repositories/feature_sets.py

@@ -1,5 +1,6 @@
 import logging
-from .. import exceptions, entities, services, miscellaneous, _api_reference
+from .. import exceptions, entities, miscellaneous, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -10,7 +11,7 @@ class FeatureSets:
     """
     URL = '/features/sets'
 
-    def __init__(self, client_api: services.ApiClient, project: entities.Project = None):
+    def __init__(self, client_api: ApiClient, project: entities.Project = None):
         self._project = project
         self._client_api = client_api
 

dtlpy/repositories/features.py

@@ -1,8 +1,7 @@
 import logging
-import time
-import numpy as np
 
-from .. import exceptions, entities, services, miscellaneous, _api_reference
+from .. import exceptions, entities, miscellaneous, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -13,7 +12,7 @@ class Features:
     """
     URL = '/features/vectors'
 
-    def __init__(self, client_api: services.ApiClient,
+    def __init__(self, client_api: ApiClient,
                  project: entities.Project = None,
                  item: entities.Item = None,
                  annotation: entities.Annotation = None,
@@ -127,7 +126,7 @@ class Features:
                parent_id: str = None,
                org_id: str = None,
                refs: dict = None
-            ):
+               ):
         """
         Create a new Feature vector
 

dtlpy/repositories/integrations.py

@@ -3,7 +3,8 @@ Integrations Repository
 """
 
 import logging
-from .. import entities, exceptions, services, miscellaneous, _api_reference
+from .. import entities, exceptions, miscellaneous, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -18,7 +19,7 @@ class Integrations:
 
     """
 
-    def __init__(self, client_api: services.ApiClient, org: entities.Organization = None,
+    def __init__(self, client_api: ApiClient, org: entities.Organization = None,
                  project: entities.Project = None):
         self._client_api = client_api
         self._org = org

dtlpy/repositories/items.py

@@ -1,6 +1,7 @@
 import logging
 
-from .. import entities, exceptions, repositories, miscellaneous, services, _api_reference
+from .. import entities, exceptions, repositories, miscellaneous, _api_reference
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -13,7 +14,7 @@ class Items:
     """
 
     def __init__(self,
-                 client_api: services.ApiClient,
+                 client_api: ApiClient,
                  datasets: repositories.Datasets = None,
                  dataset: entities.Dataset = None,
                  dataset_id=None,

dtlpy/repositories/models.py

@@ -2,7 +2,8 @@ from typing import List
 import logging
 import os
 
-from .. import entities, repositories, exceptions, miscellaneous, services
+from .. import entities, repositories, exceptions, miscellaneous
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -13,7 +14,7 @@ class Models:
     """
 
     def __init__(self,
-                 client_api: services.ApiClient,
+                 client_api: ApiClient,
                  package: entities.Package = None,
                  project: entities.Project = None,
                  project_id: str = None):
@@ -482,8 +483,9 @@ class Models:
                                                          path=f"/ml/models/{model_id}/train")
         if not success:
             raise exceptions.PlatformException(response)
-
-        return True
+        return entities.Execution.from_json(_json=response.json(),
+                                            client_api=self._client_api,
+                                            project=self._project)
 
     def predict(self, model, item_ids):
         """