dtlpy 1.73.8__py3-none-any.whl → 1.74.14__py3-none-any.whl

dtlpy/__init__.py

@@ -13,24 +13,61 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with DTLPY.  If not, see <http://www.gnu.org/licenses/>.
+import warnings
 import logging
 import sys
 import os
 
-from . import services as dtlpy_services
-from .services import DataloopLogger, DtlpyFilter, ApiClient, check_sdk, Reporter, VerboseLoggingLevel, service_defaults
+##########
+# Logger #
+##########
+from .services import DataloopLogger, DtlpyFilter
+from .__version__ import version as __version__
+
+logger = logging.getLogger(name='dtlpy')
+logging.getLogger(name='filelock').setLevel(level=logging.CRITICAL)
+if len(logger.handlers) == 0:
+    logger.setLevel(logging.DEBUG)
+    log_filepath = DataloopLogger.get_log_filepath()
+    # set file handler to save all logs to file
+    stream_formatter = logging.Formatter(
+        fmt="[%(asctime)-s][%(levelname).3s][%(name)s:v" + __version__ + "][%(relativepath)-s:%(lineno)-d] %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    file_formatter = logging.Formatter(
+        fmt="[%(asctime)s.%(msecs)03d][%(threadName)s][%(levelname).3s][%(name)s:v" + __version__ + "][%(relativepath)-s:%(lineno)-d](%(funcName)-s): %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    package_path = os.path.dirname(__file__)
+    # relative function path filtering
+    filtering = DtlpyFilter(package_path)
+    fh = DataloopLogger(log_filepath, maxBytes=(1048 * 1000 * 5))
+    fh.setLevel(logging.DEBUG)
+    fh.setFormatter(file_formatter)
+    fh.addFilter(filtering)
+    sh = logging.StreamHandler()
+    sh.setLevel(logging.WARNING)
+    sh.setFormatter(stream_formatter)
+    sh.addFilter(filtering)
+    # set handlers to main logger
+    logger.addHandler(sh)
+    logger.addHandler(fh)
+
+from .services.api_client import client as client_api
+from .services.api_client import VerboseLoggingLevel, ApiClient
+from .services import DataloopLogger, DtlpyFilter, check_sdk, Reporter, service_defaults
 from .services.api_reference import api_reference as _api_reference
 from .caches.cache import CacheConfig, CacheType
 from .exceptions import PlatformException
 from . import repositories, exceptions, entities, examples
-from .__version__ import version as __version__
 from .entities import (
     # main entities
     Project, Dataset, ExpirationOptions, ExportVersion, Trigger, Item, Execution, AnnotationCollection, Annotation,
     Recipe, IndexDriver, AttributesTypes, AttributesRange, Dpk, App,
     Ontology, Label, Task, TaskPriority, Assignment, Service, Package, Codebase, Model, PackageModule, PackageFunction,
     # annotations
-    Box, Cube, Cube3d, Point, Note, Message, Segmentation, Ellipse, Classification, Subtitle, Polyline, Pose, Description,
+    Box, Cube, Cube3d, Point, Note, Message, Segmentation, Ellipse, Classification, Subtitle, Polyline, Pose,
+    Description,
     Polygon, Text,
     # filters
     Filters, FiltersKnownFields, FiltersResource, FiltersOperations, FiltersMethod, FiltersOrderByDirection,
@@ -39,7 +76,7 @@ from .entities import (
     TriggerResource, TriggerAction, TriggerExecutionMode, TriggerType,
     # faas
     FunctionIO, KubernetesAutuscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRuntime,
-    InstanceCatalog, PackageInputType, ServiceType,
+    InstanceCatalog, PackageInputType, ServiceType, ServiceModeType,
     PackageSlot, SlotPostAction, SlotPostActionType, SlotDisplayScope, SlotDisplayScopeResource, UiBindingPanel,
     # roberto
     DatasetSubsetType, LogSample, ArtifactType, Artifact, ItemArtifact, LinkArtifact, LocalArtifact, EntityScopeLevel,
@@ -66,7 +103,6 @@ from .entities import (
 from .ml import BaseModelAdapter
 from .utilities import Converter, BaseServiceRunner, Progress, Context, AnnotationFormat
 from .repositories import FUNCTION_END_LINE, PackageCatalog
-import warnings
 
 warnings.simplefilter('once', DeprecationWarning)
 
@@ -88,43 +124,11 @@ if os.name == "nt":
 """
 Main Platform Interface module for Dataloop
 """
-##########
-# Logger #
-##########
-logger = logging.getLogger(name='dtlpy')
-logging.getLogger(name='filelock').setLevel(level=logging.CRITICAL)
-if len(logger.handlers) == 0:
-    logger.setLevel(logging.DEBUG)
-    log_filepath = DataloopLogger.get_log_filepath()
-    # set file handler to save all logs to file
-    stream_formatter = logging.Formatter(
-        fmt="[%(asctime)-s][%(levelname).3s][%(name)s:v" + __version__ + "][%(relativepath)-s:%(lineno)-d] %(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S",
-    )
-    file_formatter = logging.Formatter(
-        fmt="[%(asctime)s.%(msecs)03d][%(threadName)s][%(levelname).3s][%(name)s:v" + __version__ + "][%(relativepath)-s:%(lineno)-d](%(funcName)-s): %(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S",
-    )
-    package_path = os.path.dirname(__file__)
-    # relative function path filtering
-    filtering = DtlpyFilter(package_path)
-    fh = DataloopLogger(log_filepath, maxBytes=(1048 * 1000 * 5))
-    fh.setLevel(logging.DEBUG)
-    fh.setFormatter(file_formatter)
-    fh.addFilter(filtering)
-    sh = logging.StreamHandler()
-    sh.setLevel(logging.WARNING)
-    sh.setFormatter(stream_formatter)
-    sh.addFilter(filtering)
-    # set handlers to main logger
-    logger.addHandler(sh)
-    logger.addHandler(fh)
 
 ################
 # Repositories #
 ################
 # Create repositories instances
-client_api = ApiClient()
 projects = repositories.Projects(client_api=client_api)
 datasets = repositories.Datasets(client_api=client_api, project=None)
 items = repositories.Items(client_api=client_api, datasets=datasets)
@@ -472,3 +476,8 @@ ATTRIBUTES_TYPES_FREE_TEXT = AttributesTypes.FREE_TEXT
 TASK_PRIORITY_LOW = TaskPriority.LOW
 TASK_PRIORITY_MEDIUM = TaskPriority.MEDIUM
 TASK_PRIORITY_HIGH = TaskPriority.HIGH
+
+SERVICE_MODE_TYPE_REGULAR = ServiceModeType.REGULAR
+SERVICE_MODE_TYPE_DEBUG = ServiceModeType.DEBUG
+
+

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.73.8'
+version = '1.74.14'

dtlpy/entities/__init__.py

@@ -44,7 +44,7 @@ from .package_slot import PackageSlot, SlotPostAction, SlotPostActionType, SlotD
 from .package_function import PackageFunction, FunctionIO, PackageInputType
 from .time_series import TimeSeries
 from .service import Service, KubernetesAutuscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, \
-    InstanceCatalog, KubernetesRuntime, ServiceType
+    InstanceCatalog, KubernetesRuntime, ServiceType, ServiceModeType
 from .execution import Execution, ExecutionStatus
 from .command import Command, CommandsStatus
 from .assignment import Assignment, Workload, WorkloadUnit

dtlpy/entities/annotation.py

@@ -1216,6 +1216,13 @@ class Annotation(entities.BaseEntity):
                                     annotation=self)
 
         self.frames[frame_num] = frame
+        if not self.frames[frame_num].fixed and self.start_frame < frame_num:
+            frame_last = self.frames[frame_num - 1].to_snapshot()
+            frame_current = self.frames[frame_num].to_snapshot()
+            frame_last.pop('frame')
+            frame_current.pop('frame')
+            if frame_last == frame_current:
+                self.frames[frame_num]._interpolation = True
         self.end_frame = max(self.last_frame, frame_num)
         self.end_time = self.end_frame / self.fps
 

dtlpy/entities/annotation_definitions/note.py

@@ -2,7 +2,7 @@ import numpy as np
 import time
 
 from . import Box
-from ...services import ApiClient
+from ...services.api_client import client as api_client
 
 
 class Note(Box):
@@ -47,7 +47,6 @@ class Note(Box):
         self.create_time = create_time
         self.creator = creator
         if self.creator is None:
-            api_client = ApiClient()
             self.creator = api_client.info()['user_email']
         self.assignee = assignee
         if self.assignee is None:
@@ -119,7 +118,6 @@ class Message:
         self.body = body
         self.creator = creator
         if self.creator is None:
-            api_client = ApiClient()
             self.creator = api_client.info()['user_email']
 
     def to_json(self):

dtlpy/entities/app.py

@@ -3,7 +3,8 @@ import traceback
 import logging
 import attr
 
-from .. import entities, services, repositories
+from .. import entities, repositories
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -26,7 +27,7 @@ class App(entities.BaseEntity):
 
     # sdk
     _project = attr.ib(type=entities.Project, repr=False)
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
 
     @_repositories.default
@@ -126,7 +127,7 @@ class App(entities.BaseEntity):
         return _json
 
     @classmethod
-    def from_json(cls, _json, client_api: services.ApiClient, project: entities.Project, is_fetched=True):
+    def from_json(cls, _json, client_api: ApiClient, project: entities.Project, is_fetched=True):
         app = cls(
             id=_json.get('id', None),
             name=_json.get('name', None),

dtlpy/entities/command.py

@@ -3,7 +3,8 @@ import traceback
 from enum import Enum
 from collections import namedtuple
 
-from .. import repositories, entities, services
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 
 class CommandsStatus(str, Enum):
@@ -36,7 +37,7 @@ class Command(entities.BaseEntity):
     error = attr.ib()
 
     # sdk
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
 
     ################

dtlpy/entities/dataset.py

@@ -7,6 +7,7 @@ import attr
 import os
 
 from .. import repositories, entities, services, exceptions
+from ..services.api_client import ApiClient
 from .annotation import ViewAnnotationOptions, AnnotationType, ExportVersion
 
 logger = logging.getLogger(name='dtlpy')
@@ -72,7 +73,7 @@ class Dataset(entities.BaseEntity):
     _readonly = attr.ib(repr=False)
 
     # api
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
 
     # entities
     _project = attr.ib(default=None, repr=False)
@@ -95,7 +96,7 @@ class Dataset(entities.BaseEntity):
     @staticmethod
     def _protected_from_json(project: entities.Project,
                              _json: dict,
-                             client_api: services.ApiClient,
+                             client_api: ApiClient,
                              datasets=None,
                              is_fetched=True):
         """
@@ -124,7 +125,7 @@ class Dataset(entities.BaseEntity):
     def from_json(cls,
                   project: entities.Project,
                   _json: dict,
-                  client_api: services.ApiClient,
+                  client_api: ApiClient,
                   datasets=None,
                   is_fetched=True):
         """

dtlpy/entities/dpk.py

@@ -5,6 +5,7 @@ import traceback
 import enum
 
 from .. import entities, services, repositories
+from ..services.api_client import ApiClient
 
 
 class SlotType(str, enum.Enum):
@@ -145,7 +146,7 @@ class Dpk(entities.DlEntity):
     url: str = entities.DlProperty(location=['url'], _type=str)
 
     # sdk
-    client_api: services.ApiClient
+    client_api: ApiClient
     project: entities.Project
     __repositories = None
 
@@ -265,7 +266,7 @@ class Dpk(entities.DlEntity):
     @classmethod
     def from_json(cls,
                   _json,
-                  client_api: services.ApiClient = None,
+                  client_api: ApiClient = None,
                   project: entities.Project = None,
                   is_fetched=True) -> 'Dpk':
         """

dtlpy/entities/driver.py

@@ -2,7 +2,8 @@ import logging
 import attr
 from enum import Enum
 from collections import namedtuple
-from .. import services, entities, repositories
+from .. import entities, repositories
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -13,6 +14,7 @@ class ExternalStorage(str, Enum):
     AZUREBLOB = "azureblob"
     KEY_VALUE = "key_value"
     AWS_STS = 'aws-sts'
+    AZURE_DATALAKE_GEN2 = 'azureDatalakeGen2'
 
 
 @attr.s()
@@ -32,7 +34,7 @@ class Driver(entities.BaseEntity):
     id = attr.ib()
     path = attr.ib()
     # api
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
 
     @_repositories.default

dtlpy/entities/execution.py

@@ -4,7 +4,8 @@ import traceback
 from enum import Enum
 from collections import namedtuple
 
-from .. import repositories, entities, services
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -49,7 +50,7 @@ class Execution(entities.BaseEntity):
     package_name = attr.ib()
 
     # sdk
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _service = attr.ib(repr=False)
     _project = attr.ib(repr=False, default=None)
     _repositories = attr.ib(repr=False)

dtlpy/entities/feature.py

@@ -3,14 +3,16 @@ import traceback
 from collections import namedtuple
 from enum import Enum
 
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
-from .. import repositories, entities, services
 
 class FeatureDataType(str, Enum):
     """Available types for Feature vectors data type"""
     ANNOTATION_SCORE = 'annotationScore',
     ITEM_SCORE = 'itemScore'
 
+
 @attr.s
 class Feature(entities.BaseEntity):
     """
@@ -32,7 +34,7 @@ class Feature(entities.BaseEntity):
     refs = attr.ib()
 
     # sdk
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
 
     ################
@@ -93,8 +95,8 @@ class Feature(entities.BaseEntity):
             client_api=client_api,
             org_id=_json.get('org', None),
             creator=_json.get('creator', None),
-            refs = _json.get('refs', None),
-            data_type =_json.get('dataType', None)
+            refs=_json.get('refs', None),
+            data_type=_json.get('dataType', None)
         )
         inst.is_fetched = is_fetched
         return inst

dtlpy/entities/feature_set.py

@@ -4,7 +4,8 @@ import attr
 import traceback
 from collections import namedtuple
 
-from .. import repositories, entities, services
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 
 class FeatureEntityType(str, Enum):
@@ -35,7 +36,7 @@ class FeatureSet(entities.BaseEntity):
     data_type = attr.ib()
 
     # sdk
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
 
     ################
@@ -104,7 +105,7 @@ class FeatureSet(entities.BaseEntity):
             tags=_json.get('tags', None),
             client_api=client_api,
             org_id=_json.get('org', None),
-            data_type =_json.get('dataType', None)
+            data_type=_json.get('dataType', None)
         )
         inst.is_fetched = is_fetched
         return inst

dtlpy/entities/integration.py

@@ -1,7 +1,8 @@
 import logging
 import attr
 
-from .. import entities, services, exceptions, repositories
+from .. import entities, exceptions, repositories
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -18,13 +19,13 @@ class Integration(entities.BaseEntity):
     created_at = attr.ib()
     created_by = attr.ib()
     update_at = attr.ib()
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _project = attr.ib(default=None, repr=False)
 
     @classmethod
     def from_json(cls,
                   _json: dict,
-                  client_api: services.ApiClient,
+                  client_api: ApiClient,
                   is_fetched=True):
         """
         Build a Integration entity object from a json

dtlpy/entities/item.py

@@ -7,8 +7,10 @@ import attr
 import copy
 import os
 
-from .. import repositories, entities, services, exceptions
+from .. import repositories, entities, exceptions
 from .annotation import ViewAnnotationOptions, ExportVersion
+from ..services.api_client import ApiClient
+from ..services.api_client import client as client_api
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -53,7 +55,7 @@ class Item(entities.BaseEntity):
     annotations_count = attr.ib()
 
     # api
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _platform_dict = attr.ib(repr=False)
 
     # entities
@@ -160,7 +162,7 @@ class Item(entities.BaseEntity):
         # create a new item, and update the current one with the same state
         # this way we can have _client_api, and all the repositories and entities which are not picklable
         self.__dict__.update(entities.Item.from_json(_json=state,
-                                                     client_api=services.ApiClient()).__dict__)
+                                                     client_api=client_api).__dict__)
 
     ############
     # entities #

dtlpy/entities/model.py

@@ -5,7 +5,8 @@ import logging
 
 import attr
 
-from .. import repositories, entities, services
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -76,7 +77,7 @@ class Model(entities.BaseEntity):
     _project = attr.ib(repr=False)
     _package = attr.ib(repr=False)
     _dataset = attr.ib(repr=False)
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
     _ontology = attr.ib(repr=False, default=None)
 

dtlpy/entities/node.py

@@ -186,6 +186,7 @@ class PipelineNode:
                  metadata: dict = None,
                  config: dict = None,
                  position: tuple = (1, 1),
+                 app_id: str = None,
                  ):
         """
         :param str name: node name
@@ -198,6 +199,7 @@ class PipelineNode:
         :param str project_id: project id
         :param dict config: for the code node dict in format { package: {code : the_code}}
         :param tuple position: tuple of the node place
+        :param str app_id: app id
         """
         self.name = name
         self.node_id = node_id
@@ -209,6 +211,7 @@ class PipelineNode:
         self.project_id = project_id
         self.config = config
         self.position = position
+        self.app_id = app_id
         self._pipeline = None
 
     @property
@@ -259,7 +262,8 @@ class PipelineNode:
             namespace=namespace,
             project_id=_json.get('projectId', None),
             config=_json.get('config', None),
-            position=position
+            position=position,
+            app_id=_json.get('appId', None)
         )
 
     def to_json(self):
@@ -275,6 +279,8 @@ class PipelineNode:
         }
         if self.config is not None:
             _json['config'] = self.config
+        if self.app_id is not None:
+            _json['appId'] = self.app_id
         return _json
 
     def is_root(self):

dtlpy/entities/ontology.py

@@ -1,13 +1,13 @@
-import os
-import traceback
 from collections import namedtuple
+import traceback
 import logging
 import random
 import uuid
 import attr
+import os
 
-from .. import entities, PlatformException, repositories, services, exceptions
-
+from .. import entities, PlatformException, repositories, exceptions
+from ..services.api_client import ApiClient
 from .label import Label
 
 logger = logging.getLogger(name='dtlpy')
@@ -43,7 +43,7 @@ class Ontology(entities.BaseEntity):
     Ontology object
     """
     # api
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
 
     # params
     id = attr.ib()

dtlpy/entities/organization.py

@@ -4,7 +4,8 @@ import traceback
 import logging
 import attr
 
-from .. import repositories, services, entities
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -51,7 +52,7 @@ class Organization(entities.BaseEntity):
     created_by = attr.ib(repr=False)
 
     # api
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
 
     # repositories
     _repositories = attr.ib(repr=False)

dtlpy/entities/package.py

@@ -10,7 +10,8 @@ import os
 
 from .package_module import PackageModule
 from .package_slot import PackageSlot
-from .. import repositories, entities, exceptions, services
+from .. import repositories, entities, exceptions
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -79,7 +80,7 @@ class Package(entities.DlEntity):
                                                                                             _kls='PackageRequirement')
 
     # sdk
-    _client_api: services.ApiClient
+    _client_api: ApiClient
     _revisions = None
     __repositories = None
     _project = None

dtlpy/entities/paged_entities.py

@@ -1,11 +1,12 @@
 import logging
 import math
+import tqdm
 import copy
 import sys
 
 import attr
-from .. import services, miscellaneous
-import tqdm
+from .. import miscellaneous
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -16,7 +17,7 @@ class PagedEntities:
     Pages object
     """
     # api
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
 
     # params
     page_offset = attr.ib()

dtlpy/entities/pipeline.py

@@ -5,7 +5,8 @@ from enum import Enum
 from typing import List
 import attr
 from .node import PipelineNode, PipelineConnection, TaskNode, CodeNode, FunctionNode, DatasetNode
-from .. import repositories, entities, services
+from .. import repositories, entities
+from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -184,7 +185,7 @@ class Pipeline(entities.BaseEntity):
 
     # sdk
     _project = attr.ib(repr=False)
-    _client_api = attr.ib(type=services.ApiClient, repr=False)
+    _client_api = attr.ib(type=ApiClient, repr=False)
     _original_settings = attr.ib(repr=False, type=PipelineSettings)
     _repositories = attr.ib(repr=False)
 
@@ -452,7 +453,7 @@ class Pipeline(entities.BaseEntity):
 
             command = pipeline.execute_batch(
                         execution_inputs=dl.FunctionIO(type=dl.PackageInputType.STRING, value='test', name='string'),
-                        filters=dl.Filters(field='dir', values='/test'))
+                        filters=dl.Filters(field='dir', values='/test', context={'datasets': [dataset.id]))
         """
         command = self.pipeline_executions.create_batch(pipeline_id=self.id,
                                                         execution_inputs=execution_inputs,