dtlpy 1.91.37__py3-none-any.whl → 1.92.19__py3-none-any.whl

dtlpy/__init__.py

@@ -35,7 +35,8 @@ if len(logger.handlers) == 0:
         datefmt="%Y-%m-%d %H:%M:%S",
     )
     file_formatter = logging.Formatter(
-        fmt="[%(asctime)s.%(msecs)03d][%(threadName)s][%(levelname).3s][%(name)s:v" + __version__ + "][%(relativepath)-s:%(lineno)-d](%(funcName)-s): %(message)s",
+        fmt="[%(asctime)s.%(msecs)03d][%(threadName)s][%(levelname).3s][%(name)s:v" +
+            __version__ + "][%(relativepath)-s:%(lineno)-d](%(funcName)-s): %(message)s",
         datefmt="%Y-%m-%d %H:%M:%S",
     )
     package_path = os.path.dirname(__file__)
@@ -80,7 +81,8 @@ from .entities import (
     InstanceCatalog, PackageInputType, ServiceType, ServiceModeType,
     PackageSlot, SlotPostAction, SlotPostActionType, SlotDisplayScope, SlotDisplayScopeResource, UiBindingPanel,
     # roberto
-    DatasetSubsetType, ModelStatus, PlotSample, ArtifactType, Artifact, ItemArtifact, LinkArtifact, LocalArtifact, EntityScopeLevel,
+    DatasetSubsetType, ModelStatus, PlotSample, ArtifactType, Artifact, ItemArtifact, LinkArtifact, LocalArtifact,
+    EntityScopeLevel,
     # features
     FeatureEntityType, Feature, FeatureSet,
     #
@@ -245,6 +247,7 @@ class LoggingLevel:
 #################
 #     ENUMS     #
 #################
+CallbackEvent = client_api.callbacks.CallbackEvent
 LOGGING_LEVEL_DEBUG = LoggingLevel.DEBUG
 LOGGING_LEVEL_WARNING = LoggingLevel.WARNING
 LOGGING_LEVEL_CRITICAL = LoggingLevel.CRITICAL

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.91.37'
+version = '1.92.19'

dtlpy/entities/__init__.py

@@ -22,7 +22,7 @@ from .trigger import Trigger, TriggerResource, TriggerAction, TriggerExecutionMo
     TriggerType
 from .project import Project, MemberRole
 from .artifact import ItemArtifact, LocalArtifact, LinkArtifact, ArtifactType, Artifact
-from .dataset import Dataset, ExpirationOptions, IndexDriver
+from .dataset import Dataset, ExpirationOptions, IndexDriver, ExportType
 from .codebase import Codebase
 from .annotation import Annotation, FrameAnnotation, ViewAnnotationOptions, AnnotationStatus, AnnotationType, \
     ExportVersion

dtlpy/entities/command.py

@@ -141,7 +141,7 @@ class Command(entities.BaseEntity):
                                entities.CommandsStatus.FINALIZING,
                                entities.CommandsStatus.IN_PROGRESS]
 
-    def wait(self, timeout=0, step=None,  backoff_factor=1):
+    def wait(self, timeout=0, step=None, backoff_factor=1):
         """
         Wait for Command to finish
 
@@ -157,4 +157,5 @@ class Command(entities.BaseEntity):
                                   timeout=timeout,
                                   step=step,
                                   url=self.url,
-                                  backoff_factor=backoff_factor)
+                                  backoff_factor=backoff_factor
+                                  )

dtlpy/entities/dataset.py

@@ -18,6 +18,11 @@ class IndexDriver(str, Enum):
     V2 = "v2"
 
 
+class ExportType(str, Enum):
+    JSON = "json"
+    ZIP = "zip"
+
+
 class ExpirationOptions:
     """
     ExpirationOptions object
@@ -58,7 +63,6 @@ class Dataset(entities.BaseEntity):
     items_count = attr.ib()
     metadata = attr.ib(repr=False)
     directoryTree = attr.ib(repr=False)
-    export = attr.ib(repr=False)
     expiration_options = attr.ib()
     index_driver = attr.ib()
     enable_sync_with_cloned = attr.ib(repr=False)
@@ -165,7 +169,6 @@ class Dataset(entities.BaseEntity):
                    projects=projects,
                    creator=_json.get('creator', None),
                    items_url=_json.get('items', None),
-                   export=_json.get('export', None),
                    driver=_json.get('driver', None),
                    name=_json.get('name', None),
                    url=_json.get('url', None),
@@ -653,6 +656,53 @@ class Dataset(entities.BaseEntity):
             export_version=export_version
         )
 
+    def export(self,
+               local_path=None,
+               filters=None,
+               annotation_filters=None,
+               feature_vector_filters=None,
+               include_feature_vectors: bool = False,
+               include_annotations: bool = False,
+               export_type: ExportType = ExportType.JSON,
+               timeout: int = 0):
+        """
+        Export dataset items and annotations.
+
+        **Prerequisites**: You must be an *owner* or *developer* to use this method.
+
+        You must provide at least ONE of the following params: dataset, dataset_name, dataset_id.
+
+        :param str local_path: The local path to save the exported dataset
+        :param Union[dict, dtlpy.entities.filters.Filters] filters: Filters entity or a query dictionary
+        :param dtlpy.entities.filters.Filters annotation_filters: Filters entity
+        :param dtlpy.entities.filters.Filters feature_vector_filters: Filters entity
+        :param bool include_feature_vectors: Include item feature vectors in the export
+        :param bool include_annotations: Include item annotations in the export
+        :param entities.ExportType export_type: Type of export ('json' or 'zip')
+        :param int timeout: Maximum time in seconds to wait for the export to complete
+        :return: Exported item
+        :rtype: dtlpy.entities.item.Item
+
+        **Example**:
+
+        .. code-block:: python
+
+            export_item = dataset.export(filters=filters,
+                                         include_feature_vectors=True,
+                                         include_annotations=True,
+                                         export_type=dl.ExportType.JSON)
+        """
+
+        return self.datasets.export(dataset=self,
+                                    local_path=local_path,
+                                    filters=filters,
+                                    annotation_filters=annotation_filters,
+                                    feature_vector_filters=feature_vector_filters,
+                                    include_feature_vectors=include_feature_vectors,
+                                    include_annotations=include_annotations,
+                                    export_type=export_type,
+                                    timeout=timeout)
+
     def upload_annotations(self,
                            local_path,
                            filters=None,

dtlpy/entities/feature_set.py

@@ -31,6 +31,7 @@ class FeatureSet(entities.BaseEntity):
     set_type = attr.ib()
     entity_type = attr.ib()
     project_id = attr.ib()
+    model_id = attr.ib()
     org_id = attr.ib()
 
     # sdk
@@ -100,6 +101,7 @@ class FeatureSet(entities.BaseEntity):
             size=_json.get('size', None),
             url=_json.get('url', None),
             project_id=_json.get('project', None),
+            model_id=_json.get('modelId', None),
             created_at=_json.get('createdAt', None),
             creator=_json.get('creator', None),
             updated_by=_json.get('updatedBy', None),
@@ -121,6 +123,7 @@ class FeatureSet(entities.BaseEntity):
                  'type': self.set_type,
                  'entityType': self.entity_type,
                  'project': self.project_id,
+                 'modelId': self.model_id,
                  'creator': self.creator,
                  'createdAt': self.created_at,
                  'updatedBy': self.updated_by,

dtlpy/entities/filters.py

@@ -296,9 +296,9 @@ class Filters:
             self.join = dict()
         if 'on' not in self.join:
             if self.resource == FiltersResource.ITEM:
-                self.join['on'] = {'resource': FiltersResource.ANNOTATION, 'local': 'itemId', 'forigen': 'id'}
+                self.join['on'] = {'resource': FiltersResource.ANNOTATION.value, 'local': 'itemId', 'forigen': 'id'}
             else:
-                self.join['on'] = {'resource': FiltersResource.ITEM, 'local': 'id', 'forigen': 'itemId'}
+                self.join['on'] = {'resource': FiltersResource.ITEM.value, 'local': 'id', 'forigen': 'itemId'}
         if 'filter' not in self.join:
             self.join['filter'] = dict()
         join_method = '$' + method

dtlpy/entities/item.py

@@ -6,11 +6,13 @@ import logging
 import attr
 import copy
 import os
-
+import io
 from .. import repositories, entities, exceptions
 from .annotation import ViewAnnotationOptions, ExportVersion
 from ..services.api_client import ApiClient
 from ..services.api_client import client as client_api
+import json
+import requests
 
 logger = logging.getLogger(name='dtlpy')
 
@@ -182,6 +184,18 @@ class Item(entities.BaseEntity):
     def model(self):
         return self._model
 
+    def __update_item_binary(self, _json):
+        binary = io.BytesIO()
+        binary.write(json.dumps(_json).encode())
+        binary.seek(0)
+        binary.name = self.name
+        resp = requests.post(url=client_api.environment + f'/items/{self.id}/revisions',
+                             headers=client_api.auth,
+                             files={'file': (binary.name, binary)}
+                             )
+        if not resp.ok:
+            raise ValueError(resp.text)
+
     @property
     def project(self):
         if self._project is None:

dtlpy/entities/node.py

@@ -221,6 +221,8 @@ class PipelineNode:
                  config: dict = None,
                  position: tuple = (1, 1),
                  app_id: str = None,
+                 dpk_name: str = None,
+                 app_name: str = None,
                  ):
         """
         :param str name: node name
@@ -234,6 +236,8 @@ class PipelineNode:
         :param dict config: for the code node dict in format { package: {code : the_code}}
         :param tuple position: tuple of the node place
         :param str app_id: app id
+        :param str dpk_name: dpk name
+        :param str app_name: app name
         """
         self.name = name
         self.node_id = node_id
@@ -246,6 +250,8 @@ class PipelineNode:
         self.config = config
         self.position = position
         self.app_id = app_id
+        self.dpk_name = dpk_name
+        self.app_name = app_name
         self._pipeline = None
 
     @property
@@ -297,7 +303,9 @@ class PipelineNode:
             project_id=_json.get('projectId', None),
             config=_json.get('config', None),
             position=position,
-            app_id=_json.get('appId', None)
+            app_id=_json.get('appId', None),
+            dpk_name=_json.get('dpkName', None),
+            app_name=_json.get('appName', None),
         )
 
     def to_json(self):
@@ -310,6 +318,8 @@ class PipelineNode:
             'type': self.node_type,
             'namespace': self.namespace.to_json(),
             'projectId': self.project_id,
+            'dpkName': self.dpk_name,
+            'appName': self.app_name,
         }
         if self.config is not None:
             _json['config'] = self.config

dtlpy/entities/ontology.py

@@ -207,7 +207,14 @@ class Ontology(entities.BaseEntity):
 
     @property
     def _use_attributes_2(self):
-        return os.environ.get("USE_ATTRIBUTE_2", 'false') == 'true'
+        if isinstance(self.metadata, dict):
+            attributes = self.metadata.get("attributes", None)
+            if attributes is not None:
+                return True
+            else:
+                if isinstance(self.attributes, list) and len(self.attributes) > 0:
+                    return False
+        return True
 
     @classmethod
     def from_json(cls, _json, client_api, recipe, dataset=None, project=None, is_fetched=True):
@@ -223,10 +230,9 @@ class Ontology(entities.BaseEntity):
         :return: Ontology object
         :rtype: dtlpy.entities.ontology.Ontology
         """
-        if not os.environ.get("USE_ATTRIBUTE_2", 'false') == 'true':
-            attributes = _json.get("attributes", [])
-        else:
-            attributes = _json.get('metadata', {}).get("attributes", [])
+        attributes_v2 = _json.get('metadata', {}).get("attributes", [])
+        attributes_v1 = _json.get("attributes", [])
+        attributes = attributes_v2 if attributes_v2 else attributes_v1
 
         labels = list()
         for root in _json["roots"]:
@@ -744,10 +750,11 @@ class Ontology(entities.BaseEntity):
 
     def copy_from(self, ontology_json: dict):
         """
-        Import ontology to the platform ('ontology' is taken before 'ontology_json')
+        Import ontology to the platform.\n
+        Notice: only the following fields will be updated: `labels`, `attributes`, `instance_map` and `color_map`.
 
-        :param dict ontology_json: ontology json
-        :return: Ontology object
+        :param dict ontology_json: The source ontology json to copy from
+        :return: Ontology object: The updated ontology entity
         :rtype: dtlpy.entities.ontology.Ontology
 
         **Example**:
@@ -756,49 +763,38 @@ class Ontology(entities.BaseEntity):
 
             ontology = ontology.import_ontology(ontology_json=ontology_json)
         """
-        # TODO: Add support for import from ontology entity
-        ontology = self.from_json(_json=ontology_json, client_api=self._client_api, recipe=self.recipe)
-        attributes = ontology.attributes
-
-        # params
-        self.labels = ontology.labels
-        for key, value in ontology.metadata.items():
-            if key != "system":
-                self.metadata[key] = value
-
-        if attributes:
-            # Delete irrelevant attribute keys
-            attribute_keys = [attribute.get("key", None) for attribute in attributes]
-            to_delete_keys = [attribute.get("key", None) for attribute in self.attributes
-                              if attribute.get("key", None) not in attribute_keys]
-            self.delete_attributes(keys=to_delete_keys)
-
-            # Update attributes
-            for attribute in attributes:
-                attribute_range = attribute.get("range", None)
+        # TODO: Add support for import from ontology entity in the Future
+        if not self._use_attributes_2:
+            raise ValueError("This method is only supported for attributes 2 mode!")
+        new_ontology = self.from_json(_json=ontology_json, client_api=self._client_api, recipe=self.recipe)
+
+        # Update 'labels' and 'attributes'
+        self.labels = new_ontology.labels
+        new_attributes = new_ontology.attributes
+        if isinstance(new_attributes, list):
+            for new_attribute in new_attributes:
+                attribute_range = new_attribute.get("range", None)
                 if attribute_range is not None:
                     attribute_range = entities.AttributesRange(
                         min_range=attribute_range.get("min", None),
                         max_range=attribute_range.get("max", None),
                         step=attribute_range.get("step", None)
                     )
-
-                script_data = attribute.get("scriptData", None)
+                script_data = new_attribute.get("scriptData", None)
                 if script_data is None:
-                    raise Exception(f"Attribute '{attribute.get('key')}' scriptData is missing in the ontology json!")
+                    new_attribute_key = new_attribute.get("key", None)
+                    raise Exception(f"Attribute '{new_attribute_key}' scriptData is missing in the ontology json!")
                 self.update_attributes(
                     title=script_data.get("title", None),
-                    key=attribute.get("key", None),
-                    attribute_type=attribute.get("type", None),
-                    scope=attribute.get("scope", None),
+                    key=new_attribute.get("key", None),
+                    attribute_type=new_attribute.get("type", None),
+                    scope=new_attribute.get("scope", None),
                     optional=script_data.get("optional", None),
-                    values=attribute.get("values", None),
+                    values=new_attribute.get("values", None),
                     attribute_range=attribute_range
                 )
-        else:
-            logger.warning("No attributes were found (Make sure that you use the correct attributes mode).")
 
-        # defaults
-        self._instance_map = ontology.instance_map
-        self._color_map = ontology.color_map
+        # Update 'instance map' and 'color map'
+        self._instance_map = new_ontology.instance_map
+        self._color_map = new_ontology.color_map
         return self.update(system_metadata=True)

dtlpy/entities/pipeline.py

@@ -456,7 +456,6 @@ class Pipeline(entities.BaseEntity):
         """
         return self.pipelines.update(pipeline=self)
 
-
     def delete(self):
         """
         Delete pipeline object
@@ -560,3 +559,23 @@ class Pipeline(entities.BaseEntity):
         else:
             self.start_nodes = [{"nodeId": node.node_id,
                                  "type": "root", }]
+
+    def update_variables_values(self, **kwargs):
+        """
+        Update pipeline variables values for the given keyword arguments.
+
+        **Example**:
+
+        .. code-block:: python
+            pipeline.update_variables_values(
+                dataset=dataset.id,
+                model=model.id,
+                threshold=0.9
+            )
+            pipeline.update()
+        """
+        keys = kwargs.keys()
+        for variable in self.variables:
+            if variable.name in keys:
+                variable.value = kwargs[variable.name]
+

dtlpy/entities/pipeline_execution.py

@@ -11,6 +11,16 @@ from ..services.api_client import ApiClient
 logger = logging.getLogger(name='dtlpy')
 
 
+class PipelineExecutionStatus(str, Enum):
+    PENDING = "pending"
+    IN_PROGRESS = "in-progress"
+    FAILED = "failed"
+    SUCCESS = "success"
+    QUEUE = "queue"
+    TERMINATED = "terminated"
+    RERUN = "rerun"
+
+
 class CycleRerunMethod(str, Enum):
     START_FROM_NODES = 'startFromNodes',
     START_FROM_FAILED_EXECUTIONS = 'startFromFailedExecutions',
@@ -254,3 +264,16 @@ class PipelineExecution(entities.BaseEntity):
             filters=filters,
             wait=wait
         )
+
+    def wait(self):
+        """
+        Wait for pipeline execution
+
+        :return: Pipeline execution object
+        """
+        return self.pipeline_executions.wait(pipeline_execution_id=self.id)
+
+    def in_progress(self):
+        return self.status not in [PipelineExecutionStatus.FAILED,
+                                   PipelineExecutionStatus.SUCCESS,
+                                   PipelineExecutionStatus.TERMINATED]