dtlpy 1.85.25__py3-none-any.whl → 1.87.18__py3-none-any.whl

dtlpy/__init__.py

@@ -69,7 +69,7 @@ from .entities import (
     # annotations
     Box, Cube, Cube3d, Point, Note, Message, Segmentation, Ellipse, Classification, Subtitle, Polyline, Pose,
     Description,
-    Polygon, Text, FreeText,
+    Polygon, Text, FreeText, RefImage,
     # filters
     Filters, FiltersKnownFields, FiltersResource, FiltersOperations, FiltersMethod, FiltersOrderByDirection,
     FiltersKnownFields as KnownFields,
@@ -313,14 +313,14 @@ INSTANCE_CATALOG_REGULAR_XS = InstanceCatalog.REGULAR_XS
 INSTANCE_CATALOG_REGULAR_S = InstanceCatalog.REGULAR_S
 INSTANCE_CATALOG_REGULAR_M = InstanceCatalog.REGULAR_M
 INSTANCE_CATALOG_REGULAR_L = InstanceCatalog.REGULAR_L
-INSTANCE_CATALOG_REGULAR_XL = InstanceCatalog.REGULAR_XL
 INSTANCE_CATALOG_HIGHMEM_XS = InstanceCatalog.HIGHMEM_XS
 INSTANCE_CATALOG_HIGHMEM_S = InstanceCatalog.HIGHMEM_S
 INSTANCE_CATALOG_HIGHMEM_M = InstanceCatalog.HIGHMEM_M
 INSTANCE_CATALOG_HIGHMEM_L = InstanceCatalog.HIGHMEM_L
-INSTANCE_CATALOG_HIGHMEM_XL = InstanceCatalog.HIGHMEM_XL
 INSTANCE_CATALOG_GPU_K80_S = InstanceCatalog.GPU_K80_S
 INSTANCE_CATALOG_GPU_K80_M = InstanceCatalog.GPU_K80_M
+INSTANCE_CATALOG_GPU_T4_S = InstanceCatalog.GPU_T4_S
+INSTANCE_CATALOG_GPU_T4_M = InstanceCatalog.GPU_T4_M
 
 MODALITY_TYPE_OVERLAY = ModalityTypeEnum.OVERLAY
 MODALITY_TYPE_PREVIEW = ModalityTypeEnum.PREVIEW

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.85.25'
+version = '1.87.18'

dtlpy/entities/__init__.py

@@ -33,7 +33,7 @@ from .filters import Filters, FiltersKnownFields, FiltersResource, FiltersOperat
 from .recipe import Recipe
 from .ontology import Ontology, AttributesTypes, AttributesRange
 from .annotation_definitions import Box, Cube, Cube3d, Point, Segmentation, Polygon, Ellipse, Classification, \
-    Subtitle, Text, FreeText, \
+    Subtitle, Text, FreeText, RefImage, \
     Polyline, Comparison, UndefinedAnnotationType, Note, Message, Description, Pose
 from .label import Label
 from .codebase import Codebase, PackageCodebaseType, ItemCodebase, GitCodebase, FilesystemCodebase, LocalCodebase

dtlpy/entities/annotation.py

@@ -235,9 +235,7 @@ class Annotation(entities.BaseEntity):
     def coordinates(self):
         color = None
         if self.type in ['binary']:
-            color = self.annotation_definition._color
-            if color is None:
-                color = self.color
+            color = self.color
         coordinates = self.annotation_definition.to_coordinates(color=color)
         return coordinates
 
@@ -446,16 +444,16 @@ class Annotation(entities.BaseEntity):
     @property
     def color(self):
         # if "dataset" is not in self - this will always get the dataset
-        if self.type == 'binary' and self.annotation_definition._color is not None:
-            color = self.annotation_definition._color
+        try:
+            colors = self.dataset._get_ontology().color_map
+        except (exceptions.BadRequest, exceptions.NotFound):
+            colors = None
+            logger.warning('Cant get dataset for annotation color. using default.')
+        if colors is not None and self.label in colors:
+            color = colors[self.label]
         else:
-            try:
-                colors = self.dataset._get_ontology().color_map
-            except exceptions.BadRequest:
-                colors = None
-                logger.warning('Cant get dataset for annotation color. using default.')
-            if colors is not None and self.label in colors:
-                color = colors[self.label]
+            if self.type == 'binary' and self.annotation_definition._color is not None:
+                color = self.annotation_definition._color
             else:
                 color = (255, 255, 255)
         return color

dtlpy/entities/annotation_collection.py

@@ -75,23 +75,25 @@ class AnnotationCollection(entities.BaseEntity):
         :param metadata: optional- metadata dictionary for annotation
         :param parent_id: set a parent for this annotation (parent annotation ID)
         :param prompt_id: Connect the annotation with a specific prompt in a dl.PromptItem
-        :param model_info: optional - set model on annotation {'name',:'', 'confidence':0}
+        :param model_info: optional - set model on annotation {'confidence':0 [Mandatory], (Float between 0-1)
+                                                               'name',:'' [Optional], ('name' refers to 'model_name')
+                                                               'model_id':''[Optional]}
         :return:
         """
         if model_info is not None:
-            if not isinstance(model_info, dict) or 'name' not in model_info or 'confidence' not in model_info:
-                raise ValueError('"model_info" must be a dict with keys: "name" and "confidence"')
+            if not isinstance(model_info, dict) or 'confidence' not in model_info:
+                raise ValueError('"model_info" must be a dict with key: "confidence"')
             if metadata is None:
                 metadata = dict()
             if 'user' not in metadata:
                 metadata['user'] = dict()
-            metadata['user']['model'] = {'name': model_info['name'],
-                                         'confidence': float(model_info['confidence']),
+            confidence = float(model_info['confidence'])
+            if confidence < 0 or confidence > 1:
+                raise ValueError('"confidence" must be a float between 0 and 1')
+            metadata['user']['model'] = {'confidence': confidence,
+                                         'name': model_info.get('name'),
                                          'model_id': model_info.get('model_id'),
-                                         'snapshot_id': model_info.get('snapshot_id'),
                                          }
-            metadata['user']['annotation_type'] = 'prediction'
-
         if prompt_id is not None:
             if metadata is None:
                 metadata = dict()
@@ -512,7 +514,7 @@ class AnnotationCollection(entities.BaseEntity):
 
     @classmethod
     def from_json(cls, _json: list, item=None, is_video=None, fps=25, height=None, width=None,
-                  client_api=None, is_audio=None):
+                  client_api=None, is_audio=None) -> 'AnnotationCollection':
         """
         Create an annotation collection object from platform json
 

dtlpy/entities/annotation_definitions/__init__.py

@@ -15,4 +15,5 @@ from .description import Description
 from .undefined_annotation import UndefinedAnnotationType
 from .pose import Pose
 from .text import Text
-from .free_text import FreeText
+from .free_text import FreeText
+from .ref_image import RefImage

dtlpy/entities/annotation_definitions/ref_image.py

@@ -0,0 +1,86 @@
+from . import BaseAnnotationDefinition
+
+
+class RefImage(BaseAnnotationDefinition):
+    """
+    Create an image annotation. Reference the url or item id in this annotation type
+    """
+    type = "ref_image"
+
+    def __init__(self, ref, ref_type=None, mimetype=None, label='ref-image', attributes=None, description=None):
+
+        """
+        Create an image annotation. Used for generative model and any other algorithm where and image is the output
+
+        For type 'id', need to upload the image as item in the platform and reference the item id in the annotation.
+        For type 'url', mimetype must be provided to load the ref correctly in the platform
+
+        :param str ref: the reference to the image annotation, represented by an ‘itemId’ or ‘url’
+        :param str ref_type: one of ‘id’ | ‘url’
+        :param str mimetype: optional. in case the refType is URL, e.g. image/jpeg, video/mpeg
+        :param label: annotation label
+        :param attributes: annotation attributes
+        :param description:
+
+        :return:
+        """
+        super().__init__(description=description, attributes=attributes)
+        if ref_type is None:
+            if ref.startswith('http'):
+                ref_type = 'url'
+            else:
+                ref_type = 'id'
+        self.ref = ref
+        self.ref_type = ref_type
+        self.mimetype = mimetype
+        self.label = label
+
+    @property
+    def x(self):
+        return 0
+
+    @property
+    def y(self):
+        return 0
+
+    @property
+    def geo(self):
+        return list()
+
+    def show(self, image, thickness, with_text, height, width, annotation_format, color, alpha=1):
+        """
+        Show annotation as ndarray
+        :param image: empty or image to draw on
+        :param thickness:
+        :param with_text: not required
+        :param height: item height
+        :param width: item width
+        :param annotation_format: options: list(dl.ViewAnnotationOptions)
+        :param color: color
+        :param alpha: opacity value [0 1], default 1
+        :return: ndarray
+        """
+        # TODO over or show the image annotations
+        return self.add_text_to_image(image=image, annotation=self)
+
+    def to_coordinates(self, color):
+        coordinates = {
+            "ref": self.ref,
+            "refType": self.ref_type,
+            "mimetype": self.mimetype,
+        }
+        return coordinates
+
+    @classmethod
+    def from_json(cls, _json):
+        coordinates = _json["coordinates"]
+        ref = coordinates.get('ref')
+        ref_type = coordinates.get('refType')
+        mimetype = coordinates.get('mimetype')
+        return cls(
+            ref=ref,
+            ref_type=ref_type,
+            mimetype=mimetype,
+            label=_json["label"],
+            attributes=_json.get("attributes", None),
+        )

dtlpy/entities/command.py

@@ -141,7 +141,7 @@ class Command(entities.BaseEntity):
                                entities.CommandsStatus.FINALIZING,
                                entities.CommandsStatus.IN_PROGRESS]
 
-    def wait(self, timeout=0, step=None,  backoff_factor=0.1):
+    def wait(self, timeout=0, step=None,  backoff_factor=1):
         """
         Wait for Command to finish
 

dtlpy/entities/dataset.py

@@ -244,9 +244,8 @@ class Dataset(entities.BaseEntity):
 
     @readonly.setter
     def readonly(self, state):
-        raise exceptions.PlatformException(
-            error='400',
-            message='Cannot set attribute readonly. Please use "set_readonly({})" method'.format(state))
+        import warnings
+        warnings.warn("`readonly` flag on dataset is deprecated, doing nothing.", DeprecationWarning)
 
     @property
     def labels_flat_dict(self):
@@ -518,11 +517,8 @@ class Dataset(entities.BaseEntity):
 
             dataset.set_readonly(state=True)
         """
-        if not isinstance(state, bool):
-            raise exceptions.PlatformException(
-                error='400',
-                message='Argument "state" must be bool. input type: {}'.format(type(state)))
-        return self.datasets.set_readonly(dataset=self, state=state)
+        import warnings
+        warnings.warn("`readonly` flag on dataset is deprecated, doing nothing.", DeprecationWarning)
 
     def clone(self,
               clone_name=None,

dtlpy/entities/feature_set.py

@@ -23,7 +23,6 @@ class FeatureSet(entities.BaseEntity):
     # platform
     id = attr.ib()
     name = attr.ib()
-    tags = attr.ib()
     url = attr.ib(repr=False)
     creator = attr.ib(repr=False)
     created_at = attr.ib()
@@ -104,7 +103,6 @@ class FeatureSet(entities.BaseEntity):
             created_at=_json.get('createdAt', None),
             creator=_json.get('creator', None),
             updated_by=_json.get('updatedBy', None),
-            tags=_json.get('tags', None),
             client_api=client_api,
             org_id=_json.get('org', None),
         )
@@ -127,7 +125,6 @@ class FeatureSet(entities.BaseEntity):
                  'createdAt': self.created_at,
                  'updatedBy': self.updated_by,
                  'name': self.name,
-                 'tags': self.tags,
                  'size': self.size,
                  'url': self.url}
         if self.org_id is not None:

dtlpy/entities/filters.py

@@ -33,7 +33,7 @@ class FiltersResource(str, Enum):
     MODEL = "models"
     WEBHOOK = "webhooks"
     RECIPE = 'recipe'
-    DATASET = 'dataset'
+    DATASET = 'datasets'
     ONTOLOGY = 'ontology'
     TASK = 'tasks'
     PIPELINE = 'pipeline'
@@ -113,6 +113,7 @@ class Filters:
         self._ref_op = None
         self._ref_assignment_id = None
         self._ref_task_id = None
+        self._system_space = None
 
         self._use_defaults = use_defaults
         self.__add_defaults()
@@ -146,6 +147,14 @@ class Filters:
         self.reset()
         self.__add_defaults()
 
+    @property
+    def system_space(self):
+        return self._system_space
+
+    @system_space.setter
+    def system_space(self, val: bool):
+        self._system_space = val
+
     def reset(self):
         self.or_filter_list = list()
         self.and_filter_list = list()
@@ -443,7 +452,8 @@ class Filters:
 
         if self.context is not None:
             _json['context'] = self.context
-
+        if self._system_space is not None:
+            _json['systemSpace'] = self._system_space
         return _json
 
     def sort_by(self, field, value: FiltersOrderByDirection = FiltersOrderByDirection.ASCENDING):

dtlpy/entities/item.py

@@ -639,7 +639,6 @@ class Item(entities.BaseEntity):
         .. code-block:: python
 
             item.update_status(status='complete',
-                               operation='created',
                                task_id='task_id')
 
         """

dtlpy/entities/model.py

@@ -105,6 +105,7 @@ class Model(entities.BaseEntity):
     _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
     _ontology = attr.ib(repr=False, default=None)
+    updated_by = attr.ib(default=None)
 
     @staticmethod
     def _protected_from_json(_json, client_api, project, package, is_fetched=True):
@@ -184,7 +185,8 @@ class Model(entities.BaseEntity):
             context=_json.get('context', {}),
             input_type=_json.get('inputType', None),
             output_type=_json.get('outputType', None),
-            module_name = _json.get('moduleName', None)
+            module_name=_json.get('moduleName', None),
+            updated_by=_json.get('updatedBy', None)
         )
         inst.is_fetched = is_fetched
         return inst
@@ -212,6 +214,7 @@ class Model(entities.BaseEntity):
                                                         attr.fields(Model).updated_at,
                                                         attr.fields(Model).input_type,
                                                         attr.fields(Model).output_type,
+                                                        attr.fields(Model).updated_by
                                                         ))
         _json['packageId'] = self.package_id
         _json['datasetId'] = self.dataset_id
@@ -230,6 +233,10 @@ class Model(entities.BaseEntity):
                 artifact = artifact.to_json(as_artifact=True)
             model_artifacts.append(artifact)
         _json['artifacts'] = model_artifacts
+
+        if self.updated_by:
+            _json['updatedBy'] = self.updated_by
+
         return _json
 
     ############
@@ -307,7 +314,9 @@ class Model(entities.BaseEntity):
                  dpks=repositories.Dpks(client_api=self._client_api),
                  services=repositories.Services(client_api=self._client_api,
                                                 project=self._project,
-                                                project_id=self.project_id),
+                                                project_id=self.project_id,
+                                                model_id=self.id,
+                                                model=self),
                  )
         return r
 
@@ -391,6 +400,46 @@ class Model(entities.BaseEntity):
     ###########
     # methods #
     ###########
+
+    def add_subset(self, subset_name: str, subset_filter: entities.Filters):
+        """
+        Adds a subset for the model, specifying a subset of the model's dataset that could be used for training or
+        validation.
+
+        :param str subset_name: the name of the subset
+        :param dtlpy.entities.Filters subset_filter: the filtering operation that this subset performs in the dataset.
+
+        **Example**
+
+        .. code-block:: python
+
+            model.add_subset(subset_name='train', subset_filter=dtlpy.Filters(field='dir', values='/train'))
+            model.metadata['system']['subsets']
+                {'train': <dtlpy.entities.filters.Filters object at 0x1501dfe20>}
+
+        """
+        self.models.add_subset(self, subset_name, subset_filter)
+
+    def delete_subset(self, subset_name: str):
+        """
+        Removes a subset from the model's metadata.
+
+        :param str subset_name: the name of the subset
+
+        **Example**
+
+        .. code-block:: python
+
+            model.add_subset(subset_name='train', subset_filter=dtlpy.Filters(field='dir', values='/train'))
+            model.metadata['system']['subsets']
+                {'train': <dtlpy.entities.filters.Filters object at 0x1501dfe20>}
+            models.delete_subset(subset_name='train')
+            metadata['system']['subsets']
+                {}
+
+        """
+        self.models.delete_subset(self, subset_name)
+
     def update(self, system_metadata=False):
         """
         Update Models changes to platform

dtlpy/entities/node.py

@@ -94,7 +94,8 @@ class PipelineNodeIO:
                  action: str = None,
                  default_value=None,
                  variable_name: str = None,
-                 actions: list = None):
+                 actions: list = None,
+                 description: str = None):
         """
         Pipeline Node
 
@@ -116,6 +117,7 @@ class PipelineNodeIO:
         self.port_percentage = port_percentage
         self.default_value = default_value
         self.variable_name = variable_name
+        self.description = description
 
         if action is not None:
             warnings.warn('action param has been deprecated in version 1.80', DeprecationWarning)
@@ -141,6 +143,7 @@ class PipelineNodeIO:
             default_value=_json.get('defaultValue', None),
             variable_name=_json.get('variableName', None),
             actions=_json.get('actions', None),
+            description=_json.get('description', None),
         )
 
     def to_json(self):
@@ -158,7 +161,8 @@ class PipelineNodeIO:
             _json['actions'] = self.actions
         if self.default_value:
             _json['defaultValue'] = self.default_value
-
+        if self.description:
+            _json['description'] = self.description
         return _json
 
 
@@ -506,9 +510,9 @@ class CodeNode(PipelineNode):
         :param list inputs: list of PipelineNodeIO inputs
         :param tuple position: tuple of the node place
         """
-        if not inputs:
+        if inputs is None:
             inputs = [self._default_io()]
-        if not outputs:
+        if outputs is None:
             outputs = [self._default_io()]
 
         if method is None or not isinstance(method, Callable):
@@ -857,7 +861,12 @@ class FunctionNode(PipelineNode):
         inputs = []
         outputs = []
         package = self.service.package
-        for model in package.modules:
+        modules = []
+        if isinstance(package, entities.Package):
+            modules = package.modules
+        elif isinstance(package, entities.Dpk):
+            modules = package.components.modules
+        for model in modules:
             if model.name == self.service.module_name:
                 for func in model.functions:
                     if func.name == function_name:

dtlpy/entities/ontology.py

@@ -396,9 +396,9 @@ class Ontology(entities.BaseEntity):
     def _add_image_label(self, icon_path):
         display_data = dict()
         if self.project is not None:
-            dataset = self.project.datasets.get(dataset_name='Binaries')
+            dataset = self.project.datasets._get_binaries_dataset()
         elif self.dataset is not None:
-            dataset = self.dataset.project.datasets.get(dataset_name='Binaries')
+            dataset = self.dataset.project.datasets._get_binaries_dataset()
         else:
             raise ValueError('must have project or dataset to create with icon path')
         platform_path = "/.dataloop/ontologies/{}/labelDisplayImages/".format(self.id)

dtlpy/entities/package_function.py

@@ -95,6 +95,7 @@ class FunctionIO(entities.DlEntity):
     value = entities.DlProperty(location=['value'], _type=str)
     name = entities.DlProperty(location=['name'], _type=str)
     actions = entities.DlProperty(location=['actions'], _type=list)
+    description = entities.DlProperty(location=['description'], _type=str)
 
     def __repr__(self):
         # TODO need to move to DlEntity
@@ -164,6 +165,8 @@ class FunctionIO(entities.DlEntity):
             }
             if self.actions:
                 _json['actions'] = self.actions
+            if self.description:
+                _json['description'] = self.description
         elif resource in ['execution', 'service']:
             _json = {
                 self.name: self.value

dtlpy/entities/pipeline.py

@@ -8,6 +8,8 @@ from .node import PipelineNode, PipelineConnection, TaskNode, CodeNode, Function
 from .. import repositories, entities
 from ..services.api_client import ApiClient
 from .package_function import PackageInputType
+import copy
+
 logger = logging.getLogger(name='dtlpy')
 
 
@@ -227,6 +229,8 @@ class Pipeline(entities.BaseEntity):
     _original_variables = attr.ib(repr=False, type=List[Variable])
     _repositories = attr.ib(repr=False)
 
+    updated_by = attr.ib(default=None)
+
     @staticmethod
     def _protected_from_json(_json, client_api, project, is_fetched=True):
         """
@@ -270,7 +274,8 @@ class Pipeline(entities.BaseEntity):
         json_variables = _json.get('variables', None) or list()
         variables = list()
         if json_variables:
-            variables = [Variable.from_json(_json=v) for v in json_variables]
+            copy_json_variables = copy.deepcopy(json_variables)
+            variables = [Variable.from_json(_json=v) for v in copy_json_variables]
 
         settings = PipelineSettings.from_json(_json=_json.get('settings', dict()))
         inst = cls(
@@ -295,6 +300,7 @@ class Pipeline(entities.BaseEntity):
             status=_json.get('status', None),
             original_settings=settings,
             original_variables=json_variables,
+            updated_by=_json.get('updatedBy', None),
         )
         for node in _json.get('nodes', list()):
             inst.nodes.add(node=cls.pipeline_node(node))
@@ -349,7 +355,8 @@ class Pipeline(entities.BaseEntity):
                                                         attr.fields(Pipeline).settings,
                                                         attr.fields(Pipeline).variables,
                                                         attr.fields(Pipeline)._original_settings,
-                                                        attr.fields(Pipeline)._original_variables
+                                                        attr.fields(Pipeline)._original_variables,
+                                                        attr.fields(Pipeline).updated_by,
                                                         ))
 
         _json['projectId'] = self.project_id
@@ -374,6 +381,8 @@ class Pipeline(entities.BaseEntity):
             _json['description'] = self.description
         if self.revisions is not None:
             _json['revisions'] = self.revisions
+        if self.updated_by is not None:
+            _json['updatedBy'] = self.updated_by
 
         return _json
 

dtlpy/entities/recipe.py

@@ -252,7 +252,7 @@ class Recipe(entities.BaseEntity):
                                                message='file Must be pdf')
         for project_id in self.project_ids:
             project = repositories.Projects(client_api=self._client_api).get(project_id=project_id)
-            dataset = project.datasets.get(dataset_name='Binaries')
+            dataset = project.datasets._get_binaries_dataset()
             remote_path = '/.dataloop/recipes/{}/instructions'.format(self.id)
             instruction_item = dataset.items.upload(local_path=annotation_instruction_file,
                                                     remote_path=remote_path,