dtlpy 1.96.8__py3-none-any.whl → 1.99.11__py3-none-any.whl

dtlpy/__init__.py

@@ -68,7 +68,7 @@ from .entities import (
     Ontology, Label, Task, TaskPriority, ConsensusTaskType, Assignment, Service, Package, Codebase, Model,
     PackageModule, PackageFunction,
     # annotations
-    Box, Cube, Cube3d, Point, Note, Message, Segmentation, Ellipse, Classification, Subtitle, Polyline, Pose,
+    Box, Cube, Cube3d, Point, Note, Message, Segmentation, Ellipse, Classification, Subtitle, Polyline, Pose, Gis, GisType,
     Description,
     Polygon, Text, FreeText, RefImage,
     # filters
@@ -93,7 +93,7 @@ from .entities import (
     Webhook, HttpMethod,
     ViewAnnotationOptions, AnnotationStatus, AnnotationType,
     ItemStatus, ExecutionStatus, ExportMetadata,
-    PromptItem, Prompt, PromptType,
+    PromptItem, Prompt, PromptType, ItemGis, Layer,
     ItemLink, UrlLink, LinkTypeEnum,
     Modality, ModalityTypeEnum, ModalityRefTypeEnum,
     Workload, WorkloadUnit, ItemAction,
@@ -239,6 +239,7 @@ def checkout_state():
 
 
 def use_attributes_2(state: bool = True):
+    warnings.warn("Function 'use_attributes_2()' is deprecated as of version 1.99.11 and has been non-functional since version 1.90.39.  To work with attributes 2.0, simply use 'update_attributes()'.", DeprecationWarning)
     client_api.attributes_mode.use_attributes_2 = state
 
 

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.96.8'
+version = '1.99.11'

dtlpy/entities/__init__.py

@@ -34,7 +34,7 @@ from .recipe import Recipe
 from .ontology import Ontology, AttributesTypes, AttributesRange
 from .annotation_definitions import Box, Cube, Cube3d, Point, Segmentation, Polygon, Ellipse, Classification, \
     Subtitle, Text, FreeText, RefImage, \
-    Polyline, Comparison, UndefinedAnnotationType, Note, Message, Description, Pose
+    Polyline, Comparison, UndefinedAnnotationType, Note, Message, Description, Pose, Gis, GisType
 from .label import Label
 from .codebase import Codebase, PackageCodebaseType, ItemCodebase, GitCodebase, FilesystemCodebase, LocalCodebase
 from .package import Package, RequirementOperator, PackageRequirement
@@ -78,3 +78,4 @@ from .prompt_item import Prompt, PromptItem, PromptType
 from .compute import ClusterProvider, ComputeType, ComputeStatus, Toleration, DeploymentResource, DeploymentResources, \
     NodePool, AuthenticationIntegration, Authentication, ComputeCluster, ComputeContext, Compute, KubernetesCompute, \
     ServiceDriver
+from .gis_item import  ItemGis, Layer

dtlpy/entities/annotation.py

@@ -43,6 +43,7 @@ class AnnotationType(str, Enum):
     SEGMENTATION = "binary"
     SUBTITLE = "subtitle"
     TEXT = "text_mark"
+    GIS = "gis"
 
 
 class ViewAnnotationOptions(str, Enum):
@@ -1796,6 +1797,8 @@ class FrameAnnotation(entities.BaseEntity):
             annotation = entities.Note.from_json(_json)
         elif _json['type'] == 'pose':
             annotation = entities.Pose.from_json(_json)
+        elif _json['type'] == 'gis':
+            annotation = entities.Gis.from_json(_json)
         else:
             annotation = entities.UndefinedAnnotationType.from_json(_json)
         return annotation

dtlpy/entities/annotation_definitions/__init__.py

@@ -17,3 +17,4 @@ from .pose import Pose
 from .text import Text
 from .free_text import FreeText
 from .ref_image import RefImage
+from .gis import Gis, GisType

dtlpy/entities/annotation_definitions/gis.py

@@ -0,0 +1,69 @@
+from . import BaseAnnotationDefinition
+
+
+class GisType:
+    """
+    State enum
+    """
+    BOX = 'box'
+    POLYGON = 'polygon'
+    POLYLINE = 'polyline'
+    POINT = 'point'
+
+
+class Gis(BaseAnnotationDefinition):
+    """
+        Box annotation object
+        Can create a box using 2 point using: "top", "left", "bottom", "right" (to form a box [(left, top), (right, bottom)])
+        For rotated box add the "angel"
+    """
+    type = "gis"
+
+    def __init__(self,
+                 annotation_type: GisType,
+                 geo,
+                 label=None,
+                 attributes=None,
+                 description=None,
+                 ):
+        """
+        Can create gis annotation using points:
+
+        :param geo: list of points
+        :param label: annotation label
+        :param attributes: a list of attributes for the annotation
+        :param description:
+
+        :return:
+        """
+        super().__init__(description=description, attributes=attributes)
+
+        if geo is None:
+            raise ValueError('geo must be provided')
+        if annotation_type is None:
+            raise ValueError('annotation_type must be provided')
+        self.label = label
+        self.annotation = None
+        self.geo = geo
+        self.annotation_type = annotation_type
+
+    def to_coordinates(self, color):
+        return {
+            "geo_type": self.annotation_type,
+            "wgs84_geo_coordinates": self.geo
+        }
+
+    @classmethod
+    def from_json(cls, _json):
+        json_coordinates = _json.get("coordinates", {}) if "coordinates" in _json else _json.get("data", {})
+        coordinates = json_coordinates.get("wgs84_geo_coordinates", None)
+        annotations_type = json_coordinates.get("geo_type", None)
+        if coordinates is None:
+            raise ValueError('can not find "coordinates" or "data" in annotation. id: {}'.format(_json["id"]))
+
+        return cls(
+            annotation_type=annotations_type,
+            geo=coordinates,
+            label=_json["label"],
+            attributes=_json.get("attributes", None)
+        )

dtlpy/entities/app.py

@@ -199,7 +199,7 @@ class App(entities.BaseEntity):
             name=_json.get('name', None),
             url=_json.get('url', None),
             created_at=_json.get('createdAt', None),
-            updated_at=_json.get('updateAt', None),
+            updated_at=_json.get('updatedAt', None),
             creator=_json.get('creator', None),
             project_id=_json.get('projectId', None),
             org_id=_json.get('orgId', None),

dtlpy/entities/dpk.py

@@ -59,6 +59,7 @@ class Toolbar(entities.DlEntity):
 
 class Panel(entities.DlEntity):
     name = entities.DlProperty(location=['name'], _type=str)
+    path = entities.DlProperty(location=['path'], _type=str, default=None)
     min_role = entities.DlProperty(location=['minRole'], _type=list)
     supported_slots = entities.DlProperty(location=['supportedSlots'], _type=list)
 

dtlpy/entities/filters.py

@@ -330,7 +330,7 @@ class Filters:
                 self._unique_fields = ['type']
                 self.add(field='type',
                          values=['box', 'class', 'comparison', 'ellipse', 'point', 'segment', 'polyline', 'binary',
-                                 'subtitle', 'cube', 'cube_3d', 'pose', 'text_mark', 'text', 'ref_image'],
+                                 'subtitle', 'cube', 'cube_3d', 'pose', 'text_mark', 'text', 'ref_image', 'gis'],
                          operator=FiltersOperations.IN,
                          method=FiltersMethod.AND)
 

dtlpy/entities/gis_item.py

@@ -0,0 +1,108 @@
+import json
+from typing import List
+import logging
+import os
+
+logger = logging.getLogger(name='dtlpy')
+
+
+class Layer:
+    def __init__(self, name, layer_type, url):
+        self.name = name
+        self.type = layer_type
+        self.url = url
+
+
+class ItemGis:
+    def __init__(self,
+                 name: str,
+                 data: dict = None,
+                 layer: Layer = None,
+                 optional_layers: List[Layer] = None,
+                 zoom: int = None,
+                 min_zoom: int = None,
+                 max_zoom: int = None,
+                 epsg: str = None,
+                 bounds: list = None,
+                 aoi: list = None):
+        self.name = name
+
+        self.layer = layer or Layer(name=data.get('name', None), layer_type=data.get('type', None),
+                                    url=data.get('url', None))
+        if self.layer is None:
+            raise ValueError('layer is required')
+        elif self.layer is not None and isinstance(self.layer, dict):
+            self.layer = Layer(name=self.layer.get('name', None), layer_type=self.layer.get('type', None), url=self.layer.get('url', None))
+
+
+        self.optional_layers = optional_layers or [
+            Layer(name=layer.get('name', None), layer_type=layer.get('type', None), url=layer.get('url', None)) for
+            layer in data.get('optionalLayers', [])]
+
+        if self.optional_layers is not None and isinstance(optional_layers, list):
+            new_optional_layers = []
+            for op_layer in self.optional_layers:
+                if isinstance(op_layer, dict):
+                    new_optional_layers.append(Layer(name=op_layer.get('name', None), layer_type=op_layer.get('type', None), url=op_layer.get('url', None)))
+                else:
+                    new_optional_layers.append(op_layer)
+            self.optional_layers = new_optional_layers
+
+        self.epsg = epsg or data.get('epsg', None)
+        if self.epsg is None:
+            raise ValueError('epsg is required')
+
+        self.zoom = zoom or data.get('zoom', None)
+        self.min_zoom = min_zoom or data.get('minZoom', None)
+        self.max_zoom = max_zoom or data.get('maxZoom', None)
+        self.bounds = bounds or data.get('bounds', None)
+        self.aoi = aoi or data.get('aoi', None)
+
+    def to_json(self):
+        _json = {
+            "type": "gis",
+            "shebang": "dataloop",
+            "metadata": {
+                "dltype": "gis"
+            },
+            'layer': {
+                'name': self.layer.name,
+                'type': self.layer.type,
+                'url': self.layer.url
+            },
+            "epsg": self.epsg
+        }
+        if self.optional_layers is not None:
+            _json['optionalLayers'] = [
+                {
+                    'name': layer.name,
+                    'type': layer.type,
+                    'url': layer.url
+                } for layer in self.optional_layers
+            ]
+        if self.zoom is not None:
+            _json['zoom'] = self.zoom
+        if self.min_zoom is not None:
+            _json['minZoom'] = self.min_zoom
+        if self.max_zoom is not None:
+            _json['maxZoom'] = self.max_zoom
+        if self.bounds is not None:
+            _json['bounds'] = self.bounds
+        if self.aoi is not None:
+            _json['aoi'] = self.aoi
+        return _json
+
+    @classmethod
+    def from_local_file(cls, filepath):
+        """
+        Create a new prompt item from a file
+        :param filepath: path to the file
+        :return: PromptItem object
+        """
+        if os.path.exists(filepath) is False:
+            raise FileNotFoundError(f'File does not exists: {filepath}')
+        if 'json' not in os.path.splitext(filepath)[-1]:
+            raise ValueError(f'Expected path to json item, got {os.path.splitext(filepath)[-1]}')
+        with open(filepath, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return cls(name=os.path.basename(filepath), data=data)

dtlpy/entities/model.py

@@ -307,7 +307,7 @@ class Model(entities.BaseEntity):
     @_repositories.default
     def set_repositories(self):
         reps = namedtuple('repositories',
-                          field_names=['projects', 'datasets', 'packages', 'models', 'ontologies', 'artifacts',
+                          field_names=['projects', 'datasets', 'models', 'packages', 'ontologies', 'artifacts',
                                        'metrics', 'dpks', 'services'])
 
         r = reps(projects=repositories.Projects(client_api=self._client_api),
@@ -552,14 +552,34 @@ class Model(entities.BaseEntity):
                                     filters=filters,
                                     service_config=service_config)
 
-    def predict(self, item_ids):
+    def predict(self, item_ids=None, dataset_id=None):
         """
         Run model prediction with items
 
         :param item_ids: a list of item id to run the prediction.
+        :param dataset_id: dataset id to run the prediction on
         :return:
         """
-        return self.models.predict(model=self, item_ids=item_ids)
+        return self.models.predict(model=self, item_ids=item_ids, dataset_id=dataset_id)
+
+    def embed(self, item_ids):
+        """
+        Run model embed with items
+
+        :param item_ids: a list of item id to run the embed.
+        :return:
+        """
+        return self.models.embed(model=self, item_ids=item_ids)
+
+    def embed_datasets(self, dataset_ids, attach_trigger=False):
+        """
+        Run model embed with datasets
+
+        :param dataset_ids: a list of dataset id to run the embed.
+        :param attach_trigger: bool - True, if you want to activate the trigger
+        :return:
+        """
+        return self.models.embed_datasets(model=self, dataset_ids=dataset_ids, attach_trigger=attach_trigger)
 
     def deploy(self, service_config=None) -> entities.Service:
         """

dtlpy/entities/pipeline.py

@@ -488,26 +488,35 @@ class Pipeline(entities.BaseEntity):
         """
         return self.pipelines.pause(pipeline=self, keep_triggers_active=keep_triggers_active)
 
-    def execute(self, execution_input=None):
+    def execute(self, execution_input=None, node_id: str = None):
         """
         execute a pipeline and return to execute
 
         :param execution_input: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}
+        :param str node_id: node id to execute
         :return: entities.PipelineExecution object
         """
-        execution = self.pipeline_executions.create(pipeline_id=self.id, execution_input=execution_input)
+        execution = self.pipeline_executions.create(
+            pipeline_id=self.id,
+            execution_input=execution_input,
+            node_id=node_id
+        )
         return execution
 
-    def execute_batch(self,
-                      filters,
-                      execution_inputs=None,
-                      wait=True):
+    def execute_batch(
+            self,
+            filters,
+            execution_inputs=None,
+            wait=True,
+            node_id: str = None
+    ):
         """
         execute a pipeline and return to execute
 
         :param execution_inputs: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}, that represent the extra inputs of the function
         :param filters: Filters entity for a filtering before execute
         :param bool wait: wait until create task finish
+        :param str node_id: node id to execute
         :return: entities.PipelineExecution object
 
         **Example**:
@@ -518,10 +527,13 @@ class Pipeline(entities.BaseEntity):
                         execution_inputs=dl.FunctionIO(type=dl.PackageInputType.STRING, value='test', name='string'),
                         filters=dl.Filters(field='dir', values='/test', context={'datasets': [dataset.id]))
         """
-        command = self.pipeline_executions.create_batch(pipeline_id=self.id,
-                                                        execution_inputs=execution_inputs,
-                                                        filters=filters,
-                                                        wait=wait)
+        command = self.pipeline_executions.create_batch(
+            pipeline_id=self.id,
+            execution_inputs=execution_inputs,
+            filters=filters,
+            wait=wait,
+            node_id=node_id
+        )
         return command
 
     def reset(self, stop_if_running: bool = False):
@@ -578,4 +590,3 @@ class Pipeline(entities.BaseEntity):
         for variable in self.variables:
             if variable.name in keys:
                 variable.value = kwargs[variable.name]
-

dtlpy/entities/prompt_item.py

@@ -347,7 +347,6 @@ class PromptItem:
     def add(self,
             message: dict,
             prompt_key: str = None,
-            stream: bool = True,
             model_info: dict = None):
         """
         add a prompt to the prompt item
@@ -356,29 +355,28 @@ class PromptItem:
 
         :param message:
         :param prompt_key:
-        :param stream:
         :param model_info:
         :return:
         """
-        if prompt_key is None:
-            prompt_key = len(self.prompts) + 1
         role = message.get('role', 'user')
         content = message.get('content', list())
 
         if self.role_mapping.get(role, 'item') == 'item':
+            if prompt_key is None:
+                prompt_key = str(len(self.prompts) + 1)
             # for new prompt we need a new key
-            prompt = Prompt(key=str(prompt_key), role=role)
+            prompt = Prompt(key=prompt_key, role=role)
             for element in content:
                 prompt.add_element(value=element.get('value', ''),
                                    mimetype=element.get('mimetype', PromptType.TEXT))
 
             # create new prompt and add to prompts
             self.prompts.append(prompt)
-            if self._item is not None and stream is True:
+            if self._item is not None:
                 self._item._Item__update_item_binary(_json=self.to_json())
         else:
-            # for response - we need to assign to previous key
-            prompt_key = str(prompt_key - 1)
+            if prompt_key is None:
+                prompt_key = str(len(self.prompts))
             assistant_message = content[0]
             assistant_mimetype = assistant_message.get('mimetype', PromptType.TEXT)
             uploaded_annotation = None
@@ -421,21 +419,19 @@ class PromptItem:
                 prompt.add_element(mimetype=PromptType.METADATA,
                                    value={"model_info": model_info})
 
-                if stream:
-                    existing_annotation = entities.Annotation.new(item=self._item,
-                                                                  metadata=metadata,
-                                                                  annotation_definition=annotation_definition)
-                    uploaded_annotation = existing_annotation.upload()
-                    prompt.add_element(mimetype=PromptType.METADATA,
-                                       value={"id": uploaded_annotation.id})
+                existing_annotation = entities.Annotation.new(item=self._item,
+                                                              metadata=metadata,
+                                                              annotation_definition=annotation_definition)
+                uploaded_annotation = existing_annotation.upload()
+                prompt.add_element(mimetype=PromptType.METADATA,
+                                   value={"id": uploaded_annotation.id})
                 existing_prompt = prompt
                 self.assistant_prompts.append(prompt)
 
-            # TODO Shadi fix
             existing_prompt_element = [element for element in existing_prompt.elements if
                                        element['mimetype'] != PromptType.METADATA][-1]
             existing_prompt_element['value'] = assistant_message.get('value')
-            if stream is True and uploaded_annotation is None:
+            if uploaded_annotation is None:
                 # Creating annotation with old dict to match platform dict
                 annotation_definition = entities.FreeText(text='')
                 metadata = {'system': {'promptId': prompt_key},
@@ -451,7 +447,3 @@ class PromptItem:
                 # update the annotation with the new text
                 annotation.annotation_definition.text = existing_prompt_element['value']
                 self._item.annotations.update(annotation)
-
-    def update(self):
-        if self._item is not None:
-            self._item._Item__update_item_binary(_json=self.to_json())

dtlpy/ml/base_model_adapter.py

@@ -271,7 +271,7 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
         :param bool overwrite: overwrite the data path (download again). default is False
         """
         # define paths
-        dataloop_path = os.path.join(os.path.expanduser('~'), '.dataloop')
+        dataloop_path = service_defaults.DATALOOP_PATH
         root_path = self.adapter_defaults.resolve("root_path", root_path)
         data_path = self.adapter_defaults.resolve("data_path", data_path)
         output_path = self.adapter_defaults.resolve("output_path", output_path)

dtlpy/new_instance.py

@@ -13,7 +13,7 @@ class Dtlpy:
         Ontology, Label, Task, TaskPriority, ConsensusTaskType, Assignment, Service, Package, Codebase, Model,
         PackageModule, PackageFunction,
         # annotations
-        Box, Cube, Cube3d, Point, Note, Message, Segmentation, Ellipse, Classification, Subtitle, Polyline, Pose,
+        Box, Cube, Cube3d, Point, Note, Message, Segmentation, Ellipse, Classification, Subtitle, Polyline, Pose, Gis, GisType,
         Description,
         Polygon, Text, FreeText, RefImage,
         # filters
@@ -38,7 +38,7 @@ class Dtlpy:
         Webhook, HttpMethod,
         ViewAnnotationOptions, AnnotationStatus, AnnotationType,
         ItemStatus, ExecutionStatus, ExportMetadata,
-        PromptItem, Prompt, PromptType,
+        PromptItem, Prompt, PromptType, ItemGis, Layer,
         ItemLink, UrlLink, LinkTypeEnum,
         Modality, ModalityTypeEnum, ModalityRefTypeEnum,
         Workload, WorkloadUnit, ItemAction,

dtlpy/repositories/annotations.py

@@ -312,7 +312,7 @@ class Annotations:
         """
         # get item's annotations
         annotations = self.list()
-        if 'text' in self.item.metadata.get('system').get('mimetype', ''):
+        if 'text' in self.item.metadata.get('system').get('mimetype', '') or 'json' in self.item.metadata.get('system').get('mimetype', ''):
             annotation_format = entities.ViewAnnotationOptions.JSON
         elif 'audio' not in self.item.metadata.get('system').get('mimetype', ''):
             # height/weight