dtlpy 1.90.39__py3-none-any.whl → 1.91.37__py3-none-any.whl

dtlpy/entities/dpk.py

@@ -126,6 +126,7 @@ class DpkComputeConfig(entities.DlEntity):
     driver_id: str = entities.DlProperty(location=['driverId'], _type=str)
     versions: dict = entities.DlProperty(location=['versions'], _type=dict)
     name: str = entities.DlProperty(location=['name'], _type=str)
+    integrations: List[dict] = entities.DlProperty(location=['integrations'], _type=list)
 
     def to_json(self) -> dict:
         return self._dict.copy()
@@ -226,6 +227,7 @@ class Components(entities.DlEntity):
 class Dpk(entities.DlEntity):
     # name change
     id: str = entities.DlProperty(location=['id'], _type=str)
+    base_id: str = entities.DlProperty(location=['baseId'], _type=str)
     name: str = entities.DlProperty(location=['name'], _type=str)
     version: str = entities.DlProperty(location=['version'], _type=str)
     attributes: list = entities.DlProperty(location=['attributes'], _type=dict)
@@ -388,6 +390,19 @@ class Dpk(entities.DlEntity):
             self._revisions = self._get_revision_pages()
         return self._revisions
 
+    def get_revisions(self, version: str):
+        """
+        Get the dpk with the specified version.
+
+        :param str version: the version of the dpk to get.
+        :return: Dpk
+
+        ** Example **
+        ..code-block:: python
+        dpk = dpk.get_revisions(version='1.0.0')
+        """
+        return self.dpks.get_revisions(dpk_id=self.base_id, version=version)
+
     @staticmethod
     def _protected_from_json(_json, client_api, project, is_fetched=True):
         """

dtlpy/entities/execution.py

@@ -13,8 +13,12 @@ logger = logging.getLogger(name='dtlpy')
 class ExecutionStatus(str, Enum):
     SUCCESS = "success"
     FAILED = "failed"
-    IN_PROGRESS = "inProgress"
+    IN_PROGRESS = "in-progress"
     CREATED = "created"
+    TERMINATED = 'terminated',
+    ABORTED = 'aborted'
+    CANCELED = 'canceled'
+    SYSTEM_FAILURE = 'system-failure'
 
 
 @attr.s
@@ -377,3 +381,11 @@ class Execution(entities.BaseEntity):
         :return: Service execution object
         """
         return self.executions.wait(execution_id=self.id)
+
+    def in_progress(self):
+        return self.latest_status['status'] not in [ExecutionStatus.FAILED,
+                                                    ExecutionStatus.SUCCESS,
+                                                    ExecutionStatus.TERMINATED,
+                                                    ExecutionStatus.ABORTED,
+                                                    ExecutionStatus.CANCELED,
+                                                    ExecutionStatus.SYSTEM_FAILURE]

dtlpy/entities/filters.py

@@ -2,6 +2,7 @@ import urllib.parse
 import logging
 import json
 import os
+import io
 from enum import Enum
 
 from .. import exceptions, entities
@@ -139,7 +140,7 @@ class Filters:
 
     @property
     def resource(self):
-        return self._resource
+        return f'{self._resource.value}' if isinstance(self._resource, FiltersResource) else f'{self._resource}'
 
     @resource.setter
     def resource(self, resource):
@@ -474,7 +475,7 @@ class Filters:
         """
         if value not in [FiltersOrderByDirection.ASCENDING, FiltersOrderByDirection.DESCENDING]:
             raise exceptions.PlatformException(error='400', message='Sort can be by ascending or descending order only')
-        self.sort[field] = value
+        self.sort[field] = value.value if isinstance(value, FiltersOrderByDirection) else value
 
     def platform_url(self, resource) -> str:
         """
@@ -488,14 +489,17 @@ class Filters:
         # add the view option
         _json['view'] = 'icons'
         # convert from enum to string
-        _json["resource"] = '{}'.format(_json['resource'])
+        _json["resource"] = f'{_json["resource"]}'
         # convert the dictionary to a json string
-        _json['dqlFilter'] = json.dumps({'filter': _json.pop('filter')})
+        _json['dqlFilter'] = json.dumps({'filter': _json.pop('filter'),
+                                         'join': _json.pop('join'),
+                                         'sort': _json.get('sort')})
         # set the page size as the UI default
         _json['pageSize'] = 100
+        _json['page'] = _json['page']
         # build the url for the dataset data browser
         if isinstance(resource, entities.Dataset):
-            url = resource.platform_url + '/items?{}'.format(urllib.parse.urlencode(_json))
+            url = resource.platform_url + f'?{urllib.parse.urlencode(_json)}'
         else:
             raise NotImplementedError('Not implemented for resource type: {}'.format(type(resource)))
         return url
@@ -511,6 +515,81 @@ class Filters:
         else:
             raise NotImplementedError('Not implemented for resource type: {}'.format(type(resource)))
 
+    def save(self, project: entities.Project, filter_name: str):
+        """
+        Save the current DQL filter to the project
+
+        :param project: dl.Project
+        :param filter_name: the saved filter's name
+        :return: True if success
+        """
+        _json_filter = self.prepare()
+        shebang_dict = {"type": "dql",
+                        "shebang": "dataloop",
+                        "metadata": {
+                            "version": "1.0.0",
+                            "system": {
+                                "mimetype": "dql"
+                            },
+                            "dltype": "filter",
+                            "filterFieldsState": [],
+                            "resource": "items",
+                            "filter": _json_filter.pop('filter'),
+                            "join": _json_filter.pop('join')
+                        }
+                        }
+        b_dataset = project.datasets._get_binaries_dataset()
+        byte_io = io.BytesIO()
+        byte_io.name = filter_name
+        byte_io.write(json.dumps(shebang_dict).encode())
+        byte_io.seek(0)
+        b_dataset.items.upload(local_path=byte_io,
+                               remote_path='/.dataloop/dqlfilters/items',
+                               remote_name=filter_name)
+        return True
+
+    @classmethod
+    def load(cls, project: entities.Project, filter_name: str) -> 'Filters':
+        """
+        Load a saved filter from the project by name
+
+        :param project: dl.Project entity
+        :param filter_name: filter name
+        :return: dl.Filters
+        """
+        b_dataset = project.datasets._get_binaries_dataset()
+        f = entities.Filters(custom_filter={
+            'filter': {'$and': [{'filename': f'/.dataloop/dqlfilters/items/{filter_name}'}]},
+            'page': 0,
+            'pageSize': 1000,
+            'resource': 'items'
+        })
+        pages = b_dataset.items.list(filters=f)
+        if pages.items_count == 0:
+            raise exceptions.NotFound(
+                f'Saved filter not found: {filter_name}. Run `Filters.list()` to list existing filters')
+        with open(pages.items[0].download()) as f:
+            data = json.load(f)
+            custom_filter = data['metadata']['filter']
+            custom_filter['join'] = data['metadata']['join']
+        return cls(custom_filter=custom_filter)
+
+    @staticmethod
+    def list(project: entities.Project) -> list:
+        """
+        List all saved filters for a project
+        :param project: dl.Project entity
+        :return: a list of all the saved filters' names
+        """
+        b_dataset = project.datasets._get_binaries_dataset()
+        f = entities.Filters(use_defaults=False,
+                             field='dir',
+                             values='/.dataloop/dqlfilters/items')
+        pages = b_dataset.items.list(filters=f)
+        all_filter_items = list(pages.all())
+        names = [i.name for i in all_filter_items]
+        return names
+
 
 class SingleFilter:
     def __init__(self, field, values, operator: FiltersOperations = None):
@@ -552,6 +631,6 @@ class SingleFilter:
             _json[self.field] = value
 
         return _json
-    
+
     def print(self, indent=2):
         print(json.dumps(self.prepare(), indent=indent))

dtlpy/entities/integration.py

@@ -79,7 +79,7 @@ class Integration(entities.BaseEntity):
                    name=_json.get('name', None),
                    creator=_json.get('creator', None),
                    created_at=_json.get('createdAt', None),
-                   update_at=_json.get('updateAt', None),
+                   update_at=_json.get('updatedAt', None),
                    type=_json.get('type', None),
                    org=_json.get('org', None),
                    client_api=client_api,

dtlpy/entities/item.py

@@ -661,6 +661,32 @@ class Item(entities.BaseEntity):
         else:
             raise exceptions.PlatformException('400', 'must provide assignment_id or task_id')
 
+    def status(self, assignment_id: str = None, task_id: str = None):
+        """
+        Get item status
+
+        :param str assignment_id: assignment id
+        :param str task_id: task id
+
+        :return: status
+        :rtype: str
+
+        **Example**:
+
+        .. code-block:: python
+
+            status = item.status(task_id='task_id')
+        """
+        if not assignment_id and not task_id:
+            raise exceptions.PlatformException('400', 'must provide assignment_id or task_id')
+        status = None
+        resource_id = assignment_id if assignment_id else task_id
+        for ref in self.metadata.get('system', dict()).get('refs', []):
+            if ref.get('id') == resource_id:
+                status = ref.get('metadata', {}).get('status', None)
+                break
+        return status
+
     def set_description(self, text: str):
         """
         Update Item description

dtlpy/entities/node.py

@@ -120,7 +120,7 @@ class PipelineNodeIO:
         self.description = description
 
         if action is not None:
-            warnings.warn('action param has been deprecated in version 1.80', DeprecationWarning)
+            warnings.warn('action param has been deprecated in version 1.95', DeprecationWarning)
             if actions is None:
                 actions = []
             actions.append(action)
@@ -128,7 +128,7 @@ class PipelineNodeIO:
 
     @property
     def action(self):
-        warnings.warn('action attribute has been deprecated in version 1.80', DeprecationWarning)
+        warnings.warn('action attribute has been deprecated in version 1.95', DeprecationWarning)
         return None
 
     @staticmethod
@@ -921,12 +921,20 @@ class DatasetNode(PipelineNode):
                  project_id: str,
                  dataset_id: str,
                  dataset_folder: str = None,
+                 load_existing_data: bool = False,
+                 data_filters: entities.Filters = None,
                  position: tuple = (1, 1)):
         """
         :param str name: node name
         :param str project_id: project id
         :param str dataset_id: dataset id
         :param str dataset_folder: folder in dataset to work in it
+        :param bool load_existing_data: optional - enable to automatically load existing data into the
+                                        pipeline (executions) upon activation, based on the defined dataset,
+                                        folder, and data_filters.
+        :param entities.Filters data_filters: optional - filters entity or a dictionary containing filters parameters.
+                                              Use to filter the data items to be loaded when load_existing_data
+                                              is enabled.
         :param tuple position: tuple of the node place
         """
         inputs = [self._default_io()]
@@ -944,6 +952,8 @@ class DatasetNode(PipelineNode):
                          position=position)
         self.dataset_id = dataset_id
         self.dataset_folder = dataset_folder
+        self.load_existing_data = load_existing_data
+        self.data_filters = data_filters
 
     @property
     def dataset_id(self):
@@ -964,6 +974,32 @@ class DatasetNode(PipelineNode):
                 dataset_folder = '/' + dataset_folder
             self.metadata['dir'] = dataset_folder
 
+    @property
+    def load_existing_data(self):
+        return self.metadata.get('triggerToPipeline', {}).get('active', False)
+
+    @load_existing_data.setter
+    def load_existing_data(self, load_existing_data: bool):
+        if load_existing_data:
+            self.metadata.setdefault('triggerToPipeline', {})['active'] = True
+        else:
+            self.metadata.pop('triggerToPipeline', None)
+
+    @property
+    def data_filters(self):
+        data_filters = self.metadata.get('triggerToPipeline', {}).get('filter', None)
+        if data_filters:
+            data_filters = entities.Filters(custom_filter=json.loads(data_filters))
+        return data_filters
+
+    @data_filters.setter
+    def data_filters(self, data_filters: entities.Filters):
+        if data_filters is None:
+            filters = None
+        else:
+            filters = json.dumps(data_filters.prepare(query_only=True).get('filter'))
+        self.metadata.setdefault('triggerToPipeline', {})['filter'] = filters
+
     @staticmethod
     def from_json(_json: dict):
         parent = PipelineNode.from_json(_json)

dtlpy/entities/ontology.py

@@ -741,3 +741,64 @@ class Ontology(entities.BaseEntity):
         """
 
         return self.ontologies.delete_attributes(ontology_id=self.id, keys=keys)
+
+    def copy_from(self, ontology_json: dict):
+        """
+        Import ontology to the platform ('ontology' is taken before 'ontology_json')
+
+        :param dict ontology_json: ontology json
+        :return: Ontology object
+        :rtype: dtlpy.entities.ontology.Ontology
+
+        **Example**:
+
+        .. code-block:: python
+
+            ontology = ontology.import_ontology(ontology_json=ontology_json)
+        """
+        # TODO: Add support for import from ontology entity
+        ontology = self.from_json(_json=ontology_json, client_api=self._client_api, recipe=self.recipe)
+        attributes = ontology.attributes
+
+        # params
+        self.labels = ontology.labels
+        for key, value in ontology.metadata.items():
+            if key != "system":
+                self.metadata[key] = value
+
+        if attributes:
+            # Delete irrelevant attribute keys
+            attribute_keys = [attribute.get("key", None) for attribute in attributes]
+            to_delete_keys = [attribute.get("key", None) for attribute in self.attributes
+                              if attribute.get("key", None) not in attribute_keys]
+            self.delete_attributes(keys=to_delete_keys)
+
+            # Update attributes
+            for attribute in attributes:
+                attribute_range = attribute.get("range", None)
+                if attribute_range is not None:
+                    attribute_range = entities.AttributesRange(
+                        min_range=attribute_range.get("min", None),
+                        max_range=attribute_range.get("max", None),
+                        step=attribute_range.get("step", None)
+                    )
+
+                script_data = attribute.get("scriptData", None)
+                if script_data is None:
+                    raise Exception(f"Attribute '{attribute.get('key')}' scriptData is missing in the ontology json!")
+                self.update_attributes(
+                    title=script_data.get("title", None),
+                    key=attribute.get("key", None),
+                    attribute_type=attribute.get("type", None),
+                    scope=attribute.get("scope", None),
+                    optional=script_data.get("optional", None),
+                    values=attribute.get("values", None),
+                    attribute_range=attribute_range
+                )
+        else:
+            logger.warning("No attributes were found (Make sure that you use the correct attributes mode).")
+
+        # defaults
+        self._instance_map = ontology.instance_map
+        self._color_map = ontology.color_map
+        return self.update(system_metadata=True)

dtlpy/entities/package_function.py

@@ -97,6 +97,8 @@ class FunctionIO(entities.DlEntity):
     name = entities.DlProperty(location=['name'], _type=str)
     actions = entities.DlProperty(location=['actions'], _type=list)
     description = entities.DlProperty(location=['description'], _type=str)
+    integration = entities.DlProperty(location=['integration'], _type=dict)
+    mandatory = entities.DlProperty(location=['mandatory'], _type=bool)
 
     def __repr__(self):
         # TODO need to move to DlEntity

dtlpy/entities/package_module.py

@@ -75,7 +75,20 @@ class PackageModule(entities.DlEntity):
         for cls_name, cls_inst in inspect.getmembers(file_module, predicate=inspect.isclass):
             spec = getattr(cls_inst, '__dtlpy__', None)
             if spec is not None:
+                functions = spec['functions']
+                available_methods = [name for name in ['train', 'predict']
+                                     if 'BaseModelAdapter' not in getattr(cls_inst, name).__qualname__]
+                if "train" not in available_methods:
+                    # remove train_model from functions list if train is not available
+                    functions[:] = [d for d in functions if d.get('name') != "train_model"]
+                if "predict" not in available_methods:
+                    # remove predict_items from functions list if predict is not available
+                    functions[:] = [d for d in functions if d.get('name') != "predict_items"]
+                if "extract_features" not in available_methods:
+                    # remove extract_item_features from functions list if extract_features is not available
+                    functions[:] = [d for d in functions if d.get('name') != "extract_item_features"]
                 spec['entryPoint'] = entry_point
+                spec['functions'] = functions
                 module = cls.from_json(spec)
                 break
         if module is None:

dtlpy/entities/pipeline_execution.py

@@ -76,6 +76,7 @@ class PipelineExecution(entities.BaseEntity):
 
     # sdk
     _pipeline = attr.ib(repr=False)
+    _project = attr.ib(repr=False)
     _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
 
@@ -103,7 +104,7 @@ class PipelineExecution(entities.BaseEntity):
         return status, pipeline
 
     @classmethod
-    def from_json(cls, _json, client_api, pipeline, is_fetched=True):
+    def from_json(cls, _json, client_api, pipeline, is_fetched=True) -> 'PipelineExecution':
         """
         Turn platform representation of pipeline_execution into a pipeline_execution entity
 
@@ -112,9 +113,11 @@ class PipelineExecution(entities.BaseEntity):
         :param dtlpy.entities.pipeline.Pipeline pipeline: Pipeline entity
         :param bool is_fetched: is Entity fetched from Platform
         :return: Pipeline entity
-        :rtype: dtlpy.entities.pipeline.Pipeline
+        :rtype: dtlpy.entities.PipelineExecution
         """
+        project = None
         if pipeline is not None:
+            project = pipeline._project
             if pipeline.id != _json.get('pipelineId', None):
                 logger.warning('Pipeline has been fetched from a project that is not belong to it')
                 pipeline = None
@@ -132,6 +135,7 @@ class PipelineExecution(entities.BaseEntity):
             nodes=nodes,
             executions=_json.get('executions', dict()),
             pipeline=pipeline,
+            project=project,
             client_api=client_api,
         )
 
@@ -183,7 +187,10 @@ class PipelineExecution(entities.BaseEntity):
 
     @property
     def project(self):
-        return self.pipeline.project
+        if self._project is None:
+            self._project = self.pipeline.project
+        assert isinstance(self._pipeline.project, entities.Project)
+        return self._pipeline.project
 
     ################
     # repositories #
@@ -195,9 +202,10 @@ class PipelineExecution(entities.BaseEntity):
 
         r = reps(
             projects=repositories.Projects(client_api=self._client_api),
-            pipelines=repositories.Pipelines(client_api=self._client_api, project=self.project),
-            pipeline_executions=repositories.PipelineExecutions(client_api=self._client_api, project=self.project,
-                                                                pipeline=self.pipeline)
+            pipelines=repositories.Pipelines(client_api=self._client_api, project=self._project),
+            pipeline_executions=repositories.PipelineExecutions(client_api=self._client_api,
+                                                                project=self._project,
+                                                                pipeline=self._pipeline)
         )
         return r
 

dtlpy/entities/prompt_item.py

@@ -78,6 +78,16 @@ class PromptItem:
                 prompts_json["prompts"][prompt_key] = prompt_values
         return prompts_json
 
+    @classmethod
+    def from_json(cls, _json):
+        inst = cls(name='dummy')
+        for prompt_key, prompt_values in _json["prompts"].items():
+            prompt = Prompt(key=prompt_key)
+            for val in prompt_values:
+                prompt.add(mimetype=val['mimetype'], value=val['value'])
+            inst.prompts.append(prompt)
+        return inst
+
     def to_bytes_io(self):
         byte_io = io.BytesIO()
         byte_io.name = self.name

dtlpy/entities/recipe.py

@@ -262,3 +262,40 @@ class Recipe(entities.BaseEntity):
                                                               'datasetId': dataset.id,
                                                               'name': instruction_item.name}
             self.update(True)
+
+    def upload_annotations_verification_file(self, local_path: str, overwrite: bool = False) -> entities.Item:
+        """
+        Add Annotations Verification js file to the recipe.
+
+        :param str local_path: file path of the annotations verification js file.
+        :param bool overwrite: overwrite exiting file if the local and the remote names are matching
+        :return: annotations verification js item.
+        """
+
+        validation_file_metadata = self.metadata.get("system", dict()).get("validationFile", None)
+        if validation_file_metadata is None:
+            validation_file_metadata = dict()
+
+        remote_name = validation_file_metadata.get("name", None)
+        local_name = os.path.basename(local_path)
+        binaries_dataset = self._project.datasets._get_binaries_dataset()
+        remote_path = f"/.dataloop/recipes/{self.id}/verification/"
+
+        if remote_name is None or overwrite or remote_name != local_name:
+            validation_item = binaries_dataset.items.upload(
+                local_path=local_path,
+                remote_path=remote_path,
+                remote_name=local_name,
+                overwrite=True
+            )
+            self.metadata["system"]["validationFile"] = {
+                "itemId": validation_item.id,
+                "datasetId": binaries_dataset.id,
+                "name": local_name
+            }
+            self.update(system_metadata=True)
+        else:
+            logger.debug(f"Existing Annotations Validation Script was found.")
+            validation_item_id = self.metadata["system"]["validationFile"]["itemId"]
+            validation_item = binaries_dataset.items.get(item_id=validation_item_id)
+        return validation_item

dtlpy/entities/service.py

@@ -1,3 +1,4 @@
+import warnings
 from collections import namedtuple
 from enum import Enum
 import traceback
@@ -219,6 +220,7 @@ class Service(entities.BaseEntity):
     on_reset = attr.ib(type=OnResetAction)
     _type = attr.ib(type=ServiceType)
     project_id = attr.ib()
+    org_id = attr.ib()
     is_global = attr.ib()
     max_attempts = attr.ib()
     mode = attr.ib(repr=False)
@@ -235,6 +237,8 @@ class Service(entities.BaseEntity):
     _project = attr.ib(default=None, repr=False)
     _repositories = attr.ib(repr=False)
     updated_by = attr.ib(default=None)
+    app = attr.ib(default=None)
+    integrations = attr.ib(default=None)
 
     @property
     def createdAt(self):
@@ -334,6 +338,9 @@ class Service(entities.BaseEntity):
             updated_by=_json.get('updatedBy', None),
             config=_json.get('config', None),
             settings=_json.get('settings', None),
+            app=_json.get('app', None),
+            integrations=_json.get('integrations', None),
+            org_id=_json.get('orgId', None)
         )
         inst.is_fetched = is_fetched
         return inst
@@ -368,7 +375,19 @@ class Service(entities.BaseEntity):
                                                                                        log_error=False)
                 assert isinstance(self._package, entities.Package)
             except:
-                self._package = repositories.Dpks(client_api=self._client_api).get(dpk_id=self.package_id)
+                dpk_id = None
+                dpk_version = None
+                if self.app and isinstance(self.app, dict):
+                    dpk_id = self.app.get('dpkId', None)
+                    dpk_version = self.app.get('dpkVersion', None)
+                if dpk_id is None:
+                    self._package = repositories.Dpks(client_api=self._client_api, project=self.project).get(
+                        dpk_id=self.package_id)
+                else:
+                    self._package = repositories.Dpks(client_api=self._client_api, project=self.project).get_revisions(
+                        dpk_id=dpk_id,
+                        version=dpk_version)
+
                 assert isinstance(self._package, entities.Dpk)
         return self._package
 
@@ -462,11 +481,15 @@ class Service(entities.BaseEntity):
                 attr.fields(Service).archive,
                 attr.fields(Service).updated_by,
                 attr.fields(Service).config,
-                attr.fields(Service).settings
+                attr.fields(Service).settings,
+                attr.fields(Service).app,
+                attr.fields(Service).integrations,
+                attr.fields(Service).org_id
             )
         )
 
         _json['projectId'] = self.project_id
+        _json['orgId'] = self.org_id
         _json['packageId'] = self.package_id
         _json['initParams'] = self.init_input
         _json['moduleName'] = self.module_name
@@ -518,6 +541,12 @@ class Service(entities.BaseEntity):
         if self.settings is not None:
             _json['settings'] = self.settings
 
+        if self.app is not None:
+            _json['app'] = self.app
+
+        if self.integrations is not None:
+            _json['integrations'] = self.integrations
+
         return _json
 
     def update(self, force=False):