dtlpy 1.114.13__py3-none-any.whl → 1.114.15__py3-none-any.whl

dtlpy/repositories/pipelines.py

@@ -2,7 +2,7 @@ import logging
 from .. import entities, repositories, exceptions, miscellaneous, _api_reference
 from ..services.api_client import ApiClient
 
-logger = logging.getLogger(name='dtlpy')
+logger = logging.getLogger(name="dtlpy")
 
 
 def generate_basic_pipeline():
@@ -30,14 +30,15 @@ class Pipelines:
                 self._project = repositories.Projects(client_api=self._client_api).get()
             except exceptions.NotFound:
                 raise exceptions.PlatformException(
-                    error='2001',
-                    message='Missing "project". need to set a Project entity or use project.pipelines repository')
+                    error="2001",
+                    message='Missing "project". need to set a Project entity or use project.pipelines repository',
+                )
         return self._project
 
     @project.setter
     def project(self, project: entities.Project):
         if not isinstance(project, entities.Project):
-            raise ValueError('Must input a valid Project entity')
+            raise ValueError("Must input a valid Project entity")
         self._project = project
 
     ###########
@@ -47,10 +48,7 @@ class Pipelines:
     def platform_url(self):
         return self._client_api._get_resource_url("projects/{}/pipelines".format(self.project.id))
 
-    def open_in_web(self,
-                    pipeline: entities.Pipeline = None,
-                    pipeline_id: str = None,
-                    pipeline_name: str = None):
+    def open_in_web(self, pipeline: entities.Pipeline = None, pipeline_id: str = None, pipeline_name: str = None):
         """
         Open the pipeline in web platform.
 
@@ -71,16 +69,12 @@ class Pipelines:
         if pipeline is not None:
             pipeline.open_in_web()
         elif pipeline_id is not None:
-            self._client_api._open_in_web(url=self.platform_url + '/' + str(pipeline_id))
+            self._client_api._open_in_web(url=self.platform_url + "/" + str(pipeline_id))
         else:
             self._client_api._open_in_web(url=self.platform_url)
 
-    @_api_reference.add(path='/pipelines/{pipelineId}', method='get')
-    def get(self,
-            pipeline_name=None,
-            pipeline_id=None,
-            fetch=None
-            ) -> entities.Pipeline:
+    @_api_reference.add(path="/pipelines/{pipelineId}", method="get")
+    def get(self, pipeline_name=None, pipeline_id=None, fetch=None) -> entities.Pipeline:
         """
         Get Pipeline object to use in your code.
 
@@ -104,76 +98,63 @@ class Pipelines:
             fetch = self._client_api.fetch_entities
 
         if pipeline_name is None and pipeline_id is None:
-            raise exceptions.PlatformException(
-                error='400',
-                message='Must provide an identifier in inputs')
+            raise exceptions.PlatformException(error="400", message="Must provide an identifier in inputs")
         elif fetch:
             if pipeline_id is not None:
                 success, response = self._client_api.gen_request(
-                    req_type="get",
-                    path="/pipelines/{}".format(pipeline_id))
+                    req_type="get", path="/pipelines/{}".format(pipeline_id)
+                )
                 if not success:
                     raise exceptions.PlatformException(response)
                 pipeline = entities.Pipeline.from_json(
-                    client_api=self._client_api,
-                    _json=response.json(),
-                    project=self._project
+                    client_api=self._client_api, _json=response.json(), project=self._project
                 )
                 if pipeline_name is not None and pipeline.name != pipeline_name:
                     logger.warning(
                         "Mismatch found in pipeline.get: pipeline_name is different then pipeline.name:"
-                        " {!r} != {!r}".format(
-                            pipeline_name,
-                            pipeline.name
-                        )
+                        " {!r} != {!r}".format(pipeline_name, pipeline.name)
                     )
             elif pipeline_name is not None:
                 filters = entities.Filters(
-                    field='name',
-                    values=pipeline_name,
-                    resource=entities.FiltersResource.PIPELINE,
-                    use_defaults=False
+                    field="name", values=pipeline_name, resource=entities.FiltersResource.PIPELINE, use_defaults=False
                 )
                 if self._project is not None:
-                    filters.add(field='projectId', values=self._project.id)
+                    filters.add(field="projectId", values=self._project.id)
                 pipelines = self.list(filters=filters)
                 if pipelines.items_count == 0:
                     raise exceptions.PlatformException(
-                        error='404',
-                        message='Pipeline not found. Name: {}'.format(pipeline_name))
+                        error="404", message="Pipeline not found. Name: {}".format(pipeline_name)
+                    )
                 elif pipelines.items_count > 1:
                     raise exceptions.PlatformException(
-                        error='400',
-                        message='More than one pipelines found by the name of: {} '
-                                'Please get pipeline from a project entity'.format(pipeline_name))
+                        error="400",
+                        message="More than one pipelines found by the name of: {} "
+                        "Please get pipeline from a project entity".format(pipeline_name),
+                    )
                 pipeline = pipelines.items[0]
             else:
                 raise exceptions.PlatformException(
-                    error='400',
-                    message='No checked-out pipeline was found, must checkout or provide an identifier in inputs')
+                    error="400",
+                    message="No checked-out pipeline was found, must checkout or provide an identifier in inputs",
+                )
         else:
             pipeline = entities.Pipeline.from_json(
-                _json={'id': pipeline_id,
-                       'name': pipeline_name},
+                _json={"id": pipeline_id, "name": pipeline_name},
                 client_api=self._client_api,
                 project=self._project,
-                is_fetched=False
+                is_fetched=False,
             )
 
         return pipeline
 
     def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Pipeline]:
-        pool = self._client_api.thread_pools(pool_name='entity.create')
+        pool = self._client_api.thread_pools(pool_name="entity.create")
         jobs = [None for _ in range(len(response_items))]
 
         for i_pipeline, pipeline in enumerate(response_items):
             jobs[i_pipeline] = pool.submit(
                 entities.Pipeline._protected_from_json,
-                **{
-                    'client_api': self._client_api,
-                    '_json': pipeline,
-                    'project': self._project
-                }
+                **{"client_api": self._client_api, "_json": pipeline, "project": self._project}
             )
 
         # get all results
@@ -186,23 +167,16 @@ class Pipelines:
         return pipelines
 
     def _list(self, filters: entities.Filters):
-        url = '/pipelines/query'
+        url = "/pipelines/query"
 
         # request
-        success, response = self._client_api.gen_request(
-            req_type='post',
-            path=url,
-            json_req=filters.prepare()
-        )
+        success, response = self._client_api.gen_request(req_type="post", path=url, json_req=filters.prepare())
         if not success:
             raise exceptions.PlatformException(response)
         return response.json()
 
-    @_api_reference.add(path='/pipelines/query', method='post')
-    def list(self,
-             filters: entities.Filters = None,
-             project_id: str = None
-             ) -> entities.PagedEntities:
+    @_api_reference.add(path="/pipelines/query", method="post")
+    def list(self, filters: entities.Filters = None, project_id: str = None) -> entities.PagedEntities:
         """
         List project pipelines.
 
@@ -223,59 +197,57 @@ class Pipelines:
             filters = entities.Filters(resource=entities.FiltersResource.PIPELINE)
         # assert type filters
         elif not isinstance(filters, entities.Filters):
-            raise exceptions.PlatformException(error='400',
-                                               message='Unknown filters type: {!r}'.format(type(filters)))
+            raise exceptions.PlatformException(error="400", message="Unknown filters type: {!r}".format(type(filters)))
         if filters.resource != entities.FiltersResource.PIPELINE:
             raise exceptions.PlatformException(
-                error='400',
-                message='Filters resource must to be FiltersResource.PIPELINE. Got: {!r}'.format(filters.resource))
+                error="400",
+                message="Filters resource must to be FiltersResource.PIPELINE. Got: {!r}".format(filters.resource),
+            )
 
         if project_id is None and self._project is not None:
             project_id = self._project.id
 
         if project_id is not None:
-            filters.add(field='projectId', values=project_id)
-
-        paged = entities.PagedEntities(items_repository=self,
-                                       filters=filters,
-                                       page_offset=filters.page,
-                                       page_size=filters.page_size,
-                                       project_id=project_id,
-                                       client_api=self._client_api)
+            filters.add(field="projectId", values=project_id)
+
+        paged = entities.PagedEntities(
+            items_repository=self,
+            filters=filters,
+            page_offset=filters.page,
+            page_size=filters.page_size,
+            project_id=project_id,
+            client_api=self._client_api,
+        )
         paged.get_page()
         return paged
 
     def _name_validation(self, name: str):
-        url = '/piper-misc/naming/packages/{}'.format(name)
+        url = "/piper-misc/naming/packages/{}".format(name)
 
         # request
-        success, response = self._client_api.gen_request(req_type='get',
-                                                         path=url)
+        success, response = self._client_api.gen_request(req_type="get", path=url)
         if not success:
             raise exceptions.PlatformException(response)
 
     # @_api_reference.add(path='/pipelines/{pipelineId}', method='delete')
-    def delete(self,
-               pipeline: entities.Pipeline = None,
-               pipeline_name: str = None,
-               pipeline_id: str = None):
+    def delete(self, pipeline: entities.Pipeline = None, pipeline_name: str = None, pipeline_id: str = None):
         """
-        Delete Pipeline object.
+         Delete Pipeline object.
 
-        **prerequisites**: You must be an *owner* or *developer* to use this method.
+         **prerequisites**: You must be an *owner* or *developer* to use this method.
 
-       :param dtlpy.entities.pipeline.Pipeline pipeline: pipeline entity
-       :param str pipeline_id: pipeline id
-       :param str pipeline_name: pipeline name
-       :return: True if success
-       :rtype: bool
+        :param dtlpy.entities.pipeline.Pipeline pipeline: pipeline entity
+        :param str pipeline_id: pipeline id
+        :param str pipeline_name: pipeline name
+        :return: True if success
+        :rtype: bool
 
-       **Example**:
+        **Example**:
 
-        .. code-block:: python
+         .. code-block:: python
 
-            is_deleted = project.pipelines.delete(pipeline_id='pipeline_id')
-       """
+             is_deleted = project.pipelines.delete(pipeline_id='pipeline_id')
+        """
         # get id and name
         if pipeline_id is None:
             if pipeline is None:
@@ -283,8 +255,7 @@ class Pipelines:
             pipeline_id = pipeline.id
 
         # request
-        success, response = self._client_api.gen_request(req_type="delete",
-                                                         path="/pipelines/{}".format(pipeline_id))
+        success, response = self._client_api.gen_request(req_type="delete", path="/pipelines/{}".format(pipeline_id))
 
         # exception handling
         if not success:
@@ -293,7 +264,7 @@ class Pipelines:
         # return results
         return True
 
-    @_api_reference.add(path='/pipelines/{pipelineId}/settings', method='patch')
+    @_api_reference.add(path="/pipelines/{pipelineId}/settings", method="patch")
     def update_settings(self, pipeline: entities.Pipeline, settings: entities.PipelineSettings):
         """
         Update pipeline settings.
@@ -312,55 +283,41 @@ class Pipelines:
             pipeline = project.pipelines.update_settings(pipeline='pipeline_entity', settings=dl.PipelineSettings(keep_triggers_active=True))
         """
         # payload
-        payload = {'settings': settings.to_json()}
+        payload = {"settings": settings.to_json()}
 
         # request
         success, response = self._client_api.gen_request(
-            req_type='patch',
-            path='/pipelines/{}'.format(pipeline.id),
-            json_req=payload
+            req_type="patch", path="/pipelines/{}".format(pipeline.id), json_req=payload
         )
         if not success:
             raise exceptions.PlatformException(response)
 
         # return entity
-        return entities.Pipeline.from_json(
-            _json=response.json(),
-            client_api=self._client_api,
-            project=self._project
-        )
+        return entities.Pipeline.from_json(_json=response.json(), client_api=self._client_api, project=self._project)
 
     def __update_variables(self, pipeline: entities.Pipeline):
         pipeline_json = pipeline.to_json()
-        variables = pipeline_json.get('variables', list())
+        variables = pipeline_json.get("variables", list())
 
         for var in variables:
-            if var.get('reference', None) is None:
-                var['reference'] = pipeline.id
+            if var.get("reference", None) is None:
+                var["reference"] = pipeline.id
 
         # payload
-        payload = {'variables': variables}
+        payload = {"variables": variables}
 
         # request
         success, response = self._client_api.gen_request(
-            req_type='patch',
-            path='/pipelines/{}/variables'.format(pipeline.id),
-            json_req=payload
+            req_type="patch", path="/pipelines/{}/variables".format(pipeline.id), json_req=payload
         )
         if not success:
             raise exceptions.PlatformException(response)
 
         # return entity
-        return entities.Pipeline.from_json(
-            _json=response.json(),
-            client_api=self._client_api,
-            project=self._project
-        )
+        return entities.Pipeline.from_json(_json=response.json(), client_api=self._client_api, project=self._project)
 
-    @_api_reference.add(path='/pipelines/{pipelineId}', method='patch')
-    def update(self,
-               pipeline: entities.Pipeline = None
-               ) -> entities.Pipeline:
+    @_api_reference.add(path="/pipelines/{pipelineId}", method="patch")
+    def update(self, pipeline: entities.Pipeline = None) -> entities.Pipeline:
         """
         Update pipeline changes to platform.
 
@@ -382,17 +339,15 @@ class Pipelines:
         # update settings
         if pipeline.settings_changed():
             new_pipeline = self.update_settings(pipeline=pipeline, settings=pipeline.settings)
-            payload['settings'] = new_pipeline.to_json().get('settings', payload.get('settings'))
+            payload["settings"] = new_pipeline.to_json().get("settings", payload.get("settings"))
 
         # update variables
         if pipeline.variables_changed():
             new_pipeline = self.__update_variables(pipeline=pipeline)
-            payload['variables'] = new_pipeline.to_json().get('variables', payload.get('variables'))
+            payload["variables"] = new_pipeline.to_json().get("variables", payload.get("variables"))
 
         success, response = self._client_api.gen_request(
-            req_type='patch',
-            path='/pipelines/{}'.format(pipeline.id),
-            json_req=payload
+            req_type="patch", path="/pipelines/{}".format(pipeline.id), json_req=payload
         )
 
         # exception handling
@@ -400,18 +355,10 @@ class Pipelines:
             raise exceptions.PlatformException(response)
 
         # return entity
-        return entities.Pipeline.from_json(
-            _json=response.json(),
-            client_api=self._client_api,
-            project=self._project
-        )
+        return entities.Pipeline.from_json(_json=response.json(), client_api=self._client_api, project=self._project)
 
-    @_api_reference.add(path='/pipelines', method='post')
-    def create(self,
-               name: str = None,
-               project_id: str = None,
-               pipeline_json: dict = None
-               ) -> entities.Pipeline:
+    @_api_reference.add(path="/pipelines", method="post")
+    def create(self, name: str = None, project_id: str = None, pipeline_json: dict = None) -> entities.Pipeline:
         """
         Create a new pipeline.
 
@@ -433,27 +380,25 @@ class Pipelines:
             pipeline_json = generate_basic_pipeline()
 
         if name is not None:
-            pipeline_json['name'] = name
+            pipeline_json["name"] = name
 
         if project_id is not None:
-            pipeline_json['projectId'] = project_id
+            pipeline_json["projectId"] = project_id
         else:
-            if not pipeline_json.get('projectId', None):
-                pipeline_json['projectId'] = self.project.id
+            if not pipeline_json.get("projectId", None):
+                pipeline_json["projectId"] = self.project.id
 
-        success, response = self._client_api.gen_request(req_type='post',
-                                                         path='/pipelines',
-                                                         json_req=pipeline_json)
+        success, response = self._client_api.gen_request(req_type="post", path="/pipelines", json_req=pipeline_json)
         if success:
-            pipeline = entities.Pipeline.from_json(client_api=self._client_api,
-                                                   _json=response.json(),
-                                                   project=self.project)
+            pipeline = entities.Pipeline.from_json(
+                client_api=self._client_api, _json=response.json(), project=self.project
+            )
         else:
             raise exceptions.PlatformException(response)
         assert isinstance(pipeline, entities.Pipeline)
         return pipeline
 
-    @_api_reference.add(path='/pipelines/{pipelineId}/install', method='post')
+    @_api_reference.add(path="/pipelines/{pipelineId}/install", method="post")
     def install(self, pipeline: entities.Pipeline = None, resume_option: entities.PipelineResumeOption = None):
         """
         Install (start) a pipeline.
@@ -473,22 +418,18 @@ class Pipelines:
 
         payload = {}
         if resume_option:
-            payload['resumeOption'] = resume_option
+            payload["resumeOption"] = resume_option
 
         success, response = self._client_api.gen_request(
-            req_type='post',
-            path='/pipelines/{}/install'.format(pipeline.id),
-            json_req=payload
+            req_type="post", path="/pipelines/{}/install".format(pipeline.id), json_req=payload
         )
 
         if not success:
             raise exceptions.PlatformException(response)
 
-        return entities.Pipeline.from_json(client_api=self._client_api,
-                                           _json=response.json(),
-                                           project=self.project)
+        return entities.Pipeline.from_json(client_api=self._client_api, _json=response.json(), project=self.project)
 
-    @_api_reference.add(path='/pipelines/{pipelineId}/uninstall', method='post')
+    @_api_reference.add(path="/pipelines/{pipelineId}/uninstall", method="post")
     def pause(self, pipeline: entities.Pipeline = None, keep_triggers_active: bool = None):
         """
         Pause a pipeline.
@@ -508,23 +449,23 @@ class Pipelines:
 
         payload = {}
         if keep_triggers_active is not None:
-            payload['keepTriggersActive'] = keep_triggers_active
+            payload["keepTriggersActive"] = keep_triggers_active
 
         success, response = self._client_api.gen_request(
-            req_type='post',
-            path='/pipelines/{}/uninstall'.format(pipeline.id),
-            json_req=payload
+            req_type="post", path="/pipelines/{}/uninstall".format(pipeline.id), json_req=payload
         )
 
         if not success:
             raise exceptions.PlatformException(response)
 
-    @_api_reference.add(path='/pipelines/{pipelineId}/reset', method='post')
-    def reset(self,
-              pipeline: entities.Pipeline = None,
-              pipeline_id: str = None,
-              pipeline_name: str = None,
-              stop_if_running: bool = False):
+    @_api_reference.add(path="/pipelines/{pipelineId}/reset", method="post")
+    def reset(
+        self,
+        pipeline: entities.Pipeline = None,
+        pipeline_id: str = None,
+        pipeline_name: str = None,
+        stop_if_running: bool = False,
+    ):
         """
         Reset pipeline counters.
 
@@ -549,8 +490,7 @@ class Pipelines:
                     pipeline = self.get(pipeline_name=pipeline_name)
                 else:
                     raise exceptions.PlatformException(
-                        '400',
-                        'Must provide one of pipeline, pipeline_id or pipeline_name'
+                        "400", "Must provide one of pipeline, pipeline_id or pipeline_name"
                     )
             pipeline_id = pipeline.id
 
@@ -560,8 +500,7 @@ class Pipelines:
             pipeline.pause()
 
         success, response = self._client_api.gen_request(
-            req_type='post',
-            path='/pipelines/{}/reset'.format(pipeline_id)
+            req_type="post", path="/pipelines/{}/reset".format(pipeline_id)
         )
 
         if not success:
@@ -569,7 +508,7 @@ class Pipelines:
 
         return True
 
-    @_api_reference.add(path='/pipelines/{id}/statistics', method='get')
+    @_api_reference.add(path="/pipelines/{id}/statistics", method="get")
     def stats(self, pipeline: entities.Pipeline = None, pipeline_id: str = None, pipeline_name: str = None):
         """
         Get pipeline counters.
@@ -595,14 +534,12 @@ class Pipelines:
                     pipeline = self.get(pipeline_name=pipeline_name)
                 else:
                     raise exceptions.PlatformException(
-                        '400',
-                        'Must provide one of pipeline, pipeline_id or pipeline_name'
+                        "400", "Must provide one of pipeline, pipeline_id or pipeline_name"
                     )
             pipeline_id = pipeline.id
 
         success, response = self._client_api.gen_request(
-            req_type='get',
-            path='/pipelines/{}/statistics'.format(pipeline_id)
+            req_type="get", path="/pipelines/{}/statistics".format(pipeline_id)
         )
 
         if not success:
@@ -610,11 +547,13 @@ class Pipelines:
 
         return entities.PipelineStats.from_json(_json=response.json())
 
-    def execute(self,
-                pipeline: entities.Pipeline = None,
-                pipeline_id: str = None,
-                pipeline_name: str = None,
-                execution_input=None):
+    def execute(
+        self,
+        pipeline: entities.Pipeline = None,
+        pipeline_id: str = None,
+        pipeline_name: str = None,
+        execution_input=None,
+    ):
         """
         Execute a pipeline and return the pipeline execution as an object.
 
@@ -633,10 +572,69 @@ class Pipelines:
 
             pipeline_execution= project.pipelines.execute(pipeline='pipeline_entity', execution_input= {'item': 'item_id'} )
         """
+
+        if pipeline is None and pipeline_id is None:
+            raise ValueError("Pipeline or pipeline_id must be provided")
+
         if pipeline is None:
-            pipeline = self.get(pipeline_id=pipeline_id, pipeline_name=pipeline_name)
-        execution = repositories.PipelineExecutions(pipeline=pipeline,
-                                                    client_api=self._client_api,
-                                                    project=self._project).create(pipeline_id=pipeline.id,
-                                                                                  execution_input=execution_input)
+            pipeline = self.get(pipeline_id=pipeline_id)
+
+        execution = repositories.PipelineExecutions(
+            pipeline=pipeline, client_api=self._client_api, project=self._project
+        ).create(pipeline_id=pipeline.id, execution_input=execution_input, test_mode=False)
         return execution
+
+    def test(self, pipeline: entities.Pipeline = None, pipeline_id: str = None, execution_input=None):
+        """
+        Execute a pipeline in test mode and return the pipeline execution as an object.
+
+        **prerequisites**: You must be an *owner* or *developer* to use this method.
+
+        :param dtlpy.entities.pipeline.Pipeline pipeline: pipeline entity
+        :param str pipeline_id: pipeline id
+        :param execution_input: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}
+        :return: entities.PipelineExecution object
+        :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
+
+        **Example**:
+
+        .. code-block:: python
+
+            pipeline_execution= project.pipelines.test(pipeline='pipeline_entity', execution_input= {'item': 'item_id'} )
+        """
+        if pipeline is None and pipeline_id is None:
+            raise ValueError("Pipeline or pipeline_id must be provided")
+
+        if pipeline is None:
+            pipeline = self.get(pipeline_id=pipeline_id)
+
+        execution = repositories.PipelineExecutions(
+            pipeline=pipeline, client_api=self._client_api, project=self._project
+        ).create(pipeline_id=pipeline.id, execution_input=execution_input, test_mode=True)
+        return execution
+
+    @_api_reference.add(path="/pipelines/validate", method="post")
+    def validate(self, pipeline_json: dict):
+        """
+        Validate a pipeline configuration.
+
+        **prerequisites**: You must be an *owner* or *developer* to use this method.
+
+        :param dict pipeline_json: pipeline configuration as JSON dictionary
+        :return: Validation result
+        :rtype: dict
+
+        **Example**:
+
+        .. code-block:: python
+
+            validation_result = project.pipelines.validate(pipeline_json={'name': 'my_pipeline', 'nodes': []})
+        """
+        success, response = self._client_api.gen_request(
+            req_type="post", path="/pipelines/validate", json_req=pipeline_json
+        )
+
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        return response.json()