operaton-external-task-client-python3 1.0.0__py3-none-any.whl

operaton/external_task/external_task.py

@@ -0,0 +1,173 @@
+from operaton.variables.properties import Properties
+from operaton.variables.variables import Variables
+
+
+class ExternalTask:
+    def __init__(self, context):
+        self._context = context
+        self._variables = Variables(context.get("variables", {}))
+        self._task_result = TaskResult.empty_task_result(task=self)
+        self._extProperties = Properties(context.get("extensionProperties", {}))
+
+    def get_worker_id(self):
+        return self._context["workerId"]
+
+    def get_process_instance_id(self):
+        return self._context["processInstanceId"]
+
+    def get_variables(self):
+        return self._variables.to_dict()
+
+    def get_extension_properties(self) -> dict:
+        return self._extProperties.to_dict()
+
+    def get_task_id(self):
+        return self._context["id"]
+
+    def get_activity_id(self):
+        return self._context["activityId"]
+
+    def get_topic_name(self):
+        return self._context["topicName"]
+
+    def get_variable(self, variable_name, with_meta=False):
+        return self._variables.get_variable(variable_name, with_meta=with_meta)
+
+    def get_extension_property(self, property_name) -> str:
+        return self._extProperties.get_property(property_name)
+
+    def get_tenant_id(self):
+        return self._context.get("tenantId", None)
+
+    def get_business_key(self):
+        return self._context.get("businessKey", None)
+
+    def get_task_result(self):
+        return self._task_result
+
+    def set_task_result(self, task_result):
+        self._task_result = task_result
+
+    def complete(self, global_variables={}, local_variables={}):
+        self._task_result = TaskResult.success(self, global_variables, local_variables)
+        return self._task_result
+
+    def failure(self, error_message, error_details, max_retries, retry_timeout):
+        retries = self._calculate_retries(max_retries)
+        self._task_result = TaskResult.failure(
+            self,
+            error_message=error_message,
+            error_details=error_details,
+            retries=retries,
+            retry_timeout=retry_timeout,
+        )
+        return self._task_result
+
+    def _calculate_retries(self, max_retries):
+        retries = self._context.get("retries", None)
+        retries = int(retries - 1) if retries and retries >= 1 else max_retries
+        return retries
+
+    def bpmn_error(self, error_code, error_message, variables={}):
+        self._task_result = TaskResult.bpmn_error(
+            self,
+            error_code=error_code,
+            error_message=error_message,
+            variables=variables,
+        )
+        return self._task_result
+
+    def __str__(self):
+        return f"{self._context}"
+
+
+class TaskResult:
+    def __init__(
+        self,
+        task,
+        success=False,
+        global_variables={},
+        local_variables={},
+        bpmn_error_code=None,
+        error_message=None,
+        error_details={},
+        retries=0,
+        retry_timeout=300000,
+    ):
+        self.task = task
+        self.success_state = success
+        self.global_variables = global_variables
+        self.local_variables = local_variables
+        self.bpmn_error_code = bpmn_error_code
+        self.error_message = error_message
+        self.error_details = error_details
+        self.retries = retries
+        self.retry_timeout = retry_timeout
+
+    @classmethod
+    def success(cls, task, global_variables, local_variables={}):
+        return TaskResult(
+            task,
+            success=True,
+            global_variables=global_variables,
+            local_variables=local_variables,
+        )
+
+    @classmethod
+    def failure(cls, task, error_message, error_details, retries, retry_timeout):
+        return TaskResult(
+            task,
+            success=False,
+            error_message=error_message,
+            error_details=error_details,
+            retries=retries,
+            retry_timeout=retry_timeout,
+        )
+
+    @classmethod
+    def bpmn_error(cls, task, error_code, error_message, variables={}):
+        return TaskResult(
+            task,
+            success=False,
+            bpmn_error_code=error_code,
+            error_message=error_message,
+            global_variables=variables,
+        )
+
+    @classmethod
+    def empty_task_result(cls, task):
+        return TaskResult(task, success=False)
+
+    def is_success(self):
+        return (
+            self.success_state
+            and self.bpmn_error_code is None
+            and self.error_message is None
+        )
+
+    def is_failure(self):
+        return (
+            not self.success_state
+            and self.error_message is not None
+            and not self.is_bpmn_error()
+        )
+
+    def is_bpmn_error(self):
+        return not self.success_state and self.bpmn_error_code
+
+    def get_task(self):
+        return self.task
+
+    def __str__(self):
+        if self.is_success():
+            return f"success: task_id={self.task.get_task_id()}, global_variables={self.global_variables}, local_variables={self.local_variables}"
+        elif self.is_failure():
+            return (
+                f"failure: task_id={self.task.get_task_id()}, "
+                f"error_message={self.error_message}, error_details={self.error_details}, "
+                f"retries={self.retries}, retry_timeout={self.retry_timeout}"
+            )
+        elif self.is_bpmn_error():
+            return f"bpmn_error: task_id={self.task.get_task_id()}, error_code={self.bpmn_error_code}"
+
+        return "empty_task_result"

operaton/external_task/external_task_executor.py

@@ -0,0 +1,88 @@
+import logging
+
+from operaton.utils.log_utils import log_with_context
+
+logger = logging.getLogger(__name__)
+
+
+class ExternalTaskExecutor:
+
+    def __init__(self, worker_id, external_task_client):
+        self.worker_id = worker_id
+        self.external_task_client = external_task_client
+
+    def execute_task(self, task, action):
+        topic = task.get_topic_name()
+        task_id = task.get_task_id()
+        self._log_with_context(f"Executing external task for Topic: {topic}", task_id=task_id)
+        task_result = action(task)
+        # in case task result is not set inside action function, set it in task here
+        task.set_task_result(task_result)
+        self._handle_task_result(task_result)
+        return task_result
+
+    def _handle_task_result(self, task_result):
+        task = task_result.get_task()
+        topic = task.get_topic_name()
+        task_id = task.get_task_id()
+        if task_result.is_success():
+            self._handle_task_success(task_id, task_result, topic)
+        elif task_result.is_bpmn_error():
+            self._handle_task_bpmn_error(task_id, task_result, topic)
+        elif task_result.is_failure():
+            self._handle_task_failure(task_id, task_result, topic)
+        else:
+            err_msg = f"task result for task_id={task_id} must be either complete/failure/BPMNError"
+            self._log_with_context(err_msg, task_id=task_id, log_level='warning')
+            raise Exception(err_msg)
+
+    def _strip_long_variables(self, variables):
+        """remove value of complex variables for the dict"""
+        if not variables:
+            return variables
+        cleaned = {}
+        for k, v in variables.items():
+            if isinstance(v, dict) and v.get("type", "") in ("File", "Bytes"):
+                cleaned[k] = {**v, "value": "..."}
+            else:
+                cleaned[k] = v
+        return cleaned
+
+    def _handle_task_success(self, task_id, task_result, topic):
+        self._log_with_context(f"Marking task complete for Topic: {topic}", task_id)
+        if self.external_task_client.complete(task_id, task_result.global_variables, task_result.local_variables):
+            self._log_with_context(f"Marked task completed - Topic: {topic} "
+                                   f"global_variables: {self._strip_long_variables(task_result.global_variables)} "
+                                   f"local_variables: {self._strip_long_variables(task_result.local_variables)}", task_id)
+        else:
+            self._log_with_context(f"Not able to mark task completed - Topic: {topic} "
+                                   f"global_variables: {self._strip_long_variables(task_result.global_variables)} "
+                                   f"local_variables: {self._strip_long_variables(task_result.local_variables)}", task_id)
+            raise Exception(f"Not able to mark complete for task_id={task_id} "
+                            f"for topic={topic}, worker_id={self.worker_id}")
+
+    def _handle_task_failure(self, task_id, task_result, topic):
+        self._log_with_context(f"Marking task failed - Topic: {topic} task_result: {task_result}", task_id)
+        if self.external_task_client.failure(task_id, task_result.error_message, task_result.error_details,
+                                             task_result.retries, task_result.retry_timeout):
+            self._log_with_context(f"Marked task failed - Topic: {topic} task_result: {task_result}", task_id)
+        else:
+            self._log_with_context(f"Not able to mark task failure - Topic: {topic}", task_id=task_id)
+            raise Exception(f"Not able to mark failure for task_id={task_id} "
+                            f"for topic={topic}, worker_id={self.worker_id}")
+
+    def _handle_task_bpmn_error(self, task_id, task_result, topic):
+        bpmn_error_handled = self.external_task_client.bpmn_failure(task_id, task_result.bpmn_error_code,
+                                                                    task_result.error_message,
+                                                                    task_result.global_variables)
+        if bpmn_error_handled:
+            self._log_with_context(f"BPMN Error Handled: {bpmn_error_handled} "
+                                   f"Topic: {topic} task_result: {task_result}")
+        else:
+            self._log_with_context(f"Not able to mark BPMN error - Topic: {topic}", task_id=task_id)
+            raise Exception(f"Not able to mark BPMN Error for task_id={task_id} "
+                            f"for topic={topic}, worker_id={self.worker_id}")
+
+    def _log_with_context(self, msg, task_id=None, log_level='info', **kwargs):
+        context = {"WORKER_ID": self.worker_id, "TASK_ID": task_id}
+        log_with_context(msg, context=context, log_level=log_level, **kwargs)

operaton/external_task/external_task_worker.py

@@ -0,0 +1,92 @@
+import time
+
+from operaton.client.external_task_client import ExternalTaskClient, ENGINE_LOCAL_BASE_URL
+from operaton.external_task.external_task import ExternalTask
+from operaton.external_task.external_task_executor import ExternalTaskExecutor
+from operaton.utils.log_utils import log_with_context
+from operaton.utils.auth_basic import obfuscate_password
+from operaton.utils.utils import get_exception_detail
+
+
+class ExternalTaskWorker:
+    DEFAULT_SLEEP_SECONDS = 300
+
+    def __init__(self, worker_id, base_url=ENGINE_LOCAL_BASE_URL, config=None):
+        config = config if config is not None else {}  # To avoid to have a mutable default for a parameter
+        self.worker_id = worker_id
+        self.client = ExternalTaskClient(self.worker_id, base_url, config)
+        self.executor = ExternalTaskExecutor(self.worker_id, self.client)
+        self.config = config
+        self._log_with_context(f"Created new External Task Worker with config: {obfuscate_password(self.config)}")
+
+    def subscribe(self, topic_names, action, process_variables=None, variables=None):
+        while True:
+            self._fetch_and_execute_safe(topic_names, action, process_variables, variables)
+
+        self._log_with_context("Stopping worker")  # Fixme: This code seems to be unreachable?
+
+    def _fetch_and_execute_safe(
+        self, topic_names, action, process_variables=None, variables=None
+    ):
+        try:
+            self.fetch_and_execute(topic_names, action, process_variables, variables)
+        except NoExternalTaskFound:
+            self._log_with_context(f"no External Task found for Topics: {topic_names}, "
+                                   f"Process variables: {process_variables}", topic=topic_names)
+        except BaseException as e:
+            sleep_seconds = self._get_sleep_seconds()
+            self._log_with_context(f'error fetching and executing tasks: {get_exception_detail(e)} '
+                                   f'for topic(s)={topic_names} with Process variables: {process_variables}. '
+                                   f'retrying after {sleep_seconds} seconds', exc_info=True)
+            time.sleep(sleep_seconds)
+
+    def fetch_and_execute(self, topic_names, action, process_variables=None, variables=None):
+        self._log_with_context(f"Fetching and Executing external tasks for Topics: {topic_names} "
+                               f"with Process variables: {process_variables}")
+        resp_json = self._fetch_and_lock(topic_names, process_variables, variables)
+        tasks = self._parse_response(resp_json, topic_names, process_variables)
+        if len(tasks) == 0:
+            raise NoExternalTaskFound(f"no External Task found for Topics: {topic_names}, "
+                                      f"Process variables: {process_variables}")
+        self._execute_tasks(tasks, action)
+
+    def _fetch_and_lock(self, topic_names, process_variables=None, variables=None):
+        self._log_with_context(f"Fetching and Locking external tasks for Topics: {topic_names} "
+                               f"with Process variables: {process_variables}")
+        return self.client.fetch_and_lock(topic_names, process_variables, variables)
+
+    def _parse_response(self, resp_json, topic_names, process_variables):
+        tasks = []
+        if resp_json:
+            for context in resp_json:
+                task = ExternalTask(context)
+                tasks.append(task)
+
+        tasks_count = len(tasks)
+        self._log_with_context(f"{tasks_count} External task(s) found for "
+                               f"Topics: {topic_names}, Process variables: {process_variables}")
+        return tasks
+
+    def _execute_tasks(self, tasks, action):
+        for task in tasks:
+            self._execute_task(task, action)
+
+    def _execute_task(self, task, action):
+        try:
+            self.executor.execute_task(task, action)
+        except Exception as e:
+            self._log_with_context(f'error when executing task: {get_exception_detail(e)}',
+                                   topic=task.get_topic_name(), task_id=task.get_task_id(),
+                                   log_level='error', exc_info=True)
+            raise e
+
+    def _log_with_context(self, msg, topic=None, task_id=None, log_level='info', **kwargs):
+        context = {"WORKER_ID": str(self.worker_id), "TOPIC": topic, "TASK_ID": task_id}
+        log_with_context(msg, context=context, log_level=log_level, **kwargs)
+
+    def _get_sleep_seconds(self):
+        return self.config.get("sleepSeconds", self.DEFAULT_SLEEP_SECONDS)
+
+
+class NoExternalTaskFound(Exception):
+    pass

operaton/external_task/tests/test_async_external_task_executor.py

@@ -0,0 +1,139 @@
+import unittest
+from unittest.mock import AsyncMock
+
+from operaton.client.async_external_task_client import AsyncExternalTaskClient
+from operaton.external_task.async_external_task_executor import AsyncExternalTaskExecutor
+from operaton.external_task.external_task import ExternalTask, TaskResult
+
+
+class AsyncExternalTaskExecutorTest(unittest.IsolatedAsyncioTestCase):
+
+    async def asyncSetUp(self):
+        """
+        asyncSetUp runs before each test method in IsolatedAsyncioTestCase.
+        We instantiate an AsyncExternalTaskClient and patch/mocks as needed.
+        """
+        self.mock_client = AsyncMock(spec=AsyncExternalTaskClient)
+        self.mock_client.complete.return_value = True
+        self.mock_client.failure.return_value = True
+        self.mock_client.bpmn_failure.return_value = True
+
+        self.executor = AsyncExternalTaskExecutor(
+            worker_id="someWorker",
+            external_task_client=self.mock_client
+        )
+
+    async def test_execute_task_success(self):
+        async def success_action(task: ExternalTask):
+            return TaskResult.success(task, {"globalVar": 42}, {"localVar": "foo"})
+
+        task = ExternalTask({"id": "taskId", "topicName": "someTopic"})
+
+        result = await self.executor.execute_task(task, success_action)
+
+        # Assertions
+        self.assertTrue(result.is_success())
+        self.mock_client.complete.assert_awaited_once_with(
+            "taskId", {"globalVar": 42}, {"localVar": "foo"}
+        )
+
+    async def test_execute_task_failure(self):
+        async def fail_action(task: ExternalTask):
+            return TaskResult.failure(
+                task,
+                error_message="Some error",
+                error_details="Details here",
+                retries=3,
+                retry_timeout=1000
+            )
+
+        task = ExternalTask({"id": "taskId", "topicName": "someTopic"})
+        result = await self.executor.execute_task(task, fail_action)
+
+        # Assertions
+        self.assertTrue(result.is_failure())
+        self.mock_client.failure.assert_awaited_once_with(
+            "taskId", "Some error", "Details here", 3, 1000
+        )
+
+    async def test_execute_task_bpmn_error(self):
+        async def bpmn_error_action(task: ExternalTask):
+            return TaskResult.bpmn_error(
+                task,
+                error_code="bpmn_err_code",
+                error_message="bpmn error message",
+                variables={"varA": True}
+            )
+
+        task = ExternalTask({"id": "taskId", "topicName": "someTopic"})
+        result = await self.executor.execute_task(task, bpmn_error_action)
+
+        # Assertions
+        self.assertTrue(result.is_bpmn_error())
+        self.mock_client.bpmn_failure.assert_awaited_once_with(
+            "taskId", "bpmn_err_code", "bpmn error message", {"varA": True}
+        )
+
+    async def test_execute_task_empty_result_raises_exception(self):
+        """
+        If the action returns an "empty" TaskResult (not success/failure/BPMNError),
+        executor should raise an exception.
+        """
+
+        async def empty_action(task: ExternalTask):
+            return TaskResult.empty_task_result(task)
+
+        task = ExternalTask({"id": "taskId", "topicName": "someTopic"})
+
+        with self.assertRaises(Exception) as ctx:
+            await self.executor.execute_task(task, empty_action)
+
+        self.assertIn("must be either complete/failure/BPMNError", str(ctx.exception))
+
+    async def test_handle_task_success_when_client_returns_false_raises_exception(self):
+        """
+        If client.complete returns False, an Exception must be raised.
+        """
+        self.mock_client.complete.return_value = False
+
+        async def success_action(task: ExternalTask):
+            return TaskResult.success(task, {"var": "val"})
+
+        task = ExternalTask({"id": "taskId", "topicName": "someTopic"})
+
+        with self.assertRaises(Exception) as ctx:
+            await self.executor.execute_task(task, success_action)
+
+        self.assertIn("Not able to mark complete for task_id=taskId", str(ctx.exception))
+
+    async def test_handle_task_failure_when_client_returns_false_raises_exception(self):
+        """
+        If client.failure returns False, an Exception must be raised.
+        """
+        self.mock_client.failure.return_value = False
+
+        async def fail_action(task: ExternalTask):
+            return TaskResult.failure(task, "errMsg", "errDetails", 3, 2000)
+
+        task = ExternalTask({"id": "taskId", "topicName": "someTopic"})
+
+        with self.assertRaises(Exception) as ctx:
+            await self.executor.execute_task(task, fail_action)
+
+        self.assertIn("Not able to mark failure for task_id=taskId", str(ctx.exception))
+
+    async def test_handle_task_bpmn_error_when_client_returns_false_raises_exception(self):
+        """
+        If client.bpmn_failure returns False, an Exception must be raised.
+        """
+        self.mock_client.bpmn_failure.return_value = False
+
+        async def bpmn_error_action(task: ExternalTask):
+            return TaskResult.bpmn_error(task, "ERR_CODE", "error message")
+
+        task = ExternalTask({"id": "taskId", "topicName": "someTopic"})
+
+        with self.assertRaises(Exception) as ctx:
+            await self.executor.execute_task(task, bpmn_error_action)
+
+        self.assertIn("Not able to mark BPMN Error for task_id=taskId", str(ctx.exception))

operaton/external_task/tests/test_async_external_task_worker.py

@@ -0,0 +1,129 @@
+import asyncio
+import unittest
+from unittest.mock import AsyncMock, patch
+
+from operaton.client.async_external_task_client import AsyncExternalTaskClient
+from operaton.external_task.async_external_task_worker import AsyncExternalTaskWorker
+from operaton.external_task.external_task import ExternalTask, TaskResult
+
+
+class AsyncExternalTaskWorkerTest(unittest.IsolatedAsyncioTestCase):
+
+    async def asyncSetUp(self):
+        """
+        Setup a worker with a mock AsyncExternalTaskClient
+        """
+        self.mock_client = AsyncMock(spec=AsyncExternalTaskClient)
+        self.mock_client.fetch_and_lock.return_value = []
+
+        self.config = {"maxConcurrentTasks": 2, "sleepSeconds": 0}  # faster tests
+        self.worker = AsyncExternalTaskWorker("testWorker", config=self.config)
+        # Replace the worker's .client with our mock
+        self.worker.client = self.mock_client
+        # Similarly, replace the executor's .external_task_client
+        self.worker.executor.external_task_client = self.mock_client
+
+    async def test_fetch_and_execute_no_tasks_returns_false(self):
+        """
+        If fetch_and_lock returns [], then fetch_and_execute should return False.
+        """
+        self.mock_client.fetch_and_lock.return_value = []
+        result = await self.worker.fetch_and_execute(
+            topic_name="myTopic",
+            action=AsyncMock(return_value=None)  # doesn't matter, won't be called
+        )
+        self.assertFalse(result)
+
+    async def test_fetch_and_execute_tasks_creates_execute_task_coroutines(self):
+        """
+        If fetch_and_lock returns multiple tasks, ensure each is passed into _execute_task
+        in the background.
+        """
+        # 2 tasks with different variables
+        resp = [
+            {
+                "id": "task1",
+                "topicName": "myTopic",
+                "workerId": "aWorkerId",
+                "variables": {"foo": {"value": "bar"}}
+            },
+            {
+                "id": "task2",
+                "topicName": "myTopic",
+                "workerId": "aWorkerId2",
+                "variables": {"abc": {"value": 123}}
+            }
+        ]
+        self.mock_client.fetch_and_lock.return_value = resp
+
+        async def success_action(task: ExternalTask):
+            # Return a success result for each
+            return TaskResult.success(task, {"someGlobalVar": 99})
+
+        returned = await self.worker.fetch_and_execute("myTopic", success_action)
+        self.assertTrue(returned)
+        # confirm 2 tasks => 2 coroutines started
+        self.assertEqual(len(self.worker.running_tasks), 2)
+
+        # Let them all finish
+        await asyncio.gather(*self.worker.running_tasks, return_exceptions=True)
+
+        # Now they should be removed from running_tasks
+        self.assertEqual(len(self.worker.running_tasks), 0)
+
+    async def test_execute_task_failure_when_action_raises_exception(self):
+        """
+        If an uncaught exception occurs in the user-provided action,
+        the worker’s _execute_task wraps the result as a failure and tries to call
+        external_task_client.failure(...)
+        """
+        self.mock_client.fetch_and_lock.return_value = [
+            {"id": "task1", "topicName": "topicX", "workerId": "w1"}
+        ]
+
+        async def fail_action(task: ExternalTask):
+            raise RuntimeError("Something went wrong")
+
+        # We'll run fetch_and_execute => it should spawn one background task
+        await self.worker.fetch_and_execute("topicX", fail_action)
+
+        # Wait for background tasks to complete
+        await asyncio.gather(*self.worker.running_tasks, return_exceptions=True)
+
+        # Confirm the worker attempted to call 'failure(...)'
+        self.mock_client.failure.assert_awaited_once()
+        task_id, error_message, error_details, retries, retry_timeout = self.mock_client.failure.call_args.args
+        self.assertEqual(task_id, "task1")
+        self.assertEqual("Task execution failed", error_message)
+        self.assertEqual("An unexpected error occurred while executing the task", error_details)
+        self.assertEqual(3, retries)
+        self.assertEqual(300000, retry_timeout)
+
+    @patch.object(AsyncExternalTaskWorker, "_fetch_and_execute_safe")
+    async def test_cancel_running_tasks_single_iteration(self, mock_fetch_and_execute):
+        # Make _fetch_and_execute_safe run exactly once, then return
+        async def one_iteration(*args, **kwargs):
+            await self.worker.semaphore.acquire()
+            await self.worker.fetch_and_execute(*args, **kwargs)
+            # no 'while True', so it ends
+
+        mock_fetch_and_execute.side_effect = one_iteration
+
+        async def fake_long_action(task):
+            await asyncio.sleep(9999999)
+
+        self.mock_client.fetch_and_lock.return_value = [{"id": "taskX", "topicName": "topicA"}]
+
+        sub_task = asyncio.create_task(
+            self.worker._fetch_and_execute_safe("topicA", fake_long_action)
+        )
+        self.worker.subscriptions.append(sub_task)
+
+        # Wait for that single iteration to run
+        await asyncio.sleep(0.2)
+
+        await self.worker.stop()
+        await asyncio.sleep(0)  # let cancellation finish
+
+        for t in self.worker.running_tasks:
+            self.assertTrue(t.done())