cadence-python-client 0.1.0__py3-none-any.whl

cadence/worker/_decision_task_handler.py

@@ -0,0 +1,285 @@
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+import logging
+from typing import Optional
+
+from cadence._internal.workflow.history_event_iterator import iterate_history_events
+from cadence.api.v1.common_pb2 import Payload
+from cadence.api.v1.service_worker_pb2 import (
+    PollForDecisionTaskResponse,
+    RespondDecisionTaskCompletedRequest,
+    RespondDecisionTaskFailedRequest,
+)
+from cadence.api.v1.workflow_pb2 import DecisionTaskFailedCause
+from cadence.client import Client
+from cadence.worker._base_task_handler import BaseTaskHandler
+from cadence._internal.workflow.workflow_engine import WorkflowEngine, DecisionResult
+from cadence.workflow import WorkflowInfo
+from cadence.worker._registry import Registry
+
+logger = logging.getLogger(__name__)
+
+
+class DecisionTaskHandler(BaseTaskHandler[PollForDecisionTaskResponse]):
+    """
+    Task handler for processing decision tasks.
+
+    This handler processes decision tasks and generates decisions using workflow engines.
+    Uses a thread-safe cache to hold workflow engines for concurrent decision task handling.
+    """
+
+    def __init__(
+        self,
+        client: Client,
+        task_list: str,
+        registry: Registry,
+        identity: str = "unknown",
+        executor: Optional[ThreadPoolExecutor] = None,
+        **options,
+    ):
+        """
+        Initialize the decision task handler.
+
+        Args:
+            client: The Cadence client instance
+            task_list: The task list name
+            registry: Registry containing workflow functions
+            identity: The worker identity
+            **options: Additional options for the handler
+        """
+        super().__init__(client, task_list, identity, **options)
+        self._registry = registry
+        self._executor = executor
+
+    async def _handle_task_implementation(
+        self, task: PollForDecisionTaskResponse
+    ) -> None:
+        """
+        Handle a decision task implementation.
+
+        Args:
+            task: The decision task to handle
+        """
+        # Extract workflow execution info
+        workflow_execution = task.workflow_execution
+        workflow_type = task.workflow_type
+
+        if not workflow_execution or not workflow_type:
+            logger.error(
+                "Decision task missing workflow execution or type. Task: %r", task
+            )
+            raise ValueError("Missing workflow execution or type")
+
+        workflow_id = workflow_execution.workflow_id
+        run_id = workflow_execution.run_id
+        workflow_type_name = workflow_type.name
+
+        # This log matches the WorkflowEngine but at task handler level (like Java ReplayDecisionTaskHandler)
+        logger.info(
+            "Received decision task for workflow",
+            extra={
+                "workflow_type": workflow_type_name,
+                "workflow_id": workflow_id,
+                "run_id": run_id,
+                "started_event_id": task.started_event_id,
+                "attempt": task.attempt,
+                "task_token": task.task_token[:16].hex()
+                if task.task_token
+                else None,  # Log partial token for debugging
+            },
+        )
+
+        try:
+            workflow_definition = self._registry.get_workflow(workflow_type_name)
+        except KeyError:
+            logger.error(
+                "Workflow type not found in registry",
+                extra={
+                    "workflow_type": workflow_type_name,
+                    "workflow_id": workflow_id,
+                    "run_id": run_id,
+                    "error_type": "workflow_not_registered",
+                },
+            )
+            raise KeyError(f"Workflow type '{workflow_type_name}' not found")
+
+        # fetch full workflow history
+        # TODO sticky cache
+        workflow_events = [
+            event async for event in iterate_history_events(task, self._client)
+        ]
+
+        # Create workflow info and get or create workflow engine from cache
+        workflow_info = WorkflowInfo(
+            workflow_type=workflow_type_name,
+            workflow_domain=self._client.domain,
+            workflow_id=workflow_id,
+            workflow_run_id=run_id,
+            workflow_task_list=self.task_list,
+            data_converter=self._client.data_converter,
+        )
+
+        workflow_engine = WorkflowEngine(
+            info=workflow_info,
+            workflow_definition=workflow_definition,
+        )
+
+        decision_result = await asyncio.get_running_loop().run_in_executor(
+            self._executor, workflow_engine.process_decision, workflow_events
+        )
+
+        # Respond with the decisions
+        await self._respond_decision_task_completed(task, decision_result)
+
+        logger.info(
+            "Successfully processed decision task",
+            extra={
+                "workflow_type": workflow_type_name,
+                "workflow_id": workflow_id,
+                "run_id": run_id,
+                "started_event_id": task.started_event_id,
+            },
+        )
+
+    async def handle_task_failure(
+        self, task: PollForDecisionTaskResponse, error: Exception
+    ) -> None:
+        """
+        Handle decision task processing failure.
+
+        Args:
+            task: The task that failed
+            error: The exception that occurred
+        """
+        # Extract workflow context for error logging (matches Java ReplayDecisionTaskHandler error patterns)
+        workflow_execution = task.workflow_execution
+        workflow_id = (
+            workflow_execution.workflow_id if workflow_execution else "unknown"
+        )
+        run_id = workflow_execution.run_id if workflow_execution else "unknown"
+        workflow_type = task.workflow_type.name if task.workflow_type else "unknown"
+
+        # Log task failure with full context (matches Java error logging)
+        logger.error(
+            "Decision task processing failure",
+            extra={
+                "workflow_type": workflow_type,
+                "workflow_id": workflow_id,
+                "run_id": run_id,
+                "started_event_id": task.started_event_id,
+                "attempt": task.attempt,
+                "error_type": type(error).__name__,
+                "error_message": str(error),
+            },
+            exc_info=True,
+        )
+
+        # Determine the failure cause
+        cause = DecisionTaskFailedCause.DECISION_TASK_FAILED_CAUSE_UNHANDLED_DECISION
+        if isinstance(error, KeyError):
+            cause = DecisionTaskFailedCause.DECISION_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE
+        elif isinstance(error, ValueError):
+            cause = DecisionTaskFailedCause.DECISION_TASK_FAILED_CAUSE_BAD_SCHEDULE_ACTIVITY_ATTRIBUTES
+
+        # Create error details
+        # TODO: Use a data converter for error details serialization
+        error_message = str(error).encode("utf-8")
+        details = Payload(data=error_message)
+
+        # Respond with failure
+        try:
+            await self._client.worker_stub.RespondDecisionTaskFailed(
+                RespondDecisionTaskFailedRequest(
+                    task_token=task.task_token,
+                    cause=cause,
+                    identity=self._identity,
+                    details=details,
+                )
+            )
+            logger.info(
+                "Decision task failure response sent",
+                extra={
+                    "workflow_id": workflow_id,
+                    "run_id": run_id,
+                    "cause": cause,
+                    "task_token": task.task_token[:16].hex()
+                    if task.task_token
+                    else None,
+                },
+            )
+        except Exception as e:
+            logger.error(
+                "Error responding to decision task failure",
+                extra={
+                    "workflow_id": workflow_id,
+                    "run_id": run_id,
+                    "original_error": type(error).__name__,
+                    "response_error": type(e).__name__,
+                },
+                exc_info=True,
+            )
+
+    async def _respond_decision_task_completed(
+        self, task: PollForDecisionTaskResponse, decision_result: DecisionResult
+    ) -> None:
+        """
+        Respond to the service that the decision task has been completed.
+
+        Args:
+            task: The original decision task
+            decision_result: The result containing decisions and query results
+        """
+        try:
+            request = RespondDecisionTaskCompletedRequest(
+                task_token=task.task_token,
+                decisions=decision_result.decisions,
+                identity=self._identity,
+                return_new_decision_task=True,
+            )
+
+            await self._client.worker_stub.RespondDecisionTaskCompleted(request)
+
+            # Log completion response (matches Java ReplayDecisionTaskHandler trace/debug patterns)
+            workflow_execution = task.workflow_execution
+            logger.debug(
+                "Decision task completion response sent",
+                extra={
+                    "workflow_type": task.workflow_type.name
+                    if task.workflow_type
+                    else "unknown",
+                    "workflow_id": workflow_execution.workflow_id
+                    if workflow_execution
+                    else "unknown",
+                    "run_id": workflow_execution.run_id
+                    if workflow_execution
+                    else "unknown",
+                    "started_event_id": task.started_event_id,
+                    "decisions_count": len(decision_result.decisions),
+                    "return_new_decision_task": True,
+                    "task_token": task.task_token[:16].hex()
+                    if task.task_token
+                    else None,
+                },
+            )
+
+        except Exception as e:
+            workflow_execution = task.workflow_execution
+            logger.error(
+                "Error responding to decision task completion",
+                extra={
+                    "workflow_type": task.workflow_type.name
+                    if task.workflow_type
+                    else "unknown",
+                    "workflow_id": workflow_execution.workflow_id
+                    if workflow_execution
+                    else "unknown",
+                    "run_id": workflow_execution.run_id
+                    if workflow_execution
+                    else "unknown",
+                    "started_event_id": task.started_event_id,
+                    "decisions_count": len(decision_result.decisions),
+                    "error_type": type(e).__name__,
+                },
+                exc_info=True,
+            )
+            raise

cadence/worker/_poller.py

@@ -0,0 +1,64 @@
+import asyncio
+import logging
+from typing import Callable, TypeVar, Generic, Awaitable, Optional
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class Poller(Generic[T]):
+    def __init__(
+        self,
+        num_tasks: int,
+        permits: asyncio.Semaphore,
+        poll: Callable[[], Awaitable[Optional[T]]],
+        callback: Callable[[T], Awaitable[None]],
+    ) -> None:
+        self._num_tasks = num_tasks
+        self._permits = permits
+        self._poll = poll
+        self._callback = callback
+        self._background_tasks: set[asyncio.Task[None]] = set()
+
+    async def run(self) -> None:
+        try:
+            async with asyncio.TaskGroup() as tg:
+                for i in range(self._num_tasks):
+                    tg.create_task(self._poll_loop())
+        except asyncio.CancelledError:
+            pass
+
+    async def _poll_loop(self) -> None:
+        while True:
+            try:
+                await self._poll_and_dispatch()
+            except asyncio.CancelledError as e:
+                raise e
+            except Exception:
+                logger.exception("Exception while polling")
+
+    async def _poll_and_dispatch(self) -> None:
+        await self._permits.acquire()
+        try:
+            task = await self._poll()
+        except Exception as e:
+            self._permits.release()
+            raise e
+
+        if task is None:
+            self._permits.release()
+            return
+
+        # Need to store a reference to the async task or it may be garbage collected
+        scheduled = asyncio.create_task(self._execute_callback(task))
+        self._background_tasks.add(scheduled)
+        scheduled.add_done_callback(self._background_tasks.remove)
+
+    async def _execute_callback(self, task: T) -> None:
+        try:
+            await self._callback(task)
+        except Exception:
+            logger.exception("Exception during callback")
+        finally:
+            self._permits.release()

cadence/worker/_registry.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""
+Workflow and Activity Registry for Cadence Python Client.
+
+This module provides a registry system for managing workflows and activities,
+similar to the Go client's registry.go implementation.
+"""
+
+import logging
+from typing import (
+    Callable,
+    Dict,
+    Optional,
+    Unpack,
+    TypedDict,
+    overload,
+    Type,
+    Union,
+    TypeVar,
+)
+from cadence.activity import (
+    ActivityDefinitionOptions,
+    ActivityDefinition,
+    ActivityDecorator,
+    P,
+    T,
+)
+from cadence.workflow import WorkflowDefinition, WorkflowDefinitionOptions
+
+logger = logging.getLogger(__name__)
+
+# TypeVar for workflow class types
+W = TypeVar("W")
+
+
+class RegisterWorkflowOptions(TypedDict, total=False):
+    """Options for registering a workflow."""
+
+    name: Optional[str]
+    alias: Optional[str]
+
+
+class Registry:
+    """
+    Registry for managing workflows and activities.
+
+    This class provides functionality to register, retrieve, and manage
+    workflows and activities in a Cadence application.
+    """
+
+    def __init__(self) -> None:
+        """Initialize the registry."""
+        self._workflows: Dict[str, WorkflowDefinition] = {}
+        self._activities: Dict[str, ActivityDefinition] = {}
+        self._workflow_aliases: Dict[str, str] = {}  # alias -> name mapping
+
+    def workflow(
+        self, cls: Optional[Type[W]] = None, **kwargs: Unpack[RegisterWorkflowOptions]
+    ) -> Union[Type[W], Callable[[Type[W]], Type[W]]]:
+        """
+        Register a workflow class.
+
+        This method can be used as a decorator or called directly.
+        Only supports class-based workflows.
+
+        Args:
+            cls: The workflow class to register
+            **kwargs: Options for registration (name, alias)
+
+        Returns:
+            The decorated class
+
+        Raises:
+            KeyError: If workflow name already exists
+            ValueError: If class workflow is invalid
+        """
+        options = RegisterWorkflowOptions(**kwargs)
+
+        def decorator(target: Type[W]) -> Type[W]:
+            workflow_name = options.get("name") or target.__name__
+
+            if workflow_name in self._workflows:
+                raise KeyError(f"Workflow '{workflow_name}' is already registered")
+
+            # Create WorkflowDefinition with type information
+            workflow_opts = WorkflowDefinitionOptions(name=workflow_name)
+            workflow_def = WorkflowDefinition.wrap(target, workflow_opts)
+            self._workflows[workflow_name] = workflow_def
+
+            # Register alias if provided
+            alias = options.get("alias")
+            if alias:
+                if alias in self._workflow_aliases:
+                    raise KeyError(f"Workflow alias '{alias}' is already registered")
+                self._workflow_aliases[alias] = workflow_name
+
+            logger.info(f"Registered workflow '{workflow_name}'")
+            return target
+
+        if cls is None:
+            return decorator
+        return decorator(cls)
+
+    @overload
+    def activity(self, func: Callable[P, T]) -> ActivityDefinition[P, T]: ...
+
+    @overload
+    def activity(
+        self, **kwargs: Unpack[ActivityDefinitionOptions]
+    ) -> ActivityDecorator: ...
+
+    def activity(
+        self,
+        func: Callable[P, T] | None = None,
+        **kwargs: Unpack[ActivityDefinitionOptions],
+    ) -> ActivityDecorator | ActivityDefinition[P, T]:
+        """
+        Register an activity function.
+
+        This method can be used as a decorator or called directly.
+
+        Args:
+            func: The activity function to register
+            **kwargs: Options for registration (name, alias)
+
+        Returns:
+            The decorated function or the function itself
+
+        Raises:
+            KeyError: If activity name already exists
+        """
+        options = ActivityDefinitionOptions(**kwargs)
+
+        def decorator(f: Callable[P, T]) -> ActivityDefinition[P, T]:
+            defn = ActivityDefinition.wrap(f, options)
+
+            self._register_activity(defn)
+
+            return defn
+
+        if func is not None:
+            return decorator(func)
+
+        return decorator
+
+    def register_activities(self, obj: object) -> None:
+        activities = _find_activity_definitions(obj)
+        if not activities:
+            raise ValueError(f"No activity definitions found in '{repr(obj)}'")
+
+        for defn in activities:
+            self._register_activity(defn)
+
+    def register_activity(self, defn: Callable) -> None:
+        if not isinstance(defn, ActivityDefinition):
+            raise ValueError(f"{defn.__qualname__} must have @activity.defn decorator")
+        self._register_activity(defn)
+
+    def _register_activity(self, defn: ActivityDefinition) -> None:
+        if defn.name in self._activities:
+            raise KeyError(f"Activity '{defn.name}' is already registered")
+
+        self._activities[defn.name] = defn
+
+    def get_workflow(self, name: str) -> WorkflowDefinition:
+        """
+        Get a registered workflow by name.
+
+        Args:
+            name: Name or alias of the workflow
+
+        Returns:
+            The workflow definition
+
+        Raises:
+            KeyError: If workflow is not found
+        """
+        # Check if it's an alias
+        actual_name = self._workflow_aliases.get(name, name)
+
+        if actual_name not in self._workflows:
+            raise KeyError(f"Workflow '{name}' not found in registry")
+
+        return self._workflows[actual_name]
+
+    def get_activity(self, name: str) -> ActivityDefinition:
+        """
+        Get a registered activity by name.
+
+        Args:
+            name: Name or alias of the activity
+
+        Returns:
+            The activity function
+
+        Raises:
+            KeyError: If activity is not found
+        """
+        return self._activities[name]
+
+    def __add__(self, other: "Registry") -> "Registry":
+        result = Registry()
+        for name, fn in self._activities.items():
+            result._register_activity(fn)
+        for name, fn in other._activities.items():
+            result._register_activity(fn)
+
+        return result
+
+    @staticmethod
+    def of(*args: "Registry") -> "Registry":
+        result = Registry()
+        for other in args:
+            result += other
+
+        return result
+
+
+def _find_activity_definitions(instance: object) -> list[ActivityDefinition]:
+    attr_to_def: dict[str, ActivityDefinition] = {}
+    for t in instance.__class__.__mro__:
+        for attr in dir(t):
+            if attr.startswith("_"):
+                continue
+            value = getattr(t, attr)
+            if isinstance(value, ActivityDefinition):
+                if attr in attr_to_def:
+                    raise ValueError(
+                        f"'{attr}' was overridden with a duplicate activity definition"
+                    )
+                attr_to_def[attr] = value
+
+    result: list[ActivityDefinition] = []
+    for attr, definition in attr_to_def.items():
+        result.append(
+            ActivityDefinition(
+                getattr(instance, attr),
+                definition.name,
+                definition.strategy,
+                definition.params,
+                definition.result_type,
+            )
+        )
+
+    return result

cadence/worker/_types.py

@@ -0,0 +1,26 @@
+from typing import TypedDict
+
+
+class WorkerOptions(TypedDict, total=False):
+    max_concurrent_activity_execution_size: int
+    max_concurrent_decision_task_execution_size: int
+    task_list_activities_per_second: float
+    # Remove these in favor of introducing automatic scaling prior to release
+    activity_task_pollers: int
+    decision_task_pollers: int
+    disable_workflow_worker: bool
+    disable_activity_worker: bool
+    identity: str
+
+
+_DEFAULT_WORKER_OPTIONS: WorkerOptions = {
+    "max_concurrent_activity_execution_size": 1000,
+    "max_concurrent_decision_task_execution_size": 1000,
+    "task_list_activities_per_second": 0.0,
+    "activity_task_pollers": 2,
+    "decision_task_pollers": 2,
+    "disable_workflow_worker": False,
+    "disable_activity_worker": False,
+}
+
+_LONG_POLL_TIMEOUT = 60.0

cadence/worker/_worker.py

@@ -0,0 +1,56 @@
+import asyncio
+import uuid
+from typing import Unpack, cast
+
+from cadence.client import Client
+from cadence.worker._registry import Registry
+from cadence.worker._activity import ActivityWorker
+from cadence.worker._decision import DecisionWorker
+from cadence.worker._types import WorkerOptions, _DEFAULT_WORKER_OPTIONS
+
+
+class Worker:
+    def __init__(
+        self,
+        client: Client,
+        task_list: str,
+        registry: Registry,
+        **kwargs: Unpack[WorkerOptions],
+    ) -> None:
+        self._client = client
+        self._task_list = task_list
+
+        options = WorkerOptions(**kwargs)
+        _validate_and_copy_defaults(client, task_list, options)
+        self._options = options
+        self._activity_worker = ActivityWorker(client, task_list, registry, options)
+        self._decision_worker = DecisionWorker(client, task_list, registry, options)
+
+    @property
+    def client(self) -> Client:
+        return self._client
+
+    @property
+    def task_list(self) -> str:
+        return self._task_list
+
+    async def run(self) -> None:
+        async with asyncio.TaskGroup() as tg:
+            if not self._options["disable_workflow_worker"]:
+                tg.create_task(self._decision_worker.run())
+            if not self._options["disable_activity_worker"]:
+                tg.create_task(self._activity_worker.run())
+
+
+def _validate_and_copy_defaults(
+    client: Client, task_list: str, options: WorkerOptions
+) -> None:
+    if "identity" not in options:
+        options["identity"] = f"{client.identity}@{task_list}@{uuid.uuid4()}"
+
+    # TODO: More validation
+
+    # Set default values for missing options
+    for key, value in _DEFAULT_WORKER_OPTIONS.items():
+        if key not in options:
+            cast(dict, options)[key] = value