cadence-python-client 0.1.0__py3-none-any.whl

cadence/__init__.py

@@ -0,0 +1,18 @@
+"""
+Cadence Python Client
+
+A Python framework for authoring workflows and activities for Cadence.
+"""
+
+# Import main client functionality
+from .client import Client
+from .worker import Registry
+from . import workflow
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Client",
+    "Registry",
+    "workflow",
+]

cadence/_internal/__init__.py

@@ -0,0 +1,8 @@
+"""Private implementation details for the Cadence Python client.
+
+Modules in this package are not part of the public API surface and may change
+without notice. Public callers should import from packages like `cadence.worker`,
+`cadence.workflow`, and `cadence.activity` instead.
+"""
+
+__all__: list[str] = []

cadence/_internal/activity/__init__.py

@@ -0,0 +1,5 @@
+from ._activity_executor import ActivityExecutor
+
+__all__ = [
+    "ActivityExecutor",
+]

cadence/_internal/activity/_activity_executor.py

@@ -0,0 +1,113 @@
+from concurrent.futures import ThreadPoolExecutor
+from logging import getLogger
+from traceback import format_exception
+from typing import Any, Callable
+from google.protobuf.duration import to_timedelta
+from google.protobuf.timestamp import to_datetime
+
+from cadence._internal.activity._context import _Context, _SyncContext
+from cadence.activity import ActivityInfo, ActivityDefinition, ExecutionStrategy
+from cadence.api.v1.common_pb2 import Failure
+from cadence.api.v1.service_worker_pb2 import (
+    PollForActivityTaskResponse,
+    RespondActivityTaskFailedRequest,
+    RespondActivityTaskCompletedRequest,
+)
+from cadence.client import Client
+
+_logger = getLogger(__name__)
+
+
+class ActivityExecutor:
+    def __init__(
+        self,
+        client: Client,
+        task_list: str,
+        identity: str,
+        max_workers: int,
+        registry: Callable[[str], ActivityDefinition],
+    ):
+        self._client = client
+        self._data_converter = client.data_converter
+        self._registry = registry
+        self._identity = identity
+        self._task_list = task_list
+        self._thread_pool = ThreadPoolExecutor(
+            max_workers=max_workers, thread_name_prefix=f"{task_list}-activity-"
+        )
+
+    async def execute(self, task: PollForActivityTaskResponse):
+        try:
+            context = self._create_context(task)
+            result = await context.execute(task.input)
+            await self._report_success(task, result)
+        except Exception as e:
+            await self._report_failure(task, e)
+
+    def _create_context(self, task: PollForActivityTaskResponse) -> _Context:
+        activity_type = task.activity_type.name
+        try:
+            activity_def = self._registry(activity_type)
+        except KeyError:
+            raise KeyError(f"Activity type not found: {activity_type}") from None
+
+        info = self._create_info(task)
+
+        if activity_def.strategy == ExecutionStrategy.ASYNC:
+            return _Context(self._client, info, activity_def)
+        else:
+            return _SyncContext(self._client, info, activity_def, self._thread_pool)
+
+    async def _report_failure(
+        self, task: PollForActivityTaskResponse, error: Exception
+    ):
+        try:
+            await self._client.worker_stub.RespondActivityTaskFailed(
+                RespondActivityTaskFailedRequest(
+                    task_token=task.task_token,
+                    failure=_to_failure(error),
+                    identity=self._identity,
+                )
+            )
+        except Exception:
+            _logger.exception("Exception reporting activity failure")
+
+    async def _report_success(self, task: PollForActivityTaskResponse, result: Any):
+        as_payload = self._data_converter.to_data([result])
+
+        try:
+            await self._client.worker_stub.RespondActivityTaskCompleted(
+                RespondActivityTaskCompletedRequest(
+                    task_token=task.task_token,
+                    result=as_payload,
+                    identity=self._identity,
+                )
+            )
+        except Exception:
+            _logger.exception("Exception reporting activity complete")
+
+    def _create_info(self, task: PollForActivityTaskResponse) -> ActivityInfo:
+        return ActivityInfo(
+            task_token=task.task_token,
+            workflow_type=task.workflow_type.name,
+            workflow_domain=task.workflow_domain,
+            workflow_id=task.workflow_execution.workflow_id,
+            workflow_run_id=task.workflow_execution.run_id,
+            activity_id=task.activity_id,
+            activity_type=task.activity_type.name,
+            task_list=self._task_list,
+            heartbeat_timeout=to_timedelta(task.heartbeat_timeout),
+            scheduled_timestamp=to_datetime(task.scheduled_time),
+            started_timestamp=to_datetime(task.started_time),
+            start_to_close_timeout=to_timedelta(task.start_to_close_timeout),
+            attempt=task.attempt,
+        )
+
+
+def _to_failure(exception: Exception) -> Failure:
+    stacktrace = "".join(format_exception(exception))
+
+    return Failure(
+        reason=type(exception).__name__,
+        details=stacktrace.encode(),
+    )

cadence/_internal/activity/_context.py

@@ -0,0 +1,58 @@
+import asyncio
+from concurrent.futures.thread import ThreadPoolExecutor
+from typing import Any
+
+from cadence import Client
+from cadence.activity import ActivityInfo, ActivityContext, ActivityDefinition
+from cadence.api.v1.common_pb2 import Payload
+
+
+class _Context(ActivityContext):
+    def __init__(
+        self,
+        client: Client,
+        info: ActivityInfo,
+        activity_fn: ActivityDefinition[[Any], Any],
+    ):
+        self._client = client
+        self._info = info
+        self._activity_fn = activity_fn
+
+    async def execute(self, payload: Payload) -> Any:
+        params = self._to_params(payload)
+        with self._activate():
+            return await self._activity_fn(*params)
+
+    def _to_params(self, payload: Payload) -> list[Any]:
+        type_hints = [param.type_hint for param in self._activity_fn.params]
+        return self._client.data_converter.from_data(payload, type_hints)
+
+    def client(self) -> Client:
+        return self._client
+
+    def info(self) -> ActivityInfo:
+        return self._info
+
+
+class _SyncContext(_Context):
+    def __init__(
+        self,
+        client: Client,
+        info: ActivityInfo,
+        activity_fn: ActivityDefinition[[Any], Any],
+        executor: ThreadPoolExecutor,
+    ):
+        super().__init__(client, info, activity_fn)
+        self._executor = executor
+
+    async def execute(self, payload: Payload) -> Any:
+        params = self._to_params(payload)
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(self._executor, self._run, params)
+
+    def _run(self, args: list[Any]) -> Any:
+        with self._activate():
+            return self._activity_fn(*args)
+
+    def client(self) -> Client:
+        raise RuntimeError("client is only supported in async activities")

cadence/_internal/rpc/error.py

@@ -0,0 +1,148 @@
+from typing import Callable, Any, Optional, Generator, TypeVar
+
+import grpc
+from google.rpc.status_pb2 import Status  # type: ignore
+from grpc.aio import (
+    UnaryUnaryClientInterceptor,
+    ClientCallDetails,
+    AioRpcError,
+    UnaryUnaryCall,
+    Metadata,
+)
+from grpc_status.rpc_status import from_call  # type: ignore
+
+from cadence.api.v1 import error_pb2
+from cadence import error
+
+
+RequestType = TypeVar("RequestType")
+ResponseType = TypeVar("ResponseType")
+DoneCallbackType = Callable[[Any], None]
+
+
+# A UnaryUnaryCall is an awaitable type returned by GRPC's aio support.
+# We need to take the UnaryUnaryCall we receive and return one that remaps the exception.
+# It doesn't have any functions to compose operations together, so our only option is to wrap it.
+# If the interceptor directly throws an exception other than AioRpcError it breaks GRPC
+class CadenceErrorUnaryUnaryCall(UnaryUnaryCall[RequestType, ResponseType]):
+    def __init__(self, wrapped: UnaryUnaryCall[RequestType, ResponseType]):
+        super().__init__()
+        self._wrapped = wrapped
+
+    def __await__(self) -> Generator[Any, None, ResponseType]:
+        try:
+            response = yield from self._wrapped.__await__()  # type: ResponseType
+            return response
+        except AioRpcError as e:
+            raise map_error(e)
+
+    async def initial_metadata(self) -> Metadata:
+        return await self._wrapped.initial_metadata()
+
+    async def trailing_metadata(self) -> Metadata:
+        return await self._wrapped.trailing_metadata()
+
+    async def code(self) -> grpc.StatusCode:
+        return await self._wrapped.code()
+
+    async def details(self) -> str:
+        return await self._wrapped.details()  # type: ignore
+
+    async def wait_for_connection(self) -> None:
+        await self._wrapped.wait_for_connection()
+
+    def cancelled(self) -> bool:
+        return self._wrapped.cancelled()  # type: ignore
+
+    def done(self) -> bool:
+        return self._wrapped.done()  # type: ignore
+
+    def time_remaining(self) -> Optional[float]:
+        return self._wrapped.time_remaining()  # type: ignore
+
+    def cancel(self) -> bool:
+        return self._wrapped.cancel()  # type: ignore
+
+    def add_done_callback(self, callback: DoneCallbackType) -> None:
+        self._wrapped.add_done_callback(callback)
+
+
+class CadenceErrorInterceptor(UnaryUnaryClientInterceptor):
+    async def intercept_unary_unary(
+        self,
+        continuation: Callable[[ClientCallDetails, Any], Any],
+        client_call_details: ClientCallDetails,
+        request: Any,
+    ) -> Any:
+        rpc_call = await continuation(client_call_details, request)
+        return CadenceErrorUnaryUnaryCall(rpc_call)
+
+
+def map_error(e: AioRpcError) -> error.CadenceRpcError:
+    status: Status | None = from_call(e)
+    if not status or not status.details:
+        return error.CadenceRpcError(e.details(), e.code())
+
+    details = status.details[0]
+    if details.Is(error_pb2.WorkflowExecutionAlreadyStartedError.DESCRIPTOR):
+        already_started = error_pb2.WorkflowExecutionAlreadyStartedError()
+        details.Unpack(already_started)
+        return error.WorkflowExecutionAlreadyStartedError(
+            e.details(),
+            e.code(),
+            already_started.start_request_id,
+            already_started.run_id,
+        )
+    elif details.Is(error_pb2.EntityNotExistsError.DESCRIPTOR):
+        not_exists = error_pb2.EntityNotExistsError()
+        details.Unpack(not_exists)
+        return error.EntityNotExistsError(
+            e.details(),
+            e.code(),
+            not_exists.current_cluster,
+            not_exists.active_cluster,
+            list(not_exists.active_clusters),
+        )
+    elif details.Is(error_pb2.WorkflowExecutionAlreadyCompletedError.DESCRIPTOR):
+        return error.WorkflowExecutionAlreadyCompletedError(e.details(), e.code())
+    elif details.Is(error_pb2.DomainNotActiveError.DESCRIPTOR):
+        not_active = error_pb2.DomainNotActiveError()
+        details.Unpack(not_active)
+        return error.DomainNotActiveError(
+            e.details(),
+            e.code(),
+            not_active.domain,
+            not_active.current_cluster,
+            not_active.active_cluster,
+            list(not_active.active_clusters),
+        )
+    elif details.Is(error_pb2.ClientVersionNotSupportedError.DESCRIPTOR):
+        not_supported = error_pb2.ClientVersionNotSupportedError()
+        details.Unpack(not_supported)
+        return error.ClientVersionNotSupportedError(
+            e.details(),
+            e.code(),
+            not_supported.feature_version,
+            not_supported.client_impl,
+            not_supported.supported_versions,
+        )
+    elif details.Is(error_pb2.FeatureNotEnabledError.DESCRIPTOR):
+        not_enabled = error_pb2.FeatureNotEnabledError()
+        details.Unpack(not_enabled)
+        return error.FeatureNotEnabledError(
+            e.details(), e.code(), not_enabled.feature_flag
+        )
+    elif details.Is(error_pb2.CancellationAlreadyRequestedError.DESCRIPTOR):
+        return error.CancellationAlreadyRequestedError(e.details(), e.code())
+    elif details.Is(error_pb2.DomainAlreadyExistsError.DESCRIPTOR):
+        return error.DomainAlreadyExistsError(e.details(), e.code())
+    elif details.Is(error_pb2.LimitExceededError.DESCRIPTOR):
+        return error.LimitExceededError(e.details(), e.code())
+    elif details.Is(error_pb2.QueryFailedError.DESCRIPTOR):
+        return error.QueryFailedError(e.details(), e.code())
+    elif details.Is(error_pb2.ServiceBusyError.DESCRIPTOR):
+        service_busy = error_pb2.ServiceBusyError()
+        details.Unpack(service_busy)
+        return error.ServiceBusyError(e.details(), e.code(), service_busy.reason)
+    else:
+        return error.CadenceRpcError(e.details(), e.code())

cadence/_internal/rpc/retry.py

@@ -0,0 +1,104 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Callable, Any
+
+from grpc import StatusCode
+from grpc.aio import UnaryUnaryClientInterceptor, ClientCallDetails
+
+from cadence.error import CadenceRpcError, EntityNotExistsError
+
+RETRYABLE_CODES = {
+    StatusCode.INTERNAL,
+    StatusCode.RESOURCE_EXHAUSTED,
+    StatusCode.ABORTED,
+    StatusCode.UNAVAILABLE,
+}
+
+
+# No expiration interval, use the GRPC timeout value instead
+@dataclass
+class ExponentialRetryPolicy:
+    initial_interval: float
+    backoff_coefficient: float
+    max_interval: float
+    max_attempts: float
+
+    def next_delay(
+        self, attempts: int, elapsed: float, expiration: float
+    ) -> float | None:
+        if elapsed >= expiration:
+            return None
+        if self.max_attempts != 0 and attempts >= self.max_attempts:
+            return None
+
+        backoff = min(
+            self.initial_interval * pow(self.backoff_coefficient, attempts - 1),
+            self.max_interval,
+        )
+        if (elapsed + backoff) >= expiration:
+            return None
+
+        return backoff
+
+
+DEFAULT_RETRY_POLICY = ExponentialRetryPolicy(
+    initial_interval=0.02, backoff_coefficient=1.2, max_interval=6, max_attempts=0
+)
+GET_WORKFLOW_HISTORY = b"/uber.cadence.api.v1.WorkflowAPI/GetWorkflowExecutionHistory"
+
+
+class RetryInterceptor(UnaryUnaryClientInterceptor):
+    def __init__(self, retry_policy: ExponentialRetryPolicy = DEFAULT_RETRY_POLICY):
+        super().__init__()
+        self._retry_policy = retry_policy
+
+    async def intercept_unary_unary(
+        self,
+        continuation: Callable[[ClientCallDetails, Any], Any],
+        client_call_details: ClientCallDetails,
+        request: Any,
+    ) -> Any:
+        loop = asyncio.get_running_loop()
+        expiration_interval = client_call_details.timeout
+        start_time = loop.time()
+        deadline = start_time + expiration_interval
+
+        attempts = 0
+        while True:
+            remaining = deadline - loop.time()
+            # Namedtuple methods start with an underscore to avoid conflicts and aren't actually private
+            # noinspection PyProtectedMember
+            call_details = client_call_details._replace(timeout=remaining)
+            rpc_call = await continuation(call_details, request)
+            try:
+                # Return the result directly if success. GRPC will wrap it back into a UnaryUnaryCall
+                return await rpc_call
+            except CadenceRpcError as e:
+                err = e
+
+            attempts += 1
+            elapsed = loop.time() - start_time
+            backoff = self._retry_policy.next_delay(
+                attempts, elapsed, expiration_interval
+            )
+            if not is_retryable(err, client_call_details) or backoff is None:
+                break
+
+            await asyncio.sleep(backoff)
+
+        # On policy expiration, return the most recent UnaryUnaryCall. It has the error we want
+        return rpc_call
+
+
+def is_retryable(err: CadenceRpcError, call_details: ClientCallDetails) -> bool:
+    # Handle requests to the passive side, matching the Go and Java Clients
+    if call_details.method == GET_WORKFLOW_HISTORY and isinstance(
+        err, EntityNotExistsError
+    ):
+        return (
+            err.active_cluster is not None
+            and err.current_cluster is not None
+            and err.active_cluster != err.current_cluster
+        )
+
+    return err.code in RETRYABLE_CODES

cadence/_internal/rpc/yarpc.py

@@ -0,0 +1,42 @@
+from typing import Any, Callable
+
+from grpc.aio import Metadata
+from grpc.aio import UnaryUnaryClientInterceptor, ClientCallDetails
+
+
+SERVICE_KEY = "rpc-service"
+CALLER_KEY = "rpc-caller"
+ENCODING_KEY = "rpc-encoding"
+ENCODING_PROTO = "proto"
+
+
+class YarpcMetadataInterceptor(UnaryUnaryClientInterceptor):
+    def __init__(self, service: str, caller: str):
+        self._metadata = Metadata(
+            (SERVICE_KEY, service),
+            (CALLER_KEY, caller),
+            (ENCODING_KEY, ENCODING_PROTO),
+        )
+
+    async def intercept_unary_unary(
+        self,
+        continuation: Callable[[ClientCallDetails, Any], Any],
+        client_call_details: ClientCallDetails,
+        request: Any,
+    ) -> Any:
+        return await continuation(self._replace_details(client_call_details), request)
+
+    def _replace_details(
+        self, client_call_details: ClientCallDetails
+    ) -> ClientCallDetails:
+        metadata = client_call_details.metadata
+        if metadata is None:
+            metadata = self._metadata
+        else:
+            metadata += self._metadata
+
+        # Namedtuple methods start with an underscore to avoid conflicts and aren't actually private
+        # noinspection PyProtectedMember
+        return client_call_details._replace(
+            metadata=metadata, timeout=client_call_details.timeout or 60.0
+        )

cadence/_internal/workflow/context.py

@@ -0,0 +1,121 @@
+from contextlib import contextmanager
+from datetime import timedelta
+from math import ceil
+from typing import Iterator, Optional, Any, Unpack, Type, cast
+
+from cadence._internal.workflow.statemachine.decision_manager import DecisionManager
+from cadence._internal.workflow.decisions_helper import DecisionsHelper
+from cadence.api.v1.common_pb2 import ActivityType
+from cadence.api.v1.decision_pb2 import ScheduleActivityTaskDecisionAttributes
+from cadence.api.v1.tasklist_pb2 import TaskList, TaskListKind
+from cadence.data_converter import DataConverter
+from cadence.workflow import WorkflowContext, WorkflowInfo, ResultType, ActivityOptions
+
+
+class Context(WorkflowContext):
+    def __init__(
+        self,
+        info: WorkflowInfo,
+        decision_manager: DecisionManager,
+    ):
+        self._info = info
+        self._replay_mode = True
+        self._replay_current_time_milliseconds: Optional[int] = None
+        self._decision_helper = DecisionsHelper()
+        self._decision_manager = decision_manager
+
+    def info(self) -> WorkflowInfo:
+        return self._info
+
+    def data_converter(self) -> DataConverter:
+        return self.info().data_converter
+
+    async def execute_activity(
+        self,
+        activity: str,
+        result_type: Type[ResultType],
+        *args: Any,
+        **kwargs: Unpack[ActivityOptions],
+    ) -> ResultType:
+        opts = ActivityOptions(**kwargs)
+        if "schedule_to_close_timeout" not in opts and (
+            "schedule_to_start_timeout" not in opts
+            or "start_to_close_timeout" not in opts
+        ):
+            raise ValueError(
+                "Either schedule_to_close_timeout or both schedule_to_start_timeout and start_to_close_timeout must be specified"
+            )
+
+        schedule_to_close = opts.get("schedule_to_close_timeout", None)
+        schedule_to_start = opts.get("schedule_to_start_timeout", None)
+        start_to_close = opts.get("start_to_close_timeout", None)
+        heartbeat = opts.get("heartbeat_timeout", None)
+
+        if schedule_to_close is None:
+            schedule_to_close = schedule_to_start + start_to_close  # type: ignore
+
+        if start_to_close is None:
+            start_to_close = schedule_to_close
+
+        if schedule_to_start is None:
+            schedule_to_start = schedule_to_close
+
+        if heartbeat is None:
+            heartbeat = schedule_to_close
+
+        task_list = (
+            opts["task_list"]
+            if opts.get("task_list", None)
+            else self._info.workflow_task_list
+        )
+
+        activity_input = self.data_converter().to_data(list(args))
+        activity_id = self._decision_helper.generate_activity_id(activity)
+        schedule_attributes = ScheduleActivityTaskDecisionAttributes(
+            activity_id=activity_id,
+            activity_type=ActivityType(name=activity),
+            domain=self.info().workflow_domain,
+            task_list=TaskList(kind=TaskListKind.TASK_LIST_KIND_NORMAL, name=task_list),
+            input=activity_input,
+            schedule_to_close_timeout=_round_to_nearest_second(schedule_to_close),
+            schedule_to_start_timeout=_round_to_nearest_second(schedule_to_start),
+            start_to_close_timeout=_round_to_nearest_second(start_to_close),
+            heartbeat_timeout=_round_to_nearest_second(heartbeat),
+            retry_policy=None,
+            header=None,
+            request_local_dispatch=False,
+        )
+
+        result_payload = await self._decision_manager.schedule_activity(
+            schedule_attributes
+        )
+
+        result = self.data_converter().from_data(result_payload, [result_type])[0]
+
+        return cast(ResultType, result)
+
+    def set_replay_mode(self, replay: bool) -> None:
+        """Set whether the workflow is currently in replay mode."""
+        self._replay_mode = replay
+
+    def is_replay_mode(self) -> bool:
+        """Check if the workflow is currently in replay mode."""
+        return self._replay_mode
+
+    def set_replay_current_time_milliseconds(self, time_millis: int) -> None:
+        """Set the current replay time in milliseconds."""
+        self._replay_current_time_milliseconds = time_millis
+
+    def get_replay_current_time_milliseconds(self) -> Optional[int]:
+        """Get the current replay time in milliseconds."""
+        return self._replay_current_time_milliseconds
+
+    @contextmanager
+    def _activate(self) -> Iterator["Context"]:
+        token = WorkflowContext._var.set(self)
+        yield self
+        WorkflowContext._var.reset(token)
+
+
+def _round_to_nearest_second(delta: timedelta) -> timedelta:
+    return timedelta(seconds=ceil(delta.total_seconds()))