hatchet-sdk 1.2.6__py3-none-any.whl → 1.3.1__py3-none-any.whl

hatchet_sdk/runnables/workflow.py

@@ -1,6 +1,6 @@
 import asyncio
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Callable, Generic, Union, cast, overload
+from typing import TYPE_CHECKING, Any, Callable, Generic, cast
 
 from google.protobuf import timestamp_pb2
 from pydantic import BaseModel
@@ -12,10 +12,7 @@ from hatchet_sdk.clients.admin import (
 )
 from hatchet_sdk.clients.rest.models.cron_workflows import CronWorkflows
 from hatchet_sdk.context.context import Context, DurableContext
-from hatchet_sdk.contracts.v1.shared.condition_pb2 import TaskConditions
 from hatchet_sdk.contracts.v1.workflows_pb2 import (
-    Concurrency,
-    CreateTaskOpts,
     CreateWorkflowVersionRequest,
     DesiredWorkerLabels,
 )
@@ -36,16 +33,9 @@ from hatchet_sdk.runnables.types import (
     WorkflowConfig,
 )
 from hatchet_sdk.utils.proto_enums import convert_python_enum_to_proto
-from hatchet_sdk.utils.timedelta_to_expression import Duration, timedelta_to_expr
+from hatchet_sdk.utils.timedelta_to_expression import Duration
 from hatchet_sdk.utils.typing import JSONSerializableMapping
-from hatchet_sdk.waits import (
-    Action,
-    Condition,
-    OrGroup,
-    ParentCondition,
-    SleepCondition,
-    UserEventCondition,
-)
+from hatchet_sdk.waits import Condition, OrGroup
 from hatchet_sdk.workflow_run import WorkflowRunRef
 
 if TYPE_CHECKING:
@@ -78,16 +68,12 @@ class BaseWorkflow(Generic[TWorkflowInput]):
         self._on_success_task: Task[TWorkflowInput, Any] | None = None
         self.client = client
 
-    def _get_service_name(self, namespace: str) -> str:
-        return f"{namespace}{self.config.name.lower()}"
-
-    def _create_action_name(
-        self, namespace: str, step: Task[TWorkflowInput, Any]
-    ) -> str:
-        return self._get_service_name(namespace) + ":" + step.name
+    @property
+    def service_name(self) -> str:
+        return f"{self.client.config.namespace}{self.config.name.lower()}"
 
-    def _get_name(self, namespace: str) -> str:
-        return namespace + self.config.name
+    def _create_action_name(self, step: Task[TWorkflowInput, Any]) -> str:
+        return self.service_name + ":" + step.name
 
     def _raise_for_invalid_concurrency(
         self, concurrency: ConcurrencyExpression
@@ -106,58 +92,6 @@ class BaseWorkflow(Generic[TWorkflowInput]):
 
         return True
 
-    @overload
-    def _concurrency_to_proto(self, concurrency: None) -> None: ...
-
-    @overload
-    def _concurrency_to_proto(
-        self, concurrency: ConcurrencyExpression
-    ) -> Concurrency: ...
-
-    def _concurrency_to_proto(
-        self, concurrency: ConcurrencyExpression | None
-    ) -> Concurrency | None:
-        if not concurrency:
-            return None
-
-        self._raise_for_invalid_concurrency(concurrency)
-
-        return Concurrency(
-            expression=concurrency.expression,
-            max_runs=concurrency.max_runs,
-            limit_strategy=concurrency.limit_strategy,
-        )
-
-    @overload
-    def _validate_task(
-        self, task: "Task[TWorkflowInput, R]", service_name: str
-    ) -> CreateTaskOpts: ...
-
-    @overload
-    def _validate_task(self, task: None, service_name: str) -> None: ...
-
-    def _validate_task(
-        self, task: Union["Task[TWorkflowInput, R]", None], service_name: str
-    ) -> CreateTaskOpts | None:
-        if not task:
-            return None
-
-        return CreateTaskOpts(
-            readable_id=task.name,
-            action=service_name + ":" + task.name,
-            timeout=timedelta_to_expr(task.execution_timeout),
-            inputs="{}",
-            parents=[p.name for p in task.parents],
-            retries=task.retries,
-            rate_limits=task.rate_limits,
-            worker_labels=task.desired_worker_labels,
-            backoff_factor=task.backoff_factor,
-            backoff_max_seconds=task.backoff_max_seconds,
-            concurrency=[self._concurrency_to_proto(t) for t in task.concurrency],
-            conditions=self._conditions_to_proto(task),
-            schedule_timeout=timedelta_to_expr(task.schedule_timeout),
-        )
-
     def _validate_priority(self, default_priority: int | None) -> int | None:
         validated_priority = (
             max(1, min(3, default_priority)) if default_priority else None
@@ -169,51 +103,14 @@ class BaseWorkflow(Generic[TWorkflowInput]):
 
         return validated_priority
 
-    def _assign_action(self, condition: Condition, action: Action) -> Condition:
-        condition.base.action = action
-
-        return condition
-
-    def _conditions_to_proto(self, task: Task[TWorkflowInput, Any]) -> TaskConditions:
-        wait_for_conditions = [
-            self._assign_action(w, Action.QUEUE) for w in task.wait_for
-        ]
-
-        cancel_if_conditions = [
-            self._assign_action(c, Action.CANCEL) for c in task.cancel_if
-        ]
-        skip_if_conditions = [self._assign_action(s, Action.SKIP) for s in task.skip_if]
-
-        conditions = wait_for_conditions + cancel_if_conditions + skip_if_conditions
-
-        if len({c.base.readable_data_key for c in conditions}) != len(
-            [c.base.readable_data_key for c in conditions]
-        ):
-            raise ValueError("Conditions must have unique readable data keys.")
-
-        user_events = [
-            c.to_pb() for c in conditions if isinstance(c, UserEventCondition)
-        ]
-        parent_overrides = [
-            c.to_pb() for c in conditions if isinstance(c, ParentCondition)
-        ]
-        sleep_conditions = [
-            c.to_pb() for c in conditions if isinstance(c, SleepCondition)
-        ]
-
-        return TaskConditions(
-            parent_override_conditions=parent_overrides,
-            sleep_conditions=sleep_conditions,
-            user_event_conditions=user_events,
-        )
-
     def _is_leaf_task(self, task: Task[TWorkflowInput, Any]) -> bool:
         return not any(task in t.parents for t in self.tasks if task != t)
 
-    def _get_create_opts(self, namespace: str) -> CreateWorkflowVersionRequest:
-        service_name = self._get_service_name(namespace)
+    def to_proto(self) -> CreateWorkflowVersionRequest:
+        namespace = self.client.config.namespace
+        service_name = self.service_name
 
-        name = self._get_name(namespace)
+        name = self.name
         event_triggers = [namespace + event for event in self.config.on_events]
 
         if self._on_success_task:
@@ -223,10 +120,12 @@ class BaseWorkflow(Generic[TWorkflowInput]):
                 if task.type == StepType.DEFAULT and self._is_leaf_task(task)
             ]
 
-        on_success_task = self._validate_task(self._on_success_task, service_name)
+        on_success_task = (
+            t.to_proto(service_name) if (t := self._on_success_task) else None
+        )
 
         tasks = [
-            self._validate_task(task, service_name)
+            task.to_proto(service_name)
             for task in self.tasks
             if task.type == StepType.DEFAULT
         ]
@@ -234,7 +133,9 @@ class BaseWorkflow(Generic[TWorkflowInput]):
         if on_success_task:
             tasks += [on_success_task]
 
-        on_failure_task = self._validate_task(self._on_failure_task, service_name)
+        on_failure_task = (
+            t.to_proto(service_name) if (t := self._on_failure_task) else None
+        )
 
         return CreateWorkflowVersionRequest(
             name=name,
@@ -243,7 +144,7 @@ class BaseWorkflow(Generic[TWorkflowInput]):
             event_triggers=event_triggers,
             cron_triggers=self.config.on_crons,
             tasks=tasks,
-            concurrency=self._concurrency_to_proto(self.config.concurrency),
+            concurrency=(c.to_proto() if (c := self.config.concurrency) else None),
             ## TODO: Fix this
             cron_input=None,
             on_failure_task=on_failure_task,
@@ -274,11 +175,11 @@ class BaseWorkflow(Generic[TWorkflowInput]):
 
     @property
     def name(self) -> str:
-        return self._get_name(self.client.config.namespace)
+        return self.client.config.namespace + self.config.name
 
     def create_bulk_run_item(
         self,
-        input: TWorkflowInput | None = None,
+        input: TWorkflowInput = cast(TWorkflowInput, EmptyModel()),
         key: str | None = None,
         options: TriggerWorkflowOptions = TriggerWorkflowOptions(),
     ) -> WorkflowRunTriggerConfig:
@@ -394,7 +295,7 @@ class Workflow(BaseWorkflow[TWorkflowInput]):
     def schedule(
         self,
         run_at: datetime,
-        input: TWorkflowInput | None = None,
+        input: TWorkflowInput = cast(TWorkflowInput, EmptyModel()),
         options: ScheduleTriggerWorkflowOptions = ScheduleTriggerWorkflowOptions(),
     ) -> WorkflowVersion:
         return self.client._client.admin.schedule_workflow(
@@ -407,7 +308,7 @@ class Workflow(BaseWorkflow[TWorkflowInput]):
     async def aio_schedule(
         self,
         run_at: datetime,
-        input: TWorkflowInput | None = None,
+        input: TWorkflowInput = cast(TWorkflowInput, EmptyModel()),
         options: ScheduleTriggerWorkflowOptions = ScheduleTriggerWorkflowOptions(),
     ) -> WorkflowVersion:
         return await self.client._client.admin.aio_schedule_workflow(
@@ -421,7 +322,7 @@ class Workflow(BaseWorkflow[TWorkflowInput]):
         self,
         cron_name: str,
         expression: str,
-        input: TWorkflowInput | None = None,
+        input: TWorkflowInput = cast(TWorkflowInput, EmptyModel()),
         additional_metadata: JSONSerializableMapping = {},
     ) -> CronWorkflows:
         return self.client.cron.create(
@@ -436,7 +337,7 @@ class Workflow(BaseWorkflow[TWorkflowInput]):
         self,
         cron_name: str,
         expression: str,
-        input: TWorkflowInput | None = None,
+        input: TWorkflowInput = cast(TWorkflowInput, EmptyModel()),
         additional_metadata: JSONSerializableMapping = {},
     ) -> CronWorkflows:
         return await self.client.cron.aio_create(

hatchet_sdk/waits.py

@@ -37,7 +37,7 @@ class BaseCondition(BaseModel):
     or_group_id: str = Field(default_factory=generate_or_group_id)
     expression: str | None = None
 
-    def to_pb(self) -> BaseMatchCondition:
+    def to_proto(self) -> BaseMatchCondition:
         return BaseMatchCondition(
             readable_data_key=self.readable_data_key,
             action=convert_python_enum_to_proto(self.action, ProtoAction),  # type: ignore[arg-type]
@@ -51,7 +51,7 @@ class Condition(ABC):
         self.base = base
 
     @abstractmethod
-    def to_pb(
+    def to_proto(
         self,
     ) -> UserEventMatchCondition | ParentOverrideMatchCondition | SleepMatchCondition:
         pass
@@ -67,9 +67,9 @@ class SleepCondition(Condition):
 
         self.duration = duration
 
-    def to_pb(self) -> SleepMatchCondition:
+    def to_proto(self) -> SleepMatchCondition:
         return SleepMatchCondition(
-            base=self.base.to_pb(),
+            base=self.base.to_proto(),
             sleep_for=timedelta_to_expr(self.duration),
         )
 
@@ -86,9 +86,9 @@ class UserEventCondition(Condition):
         self.event_key = event_key
         self.expression = expression
 
-    def to_pb(self) -> UserEventMatchCondition:
+    def to_proto(self) -> UserEventMatchCondition:
         return UserEventMatchCondition(
-            base=self.base.to_pb(),
+            base=self.base.to_proto(),
             user_event_key=self.event_key,
         )
 
@@ -103,9 +103,9 @@ class ParentCondition(Condition):
 
         self.parent = parent
 
-    def to_pb(self) -> ParentOverrideMatchCondition:
+    def to_proto(self) -> ParentOverrideMatchCondition:
         return ParentOverrideMatchCondition(
-            base=self.base.to_pb(),
+            base=self.base.to_proto(),
             parent_readable_id=self.parent.name,
         )
 

hatchet_sdk/worker/runner/run_loop_manager.py

@@ -55,17 +55,17 @@ class WorkerActionRunLoopManager:
         self.client = Client(config=self.config, debug=self.debug)
         self.start()
 
-    def start(self, retry_count: int = 1) -> None:
-        k = self.loop.create_task(self.aio_start(retry_count))  # noqa: F841
+    def start(self) -> None:
+        k = self.loop.create_task(self.aio_start())  # noqa: F841
 
     async def aio_start(self, retry_count: int = 1) -> None:
         await capture_logs(
             self.client.log_interceptor,
             self.client.event,
             self._async_start,
-        )(retry_count=retry_count)
+        )()
 
-    async def _async_start(self, retry_count: int = 1) -> None:
+    async def _async_start(self) -> None:
         logger.info("starting runner...")
         self.loop = asyncio.get_running_loop()
         # needed for graceful termination

hatchet_sdk/worker/runner/runner.py

@@ -16,9 +16,10 @@ from hatchet_sdk.client import Client
 from hatchet_sdk.clients.admin import AdminClient
 from hatchet_sdk.clients.dispatcher.action_listener import Action, ActionType
 from hatchet_sdk.clients.dispatcher.dispatcher import DispatcherClient
-from hatchet_sdk.clients.durable_event_listener import DurableEventListener
-from hatchet_sdk.clients.run_event_listener import RunEventListenerClient
-from hatchet_sdk.clients.workflow_listener import PooledWorkflowRunListener
+from hatchet_sdk.clients.events import EventClient
+from hatchet_sdk.clients.listeners.durable_event_listener import DurableEventListener
+from hatchet_sdk.clients.listeners.run_event_listener import RunEventListenerClient
+from hatchet_sdk.clients.listeners.workflow_listener import PooledWorkflowRunListener
 from hatchet_sdk.config import ClientConfig
 from hatchet_sdk.context.context import Context, DurableContext
 from hatchet_sdk.context.worker_context import WorkerContext
@@ -31,6 +32,7 @@ from hatchet_sdk.contracts.dispatcher_pb2 import (
     STEP_EVENT_TYPE_STARTED,
 )
 from hatchet_sdk.exceptions import NonRetryableException
+from hatchet_sdk.features.runs import RunsClient
 from hatchet_sdk.logger import logger
 from hatchet_sdk.runnables.contextvars import (
     ctx_step_run_id,
@@ -66,7 +68,7 @@ class Runner:
     ):
         # We store the config so we can dynamically create clients for the dispatcher client.
         self.config = config
-        self.client = Client(config)
+
         self.slots = slots
         self.tasks: dict[str, asyncio.Task[Any]] = {}  # Store run ids and futures
         self.contexts: dict[str, Context] = {}  # Store run ids and contexts
@@ -82,12 +84,21 @@ class Runner:
         self.killing = False
         self.handle_kill = handle_kill
 
-        # We need to initialize a new admin and dispatcher client *after* we've started the event loop,
-        # otherwise the grpc.aio methods will use a different event loop and we'll get a bunch of errors.
         self.dispatcher_client = DispatcherClient(self.config)
-        self.admin_client = AdminClient(self.config)
         self.workflow_run_event_listener = RunEventListenerClient(self.config)
-        self.client.workflow_listener = PooledWorkflowRunListener(self.config)
+        self.workflow_listener = PooledWorkflowRunListener(self.config)
+        self.runs_client = RunsClient(
+            config=self.config,
+            workflow_run_event_listener=self.workflow_run_event_listener,
+            workflow_run_listener=self.workflow_listener,
+        )
+        self.admin_client = AdminClient(
+            self.config,
+            self.workflow_listener,
+            self.workflow_run_event_listener,
+            self.runs_client,
+        )
+        self.event_client = EventClient(self.config)
         self.durable_event_listener = DurableEventListener(self.config)
 
         self.worker_context = WorkerContext(
@@ -291,11 +302,11 @@ class Runner:
             action=action,
             dispatcher_client=self.dispatcher_client,
             admin_client=self.admin_client,
-            event_client=self.client.event,
+            event_client=self.event_client,
             durable_event_listener=self.durable_event_listener,
             worker=self.worker_context,
             validator_registry=self.validator_registry,
-            runs_client=self.client.runs,
+            runs_client=self.runs_client,
         )
 
     ## IMPORTANT: Keep this method's signature in sync with the wrapper in the OTel instrumentor
@@ -430,7 +441,7 @@ class Runner:
             # check if thread is still running, if so, print a warning
             if run_id in self.threads:
                 thread = self.threads.get(run_id)
-                if thread and self.client.config.enable_force_kill_sync_threads:
+                if thread and self.config.enable_force_kill_sync_threads:
                     self.force_kill_thread(thread)
                     await asyncio.sleep(1)
 

hatchet_sdk/worker/worker.py

@@ -11,6 +11,7 @@ from multiprocessing import Queue
 from multiprocessing.process import BaseProcess
 from types import FrameType
 from typing import Any, TypeVar, get_type_hints
+from warnings import warn
 
 from aiohttp import web
 from aiohttp.web_request import Request
@@ -38,6 +39,10 @@ from hatchet_sdk.worker.runner.run_loop_manager import (
 T = TypeVar("T")
 
 
+class LoopAlreadyRunningException(Exception):
+    pass
+
+
 class WorkerStatus(Enum):
     INITIALIZED = 1
     STARTING = 2
@@ -128,24 +133,20 @@ class Worker:
             sys.exit(1)
 
     def register_workflow(self, workflow: BaseWorkflow[Any]) -> None:
-        namespace = self.client.config.namespace
-
-        opts = workflow._get_create_opts(namespace)
-        name = workflow._get_name(namespace)
+        opts = workflow.to_proto()
+        name = workflow.name
 
         try:
             self.client.admin.put_workflow(name, opts)
         except Exception as e:
-            logger.error(
-                f"failed to register workflow: {workflow._get_name(namespace)}"
-            )
+            logger.error(f"failed to register workflow: {workflow.name}")
             logger.error(e)
             sys.exit(1)
 
         for step in workflow.tasks:
-            action_name = workflow._create_action_name(namespace, step)
+            action_name = workflow._create_action_name(step)
 
-            if workflow.is_durable:
+            if step.is_durable:
                 self.has_any_durable = True
                 self.durable_action_registry[action_name] = step
             else:
@@ -167,22 +168,20 @@ class Worker:
     def status(self) -> WorkerStatus:
         return self._status
 
-    def _setup_loop(self, loop: asyncio.AbstractEventLoop | None = None) -> bool:
+    def _setup_loop(self) -> None:
         try:
-            self.loop = loop or asyncio.get_running_loop()
-            logger.debug("using existing event loop")
-
-            created_loop = False
+            asyncio.get_running_loop()
+            raise LoopAlreadyRunningException(
+                "An event loop is already running. This worker requires its own dedicated event loop. "
+                "Make sure you're not using asyncio.run() or other loop-creating functions in the main thread."
+            )
         except RuntimeError:
-            self.loop = asyncio.new_event_loop()
-
-            logger.debug("creating new event loop")
-            created_loop = True
+            pass
 
+        logger.debug("Creating new event loop")
+        self.loop = asyncio.new_event_loop()
         asyncio.set_event_loop(self.loop)
 
-        return created_loop
-
     async def _health_check_handler(self, request: Request) -> Response:
         response = HealthCheckResponse(
             status=self.status.name,
@@ -224,7 +223,13 @@ class Worker:
         logger.info(f"healthcheck server running on port {port}")
 
     def start(self, options: WorkerStartOptions = WorkerStartOptions()) -> None:
-        self.owned_loop = self._setup_loop(options.loop)
+        if options.loop is not None:
+            warn(
+                "Passing a custom event loop is deprecated and will be removed in the future. This option no longer has any effect",
+                DeprecationWarning,
+            )
+
+        self._setup_loop()
 
         if not self.loop:
             raise RuntimeError("event loop not set, cannot start worker")
@@ -232,11 +237,10 @@ class Worker:
         asyncio.run_coroutine_threadsafe(self._aio_start(), self.loop)
 
         # start the loop and wait until its closed
-        if self.owned_loop:
-            self.loop.run_forever()
+        self.loop.run_forever()
 
-            if self.handle_kill:
-                sys.exit(0)
+        if self.handle_kill:
+            sys.exit(0)
 
     async def _aio_start(self) -> None:
         main_pid = os.getpid()

hatchet_sdk/workflow_run.py

@@ -1,23 +1,27 @@
+import time
 from typing import Any
 
-from hatchet_sdk.clients.run_event_listener import (
+from hatchet_sdk.clients.listeners.run_event_listener import (
     RunEventListener,
     RunEventListenerClient,
 )
-from hatchet_sdk.clients.workflow_listener import PooledWorkflowRunListener
-from hatchet_sdk.config import ClientConfig
-from hatchet_sdk.utils.aio import run_async_from_sync
+from hatchet_sdk.clients.listeners.workflow_listener import PooledWorkflowRunListener
+from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus
+from hatchet_sdk.features.runs import RunsClient
 
 
 class WorkflowRunRef:
     def __init__(
         self,
         workflow_run_id: str,
-        config: ClientConfig,
+        workflow_run_listener: PooledWorkflowRunListener,
+        workflow_run_event_listener: RunEventListenerClient,
+        runs_client: RunsClient,
     ):
         self.workflow_run_id = workflow_run_id
-        self.workflow_listener = PooledWorkflowRunListener(config)
-        self.workflow_run_event_listener = RunEventListenerClient(config=config)
+        self.workflow_run_listener = workflow_run_listener
+        self.workflow_run_event_listener = workflow_run_event_listener
+        self.runs_client = runs_client
 
     def __str__(self) -> str:
         return self.workflow_run_id
@@ -26,7 +30,52 @@ class WorkflowRunRef:
         return self.workflow_run_event_listener.stream(self.workflow_run_id)
 
     async def aio_result(self) -> dict[str, Any]:
-        return await self.workflow_listener.aio_result(self.workflow_run_id)
+        return await self.workflow_run_listener.aio_result(self.workflow_run_id)
+
+    def _safely_get_action_name(self, action_id: str | None) -> str | None:
+        if not action_id:
+            return None
+
+        try:
+            return action_id.split(":", maxsplit=1)[1]
+        except IndexError:
+            return None
 
     def result(self) -> dict[str, Any]:
-        return run_async_from_sync(self.aio_result)
+        retries = 0
+
+        while True:
+            try:
+                details = self.runs_client.get(self.workflow_run_id)
+            except Exception:
+                retries += 1
+
+                if retries > 10:
+                    raise ValueError(f"Workflow run {self.workflow_run_id} not found")
+
+                time.sleep(1)
+                continue
+
+            match details.run.status:
+                case V1TaskStatus.RUNNING:
+                    time.sleep(1)
+                case V1TaskStatus.FAILED:
+                    raise ValueError(
+                        f"Workflow run failed: {details.run.error_message}"
+                    )
+                case V1TaskStatus.COMPLETED:
+                    return {
+                        name: t.output
+                        for t in details.tasks
+                        if (name := self._safely_get_action_name(t.action_id))
+                    }
+                case V1TaskStatus.QUEUED:
+                    time.sleep(1)
+                case V1TaskStatus.CANCELLED:
+                    raise ValueError(
+                        f"Workflow run cancelled: {details.run.error_message}"
+                    )
+                case _:
+                    raise ValueError(
+                        f"Unknown workflow run status: {details.run.status}"
+                    )

{hatchet_sdk-1.2.6.dist-info → hatchet_sdk-1.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hatchet-sdk
-Version: 1.2.6
+Version: 1.3.1
 Summary: 
 Author: Alexander Belanger
 Author-email: alexander@hatchet.run