hatchet-sdk 0.42.3__py3-none-any.whl → 0.42.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hatchet-sdk might be problematic. Click here for more details.
- hatchet_sdk/clients/admin.py +6 -6
- hatchet_sdk/clients/dispatcher/dispatcher.py +22 -15
- hatchet_sdk/clients/events.py +3 -3
- hatchet_sdk/clients/rest/models/workflow_list.py +4 -4
- hatchet_sdk/clients/rest/models/workflow_run.py +10 -10
- hatchet_sdk/context/context.py +9 -7
- hatchet_sdk/context/worker_context.py +5 -5
- hatchet_sdk/hatchet.py +64 -48
- hatchet_sdk/labels.py +1 -1
- hatchet_sdk/utils/aio_utils.py +13 -4
- hatchet_sdk/utils/typing.py +4 -1
- hatchet_sdk/v2/callable.py +23 -20
- hatchet_sdk/v2/concurrency.py +10 -8
- hatchet_sdk/v2/hatchet.py +38 -36
- hatchet_sdk/worker/runner/runner.py +1 -1
- hatchet_sdk/worker/worker.py +16 -9
- hatchet_sdk/workflow.py +21 -9
- {hatchet_sdk-0.42.3.dist-info → hatchet_sdk-0.42.4.dist-info}/METADATA +1 -1
- {hatchet_sdk-0.42.3.dist-info → hatchet_sdk-0.42.4.dist-info}/RECORD +21 -21
- {hatchet_sdk-0.42.3.dist-info → hatchet_sdk-0.42.4.dist-info}/WHEEL +0 -0
- {hatchet_sdk-0.42.3.dist-info → hatchet_sdk-0.42.4.dist-info}/entry_points.txt +0 -0
hatchet_sdk/clients/admin.py
CHANGED
|
@@ -41,7 +41,7 @@ def new_admin(config: ClientConfig):
|
|
|
41
41
|
return AdminClient(config)
|
|
42
42
|
|
|
43
43
|
|
|
44
|
-
class ScheduleTriggerWorkflowOptions(TypedDict):
|
|
44
|
+
class ScheduleTriggerWorkflowOptions(TypedDict, total=False):
|
|
45
45
|
parent_id: Optional[str]
|
|
46
46
|
parent_step_run_id: Optional[str]
|
|
47
47
|
child_index: Optional[int]
|
|
@@ -49,25 +49,25 @@ class ScheduleTriggerWorkflowOptions(TypedDict):
|
|
|
49
49
|
namespace: Optional[str]
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
class ChildTriggerWorkflowOptions(TypedDict):
|
|
52
|
+
class ChildTriggerWorkflowOptions(TypedDict, total=False):
|
|
53
53
|
additional_metadata: Dict[str, str] | None = None
|
|
54
54
|
sticky: bool | None = None
|
|
55
55
|
|
|
56
56
|
|
|
57
|
-
class ChildWorkflowRunDict(TypedDict):
|
|
57
|
+
class ChildWorkflowRunDict(TypedDict, total=False):
|
|
58
58
|
workflow_name: str
|
|
59
59
|
input: Any
|
|
60
60
|
options: ChildTriggerWorkflowOptions
|
|
61
61
|
key: str | None = None
|
|
62
62
|
|
|
63
63
|
|
|
64
|
-
class TriggerWorkflowOptions(ScheduleTriggerWorkflowOptions,
|
|
64
|
+
class TriggerWorkflowOptions(ScheduleTriggerWorkflowOptions, total=False):
|
|
65
65
|
additional_metadata: Dict[str, str] | None = None
|
|
66
66
|
desired_worker_id: str | None = None
|
|
67
67
|
namespace: str | None = None
|
|
68
68
|
|
|
69
69
|
|
|
70
|
-
class WorkflowRunDict(TypedDict):
|
|
70
|
+
class WorkflowRunDict(TypedDict, total=False):
|
|
71
71
|
workflow_name: str
|
|
72
72
|
input: Any
|
|
73
73
|
options: TriggerWorkflowOptions | None
|
|
@@ -425,7 +425,7 @@ class AdminClient(AdminClientBase):
|
|
|
425
425
|
self,
|
|
426
426
|
key: str,
|
|
427
427
|
limit: int,
|
|
428
|
-
duration: RateLimitDuration = RateLimitDuration.SECOND,
|
|
428
|
+
duration: Union[RateLimitDuration.Value, str] = RateLimitDuration.SECOND,
|
|
429
429
|
):
|
|
430
430
|
try:
|
|
431
431
|
self.client.PutRateLimit(
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from typing import Any, cast
|
|
2
|
+
|
|
1
3
|
from google.protobuf.timestamp_pb2 import Timestamp
|
|
2
4
|
|
|
3
5
|
from hatchet_sdk.clients.dispatcher.action_listener import (
|
|
@@ -31,7 +33,7 @@ from ...metadata import get_metadata
|
|
|
31
33
|
DEFAULT_REGISTER_TIMEOUT = 30
|
|
32
34
|
|
|
33
35
|
|
|
34
|
-
def new_dispatcher(config: ClientConfig):
|
|
36
|
+
def new_dispatcher(config: ClientConfig) -> "DispatcherClient":
|
|
35
37
|
return DispatcherClient(config=config)
|
|
36
38
|
|
|
37
39
|
|
|
@@ -40,10 +42,10 @@ class DispatcherClient:
|
|
|
40
42
|
|
|
41
43
|
def __init__(self, config: ClientConfig):
|
|
42
44
|
conn = new_conn(config)
|
|
43
|
-
self.client = DispatcherStub(conn)
|
|
45
|
+
self.client = DispatcherStub(conn) # type: ignore[no-untyped-call]
|
|
44
46
|
|
|
45
47
|
aio_conn = new_conn(config, True)
|
|
46
|
-
self.aio_client = DispatcherStub(aio_conn)
|
|
48
|
+
self.aio_client = DispatcherStub(aio_conn) # type: ignore[no-untyped-call]
|
|
47
49
|
self.token = config.token
|
|
48
50
|
self.config = config
|
|
49
51
|
|
|
@@ -67,7 +69,7 @@ class DispatcherClient:
|
|
|
67
69
|
|
|
68
70
|
async def send_step_action_event(
|
|
69
71
|
self, action: Action, event_type: StepActionEventType, payload: str
|
|
70
|
-
):
|
|
72
|
+
) -> Any:
|
|
71
73
|
try:
|
|
72
74
|
return await self._try_send_step_action_event(action, event_type, payload)
|
|
73
75
|
except Exception as e:
|
|
@@ -87,7 +89,7 @@ class DispatcherClient:
|
|
|
87
89
|
@tenacity_retry
|
|
88
90
|
async def _try_send_step_action_event(
|
|
89
91
|
self, action: Action, event_type: StepActionEventType, payload: str
|
|
90
|
-
):
|
|
92
|
+
) -> Any:
|
|
91
93
|
eventTimestamp = Timestamp()
|
|
92
94
|
eventTimestamp.GetCurrentTime()
|
|
93
95
|
|
|
@@ -103,6 +105,7 @@ class DispatcherClient:
|
|
|
103
105
|
eventPayload=payload,
|
|
104
106
|
)
|
|
105
107
|
|
|
108
|
+
## TODO: What does this return?
|
|
106
109
|
return await self.aio_client.SendStepActionEvent(
|
|
107
110
|
event,
|
|
108
111
|
metadata=get_metadata(self.token),
|
|
@@ -110,7 +113,7 @@ class DispatcherClient:
|
|
|
110
113
|
|
|
111
114
|
async def send_group_key_action_event(
|
|
112
115
|
self, action: Action, event_type: GroupKeyActionEventType, payload: str
|
|
113
|
-
):
|
|
116
|
+
) -> Any:
|
|
114
117
|
eventTimestamp = Timestamp()
|
|
115
118
|
eventTimestamp.GetCurrentTime()
|
|
116
119
|
|
|
@@ -124,19 +127,21 @@ class DispatcherClient:
|
|
|
124
127
|
eventPayload=payload,
|
|
125
128
|
)
|
|
126
129
|
|
|
130
|
+
## TODO: What does this return?
|
|
127
131
|
return await self.aio_client.SendGroupKeyActionEvent(
|
|
128
132
|
event,
|
|
129
133
|
metadata=get_metadata(self.token),
|
|
130
134
|
)
|
|
131
135
|
|
|
132
|
-
def put_overrides_data(self, data: OverridesData):
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
+
def put_overrides_data(self, data: OverridesData) -> ActionEventResponse:
|
|
137
|
+
return cast(
|
|
138
|
+
ActionEventResponse,
|
|
139
|
+
self.client.PutOverridesData(
|
|
140
|
+
data,
|
|
141
|
+
metadata=get_metadata(self.token),
|
|
142
|
+
),
|
|
136
143
|
)
|
|
137
144
|
|
|
138
|
-
return response
|
|
139
|
-
|
|
140
145
|
def release_slot(self, step_run_id: str) -> None:
|
|
141
146
|
self.client.ReleaseSlot(
|
|
142
147
|
ReleaseSlotRequest(stepRunId=step_run_id),
|
|
@@ -154,7 +159,9 @@ class DispatcherClient:
|
|
|
154
159
|
metadata=get_metadata(self.token),
|
|
155
160
|
)
|
|
156
161
|
|
|
157
|
-
def upsert_worker_labels(
|
|
162
|
+
def upsert_worker_labels(
|
|
163
|
+
self, worker_id: str | None, labels: dict[str, str | int]
|
|
164
|
+
) -> None:
|
|
158
165
|
worker_labels = {}
|
|
159
166
|
|
|
160
167
|
for key, value in labels.items():
|
|
@@ -171,9 +178,9 @@ class DispatcherClient:
|
|
|
171
178
|
|
|
172
179
|
async def async_upsert_worker_labels(
|
|
173
180
|
self,
|
|
174
|
-
worker_id: str,
|
|
181
|
+
worker_id: str | None,
|
|
175
182
|
labels: dict[str, str | int],
|
|
176
|
-
):
|
|
183
|
+
) -> None:
|
|
177
184
|
worker_labels = {}
|
|
178
185
|
|
|
179
186
|
for key, value in labels.items():
|
hatchet_sdk/clients/events.py
CHANGED
|
@@ -43,16 +43,16 @@ def proto_timestamp_now():
|
|
|
43
43
|
return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
class PushEventOptions(TypedDict):
|
|
46
|
+
class PushEventOptions(TypedDict, total=False):
|
|
47
47
|
additional_metadata: Dict[str, str] | None = None
|
|
48
48
|
namespace: str | None = None
|
|
49
49
|
|
|
50
50
|
|
|
51
|
-
class BulkPushEventOptions(TypedDict):
|
|
51
|
+
class BulkPushEventOptions(TypedDict, total=False):
|
|
52
52
|
namespace: str | None = None
|
|
53
53
|
|
|
54
54
|
|
|
55
|
-
class BulkPushEventWithMetadata(TypedDict):
|
|
55
|
+
class BulkPushEventWithMetadata(TypedDict, total=False):
|
|
56
56
|
key: str
|
|
57
57
|
payload: Any
|
|
58
58
|
additional_metadata: Optional[Dict[str, Any]] # Optional metadata
|
|
@@ -32,10 +32,10 @@ class WorkflowList(BaseModel):
|
|
|
32
32
|
WorkflowList
|
|
33
33
|
""" # noqa: E501
|
|
34
34
|
|
|
35
|
-
metadata:
|
|
36
|
-
rows:
|
|
37
|
-
pagination:
|
|
38
|
-
__properties: ClassVar[
|
|
35
|
+
metadata: APIResourceMeta | None = None
|
|
36
|
+
rows: list[Workflow] | None = None
|
|
37
|
+
pagination: PaginationResponse | None = None
|
|
38
|
+
__properties: ClassVar[list[str]] = ["metadata", "rows", "pagination"]
|
|
39
39
|
|
|
40
40
|
model_config = ConfigDict(
|
|
41
41
|
populate_by_name=True,
|
|
@@ -39,28 +39,28 @@ class WorkflowRun(BaseModel):
|
|
|
39
39
|
metadata: APIResourceMeta
|
|
40
40
|
tenant_id: StrictStr = Field(alias="tenantId")
|
|
41
41
|
workflow_version_id: StrictStr = Field(alias="workflowVersionId")
|
|
42
|
-
workflow_version:
|
|
42
|
+
workflow_version: WorkflowVersion | None = Field(
|
|
43
43
|
default=None, alias="workflowVersion"
|
|
44
44
|
)
|
|
45
45
|
status: WorkflowRunStatus
|
|
46
|
-
display_name:
|
|
47
|
-
job_runs:
|
|
46
|
+
display_name: StrictStr | None = Field(default=None, alias="displayName")
|
|
47
|
+
job_runs: list[JobRun] | None = Field(default=None, alias="jobRuns")
|
|
48
48
|
triggered_by: WorkflowRunTriggeredBy = Field(alias="triggeredBy")
|
|
49
|
-
input:
|
|
50
|
-
error:
|
|
51
|
-
started_at:
|
|
52
|
-
finished_at:
|
|
53
|
-
duration:
|
|
49
|
+
input: dict[str, Any] | None = None
|
|
50
|
+
error: StrictStr | None = None
|
|
51
|
+
started_at: datetime | None = Field(default=None, alias="startedAt")
|
|
52
|
+
finished_at: datetime | None = Field(default=None, alias="finishedAt")
|
|
53
|
+
duration: StrictInt | None = None
|
|
54
54
|
parent_id: Optional[
|
|
55
55
|
Annotated[str, Field(min_length=36, strict=True, max_length=36)]
|
|
56
56
|
] = Field(default=None, alias="parentId")
|
|
57
57
|
parent_step_run_id: Optional[
|
|
58
58
|
Annotated[str, Field(min_length=36, strict=True, max_length=36)]
|
|
59
59
|
] = Field(default=None, alias="parentStepRunId")
|
|
60
|
-
additional_metadata:
|
|
60
|
+
additional_metadata: dict[str, Any] | None = Field(
|
|
61
61
|
default=None, alias="additionalMetadata"
|
|
62
62
|
)
|
|
63
|
-
__properties: ClassVar[
|
|
63
|
+
__properties: ClassVar[list[str]] = [
|
|
64
64
|
"metadata",
|
|
65
65
|
"tenantId",
|
|
66
66
|
"workflowVersionId",
|
hatchet_sdk/context/context.py
CHANGED
|
@@ -13,8 +13,8 @@ from hatchet_sdk.clients.rest_client import RestApi
|
|
|
13
13
|
from hatchet_sdk.clients.run_event_listener import RunEventListenerClient
|
|
14
14
|
from hatchet_sdk.clients.workflow_listener import PooledWorkflowRunListener
|
|
15
15
|
from hatchet_sdk.context.worker_context import WorkerContext
|
|
16
|
-
from hatchet_sdk.contracts.dispatcher_pb2 import OverridesData
|
|
17
|
-
from hatchet_sdk.contracts.workflows_pb2 import (
|
|
16
|
+
from hatchet_sdk.contracts.dispatcher_pb2 import OverridesData
|
|
17
|
+
from hatchet_sdk.contracts.workflows_pb2 import (
|
|
18
18
|
BulkTriggerWorkflowRequest,
|
|
19
19
|
TriggerWorkflowRequest,
|
|
20
20
|
)
|
|
@@ -69,7 +69,7 @@ class BaseContext:
|
|
|
69
69
|
meta = options["additional_metadata"]
|
|
70
70
|
|
|
71
71
|
## TODO: Pydantic here to simplify this
|
|
72
|
-
trigger_options: TriggerWorkflowOptions = {
|
|
72
|
+
trigger_options: TriggerWorkflowOptions = {
|
|
73
73
|
"parent_id": workflow_run_id,
|
|
74
74
|
"parent_step_run_id": step_run_id,
|
|
75
75
|
"child_key": key,
|
|
@@ -149,8 +149,7 @@ class ContextAioImpl(BaseContext):
|
|
|
149
149
|
key = child_workflow_run.get("key")
|
|
150
150
|
options = child_workflow_run.get("options", {})
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
trigger_options = self._prepare_workflow_options(key, options, worker_id) # type: ignore[arg-type]
|
|
152
|
+
trigger_options = self._prepare_workflow_options(key, options, worker_id)
|
|
154
153
|
|
|
155
154
|
bulk_trigger_workflow_runs.append(
|
|
156
155
|
WorkflowRunDict(
|
|
@@ -238,14 +237,17 @@ class Context(BaseContext):
|
|
|
238
237
|
self.input = self.data.get("input", {})
|
|
239
238
|
|
|
240
239
|
def step_output(self, step: str) -> dict[str, Any] | BaseModel:
|
|
241
|
-
|
|
240
|
+
workflow_validator = next(
|
|
241
|
+
(v for k, v in self.validator_registry.items() if k.split(":")[-1] == step),
|
|
242
|
+
None,
|
|
243
|
+
)
|
|
242
244
|
|
|
243
245
|
try:
|
|
244
246
|
parent_step_data = cast(dict[str, Any], self.data["parents"][step])
|
|
245
247
|
except KeyError:
|
|
246
248
|
raise ValueError(f"Step output for '{step}' not found")
|
|
247
249
|
|
|
248
|
-
if
|
|
250
|
+
if workflow_validator and (v := workflow_validator.step_output):
|
|
249
251
|
return v.model_validate(parent_step_data)
|
|
250
252
|
|
|
251
253
|
return parent_step_data
|
|
@@ -2,7 +2,7 @@ from hatchet_sdk.clients.dispatcher.dispatcher import DispatcherClient
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class WorkerContext:
|
|
5
|
-
_worker_id: str = None
|
|
5
|
+
_worker_id: str | None = None
|
|
6
6
|
_registered_workflow_names: list[str] = []
|
|
7
7
|
_labels: dict[str, str | int] = {}
|
|
8
8
|
|
|
@@ -10,18 +10,18 @@ class WorkerContext:
|
|
|
10
10
|
self._labels = labels
|
|
11
11
|
self.client = client
|
|
12
12
|
|
|
13
|
-
def labels(self):
|
|
13
|
+
def labels(self) -> dict[str, str | int]:
|
|
14
14
|
return self._labels
|
|
15
15
|
|
|
16
|
-
def upsert_labels(self, labels: dict[str, str | int]):
|
|
16
|
+
def upsert_labels(self, labels: dict[str, str | int]) -> None:
|
|
17
17
|
self.client.upsert_worker_labels(self._worker_id, labels)
|
|
18
18
|
self._labels.update(labels)
|
|
19
19
|
|
|
20
|
-
async def async_upsert_labels(self, labels: dict[str, str | int]):
|
|
20
|
+
async def async_upsert_labels(self, labels: dict[str, str | int]) -> None:
|
|
21
21
|
await self.client.async_upsert_worker_labels(self._worker_id, labels)
|
|
22
22
|
self._labels.update(labels)
|
|
23
23
|
|
|
24
|
-
def id(self) -> str:
|
|
24
|
+
def id(self) -> str | None:
|
|
25
25
|
return self._worker_id
|
|
26
26
|
|
|
27
27
|
# def has_workflow(self, workflow_name: str):
|
hatchet_sdk/hatchet.py
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import logging
|
|
3
|
-
from typing import Any, Callable, Optional, Type, TypeVar,
|
|
3
|
+
from typing import Any, Callable, Optional, Type, TypeVar, Union
|
|
4
4
|
|
|
5
5
|
from pydantic import BaseModel
|
|
6
6
|
from typing_extensions import deprecated
|
|
7
7
|
|
|
8
8
|
from hatchet_sdk.clients.rest_client import RestApi
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
## There are file-level type ignore lines in the corresponding .pyi files.
|
|
12
|
-
from hatchet_sdk.contracts.workflows_pb2 import ( # type: ignore[attr-defined]
|
|
9
|
+
from hatchet_sdk.context.context import Context
|
|
10
|
+
from hatchet_sdk.contracts.workflows_pb2 import (
|
|
13
11
|
ConcurrencyLimitStrategy,
|
|
14
12
|
CreateStepRateLimit,
|
|
15
13
|
DesiredWorkerLabels,
|
|
@@ -20,6 +18,7 @@ from hatchet_sdk.features.scheduled import ScheduledClient
|
|
|
20
18
|
from hatchet_sdk.labels import DesiredWorkerLabel
|
|
21
19
|
from hatchet_sdk.loader import ClientConfig, ConfigLoader
|
|
22
20
|
from hatchet_sdk.rate_limit import RateLimit
|
|
21
|
+
from hatchet_sdk.v2.callable import HatchetCallable
|
|
23
22
|
|
|
24
23
|
from .client import Client, new_client, new_client_raw
|
|
25
24
|
from .clients.admin import AdminClient
|
|
@@ -36,6 +35,7 @@ from .workflow import (
|
|
|
36
35
|
)
|
|
37
36
|
|
|
38
37
|
T = TypeVar("T", bound=BaseModel)
|
|
38
|
+
TWorkflow = TypeVar("TWorkflow", bound=object)
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
def workflow(
|
|
@@ -45,30 +45,35 @@ def workflow(
|
|
|
45
45
|
version: str = "",
|
|
46
46
|
timeout: str = "60m",
|
|
47
47
|
schedule_timeout: str = "5m",
|
|
48
|
-
sticky: StickyStrategy = None,
|
|
48
|
+
sticky: Union[StickyStrategy.Value, None] = None, # type: ignore[name-defined]
|
|
49
49
|
default_priority: int | None = None,
|
|
50
50
|
concurrency: ConcurrencyExpression | None = None,
|
|
51
51
|
input_validator: Type[T] | None = None,
|
|
52
|
-
) -> Callable[[Type[
|
|
52
|
+
) -> Callable[[Type[TWorkflow]], WorkflowMeta]:
|
|
53
53
|
on_events = on_events or []
|
|
54
54
|
on_crons = on_crons or []
|
|
55
55
|
|
|
56
|
-
def inner(cls: Type[
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
cls
|
|
61
|
-
cls
|
|
62
|
-
cls
|
|
63
|
-
cls
|
|
64
|
-
cls
|
|
65
|
-
cls
|
|
56
|
+
def inner(cls: Type[TWorkflow]) -> WorkflowMeta:
|
|
57
|
+
nonlocal name
|
|
58
|
+
name = name or str(cls.__name__)
|
|
59
|
+
|
|
60
|
+
setattr(cls, "on_events", on_events)
|
|
61
|
+
setattr(cls, "on_crons", on_crons)
|
|
62
|
+
setattr(cls, "name", name)
|
|
63
|
+
setattr(cls, "version", version)
|
|
64
|
+
setattr(cls, "timeout", timeout)
|
|
65
|
+
setattr(cls, "schedule_timeout", schedule_timeout)
|
|
66
|
+
setattr(cls, "sticky", sticky)
|
|
67
|
+
setattr(cls, "default_priority", default_priority)
|
|
68
|
+
setattr(cls, "concurrency_expression", concurrency)
|
|
69
|
+
|
|
66
70
|
# Define a new class with the same name and bases as the original, but
|
|
67
71
|
# with WorkflowMeta as its metaclass
|
|
68
72
|
|
|
69
73
|
## TODO: Figure out how to type this metaclass correctly
|
|
70
|
-
cls
|
|
71
|
-
|
|
74
|
+
setattr(cls, "input_validator", input_validator)
|
|
75
|
+
|
|
76
|
+
return WorkflowMeta(name, cls.__bases__, dict(cls.__dict__))
|
|
72
77
|
|
|
73
78
|
return inner
|
|
74
79
|
|
|
@@ -82,34 +87,39 @@ def step(
|
|
|
82
87
|
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
83
88
|
backoff_factor: float | None = None,
|
|
84
89
|
backoff_max_seconds: int | None = None,
|
|
85
|
-
) -> Callable[
|
|
90
|
+
) -> Callable[..., Any]:
|
|
86
91
|
parents = parents or []
|
|
87
92
|
|
|
88
|
-
def inner(func:
|
|
93
|
+
def inner(func: Callable[[Context], Any]) -> Callable[[Context], Any]:
|
|
89
94
|
limits = None
|
|
90
95
|
if rate_limits:
|
|
91
96
|
limits = [rate_limit._req for rate_limit in rate_limits or []]
|
|
92
97
|
|
|
93
|
-
func
|
|
94
|
-
func
|
|
95
|
-
func
|
|
96
|
-
func
|
|
97
|
-
func
|
|
98
|
-
func
|
|
99
|
-
func
|
|
100
|
-
|
|
101
|
-
func._step_desired_worker_labels = {}
|
|
98
|
+
setattr(func, "_step_name", name.lower() or str(func.__name__).lower())
|
|
99
|
+
setattr(func, "_step_parents", parents)
|
|
100
|
+
setattr(func, "_step_timeout", timeout)
|
|
101
|
+
setattr(func, "_step_retries", retries)
|
|
102
|
+
setattr(func, "_step_rate_limits", retries)
|
|
103
|
+
setattr(func, "_step_rate_limits", limits)
|
|
104
|
+
setattr(func, "_step_backoff_factor", backoff_factor)
|
|
105
|
+
setattr(func, "_step_backoff_max_seconds", backoff_max_seconds)
|
|
102
106
|
|
|
103
|
-
|
|
107
|
+
def create_label(d: DesiredWorkerLabel) -> DesiredWorkerLabels:
|
|
104
108
|
value = d["value"] if "value" in d else None
|
|
105
|
-
|
|
109
|
+
return DesiredWorkerLabels(
|
|
106
110
|
strValue=str(value) if not isinstance(value, int) else None,
|
|
107
111
|
intValue=value if isinstance(value, int) else None,
|
|
108
|
-
required=d["required"] if "required" in d else None,
|
|
112
|
+
required=d["required"] if "required" in d else None, # type: ignore[arg-type]
|
|
109
113
|
weight=d["weight"] if "weight" in d else None,
|
|
110
|
-
comparator=d["comparator"] if "comparator" in d else None,
|
|
114
|
+
comparator=d["comparator"] if "comparator" in d else None, # type: ignore[arg-type]
|
|
111
115
|
)
|
|
112
116
|
|
|
117
|
+
setattr(
|
|
118
|
+
func,
|
|
119
|
+
"_step_desired_worker_labels",
|
|
120
|
+
{key: create_label(d) for key, d in desired_worker_labels.items()},
|
|
121
|
+
)
|
|
122
|
+
|
|
113
123
|
return func
|
|
114
124
|
|
|
115
125
|
return inner
|
|
@@ -122,21 +132,23 @@ def on_failure_step(
|
|
|
122
132
|
rate_limits: list[RateLimit] | None = None,
|
|
123
133
|
backoff_factor: float | None = None,
|
|
124
134
|
backoff_max_seconds: int | None = None,
|
|
125
|
-
) -> Callable[
|
|
126
|
-
def inner(func:
|
|
135
|
+
) -> Callable[..., Any]:
|
|
136
|
+
def inner(func: Callable[[Context], Any]) -> Callable[[Context], Any]:
|
|
127
137
|
limits = None
|
|
128
138
|
if rate_limits:
|
|
129
139
|
limits = [
|
|
130
|
-
CreateStepRateLimit(key=rate_limit.static_key, units=rate_limit.units)
|
|
140
|
+
CreateStepRateLimit(key=rate_limit.static_key, units=rate_limit.units) # type: ignore[arg-type]
|
|
131
141
|
for rate_limit in rate_limits or []
|
|
132
142
|
]
|
|
133
143
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
func
|
|
138
|
-
func
|
|
139
|
-
func
|
|
144
|
+
setattr(
|
|
145
|
+
func, "_on_failure_step_name", name.lower() or str(func.__name__).lower()
|
|
146
|
+
)
|
|
147
|
+
setattr(func, "_on_failure_step_timeout", timeout)
|
|
148
|
+
setattr(func, "_on_failure_step_retries", retries)
|
|
149
|
+
setattr(func, "_on_failure_step_rate_limits", limits)
|
|
150
|
+
setattr(func, "_on_failure_step_backoff_factor", backoff_factor)
|
|
151
|
+
setattr(func, "_on_failure_step_backoff_max_seconds", backoff_max_seconds)
|
|
140
152
|
|
|
141
153
|
return func
|
|
142
154
|
|
|
@@ -147,11 +159,15 @@ def concurrency(
|
|
|
147
159
|
name: str = "",
|
|
148
160
|
max_runs: int = 1,
|
|
149
161
|
limit_strategy: ConcurrencyLimitStrategy = ConcurrencyLimitStrategy.CANCEL_IN_PROGRESS,
|
|
150
|
-
) -> Callable[
|
|
151
|
-
def inner(func:
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
162
|
+
) -> Callable[..., Any]:
|
|
163
|
+
def inner(func: Callable[[Context], Any]) -> Callable[[Context], Any]:
|
|
164
|
+
setattr(
|
|
165
|
+
func,
|
|
166
|
+
"_concurrency_fn_name",
|
|
167
|
+
name.lower() or str(func.__name__).lower(),
|
|
168
|
+
)
|
|
169
|
+
setattr(func, "_concurrency_max_runs", max_runs)
|
|
170
|
+
setattr(func, "_concurrency_limit_strategy", limit_strategy)
|
|
155
171
|
|
|
156
172
|
return func
|
|
157
173
|
|
hatchet_sdk/labels.py
CHANGED
hatchet_sdk/utils/aio_utils.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import inspect
|
|
3
|
+
from concurrent.futures import Executor
|
|
3
4
|
from functools import partial, wraps
|
|
4
5
|
from threading import Thread
|
|
6
|
+
from typing import Any
|
|
5
7
|
|
|
6
8
|
|
|
7
|
-
|
|
9
|
+
## TODO: Stricter typing here
|
|
10
|
+
def sync_to_async(func: Any) -> Any:
|
|
8
11
|
"""
|
|
9
12
|
A decorator to run a synchronous function or coroutine in an asynchronous context with added
|
|
10
13
|
asyncio loop safety.
|
|
@@ -40,8 +43,14 @@ def sync_to_async(func):
|
|
|
40
43
|
asyncio.run(main())
|
|
41
44
|
"""
|
|
42
45
|
|
|
46
|
+
## TODO: Stricter typing here
|
|
43
47
|
@wraps(func)
|
|
44
|
-
async def run(
|
|
48
|
+
async def run(
|
|
49
|
+
*args: Any,
|
|
50
|
+
loop: asyncio.AbstractEventLoop | None = None,
|
|
51
|
+
executor: Executor | None = None,
|
|
52
|
+
**kwargs: Any
|
|
53
|
+
) -> Any:
|
|
45
54
|
"""
|
|
46
55
|
The asynchronous wrapper function that runs the given function in an executor.
|
|
47
56
|
|
|
@@ -59,7 +68,7 @@ def sync_to_async(func):
|
|
|
59
68
|
|
|
60
69
|
if inspect.iscoroutinefunction(func):
|
|
61
70
|
# Wrap the coroutine to run it in an executor
|
|
62
|
-
async def wrapper():
|
|
71
|
+
async def wrapper() -> Any:
|
|
63
72
|
return await func(*args, **kwargs)
|
|
64
73
|
|
|
65
74
|
pfunc = partial(asyncio.run, wrapper())
|
|
@@ -75,7 +84,7 @@ def sync_to_async(func):
|
|
|
75
84
|
class EventLoopThread:
|
|
76
85
|
"""A class that manages an asyncio event loop running in a separate thread."""
|
|
77
86
|
|
|
78
|
-
def __init__(self):
|
|
87
|
+
def __init__(self) -> None:
|
|
79
88
|
"""
|
|
80
89
|
Initializes the EventLoopThread by creating an event loop
|
|
81
90
|
and setting up a thread to run the loop.
|
hatchet_sdk/utils/typing.py
CHANGED
hatchet_sdk/v2/callable.py
CHANGED
|
@@ -1,8 +1,19 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
from typing import
|
|
2
|
+
from typing import (
|
|
3
|
+
Any,
|
|
4
|
+
Callable,
|
|
5
|
+
Dict,
|
|
6
|
+
Generic,
|
|
7
|
+
List,
|
|
8
|
+
Optional,
|
|
9
|
+
TypedDict,
|
|
10
|
+
TypeVar,
|
|
11
|
+
Union,
|
|
12
|
+
)
|
|
3
13
|
|
|
4
|
-
from hatchet_sdk.
|
|
5
|
-
from hatchet_sdk.
|
|
14
|
+
from hatchet_sdk.clients.admin import ChildTriggerWorkflowOptions
|
|
15
|
+
from hatchet_sdk.context.context import Context
|
|
16
|
+
from hatchet_sdk.contracts.workflows_pb2 import ( # type: ignore[attr-defined]
|
|
6
17
|
CreateStepRateLimit,
|
|
7
18
|
CreateWorkflowJobOpts,
|
|
8
19
|
CreateWorkflowStepOpts,
|
|
@@ -28,8 +39,8 @@ class HatchetCallable(Generic[T]):
|
|
|
28
39
|
durable: bool = False,
|
|
29
40
|
name: str = "",
|
|
30
41
|
auto_register: bool = True,
|
|
31
|
-
on_events: list | None = None,
|
|
32
|
-
on_crons: list | None = None,
|
|
42
|
+
on_events: list[str] | None = None,
|
|
43
|
+
on_crons: list[str] | None = None,
|
|
33
44
|
version: str = "",
|
|
34
45
|
timeout: str = "60m",
|
|
35
46
|
schedule_timeout: str = "5m",
|
|
@@ -37,8 +48,8 @@ class HatchetCallable(Generic[T]):
|
|
|
37
48
|
retries: int = 0,
|
|
38
49
|
rate_limits: List[RateLimit] | None = None,
|
|
39
50
|
concurrency: ConcurrencyFunction | None = None,
|
|
40
|
-
on_failure:
|
|
41
|
-
desired_worker_labels: dict[str
|
|
51
|
+
on_failure: Union["HatchetCallable[T]", None] = None,
|
|
52
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
42
53
|
default_priority: int | None = None,
|
|
43
54
|
):
|
|
44
55
|
self.func = func
|
|
@@ -85,7 +96,7 @@ class HatchetCallable(Generic[T]):
|
|
|
85
96
|
def __call__(self, context: Context) -> T:
|
|
86
97
|
return self.func(context)
|
|
87
98
|
|
|
88
|
-
def with_namespace(self, namespace: str):
|
|
99
|
+
def with_namespace(self, namespace: str) -> None:
|
|
89
100
|
if namespace is not None and namespace != "":
|
|
90
101
|
self.function_namespace = namespace
|
|
91
102
|
self.function_name = namespace + self.function_name
|
|
@@ -161,21 +172,13 @@ class HatchetCallable(Generic[T]):
|
|
|
161
172
|
return self.function_namespace + ":" + self.function_name
|
|
162
173
|
|
|
163
174
|
|
|
164
|
-
T = TypeVar("T")
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
class TriggerOptions(TypedDict):
|
|
168
|
-
additional_metadata: Dict[str, str] | None = None
|
|
169
|
-
sticky: bool | None = None
|
|
170
|
-
|
|
171
|
-
|
|
172
175
|
class DurableContext(Context):
|
|
173
176
|
def run(
|
|
174
177
|
self,
|
|
175
|
-
function:
|
|
176
|
-
input: dict = {},
|
|
177
|
-
key: str = None,
|
|
178
|
-
options:
|
|
178
|
+
function: str | Callable[[Context], Any],
|
|
179
|
+
input: dict[Any, Any] = {},
|
|
180
|
+
key: str | None = None,
|
|
181
|
+
options: ChildTriggerWorkflowOptions | None = None,
|
|
179
182
|
) -> "RunRef[T]":
|
|
180
183
|
worker_id = self.worker.id()
|
|
181
184
|
|
hatchet_sdk/v2/concurrency.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
from typing import Callable
|
|
1
|
+
from typing import Any, Callable
|
|
2
2
|
|
|
3
|
-
from hatchet_sdk.context import Context
|
|
4
|
-
from hatchet_sdk.contracts.workflows_pb2 import
|
|
3
|
+
from hatchet_sdk.context.context import Context
|
|
4
|
+
from hatchet_sdk.contracts.workflows_pb2 import ( # type: ignore[attr-defined]
|
|
5
|
+
ConcurrencyLimitStrategy,
|
|
6
|
+
)
|
|
5
7
|
|
|
6
8
|
|
|
7
9
|
class ConcurrencyFunction:
|
|
@@ -18,19 +20,19 @@ class ConcurrencyFunction:
|
|
|
18
20
|
self.limit_strategy = limit_strategy
|
|
19
21
|
self.namespace = "default"
|
|
20
22
|
|
|
21
|
-
def set_namespace(self, namespace: str):
|
|
23
|
+
def set_namespace(self, namespace: str) -> None:
|
|
22
24
|
self.namespace = namespace
|
|
23
25
|
|
|
24
26
|
def get_action_name(self) -> str:
|
|
25
27
|
return self.namespace + ":" + self.name
|
|
26
28
|
|
|
27
|
-
def __call__(self, *args, **kwargs):
|
|
29
|
+
def __call__(self, *args: Any, **kwargs: Any) -> str:
|
|
28
30
|
return self.func(*args, **kwargs)
|
|
29
31
|
|
|
30
|
-
def __str__(self):
|
|
32
|
+
def __str__(self) -> str:
|
|
31
33
|
return f"{self.name}({self.max_runs})"
|
|
32
34
|
|
|
33
|
-
def __repr__(self):
|
|
35
|
+
def __repr__(self) -> str:
|
|
34
36
|
return f"{self.name}({self.max_runs})"
|
|
35
37
|
|
|
36
38
|
|
|
@@ -38,7 +40,7 @@ def concurrency(
|
|
|
38
40
|
name: str = "",
|
|
39
41
|
max_runs: int = 1,
|
|
40
42
|
limit_strategy: ConcurrencyLimitStrategy = ConcurrencyLimitStrategy.GROUP_ROUND_ROBIN,
|
|
41
|
-
):
|
|
43
|
+
) -> Callable[[Callable[[Context], str]], ConcurrencyFunction]:
|
|
42
44
|
def inner(func: Callable[[Context], str]) -> ConcurrencyFunction:
|
|
43
45
|
return ConcurrencyFunction(func, name, max_runs, limit_strategy)
|
|
44
46
|
|
hatchet_sdk/v2/hatchet.py
CHANGED
|
@@ -1,36 +1,38 @@
|
|
|
1
|
-
from typing import
|
|
2
|
-
|
|
3
|
-
from hatchet_sdk
|
|
4
|
-
from hatchet_sdk.
|
|
1
|
+
from typing import Any, Callable, TypeVar, Union
|
|
2
|
+
|
|
3
|
+
from hatchet_sdk import Worker
|
|
4
|
+
from hatchet_sdk.context.context import Context
|
|
5
|
+
from hatchet_sdk.contracts.workflows_pb2 import ( # type: ignore[attr-defined]
|
|
6
|
+
ConcurrencyLimitStrategy,
|
|
7
|
+
StickyStrategy,
|
|
8
|
+
)
|
|
5
9
|
from hatchet_sdk.hatchet import Hatchet as HatchetV1
|
|
6
10
|
from hatchet_sdk.hatchet import workflow
|
|
7
11
|
from hatchet_sdk.labels import DesiredWorkerLabel
|
|
8
12
|
from hatchet_sdk.rate_limit import RateLimit
|
|
9
|
-
from hatchet_sdk.v2.callable import HatchetCallable
|
|
13
|
+
from hatchet_sdk.v2.callable import DurableContext, HatchetCallable
|
|
10
14
|
from hatchet_sdk.v2.concurrency import ConcurrencyFunction
|
|
11
15
|
from hatchet_sdk.worker.worker import register_on_worker
|
|
12
16
|
|
|
13
|
-
from ..worker import Worker
|
|
14
|
-
|
|
15
17
|
T = TypeVar("T")
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
def function(
|
|
19
21
|
name: str = "",
|
|
20
22
|
auto_register: bool = True,
|
|
21
|
-
on_events: list | None = None,
|
|
22
|
-
on_crons: list | None = None,
|
|
23
|
+
on_events: list[str] | None = None,
|
|
24
|
+
on_crons: list[str] | None = None,
|
|
23
25
|
version: str = "",
|
|
24
26
|
timeout: str = "60m",
|
|
25
27
|
schedule_timeout: str = "5m",
|
|
26
28
|
sticky: StickyStrategy = None,
|
|
27
29
|
retries: int = 0,
|
|
28
|
-
rate_limits:
|
|
29
|
-
desired_worker_labels: dict[str
|
|
30
|
+
rate_limits: list[RateLimit] | None = None,
|
|
31
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
30
32
|
concurrency: ConcurrencyFunction | None = None,
|
|
31
|
-
on_failure:
|
|
33
|
+
on_failure: Union["HatchetCallable[T]", None] = None,
|
|
32
34
|
default_priority: int | None = None,
|
|
33
|
-
):
|
|
35
|
+
) -> Callable[[Callable[[Context], str]], HatchetCallable[T]]:
|
|
34
36
|
def inner(func: Callable[[Context], T]) -> HatchetCallable[T]:
|
|
35
37
|
return HatchetCallable(
|
|
36
38
|
func=func,
|
|
@@ -56,20 +58,20 @@ def function(
|
|
|
56
58
|
def durable(
|
|
57
59
|
name: str = "",
|
|
58
60
|
auto_register: bool = True,
|
|
59
|
-
on_events: list | None = None,
|
|
60
|
-
on_crons: list | None = None,
|
|
61
|
+
on_events: list[str] | None = None,
|
|
62
|
+
on_crons: list[str] | None = None,
|
|
61
63
|
version: str = "",
|
|
62
64
|
timeout: str = "60m",
|
|
63
65
|
schedule_timeout: str = "5m",
|
|
64
66
|
sticky: StickyStrategy = None,
|
|
65
67
|
retries: int = 0,
|
|
66
|
-
rate_limits:
|
|
67
|
-
desired_worker_labels: dict[str
|
|
68
|
+
rate_limits: list[RateLimit] | None = None,
|
|
69
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
68
70
|
concurrency: ConcurrencyFunction | None = None,
|
|
69
|
-
on_failure: HatchetCallable | None = None,
|
|
71
|
+
on_failure: HatchetCallable[T] | None = None,
|
|
70
72
|
default_priority: int | None = None,
|
|
71
|
-
):
|
|
72
|
-
def inner(func: HatchetCallable) -> HatchetCallable:
|
|
73
|
+
) -> Callable[[HatchetCallable[T]], HatchetCallable[T]]:
|
|
74
|
+
def inner(func: HatchetCallable[T]) -> HatchetCallable[T]:
|
|
73
75
|
func.durable = True
|
|
74
76
|
|
|
75
77
|
f = function(
|
|
@@ -102,7 +104,7 @@ def concurrency(
|
|
|
102
104
|
name: str = "concurrency",
|
|
103
105
|
max_runs: int = 1,
|
|
104
106
|
limit_strategy: ConcurrencyLimitStrategy = ConcurrencyLimitStrategy.GROUP_ROUND_ROBIN,
|
|
105
|
-
):
|
|
107
|
+
) -> Callable[[Callable[[Context], str]], ConcurrencyFunction]:
|
|
106
108
|
def inner(func: Callable[[Context], str]) -> ConcurrencyFunction:
|
|
107
109
|
return ConcurrencyFunction(func, name, max_runs, limit_strategy)
|
|
108
110
|
|
|
@@ -113,24 +115,24 @@ class Hatchet(HatchetV1):
|
|
|
113
115
|
dag = staticmethod(workflow)
|
|
114
116
|
concurrency = staticmethod(concurrency)
|
|
115
117
|
|
|
116
|
-
functions:
|
|
118
|
+
functions: list[HatchetCallable[T]] = []
|
|
117
119
|
|
|
118
120
|
def function(
|
|
119
121
|
self,
|
|
120
122
|
name: str = "",
|
|
121
123
|
auto_register: bool = True,
|
|
122
|
-
on_events: list | None = None,
|
|
123
|
-
on_crons: list | None = None,
|
|
124
|
+
on_events: list[str] | None = None,
|
|
125
|
+
on_crons: list[str] | None = None,
|
|
124
126
|
version: str = "",
|
|
125
127
|
timeout: str = "60m",
|
|
126
128
|
schedule_timeout: str = "5m",
|
|
127
129
|
retries: int = 0,
|
|
128
|
-
rate_limits:
|
|
129
|
-
desired_worker_labels: dict[str
|
|
130
|
+
rate_limits: list[RateLimit] | None = None,
|
|
131
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
130
132
|
concurrency: ConcurrencyFunction | None = None,
|
|
131
|
-
on_failure:
|
|
133
|
+
on_failure: Union["HatchetCallable[T]", None] = None,
|
|
132
134
|
default_priority: int | None = None,
|
|
133
|
-
):
|
|
135
|
+
) -> Callable[[Callable[[Context], Any]], Callable[[Context], Any]]:
|
|
134
136
|
resp = function(
|
|
135
137
|
name=name,
|
|
136
138
|
auto_register=auto_register,
|
|
@@ -147,7 +149,7 @@ class Hatchet(HatchetV1):
|
|
|
147
149
|
default_priority=default_priority,
|
|
148
150
|
)
|
|
149
151
|
|
|
150
|
-
def wrapper(func: Callable[[Context],
|
|
152
|
+
def wrapper(func: Callable[[Context], str]) -> HatchetCallable[T]:
|
|
151
153
|
wrapped_resp = resp(func)
|
|
152
154
|
|
|
153
155
|
if wrapped_resp.function_auto_register:
|
|
@@ -163,19 +165,19 @@ class Hatchet(HatchetV1):
|
|
|
163
165
|
self,
|
|
164
166
|
name: str = "",
|
|
165
167
|
auto_register: bool = True,
|
|
166
|
-
on_events: list | None = None,
|
|
167
|
-
on_crons: list | None = None,
|
|
168
|
+
on_events: list[str] | None = None,
|
|
169
|
+
on_crons: list[str] | None = None,
|
|
168
170
|
version: str = "",
|
|
169
171
|
timeout: str = "60m",
|
|
170
172
|
schedule_timeout: str = "5m",
|
|
171
173
|
sticky: StickyStrategy = None,
|
|
172
174
|
retries: int = 0,
|
|
173
|
-
rate_limits:
|
|
174
|
-
desired_worker_labels: dict[str
|
|
175
|
+
rate_limits: list[RateLimit] | None = None,
|
|
176
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
175
177
|
concurrency: ConcurrencyFunction | None = None,
|
|
176
|
-
on_failure:
|
|
178
|
+
on_failure: Union["HatchetCallable[T]", None] = None,
|
|
177
179
|
default_priority: int | None = None,
|
|
178
|
-
) -> Callable[[
|
|
180
|
+
) -> Callable[[Callable[[DurableContext], Any]], Callable[[DurableContext], Any]]:
|
|
179
181
|
resp = durable(
|
|
180
182
|
name=name,
|
|
181
183
|
auto_register=auto_register,
|
|
@@ -193,7 +195,7 @@ class Hatchet(HatchetV1):
|
|
|
193
195
|
default_priority=default_priority,
|
|
194
196
|
)
|
|
195
197
|
|
|
196
|
-
def wrapper(func:
|
|
198
|
+
def wrapper(func: HatchetCallable[T]) -> HatchetCallable[T]:
|
|
197
199
|
wrapped_resp = resp(func)
|
|
198
200
|
|
|
199
201
|
if wrapped_resp.function_auto_register:
|
|
@@ -21,7 +21,7 @@ from hatchet_sdk.clients.run_event_listener import new_listener
|
|
|
21
21
|
from hatchet_sdk.clients.workflow_listener import PooledWorkflowRunListener
|
|
22
22
|
from hatchet_sdk.context import Context # type: ignore[attr-defined]
|
|
23
23
|
from hatchet_sdk.context.worker_context import WorkerContext
|
|
24
|
-
from hatchet_sdk.contracts.dispatcher_pb2 import (
|
|
24
|
+
from hatchet_sdk.contracts.dispatcher_pb2 import (
|
|
25
25
|
GROUP_KEY_EVENT_TYPE_COMPLETED,
|
|
26
26
|
GROUP_KEY_EVENT_TYPE_FAILED,
|
|
27
27
|
GROUP_KEY_EVENT_TYPE_STARTED,
|
hatchet_sdk/worker/worker.py
CHANGED
|
@@ -12,14 +12,15 @@ from multiprocessing.process import BaseProcess
|
|
|
12
12
|
from types import FrameType
|
|
13
13
|
from typing import Any, Callable, TypeVar, get_type_hints
|
|
14
14
|
|
|
15
|
+
from pydantic import BaseModel
|
|
16
|
+
|
|
15
17
|
from hatchet_sdk import Context
|
|
16
18
|
from hatchet_sdk.client import Client, new_client_raw
|
|
17
|
-
from hatchet_sdk.contracts.workflows_pb2 import
|
|
18
|
-
CreateWorkflowVersionOpts,
|
|
19
|
-
)
|
|
19
|
+
from hatchet_sdk.contracts.workflows_pb2 import CreateWorkflowVersionOpts
|
|
20
20
|
from hatchet_sdk.loader import ClientConfig
|
|
21
21
|
from hatchet_sdk.logger import logger
|
|
22
22
|
from hatchet_sdk.utils.types import WorkflowValidator
|
|
23
|
+
from hatchet_sdk.utils.typing import is_basemodel_subclass
|
|
23
24
|
from hatchet_sdk.v2.callable import HatchetCallable
|
|
24
25
|
from hatchet_sdk.v2.concurrency import ConcurrencyFunction
|
|
25
26
|
from hatchet_sdk.worker.action_listener_process import worker_action_listener_process
|
|
@@ -41,6 +42,9 @@ class WorkerStartOptions:
|
|
|
41
42
|
loop: asyncio.AbstractEventLoop | None = field(default=None)
|
|
42
43
|
|
|
43
44
|
|
|
45
|
+
TWorkflow = TypeVar("TWorkflow", bound=object)
|
|
46
|
+
|
|
47
|
+
|
|
44
48
|
class Worker:
|
|
45
49
|
def __init__(
|
|
46
50
|
self,
|
|
@@ -62,7 +66,7 @@ class Worker:
|
|
|
62
66
|
|
|
63
67
|
self.client: Client
|
|
64
68
|
|
|
65
|
-
self.action_registry: dict[str, Callable[[Context],
|
|
69
|
+
self.action_registry: dict[str, Callable[[Context], Any]] = {}
|
|
66
70
|
self.validator_registry: dict[str, WorkflowValidator] = {}
|
|
67
71
|
|
|
68
72
|
self.killing: bool = False
|
|
@@ -84,9 +88,7 @@ class Worker:
|
|
|
84
88
|
|
|
85
89
|
self._setup_signal_handlers()
|
|
86
90
|
|
|
87
|
-
def register_function(
|
|
88
|
-
self, action: str, func: HatchetCallable[Any] | ConcurrencyFunction
|
|
89
|
-
) -> None:
|
|
91
|
+
def register_function(self, action: str, func: Callable[[Context], Any]) -> None:
|
|
90
92
|
self.action_registry[action] = func
|
|
91
93
|
|
|
92
94
|
def register_workflow_from_opts(
|
|
@@ -99,7 +101,10 @@ class Worker:
|
|
|
99
101
|
logger.error(e)
|
|
100
102
|
sys.exit(1)
|
|
101
103
|
|
|
102
|
-
def register_workflow(self, workflow:
|
|
104
|
+
def register_workflow(self, workflow: TWorkflow) -> None:
|
|
105
|
+
## Hack for typing
|
|
106
|
+
assert isinstance(workflow, WorkflowInterface)
|
|
107
|
+
|
|
103
108
|
namespace = self.client.config.namespace
|
|
104
109
|
|
|
105
110
|
try:
|
|
@@ -127,8 +132,10 @@ class Worker:
|
|
|
127
132
|
for action_name, action_func in workflow.get_actions(namespace):
|
|
128
133
|
self.action_registry[action_name] = create_action_function(action_func)
|
|
129
134
|
return_type = get_type_hints(action_func).get("return")
|
|
135
|
+
|
|
130
136
|
self.validator_registry[action_name] = WorkflowValidator(
|
|
131
|
-
workflow_input=workflow.input_validator,
|
|
137
|
+
workflow_input=workflow.input_validator,
|
|
138
|
+
step_output=return_type if is_basemodel_subclass(return_type) else None,
|
|
132
139
|
)
|
|
133
140
|
|
|
134
141
|
def status(self) -> WorkerStatus:
|
hatchet_sdk/workflow.py
CHANGED
|
@@ -1,10 +1,20 @@
|
|
|
1
1
|
import functools
|
|
2
|
-
from typing import
|
|
2
|
+
from typing import (
|
|
3
|
+
Any,
|
|
4
|
+
Callable,
|
|
5
|
+
Protocol,
|
|
6
|
+
Type,
|
|
7
|
+
TypeVar,
|
|
8
|
+
Union,
|
|
9
|
+
cast,
|
|
10
|
+
get_type_hints,
|
|
11
|
+
runtime_checkable,
|
|
12
|
+
)
|
|
3
13
|
|
|
4
14
|
from pydantic import BaseModel
|
|
5
15
|
|
|
6
16
|
from hatchet_sdk import ConcurrencyLimitStrategy
|
|
7
|
-
from hatchet_sdk.contracts.workflows_pb2 import (
|
|
17
|
+
from hatchet_sdk.contracts.workflows_pb2 import (
|
|
8
18
|
CreateWorkflowJobOpts,
|
|
9
19
|
CreateWorkflowStepOpts,
|
|
10
20
|
CreateWorkflowVersionOpts,
|
|
@@ -69,6 +79,7 @@ class ConcurrencyExpression:
|
|
|
69
79
|
self.limit_strategy = limit_strategy
|
|
70
80
|
|
|
71
81
|
|
|
82
|
+
@runtime_checkable
|
|
72
83
|
class WorkflowInterface(Protocol):
|
|
73
84
|
def get_name(self, namespace: str) -> str: ...
|
|
74
85
|
|
|
@@ -76,13 +87,13 @@ class WorkflowInterface(Protocol):
|
|
|
76
87
|
|
|
77
88
|
def get_create_opts(self, namespace: str) -> Any: ...
|
|
78
89
|
|
|
79
|
-
on_events: list[str]
|
|
80
|
-
on_crons: list[str]
|
|
90
|
+
on_events: list[str] | None
|
|
91
|
+
on_crons: list[str] | None
|
|
81
92
|
name: str
|
|
82
93
|
version: str
|
|
83
94
|
timeout: str
|
|
84
95
|
schedule_timeout: str
|
|
85
|
-
sticky: Union[StickyStrategy.Value, None]
|
|
96
|
+
sticky: Union[StickyStrategy.Value, None] # type: ignore[name-defined]
|
|
86
97
|
default_priority: int | None
|
|
87
98
|
concurrency_expression: ConcurrencyExpression | None
|
|
88
99
|
input_validator: Type[BaseModel] | None
|
|
@@ -120,6 +131,7 @@ class WorkflowMeta(type):
|
|
|
120
131
|
@functools.cache
|
|
121
132
|
def get_actions(self: TW, namespace: str) -> StepsType:
|
|
122
133
|
serviceName = get_service_name(namespace)
|
|
134
|
+
|
|
123
135
|
func_actions = [
|
|
124
136
|
(serviceName + ":" + func_name, func) for func_name, func in steps
|
|
125
137
|
]
|
|
@@ -165,8 +177,8 @@ class WorkflowMeta(type):
|
|
|
165
177
|
inputs="{}",
|
|
166
178
|
parents=[x for x in func._step_parents],
|
|
167
179
|
retries=func._step_retries,
|
|
168
|
-
rate_limits=func._step_rate_limits,
|
|
169
|
-
worker_labels=func._step_desired_worker_labels,
|
|
180
|
+
rate_limits=func._step_rate_limits, # type: ignore[arg-type]
|
|
181
|
+
worker_labels=func._step_desired_worker_labels, # type: ignore[arg-type]
|
|
170
182
|
backoff_factor=func._step_backoff_factor,
|
|
171
183
|
backoff_max_seconds=func._step_backoff_max_seconds,
|
|
172
184
|
)
|
|
@@ -196,7 +208,7 @@ class WorkflowMeta(type):
|
|
|
196
208
|
"Error: Both concurrencyActions and concurrency_expression are defined. Please use only one concurrency configuration method."
|
|
197
209
|
)
|
|
198
210
|
|
|
199
|
-
on_failure_job:
|
|
211
|
+
on_failure_job: CreateWorkflowJobOpts | None = None
|
|
200
212
|
|
|
201
213
|
if len(onFailureSteps) > 0:
|
|
202
214
|
func_name, func = onFailureSteps[0]
|
|
@@ -210,7 +222,7 @@ class WorkflowMeta(type):
|
|
|
210
222
|
inputs="{}",
|
|
211
223
|
parents=[],
|
|
212
224
|
retries=func._on_failure_step_retries,
|
|
213
|
-
rate_limits=func._on_failure_step_rate_limits,
|
|
225
|
+
rate_limits=func._on_failure_step_rate_limits, # type: ignore[arg-type]
|
|
214
226
|
backoff_factor=func._on_failure_step_backoff_factor,
|
|
215
227
|
backoff_max_seconds=func._on_failure_step_backoff_max_seconds,
|
|
216
228
|
)
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
hatchet_sdk/__init__.py,sha256=R5ogd_Dn_6gA_u9a5W2URNq6eDtN1i56cObBv1tOwyU,9408
|
|
2
2
|
hatchet_sdk/client.py,sha256=ajjLd-gZptVuAx25gG_SdAW8xDA4V7HMIhgYuh9MkVY,3486
|
|
3
|
-
hatchet_sdk/clients/admin.py,sha256=
|
|
3
|
+
hatchet_sdk/clients/admin.py,sha256=RGi-cXGTn54EmgiqbUsnb9a9gbA3WFr9Qo2VdBHdSzQ,21530
|
|
4
4
|
hatchet_sdk/clients/dispatcher/action_listener.py,sha256=eOq5z29MhC7ynbOOegDRQr-RqDhpS3gfeLswIlZbuGg,15378
|
|
5
|
-
hatchet_sdk/clients/dispatcher/dispatcher.py,sha256=
|
|
5
|
+
hatchet_sdk/clients/dispatcher/dispatcher.py,sha256=qTcfIXn9EpfUlafff8Wun_fWTvz-UxOa4i4m6ZVIAnM,6402
|
|
6
6
|
hatchet_sdk/clients/event_ts.py,sha256=ACGvDdfhvK6ZLKdsPxy-PksLhjIU69P9cdH3AxX-X10,728
|
|
7
|
-
hatchet_sdk/clients/events.py,sha256=
|
|
7
|
+
hatchet_sdk/clients/events.py,sha256=8QOHv4ObSNXb6t5EnaXPwkwr1hN4D087iWn02TRtXb0,7505
|
|
8
8
|
hatchet_sdk/clients/rest/__init__.py,sha256=AQBp3fX79IKrcjUmzLRGFaFKMYBnEbQD9DnwYOHU0jQ,14157
|
|
9
9
|
hatchet_sdk/clients/rest/api/__init__.py,sha256=LaTEK7cYklb8R0iYj727C-jVk-MGHenADN8TJpoIASs,1067
|
|
10
10
|
hatchet_sdk/clients/rest/api/api_token_api.py,sha256=C10FEIHHGBpwq-bIKkrBhvPlg6az4aHlREWEUlJHWl0,33577
|
|
@@ -160,9 +160,9 @@ hatchet_sdk/clients/rest/models/workflow.py,sha256=8eRB3hq4U3aipNhpdLZfnDPoBhU6n
|
|
|
160
160
|
hatchet_sdk/clients/rest/models/workflow_concurrency.py,sha256=RtyT3zGwQtd8upt3xWwqN9O_Frdcu9jQ0Hq9DWmygQ4,3199
|
|
161
161
|
hatchet_sdk/clients/rest/models/workflow_deployment_config.py,sha256=_fjNErg4HnlpHWtapDlKqhMpmUkAdWLQ7DLDMbYkSNE,4510
|
|
162
162
|
hatchet_sdk/clients/rest/models/workflow_kind.py,sha256=JOAMn9bJFWNeR9vKWIbTnrIpCP8d_UXRuqDCjdeqI30,684
|
|
163
|
-
hatchet_sdk/clients/rest/models/workflow_list.py,sha256=
|
|
163
|
+
hatchet_sdk/clients/rest/models/workflow_list.py,sha256=HnWvKZe0PuySA75gZMK7-o6PV4RjWerh_G0SKLAWSn8,3948
|
|
164
164
|
hatchet_sdk/clients/rest/models/workflow_metrics.py,sha256=mslUc2ChUk4hKna-kyflAzgdOcbGk5XqFv_hJ1ujMsI,2877
|
|
165
|
-
hatchet_sdk/clients/rest/models/workflow_run.py,sha256=
|
|
165
|
+
hatchet_sdk/clients/rest/models/workflow_run.py,sha256=nOo94uJw8esygeLTcJRx8TpJN1zq_3Qs-yf9kRTyKk0,6758
|
|
166
166
|
hatchet_sdk/clients/rest/models/workflow_run_cancel200_response.py,sha256=nS_T_uP_ZRwMz-y9DellHrO74hGiLkdlE9z21uH3mh4,2573
|
|
167
167
|
hatchet_sdk/clients/rest/models/workflow_run_list.py,sha256=ZDq7OSw3aLJCSFPPwhBWtGlMpH00B_pns9Ynqdw_QKc,3479
|
|
168
168
|
hatchet_sdk/clients/rest/models/workflow_run_order_by_direction.py,sha256=P-QYH8bwfZmswc1-VxLTmL_bbxiioDlPSQ3TP2350UQ,697
|
|
@@ -190,8 +190,8 @@ hatchet_sdk/clients/run_event_listener.py,sha256=51WTg52_aISgYPOFPHJ21rb4IO6aEd7
|
|
|
190
190
|
hatchet_sdk/clients/workflow_listener.py,sha256=Q_WJcGlZNHJGSpxzDac9wELjgxhP0vLaNTXRy_xnxc8,9466
|
|
191
191
|
hatchet_sdk/connection.py,sha256=593aUGAj7Ouf00lcVwx_pmhdQ9NOC5ANT1Jrf8nwkHs,2165
|
|
192
192
|
hatchet_sdk/context/__init__.py,sha256=Pl_seJ_SJpW34BBZp4KixuZ8GiRK9sJFfegf9u3m7zk,29
|
|
193
|
-
hatchet_sdk/context/context.py,sha256=
|
|
194
|
-
hatchet_sdk/context/worker_context.py,sha256=
|
|
193
|
+
hatchet_sdk/context/context.py,sha256=R2RsPgzBNYi1FJvg4MOowSY-V1Yf69qUmXe3MsAWkyo,13518
|
|
194
|
+
hatchet_sdk/context/worker_context.py,sha256=OVcEWvdT_Kpd0nlg61VAPUgIPSFzSLs0aSrXWj-1GX4,974
|
|
195
195
|
hatchet_sdk/contracts/dispatcher_pb2.py,sha256=W9_Us2_HIdUV5idN2rPwzt4J06JfDeogxwDVvjerk_U,14320
|
|
196
196
|
hatchet_sdk/contracts/dispatcher_pb2.pyi,sha256=2dvxvN4OETOc-OhLNWDntN7GvI4QUxbMiGLdK7FM1wE,18224
|
|
197
197
|
hatchet_sdk/contracts/dispatcher_pb2_grpc.py,sha256=4uJkNig8nssJwS7P-lea1YqE4wl0cVWQaJpOndOdaDs,21453
|
|
@@ -203,33 +203,33 @@ hatchet_sdk/contracts/workflows_pb2.pyi,sha256=vq3r4C60MW7qx9735q1kAyGENgfbHZtKe
|
|
|
203
203
|
hatchet_sdk/contracts/workflows_pb2_grpc.py,sha256=rOC2yF2VqxyGsxHnr1XbKFFh8-eLGic2SXgfS1gU3xM,8864
|
|
204
204
|
hatchet_sdk/features/cron.py,sha256=4lKMH0MqiN8cHJk2jhF0Ueqs6z5ozwJzlOeSeaWqvO0,10217
|
|
205
205
|
hatchet_sdk/features/scheduled.py,sha256=YhEbNWl8dWOH61rXVjAyu8iG1BZqpSkD4kgaxkKIHgY,9504
|
|
206
|
-
hatchet_sdk/hatchet.py,sha256=
|
|
207
|
-
hatchet_sdk/labels.py,sha256=
|
|
206
|
+
hatchet_sdk/hatchet.py,sha256=UpCDv3q-Mrbwi7tFHS2FonBwwXlbs5kRFfCfx4DJVeI,10027
|
|
207
|
+
hatchet_sdk/labels.py,sha256=Axfp1yUNowzE9mL8AQA1ADqwOaNmq3QP_45wb1Ed1aI,272
|
|
208
208
|
hatchet_sdk/loader.py,sha256=dGs6Tt8wdEgHeCBesbbQsP6ZYiARBvoqwIdXIO1P5Os,7913
|
|
209
209
|
hatchet_sdk/logger.py,sha256=5uOr52T4mImSQm1QvWT8HvZFK5WfPNh3Y1cBQZRFgUQ,333
|
|
210
210
|
hatchet_sdk/metadata.py,sha256=M_Cb-CXRKitzVMQHeaHUtbY28ET__fAbyGX1YKaeN4I,80
|
|
211
211
|
hatchet_sdk/rate_limit.py,sha256=IIzpe65i-518t9kQcZVEykDQ2VY8sOw2F7qlQ4wlAjw,4421
|
|
212
212
|
hatchet_sdk/semver.py,sha256=PrgBL0TnyXl3p_OK1iSMk9Gpujfh5asQpJ4DHJLCW2k,998
|
|
213
213
|
hatchet_sdk/token.py,sha256=Ap3jnbaPAL10F2G_D71wj7OpBcvrI3RuE0keqXx1lAE,698
|
|
214
|
-
hatchet_sdk/utils/aio_utils.py,sha256=
|
|
214
|
+
hatchet_sdk/utils/aio_utils.py,sha256=D92Z1lc84u8f1mVE3XzOv9XtRKfCIvwfdfL-Bo5fJA8,4336
|
|
215
215
|
hatchet_sdk/utils/backoff.py,sha256=6B5Rb5nLKw_TqqgpJMYjIBV1PTTtbOMRZCveisVhg_I,353
|
|
216
216
|
hatchet_sdk/utils/serialization.py,sha256=P2Uq0yxg-Cea5Lmf6IOh2r7W17MNF1Hv2qxSny6BUk8,451
|
|
217
217
|
hatchet_sdk/utils/tracing.py,sha256=UjTw2vMLg1p6GjOGwt634nYJaNZh8c70PBxOBVLiDQ4,2261
|
|
218
218
|
hatchet_sdk/utils/types.py,sha256=qhx1OoeXh77AN6s4SMaGpB3zK3hPm7ocv-23JFa6_wE,191
|
|
219
|
-
hatchet_sdk/utils/typing.py,sha256=
|
|
220
|
-
hatchet_sdk/v2/callable.py,sha256=
|
|
221
|
-
hatchet_sdk/v2/concurrency.py,sha256=
|
|
222
|
-
hatchet_sdk/v2/hatchet.py,sha256=
|
|
219
|
+
hatchet_sdk/utils/typing.py,sha256=ab1FdF8OAXjcFq9Ui7Bd6yi8ZX473rIqFAqXJnh5AEM,261
|
|
220
|
+
hatchet_sdk/v2/callable.py,sha256=DqBc3VS7WAk5qU9Ef1HnyAzdgxUq1hDzj-CwQYQrvcA,6948
|
|
221
|
+
hatchet_sdk/v2/concurrency.py,sha256=mRsbCj3G2kwkAJjHOg67y226mYdXVmd0uka3ypdRKUQ,1434
|
|
222
|
+
hatchet_sdk/v2/hatchet.py,sha256=mQZoloWuRCIEwDwXeTaJL2kBjgs9YLHy7UPWxBD5070,7282
|
|
223
223
|
hatchet_sdk/worker/__init__.py,sha256=1Ze1seDuXx5yD1IfHmqGFgK5qrRazVW4ZcDVGl-Pddw,61
|
|
224
224
|
hatchet_sdk/worker/action_listener_process.py,sha256=tmlzDgyHWxGl8fJWE9NKqjvhqpGi9SMmOh5dFyiVL-Q,9979
|
|
225
225
|
hatchet_sdk/worker/runner/run_loop_manager.py,sha256=nV7fhNxJKCcrBm0ci118aszF_7AxenBkOTIe1UsBEt4,3490
|
|
226
|
-
hatchet_sdk/worker/runner/runner.py,sha256=
|
|
226
|
+
hatchet_sdk/worker/runner/runner.py,sha256=uNHPViiQvtwvX4uJ9TqcbFPonJnhhoEM3WNOWEKyop0,18517
|
|
227
227
|
hatchet_sdk/worker/runner/utils/capture_logs.py,sha256=s_BGxeykelVbusx6u31EPx3vv9c2BHkuBnYcaLW680E,2381
|
|
228
228
|
hatchet_sdk/worker/runner/utils/error_with_traceback.py,sha256=Iih_s8JNqrinXETFJ3ZS88EhaTekfM6m5fqIP7QWoIM,181
|
|
229
|
-
hatchet_sdk/worker/worker.py,sha256=
|
|
230
|
-
hatchet_sdk/workflow.py,sha256=
|
|
229
|
+
hatchet_sdk/worker/worker.py,sha256=R4UEf_1e7qe1LrXS8-0fjfriwcSb3W_38RUR6pOxBCo,11547
|
|
230
|
+
hatchet_sdk/workflow.py,sha256=XRj5jcCQSvPQMXxBipf-ZlARua2E8Z9igRzGcQ5alkI,9375
|
|
231
231
|
hatchet_sdk/workflow_run.py,sha256=BwK5cefvXXvyQ1Ednj_7LeejMwQJqWnvUC_FTBmJNxk,1805
|
|
232
|
-
hatchet_sdk-0.42.
|
|
233
|
-
hatchet_sdk-0.42.
|
|
234
|
-
hatchet_sdk-0.42.
|
|
235
|
-
hatchet_sdk-0.42.
|
|
232
|
+
hatchet_sdk-0.42.4.dist-info/METADATA,sha256=K_UnsT2BbAX5LRTC9afZNWJfA9wYMS9rk17zhZQ8Aqo,1514
|
|
233
|
+
hatchet_sdk-0.42.4.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
|
|
234
|
+
hatchet_sdk-0.42.4.dist-info/entry_points.txt,sha256=LTtQRABmSGYOxRI68cUVEz5dp9Qb57eqXGic9lU8RMo,1023
|
|
235
|
+
hatchet_sdk-0.42.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|