hatchet-sdk 0.42.2__py3-none-any.whl → 0.42.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hatchet-sdk might be problematic. Click here for more details.
- hatchet_sdk/clients/admin.py +6 -6
- hatchet_sdk/clients/dispatcher/dispatcher.py +22 -15
- hatchet_sdk/clients/events.py +3 -3
- hatchet_sdk/clients/rest/__init__.py +9 -3
- hatchet_sdk/clients/rest/api/event_api.py +6 -8
- hatchet_sdk/clients/rest/api/workflow_api.py +69 -0
- hatchet_sdk/clients/rest/models/__init__.py +9 -3
- hatchet_sdk/clients/rest/models/concurrency_limit_strategy.py +39 -0
- hatchet_sdk/clients/rest/models/cron_workflows.py +3 -16
- hatchet_sdk/clients/rest/models/cron_workflows_method.py +37 -0
- hatchet_sdk/clients/rest/models/events.py +110 -0
- hatchet_sdk/clients/rest/models/scheduled_workflows.py +5 -9
- hatchet_sdk/clients/rest/models/scheduled_workflows_method.py +37 -0
- hatchet_sdk/clients/rest/models/worker.py +2 -10
- hatchet_sdk/clients/rest/models/worker_type.py +38 -0
- hatchet_sdk/clients/rest/models/workflow_concurrency.py +6 -13
- hatchet_sdk/clients/rest/models/workflow_list.py +4 -4
- hatchet_sdk/clients/rest/models/workflow_run.py +10 -10
- hatchet_sdk/context/context.py +9 -7
- hatchet_sdk/context/worker_context.py +5 -5
- hatchet_sdk/contracts/dispatcher_pb2.pyi +0 -2
- hatchet_sdk/contracts/events_pb2.pyi +0 -2
- hatchet_sdk/contracts/workflows_pb2.pyi +0 -2
- hatchet_sdk/hatchet.py +67 -54
- hatchet_sdk/labels.py +1 -1
- hatchet_sdk/rate_limit.py +117 -4
- hatchet_sdk/utils/aio_utils.py +13 -4
- hatchet_sdk/utils/typing.py +4 -1
- hatchet_sdk/v2/callable.py +24 -24
- hatchet_sdk/v2/concurrency.py +10 -8
- hatchet_sdk/v2/hatchet.py +38 -36
- hatchet_sdk/worker/runner/runner.py +1 -1
- hatchet_sdk/worker/worker.py +16 -9
- hatchet_sdk/workflow.py +21 -9
- {hatchet_sdk-0.42.2.dist-info → hatchet_sdk-0.42.4.dist-info}/METADATA +2 -1
- {hatchet_sdk-0.42.2.dist-info → hatchet_sdk-0.42.4.dist-info}/RECORD +38 -33
- {hatchet_sdk-0.42.2.dist-info → hatchet_sdk-0.42.4.dist-info}/entry_points.txt +1 -0
- {hatchet_sdk-0.42.2.dist-info → hatchet_sdk-0.42.4.dist-info}/WHEEL +0 -0
hatchet_sdk/rate_limit.py
CHANGED
|
@@ -1,13 +1,126 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
|
+
from typing import Union
|
|
2
3
|
|
|
4
|
+
from celpy import CELEvalError, Environment
|
|
3
5
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
from hatchet_sdk.contracts.workflows_pb2 import CreateStepRateLimit
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def validate_cel_expression(expr: str) -> bool:
|
|
10
|
+
env = Environment()
|
|
11
|
+
try:
|
|
12
|
+
env.compile(expr)
|
|
13
|
+
return True
|
|
14
|
+
except CELEvalError:
|
|
15
|
+
return False
|
|
8
16
|
|
|
9
17
|
|
|
10
18
|
class RateLimitDuration:
|
|
11
19
|
SECOND = "SECOND"
|
|
12
20
|
MINUTE = "MINUTE"
|
|
13
21
|
HOUR = "HOUR"
|
|
22
|
+
DAY = "DAY"
|
|
23
|
+
WEEK = "WEEK"
|
|
24
|
+
MONTH = "MONTH"
|
|
25
|
+
YEAR = "YEAR"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class RateLimit:
|
|
30
|
+
"""
|
|
31
|
+
Represents a rate limit configuration for a step in a workflow.
|
|
32
|
+
|
|
33
|
+
This class allows for both static and dynamic rate limiting based on various parameters.
|
|
34
|
+
It supports both simple integer values and Common Expression Language (CEL) expressions
|
|
35
|
+
for dynamic evaluation.
|
|
36
|
+
|
|
37
|
+
Attributes:
|
|
38
|
+
static_key (str, optional): A static key for rate limiting.
|
|
39
|
+
dynamic_key (str, optional): A CEL expression for dynamic key evaluation.
|
|
40
|
+
units (int or str, default=1): The number of units or a CEL expression for dynamic unit calculation.
|
|
41
|
+
limit (int or str, optional): The rate limit value or a CEL expression for dynamic limit calculation.
|
|
42
|
+
duration (str, default=RateLimitDuration.MINUTE): The window duration of the rate limit.
|
|
43
|
+
key (str, optional): Deprecated. Use static_key instead.
|
|
44
|
+
|
|
45
|
+
Usage:
|
|
46
|
+
1. Static rate limit:
|
|
47
|
+
rate_limit = RateLimit(static_key="external-api", units=100)
|
|
48
|
+
> NOTE: if you want to use a static key, you must first put the rate limit: hatchet.admin.put_rate_limit("external-api", 200, RateLimitDuration.SECOND)
|
|
49
|
+
|
|
50
|
+
2. Dynamic rate limit with CEL expressions:
|
|
51
|
+
rate_limit = RateLimit(
|
|
52
|
+
dynamic_key="input.user_id",
|
|
53
|
+
units="input.units",
|
|
54
|
+
limit="input.limit * input.user_tier"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
Note:
|
|
58
|
+
- Either static_key or dynamic_key must be set, but not both.
|
|
59
|
+
- When using dynamic_key, limit must also be set.
|
|
60
|
+
- CEL expressions are validated upon instantiation.
|
|
61
|
+
|
|
62
|
+
Raises:
|
|
63
|
+
ValueError: If invalid combinations of attributes are provided or if CEL expressions are invalid.
|
|
64
|
+
DeprecationWarning: If the deprecated 'key' attribute is used.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
key: Union[str, None] = None
|
|
68
|
+
static_key: Union[str, None] = None
|
|
69
|
+
dynamic_key: Union[str, None] = None
|
|
70
|
+
units: Union[int, str] = 1
|
|
71
|
+
limit: Union[int, str, None] = None
|
|
72
|
+
duration: RateLimitDuration = RateLimitDuration.MINUTE
|
|
73
|
+
|
|
74
|
+
_req: CreateStepRateLimit = None
|
|
75
|
+
|
|
76
|
+
def __post_init__(self):
|
|
77
|
+
# juggle the key and key_expr fields
|
|
78
|
+
key = self.static_key
|
|
79
|
+
key_expression = self.dynamic_key
|
|
80
|
+
|
|
81
|
+
if self.key is not None:
|
|
82
|
+
DeprecationWarning(
|
|
83
|
+
"key is deprecated and will be removed in a future release, please use static_key instead"
|
|
84
|
+
)
|
|
85
|
+
key = self.key
|
|
86
|
+
|
|
87
|
+
if key_expression is not None:
|
|
88
|
+
if key is not None:
|
|
89
|
+
raise ValueError("Cannot have both static key and dynamic key set")
|
|
90
|
+
|
|
91
|
+
key = key_expression
|
|
92
|
+
if not validate_cel_expression(key_expression):
|
|
93
|
+
raise ValueError(f"Invalid CEL expression: {key_expression}")
|
|
94
|
+
|
|
95
|
+
# juggle the units and units_expr fields
|
|
96
|
+
units = None
|
|
97
|
+
units_expression = None
|
|
98
|
+
if isinstance(self.units, int):
|
|
99
|
+
units = self.units
|
|
100
|
+
else:
|
|
101
|
+
if not validate_cel_expression(self.units):
|
|
102
|
+
raise ValueError(f"Invalid CEL expression: {self.units}")
|
|
103
|
+
units_expression = self.units
|
|
104
|
+
|
|
105
|
+
# juggle the limit and limit_expr fields
|
|
106
|
+
limit_expression = None
|
|
107
|
+
|
|
108
|
+
if self.limit:
|
|
109
|
+
if isinstance(self.limit, int):
|
|
110
|
+
limit_expression = f"{self.limit}"
|
|
111
|
+
else:
|
|
112
|
+
if not validate_cel_expression(self.limit):
|
|
113
|
+
raise ValueError(f"Invalid CEL expression: {self.limit}")
|
|
114
|
+
limit_expression = self.limit
|
|
115
|
+
|
|
116
|
+
if key_expression is not None and limit_expression is None:
|
|
117
|
+
raise ValueError("CEL based keys requires limit to be set")
|
|
118
|
+
|
|
119
|
+
self._req = CreateStepRateLimit(
|
|
120
|
+
key=key,
|
|
121
|
+
key_expr=key_expression,
|
|
122
|
+
units=units,
|
|
123
|
+
units_expr=units_expression,
|
|
124
|
+
limit_values_expr=limit_expression,
|
|
125
|
+
duration=self.duration,
|
|
126
|
+
)
|
hatchet_sdk/utils/aio_utils.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import inspect
|
|
3
|
+
from concurrent.futures import Executor
|
|
3
4
|
from functools import partial, wraps
|
|
4
5
|
from threading import Thread
|
|
6
|
+
from typing import Any
|
|
5
7
|
|
|
6
8
|
|
|
7
|
-
|
|
9
|
+
## TODO: Stricter typing here
|
|
10
|
+
def sync_to_async(func: Any) -> Any:
|
|
8
11
|
"""
|
|
9
12
|
A decorator to run a synchronous function or coroutine in an asynchronous context with added
|
|
10
13
|
asyncio loop safety.
|
|
@@ -40,8 +43,14 @@ def sync_to_async(func):
|
|
|
40
43
|
asyncio.run(main())
|
|
41
44
|
"""
|
|
42
45
|
|
|
46
|
+
## TODO: Stricter typing here
|
|
43
47
|
@wraps(func)
|
|
44
|
-
async def run(
|
|
48
|
+
async def run(
|
|
49
|
+
*args: Any,
|
|
50
|
+
loop: asyncio.AbstractEventLoop | None = None,
|
|
51
|
+
executor: Executor | None = None,
|
|
52
|
+
**kwargs: Any
|
|
53
|
+
) -> Any:
|
|
45
54
|
"""
|
|
46
55
|
The asynchronous wrapper function that runs the given function in an executor.
|
|
47
56
|
|
|
@@ -59,7 +68,7 @@ def sync_to_async(func):
|
|
|
59
68
|
|
|
60
69
|
if inspect.iscoroutinefunction(func):
|
|
61
70
|
# Wrap the coroutine to run it in an executor
|
|
62
|
-
async def wrapper():
|
|
71
|
+
async def wrapper() -> Any:
|
|
63
72
|
return await func(*args, **kwargs)
|
|
64
73
|
|
|
65
74
|
pfunc = partial(asyncio.run, wrapper())
|
|
@@ -75,7 +84,7 @@ def sync_to_async(func):
|
|
|
75
84
|
class EventLoopThread:
|
|
76
85
|
"""A class that manages an asyncio event loop running in a separate thread."""
|
|
77
86
|
|
|
78
|
-
def __init__(self):
|
|
87
|
+
def __init__(self) -> None:
|
|
79
88
|
"""
|
|
80
89
|
Initializes the EventLoopThread by creating an event loop
|
|
81
90
|
and setting up a thread to run the loop.
|
hatchet_sdk/utils/typing.py
CHANGED
hatchet_sdk/v2/callable.py
CHANGED
|
@@ -1,8 +1,19 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
from typing import
|
|
2
|
+
from typing import (
|
|
3
|
+
Any,
|
|
4
|
+
Callable,
|
|
5
|
+
Dict,
|
|
6
|
+
Generic,
|
|
7
|
+
List,
|
|
8
|
+
Optional,
|
|
9
|
+
TypedDict,
|
|
10
|
+
TypeVar,
|
|
11
|
+
Union,
|
|
12
|
+
)
|
|
3
13
|
|
|
4
|
-
from hatchet_sdk.
|
|
5
|
-
from hatchet_sdk.
|
|
14
|
+
from hatchet_sdk.clients.admin import ChildTriggerWorkflowOptions
|
|
15
|
+
from hatchet_sdk.context.context import Context
|
|
16
|
+
from hatchet_sdk.contracts.workflows_pb2 import ( # type: ignore[attr-defined]
|
|
6
17
|
CreateStepRateLimit,
|
|
7
18
|
CreateWorkflowJobOpts,
|
|
8
19
|
CreateWorkflowStepOpts,
|
|
@@ -28,8 +39,8 @@ class HatchetCallable(Generic[T]):
|
|
|
28
39
|
durable: bool = False,
|
|
29
40
|
name: str = "",
|
|
30
41
|
auto_register: bool = True,
|
|
31
|
-
on_events: list | None = None,
|
|
32
|
-
on_crons: list | None = None,
|
|
42
|
+
on_events: list[str] | None = None,
|
|
43
|
+
on_crons: list[str] | None = None,
|
|
33
44
|
version: str = "",
|
|
34
45
|
timeout: str = "60m",
|
|
35
46
|
schedule_timeout: str = "5m",
|
|
@@ -37,8 +48,8 @@ class HatchetCallable(Generic[T]):
|
|
|
37
48
|
retries: int = 0,
|
|
38
49
|
rate_limits: List[RateLimit] | None = None,
|
|
39
50
|
concurrency: ConcurrencyFunction | None = None,
|
|
40
|
-
on_failure:
|
|
41
|
-
desired_worker_labels: dict[str
|
|
51
|
+
on_failure: Union["HatchetCallable[T]", None] = None,
|
|
52
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
42
53
|
default_priority: int | None = None,
|
|
43
54
|
):
|
|
44
55
|
self.func = func
|
|
@@ -48,10 +59,7 @@ class HatchetCallable(Generic[T]):
|
|
|
48
59
|
|
|
49
60
|
limits = None
|
|
50
61
|
if rate_limits:
|
|
51
|
-
limits = [
|
|
52
|
-
CreateStepRateLimit(key=rate_limit.key, units=rate_limit.units)
|
|
53
|
-
for rate_limit in rate_limits or []
|
|
54
|
-
]
|
|
62
|
+
limits = [rate_limit._req for rate_limit in rate_limits or []]
|
|
55
63
|
|
|
56
64
|
self.function_desired_worker_labels = {}
|
|
57
65
|
|
|
@@ -88,7 +96,7 @@ class HatchetCallable(Generic[T]):
|
|
|
88
96
|
def __call__(self, context: Context) -> T:
|
|
89
97
|
return self.func(context)
|
|
90
98
|
|
|
91
|
-
def with_namespace(self, namespace: str):
|
|
99
|
+
def with_namespace(self, namespace: str) -> None:
|
|
92
100
|
if namespace is not None and namespace != "":
|
|
93
101
|
self.function_namespace = namespace
|
|
94
102
|
self.function_name = namespace + self.function_name
|
|
@@ -164,21 +172,13 @@ class HatchetCallable(Generic[T]):
|
|
|
164
172
|
return self.function_namespace + ":" + self.function_name
|
|
165
173
|
|
|
166
174
|
|
|
167
|
-
T = TypeVar("T")
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
class TriggerOptions(TypedDict):
|
|
171
|
-
additional_metadata: Dict[str, str] | None = None
|
|
172
|
-
sticky: bool | None = None
|
|
173
|
-
|
|
174
|
-
|
|
175
175
|
class DurableContext(Context):
|
|
176
176
|
def run(
|
|
177
177
|
self,
|
|
178
|
-
function:
|
|
179
|
-
input: dict = {},
|
|
180
|
-
key: str = None,
|
|
181
|
-
options:
|
|
178
|
+
function: str | Callable[[Context], Any],
|
|
179
|
+
input: dict[Any, Any] = {},
|
|
180
|
+
key: str | None = None,
|
|
181
|
+
options: ChildTriggerWorkflowOptions | None = None,
|
|
182
182
|
) -> "RunRef[T]":
|
|
183
183
|
worker_id = self.worker.id()
|
|
184
184
|
|
hatchet_sdk/v2/concurrency.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
from typing import Callable
|
|
1
|
+
from typing import Any, Callable
|
|
2
2
|
|
|
3
|
-
from hatchet_sdk.context import Context
|
|
4
|
-
from hatchet_sdk.contracts.workflows_pb2 import
|
|
3
|
+
from hatchet_sdk.context.context import Context
|
|
4
|
+
from hatchet_sdk.contracts.workflows_pb2 import ( # type: ignore[attr-defined]
|
|
5
|
+
ConcurrencyLimitStrategy,
|
|
6
|
+
)
|
|
5
7
|
|
|
6
8
|
|
|
7
9
|
class ConcurrencyFunction:
|
|
@@ -18,19 +20,19 @@ class ConcurrencyFunction:
|
|
|
18
20
|
self.limit_strategy = limit_strategy
|
|
19
21
|
self.namespace = "default"
|
|
20
22
|
|
|
21
|
-
def set_namespace(self, namespace: str):
|
|
23
|
+
def set_namespace(self, namespace: str) -> None:
|
|
22
24
|
self.namespace = namespace
|
|
23
25
|
|
|
24
26
|
def get_action_name(self) -> str:
|
|
25
27
|
return self.namespace + ":" + self.name
|
|
26
28
|
|
|
27
|
-
def __call__(self, *args, **kwargs):
|
|
29
|
+
def __call__(self, *args: Any, **kwargs: Any) -> str:
|
|
28
30
|
return self.func(*args, **kwargs)
|
|
29
31
|
|
|
30
|
-
def __str__(self):
|
|
32
|
+
def __str__(self) -> str:
|
|
31
33
|
return f"{self.name}({self.max_runs})"
|
|
32
34
|
|
|
33
|
-
def __repr__(self):
|
|
35
|
+
def __repr__(self) -> str:
|
|
34
36
|
return f"{self.name}({self.max_runs})"
|
|
35
37
|
|
|
36
38
|
|
|
@@ -38,7 +40,7 @@ def concurrency(
|
|
|
38
40
|
name: str = "",
|
|
39
41
|
max_runs: int = 1,
|
|
40
42
|
limit_strategy: ConcurrencyLimitStrategy = ConcurrencyLimitStrategy.GROUP_ROUND_ROBIN,
|
|
41
|
-
):
|
|
43
|
+
) -> Callable[[Callable[[Context], str]], ConcurrencyFunction]:
|
|
42
44
|
def inner(func: Callable[[Context], str]) -> ConcurrencyFunction:
|
|
43
45
|
return ConcurrencyFunction(func, name, max_runs, limit_strategy)
|
|
44
46
|
|
hatchet_sdk/v2/hatchet.py
CHANGED
|
@@ -1,36 +1,38 @@
|
|
|
1
|
-
from typing import
|
|
2
|
-
|
|
3
|
-
from hatchet_sdk
|
|
4
|
-
from hatchet_sdk.
|
|
1
|
+
from typing import Any, Callable, TypeVar, Union
|
|
2
|
+
|
|
3
|
+
from hatchet_sdk import Worker
|
|
4
|
+
from hatchet_sdk.context.context import Context
|
|
5
|
+
from hatchet_sdk.contracts.workflows_pb2 import ( # type: ignore[attr-defined]
|
|
6
|
+
ConcurrencyLimitStrategy,
|
|
7
|
+
StickyStrategy,
|
|
8
|
+
)
|
|
5
9
|
from hatchet_sdk.hatchet import Hatchet as HatchetV1
|
|
6
10
|
from hatchet_sdk.hatchet import workflow
|
|
7
11
|
from hatchet_sdk.labels import DesiredWorkerLabel
|
|
8
12
|
from hatchet_sdk.rate_limit import RateLimit
|
|
9
|
-
from hatchet_sdk.v2.callable import HatchetCallable
|
|
13
|
+
from hatchet_sdk.v2.callable import DurableContext, HatchetCallable
|
|
10
14
|
from hatchet_sdk.v2.concurrency import ConcurrencyFunction
|
|
11
15
|
from hatchet_sdk.worker.worker import register_on_worker
|
|
12
16
|
|
|
13
|
-
from ..worker import Worker
|
|
14
|
-
|
|
15
17
|
T = TypeVar("T")
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
def function(
|
|
19
21
|
name: str = "",
|
|
20
22
|
auto_register: bool = True,
|
|
21
|
-
on_events: list | None = None,
|
|
22
|
-
on_crons: list | None = None,
|
|
23
|
+
on_events: list[str] | None = None,
|
|
24
|
+
on_crons: list[str] | None = None,
|
|
23
25
|
version: str = "",
|
|
24
26
|
timeout: str = "60m",
|
|
25
27
|
schedule_timeout: str = "5m",
|
|
26
28
|
sticky: StickyStrategy = None,
|
|
27
29
|
retries: int = 0,
|
|
28
|
-
rate_limits:
|
|
29
|
-
desired_worker_labels: dict[str
|
|
30
|
+
rate_limits: list[RateLimit] | None = None,
|
|
31
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
30
32
|
concurrency: ConcurrencyFunction | None = None,
|
|
31
|
-
on_failure:
|
|
33
|
+
on_failure: Union["HatchetCallable[T]", None] = None,
|
|
32
34
|
default_priority: int | None = None,
|
|
33
|
-
):
|
|
35
|
+
) -> Callable[[Callable[[Context], str]], HatchetCallable[T]]:
|
|
34
36
|
def inner(func: Callable[[Context], T]) -> HatchetCallable[T]:
|
|
35
37
|
return HatchetCallable(
|
|
36
38
|
func=func,
|
|
@@ -56,20 +58,20 @@ def function(
|
|
|
56
58
|
def durable(
|
|
57
59
|
name: str = "",
|
|
58
60
|
auto_register: bool = True,
|
|
59
|
-
on_events: list | None = None,
|
|
60
|
-
on_crons: list | None = None,
|
|
61
|
+
on_events: list[str] | None = None,
|
|
62
|
+
on_crons: list[str] | None = None,
|
|
61
63
|
version: str = "",
|
|
62
64
|
timeout: str = "60m",
|
|
63
65
|
schedule_timeout: str = "5m",
|
|
64
66
|
sticky: StickyStrategy = None,
|
|
65
67
|
retries: int = 0,
|
|
66
|
-
rate_limits:
|
|
67
|
-
desired_worker_labels: dict[str
|
|
68
|
+
rate_limits: list[RateLimit] | None = None,
|
|
69
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
68
70
|
concurrency: ConcurrencyFunction | None = None,
|
|
69
|
-
on_failure: HatchetCallable | None = None,
|
|
71
|
+
on_failure: HatchetCallable[T] | None = None,
|
|
70
72
|
default_priority: int | None = None,
|
|
71
|
-
):
|
|
72
|
-
def inner(func: HatchetCallable) -> HatchetCallable:
|
|
73
|
+
) -> Callable[[HatchetCallable[T]], HatchetCallable[T]]:
|
|
74
|
+
def inner(func: HatchetCallable[T]) -> HatchetCallable[T]:
|
|
73
75
|
func.durable = True
|
|
74
76
|
|
|
75
77
|
f = function(
|
|
@@ -102,7 +104,7 @@ def concurrency(
|
|
|
102
104
|
name: str = "concurrency",
|
|
103
105
|
max_runs: int = 1,
|
|
104
106
|
limit_strategy: ConcurrencyLimitStrategy = ConcurrencyLimitStrategy.GROUP_ROUND_ROBIN,
|
|
105
|
-
):
|
|
107
|
+
) -> Callable[[Callable[[Context], str]], ConcurrencyFunction]:
|
|
106
108
|
def inner(func: Callable[[Context], str]) -> ConcurrencyFunction:
|
|
107
109
|
return ConcurrencyFunction(func, name, max_runs, limit_strategy)
|
|
108
110
|
|
|
@@ -113,24 +115,24 @@ class Hatchet(HatchetV1):
|
|
|
113
115
|
dag = staticmethod(workflow)
|
|
114
116
|
concurrency = staticmethod(concurrency)
|
|
115
117
|
|
|
116
|
-
functions:
|
|
118
|
+
functions: list[HatchetCallable[T]] = []
|
|
117
119
|
|
|
118
120
|
def function(
|
|
119
121
|
self,
|
|
120
122
|
name: str = "",
|
|
121
123
|
auto_register: bool = True,
|
|
122
|
-
on_events: list | None = None,
|
|
123
|
-
on_crons: list | None = None,
|
|
124
|
+
on_events: list[str] | None = None,
|
|
125
|
+
on_crons: list[str] | None = None,
|
|
124
126
|
version: str = "",
|
|
125
127
|
timeout: str = "60m",
|
|
126
128
|
schedule_timeout: str = "5m",
|
|
127
129
|
retries: int = 0,
|
|
128
|
-
rate_limits:
|
|
129
|
-
desired_worker_labels: dict[str
|
|
130
|
+
rate_limits: list[RateLimit] | None = None,
|
|
131
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
130
132
|
concurrency: ConcurrencyFunction | None = None,
|
|
131
|
-
on_failure:
|
|
133
|
+
on_failure: Union["HatchetCallable[T]", None] = None,
|
|
132
134
|
default_priority: int | None = None,
|
|
133
|
-
):
|
|
135
|
+
) -> Callable[[Callable[[Context], Any]], Callable[[Context], Any]]:
|
|
134
136
|
resp = function(
|
|
135
137
|
name=name,
|
|
136
138
|
auto_register=auto_register,
|
|
@@ -147,7 +149,7 @@ class Hatchet(HatchetV1):
|
|
|
147
149
|
default_priority=default_priority,
|
|
148
150
|
)
|
|
149
151
|
|
|
150
|
-
def wrapper(func: Callable[[Context],
|
|
152
|
+
def wrapper(func: Callable[[Context], str]) -> HatchetCallable[T]:
|
|
151
153
|
wrapped_resp = resp(func)
|
|
152
154
|
|
|
153
155
|
if wrapped_resp.function_auto_register:
|
|
@@ -163,19 +165,19 @@ class Hatchet(HatchetV1):
|
|
|
163
165
|
self,
|
|
164
166
|
name: str = "",
|
|
165
167
|
auto_register: bool = True,
|
|
166
|
-
on_events: list | None = None,
|
|
167
|
-
on_crons: list | None = None,
|
|
168
|
+
on_events: list[str] | None = None,
|
|
169
|
+
on_crons: list[str] | None = None,
|
|
168
170
|
version: str = "",
|
|
169
171
|
timeout: str = "60m",
|
|
170
172
|
schedule_timeout: str = "5m",
|
|
171
173
|
sticky: StickyStrategy = None,
|
|
172
174
|
retries: int = 0,
|
|
173
|
-
rate_limits:
|
|
174
|
-
desired_worker_labels: dict[str
|
|
175
|
+
rate_limits: list[RateLimit] | None = None,
|
|
176
|
+
desired_worker_labels: dict[str, DesiredWorkerLabel] = {},
|
|
175
177
|
concurrency: ConcurrencyFunction | None = None,
|
|
176
|
-
on_failure:
|
|
178
|
+
on_failure: Union["HatchetCallable[T]", None] = None,
|
|
177
179
|
default_priority: int | None = None,
|
|
178
|
-
) -> Callable[[
|
|
180
|
+
) -> Callable[[Callable[[DurableContext], Any]], Callable[[DurableContext], Any]]:
|
|
179
181
|
resp = durable(
|
|
180
182
|
name=name,
|
|
181
183
|
auto_register=auto_register,
|
|
@@ -193,7 +195,7 @@ class Hatchet(HatchetV1):
|
|
|
193
195
|
default_priority=default_priority,
|
|
194
196
|
)
|
|
195
197
|
|
|
196
|
-
def wrapper(func:
|
|
198
|
+
def wrapper(func: HatchetCallable[T]) -> HatchetCallable[T]:
|
|
197
199
|
wrapped_resp = resp(func)
|
|
198
200
|
|
|
199
201
|
if wrapped_resp.function_auto_register:
|
|
@@ -21,7 +21,7 @@ from hatchet_sdk.clients.run_event_listener import new_listener
|
|
|
21
21
|
from hatchet_sdk.clients.workflow_listener import PooledWorkflowRunListener
|
|
22
22
|
from hatchet_sdk.context import Context # type: ignore[attr-defined]
|
|
23
23
|
from hatchet_sdk.context.worker_context import WorkerContext
|
|
24
|
-
from hatchet_sdk.contracts.dispatcher_pb2 import (
|
|
24
|
+
from hatchet_sdk.contracts.dispatcher_pb2 import (
|
|
25
25
|
GROUP_KEY_EVENT_TYPE_COMPLETED,
|
|
26
26
|
GROUP_KEY_EVENT_TYPE_FAILED,
|
|
27
27
|
GROUP_KEY_EVENT_TYPE_STARTED,
|
hatchet_sdk/worker/worker.py
CHANGED
|
@@ -12,14 +12,15 @@ from multiprocessing.process import BaseProcess
|
|
|
12
12
|
from types import FrameType
|
|
13
13
|
from typing import Any, Callable, TypeVar, get_type_hints
|
|
14
14
|
|
|
15
|
+
from pydantic import BaseModel
|
|
16
|
+
|
|
15
17
|
from hatchet_sdk import Context
|
|
16
18
|
from hatchet_sdk.client import Client, new_client_raw
|
|
17
|
-
from hatchet_sdk.contracts.workflows_pb2 import
|
|
18
|
-
CreateWorkflowVersionOpts,
|
|
19
|
-
)
|
|
19
|
+
from hatchet_sdk.contracts.workflows_pb2 import CreateWorkflowVersionOpts
|
|
20
20
|
from hatchet_sdk.loader import ClientConfig
|
|
21
21
|
from hatchet_sdk.logger import logger
|
|
22
22
|
from hatchet_sdk.utils.types import WorkflowValidator
|
|
23
|
+
from hatchet_sdk.utils.typing import is_basemodel_subclass
|
|
23
24
|
from hatchet_sdk.v2.callable import HatchetCallable
|
|
24
25
|
from hatchet_sdk.v2.concurrency import ConcurrencyFunction
|
|
25
26
|
from hatchet_sdk.worker.action_listener_process import worker_action_listener_process
|
|
@@ -41,6 +42,9 @@ class WorkerStartOptions:
|
|
|
41
42
|
loop: asyncio.AbstractEventLoop | None = field(default=None)
|
|
42
43
|
|
|
43
44
|
|
|
45
|
+
TWorkflow = TypeVar("TWorkflow", bound=object)
|
|
46
|
+
|
|
47
|
+
|
|
44
48
|
class Worker:
|
|
45
49
|
def __init__(
|
|
46
50
|
self,
|
|
@@ -62,7 +66,7 @@ class Worker:
|
|
|
62
66
|
|
|
63
67
|
self.client: Client
|
|
64
68
|
|
|
65
|
-
self.action_registry: dict[str, Callable[[Context],
|
|
69
|
+
self.action_registry: dict[str, Callable[[Context], Any]] = {}
|
|
66
70
|
self.validator_registry: dict[str, WorkflowValidator] = {}
|
|
67
71
|
|
|
68
72
|
self.killing: bool = False
|
|
@@ -84,9 +88,7 @@ class Worker:
|
|
|
84
88
|
|
|
85
89
|
self._setup_signal_handlers()
|
|
86
90
|
|
|
87
|
-
def register_function(
|
|
88
|
-
self, action: str, func: HatchetCallable[Any] | ConcurrencyFunction
|
|
89
|
-
) -> None:
|
|
91
|
+
def register_function(self, action: str, func: Callable[[Context], Any]) -> None:
|
|
90
92
|
self.action_registry[action] = func
|
|
91
93
|
|
|
92
94
|
def register_workflow_from_opts(
|
|
@@ -99,7 +101,10 @@ class Worker:
|
|
|
99
101
|
logger.error(e)
|
|
100
102
|
sys.exit(1)
|
|
101
103
|
|
|
102
|
-
def register_workflow(self, workflow:
|
|
104
|
+
def register_workflow(self, workflow: TWorkflow) -> None:
|
|
105
|
+
## Hack for typing
|
|
106
|
+
assert isinstance(workflow, WorkflowInterface)
|
|
107
|
+
|
|
103
108
|
namespace = self.client.config.namespace
|
|
104
109
|
|
|
105
110
|
try:
|
|
@@ -127,8 +132,10 @@ class Worker:
|
|
|
127
132
|
for action_name, action_func in workflow.get_actions(namespace):
|
|
128
133
|
self.action_registry[action_name] = create_action_function(action_func)
|
|
129
134
|
return_type = get_type_hints(action_func).get("return")
|
|
135
|
+
|
|
130
136
|
self.validator_registry[action_name] = WorkflowValidator(
|
|
131
|
-
workflow_input=workflow.input_validator,
|
|
137
|
+
workflow_input=workflow.input_validator,
|
|
138
|
+
step_output=return_type if is_basemodel_subclass(return_type) else None,
|
|
132
139
|
)
|
|
133
140
|
|
|
134
141
|
def status(self) -> WorkerStatus:
|
hatchet_sdk/workflow.py
CHANGED
|
@@ -1,10 +1,20 @@
|
|
|
1
1
|
import functools
|
|
2
|
-
from typing import
|
|
2
|
+
from typing import (
|
|
3
|
+
Any,
|
|
4
|
+
Callable,
|
|
5
|
+
Protocol,
|
|
6
|
+
Type,
|
|
7
|
+
TypeVar,
|
|
8
|
+
Union,
|
|
9
|
+
cast,
|
|
10
|
+
get_type_hints,
|
|
11
|
+
runtime_checkable,
|
|
12
|
+
)
|
|
3
13
|
|
|
4
14
|
from pydantic import BaseModel
|
|
5
15
|
|
|
6
16
|
from hatchet_sdk import ConcurrencyLimitStrategy
|
|
7
|
-
from hatchet_sdk.contracts.workflows_pb2 import (
|
|
17
|
+
from hatchet_sdk.contracts.workflows_pb2 import (
|
|
8
18
|
CreateWorkflowJobOpts,
|
|
9
19
|
CreateWorkflowStepOpts,
|
|
10
20
|
CreateWorkflowVersionOpts,
|
|
@@ -69,6 +79,7 @@ class ConcurrencyExpression:
|
|
|
69
79
|
self.limit_strategy = limit_strategy
|
|
70
80
|
|
|
71
81
|
|
|
82
|
+
@runtime_checkable
|
|
72
83
|
class WorkflowInterface(Protocol):
|
|
73
84
|
def get_name(self, namespace: str) -> str: ...
|
|
74
85
|
|
|
@@ -76,13 +87,13 @@ class WorkflowInterface(Protocol):
|
|
|
76
87
|
|
|
77
88
|
def get_create_opts(self, namespace: str) -> Any: ...
|
|
78
89
|
|
|
79
|
-
on_events: list[str]
|
|
80
|
-
on_crons: list[str]
|
|
90
|
+
on_events: list[str] | None
|
|
91
|
+
on_crons: list[str] | None
|
|
81
92
|
name: str
|
|
82
93
|
version: str
|
|
83
94
|
timeout: str
|
|
84
95
|
schedule_timeout: str
|
|
85
|
-
sticky: Union[StickyStrategy.Value, None]
|
|
96
|
+
sticky: Union[StickyStrategy.Value, None] # type: ignore[name-defined]
|
|
86
97
|
default_priority: int | None
|
|
87
98
|
concurrency_expression: ConcurrencyExpression | None
|
|
88
99
|
input_validator: Type[BaseModel] | None
|
|
@@ -120,6 +131,7 @@ class WorkflowMeta(type):
|
|
|
120
131
|
@functools.cache
|
|
121
132
|
def get_actions(self: TW, namespace: str) -> StepsType:
|
|
122
133
|
serviceName = get_service_name(namespace)
|
|
134
|
+
|
|
123
135
|
func_actions = [
|
|
124
136
|
(serviceName + ":" + func_name, func) for func_name, func in steps
|
|
125
137
|
]
|
|
@@ -165,8 +177,8 @@ class WorkflowMeta(type):
|
|
|
165
177
|
inputs="{}",
|
|
166
178
|
parents=[x for x in func._step_parents],
|
|
167
179
|
retries=func._step_retries,
|
|
168
|
-
rate_limits=func._step_rate_limits,
|
|
169
|
-
worker_labels=func._step_desired_worker_labels,
|
|
180
|
+
rate_limits=func._step_rate_limits, # type: ignore[arg-type]
|
|
181
|
+
worker_labels=func._step_desired_worker_labels, # type: ignore[arg-type]
|
|
170
182
|
backoff_factor=func._step_backoff_factor,
|
|
171
183
|
backoff_max_seconds=func._step_backoff_max_seconds,
|
|
172
184
|
)
|
|
@@ -196,7 +208,7 @@ class WorkflowMeta(type):
|
|
|
196
208
|
"Error: Both concurrencyActions and concurrency_expression are defined. Please use only one concurrency configuration method."
|
|
197
209
|
)
|
|
198
210
|
|
|
199
|
-
on_failure_job:
|
|
211
|
+
on_failure_job: CreateWorkflowJobOpts | None = None
|
|
200
212
|
|
|
201
213
|
if len(onFailureSteps) > 0:
|
|
202
214
|
func_name, func = onFailureSteps[0]
|
|
@@ -210,7 +222,7 @@ class WorkflowMeta(type):
|
|
|
210
222
|
inputs="{}",
|
|
211
223
|
parents=[],
|
|
212
224
|
retries=func._on_failure_step_retries,
|
|
213
|
-
rate_limits=func._on_failure_step_rate_limits,
|
|
225
|
+
rate_limits=func._on_failure_step_rate_limits, # type: ignore[arg-type]
|
|
214
226
|
backoff_factor=func._on_failure_step_backoff_factor,
|
|
215
227
|
backoff_max_seconds=func._on_failure_step_backoff_max_seconds,
|
|
216
228
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: hatchet-sdk
|
|
3
|
-
Version: 0.42.
|
|
3
|
+
Version: 0.42.4
|
|
4
4
|
Summary:
|
|
5
5
|
Author: Alexander Belanger
|
|
6
6
|
Author-email: alexander@hatchet.run
|
|
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
12
12
|
Requires-Dist: aiohttp (>=3.10.5,<4.0.0)
|
|
13
13
|
Requires-Dist: aiohttp-retry (>=2.8.3,<3.0.0)
|
|
14
14
|
Requires-Dist: aiostream (>=0.5.2,<0.6.0)
|
|
15
|
+
Requires-Dist: cel-python (>=0.1.5,<0.2.0)
|
|
15
16
|
Requires-Dist: grpcio (>=1.64.1,<2.0.0)
|
|
16
17
|
Requires-Dist: grpcio-tools (>=1.60.0,<2.0.0)
|
|
17
18
|
Requires-Dist: loguru (>=0.7.2,<0.8.0)
|