joobq 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- joobq/__init__.py +1645 -0
- joobq/py.typed +0 -0
- joobq-0.1.0.dist-info/METADATA +46 -0
- joobq-0.1.0.dist-info/RECORD +7 -0
- joobq-0.1.0.dist-info/WHEEL +5 -0
- joobq-0.1.0.dist-info/licenses/LICENSE +21 -0
- joobq-0.1.0.dist-info/top_level.txt +1 -0
joobq/__init__.py
ADDED
|
@@ -0,0 +1,1645 @@
|
|
|
1
|
+
# Code generated by joobqui-selfhosted sdk:generate (v0.1.0) from JoobQ Polyglot SDK Gateway API. DO NOT EDIT.
|
|
2
|
+
# Supported Gateway Operations:
|
|
3
|
+
# - POST /api/v1/sdk/batches: Create a Pro batch of jobs with completion callbacks, filled and sealed in one call (Pro)
|
|
4
|
+
# - GET /api/v1/sdk/batches/{id}: Read a Pro batch's progress: pending, succeeded and failed members (Pro)
|
|
5
|
+
# - POST /api/v1/sdk/blueprints: Register a workflow blueprint, versioned by its content hash
|
|
6
|
+
# - POST /api/v1/sdk/jobs: Enqueue a background job onto a queue
|
|
7
|
+
# - POST /api/v1/sdk/jobs/batch: Enqueue up to 500 background jobs in one call
|
|
8
|
+
# - GET /api/v1/sdk/openapi.json: Describe the Polyglot SDK Gateway API as an OpenAPI 3.1 document
|
|
9
|
+
# - POST /api/v1/sdk/rate-limit: Reserve one slot of a sliding-window rate limit (Pro)
|
|
10
|
+
# - POST /api/v1/sdk/schedules: Register a code-declared cron/recurring job or workflow schedule, idempotent by name
|
|
11
|
+
# - POST /api/v1/sdk/tasks/ack: Settle up to 100 task leases in one call, each completed or failed
|
|
12
|
+
# - POST /api/v1/sdk/tasks/heartbeat: Extend every lease a worker holds in one call and check each for cancellation
|
|
13
|
+
# - POST /api/v1/sdk/tasks/poll: Claim due jobs and workflow steps from unpaused queues
|
|
14
|
+
# - POST /api/v1/sdk/tasks/{token}/complete: Commit task completion and output
|
|
15
|
+
# - POST /api/v1/sdk/tasks/{token}/fail: Record structured polyglot task failure and stack frames
|
|
16
|
+
# - POST /api/v1/sdk/tasks/{token}/heartbeat: Extend an active task lease and check for cancellation
|
|
17
|
+
# - POST /api/v1/sdk/workers/register: Register or heartbeat a polyglot SDK worker
|
|
18
|
+
# - POST /api/v1/sdk/workers/{id}/deregister: Deregister a polyglot SDK worker on shutdown
|
|
19
|
+
# - POST /api/v1/sdk/workflows: Start a durable workflow execution
|
|
20
|
+
# - GET /api/v1/sdk/workflows/{id}: Describe one workflow run: status, output, failure and search attributes
|
|
21
|
+
# - GET /api/v1/sdk/workflows/{id}/history: The events of a workflow run, after after_seq, at most limit
|
|
22
|
+
# - POST /api/v1/sdk/workflows/{id}/query: Ask a workflow run a read-only question
|
|
23
|
+
# - GET /api/v1/sdk/workflows/{id}/result: The outcome of a workflow, following continue-as-new; waits up to wait_ms for it to close
|
|
24
|
+
# - POST /api/v1/sdk/workflows/{id}/search-attributes: Set and remove search attributes of a workflow's open run
|
|
25
|
+
# - POST /api/v1/sdk/workflows/{id}/signal: Deliver an asynchronous signal to a running workflow
|
|
26
|
+
# - POST /api/v1/sdk/workflows/{id}/updates: Send an update to a running workflow and wait for its outcome
|
|
27
|
+
# NOTE: Queue & workflow management (pause, resume, redrive, cancel, terminate) is intentionally
|
|
28
|
+
# excluded from SDKs and must be performed via the JoobQUI Operator Console (/ui/).
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
import http.client
|
|
32
|
+
import json
|
|
33
|
+
import os
|
|
34
|
+
import queue as _queue
|
|
35
|
+
import random
|
|
36
|
+
import signal as _signal
|
|
37
|
+
import socket
|
|
38
|
+
import sys
|
|
39
|
+
import threading
|
|
40
|
+
import time
|
|
41
|
+
import traceback
|
|
42
|
+
import urllib.parse
|
|
43
|
+
import urllib.request
|
|
44
|
+
from concurrent.futures import Future, ThreadPoolExecutor
|
|
45
|
+
from dataclasses import dataclass, field
|
|
46
|
+
from typing import Any, Callable, Dict, List, Optional, Type, Union
|
|
47
|
+
|
|
48
|
+
SDK_VERSION = "0.1.0"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class NonRetryableError(Exception):
|
|
53
|
+
"""Raised by a Job or Workflow Activity when a failure is permanent and must not be retried."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class RetryPolicy:
|
|
58
|
+
attempts: int = 3
|
|
59
|
+
initial_ms: int = 1000
|
|
60
|
+
backoff: float = 2.0
|
|
61
|
+
maximum_ms: int = 30000
|
|
62
|
+
# Error types (the exception's class name, as the worker reports it) that end the step without another attempt.
|
|
63
|
+
non_retryable: List[str] = field(default_factory=list)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def retry_policy(
|
|
67
|
+
attempts: int = 3,
|
|
68
|
+
initial_ms: int = 1000,
|
|
69
|
+
backoff: float = 2.0,
|
|
70
|
+
maximum_ms: int = 30000,
|
|
71
|
+
non_retryable: Optional[List[str]] = None,
|
|
72
|
+
) -> RetryPolicy:
|
|
73
|
+
return RetryPolicy(attempts=attempts, initial_ms=initial_ms, backoff=backoff, maximum_ms=maximum_ms,
|
|
74
|
+
non_retryable=list(non_retryable or []))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class DataRef:
|
|
79
|
+
expr: str
|
|
80
|
+
|
|
81
|
+
def to_expr(self) -> str:
|
|
82
|
+
return self.expr
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def input(key: Optional[str] = None) -> DataRef:
|
|
86
|
+
return DataRef("${input}" if not key else f"${{input.{key}}}")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def output(step_name: str, key: Optional[str] = None) -> DataRef:
|
|
90
|
+
return DataRef(
|
|
91
|
+
f"${{steps.{step_name}.output}}"
|
|
92
|
+
if not key
|
|
93
|
+
else f"${{steps.{step_name}.output.{key}}}"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def update_input(key: Optional[str] = None) -> DataRef:
|
|
98
|
+
"""The input of the update being handled (`${update.input...}`): for an update's `validate`, `input` and `result`."""
|
|
99
|
+
return DataRef("${update.input}" if not key else f"${{update.input.{key}}}")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def ref(path: str) -> DataRef:
|
|
103
|
+
clean = path.strip()
|
|
104
|
+
if clean.startswith("${") and clean.endswith("}"):
|
|
105
|
+
return DataRef(clean)
|
|
106
|
+
return DataRef(f"${{{clean}}}")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _lookup_path(root: Any, dotted: str) -> Any:
|
|
110
|
+
cur = root
|
|
111
|
+
for part in dotted.split("."):
|
|
112
|
+
if not part:
|
|
113
|
+
continue
|
|
114
|
+
if isinstance(cur, dict):
|
|
115
|
+
cur = cur.get(part)
|
|
116
|
+
else:
|
|
117
|
+
cur = getattr(cur, part, None)
|
|
118
|
+
if cur is None:
|
|
119
|
+
return None
|
|
120
|
+
return cur
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def resolve_references(spec: Any, workflow_input: Any, checkpoints: Dict[str, Any]) -> Any:
|
|
124
|
+
if spec is None:
|
|
125
|
+
return workflow_input
|
|
126
|
+
if isinstance(spec, DataRef):
|
|
127
|
+
expr = spec.expr
|
|
128
|
+
if expr == "${input}":
|
|
129
|
+
return workflow_input
|
|
130
|
+
if expr.startswith("${input.") and expr.endswith("}"):
|
|
131
|
+
return _lookup_path(workflow_input, expr[8:-1])
|
|
132
|
+
if expr.startswith("${steps.") and expr.endswith("}"):
|
|
133
|
+
rest = expr[8:-1]
|
|
134
|
+
parts = rest.split(".")
|
|
135
|
+
step_name = parts[0]
|
|
136
|
+
step_out = checkpoints.get(step_name)
|
|
137
|
+
if len(parts) <= 2:
|
|
138
|
+
return step_out
|
|
139
|
+
return _lookup_path(step_out, ".".join(parts[2:]))
|
|
140
|
+
if expr == "${steps}":
|
|
141
|
+
return dict(checkpoints)
|
|
142
|
+
if expr.startswith("${") and expr.endswith("}"):
|
|
143
|
+
return _lookup_path({"input": workflow_input, "steps": checkpoints}, expr[2:-1])
|
|
144
|
+
return expr
|
|
145
|
+
if isinstance(spec, dict):
|
|
146
|
+
return {k: resolve_references(v, workflow_input, checkpoints) for k, v in spec.items()}
|
|
147
|
+
if isinstance(spec, list):
|
|
148
|
+
return [resolve_references(v, workflow_input, checkpoints) for v in spec]
|
|
149
|
+
return spec
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _extract_step_dependencies(spec: Any) -> List[str]:
|
|
153
|
+
deps: List[str] = []
|
|
154
|
+
if isinstance(spec, DataRef):
|
|
155
|
+
if spec.expr.startswith("${steps."):
|
|
156
|
+
step_name = spec.expr[8:-1].split(".")[0]
|
|
157
|
+
if step_name and step_name not in deps:
|
|
158
|
+
deps.append(step_name)
|
|
159
|
+
elif isinstance(spec, dict):
|
|
160
|
+
for v in spec.values():
|
|
161
|
+
for d in _extract_step_dependencies(v):
|
|
162
|
+
if d not in deps:
|
|
163
|
+
deps.append(d)
|
|
164
|
+
elif isinstance(spec, list):
|
|
165
|
+
for v in spec:
|
|
166
|
+
for d in _extract_step_dependencies(v):
|
|
167
|
+
if d not in deps:
|
|
168
|
+
deps.append(d)
|
|
169
|
+
return deps
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _collect_references(spec: Any) -> List[str]:
|
|
173
|
+
"""The `${...}` references a step input makes (what a blueprint sends: never the literal values around them)."""
|
|
174
|
+
found: List[str] = []
|
|
175
|
+
if isinstance(spec, DataRef):
|
|
176
|
+
found.append(spec.expr)
|
|
177
|
+
elif isinstance(spec, dict):
|
|
178
|
+
for v in spec.values():
|
|
179
|
+
found.extend(r for r in _collect_references(v) if r not in found)
|
|
180
|
+
elif isinstance(spec, (list, tuple)):
|
|
181
|
+
for v in spec:
|
|
182
|
+
found.extend(r for r in _collect_references(v) if r not in found)
|
|
183
|
+
return found
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _template(spec: Any, where: str) -> Any:
|
|
187
|
+
"""A child's or continued run's input as the engine resolves it: references (`input()`, `output()`, `ref()`) in
|
|
188
|
+
dicts and lists. Literal values are refused: compute a constant in a step and refer to its output."""
|
|
189
|
+
if isinstance(spec, DataRef):
|
|
190
|
+
return spec.expr
|
|
191
|
+
if isinstance(spec, dict):
|
|
192
|
+
return {k: _template(v, f"{where}.{k}") for k, v in spec.items()}
|
|
193
|
+
if isinstance(spec, (list, tuple)):
|
|
194
|
+
return [_template(v, f"{where}[{i}]") for i, v in enumerate(spec)]
|
|
195
|
+
raise ValueError(f"{where}: {spec!r} is not a reference; the engine resolves this input, so only references are sent")
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _callable_name(target: Any) -> Optional[str]:
|
|
199
|
+
if isinstance(target, type):
|
|
200
|
+
return getattr(target, "job_name", None) or target.__name__
|
|
201
|
+
name = getattr(target, "__name__", None) or getattr(type(target), "__name__", None)
|
|
202
|
+
return None if not name or name == "<lambda>" or name == "function" else name
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@dataclass
|
|
206
|
+
class Gate:
|
|
207
|
+
"""Puts a step on one side of a change (versioning): the step runs only for runs whose version of `change` is within
|
|
208
|
+
`min..max` (open ended when None), and is skipped otherwise. A new run takes the change's `max`."""
|
|
209
|
+
change: str
|
|
210
|
+
min: Optional[int] = None
|
|
211
|
+
max: Optional[int] = None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def gate(change: str, *, min: Optional[int] = None, max: Optional[int] = None) -> Gate:
|
|
215
|
+
if min is None and max is None:
|
|
216
|
+
raise ValueError(f"gate on {change!r} needs a min or a max")
|
|
217
|
+
return Gate(change, min, max)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@dataclass
|
|
221
|
+
class UpdateDef:
|
|
222
|
+
name: str
|
|
223
|
+
handler: Optional[Callable[[Any, "ExecutionContext"], Any]] = None
|
|
224
|
+
validate: Optional[DataRef] = None
|
|
225
|
+
message: Optional[str] = None
|
|
226
|
+
input: Any = None
|
|
227
|
+
result: Any = None
|
|
228
|
+
timeout_ms: Optional[int] = None
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@dataclass
|
|
232
|
+
class Compensation:
|
|
233
|
+
handler: Callable[[Any, "ExecutionContext"], Any]
|
|
234
|
+
input: Any = None
|
|
235
|
+
activity_name: Optional[str] = None
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def compensation(
|
|
239
|
+
handler: Union[Callable[[Any, "ExecutionContext"], Any], Type[Any], Any],
|
|
240
|
+
*,
|
|
241
|
+
input: Any = None,
|
|
242
|
+
) -> Compensation:
|
|
243
|
+
fn = _normalize_callable(handler)
|
|
244
|
+
return Compensation(handler=fn, input=input, activity_name=_callable_name(handler))
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _normalize_callable(target: Any) -> Callable[[Any, "ExecutionContext"], Any]:
|
|
248
|
+
if isinstance(target, type) and hasattr(target, "perform"):
|
|
249
|
+
def _invoke_cls(payload: Any, ctx: "ExecutionContext") -> Any:
|
|
250
|
+
inst = target()
|
|
251
|
+
return inst.perform(payload, ctx)
|
|
252
|
+
return _invoke_cls
|
|
253
|
+
if hasattr(target, "perform") and callable(getattr(target, "perform")):
|
|
254
|
+
return getattr(target, "perform")
|
|
255
|
+
if callable(target):
|
|
256
|
+
return target
|
|
257
|
+
raise TypeError(f"Expected callable or Activity/Job with perform(payload, ctx), got {target!r}")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
class Job:
|
|
261
|
+
"""Declarative Job base class mirroring Crystal's `JoobQ::Job`."""
|
|
262
|
+
|
|
263
|
+
job_name: Optional[str] = None
|
|
264
|
+
queue: str = "default"
|
|
265
|
+
retries: int = 3
|
|
266
|
+
expires_ms: Optional[int] = None
|
|
267
|
+
encrypted_fields: List[str] = []
|
|
268
|
+
unique_ttl_ms: Optional[int] = None
|
|
269
|
+
|
|
270
|
+
@classmethod
|
|
271
|
+
def resolved_name(cls) -> str:
|
|
272
|
+
return cls.job_name or cls.__name__
|
|
273
|
+
|
|
274
|
+
@staticmethod
|
|
275
|
+
def perform(payload: Any, ctx: "ExecutionContext") -> Any:
|
|
276
|
+
raise NotImplementedError("Subclasses of Job must implement perform(payload, ctx)")
|
|
277
|
+
|
|
278
|
+
@classmethod
|
|
279
|
+
def enqueue(cls, client: "JoobQClient", payload: Any = None, **kwargs: Any) -> Dict[str, Any]:
|
|
280
|
+
return client.enqueue(cls, payload, **kwargs)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def job(
|
|
284
|
+
name: Optional[str] = None,
|
|
285
|
+
*,
|
|
286
|
+
queue: str = "default",
|
|
287
|
+
retries: int = 3,
|
|
288
|
+
expires_ms: Optional[int] = None,
|
|
289
|
+
encrypted_fields: Optional[List[str]] = None,
|
|
290
|
+
unique_ttl_ms: Optional[int] = None,
|
|
291
|
+
) -> Callable[[Callable[[Any, "ExecutionContext"], Any]], Type[Job]]:
|
|
292
|
+
"""Decorator for defining a `Job` from a function with `JoobQ::Job` metadata."""
|
|
293
|
+
|
|
294
|
+
def _decorator(fn: Callable[[Any, "ExecutionContext"], Any]) -> Type[Job]:
|
|
295
|
+
job_type_name = name or fn.__name__
|
|
296
|
+
attrs = {
|
|
297
|
+
"job_name": job_type_name,
|
|
298
|
+
"queue": queue,
|
|
299
|
+
"retries": retries,
|
|
300
|
+
"expires_ms": expires_ms,
|
|
301
|
+
"encrypted_fields": list(encrypted_fields or []),
|
|
302
|
+
"unique_ttl_ms": unique_ttl_ms,
|
|
303
|
+
"perform": staticmethod(fn),
|
|
304
|
+
}
|
|
305
|
+
return type(job_type_name, (Job,), attrs)
|
|
306
|
+
|
|
307
|
+
return _decorator
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
@dataclass
|
|
311
|
+
class WorkflowStepDef:
|
|
312
|
+
kind: str # "step" | "wait_for" | "timer" | "child" | "parallel" | "sequence"
|
|
313
|
+
name: str
|
|
314
|
+
handler: Optional[Callable[[Any, "ExecutionContext"], Any]] = None
|
|
315
|
+
input_spec: Any = None
|
|
316
|
+
retry: Optional[RetryPolicy] = None
|
|
317
|
+
compensate: Optional[Compensation] = None
|
|
318
|
+
signal_name: Optional[str] = None
|
|
319
|
+
timeout_ms: Optional[int] = None
|
|
320
|
+
duration_ms: Optional[int] = None
|
|
321
|
+
children: List["WorkflowStepDef"] = field(default_factory=list)
|
|
322
|
+
activity_name: Optional[str] = None
|
|
323
|
+
after: List[str] = field(default_factory=list)
|
|
324
|
+
heartbeat_ms: Optional[int] = None
|
|
325
|
+
schedule_to_start_ms: Optional[int] = None
|
|
326
|
+
only_if: Optional[DataRef] = None
|
|
327
|
+
workflow_name: Optional[str] = None
|
|
328
|
+
gate: Optional[Gate] = None
|
|
329
|
+
non_cancellable: bool = False
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
class WorkflowBlueprint:
|
|
333
|
+
"""Declarative workflow builder (the engine conducts the run and its compensations), mirroring `JoobQPro::Workflows::DSL`."""
|
|
334
|
+
|
|
335
|
+
def __init__(
|
|
336
|
+
self,
|
|
337
|
+
name: str,
|
|
338
|
+
*,
|
|
339
|
+
version: int = 1,
|
|
340
|
+
timeout_ms: Optional[int] = None,
|
|
341
|
+
task_queue: str = "workflows",
|
|
342
|
+
) -> None:
|
|
343
|
+
self.name = name
|
|
344
|
+
self.version = version
|
|
345
|
+
self.timeout_ms = timeout_ms
|
|
346
|
+
self.task_queue = task_queue
|
|
347
|
+
self.steps: List[WorkflowStepDef] = []
|
|
348
|
+
self.queries: Dict[str, DataRef] = {}
|
|
349
|
+
self._declared_steps: set[str] = set()
|
|
350
|
+
self.continuation: Optional[Dict[str, Any]] = None
|
|
351
|
+
self.changes: Dict[str, Dict[str, int]] = {}
|
|
352
|
+
self.updates: Dict[str, UpdateDef] = {}
|
|
353
|
+
self._in_scope = False
|
|
354
|
+
|
|
355
|
+
def _validate_and_register(self, step_name: str, input_spec: Any, after: Optional[List[str]] = None) -> None:
|
|
356
|
+
if step_name in self._declared_steps:
|
|
357
|
+
raise ValueError(f"Duplicate workflow step name '{step_name}' in workflow '{self.name}'")
|
|
358
|
+
for dep in after or []:
|
|
359
|
+
if dep not in self._declared_steps:
|
|
360
|
+
raise ValueError(f"Step '{step_name}' in workflow '{self.name}' waits (after=) for unknown or downstream step '{dep}'")
|
|
361
|
+
for dep in _extract_step_dependencies(input_spec):
|
|
362
|
+
if dep not in self._declared_steps:
|
|
363
|
+
raise ValueError(
|
|
364
|
+
f"Step '{step_name}' in workflow '{self.name}' references output of unknown or downstream step '{dep}'"
|
|
365
|
+
)
|
|
366
|
+
self._declared_steps.add(step_name)
|
|
367
|
+
|
|
368
|
+
def query(self, name: str, expr: DataRef) -> "WorkflowBlueprint":
|
|
369
|
+
self.queries[name] = expr
|
|
370
|
+
return self
|
|
371
|
+
|
|
372
|
+
def step(
|
|
373
|
+
self,
|
|
374
|
+
name: str,
|
|
375
|
+
activity: Union[Callable[[Any, "ExecutionContext"], Any], Type[Any], Any],
|
|
376
|
+
*,
|
|
377
|
+
input: Any = None,
|
|
378
|
+
retry: Optional[RetryPolicy] = None,
|
|
379
|
+
compensate: Optional[Compensation] = None,
|
|
380
|
+
after: Optional[List[str]] = None,
|
|
381
|
+
timeout_ms: Optional[int] = None,
|
|
382
|
+
heartbeat_ms: Optional[int] = None,
|
|
383
|
+
schedule_to_start_ms: Optional[int] = None,
|
|
384
|
+
only_if: Optional[DataRef] = None,
|
|
385
|
+
gate: Optional[Gate] = None,
|
|
386
|
+
) -> "WorkflowBlueprint":
|
|
387
|
+
"""A step. `after` names earlier steps it also waits for (on top of what precedes it, e.g. a sibling in a
|
|
388
|
+
`parallel` group); `timeout_ms` limits one attempt, `heartbeat_ms` how long the handler may go without calling
|
|
389
|
+
`ctx.heartbeat()`, `schedule_to_start_ms` how long the step may wait for a worker; `only_if` (a reference) runs
|
|
390
|
+
the step only when it is truthy, else the step is skipped; `gate` (see `gate()`) runs it only on one side of a
|
|
391
|
+
change. The engine enforces them."""
|
|
392
|
+
self._check_condition(name, only_if) # before the name is declared: a step cannot gate on itself
|
|
393
|
+
self._validate_and_register(name, input, after)
|
|
394
|
+
self.steps.append(
|
|
395
|
+
WorkflowStepDef(
|
|
396
|
+
kind="step",
|
|
397
|
+
name=name,
|
|
398
|
+
handler=_normalize_callable(activity),
|
|
399
|
+
activity_name=_callable_name(activity),
|
|
400
|
+
input_spec=input,
|
|
401
|
+
retry=retry,
|
|
402
|
+
compensate=compensate,
|
|
403
|
+
after=list(after or []),
|
|
404
|
+
timeout_ms=timeout_ms,
|
|
405
|
+
heartbeat_ms=heartbeat_ms,
|
|
406
|
+
schedule_to_start_ms=schedule_to_start_ms,
|
|
407
|
+
only_if=only_if,
|
|
408
|
+
gate=gate,
|
|
409
|
+
)
|
|
410
|
+
)
|
|
411
|
+
return self
|
|
412
|
+
|
|
413
|
+
def _check_condition(self, name: str, only_if: Optional[DataRef]) -> None:
|
|
414
|
+
if only_if is None:
|
|
415
|
+
return
|
|
416
|
+
if not isinstance(only_if, DataRef):
|
|
417
|
+
raise ValueError(f"Step '{name}' in workflow '{self.name}': only_if must be a reference (input(), output(), ref())")
|
|
418
|
+
for dep in _extract_step_dependencies(only_if):
|
|
419
|
+
if dep not in self._declared_steps:
|
|
420
|
+
raise ValueError(f"Step '{name}' in workflow '{self.name}': only_if refers to unknown or downstream step '{dep}'")
|
|
421
|
+
|
|
422
|
+
def wait_for(self, name: str, *, signal: str, timeout_ms: int = 600_000,
|
|
423
|
+
after: Optional[List[str]] = None, only_if: Optional[DataRef] = None,
|
|
424
|
+
compensate: Optional[Compensation] = None, gate: Optional[Gate] = None) -> "WorkflowBlueprint":
|
|
425
|
+
"""Waits for a signal. `compensate` undoes it (its input defaults to the signal's payload) when the run fails."""
|
|
426
|
+
self._check_condition(name, only_if)
|
|
427
|
+
self._validate_and_register(name, None, after)
|
|
428
|
+
self.steps.append(
|
|
429
|
+
WorkflowStepDef(kind="wait_for", name=name, signal_name=signal, timeout_ms=timeout_ms, after=list(after or []),
|
|
430
|
+
only_if=only_if, compensate=compensate, gate=gate)
|
|
431
|
+
)
|
|
432
|
+
return self
|
|
433
|
+
|
|
434
|
+
def timer(self, name: str, *, for_ms: int, after: Optional[List[str]] = None, only_if: Optional[DataRef] = None,
|
|
435
|
+
compensate: Optional[Compensation] = None, gate: Optional[Gate] = None) -> "WorkflowBlueprint":
|
|
436
|
+
"""Waits for a time. `compensate` undoes it (its input defaults to the timer's output) when the run fails."""
|
|
437
|
+
self._check_condition(name, only_if)
|
|
438
|
+
self._validate_and_register(name, None, after)
|
|
439
|
+
self.steps.append(WorkflowStepDef(kind="timer", name=name, duration_ms=for_ms, after=list(after or []),
|
|
440
|
+
only_if=only_if, compensate=compensate, gate=gate))
|
|
441
|
+
return self
|
|
442
|
+
|
|
443
|
+
def child(self, name: str, workflow_def: Union["WorkflowBlueprint", str], *, input: Any = None,
|
|
444
|
+
compensate: Optional[Compensation] = None, after: Optional[List[str]] = None,
|
|
445
|
+
only_if: Optional[DataRef] = None, gate: Optional[Gate] = None) -> "WorkflowBlueprint":
|
|
446
|
+
"""Runs another workflow (a blueprint or its name) as a step; its output is the child run's output, its failure
|
|
447
|
+
the step's. The engine starts the child and resolves `input` (references only: `input()`, `output()`, `ref()`,
|
|
448
|
+
in dicts and lists; the run's input when omitted). A child is not retried: give the child's own steps retries,
|
|
449
|
+
and the child workflow a `timeout_ms`. The child workflow must be registered before a run reaches this step."""
|
|
450
|
+
if input is not None:
|
|
451
|
+
_template(input, f"{name}.input")
|
|
452
|
+
self._check_condition(name, only_if)
|
|
453
|
+
self._validate_and_register(name, input, after)
|
|
454
|
+
child_name = workflow_def.name if isinstance(workflow_def, WorkflowBlueprint) else str(workflow_def)
|
|
455
|
+
self.steps.append(WorkflowStepDef(kind="child", name=name, workflow_name=child_name, input_spec=input,
|
|
456
|
+
compensate=compensate, after=list(after or []), only_if=only_if, gate=gate))
|
|
457
|
+
return self
|
|
458
|
+
|
|
459
|
+
def continue_as_new(self, *, input: Any = None, only_if: Optional[DataRef] = None, max_runs: Optional[int] = None,
|
|
460
|
+
max_events: Optional[int] = None) -> "WorkflowBlueprint":
|
|
461
|
+
"""When the run ends (and `only_if` is truthy), it closes `continued_as_new` and a fresh run of this workflow starts
|
|
462
|
+
with `input` (references only, resolved by the engine; the same input when omitted), at most `max_runs` runs."""
|
|
463
|
+
policy: Dict[str, Any] = {}
|
|
464
|
+
if input is not None:
|
|
465
|
+
policy["input"] = _template(input, "continue_as_new.input")
|
|
466
|
+
if only_if is not None:
|
|
467
|
+
if not isinstance(only_if, DataRef):
|
|
468
|
+
raise ValueError("continue_as_new: only_if must be a reference (input(), output(), ref())")
|
|
469
|
+
policy["only_if"] = only_if.expr
|
|
470
|
+
if max_runs is not None:
|
|
471
|
+
policy["max_runs"] = max_runs
|
|
472
|
+
if max_events is not None:
|
|
473
|
+
policy["max_events"] = max_events
|
|
474
|
+
self.continuation = policy
|
|
475
|
+
return self
|
|
476
|
+
|
|
477
|
+
def change(self, change_id: str, *, min: int = 0, max: int = 1) -> "WorkflowBlueprint":
|
|
478
|
+
"""Declares a change (versioning, like Temporal's getVersion): runs that reach a gated step fresh take `max`,
|
|
479
|
+
runs that went past it on the old path keep 0; steps pick a side with `gate=gate(change_id, min=..., max=...)`.
|
|
480
|
+
Raise `min` only once no open run is on an older path."""
|
|
481
|
+
if min > max:
|
|
482
|
+
raise ValueError(f"change {change_id!r} in workflow {self.name!r}: min is above max")
|
|
483
|
+
self.changes[change_id] = {"min": min, "max": max}
|
|
484
|
+
return self
|
|
485
|
+
|
|
486
|
+
def update(self, name: str, handler: Optional[Union[Callable[[Any, "ExecutionContext"], Any], Type[Any], Any]] = None, *,
|
|
487
|
+
validate: Optional[DataRef] = None, message: Optional[str] = None, input: Any = None, result: Any = None,
|
|
488
|
+
timeout_ms: Optional[int] = None) -> "WorkflowBlueprint":
|
|
489
|
+
"""An update handler (`JoobQClient.update_workflow`): `validate` (a reference, e.g. `update_input("name")`) must be
|
|
490
|
+
truthy or the update is rejected with `message`; then `handler` runs as a step of the run (its payload is `input`,
|
|
491
|
+
by default the update's input, and its return value is the result), or, without one, `result` (a template of
|
|
492
|
+
references) is the result. References may read the run (`input()`, `output()`) and `update_input()`."""
|
|
493
|
+
if name in self.updates:
|
|
494
|
+
raise ValueError(f"Duplicate update {name!r} in workflow {self.name!r}")
|
|
495
|
+
if validate is not None and not isinstance(validate, DataRef):
|
|
496
|
+
raise ValueError(f"update {name!r}: validate must be a reference (update_input(), input(), output(), ref())")
|
|
497
|
+
if input is not None:
|
|
498
|
+
_template(input, f"update {name}.input")
|
|
499
|
+
if result is not None:
|
|
500
|
+
_template(result, f"update {name}.result")
|
|
501
|
+
fn = _normalize_callable(handler) if handler is not None else None
|
|
502
|
+
self.updates[name] = UpdateDef(name, fn, validate, message, input, result, timeout_ms)
|
|
503
|
+
return self
|
|
504
|
+
|
|
505
|
+
def scope(self, name: str, build_fn: Callable[["WorkflowBlueprint"], Any], *, timeout_ms: Optional[int] = None,
|
|
506
|
+
non_cancellable: bool = False) -> "WorkflowBlueprint":
|
|
507
|
+
"""A cancellation scope: its steps run in sequence and can be cancelled as a unit (from the console or API,
|
|
508
|
+
or after `timeout_ms` from its first step's start) while the rest of the run goes on; a cancelled step counts as
|
|
509
|
+
finished with a null output. A `non_cancellable` scope holds cleanup that still runs when the workflow is cancelled,
|
|
510
|
+
times out or fails (it cannot have a timeout). A running step sees a cancel through `ctx.heartbeat()`.
|
|
511
|
+
Scopes do not nest."""
|
|
512
|
+
if self._in_scope:
|
|
513
|
+
raise ValueError(f"scope {name!r} in workflow {self.name!r} is inside another scope: scopes do not nest")
|
|
514
|
+
if non_cancellable and timeout_ms is not None:
|
|
515
|
+
raise ValueError(f"scope {name!r} is non_cancellable and cannot time out")
|
|
516
|
+
inner = WorkflowBlueprint(f"{self.name}:scope", version=self.version, timeout_ms=self.timeout_ms, task_queue=self.task_queue)
|
|
517
|
+
inner._declared_steps = set(self._declared_steps)
|
|
518
|
+
inner._in_scope = True
|
|
519
|
+
build_fn(inner)
|
|
520
|
+
if not inner.steps:
|
|
521
|
+
raise ValueError(f"scope {name!r} in workflow {self.name!r} has no steps")
|
|
522
|
+
self._declared_steps.update(inner._declared_steps)
|
|
523
|
+
self.steps.append(WorkflowStepDef(kind="scope", name=name, children=inner.steps, timeout_ms=timeout_ms,
|
|
524
|
+
non_cancellable=non_cancellable))
|
|
525
|
+
return self
|
|
526
|
+
|
|
527
|
+
def parallel(self, build_fn: Callable[["WorkflowBlueprint"], Any]) -> "WorkflowBlueprint":
|
|
528
|
+
branch = WorkflowBlueprint(f"{self.name}:parallel", version=self.version, timeout_ms=self.timeout_ms, task_queue=self.task_queue)
|
|
529
|
+
branch._declared_steps = set(self._declared_steps)
|
|
530
|
+
branch._in_scope = self._in_scope
|
|
531
|
+
build_fn(branch)
|
|
532
|
+
self._declared_steps.update(branch._declared_steps)
|
|
533
|
+
self.steps.append(WorkflowStepDef(kind="parallel", name=f"parallel_{len(self.steps) + 1}", children=branch.steps))
|
|
534
|
+
return self
|
|
535
|
+
|
|
536
|
+
def sequence(self, build_fn: Callable[["WorkflowBlueprint"], Any]) -> "WorkflowBlueprint":
|
|
537
|
+
seq = WorkflowBlueprint(f"{self.name}:sequence", version=self.version, timeout_ms=self.timeout_ms, task_queue=self.task_queue)
|
|
538
|
+
seq._declared_steps = set(self._declared_steps)
|
|
539
|
+
seq._in_scope = self._in_scope
|
|
540
|
+
build_fn(seq)
|
|
541
|
+
self._declared_steps.update(seq._declared_steps)
|
|
542
|
+
self.steps.append(WorkflowStepDef(kind="sequence", name=f"sequence_{len(self.steps) + 1}", children=seq.steps))
|
|
543
|
+
return self
|
|
544
|
+
|
|
545
|
+
def to_blueprint(self) -> Dict[str, Any]:
|
|
546
|
+
"""The workflow's structure as data, as the worker registers it with the gateway (`CONTRACT.md`)."""
|
|
547
|
+
|
|
548
|
+
def node(s: WorkflowStepDef) -> Dict[str, Any]:
|
|
549
|
+
out: Dict[str, Any] = {"name": s.name, "kind": s.kind}
|
|
550
|
+
if s.kind in ("parallel", "sequence", "scope"):
|
|
551
|
+
out["children"] = [node(c) for c in s.children]
|
|
552
|
+
if s.kind == "scope":
|
|
553
|
+
if s.timeout_ms is not None:
|
|
554
|
+
out["timeout_ms"] = s.timeout_ms
|
|
555
|
+
if s.non_cancellable:
|
|
556
|
+
out["non_cancellable"] = True
|
|
557
|
+
elif s.kind == "wait_for":
|
|
558
|
+
out.update({"signal": s.signal_name, "timeout_ms": s.timeout_ms})
|
|
559
|
+
elif s.kind == "timer":
|
|
560
|
+
out["for_ms"] = s.duration_ms
|
|
561
|
+
elif s.kind == "child":
|
|
562
|
+
out["workflow"] = s.workflow_name
|
|
563
|
+
if s.input_spec is not None:
|
|
564
|
+
out["input"] = _template(s.input_spec, f"{s.name}.input")
|
|
565
|
+
else:
|
|
566
|
+
out["activity"] = s.activity_name
|
|
567
|
+
out["references"] = _collect_references(s.input_spec) if s.input_spec is not None else ["${input}"]
|
|
568
|
+
for key, value in (("timeout_ms", s.timeout_ms), ("heartbeat_ms", s.heartbeat_ms),
|
|
569
|
+
("schedule_to_start_ms", s.schedule_to_start_ms)):
|
|
570
|
+
if value is not None:
|
|
571
|
+
out[key] = value
|
|
572
|
+
if s.retry is not None:
|
|
573
|
+
out["retry"] = {"attempts": s.retry.attempts, "initial_ms": s.retry.initial_ms,
|
|
574
|
+
"backoff": s.retry.backoff, "maximum_ms": s.retry.maximum_ms}
|
|
575
|
+
if s.retry.non_retryable:
|
|
576
|
+
out["retry"]["non_retryable"] = list(s.retry.non_retryable)
|
|
577
|
+
if s.kind in ("step", "wait_for", "timer", "child"):
|
|
578
|
+
if s.compensate is not None:
|
|
579
|
+
comp_in = s.compensate.input if s.compensate.input is not None else output(s.name)
|
|
580
|
+
out["compensate"] = {"activity": s.compensate.activity_name, "references": _collect_references(comp_in)}
|
|
581
|
+
if s.only_if is not None:
|
|
582
|
+
out["only_if"] = s.only_if.expr
|
|
583
|
+
if s.gate is not None:
|
|
584
|
+
out["gate"] = {"change": s.gate.change, **({"min": s.gate.min} if s.gate.min is not None else {}),
|
|
585
|
+
**({"max": s.gate.max} if s.gate.max is not None else {})}
|
|
586
|
+
if s.after:
|
|
587
|
+
out["after"] = list(s.after)
|
|
588
|
+
return out
|
|
589
|
+
|
|
590
|
+
blueprint: Dict[str, Any] = {
|
|
591
|
+
"name": self.name,
|
|
592
|
+
"version": self.version,
|
|
593
|
+
"task_queue": self.task_queue,
|
|
594
|
+
"queries": {k: v.to_expr() for k, v in self.queries.items()},
|
|
595
|
+
"nodes": [node(s) for s in self.steps],
|
|
596
|
+
}
|
|
597
|
+
if self.timeout_ms is not None:
|
|
598
|
+
blueprint["timeout_ms"] = self.timeout_ms # the whole run's deadline, enforced by the engine
|
|
599
|
+
if self.continuation is not None:
|
|
600
|
+
blueprint["continue_as_new"] = dict(self.continuation)
|
|
601
|
+
if self.changes:
|
|
602
|
+
blueprint["changes"] = [{"id": cid, **bounds} for cid, bounds in self.changes.items()]
|
|
603
|
+
if self.updates:
|
|
604
|
+
blueprint["updates"] = {u.name: _update_spec(u) for u in self.updates.values()}
|
|
605
|
+
return blueprint
|
|
606
|
+
|
|
607
|
+
def activities(self) -> Dict[str, Callable[[Any, "ExecutionContext"], Any]]:
|
|
608
|
+
"""The handlers of this workflow's steps, by the activity name the engine sends them under (`<workflow>/<step>`,
|
|
609
|
+
compensations `<workflow>/<step>/compensate`, update handlers `<workflow>/@update/<name>`). Each runs its step
|
|
610
|
+
once: the engine conducts the run (it retries, compensates and waits). A step's input is resolved against the
|
|
611
|
+
run's input (the task's payload) and the outputs of the steps that completed (the task's checkpoints)."""
|
|
612
|
+
found: Dict[str, Callable[[Any, "ExecutionContext"], Any]] = {}
|
|
613
|
+
|
|
614
|
+
def step_activity(node: WorkflowStepDef) -> Callable[[Any, "ExecutionContext"], Any]:
|
|
615
|
+
def run(workflow_input: Any, ctx: "ExecutionContext") -> Any:
|
|
616
|
+
assert node.handler is not None
|
|
617
|
+
return node.handler(resolve_references(node.input_spec, workflow_input, ctx.task.checkpoints), ctx)
|
|
618
|
+
return run
|
|
619
|
+
|
|
620
|
+
def compensation_activity(node: WorkflowStepDef, comp: Compensation) -> Callable[[Any, "ExecutionContext"], Any]:
|
|
621
|
+
def run(workflow_input: Any, ctx: "ExecutionContext") -> Any:
|
|
622
|
+
spec = comp.input if comp.input is not None else output(node.name)
|
|
623
|
+
return comp.handler(resolve_references(spec, workflow_input, ctx.task.checkpoints), ctx)
|
|
624
|
+
return run
|
|
625
|
+
|
|
626
|
+
def update_activity(update: UpdateDef) -> Callable[[Any, "ExecutionContext"], Any]:
|
|
627
|
+
def run(update_payload: Any, ctx: "ExecutionContext") -> Any:
|
|
628
|
+
assert update.handler is not None
|
|
629
|
+
return update.handler(update_payload, ctx) # the engine resolved the handler's input already
|
|
630
|
+
return run
|
|
631
|
+
|
|
632
|
+
def visit(nodes: List[WorkflowStepDef]) -> None:
|
|
633
|
+
for node in nodes:
|
|
634
|
+
if node.kind in ("sequence", "parallel", "scope"):
|
|
635
|
+
visit(node.children)
|
|
636
|
+
else:
|
|
637
|
+
if node.kind == "step":
|
|
638
|
+
found[_activity_name(self.name, node.name)] = step_activity(node)
|
|
639
|
+
if node.compensate is not None: # a step, wait, timer or child can be undone
|
|
640
|
+
found[_activity_name(self.name, node.name, True)] = compensation_activity(node, node.compensate)
|
|
641
|
+
|
|
642
|
+
visit(self.steps)
|
|
643
|
+
for update in self.updates.values():
|
|
644
|
+
if update.handler is not None:
|
|
645
|
+
found[_update_activity(self.name, update.name)] = update_activity(update)
|
|
646
|
+
return found
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def _update_spec(update: UpdateDef) -> Dict[str, Any]:
|
|
650
|
+
spec: Dict[str, Any] = {}
|
|
651
|
+
if update.handler is not None:
|
|
652
|
+
spec["handler"] = True
|
|
653
|
+
if update.validate is not None:
|
|
654
|
+
spec["validate"] = update.validate.expr
|
|
655
|
+
if update.message:
|
|
656
|
+
spec["message"] = update.message
|
|
657
|
+
if update.input is not None:
|
|
658
|
+
spec["input"] = _template(update.input, f"update {update.name}.input")
|
|
659
|
+
if update.result is not None:
|
|
660
|
+
spec["result"] = _template(update.result, f"update {update.name}.result")
|
|
661
|
+
if update.timeout_ms is not None:
|
|
662
|
+
spec["timeout_ms"] = update.timeout_ms
|
|
663
|
+
return spec
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _update_activity(workflow_name: str, update: str) -> str:
|
|
667
|
+
"""The activity the engine sends an update handler of an SDK workflow under."""
|
|
668
|
+
return f"{workflow_name}/@update/{update}"
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def _activity_name(workflow_name: str, step: str, compensate: bool = False) -> str:
|
|
672
|
+
"""The activity the engine sends a step (or its compensation) of an SDK workflow under."""
|
|
673
|
+
return f"{workflow_name}/{step}/compensate" if compensate else f"{workflow_name}/{step}"
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def workflow(
|
|
677
|
+
name: str,
|
|
678
|
+
*,
|
|
679
|
+
version: int = 1,
|
|
680
|
+
timeout_ms: Optional[int] = None,
|
|
681
|
+
task_queue: str = "workflows",
|
|
682
|
+
) -> WorkflowBlueprint:
|
|
683
|
+
"""A workflow blueprint. `timeout_ms` is the whole run's deadline (none by default), enforced by the engine."""
|
|
684
|
+
return WorkflowBlueprint(name, version=version, timeout_ms=timeout_ms, task_queue=task_queue)
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
@dataclass
|
|
688
|
+
class LeasedTask:
|
|
689
|
+
lease_token: str
|
|
690
|
+
kind: str
|
|
691
|
+
id: str
|
|
692
|
+
queue: str
|
|
693
|
+
name: str
|
|
694
|
+
attempt: int
|
|
695
|
+
max_retries: int
|
|
696
|
+
payload: Any
|
|
697
|
+
context: Dict[str, Any] = field(default_factory=dict)
|
|
698
|
+
workflow_id: Optional[str] = None
|
|
699
|
+
step_id: Optional[str] = None
|
|
700
|
+
checkpoints: Dict[str, Any] = field(default_factory=dict) # a step's: the outputs of the run's completed steps
|
|
701
|
+
lease_until_ms: int = 0
|
|
702
|
+
heartbeat_ms: Optional[int] = None # an engine step's heartbeat timeout: call ctx.heartbeat() more often than this
|
|
703
|
+
|
|
704
|
+
@classmethod
|
|
705
|
+
def from_dict(cls, raw: Dict[str, Any]) -> "LeasedTask":
|
|
706
|
+
known = {f for f in cls.__dataclass_fields__}
|
|
707
|
+
return cls(**{k: v for k, v in raw.items() if k in known})
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
class ExecutionContext:
|
|
711
|
+
"""Carries the propagated request `context`, heartbeating, and downstream context-forwarding helpers. Handlers
|
|
712
|
+
reach the application's own dependencies directly: the worker runs inside the booted application."""
|
|
713
|
+
|
|
714
|
+
def __init__(self, task: LeasedTask, worker_id: str, client: "JoobQClient", lease_ttl_ms: int = 30000,
|
|
715
|
+
transport: Optional["_Transport"] = None) -> None:
|
|
716
|
+
self.task = task
|
|
717
|
+
self.worker_id = worker_id
|
|
718
|
+
self.client = client
|
|
719
|
+
self.lease_ttl_ms = lease_ttl_ms
|
|
720
|
+
self._transport = transport
|
|
721
|
+
self.context: Dict[str, Any] = dict(task.context or {})
|
|
722
|
+
self._last_beat_ms = float("-inf")
|
|
723
|
+
self._beat_cancel = False
|
|
724
|
+
|
|
725
|
+
@property
|
|
726
|
+
def attempt(self) -> int:
|
|
727
|
+
return self.task.attempt
|
|
728
|
+
|
|
729
|
+
@property
|
|
730
|
+
def dedupe_key(self) -> str:
|
|
731
|
+
wf = self.task.workflow_id or self.task.id
|
|
732
|
+
step = self.task.step_id or self.task.name
|
|
733
|
+
return f"{wf}:{step}:{self.task.attempt}"
|
|
734
|
+
|
|
735
|
+
def heartbeat(self, lease_ttl_ms: Optional[int] = None) -> bool:
|
|
736
|
+
"""True when the run was cancelled. Inside a running worker this answers what the worker's last lease renewal
|
|
737
|
+
said, without waiting for the gateway (the worker renews every lease on its own). For a step with
|
|
738
|
+
`heartbeat_ms` it also tells the engine the step is alive (at most every third of `heartbeat_ms`): the worker's
|
|
739
|
+
own renewals do not, so a handler that stops calling this is timed out."""
|
|
740
|
+
if self.task.heartbeat_ms:
|
|
741
|
+
now = time.monotonic() * 1000
|
|
742
|
+
if now - self._last_beat_ms >= max(100, self.task.heartbeat_ms // 3):
|
|
743
|
+
self._last_beat_ms = now
|
|
744
|
+
try:
|
|
745
|
+
res = self.client.heartbeat_task(self.task.lease_token, self.worker_id, lease_ttl_ms or self.lease_ttl_ms)
|
|
746
|
+
self._beat_cancel = bool(res.get("cancel_requested", False))
|
|
747
|
+
except Exception:
|
|
748
|
+
pass # the next window's beat tries again; the engine's deadline decides
|
|
749
|
+
if self._beat_cancel:
|
|
750
|
+
return True
|
|
751
|
+
if self._transport is not None:
|
|
752
|
+
return self._transport.cancel_requested(self.task.lease_token)
|
|
753
|
+
res = self.client.heartbeat_task(self.task.lease_token, self.worker_id, lease_ttl_ms or self.lease_ttl_ms)
|
|
754
|
+
return bool(res.get("cancel_requested", False))
|
|
755
|
+
|
|
756
|
+
def enqueue(self, job_cls: Type[Job], payload: Any = None, **kwargs: Any) -> Dict[str, Any]:
|
|
757
|
+
merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
|
|
758
|
+
return self.client.enqueue(job_cls, payload, context=merged_ctx, **kwargs)
|
|
759
|
+
|
|
760
|
+
def enqueue_job(self, queue: str, job_type: str, payload: Any = None, **kwargs: Any) -> Dict[str, Any]:
|
|
761
|
+
merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
|
|
762
|
+
return self.client.enqueue_job(queue, job_type, payload, context=merged_ctx, **kwargs)
|
|
763
|
+
|
|
764
|
+
def start(self, workflow_def: WorkflowBlueprint, input_data: Any = None, **kwargs: Any) -> Dict[str, Any]:
|
|
765
|
+
merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
|
|
766
|
+
return self.client.start(workflow_def, input_data, context=merged_ctx, **kwargs)
|
|
767
|
+
|
|
768
|
+
def start_workflow(self, name: str, input_data: Any = None, **kwargs: Any) -> Dict[str, Any]:
|
|
769
|
+
merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
|
|
770
|
+
return self.client.start_workflow(name, input_data, context=merged_ctx, **kwargs)
|
|
771
|
+
|
|
772
|
+
def register_schedule(self, name: str, job_type: str, queue: str, **kwargs: Any) -> Dict[str, Any]:
|
|
773
|
+
return self.client.register_schedule(name, job_type, queue, **kwargs)
|
|
774
|
+
|
|
775
|
+
def rate_limit(self, key: str, limit: int, period_ms: int) -> Dict[str, Any]:
|
|
776
|
+
return self.client.rate_limit(key, limit, period_ms)
|
|
777
|
+
|
|
778
|
+
def create_batch(self, jobs: List[Any], **kwargs: Any) -> Dict[str, Any]:
|
|
779
|
+
"""`JoobQClient.create_batch`; the members and callbacks carry this handler's context."""
|
|
780
|
+
merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
|
|
781
|
+
return self.client.create_batch(jobs, context=merged_ctx, **kwargs)
|
|
782
|
+
|
|
783
|
+
def get_batch(self, batch_id: str) -> Dict[str, Any]:
|
|
784
|
+
return self.client.get_batch(batch_id)
|
|
785
|
+
|
|
786
|
+
def describe_workflow(self, workflow_id: str) -> Dict[str, Any]:
|
|
787
|
+
return self.client.describe_workflow(workflow_id)
|
|
788
|
+
|
|
789
|
+
def workflow_result(self, workflow_id: str, wait_ms: int = 0) -> Dict[str, Any]:
|
|
790
|
+
return self.client.workflow_result(workflow_id, wait_ms)
|
|
791
|
+
|
|
792
|
+
def workflow_history(self, workflow_id: str, after_seq: int = 0, limit: int = 200) -> List[Dict[str, Any]]:
|
|
793
|
+
return self.client.workflow_history(workflow_id, after_seq, limit)
|
|
794
|
+
|
|
795
|
+
def query_workflow(self, workflow_id: str, name: str) -> Any:
|
|
796
|
+
return self.client.query_workflow(workflow_id, name)
|
|
797
|
+
|
|
798
|
+
def signal_workflow(self, workflow_id: str, name: str, payload: Any = None) -> Dict[str, Any]:
|
|
799
|
+
return self.client.signal_workflow(workflow_id, name, payload)
|
|
800
|
+
|
|
801
|
+
def update_workflow(self, workflow_id: str, name: str, input: Any = None, *, update_id: Optional[str] = None,
|
|
802
|
+
wait_ms: Optional[int] = None) -> Dict[str, Any]:
|
|
803
|
+
return self.client.update_workflow(workflow_id, name, input, update_id=update_id, wait_ms=wait_ms)
|
|
804
|
+
|
|
805
|
+
def upsert_workflow_attributes(self, workflow_id: str, set: Optional[Dict[str, Any]] = None,
|
|
806
|
+
remove: Optional[List[str]] = None) -> Dict[str, Any]:
|
|
807
|
+
return self.client.upsert_workflow_attributes(workflow_id, set, remove)
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def _seg(value: str) -> str:
|
|
811
|
+
"""A workflow id as one URL path segment."""
|
|
812
|
+
return urllib.parse.quote(str(value), safe="")
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def failure(exc: BaseException, retryable: bool = True) -> Dict[str, Any]:
|
|
816
|
+
"""The fields that report `exc` to the gateway (`fail`, and a `fail` item of `ack_tasks`)."""
|
|
817
|
+
frames = [
|
|
818
|
+
{"file": f.filename, "line": f.lineno or 0, "function": f.name}
|
|
819
|
+
for f in traceback.extract_tb(exc.__traceback__)
|
|
820
|
+
]
|
|
821
|
+
return {
|
|
822
|
+
"error_class": exc.__class__.__name__,
|
|
823
|
+
"error_message": (str(exc) or exc.__class__.__name__)[:4000],
|
|
824
|
+
"retryable": retryable,
|
|
825
|
+
"sdk_language": "python",
|
|
826
|
+
"stack_frames": frames,
|
|
827
|
+
"raw_stacktrace": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))[-4000:],
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
class GatewayError(Exception):
|
|
832
|
+
"""The gateway answered with an error status (`status`, the JSON error envelope as `body`)."""
|
|
833
|
+
|
|
834
|
+
def __init__(self, status: int, body: str) -> None:
|
|
835
|
+
super().__init__(f"gateway answered {status}: {body[:300]}")
|
|
836
|
+
self.status = status
|
|
837
|
+
self.body = body
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
class JoobQClient:
|
|
841
|
+
def __init__(
|
|
842
|
+
self,
|
|
843
|
+
gateway_url: str,
|
|
844
|
+
token: str,
|
|
845
|
+
context_provider: Optional[Callable[[], Dict[str, Any]]] = None,
|
|
846
|
+
) -> None:
|
|
847
|
+
self.gateway_url = gateway_url.rstrip("/")
|
|
848
|
+
self.token = token
|
|
849
|
+
self.context_provider = context_provider
|
|
850
|
+
self._url = urllib.parse.urlsplit(self.gateway_url)
|
|
851
|
+
self._local = threading.local() # one kept-alive connection per thread
|
|
852
|
+
# Seconds one gateway call may take. A worker lowers it (`JoobQWorker`): a call that hangs for longer than a
|
|
853
|
+
# lease would let every lease the worker holds expire while its tasks still run.
|
|
854
|
+
self.request_timeout: float = 60.0
|
|
855
|
+
self._async: Optional[ThreadPoolExecutor] = None
|
|
856
|
+
self._async_lock = threading.Lock()
|
|
857
|
+
|
|
858
|
+
def _resolve_context(self, explicit: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
859
|
+
ambient = self.context_provider() if self.context_provider else {}
|
|
860
|
+
return {**(ambient or {}), **(explicit or {})}
|
|
861
|
+
|
|
862
|
+
def enqueue(
|
|
863
|
+
self,
|
|
864
|
+
job_cls: Type[Job],
|
|
865
|
+
payload: Any = None,
|
|
866
|
+
*,
|
|
867
|
+
context: Optional[Dict[str, Any]] = None,
|
|
868
|
+
jid: Optional[str] = None,
|
|
869
|
+
delay_ms: int = 0,
|
|
870
|
+
run_at_ms: Optional[int] = None,
|
|
871
|
+
expires_ms: Optional[int] = None,
|
|
872
|
+
timeout_ms: Optional[int] = None,
|
|
873
|
+
unique_key: Optional[str] = None,
|
|
874
|
+
unique_mode: str = "while_locked",
|
|
875
|
+
) -> Dict[str, Any]:
|
|
876
|
+
return self.enqueue_job(
|
|
877
|
+
queue=job_cls.queue,
|
|
878
|
+
job_type=job_cls.resolved_name(),
|
|
879
|
+
payload=payload,
|
|
880
|
+
context=context,
|
|
881
|
+
jid=jid,
|
|
882
|
+
max_retries=job_cls.retries,
|
|
883
|
+
delay_ms=delay_ms,
|
|
884
|
+
run_at_ms=run_at_ms,
|
|
885
|
+
expires_ms=expires_ms if expires_ms is not None else job_cls.expires_ms,
|
|
886
|
+
timeout_ms=timeout_ms,
|
|
887
|
+
encrypt_fields=list(job_cls.encrypted_fields or []),
|
|
888
|
+
unique_key=unique_key,
|
|
889
|
+
unique_ttl_ms=job_cls.unique_ttl_ms,
|
|
890
|
+
unique_mode=unique_mode,
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
def enqueue_job(
|
|
894
|
+
self,
|
|
895
|
+
queue: str,
|
|
896
|
+
job_type: str,
|
|
897
|
+
payload: Any = None,
|
|
898
|
+
*,
|
|
899
|
+
context: Optional[Dict[str, Any]] = None,
|
|
900
|
+
jid: Optional[str] = None,
|
|
901
|
+
max_retries: int = 3,
|
|
902
|
+
delay_ms: int = 0,
|
|
903
|
+
run_at_ms: Optional[int] = None,
|
|
904
|
+
expires_ms: Optional[int] = None,
|
|
905
|
+
timeout_ms: Optional[int] = None,
|
|
906
|
+
encrypt_fields: Optional[List[str]] = None,
|
|
907
|
+
unique_key: Optional[str] = None,
|
|
908
|
+
unique_ttl_ms: Optional[int] = None,
|
|
909
|
+
unique_mode: str = "while_locked",
|
|
910
|
+
) -> Dict[str, Any]:
|
|
911
|
+
return self._post("/api/v1/sdk/jobs", self._job_body(queue, job_type, payload, context=context, jid=jid,
|
|
912
|
+
max_retries=max_retries, delay_ms=delay_ms,
|
|
913
|
+
run_at_ms=run_at_ms, expires_ms=expires_ms,
|
|
914
|
+
timeout_ms=timeout_ms,
|
|
915
|
+
encrypt_fields=encrypt_fields, unique_key=unique_key,
|
|
916
|
+
unique_ttl_ms=unique_ttl_ms, unique_mode=unique_mode))
|
|
917
|
+
|
|
918
|
+
def _job_body(self, queue: str, job_type: str, payload: Any = None, *, context: Optional[Dict[str, Any]] = None,
|
|
919
|
+
jid: Optional[str] = None, max_retries: int = 3, delay_ms: int = 0,
|
|
920
|
+
run_at_ms: Optional[int] = None, expires_ms: Optional[int] = None, timeout_ms: Optional[int] = None,
|
|
921
|
+
encrypt_fields: Optional[List[str]] = None, unique_key: Optional[str] = None,
|
|
922
|
+
unique_ttl_ms: Optional[int] = None, unique_mode: str = "while_locked") -> Dict[str, Any]:
|
|
923
|
+
return {
|
|
924
|
+
"queue": queue,
|
|
925
|
+
"job_type": job_type,
|
|
926
|
+
"payload": payload if payload is not None else {},
|
|
927
|
+
"context": self._resolve_context(context),
|
|
928
|
+
"jid": jid,
|
|
929
|
+
"max_retries": max_retries,
|
|
930
|
+
"delay_ms": delay_ms,
|
|
931
|
+
"run_at_ms": run_at_ms,
|
|
932
|
+
"expires_ms": expires_ms,
|
|
933
|
+
"timeout_ms": timeout_ms,
|
|
934
|
+
"encrypt_fields": encrypt_fields or [],
|
|
935
|
+
"unique_key": unique_key,
|
|
936
|
+
"unique_ttl_ms": unique_ttl_ms,
|
|
937
|
+
"unique_mode": unique_mode,
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
def enqueue_many(self, jobs: List[Any], *, context: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
|
|
941
|
+
"""Enqueues many jobs with `POST /sdk/jobs/batch` (500 per call). Each item is `(JobClass, payload)`,
|
|
942
|
+
`(JobClass, payload, {"delay_ms": ..., "jid": ..., "unique_key": ...})` or a dict of `enqueue_job` arguments
|
|
943
|
+
(`queue`, `job_type`, `payload`, ...). Answers one `{jid, queue, status}` per job, in order."""
|
|
944
|
+
bodies = [self._item_body(item, context) for item in jobs]
|
|
945
|
+
out: List[Dict[str, Any]] = []
|
|
946
|
+
for start in range(0, len(bodies), 500):
|
|
947
|
+
out.extend(self._post("/api/v1/sdk/jobs/batch", {"jobs": bodies[start:start + 500]}).get("jobs", []))
|
|
948
|
+
return out
|
|
949
|
+
|
|
950
|
+
def _item_body(self, item: Any, context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
951
|
+
"""One job of `enqueue_many` or `create_batch`: a dict of `enqueue_job` arguments or `(JobClass, payload[, opts])`."""
|
|
952
|
+
if isinstance(item, dict):
|
|
953
|
+
spec = dict(item)
|
|
954
|
+
spec.setdefault("context", context)
|
|
955
|
+
return self._job_body(spec.pop("queue"), spec.pop("job_type"), spec.pop("payload", None), **spec)
|
|
956
|
+
job_cls, payload, opts = (tuple(item) + ({},))[:3]
|
|
957
|
+
return self._job_body(job_cls.queue, job_cls.resolved_name(), payload,
|
|
958
|
+
context=opts.get("context", context), jid=opts.get("jid"),
|
|
959
|
+
max_retries=job_cls.retries, delay_ms=opts.get("delay_ms", 0),
|
|
960
|
+
run_at_ms=opts.get("run_at_ms"),
|
|
961
|
+
expires_ms=opts.get("expires_ms", job_cls.expires_ms),
|
|
962
|
+
timeout_ms=opts.get("timeout_ms"),
|
|
963
|
+
encrypt_fields=list(job_cls.encrypted_fields or []),
|
|
964
|
+
unique_key=opts.get("unique_key"), unique_ttl_ms=job_cls.unique_ttl_ms,
|
|
965
|
+
unique_mode=opts.get("unique_mode", "while_locked"))
|
|
966
|
+
|
|
967
|
+
def create_batch(self, jobs: List[Any], *, on_complete: Any = None, on_success: Any = None, on_failure: Any = None,
|
|
968
|
+
description: Optional[str] = None, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
969
|
+
"""Creates a Pro batch with `POST /sdk/batches`: 1 to 1000 member jobs (the shapes `enqueue_many` takes, without a
|
|
970
|
+
`unique_key`) and callback jobs (the same shapes). `on_failure` is enqueued at the first dead member,
|
|
971
|
+
`on_complete` once every member completed or was dead-lettered, `on_success` then too when none was dead; each
|
|
972
|
+
once, with `{"batch": {id, kind, total, succeeded, failed}}` merged into its payload. Answers
|
|
973
|
+
`{batch_id, total, jobs: [{jid, queue, status}]}`. 402 pro_required without a Pro licence."""
|
|
974
|
+
body: Dict[str, Any] = {"description": description, "jobs": [self._batch_item_body(item, context) for item in jobs]}
|
|
975
|
+
for kind, callback in (("on_complete", on_complete), ("on_success", on_success), ("on_failure", on_failure)):
|
|
976
|
+
if callback is not None:
|
|
977
|
+
body[kind] = self._batch_item_body(callback, context)
|
|
978
|
+
return self._post("/api/v1/sdk/batches", body)
|
|
979
|
+
|
|
980
|
+
def _batch_item_body(self, item: Any, context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
981
|
+
"""One member or callback of `create_batch`: the batch's `context` merged key by key under the job's own (the
|
|
982
|
+
job's keys win)."""
|
|
983
|
+
base = dict(context or {})
|
|
984
|
+
if isinstance(item, dict):
|
|
985
|
+
spec = dict(item)
|
|
986
|
+
spec["context"] = {**base, **(spec.get("context") or {})}
|
|
987
|
+
return self._item_body(spec, None)
|
|
988
|
+
job_cls, payload, opts = (tuple(item) + ({},))[:3]
|
|
989
|
+
opts = dict(opts or {})
|
|
990
|
+
opts["context"] = {**base, **(opts.get("context") or {})}
|
|
991
|
+
return self._item_body((job_cls, payload, opts), None)
|
|
992
|
+
|
|
993
|
+
def get_batch(self, batch_id: str) -> Dict[str, Any]:
|
|
994
|
+
"""A Pro batch's progress: `{batch_id, description, status, total, pending, succeeded, failed, created_at,
|
|
995
|
+
finished_at}`; `status` is pending, then success (no member dead) or complete."""
|
|
996
|
+
return self._get(f"/api/v1/sdk/batches/{_seg(batch_id)}")
|
|
997
|
+
|
|
998
|
+
def enqueue_async(self, job_cls: Type[Job], payload: Any = None, **kwargs: Any) -> "Future[Dict[str, Any]]":
|
|
999
|
+
"""`enqueue` on a background thread: the future holds `{jid, queue, status}` once the gateway stored the job
|
|
1000
|
+
(or the error). Use it when the caller must not wait; `enqueue` answers only once the job is stored."""
|
|
1001
|
+
with self._async_lock:
|
|
1002
|
+
if self._async is None:
|
|
1003
|
+
self._async = ThreadPoolExecutor(max_workers=8, thread_name_prefix="joobq-enqueue")
|
|
1004
|
+
return self._async.submit(self.enqueue, job_cls, payload, **kwargs)
|
|
1005
|
+
|
|
1006
|
+
def start(
|
|
1007
|
+
self,
|
|
1008
|
+
workflow_def: WorkflowBlueprint,
|
|
1009
|
+
input_data: Any = None,
|
|
1010
|
+
*,
|
|
1011
|
+
context: Optional[Dict[str, Any]] = None,
|
|
1012
|
+
workflow_id: Optional[str] = None,
|
|
1013
|
+
search_attributes: Optional[Dict[str, Any]] = None,
|
|
1014
|
+
build: Optional[str] = None,
|
|
1015
|
+
) -> Dict[str, Any]:
|
|
1016
|
+
return self.start_workflow(
|
|
1017
|
+
name=workflow_def.name,
|
|
1018
|
+
input_data=input_data,
|
|
1019
|
+
context=context,
|
|
1020
|
+
workflow_id=workflow_id,
|
|
1021
|
+
search_attributes=search_attributes,
|
|
1022
|
+
build=build,
|
|
1023
|
+
)
|
|
1024
|
+
|
|
1025
|
+
def start_workflow(
|
|
1026
|
+
self,
|
|
1027
|
+
name: str,
|
|
1028
|
+
input_data: Any = None,
|
|
1029
|
+
*,
|
|
1030
|
+
context: Optional[Dict[str, Any]] = None,
|
|
1031
|
+
workflow_id: Optional[str] = None,
|
|
1032
|
+
search_attributes: Optional[Dict[str, Any]] = None,
|
|
1033
|
+
build: Optional[str] = None,
|
|
1034
|
+
) -> Dict[str, Any]:
|
|
1035
|
+
return self._post("/api/v1/sdk/workflows", {
|
|
1036
|
+
"name": name,
|
|
1037
|
+
"input": input_data,
|
|
1038
|
+
"context": self._resolve_context(context),
|
|
1039
|
+
"id": workflow_id,
|
|
1040
|
+
"search_attributes": search_attributes,
|
|
1041
|
+
"build": build,
|
|
1042
|
+
})
|
|
1043
|
+
|
|
1044
|
+
def signal_workflow(self, workflow_id: str, name: str, payload: Any = None) -> Dict[str, Any]:
|
|
1045
|
+
return self._post(f"/api/v1/sdk/workflows/{workflow_id}/signal", {
|
|
1046
|
+
"name": name,
|
|
1047
|
+
"payload": payload,
|
|
1048
|
+
})
|
|
1049
|
+
|
|
1050
|
+
def update_workflow(self, workflow_id: str, name: str, input: Any = None, *, update_id: Optional[str] = None,
|
|
1051
|
+
wait_ms: Optional[int] = None) -> Dict[str, Any]:
|
|
1052
|
+
"""Sends an update to a running workflow's update handler (Pro) and waits up to `wait_ms` (100 to 30000, default
|
|
1053
|
+
10000) for its outcome: {update_id, name, status, result, error, reason, message}. `status` is completed,
|
|
1054
|
+
rejected (validate failed: `reason`, `message`), failed, or accepted when the handler had not answered in time
|
|
1055
|
+
(send again with the same `update_id` to wait for the same update)."""
|
|
1056
|
+
body: Dict[str, Any] = {"name": name, "input": input}
|
|
1057
|
+
if update_id is not None:
|
|
1058
|
+
body["update_id"] = update_id
|
|
1059
|
+
if wait_ms is not None:
|
|
1060
|
+
body["wait_ms"] = int(wait_ms)
|
|
1061
|
+
return self._post(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/updates", body)
|
|
1062
|
+
|
|
1063
|
+
def describe_workflow(self, workflow_id: str) -> Dict[str, Any]:
|
|
1064
|
+
"""One run (Pro): {id, name, status, closed, run, started_at, closed_at, continued_to, output, failure,
|
|
1065
|
+
attributes, queries}. Payload fields with secret-looking names are masked, as in the console."""
|
|
1066
|
+
return self._get(f"/api/v1/sdk/workflows/{_seg(workflow_id)}")
|
|
1067
|
+
|
|
1068
|
+
def workflow_result(self, workflow_id: str, wait_ms: int = 0) -> Dict[str, Any]:
|
|
1069
|
+
"""The run at the end of the workflow's continue-as-new chain (Pro), waiting up to `wait_ms` (at most 25000)
|
|
1070
|
+
for it to close; same shape as `describe_workflow`. `closed` false means it was still open when the wait ended."""
|
|
1071
|
+
return self._get(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/result?wait_ms={int(wait_ms)}")
|
|
1072
|
+
|
|
1073
|
+
def workflow_history(self, workflow_id: str, after_seq: int = 0, limit: int = 200) -> List[Dict[str, Any]]:
|
|
1074
|
+
"""The run's events after `after_seq` (Pro), at most `limit`, oldest first."""
|
|
1075
|
+
res = self._get(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/history?after_seq={int(after_seq)}&limit={int(limit)}")
|
|
1076
|
+
return res.get("items", [])
|
|
1077
|
+
|
|
1078
|
+
def query_workflow(self, workflow_id: str, name: str) -> Any:
|
|
1079
|
+
"""Asks the run a read-only question (Pro); returns the answer."""
|
|
1080
|
+
return self._post(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/query", {"name": name}).get("result")
|
|
1081
|
+
|
|
1082
|
+
def upsert_workflow_attributes(self, workflow_id: str, set: Optional[Dict[str, Any]] = None,
|
|
1083
|
+
remove: Optional[List[str]] = None) -> Dict[str, Any]:
|
|
1084
|
+
"""Sets and/or removes search attributes of the workflow's open run (Pro); returns {id, attributes}."""
|
|
1085
|
+
return self._post(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/search-attributes", {"set": set, "remove": remove})
|
|
1086
|
+
|
|
1087
|
+
def register_schedule(
|
|
1088
|
+
self,
|
|
1089
|
+
name: str,
|
|
1090
|
+
job_type: str,
|
|
1091
|
+
queue: str,
|
|
1092
|
+
*,
|
|
1093
|
+
cron: Optional[str] = None,
|
|
1094
|
+
every: Optional[str] = None,
|
|
1095
|
+
timezone: str = "UTC",
|
|
1096
|
+
payload: Any = None,
|
|
1097
|
+
max_retries: int = 3,
|
|
1098
|
+
encrypt_fields: Optional[List[str]] = None,
|
|
1099
|
+
unique_key: Optional[str] = None,
|
|
1100
|
+
) -> Dict[str, Any]:
|
|
1101
|
+
"""Registers a cron/recurring job schedule, idempotent by `name`. Pass exactly one of `cron` or `every`
|
|
1102
|
+
(`every` is "<count> <unit>", e.g. "30 seconds", "5 minutes"). Returns {id, name, kind, created}."""
|
|
1103
|
+
return self._post("/api/v1/sdk/schedules", {
|
|
1104
|
+
"name": name,
|
|
1105
|
+
"kind": "job",
|
|
1106
|
+
"cron": cron,
|
|
1107
|
+
"every": every,
|
|
1108
|
+
"timezone": timezone,
|
|
1109
|
+
"queue": queue,
|
|
1110
|
+
"job_type": job_type,
|
|
1111
|
+
"payload": payload if payload is not None else {},
|
|
1112
|
+
"max_retries": max_retries,
|
|
1113
|
+
"encrypt_fields": encrypt_fields or [],
|
|
1114
|
+
"unique_key": unique_key,
|
|
1115
|
+
})
|
|
1116
|
+
|
|
1117
|
+
def rate_limit(self, key: str, limit: int, period_ms: int) -> Dict[str, Any]:
|
|
1118
|
+
"""Reserves one slot of a sliding-window rate limit (Pro), returning {allowed, retry_after_ms}. `allowed` false
|
|
1119
|
+
gives the ms until a slot frees up. 402 pro_required without a Pro licence."""
|
|
1120
|
+
return self._post("/api/v1/sdk/rate-limit", {"key": key, "limit": limit, "period_ms": period_ms})
|
|
1121
|
+
|
|
1122
|
+
def register_blueprint(self, workflow_def: WorkflowBlueprint, worker_id: Optional[str] = None) -> Dict[str, Any]:
|
|
1123
|
+
"""Sends the workflow's blueprint; the gateway keeps each new content hash (`created`) and ignores a known one."""
|
|
1124
|
+
return self._post("/api/v1/sdk/blueprints", {
|
|
1125
|
+
"name": workflow_def.name,
|
|
1126
|
+
"sdk_language": "python",
|
|
1127
|
+
"worker_id": worker_id,
|
|
1128
|
+
"blueprint": workflow_def.to_blueprint(),
|
|
1129
|
+
})
|
|
1130
|
+
|
|
1131
|
+
def register_worker(
|
|
1132
|
+
self,
|
|
1133
|
+
worker_id: str,
|
|
1134
|
+
task_queues: List[str],
|
|
1135
|
+
*,
|
|
1136
|
+
concurrency: int = 4,
|
|
1137
|
+
service_name: Optional[str] = None,
|
|
1138
|
+
team: Optional[str] = None,
|
|
1139
|
+
) -> Dict[str, Any]:
|
|
1140
|
+
return self._post("/api/v1/sdk/workers/register", {
|
|
1141
|
+
"worker_id": worker_id,
|
|
1142
|
+
"sdk_language": "python",
|
|
1143
|
+
"sdk_version": SDK_VERSION,
|
|
1144
|
+
"service_name": service_name,
|
|
1145
|
+
"team": team or os.environ.get("JOOBQ_TEAM"),
|
|
1146
|
+
"host": socket.gethostname(),
|
|
1147
|
+
"pid": os.getpid(),
|
|
1148
|
+
"task_queues": task_queues,
|
|
1149
|
+
"concurrency": concurrency,
|
|
1150
|
+
"delivery_mode": "poll",
|
|
1151
|
+
})
|
|
1152
|
+
|
|
1153
|
+
def deregister_worker(self, worker_id: str) -> Dict[str, Any]:
|
|
1154
|
+
return self._post(f"/api/v1/sdk/workers/{worker_id}/deregister", {})
|
|
1155
|
+
|
|
1156
|
+
def poll_tasks(
|
|
1157
|
+
self,
|
|
1158
|
+
worker_id: str,
|
|
1159
|
+
task_queues: List[str],
|
|
1160
|
+
*,
|
|
1161
|
+
max_tasks: int = 1,
|
|
1162
|
+
timeout_ms: int = 0,
|
|
1163
|
+
lease_ttl_ms: int = 30000,
|
|
1164
|
+
) -> List[LeasedTask]:
|
|
1165
|
+
res = self._post("/api/v1/sdk/tasks/poll", {
|
|
1166
|
+
"worker_id": worker_id,
|
|
1167
|
+
"task_queues": task_queues,
|
|
1168
|
+
"max_tasks": max_tasks,
|
|
1169
|
+
"timeout_ms": timeout_ms,
|
|
1170
|
+
"lease_ttl_ms": lease_ttl_ms,
|
|
1171
|
+
})
|
|
1172
|
+
return [LeasedTask.from_dict(t) for t in res.get("tasks", [])]
|
|
1173
|
+
|
|
1174
|
+
def heartbeat_task(
|
|
1175
|
+
self,
|
|
1176
|
+
lease_token: str,
|
|
1177
|
+
worker_id: str,
|
|
1178
|
+
lease_ttl_ms: int = 30000,
|
|
1179
|
+
) -> Dict[str, Any]:
|
|
1180
|
+
return self._post(f"/api/v1/sdk/tasks/{lease_token}/heartbeat", {
|
|
1181
|
+
"worker_id": worker_id,
|
|
1182
|
+
"lease_ttl_ms": lease_ttl_ms,
|
|
1183
|
+
})
|
|
1184
|
+
|
|
1185
|
+
def complete_task(
|
|
1186
|
+
self,
|
|
1187
|
+
lease_token: str,
|
|
1188
|
+
worker_id: str,
|
|
1189
|
+
output: Any = None,
|
|
1190
|
+
) -> Dict[str, Any]:
|
|
1191
|
+
return self._post(f"/api/v1/sdk/tasks/{lease_token}/complete", {
|
|
1192
|
+
"worker_id": worker_id,
|
|
1193
|
+
"output": output,
|
|
1194
|
+
})
|
|
1195
|
+
|
|
1196
|
+
def fail_task(
|
|
1197
|
+
self,
|
|
1198
|
+
lease_token: str,
|
|
1199
|
+
worker_id: str,
|
|
1200
|
+
exc: BaseException,
|
|
1201
|
+
*,
|
|
1202
|
+
retryable: bool = True,
|
|
1203
|
+
) -> Dict[str, Any]:
|
|
1204
|
+
return self._post(f"/api/v1/sdk/tasks/{lease_token}/fail",
|
|
1205
|
+
{"worker_id": worker_id, **failure(exc, retryable)})
|
|
1206
|
+
|
|
1207
|
+
def ack_tasks(self, worker_id: str, items: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
1208
|
+
"""Settles up to 100 leases in one call (`POST /sdk/tasks/ack`): items are `{token, outcome: "complete", output}`
|
|
1209
|
+
or `{token, outcome: "fail", **failure(exc)}`; each gets its own result."""
|
|
1210
|
+
return self._post("/api/v1/sdk/tasks/ack", {"worker_id": worker_id, "items": items})
|
|
1211
|
+
|
|
1212
|
+
def heartbeat_tasks(self, worker_id: str, leases: List[Dict[str, Any]], lease_ttl_ms: int = 30000) -> Dict[str, Any]:
|
|
1213
|
+
"""Renews many leases in one call (`POST /sdk/tasks/heartbeat`): `leases` are `{token}`."""
|
|
1214
|
+
return self._post("/api/v1/sdk/tasks/heartbeat", {"worker_id": worker_id, "lease_ttl_ms": lease_ttl_ms,
|
|
1215
|
+
"leases": leases})
|
|
1216
|
+
|
|
1217
|
+
def _connection(self) -> http.client.HTTPConnection:
|
|
1218
|
+
"""This thread's kept-alive connection to the gateway (through the environment's HTTP proxy, if any)."""
|
|
1219
|
+
conn = getattr(self._local, "conn", None)
|
|
1220
|
+
if conn is not None:
|
|
1221
|
+
return conn
|
|
1222
|
+
scheme, host = self._url.scheme or "http", self._url.hostname or "localhost"
|
|
1223
|
+
port = self._url.port or (443 if scheme == "https" else 80)
|
|
1224
|
+
proxy = urllib.request.getproxies().get(scheme)
|
|
1225
|
+
if proxy and urllib.request.proxy_bypass(host):
|
|
1226
|
+
proxy = None
|
|
1227
|
+
self._local.absolute = False
|
|
1228
|
+
if proxy:
|
|
1229
|
+
p = urllib.parse.urlsplit(proxy if "://" in proxy else "http://" + proxy)
|
|
1230
|
+
if scheme == "https":
|
|
1231
|
+
conn = http.client.HTTPSConnection(p.hostname, p.port or 80, timeout=self.request_timeout)
|
|
1232
|
+
conn.set_tunnel(host, port)
|
|
1233
|
+
else:
|
|
1234
|
+
conn = http.client.HTTPConnection(p.hostname, p.port or 80, timeout=self.request_timeout)
|
|
1235
|
+
self._local.absolute = True
|
|
1236
|
+
else:
|
|
1237
|
+
cls = http.client.HTTPSConnection if scheme == "https" else http.client.HTTPConnection
|
|
1238
|
+
conn = cls(host, port, timeout=self.request_timeout)
|
|
1239
|
+
self._local.conn = conn
|
|
1240
|
+
self._local.used = False
|
|
1241
|
+
return conn
|
|
1242
|
+
|
|
1243
|
+
def _drop_connection(self) -> None:
|
|
1244
|
+
conn = getattr(self._local, "conn", None)
|
|
1245
|
+
if conn is not None:
|
|
1246
|
+
conn.close()
|
|
1247
|
+
self._local.conn = None
|
|
1248
|
+
|
|
1249
|
+
def _post(self, path: str, body: Dict[str, Any]) -> Dict[str, Any]:
|
|
1250
|
+
return self._send("POST", path, body)
|
|
1251
|
+
|
|
1252
|
+
def _get(self, path: str) -> Dict[str, Any]:
|
|
1253
|
+
return self._send("GET", path, None)
|
|
1254
|
+
|
|
1255
|
+
def _send(self, method: str, path: str, body: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
1256
|
+
data = json.dumps(body).encode("utf-8") if body is not None else None
|
|
1257
|
+
headers = {"Authorization": f"Bearer {self.token}"}
|
|
1258
|
+
if data is not None:
|
|
1259
|
+
headers["Content-Type"] = "application/json"
|
|
1260
|
+
target = (self._url.path or "") + path
|
|
1261
|
+
while True:
|
|
1262
|
+
conn = self._connection()
|
|
1263
|
+
reused = self._local.used
|
|
1264
|
+
try:
|
|
1265
|
+
conn.request(method, f"{self.gateway_url}{path}" if self._local.absolute else target, data, headers)
|
|
1266
|
+
resp = conn.getresponse()
|
|
1267
|
+
raw = resp.read()
|
|
1268
|
+
except (http.client.RemoteDisconnected, BrokenPipeError, ConnectionResetError):
|
|
1269
|
+
self._drop_connection()
|
|
1270
|
+
if reused:
|
|
1271
|
+
continue # the gateway closed the kept-alive connection while it was idle: once more, on a new one
|
|
1272
|
+
raise
|
|
1273
|
+
except Exception:
|
|
1274
|
+
self._drop_connection()
|
|
1275
|
+
raise
|
|
1276
|
+
self._local.used = True
|
|
1277
|
+
if resp.will_close:
|
|
1278
|
+
self._drop_connection()
|
|
1279
|
+
if resp.status >= 400:
|
|
1280
|
+
raise GatewayError(resp.status, raw.decode("utf-8", "replace"))
|
|
1281
|
+
return json.loads(raw.decode("utf-8")) if raw else {}
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
class _Transport:
|
|
1285
|
+
"""Every gateway call a running worker makes, on threads of its own: handler threads never wait for HTTP.
|
|
1286
|
+
|
|
1287
|
+
- the poller long-polls for leases into a small buffer (free slots plus `prefetch`) that handlers take from;
|
|
1288
|
+
- the acker sends the outcomes handlers queue (`POST /sdk/tasks/ack`, up to 100 per call, after at most
|
|
1289
|
+
`ack_linger_ms`), retrying while the gateway is unavailable: a task stays leased (and renewed) until its outcome
|
|
1290
|
+
is acknowledged. The outcome queue holds `ack_buffer` items: when it is full, handlers wait;
|
|
1291
|
+
- the heartbeater renews every lease the worker holds in one call (`POST /sdk/tasks/heartbeat`) every third of the
|
|
1292
|
+
lease, and registers the worker every 10 s."""
|
|
1293
|
+
|
|
1294
|
+
def __init__(self, worker: "JoobQWorker") -> None:
|
|
1295
|
+
self.worker = worker
|
|
1296
|
+
self.client = worker.client
|
|
1297
|
+
self.lock = threading.Condition()
|
|
1298
|
+
self.ready: "_queue.Queue[Optional[LeasedTask]]" = _queue.Queue()
|
|
1299
|
+
self.acks: "_queue.Queue[Dict[str, Any]]" = _queue.Queue(maxsize=worker.ack_buffer)
|
|
1300
|
+
self.leased: Dict[str, LeasedTask] = {} # every lease held: buffered, running, or waiting for its ack
|
|
1301
|
+
self.held = 0 # buffered + running
|
|
1302
|
+
self.cancelled: Dict[str, bool] = {}
|
|
1303
|
+
self.acker_done = threading.Event() # set: the acker exits once the outcome queue is empty
|
|
1304
|
+
self.give_up = threading.Event() # set: the drain timeout passed, unsent outcomes are left to their leases
|
|
1305
|
+
self.stopped = threading.Event()
|
|
1306
|
+
self.running = False
|
|
1307
|
+
|
|
1308
|
+
# -- handler side ---------------------------------------------------------------------------------------------
|
|
1309
|
+
def settle(self, item: Dict[str, Any]) -> None:
|
|
1310
|
+
"""Queues an outcome for the acker; waits while the queue is full (back-pressure)."""
|
|
1311
|
+
while True:
|
|
1312
|
+
try:
|
|
1313
|
+
self.acks.put(item, timeout=0.5)
|
|
1314
|
+
return
|
|
1315
|
+
except _queue.Full:
|
|
1316
|
+
if self.give_up.is_set():
|
|
1317
|
+
return # left to its lease
|
|
1318
|
+
|
|
1319
|
+
def cancel_requested(self, token: str) -> bool:
|
|
1320
|
+
with self.lock:
|
|
1321
|
+
return self.cancelled.get(token, False)
|
|
1322
|
+
|
|
1323
|
+
def release(self, token: str) -> None:
|
|
1324
|
+
with self.lock:
|
|
1325
|
+
self.leased.pop(token, None)
|
|
1326
|
+
self.cancelled.pop(token, None)
|
|
1327
|
+
|
|
1328
|
+
# -- threads --------------------------------------------------------------------------------------------------
|
|
1329
|
+
def handle(self) -> None:
|
|
1330
|
+
while True:
|
|
1331
|
+
task = self.ready.get()
|
|
1332
|
+
if task is None:
|
|
1333
|
+
return
|
|
1334
|
+
try:
|
|
1335
|
+
self.worker.process_task(task)
|
|
1336
|
+
except Exception as exc: # process_task reports handler errors itself: this is a bug in the worker
|
|
1337
|
+
print(f"joobq: task {task.id} could not be settled: {exc!r}", file=sys.stderr)
|
|
1338
|
+
finally:
|
|
1339
|
+
with self.lock:
|
|
1340
|
+
self.held -= 1
|
|
1341
|
+
self.lock.notify_all()
|
|
1342
|
+
|
|
1343
|
+
def poll(self, poll_interval_sec: float) -> None:
|
|
1344
|
+
w = self.worker
|
|
1345
|
+
capacity = w.concurrency + w.prefetch
|
|
1346
|
+
backoff = poll_interval_sec
|
|
1347
|
+
while not w._stopping.is_set():
|
|
1348
|
+
with self.lock:
|
|
1349
|
+
while self.held >= capacity and not w._stopping.is_set():
|
|
1350
|
+
self.lock.wait(0.5)
|
|
1351
|
+
room = capacity - self.held
|
|
1352
|
+
if w._stopping.is_set():
|
|
1353
|
+
return
|
|
1354
|
+
try:
|
|
1355
|
+
tasks = self.client.poll_tasks(w.worker_id, w.task_queues, max_tasks=min(room, 100),
|
|
1356
|
+
timeout_ms=w.poll_timeout_ms, lease_ttl_ms=w.lease_ttl_ms)
|
|
1357
|
+
backoff = poll_interval_sec
|
|
1358
|
+
w._refused = None
|
|
1359
|
+
except Exception as exc:
|
|
1360
|
+
w._report_refusal("poll for tasks", exc)
|
|
1361
|
+
w._stopping.wait(backoff + random.random() * 0.1)
|
|
1362
|
+
backoff = min(backoff * 2, 5.0)
|
|
1363
|
+
continue
|
|
1364
|
+
with self.lock:
|
|
1365
|
+
for task in tasks:
|
|
1366
|
+
self.leased[task.lease_token] = task
|
|
1367
|
+
self.held += len(tasks)
|
|
1368
|
+
for task in tasks:
|
|
1369
|
+
self.ready.put(task)
|
|
1370
|
+
|
|
1371
|
+
def ack(self) -> None:
|
|
1372
|
+
linger = self.worker.ack_linger_ms / 1000.0
|
|
1373
|
+
while True:
|
|
1374
|
+
try:
|
|
1375
|
+
first = self.acks.get(timeout=0.1)
|
|
1376
|
+
except _queue.Empty:
|
|
1377
|
+
if self.acker_done.is_set():
|
|
1378
|
+
return
|
|
1379
|
+
continue
|
|
1380
|
+
batch = [first]
|
|
1381
|
+
deadline = time.monotonic() + linger
|
|
1382
|
+
while len(batch) < 100:
|
|
1383
|
+
left = deadline - time.monotonic()
|
|
1384
|
+
try:
|
|
1385
|
+
batch.append(self.acks.get(timeout=left) if left > 0 else self.acks.get_nowait())
|
|
1386
|
+
except _queue.Empty:
|
|
1387
|
+
break
|
|
1388
|
+
self.send_acks(batch)
|
|
1389
|
+
|
|
1390
|
+
def send_acks(self, batch: List[Dict[str, Any]]) -> None:
|
|
1391
|
+
delay = 0.25
|
|
1392
|
+
while True:
|
|
1393
|
+
try:
|
|
1394
|
+
self.client.ack_tasks(self.worker.worker_id, batch)
|
|
1395
|
+
break # a refused item (its lease is gone) is not sent again either
|
|
1396
|
+
except GatewayError as err:
|
|
1397
|
+
if err.status == 413 and len(batch) > 1: # too large for one call: in halves
|
|
1398
|
+
half = len(batch) // 2
|
|
1399
|
+
self.send_acks(batch[:half])
|
|
1400
|
+
self.send_acks(batch[half:])
|
|
1401
|
+
return
|
|
1402
|
+
if err.status < 500 and err.status != 429:
|
|
1403
|
+
print(f"joobq: the gateway refused {len(batch)} outcomes: {err}", file=sys.stderr)
|
|
1404
|
+
break
|
|
1405
|
+
except Exception:
|
|
1406
|
+
pass
|
|
1407
|
+
if self.give_up.wait(delay):
|
|
1408
|
+
break # shutting down: the leases bring these tasks back
|
|
1409
|
+
delay = min(delay * 2, 5.0)
|
|
1410
|
+
for item in batch:
|
|
1411
|
+
self.release(item["token"])
|
|
1412
|
+
|
|
1413
|
+
def heartbeat(self) -> None:
|
|
1414
|
+
w = self.worker
|
|
1415
|
+
every = max(0.5, w.lease_ttl_ms / 3000.0)
|
|
1416
|
+
last_reg = time.monotonic()
|
|
1417
|
+
while not self.stopped.wait(every):
|
|
1418
|
+
with self.lock:
|
|
1419
|
+
tokens = list(self.leased)
|
|
1420
|
+
for start in range(0, len(tokens), 500):
|
|
1421
|
+
chunk = tokens[start:start + 500]
|
|
1422
|
+
try:
|
|
1423
|
+
res = self.client.heartbeat_tasks(w.worker_id, [{"token": t} for t in chunk], w.lease_ttl_ms)
|
|
1424
|
+
except Exception:
|
|
1425
|
+
continue # renewed with the next heartbeat
|
|
1426
|
+
with self.lock:
|
|
1427
|
+
for lease in res.get("leases", []):
|
|
1428
|
+
token = lease.get("lease_token")
|
|
1429
|
+
if lease.get("error"):
|
|
1430
|
+
self.leased.pop(token, None) # gone: its outcome will be refused, nothing to renew
|
|
1431
|
+
elif token in self.leased:
|
|
1432
|
+
self.cancelled[token] = bool(lease.get("cancel_requested"))
|
|
1433
|
+
if time.monotonic() - last_reg > 10.0 and not w._stopping.is_set():
|
|
1434
|
+
try:
|
|
1435
|
+
w._register_blueprints()
|
|
1436
|
+
w._register()
|
|
1437
|
+
except Exception:
|
|
1438
|
+
pass
|
|
1439
|
+
last_reg = time.monotonic()
|
|
1440
|
+
|
|
1441
|
+
def run(self, poll_interval_sec: float) -> None:
|
|
1442
|
+
w = self.worker
|
|
1443
|
+
self.running = True
|
|
1444
|
+
threads = [threading.Thread(target=self.handle, name=f"joobq-task-{i}", daemon=True) for i in range(w.concurrency)]
|
|
1445
|
+
acker = threading.Thread(target=self.ack, name="joobq-acker", daemon=True)
|
|
1446
|
+
beat = threading.Thread(target=self.heartbeat, name="joobq-heartbeat", daemon=True)
|
|
1447
|
+
for t in threads + [acker, beat]:
|
|
1448
|
+
t.start()
|
|
1449
|
+
self.poll(poll_interval_sec) # until stop()
|
|
1450
|
+
deadline = time.monotonic() + w.drain_timeout_sec
|
|
1451
|
+
with self.lock:
|
|
1452
|
+
while self.held > 0 and time.monotonic() < deadline:
|
|
1453
|
+
self.lock.wait(0.05)
|
|
1454
|
+
for _ in threads:
|
|
1455
|
+
self.ready.put(None)
|
|
1456
|
+
self.acker_done.set()
|
|
1457
|
+
acker.join(max(0.0, deadline - time.monotonic()))
|
|
1458
|
+
if acker.is_alive():
|
|
1459
|
+
self.give_up.set()
|
|
1460
|
+
acker.join(5.0)
|
|
1461
|
+
self.give_up.set()
|
|
1462
|
+
self.stopped.set()
|
|
1463
|
+
beat.join(5.0)
|
|
1464
|
+
self.running = False
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
class JoobQWorker:
|
|
1468
|
+
"""In-app worker: runs up to `concurrency` jobs and workflow steps at once inside the booted application, and
|
|
1469
|
+
drains on SIGTERM/SIGINT. Its gateway calls (poll, outcomes, heartbeats) run
|
|
1470
|
+
on threads of their own (`_Transport`), so a handler never waits for HTTP."""
|
|
1471
|
+
|
|
1472
|
+
def __init__(
|
|
1473
|
+
self,
|
|
1474
|
+
client: JoobQClient,
|
|
1475
|
+
worker_id: str,
|
|
1476
|
+
task_queues: Optional[List[str]] = None,
|
|
1477
|
+
*,
|
|
1478
|
+
concurrency: int = 4,
|
|
1479
|
+
service_name: Optional[str] = None,
|
|
1480
|
+
team: Optional[str] = None,
|
|
1481
|
+
lease_ttl_ms: int = 30000,
|
|
1482
|
+
poll_timeout_ms: int = 5000,
|
|
1483
|
+
drain_timeout_sec: float = 30.0,
|
|
1484
|
+
prefetch: Optional[int] = None,
|
|
1485
|
+
ack_buffer: Optional[int] = None,
|
|
1486
|
+
ack_linger_ms: int = 5,
|
|
1487
|
+
) -> None:
|
|
1488
|
+
self.client = client
|
|
1489
|
+
self.worker_id = worker_id
|
|
1490
|
+
self.task_queues = list(task_queues or [])
|
|
1491
|
+
self.concurrency = max(1, concurrency)
|
|
1492
|
+
self.service_name = service_name
|
|
1493
|
+
self.team = team
|
|
1494
|
+
self.lease_ttl_ms = lease_ttl_ms
|
|
1495
|
+
self.poll_timeout_ms = poll_timeout_ms
|
|
1496
|
+
# One hung call must not outlast the leases: a poll waits `poll_timeout_ms` at the gateway, and a sixth of a
|
|
1497
|
+
# lease on top keeps a renewal that hangs well inside the lease (10 s with the defaults).
|
|
1498
|
+
bound = poll_timeout_ms / 1000.0 + lease_ttl_ms / 6000.0
|
|
1499
|
+
if getattr(client, "request_timeout", None) is not None and client.request_timeout > bound:
|
|
1500
|
+
client.request_timeout = bound
|
|
1501
|
+
self.drain_timeout_sec = drain_timeout_sec
|
|
1502
|
+
# Leases held beyond the free slots, so a handler that finishes finds its next task at once.
|
|
1503
|
+
self.prefetch = max(0, prefetch if prefetch is not None else max(1, self.concurrency // 2))
|
|
1504
|
+
# Outcomes queued for the acker before handlers wait for it.
|
|
1505
|
+
self.ack_buffer = max(1, ack_buffer if ack_buffer is not None else 4 * self.concurrency)
|
|
1506
|
+
self.ack_linger_ms = max(0, ack_linger_ms)
|
|
1507
|
+
self._handlers: Dict[str, Callable[[Any, ExecutionContext], Any]] = {}
|
|
1508
|
+
self._workflows: Dict[str, WorkflowBlueprint] = {}
|
|
1509
|
+
self._middlewares: List[Callable[[ExecutionContext, Callable[[], Any]], Any]] = []
|
|
1510
|
+
self._stopping = threading.Event()
|
|
1511
|
+
self._blueprints_sent: set = set()
|
|
1512
|
+
self._transport = _Transport(self)
|
|
1513
|
+
|
|
1514
|
+
def use(self, middleware: Callable[[ExecutionContext, Callable[[], Any]], Any]) -> "JoobQWorker":
|
|
1515
|
+
self._middlewares.append(middleware)
|
|
1516
|
+
return self
|
|
1517
|
+
|
|
1518
|
+
def register_job(self, job_cls: Type[Job]) -> "JoobQWorker":
|
|
1519
|
+
if job_cls.queue and job_cls.queue not in self.task_queues:
|
|
1520
|
+
self.task_queues.append(job_cls.queue)
|
|
1521
|
+
self._handlers[job_cls.resolved_name()] = _normalize_callable(job_cls)
|
|
1522
|
+
return self
|
|
1523
|
+
|
|
1524
|
+
def register_workflow(self, workflow_def: WorkflowBlueprint) -> "JoobQWorker":
|
|
1525
|
+
if workflow_def.task_queue and workflow_def.task_queue not in self.task_queues:
|
|
1526
|
+
self.task_queues.append(workflow_def.task_queue)
|
|
1527
|
+
self._workflows[workflow_def.name] = workflow_def
|
|
1528
|
+
# The engine conducts the workflow's runs and sends each step as its own task.
|
|
1529
|
+
self._handlers.update(workflow_def.activities())
|
|
1530
|
+
return self
|
|
1531
|
+
|
|
1532
|
+
def handle_job(self, job_type: str, handler: Callable[[Any, ExecutionContext], Any]) -> "JoobQWorker":
|
|
1533
|
+
self._handlers[job_type] = handler
|
|
1534
|
+
return self
|
|
1535
|
+
|
|
1536
|
+
def process_task(self, task: LeasedTask) -> Dict[str, Any]:
|
|
1537
|
+
handler = self._handlers.get(task.name)
|
|
1538
|
+
if not handler:
|
|
1539
|
+
self._settle(task, {"outcome": "fail", **failure(NonRetryableError(f"No handler registered for {task.name!r}"), False)})
|
|
1540
|
+
return {"status": "failed"}
|
|
1541
|
+
# Everything that can raise for this task is inside the try: a task with no outcome keeps its lease renewed for
|
|
1542
|
+
# ever. BaseException, so that a handler calling sys.exit() (or raising anything else that is not an Exception)
|
|
1543
|
+
# fails its task instead of ending the handler thread without a word.
|
|
1544
|
+
try:
|
|
1545
|
+
transport = self._transport if self._transport.running else None
|
|
1546
|
+
ctx = ExecutionContext(task, self.worker_id, self.client, self.lease_ttl_ms, transport)
|
|
1547
|
+
|
|
1548
|
+
def invoke(idx: int) -> Any:
|
|
1549
|
+
if idx < len(self._middlewares):
|
|
1550
|
+
return self._middlewares[idx](ctx, lambda: invoke(idx + 1))
|
|
1551
|
+
return handler(task.payload, ctx)
|
|
1552
|
+
|
|
1553
|
+
output = invoke(0)
|
|
1554
|
+
except BaseException as exc:
|
|
1555
|
+
if isinstance(exc, KeyboardInterrupt):
|
|
1556
|
+
raise
|
|
1557
|
+
retryable = not isinstance(exc, NonRetryableError)
|
|
1558
|
+
self._settle(task, {"outcome": "fail", **failure(exc, retryable)})
|
|
1559
|
+
return {"status": "failed"}
|
|
1560
|
+
self._settle(task, {"outcome": "complete", "output": output})
|
|
1561
|
+
return {"status": "completed", "output": output}
|
|
1562
|
+
|
|
1563
|
+
def _report_refusal(self, what: str, exc: BaseException) -> None:
|
|
1564
|
+
"""Says once, on stderr, that the gateway refuses this worker (a wrong or rotated token, no Pro key): it goes
|
|
1565
|
+
on retrying, and without this line it would sit idle with nothing in its output."""
|
|
1566
|
+
status = getattr(exc, "status", None)
|
|
1567
|
+
if not isinstance(status, int) or status < 400 or status >= 500 or status in (408, 429):
|
|
1568
|
+
return
|
|
1569
|
+
if getattr(self, "_refused", None) != status:
|
|
1570
|
+
self._refused = status
|
|
1571
|
+
print(f"joobq: the gateway refuses this worker ({what}): {exc}. Retrying; check the SDK token and the server's licence.", file=sys.stderr)
|
|
1572
|
+
|
|
1573
|
+
def _settle(self, task: LeasedTask, item: Dict[str, Any]) -> None:
|
|
1574
|
+
"""Hands the outcome to the acker; outside a running worker (`process_task` called directly) it is sent here."""
|
|
1575
|
+
item = {"token": task.lease_token, **item}
|
|
1576
|
+
if self._transport.running:
|
|
1577
|
+
self._transport.settle(item)
|
|
1578
|
+
else:
|
|
1579
|
+
self._report(lambda: self.client.ack_tasks(self.worker_id, [item]))
|
|
1580
|
+
|
|
1581
|
+
def _report(self, call: Callable[[], Any]) -> None:
|
|
1582
|
+
"""Sends an outcome, retrying while the gateway is unavailable; a lost outcome is left to the lease."""
|
|
1583
|
+
delay = 0.25
|
|
1584
|
+
for _ in range(6):
|
|
1585
|
+
try:
|
|
1586
|
+
call()
|
|
1587
|
+
return
|
|
1588
|
+
except GatewayError as err:
|
|
1589
|
+
if err.status < 500:
|
|
1590
|
+
return # the outcome was refused: nothing to retry
|
|
1591
|
+
except Exception:
|
|
1592
|
+
pass
|
|
1593
|
+
time.sleep(delay)
|
|
1594
|
+
delay = min(delay * 2, 5.0)
|
|
1595
|
+
|
|
1596
|
+
def stop(self) -> None:
|
|
1597
|
+
"""Stops polling; `start` returns once the running tasks have finished and their outcomes are acknowledged
|
|
1598
|
+
(or `drain_timeout_sec` passed)."""
|
|
1599
|
+
self._stopping.set()
|
|
1600
|
+
with self._transport.lock:
|
|
1601
|
+
self._transport.lock.notify_all()
|
|
1602
|
+
|
|
1603
|
+
def _register(self) -> None:
|
|
1604
|
+
self.client.register_worker(self.worker_id, self.task_queues, concurrency=self.concurrency,
|
|
1605
|
+
service_name=self.service_name, team=self.team)
|
|
1606
|
+
|
|
1607
|
+
def _register_blueprints(self) -> None:
|
|
1608
|
+
"""Registers every workflow's blueprint once; one the gateway refuses (4xx) is logged and not sent again."""
|
|
1609
|
+
for name, wf in list(self._workflows.items()):
|
|
1610
|
+
if name in self._blueprints_sent:
|
|
1611
|
+
continue
|
|
1612
|
+
try:
|
|
1613
|
+
self.client.register_blueprint(wf, self.worker_id)
|
|
1614
|
+
self._blueprints_sent.add(name)
|
|
1615
|
+
except GatewayError as err:
|
|
1616
|
+
if err.status >= 500:
|
|
1617
|
+
raise
|
|
1618
|
+
self._blueprints_sent.add(name)
|
|
1619
|
+
print(f"joobq: blueprint of {name!r} refused: {err}", file=sys.stderr)
|
|
1620
|
+
|
|
1621
|
+
def start(self, poll_interval_sec: float = 0.25, install_signal_handlers: bool = True) -> None:
|
|
1622
|
+
"""Runs the worker until SIGTERM/SIGINT or `stop()`, then drains the running tasks, sends their outcomes and
|
|
1623
|
+
deregisters."""
|
|
1624
|
+
if install_signal_handlers and threading.current_thread() is threading.main_thread():
|
|
1625
|
+
for sig in (_signal.SIGTERM, _signal.SIGINT):
|
|
1626
|
+
_signal.signal(sig, lambda *_: self.stop())
|
|
1627
|
+
backoff = poll_interval_sec
|
|
1628
|
+
while not self._stopping.is_set():
|
|
1629
|
+
try:
|
|
1630
|
+
self._register_blueprints() # first: a worker that shows up has its blueprints in
|
|
1631
|
+
self._register()
|
|
1632
|
+
self._refused = None
|
|
1633
|
+
break
|
|
1634
|
+
except Exception as exc:
|
|
1635
|
+
self._report_refusal("register", exc)
|
|
1636
|
+
self._stopping.wait(backoff + random.random() * 0.1)
|
|
1637
|
+
backoff = min(backoff * 2, 5.0)
|
|
1638
|
+
self._transport.run(poll_interval_sec)
|
|
1639
|
+
try:
|
|
1640
|
+
self.client.deregister_worker(self.worker_id)
|
|
1641
|
+
except Exception:
|
|
1642
|
+
pass
|
|
1643
|
+
|
|
1644
|
+
run = start
|
|
1645
|
+
|