joobq 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
joobq/__init__.py ADDED
@@ -0,0 +1,1645 @@
1
+ # Code generated by joobqui-selfhosted sdk:generate (v0.1.0) from JoobQ Polyglot SDK Gateway API. DO NOT EDIT.
2
+ # Supported Gateway Operations:
3
+ # - POST /api/v1/sdk/batches: Create a Pro batch of jobs with completion callbacks, filled and sealed in one call (Pro)
4
+ # - GET /api/v1/sdk/batches/{id}: Read a Pro batch's progress: pending, succeeded and failed members (Pro)
5
+ # - POST /api/v1/sdk/blueprints: Register a workflow blueprint, versioned by its content hash
6
+ # - POST /api/v1/sdk/jobs: Enqueue a background job onto a queue
7
+ # - POST /api/v1/sdk/jobs/batch: Enqueue up to 500 background jobs in one call
8
+ # - GET /api/v1/sdk/openapi.json: Describe the Polyglot SDK Gateway API as an OpenAPI 3.1 document
9
+ # - POST /api/v1/sdk/rate-limit: Reserve one slot of a sliding-window rate limit (Pro)
10
+ # - POST /api/v1/sdk/schedules: Register a code-declared cron/recurring job or workflow schedule, idempotent by name
11
+ # - POST /api/v1/sdk/tasks/ack: Settle up to 100 task leases in one call, each completed or failed
12
+ # - POST /api/v1/sdk/tasks/heartbeat: Extend every lease a worker holds in one call and check each for cancellation
13
+ # - POST /api/v1/sdk/tasks/poll: Claim due jobs and workflow steps from unpaused queues
14
+ # - POST /api/v1/sdk/tasks/{token}/complete: Commit task completion and output
15
+ # - POST /api/v1/sdk/tasks/{token}/fail: Record structured polyglot task failure and stack frames
16
+ # - POST /api/v1/sdk/tasks/{token}/heartbeat: Extend an active task lease and check for cancellation
17
+ # - POST /api/v1/sdk/workers/register: Register or heartbeat a polyglot SDK worker
18
+ # - POST /api/v1/sdk/workers/{id}/deregister: Deregister a polyglot SDK worker on shutdown
19
+ # - POST /api/v1/sdk/workflows: Start a durable workflow execution
20
+ # - GET /api/v1/sdk/workflows/{id}: Describe one workflow run: status, output, failure and search attributes
21
+ # - GET /api/v1/sdk/workflows/{id}/history: The events of a workflow run, after after_seq, at most limit
22
+ # - POST /api/v1/sdk/workflows/{id}/query: Ask a workflow run a read-only question
23
+ # - GET /api/v1/sdk/workflows/{id}/result: The outcome of a workflow, following continue-as-new; waits up to wait_ms for it to close
24
+ # - POST /api/v1/sdk/workflows/{id}/search-attributes: Set and remove search attributes of a workflow's open run
25
+ # - POST /api/v1/sdk/workflows/{id}/signal: Deliver an asynchronous signal to a running workflow
26
+ # - POST /api/v1/sdk/workflows/{id}/updates: Send an update to a running workflow and wait for its outcome
27
+ # NOTE: Queue & workflow management (pause, resume, redrive, cancel, terminate) is intentionally
28
+ # excluded from SDKs and must be performed via the JoobQUI Operator Console (/ui/).
29
+
30
+ from __future__ import annotations
31
+ import http.client
32
+ import json
33
+ import os
34
+ import queue as _queue
35
+ import random
36
+ import signal as _signal
37
+ import socket
38
+ import sys
39
+ import threading
40
+ import time
41
+ import traceback
42
+ import urllib.parse
43
+ import urllib.request
44
+ from concurrent.futures import Future, ThreadPoolExecutor
45
+ from dataclasses import dataclass, field
46
+ from typing import Any, Callable, Dict, List, Optional, Type, Union
47
+
48
+ SDK_VERSION = "0.1.0"
49
+
50
+
51
+
52
+ class NonRetryableError(Exception):
53
+ """Raised by a Job or Workflow Activity when a failure is permanent and must not be retried."""
54
+
55
+
56
+ @dataclass
57
+ class RetryPolicy:
58
+ attempts: int = 3
59
+ initial_ms: int = 1000
60
+ backoff: float = 2.0
61
+ maximum_ms: int = 30000
62
+ # Error types (the exception's class name, as the worker reports it) that end the step without another attempt.
63
+ non_retryable: List[str] = field(default_factory=list)
64
+
65
+
66
+ def retry_policy(
67
+ attempts: int = 3,
68
+ initial_ms: int = 1000,
69
+ backoff: float = 2.0,
70
+ maximum_ms: int = 30000,
71
+ non_retryable: Optional[List[str]] = None,
72
+ ) -> RetryPolicy:
73
+ return RetryPolicy(attempts=attempts, initial_ms=initial_ms, backoff=backoff, maximum_ms=maximum_ms,
74
+ non_retryable=list(non_retryable or []))
75
+
76
+
77
+ @dataclass
78
+ class DataRef:
79
+ expr: str
80
+
81
+ def to_expr(self) -> str:
82
+ return self.expr
83
+
84
+
85
+ def input(key: Optional[str] = None) -> DataRef:
86
+ return DataRef("${input}" if not key else f"${{input.{key}}}")
87
+
88
+
89
+ def output(step_name: str, key: Optional[str] = None) -> DataRef:
90
+ return DataRef(
91
+ f"${{steps.{step_name}.output}}"
92
+ if not key
93
+ else f"${{steps.{step_name}.output.{key}}}"
94
+ )
95
+
96
+
97
+ def update_input(key: Optional[str] = None) -> DataRef:
98
+ """The input of the update being handled (`${update.input...}`): for an update's `validate`, `input` and `result`."""
99
+ return DataRef("${update.input}" if not key else f"${{update.input.{key}}}")
100
+
101
+
102
+ def ref(path: str) -> DataRef:
103
+ clean = path.strip()
104
+ if clean.startswith("${") and clean.endswith("}"):
105
+ return DataRef(clean)
106
+ return DataRef(f"${{{clean}}}")
107
+
108
+
109
+ def _lookup_path(root: Any, dotted: str) -> Any:
110
+ cur = root
111
+ for part in dotted.split("."):
112
+ if not part:
113
+ continue
114
+ if isinstance(cur, dict):
115
+ cur = cur.get(part)
116
+ else:
117
+ cur = getattr(cur, part, None)
118
+ if cur is None:
119
+ return None
120
+ return cur
121
+
122
+
123
+ def resolve_references(spec: Any, workflow_input: Any, checkpoints: Dict[str, Any]) -> Any:
124
+ if spec is None:
125
+ return workflow_input
126
+ if isinstance(spec, DataRef):
127
+ expr = spec.expr
128
+ if expr == "${input}":
129
+ return workflow_input
130
+ if expr.startswith("${input.") and expr.endswith("}"):
131
+ return _lookup_path(workflow_input, expr[8:-1])
132
+ if expr.startswith("${steps.") and expr.endswith("}"):
133
+ rest = expr[8:-1]
134
+ parts = rest.split(".")
135
+ step_name = parts[0]
136
+ step_out = checkpoints.get(step_name)
137
+ if len(parts) <= 2:
138
+ return step_out
139
+ return _lookup_path(step_out, ".".join(parts[2:]))
140
+ if expr == "${steps}":
141
+ return dict(checkpoints)
142
+ if expr.startswith("${") and expr.endswith("}"):
143
+ return _lookup_path({"input": workflow_input, "steps": checkpoints}, expr[2:-1])
144
+ return expr
145
+ if isinstance(spec, dict):
146
+ return {k: resolve_references(v, workflow_input, checkpoints) for k, v in spec.items()}
147
+ if isinstance(spec, list):
148
+ return [resolve_references(v, workflow_input, checkpoints) for v in spec]
149
+ return spec
150
+
151
+
152
+ def _extract_step_dependencies(spec: Any) -> List[str]:
153
+ deps: List[str] = []
154
+ if isinstance(spec, DataRef):
155
+ if spec.expr.startswith("${steps."):
156
+ step_name = spec.expr[8:-1].split(".")[0]
157
+ if step_name and step_name not in deps:
158
+ deps.append(step_name)
159
+ elif isinstance(spec, dict):
160
+ for v in spec.values():
161
+ for d in _extract_step_dependencies(v):
162
+ if d not in deps:
163
+ deps.append(d)
164
+ elif isinstance(spec, list):
165
+ for v in spec:
166
+ for d in _extract_step_dependencies(v):
167
+ if d not in deps:
168
+ deps.append(d)
169
+ return deps
170
+
171
+
172
+ def _collect_references(spec: Any) -> List[str]:
173
+ """The `${...}` references a step input makes (what a blueprint sends: never the literal values around them)."""
174
+ found: List[str] = []
175
+ if isinstance(spec, DataRef):
176
+ found.append(spec.expr)
177
+ elif isinstance(spec, dict):
178
+ for v in spec.values():
179
+ found.extend(r for r in _collect_references(v) if r not in found)
180
+ elif isinstance(spec, (list, tuple)):
181
+ for v in spec:
182
+ found.extend(r for r in _collect_references(v) if r not in found)
183
+ return found
184
+
185
+
186
+ def _template(spec: Any, where: str) -> Any:
187
+ """A child's or continued run's input as the engine resolves it: references (`input()`, `output()`, `ref()`) in
188
+ dicts and lists. Literal values are refused: compute a constant in a step and refer to its output."""
189
+ if isinstance(spec, DataRef):
190
+ return spec.expr
191
+ if isinstance(spec, dict):
192
+ return {k: _template(v, f"{where}.{k}") for k, v in spec.items()}
193
+ if isinstance(spec, (list, tuple)):
194
+ return [_template(v, f"{where}[{i}]") for i, v in enumerate(spec)]
195
+ raise ValueError(f"{where}: {spec!r} is not a reference; the engine resolves this input, so only references are sent")
196
+
197
+
198
+ def _callable_name(target: Any) -> Optional[str]:
199
+ if isinstance(target, type):
200
+ return getattr(target, "job_name", None) or target.__name__
201
+ name = getattr(target, "__name__", None) or getattr(type(target), "__name__", None)
202
+ return None if not name or name == "<lambda>" or name == "function" else name
203
+
204
+
205
+ @dataclass
206
+ class Gate:
207
+ """Puts a step on one side of a change (versioning): the step runs only for runs whose version of `change` is within
208
+ `min..max` (open ended when None), and is skipped otherwise. A new run takes the change's `max`."""
209
+ change: str
210
+ min: Optional[int] = None
211
+ max: Optional[int] = None
212
+
213
+
214
+ def gate(change: str, *, min: Optional[int] = None, max: Optional[int] = None) -> Gate:
215
+ if min is None and max is None:
216
+ raise ValueError(f"gate on {change!r} needs a min or a max")
217
+ return Gate(change, min, max)
218
+
219
+
220
+ @dataclass
221
+ class UpdateDef:
222
+ name: str
223
+ handler: Optional[Callable[[Any, "ExecutionContext"], Any]] = None
224
+ validate: Optional[DataRef] = None
225
+ message: Optional[str] = None
226
+ input: Any = None
227
+ result: Any = None
228
+ timeout_ms: Optional[int] = None
229
+
230
+
231
+ @dataclass
232
+ class Compensation:
233
+ handler: Callable[[Any, "ExecutionContext"], Any]
234
+ input: Any = None
235
+ activity_name: Optional[str] = None
236
+
237
+
238
+ def compensation(
239
+ handler: Union[Callable[[Any, "ExecutionContext"], Any], Type[Any], Any],
240
+ *,
241
+ input: Any = None,
242
+ ) -> Compensation:
243
+ fn = _normalize_callable(handler)
244
+ return Compensation(handler=fn, input=input, activity_name=_callable_name(handler))
245
+
246
+
247
+ def _normalize_callable(target: Any) -> Callable[[Any, "ExecutionContext"], Any]:
248
+ if isinstance(target, type) and hasattr(target, "perform"):
249
+ def _invoke_cls(payload: Any, ctx: "ExecutionContext") -> Any:
250
+ inst = target()
251
+ return inst.perform(payload, ctx)
252
+ return _invoke_cls
253
+ if hasattr(target, "perform") and callable(getattr(target, "perform")):
254
+ return getattr(target, "perform")
255
+ if callable(target):
256
+ return target
257
+ raise TypeError(f"Expected callable or Activity/Job with perform(payload, ctx), got {target!r}")
258
+
259
+
260
+ class Job:
261
+ """Declarative Job base class mirroring Crystal's `JoobQ::Job`."""
262
+
263
+ job_name: Optional[str] = None
264
+ queue: str = "default"
265
+ retries: int = 3
266
+ expires_ms: Optional[int] = None
267
+ encrypted_fields: List[str] = []
268
+ unique_ttl_ms: Optional[int] = None
269
+
270
+ @classmethod
271
+ def resolved_name(cls) -> str:
272
+ return cls.job_name or cls.__name__
273
+
274
+ @staticmethod
275
+ def perform(payload: Any, ctx: "ExecutionContext") -> Any:
276
+ raise NotImplementedError("Subclasses of Job must implement perform(payload, ctx)")
277
+
278
+ @classmethod
279
+ def enqueue(cls, client: "JoobQClient", payload: Any = None, **kwargs: Any) -> Dict[str, Any]:
280
+ return client.enqueue(cls, payload, **kwargs)
281
+
282
+
283
+ def job(
284
+ name: Optional[str] = None,
285
+ *,
286
+ queue: str = "default",
287
+ retries: int = 3,
288
+ expires_ms: Optional[int] = None,
289
+ encrypted_fields: Optional[List[str]] = None,
290
+ unique_ttl_ms: Optional[int] = None,
291
+ ) -> Callable[[Callable[[Any, "ExecutionContext"], Any]], Type[Job]]:
292
+ """Decorator for defining a `Job` from a function with `JoobQ::Job` metadata."""
293
+
294
+ def _decorator(fn: Callable[[Any, "ExecutionContext"], Any]) -> Type[Job]:
295
+ job_type_name = name or fn.__name__
296
+ attrs = {
297
+ "job_name": job_type_name,
298
+ "queue": queue,
299
+ "retries": retries,
300
+ "expires_ms": expires_ms,
301
+ "encrypted_fields": list(encrypted_fields or []),
302
+ "unique_ttl_ms": unique_ttl_ms,
303
+ "perform": staticmethod(fn),
304
+ }
305
+ return type(job_type_name, (Job,), attrs)
306
+
307
+ return _decorator
308
+
309
+
310
+ @dataclass
311
+ class WorkflowStepDef:
312
+ kind: str # "step" | "wait_for" | "timer" | "child" | "parallel" | "sequence"
313
+ name: str
314
+ handler: Optional[Callable[[Any, "ExecutionContext"], Any]] = None
315
+ input_spec: Any = None
316
+ retry: Optional[RetryPolicy] = None
317
+ compensate: Optional[Compensation] = None
318
+ signal_name: Optional[str] = None
319
+ timeout_ms: Optional[int] = None
320
+ duration_ms: Optional[int] = None
321
+ children: List["WorkflowStepDef"] = field(default_factory=list)
322
+ activity_name: Optional[str] = None
323
+ after: List[str] = field(default_factory=list)
324
+ heartbeat_ms: Optional[int] = None
325
+ schedule_to_start_ms: Optional[int] = None
326
+ only_if: Optional[DataRef] = None
327
+ workflow_name: Optional[str] = None
328
+ gate: Optional[Gate] = None
329
+ non_cancellable: bool = False
330
+
331
+
332
+ class WorkflowBlueprint:
333
+ """Declarative workflow builder (the engine conducts the run and its compensations), mirroring `JoobQPro::Workflows::DSL`."""
334
+
335
+ def __init__(
336
+ self,
337
+ name: str,
338
+ *,
339
+ version: int = 1,
340
+ timeout_ms: Optional[int] = None,
341
+ task_queue: str = "workflows",
342
+ ) -> None:
343
+ self.name = name
344
+ self.version = version
345
+ self.timeout_ms = timeout_ms
346
+ self.task_queue = task_queue
347
+ self.steps: List[WorkflowStepDef] = []
348
+ self.queries: Dict[str, DataRef] = {}
349
+ self._declared_steps: set[str] = set()
350
+ self.continuation: Optional[Dict[str, Any]] = None
351
+ self.changes: Dict[str, Dict[str, int]] = {}
352
+ self.updates: Dict[str, UpdateDef] = {}
353
+ self._in_scope = False
354
+
355
+ def _validate_and_register(self, step_name: str, input_spec: Any, after: Optional[List[str]] = None) -> None:
356
+ if step_name in self._declared_steps:
357
+ raise ValueError(f"Duplicate workflow step name '{step_name}' in workflow '{self.name}'")
358
+ for dep in after or []:
359
+ if dep not in self._declared_steps:
360
+ raise ValueError(f"Step '{step_name}' in workflow '{self.name}' waits (after=) for unknown or downstream step '{dep}'")
361
+ for dep in _extract_step_dependencies(input_spec):
362
+ if dep not in self._declared_steps:
363
+ raise ValueError(
364
+ f"Step '{step_name}' in workflow '{self.name}' references output of unknown or downstream step '{dep}'"
365
+ )
366
+ self._declared_steps.add(step_name)
367
+
368
+ def query(self, name: str, expr: DataRef) -> "WorkflowBlueprint":
369
+ self.queries[name] = expr
370
+ return self
371
+
372
+ def step(
373
+ self,
374
+ name: str,
375
+ activity: Union[Callable[[Any, "ExecutionContext"], Any], Type[Any], Any],
376
+ *,
377
+ input: Any = None,
378
+ retry: Optional[RetryPolicy] = None,
379
+ compensate: Optional[Compensation] = None,
380
+ after: Optional[List[str]] = None,
381
+ timeout_ms: Optional[int] = None,
382
+ heartbeat_ms: Optional[int] = None,
383
+ schedule_to_start_ms: Optional[int] = None,
384
+ only_if: Optional[DataRef] = None,
385
+ gate: Optional[Gate] = None,
386
+ ) -> "WorkflowBlueprint":
387
+ """A step. `after` names earlier steps it also waits for (on top of what precedes it, e.g. a sibling in a
388
+ `parallel` group); `timeout_ms` limits one attempt, `heartbeat_ms` how long the handler may go without calling
389
+ `ctx.heartbeat()`, `schedule_to_start_ms` how long the step may wait for a worker; `only_if` (a reference) runs
390
+ the step only when it is truthy, else the step is skipped; `gate` (see `gate()`) runs it only on one side of a
391
+ change. The engine enforces them."""
392
+ self._check_condition(name, only_if) # before the name is declared: a step cannot gate on itself
393
+ self._validate_and_register(name, input, after)
394
+ self.steps.append(
395
+ WorkflowStepDef(
396
+ kind="step",
397
+ name=name,
398
+ handler=_normalize_callable(activity),
399
+ activity_name=_callable_name(activity),
400
+ input_spec=input,
401
+ retry=retry,
402
+ compensate=compensate,
403
+ after=list(after or []),
404
+ timeout_ms=timeout_ms,
405
+ heartbeat_ms=heartbeat_ms,
406
+ schedule_to_start_ms=schedule_to_start_ms,
407
+ only_if=only_if,
408
+ gate=gate,
409
+ )
410
+ )
411
+ return self
412
+
413
+ def _check_condition(self, name: str, only_if: Optional[DataRef]) -> None:
414
+ if only_if is None:
415
+ return
416
+ if not isinstance(only_if, DataRef):
417
+ raise ValueError(f"Step '{name}' in workflow '{self.name}': only_if must be a reference (input(), output(), ref())")
418
+ for dep in _extract_step_dependencies(only_if):
419
+ if dep not in self._declared_steps:
420
+ raise ValueError(f"Step '{name}' in workflow '{self.name}': only_if refers to unknown or downstream step '{dep}'")
421
+
422
+ def wait_for(self, name: str, *, signal: str, timeout_ms: int = 600_000,
423
+ after: Optional[List[str]] = None, only_if: Optional[DataRef] = None,
424
+ compensate: Optional[Compensation] = None, gate: Optional[Gate] = None) -> "WorkflowBlueprint":
425
+ """Waits for a signal. `compensate` undoes it (its input defaults to the signal's payload) when the run fails."""
426
+ self._check_condition(name, only_if)
427
+ self._validate_and_register(name, None, after)
428
+ self.steps.append(
429
+ WorkflowStepDef(kind="wait_for", name=name, signal_name=signal, timeout_ms=timeout_ms, after=list(after or []),
430
+ only_if=only_if, compensate=compensate, gate=gate)
431
+ )
432
+ return self
433
+
434
+ def timer(self, name: str, *, for_ms: int, after: Optional[List[str]] = None, only_if: Optional[DataRef] = None,
435
+ compensate: Optional[Compensation] = None, gate: Optional[Gate] = None) -> "WorkflowBlueprint":
436
+ """Waits for a time. `compensate` undoes it (its input defaults to the timer's output) when the run fails."""
437
+ self._check_condition(name, only_if)
438
+ self._validate_and_register(name, None, after)
439
+ self.steps.append(WorkflowStepDef(kind="timer", name=name, duration_ms=for_ms, after=list(after or []),
440
+ only_if=only_if, compensate=compensate, gate=gate))
441
+ return self
442
+
443
+ def child(self, name: str, workflow_def: Union["WorkflowBlueprint", str], *, input: Any = None,
444
+ compensate: Optional[Compensation] = None, after: Optional[List[str]] = None,
445
+ only_if: Optional[DataRef] = None, gate: Optional[Gate] = None) -> "WorkflowBlueprint":
446
+ """Runs another workflow (a blueprint or its name) as a step; its output is the child run's output, its failure
447
+ the step's. The engine starts the child and resolves `input` (references only: `input()`, `output()`, `ref()`,
448
+ in dicts and lists; the run's input when omitted). A child is not retried: give the child's own steps retries,
449
+ and the child workflow a `timeout_ms`. The child workflow must be registered before a run reaches this step."""
450
+ if input is not None:
451
+ _template(input, f"{name}.input")
452
+ self._check_condition(name, only_if)
453
+ self._validate_and_register(name, input, after)
454
+ child_name = workflow_def.name if isinstance(workflow_def, WorkflowBlueprint) else str(workflow_def)
455
+ self.steps.append(WorkflowStepDef(kind="child", name=name, workflow_name=child_name, input_spec=input,
456
+ compensate=compensate, after=list(after or []), only_if=only_if, gate=gate))
457
+ return self
458
+
459
+ def continue_as_new(self, *, input: Any = None, only_if: Optional[DataRef] = None, max_runs: Optional[int] = None,
460
+ max_events: Optional[int] = None) -> "WorkflowBlueprint":
461
+ """When the run ends (and `only_if` is truthy), it closes `continued_as_new` and a fresh run of this workflow starts
462
+ with `input` (references only, resolved by the engine; the same input when omitted), at most `max_runs` runs."""
463
+ policy: Dict[str, Any] = {}
464
+ if input is not None:
465
+ policy["input"] = _template(input, "continue_as_new.input")
466
+ if only_if is not None:
467
+ if not isinstance(only_if, DataRef):
468
+ raise ValueError("continue_as_new: only_if must be a reference (input(), output(), ref())")
469
+ policy["only_if"] = only_if.expr
470
+ if max_runs is not None:
471
+ policy["max_runs"] = max_runs
472
+ if max_events is not None:
473
+ policy["max_events"] = max_events
474
+ self.continuation = policy
475
+ return self
476
+
477
+ def change(self, change_id: str, *, min: int = 0, max: int = 1) -> "WorkflowBlueprint":
478
+ """Declares a change (versioning, like Temporal's getVersion): runs that reach a gated step fresh take `max`,
479
+ runs that went past it on the old path keep 0; steps pick a side with `gate=gate(change_id, min=..., max=...)`.
480
+ Raise `min` only once no open run is on an older path."""
481
+ if min > max:
482
+ raise ValueError(f"change {change_id!r} in workflow {self.name!r}: min is above max")
483
+ self.changes[change_id] = {"min": min, "max": max}
484
+ return self
485
+
486
+ def update(self, name: str, handler: Optional[Union[Callable[[Any, "ExecutionContext"], Any], Type[Any], Any]] = None, *,
487
+ validate: Optional[DataRef] = None, message: Optional[str] = None, input: Any = None, result: Any = None,
488
+ timeout_ms: Optional[int] = None) -> "WorkflowBlueprint":
489
+ """An update handler (`JoobQClient.update_workflow`): `validate` (a reference, e.g. `update_input("name")`) must be
490
+ truthy or the update is rejected with `message`; then `handler` runs as a step of the run (its payload is `input`,
491
+ by default the update's input, and its return value is the result), or, without one, `result` (a template of
492
+ references) is the result. References may read the run (`input()`, `output()`) and `update_input()`."""
493
+ if name in self.updates:
494
+ raise ValueError(f"Duplicate update {name!r} in workflow {self.name!r}")
495
+ if validate is not None and not isinstance(validate, DataRef):
496
+ raise ValueError(f"update {name!r}: validate must be a reference (update_input(), input(), output(), ref())")
497
+ if input is not None:
498
+ _template(input, f"update {name}.input")
499
+ if result is not None:
500
+ _template(result, f"update {name}.result")
501
+ fn = _normalize_callable(handler) if handler is not None else None
502
+ self.updates[name] = UpdateDef(name, fn, validate, message, input, result, timeout_ms)
503
+ return self
504
+
505
+ def scope(self, name: str, build_fn: Callable[["WorkflowBlueprint"], Any], *, timeout_ms: Optional[int] = None,
506
+ non_cancellable: bool = False) -> "WorkflowBlueprint":
507
+ """A cancellation scope: its steps run in sequence and can be cancelled as a unit (from the console or API,
508
+ or after `timeout_ms` from its first step's start) while the rest of the run goes on; a cancelled step counts as
509
+ finished with a null output. A `non_cancellable` scope holds cleanup that still runs when the workflow is cancelled,
510
+ times out or fails (it cannot have a timeout). A running step sees a cancel through `ctx.heartbeat()`.
511
+ Scopes do not nest."""
512
+ if self._in_scope:
513
+ raise ValueError(f"scope {name!r} in workflow {self.name!r} is inside another scope: scopes do not nest")
514
+ if non_cancellable and timeout_ms is not None:
515
+ raise ValueError(f"scope {name!r} is non_cancellable and cannot time out")
516
+ inner = WorkflowBlueprint(f"{self.name}:scope", version=self.version, timeout_ms=self.timeout_ms, task_queue=self.task_queue)
517
+ inner._declared_steps = set(self._declared_steps)
518
+ inner._in_scope = True
519
+ build_fn(inner)
520
+ if not inner.steps:
521
+ raise ValueError(f"scope {name!r} in workflow {self.name!r} has no steps")
522
+ self._declared_steps.update(inner._declared_steps)
523
+ self.steps.append(WorkflowStepDef(kind="scope", name=name, children=inner.steps, timeout_ms=timeout_ms,
524
+ non_cancellable=non_cancellable))
525
+ return self
526
+
527
+ def parallel(self, build_fn: Callable[["WorkflowBlueprint"], Any]) -> "WorkflowBlueprint":
528
+ branch = WorkflowBlueprint(f"{self.name}:parallel", version=self.version, timeout_ms=self.timeout_ms, task_queue=self.task_queue)
529
+ branch._declared_steps = set(self._declared_steps)
530
+ branch._in_scope = self._in_scope
531
+ build_fn(branch)
532
+ self._declared_steps.update(branch._declared_steps)
533
+ self.steps.append(WorkflowStepDef(kind="parallel", name=f"parallel_{len(self.steps) + 1}", children=branch.steps))
534
+ return self
535
+
536
+ def sequence(self, build_fn: Callable[["WorkflowBlueprint"], Any]) -> "WorkflowBlueprint":
537
+ seq = WorkflowBlueprint(f"{self.name}:sequence", version=self.version, timeout_ms=self.timeout_ms, task_queue=self.task_queue)
538
+ seq._declared_steps = set(self._declared_steps)
539
+ seq._in_scope = self._in_scope
540
+ build_fn(seq)
541
+ self._declared_steps.update(seq._declared_steps)
542
+ self.steps.append(WorkflowStepDef(kind="sequence", name=f"sequence_{len(self.steps) + 1}", children=seq.steps))
543
+ return self
544
+
545
+ def to_blueprint(self) -> Dict[str, Any]:
546
+ """The workflow's structure as data, as the worker registers it with the gateway (`CONTRACT.md`)."""
547
+
548
+ def node(s: WorkflowStepDef) -> Dict[str, Any]:
549
+ out: Dict[str, Any] = {"name": s.name, "kind": s.kind}
550
+ if s.kind in ("parallel", "sequence", "scope"):
551
+ out["children"] = [node(c) for c in s.children]
552
+ if s.kind == "scope":
553
+ if s.timeout_ms is not None:
554
+ out["timeout_ms"] = s.timeout_ms
555
+ if s.non_cancellable:
556
+ out["non_cancellable"] = True
557
+ elif s.kind == "wait_for":
558
+ out.update({"signal": s.signal_name, "timeout_ms": s.timeout_ms})
559
+ elif s.kind == "timer":
560
+ out["for_ms"] = s.duration_ms
561
+ elif s.kind == "child":
562
+ out["workflow"] = s.workflow_name
563
+ if s.input_spec is not None:
564
+ out["input"] = _template(s.input_spec, f"{s.name}.input")
565
+ else:
566
+ out["activity"] = s.activity_name
567
+ out["references"] = _collect_references(s.input_spec) if s.input_spec is not None else ["${input}"]
568
+ for key, value in (("timeout_ms", s.timeout_ms), ("heartbeat_ms", s.heartbeat_ms),
569
+ ("schedule_to_start_ms", s.schedule_to_start_ms)):
570
+ if value is not None:
571
+ out[key] = value
572
+ if s.retry is not None:
573
+ out["retry"] = {"attempts": s.retry.attempts, "initial_ms": s.retry.initial_ms,
574
+ "backoff": s.retry.backoff, "maximum_ms": s.retry.maximum_ms}
575
+ if s.retry.non_retryable:
576
+ out["retry"]["non_retryable"] = list(s.retry.non_retryable)
577
+ if s.kind in ("step", "wait_for", "timer", "child"):
578
+ if s.compensate is not None:
579
+ comp_in = s.compensate.input if s.compensate.input is not None else output(s.name)
580
+ out["compensate"] = {"activity": s.compensate.activity_name, "references": _collect_references(comp_in)}
581
+ if s.only_if is not None:
582
+ out["only_if"] = s.only_if.expr
583
+ if s.gate is not None:
584
+ out["gate"] = {"change": s.gate.change, **({"min": s.gate.min} if s.gate.min is not None else {}),
585
+ **({"max": s.gate.max} if s.gate.max is not None else {})}
586
+ if s.after:
587
+ out["after"] = list(s.after)
588
+ return out
589
+
590
+ blueprint: Dict[str, Any] = {
591
+ "name": self.name,
592
+ "version": self.version,
593
+ "task_queue": self.task_queue,
594
+ "queries": {k: v.to_expr() for k, v in self.queries.items()},
595
+ "nodes": [node(s) for s in self.steps],
596
+ }
597
+ if self.timeout_ms is not None:
598
+ blueprint["timeout_ms"] = self.timeout_ms # the whole run's deadline, enforced by the engine
599
+ if self.continuation is not None:
600
+ blueprint["continue_as_new"] = dict(self.continuation)
601
+ if self.changes:
602
+ blueprint["changes"] = [{"id": cid, **bounds} for cid, bounds in self.changes.items()]
603
+ if self.updates:
604
+ blueprint["updates"] = {u.name: _update_spec(u) for u in self.updates.values()}
605
+ return blueprint
606
+
607
+ def activities(self) -> Dict[str, Callable[[Any, "ExecutionContext"], Any]]:
608
+ """The handlers of this workflow's steps, by the activity name the engine sends them under (`<workflow>/<step>`,
609
+ compensations `<workflow>/<step>/compensate`, update handlers `<workflow>/@update/<name>`). Each runs its step
610
+ once: the engine conducts the run (it retries, compensates and waits). A step's input is resolved against the
611
+ run's input (the task's payload) and the outputs of the steps that completed (the task's checkpoints)."""
612
+ found: Dict[str, Callable[[Any, "ExecutionContext"], Any]] = {}
613
+
614
+ def step_activity(node: WorkflowStepDef) -> Callable[[Any, "ExecutionContext"], Any]:
615
+ def run(workflow_input: Any, ctx: "ExecutionContext") -> Any:
616
+ assert node.handler is not None
617
+ return node.handler(resolve_references(node.input_spec, workflow_input, ctx.task.checkpoints), ctx)
618
+ return run
619
+
620
+ def compensation_activity(node: WorkflowStepDef, comp: Compensation) -> Callable[[Any, "ExecutionContext"], Any]:
621
+ def run(workflow_input: Any, ctx: "ExecutionContext") -> Any:
622
+ spec = comp.input if comp.input is not None else output(node.name)
623
+ return comp.handler(resolve_references(spec, workflow_input, ctx.task.checkpoints), ctx)
624
+ return run
625
+
626
+ def update_activity(update: UpdateDef) -> Callable[[Any, "ExecutionContext"], Any]:
627
+ def run(update_payload: Any, ctx: "ExecutionContext") -> Any:
628
+ assert update.handler is not None
629
+ return update.handler(update_payload, ctx) # the engine resolved the handler's input already
630
+ return run
631
+
632
+ def visit(nodes: List[WorkflowStepDef]) -> None:
633
+ for node in nodes:
634
+ if node.kind in ("sequence", "parallel", "scope"):
635
+ visit(node.children)
636
+ else:
637
+ if node.kind == "step":
638
+ found[_activity_name(self.name, node.name)] = step_activity(node)
639
+ if node.compensate is not None: # a step, wait, timer or child can be undone
640
+ found[_activity_name(self.name, node.name, True)] = compensation_activity(node, node.compensate)
641
+
642
+ visit(self.steps)
643
+ for update in self.updates.values():
644
+ if update.handler is not None:
645
+ found[_update_activity(self.name, update.name)] = update_activity(update)
646
+ return found
647
+
648
+
649
+ def _update_spec(update: UpdateDef) -> Dict[str, Any]:
650
+ spec: Dict[str, Any] = {}
651
+ if update.handler is not None:
652
+ spec["handler"] = True
653
+ if update.validate is not None:
654
+ spec["validate"] = update.validate.expr
655
+ if update.message:
656
+ spec["message"] = update.message
657
+ if update.input is not None:
658
+ spec["input"] = _template(update.input, f"update {update.name}.input")
659
+ if update.result is not None:
660
+ spec["result"] = _template(update.result, f"update {update.name}.result")
661
+ if update.timeout_ms is not None:
662
+ spec["timeout_ms"] = update.timeout_ms
663
+ return spec
664
+
665
+
666
+ def _update_activity(workflow_name: str, update: str) -> str:
667
+ """The activity the engine sends an update handler of an SDK workflow under."""
668
+ return f"{workflow_name}/@update/{update}"
669
+
670
+
671
+ def _activity_name(workflow_name: str, step: str, compensate: bool = False) -> str:
672
+ """The activity the engine sends a step (or its compensation) of an SDK workflow under."""
673
+ return f"{workflow_name}/{step}/compensate" if compensate else f"{workflow_name}/{step}"
674
+
675
+
676
+ def workflow(
677
+ name: str,
678
+ *,
679
+ version: int = 1,
680
+ timeout_ms: Optional[int] = None,
681
+ task_queue: str = "workflows",
682
+ ) -> WorkflowBlueprint:
683
+ """A workflow blueprint. `timeout_ms` is the whole run's deadline (none by default), enforced by the engine."""
684
+ return WorkflowBlueprint(name, version=version, timeout_ms=timeout_ms, task_queue=task_queue)
685
+
686
+
687
+ @dataclass
688
+ class LeasedTask:
689
+ lease_token: str
690
+ kind: str
691
+ id: str
692
+ queue: str
693
+ name: str
694
+ attempt: int
695
+ max_retries: int
696
+ payload: Any
697
+ context: Dict[str, Any] = field(default_factory=dict)
698
+ workflow_id: Optional[str] = None
699
+ step_id: Optional[str] = None
700
+ checkpoints: Dict[str, Any] = field(default_factory=dict) # a step's: the outputs of the run's completed steps
701
+ lease_until_ms: int = 0
702
+ heartbeat_ms: Optional[int] = None # an engine step's heartbeat timeout: call ctx.heartbeat() more often than this
703
+
704
+ @classmethod
705
+ def from_dict(cls, raw: Dict[str, Any]) -> "LeasedTask":
706
+ known = {f for f in cls.__dataclass_fields__}
707
+ return cls(**{k: v for k, v in raw.items() if k in known})
708
+
709
+
710
+ class ExecutionContext:
711
+ """Carries the propagated request `context`, heartbeating, and downstream context-forwarding helpers. Handlers
712
+ reach the application's own dependencies directly: the worker runs inside the booted application."""
713
+
714
+ def __init__(self, task: LeasedTask, worker_id: str, client: "JoobQClient", lease_ttl_ms: int = 30000,
715
+ transport: Optional["_Transport"] = None) -> None:
716
+ self.task = task
717
+ self.worker_id = worker_id
718
+ self.client = client
719
+ self.lease_ttl_ms = lease_ttl_ms
720
+ self._transport = transport
721
+ self.context: Dict[str, Any] = dict(task.context or {})
722
+ self._last_beat_ms = float("-inf")
723
+ self._beat_cancel = False
724
+
725
+ @property
726
+ def attempt(self) -> int:
727
+ return self.task.attempt
728
+
729
+ @property
730
+ def dedupe_key(self) -> str:
731
+ wf = self.task.workflow_id or self.task.id
732
+ step = self.task.step_id or self.task.name
733
+ return f"{wf}:{step}:{self.task.attempt}"
734
+
735
+ def heartbeat(self, lease_ttl_ms: Optional[int] = None) -> bool:
736
+ """True when the run was cancelled. Inside a running worker this answers what the worker's last lease renewal
737
+ said, without waiting for the gateway (the worker renews every lease on its own). For a step with
738
+ `heartbeat_ms` it also tells the engine the step is alive (at most every third of `heartbeat_ms`): the worker's
739
+ own renewals do not, so a handler that stops calling this is timed out."""
740
+ if self.task.heartbeat_ms:
741
+ now = time.monotonic() * 1000
742
+ if now - self._last_beat_ms >= max(100, self.task.heartbeat_ms // 3):
743
+ self._last_beat_ms = now
744
+ try:
745
+ res = self.client.heartbeat_task(self.task.lease_token, self.worker_id, lease_ttl_ms or self.lease_ttl_ms)
746
+ self._beat_cancel = bool(res.get("cancel_requested", False))
747
+ except Exception:
748
+ pass # the next window's beat tries again; the engine's deadline decides
749
+ if self._beat_cancel:
750
+ return True
751
+ if self._transport is not None:
752
+ return self._transport.cancel_requested(self.task.lease_token)
753
+ res = self.client.heartbeat_task(self.task.lease_token, self.worker_id, lease_ttl_ms or self.lease_ttl_ms)
754
+ return bool(res.get("cancel_requested", False))
755
+
756
+ def enqueue(self, job_cls: Type[Job], payload: Any = None, **kwargs: Any) -> Dict[str, Any]:
757
+ merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
758
+ return self.client.enqueue(job_cls, payload, context=merged_ctx, **kwargs)
759
+
760
+ def enqueue_job(self, queue: str, job_type: str, payload: Any = None, **kwargs: Any) -> Dict[str, Any]:
761
+ merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
762
+ return self.client.enqueue_job(queue, job_type, payload, context=merged_ctx, **kwargs)
763
+
764
+ def start(self, workflow_def: WorkflowBlueprint, input_data: Any = None, **kwargs: Any) -> Dict[str, Any]:
765
+ merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
766
+ return self.client.start(workflow_def, input_data, context=merged_ctx, **kwargs)
767
+
768
+ def start_workflow(self, name: str, input_data: Any = None, **kwargs: Any) -> Dict[str, Any]:
769
+ merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
770
+ return self.client.start_workflow(name, input_data, context=merged_ctx, **kwargs)
771
+
772
+ def register_schedule(self, name: str, job_type: str, queue: str, **kwargs: Any) -> Dict[str, Any]:
773
+ return self.client.register_schedule(name, job_type, queue, **kwargs)
774
+
775
+ def rate_limit(self, key: str, limit: int, period_ms: int) -> Dict[str, Any]:
776
+ return self.client.rate_limit(key, limit, period_ms)
777
+
778
+ def create_batch(self, jobs: List[Any], **kwargs: Any) -> Dict[str, Any]:
779
+ """`JoobQClient.create_batch`; the members and callbacks carry this handler's context."""
780
+ merged_ctx = {**self.context, **(kwargs.pop("context", None) or {})}
781
+ return self.client.create_batch(jobs, context=merged_ctx, **kwargs)
782
+
783
+ def get_batch(self, batch_id: str) -> Dict[str, Any]:
784
+ return self.client.get_batch(batch_id)
785
+
786
+ def describe_workflow(self, workflow_id: str) -> Dict[str, Any]:
787
+ return self.client.describe_workflow(workflow_id)
788
+
789
+ def workflow_result(self, workflow_id: str, wait_ms: int = 0) -> Dict[str, Any]:
790
+ return self.client.workflow_result(workflow_id, wait_ms)
791
+
792
+ def workflow_history(self, workflow_id: str, after_seq: int = 0, limit: int = 200) -> List[Dict[str, Any]]:
793
+ return self.client.workflow_history(workflow_id, after_seq, limit)
794
+
795
+ def query_workflow(self, workflow_id: str, name: str) -> Any:
796
+ return self.client.query_workflow(workflow_id, name)
797
+
798
+ def signal_workflow(self, workflow_id: str, name: str, payload: Any = None) -> Dict[str, Any]:
799
+ return self.client.signal_workflow(workflow_id, name, payload)
800
+
801
+ def update_workflow(self, workflow_id: str, name: str, input: Any = None, *, update_id: Optional[str] = None,
802
+ wait_ms: Optional[int] = None) -> Dict[str, Any]:
803
+ return self.client.update_workflow(workflow_id, name, input, update_id=update_id, wait_ms=wait_ms)
804
+
805
+ def upsert_workflow_attributes(self, workflow_id: str, set: Optional[Dict[str, Any]] = None,
806
+ remove: Optional[List[str]] = None) -> Dict[str, Any]:
807
+ return self.client.upsert_workflow_attributes(workflow_id, set, remove)
808
+
809
+
810
+ def _seg(value: str) -> str:
811
+ """A workflow id as one URL path segment."""
812
+ return urllib.parse.quote(str(value), safe="")
813
+
814
+
815
+ def failure(exc: BaseException, retryable: bool = True) -> Dict[str, Any]:
816
+ """The fields that report `exc` to the gateway (`fail`, and a `fail` item of `ack_tasks`)."""
817
+ frames = [
818
+ {"file": f.filename, "line": f.lineno or 0, "function": f.name}
819
+ for f in traceback.extract_tb(exc.__traceback__)
820
+ ]
821
+ return {
822
+ "error_class": exc.__class__.__name__,
823
+ "error_message": (str(exc) or exc.__class__.__name__)[:4000],
824
+ "retryable": retryable,
825
+ "sdk_language": "python",
826
+ "stack_frames": frames,
827
+ "raw_stacktrace": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))[-4000:],
828
+ }
829
+
830
+
831
+ class GatewayError(Exception):
832
+ """The gateway answered with an error status (`status`, the JSON error envelope as `body`)."""
833
+
834
+ def __init__(self, status: int, body: str) -> None:
835
+ super().__init__(f"gateway answered {status}: {body[:300]}")
836
+ self.status = status
837
+ self.body = body
838
+
839
+
840
+ class JoobQClient:
841
+ def __init__(
842
+ self,
843
+ gateway_url: str,
844
+ token: str,
845
+ context_provider: Optional[Callable[[], Dict[str, Any]]] = None,
846
+ ) -> None:
847
+ self.gateway_url = gateway_url.rstrip("/")
848
+ self.token = token
849
+ self.context_provider = context_provider
850
+ self._url = urllib.parse.urlsplit(self.gateway_url)
851
+ self._local = threading.local() # one kept-alive connection per thread
852
+ # Seconds one gateway call may take. A worker lowers it (`JoobQWorker`): a call that hangs for longer than a
853
+ # lease would let every lease the worker holds expire while its tasks still run.
854
+ self.request_timeout: float = 60.0
855
+ self._async: Optional[ThreadPoolExecutor] = None
856
+ self._async_lock = threading.Lock()
857
+
858
+ def _resolve_context(self, explicit: Optional[Dict[str, Any]]) -> Dict[str, Any]:
859
+ ambient = self.context_provider() if self.context_provider else {}
860
+ return {**(ambient or {}), **(explicit or {})}
861
+
862
+ def enqueue(
863
+ self,
864
+ job_cls: Type[Job],
865
+ payload: Any = None,
866
+ *,
867
+ context: Optional[Dict[str, Any]] = None,
868
+ jid: Optional[str] = None,
869
+ delay_ms: int = 0,
870
+ run_at_ms: Optional[int] = None,
871
+ expires_ms: Optional[int] = None,
872
+ timeout_ms: Optional[int] = None,
873
+ unique_key: Optional[str] = None,
874
+ unique_mode: str = "while_locked",
875
+ ) -> Dict[str, Any]:
876
+ return self.enqueue_job(
877
+ queue=job_cls.queue,
878
+ job_type=job_cls.resolved_name(),
879
+ payload=payload,
880
+ context=context,
881
+ jid=jid,
882
+ max_retries=job_cls.retries,
883
+ delay_ms=delay_ms,
884
+ run_at_ms=run_at_ms,
885
+ expires_ms=expires_ms if expires_ms is not None else job_cls.expires_ms,
886
+ timeout_ms=timeout_ms,
887
+ encrypt_fields=list(job_cls.encrypted_fields or []),
888
+ unique_key=unique_key,
889
+ unique_ttl_ms=job_cls.unique_ttl_ms,
890
+ unique_mode=unique_mode,
891
+ )
892
+
893
+ def enqueue_job(
894
+ self,
895
+ queue: str,
896
+ job_type: str,
897
+ payload: Any = None,
898
+ *,
899
+ context: Optional[Dict[str, Any]] = None,
900
+ jid: Optional[str] = None,
901
+ max_retries: int = 3,
902
+ delay_ms: int = 0,
903
+ run_at_ms: Optional[int] = None,
904
+ expires_ms: Optional[int] = None,
905
+ timeout_ms: Optional[int] = None,
906
+ encrypt_fields: Optional[List[str]] = None,
907
+ unique_key: Optional[str] = None,
908
+ unique_ttl_ms: Optional[int] = None,
909
+ unique_mode: str = "while_locked",
910
+ ) -> Dict[str, Any]:
911
+ return self._post("/api/v1/sdk/jobs", self._job_body(queue, job_type, payload, context=context, jid=jid,
912
+ max_retries=max_retries, delay_ms=delay_ms,
913
+ run_at_ms=run_at_ms, expires_ms=expires_ms,
914
+ timeout_ms=timeout_ms,
915
+ encrypt_fields=encrypt_fields, unique_key=unique_key,
916
+ unique_ttl_ms=unique_ttl_ms, unique_mode=unique_mode))
917
+
918
+ def _job_body(self, queue: str, job_type: str, payload: Any = None, *, context: Optional[Dict[str, Any]] = None,
919
+ jid: Optional[str] = None, max_retries: int = 3, delay_ms: int = 0,
920
+ run_at_ms: Optional[int] = None, expires_ms: Optional[int] = None, timeout_ms: Optional[int] = None,
921
+ encrypt_fields: Optional[List[str]] = None, unique_key: Optional[str] = None,
922
+ unique_ttl_ms: Optional[int] = None, unique_mode: str = "while_locked") -> Dict[str, Any]:
923
+ return {
924
+ "queue": queue,
925
+ "job_type": job_type,
926
+ "payload": payload if payload is not None else {},
927
+ "context": self._resolve_context(context),
928
+ "jid": jid,
929
+ "max_retries": max_retries,
930
+ "delay_ms": delay_ms,
931
+ "run_at_ms": run_at_ms,
932
+ "expires_ms": expires_ms,
933
+ "timeout_ms": timeout_ms,
934
+ "encrypt_fields": encrypt_fields or [],
935
+ "unique_key": unique_key,
936
+ "unique_ttl_ms": unique_ttl_ms,
937
+ "unique_mode": unique_mode,
938
+ }
939
+
940
+ def enqueue_many(self, jobs: List[Any], *, context: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
941
+ """Enqueues many jobs with `POST /sdk/jobs/batch` (500 per call). Each item is `(JobClass, payload)`,
942
+ `(JobClass, payload, {"delay_ms": ..., "jid": ..., "unique_key": ...})` or a dict of `enqueue_job` arguments
943
+ (`queue`, `job_type`, `payload`, ...). Answers one `{jid, queue, status}` per job, in order."""
944
+ bodies = [self._item_body(item, context) for item in jobs]
945
+ out: List[Dict[str, Any]] = []
946
+ for start in range(0, len(bodies), 500):
947
+ out.extend(self._post("/api/v1/sdk/jobs/batch", {"jobs": bodies[start:start + 500]}).get("jobs", []))
948
+ return out
949
+
950
+ def _item_body(self, item: Any, context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
951
+ """One job of `enqueue_many` or `create_batch`: a dict of `enqueue_job` arguments or `(JobClass, payload[, opts])`."""
952
+ if isinstance(item, dict):
953
+ spec = dict(item)
954
+ spec.setdefault("context", context)
955
+ return self._job_body(spec.pop("queue"), spec.pop("job_type"), spec.pop("payload", None), **spec)
956
+ job_cls, payload, opts = (tuple(item) + ({},))[:3]
957
+ return self._job_body(job_cls.queue, job_cls.resolved_name(), payload,
958
+ context=opts.get("context", context), jid=opts.get("jid"),
959
+ max_retries=job_cls.retries, delay_ms=opts.get("delay_ms", 0),
960
+ run_at_ms=opts.get("run_at_ms"),
961
+ expires_ms=opts.get("expires_ms", job_cls.expires_ms),
962
+ timeout_ms=opts.get("timeout_ms"),
963
+ encrypt_fields=list(job_cls.encrypted_fields or []),
964
+ unique_key=opts.get("unique_key"), unique_ttl_ms=job_cls.unique_ttl_ms,
965
+ unique_mode=opts.get("unique_mode", "while_locked"))
966
+
967
+ def create_batch(self, jobs: List[Any], *, on_complete: Any = None, on_success: Any = None, on_failure: Any = None,
968
+ description: Optional[str] = None, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
969
+ """Creates a Pro batch with `POST /sdk/batches`: 1 to 1000 member jobs (the shapes `enqueue_many` takes, without a
970
+ `unique_key`) and callback jobs (the same shapes). `on_failure` is enqueued at the first dead member,
971
+ `on_complete` once every member completed or was dead-lettered, `on_success` then too when none was dead; each
972
+ once, with `{"batch": {id, kind, total, succeeded, failed}}` merged into its payload. Answers
973
+ `{batch_id, total, jobs: [{jid, queue, status}]}`. 402 pro_required without a Pro licence."""
974
+ body: Dict[str, Any] = {"description": description, "jobs": [self._batch_item_body(item, context) for item in jobs]}
975
+ for kind, callback in (("on_complete", on_complete), ("on_success", on_success), ("on_failure", on_failure)):
976
+ if callback is not None:
977
+ body[kind] = self._batch_item_body(callback, context)
978
+ return self._post("/api/v1/sdk/batches", body)
979
+
980
+ def _batch_item_body(self, item: Any, context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
981
+ """One member or callback of `create_batch`: the batch's `context` merged key by key under the job's own (the
982
+ job's keys win)."""
983
+ base = dict(context or {})
984
+ if isinstance(item, dict):
985
+ spec = dict(item)
986
+ spec["context"] = {**base, **(spec.get("context") or {})}
987
+ return self._item_body(spec, None)
988
+ job_cls, payload, opts = (tuple(item) + ({},))[:3]
989
+ opts = dict(opts or {})
990
+ opts["context"] = {**base, **(opts.get("context") or {})}
991
+ return self._item_body((job_cls, payload, opts), None)
992
+
993
+ def get_batch(self, batch_id: str) -> Dict[str, Any]:
994
+ """A Pro batch's progress: `{batch_id, description, status, total, pending, succeeded, failed, created_at,
995
+ finished_at}`; `status` is pending, then success (no member dead) or complete."""
996
+ return self._get(f"/api/v1/sdk/batches/{_seg(batch_id)}")
997
+
998
+ def enqueue_async(self, job_cls: Type[Job], payload: Any = None, **kwargs: Any) -> "Future[Dict[str, Any]]":
999
+ """`enqueue` on a background thread: the future holds `{jid, queue, status}` once the gateway stored the job
1000
+ (or the error). Use it when the caller must not wait; `enqueue` answers only once the job is stored."""
1001
+ with self._async_lock:
1002
+ if self._async is None:
1003
+ self._async = ThreadPoolExecutor(max_workers=8, thread_name_prefix="joobq-enqueue")
1004
+ return self._async.submit(self.enqueue, job_cls, payload, **kwargs)
1005
+
1006
+ def start(
1007
+ self,
1008
+ workflow_def: WorkflowBlueprint,
1009
+ input_data: Any = None,
1010
+ *,
1011
+ context: Optional[Dict[str, Any]] = None,
1012
+ workflow_id: Optional[str] = None,
1013
+ search_attributes: Optional[Dict[str, Any]] = None,
1014
+ build: Optional[str] = None,
1015
+ ) -> Dict[str, Any]:
1016
+ return self.start_workflow(
1017
+ name=workflow_def.name,
1018
+ input_data=input_data,
1019
+ context=context,
1020
+ workflow_id=workflow_id,
1021
+ search_attributes=search_attributes,
1022
+ build=build,
1023
+ )
1024
+
1025
+ def start_workflow(
1026
+ self,
1027
+ name: str,
1028
+ input_data: Any = None,
1029
+ *,
1030
+ context: Optional[Dict[str, Any]] = None,
1031
+ workflow_id: Optional[str] = None,
1032
+ search_attributes: Optional[Dict[str, Any]] = None,
1033
+ build: Optional[str] = None,
1034
+ ) -> Dict[str, Any]:
1035
+ return self._post("/api/v1/sdk/workflows", {
1036
+ "name": name,
1037
+ "input": input_data,
1038
+ "context": self._resolve_context(context),
1039
+ "id": workflow_id,
1040
+ "search_attributes": search_attributes,
1041
+ "build": build,
1042
+ })
1043
+
1044
+ def signal_workflow(self, workflow_id: str, name: str, payload: Any = None) -> Dict[str, Any]:
1045
+ return self._post(f"/api/v1/sdk/workflows/{workflow_id}/signal", {
1046
+ "name": name,
1047
+ "payload": payload,
1048
+ })
1049
+
1050
+ def update_workflow(self, workflow_id: str, name: str, input: Any = None, *, update_id: Optional[str] = None,
1051
+ wait_ms: Optional[int] = None) -> Dict[str, Any]:
1052
+ """Sends an update to a running workflow's update handler (Pro) and waits up to `wait_ms` (100 to 30000, default
1053
+ 10000) for its outcome: {update_id, name, status, result, error, reason, message}. `status` is completed,
1054
+ rejected (validate failed: `reason`, `message`), failed, or accepted when the handler had not answered in time
1055
+ (send again with the same `update_id` to wait for the same update)."""
1056
+ body: Dict[str, Any] = {"name": name, "input": input}
1057
+ if update_id is not None:
1058
+ body["update_id"] = update_id
1059
+ if wait_ms is not None:
1060
+ body["wait_ms"] = int(wait_ms)
1061
+ return self._post(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/updates", body)
1062
+
1063
+ def describe_workflow(self, workflow_id: str) -> Dict[str, Any]:
1064
+ """One run (Pro): {id, name, status, closed, run, started_at, closed_at, continued_to, output, failure,
1065
+ attributes, queries}. Payload fields with secret-looking names are masked, as in the console."""
1066
+ return self._get(f"/api/v1/sdk/workflows/{_seg(workflow_id)}")
1067
+
1068
+ def workflow_result(self, workflow_id: str, wait_ms: int = 0) -> Dict[str, Any]:
1069
+ """The run at the end of the workflow's continue-as-new chain (Pro), waiting up to `wait_ms` (at most 25000)
1070
+ for it to close; same shape as `describe_workflow`. `closed` false means it was still open when the wait ended."""
1071
+ return self._get(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/result?wait_ms={int(wait_ms)}")
1072
+
1073
+ def workflow_history(self, workflow_id: str, after_seq: int = 0, limit: int = 200) -> List[Dict[str, Any]]:
1074
+ """The run's events after `after_seq` (Pro), at most `limit`, oldest first."""
1075
+ res = self._get(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/history?after_seq={int(after_seq)}&limit={int(limit)}")
1076
+ return res.get("items", [])
1077
+
1078
+ def query_workflow(self, workflow_id: str, name: str) -> Any:
1079
+ """Asks the run a read-only question (Pro); returns the answer."""
1080
+ return self._post(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/query", {"name": name}).get("result")
1081
+
1082
+ def upsert_workflow_attributes(self, workflow_id: str, set: Optional[Dict[str, Any]] = None,
1083
+ remove: Optional[List[str]] = None) -> Dict[str, Any]:
1084
+ """Sets and/or removes search attributes of the workflow's open run (Pro); returns {id, attributes}."""
1085
+ return self._post(f"/api/v1/sdk/workflows/{_seg(workflow_id)}/search-attributes", {"set": set, "remove": remove})
1086
+
1087
+ def register_schedule(
1088
+ self,
1089
+ name: str,
1090
+ job_type: str,
1091
+ queue: str,
1092
+ *,
1093
+ cron: Optional[str] = None,
1094
+ every: Optional[str] = None,
1095
+ timezone: str = "UTC",
1096
+ payload: Any = None,
1097
+ max_retries: int = 3,
1098
+ encrypt_fields: Optional[List[str]] = None,
1099
+ unique_key: Optional[str] = None,
1100
+ ) -> Dict[str, Any]:
1101
+ """Registers a cron/recurring job schedule, idempotent by `name`. Pass exactly one of `cron` or `every`
1102
+ (`every` is "<count> <unit>", e.g. "30 seconds", "5 minutes"). Returns {id, name, kind, created}."""
1103
+ return self._post("/api/v1/sdk/schedules", {
1104
+ "name": name,
1105
+ "kind": "job",
1106
+ "cron": cron,
1107
+ "every": every,
1108
+ "timezone": timezone,
1109
+ "queue": queue,
1110
+ "job_type": job_type,
1111
+ "payload": payload if payload is not None else {},
1112
+ "max_retries": max_retries,
1113
+ "encrypt_fields": encrypt_fields or [],
1114
+ "unique_key": unique_key,
1115
+ })
1116
+
1117
+ def rate_limit(self, key: str, limit: int, period_ms: int) -> Dict[str, Any]:
1118
+ """Reserves one slot of a sliding-window rate limit (Pro), returning {allowed, retry_after_ms}. `allowed` false
1119
+ gives the ms until a slot frees up. 402 pro_required without a Pro licence."""
1120
+ return self._post("/api/v1/sdk/rate-limit", {"key": key, "limit": limit, "period_ms": period_ms})
1121
+
1122
+ def register_blueprint(self, workflow_def: WorkflowBlueprint, worker_id: Optional[str] = None) -> Dict[str, Any]:
1123
+ """Sends the workflow's blueprint; the gateway keeps each new content hash (`created`) and ignores a known one."""
1124
+ return self._post("/api/v1/sdk/blueprints", {
1125
+ "name": workflow_def.name,
1126
+ "sdk_language": "python",
1127
+ "worker_id": worker_id,
1128
+ "blueprint": workflow_def.to_blueprint(),
1129
+ })
1130
+
1131
+ def register_worker(
1132
+ self,
1133
+ worker_id: str,
1134
+ task_queues: List[str],
1135
+ *,
1136
+ concurrency: int = 4,
1137
+ service_name: Optional[str] = None,
1138
+ team: Optional[str] = None,
1139
+ ) -> Dict[str, Any]:
1140
+ return self._post("/api/v1/sdk/workers/register", {
1141
+ "worker_id": worker_id,
1142
+ "sdk_language": "python",
1143
+ "sdk_version": SDK_VERSION,
1144
+ "service_name": service_name,
1145
+ "team": team or os.environ.get("JOOBQ_TEAM"),
1146
+ "host": socket.gethostname(),
1147
+ "pid": os.getpid(),
1148
+ "task_queues": task_queues,
1149
+ "concurrency": concurrency,
1150
+ "delivery_mode": "poll",
1151
+ })
1152
+
1153
+ def deregister_worker(self, worker_id: str) -> Dict[str, Any]:
1154
+ return self._post(f"/api/v1/sdk/workers/{worker_id}/deregister", {})
1155
+
1156
+ def poll_tasks(
1157
+ self,
1158
+ worker_id: str,
1159
+ task_queues: List[str],
1160
+ *,
1161
+ max_tasks: int = 1,
1162
+ timeout_ms: int = 0,
1163
+ lease_ttl_ms: int = 30000,
1164
+ ) -> List[LeasedTask]:
1165
+ res = self._post("/api/v1/sdk/tasks/poll", {
1166
+ "worker_id": worker_id,
1167
+ "task_queues": task_queues,
1168
+ "max_tasks": max_tasks,
1169
+ "timeout_ms": timeout_ms,
1170
+ "lease_ttl_ms": lease_ttl_ms,
1171
+ })
1172
+ return [LeasedTask.from_dict(t) for t in res.get("tasks", [])]
1173
+
1174
+ def heartbeat_task(
1175
+ self,
1176
+ lease_token: str,
1177
+ worker_id: str,
1178
+ lease_ttl_ms: int = 30000,
1179
+ ) -> Dict[str, Any]:
1180
+ return self._post(f"/api/v1/sdk/tasks/{lease_token}/heartbeat", {
1181
+ "worker_id": worker_id,
1182
+ "lease_ttl_ms": lease_ttl_ms,
1183
+ })
1184
+
1185
+ def complete_task(
1186
+ self,
1187
+ lease_token: str,
1188
+ worker_id: str,
1189
+ output: Any = None,
1190
+ ) -> Dict[str, Any]:
1191
+ return self._post(f"/api/v1/sdk/tasks/{lease_token}/complete", {
1192
+ "worker_id": worker_id,
1193
+ "output": output,
1194
+ })
1195
+
1196
+ def fail_task(
1197
+ self,
1198
+ lease_token: str,
1199
+ worker_id: str,
1200
+ exc: BaseException,
1201
+ *,
1202
+ retryable: bool = True,
1203
+ ) -> Dict[str, Any]:
1204
+ return self._post(f"/api/v1/sdk/tasks/{lease_token}/fail",
1205
+ {"worker_id": worker_id, **failure(exc, retryable)})
1206
+
1207
+ def ack_tasks(self, worker_id: str, items: List[Dict[str, Any]]) -> Dict[str, Any]:
1208
+ """Settles up to 100 leases in one call (`POST /sdk/tasks/ack`): items are `{token, outcome: "complete", output}`
1209
+ or `{token, outcome: "fail", **failure(exc)}`; each gets its own result."""
1210
+ return self._post("/api/v1/sdk/tasks/ack", {"worker_id": worker_id, "items": items})
1211
+
1212
+ def heartbeat_tasks(self, worker_id: str, leases: List[Dict[str, Any]], lease_ttl_ms: int = 30000) -> Dict[str, Any]:
1213
+ """Renews many leases in one call (`POST /sdk/tasks/heartbeat`): `leases` are `{token}`."""
1214
+ return self._post("/api/v1/sdk/tasks/heartbeat", {"worker_id": worker_id, "lease_ttl_ms": lease_ttl_ms,
1215
+ "leases": leases})
1216
+
1217
+ def _connection(self) -> http.client.HTTPConnection:
1218
+ """This thread's kept-alive connection to the gateway (through the environment's HTTP proxy, if any)."""
1219
+ conn = getattr(self._local, "conn", None)
1220
+ if conn is not None:
1221
+ return conn
1222
+ scheme, host = self._url.scheme or "http", self._url.hostname or "localhost"
1223
+ port = self._url.port or (443 if scheme == "https" else 80)
1224
+ proxy = urllib.request.getproxies().get(scheme)
1225
+ if proxy and urllib.request.proxy_bypass(host):
1226
+ proxy = None
1227
+ self._local.absolute = False
1228
+ if proxy:
1229
+ p = urllib.parse.urlsplit(proxy if "://" in proxy else "http://" + proxy)
1230
+ if scheme == "https":
1231
+ conn = http.client.HTTPSConnection(p.hostname, p.port or 80, timeout=self.request_timeout)
1232
+ conn.set_tunnel(host, port)
1233
+ else:
1234
+ conn = http.client.HTTPConnection(p.hostname, p.port or 80, timeout=self.request_timeout)
1235
+ self._local.absolute = True
1236
+ else:
1237
+ cls = http.client.HTTPSConnection if scheme == "https" else http.client.HTTPConnection
1238
+ conn = cls(host, port, timeout=self.request_timeout)
1239
+ self._local.conn = conn
1240
+ self._local.used = False
1241
+ return conn
1242
+
1243
+ def _drop_connection(self) -> None:
1244
+ conn = getattr(self._local, "conn", None)
1245
+ if conn is not None:
1246
+ conn.close()
1247
+ self._local.conn = None
1248
+
1249
+ def _post(self, path: str, body: Dict[str, Any]) -> Dict[str, Any]:
1250
+ return self._send("POST", path, body)
1251
+
1252
+ def _get(self, path: str) -> Dict[str, Any]:
1253
+ return self._send("GET", path, None)
1254
+
1255
+ def _send(self, method: str, path: str, body: Optional[Dict[str, Any]]) -> Dict[str, Any]:
1256
+ data = json.dumps(body).encode("utf-8") if body is not None else None
1257
+ headers = {"Authorization": f"Bearer {self.token}"}
1258
+ if data is not None:
1259
+ headers["Content-Type"] = "application/json"
1260
+ target = (self._url.path or "") + path
1261
+ while True:
1262
+ conn = self._connection()
1263
+ reused = self._local.used
1264
+ try:
1265
+ conn.request(method, f"{self.gateway_url}{path}" if self._local.absolute else target, data, headers)
1266
+ resp = conn.getresponse()
1267
+ raw = resp.read()
1268
+ except (http.client.RemoteDisconnected, BrokenPipeError, ConnectionResetError):
1269
+ self._drop_connection()
1270
+ if reused:
1271
+ continue # the gateway closed the kept-alive connection while it was idle: once more, on a new one
1272
+ raise
1273
+ except Exception:
1274
+ self._drop_connection()
1275
+ raise
1276
+ self._local.used = True
1277
+ if resp.will_close:
1278
+ self._drop_connection()
1279
+ if resp.status >= 400:
1280
+ raise GatewayError(resp.status, raw.decode("utf-8", "replace"))
1281
+ return json.loads(raw.decode("utf-8")) if raw else {}
1282
+
1283
+
1284
+ class _Transport:
1285
+ """Every gateway call a running worker makes, on threads of its own: handler threads never wait for HTTP.
1286
+
1287
+ - the poller long-polls for leases into a small buffer (free slots plus `prefetch`) that handlers take from;
1288
+ - the acker sends the outcomes handlers queue (`POST /sdk/tasks/ack`, up to 100 per call, after at most
1289
+ `ack_linger_ms`), retrying while the gateway is unavailable: a task stays leased (and renewed) until its outcome
1290
+ is acknowledged. The outcome queue holds `ack_buffer` items: when it is full, handlers wait;
1291
+ - the heartbeater renews every lease the worker holds in one call (`POST /sdk/tasks/heartbeat`) every third of the
1292
+ lease, and registers the worker every 10 s."""
1293
+
1294
+ def __init__(self, worker: "JoobQWorker") -> None:
1295
+ self.worker = worker
1296
+ self.client = worker.client
1297
+ self.lock = threading.Condition()
1298
+ self.ready: "_queue.Queue[Optional[LeasedTask]]" = _queue.Queue()
1299
+ self.acks: "_queue.Queue[Dict[str, Any]]" = _queue.Queue(maxsize=worker.ack_buffer)
1300
+ self.leased: Dict[str, LeasedTask] = {} # every lease held: buffered, running, or waiting for its ack
1301
+ self.held = 0 # buffered + running
1302
+ self.cancelled: Dict[str, bool] = {}
1303
+ self.acker_done = threading.Event() # set: the acker exits once the outcome queue is empty
1304
+ self.give_up = threading.Event() # set: the drain timeout passed, unsent outcomes are left to their leases
1305
+ self.stopped = threading.Event()
1306
+ self.running = False
1307
+
1308
+ # -- handler side ---------------------------------------------------------------------------------------------
1309
+ def settle(self, item: Dict[str, Any]) -> None:
1310
+ """Queues an outcome for the acker; waits while the queue is full (back-pressure)."""
1311
+ while True:
1312
+ try:
1313
+ self.acks.put(item, timeout=0.5)
1314
+ return
1315
+ except _queue.Full:
1316
+ if self.give_up.is_set():
1317
+ return # left to its lease
1318
+
1319
+ def cancel_requested(self, token: str) -> bool:
1320
+ with self.lock:
1321
+ return self.cancelled.get(token, False)
1322
+
1323
+ def release(self, token: str) -> None:
1324
+ with self.lock:
1325
+ self.leased.pop(token, None)
1326
+ self.cancelled.pop(token, None)
1327
+
1328
+ # -- threads --------------------------------------------------------------------------------------------------
1329
+ def handle(self) -> None:
1330
+ while True:
1331
+ task = self.ready.get()
1332
+ if task is None:
1333
+ return
1334
+ try:
1335
+ self.worker.process_task(task)
1336
+ except Exception as exc: # process_task reports handler errors itself: this is a bug in the worker
1337
+ print(f"joobq: task {task.id} could not be settled: {exc!r}", file=sys.stderr)
1338
+ finally:
1339
+ with self.lock:
1340
+ self.held -= 1
1341
+ self.lock.notify_all()
1342
+
1343
+ def poll(self, poll_interval_sec: float) -> None:
1344
+ w = self.worker
1345
+ capacity = w.concurrency + w.prefetch
1346
+ backoff = poll_interval_sec
1347
+ while not w._stopping.is_set():
1348
+ with self.lock:
1349
+ while self.held >= capacity and not w._stopping.is_set():
1350
+ self.lock.wait(0.5)
1351
+ room = capacity - self.held
1352
+ if w._stopping.is_set():
1353
+ return
1354
+ try:
1355
+ tasks = self.client.poll_tasks(w.worker_id, w.task_queues, max_tasks=min(room, 100),
1356
+ timeout_ms=w.poll_timeout_ms, lease_ttl_ms=w.lease_ttl_ms)
1357
+ backoff = poll_interval_sec
1358
+ w._refused = None
1359
+ except Exception as exc:
1360
+ w._report_refusal("poll for tasks", exc)
1361
+ w._stopping.wait(backoff + random.random() * 0.1)
1362
+ backoff = min(backoff * 2, 5.0)
1363
+ continue
1364
+ with self.lock:
1365
+ for task in tasks:
1366
+ self.leased[task.lease_token] = task
1367
+ self.held += len(tasks)
1368
+ for task in tasks:
1369
+ self.ready.put(task)
1370
+
1371
+ def ack(self) -> None:
1372
+ linger = self.worker.ack_linger_ms / 1000.0
1373
+ while True:
1374
+ try:
1375
+ first = self.acks.get(timeout=0.1)
1376
+ except _queue.Empty:
1377
+ if self.acker_done.is_set():
1378
+ return
1379
+ continue
1380
+ batch = [first]
1381
+ deadline = time.monotonic() + linger
1382
+ while len(batch) < 100:
1383
+ left = deadline - time.monotonic()
1384
+ try:
1385
+ batch.append(self.acks.get(timeout=left) if left > 0 else self.acks.get_nowait())
1386
+ except _queue.Empty:
1387
+ break
1388
+ self.send_acks(batch)
1389
+
1390
+ def send_acks(self, batch: List[Dict[str, Any]]) -> None:
1391
+ delay = 0.25
1392
+ while True:
1393
+ try:
1394
+ self.client.ack_tasks(self.worker.worker_id, batch)
1395
+ break # a refused item (its lease is gone) is not sent again either
1396
+ except GatewayError as err:
1397
+ if err.status == 413 and len(batch) > 1: # too large for one call: in halves
1398
+ half = len(batch) // 2
1399
+ self.send_acks(batch[:half])
1400
+ self.send_acks(batch[half:])
1401
+ return
1402
+ if err.status < 500 and err.status != 429:
1403
+ print(f"joobq: the gateway refused {len(batch)} outcomes: {err}", file=sys.stderr)
1404
+ break
1405
+ except Exception:
1406
+ pass
1407
+ if self.give_up.wait(delay):
1408
+ break # shutting down: the leases bring these tasks back
1409
+ delay = min(delay * 2, 5.0)
1410
+ for item in batch:
1411
+ self.release(item["token"])
1412
+
1413
+ def heartbeat(self) -> None:
1414
+ w = self.worker
1415
+ every = max(0.5, w.lease_ttl_ms / 3000.0)
1416
+ last_reg = time.monotonic()
1417
+ while not self.stopped.wait(every):
1418
+ with self.lock:
1419
+ tokens = list(self.leased)
1420
+ for start in range(0, len(tokens), 500):
1421
+ chunk = tokens[start:start + 500]
1422
+ try:
1423
+ res = self.client.heartbeat_tasks(w.worker_id, [{"token": t} for t in chunk], w.lease_ttl_ms)
1424
+ except Exception:
1425
+ continue # renewed with the next heartbeat
1426
+ with self.lock:
1427
+ for lease in res.get("leases", []):
1428
+ token = lease.get("lease_token")
1429
+ if lease.get("error"):
1430
+ self.leased.pop(token, None) # gone: its outcome will be refused, nothing to renew
1431
+ elif token in self.leased:
1432
+ self.cancelled[token] = bool(lease.get("cancel_requested"))
1433
+ if time.monotonic() - last_reg > 10.0 and not w._stopping.is_set():
1434
+ try:
1435
+ w._register_blueprints()
1436
+ w._register()
1437
+ except Exception:
1438
+ pass
1439
+ last_reg = time.monotonic()
1440
+
1441
+ def run(self, poll_interval_sec: float) -> None:
1442
+ w = self.worker
1443
+ self.running = True
1444
+ threads = [threading.Thread(target=self.handle, name=f"joobq-task-{i}", daemon=True) for i in range(w.concurrency)]
1445
+ acker = threading.Thread(target=self.ack, name="joobq-acker", daemon=True)
1446
+ beat = threading.Thread(target=self.heartbeat, name="joobq-heartbeat", daemon=True)
1447
+ for t in threads + [acker, beat]:
1448
+ t.start()
1449
+ self.poll(poll_interval_sec) # until stop()
1450
+ deadline = time.monotonic() + w.drain_timeout_sec
1451
+ with self.lock:
1452
+ while self.held > 0 and time.monotonic() < deadline:
1453
+ self.lock.wait(0.05)
1454
+ for _ in threads:
1455
+ self.ready.put(None)
1456
+ self.acker_done.set()
1457
+ acker.join(max(0.0, deadline - time.monotonic()))
1458
+ if acker.is_alive():
1459
+ self.give_up.set()
1460
+ acker.join(5.0)
1461
+ self.give_up.set()
1462
+ self.stopped.set()
1463
+ beat.join(5.0)
1464
+ self.running = False
1465
+
1466
+
1467
+ class JoobQWorker:
1468
+ """In-app worker: runs up to `concurrency` jobs and workflow steps at once inside the booted application, and
1469
+ drains on SIGTERM/SIGINT. Its gateway calls (poll, outcomes, heartbeats) run
1470
+ on threads of their own (`_Transport`), so a handler never waits for HTTP."""
1471
+
1472
+ def __init__(
1473
+ self,
1474
+ client: JoobQClient,
1475
+ worker_id: str,
1476
+ task_queues: Optional[List[str]] = None,
1477
+ *,
1478
+ concurrency: int = 4,
1479
+ service_name: Optional[str] = None,
1480
+ team: Optional[str] = None,
1481
+ lease_ttl_ms: int = 30000,
1482
+ poll_timeout_ms: int = 5000,
1483
+ drain_timeout_sec: float = 30.0,
1484
+ prefetch: Optional[int] = None,
1485
+ ack_buffer: Optional[int] = None,
1486
+ ack_linger_ms: int = 5,
1487
+ ) -> None:
1488
+ self.client = client
1489
+ self.worker_id = worker_id
1490
+ self.task_queues = list(task_queues or [])
1491
+ self.concurrency = max(1, concurrency)
1492
+ self.service_name = service_name
1493
+ self.team = team
1494
+ self.lease_ttl_ms = lease_ttl_ms
1495
+ self.poll_timeout_ms = poll_timeout_ms
1496
+ # One hung call must not outlast the leases: a poll waits `poll_timeout_ms` at the gateway, and a sixth of a
1497
+ # lease on top keeps a renewal that hangs well inside the lease (10 s with the defaults).
1498
+ bound = poll_timeout_ms / 1000.0 + lease_ttl_ms / 6000.0
1499
+ if getattr(client, "request_timeout", None) is not None and client.request_timeout > bound:
1500
+ client.request_timeout = bound
1501
+ self.drain_timeout_sec = drain_timeout_sec
1502
+ # Leases held beyond the free slots, so a handler that finishes finds its next task at once.
1503
+ self.prefetch = max(0, prefetch if prefetch is not None else max(1, self.concurrency // 2))
1504
+ # Outcomes queued for the acker before handlers wait for it.
1505
+ self.ack_buffer = max(1, ack_buffer if ack_buffer is not None else 4 * self.concurrency)
1506
+ self.ack_linger_ms = max(0, ack_linger_ms)
1507
+ self._handlers: Dict[str, Callable[[Any, ExecutionContext], Any]] = {}
1508
+ self._workflows: Dict[str, WorkflowBlueprint] = {}
1509
+ self._middlewares: List[Callable[[ExecutionContext, Callable[[], Any]], Any]] = []
1510
+ self._stopping = threading.Event()
1511
+ self._blueprints_sent: set = set()
1512
+ self._transport = _Transport(self)
1513
+
1514
+ def use(self, middleware: Callable[[ExecutionContext, Callable[[], Any]], Any]) -> "JoobQWorker":
1515
+ self._middlewares.append(middleware)
1516
+ return self
1517
+
1518
+ def register_job(self, job_cls: Type[Job]) -> "JoobQWorker":
1519
+ if job_cls.queue and job_cls.queue not in self.task_queues:
1520
+ self.task_queues.append(job_cls.queue)
1521
+ self._handlers[job_cls.resolved_name()] = _normalize_callable(job_cls)
1522
+ return self
1523
+
1524
+ def register_workflow(self, workflow_def: WorkflowBlueprint) -> "JoobQWorker":
1525
+ if workflow_def.task_queue and workflow_def.task_queue not in self.task_queues:
1526
+ self.task_queues.append(workflow_def.task_queue)
1527
+ self._workflows[workflow_def.name] = workflow_def
1528
+ # The engine conducts the workflow's runs and sends each step as its own task.
1529
+ self._handlers.update(workflow_def.activities())
1530
+ return self
1531
+
1532
+ def handle_job(self, job_type: str, handler: Callable[[Any, ExecutionContext], Any]) -> "JoobQWorker":
1533
+ self._handlers[job_type] = handler
1534
+ return self
1535
+
1536
+ def process_task(self, task: LeasedTask) -> Dict[str, Any]:
1537
+ handler = self._handlers.get(task.name)
1538
+ if not handler:
1539
+ self._settle(task, {"outcome": "fail", **failure(NonRetryableError(f"No handler registered for {task.name!r}"), False)})
1540
+ return {"status": "failed"}
1541
+ # Everything that can raise for this task is inside the try: a task with no outcome keeps its lease renewed for
1542
+ # ever. BaseException, so that a handler calling sys.exit() (or raising anything else that is not an Exception)
1543
+ # fails its task instead of ending the handler thread without a word.
1544
+ try:
1545
+ transport = self._transport if self._transport.running else None
1546
+ ctx = ExecutionContext(task, self.worker_id, self.client, self.lease_ttl_ms, transport)
1547
+
1548
+ def invoke(idx: int) -> Any:
1549
+ if idx < len(self._middlewares):
1550
+ return self._middlewares[idx](ctx, lambda: invoke(idx + 1))
1551
+ return handler(task.payload, ctx)
1552
+
1553
+ output = invoke(0)
1554
+ except BaseException as exc:
1555
+ if isinstance(exc, KeyboardInterrupt):
1556
+ raise
1557
+ retryable = not isinstance(exc, NonRetryableError)
1558
+ self._settle(task, {"outcome": "fail", **failure(exc, retryable)})
1559
+ return {"status": "failed"}
1560
+ self._settle(task, {"outcome": "complete", "output": output})
1561
+ return {"status": "completed", "output": output}
1562
+
1563
+ def _report_refusal(self, what: str, exc: BaseException) -> None:
1564
+ """Says once, on stderr, that the gateway refuses this worker (a wrong or rotated token, no Pro key): it goes
1565
+ on retrying, and without this line it would sit idle with nothing in its output."""
1566
+ status = getattr(exc, "status", None)
1567
+ if not isinstance(status, int) or status < 400 or status >= 500 or status in (408, 429):
1568
+ return
1569
+ if getattr(self, "_refused", None) != status:
1570
+ self._refused = status
1571
+ print(f"joobq: the gateway refuses this worker ({what}): {exc}. Retrying; check the SDK token and the server's licence.", file=sys.stderr)
1572
+
1573
+ def _settle(self, task: LeasedTask, item: Dict[str, Any]) -> None:
1574
+ """Hands the outcome to the acker; outside a running worker (`process_task` called directly) it is sent here."""
1575
+ item = {"token": task.lease_token, **item}
1576
+ if self._transport.running:
1577
+ self._transport.settle(item)
1578
+ else:
1579
+ self._report(lambda: self.client.ack_tasks(self.worker_id, [item]))
1580
+
1581
+ def _report(self, call: Callable[[], Any]) -> None:
1582
+ """Sends an outcome, retrying while the gateway is unavailable; a lost outcome is left to the lease."""
1583
+ delay = 0.25
1584
+ for _ in range(6):
1585
+ try:
1586
+ call()
1587
+ return
1588
+ except GatewayError as err:
1589
+ if err.status < 500:
1590
+ return # the outcome was refused: nothing to retry
1591
+ except Exception:
1592
+ pass
1593
+ time.sleep(delay)
1594
+ delay = min(delay * 2, 5.0)
1595
+
1596
+ def stop(self) -> None:
1597
+ """Stops polling; `start` returns once the running tasks have finished and their outcomes are acknowledged
1598
+ (or `drain_timeout_sec` passed)."""
1599
+ self._stopping.set()
1600
+ with self._transport.lock:
1601
+ self._transport.lock.notify_all()
1602
+
1603
+ def _register(self) -> None:
1604
+ self.client.register_worker(self.worker_id, self.task_queues, concurrency=self.concurrency,
1605
+ service_name=self.service_name, team=self.team)
1606
+
1607
+ def _register_blueprints(self) -> None:
1608
+ """Registers every workflow's blueprint once; one the gateway refuses (4xx) is logged and not sent again."""
1609
+ for name, wf in list(self._workflows.items()):
1610
+ if name in self._blueprints_sent:
1611
+ continue
1612
+ try:
1613
+ self.client.register_blueprint(wf, self.worker_id)
1614
+ self._blueprints_sent.add(name)
1615
+ except GatewayError as err:
1616
+ if err.status >= 500:
1617
+ raise
1618
+ self._blueprints_sent.add(name)
1619
+ print(f"joobq: blueprint of {name!r} refused: {err}", file=sys.stderr)
1620
+
1621
+ def start(self, poll_interval_sec: float = 0.25, install_signal_handlers: bool = True) -> None:
1622
+ """Runs the worker until SIGTERM/SIGINT or `stop()`, then drains the running tasks, sends their outcomes and
1623
+ deregisters."""
1624
+ if install_signal_handlers and threading.current_thread() is threading.main_thread():
1625
+ for sig in (_signal.SIGTERM, _signal.SIGINT):
1626
+ _signal.signal(sig, lambda *_: self.stop())
1627
+ backoff = poll_interval_sec
1628
+ while not self._stopping.is_set():
1629
+ try:
1630
+ self._register_blueprints() # first: a worker that shows up has its blueprints in
1631
+ self._register()
1632
+ self._refused = None
1633
+ break
1634
+ except Exception as exc:
1635
+ self._report_refusal("register", exc)
1636
+ self._stopping.wait(backoff + random.random() * 0.1)
1637
+ backoff = min(backoff * 2, 5.0)
1638
+ self._transport.run(poll_interval_sec)
1639
+ try:
1640
+ self.client.deregister_worker(self.worker_id)
1641
+ except Exception:
1642
+ pass
1643
+
1644
+ run = start
1645
+