continuum-task-server-sdk 1.4.0__tar.gz → 1.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: continuum-task-server-sdk
3
- Version: 1.4.0
3
+ Version: 1.6.0
4
4
  Summary: Python SDK for the Continuum Task Server
5
5
  Project-URL: Homepage, https://github.com/ContinuumWorkflow/continuum-task-server-sdk-python
6
6
  Project-URL: Issues, https://github.com/ContinuumWorkflow/continuum-task-server-sdk-python/issues
@@ -77,4 +77,4 @@ __all__ = [
77
77
  "WorkSubscriptionResult",
78
78
  ]
79
79
 
80
- __version__ = "1.4.0"
80
+ __version__ = "1.6.0"
@@ -46,6 +46,21 @@ class HttpClient:
46
46
  def close(self) -> None:
47
47
  self._client.close()
48
48
 
49
+ def set_task_server_token(self, token: str | None) -> None:
50
+ if token:
51
+ self._client.headers["Task-Server-Token"] = token
52
+ else:
53
+ self._client.headers.pop("Task-Server-Token", None)
54
+
55
+ def delete(self, path: str) -> None:
56
+ try:
57
+ response = self._client.delete(path)
58
+ except httpx.HTTPError as e:
59
+ raise ContinuumError(f"Request failed: {e}") from e
60
+ if response.status_code == 204:
61
+ return
62
+ self._raise_for_status(response)
63
+
49
64
  def __enter__(self) -> HttpClient:
50
65
  return self
51
66
 
@@ -240,6 +240,8 @@ class QueueApi:
240
240
  priority: int | None = None,
241
241
  parent_id: UUID | str | None = None,
242
242
  idempotency_token: str | None = None,
243
+ layer: str | None = None,
244
+ strict_layer: bool | None = None,
243
245
  ) -> QueueItem:
244
246
  """Add a queue item. At least one of task_name or task_item_name is required."""
245
247
  if task_name is None and task_item_name is None:
@@ -252,6 +254,8 @@ class QueueApi:
252
254
  "priority": priority,
253
255
  "inputData": _encode_input(input_data),
254
256
  "idempotencyToken": idempotency_token,
257
+ "layer": layer,
258
+ "strictLayer": strict_layer,
255
259
  }
256
260
  )
257
261
  return QueueItem.model_validate(self._http.post_json(self._MGMT, body))
@@ -266,9 +270,11 @@ class QueueApi:
266
270
  data, mime = result
267
271
  return Content(data=data, mime_type=mime)
268
272
 
269
- def claim(self, task_name: str) -> QueueItem | None:
273
+ def claim(self, task_name: str, *, layer: str | None = None) -> QueueItem | None:
270
274
  """Claim one OPEN queue item for the given task name. Returns None if none available."""
271
- result = self._http.post_optional(f"{self._QUEUE}/claim", {"taskName": task_name})
275
+ result = self._http.post_optional(
276
+ f"{self._QUEUE}/claim", drop_none({"taskName": task_name, "layer": layer})
277
+ )
272
278
  if result is None:
273
279
  return None
274
280
  return QueueItem.model_validate(result)
@@ -408,6 +414,16 @@ class ContinuumClient:
408
414
  def base_url(self) -> str:
409
415
  return self._http.base_url
410
416
 
417
+ def set_task_server_token(self, token: str | None) -> None:
418
+ """Attach the secret from task-server registration. Blank clears it."""
419
+ self._http.set_task_server_token(token)
420
+
421
+ def post_raw(self, path: str, body: Any) -> Any:
422
+ return self._http.post_json(path, body)
423
+
424
+ def delete(self, path: str) -> None:
425
+ self._http.delete(path)
426
+
411
427
  def close(self) -> None:
412
428
  self._http.close()
413
429
 
@@ -0,0 +1,178 @@
1
+ """Registers this process with the queue API and refreshes the lease.
2
+
3
+ A missing or failing registration never stops the task server from claiming work.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import logging
9
+ import socket
10
+ import threading
11
+ import uuid
12
+ from collections.abc import Callable, Iterable
13
+ from importlib.metadata import PackageNotFoundError, version
14
+
15
+ from .client import ContinuumClient
16
+ from .exceptions import ContinuumError, NotFoundError
17
+ from .websocket import ContinuumWebSocketClient
18
+
19
+ logger = logging.getLogger("continuum_task_server")
20
+
21
+ _PATH = "/api/queue/task-servers"
22
+
23
+
24
+ def sdk_version() -> str:
25
+ try:
26
+ return version("continuum-task-server-sdk")
27
+ except PackageNotFoundError:
28
+ return "unknown"
29
+
30
+
31
+ class TaskServerPresence:
32
+ def __init__(
33
+ self,
34
+ client: ContinuumClient,
35
+ *,
36
+ name: str | None,
37
+ instance_id: str | None,
38
+ labels: dict[str, str] | None,
39
+ task_names: Callable[[], Iterable[str]],
40
+ layer: str | None = None,
41
+ ) -> None:
42
+ self._client = client
43
+ self._name = name or None
44
+ self._instance_id = instance_id or str(uuid.uuid4())
45
+ self._labels = _labels(labels)
46
+ self._task_names = task_names
47
+ self._layer = layer.strip() if isinstance(layer, str) and layer.strip() else None
48
+ self._enabled = True
49
+ self._stopped = threading.Event()
50
+ self._token: str | None = None
51
+ self._heartbeat_seconds = 10.0
52
+ self._websocket: ContinuumWebSocketClient | None = None
53
+ self._thread: threading.Thread | None = None
54
+
55
+ def attach(self, websocket: ContinuumWebSocketClient | None) -> None:
56
+ self._websocket = websocket
57
+ if websocket is not None and self._token:
58
+ websocket.set_task_server_token(self._token)
59
+
60
+ def start(self) -> None:
61
+ if self._thread is not None:
62
+ return
63
+ self._thread = threading.Thread(target=self._loop, name="continuum-presence", daemon=True)
64
+ self._thread.start()
65
+
66
+ def close(self) -> None:
67
+ self._stopped.set()
68
+ if self._thread is not None:
69
+ self._thread.join(timeout=1)
70
+ current = self._token
71
+ self._token = None
72
+ if not current:
73
+ self._client.set_task_server_token(None)
74
+ return
75
+ self._client.set_task_server_token(current)
76
+ try:
77
+ self._client.delete(_PATH)
78
+ except ContinuumError as exc:
79
+ logger.debug("task server deregister failed: %s", exc)
80
+ finally:
81
+ self._client.set_task_server_token(None)
82
+ if self._websocket is not None:
83
+ self._websocket.set_task_server_token(None)
84
+
85
+ def reregister(self) -> bool:
86
+ if not self._enabled or self._stopped.is_set():
87
+ return False
88
+ self._token = None
89
+ self._client.set_task_server_token(None)
90
+ return self._register()
91
+
92
+ def _loop(self) -> None:
93
+ backoff = 1.0
94
+ while not self._stopped.is_set() and self._enabled:
95
+ if self._token is None:
96
+ if not self._register():
97
+ if not self._enabled or self._stopped.is_set():
98
+ return
99
+ if self._stopped.wait(backoff):
100
+ return
101
+ backoff = min(backoff * 2, 30.0)
102
+ continue
103
+ backoff = 1.0
104
+ if self._stopped.wait(self._heartbeat_seconds) or self._token is None:
105
+ continue
106
+ if not self._heartbeat():
107
+ self._token = None
108
+ self._client.set_task_server_token(None)
109
+
110
+ def _register(self) -> bool:
111
+ body = {
112
+ "instanceId": self._instance_id,
113
+ "name": self._name,
114
+ "taskNames": list(self._task_names()),
115
+ "labels": self._labels,
116
+ "layer": self._layer,
117
+ }
118
+ try:
119
+ payload = self._client.post_raw(_PATH, body) or {}
120
+ if payload.get("status") != 0:
121
+ logger.warning("task server register rejected: %s", payload.get("message"))
122
+ return False
123
+ issued = payload.get("token")
124
+ if not isinstance(issued, str) or not issued:
125
+ logger.warning("task server register returned no token")
126
+ return False
127
+ seconds = payload.get("heartbeatSeconds")
128
+ self._heartbeat_seconds = (
129
+ float(seconds) if isinstance(seconds, (int, float)) and seconds > 0 else 10.0
130
+ )
131
+ self._token = issued
132
+ self._client.set_task_server_token(issued)
133
+ if self._websocket is not None:
134
+ self._websocket.set_task_server_token(issued)
135
+ server = (
136
+ payload.get("taskServer") if isinstance(payload.get("taskServer"), dict) else {}
137
+ )
138
+ logger.info("Registered task server %s", server.get("id"))
139
+ return True
140
+ except NotFoundError:
141
+ self._enabled = False
142
+ self._token = None
143
+ self._client.set_task_server_token(None)
144
+ logger.info("Queue API does not support task server presence; claiming without it")
145
+ return False
146
+ except Exception as exc:
147
+ logger.warning("task server register failed: %s", exc)
148
+ return False
149
+
150
+ def _heartbeat(self) -> bool:
151
+ try:
152
+ self._client.post_raw(_PATH + "/heartbeat", {})
153
+ return True
154
+ except NotFoundError:
155
+ self._enabled = False
156
+ logger.info("Queue API does not support task server presence; claiming without it")
157
+ return False
158
+ except Exception as exc:
159
+ logger.warning("task server heartbeat failed: %s", exc)
160
+ return False
161
+
162
+
163
+ def _labels(user: dict[str, str] | None) -> dict[str, str]:
164
+ merged: dict[str, str] = {}
165
+ if user:
166
+ merged.update(user)
167
+ merged["continuum.sdk"] = "python"
168
+ merged["continuum.sdkVersion"] = sdk_version()
169
+ merged["continuum.hostname"] = _hostname()
170
+ return merged
171
+
172
+
173
+ def _hostname() -> str:
174
+ try:
175
+ host = socket.gethostname()
176
+ except OSError:
177
+ return "unknown"
178
+ return host or "unknown"
@@ -34,8 +34,9 @@ import httpx
34
34
 
35
35
  from .client import ContinuumClient
36
36
  from .context import TaskExecutionContext
37
- from .exceptions import ContinuumError
37
+ from .exceptions import ConflictError, ContinuumError
38
38
  from .models import QueueEvent, QueueEventType, QueueItem, TaskStatus, TransportMode
39
+ from .presence import TaskServerPresence
39
40
  from .websocket import ContinuumWebSocketClient, WebSocketOptions
40
41
 
41
42
  logger = logging.getLogger("continuum_task_server")
@@ -92,6 +93,11 @@ class TaskServer:
92
93
  transport: TransportMode | str | httpx.BaseTransport | None = None,
93
94
  websocket_options: WebSocketOptions | None = None,
94
95
  websocket_client: Any = None,
96
+ name: str | None = None,
97
+ labels: dict[str, str] | None = None,
98
+ instance_id: str | None = None,
99
+ presence: bool = True,
100
+ layer: str | None = None,
95
101
  ) -> None:
96
102
  http_transport: httpx.BaseTransport | None = None
97
103
  if isinstance(transport, httpx.BaseTransport):
@@ -129,6 +135,21 @@ class TaskServer:
129
135
  self._base_url = base_url
130
136
  self._api_key = api_key
131
137
  self._active: dict[uuid.UUID, _ActiveClaim] = {}
138
+ self._layer = layer.strip() if isinstance(layer, str) and layer.strip() else None
139
+ self._presence = (
140
+ TaskServerPresence(
141
+ self._client,
142
+ name=name,
143
+ instance_id=instance_id,
144
+ labels=labels,
145
+ task_names=self._handlers.keys,
146
+ layer=self._layer,
147
+ )
148
+ if presence
149
+ else None
150
+ )
151
+ if self._presence is not None and self._ws is not None:
152
+ self._presence.attach(self._ws)
132
153
 
133
154
  @property
134
155
  def client(self) -> ContinuumClient:
@@ -269,6 +290,8 @@ class TaskServer:
269
290
  )
270
291
 
271
292
  try:
293
+ if self._presence is not None:
294
+ self._presence.start()
272
295
  if self._mode is TransportMode.WEBSOCKET:
273
296
  self._websocket_loop()
274
297
  else:
@@ -310,7 +333,7 @@ class TaskServer:
310
333
  continue
311
334
 
312
335
  try:
313
- item = self._client.queue.claim(registration.name)
336
+ item = self._claim_http(registration.name)
314
337
  except ContinuumError as e:
315
338
  registration.semaphore.release()
316
339
  logger.error("claim(%s) failed: %s", registration.name, e)
@@ -336,13 +359,16 @@ class TaskServer:
336
359
  options=self._ws_options,
337
360
  )
338
361
  self._owns_ws = True
362
+ if self._presence is not None:
363
+ self._presence.attach(self._ws)
339
364
  return self._ws
340
365
 
341
366
  def _websocket_loop(self) -> None:
342
367
  ws = self._ensure_websocket()
343
368
  ws.connect()
344
369
  for name in self._handlers:
345
- ws.subscribe_work(name)
370
+ ws.set_layer(self._layer)
371
+ ws.subscribe_work(name, layer=self._layer)
346
372
  logger.info("WebSocket subscribed to work: %s", list(self._handlers))
347
373
  while not self._stop_event.is_set():
348
374
  event = ws.next_event(timeout=0.25)
@@ -406,7 +432,7 @@ class TaskServer:
406
432
  if registration is None:
407
433
  return
408
434
  try:
409
- item = ws.claim(registration.name)
435
+ item = self._claim_socket(ws, registration.name)
410
436
  except ContinuumError as e:
411
437
  registration.semaphore.release()
412
438
  logger.error("claim(%s) failed: %s", registration.name, e)
@@ -630,6 +656,22 @@ class TaskServer:
630
656
  logger.error("update_status(%s, %s) failed: %s", queue_item_id, status.value, e)
631
657
  return None
632
658
 
659
+ def _claim_http(self, task_name: str) -> QueueItem | None:
660
+ try:
661
+ return self._client.queue.claim(task_name, layer=self._layer)
662
+ except ConflictError:
663
+ if self._presence is None or not self._presence.reregister():
664
+ raise
665
+ return self._client.queue.claim(task_name, layer=self._layer)
666
+
667
+ def _claim_socket(self, ws: ContinuumWebSocketClient, task_name: str) -> QueueItem | None:
668
+ try:
669
+ return ws.claim(task_name, layer=self._layer)
670
+ except ConflictError:
671
+ if self._presence is None or not self._presence.reregister():
672
+ raise
673
+ return ws.claim(task_name, layer=self._layer)
674
+
633
675
  def _drain(self) -> None:
634
676
  if self._executor is None:
635
677
  return
@@ -645,6 +687,8 @@ class TaskServer:
645
687
  logger.debug("drain: handler future raised", exc_info=True)
646
688
  self._executor.shutdown(wait=False, cancel_futures=True)
647
689
  self._executor = None
690
+ if self._presence is not None:
691
+ self._presence.close()
648
692
  if self._ws is not None:
649
693
  self._ws.close()
650
694
  if self._owns_ws:
@@ -127,6 +127,7 @@ class ContinuumWebSocketClient:
127
127
  self._events_cv = threading.Condition()
128
128
  self._event_callbacks: list[EventCallback] = []
129
129
  self._work_subscriptions: set[str] = set()
130
+ self._layer: str | None = None
130
131
  self._control_subscriptions: set[tuple[str, str | None]] = set()
131
132
  self._wait_subscribers: set[str] = set()
132
133
  self._seen_completed: OrderedDict[str, None] = OrderedDict()
@@ -137,6 +138,7 @@ class ContinuumWebSocketClient:
137
138
  self._heartbeat_stop = threading.Event()
138
139
  self._decoder = _stomp.StompDecoder()
139
140
  self._reconnect_attempt = 0
141
+ self._task_server_token: str | None = None
140
142
  self._client_heartbeat_ms = self._options.heartbeat_ms
141
143
  self._generation = 0
142
144
 
@@ -227,6 +229,8 @@ class ContinuumWebSocketClient:
227
229
  parent_id: UUID | str | None = None,
228
230
  request_id: str | None = None,
229
231
  idempotency_token: str | None = None,
232
+ layer: str | None = None,
233
+ strict_layer: bool | None = None,
230
234
  ) -> QueueItem:
231
235
  if task_name is None and task_item_name is None:
232
236
  raise ValueError("at least one of task_name or task_item_name is required")
@@ -238,6 +242,8 @@ class ContinuumWebSocketClient:
238
242
  "priority": priority,
239
243
  "inputData": _encode_input(input_data),
240
244
  "idempotencyToken": idempotency_token,
245
+ "layer": layer,
246
+ "strictLayer": strict_layer,
241
247
  }
242
248
  )
243
249
  payload = self._command("/app/enqueue", body, request_id=request_id, retry=False)
@@ -259,6 +265,8 @@ class ContinuumWebSocketClient:
259
265
  correlation: str | None = None,
260
266
  request_id: str | None = None,
261
267
  idempotency_token: str | None = None,
268
+ layer: str | None = None,
269
+ strict_layer: bool | None = None,
262
270
  ) -> EnqueueAndSubscribeWaitResult:
263
271
  if task_name is None and task_item_name is None:
264
272
  raise ValueError("at least one of task_name or task_item_name is required")
@@ -276,6 +284,8 @@ class ContinuumWebSocketClient:
276
284
  "mode": mode_value,
277
285
  "correlation": correlation,
278
286
  "idempotencyToken": idempotency_token,
287
+ "layer": layer,
288
+ "strictLayer": strict_layer,
279
289
  }
280
290
  )
281
291
  payload = self._command(
@@ -288,10 +298,16 @@ class ContinuumWebSocketClient:
288
298
  self._wait_subscribers.add(subscriber_id)
289
299
  return result
290
300
 
291
- def claim(self, task_name: str, *, request_id: str | None = None) -> QueueItem | None:
301
+ def set_layer(self, layer: str | None) -> None:
302
+ self._layer = layer.strip() if isinstance(layer, str) and layer.strip() else None
303
+
304
+ def claim(
305
+ self, task_name: str, *, request_id: str | None = None, layer: str | None = None
306
+ ) -> QueueItem | None:
307
+ chosen = layer if layer else self._layer
292
308
  payload = self._command(
293
309
  "/app/claim",
294
- {"taskName": task_name},
310
+ drop_none({"taskName": task_name, "layer": chosen}),
295
311
  request_id=request_id,
296
312
  retry=False,
297
313
  )
@@ -342,11 +358,12 @@ class ContinuumWebSocketClient:
342
358
  return QueueItem.model_validate(payload["item"])
343
359
 
344
360
  def subscribe_work(
345
- self, task_name: str, *, request_id: str | None = None
361
+ self, task_name: str, *, request_id: str | None = None, layer: str | None = None
346
362
  ) -> WorkSubscriptionResult:
363
+ chosen = layer if layer else self._layer
347
364
  payload = self._command(
348
365
  "/app/subscribeWork",
349
- {"taskName": task_name},
366
+ drop_none({"taskName": task_name, "layer": chosen}),
350
367
  request_id=request_id,
351
368
  retry=True,
352
369
  )
@@ -603,24 +620,35 @@ class ContinuumWebSocketClient:
603
620
  return
604
621
  backoff = min(backoff * 2, self._options.max_reconnect_backoff)
605
622
 
623
+ def set_task_server_token(self, token: str | None) -> None:
624
+ """Secret from registration. The next CONNECT sends it, including after reconnect."""
625
+ self._task_server_token = token or None
626
+
606
627
  def _open_and_handshake(self) -> None:
607
628
  url = derive_websocket_url(self._base_url, self._options.endpoint)
629
+ headers = {"Api-Key": self._api_key}
630
+ token = self._task_server_token
631
+ if token:
632
+ headers["Task-Server-Token"] = token
608
633
  socket = self._connector(
609
634
  url,
610
- {"Api-Key": self._api_key},
635
+ headers,
611
636
  self._options.connect_timeout,
612
637
  )
613
638
  self._socket = socket
614
639
  self._decoder = _stomp.StompDecoder()
615
640
  self._generation += 1
616
641
  cx = max(0, int(self._options.heartbeat_ms))
642
+ connect_headers = {
643
+ "accept-version": "1.2,1.1,1.0",
644
+ "heart-beat": f"{cx},{cx}",
645
+ "host": urlparse(url).hostname or "localhost",
646
+ }
647
+ if token:
648
+ connect_headers["Task-Server-Token"] = token
617
649
  connect = _stomp.StompFrame(
618
650
  command="CONNECT",
619
- headers={
620
- "accept-version": "1.2,1.1,1.0",
621
- "heart-beat": f"{cx},{cx}",
622
- "host": urlparse(url).hostname or "localhost",
623
- },
651
+ headers=connect_headers,
624
652
  )
625
653
  self._send_raw(_stomp.encode_frame(connect))
626
654
  connected = self._recv_until_command("CONNECTED", timeout=self._options.connect_timeout)
@@ -683,7 +711,9 @@ class ContinuumWebSocketClient:
683
711
  controls = list(self._control_subscriptions)
684
712
  for task_name in work:
685
713
  try:
686
- self._command_inline("/app/subscribeWork", {"taskName": task_name})
714
+ self._command_inline(
715
+ "/app/subscribeWork", drop_none({"taskName": task_name, "layer": self._layer})
716
+ )
687
717
  except ContinuumError:
688
718
  logger.error("Failed to restore work subscription %s", task_name, exc_info=True)
689
719
  for subscriber_id in waits:
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "continuum-task-server-sdk"
7
- version = "1.4.0"
7
+ version = "1.6.0"
8
8
  description = "Python SDK for the Continuum Task Server"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"