requence 0.9.0__tar.gz → 0.9.1.dev154__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {requence-0.9.0/requence.egg-info → requence-0.9.1.dev154}/PKG-INFO +1 -1
  2. {requence-0.9.0 → requence-0.9.1.dev154}/pyproject.toml +1 -1
  3. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/__init__.py +6 -1
  4. requence-0.9.1.dev154/requence/task/ack_task.py +46 -0
  5. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/automations.py +8 -0
  6. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/task.py +6 -13
  7. requence-0.9.1.dev154/requence/task/task_watcher.py +343 -0
  8. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/updates.py +10 -1
  9. {requence-0.9.0 → requence-0.9.1.dev154/requence.egg-info}/PKG-INFO +1 -1
  10. requence-0.9.0/requence/task/ack_task.py +0 -15
  11. requence-0.9.0/requence/task/task_watcher.py +0 -213
  12. {requence-0.9.0 → requence-0.9.1.dev154}/MANIFEST.in +0 -0
  13. {requence-0.9.0 → requence-0.9.1.dev154}/README.md +0 -0
  14. {requence-0.9.0 → requence-0.9.1.dev154}/requence/__init__.py +0 -0
  15. {requence-0.9.0 → requence-0.9.1.dev154}/requence/callbacks/__init__.py +0 -0
  16. {requence-0.9.0 → requence-0.9.1.dev154}/requence/callbacks/context.py +0 -0
  17. {requence-0.9.0 → requence-0.9.1.dev154}/requence/callbacks/handler.py +0 -0
  18. {requence-0.9.0 → requence-0.9.1.dev154}/requence/client_properties.py +0 -0
  19. {requence-0.9.0 → requence-0.9.1.dev154}/requence/context.py +0 -0
  20. {requence-0.9.0 → requence-0.9.1.dev154}/requence/exceptions.py +0 -0
  21. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/__init__.py +0 -0
  22. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/errors.py +0 -0
  23. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/handlers/__init__.py +0 -0
  24. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/handlers/claim_check_handler.py +0 -0
  25. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/handlers/context.py +0 -0
  26. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/handlers/file_handler.py +0 -0
  27. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/handlers/stream_handler.py +0 -0
  28. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/pending_uploads.py +0 -0
  29. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/requence_file.py +0 -0
  30. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/requence_stream.py +0 -0
  31. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/sanitize_log_data.py +0 -0
  32. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/stream_framing.py +0 -0
  33. {requence-0.9.0 → requence-0.9.1.dev154}/requence/files/transport.py +0 -0
  34. {requence-0.9.0 → requence-0.9.1.dev154}/requence/internal_types.py +0 -0
  35. {requence-0.9.0 → requence-0.9.1.dev154}/requence/logger.py +0 -0
  36. {requence-0.9.0 → requence-0.9.1.dev154}/requence/obfuscase.py +0 -0
  37. {requence-0.9.0 → requence-0.9.1.dev154}/requence/py.typed +0 -0
  38. {requence-0.9.0 → requence-0.9.1.dev154}/requence/reachable.py +0 -0
  39. {requence-0.9.0 → requence-0.9.1.dev154}/requence/requence_callback.py +0 -0
  40. {requence-0.9.0 → requence-0.9.1.dev154}/requence/requence_file.py +0 -0
  41. {requence-0.9.0 → requence-0.9.1.dev154}/requence/requence_stream.py +0 -0
  42. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/__init__.py +0 -0
  43. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/boundary.py +0 -0
  44. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/callback_growth.py +0 -0
  45. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/cli.py +0 -0
  46. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/dev_session.py +0 -0
  47. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/durable_callback.py +0 -0
  48. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/surface_handler.py +0 -0
  49. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/transformer.py +0 -0
  50. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/types.py +0 -0
  51. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/ui.py +0 -0
  52. {requence-0.9.0 → requence-0.9.1.dev154}/requence/service/ui_watch.py +0 -0
  53. {requence-0.9.0 → requence-0.9.1.dev154}/requence/store.py +0 -0
  54. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/abort_task.py +0 -0
  55. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/cli.py +0 -0
  56. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/get_task.py +0 -0
  57. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/ingress.py +0 -0
  58. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/protect_task.py +0 -0
  59. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/types.py +0 -0
  60. {requence-0.9.0 → requence-0.9.1.dev154}/requence/task/utils.py +0 -0
  61. {requence-0.9.0 → requence-0.9.1.dev154}/requence/transformer/__init__.py +0 -0
  62. {requence-0.9.0 → requence-0.9.1.dev154}/requence/transformer/context.py +0 -0
  63. {requence-0.9.0 → requence-0.9.1.dev154}/requence/transformer/errors.py +0 -0
  64. {requence-0.9.0 → requence-0.9.1.dev154}/requence/transformer/guards.py +0 -0
  65. {requence-0.9.0 → requence-0.9.1.dev154}/requence/transformer/message_handler.py +0 -0
  66. {requence-0.9.0 → requence-0.9.1.dev154}/requence/transformer/transform.py +0 -0
  67. {requence-0.9.0 → requence-0.9.1.dev154}/requence/transformer/value_handler.py +0 -0
  68. {requence-0.9.0 → requence-0.9.1.dev154}/requence/transformer/walk.py +0 -0
  69. {requence-0.9.0 → requence-0.9.1.dev154}/requence/utils.py +0 -0
  70. {requence-0.9.0 → requence-0.9.1.dev154}/requence/wire.py +0 -0
  71. {requence-0.9.0 → requence-0.9.1.dev154}/requence/wire_codec.py +0 -0
  72. {requence-0.9.0 → requence-0.9.1.dev154}/requence.egg-info/SOURCES.txt +0 -0
  73. {requence-0.9.0 → requence-0.9.1.dev154}/requence.egg-info/dependency_links.txt +0 -0
  74. {requence-0.9.0 → requence-0.9.1.dev154}/requence.egg-info/entry_points.txt +0 -0
  75. {requence-0.9.0 → requence-0.9.1.dev154}/requence.egg-info/requires.txt +0 -0
  76. {requence-0.9.0 → requence-0.9.1.dev154}/requence.egg-info/top_level.txt +0 -0
  77. {requence-0.9.0 → requence-0.9.1.dev154}/setup.cfg +0 -0
  78. {requence-0.9.0 → requence-0.9.1.dev154}/tests/test_client_properties.py +0 -0
  79. {requence-0.9.0 → requence-0.9.1.dev154}/tests/test_logger.py +0 -0
  80. {requence-0.9.0 → requence-0.9.1.dev154}/tests/test_store_parity.py +0 -0
  81. {requence-0.9.0 → requence-0.9.1.dev154}/tests/test_wire.py +0 -0
  82. {requence-0.9.0 → requence-0.9.1.dev154}/tests/test_wire_codec.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: requence
3
- Version: 0.9.0
3
+ Version: 0.9.1.dev154
4
4
  Summary: Requence Service
5
5
  Author: Torsten Blindert
6
6
  Requires-Python: >=3.12
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "requence"
7
- version = "0.9.0"
7
+ version = "0.9.1.dev154"
8
8
  description = "Requence Service"
9
9
  readme = { file = "README.md", content-type = "text/markdown" }
10
10
  authors = [{ name = "Torsten Blindert" }]
@@ -1,8 +1,9 @@
1
1
  from .task import Task
2
2
  from .get_task import get_task
3
3
  from .abort_task import abort_task, TaskAbortException
4
+ from .ack_task import ack_task, RequireAck
4
5
  from .protect_task import protect_task, TaskProtectException
5
- from .task_watcher import TaskWatcher
6
+ from .task_watcher import TaskWatcher, WatchFilter, FilteredOutException
6
7
  from .automations import (
7
8
  Automation,
8
9
  AutomationException,
@@ -21,9 +22,13 @@ __all__ = [
21
22
  "get_task",
22
23
  "abort_task",
23
24
  "TaskAbortException",
25
+ "ack_task",
26
+ "RequireAck",
24
27
  "protect_task",
25
28
  "TaskProtectException",
26
29
  "TaskWatcher",
30
+ "WatchFilter",
31
+ "FilteredOutException",
27
32
  "Automation",
28
33
  "AutomationException",
29
34
  "AutomationExistsException",
@@ -0,0 +1,46 @@
1
+ from typing import Any, Callable, Literal, Optional
2
+ from .utils import get_key, reachable_request
3
+
4
+ type RequireAck = bool | Literal["manual"]
5
+
6
+
7
+ def ack_task(task_id: str, access_token: Optional[str] = None):
8
+ token, api_base_url = get_key(access_token)
9
+
10
+ response = reachable_request(
11
+ method="POST",
12
+ url=f"{api_base_url}/task/ack/{task_id}",
13
+ headers={"Authorization": f"Bearer {token}"},
14
+ )
15
+
16
+ if not response.ok:
17
+ raise Exception(f"Failed to acknowledge task delivery: {response.text}")
18
+
19
+
20
+ def receive_terminal_update(task_id: str, require_ack: Any, access_token: Optional[str] = None) -> Callable[[], None]:
21
+ """Runs on receive of a terminal update (taskEnd, taskError, taskAborted).
22
+
23
+ A `require_ack=True` task is acked right here, before the update reaches the
24
+ app. Returns the update's `ack`: a request for "manual", a no-op for True and
25
+ False. Once it has succeeded, calling it again sends nothing; after a
26
+ failure, calling it again retries.
27
+ """
28
+ if require_ack is True:
29
+ try:
30
+ ack_task(task_id, access_token)
31
+ except Exception:
32
+ pass
33
+
34
+ if require_ack != "manual":
35
+ return lambda: None
36
+
37
+ sent = False
38
+
39
+ def ack() -> None:
40
+ nonlocal sent
41
+ if sent:
42
+ return
43
+ ack_task(task_id, access_token)
44
+ sent = True
45
+
46
+ return ack
@@ -19,6 +19,7 @@ class Automation(TypedDict):
19
19
  previousTaskIds: list[str]
20
20
  triggerTemplateId: str
21
21
  actionTemplateId: str
22
+ requireAck: bool | Literal["manual"]
22
23
  failureReason: Optional[str]
23
24
  outdatedReason: Optional[str]
24
25
  createdAt: str
@@ -91,6 +92,7 @@ def create_automation(
91
92
  id: Optional[str] = None,
92
93
  mapping_function: Optional[str] = None,
93
94
  input: Any = None,
95
+ require_ack: bool | Literal["manual"] = False,
94
96
  access_token: Optional[str] = None,
95
97
  ) -> Automation:
96
98
  """Creates an automation.
@@ -100,6 +102,11 @@ def create_automation(
100
102
  names the same id, and AutomationExistsException means it is done. Never
101
103
  derive it from your own user id: once that automation is deleted, the id can
102
104
  never be used again.
105
+
106
+ `require_ack` takes the same values as on a task. Only the action sub-tasks
107
+ inherit it: each finished action run waits in AWAITING_DELIVERY until it is
108
+ acked. Default False: an automation nobody watches would pile up waiting
109
+ results.
103
110
  """
104
111
  body: dict[str, Any] = {
105
112
  "scope": scope,
@@ -108,6 +115,7 @@ def create_automation(
108
115
  "actionTemplateId": action_template_id,
109
116
  "mappingFunction": mapping_function,
110
117
  "input": input,
118
+ "requireAck": require_ack,
111
119
  }
112
120
  if id is not None:
113
121
  body["id"] = id
@@ -8,7 +8,7 @@ from ..obfuscase import deobfuscate
8
8
  from ..logger import logger
9
9
  from .ingress import forbid_callbacks, inbound, observe_only_callbacks, outbound
10
10
  from ..exceptions import TaskException, RequenceUnreachableException
11
- from .ack_task import ack_task
11
+ from .ack_task import RequireAck, receive_terminal_update
12
12
  from .utils import is_unreachable_status, reachable_request
13
13
  import requests
14
14
  import threading
@@ -85,7 +85,7 @@ class FinalizeResult(TypedDict):
85
85
  urls: NotRequired[TaskUrls]
86
86
 
87
87
  class Task[I: Mapping[str, Any] | None, R: Mapping[str, Any] | None, D: Mapping[str, Any], E: Mapping[str, Any]]:
88
- def __init__(self, task_template: str, input: dict[str, Any] = {}, name: Optional[str] = None, priority: int = 2, access_token: Optional[str] = None, suppress_branch_warning: bool = False, require_ack: bool = False):
88
+ def __init__(self, task_template: str, input: dict[str, Any] = {}, name: Optional[str] = None, priority: int = 2, access_token: Optional[str] = None, suppress_branch_warning: bool = False, require_ack: RequireAck = False):
89
89
 
90
90
  key = access_token or os.getenv("REQUENCE_TASK_ACCESS_TOKEN") or os.getenv("REQUENCE_ACCESS_TOKEN")
91
91
  if not key:
@@ -324,12 +324,10 @@ class Task[I: Mapping[str, Any] | None, R: Mapping[str, Any] | None, D: Mapping[
324
324
 
325
325
  type = task_update.get('type')
326
326
 
327
+ if type in ('taskEnd', 'taskError', 'taskAborted'):
328
+ task_update['ack'] = receive_terminal_update(cast(str, update.get('taskId')), update.get('requireAck'), self._access_key) # type: ignore
329
+
327
330
  if type == 'taskEnd':
328
- if update.get('requireAck'):
329
- try:
330
- ack_task(cast(str, update.get('taskId')), self._access_key)
331
- except Exception:
332
- pass
333
331
  self._task_result_future.set_result(TaskResult(
334
332
  task_id=cast(str, update.get("taskId")),
335
333
  input=task_storage.get("input"),
@@ -340,13 +338,8 @@ class Task[I: Mapping[str, Any] | None, R: Mapping[str, Any] | None, D: Mapping[
340
338
  ))
341
339
 
342
340
  if type == "taskError" or type == "taskAborted":
343
- if update.get('requireAck'):
344
- try:
345
- ack_task(cast(str, update.get('taskId')), self._access_key)
346
- except Exception:
347
- pass
348
341
  self._task_result_future.set_exception(TaskException(
349
- message=task_update.get('reason') or "Task failed" if type == "taskError" else "Task aborted",
342
+ message=task_update.get('reason') or ("Task failed" if type == "taskError" else "Task aborted"),
350
343
  task_template=self._task_template
351
344
  ))
352
345
 
@@ -0,0 +1,343 @@
1
+ from typing import Optional, TypedDict, Any, Callable, Literal, NoReturn, cast, Tuple, Generator
2
+ from .utils import get_key, is_unreachable_status
3
+ from ..exceptions import RequenceUnreachableException
4
+ from ..logger import logger
5
+ from .ack_task import receive_terminal_update
6
+ from datetime import date, datetime
7
+ import sseclient
8
+ import threading
9
+ import asyncio
10
+ import requests
11
+ import json
12
+ from .ingress import forbid_callbacks, inbound, observe_only_callbacks
13
+ from .updates import TaskContext, Update
14
+ import queue
15
+
16
+ type WatchUpdate = Update[Any, Any, dict[str, Any], dict[str, str]]
17
+
18
+ UpdateType = Literal[
19
+ "taskStart",
20
+ "taskEnd",
21
+ "taskError",
22
+ "taskAborted",
23
+ "nodeStart",
24
+ "nodeUpdate",
25
+ "nodeError",
26
+ "nodeDefer",
27
+ "nodeEnd",
28
+ ]
29
+
30
+ TERMINAL_TYPES: tuple[UpdateType, ...] = ("taskEnd", "taskError", "taskAborted")
31
+
32
+ RECONNECT_DELAY_SECONDS = 2.0
33
+
34
+
35
+ class WatchFilter(TypedDict, total=False):
36
+ only: list[UpdateType]
37
+
38
+
39
+ class WatchTaskSorage(TypedDict):
40
+ input: Any
41
+ data: dict[str, Any]
42
+ errors: dict[str, str]
43
+ result: Any
44
+ incomplete: bool
45
+
46
+
47
+ def make_default_entry() -> WatchTaskSorage:
48
+ return {
49
+ "input": None,
50
+ "data": {},
51
+ "errors": {},
52
+ "result": None,
53
+ "incomplete": True,
54
+ }
55
+
56
+
57
+ class FilteredOutException(Exception):
58
+ """A value built from an update type that `filter["only"]` leaves out is
59
+ unknown, not empty, so reading it raises instead of returning `None`."""
60
+
61
+ def __init__(self, field: str, update_type: UpdateType):
62
+ super().__init__(f"{field} is not available: filter.only leaves out '{update_type}'")
63
+ self.field = field
64
+ self.update_type = update_type
65
+
66
+
67
+ def _filtered_out(field: str, update_type: UpdateType) -> NoReturn:
68
+ raise FilteredOutException(field, update_type)
69
+
70
+
71
+ class _WatchContext(dict[str, Any]):
72
+ """The keys the filter leaves out are absent, and reading one raises."""
73
+
74
+ def __init__(self, values: dict[str, Any], filtered_out: dict[str, UpdateType]):
75
+ super().__init__(values)
76
+ self._filtered_out = filtered_out
77
+
78
+ def __missing__(self, key: str) -> Any:
79
+ if key in self._filtered_out:
80
+ _filtered_out(f'context["{key}"]', self._filtered_out[key])
81
+ raise KeyError(key)
82
+
83
+ def get(self, key: str, default: Any = None) -> Any: # type: ignore[override]
84
+ if key not in self and key in self._filtered_out:
85
+ _filtered_out(f'context["{key}"]', self._filtered_out[key])
86
+ return super().get(key, default)
87
+
88
+
89
+ class _FilteredOutFlag:
90
+ """Stands in for the `incomplete` bool when the filter leaves out
91
+ `taskStart`: the tuple item can be unpacked, but testing it raises."""
92
+
93
+ __slots__ = ()
94
+
95
+ def __bool__(self) -> bool:
96
+ _filtered_out("incomplete", "taskStart")
97
+
98
+ def __eq__(self, other: object) -> bool:
99
+ _filtered_out("incomplete", "taskStart")
100
+
101
+ __hash__ = None # type: ignore[assignment]
102
+
103
+ def __repr__(self) -> str:
104
+ return "<incomplete: filtered out>"
105
+
106
+
107
+ class SyncWatch:
108
+ def __init__(self, updates: queue.Queue[Tuple[WatchUpdate, bool]]):
109
+ self._updates = updates
110
+
111
+ @property
112
+ def updates(self) -> Generator[Tuple[WatchUpdate, bool], Any, None]:
113
+ while True:
114
+ item = self._updates.get()
115
+ yield item
116
+
117
+
118
+ class TaskWatcher:
119
+
120
+ def __init__(
121
+ self,
122
+ since: date,
123
+ access_token: Optional[str] = None,
124
+ filter: Optional[WatchFilter] = None,
125
+ on_connect: Optional[Callable[[], None]] = None,
126
+ on_unreachable: Optional[Callable[[RequenceUnreachableException], None]] = None,
127
+ on_error: Optional[Callable[[int, Optional[str]], None]] = None,
128
+ on_reconnecting: Optional[Callable[[], None]] = None,
129
+ ):
130
+ token, api_base_url = get_key(access_token)
131
+
132
+ self._stop_event = threading.Event()
133
+ self._monitor_thread: Optional[threading.Thread] = None
134
+ self._client: Optional[Any] = None
135
+
136
+ only = filter.get("only") if filter is not None else None
137
+
138
+ def receives(update_type: UpdateType) -> bool:
139
+ return only is None or update_type in only
140
+
141
+ receives_start = receives("taskStart")
142
+ receives_node_data = receives("nodeUpdate")
143
+ receives_node_errors = receives("nodeError")
144
+
145
+ # Only these three fill a task's entry; without them there is nothing to
146
+ # keep between updates, and so nothing that has to be released.
147
+ keeps_entries = receives_start or receives_node_data or receives_node_errors
148
+ missed_terminals = [t for t in TERMINAL_TYPES if not receives(t)]
149
+
150
+ if keeps_entries and missed_terminals:
151
+ logger.warn(
152
+ "TaskWatcher keeps each task's context until the task ends, but filter.only leaves out "
153
+ + ", ".join(f"'{t}'" for t in missed_terminals)
154
+ + ": a task that ends that way is never released"
155
+ )
156
+
157
+ task_storage: dict[str, WatchTaskSorage] = dict()
158
+ self._task_update_queue: queue.Queue[Tuple[WatchUpdate, bool]] = queue.Queue()
159
+ start_date = since
160
+
161
+ async def monitor():
162
+ nonlocal start_date
163
+ watch_url = f"{api_base_url}/task/watch-all"
164
+ last_position: Optional[int] = None
165
+
166
+ def connect() -> Optional[Any]:
167
+ body: dict[str, Any] = {"since": start_date.isoformat()}
168
+ if last_position is not None:
169
+ body["lastPosition"] = last_position
170
+ if filter is not None:
171
+ body["filter"] = filter
172
+ response = requests.request(
173
+ url=watch_url,
174
+ method="POST",
175
+ headers={
176
+ "Authorization": f"Bearer {token}",
177
+ "Accept": "text/event-stream"
178
+ },
179
+ json=body,
180
+ stream=True,
181
+ )
182
+
183
+ if not response.ok:
184
+ # A gateway status signals the requence system is unreachable (e.g.
185
+ # during maintenance). The reconnect loop keeps retrying regardless.
186
+ if is_unreachable_status(response.status_code) and on_unreachable:
187
+ on_unreachable(RequenceUnreachableException(response.status_code))
188
+ if on_error:
189
+ on_error(response.status_code, response.reason)
190
+ response.close()
191
+ return None
192
+
193
+ if on_connect:
194
+ on_connect()
195
+ return sseclient.SSEClient(response) # type: ignore
196
+
197
+ def apply_event(event: Any) -> None:
198
+ nonlocal last_position
199
+ if event.id:
200
+ try:
201
+ last_position = int(event.id)
202
+ except (ValueError, TypeError):
203
+ pass
204
+
205
+ raw_update = json.loads(event.data)
206
+ type = event.event
207
+
208
+ if not isinstance(raw_update, dict):
209
+ return
210
+
211
+ update = cast(dict[str, Any], raw_update)
212
+ task_id = update.get("taskId")
213
+
214
+ if not isinstance(task_id, str) or not task_id:
215
+ return
216
+
217
+ storage_entry = task_storage.get(task_id)
218
+ if storage_entry is None:
219
+ storage_entry = make_default_entry()
220
+ if keeps_entries:
221
+ task_storage[task_id] = storage_entry
222
+ ack: Optional[Callable[[], None]] = None
223
+
224
+ match type:
225
+ case "taskStart":
226
+ storage_entry["input"] = inbound(update.get("input"), on_callback_invoke=forbid_callbacks)
227
+ storage_entry["incomplete"] = False
228
+ case "nodeUpdate":
229
+ node_alias = update.get("node", {}).get("alias")
230
+ if node_alias:
231
+ storage_entry["data"][node_alias] = inbound(update.get("data"), on_callback_invoke=observe_only_callbacks)
232
+
233
+ case "nodeError":
234
+ node_alias = update.get("node", {}).get("alias")
235
+ if node_alias:
236
+ storage_entry["errors"][node_alias] = cast(str, update.get("error"))
237
+
238
+ case "taskEnd":
239
+ storage_entry["result"] = inbound(update.get("result"), on_callback_invoke=observe_only_callbacks)
240
+ task_storage.pop(task_id, None)
241
+ ack = receive_terminal_update(task_id, update.get("requireAck"), access_token)
242
+
243
+ case "taskError" | "taskAborted":
244
+ task_storage.pop(task_id, None)
245
+ ack = receive_terminal_update(task_id, update.get("requireAck"), access_token)
246
+
247
+ case _:
248
+ pass
249
+
250
+ values: dict[str, Any] = {"task_id": task_id, "result": storage_entry["result"]}
251
+ filtered_out: dict[str, UpdateType] = {}
252
+ if receives_start:
253
+ values["input"] = storage_entry["input"]
254
+ else:
255
+ filtered_out["input"] = "taskStart"
256
+ if receives_node_data:
257
+ values["node_data"] = storage_entry["data"]
258
+ else:
259
+ filtered_out["node_data"] = "nodeUpdate"
260
+ if receives_node_errors:
261
+ values["node_error"] = storage_entry["errors"]
262
+ else:
263
+ filtered_out["node_error"] = "nodeError"
264
+
265
+ context = cast(
266
+ TaskContext[Any, Any, dict[str, Any], dict[str, str]],
267
+ _WatchContext(values, filtered_out),
268
+ )
269
+
270
+ context_update: WatchUpdate = {
271
+ "context": context,
272
+ "type": type,
273
+ "root_task_id": update.get("rootTaskId"),
274
+ **update
275
+ } # type: ignore
276
+ if ack:
277
+ context_update["ack"] = ack # type: ignore
278
+
279
+ incomplete = storage_entry["incomplete"] if receives_start else cast(bool, _FilteredOutFlag())
280
+ self._task_update_queue.put((context_update, incomplete))
281
+
282
+ attempt = 0
283
+ while not self._stop_event.is_set():
284
+ if attempt > 0:
285
+ # Once an event arrived, lastPosition is the cursor. Before that,
286
+ # advance the start so a pre-connect drop does not replay history.
287
+ if last_position is None:
288
+ start_date = datetime.now()
289
+ if on_reconnecting:
290
+ on_reconnecting()
291
+ if self._stop_event.wait(RECONNECT_DELAY_SECONDS):
292
+ break
293
+ attempt += 1
294
+
295
+ try:
296
+ client = connect()
297
+ if client is None:
298
+ continue
299
+ self._client = client
300
+ for event in client.events():
301
+ if self._stop_event.is_set():
302
+ break
303
+ apply_event(event)
304
+ except (requests.RequestException, EOFError):
305
+ pass
306
+ except Exception:
307
+ # Closing the stream from `stop()` can surface as any read error.
308
+ if self._stop_event.is_set():
309
+ break
310
+ raise
311
+ finally:
312
+ if self._client is not None:
313
+ self._client.close()
314
+ self._client = None
315
+
316
+ def start_monitoring():
317
+ asyncio.run(monitor())
318
+
319
+ self._monitor_thread = threading.Thread(target=start_monitoring, daemon=True)
320
+ self._monitor_thread.start()
321
+
322
+ @property
323
+ def updates(self):
324
+ async def generate():
325
+ while True:
326
+ item = await asyncio.to_thread(self._task_update_queue.get)
327
+ yield item
328
+
329
+ return generate()
330
+
331
+ def sync(self):
332
+ return SyncWatch(self._task_update_queue)
333
+
334
+ def stop(self):
335
+ self._stop_event.set()
336
+
337
+ if self._client is not None:
338
+ self._client.close()
339
+
340
+ self._client = None
341
+
342
+ if self._monitor_thread is not None and self._monitor_thread.is_alive():
343
+ self._monitor_thread.join(timeout=1.0)
@@ -1,5 +1,5 @@
1
1
  from collections.abc import Mapping
2
- from typing import Optional, Literal, Any, Union, TypedDict
2
+ from typing import Callable, NotRequired, Optional, Literal, Any, Union, TypedDict
3
3
  from datetime import datetime
4
4
  from uuid import UUID
5
5
 
@@ -33,15 +33,24 @@ class TaskStartUpdate[I: Mapping[str, Any] | None, R: Mapping[str, Any] | None,
33
33
  class TaskErrorUpdate[I: Mapping[str, Any] | None, R: Mapping[str, Any] | None, D: Mapping[str, Any], E: Mapping[str, Any]](BaseUpdate[I,R,D,E]):
34
34
  type: Literal['taskError']
35
35
  reason: Optional[str]
36
+ requireAck: NotRequired[bool | Literal['manual']]
37
+ # Sends the delivery ack of a task started with require_ack="manual"; a no-op otherwise.
38
+ ack: Callable[[], None]
36
39
 
37
40
  class TaskAbortUpdate[I: Mapping[str, Any] | None, R: Mapping[str, Any] | None, D: Mapping[str, Any], E: Mapping[str, Any]](BaseUpdate[I,R,D,E]):
38
41
  type: Literal['taskAborted']
39
42
  reason: Optional[str]
43
+ requireAck: NotRequired[bool | Literal['manual']]
44
+ # Sends the delivery ack of a task started with require_ack="manual"; a no-op otherwise.
45
+ ack: Callable[[], None]
40
46
 
41
47
  class TaskEndUpdate[I: Mapping[str, Any] | None, R: Mapping[str, Any] | None, D: Mapping[str, Any], E: Mapping[str, Any]](BaseUpdate[I,R,D,E]):
42
48
  type: Literal['taskEnd']
43
49
  result: Mapping[str, Any]
44
50
  exit: Union[None, str]
51
+ requireAck: NotRequired[bool | Literal['manual']]
52
+ # Sends the delivery ack of a task started with require_ack="manual"; a no-op otherwise.
53
+ ack: Callable[[], None]
45
54
 
46
55
  class NodeStartUpdate[I: Mapping[str, Any] | None, R: Mapping[str, Any] | None, D: Mapping[str, Any], E: Mapping[str, Any]](BaseUpdate[I,R,D,E]):
47
56
  type: Literal['nodeStart']
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: requence
3
- Version: 0.9.0
3
+ Version: 0.9.1.dev154
4
4
  Summary: Requence Service
5
5
  Author: Torsten Blindert
6
6
  Requires-Python: >=3.12
@@ -1,15 +0,0 @@
1
- from typing import Optional
2
- from .utils import get_key, reachable_request
3
-
4
-
5
- def ack_task(task_id: str, access_token: Optional[str] = None):
6
- token, api_base_url = get_key(access_token)
7
-
8
- response = reachable_request(
9
- method="POST",
10
- url=f"{api_base_url}/task/ack/{task_id}",
11
- headers={"Authorization": f"Bearer {token}"},
12
- )
13
-
14
- if not response.ok:
15
- raise Exception(f"Failed to acknowledge task delivery: {response.text}")
@@ -1,213 +0,0 @@
1
- from typing import Optional, TypedDict, Any, Callable, cast, Tuple, Generator
2
- from .utils import get_key, is_unreachable_status
3
- from ..exceptions import RequenceUnreachableException
4
- from .ack_task import ack_task
5
- from datetime import date, datetime
6
- import sseclient
7
- import threading
8
- import asyncio
9
- import requests
10
- import json
11
- from .ingress import forbid_callbacks, inbound, observe_only_callbacks
12
- from .updates import TaskContext, Update
13
- import queue
14
-
15
- type WatchUpdate = Update[Any, Any, dict[str, Any], dict[str, str]]
16
- class WatchTaskSorage(TypedDict):
17
- input: Any
18
- data: dict[str, Any]
19
- errors: dict[str, str]
20
- result: Any
21
- incomplete: bool
22
- root_task_id: Optional[str]
23
-
24
-
25
- def make_default_entry() -> WatchTaskSorage:
26
- return {
27
- "input": None,
28
- "data": {},
29
- "errors": {},
30
- "result": None,
31
- "incomplete": True,
32
- "root_task_id": None
33
- }
34
-
35
- class SyncWatch:
36
- def __init__(self, updates: queue.Queue[Tuple[WatchUpdate, bool]]):
37
- self._updates = updates
38
-
39
- @property
40
- def updates(self) -> Generator[Tuple[WatchUpdate, bool], Any, None]:
41
- while True:
42
- item = self._updates.get()
43
- yield item
44
-
45
-
46
- class TaskWatcher:
47
-
48
- def __init__(self, since: date, access_token: Optional[str] = None, on_connect: Optional[Callable[[], None]] = None, on_unreachable: Optional[Callable[[RequenceUnreachableException], None]] = None):
49
- token, api_base_url = get_key(access_token)
50
-
51
- self._stop_event = threading.Event()
52
- self._monitor_thread: Optional[threading.Thread] = None
53
- self._client: Optional[Any] = None
54
-
55
- task_storage: dict[str, WatchTaskSorage] = dict()
56
- self._task_update_queue: queue.Queue[Tuple[WatchUpdate, bool]] = queue.Queue()
57
- start_date = since
58
-
59
- async def monitor():
60
- nonlocal start_date
61
- watch_url = f"{api_base_url}/task/watch-all"
62
- last_position: Optional[int] = None
63
-
64
- def connect():
65
- body: dict[str, Any] = {"since": start_date.isoformat()}
66
- if last_position is not None:
67
- body["lastPosition"] = last_position
68
- response = requests.request(
69
- url=watch_url,
70
- method="POST",
71
- headers={
72
- "Authorization": f"Bearer {token}",
73
- "Accept": "text/event-stream"
74
- },
75
- json=body,
76
- stream=True,
77
- )
78
-
79
- # A gateway status signals the requence system is unreachable (e.g.
80
- # during maintenance). Notify the caller so it can surface a "retry
81
- # later" state; the reconnect loop keeps retrying regardless.
82
- if is_unreachable_status(response.status_code) and on_unreachable:
83
- on_unreachable(RequenceUnreachableException(response.status_code))
84
-
85
- return sseclient.SSEClient(response) # type: ignore
86
-
87
- client = connect()
88
- self._client = client
89
- if on_connect:
90
- on_connect()
91
-
92
- while not self._stop_event.is_set():
93
- try:
94
- for event in client.events():
95
- if self._stop_event.is_set():
96
- break
97
-
98
- if event.id:
99
- try:
100
- last_position = int(event.id)
101
- except (ValueError, TypeError):
102
- pass
103
-
104
- raw_update = json.loads(event.data)
105
- type = event.event
106
-
107
- if not isinstance(raw_update, dict):
108
- continue
109
-
110
- update = cast(dict[str, Any], raw_update)
111
- task_id = update.get("task_id")
112
-
113
- if not isinstance(task_id, str) or not task_id:
114
- continue
115
-
116
- storage_entry = task_storage.get(task_id, make_default_entry())
117
-
118
- match type:
119
- case "taskStart":
120
- storage_entry["input"] = inbound(update.get("input"), on_callback_invoke=forbid_callbacks)
121
- storage_entry["incomplete"] = False
122
- storage_entry["root_task_id"] = update.get("rootTaskId")
123
- case "nodeUpdate":
124
- node_alias = update.get("node", {}).get("alias")
125
- if node_alias:
126
- storage_entry["data"][node_alias] = inbound(update.get("data"), on_callback_invoke=observe_only_callbacks)
127
-
128
- case "nodeError":
129
- node_alias = update.get("node", {}).get("alias")
130
- if node_alias:
131
- storage_entry["errors"][node_alias] = cast(str, update.get("error"))
132
-
133
- case "taskEnd":
134
- storage_entry["result"] = inbound(update.get("result"), on_callback_invoke=observe_only_callbacks)
135
- del task_storage[task_id]
136
- if update.get("requireAck"):
137
- try:
138
- ack_task(task_id, access_token)
139
- except Exception:
140
- pass
141
-
142
- case "taskError" | "taskAborted":
143
- del task_storage[task_id]
144
- if update.get("requireAck"):
145
- try:
146
- ack_task(task_id, access_token)
147
- except Exception:
148
- pass
149
-
150
- case _:
151
- pass
152
-
153
- context = TaskContext[Any, Any, dict[str, Any], dict[str, str]](
154
- task_id=task_id,
155
- input=storage_entry.get("input"),
156
- node_data=storage_entry.get("data"),
157
- node_error=storage_entry.get("errors"),
158
- result=storage_entry.get("result")
159
- )
160
-
161
- context_update: WatchUpdate = {
162
- "context": context,
163
- "type": type,
164
- "root_task_id": storage_entry.get("root_task_id"),
165
- **update
166
- } # type: ignore
167
-
168
- self._task_update_queue.put((context_update, storage_entry.get("incomplete")))
169
-
170
- if self._stop_event.is_set():
171
- break
172
-
173
- except (requests.RequestException, EOFError):
174
- if self._stop_event.is_set():
175
- break
176
-
177
- if last_position is None:
178
- start_date = datetime.now()
179
- client = connect()
180
- self._client = client
181
- continue
182
-
183
- client.close()
184
- self._client = None
185
-
186
- def start_monitoring():
187
- asyncio.run(monitor())
188
-
189
- self._monitor_thread = threading.Thread(target=start_monitoring, daemon=True)
190
- self._monitor_thread.start()
191
-
192
- @property
193
- def updates(self):
194
- async def generate():
195
- while True:
196
- item = await asyncio.to_thread(self._task_update_queue.get)
197
- yield item
198
-
199
- return generate()
200
-
201
- def sync(self):
202
- return SyncWatch(self._task_update_queue)
203
-
204
- def stop(self):
205
- self._stop_event.set()
206
-
207
- if self._client is not None:
208
- self._client.close()
209
-
210
- self._client = None
211
-
212
- if self._monitor_thread is not None and self._monitor_thread.is_alive():
213
- self._monitor_thread.join(timeout=1.0)
File without changes
File without changes
File without changes