temporal-flowrensic-python-sdk 0.6.63__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. temporal_flowrensic/__init__.py +2 -0
  2. temporal_flowrensic/_calls.py +185 -0
  3. temporal_flowrensic/_ci.py +56 -0
  4. temporal_flowrensic/_deploy.py +65 -0
  5. temporal_flowrensic/_gen/flowrensic/v1/activity_metrics_pb2.py +76 -0
  6. temporal_flowrensic/_gen/flowrensic/v1/activity_metrics_pb2.pyi +301 -0
  7. temporal_flowrensic/_gen/flowrensic/v1/catalog_pb2.py +60 -0
  8. temporal_flowrensic/_gen/flowrensic/v1/catalog_pb2.pyi +127 -0
  9. temporal_flowrensic/_gen/flowrensic/v1/catch_up_pb2.py +42 -0
  10. temporal_flowrensic/_gen/flowrensic/v1/catch_up_pb2.pyi +71 -0
  11. temporal_flowrensic/_gen/flowrensic/v1/client_calls_pb2.py +49 -0
  12. temporal_flowrensic/_gen/flowrensic/v1/client_calls_pb2.pyi +90 -0
  13. temporal_flowrensic/_gen/flowrensic/v1/cost_pb2.py +52 -0
  14. temporal_flowrensic/_gen/flowrensic/v1/cost_pb2.pyi +107 -0
  15. temporal_flowrensic/_gen/flowrensic/v1/deploy_pb2.py +51 -0
  16. temporal_flowrensic/_gen/flowrensic/v1/deploy_pb2.pyi +92 -0
  17. temporal_flowrensic/_gen/flowrensic/v1/ephemeral_run_pb2.py +107 -0
  18. temporal_flowrensic/_gen/flowrensic/v1/ephemeral_run_pb2.pyi +344 -0
  19. temporal_flowrensic/_gen/flowrensic/v1/finding_pb2.py +76 -0
  20. temporal_flowrensic/_gen/flowrensic/v1/finding_pb2.pyi +371 -0
  21. temporal_flowrensic/_gen/flowrensic/v1/job_pb2.py +46 -0
  22. temporal_flowrensic/_gen/flowrensic/v1/job_pb2.pyi +51 -0
  23. temporal_flowrensic/_gen/flowrensic/v1/load_pb2.py +51 -0
  24. temporal_flowrensic/_gen/flowrensic/v1/load_pb2.pyi +118 -0
  25. temporal_flowrensic/_gen/flowrensic/v1/page_pb2.py +38 -0
  26. temporal_flowrensic/_gen/flowrensic/v1/page_pb2.pyi +23 -0
  27. temporal_flowrensic/_gen/flowrensic/v1/remediation_pb2.py +60 -0
  28. temporal_flowrensic/_gen/flowrensic/v1/remediation_pb2.pyi +157 -0
  29. temporal_flowrensic/_gen/flowrensic/v1/replay_pb2.py +84 -0
  30. temporal_flowrensic/_gen/flowrensic/v1/replay_pb2.pyi +309 -0
  31. temporal_flowrensic/_gen/flowrensic/v1/server_config_pb2.py +48 -0
  32. temporal_flowrensic/_gen/flowrensic/v1/server_config_pb2.pyi +97 -0
  33. temporal_flowrensic/_gen/flowrensic/v1/services_connect.py +3621 -0
  34. temporal_flowrensic/_gen/flowrensic/v1/services_pb2.py +60 -0
  35. temporal_flowrensic/_gen/flowrensic/v1/services_pb2.pyi +20 -0
  36. temporal_flowrensic/_gen/flowrensic/v1/stored_data_pb2.py +43 -0
  37. temporal_flowrensic/_gen/flowrensic/v1/stored_data_pb2.pyi +50 -0
  38. temporal_flowrensic/_gen/flowrensic/v1/workflow_history_pb2.py +47 -0
  39. temporal_flowrensic/_gen/flowrensic/v1/workflow_history_pb2.pyi +102 -0
  40. temporal_flowrensic/_gen/flowrensic/v1/workflow_types_pb2.py +55 -0
  41. temporal_flowrensic/_gen/flowrensic/v1/workflow_types_pb2.pyi +197 -0
  42. temporal_flowrensic/_names.py +42 -0
  43. temporal_flowrensic/_queries.py +61 -0
  44. temporal_flowrensic/_replay.py +378 -0
  45. temporal_flowrensic/_sink.py +574 -0
  46. temporal_flowrensic/_telemetry.py +559 -0
  47. temporal_flowrensic/activity.py +136 -0
  48. temporal_flowrensic/client.py +166 -0
  49. temporal_flowrensic/py.typed +0 -0
  50. temporal_flowrensic/workflow.py +103 -0
  51. temporal_flowrensic_python_sdk-0.6.63.dist-info/METADATA +74 -0
  52. temporal_flowrensic_python_sdk-0.6.63.dist-info/RECORD +54 -0
  53. temporal_flowrensic_python_sdk-0.6.63.dist-info/WHEEL +4 -0
  54. temporal_flowrensic_python_sdk-0.6.63.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,559 @@
1
+ """The worker telemetry: the hello with its answer, the beats, the forwarding
2
+ of the engine's and the worker's metrics, the histories of the runs the
3
+ activity interceptor saw, and the handover at shutdown. It runs as tasks on
4
+ the worker's event loop, and every call it makes is async."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import asyncio
9
+ import urllib.request
10
+ import warnings
11
+ from collections import OrderedDict
12
+ from dataclasses import dataclass
13
+ from datetime import timedelta
14
+ from typing import Any, Awaitable, Callable, Optional, Sequence, Union
15
+
16
+ from connectrpc.code import Code
17
+ from connectrpc.errors import ConnectError
18
+
19
+ from temporal_flowrensic._gen.flowrensic.v1.activity_metrics_pb2 import ActivityAttempt
20
+ from temporal_flowrensic._gen.flowrensic.v1.ephemeral_run_pb2 import (
21
+ EphemeralRunHandoverRequest,
22
+ EphemeralRunHandshakeRequest,
23
+ EphemeralRunHandshakeResponse,
24
+ IngestEngineHistoryRequest,
25
+ IngestEphemeralRunMetricsRequest,
26
+ IngestWorkerMetricsRequest,
27
+ WorkerBeatRequest,
28
+ )
29
+ from temporal_flowrensic._gen.flowrensic.v1.services_connect import IngestServiceClient
30
+ from temporal_flowrensic._names import interceptor_list
31
+ from temporal_flowrensic._sink import (
32
+ DEFAULT_TIMEOUT_MS,
33
+ AttemptSink,
34
+ RecordSink,
35
+ SinkCounts,
36
+ create_attempt_sink,
37
+ default_worker_id,
38
+ client_headers,
39
+ )
40
+
41
+ __all__ = [
42
+ "FlowrensicWarning",
43
+ "ForwardAnyway",
44
+ "Fail",
45
+ "GiveUpQuietly",
46
+ "KeepTrying",
47
+ "NoAnswerStrategy",
48
+ "Retry",
49
+ "Telemetry",
50
+ "start_telemetry",
51
+ ]
52
+
53
+
54
+ class FlowrensicWarning(UserWarning):
55
+ """A warning about what Flowrensic answered, such as an interval out of range."""
56
+
57
+
58
+ @dataclass(frozen=True)
59
+ class KeepTrying:
60
+ """Start the worker at once and ask again in the background, delay seconds
61
+ between two asks, and once answered do what the answer says."""
62
+
63
+ delay: float = 5.0
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class Fail:
68
+ """Raise at once."""
69
+
70
+
71
+ @dataclass(frozen=True)
72
+ class Retry:
73
+ """Ask again, attempts times in all with delay seconds between two asks, and then raise as Fail does."""
74
+
75
+ attempts: int
76
+ delay: float
77
+
78
+
79
+ @dataclass(frozen=True)
80
+ class GiveUpQuietly:
81
+ """Start the worker with nothing forwarded on a timer."""
82
+
83
+
84
+ @dataclass(frozen=True)
85
+ class ForwardAnyway:
86
+ """Start as if told to forward when there is a metrics address and an ephemeral run id, at the default interval."""
87
+
88
+
89
+ NoAnswerStrategy = Union[KeepTrying, Fail, Retry, GiveUpQuietly, ForwardAnyway]
90
+ """What the worker does when the hello gets no answer."""
91
+
92
+ _DEFAULT_INTERVAL = 5
93
+ _MIN_INTERVAL = 1
94
+ _MAX_INTERVAL = 60
95
+ _SEEN_CAP = 10_000
96
+
97
+ # How many histories a worker reads and sends at once.
98
+ _HISTORIES_AT_ONCE = 16
99
+
100
+ DEFAULT_SHUTDOWN_TIMEOUT_MS = 20_000
101
+ """Milliseconds the shutdown may send histories when start_telemetry is given no shutdown_timeout_ms."""
102
+
103
+ # The codes Flowrensic refuses a hello with on purpose, such as an env it does
104
+ # not know or a wrong key. Any other error is no answer.
105
+ _REFUSALS = {
106
+ Code.INVALID_ARGUMENT,
107
+ Code.NOT_FOUND,
108
+ Code.ALREADY_EXISTS,
109
+ Code.PERMISSION_DENIED,
110
+ Code.FAILED_PRECONDITION,
111
+ Code.OUT_OF_RANGE,
112
+ Code.UNAUTHENTICATED,
113
+ }
114
+
115
+ # The events that end a run, so nothing is added to its history after them.
116
+ _CLOSING = {
117
+ "workflow_execution_completed_event_attributes",
118
+ "workflow_execution_failed_event_attributes",
119
+ "workflow_execution_timed_out_event_attributes",
120
+ "workflow_execution_canceled_event_attributes",
121
+ "workflow_execution_terminated_event_attributes",
122
+ "workflow_execution_continued_as_new_event_attributes",
123
+ }
124
+
125
+
126
+ def _no_answer(error: BaseException) -> bool:
127
+ """Whether the call got no answer a worker can act on. Only a refusal on purpose is an answer."""
128
+ return not isinstance(error, ConnectError) or error.code not in _REFUSALS
129
+
130
+
131
+ def _checked_interval(seconds: int) -> int:
132
+ """The interval of an answer, or the default when it is out of range, with a warning."""
133
+ if _MIN_INTERVAL <= seconds <= _MAX_INTERVAL:
134
+ return seconds
135
+ warnings.warn(
136
+ f"Flowrensic sent a scrape interval of {seconds} seconds, out of {_MIN_INTERVAL} to {_MAX_INTERVAL}; "
137
+ f"the worker keeps {_DEFAULT_INTERVAL} seconds",
138
+ FlowrensicWarning,
139
+ stacklevel=2,
140
+ )
141
+ return _DEFAULT_INTERVAL
142
+
143
+
144
+ async def _ignoring_failure(job: Callable[[], Awaitable[Any]]) -> None:
145
+ try:
146
+ await job()
147
+ except asyncio.CancelledError:
148
+ raise
149
+ except Exception: # noqa: BLE001
150
+ return
151
+
152
+
153
+ def _scrape(address: str, timeout_ms: int) -> str:
154
+ with urllib.request.urlopen(f"http://{address}/metrics", timeout=timeout_ms / 1000) as res: # noqa: S310
155
+ if res.status != 200:
156
+ raise RuntimeError(f"metrics endpoint returned {res.status}")
157
+ return res.read().decode("utf-8")
158
+
159
+
160
+ _HISTOGRAM_SERIES = ("_bucket", "_sum", "_count")
161
+
162
+
163
+ def _only_named(page: str, names: frozenset[str]) -> str:
164
+ """The lines of the engine's page the server reads: a series line whose name
165
+ is named, and a HELP or TYPE line whose name, or that name with a histogram
166
+ suffix, is named. No names, from a server that gives none, keeps the whole
167
+ page."""
168
+ if not names:
169
+ return page
170
+ kept: list[str] = []
171
+ for line in page.split("\n"):
172
+ text = line.strip()
173
+ if text.startswith("#"):
174
+ words = text.split(None, 3)
175
+ if (
176
+ len(words) >= 3
177
+ and words[1] in ("HELP", "TYPE")
178
+ and (words[2] in names or any(words[2] + s in names for s in _HISTOGRAM_SERIES))
179
+ ):
180
+ kept.append(line)
181
+ continue
182
+ end = next((i for i, c in enumerate(text) if c.isspace() or c == "{"), len(text))
183
+ if text and text[:end] in names:
184
+ kept.append(line)
185
+ return "\n".join(kept) + "\n" if kept else ""
186
+
187
+
188
+ class _SeenSink(RecordSink[ActivityAttempt]):
189
+ """The sink the activity interceptor reports to: it notes the run of each
190
+ attempt, so its history is sent, and passes the attempt on."""
191
+
192
+ def __init__(self, telemetry: Telemetry, inner: AttemptSink) -> None: # noqa: D107
193
+ # Not a buffer of its own: every call goes to inner.
194
+ self._telemetry = telemetry
195
+ self._inner = inner
196
+
197
+ def record(self, one: ActivityAttempt) -> None:
198
+ try:
199
+ self._telemetry._saw(one)
200
+ except Exception: # noqa: BLE001
201
+ pass
202
+ self._inner.record(one)
203
+
204
+ def counts(self) -> SinkCounts:
205
+ return self._inner.counts()
206
+
207
+ def flush_sync(self) -> None:
208
+ self._inner.flush_sync()
209
+
210
+ async def flush(self) -> None:
211
+ await self._inner.flush()
212
+
213
+ def close_sync(self) -> None:
214
+ self._inner.close_sync()
215
+
216
+ async def close(self) -> None:
217
+ await self._inner.close()
218
+
219
+ def __getattr__(self, name: str) -> Any:
220
+ return getattr(self._inner, name)
221
+
222
+
223
+ class Telemetry:
224
+ """A worker's telemetry, from start_telemetry."""
225
+
226
+ def __init__(self) -> None: # noqa: D107
227
+ self.answered = False
228
+ """Whether Flowrensic answered the hello, at start or later in the background."""
229
+ self.forwarding = False
230
+ """Whether the answer told this worker to forward the engine's metrics."""
231
+ self.interval_seconds = 0
232
+ """The interval the worker sends on, in seconds."""
233
+ self.sending_histories = True
234
+ """Whether the worker sends the histories of the runs it saw."""
235
+ self.engine_metric_names: frozenset[str] = frozenset()
236
+ """The series of the engine's page the server reads. Empty sends the whole page."""
237
+ self.when_answered: asyncio.Future[bool]
238
+ """Done with True once Flowrensic answers, and with False when it refuses
239
+ in the background, the strategy ran with no answer, or shutdown came first."""
240
+ self.sink: AttemptSink
241
+ """The sink to give the activity interceptor."""
242
+ self._seen: OrderedDict[str, str] = OrderedDict()
243
+ # The length of the history last sent for each run seen.
244
+ self._sent: dict[str, int] = {}
245
+ self._shutdown: Callable[[], Awaitable[None]]
246
+
247
+ def _saw(self, attempt: ActivityAttempt) -> None:
248
+ if self.sending_histories and attempt.run_key and attempt.workflow_execution_id:
249
+ self._seen[attempt.run_key] = attempt.workflow_execution_id
250
+ self._seen.move_to_end(attempt.run_key)
251
+ while len(self._seen) > _SEEN_CAP:
252
+ gone, _ = self._seen.popitem(last=False)
253
+ self._sent.pop(gone, None)
254
+
255
+ async def shutdown(self) -> None:
256
+ """The last beat and scrape, the histories of the runs this worker saw,
257
+ the attempts still waiting, and the handover. Returns once they landed."""
258
+ await self._shutdown()
259
+
260
+
261
+ async def start_telemetry(
262
+ *,
263
+ server_base_url: str,
264
+ current_env: str,
265
+ namespace: str,
266
+ temporal_address: str = "",
267
+ temporal_client: Any = None,
268
+ env_access_key: str = "",
269
+ current_ephemeral_run: str = "",
270
+ metrics_address: str = "",
271
+ worker_metrics_address: str = "",
272
+ worker_id: str = "",
273
+ worker_name: str = "",
274
+ version: str = "",
275
+ workflow_interceptors: Optional[Sequence[Any]] = None,
276
+ no_answer: Optional[NoAnswerStrategy] = None,
277
+ timeout_ms: int = DEFAULT_TIMEOUT_MS,
278
+ shutdown_timeout_ms: int = DEFAULT_SHUTDOWN_TIMEOUT_MS,
279
+ ) -> Telemetry:
280
+ """Says hello to Flowrensic with the env and the ephemeral run, and does
281
+ what the answer says. When the hello gets no answer it does what no_answer
282
+ says, KeepTrying when left out. The answer says whether this worker
283
+ forwards the engine's metrics and how often it sends. On that interval it
284
+ sends a beat, the engine's metrics when told to, the worker's own metrics
285
+ when it has their address, and the history of every run the activity
286
+ interceptor reported through telemetry.sink, read from Temporal.
287
+
288
+ temporal_client is the worker's own temporalio Client, used to read the
289
+ histories; it is needed for an engine behind TLS or an API key. Left out, a
290
+ plain client to temporal_address is opened the first time a history is
291
+ read. A worker told that Flowrensic pulls the histories itself never opens
292
+ one. worker_name is the name of the worker, which the sink sends with each
293
+ attempt. workflow_interceptors are the workflow interceptors the worker loads,
294
+ sent by name; left out, no list is sent. timeout_ms is in
295
+ milliseconds, the other times in seconds. shutdown_timeout_ms is how long
296
+ the shutdown may send histories, in milliseconds, counted from when it
297
+ starts: the beat and the last scrape go first, past it no more histories
298
+ are started, and then the sink is closed and the handover sent."""
299
+ if not server_base_url:
300
+ raise ValueError("flowrensic: server_base_url is required")
301
+ strategy: NoAnswerStrategy = no_answer if no_answer is not None else KeepTrying()
302
+ worker_id = worker_id or default_worker_id()
303
+ interceptors = interceptor_list(workflow_interceptors)
304
+ timeout_ms = max(1, timeout_ms)
305
+ shutdown_timeout_ms = max(1, shutdown_timeout_ms)
306
+ headers = client_headers(current_env, env_access_key, current_ephemeral_run)
307
+ ingest = IngestServiceClient(server_base_url.rstrip("/"), send_compression=None)
308
+ t = Telemetry()
309
+
310
+ def listed(message: Any) -> Any:
311
+ if interceptors is not None:
312
+ message.workflow_interceptors.CopyFrom(interceptors)
313
+ return message
314
+
315
+ async def ask() -> EphemeralRunHandshakeResponse:
316
+ return await ingest.ephemeral_run_handshake(
317
+ listed(
318
+ EphemeralRunHandshakeRequest(
319
+ metrics_address=metrics_address,
320
+ worker_id=worker_id,
321
+ version=version,
322
+ )
323
+ ),
324
+ headers=headers,
325
+ timeout_ms=timeout_ms,
326
+ )
327
+
328
+ def take(answer: EphemeralRunHandshakeResponse) -> None:
329
+ t.answered = True
330
+ t.forwarding = answer.forward
331
+ t.interval_seconds = _checked_interval(answer.interval_seconds)
332
+ t.sending_histories = not answer.skip_histories
333
+ t.engine_metric_names = frozenset(answer.engine_metric_names)
334
+
335
+ attempts = max(1, strategy.attempts) if isinstance(strategy, Retry) else 1
336
+ unanswered = False
337
+ for attempt in range(1, attempts + 1):
338
+ try:
339
+ take(await ask())
340
+ break
341
+ except Exception as error: # noqa: BLE001
342
+ if isinstance(strategy, Retry) and attempt < attempts and _no_answer(error):
343
+ await asyncio.sleep(strategy.delay)
344
+ continue
345
+ if not _no_answer(error) or isinstance(strategy, (Fail, Retry)):
346
+ await ingest.close()
347
+ raise
348
+ if isinstance(strategy, KeepTrying):
349
+ unanswered = True
350
+ elif isinstance(strategy, GiveUpQuietly):
351
+ t.forwarding = False
352
+ t.interval_seconds = 0
353
+ else:
354
+ t.forwarding = bool(metrics_address) and bool(current_ephemeral_run)
355
+ t.interval_seconds = _DEFAULT_INTERVAL
356
+ break
357
+
358
+ t.sink = _SeenSink(
359
+ t, create_attempt_sink(
360
+ server_base_url,
361
+ worker_id=worker_id,
362
+ worker_name=worker_name,
363
+ current_env=current_env,
364
+ env_access_key=env_access_key,
365
+ current_ephemeral_run=current_ephemeral_run,
366
+ timeout_ms=timeout_ms,
367
+ )
368
+ )
369
+
370
+ temporal: list[Any] = [temporal_client]
371
+
372
+ async def connect() -> Any:
373
+ if temporal[0] is None:
374
+ from temporalio.client import Client
375
+
376
+ temporal[0] = await Client.connect(temporal_address, namespace=namespace)
377
+ return temporal[0]
378
+
379
+ async def scrape() -> None:
380
+ text = await asyncio.get_running_loop().run_in_executor(None, _scrape, metrics_address, timeout_ms)
381
+ await ingest.ingest_ephemeral_run_metrics(
382
+ IngestEphemeralRunMetricsRequest(worker_id=worker_id, exposition=_only_named(text, t.engine_metric_names)),
383
+ headers=headers,
384
+ timeout_ms=timeout_ms,
385
+ )
386
+
387
+ async def scrape_worker() -> None:
388
+ text = await asyncio.get_running_loop().run_in_executor(None, _scrape, worker_metrics_address, timeout_ms)
389
+ await ingest.ingest_worker_metrics(
390
+ IngestWorkerMetricsRequest(worker_id=worker_id, exposition=text),
391
+ headers=headers,
392
+ timeout_ms=timeout_ms,
393
+ )
394
+
395
+ async def forward_history(run_id: str, workflow_id: str) -> None:
396
+ from temporalio.api.common.v1 import WorkflowExecution
397
+ from temporalio.api.history.v1 import History
398
+ from temporalio.api.workflowservice.v1 import (
399
+ DescribeWorkflowExecutionRequest,
400
+ GetWorkflowExecutionHistoryRequest,
401
+ )
402
+
403
+ client = await connect()
404
+ # Sent again only when the engine holds more events than were sent last.
405
+ described = await client.workflow_service.describe_workflow_execution(
406
+ DescribeWorkflowExecutionRequest(
407
+ namespace=namespace,
408
+ execution=WorkflowExecution(workflow_id=workflow_id, run_id=run_id),
409
+ ),
410
+ timeout=timedelta(milliseconds=timeout_ms),
411
+ )
412
+ length = described.workflow_execution_info.history_length
413
+ if length > 0 and length == t._sent.get(run_id, 0):
414
+ return
415
+ events: list[Any] = []
416
+ token = b""
417
+ while True:
418
+ page = await client.workflow_service.get_workflow_execution_history(
419
+ GetWorkflowExecutionHistoryRequest(
420
+ namespace=namespace,
421
+ execution=WorkflowExecution(workflow_id=workflow_id, run_id=run_id),
422
+ next_page_token=token,
423
+ ),
424
+ timeout=timedelta(milliseconds=timeout_ms),
425
+ )
426
+ events.extend(page.history.events)
427
+ if not page.next_page_token:
428
+ break
429
+ token = page.next_page_token
430
+ await ingest.ingest_engine_history(
431
+ IngestEngineHistoryRequest(
432
+ run_id=run_id,
433
+ namespace=namespace,
434
+ workflow_id=workflow_id,
435
+ task_attempt=described.pending_workflow_task.attempt,
436
+ history=History(events=events).SerializeToString(),
437
+ ),
438
+ headers=headers,
439
+ timeout_ms=timeout_ms,
440
+ )
441
+ t._sent[run_id] = len(events)
442
+ if events and events[-1].WhichOneof("attributes") in _CLOSING:
443
+ t._seen.pop(run_id, None)
444
+ t._sent.pop(run_id, None)
445
+
446
+ async def collect(deadline: Optional[float] = None) -> None:
447
+ # deadline is a time of the loop; past it no more histories are started.
448
+ async def beat() -> None:
449
+ await ingest.worker_beat(
450
+ listed(
451
+ WorkerBeatRequest(worker_id=worker_id, version=version)
452
+ ),
453
+ headers=headers,
454
+ timeout_ms=timeout_ms,
455
+ )
456
+
457
+ await _ignoring_failure(beat)
458
+ if t.forwarding:
459
+ await _ignoring_failure(scrape)
460
+ if worker_metrics_address:
461
+ await _ignoring_failure(scrape_worker)
462
+ if not t.sending_histories:
463
+ return
464
+ runs = list(t._seen.items())
465
+ queue = iter(runs)
466
+
467
+ async def send_more() -> None:
468
+ for run_id, workflow_id in queue:
469
+ if deadline is not None and loop.time() >= deadline:
470
+ return
471
+ await _ignoring_failure(lambda r=run_id, w=workflow_id: forward_history(r, w))
472
+
473
+ await asyncio.gather(*(send_more() for _ in range(min(_HISTORIES_AT_ONCE, len(runs)))))
474
+
475
+ loop = asyncio.get_running_loop()
476
+ state: dict[str, Any] = {"working": None, "ticker": None}
477
+
478
+ def tick() -> None:
479
+ # A tick that comes while the one before it still works is skipped.
480
+ working = state["working"]
481
+ if working is not None and not working.done():
482
+ return
483
+ state["working"] = loop.create_task(collect())
484
+
485
+ async def ticking() -> None:
486
+ while True:
487
+ await asyncio.sleep(t.interval_seconds)
488
+ tick()
489
+
490
+ def start_ticking() -> None:
491
+ if t.interval_seconds > 0 and state["ticker"] is None:
492
+ state["ticker"] = loop.create_task(ticking())
493
+
494
+ start_ticking()
495
+
496
+ stop_asking = asyncio.Event()
497
+
498
+ async def keep_asking() -> bool:
499
+ delay = strategy.delay if isinstance(strategy, KeepTrying) else 0.0
500
+ while True:
501
+ try:
502
+ await asyncio.wait_for(stop_asking.wait(), timeout=delay)
503
+ return False
504
+ except asyncio.TimeoutError:
505
+ pass
506
+ try:
507
+ answer = await ask()
508
+ except Exception as error: # noqa: BLE001
509
+ if stop_asking.is_set() or not _no_answer(error):
510
+ return False
511
+ continue
512
+ if stop_asking.is_set():
513
+ return False
514
+ take(answer)
515
+ if not t.sending_histories:
516
+ t._seen.clear()
517
+ t._sent.clear()
518
+ start_ticking()
519
+ tick()
520
+ return True
521
+
522
+ if unanswered:
523
+ t.when_answered = loop.create_task(keep_asking())
524
+ else:
525
+ done: asyncio.Future[bool] = loop.create_future()
526
+ done.set_result(t.answered)
527
+ t.when_answered = done
528
+
529
+ async def shutdown() -> None:
530
+ deadline = loop.time() + shutdown_timeout_ms / 1000
531
+ stop_asking.set()
532
+ await asyncio.shield(t.when_answered)
533
+ ticker = state["ticker"]
534
+ if ticker is not None:
535
+ ticker.cancel()
536
+ working = state["working"]
537
+ if working is not None and not working.done():
538
+ working.cancel()
539
+ try:
540
+ await working
541
+ except BaseException: # noqa: BLE001
542
+ pass
543
+ # keep trying with no answer has no Flowrensic to hand over to.
544
+ hand_over = t.answered or not unanswered
545
+ if hand_over:
546
+ await collect(deadline)
547
+ await t.sink.close()
548
+ if hand_over:
549
+ await _ignoring_failure(
550
+ lambda: ingest.ephemeral_run_handover(
551
+ EphemeralRunHandoverRequest(worker_id=worker_id, version=version),
552
+ headers=headers,
553
+ timeout_ms=timeout_ms,
554
+ )
555
+ )
556
+ await _ignoring_failure(ingest.close)
557
+
558
+ t._shutdown = shutdown
559
+ return t
@@ -0,0 +1,136 @@
1
+ """The Flowrensic activity interceptor for a Temporal Python worker, beside
2
+ temporalio.activity."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import asyncio
7
+ import time
8
+ from datetime import timedelta
9
+ from typing import Any, Optional
10
+
11
+ from google.protobuf.timestamp_pb2 import Timestamp
12
+ from temporalio import activity as _activity
13
+ from temporalio.exceptions import ApplicationError, CancelledError
14
+ from temporalio.worker import (
15
+ ActivityInboundInterceptor,
16
+ ActivityOutboundInterceptor,
17
+ ExecuteActivityInput,
18
+ Interceptor as _TemporalInterceptor,
19
+ )
20
+
21
+ from temporal_flowrensic._gen.flowrensic.v1.activity_metrics_pb2 import (
22
+ OUTCOME_CANCELED,
23
+ OUTCOME_COMPLETED,
24
+ OUTCOME_FAILED,
25
+ ActivityAttempt,
26
+ )
27
+ from temporal_flowrensic._sink import RecordSink
28
+
29
+ __all__ = ["Interceptor"]
30
+
31
+
32
+ def _classify(err: BaseException) -> tuple[int, str]:
33
+ """The outcome and error type of a failed attempt."""
34
+ try:
35
+ if isinstance(err, (CancelledError, asyncio.CancelledError)):
36
+ return OUTCOME_CANCELED, "CancelledFailure"
37
+ if isinstance(err, ApplicationError):
38
+ return OUTCOME_FAILED, err.type or "ApplicationFailure"
39
+ return OUTCOME_FAILED, type(err).__name__
40
+ except Exception: # noqa: BLE001
41
+ return OUTCOME_FAILED, "Unknown"
42
+
43
+
44
+ def _ms(value: Optional[timedelta]) -> int:
45
+ return 0 if value is None else round(value.total_seconds() * 1000)
46
+
47
+
48
+ def _timestamp(seconds: float) -> Timestamp:
49
+ out = Timestamp()
50
+ out.FromNanoseconds(int(seconds * 1_000_000_000))
51
+ return out
52
+
53
+
54
+ class Interceptor(_TemporalInterceptor):
55
+ """The Flowrensic interceptor, to put in Worker(interceptors=[...]). It
56
+ records one ActivityAttempt per activity attempt into sink, and never
57
+ changes what the activity returns or raises. It adds nothing to workflows.
58
+
59
+ The namespace of each attempt is the one Temporal gives in the activity
60
+ info. current_env is the env the worker runs in, current_ephemeral_run which run of
61
+ that env this is, and version the version this worker serves. Flowrensic
62
+ refuses an attempt that names an env and no ephemeral run."""
63
+
64
+ def __init__(
65
+ self,
66
+ sink: RecordSink[ActivityAttempt],
67
+ *,
68
+ current_env: str = "",
69
+ current_ephemeral_run: str = "",
70
+ version: str = "",
71
+ ) -> None:
72
+ self._sink = sink
73
+ self._env = current_env
74
+ self._ephemeral_run = current_ephemeral_run
75
+ self._version = version
76
+
77
+ def intercept_activity(self, next: ActivityInboundInterceptor) -> ActivityInboundInterceptor:
78
+ return _ActivityInbound(next, self)
79
+
80
+ def _emit(self, started: float, outcome: int, error_type: str) -> None:
81
+ """Records one attempt. An error inside it is swallowed."""
82
+ sink = self._sink
83
+ try:
84
+ ended = time.time()
85
+ info = _activity.info()
86
+ policy = info.retry_policy
87
+ scheduled = info.scheduled_time
88
+ attempt = ActivityAttempt(
89
+ namespace=getattr(info, "namespace", "") or info.workflow_namespace,
90
+ env=self._env,
91
+ ephemeral_run=self._ephemeral_run,
92
+ version=self._version,
93
+ workflow_type=info.workflow_type or "",
94
+ workflow_execution_id=info.workflow_id or "",
95
+ run_key=info.workflow_run_id or "",
96
+ activity_type=info.activity_type,
97
+ activity_id=info.activity_id,
98
+ attempt=info.attempt,
99
+ task_queue=info.task_queue,
100
+ outcome=outcome,
101
+ error_type=error_type,
102
+ started_at=_timestamp(started),
103
+ ended_at=_timestamp(ended),
104
+ duration_ms=round((ended - started) * 1000),
105
+ start_to_close_timeout_ms=_ms(info.start_to_close_timeout),
106
+ maximum_attempts=(policy.maximum_attempts if policy is not None else 0) or 0,
107
+ heartbeat_timeout_ms=_ms(info.heartbeat_timeout),
108
+ )
109
+ if scheduled is not None:
110
+ attempt.scheduled_at.FromDatetime(scheduled)
111
+ sink.record(attempt)
112
+ except Exception: # noqa: BLE001
113
+ return
114
+
115
+
116
+ class _ActivityInbound(ActivityInboundInterceptor):
117
+ def __init__(self, next: ActivityInboundInterceptor, owner: Interceptor) -> None:
118
+ super().__init__(next)
119
+ self._owner = owner
120
+
121
+ def init(self, outbound: ActivityOutboundInterceptor) -> None:
122
+ super().init(_ActivityOutbound(outbound))
123
+
124
+ async def execute_activity(self, input: ExecuteActivityInput) -> Any:
125
+ started = time.time()
126
+ try:
127
+ result = await super().execute_activity(input)
128
+ except BaseException as err:
129
+ self._owner._emit(started, *_classify(err))
130
+ raise
131
+ self._owner._emit(started, OUTCOME_COMPLETED, "")
132
+ return result
133
+
134
+
135
+ class _ActivityOutbound(ActivityOutboundInterceptor):
136
+ pass