temporal-flowrensic-python-sdk 0.6.63__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- temporal_flowrensic/__init__.py +2 -0
- temporal_flowrensic/_calls.py +185 -0
- temporal_flowrensic/_ci.py +56 -0
- temporal_flowrensic/_deploy.py +65 -0
- temporal_flowrensic/_gen/flowrensic/v1/activity_metrics_pb2.py +76 -0
- temporal_flowrensic/_gen/flowrensic/v1/activity_metrics_pb2.pyi +301 -0
- temporal_flowrensic/_gen/flowrensic/v1/catalog_pb2.py +60 -0
- temporal_flowrensic/_gen/flowrensic/v1/catalog_pb2.pyi +127 -0
- temporal_flowrensic/_gen/flowrensic/v1/catch_up_pb2.py +42 -0
- temporal_flowrensic/_gen/flowrensic/v1/catch_up_pb2.pyi +71 -0
- temporal_flowrensic/_gen/flowrensic/v1/client_calls_pb2.py +49 -0
- temporal_flowrensic/_gen/flowrensic/v1/client_calls_pb2.pyi +90 -0
- temporal_flowrensic/_gen/flowrensic/v1/cost_pb2.py +52 -0
- temporal_flowrensic/_gen/flowrensic/v1/cost_pb2.pyi +107 -0
- temporal_flowrensic/_gen/flowrensic/v1/deploy_pb2.py +51 -0
- temporal_flowrensic/_gen/flowrensic/v1/deploy_pb2.pyi +92 -0
- temporal_flowrensic/_gen/flowrensic/v1/ephemeral_run_pb2.py +107 -0
- temporal_flowrensic/_gen/flowrensic/v1/ephemeral_run_pb2.pyi +344 -0
- temporal_flowrensic/_gen/flowrensic/v1/finding_pb2.py +76 -0
- temporal_flowrensic/_gen/flowrensic/v1/finding_pb2.pyi +371 -0
- temporal_flowrensic/_gen/flowrensic/v1/job_pb2.py +46 -0
- temporal_flowrensic/_gen/flowrensic/v1/job_pb2.pyi +51 -0
- temporal_flowrensic/_gen/flowrensic/v1/load_pb2.py +51 -0
- temporal_flowrensic/_gen/flowrensic/v1/load_pb2.pyi +118 -0
- temporal_flowrensic/_gen/flowrensic/v1/page_pb2.py +38 -0
- temporal_flowrensic/_gen/flowrensic/v1/page_pb2.pyi +23 -0
- temporal_flowrensic/_gen/flowrensic/v1/remediation_pb2.py +60 -0
- temporal_flowrensic/_gen/flowrensic/v1/remediation_pb2.pyi +157 -0
- temporal_flowrensic/_gen/flowrensic/v1/replay_pb2.py +84 -0
- temporal_flowrensic/_gen/flowrensic/v1/replay_pb2.pyi +309 -0
- temporal_flowrensic/_gen/flowrensic/v1/server_config_pb2.py +48 -0
- temporal_flowrensic/_gen/flowrensic/v1/server_config_pb2.pyi +97 -0
- temporal_flowrensic/_gen/flowrensic/v1/services_connect.py +3621 -0
- temporal_flowrensic/_gen/flowrensic/v1/services_pb2.py +60 -0
- temporal_flowrensic/_gen/flowrensic/v1/services_pb2.pyi +20 -0
- temporal_flowrensic/_gen/flowrensic/v1/stored_data_pb2.py +43 -0
- temporal_flowrensic/_gen/flowrensic/v1/stored_data_pb2.pyi +50 -0
- temporal_flowrensic/_gen/flowrensic/v1/workflow_history_pb2.py +47 -0
- temporal_flowrensic/_gen/flowrensic/v1/workflow_history_pb2.pyi +102 -0
- temporal_flowrensic/_gen/flowrensic/v1/workflow_types_pb2.py +55 -0
- temporal_flowrensic/_gen/flowrensic/v1/workflow_types_pb2.pyi +197 -0
- temporal_flowrensic/_names.py +42 -0
- temporal_flowrensic/_queries.py +61 -0
- temporal_flowrensic/_replay.py +378 -0
- temporal_flowrensic/_sink.py +574 -0
- temporal_flowrensic/_telemetry.py +559 -0
- temporal_flowrensic/activity.py +136 -0
- temporal_flowrensic/client.py +166 -0
- temporal_flowrensic/py.typed +0 -0
- temporal_flowrensic/workflow.py +103 -0
- temporal_flowrensic_python_sdk-0.6.63.dist-info/METADATA +74 -0
- temporal_flowrensic_python_sdk-0.6.63.dist-info/RECORD +54 -0
- temporal_flowrensic_python_sdk-0.6.63.dist-info/WHEEL +4 -0
- temporal_flowrensic_python_sdk-0.6.63.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
"""The worker telemetry: the hello with its answer, the beats, the forwarding
|
|
2
|
+
of the engine's and the worker's metrics, the histories of the runs the
|
|
3
|
+
activity interceptor saw, and the handover at shutdown. It runs as tasks on
|
|
4
|
+
the worker's event loop, and every call it makes is async."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import urllib.request
|
|
10
|
+
import warnings
|
|
11
|
+
from collections import OrderedDict
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from datetime import timedelta
|
|
14
|
+
from typing import Any, Awaitable, Callable, Optional, Sequence, Union
|
|
15
|
+
|
|
16
|
+
from connectrpc.code import Code
|
|
17
|
+
from connectrpc.errors import ConnectError
|
|
18
|
+
|
|
19
|
+
from temporal_flowrensic._gen.flowrensic.v1.activity_metrics_pb2 import ActivityAttempt
|
|
20
|
+
from temporal_flowrensic._gen.flowrensic.v1.ephemeral_run_pb2 import (
|
|
21
|
+
EphemeralRunHandoverRequest,
|
|
22
|
+
EphemeralRunHandshakeRequest,
|
|
23
|
+
EphemeralRunHandshakeResponse,
|
|
24
|
+
IngestEngineHistoryRequest,
|
|
25
|
+
IngestEphemeralRunMetricsRequest,
|
|
26
|
+
IngestWorkerMetricsRequest,
|
|
27
|
+
WorkerBeatRequest,
|
|
28
|
+
)
|
|
29
|
+
from temporal_flowrensic._gen.flowrensic.v1.services_connect import IngestServiceClient
|
|
30
|
+
from temporal_flowrensic._names import interceptor_list
|
|
31
|
+
from temporal_flowrensic._sink import (
|
|
32
|
+
DEFAULT_TIMEOUT_MS,
|
|
33
|
+
AttemptSink,
|
|
34
|
+
RecordSink,
|
|
35
|
+
SinkCounts,
|
|
36
|
+
create_attempt_sink,
|
|
37
|
+
default_worker_id,
|
|
38
|
+
client_headers,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
__all__ = [
|
|
42
|
+
"FlowrensicWarning",
|
|
43
|
+
"ForwardAnyway",
|
|
44
|
+
"Fail",
|
|
45
|
+
"GiveUpQuietly",
|
|
46
|
+
"KeepTrying",
|
|
47
|
+
"NoAnswerStrategy",
|
|
48
|
+
"Retry",
|
|
49
|
+
"Telemetry",
|
|
50
|
+
"start_telemetry",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class FlowrensicWarning(UserWarning):
|
|
55
|
+
"""A warning about what Flowrensic answered, such as an interval out of range."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class KeepTrying:
|
|
60
|
+
"""Start the worker at once and ask again in the background, delay seconds
|
|
61
|
+
between two asks, and once answered do what the answer says."""
|
|
62
|
+
|
|
63
|
+
delay: float = 5.0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Fail:
|
|
68
|
+
"""Raise at once."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class Retry:
|
|
73
|
+
"""Ask again, attempts times in all with delay seconds between two asks, and then raise as Fail does."""
|
|
74
|
+
|
|
75
|
+
attempts: int
|
|
76
|
+
delay: float
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True)
|
|
80
|
+
class GiveUpQuietly:
|
|
81
|
+
"""Start the worker with nothing forwarded on a timer."""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class ForwardAnyway:
|
|
86
|
+
"""Start as if told to forward when there is a metrics address and an ephemeral run id, at the default interval."""
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
NoAnswerStrategy = Union[KeepTrying, Fail, Retry, GiveUpQuietly, ForwardAnyway]
|
|
90
|
+
"""What the worker does when the hello gets no answer."""
|
|
91
|
+
|
|
92
|
+
_DEFAULT_INTERVAL = 5
|
|
93
|
+
_MIN_INTERVAL = 1
|
|
94
|
+
_MAX_INTERVAL = 60
|
|
95
|
+
_SEEN_CAP = 10_000
|
|
96
|
+
|
|
97
|
+
# How many histories a worker reads and sends at once.
|
|
98
|
+
_HISTORIES_AT_ONCE = 16
|
|
99
|
+
|
|
100
|
+
DEFAULT_SHUTDOWN_TIMEOUT_MS = 20_000
|
|
101
|
+
"""Milliseconds the shutdown may send histories when start_telemetry is given no shutdown_timeout_ms."""
|
|
102
|
+
|
|
103
|
+
# The codes Flowrensic refuses a hello with on purpose, such as an env it does
|
|
104
|
+
# not know or a wrong key. Any other error is no answer.
|
|
105
|
+
_REFUSALS = {
|
|
106
|
+
Code.INVALID_ARGUMENT,
|
|
107
|
+
Code.NOT_FOUND,
|
|
108
|
+
Code.ALREADY_EXISTS,
|
|
109
|
+
Code.PERMISSION_DENIED,
|
|
110
|
+
Code.FAILED_PRECONDITION,
|
|
111
|
+
Code.OUT_OF_RANGE,
|
|
112
|
+
Code.UNAUTHENTICATED,
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
# The events that end a run, so nothing is added to its history after them.
|
|
116
|
+
_CLOSING = {
|
|
117
|
+
"workflow_execution_completed_event_attributes",
|
|
118
|
+
"workflow_execution_failed_event_attributes",
|
|
119
|
+
"workflow_execution_timed_out_event_attributes",
|
|
120
|
+
"workflow_execution_canceled_event_attributes",
|
|
121
|
+
"workflow_execution_terminated_event_attributes",
|
|
122
|
+
"workflow_execution_continued_as_new_event_attributes",
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _no_answer(error: BaseException) -> bool:
|
|
127
|
+
"""Whether the call got no answer a worker can act on. Only a refusal on purpose is an answer."""
|
|
128
|
+
return not isinstance(error, ConnectError) or error.code not in _REFUSALS
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _checked_interval(seconds: int) -> int:
|
|
132
|
+
"""The interval of an answer, or the default when it is out of range, with a warning."""
|
|
133
|
+
if _MIN_INTERVAL <= seconds <= _MAX_INTERVAL:
|
|
134
|
+
return seconds
|
|
135
|
+
warnings.warn(
|
|
136
|
+
f"Flowrensic sent a scrape interval of {seconds} seconds, out of {_MIN_INTERVAL} to {_MAX_INTERVAL}; "
|
|
137
|
+
f"the worker keeps {_DEFAULT_INTERVAL} seconds",
|
|
138
|
+
FlowrensicWarning,
|
|
139
|
+
stacklevel=2,
|
|
140
|
+
)
|
|
141
|
+
return _DEFAULT_INTERVAL
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
async def _ignoring_failure(job: Callable[[], Awaitable[Any]]) -> None:
|
|
145
|
+
try:
|
|
146
|
+
await job()
|
|
147
|
+
except asyncio.CancelledError:
|
|
148
|
+
raise
|
|
149
|
+
except Exception: # noqa: BLE001
|
|
150
|
+
return
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _scrape(address: str, timeout_ms: int) -> str:
|
|
154
|
+
with urllib.request.urlopen(f"http://{address}/metrics", timeout=timeout_ms / 1000) as res: # noqa: S310
|
|
155
|
+
if res.status != 200:
|
|
156
|
+
raise RuntimeError(f"metrics endpoint returned {res.status}")
|
|
157
|
+
return res.read().decode("utf-8")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
_HISTOGRAM_SERIES = ("_bucket", "_sum", "_count")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _only_named(page: str, names: frozenset[str]) -> str:
|
|
164
|
+
"""The lines of the engine's page the server reads: a series line whose name
|
|
165
|
+
is named, and a HELP or TYPE line whose name, or that name with a histogram
|
|
166
|
+
suffix, is named. No names, from a server that gives none, keeps the whole
|
|
167
|
+
page."""
|
|
168
|
+
if not names:
|
|
169
|
+
return page
|
|
170
|
+
kept: list[str] = []
|
|
171
|
+
for line in page.split("\n"):
|
|
172
|
+
text = line.strip()
|
|
173
|
+
if text.startswith("#"):
|
|
174
|
+
words = text.split(None, 3)
|
|
175
|
+
if (
|
|
176
|
+
len(words) >= 3
|
|
177
|
+
and words[1] in ("HELP", "TYPE")
|
|
178
|
+
and (words[2] in names or any(words[2] + s in names for s in _HISTOGRAM_SERIES))
|
|
179
|
+
):
|
|
180
|
+
kept.append(line)
|
|
181
|
+
continue
|
|
182
|
+
end = next((i for i, c in enumerate(text) if c.isspace() or c == "{"), len(text))
|
|
183
|
+
if text and text[:end] in names:
|
|
184
|
+
kept.append(line)
|
|
185
|
+
return "\n".join(kept) + "\n" if kept else ""
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class _SeenSink(RecordSink[ActivityAttempt]):
|
|
189
|
+
"""The sink the activity interceptor reports to: it notes the run of each
|
|
190
|
+
attempt, so its history is sent, and passes the attempt on."""
|
|
191
|
+
|
|
192
|
+
def __init__(self, telemetry: Telemetry, inner: AttemptSink) -> None: # noqa: D107
|
|
193
|
+
# Not a buffer of its own: every call goes to inner.
|
|
194
|
+
self._telemetry = telemetry
|
|
195
|
+
self._inner = inner
|
|
196
|
+
|
|
197
|
+
def record(self, one: ActivityAttempt) -> None:
|
|
198
|
+
try:
|
|
199
|
+
self._telemetry._saw(one)
|
|
200
|
+
except Exception: # noqa: BLE001
|
|
201
|
+
pass
|
|
202
|
+
self._inner.record(one)
|
|
203
|
+
|
|
204
|
+
def counts(self) -> SinkCounts:
|
|
205
|
+
return self._inner.counts()
|
|
206
|
+
|
|
207
|
+
def flush_sync(self) -> None:
|
|
208
|
+
self._inner.flush_sync()
|
|
209
|
+
|
|
210
|
+
async def flush(self) -> None:
|
|
211
|
+
await self._inner.flush()
|
|
212
|
+
|
|
213
|
+
def close_sync(self) -> None:
|
|
214
|
+
self._inner.close_sync()
|
|
215
|
+
|
|
216
|
+
async def close(self) -> None:
|
|
217
|
+
await self._inner.close()
|
|
218
|
+
|
|
219
|
+
def __getattr__(self, name: str) -> Any:
|
|
220
|
+
return getattr(self._inner, name)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class Telemetry:
|
|
224
|
+
"""A worker's telemetry, from start_telemetry."""
|
|
225
|
+
|
|
226
|
+
def __init__(self) -> None: # noqa: D107
|
|
227
|
+
self.answered = False
|
|
228
|
+
"""Whether Flowrensic answered the hello, at start or later in the background."""
|
|
229
|
+
self.forwarding = False
|
|
230
|
+
"""Whether the answer told this worker to forward the engine's metrics."""
|
|
231
|
+
self.interval_seconds = 0
|
|
232
|
+
"""The interval the worker sends on, in seconds."""
|
|
233
|
+
self.sending_histories = True
|
|
234
|
+
"""Whether the worker sends the histories of the runs it saw."""
|
|
235
|
+
self.engine_metric_names: frozenset[str] = frozenset()
|
|
236
|
+
"""The series of the engine's page the server reads. Empty sends the whole page."""
|
|
237
|
+
self.when_answered: asyncio.Future[bool]
|
|
238
|
+
"""Done with True once Flowrensic answers, and with False when it refuses
|
|
239
|
+
in the background, the strategy ran with no answer, or shutdown came first."""
|
|
240
|
+
self.sink: AttemptSink
|
|
241
|
+
"""The sink to give the activity interceptor."""
|
|
242
|
+
self._seen: OrderedDict[str, str] = OrderedDict()
|
|
243
|
+
# The length of the history last sent for each run seen.
|
|
244
|
+
self._sent: dict[str, int] = {}
|
|
245
|
+
self._shutdown: Callable[[], Awaitable[None]]
|
|
246
|
+
|
|
247
|
+
def _saw(self, attempt: ActivityAttempt) -> None:
|
|
248
|
+
if self.sending_histories and attempt.run_key and attempt.workflow_execution_id:
|
|
249
|
+
self._seen[attempt.run_key] = attempt.workflow_execution_id
|
|
250
|
+
self._seen.move_to_end(attempt.run_key)
|
|
251
|
+
while len(self._seen) > _SEEN_CAP:
|
|
252
|
+
gone, _ = self._seen.popitem(last=False)
|
|
253
|
+
self._sent.pop(gone, None)
|
|
254
|
+
|
|
255
|
+
async def shutdown(self) -> None:
|
|
256
|
+
"""The last beat and scrape, the histories of the runs this worker saw,
|
|
257
|
+
the attempts still waiting, and the handover. Returns once they landed."""
|
|
258
|
+
await self._shutdown()
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
async def start_telemetry(
|
|
262
|
+
*,
|
|
263
|
+
server_base_url: str,
|
|
264
|
+
current_env: str,
|
|
265
|
+
namespace: str,
|
|
266
|
+
temporal_address: str = "",
|
|
267
|
+
temporal_client: Any = None,
|
|
268
|
+
env_access_key: str = "",
|
|
269
|
+
current_ephemeral_run: str = "",
|
|
270
|
+
metrics_address: str = "",
|
|
271
|
+
worker_metrics_address: str = "",
|
|
272
|
+
worker_id: str = "",
|
|
273
|
+
worker_name: str = "",
|
|
274
|
+
version: str = "",
|
|
275
|
+
workflow_interceptors: Optional[Sequence[Any]] = None,
|
|
276
|
+
no_answer: Optional[NoAnswerStrategy] = None,
|
|
277
|
+
timeout_ms: int = DEFAULT_TIMEOUT_MS,
|
|
278
|
+
shutdown_timeout_ms: int = DEFAULT_SHUTDOWN_TIMEOUT_MS,
|
|
279
|
+
) -> Telemetry:
|
|
280
|
+
"""Says hello to Flowrensic with the env and the ephemeral run, and does
|
|
281
|
+
what the answer says. When the hello gets no answer it does what no_answer
|
|
282
|
+
says, KeepTrying when left out. The answer says whether this worker
|
|
283
|
+
forwards the engine's metrics and how often it sends. On that interval it
|
|
284
|
+
sends a beat, the engine's metrics when told to, the worker's own metrics
|
|
285
|
+
when it has their address, and the history of every run the activity
|
|
286
|
+
interceptor reported through telemetry.sink, read from Temporal.
|
|
287
|
+
|
|
288
|
+
temporal_client is the worker's own temporalio Client, used to read the
|
|
289
|
+
histories; it is needed for an engine behind TLS or an API key. Left out, a
|
|
290
|
+
plain client to temporal_address is opened the first time a history is
|
|
291
|
+
read. A worker told that Flowrensic pulls the histories itself never opens
|
|
292
|
+
one. worker_name is the name of the worker, which the sink sends with each
|
|
293
|
+
attempt. workflow_interceptors are the workflow interceptors the worker loads,
|
|
294
|
+
sent by name; left out, no list is sent. timeout_ms is in
|
|
295
|
+
milliseconds, the other times in seconds. shutdown_timeout_ms is how long
|
|
296
|
+
the shutdown may send histories, in milliseconds, counted from when it
|
|
297
|
+
starts: the beat and the last scrape go first, past it no more histories
|
|
298
|
+
are started, and then the sink is closed and the handover sent."""
|
|
299
|
+
if not server_base_url:
|
|
300
|
+
raise ValueError("flowrensic: server_base_url is required")
|
|
301
|
+
strategy: NoAnswerStrategy = no_answer if no_answer is not None else KeepTrying()
|
|
302
|
+
worker_id = worker_id or default_worker_id()
|
|
303
|
+
interceptors = interceptor_list(workflow_interceptors)
|
|
304
|
+
timeout_ms = max(1, timeout_ms)
|
|
305
|
+
shutdown_timeout_ms = max(1, shutdown_timeout_ms)
|
|
306
|
+
headers = client_headers(current_env, env_access_key, current_ephemeral_run)
|
|
307
|
+
ingest = IngestServiceClient(server_base_url.rstrip("/"), send_compression=None)
|
|
308
|
+
t = Telemetry()
|
|
309
|
+
|
|
310
|
+
def listed(message: Any) -> Any:
|
|
311
|
+
if interceptors is not None:
|
|
312
|
+
message.workflow_interceptors.CopyFrom(interceptors)
|
|
313
|
+
return message
|
|
314
|
+
|
|
315
|
+
async def ask() -> EphemeralRunHandshakeResponse:
|
|
316
|
+
return await ingest.ephemeral_run_handshake(
|
|
317
|
+
listed(
|
|
318
|
+
EphemeralRunHandshakeRequest(
|
|
319
|
+
metrics_address=metrics_address,
|
|
320
|
+
worker_id=worker_id,
|
|
321
|
+
version=version,
|
|
322
|
+
)
|
|
323
|
+
),
|
|
324
|
+
headers=headers,
|
|
325
|
+
timeout_ms=timeout_ms,
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
def take(answer: EphemeralRunHandshakeResponse) -> None:
|
|
329
|
+
t.answered = True
|
|
330
|
+
t.forwarding = answer.forward
|
|
331
|
+
t.interval_seconds = _checked_interval(answer.interval_seconds)
|
|
332
|
+
t.sending_histories = not answer.skip_histories
|
|
333
|
+
t.engine_metric_names = frozenset(answer.engine_metric_names)
|
|
334
|
+
|
|
335
|
+
attempts = max(1, strategy.attempts) if isinstance(strategy, Retry) else 1
|
|
336
|
+
unanswered = False
|
|
337
|
+
for attempt in range(1, attempts + 1):
|
|
338
|
+
try:
|
|
339
|
+
take(await ask())
|
|
340
|
+
break
|
|
341
|
+
except Exception as error: # noqa: BLE001
|
|
342
|
+
if isinstance(strategy, Retry) and attempt < attempts and _no_answer(error):
|
|
343
|
+
await asyncio.sleep(strategy.delay)
|
|
344
|
+
continue
|
|
345
|
+
if not _no_answer(error) or isinstance(strategy, (Fail, Retry)):
|
|
346
|
+
await ingest.close()
|
|
347
|
+
raise
|
|
348
|
+
if isinstance(strategy, KeepTrying):
|
|
349
|
+
unanswered = True
|
|
350
|
+
elif isinstance(strategy, GiveUpQuietly):
|
|
351
|
+
t.forwarding = False
|
|
352
|
+
t.interval_seconds = 0
|
|
353
|
+
else:
|
|
354
|
+
t.forwarding = bool(metrics_address) and bool(current_ephemeral_run)
|
|
355
|
+
t.interval_seconds = _DEFAULT_INTERVAL
|
|
356
|
+
break
|
|
357
|
+
|
|
358
|
+
t.sink = _SeenSink(
|
|
359
|
+
t, create_attempt_sink(
|
|
360
|
+
server_base_url,
|
|
361
|
+
worker_id=worker_id,
|
|
362
|
+
worker_name=worker_name,
|
|
363
|
+
current_env=current_env,
|
|
364
|
+
env_access_key=env_access_key,
|
|
365
|
+
current_ephemeral_run=current_ephemeral_run,
|
|
366
|
+
timeout_ms=timeout_ms,
|
|
367
|
+
)
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
temporal: list[Any] = [temporal_client]
|
|
371
|
+
|
|
372
|
+
async def connect() -> Any:
|
|
373
|
+
if temporal[0] is None:
|
|
374
|
+
from temporalio.client import Client
|
|
375
|
+
|
|
376
|
+
temporal[0] = await Client.connect(temporal_address, namespace=namespace)
|
|
377
|
+
return temporal[0]
|
|
378
|
+
|
|
379
|
+
async def scrape() -> None:
|
|
380
|
+
text = await asyncio.get_running_loop().run_in_executor(None, _scrape, metrics_address, timeout_ms)
|
|
381
|
+
await ingest.ingest_ephemeral_run_metrics(
|
|
382
|
+
IngestEphemeralRunMetricsRequest(worker_id=worker_id, exposition=_only_named(text, t.engine_metric_names)),
|
|
383
|
+
headers=headers,
|
|
384
|
+
timeout_ms=timeout_ms,
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
async def scrape_worker() -> None:
|
|
388
|
+
text = await asyncio.get_running_loop().run_in_executor(None, _scrape, worker_metrics_address, timeout_ms)
|
|
389
|
+
await ingest.ingest_worker_metrics(
|
|
390
|
+
IngestWorkerMetricsRequest(worker_id=worker_id, exposition=text),
|
|
391
|
+
headers=headers,
|
|
392
|
+
timeout_ms=timeout_ms,
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
async def forward_history(run_id: str, workflow_id: str) -> None:
|
|
396
|
+
from temporalio.api.common.v1 import WorkflowExecution
|
|
397
|
+
from temporalio.api.history.v1 import History
|
|
398
|
+
from temporalio.api.workflowservice.v1 import (
|
|
399
|
+
DescribeWorkflowExecutionRequest,
|
|
400
|
+
GetWorkflowExecutionHistoryRequest,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
client = await connect()
|
|
404
|
+
# Sent again only when the engine holds more events than were sent last.
|
|
405
|
+
described = await client.workflow_service.describe_workflow_execution(
|
|
406
|
+
DescribeWorkflowExecutionRequest(
|
|
407
|
+
namespace=namespace,
|
|
408
|
+
execution=WorkflowExecution(workflow_id=workflow_id, run_id=run_id),
|
|
409
|
+
),
|
|
410
|
+
timeout=timedelta(milliseconds=timeout_ms),
|
|
411
|
+
)
|
|
412
|
+
length = described.workflow_execution_info.history_length
|
|
413
|
+
if length > 0 and length == t._sent.get(run_id, 0):
|
|
414
|
+
return
|
|
415
|
+
events: list[Any] = []
|
|
416
|
+
token = b""
|
|
417
|
+
while True:
|
|
418
|
+
page = await client.workflow_service.get_workflow_execution_history(
|
|
419
|
+
GetWorkflowExecutionHistoryRequest(
|
|
420
|
+
namespace=namespace,
|
|
421
|
+
execution=WorkflowExecution(workflow_id=workflow_id, run_id=run_id),
|
|
422
|
+
next_page_token=token,
|
|
423
|
+
),
|
|
424
|
+
timeout=timedelta(milliseconds=timeout_ms),
|
|
425
|
+
)
|
|
426
|
+
events.extend(page.history.events)
|
|
427
|
+
if not page.next_page_token:
|
|
428
|
+
break
|
|
429
|
+
token = page.next_page_token
|
|
430
|
+
await ingest.ingest_engine_history(
|
|
431
|
+
IngestEngineHistoryRequest(
|
|
432
|
+
run_id=run_id,
|
|
433
|
+
namespace=namespace,
|
|
434
|
+
workflow_id=workflow_id,
|
|
435
|
+
task_attempt=described.pending_workflow_task.attempt,
|
|
436
|
+
history=History(events=events).SerializeToString(),
|
|
437
|
+
),
|
|
438
|
+
headers=headers,
|
|
439
|
+
timeout_ms=timeout_ms,
|
|
440
|
+
)
|
|
441
|
+
t._sent[run_id] = len(events)
|
|
442
|
+
if events and events[-1].WhichOneof("attributes") in _CLOSING:
|
|
443
|
+
t._seen.pop(run_id, None)
|
|
444
|
+
t._sent.pop(run_id, None)
|
|
445
|
+
|
|
446
|
+
async def collect(deadline: Optional[float] = None) -> None:
|
|
447
|
+
# deadline is a time of the loop; past it no more histories are started.
|
|
448
|
+
async def beat() -> None:
|
|
449
|
+
await ingest.worker_beat(
|
|
450
|
+
listed(
|
|
451
|
+
WorkerBeatRequest(worker_id=worker_id, version=version)
|
|
452
|
+
),
|
|
453
|
+
headers=headers,
|
|
454
|
+
timeout_ms=timeout_ms,
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
await _ignoring_failure(beat)
|
|
458
|
+
if t.forwarding:
|
|
459
|
+
await _ignoring_failure(scrape)
|
|
460
|
+
if worker_metrics_address:
|
|
461
|
+
await _ignoring_failure(scrape_worker)
|
|
462
|
+
if not t.sending_histories:
|
|
463
|
+
return
|
|
464
|
+
runs = list(t._seen.items())
|
|
465
|
+
queue = iter(runs)
|
|
466
|
+
|
|
467
|
+
async def send_more() -> None:
|
|
468
|
+
for run_id, workflow_id in queue:
|
|
469
|
+
if deadline is not None and loop.time() >= deadline:
|
|
470
|
+
return
|
|
471
|
+
await _ignoring_failure(lambda r=run_id, w=workflow_id: forward_history(r, w))
|
|
472
|
+
|
|
473
|
+
await asyncio.gather(*(send_more() for _ in range(min(_HISTORIES_AT_ONCE, len(runs)))))
|
|
474
|
+
|
|
475
|
+
loop = asyncio.get_running_loop()
|
|
476
|
+
state: dict[str, Any] = {"working": None, "ticker": None}
|
|
477
|
+
|
|
478
|
+
def tick() -> None:
|
|
479
|
+
# A tick that comes while the one before it still works is skipped.
|
|
480
|
+
working = state["working"]
|
|
481
|
+
if working is not None and not working.done():
|
|
482
|
+
return
|
|
483
|
+
state["working"] = loop.create_task(collect())
|
|
484
|
+
|
|
485
|
+
async def ticking() -> None:
|
|
486
|
+
while True:
|
|
487
|
+
await asyncio.sleep(t.interval_seconds)
|
|
488
|
+
tick()
|
|
489
|
+
|
|
490
|
+
def start_ticking() -> None:
|
|
491
|
+
if t.interval_seconds > 0 and state["ticker"] is None:
|
|
492
|
+
state["ticker"] = loop.create_task(ticking())
|
|
493
|
+
|
|
494
|
+
start_ticking()
|
|
495
|
+
|
|
496
|
+
stop_asking = asyncio.Event()
|
|
497
|
+
|
|
498
|
+
async def keep_asking() -> bool:
|
|
499
|
+
delay = strategy.delay if isinstance(strategy, KeepTrying) else 0.0
|
|
500
|
+
while True:
|
|
501
|
+
try:
|
|
502
|
+
await asyncio.wait_for(stop_asking.wait(), timeout=delay)
|
|
503
|
+
return False
|
|
504
|
+
except asyncio.TimeoutError:
|
|
505
|
+
pass
|
|
506
|
+
try:
|
|
507
|
+
answer = await ask()
|
|
508
|
+
except Exception as error: # noqa: BLE001
|
|
509
|
+
if stop_asking.is_set() or not _no_answer(error):
|
|
510
|
+
return False
|
|
511
|
+
continue
|
|
512
|
+
if stop_asking.is_set():
|
|
513
|
+
return False
|
|
514
|
+
take(answer)
|
|
515
|
+
if not t.sending_histories:
|
|
516
|
+
t._seen.clear()
|
|
517
|
+
t._sent.clear()
|
|
518
|
+
start_ticking()
|
|
519
|
+
tick()
|
|
520
|
+
return True
|
|
521
|
+
|
|
522
|
+
if unanswered:
|
|
523
|
+
t.when_answered = loop.create_task(keep_asking())
|
|
524
|
+
else:
|
|
525
|
+
done: asyncio.Future[bool] = loop.create_future()
|
|
526
|
+
done.set_result(t.answered)
|
|
527
|
+
t.when_answered = done
|
|
528
|
+
|
|
529
|
+
async def shutdown() -> None:
|
|
530
|
+
deadline = loop.time() + shutdown_timeout_ms / 1000
|
|
531
|
+
stop_asking.set()
|
|
532
|
+
await asyncio.shield(t.when_answered)
|
|
533
|
+
ticker = state["ticker"]
|
|
534
|
+
if ticker is not None:
|
|
535
|
+
ticker.cancel()
|
|
536
|
+
working = state["working"]
|
|
537
|
+
if working is not None and not working.done():
|
|
538
|
+
working.cancel()
|
|
539
|
+
try:
|
|
540
|
+
await working
|
|
541
|
+
except BaseException: # noqa: BLE001
|
|
542
|
+
pass
|
|
543
|
+
# keep trying with no answer has no Flowrensic to hand over to.
|
|
544
|
+
hand_over = t.answered or not unanswered
|
|
545
|
+
if hand_over:
|
|
546
|
+
await collect(deadline)
|
|
547
|
+
await t.sink.close()
|
|
548
|
+
if hand_over:
|
|
549
|
+
await _ignoring_failure(
|
|
550
|
+
lambda: ingest.ephemeral_run_handover(
|
|
551
|
+
EphemeralRunHandoverRequest(worker_id=worker_id, version=version),
|
|
552
|
+
headers=headers,
|
|
553
|
+
timeout_ms=timeout_ms,
|
|
554
|
+
)
|
|
555
|
+
)
|
|
556
|
+
await _ignoring_failure(ingest.close)
|
|
557
|
+
|
|
558
|
+
t._shutdown = shutdown
|
|
559
|
+
return t
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""The Flowrensic activity interceptor for a Temporal Python worker, beside
|
|
2
|
+
temporalio.activity."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
import time
|
|
8
|
+
from datetime import timedelta
|
|
9
|
+
from typing import Any, Optional
|
|
10
|
+
|
|
11
|
+
from google.protobuf.timestamp_pb2 import Timestamp
|
|
12
|
+
from temporalio import activity as _activity
|
|
13
|
+
from temporalio.exceptions import ApplicationError, CancelledError
|
|
14
|
+
from temporalio.worker import (
|
|
15
|
+
ActivityInboundInterceptor,
|
|
16
|
+
ActivityOutboundInterceptor,
|
|
17
|
+
ExecuteActivityInput,
|
|
18
|
+
Interceptor as _TemporalInterceptor,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
from temporal_flowrensic._gen.flowrensic.v1.activity_metrics_pb2 import (
|
|
22
|
+
OUTCOME_CANCELED,
|
|
23
|
+
OUTCOME_COMPLETED,
|
|
24
|
+
OUTCOME_FAILED,
|
|
25
|
+
ActivityAttempt,
|
|
26
|
+
)
|
|
27
|
+
from temporal_flowrensic._sink import RecordSink
|
|
28
|
+
|
|
29
|
+
__all__ = ["Interceptor"]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _classify(err: BaseException) -> tuple[int, str]:
|
|
33
|
+
"""The outcome and error type of a failed attempt."""
|
|
34
|
+
try:
|
|
35
|
+
if isinstance(err, (CancelledError, asyncio.CancelledError)):
|
|
36
|
+
return OUTCOME_CANCELED, "CancelledFailure"
|
|
37
|
+
if isinstance(err, ApplicationError):
|
|
38
|
+
return OUTCOME_FAILED, err.type or "ApplicationFailure"
|
|
39
|
+
return OUTCOME_FAILED, type(err).__name__
|
|
40
|
+
except Exception: # noqa: BLE001
|
|
41
|
+
return OUTCOME_FAILED, "Unknown"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _ms(value: Optional[timedelta]) -> int:
|
|
45
|
+
return 0 if value is None else round(value.total_seconds() * 1000)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _timestamp(seconds: float) -> Timestamp:
|
|
49
|
+
out = Timestamp()
|
|
50
|
+
out.FromNanoseconds(int(seconds * 1_000_000_000))
|
|
51
|
+
return out
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Interceptor(_TemporalInterceptor):
|
|
55
|
+
"""The Flowrensic interceptor, to put in Worker(interceptors=[...]). It
|
|
56
|
+
records one ActivityAttempt per activity attempt into sink, and never
|
|
57
|
+
changes what the activity returns or raises. It adds nothing to workflows.
|
|
58
|
+
|
|
59
|
+
The namespace of each attempt is the one Temporal gives in the activity
|
|
60
|
+
info. current_env is the env the worker runs in, current_ephemeral_run which run of
|
|
61
|
+
that env this is, and version the version this worker serves. Flowrensic
|
|
62
|
+
refuses an attempt that names an env and no ephemeral run."""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
sink: RecordSink[ActivityAttempt],
|
|
67
|
+
*,
|
|
68
|
+
current_env: str = "",
|
|
69
|
+
current_ephemeral_run: str = "",
|
|
70
|
+
version: str = "",
|
|
71
|
+
) -> None:
|
|
72
|
+
self._sink = sink
|
|
73
|
+
self._env = current_env
|
|
74
|
+
self._ephemeral_run = current_ephemeral_run
|
|
75
|
+
self._version = version
|
|
76
|
+
|
|
77
|
+
def intercept_activity(self, next: ActivityInboundInterceptor) -> ActivityInboundInterceptor:
|
|
78
|
+
return _ActivityInbound(next, self)
|
|
79
|
+
|
|
80
|
+
def _emit(self, started: float, outcome: int, error_type: str) -> None:
|
|
81
|
+
"""Records one attempt. An error inside it is swallowed."""
|
|
82
|
+
sink = self._sink
|
|
83
|
+
try:
|
|
84
|
+
ended = time.time()
|
|
85
|
+
info = _activity.info()
|
|
86
|
+
policy = info.retry_policy
|
|
87
|
+
scheduled = info.scheduled_time
|
|
88
|
+
attempt = ActivityAttempt(
|
|
89
|
+
namespace=getattr(info, "namespace", "") or info.workflow_namespace,
|
|
90
|
+
env=self._env,
|
|
91
|
+
ephemeral_run=self._ephemeral_run,
|
|
92
|
+
version=self._version,
|
|
93
|
+
workflow_type=info.workflow_type or "",
|
|
94
|
+
workflow_execution_id=info.workflow_id or "",
|
|
95
|
+
run_key=info.workflow_run_id or "",
|
|
96
|
+
activity_type=info.activity_type,
|
|
97
|
+
activity_id=info.activity_id,
|
|
98
|
+
attempt=info.attempt,
|
|
99
|
+
task_queue=info.task_queue,
|
|
100
|
+
outcome=outcome,
|
|
101
|
+
error_type=error_type,
|
|
102
|
+
started_at=_timestamp(started),
|
|
103
|
+
ended_at=_timestamp(ended),
|
|
104
|
+
duration_ms=round((ended - started) * 1000),
|
|
105
|
+
start_to_close_timeout_ms=_ms(info.start_to_close_timeout),
|
|
106
|
+
maximum_attempts=(policy.maximum_attempts if policy is not None else 0) or 0,
|
|
107
|
+
heartbeat_timeout_ms=_ms(info.heartbeat_timeout),
|
|
108
|
+
)
|
|
109
|
+
if scheduled is not None:
|
|
110
|
+
attempt.scheduled_at.FromDatetime(scheduled)
|
|
111
|
+
sink.record(attempt)
|
|
112
|
+
except Exception: # noqa: BLE001
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class _ActivityInbound(ActivityInboundInterceptor):
|
|
117
|
+
def __init__(self, next: ActivityInboundInterceptor, owner: Interceptor) -> None:
|
|
118
|
+
super().__init__(next)
|
|
119
|
+
self._owner = owner
|
|
120
|
+
|
|
121
|
+
def init(self, outbound: ActivityOutboundInterceptor) -> None:
|
|
122
|
+
super().init(_ActivityOutbound(outbound))
|
|
123
|
+
|
|
124
|
+
async def execute_activity(self, input: ExecuteActivityInput) -> Any:
|
|
125
|
+
started = time.time()
|
|
126
|
+
try:
|
|
127
|
+
result = await super().execute_activity(input)
|
|
128
|
+
except BaseException as err:
|
|
129
|
+
self._owner._emit(started, *_classify(err))
|
|
130
|
+
raise
|
|
131
|
+
self._owner._emit(started, OUTCOME_COMPLETED, "")
|
|
132
|
+
return result
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class _ActivityOutbound(ActivityOutboundInterceptor):
|
|
136
|
+
pass
|