runsignal 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
runsignal/__init__.py ADDED
@@ -0,0 +1,53 @@
1
+ """
2
+ RunSignal records runs, items and heartbeats of Python background work (Celery tasks, APScheduler jobs, cron
3
+ scripts, queue consumers, sync loops) as OpenTelemetry metrics and spans, so you can alert on work that no HTTP
4
+ status code describes: missed, failed, overrunning, stalled or low-output jobs.
5
+
6
+ The metric names, attributes and manifest format are the same in every RunSignal SDK, so one set of alert rules
7
+ covers .NET, Go, Java, Node.js, Python and Rust services.
8
+
9
+ from runsignal import default_registry
10
+
11
+ invoices = default_registry.workload("nightly-invoice", schedule="0 2 * * *", max_duration="30m",
12
+ min_processed=100, severity="critical")
13
+
14
+ with invoices.run():
15
+ for order in unbilled_orders():
16
+ create_invoice(order)
17
+ invoices.processed()
18
+ """
19
+
20
+ from ._naming import InvalidWorkloadNameError, to_workload_name
21
+ from ._registry import (
22
+ Duration,
23
+ Expectations,
24
+ Run,
25
+ Severity,
26
+ Workload,
27
+ WorkloadConfigurationError,
28
+ WorkloadRegistry,
29
+ default_registry,
30
+ monitor,
31
+ )
32
+ from ._telemetry import DURATION_BUCKETS, SCOPE_NAME, VERSION, Attributes, Metrics
33
+
34
+ __version__ = VERSION
35
+
36
+ __all__ = [
37
+ "Attributes",
38
+ "DURATION_BUCKETS",
39
+ "Duration",
40
+ "Expectations",
41
+ "InvalidWorkloadNameError",
42
+ "Metrics",
43
+ "Run",
44
+ "SCOPE_NAME",
45
+ "Severity",
46
+ "VERSION",
47
+ "Workload",
48
+ "WorkloadConfigurationError",
49
+ "WorkloadRegistry",
50
+ "default_registry",
51
+ "monitor",
52
+ "to_workload_name",
53
+ ]
runsignal/_naming.py ADDED
@@ -0,0 +1,73 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ import unicodedata
5
+
6
+ _NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$")
7
+
8
+ # Letters that do not decompose into a base letter and a combining mark.
9
+ _TRANSLITERATIONS = {
10
+ "ı": "i", "ß": "ss", "ø": "o", "Ø": "O", "æ": "ae", "Æ": "Ae", "đ": "d", "Đ": "D", "ł": "l", "Ł": "L",
11
+ }
12
+
13
+
14
+ class InvalidWorkloadNameError(ValueError):
15
+ """Raised for workload names that are not 1-128 lowercase letters, digits, '.', '_' or '-'."""
16
+
17
+
18
+ def validate_name(name: str) -> None:
19
+ if not isinstance(name, str) or not _NAME_PATTERN.match(name):
20
+ raise InvalidWorkloadNameError(
21
+ f"Workload name '{name}' is invalid. Use 1-128 lowercase letters, digits, '.', '_' or '-', "
22
+ "starting with a letter or digit."
23
+ )
24
+
25
+
26
+ def _is_lower(c: str | None) -> bool:
27
+ return c is not None and "a" <= c <= "z"
28
+
29
+
30
+ def _is_upper(c: str | None) -> bool:
31
+ return c is not None and "A" <= c <= "Z"
32
+
33
+
34
+ def _is_digit(c: str | None) -> bool:
35
+ return c is not None and "0" <= c <= "9"
36
+
37
+
38
+ def to_workload_name(identifier: str) -> str:
39
+ """
40
+ Turns an arbitrary job identifier, such as ``InvoiceJobs.GenerateInvoices`` or ``Reports/Daily Sales``, into a
41
+ valid workload name such as ``invoice-jobs.generate-invoices`` or ``reports-daily-sales``. Every RunSignal SDK
42
+ derives names the same way (spec/conformance/workload-names.json).
43
+ """
44
+ if not identifier.strip():
45
+ raise InvalidWorkloadNameError("The identifier is empty.")
46
+
47
+ # "e-Arşiv gönderimi" -> "e-Arsiv gonderimi", so names stay readable instead of losing letters.
48
+ decomposed = unicodedata.normalize("NFD", identifier)
49
+ chars = [_TRANSLITERATIONS.get(c, c) for c in decomposed if unicodedata.category(c) != "Mn"]
50
+ chars = list("".join(chars))
51
+
52
+ out: list[str] = []
53
+ for i, c in enumerate(chars):
54
+ if _is_upper(c):
55
+ # Break camel case: "GenerateInvoices" -> "generate-invoices", "HTTPSync" -> "http-sync".
56
+ prev = chars[i - 1] if i > 0 else None
57
+ nxt = chars[i + 1] if i + 1 < len(chars) else None
58
+ if out and out[-1] not in "-._" and (
59
+ _is_lower(prev) or _is_digit(prev) or (_is_upper(prev) and _is_lower(nxt))
60
+ ):
61
+ out.append("-")
62
+ out.append(c.lower())
63
+ elif _is_lower(c) or _is_digit(c) or c in "._":
64
+ out.append(c)
65
+ elif out and out[-1] != "-":
66
+ out.append("-")
67
+
68
+ name = "".join(out).strip("-._")
69
+ if len(name) > 128:
70
+ name = name[:128].rstrip("-._")
71
+ if not name:
72
+ raise InvalidWorkloadNameError(f"'{identifier}' contains no letters or digits to name a workload after.")
73
+ return name
runsignal/_registry.py ADDED
@@ -0,0 +1,442 @@
1
+ from __future__ import annotations
2
+
3
+ import functools
4
+ import inspect
5
+ import json
6
+ import os
7
+ import re
8
+ import sys
9
+ import threading
10
+ import time
11
+ from datetime import timedelta
12
+ from typing import Any, Callable, Iterable, Literal, TypeVar, Union
13
+
14
+ from opentelemetry import context, metrics, trace
15
+ from opentelemetry.metrics import CallbackOptions, MeterProvider, Observation
16
+ from opentelemetry.trace import Span, SpanKind, Status, StatusCode, TracerProvider
17
+
18
+ from ._naming import to_workload_name, validate_name
19
+ from ._telemetry import DURATION_BUCKETS, SCOPE_NAME, VERSION, Attributes, Metrics
20
+
21
+ Severity = Literal["info", "warning", "critical"]
22
+ """How urgent a workload's alerts are; it routes notifications."""
23
+
24
+ Duration = Union[float, int, timedelta, str]
25
+ """Seconds, a ``timedelta``, or a string such as ``'90s'``, ``'30m'``, ``'2h'`` or ``'1d'``."""
26
+
27
+ F = TypeVar("F", bound=Callable[..., Any])
28
+
29
+ _SEVERITIES = ("info", "warning", "critical")
30
+ _UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400}
31
+
32
+
33
+ class WorkloadConfigurationError(ValueError):
34
+ """Raised when a workload's options are invalid or declared twice."""
35
+
36
+
37
+ def _to_seconds(value: Duration, option: str) -> float:
38
+ if isinstance(value, timedelta):
39
+ seconds = value.total_seconds()
40
+ elif isinstance(value, str):
41
+ match = re.fullmatch(r"(\d+(?:\.\d+)?)([smhd])", value.strip())
42
+ if not match:
43
+ raise WorkloadConfigurationError(
44
+ f"{option} '{value}' is not a duration. Use seconds, a timedelta or a number with s, m, h or d, such as '30m'."
45
+ )
46
+ seconds = float(match.group(1)) * _UNIT_SECONDS[match.group(2)]
47
+ elif isinstance(value, (int, float)) and not isinstance(value, bool):
48
+ seconds = float(value)
49
+ else:
50
+ raise WorkloadConfigurationError(f"{option} must be seconds, a timedelta or a string such as '30m'.")
51
+ if not seconds > 0 or seconds == float("inf"):
52
+ raise WorkloadConfigurationError(f"{option} must be positive.")
53
+ return seconds
54
+
55
+
56
+ class Expectations:
57
+ """The validated expectations of a workload, with durations in seconds. They define the alerts RunSignal
58
+ provisions and do not change how signals are recorded."""
59
+
60
+ __slots__ = ("schedule", "max_duration", "expect_every", "min_processed", "failure_rate_threshold", "severity")
61
+
62
+ def __init__(
63
+ self,
64
+ schedule: str | None = None,
65
+ max_duration: Duration | None = None,
66
+ expect_every: Duration | None = None,
67
+ min_processed: int | None = None,
68
+ failure_rate_threshold: float | None = None,
69
+ severity: Severity | None = None,
70
+ ) -> None:
71
+ if failure_rate_threshold is not None and not 0 <= failure_rate_threshold <= 1:
72
+ raise WorkloadConfigurationError(f"failure_rate_threshold must be between 0 and 1, got {failure_rate_threshold}.")
73
+ if min_processed is not None and (not isinstance(min_processed, int) or isinstance(min_processed, bool) or min_processed < 0):
74
+ raise WorkloadConfigurationError("min_processed must be a non-negative integer.")
75
+ severity = severity or "warning"
76
+ if severity not in _SEVERITIES:
77
+ raise WorkloadConfigurationError(f"Unknown severity '{severity}'.")
78
+ self.schedule = schedule
79
+ self.max_duration = None if max_duration is None else _to_seconds(max_duration, "max_duration")
80
+ self.expect_every = None if expect_every is None else _to_seconds(expect_every, "expect_every")
81
+ self.min_processed = min_processed
82
+ self.failure_rate_threshold = failure_rate_threshold
83
+ self.severity: Severity = severity
84
+
85
+
86
+ class _Instruments:
87
+ def __init__(self, registry: WorkloadRegistry, provider: MeterProvider) -> None:
88
+ meter = provider.get_meter(SCOPE_NAME, VERSION)
89
+ self.runs = meter.create_counter(Metrics.RUNS, unit="{run}", description="Completed runs of a workload.")
90
+ self.duration = meter.create_histogram(
91
+ Metrics.DURATION,
92
+ unit="s",
93
+ description="Duration of workload runs.",
94
+ explicit_bucket_boundaries_advisory=list(DURATION_BUCKETS),
95
+ )
96
+ self.items = meter.create_counter(Metrics.ITEMS, unit="{item}", description="Items handled by workload runs.")
97
+ self.heartbeats = meter.create_counter(
98
+ Metrics.HEARTBEATS, unit="{heartbeat}", description="Heartbeats of long-running workloads."
99
+ )
100
+ meter.create_observable_gauge(
101
+ Metrics.SINCE_SUCCESS,
102
+ callbacks=[registry._observe_since_success],
103
+ unit="s",
104
+ description="Seconds since the last successful run of a workload, or since it was registered.",
105
+ )
106
+ meter.create_observable_gauge(
107
+ Metrics.SINCE_HEARTBEAT,
108
+ callbacks=[registry._observe_since_heartbeat],
109
+ unit="s",
110
+ description="Seconds since the last heartbeat of a workload, or since it was registered.",
111
+ )
112
+
113
+
114
+ class WorkloadRegistry:
115
+ """Creates and holds the workloads of an application and the instruments they record on.
116
+
117
+ By default it records on the global OpenTelemetry providers. They forward to the SDK once it is configured,
118
+ so a registry created by an import before the SDK starts still records.
119
+ """
120
+
121
+ def __init__(
122
+ self,
123
+ *,
124
+ meter_provider: MeterProvider | None = None,
125
+ tracer_provider: TracerProvider | None = None,
126
+ tracing: bool = True,
127
+ ) -> None:
128
+ self._workloads: dict[str, Workload] = {}
129
+ self._lock = threading.Lock()
130
+ self._instruments = _Instruments(self, meter_provider or metrics.get_meter_provider())
131
+ self._tracer = (tracer_provider or trace.get_tracer_provider()).get_tracer(SCOPE_NAME, VERSION) if tracing else None
132
+
133
+ def workload(
134
+ self,
135
+ name: str,
136
+ *,
137
+ schedule: str | None = None,
138
+ max_duration: Duration | None = None,
139
+ expect_every: Duration | None = None,
140
+ min_processed: int | None = None,
141
+ failure_rate_threshold: float | None = None,
142
+ severity: Severity | None = None,
143
+ ) -> Workload:
144
+ """Returns the workload called ``name``, creating it with the given expectations on first use. Call it once
145
+ and keep the result. Passing expectations for a workload that already exists raises, so expectations are
146
+ declared in exactly one place."""
147
+ options = dict(schedule=schedule, max_duration=max_duration, expect_every=expect_every, min_processed=min_processed,
148
+ failure_rate_threshold=failure_rate_threshold, severity=severity)
149
+ has_options = any(v is not None for v in options.values())
150
+ with self._lock:
151
+ existing = self._workloads.get(name)
152
+ if existing is not None:
153
+ if has_options:
154
+ raise WorkloadConfigurationError(
155
+ f"Workload '{name}' is already registered; set its options only where it is first created."
156
+ )
157
+ return existing
158
+ return self._add(name, Expectations(**options))
159
+
160
+ def get_or_add(self, name: str, **options: Any) -> Workload:
161
+ """Returns the workload called ``name``, creating it with ``options`` if it does not exist yet. Unlike
162
+ ``workload``, an existing workload is returned as is: options declared elsewhere take precedence. Scheduler
163
+ integrations use it to name workloads after jobs."""
164
+ with self._lock:
165
+ existing = self._workloads.get(name)
166
+ if existing is not None:
167
+ return existing
168
+ return self._add(name, Expectations(**options))
169
+
170
+ def _add(self, name: str, expectations: Expectations) -> Workload:
171
+ validate_name(name)
172
+ created = Workload(self, name, expectations)
173
+ self._workloads[name] = created
174
+ return created
175
+
176
+ def monitor(self, name: str, **options: Any) -> Callable[[F], F]:
177
+ """Decorates a function, sync or async, so every call is a run of the workload ``name``, declared with
178
+ ``options``. The decorated function has a ``workload`` attribute, for counting items."""
179
+ workload = self.workload(name, **options)
180
+ return workload.monitor
181
+
182
+ @property
183
+ def workloads(self) -> list[Workload]:
184
+ """Every workload created so far, sorted by name."""
185
+ with self._lock:
186
+ return sorted(self._workloads.values(), key=lambda w: w.name)
187
+
188
+ def manifest(self, service: str) -> dict[str, Any]:
189
+ """Describes every workload, for alert provisioning. ``service`` must match the OpenTelemetry ``service.name``."""
190
+ entries = []
191
+ for w in self.workloads:
192
+ e = w.expectations
193
+ entry: dict[str, Any] = {"name": w.name}
194
+ if e.schedule is not None:
195
+ entry["schedule"] = e.schedule
196
+ if e.max_duration is not None:
197
+ entry["maxDurationSeconds"] = _number(e.max_duration)
198
+ if e.expect_every is not None:
199
+ entry["expectEverySeconds"] = _number(e.expect_every)
200
+ if e.min_processed:
201
+ entry["minProcessed"] = e.min_processed
202
+ if e.failure_rate_threshold:
203
+ entry["failureRateThreshold"] = _number(e.failure_rate_threshold)
204
+ entry["severity"] = e.severity
205
+ entries.append(entry)
206
+ return {"version": 1, "service": service, "workloads": entries}
207
+
208
+ def write_manifest(self, path: str, service: str) -> None:
209
+ """Writes the manifest as indented JSON, atomically, so a reader never sees half a manifest."""
210
+ directory = os.path.dirname(path)
211
+ if directory:
212
+ os.makedirs(directory, exist_ok=True)
213
+ with open(path + ".tmp", "w", encoding="utf-8") as f:
214
+ f.write(json.dumps(self.manifest(service), indent=2, ensure_ascii=False))
215
+ os.replace(path + ".tmp", path)
216
+
217
+ def write_manifest_from_env(self, service: str | None = None) -> str | None:
218
+ """Writes the manifest if the RUNSIGNAL_MANIFEST_PATH environment variable is set, so CI can ask for one
219
+ without code changes. Call it once every workload has been registered. The service name is ``service``, or
220
+ else OTEL_SERVICE_NAME, or else the script name. Returns the path written, or None."""
221
+ path = os.environ.get("RUNSIGNAL_MANIFEST_PATH", "").strip()
222
+ if not path:
223
+ return None
224
+ script = os.path.splitext(os.path.basename(sys.argv[0] if sys.argv and sys.argv[0] else "python"))[0]
225
+ self.write_manifest(path, service or os.environ.get("OTEL_SERVICE_NAME") or script)
226
+ return path
227
+
228
+ def _observe_since_success(self, options: CallbackOptions) -> Iterable[Observation]:
229
+ now = time.monotonic()
230
+ for w in self.workloads:
231
+ seconds = w._seconds_since_success(now)
232
+ if seconds is not None:
233
+ yield Observation(seconds, w._name_attributes)
234
+
235
+ def _observe_since_heartbeat(self, options: CallbackOptions) -> Iterable[Observation]:
236
+ now = time.monotonic()
237
+ for w in self.workloads:
238
+ seconds = w._seconds_since_heartbeat(now)
239
+ if seconds is not None:
240
+ yield Observation(seconds, w._name_attributes)
241
+
242
+
243
+ def _number(value: float) -> float | int:
244
+ return int(value) if float(value).is_integer() else value
245
+
246
+
247
+ class Workload:
248
+ """A unit of background work, such as a scheduled job, a queue consumer or a sync loop. Get one from a
249
+ WorkloadRegistry once and keep it."""
250
+
251
+ def __init__(self, registry: WorkloadRegistry, name: str, expectations: Expectations) -> None:
252
+ self._registry = registry
253
+ self.name = name
254
+ self.expectations = expectations
255
+ self._name_attributes = {Attributes.WORKLOAD: name}
256
+ self._outcome_attributes = {
257
+ outcome: {Attributes.WORKLOAD: name, Attributes.OUTCOME: outcome} for outcome in ("success", "failure", "cancelled")
258
+ }
259
+ self._processed = {Attributes.WORKLOAD: name, Attributes.ITEM_RESULT: "processed"}
260
+ self._failed = {Attributes.WORKLOAD: name, Attributes.ITEM_RESULT: "failed"}
261
+ # Until the first success or heartbeat these hold the time the workload was created, so a workload that
262
+ # never succeeds or never beats still reports a growing age.
263
+ self._last_success = self._last_heartbeat = time.monotonic()
264
+ self._has_started = False
265
+ self._has_beaten = False
266
+
267
+ def start(self) -> Run:
268
+ """Starts a run. End it with exactly one call to ``succeed``, ``fail``, ``fail_code`` or ``cancel``; later
269
+ calls are ignored. Prefer ``run()`` as a context manager, which ends the run for you."""
270
+ self._has_started = True
271
+ tracer = self._registry._tracer
272
+ span = tracer.start_span(self.name, kind=SpanKind.INTERNAL, attributes=self._name_attributes) if tracer else None
273
+ return Run(self, time.monotonic(), span)
274
+
275
+ def run(self) -> Run:
276
+ """A run to use as a context manager, ``with`` or ``async with``. It succeeds when the block completes,
277
+ is cancelled on ``asyncio.CancelledError``, ``KeyboardInterrupt`` or ``SystemExit``, and fails on any other
278
+ exception, which propagates. The run's span is current inside the block."""
279
+ return self.start()
280
+
281
+ def monitor(self, fn: F) -> F:
282
+ """Decorates a function, sync or async, so every call is a run of this workload."""
283
+ if inspect.iscoroutinefunction(fn):
284
+ @functools.wraps(fn)
285
+ async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
286
+ with self.run():
287
+ return await fn(*args, **kwargs)
288
+ async_wrapper.workload = self # type: ignore[attr-defined]
289
+ return async_wrapper # type: ignore[return-value]
290
+
291
+ @functools.wraps(fn)
292
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
293
+ with self.run():
294
+ return fn(*args, **kwargs)
295
+ wrapper.workload = self # type: ignore[attr-defined]
296
+ return wrapper # type: ignore[return-value]
297
+
298
+ def processed(self, count: int = 1) -> None:
299
+ """Records ``count`` successfully processed items. Recording 0 is useful: it reports that a run did nothing."""
300
+ if count >= 0:
301
+ self._registry._instruments.items.add(count, self._processed)
302
+
303
+ def item_failed(self, count: int = 1) -> None:
304
+ """Records ``count`` items that could not be processed."""
305
+ if count >= 0:
306
+ self._registry._instruments.items.add(count, self._failed)
307
+
308
+ def beat(self) -> None:
309
+ """Records a heartbeat: the workload is alive and making progress."""
310
+ self._last_heartbeat = time.monotonic()
311
+ self._has_beaten = True
312
+ self._registry._instruments.heartbeats.add(1, self._name_attributes)
313
+
314
+ def _complete(self, start: float, span: Span | None, outcome: str, error_type: str | None = None,
315
+ error: BaseException | None = None, description: str | None = None) -> None:
316
+ now = time.monotonic()
317
+ instruments = self._registry._instruments
318
+ if error_type is None:
319
+ instruments.runs.add(1, self._outcome_attributes[outcome])
320
+ else:
321
+ instruments.runs.add(1, {**self._outcome_attributes["failure"], Attributes.ERROR_TYPE: error_type})
322
+ instruments.duration.record(now - start, self._outcome_attributes[outcome])
323
+ if outcome == "success":
324
+ self._last_success = now
325
+
326
+ if span is not None:
327
+ if span.is_recording():
328
+ span.set_attribute(Attributes.OUTCOME, outcome)
329
+ if outcome == "failure":
330
+ span.set_attribute(Attributes.ERROR_TYPE, error_type or "unknown")
331
+ if error is not None:
332
+ span.record_exception(error)
333
+ span.set_status(Status(StatusCode.ERROR, description))
334
+ elif outcome == "success":
335
+ span.set_status(Status(StatusCode.OK))
336
+ span.end()
337
+
338
+ # Reported for workloads that are scheduled or have run, so alerts can tell a missed run from a workload that
339
+ # simply has no runs.
340
+ def _seconds_since_success(self, now: float) -> float | None:
341
+ if self._has_started or self.expectations.schedule is not None:
342
+ return now - self._last_success
343
+ return None
344
+
345
+ # Reported for workloads that expect heartbeats or have sent one.
346
+ def _seconds_since_heartbeat(self, now: float) -> float | None:
347
+ if self._has_beaten or self.expectations.expect_every is not None:
348
+ return now - self._last_heartbeat
349
+ return None
350
+
351
+ def __repr__(self) -> str:
352
+ return f"Workload({self.name!r})"
353
+
354
+
355
+ class Run:
356
+ """One run of a Workload, from ``Workload.start`` or ``Workload.run``."""
357
+
358
+ __slots__ = ("workload", "span", "_started", "_ended", "_token")
359
+
360
+ def __init__(self, workload: Workload, started: float, span: Span | None) -> None:
361
+ self.workload = workload
362
+ self.span = span
363
+ self._started = started
364
+ self._ended = False
365
+ self._token: object | None = None
366
+
367
+ @property
368
+ def elapsed(self) -> float:
369
+ """Seconds since the run started."""
370
+ return time.monotonic() - self._started
371
+
372
+ def succeed(self) -> None:
373
+ """Ends the run as successful."""
374
+ if self._end():
375
+ self.workload._complete(self._started, self.span, "success")
376
+
377
+ def fail(self, error: BaseException) -> None:
378
+ """Ends the run as failed because of ``error``. Its ``error.type`` is the exception's ``error_type``
379
+ attribute if it has one, otherwise its fully qualified class name."""
380
+ if self._end():
381
+ self.workload._complete(self._started, self.span, "failure", _error_type(error), error, str(error) or None)
382
+
383
+ def fail_code(self, code: str, description: str | None = None) -> None:
384
+ """Ends the run as failed without an exception, for example when an expected input file is missing.
385
+ ``code`` must be a short, stable value such as ``'statement_missing'``; never put messages or IDs in it."""
386
+ if self._end():
387
+ self.workload._complete(self._started, self.span, "failure", code or "unknown", None, description)
388
+
389
+ def cancel(self) -> None:
390
+ """Ends the run as cancelled, for example on shutdown. Cancelled runs do not count as failures."""
391
+ if self._end():
392
+ self.workload._complete(self._started, self.span, "cancelled")
393
+
394
+ def _end(self) -> bool:
395
+ if self._ended:
396
+ return False
397
+ self._ended = True
398
+ return True
399
+
400
+ def __enter__(self) -> Run:
401
+ if self.span is not None:
402
+ self._token = context.attach(trace.set_span_in_context(self.span))
403
+ return self
404
+
405
+ def __exit__(self, exc_type: Any, exc: BaseException | None, tb: Any) -> Literal[False]:
406
+ if self._token is not None:
407
+ context.detach(self._token) # type: ignore[arg-type]
408
+ self._token = None
409
+ if exc is None:
410
+ self.succeed()
411
+ elif isinstance(exc, Exception):
412
+ self.fail(exc)
413
+ else:
414
+ # asyncio.CancelledError, KeyboardInterrupt, SystemExit: the run was stopped, not broken.
415
+ self.cancel()
416
+ return False
417
+
418
+ async def __aenter__(self) -> Run:
419
+ return self.__enter__()
420
+
421
+ async def __aexit__(self, exc_type: Any, exc: BaseException | None, tb: Any) -> Literal[False]:
422
+ return self.__exit__(exc_type, exc, tb)
423
+
424
+
425
+ def _error_type(error: BaseException) -> str:
426
+ error_type = getattr(error, "error_type", None)
427
+ if isinstance(error_type, str) and error_type:
428
+ return error_type
429
+ cls = type(error)
430
+ return cls.__qualname__ if cls.__module__ == "builtins" else f"{cls.__module__}.{cls.__qualname__}"
431
+
432
+
433
+ default_registry = WorkloadRegistry()
434
+ """A process-wide registry on the global OpenTelemetry providers."""
435
+
436
+
437
+ def monitor(name: str, **options: Any) -> Callable[[F], F]:
438
+ """Decorates a function, sync or async, so every call is a run of the workload ``name`` in the default
439
+ registry, declared with ``options`` (``schedule``, ``max_duration``, ``expect_every``, ``min_processed``,
440
+ ``failure_rate_threshold``, ``severity``)."""
441
+ return default_registry.monitor(name, **options)
442
+
@@ -0,0 +1,30 @@
1
+ """
2
+ Names of the meter, tracer, instruments and attributes RunSignal emits. They are identical in every RunSignal SDK
3
+ (see spec/README.md in the repository), so one set of alert rules covers every language.
4
+ """
5
+
6
+ SCOPE_NAME = "RunSignal"
7
+ VERSION = "0.2.0"
8
+
9
+
10
+ class Metrics:
11
+ RUNS = "runsignal.workload.runs"
12
+ DURATION = "runsignal.workload.duration"
13
+ ITEMS = "runsignal.workload.items"
14
+ HEARTBEATS = "runsignal.workload.heartbeats"
15
+ SINCE_SUCCESS = "runsignal.workload.since_success"
16
+ SINCE_HEARTBEAT = "runsignal.workload.since_heartbeat"
17
+
18
+
19
+ class Attributes:
20
+ WORKLOAD = "runsignal.workload"
21
+ OUTCOME = "runsignal.outcome"
22
+ ITEM_RESULT = "runsignal.item.result"
23
+ ERROR_TYPE = "error.type"
24
+
25
+
26
+ # Histogram bucket boundaries, in seconds, for the duration metric: background work runs from milliseconds to
27
+ # hours, while the OpenTelemetry defaults stop at 10 seconds.
28
+ DURATION_BUCKETS = (
29
+ 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 120, 300, 600, 900, 1800, 3600, 7200, 14400, 28800,
30
+ )
@@ -0,0 +1,129 @@
1
+ """
2
+ APScheduler 3 integration: records every job execution as a run of its workload, with no code in your jobs, and
3
+ registers cron-triggered jobs with their schedules, so a job that never fires still gets a missed alert.
4
+
5
+ from runsignal.apscheduler import instrument
6
+
7
+ scheduler = BackgroundScheduler()
8
+ scheduler.add_job(generate_invoices, "cron", hour=2, id="nightly-invoice")
9
+ instrument(scheduler, workloads={"nightly-invoice": {"max_duration": "30m", "severity": "critical"}})
10
+ scheduler.start()
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import re
16
+ import warnings
17
+ from typing import Any, Mapping
18
+
19
+ from apscheduler.events import EVENT_JOB_ADDED, JobEvent
20
+ from apscheduler.job import Job
21
+ from apscheduler.schedulers.base import BaseScheduler
22
+ from apscheduler.triggers.cron import CronTrigger
23
+
24
+ from ._naming import to_workload_name
25
+ from ._registry import Workload, WorkloadRegistry, default_registry
26
+
27
+ __all__ = ["instrument", "cron_of", "workload_name_of"]
28
+
29
+ _GENERATED_ID = re.compile(r"^[0-9a-f]{32}$")
30
+ _DAYS = {"mon": 1, "tue": 2, "wed": 3, "thu": 4, "fri": 5, "sat": 6, "sun": 7}
31
+ _MARK = "__runsignal_workload__"
32
+
33
+
34
+ def workload_name_of(job: Job) -> str:
35
+ """The workload a job runs as: its ``id``, or its ``name`` when the id was generated by APScheduler."""
36
+ return to_workload_name(job.name if _GENERATED_ID.match(job.id) else job.id)
37
+
38
+
39
+ def _day_of_week(expression: str) -> str:
40
+ # APScheduler counts days from Monday = 0; Unix cron from Sunday = 0, with 7 also Sunday. Monday..Sunday become
41
+ # 1..7, so ranges such as sat-sun stay ascending.
42
+ def value(v: str) -> str:
43
+ v = v.lower()
44
+ if v in _DAYS:
45
+ return str(_DAYS[v])
46
+ if v.isdigit():
47
+ return str(int(v) + 1)
48
+ raise ValueError(v)
49
+
50
+ parts = []
51
+ for token in expression.split(","):
52
+ base, _, step = token.partition("/")
53
+ if base == "*":
54
+ base = "1-7" if step else "*"
55
+ else:
56
+ base = "-".join(value(v) for v in base.split("-"))
57
+ parts.append(f"{base}/{step}" if step else base)
58
+ return ",".join(parts)
59
+
60
+
61
+ def cron_of(trigger: CronTrigger) -> str | None:
62
+ """A cron trigger as a cron expression, 5 fields or 6 with seconds first, or None if it uses fields cron
63
+ cannot express (``year``, ``week``, or days such as ``1st mon``)."""
64
+ fields = {f.name: str(f) for f in trigger.fields}
65
+ if fields["year"] != "*" or fields["week"] != "*":
66
+ return None
67
+ day = "L" if fields["day"] == "last" else fields["day"]
68
+ if re.search(r"[a-zA-KM-Z]", day):
69
+ return None
70
+ try:
71
+ day_of_week = _day_of_week(fields["day_of_week"])
72
+ except ValueError:
73
+ return None
74
+ cron = f"{fields['minute']} {fields['hour']} {day} {fields['month']} {day_of_week}"
75
+ return cron if fields["second"] == "0" else f"{fields['second']} {cron}"
76
+
77
+
78
+ class _Instrumentation:
79
+ def __init__(self, scheduler: BaseScheduler, registry: WorkloadRegistry, workloads: Mapping[str, Mapping[str, Any]]) -> None:
80
+ self.scheduler = scheduler
81
+ self.registry = registry
82
+ self.workloads = workloads
83
+
84
+ def register(self, job: Job) -> Workload:
85
+ name = workload_name_of(job)
86
+ options = dict(self.workloads.get(name, {}))
87
+ if isinstance(job.trigger, CronTrigger) and "schedule" not in options:
88
+ cron = cron_of(job.trigger)
89
+ if cron is not None:
90
+ options["schedule"] = cron
91
+ else:
92
+ warnings.warn(f"runsignal: the trigger of job '{job.id}' cannot be written as cron, so it gets no missed alert.",
93
+ stacklevel=3)
94
+ workload = self.registry.get_or_add(name, **options)
95
+
96
+ if getattr(job.func, _MARK, None) is None:
97
+ wrapped = workload.monitor(job.func)
98
+ setattr(wrapped, _MARK, workload)
99
+ try:
100
+ self.scheduler.modify_job(job.id, getattr(job, "_jobstore_alias", None), func=wrapped)
101
+ except Exception as error: # a persistent job store cannot store a wrapper it cannot import
102
+ warnings.warn(f"runsignal: could not instrument job '{job.id}' ({error}). Decorate its function with "
103
+ f"@registry.monitor('{name}') instead.", stacklevel=3)
104
+ return workload
105
+
106
+ def on_added(self, event: JobEvent) -> None:
107
+ job = self.scheduler.get_job(event.job_id, event.jobstore)
108
+ if job is not None:
109
+ self.register(job)
110
+
111
+
112
+ def instrument(
113
+ scheduler: BaseScheduler,
114
+ *,
115
+ registry: WorkloadRegistry | None = None,
116
+ workloads: Mapping[str, Mapping[str, Any]] | None = None,
117
+ ) -> list[Workload]:
118
+ """Records every run of every job of ``scheduler``, now and added later: success, failure with the exception's
119
+ type, and duration. Cron-triggered jobs are registered with their schedule. ``workloads`` maps workload names
120
+ to expectations, such as ``{"nightly-invoice": {"max_duration": "30m"}}``. Time zones are not carried over:
121
+ pass ``--time-zone`` to the CLI if the triggers use one.
122
+
123
+ Jobs in a persistent job store (SQLAlchemy, Redis, MongoDB) cannot hold a wrapped function; decorate their
124
+ functions with ``@registry.monitor(name)`` instead, and this still registers their schedules.
125
+ """
126
+ handler = _Instrumentation(scheduler, registry or default_registry, workloads or {})
127
+ found = [handler.register(job) for job in scheduler.get_jobs()]
128
+ scheduler.add_listener(handler.on_added, EVENT_JOB_ADDED)
129
+ return found
runsignal/celery.py ADDED
@@ -0,0 +1,176 @@
1
+ """
2
+ Celery integration: records every task execution as a run of its workload, with no code in your tasks, and
3
+ registers Celery beat's crontab entries as workloads with their schedules, so a periodic task that never runs
4
+ still gets a missed alert.
5
+
6
+ from runsignal.celery import instrument
7
+
8
+ app = Celery("billing")
9
+ app.conf.beat_schedule = {"nightly-invoice": {"task": "billing.tasks.generate_invoices",
10
+ "schedule": crontab(hour=2, minute=0)}}
11
+ instrument(app, workloads={"billing.tasks.generate_invoices": {"max_duration": "30m", "severity": "critical"}})
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import os
17
+ import threading
18
+ from typing import Any, Mapping
19
+
20
+ from celery import Celery, Task, signals
21
+ from celery.schedules import crontab
22
+
23
+ from ._naming import to_workload_name
24
+ from ._registry import Run, Workload, WorkloadRegistry, default_registry
25
+
26
+ __all__ = ["instrument", "discover_beat_schedule", "workload_name_of", "workload_of", "beat_cron"]
27
+
28
+
29
+ def workload_name_of(task: Task | str) -> str:
30
+ """The workload a task runs as: its task name, such as ``billing.tasks.generate_invoices``."""
31
+ return to_workload_name(task if isinstance(task, str) else task.name)
32
+
33
+
34
+ def beat_cron(schedule: crontab) -> str:
35
+ """A Celery ``crontab`` as a 5-field cron expression. Celery numbers days of the week like Unix cron
36
+ (0 = Sunday), so fields carry over unchanged."""
37
+
38
+ def field(value: Any) -> str:
39
+ if isinstance(value, (set, frozenset, list, tuple)):
40
+ return ",".join(str(v) for v in sorted(value))
41
+ return str(value)
42
+
43
+ return " ".join(field(getattr(schedule, f"_orig_{name}")) for name in
44
+ ("minute", "hour", "day_of_month", "month_of_year", "day_of_week"))
45
+
46
+
47
+ def discover_beat_schedule(
48
+ app: Celery,
49
+ *,
50
+ registry: WorkloadRegistry | None = None,
51
+ workloads: Mapping[str, Mapping[str, Any]] | None = None,
52
+ ) -> list[Workload]:
53
+ """Registers a workload for every task in ``app.conf.beat_schedule``, with its crontab as the schedule, so a
54
+ periodic task that never runs is still alerted on. Entries with an interval instead of a crontab get no
55
+ schedule. Time zones are not carried over: pass ``--time-zone`` to the CLI if beat uses one."""
56
+ registry = registry or default_registry
57
+ found: list[Workload] = []
58
+ for entry in (app.conf.beat_schedule or {}).values():
59
+ task = entry.get("task")
60
+ if not task or task.startswith("celery."):
61
+ continue
62
+ name = workload_name_of(task)
63
+ options = dict((workloads or {}).get(name, {}))
64
+ schedule = entry.get("schedule")
65
+ if isinstance(schedule, crontab):
66
+ options.setdefault("schedule", beat_cron(schedule))
67
+ found.append(registry.get_or_add(name, **options))
68
+ return found
69
+
70
+
71
+ _task_workloads: dict[str, Workload] = {}
72
+
73
+
74
+ def workload_of(task: Task) -> Workload | None:
75
+ """The workload a task runs as, for counting items: ``workload_of(self).processed()`` in a bound task."""
76
+ return _task_workloads.get(task.name)
77
+
78
+
79
+ class _Instrumentation:
80
+ def __init__(self, registry: WorkloadRegistry, workloads: Mapping[str, Mapping[str, Any]]) -> None:
81
+ self.registry = registry
82
+ self.workloads = workloads
83
+ self.runs: dict[str, Run] = {}
84
+ self.errors: dict[str, BaseException] = {}
85
+ self.lock = threading.Lock()
86
+
87
+ def workload(self, task: Task) -> Workload | None:
88
+ workload = _task_workloads.get(task.name)
89
+ if workload is None:
90
+ if task.name.startswith("celery."):
91
+ return None
92
+ name = workload_name_of(task)
93
+ workload = self.registry.get_or_add(name, **self.workloads.get(name, {}))
94
+ _task_workloads[task.name] = workload
95
+ return workload
96
+
97
+ def prerun(self, task_id: str = "", task: Task | None = None, **_: Any) -> None:
98
+ workload = self.workload(task) if task is not None else None
99
+ if workload is not None:
100
+ with self.lock:
101
+ self.runs[task_id] = workload.start()
102
+
103
+ def failure(self, task_id: str = "", exception: BaseException | None = None, **_: Any) -> None:
104
+ if exception is not None:
105
+ with self.lock:
106
+ self.errors[task_id] = exception
107
+
108
+ def retry(self, request: Any = None, reason: Any = None, **_: Any) -> None:
109
+ if request is not None and isinstance(reason, BaseException):
110
+ with self.lock:
111
+ self.errors[request.id] = reason
112
+
113
+ def postrun(self, task_id: str = "", state: str | None = None, **_: Any) -> None:
114
+ with self.lock:
115
+ run = self.runs.pop(task_id, None)
116
+ error = self.errors.pop(task_id, None)
117
+ if run is None:
118
+ return
119
+ if state == "SUCCESS":
120
+ run.succeed()
121
+ elif state in ("FAILURE", "RETRY"):
122
+ if error is not None:
123
+ run.fail(error)
124
+ else:
125
+ run.fail_code(state.lower())
126
+ else:
127
+ run.cancel()
128
+
129
+ def revoked(self, request: Any = None, **_: Any) -> None:
130
+ if request is None:
131
+ return
132
+ with self.lock:
133
+ run = self.runs.pop(request.id, None)
134
+ self.errors.pop(request.id, None)
135
+ if run is not None:
136
+ run.cancel()
137
+
138
+
139
+ _instrumented: dict[int, _Instrumentation] = {}
140
+
141
+
142
+ def instrument(
143
+ app: Celery | None = None,
144
+ *,
145
+ registry: WorkloadRegistry | None = None,
146
+ workloads: Mapping[str, Mapping[str, Any]] | None = None,
147
+ write_manifest_on_ready: bool = True,
148
+ ) -> list[Workload]:
149
+ """Records every Celery task execution as a run: success, failure with the exception's type (each retry counts
150
+ as a failed run), and cancellation when the task is revoked or ignored. Celery's own ``celery.*`` tasks are
151
+ skipped. Call it once per process, where the app is configured.
152
+
153
+ With ``app``, it also registers the beat schedule's tasks (``discover_beat_schedule``) and, when
154
+ ``write_manifest_on_ready`` is true, writes the manifest when the worker is ready if RUNSIGNAL_MANIFEST_PATH is
155
+ set. ``workloads`` maps workload names to expectations, such as ``{"billing.tasks.generate_invoices":
156
+ {"max_duration": "30m"}}``. Returns the workloads discovered from the beat schedule.
157
+ """
158
+ registry = registry or default_registry
159
+ workloads = workloads or {}
160
+ if id(registry) not in _instrumented:
161
+ handler = _Instrumentation(registry, workloads)
162
+ _instrumented[id(registry)] = handler
163
+ signals.task_prerun.connect(handler.prerun, weak=False)
164
+ signals.task_failure.connect(handler.failure, weak=False)
165
+ signals.task_retry.connect(handler.retry, weak=False)
166
+ signals.task_postrun.connect(handler.postrun, weak=False)
167
+ signals.task_revoked.connect(handler.revoked, weak=False)
168
+
169
+ if app is None:
170
+ return []
171
+ found = discover_beat_schedule(app, registry=registry, workloads=workloads)
172
+ if write_manifest_on_ready:
173
+ def write_manifest(**_: Any) -> None:
174
+ registry.write_manifest_from_env(os.environ.get("OTEL_SERVICE_NAME") or app.main or None)
175
+ signals.worker_ready.connect(write_manifest, weak=False)
176
+ return found
runsignal/py.typed ADDED
File without changes
@@ -0,0 +1,205 @@
1
+ Metadata-Version: 2.5
2
+ Name: runsignal
3
+ Version: 0.2.0
4
+ Summary: Cron job and background job monitoring for Python with OpenTelemetry: alert when a Celery task, APScheduler job or cron script is missed, fails, overruns, stalls or processes nothing, in New Relic, Prometheus and Grafana.
5
+ Project-URL: Homepage, https://github.com/gokhanertogan/RunSignal
6
+ Project-URL: Documentation, https://github.com/gokhanertogan/RunSignal/tree/main/src/RunSignal.Python#readme
7
+ Project-URL: Repository, https://github.com/gokhanertogan/RunSignal
8
+ Project-URL: Issues, https://github.com/gokhanertogan/RunSignal/issues
9
+ Project-URL: Changelog, https://github.com/gokhanertogan/RunSignal/blob/main/CHANGELOG.md
10
+ Author: Gokhan Ertogan
11
+ License-Expression: MIT
12
+ License-File: LICENSE
13
+ Keywords: alerting,apscheduler,background-jobs,celery,celery-beat,cron,cron-monitoring,dead-mans-switch,grafana,healthcheck,heartbeat,job-monitoring,monitoring,newrelic,observability,opentelemetry,otel,prometheus,scheduled-jobs,worker
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Framework :: Celery
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Intended Audience :: System Administrators
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3 :: Only
21
+ Classifier: Topic :: System :: Logging
22
+ Classifier: Topic :: System :: Monitoring
23
+ Classifier: Typing :: Typed
24
+ Requires-Python: >=3.9
25
+ Requires-Dist: opentelemetry-api>=1.27
26
+ Provides-Extra: apscheduler
27
+ Requires-Dist: apscheduler<4,>=3.10; extra == 'apscheduler'
28
+ Provides-Extra: celery
29
+ Requires-Dist: celery>=5.3; extra == 'celery'
30
+ Description-Content-Type: text/markdown
31
+
32
+ # RunSignal for Python
33
+
34
+ [![PyPI](https://img.shields.io/pypi/v/runsignal)](https://pypi.org/project/runsignal/)
35
+ [![Python versions](https://img.shields.io/pypi/pyversions/runsignal)](https://pypi.org/project/runsignal/)
36
+ [![license](https://img.shields.io/pypi/l/runsignal)](https://github.com/gokhanertogan/RunSignal/blob/main/LICENSE)
37
+
38
+ **Cron job and background job monitoring for Python, on OpenTelemetry.** Get an alert when a Celery task or
39
+ Celery beat schedule, an APScheduler job or a cron script is missed, fails, runs too long or processes nothing,
40
+ and when a consumer or worker loop stops making progress: the failures no HTTP status code describes. No hosted
41
+ service and no ping URLs: the alerts live in the backend you already use.
42
+
43
+ RunSignal records runs, items and heartbeats as OpenTelemetry metrics, with the same names as every other
44
+ RunSignal SDK, so the `runsignal` CLI provisions the same alerts for Python services as for .NET, Go, Java,
45
+ Node.js and Rust ones: New Relic alerts and dashboards, or Prometheus rules and a Grafana dashboard. The
46
+ [main README](https://github.com/gokhanertogan/RunSignal#readme) covers the concepts, the alerts and the
47
+ backends.
48
+
49
+ ## Installation
50
+
51
+ ```bash
52
+ pip install runsignal # core, depends only on opentelemetry-api
53
+ pip install "runsignal[celery]" # with Celery 5.3+
54
+ pip install "runsignal[apscheduler]" # with APScheduler 3.10+
55
+ ```
56
+
57
+ Requires Python 3.9 or later. Install and configure the OpenTelemetry SDK and an exporter as usual
58
+ (`opentelemetry-sdk`, `opentelemetry-exporter-otlp`).
59
+
60
+ ## Quick start
61
+
62
+ ```python
63
+ from runsignal import default_registry
64
+
65
+ invoices = default_registry.workload(
66
+ "nightly-invoice",
67
+ schedule="0 2 * * *",
68
+ max_duration="30m", # or seconds, or a timedelta
69
+ min_processed=100,
70
+ severity="critical",
71
+ )
72
+
73
+ with invoices.run():
74
+ for order in unbilled_orders():
75
+ create_invoice(order)
76
+ invoices.processed()
77
+ ```
78
+
79
+ Or as a decorator, for sync and async functions:
80
+
81
+ ```python
82
+ from runsignal import monitor
83
+
84
+ @monitor("nightly-invoice", schedule="0 2 * * *", max_duration="30m")
85
+ def generate_invoices():
86
+ for order in unbilled_orders():
87
+ create_invoice(order)
88
+ generate_invoices.workload.processed()
89
+ ```
90
+
91
+ `default_registry` records on the global OpenTelemetry providers, which forward to the SDK once it is configured,
92
+ so it works even when it is created by an import first. For explicit providers, use
93
+ `WorkloadRegistry(meter_provider=..., tracer_provider=...)`.
94
+
95
+ ## Recording work
96
+
97
+ | Call | Records |
98
+ |---|---|
99
+ | `with w.run():` / `async with w.run():` | One run. It succeeds when the block completes, is cancelled on `asyncio.CancelledError`, `KeyboardInterrupt` or `SystemExit`, and fails on any other exception, which propagates. The run's span is current inside the block. |
100
+ | `@w.monitor` / `@monitor(name, **expectations)` | One run per call, for sync and async functions. |
101
+ | `run = w.start()` then `run.succeed()` / `fail(exc)` / `fail_code(code, description)` / `cancel()` | A run ended by hand. Later calls are ignored. |
102
+ | `w.processed(n)` / `w.item_failed(n)` | Items handled. |
103
+ | `w.beat()` | A heartbeat from a long-running loop. |
104
+
105
+ `error.type` is the exception's `error_type` attribute if it has one (such as `"statement_missing"`), otherwise
106
+ its fully qualified class name (`ValueError`, `requests.exceptions.ConnectionError`).
107
+
108
+ ## Celery
109
+
110
+ ```python
111
+ from celery import Celery
112
+ from celery.schedules import crontab
113
+ from runsignal.celery import instrument, workload_of
114
+
115
+ app = Celery("billing")
116
+ app.conf.beat_schedule = {
117
+ "nightly-invoice": {"task": "billing.tasks.generate_invoices", "schedule": crontab(hour=2, minute=0)},
118
+ }
119
+ instrument(app, workloads={"billing.tasks.generate_invoices": {"max_duration": "30m", "severity": "critical"}})
120
+
121
+ @app.task(bind=True)
122
+ def generate_invoices(self):
123
+ for order in unbilled_orders():
124
+ create_invoice(order)
125
+ workload_of(self).processed()
126
+ ```
127
+
128
+ - **What is recorded:** every task execution, with no code in your tasks: success, failure with the exception's type (each retry counts as a failed run), cancellation when a task is revoked, and duration. Celery's own `celery.*` tasks are skipped.
129
+ - **Naming:** tasks are named after their task name (`billing.tasks.generate_invoices`).
130
+ - **Celery beat:** `instrument(app)` registers every task in `beat_schedule` with its `crontab` as the schedule, so a periodic task that never runs gets a `missed` alert. Interval schedules get no schedule.
131
+ - **Manifest:** the worker writes it when it is ready if `RUNSIGNAL_MANIFEST_PATH` is set. In CI, import the app and call `default_registry.write_manifest_from_env("billing")` instead; no broker is needed.
132
+ - **Prefork workers:** configure the OpenTelemetry SDK in `worker_process_init`, as for any OpenTelemetry instrumentation of Celery.
133
+
134
+ ## APScheduler
135
+
136
+ ```python
137
+ from apscheduler.schedulers.background import BackgroundScheduler
138
+ from runsignal.apscheduler import instrument
139
+
140
+ scheduler = BackgroundScheduler()
141
+ scheduler.add_job(generate_invoices, "cron", hour=2, minute=0, id="nightly-invoice")
142
+ instrument(scheduler, workloads={"nightly-invoice": {"max_duration": "30m", "severity": "critical"}})
143
+ scheduler.start()
144
+ ```
145
+
146
+ - **What is recorded:** every run of every job, including jobs added after `instrument`, for all schedulers (`BackgroundScheduler`, `BlockingScheduler`, `AsyncIOScheduler`, ...).
147
+ - **Naming:** jobs are named after the `id` you gave them, or their `name` when APScheduler generated the id.
148
+ - **Schedules:** cron triggers become the workload's schedule, with APScheduler's Monday = 0 converted to cron's numbering. Triggers that use `year`, `week` or days such as `1st mon` get no schedule, with a warning.
149
+ - **Persistent job stores** (SQLAlchemy, Redis, MongoDB) cannot store a wrapped function: decorate the job function with `@default_registry.monitor("nightly-invoice")` instead. `instrument` still registers the schedule.
150
+
151
+ ## Cron scripts and loops
152
+
153
+ A script run by cron, a Kubernetes CronJob or systemd timer is one run per invocation:
154
+
155
+ ```python
156
+ with default_registry.workload("nightly-invoice", schedule="0 2 * * *").run():
157
+ main()
158
+ ```
159
+
160
+ The `missed` alert needs a long-running process to report `since_success`, so for short-lived scripts declare
161
+ the workload in the service that exports continuously, or rely on the `failed`, `overrun` and `low output`
162
+ alerts. Long-running loops send heartbeats:
163
+
164
+ ```python
165
+ sync = default_registry.workload("stock-sync", expect_every="5m")
166
+ while True:
167
+ sync_stock()
168
+ sync.beat()
169
+ ```
170
+
171
+ ## Manifest
172
+
173
+ ```python
174
+ default_registry.write_manifest_from_env("billing-service") # writes to RUNSIGNAL_MANIFEST_PATH when it is set
175
+ ```
176
+
177
+ Then provision the alerts from CI with the [`runsignal` CLI](https://github.com/gokhanertogan/RunSignal#cli).
178
+
179
+ ## Exporting
180
+
181
+ **New Relic** needs delta temporality. With cumulative, a stopped workload never looks stopped:
182
+
183
+ ```bash
184
+ export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.eu01.nr-data.net:4317 # US: https://otlp.nr-data.net:4317
185
+ export OTEL_EXPORTER_OTLP_HEADERS=api-key=<your license key>
186
+ export OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=delta
187
+ ```
188
+
189
+ **Prometheus 3** receives OTLP at `/api/v1/otlp/v1/metrics` with the default (cumulative) temporality. Generate
190
+ its rules with `runsignal export prometheus` and a dashboard with `runsignal export grafana`.
191
+
192
+ ## Development
193
+
194
+ ```bash
195
+ python -m venv .venv && .venv/bin/pip install -e ".[celery,apscheduler]" opentelemetry-sdk pytest
196
+ .venv/bin/pytest
197
+ ```
198
+
199
+ The tests include the cross-language conformance vectors in `spec/conformance`.
200
+
201
+ ## Links
202
+
203
+ [Specification](https://github.com/gokhanertogan/RunSignal/blob/main/spec/README.md) ·
204
+ [Changelog](https://github.com/gokhanertogan/RunSignal/blob/main/CHANGELOG.md) ·
205
+ [Other languages](https://github.com/gokhanertogan/RunSignal#quick-start)
@@ -0,0 +1,11 @@
1
+ runsignal/__init__.py,sha256=XDMceKKycUNMyuA9RwSeueBPmlJA-lxauUfqyk64uk4,1534
2
+ runsignal/_naming.py,sha256=qMvvwomBmqlAjhUWfpFDXPfNz7dAonEruc1ZTruJlx4,2722
3
+ runsignal/_registry.py,sha256=1HQ24PtHEQmQPVKonTrDss59QUktgu9w18Xeghak_Ys,20343
4
+ runsignal/_telemetry.py,sha256=BdOfsmlZQt1eYJv9CjB0PLiTn_QbDZF-fD5xaPeJcO4,1011
5
+ runsignal/apscheduler.py,sha256=RazOZ7nerXa_6apw8NwF61ywWHr7aRxb6jQceti1z3k,5516
6
+ runsignal/celery.py,sha256=HpePY-BD6UPjXWoQPfyf4TvB2HcO09OoWoiWdVqXCwk,7255
7
+ runsignal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ runsignal-0.2.0.dist-info/METADATA,sha256=K_bLFcDWgAOmeZpbbnQb382ZGgZlaojrMHAQ7TSRaG0,9539
9
+ runsignal-0.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
10
+ runsignal-0.2.0.dist-info/licenses/LICENSE,sha256=mHu_gwPqV0P9YgCWEaGbMR7u19MI6ac1gJoP1HSTbB4,1071
11
+ runsignal-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Gokhan Ertogan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.