runsignal 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runsignal/__init__.py +53 -0
- runsignal/_naming.py +73 -0
- runsignal/_registry.py +442 -0
- runsignal/_telemetry.py +30 -0
- runsignal/apscheduler.py +129 -0
- runsignal/celery.py +176 -0
- runsignal/py.typed +0 -0
- runsignal-0.2.0.dist-info/METADATA +205 -0
- runsignal-0.2.0.dist-info/RECORD +11 -0
- runsignal-0.2.0.dist-info/WHEEL +4 -0
- runsignal-0.2.0.dist-info/licenses/LICENSE +21 -0
runsignal/__init__.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""
|
|
2
|
+
RunSignal records runs, items and heartbeats of Python background work (Celery tasks, APScheduler jobs, cron
|
|
3
|
+
scripts, queue consumers, sync loops) as OpenTelemetry metrics and spans, so you can alert on work that no HTTP
|
|
4
|
+
status code describes: missed, failed, overrunning, stalled or low-output jobs.
|
|
5
|
+
|
|
6
|
+
The metric names, attributes and manifest format are the same in every RunSignal SDK, so one set of alert rules
|
|
7
|
+
covers .NET, Go, Java, Node.js, Python and Rust services.
|
|
8
|
+
|
|
9
|
+
from runsignal import default_registry
|
|
10
|
+
|
|
11
|
+
invoices = default_registry.workload("nightly-invoice", schedule="0 2 * * *", max_duration="30m",
|
|
12
|
+
min_processed=100, severity="critical")
|
|
13
|
+
|
|
14
|
+
with invoices.run():
|
|
15
|
+
for order in unbilled_orders():
|
|
16
|
+
create_invoice(order)
|
|
17
|
+
invoices.processed()
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from ._naming import InvalidWorkloadNameError, to_workload_name
|
|
21
|
+
from ._registry import (
|
|
22
|
+
Duration,
|
|
23
|
+
Expectations,
|
|
24
|
+
Run,
|
|
25
|
+
Severity,
|
|
26
|
+
Workload,
|
|
27
|
+
WorkloadConfigurationError,
|
|
28
|
+
WorkloadRegistry,
|
|
29
|
+
default_registry,
|
|
30
|
+
monitor,
|
|
31
|
+
)
|
|
32
|
+
from ._telemetry import DURATION_BUCKETS, SCOPE_NAME, VERSION, Attributes, Metrics
|
|
33
|
+
|
|
34
|
+
__version__ = VERSION
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"Attributes",
|
|
38
|
+
"DURATION_BUCKETS",
|
|
39
|
+
"Duration",
|
|
40
|
+
"Expectations",
|
|
41
|
+
"InvalidWorkloadNameError",
|
|
42
|
+
"Metrics",
|
|
43
|
+
"Run",
|
|
44
|
+
"SCOPE_NAME",
|
|
45
|
+
"Severity",
|
|
46
|
+
"VERSION",
|
|
47
|
+
"Workload",
|
|
48
|
+
"WorkloadConfigurationError",
|
|
49
|
+
"WorkloadRegistry",
|
|
50
|
+
"default_registry",
|
|
51
|
+
"monitor",
|
|
52
|
+
"to_workload_name",
|
|
53
|
+
]
|
runsignal/_naming.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import unicodedata
|
|
5
|
+
|
|
6
|
+
_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$")
|
|
7
|
+
|
|
8
|
+
# Letters that do not decompose into a base letter and a combining mark.
|
|
9
|
+
_TRANSLITERATIONS = {
|
|
10
|
+
"ı": "i", "ß": "ss", "ø": "o", "Ø": "O", "æ": "ae", "Æ": "Ae", "đ": "d", "Đ": "D", "ł": "l", "Ł": "L",
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class InvalidWorkloadNameError(ValueError):
|
|
15
|
+
"""Raised for workload names that are not 1-128 lowercase letters, digits, '.', '_' or '-'."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def validate_name(name: str) -> None:
|
|
19
|
+
if not isinstance(name, str) or not _NAME_PATTERN.match(name):
|
|
20
|
+
raise InvalidWorkloadNameError(
|
|
21
|
+
f"Workload name '{name}' is invalid. Use 1-128 lowercase letters, digits, '.', '_' or '-', "
|
|
22
|
+
"starting with a letter or digit."
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _is_lower(c: str | None) -> bool:
|
|
27
|
+
return c is not None and "a" <= c <= "z"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _is_upper(c: str | None) -> bool:
|
|
31
|
+
return c is not None and "A" <= c <= "Z"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _is_digit(c: str | None) -> bool:
|
|
35
|
+
return c is not None and "0" <= c <= "9"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def to_workload_name(identifier: str) -> str:
|
|
39
|
+
"""
|
|
40
|
+
Turns an arbitrary job identifier, such as ``InvoiceJobs.GenerateInvoices`` or ``Reports/Daily Sales``, into a
|
|
41
|
+
valid workload name such as ``invoice-jobs.generate-invoices`` or ``reports-daily-sales``. Every RunSignal SDK
|
|
42
|
+
derives names the same way (spec/conformance/workload-names.json).
|
|
43
|
+
"""
|
|
44
|
+
if not identifier.strip():
|
|
45
|
+
raise InvalidWorkloadNameError("The identifier is empty.")
|
|
46
|
+
|
|
47
|
+
# "e-Arşiv gönderimi" -> "e-Arsiv gonderimi", so names stay readable instead of losing letters.
|
|
48
|
+
decomposed = unicodedata.normalize("NFD", identifier)
|
|
49
|
+
chars = [_TRANSLITERATIONS.get(c, c) for c in decomposed if unicodedata.category(c) != "Mn"]
|
|
50
|
+
chars = list("".join(chars))
|
|
51
|
+
|
|
52
|
+
out: list[str] = []
|
|
53
|
+
for i, c in enumerate(chars):
|
|
54
|
+
if _is_upper(c):
|
|
55
|
+
# Break camel case: "GenerateInvoices" -> "generate-invoices", "HTTPSync" -> "http-sync".
|
|
56
|
+
prev = chars[i - 1] if i > 0 else None
|
|
57
|
+
nxt = chars[i + 1] if i + 1 < len(chars) else None
|
|
58
|
+
if out and out[-1] not in "-._" and (
|
|
59
|
+
_is_lower(prev) or _is_digit(prev) or (_is_upper(prev) and _is_lower(nxt))
|
|
60
|
+
):
|
|
61
|
+
out.append("-")
|
|
62
|
+
out.append(c.lower())
|
|
63
|
+
elif _is_lower(c) or _is_digit(c) or c in "._":
|
|
64
|
+
out.append(c)
|
|
65
|
+
elif out and out[-1] != "-":
|
|
66
|
+
out.append("-")
|
|
67
|
+
|
|
68
|
+
name = "".join(out).strip("-._")
|
|
69
|
+
if len(name) > 128:
|
|
70
|
+
name = name[:128].rstrip("-._")
|
|
71
|
+
if not name:
|
|
72
|
+
raise InvalidWorkloadNameError(f"'{identifier}' contains no letters or digits to name a workload after.")
|
|
73
|
+
return name
|
runsignal/_registry.py
ADDED
|
@@ -0,0 +1,442 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import functools
|
|
4
|
+
import inspect
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import sys
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
from datetime import timedelta
|
|
12
|
+
from typing import Any, Callable, Iterable, Literal, TypeVar, Union
|
|
13
|
+
|
|
14
|
+
from opentelemetry import context, metrics, trace
|
|
15
|
+
from opentelemetry.metrics import CallbackOptions, MeterProvider, Observation
|
|
16
|
+
from opentelemetry.trace import Span, SpanKind, Status, StatusCode, TracerProvider
|
|
17
|
+
|
|
18
|
+
from ._naming import to_workload_name, validate_name
|
|
19
|
+
from ._telemetry import DURATION_BUCKETS, SCOPE_NAME, VERSION, Attributes, Metrics
|
|
20
|
+
|
|
21
|
+
Severity = Literal["info", "warning", "critical"]
|
|
22
|
+
"""How urgent a workload's alerts are; it routes notifications."""
|
|
23
|
+
|
|
24
|
+
Duration = Union[float, int, timedelta, str]
|
|
25
|
+
"""Seconds, a ``timedelta``, or a string such as ``'90s'``, ``'30m'``, ``'2h'`` or ``'1d'``."""
|
|
26
|
+
|
|
27
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
28
|
+
|
|
29
|
+
_SEVERITIES = ("info", "warning", "critical")
|
|
30
|
+
_UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class WorkloadConfigurationError(ValueError):
|
|
34
|
+
"""Raised when a workload's options are invalid or declared twice."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _to_seconds(value: Duration, option: str) -> float:
|
|
38
|
+
if isinstance(value, timedelta):
|
|
39
|
+
seconds = value.total_seconds()
|
|
40
|
+
elif isinstance(value, str):
|
|
41
|
+
match = re.fullmatch(r"(\d+(?:\.\d+)?)([smhd])", value.strip())
|
|
42
|
+
if not match:
|
|
43
|
+
raise WorkloadConfigurationError(
|
|
44
|
+
f"{option} '{value}' is not a duration. Use seconds, a timedelta or a number with s, m, h or d, such as '30m'."
|
|
45
|
+
)
|
|
46
|
+
seconds = float(match.group(1)) * _UNIT_SECONDS[match.group(2)]
|
|
47
|
+
elif isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
48
|
+
seconds = float(value)
|
|
49
|
+
else:
|
|
50
|
+
raise WorkloadConfigurationError(f"{option} must be seconds, a timedelta or a string such as '30m'.")
|
|
51
|
+
if not seconds > 0 or seconds == float("inf"):
|
|
52
|
+
raise WorkloadConfigurationError(f"{option} must be positive.")
|
|
53
|
+
return seconds
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class Expectations:
|
|
57
|
+
"""The validated expectations of a workload, with durations in seconds. They define the alerts RunSignal
|
|
58
|
+
provisions and do not change how signals are recorded."""
|
|
59
|
+
|
|
60
|
+
__slots__ = ("schedule", "max_duration", "expect_every", "min_processed", "failure_rate_threshold", "severity")
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
schedule: str | None = None,
|
|
65
|
+
max_duration: Duration | None = None,
|
|
66
|
+
expect_every: Duration | None = None,
|
|
67
|
+
min_processed: int | None = None,
|
|
68
|
+
failure_rate_threshold: float | None = None,
|
|
69
|
+
severity: Severity | None = None,
|
|
70
|
+
) -> None:
|
|
71
|
+
if failure_rate_threshold is not None and not 0 <= failure_rate_threshold <= 1:
|
|
72
|
+
raise WorkloadConfigurationError(f"failure_rate_threshold must be between 0 and 1, got {failure_rate_threshold}.")
|
|
73
|
+
if min_processed is not None and (not isinstance(min_processed, int) or isinstance(min_processed, bool) or min_processed < 0):
|
|
74
|
+
raise WorkloadConfigurationError("min_processed must be a non-negative integer.")
|
|
75
|
+
severity = severity or "warning"
|
|
76
|
+
if severity not in _SEVERITIES:
|
|
77
|
+
raise WorkloadConfigurationError(f"Unknown severity '{severity}'.")
|
|
78
|
+
self.schedule = schedule
|
|
79
|
+
self.max_duration = None if max_duration is None else _to_seconds(max_duration, "max_duration")
|
|
80
|
+
self.expect_every = None if expect_every is None else _to_seconds(expect_every, "expect_every")
|
|
81
|
+
self.min_processed = min_processed
|
|
82
|
+
self.failure_rate_threshold = failure_rate_threshold
|
|
83
|
+
self.severity: Severity = severity
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class _Instruments:
|
|
87
|
+
def __init__(self, registry: WorkloadRegistry, provider: MeterProvider) -> None:
|
|
88
|
+
meter = provider.get_meter(SCOPE_NAME, VERSION)
|
|
89
|
+
self.runs = meter.create_counter(Metrics.RUNS, unit="{run}", description="Completed runs of a workload.")
|
|
90
|
+
self.duration = meter.create_histogram(
|
|
91
|
+
Metrics.DURATION,
|
|
92
|
+
unit="s",
|
|
93
|
+
description="Duration of workload runs.",
|
|
94
|
+
explicit_bucket_boundaries_advisory=list(DURATION_BUCKETS),
|
|
95
|
+
)
|
|
96
|
+
self.items = meter.create_counter(Metrics.ITEMS, unit="{item}", description="Items handled by workload runs.")
|
|
97
|
+
self.heartbeats = meter.create_counter(
|
|
98
|
+
Metrics.HEARTBEATS, unit="{heartbeat}", description="Heartbeats of long-running workloads."
|
|
99
|
+
)
|
|
100
|
+
meter.create_observable_gauge(
|
|
101
|
+
Metrics.SINCE_SUCCESS,
|
|
102
|
+
callbacks=[registry._observe_since_success],
|
|
103
|
+
unit="s",
|
|
104
|
+
description="Seconds since the last successful run of a workload, or since it was registered.",
|
|
105
|
+
)
|
|
106
|
+
meter.create_observable_gauge(
|
|
107
|
+
Metrics.SINCE_HEARTBEAT,
|
|
108
|
+
callbacks=[registry._observe_since_heartbeat],
|
|
109
|
+
unit="s",
|
|
110
|
+
description="Seconds since the last heartbeat of a workload, or since it was registered.",
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class WorkloadRegistry:
|
|
115
|
+
"""Creates and holds the workloads of an application and the instruments they record on.
|
|
116
|
+
|
|
117
|
+
By default it records on the global OpenTelemetry providers. They forward to the SDK once it is configured,
|
|
118
|
+
so a registry created by an import before the SDK starts still records.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
def __init__(
|
|
122
|
+
self,
|
|
123
|
+
*,
|
|
124
|
+
meter_provider: MeterProvider | None = None,
|
|
125
|
+
tracer_provider: TracerProvider | None = None,
|
|
126
|
+
tracing: bool = True,
|
|
127
|
+
) -> None:
|
|
128
|
+
self._workloads: dict[str, Workload] = {}
|
|
129
|
+
self._lock = threading.Lock()
|
|
130
|
+
self._instruments = _Instruments(self, meter_provider or metrics.get_meter_provider())
|
|
131
|
+
self._tracer = (tracer_provider or trace.get_tracer_provider()).get_tracer(SCOPE_NAME, VERSION) if tracing else None
|
|
132
|
+
|
|
133
|
+
def workload(
|
|
134
|
+
self,
|
|
135
|
+
name: str,
|
|
136
|
+
*,
|
|
137
|
+
schedule: str | None = None,
|
|
138
|
+
max_duration: Duration | None = None,
|
|
139
|
+
expect_every: Duration | None = None,
|
|
140
|
+
min_processed: int | None = None,
|
|
141
|
+
failure_rate_threshold: float | None = None,
|
|
142
|
+
severity: Severity | None = None,
|
|
143
|
+
) -> Workload:
|
|
144
|
+
"""Returns the workload called ``name``, creating it with the given expectations on first use. Call it once
|
|
145
|
+
and keep the result. Passing expectations for a workload that already exists raises, so expectations are
|
|
146
|
+
declared in exactly one place."""
|
|
147
|
+
options = dict(schedule=schedule, max_duration=max_duration, expect_every=expect_every, min_processed=min_processed,
|
|
148
|
+
failure_rate_threshold=failure_rate_threshold, severity=severity)
|
|
149
|
+
has_options = any(v is not None for v in options.values())
|
|
150
|
+
with self._lock:
|
|
151
|
+
existing = self._workloads.get(name)
|
|
152
|
+
if existing is not None:
|
|
153
|
+
if has_options:
|
|
154
|
+
raise WorkloadConfigurationError(
|
|
155
|
+
f"Workload '{name}' is already registered; set its options only where it is first created."
|
|
156
|
+
)
|
|
157
|
+
return existing
|
|
158
|
+
return self._add(name, Expectations(**options))
|
|
159
|
+
|
|
160
|
+
def get_or_add(self, name: str, **options: Any) -> Workload:
|
|
161
|
+
"""Returns the workload called ``name``, creating it with ``options`` if it does not exist yet. Unlike
|
|
162
|
+
``workload``, an existing workload is returned as is: options declared elsewhere take precedence. Scheduler
|
|
163
|
+
integrations use it to name workloads after jobs."""
|
|
164
|
+
with self._lock:
|
|
165
|
+
existing = self._workloads.get(name)
|
|
166
|
+
if existing is not None:
|
|
167
|
+
return existing
|
|
168
|
+
return self._add(name, Expectations(**options))
|
|
169
|
+
|
|
170
|
+
def _add(self, name: str, expectations: Expectations) -> Workload:
|
|
171
|
+
validate_name(name)
|
|
172
|
+
created = Workload(self, name, expectations)
|
|
173
|
+
self._workloads[name] = created
|
|
174
|
+
return created
|
|
175
|
+
|
|
176
|
+
def monitor(self, name: str, **options: Any) -> Callable[[F], F]:
|
|
177
|
+
"""Decorates a function, sync or async, so every call is a run of the workload ``name``, declared with
|
|
178
|
+
``options``. The decorated function has a ``workload`` attribute, for counting items."""
|
|
179
|
+
workload = self.workload(name, **options)
|
|
180
|
+
return workload.monitor
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def workloads(self) -> list[Workload]:
|
|
184
|
+
"""Every workload created so far, sorted by name."""
|
|
185
|
+
with self._lock:
|
|
186
|
+
return sorted(self._workloads.values(), key=lambda w: w.name)
|
|
187
|
+
|
|
188
|
+
def manifest(self, service: str) -> dict[str, Any]:
|
|
189
|
+
"""Describes every workload, for alert provisioning. ``service`` must match the OpenTelemetry ``service.name``."""
|
|
190
|
+
entries = []
|
|
191
|
+
for w in self.workloads:
|
|
192
|
+
e = w.expectations
|
|
193
|
+
entry: dict[str, Any] = {"name": w.name}
|
|
194
|
+
if e.schedule is not None:
|
|
195
|
+
entry["schedule"] = e.schedule
|
|
196
|
+
if e.max_duration is not None:
|
|
197
|
+
entry["maxDurationSeconds"] = _number(e.max_duration)
|
|
198
|
+
if e.expect_every is not None:
|
|
199
|
+
entry["expectEverySeconds"] = _number(e.expect_every)
|
|
200
|
+
if e.min_processed:
|
|
201
|
+
entry["minProcessed"] = e.min_processed
|
|
202
|
+
if e.failure_rate_threshold:
|
|
203
|
+
entry["failureRateThreshold"] = _number(e.failure_rate_threshold)
|
|
204
|
+
entry["severity"] = e.severity
|
|
205
|
+
entries.append(entry)
|
|
206
|
+
return {"version": 1, "service": service, "workloads": entries}
|
|
207
|
+
|
|
208
|
+
def write_manifest(self, path: str, service: str) -> None:
|
|
209
|
+
"""Writes the manifest as indented JSON, atomically, so a reader never sees half a manifest."""
|
|
210
|
+
directory = os.path.dirname(path)
|
|
211
|
+
if directory:
|
|
212
|
+
os.makedirs(directory, exist_ok=True)
|
|
213
|
+
with open(path + ".tmp", "w", encoding="utf-8") as f:
|
|
214
|
+
f.write(json.dumps(self.manifest(service), indent=2, ensure_ascii=False))
|
|
215
|
+
os.replace(path + ".tmp", path)
|
|
216
|
+
|
|
217
|
+
def write_manifest_from_env(self, service: str | None = None) -> str | None:
|
|
218
|
+
"""Writes the manifest if the RUNSIGNAL_MANIFEST_PATH environment variable is set, so CI can ask for one
|
|
219
|
+
without code changes. Call it once every workload has been registered. The service name is ``service``, or
|
|
220
|
+
else OTEL_SERVICE_NAME, or else the script name. Returns the path written, or None."""
|
|
221
|
+
path = os.environ.get("RUNSIGNAL_MANIFEST_PATH", "").strip()
|
|
222
|
+
if not path:
|
|
223
|
+
return None
|
|
224
|
+
script = os.path.splitext(os.path.basename(sys.argv[0] if sys.argv and sys.argv[0] else "python"))[0]
|
|
225
|
+
self.write_manifest(path, service or os.environ.get("OTEL_SERVICE_NAME") or script)
|
|
226
|
+
return path
|
|
227
|
+
|
|
228
|
+
def _observe_since_success(self, options: CallbackOptions) -> Iterable[Observation]:
|
|
229
|
+
now = time.monotonic()
|
|
230
|
+
for w in self.workloads:
|
|
231
|
+
seconds = w._seconds_since_success(now)
|
|
232
|
+
if seconds is not None:
|
|
233
|
+
yield Observation(seconds, w._name_attributes)
|
|
234
|
+
|
|
235
|
+
def _observe_since_heartbeat(self, options: CallbackOptions) -> Iterable[Observation]:
|
|
236
|
+
now = time.monotonic()
|
|
237
|
+
for w in self.workloads:
|
|
238
|
+
seconds = w._seconds_since_heartbeat(now)
|
|
239
|
+
if seconds is not None:
|
|
240
|
+
yield Observation(seconds, w._name_attributes)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _number(value: float) -> float | int:
|
|
244
|
+
return int(value) if float(value).is_integer() else value
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class Workload:
|
|
248
|
+
"""A unit of background work, such as a scheduled job, a queue consumer or a sync loop. Get one from a
|
|
249
|
+
WorkloadRegistry once and keep it."""
|
|
250
|
+
|
|
251
|
+
def __init__(self, registry: WorkloadRegistry, name: str, expectations: Expectations) -> None:
|
|
252
|
+
self._registry = registry
|
|
253
|
+
self.name = name
|
|
254
|
+
self.expectations = expectations
|
|
255
|
+
self._name_attributes = {Attributes.WORKLOAD: name}
|
|
256
|
+
self._outcome_attributes = {
|
|
257
|
+
outcome: {Attributes.WORKLOAD: name, Attributes.OUTCOME: outcome} for outcome in ("success", "failure", "cancelled")
|
|
258
|
+
}
|
|
259
|
+
self._processed = {Attributes.WORKLOAD: name, Attributes.ITEM_RESULT: "processed"}
|
|
260
|
+
self._failed = {Attributes.WORKLOAD: name, Attributes.ITEM_RESULT: "failed"}
|
|
261
|
+
# Until the first success or heartbeat these hold the time the workload was created, so a workload that
|
|
262
|
+
# never succeeds or never beats still reports a growing age.
|
|
263
|
+
self._last_success = self._last_heartbeat = time.monotonic()
|
|
264
|
+
self._has_started = False
|
|
265
|
+
self._has_beaten = False
|
|
266
|
+
|
|
267
|
+
def start(self) -> Run:
|
|
268
|
+
"""Starts a run. End it with exactly one call to ``succeed``, ``fail``, ``fail_code`` or ``cancel``; later
|
|
269
|
+
calls are ignored. Prefer ``run()`` as a context manager, which ends the run for you."""
|
|
270
|
+
self._has_started = True
|
|
271
|
+
tracer = self._registry._tracer
|
|
272
|
+
span = tracer.start_span(self.name, kind=SpanKind.INTERNAL, attributes=self._name_attributes) if tracer else None
|
|
273
|
+
return Run(self, time.monotonic(), span)
|
|
274
|
+
|
|
275
|
+
def run(self) -> Run:
|
|
276
|
+
"""A run to use as a context manager, ``with`` or ``async with``. It succeeds when the block completes,
|
|
277
|
+
is cancelled on ``asyncio.CancelledError``, ``KeyboardInterrupt`` or ``SystemExit``, and fails on any other
|
|
278
|
+
exception, which propagates. The run's span is current inside the block."""
|
|
279
|
+
return self.start()
|
|
280
|
+
|
|
281
|
+
def monitor(self, fn: F) -> F:
|
|
282
|
+
"""Decorates a function, sync or async, so every call is a run of this workload."""
|
|
283
|
+
if inspect.iscoroutinefunction(fn):
|
|
284
|
+
@functools.wraps(fn)
|
|
285
|
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
286
|
+
with self.run():
|
|
287
|
+
return await fn(*args, **kwargs)
|
|
288
|
+
async_wrapper.workload = self # type: ignore[attr-defined]
|
|
289
|
+
return async_wrapper # type: ignore[return-value]
|
|
290
|
+
|
|
291
|
+
@functools.wraps(fn)
|
|
292
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
293
|
+
with self.run():
|
|
294
|
+
return fn(*args, **kwargs)
|
|
295
|
+
wrapper.workload = self # type: ignore[attr-defined]
|
|
296
|
+
return wrapper # type: ignore[return-value]
|
|
297
|
+
|
|
298
|
+
def processed(self, count: int = 1) -> None:
|
|
299
|
+
"""Records ``count`` successfully processed items. Recording 0 is useful: it reports that a run did nothing."""
|
|
300
|
+
if count >= 0:
|
|
301
|
+
self._registry._instruments.items.add(count, self._processed)
|
|
302
|
+
|
|
303
|
+
def item_failed(self, count: int = 1) -> None:
|
|
304
|
+
"""Records ``count`` items that could not be processed."""
|
|
305
|
+
if count >= 0:
|
|
306
|
+
self._registry._instruments.items.add(count, self._failed)
|
|
307
|
+
|
|
308
|
+
def beat(self) -> None:
|
|
309
|
+
"""Records a heartbeat: the workload is alive and making progress."""
|
|
310
|
+
self._last_heartbeat = time.monotonic()
|
|
311
|
+
self._has_beaten = True
|
|
312
|
+
self._registry._instruments.heartbeats.add(1, self._name_attributes)
|
|
313
|
+
|
|
314
|
+
def _complete(self, start: float, span: Span | None, outcome: str, error_type: str | None = None,
|
|
315
|
+
error: BaseException | None = None, description: str | None = None) -> None:
|
|
316
|
+
now = time.monotonic()
|
|
317
|
+
instruments = self._registry._instruments
|
|
318
|
+
if error_type is None:
|
|
319
|
+
instruments.runs.add(1, self._outcome_attributes[outcome])
|
|
320
|
+
else:
|
|
321
|
+
instruments.runs.add(1, {**self._outcome_attributes["failure"], Attributes.ERROR_TYPE: error_type})
|
|
322
|
+
instruments.duration.record(now - start, self._outcome_attributes[outcome])
|
|
323
|
+
if outcome == "success":
|
|
324
|
+
self._last_success = now
|
|
325
|
+
|
|
326
|
+
if span is not None:
|
|
327
|
+
if span.is_recording():
|
|
328
|
+
span.set_attribute(Attributes.OUTCOME, outcome)
|
|
329
|
+
if outcome == "failure":
|
|
330
|
+
span.set_attribute(Attributes.ERROR_TYPE, error_type or "unknown")
|
|
331
|
+
if error is not None:
|
|
332
|
+
span.record_exception(error)
|
|
333
|
+
span.set_status(Status(StatusCode.ERROR, description))
|
|
334
|
+
elif outcome == "success":
|
|
335
|
+
span.set_status(Status(StatusCode.OK))
|
|
336
|
+
span.end()
|
|
337
|
+
|
|
338
|
+
# Reported for workloads that are scheduled or have run, so alerts can tell a missed run from a workload that
|
|
339
|
+
# simply has no runs.
|
|
340
|
+
def _seconds_since_success(self, now: float) -> float | None:
|
|
341
|
+
if self._has_started or self.expectations.schedule is not None:
|
|
342
|
+
return now - self._last_success
|
|
343
|
+
return None
|
|
344
|
+
|
|
345
|
+
# Reported for workloads that expect heartbeats or have sent one.
|
|
346
|
+
def _seconds_since_heartbeat(self, now: float) -> float | None:
|
|
347
|
+
if self._has_beaten or self.expectations.expect_every is not None:
|
|
348
|
+
return now - self._last_heartbeat
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
def __repr__(self) -> str:
|
|
352
|
+
return f"Workload({self.name!r})"
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
class Run:
|
|
356
|
+
"""One run of a Workload, from ``Workload.start`` or ``Workload.run``."""
|
|
357
|
+
|
|
358
|
+
__slots__ = ("workload", "span", "_started", "_ended", "_token")
|
|
359
|
+
|
|
360
|
+
def __init__(self, workload: Workload, started: float, span: Span | None) -> None:
|
|
361
|
+
self.workload = workload
|
|
362
|
+
self.span = span
|
|
363
|
+
self._started = started
|
|
364
|
+
self._ended = False
|
|
365
|
+
self._token: object | None = None
|
|
366
|
+
|
|
367
|
+
@property
|
|
368
|
+
def elapsed(self) -> float:
|
|
369
|
+
"""Seconds since the run started."""
|
|
370
|
+
return time.monotonic() - self._started
|
|
371
|
+
|
|
372
|
+
def succeed(self) -> None:
|
|
373
|
+
"""Ends the run as successful."""
|
|
374
|
+
if self._end():
|
|
375
|
+
self.workload._complete(self._started, self.span, "success")
|
|
376
|
+
|
|
377
|
+
def fail(self, error: BaseException) -> None:
|
|
378
|
+
"""Ends the run as failed because of ``error``. Its ``error.type`` is the exception's ``error_type``
|
|
379
|
+
attribute if it has one, otherwise its fully qualified class name."""
|
|
380
|
+
if self._end():
|
|
381
|
+
self.workload._complete(self._started, self.span, "failure", _error_type(error), error, str(error) or None)
|
|
382
|
+
|
|
383
|
+
def fail_code(self, code: str, description: str | None = None) -> None:
|
|
384
|
+
"""Ends the run as failed without an exception, for example when an expected input file is missing.
|
|
385
|
+
``code`` must be a short, stable value such as ``'statement_missing'``; never put messages or IDs in it."""
|
|
386
|
+
if self._end():
|
|
387
|
+
self.workload._complete(self._started, self.span, "failure", code or "unknown", None, description)
|
|
388
|
+
|
|
389
|
+
def cancel(self) -> None:
|
|
390
|
+
"""Ends the run as cancelled, for example on shutdown. Cancelled runs do not count as failures."""
|
|
391
|
+
if self._end():
|
|
392
|
+
self.workload._complete(self._started, self.span, "cancelled")
|
|
393
|
+
|
|
394
|
+
def _end(self) -> bool:
|
|
395
|
+
if self._ended:
|
|
396
|
+
return False
|
|
397
|
+
self._ended = True
|
|
398
|
+
return True
|
|
399
|
+
|
|
400
|
+
def __enter__(self) -> Run:
|
|
401
|
+
if self.span is not None:
|
|
402
|
+
self._token = context.attach(trace.set_span_in_context(self.span))
|
|
403
|
+
return self
|
|
404
|
+
|
|
405
|
+
def __exit__(self, exc_type: Any, exc: BaseException | None, tb: Any) -> Literal[False]:
|
|
406
|
+
if self._token is not None:
|
|
407
|
+
context.detach(self._token) # type: ignore[arg-type]
|
|
408
|
+
self._token = None
|
|
409
|
+
if exc is None:
|
|
410
|
+
self.succeed()
|
|
411
|
+
elif isinstance(exc, Exception):
|
|
412
|
+
self.fail(exc)
|
|
413
|
+
else:
|
|
414
|
+
# asyncio.CancelledError, KeyboardInterrupt, SystemExit: the run was stopped, not broken.
|
|
415
|
+
self.cancel()
|
|
416
|
+
return False
|
|
417
|
+
|
|
418
|
+
async def __aenter__(self) -> Run:
|
|
419
|
+
return self.__enter__()
|
|
420
|
+
|
|
421
|
+
async def __aexit__(self, exc_type: Any, exc: BaseException | None, tb: Any) -> Literal[False]:
|
|
422
|
+
return self.__exit__(exc_type, exc, tb)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _error_type(error: BaseException) -> str:
|
|
426
|
+
error_type = getattr(error, "error_type", None)
|
|
427
|
+
if isinstance(error_type, str) and error_type:
|
|
428
|
+
return error_type
|
|
429
|
+
cls = type(error)
|
|
430
|
+
return cls.__qualname__ if cls.__module__ == "builtins" else f"{cls.__module__}.{cls.__qualname__}"
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
default_registry = WorkloadRegistry()
|
|
434
|
+
"""A process-wide registry on the global OpenTelemetry providers."""
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def monitor(name: str, **options: Any) -> Callable[[F], F]:
|
|
438
|
+
"""Decorates a function, sync or async, so every call is a run of the workload ``name`` in the default
|
|
439
|
+
registry, declared with ``options`` (``schedule``, ``max_duration``, ``expect_every``, ``min_processed``,
|
|
440
|
+
``failure_rate_threshold``, ``severity``)."""
|
|
441
|
+
return default_registry.monitor(name, **options)
|
|
442
|
+
|
runsignal/_telemetry.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Names of the meter, tracer, instruments and attributes RunSignal emits. They are identical in every RunSignal SDK
|
|
3
|
+
(see spec/README.md in the repository), so one set of alert rules covers every language.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
SCOPE_NAME = "RunSignal"
|
|
7
|
+
VERSION = "0.2.0"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Metrics:
|
|
11
|
+
RUNS = "runsignal.workload.runs"
|
|
12
|
+
DURATION = "runsignal.workload.duration"
|
|
13
|
+
ITEMS = "runsignal.workload.items"
|
|
14
|
+
HEARTBEATS = "runsignal.workload.heartbeats"
|
|
15
|
+
SINCE_SUCCESS = "runsignal.workload.since_success"
|
|
16
|
+
SINCE_HEARTBEAT = "runsignal.workload.since_heartbeat"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Attributes:
|
|
20
|
+
WORKLOAD = "runsignal.workload"
|
|
21
|
+
OUTCOME = "runsignal.outcome"
|
|
22
|
+
ITEM_RESULT = "runsignal.item.result"
|
|
23
|
+
ERROR_TYPE = "error.type"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# Histogram bucket boundaries, in seconds, for the duration metric: background work runs from milliseconds to
|
|
27
|
+
# hours, while the OpenTelemetry defaults stop at 10 seconds.
|
|
28
|
+
DURATION_BUCKETS = (
|
|
29
|
+
0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 120, 300, 600, 900, 1800, 3600, 7200, 14400, 28800,
|
|
30
|
+
)
|
runsignal/apscheduler.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""
|
|
2
|
+
APScheduler 3 integration: records every job execution as a run of its workload, with no code in your jobs, and
|
|
3
|
+
registers cron-triggered jobs with their schedules, so a job that never fires still gets a missed alert.
|
|
4
|
+
|
|
5
|
+
from runsignal.apscheduler import instrument
|
|
6
|
+
|
|
7
|
+
scheduler = BackgroundScheduler()
|
|
8
|
+
scheduler.add_job(generate_invoices, "cron", hour=2, id="nightly-invoice")
|
|
9
|
+
instrument(scheduler, workloads={"nightly-invoice": {"max_duration": "30m", "severity": "critical"}})
|
|
10
|
+
scheduler.start()
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
import warnings
|
|
17
|
+
from typing import Any, Mapping
|
|
18
|
+
|
|
19
|
+
from apscheduler.events import EVENT_JOB_ADDED, JobEvent
|
|
20
|
+
from apscheduler.job import Job
|
|
21
|
+
from apscheduler.schedulers.base import BaseScheduler
|
|
22
|
+
from apscheduler.triggers.cron import CronTrigger
|
|
23
|
+
|
|
24
|
+
from ._naming import to_workload_name
|
|
25
|
+
from ._registry import Workload, WorkloadRegistry, default_registry
|
|
26
|
+
|
|
27
|
+
__all__ = ["instrument", "cron_of", "workload_name_of"]
|
|
28
|
+
|
|
29
|
+
_GENERATED_ID = re.compile(r"^[0-9a-f]{32}$")
|
|
30
|
+
_DAYS = {"mon": 1, "tue": 2, "wed": 3, "thu": 4, "fri": 5, "sat": 6, "sun": 7}
|
|
31
|
+
_MARK = "__runsignal_workload__"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def workload_name_of(job: Job) -> str:
|
|
35
|
+
"""The workload a job runs as: its ``id``, or its ``name`` when the id was generated by APScheduler."""
|
|
36
|
+
return to_workload_name(job.name if _GENERATED_ID.match(job.id) else job.id)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _day_of_week(expression: str) -> str:
|
|
40
|
+
# APScheduler counts days from Monday = 0; Unix cron from Sunday = 0, with 7 also Sunday. Monday..Sunday become
|
|
41
|
+
# 1..7, so ranges such as sat-sun stay ascending.
|
|
42
|
+
def value(v: str) -> str:
|
|
43
|
+
v = v.lower()
|
|
44
|
+
if v in _DAYS:
|
|
45
|
+
return str(_DAYS[v])
|
|
46
|
+
if v.isdigit():
|
|
47
|
+
return str(int(v) + 1)
|
|
48
|
+
raise ValueError(v)
|
|
49
|
+
|
|
50
|
+
parts = []
|
|
51
|
+
for token in expression.split(","):
|
|
52
|
+
base, _, step = token.partition("/")
|
|
53
|
+
if base == "*":
|
|
54
|
+
base = "1-7" if step else "*"
|
|
55
|
+
else:
|
|
56
|
+
base = "-".join(value(v) for v in base.split("-"))
|
|
57
|
+
parts.append(f"{base}/{step}" if step else base)
|
|
58
|
+
return ",".join(parts)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def cron_of(trigger: CronTrigger) -> str | None:
|
|
62
|
+
"""A cron trigger as a cron expression, 5 fields or 6 with seconds first, or None if it uses fields cron
|
|
63
|
+
cannot express (``year``, ``week``, or days such as ``1st mon``)."""
|
|
64
|
+
fields = {f.name: str(f) for f in trigger.fields}
|
|
65
|
+
if fields["year"] != "*" or fields["week"] != "*":
|
|
66
|
+
return None
|
|
67
|
+
day = "L" if fields["day"] == "last" else fields["day"]
|
|
68
|
+
if re.search(r"[a-zA-KM-Z]", day):
|
|
69
|
+
return None
|
|
70
|
+
try:
|
|
71
|
+
day_of_week = _day_of_week(fields["day_of_week"])
|
|
72
|
+
except ValueError:
|
|
73
|
+
return None
|
|
74
|
+
cron = f"{fields['minute']} {fields['hour']} {day} {fields['month']} {day_of_week}"
|
|
75
|
+
return cron if fields["second"] == "0" else f"{fields['second']} {cron}"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class _Instrumentation:
|
|
79
|
+
def __init__(self, scheduler: BaseScheduler, registry: WorkloadRegistry, workloads: Mapping[str, Mapping[str, Any]]) -> None:
|
|
80
|
+
self.scheduler = scheduler
|
|
81
|
+
self.registry = registry
|
|
82
|
+
self.workloads = workloads
|
|
83
|
+
|
|
84
|
+
def register(self, job: Job) -> Workload:
|
|
85
|
+
name = workload_name_of(job)
|
|
86
|
+
options = dict(self.workloads.get(name, {}))
|
|
87
|
+
if isinstance(job.trigger, CronTrigger) and "schedule" not in options:
|
|
88
|
+
cron = cron_of(job.trigger)
|
|
89
|
+
if cron is not None:
|
|
90
|
+
options["schedule"] = cron
|
|
91
|
+
else:
|
|
92
|
+
warnings.warn(f"runsignal: the trigger of job '{job.id}' cannot be written as cron, so it gets no missed alert.",
|
|
93
|
+
stacklevel=3)
|
|
94
|
+
workload = self.registry.get_or_add(name, **options)
|
|
95
|
+
|
|
96
|
+
if getattr(job.func, _MARK, None) is None:
|
|
97
|
+
wrapped = workload.monitor(job.func)
|
|
98
|
+
setattr(wrapped, _MARK, workload)
|
|
99
|
+
try:
|
|
100
|
+
self.scheduler.modify_job(job.id, getattr(job, "_jobstore_alias", None), func=wrapped)
|
|
101
|
+
except Exception as error: # a persistent job store cannot store a wrapper it cannot import
|
|
102
|
+
warnings.warn(f"runsignal: could not instrument job '{job.id}' ({error}). Decorate its function with "
|
|
103
|
+
f"@registry.monitor('{name}') instead.", stacklevel=3)
|
|
104
|
+
return workload
|
|
105
|
+
|
|
106
|
+
def on_added(self, event: JobEvent) -> None:
|
|
107
|
+
job = self.scheduler.get_job(event.job_id, event.jobstore)
|
|
108
|
+
if job is not None:
|
|
109
|
+
self.register(job)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def instrument(
|
|
113
|
+
scheduler: BaseScheduler,
|
|
114
|
+
*,
|
|
115
|
+
registry: WorkloadRegistry | None = None,
|
|
116
|
+
workloads: Mapping[str, Mapping[str, Any]] | None = None,
|
|
117
|
+
) -> list[Workload]:
|
|
118
|
+
"""Records every run of every job of ``scheduler``, now and added later: success, failure with the exception's
|
|
119
|
+
type, and duration. Cron-triggered jobs are registered with their schedule. ``workloads`` maps workload names
|
|
120
|
+
to expectations, such as ``{"nightly-invoice": {"max_duration": "30m"}}``. Time zones are not carried over:
|
|
121
|
+
pass ``--time-zone`` to the CLI if the triggers use one.
|
|
122
|
+
|
|
123
|
+
Jobs in a persistent job store (SQLAlchemy, Redis, MongoDB) cannot hold a wrapped function; decorate their
|
|
124
|
+
functions with ``@registry.monitor(name)`` instead, and this still registers their schedules.
|
|
125
|
+
"""
|
|
126
|
+
handler = _Instrumentation(scheduler, registry or default_registry, workloads or {})
|
|
127
|
+
found = [handler.register(job) for job in scheduler.get_jobs()]
|
|
128
|
+
scheduler.add_listener(handler.on_added, EVENT_JOB_ADDED)
|
|
129
|
+
return found
|
runsignal/celery.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Celery integration: records every task execution as a run of its workload, with no code in your tasks, and
|
|
3
|
+
registers Celery beat's crontab entries as workloads with their schedules, so a periodic task that never runs
|
|
4
|
+
still gets a missed alert.
|
|
5
|
+
|
|
6
|
+
from runsignal.celery import instrument
|
|
7
|
+
|
|
8
|
+
app = Celery("billing")
|
|
9
|
+
app.conf.beat_schedule = {"nightly-invoice": {"task": "billing.tasks.generate_invoices",
|
|
10
|
+
"schedule": crontab(hour=2, minute=0)}}
|
|
11
|
+
instrument(app, workloads={"billing.tasks.generate_invoices": {"max_duration": "30m", "severity": "critical"}})
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import threading
|
|
18
|
+
from typing import Any, Mapping
|
|
19
|
+
|
|
20
|
+
from celery import Celery, Task, signals
|
|
21
|
+
from celery.schedules import crontab
|
|
22
|
+
|
|
23
|
+
from ._naming import to_workload_name
|
|
24
|
+
from ._registry import Run, Workload, WorkloadRegistry, default_registry
|
|
25
|
+
|
|
26
|
+
__all__ = ["instrument", "discover_beat_schedule", "workload_name_of", "workload_of", "beat_cron"]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def workload_name_of(task: Task | str) -> str:
|
|
30
|
+
"""The workload a task runs as: its task name, such as ``billing.tasks.generate_invoices``."""
|
|
31
|
+
return to_workload_name(task if isinstance(task, str) else task.name)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def beat_cron(schedule: crontab) -> str:
|
|
35
|
+
"""A Celery ``crontab`` as a 5-field cron expression. Celery numbers days of the week like Unix cron
|
|
36
|
+
(0 = Sunday), so fields carry over unchanged."""
|
|
37
|
+
|
|
38
|
+
def field(value: Any) -> str:
|
|
39
|
+
if isinstance(value, (set, frozenset, list, tuple)):
|
|
40
|
+
return ",".join(str(v) for v in sorted(value))
|
|
41
|
+
return str(value)
|
|
42
|
+
|
|
43
|
+
return " ".join(field(getattr(schedule, f"_orig_{name}")) for name in
|
|
44
|
+
("minute", "hour", "day_of_month", "month_of_year", "day_of_week"))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def discover_beat_schedule(
|
|
48
|
+
app: Celery,
|
|
49
|
+
*,
|
|
50
|
+
registry: WorkloadRegistry | None = None,
|
|
51
|
+
workloads: Mapping[str, Mapping[str, Any]] | None = None,
|
|
52
|
+
) -> list[Workload]:
|
|
53
|
+
"""Registers a workload for every task in ``app.conf.beat_schedule``, with its crontab as the schedule, so a
|
|
54
|
+
periodic task that never runs is still alerted on. Entries with an interval instead of a crontab get no
|
|
55
|
+
schedule. Time zones are not carried over: pass ``--time-zone`` to the CLI if beat uses one."""
|
|
56
|
+
registry = registry or default_registry
|
|
57
|
+
found: list[Workload] = []
|
|
58
|
+
for entry in (app.conf.beat_schedule or {}).values():
|
|
59
|
+
task = entry.get("task")
|
|
60
|
+
if not task or task.startswith("celery."):
|
|
61
|
+
continue
|
|
62
|
+
name = workload_name_of(task)
|
|
63
|
+
options = dict((workloads or {}).get(name, {}))
|
|
64
|
+
schedule = entry.get("schedule")
|
|
65
|
+
if isinstance(schedule, crontab):
|
|
66
|
+
options.setdefault("schedule", beat_cron(schedule))
|
|
67
|
+
found.append(registry.get_or_add(name, **options))
|
|
68
|
+
return found
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
_task_workloads: dict[str, Workload] = {}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def workload_of(task: Task) -> Workload | None:
|
|
75
|
+
"""The workload a task runs as, for counting items: ``workload_of(self).processed()`` in a bound task."""
|
|
76
|
+
return _task_workloads.get(task.name)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class _Instrumentation:
|
|
80
|
+
def __init__(self, registry: WorkloadRegistry, workloads: Mapping[str, Mapping[str, Any]]) -> None:
|
|
81
|
+
self.registry = registry
|
|
82
|
+
self.workloads = workloads
|
|
83
|
+
self.runs: dict[str, Run] = {}
|
|
84
|
+
self.errors: dict[str, BaseException] = {}
|
|
85
|
+
self.lock = threading.Lock()
|
|
86
|
+
|
|
87
|
+
def workload(self, task: Task) -> Workload | None:
|
|
88
|
+
workload = _task_workloads.get(task.name)
|
|
89
|
+
if workload is None:
|
|
90
|
+
if task.name.startswith("celery."):
|
|
91
|
+
return None
|
|
92
|
+
name = workload_name_of(task)
|
|
93
|
+
workload = self.registry.get_or_add(name, **self.workloads.get(name, {}))
|
|
94
|
+
_task_workloads[task.name] = workload
|
|
95
|
+
return workload
|
|
96
|
+
|
|
97
|
+
def prerun(self, task_id: str = "", task: Task | None = None, **_: Any) -> None:
|
|
98
|
+
workload = self.workload(task) if task is not None else None
|
|
99
|
+
if workload is not None:
|
|
100
|
+
with self.lock:
|
|
101
|
+
self.runs[task_id] = workload.start()
|
|
102
|
+
|
|
103
|
+
def failure(self, task_id: str = "", exception: BaseException | None = None, **_: Any) -> None:
|
|
104
|
+
if exception is not None:
|
|
105
|
+
with self.lock:
|
|
106
|
+
self.errors[task_id] = exception
|
|
107
|
+
|
|
108
|
+
def retry(self, request: Any = None, reason: Any = None, **_: Any) -> None:
|
|
109
|
+
if request is not None and isinstance(reason, BaseException):
|
|
110
|
+
with self.lock:
|
|
111
|
+
self.errors[request.id] = reason
|
|
112
|
+
|
|
113
|
+
def postrun(self, task_id: str = "", state: str | None = None, **_: Any) -> None:
|
|
114
|
+
with self.lock:
|
|
115
|
+
run = self.runs.pop(task_id, None)
|
|
116
|
+
error = self.errors.pop(task_id, None)
|
|
117
|
+
if run is None:
|
|
118
|
+
return
|
|
119
|
+
if state == "SUCCESS":
|
|
120
|
+
run.succeed()
|
|
121
|
+
elif state in ("FAILURE", "RETRY"):
|
|
122
|
+
if error is not None:
|
|
123
|
+
run.fail(error)
|
|
124
|
+
else:
|
|
125
|
+
run.fail_code(state.lower())
|
|
126
|
+
else:
|
|
127
|
+
run.cancel()
|
|
128
|
+
|
|
129
|
+
def revoked(self, request: Any = None, **_: Any) -> None:
|
|
130
|
+
if request is None:
|
|
131
|
+
return
|
|
132
|
+
with self.lock:
|
|
133
|
+
run = self.runs.pop(request.id, None)
|
|
134
|
+
self.errors.pop(request.id, None)
|
|
135
|
+
if run is not None:
|
|
136
|
+
run.cancel()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
_instrumented: dict[int, _Instrumentation] = {}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def instrument(
|
|
143
|
+
app: Celery | None = None,
|
|
144
|
+
*,
|
|
145
|
+
registry: WorkloadRegistry | None = None,
|
|
146
|
+
workloads: Mapping[str, Mapping[str, Any]] | None = None,
|
|
147
|
+
write_manifest_on_ready: bool = True,
|
|
148
|
+
) -> list[Workload]:
|
|
149
|
+
"""Records every Celery task execution as a run: success, failure with the exception's type (each retry counts
|
|
150
|
+
as a failed run), and cancellation when the task is revoked or ignored. Celery's own ``celery.*`` tasks are
|
|
151
|
+
skipped. Call it once per process, where the app is configured.
|
|
152
|
+
|
|
153
|
+
With ``app``, it also registers the beat schedule's tasks (``discover_beat_schedule``) and, when
|
|
154
|
+
``write_manifest_on_ready`` is true, writes the manifest when the worker is ready if RUNSIGNAL_MANIFEST_PATH is
|
|
155
|
+
set. ``workloads`` maps workload names to expectations, such as ``{"billing.tasks.generate_invoices":
|
|
156
|
+
{"max_duration": "30m"}}``. Returns the workloads discovered from the beat schedule.
|
|
157
|
+
"""
|
|
158
|
+
registry = registry or default_registry
|
|
159
|
+
workloads = workloads or {}
|
|
160
|
+
if id(registry) not in _instrumented:
|
|
161
|
+
handler = _Instrumentation(registry, workloads)
|
|
162
|
+
_instrumented[id(registry)] = handler
|
|
163
|
+
signals.task_prerun.connect(handler.prerun, weak=False)
|
|
164
|
+
signals.task_failure.connect(handler.failure, weak=False)
|
|
165
|
+
signals.task_retry.connect(handler.retry, weak=False)
|
|
166
|
+
signals.task_postrun.connect(handler.postrun, weak=False)
|
|
167
|
+
signals.task_revoked.connect(handler.revoked, weak=False)
|
|
168
|
+
|
|
169
|
+
if app is None:
|
|
170
|
+
return []
|
|
171
|
+
found = discover_beat_schedule(app, registry=registry, workloads=workloads)
|
|
172
|
+
if write_manifest_on_ready:
|
|
173
|
+
def write_manifest(**_: Any) -> None:
|
|
174
|
+
registry.write_manifest_from_env(os.environ.get("OTEL_SERVICE_NAME") or app.main or None)
|
|
175
|
+
signals.worker_ready.connect(write_manifest, weak=False)
|
|
176
|
+
return found
|
runsignal/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: runsignal
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Cron job and background job monitoring for Python with OpenTelemetry: alert when a Celery task, APScheduler job or cron script is missed, fails, overruns, stalls or processes nothing, in New Relic, Prometheus and Grafana.
|
|
5
|
+
Project-URL: Homepage, https://github.com/gokhanertogan/RunSignal
|
|
6
|
+
Project-URL: Documentation, https://github.com/gokhanertogan/RunSignal/tree/main/src/RunSignal.Python#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/gokhanertogan/RunSignal
|
|
8
|
+
Project-URL: Issues, https://github.com/gokhanertogan/RunSignal/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/gokhanertogan/RunSignal/blob/main/CHANGELOG.md
|
|
10
|
+
Author: Gokhan Ertogan
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: alerting,apscheduler,background-jobs,celery,celery-beat,cron,cron-monitoring,dead-mans-switch,grafana,healthcheck,heartbeat,job-monitoring,monitoring,newrelic,observability,opentelemetry,otel,prometheus,scheduled-jobs,worker
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Framework :: Celery
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Intended Audience :: System Administrators
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Classifier: Topic :: System :: Logging
|
|
22
|
+
Classifier: Topic :: System :: Monitoring
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Requires-Dist: opentelemetry-api>=1.27
|
|
26
|
+
Provides-Extra: apscheduler
|
|
27
|
+
Requires-Dist: apscheduler<4,>=3.10; extra == 'apscheduler'
|
|
28
|
+
Provides-Extra: celery
|
|
29
|
+
Requires-Dist: celery>=5.3; extra == 'celery'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# RunSignal for Python
|
|
33
|
+
|
|
34
|
+
[](https://pypi.org/project/runsignal/)
|
|
35
|
+
[](https://pypi.org/project/runsignal/)
|
|
36
|
+
[](https://github.com/gokhanertogan/RunSignal/blob/main/LICENSE)
|
|
37
|
+
|
|
38
|
+
**Cron job and background job monitoring for Python, on OpenTelemetry.** Get an alert when a Celery task or
|
|
39
|
+
Celery beat schedule, an APScheduler job or a cron script is missed, fails, runs too long or processes nothing,
|
|
40
|
+
and when a consumer or worker loop stops making progress: the failures no HTTP status code describes. No hosted
|
|
41
|
+
service and no ping URLs: the alerts live in the backend you already use.
|
|
42
|
+
|
|
43
|
+
RunSignal records runs, items and heartbeats as OpenTelemetry metrics, with the same names as every other
|
|
44
|
+
RunSignal SDK, so the `runsignal` CLI provisions the same alerts for Python services as for .NET, Go, Java,
|
|
45
|
+
Node.js and Rust ones: New Relic alerts and dashboards, or Prometheus rules and a Grafana dashboard. The
|
|
46
|
+
[main README](https://github.com/gokhanertogan/RunSignal#readme) covers the concepts, the alerts and the
|
|
47
|
+
backends.
|
|
48
|
+
|
|
49
|
+
## Installation
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install runsignal # core, depends only on opentelemetry-api
|
|
53
|
+
pip install "runsignal[celery]" # with Celery 5.3+
|
|
54
|
+
pip install "runsignal[apscheduler]" # with APScheduler 3.10+
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Requires Python 3.9 or later. Install and configure the OpenTelemetry SDK and an exporter as usual
|
|
58
|
+
(`opentelemetry-sdk`, `opentelemetry-exporter-otlp`).
|
|
59
|
+
|
|
60
|
+
## Quick start
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from runsignal import default_registry
|
|
64
|
+
|
|
65
|
+
invoices = default_registry.workload(
|
|
66
|
+
"nightly-invoice",
|
|
67
|
+
schedule="0 2 * * *",
|
|
68
|
+
max_duration="30m", # or seconds, or a timedelta
|
|
69
|
+
min_processed=100,
|
|
70
|
+
severity="critical",
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
with invoices.run():
|
|
74
|
+
for order in unbilled_orders():
|
|
75
|
+
create_invoice(order)
|
|
76
|
+
invoices.processed()
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Or as a decorator, for sync and async functions:
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from runsignal import monitor
|
|
83
|
+
|
|
84
|
+
@monitor("nightly-invoice", schedule="0 2 * * *", max_duration="30m")
|
|
85
|
+
def generate_invoices():
|
|
86
|
+
for order in unbilled_orders():
|
|
87
|
+
create_invoice(order)
|
|
88
|
+
generate_invoices.workload.processed()
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`default_registry` records on the global OpenTelemetry providers, which forward to the SDK once it is configured,
|
|
92
|
+
so it works even when it is created by an import first. For explicit providers, use
|
|
93
|
+
`WorkloadRegistry(meter_provider=..., tracer_provider=...)`.
|
|
94
|
+
|
|
95
|
+
## Recording work
|
|
96
|
+
|
|
97
|
+
| Call | Records |
|
|
98
|
+
|---|---|
|
|
99
|
+
| `with w.run():` / `async with w.run():` | One run. It succeeds when the block completes, is cancelled on `asyncio.CancelledError`, `KeyboardInterrupt` or `SystemExit`, and fails on any other exception, which propagates. The run's span is current inside the block. |
|
|
100
|
+
| `@w.monitor` / `@monitor(name, **expectations)` | One run per call, for sync and async functions. |
|
|
101
|
+
| `run = w.start()` then `run.succeed()` / `fail(exc)` / `fail_code(code, description)` / `cancel()` | A run ended by hand. Later calls are ignored. |
|
|
102
|
+
| `w.processed(n)` / `w.item_failed(n)` | Items handled. |
|
|
103
|
+
| `w.beat()` | A heartbeat from a long-running loop. |
|
|
104
|
+
|
|
105
|
+
`error.type` is the exception's `error_type` attribute if it has one (such as `"statement_missing"`), otherwise
|
|
106
|
+
its fully qualified class name (`ValueError`, `requests.exceptions.ConnectionError`).
|
|
107
|
+
|
|
108
|
+
## Celery
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from celery import Celery
|
|
112
|
+
from celery.schedules import crontab
|
|
113
|
+
from runsignal.celery import instrument, workload_of
|
|
114
|
+
|
|
115
|
+
app = Celery("billing")
|
|
116
|
+
app.conf.beat_schedule = {
|
|
117
|
+
"nightly-invoice": {"task": "billing.tasks.generate_invoices", "schedule": crontab(hour=2, minute=0)},
|
|
118
|
+
}
|
|
119
|
+
instrument(app, workloads={"billing.tasks.generate_invoices": {"max_duration": "30m", "severity": "critical"}})
|
|
120
|
+
|
|
121
|
+
@app.task(bind=True)
|
|
122
|
+
def generate_invoices(self):
|
|
123
|
+
for order in unbilled_orders():
|
|
124
|
+
create_invoice(order)
|
|
125
|
+
workload_of(self).processed()
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
- **What is recorded:** every task execution, with no code in your tasks: success, failure with the exception's type (each retry counts as a failed run), cancellation when a task is revoked, and duration. Celery's own `celery.*` tasks are skipped.
|
|
129
|
+
- **Naming:** tasks are named after their task name (`billing.tasks.generate_invoices`).
|
|
130
|
+
- **Celery beat:** `instrument(app)` registers every task in `beat_schedule` with its `crontab` as the schedule, so a periodic task that never runs gets a `missed` alert. Interval schedules get no schedule.
|
|
131
|
+
- **Manifest:** the worker writes it when it is ready if `RUNSIGNAL_MANIFEST_PATH` is set. In CI, import the app and call `default_registry.write_manifest_from_env("billing")` instead; no broker is needed.
|
|
132
|
+
- **Prefork workers:** configure the OpenTelemetry SDK in `worker_process_init`, as for any OpenTelemetry instrumentation of Celery.
|
|
133
|
+
|
|
134
|
+
## APScheduler
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
from apscheduler.schedulers.background import BackgroundScheduler
|
|
138
|
+
from runsignal.apscheduler import instrument
|
|
139
|
+
|
|
140
|
+
scheduler = BackgroundScheduler()
|
|
141
|
+
scheduler.add_job(generate_invoices, "cron", hour=2, minute=0, id="nightly-invoice")
|
|
142
|
+
instrument(scheduler, workloads={"nightly-invoice": {"max_duration": "30m", "severity": "critical"}})
|
|
143
|
+
scheduler.start()
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
- **What is recorded:** every run of every job, including jobs added after `instrument`, for all schedulers (`BackgroundScheduler`, `BlockingScheduler`, `AsyncIOScheduler`, ...).
|
|
147
|
+
- **Naming:** jobs are named after the `id` you gave them, or their `name` when APScheduler generated the id.
|
|
148
|
+
- **Schedules:** cron triggers become the workload's schedule, with APScheduler's Monday = 0 converted to cron's numbering. Triggers that use `year`, `week` or days such as `1st mon` get no schedule, with a warning.
|
|
149
|
+
- **Persistent job stores** (SQLAlchemy, Redis, MongoDB) cannot store a wrapped function: decorate the job function with `@default_registry.monitor("nightly-invoice")` instead. `instrument` still registers the schedule.
|
|
150
|
+
|
|
151
|
+
## Cron scripts and loops
|
|
152
|
+
|
|
153
|
+
A script run by cron, a Kubernetes CronJob or systemd timer is one run per invocation:
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
with default_registry.workload("nightly-invoice", schedule="0 2 * * *").run():
|
|
157
|
+
main()
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
The `missed` alert needs a long-running process to report `since_success`, so for short-lived scripts declare
|
|
161
|
+
the workload in the service that exports continuously, or rely on the `failed`, `overrun` and `low output`
|
|
162
|
+
alerts. Long-running loops send heartbeats:
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
sync = default_registry.workload("stock-sync", expect_every="5m")
|
|
166
|
+
while True:
|
|
167
|
+
sync_stock()
|
|
168
|
+
sync.beat()
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
## Manifest
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
default_registry.write_manifest_from_env("billing-service") # writes to RUNSIGNAL_MANIFEST_PATH when it is set
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Then provision the alerts from CI with the [`runsignal` CLI](https://github.com/gokhanertogan/RunSignal#cli).
|
|
178
|
+
|
|
179
|
+
## Exporting
|
|
180
|
+
|
|
181
|
+
**New Relic** needs delta temporality. With cumulative, a stopped workload never looks stopped:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.eu01.nr-data.net:4317 # US: https://otlp.nr-data.net:4317
|
|
185
|
+
export OTEL_EXPORTER_OTLP_HEADERS=api-key=<your license key>
|
|
186
|
+
export OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=delta
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
**Prometheus 3** receives OTLP at `/api/v1/otlp/v1/metrics` with the default (cumulative) temporality. Generate
|
|
190
|
+
its rules with `runsignal export prometheus` and a dashboard with `runsignal export grafana`.
|
|
191
|
+
|
|
192
|
+
## Development
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
python -m venv .venv && .venv/bin/pip install -e ".[celery,apscheduler]" opentelemetry-sdk pytest
|
|
196
|
+
.venv/bin/pytest
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
The tests include the cross-language conformance vectors in `spec/conformance`.
|
|
200
|
+
|
|
201
|
+
## Links
|
|
202
|
+
|
|
203
|
+
[Specification](https://github.com/gokhanertogan/RunSignal/blob/main/spec/README.md) ·
|
|
204
|
+
[Changelog](https://github.com/gokhanertogan/RunSignal/blob/main/CHANGELOG.md) ·
|
|
205
|
+
[Other languages](https://github.com/gokhanertogan/RunSignal#quick-start)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
runsignal/__init__.py,sha256=XDMceKKycUNMyuA9RwSeueBPmlJA-lxauUfqyk64uk4,1534
|
|
2
|
+
runsignal/_naming.py,sha256=qMvvwomBmqlAjhUWfpFDXPfNz7dAonEruc1ZTruJlx4,2722
|
|
3
|
+
runsignal/_registry.py,sha256=1HQ24PtHEQmQPVKonTrDss59QUktgu9w18Xeghak_Ys,20343
|
|
4
|
+
runsignal/_telemetry.py,sha256=BdOfsmlZQt1eYJv9CjB0PLiTn_QbDZF-fD5xaPeJcO4,1011
|
|
5
|
+
runsignal/apscheduler.py,sha256=RazOZ7nerXa_6apw8NwF61ywWHr7aRxb6jQceti1z3k,5516
|
|
6
|
+
runsignal/celery.py,sha256=HpePY-BD6UPjXWoQPfyf4TvB2HcO09OoWoiWdVqXCwk,7255
|
|
7
|
+
runsignal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
runsignal-0.2.0.dist-info/METADATA,sha256=K_bLFcDWgAOmeZpbbnQb382ZGgZlaojrMHAQ7TSRaG0,9539
|
|
9
|
+
runsignal-0.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
10
|
+
runsignal-0.2.0.dist-info/licenses/LICENSE,sha256=mHu_gwPqV0P9YgCWEaGbMR7u19MI6ac1gJoP1HSTbB4,1071
|
|
11
|
+
runsignal-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gokhan Ertogan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|