devalerts 0.1.5__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devalerts-0.1.5 → devalerts-0.2.0}/PKG-INFO +73 -4
- {devalerts-0.1.5 → devalerts-0.2.0}/README.md +72 -3
- {devalerts-0.1.5 → devalerts-0.2.0}/pyproject.toml +2 -1
- {devalerts-0.1.5 → devalerts-0.2.0}/src/devalerts/__init__.py +19 -6
- {devalerts-0.1.5 → devalerts-0.2.0}/src/devalerts/_alert.py +16 -2
- devalerts-0.2.0/src/devalerts/_celery.py +45 -0
- devalerts-0.2.0/src/devalerts/_store.py +191 -0
- devalerts-0.2.0/src/devalerts/_telegram.py +72 -0
- devalerts-0.2.0/src/devalerts/cli.py +317 -0
- devalerts-0.1.5/src/devalerts/_store.py +0 -98
- devalerts-0.1.5/src/devalerts/_telegram.py +0 -25
- devalerts-0.1.5/src/devalerts/cli.py +0 -156
- {devalerts-0.1.5 → devalerts-0.2.0}/LICENSE +0 -0
- {devalerts-0.1.5 → devalerts-0.2.0}/src/devalerts/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: devalerts
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Send unhandled Python exceptions straight to a Telegram chat. No backend, no account, no database — just your own bot token.
|
|
5
5
|
Keywords: telegram,error-tracking,exception-monitoring,alerts,logging,monitoring,asgi
|
|
6
6
|
Author: sslinNn
|
|
@@ -77,7 +77,13 @@ chat instead of a log file nobody's watching.
|
|
|
77
77
|
2. Message your bot once (or add it to a group) so it's allowed to message you back.
|
|
78
78
|
3. Get your chat id — message [@userinfobot](https://t.me/userinfobot), or call
|
|
79
79
|
`https://api.telegram.org/bot<TOKEN>/getUpdates` after step 2 and read `message.chat.id`.
|
|
80
|
-
4.
|
|
80
|
+
4. Verify it's wired up correctly before touching any code:
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
uv run devalerts test --bot-token 123456:ABC-DEF... --chat-id 123456789
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
5. In your app, as early as possible:
|
|
81
87
|
|
|
82
88
|
```python
|
|
83
89
|
import devalerts
|
|
@@ -109,6 +115,48 @@ uv run devalerts dashboard
|
|
|
109
115
|
|
|
110
116
|

|
|
111
117
|
|
|
118
|
+
Pass `--json` for machine-readable output. Silence a noisy group without
|
|
119
|
+
touching code — the `ID` column accepts any unique prefix:
|
|
120
|
+
|
|
121
|
+
```
|
|
122
|
+
uv run devalerts mute abc12345
|
|
123
|
+
uv run devalerts unmute abc12345
|
|
124
|
+
uv run devalerts clear abc12345 # or: devalerts clear --all
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Unmuting resends the next occurrence with the accumulated skip count, same
|
|
128
|
+
as a rate-limit window expiring.
|
|
129
|
+
|
|
130
|
+
Error groups that keep piling up while suppressed are chronic: each such
|
|
131
|
+
resend doubles the effective `rate_limit_seconds` for that group (capped at
|
|
132
|
+
8x), so a crash loop backs itself off instead of paging you every window
|
|
133
|
+
forever. A group that goes quiet and reappears once resets to the base rate
|
|
134
|
+
immediately. The dashboard shows the active multiplier (`● sending ×4`).
|
|
135
|
+
|
|
136
|
+
## Context: hostname and tags
|
|
137
|
+
|
|
138
|
+
Every alert automatically includes the sending host, so you can tell which
|
|
139
|
+
process/server it came from when one bot serves several:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
devalerts.init(bot_token="...", chat_id=123456789, tags={"env": "production"})
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
```
|
|
146
|
+
🔴 ValueError: boom
|
|
147
|
+
🖥️ prod-web-2 (env=production)
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Add ad-hoc tags to a single call — they override `init()`'s tags on a key
|
|
151
|
+
collision:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
devalerts.report(extra={"request_id": "abc123"})
|
|
155
|
+
|
|
156
|
+
@devalerts.capture(extra={"job": "nightly-sync"})
|
|
157
|
+
def run_job(): ...
|
|
158
|
+
```
|
|
159
|
+
|
|
112
160
|
## Manually reporting a caught exception
|
|
113
161
|
|
|
114
162
|
```python
|
|
@@ -148,6 +196,24 @@ Only exceptions that actually escape as server errors get reported — routing
|
|
|
148
196
|
404s and raised `HTTPException`s are already turned into responses by the
|
|
149
197
|
framework before the middleware sees them.
|
|
150
198
|
|
|
199
|
+
## Celery
|
|
200
|
+
|
|
201
|
+
Same problem as ASGI apps: `init()`'s excepthook never sees exceptions raised
|
|
202
|
+
inside a task, because Celery catches them itself to record the task's
|
|
203
|
+
`FAILURE` state. Call `init_celery()` in addition to `init()`:
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
devalerts.init(bot_token="...", chat_id=123456789)
|
|
207
|
+
devalerts.init_celery()
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
This connects to Celery's `task_failure` signal (fired once a task has
|
|
211
|
+
genuinely failed — retries exhausted or none configured, so retried tasks
|
|
212
|
+
don't spam an alert per attempt) and reports through the same
|
|
213
|
+
grouping/rate-limiting/redaction path as everything else, tagged with the
|
|
214
|
+
task name and id automatically. Requires Celery to already be installed in
|
|
215
|
+
the worker process — it's imported lazily, not a devalerts dependency.
|
|
216
|
+
|
|
151
217
|
## Why not Sentry?
|
|
152
218
|
|
|
153
219
|
If you already run Sentry/Rollbar/etc., keep using it — devalerts isn't a
|
|
@@ -178,8 +244,8 @@ you want dedup state to survive restarts — it self-recreates otherwise.
|
|
|
178
244
|
Yes — `init()` installs both `sys.excepthook` and `threading.excepthook`.
|
|
179
245
|
|
|
180
246
|
**Works with Celery / background workers?**
|
|
181
|
-
Yes
|
|
182
|
-
|
|
247
|
+
Yes — call [`init_celery()`](#celery) in addition to `init()` to catch
|
|
248
|
+
exceptions raised inside tasks, which the excepthook alone won't see.
|
|
183
249
|
|
|
184
250
|
**Works on Windows / Linux / macOS?**
|
|
185
251
|
Yes — stdlib only (`urllib`, `sqlite3`, `threading`), no OS-specific code
|
|
@@ -196,6 +262,9 @@ paths.
|
|
|
196
262
|
- Basic secret redaction only (a few common token/key patterns) — do not
|
|
197
263
|
rely on this for sensitive production data; scrub what you can before it
|
|
198
264
|
ever reaches an exception message.
|
|
265
|
+
- If Telegram delivery fails after retrying, the alert (already redacted, if
|
|
266
|
+
`redact=True`) is appended to `~/.devalerts/failed.log` instead of being
|
|
267
|
+
dropped — clean it up like any other local log file.
|
|
199
268
|
|
|
200
269
|
## What this does NOT do (by design)
|
|
201
270
|
|
|
@@ -50,7 +50,13 @@ chat instead of a log file nobody's watching.
|
|
|
50
50
|
2. Message your bot once (or add it to a group) so it's allowed to message you back.
|
|
51
51
|
3. Get your chat id — message [@userinfobot](https://t.me/userinfobot), or call
|
|
52
52
|
`https://api.telegram.org/bot<TOKEN>/getUpdates` after step 2 and read `message.chat.id`.
|
|
53
|
-
4.
|
|
53
|
+
4. Verify it's wired up correctly before touching any code:
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
uv run devalerts test --bot-token 123456:ABC-DEF... --chat-id 123456789
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
5. In your app, as early as possible:
|
|
54
60
|
|
|
55
61
|
```python
|
|
56
62
|
import devalerts
|
|
@@ -82,6 +88,48 @@ uv run devalerts dashboard
|
|
|
82
88
|
|
|
83
89
|

|
|
84
90
|
|
|
91
|
+
Pass `--json` for machine-readable output. Silence a noisy group without
|
|
92
|
+
touching code — the `ID` column accepts any unique prefix:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
uv run devalerts mute abc12345
|
|
96
|
+
uv run devalerts unmute abc12345
|
|
97
|
+
uv run devalerts clear abc12345 # or: devalerts clear --all
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Unmuting resends the next occurrence with the accumulated skip count, same
|
|
101
|
+
as a rate-limit window expiring.
|
|
102
|
+
|
|
103
|
+
Error groups that keep piling up while suppressed are chronic: each such
|
|
104
|
+
resend doubles the effective `rate_limit_seconds` for that group (capped at
|
|
105
|
+
8x), so a crash loop backs itself off instead of paging you every window
|
|
106
|
+
forever. A group that goes quiet and reappears once resets to the base rate
|
|
107
|
+
immediately. The dashboard shows the active multiplier (`● sending ×4`).
|
|
108
|
+
|
|
109
|
+
## Context: hostname and tags
|
|
110
|
+
|
|
111
|
+
Every alert automatically includes the sending host, so you can tell which
|
|
112
|
+
process/server it came from when one bot serves several:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
devalerts.init(bot_token="...", chat_id=123456789, tags={"env": "production"})
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
🔴 ValueError: boom
|
|
120
|
+
🖥️ prod-web-2 (env=production)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Add ad-hoc tags to a single call — they override `init()`'s tags on a key
|
|
124
|
+
collision:
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
devalerts.report(extra={"request_id": "abc123"})
|
|
128
|
+
|
|
129
|
+
@devalerts.capture(extra={"job": "nightly-sync"})
|
|
130
|
+
def run_job(): ...
|
|
131
|
+
```
|
|
132
|
+
|
|
85
133
|
## Manually reporting a caught exception
|
|
86
134
|
|
|
87
135
|
```python
|
|
@@ -121,6 +169,24 @@ Only exceptions that actually escape as server errors get reported — routing
|
|
|
121
169
|
404s and raised `HTTPException`s are already turned into responses by the
|
|
122
170
|
framework before the middleware sees them.
|
|
123
171
|
|
|
172
|
+
## Celery
|
|
173
|
+
|
|
174
|
+
Same problem as ASGI apps: `init()`'s excepthook never sees exceptions raised
|
|
175
|
+
inside a task, because Celery catches them itself to record the task's
|
|
176
|
+
`FAILURE` state. Call `init_celery()` in addition to `init()`:
|
|
177
|
+
|
|
178
|
+
```python
|
|
179
|
+
devalerts.init(bot_token="...", chat_id=123456789)
|
|
180
|
+
devalerts.init_celery()
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
This connects to Celery's `task_failure` signal (fired once a task has
|
|
184
|
+
genuinely failed — retries exhausted or none configured, so retried tasks
|
|
185
|
+
don't spam an alert per attempt) and reports through the same
|
|
186
|
+
grouping/rate-limiting/redaction path as everything else, tagged with the
|
|
187
|
+
task name and id automatically. Requires Celery to already be installed in
|
|
188
|
+
the worker process — it's imported lazily, not a devalerts dependency.
|
|
189
|
+
|
|
124
190
|
## Why not Sentry?
|
|
125
191
|
|
|
126
192
|
If you already run Sentry/Rollbar/etc., keep using it — devalerts isn't a
|
|
@@ -151,8 +217,8 @@ you want dedup state to survive restarts — it self-recreates otherwise.
|
|
|
151
217
|
Yes — `init()` installs both `sys.excepthook` and `threading.excepthook`.
|
|
152
218
|
|
|
153
219
|
**Works with Celery / background workers?**
|
|
154
|
-
Yes
|
|
155
|
-
|
|
220
|
+
Yes — call [`init_celery()`](#celery) in addition to `init()` to catch
|
|
221
|
+
exceptions raised inside tasks, which the excepthook alone won't see.
|
|
156
222
|
|
|
157
223
|
**Works on Windows / Linux / macOS?**
|
|
158
224
|
Yes — stdlib only (`urllib`, `sqlite3`, `threading`), no OS-specific code
|
|
@@ -169,6 +235,9 @@ paths.
|
|
|
169
235
|
- Basic secret redaction only (a few common token/key patterns) — do not
|
|
170
236
|
rely on this for sensitive production data; scrub what you can before it
|
|
171
237
|
ever reaches an exception message.
|
|
238
|
+
- If Telegram delivery fails after retrying, the alert (already redacted, if
|
|
239
|
+
`redact=True`) is appended to `~/.devalerts/failed.log` instead of being
|
|
240
|
+
dropped — clean it up like any other local log file.
|
|
172
241
|
|
|
173
242
|
## What this does NOT do (by design)
|
|
174
243
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "devalerts"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "Send unhandled Python exceptions straight to a Telegram chat. No backend, no account, no database — just your own bot token."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -48,6 +48,7 @@ build-backend = "uv_build"
|
|
|
48
48
|
|
|
49
49
|
[dependency-groups]
|
|
50
50
|
dev = [
|
|
51
|
+
"celery>=5.3.0",
|
|
51
52
|
"mypy>=1.19.1",
|
|
52
53
|
"pre-commit>=4.3.0",
|
|
53
54
|
"pytest>=8.4.2",
|
|
@@ -9,10 +9,11 @@ from types import TracebackType
|
|
|
9
9
|
from typing import Callable, Literal, Optional, TypedDict
|
|
10
10
|
|
|
11
11
|
from ._alert import _format_alert, _redact
|
|
12
|
+
from ._celery import init_celery
|
|
12
13
|
from ._store import _DEFAULT_RATE_LIMIT_SECONDS, _fingerprint, _should_send
|
|
13
14
|
from ._telegram import _send_telegram_message
|
|
14
15
|
|
|
15
|
-
__all__: list[str] = ["init", "report", "capture", "ASGIMiddleware"]
|
|
16
|
+
__all__: list[str] = ["init", "report", "capture", "ASGIMiddleware", "init_celery"]
|
|
16
17
|
|
|
17
18
|
_ExceptHook = Callable[
|
|
18
19
|
[type[BaseException], BaseException, Optional[TracebackType]], object
|
|
@@ -25,6 +26,7 @@ class _State(TypedDict):
|
|
|
25
26
|
chat_id: int | str | None
|
|
26
27
|
redact: bool
|
|
27
28
|
rate_limit_seconds: int
|
|
29
|
+
tags: dict[str, str]
|
|
28
30
|
# Seeded with the real default hooks below (never None) -- _excepthook/
|
|
29
31
|
# _threading_excepthook can only ever fire after init() has replaced
|
|
30
32
|
# sys.excepthook/threading.excepthook, by which point these are always
|
|
@@ -38,12 +40,15 @@ _state: _State = {
|
|
|
38
40
|
"chat_id": None,
|
|
39
41
|
"redact": True,
|
|
40
42
|
"rate_limit_seconds": _DEFAULT_RATE_LIMIT_SECONDS,
|
|
43
|
+
"tags": {},
|
|
41
44
|
"prev_excepthook": sys.excepthook,
|
|
42
45
|
"prev_threading_excepthook": threading.excepthook,
|
|
43
46
|
}
|
|
44
47
|
|
|
45
48
|
|
|
46
|
-
def _send_exception(
|
|
49
|
+
def _send_exception(
|
|
50
|
+
exc_type, exc_value, tb, extra: dict[str, str] | None = None
|
|
51
|
+
) -> None:
|
|
47
52
|
if _state["bot_token"] is None or _state["chat_id"] is None:
|
|
48
53
|
print("devalerts: init() was not called, dropping alert", file=sys.stderr)
|
|
49
54
|
return
|
|
@@ -53,7 +58,8 @@ def _send_exception(exc_type, exc_value, tb) -> None:
|
|
|
53
58
|
)
|
|
54
59
|
if not send:
|
|
55
60
|
return
|
|
56
|
-
|
|
61
|
+
tags = {**_state["tags"], **(extra or {})}
|
|
62
|
+
message = _format_alert(exc_type, exc_value, tb, skipped=skipped, tags=tags)
|
|
57
63
|
if _state["redact"]:
|
|
58
64
|
message = _redact(message)
|
|
59
65
|
_send_telegram_message(_state["bot_token"], _state["chat_id"], message)
|
|
@@ -87,12 +93,14 @@ def init(
|
|
|
87
93
|
*,
|
|
88
94
|
redact: bool = True,
|
|
89
95
|
rate_limit_seconds: int = _DEFAULT_RATE_LIMIT_SECONDS,
|
|
96
|
+
tags: dict[str, str] | None = None,
|
|
90
97
|
) -> None:
|
|
91
98
|
"""Install a global exception hook that sends unhandled exceptions to Telegram."""
|
|
92
99
|
_state["bot_token"] = bot_token
|
|
93
100
|
_state["chat_id"] = chat_id
|
|
94
101
|
_state["redact"] = redact
|
|
95
102
|
_state["rate_limit_seconds"] = rate_limit_seconds
|
|
103
|
+
_state["tags"] = tags or {}
|
|
96
104
|
# ponytail: guard against calling init() twice capturing our own hook as
|
|
97
105
|
# "previous" -- that would make _excepthook chain to itself and recurse
|
|
98
106
|
# forever on the next crash, violating "must never raise".
|
|
@@ -104,7 +112,9 @@ def init(
|
|
|
104
112
|
threading.excepthook = _threading_excepthook
|
|
105
113
|
|
|
106
114
|
|
|
107
|
-
def report(
|
|
115
|
+
def report(
|
|
116
|
+
exc: BaseException | None = None, *, extra: dict[str, str] | None = None
|
|
117
|
+
) -> None:
|
|
108
118
|
"""Manually send a caught exception to Telegram."""
|
|
109
119
|
if exc is None:
|
|
110
120
|
exc_type, exc_value, tb = sys.exc_info()
|
|
@@ -114,7 +124,7 @@ def report(exc: BaseException | None = None) -> None:
|
|
|
114
124
|
)
|
|
115
125
|
else:
|
|
116
126
|
exc_type, exc_value, tb = type(exc), exc, exc.__traceback__
|
|
117
|
-
_send_exception(exc_type, exc_value, tb)
|
|
127
|
+
_send_exception(exc_type, exc_value, tb, extra=extra)
|
|
118
128
|
|
|
119
129
|
|
|
120
130
|
class capture(contextlib.ContextDecorator):
|
|
@@ -122,12 +132,15 @@ class capture(contextlib.ContextDecorator):
|
|
|
122
132
|
function, then re-raise it. Use ``@capture()`` on a function instead of wrapping
|
|
123
133
|
its body in a manual ``try/except`` or ``with`` block."""
|
|
124
134
|
|
|
135
|
+
def __init__(self, *, extra: dict[str, str] | None = None) -> None:
|
|
136
|
+
self._extra = extra
|
|
137
|
+
|
|
125
138
|
def __enter__(self) -> "capture":
|
|
126
139
|
return self
|
|
127
140
|
|
|
128
141
|
def __exit__(self, exc_type, exc_value, tb) -> Literal[False]:
|
|
129
142
|
if exc_type is not None:
|
|
130
|
-
_send_exception(exc_type, exc_value, tb)
|
|
143
|
+
_send_exception(exc_type, exc_value, tb, extra=self._extra)
|
|
131
144
|
return False
|
|
132
145
|
|
|
133
146
|
|
|
@@ -3,13 +3,27 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import re
|
|
6
|
+
import socket
|
|
6
7
|
import traceback
|
|
7
8
|
|
|
8
9
|
_MAX_MESSAGE_LENGTH = 4096
|
|
9
10
|
|
|
10
11
|
|
|
11
|
-
def
|
|
12
|
-
|
|
12
|
+
def _format_context(tags: dict[str, str] | None) -> str:
|
|
13
|
+
try:
|
|
14
|
+
hostname = socket.gethostname()
|
|
15
|
+
except OSError:
|
|
16
|
+
hostname = "unknown-host"
|
|
17
|
+
line = f"🖥️ {hostname}"
|
|
18
|
+
if tags:
|
|
19
|
+
line += " (" + ", ".join(f"{key}={value}" for key, value in tags.items()) + ")"
|
|
20
|
+
return line
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _format_alert(
|
|
24
|
+
exc_type, exc_value, tb, skipped: int = 0, tags: dict[str, str] | None = None
|
|
25
|
+
) -> str:
|
|
26
|
+
header = f"\U0001f534 {exc_type.__name__}: {exc_value}\n{_format_context(tags)}"
|
|
13
27
|
if skipped:
|
|
14
28
|
header += f"\n⚠️ Повторилась ещё {skipped} раз(а) с последнего алерта"
|
|
15
29
|
body = "".join(traceback.format_exception(exc_type, exc_value, tb))
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Optional Celery integration: report task failures that Celery's own trace
|
|
2
|
+
machinery swallows before they ever reach sys.excepthook -- the same problem
|
|
3
|
+
ASGIMiddleware solves for FastAPI/Starlette request errors."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
_connected = False
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def init_celery() -> None:
|
|
11
|
+
"""Report Celery task failures to Telegram. Call in addition to init(),
|
|
12
|
+
which never sees exceptions raised inside a task -- Celery catches those
|
|
13
|
+
itself to record the task's FAILURE state.
|
|
14
|
+
|
|
15
|
+
Usage::
|
|
16
|
+
|
|
17
|
+
devalerts.init(bot_token="...", chat_id=123456789)
|
|
18
|
+
devalerts.init_celery()
|
|
19
|
+
|
|
20
|
+
Requires Celery to already be installed in the worker process (not a
|
|
21
|
+
devalerts dependency -- imported lazily here).
|
|
22
|
+
"""
|
|
23
|
+
global _connected
|
|
24
|
+
if _connected:
|
|
25
|
+
return
|
|
26
|
+
from celery.signals import task_failure # type: ignore[import-untyped]
|
|
27
|
+
|
|
28
|
+
task_failure.connect(_on_task_failure, weak=False)
|
|
29
|
+
_connected = True
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _on_task_failure(
|
|
33
|
+
sender=None, task_id=None, exception=None, traceback=None, **_kwargs
|
|
34
|
+
) -> None:
|
|
35
|
+
from . import _send_exception
|
|
36
|
+
|
|
37
|
+
if exception is None:
|
|
38
|
+
return
|
|
39
|
+
tags: dict[str, str] = {}
|
|
40
|
+
if task_id is not None:
|
|
41
|
+
tags["task_id"] = str(task_id)
|
|
42
|
+
if sender is not None:
|
|
43
|
+
tags["task"] = getattr(sender, "name", type(sender).__name__)
|
|
44
|
+
exc = exception.with_traceback(traceback) if traceback is not None else exception
|
|
45
|
+
_send_exception(type(exc), exc, exc.__traceback__, extra=tags)
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Local SQLite state: fingerprint-based grouping, dedup, and rate limiting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import sqlite3
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
import traceback
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
_DB_PATH = Path.home() / ".devalerts" / "state.db"
|
|
13
|
+
_RETENTION_SECONDS = 7 * 24 * 3600
|
|
14
|
+
_DEFAULT_RATE_LIMIT_SECONDS = 300
|
|
15
|
+
# ponytail: fixed cap, not configurable -- upgrade to an init() param if
|
|
16
|
+
# users report the ceiling being wrong for their crash-loop cadence.
|
|
17
|
+
_MAX_BACKOFF_MULTIPLIER = 8
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _fingerprint(exc_type, tb) -> tuple[str, str]:
|
|
21
|
+
frames = traceback.extract_tb(tb)
|
|
22
|
+
location = f"{frames[-1].filename}:{frames[-1].lineno}" if frames else "unknown"
|
|
23
|
+
raw = f"{exc_type.__name__}:{location}"
|
|
24
|
+
return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16], location
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _get_connection() -> sqlite3.Connection:
|
|
28
|
+
_DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
29
|
+
conn = sqlite3.connect(_DB_PATH, timeout=5)
|
|
30
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
31
|
+
conn.execute(
|
|
32
|
+
"""
|
|
33
|
+
CREATE TABLE IF NOT EXISTS error_groups (
|
|
34
|
+
fingerprint TEXT PRIMARY KEY,
|
|
35
|
+
exc_type TEXT NOT NULL,
|
|
36
|
+
location TEXT NOT NULL,
|
|
37
|
+
first_seen REAL NOT NULL,
|
|
38
|
+
last_seen REAL NOT NULL,
|
|
39
|
+
last_sent REAL,
|
|
40
|
+
count_since_last_sent INTEGER NOT NULL DEFAULT 0,
|
|
41
|
+
total_count INTEGER NOT NULL DEFAULT 0,
|
|
42
|
+
rate_limit_seconds INTEGER,
|
|
43
|
+
muted INTEGER NOT NULL DEFAULT 0,
|
|
44
|
+
backoff_multiplier INTEGER NOT NULL DEFAULT 1
|
|
45
|
+
)
|
|
46
|
+
"""
|
|
47
|
+
)
|
|
48
|
+
# ponytail: lazy migration for DBs created before these columns existed --
|
|
49
|
+
# ADD COLUMN has no "IF NOT EXISTS", so swallow the duplicate-column error
|
|
50
|
+
# on every subsequent call instead of tracking schema version.
|
|
51
|
+
for ddl in (
|
|
52
|
+
"ALTER TABLE error_groups ADD COLUMN rate_limit_seconds INTEGER",
|
|
53
|
+
"ALTER TABLE error_groups ADD COLUMN muted INTEGER NOT NULL DEFAULT 0",
|
|
54
|
+
"ALTER TABLE error_groups ADD COLUMN backoff_multiplier INTEGER NOT NULL DEFAULT 1",
|
|
55
|
+
):
|
|
56
|
+
try:
|
|
57
|
+
conn.execute(ddl)
|
|
58
|
+
except sqlite3.OperationalError:
|
|
59
|
+
pass
|
|
60
|
+
return conn
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _should_send(
|
|
64
|
+
fingerprint: str, exc_type_name: str, location: str, rate_limit_seconds: int
|
|
65
|
+
) -> tuple[bool, int]:
|
|
66
|
+
now = time.time()
|
|
67
|
+
try:
|
|
68
|
+
conn = _get_connection()
|
|
69
|
+
try:
|
|
70
|
+
with conn:
|
|
71
|
+
row = conn.execute(
|
|
72
|
+
"SELECT last_sent, count_since_last_sent, muted, backoff_multiplier "
|
|
73
|
+
"FROM error_groups WHERE fingerprint = ?",
|
|
74
|
+
(fingerprint,),
|
|
75
|
+
).fetchone()
|
|
76
|
+
muted = bool(row[2]) if row else False
|
|
77
|
+
multiplier = row[3] if row else 1
|
|
78
|
+
effective_window = rate_limit_seconds * multiplier
|
|
79
|
+
if not muted and (
|
|
80
|
+
row is None or row[0] is None or now - row[0] >= effective_window
|
|
81
|
+
):
|
|
82
|
+
send, skipped = True, (row[1] if row else 0)
|
|
83
|
+
# Chronic (occurrences piled up while suppressed) doubles the
|
|
84
|
+
# backoff, capped at _MAX_BACKOFF_MULTIPLIER; a single fresh
|
|
85
|
+
# occurrence after a genuine quiet spell resets it to 1.
|
|
86
|
+
new_multiplier = (
|
|
87
|
+
min(multiplier * 2, _MAX_BACKOFF_MULTIPLIER) if skipped else 1
|
|
88
|
+
)
|
|
89
|
+
conn.execute(
|
|
90
|
+
"""
|
|
91
|
+
INSERT INTO error_groups
|
|
92
|
+
(fingerprint, exc_type, location, first_seen, last_seen,
|
|
93
|
+
last_sent, count_since_last_sent, total_count,
|
|
94
|
+
rate_limit_seconds, backoff_multiplier)
|
|
95
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, 1, ?, ?)
|
|
96
|
+
ON CONFLICT(fingerprint) DO UPDATE SET
|
|
97
|
+
last_seen = excluded.last_seen,
|
|
98
|
+
last_sent = excluded.last_sent,
|
|
99
|
+
count_since_last_sent = 0,
|
|
100
|
+
total_count = total_count + 1,
|
|
101
|
+
rate_limit_seconds = excluded.rate_limit_seconds,
|
|
102
|
+
backoff_multiplier = excluded.backoff_multiplier
|
|
103
|
+
""",
|
|
104
|
+
(
|
|
105
|
+
fingerprint,
|
|
106
|
+
exc_type_name,
|
|
107
|
+
location,
|
|
108
|
+
now,
|
|
109
|
+
now,
|
|
110
|
+
now,
|
|
111
|
+
rate_limit_seconds,
|
|
112
|
+
new_multiplier,
|
|
113
|
+
),
|
|
114
|
+
)
|
|
115
|
+
else:
|
|
116
|
+
# Covers both "still rate-limited" and "muted" -- neither sends,
|
|
117
|
+
# both keep counting so an eventual unmute/window-expiry reports
|
|
118
|
+
# the accumulated skip count via count_since_last_sent.
|
|
119
|
+
send, skipped = False, 0
|
|
120
|
+
conn.execute(
|
|
121
|
+
"""
|
|
122
|
+
UPDATE error_groups
|
|
123
|
+
SET last_seen = ?, count_since_last_sent = count_since_last_sent + 1,
|
|
124
|
+
total_count = total_count + 1, rate_limit_seconds = ?
|
|
125
|
+
WHERE fingerprint = ?
|
|
126
|
+
""",
|
|
127
|
+
(now, rate_limit_seconds, fingerprint),
|
|
128
|
+
)
|
|
129
|
+
conn.execute(
|
|
130
|
+
"DELETE FROM error_groups WHERE last_seen < ?",
|
|
131
|
+
(now - _RETENTION_SECONDS,),
|
|
132
|
+
)
|
|
133
|
+
return send, skipped
|
|
134
|
+
finally:
|
|
135
|
+
conn.close()
|
|
136
|
+
except (sqlite3.Error, OSError) as error:
|
|
137
|
+
# ponytail: dedup/rate-limit state must never block an alert -- fail
|
|
138
|
+
# open (send, as if this were the first occurrence) on any DB error.
|
|
139
|
+
print(
|
|
140
|
+
f"devalerts: dedup/rate-limit state error, sending anyway: {error}",
|
|
141
|
+
file=sys.stderr,
|
|
142
|
+
)
|
|
143
|
+
return True, 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _match_fingerprints(prefix: str) -> list[str]:
|
|
147
|
+
"""Fingerprints starting with prefix, for CLI mute/unmute/clear commands."""
|
|
148
|
+
conn = _get_connection()
|
|
149
|
+
try:
|
|
150
|
+
rows = conn.execute(
|
|
151
|
+
"SELECT fingerprint FROM error_groups WHERE fingerprint LIKE ? ESCAPE '\\'",
|
|
152
|
+
(
|
|
153
|
+
prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
154
|
+
+ "%",
|
|
155
|
+
),
|
|
156
|
+
).fetchall()
|
|
157
|
+
finally:
|
|
158
|
+
conn.close()
|
|
159
|
+
return [row[0] for row in rows]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _set_muted(fingerprint: str, muted: bool) -> None:
|
|
163
|
+
conn = _get_connection()
|
|
164
|
+
try:
|
|
165
|
+
with conn:
|
|
166
|
+
conn.execute(
|
|
167
|
+
"UPDATE error_groups SET muted = ? WHERE fingerprint = ?",
|
|
168
|
+
(int(muted), fingerprint),
|
|
169
|
+
)
|
|
170
|
+
finally:
|
|
171
|
+
conn.close()
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _clear(fingerprint: str) -> None:
|
|
175
|
+
conn = _get_connection()
|
|
176
|
+
try:
|
|
177
|
+
with conn:
|
|
178
|
+
conn.execute(
|
|
179
|
+
"DELETE FROM error_groups WHERE fingerprint = ?", (fingerprint,)
|
|
180
|
+
)
|
|
181
|
+
finally:
|
|
182
|
+
conn.close()
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _clear_all() -> None:
|
|
186
|
+
conn = _get_connection()
|
|
187
|
+
try:
|
|
188
|
+
with conn:
|
|
189
|
+
conn.execute("DELETE FROM error_groups")
|
|
190
|
+
finally:
|
|
191
|
+
conn.close()
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Telegram Bot API delivery -- retries transient failures, logs to a local
|
|
2
|
+
fallback file if every attempt fails. Never raises."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.request
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
_TIMEOUT_SECONDS = 5
|
|
14
|
+
_MAX_ATTEMPTS = 2
|
|
15
|
+
_RETRY_BACKOFF_SECONDS = 1.0
|
|
16
|
+
# ponytail: cap how long we'll honor Telegram's requested Retry-After --
|
|
17
|
+
# beyond this we're blocking an excepthook that's about to exit the process,
|
|
18
|
+
# so give up and fall back to the local log instead of stalling shutdown.
|
|
19
|
+
_MAX_RETRY_AFTER_SECONDS = 10.0
|
|
20
|
+
_FAILED_LOG_PATH = Path.home() / ".devalerts" / "failed.log"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _retry_after_seconds(error: urllib.error.HTTPError) -> float | None:
|
|
24
|
+
try:
|
|
25
|
+
body = json.loads(error.read())
|
|
26
|
+
retry_after = body.get("parameters", {}).get("retry_after")
|
|
27
|
+
return float(retry_after) if retry_after is not None else None
|
|
28
|
+
except (ValueError, AttributeError, TypeError):
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _log_failed_delivery(text: str) -> None:
|
|
33
|
+
try:
|
|
34
|
+
_FAILED_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
with _FAILED_LOG_PATH.open("a", encoding="utf-8") as fh:
|
|
36
|
+
fh.write(f"--- {time.strftime('%Y-%m-%d %H:%M:%S')} ---\n{text}\n\n")
|
|
37
|
+
except OSError as error:
|
|
38
|
+
print(f"devalerts: failed to write fallback log: {error}", file=sys.stderr)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _send_telegram_message(bot_token: str, chat_id: int | str, text: str) -> bool:
|
|
42
|
+
url = f"https://api.telegram.org/bot{bot_token}/sendMessage"
|
|
43
|
+
payload = json.dumps({"chat_id": chat_id, "text": text}).encode("utf-8")
|
|
44
|
+
last_error: Exception | None = None
|
|
45
|
+
|
|
46
|
+
for attempt in range(_MAX_ATTEMPTS):
|
|
47
|
+
request = urllib.request.Request(
|
|
48
|
+
url,
|
|
49
|
+
data=payload,
|
|
50
|
+
headers={"Content-Type": "application/json"},
|
|
51
|
+
method="POST",
|
|
52
|
+
)
|
|
53
|
+
wait: float = _RETRY_BACKOFF_SECONDS
|
|
54
|
+
try:
|
|
55
|
+
urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS)
|
|
56
|
+
return True
|
|
57
|
+
except urllib.error.HTTPError as error:
|
|
58
|
+
last_error = error
|
|
59
|
+
retry_after = _retry_after_seconds(error)
|
|
60
|
+
if retry_after is not None:
|
|
61
|
+
if retry_after > _MAX_RETRY_AFTER_SECONDS:
|
|
62
|
+
break
|
|
63
|
+
wait = retry_after
|
|
64
|
+
except (urllib.error.URLError, OSError, ValueError) as error:
|
|
65
|
+
last_error = error
|
|
66
|
+
|
|
67
|
+
if attempt < _MAX_ATTEMPTS - 1:
|
|
68
|
+
time.sleep(wait)
|
|
69
|
+
|
|
70
|
+
print(f"devalerts: failed to send Telegram alert: {last_error}", file=sys.stderr)
|
|
71
|
+
_log_failed_delivery(text)
|
|
72
|
+
return False
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""CLI: `devalerts dashboard` reports grouped/rate-limited errors from the local
|
|
2
|
+
state DB; `devalerts test` sends a one-off message to verify bot_token/chat_id."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import sqlite3
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from importlib.metadata import version as _pkg_version
|
|
13
|
+
|
|
14
|
+
from ._store import (
|
|
15
|
+
_DB_PATH,
|
|
16
|
+
_DEFAULT_RATE_LIMIT_SECONDS,
|
|
17
|
+
_clear,
|
|
18
|
+
_clear_all,
|
|
19
|
+
_match_fingerprints,
|
|
20
|
+
_set_muted,
|
|
21
|
+
)
|
|
22
|
+
from ._telegram import _send_telegram_message
|
|
23
|
+
|
|
24
|
+
_LOCATION_WIDTH = 34
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _supports_unicode() -> bool:
|
|
28
|
+
encoding = getattr(sys.stdout, "encoding", None) or ""
|
|
29
|
+
try:
|
|
30
|
+
"─●…".encode(encoding)
|
|
31
|
+
return True
|
|
32
|
+
except (LookupError, UnicodeEncodeError, TypeError):
|
|
33
|
+
return False
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _truncate(text: str, width: int, ellipsis: str) -> str:
|
|
37
|
+
"""Keep the tail (file:line matters more than leading directories)."""
|
|
38
|
+
if len(text) <= width:
|
|
39
|
+
return text
|
|
40
|
+
keep = width - len(ellipsis)
|
|
41
|
+
return ellipsis + text[-keep:] if keep > 0 else ellipsis[:width]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _relative_time(ts: float, now: float) -> str:
|
|
45
|
+
delta = now - ts
|
|
46
|
+
if delta < 60:
|
|
47
|
+
return "just now"
|
|
48
|
+
if delta < 3600:
|
|
49
|
+
return f"{int(delta // 60)}m ago"
|
|
50
|
+
if delta < 86400:
|
|
51
|
+
return f"{int(delta // 3600)}h ago"
|
|
52
|
+
return f"{int(delta // 86400)}d ago"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _color_enabled() -> bool:
|
|
56
|
+
if os.environ.get("NO_COLOR") is not None or not sys.stdout.isatty():
|
|
57
|
+
return False
|
|
58
|
+
if sys.platform == "win32":
|
|
59
|
+
# ponytail: enable VT100 processing for legacy conhost.exe -- Windows
|
|
60
|
+
# Terminal / PowerShell 7 already support ANSI without this.
|
|
61
|
+
import ctypes
|
|
62
|
+
|
|
63
|
+
kernel32 = ctypes.windll.kernel32
|
|
64
|
+
handle = kernel32.GetStdHandle(-11)
|
|
65
|
+
mode = ctypes.c_uint32()
|
|
66
|
+
if not kernel32.GetConsoleMode(handle, ctypes.byref(mode)):
|
|
67
|
+
return False
|
|
68
|
+
kernel32.SetConsoleMode(handle, mode.value | 0x0004)
|
|
69
|
+
return True
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class _Style:
|
|
73
|
+
def __init__(self, enabled: bool) -> None:
|
|
74
|
+
self._enabled = enabled
|
|
75
|
+
|
|
76
|
+
def _wrap(self, code: str, text: str) -> str:
|
|
77
|
+
return f"\033[{code}m{text}\033[0m" if self._enabled else text
|
|
78
|
+
|
|
79
|
+
def bold(self, text: str) -> str:
|
|
80
|
+
return self._wrap("1", text)
|
|
81
|
+
|
|
82
|
+
def dim(self, text: str) -> str:
|
|
83
|
+
return self._wrap("2", text)
|
|
84
|
+
|
|
85
|
+
def red(self, text: str) -> str:
|
|
86
|
+
return self._wrap("31", text)
|
|
87
|
+
|
|
88
|
+
def green(self, text: str) -> str:
|
|
89
|
+
return self._wrap("32", text)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _dashboard(as_json: bool = False) -> int:
|
|
93
|
+
if not _DB_PATH.exists():
|
|
94
|
+
print("[]" if as_json else "No errors recorded yet.")
|
|
95
|
+
return 0
|
|
96
|
+
|
|
97
|
+
conn = sqlite3.connect(_DB_PATH)
|
|
98
|
+
try:
|
|
99
|
+
rows = conn.execute(
|
|
100
|
+
"SELECT fingerprint, exc_type, location, last_seen, last_sent, total_count, "
|
|
101
|
+
"count_since_last_sent, rate_limit_seconds, muted, backoff_multiplier "
|
|
102
|
+
"FROM error_groups ORDER BY last_seen DESC"
|
|
103
|
+
).fetchall()
|
|
104
|
+
finally:
|
|
105
|
+
conn.close()
|
|
106
|
+
|
|
107
|
+
if not rows:
|
|
108
|
+
print("[]" if as_json else "No errors recorded yet.")
|
|
109
|
+
return 0
|
|
110
|
+
|
|
111
|
+
now = time.time()
|
|
112
|
+
|
|
113
|
+
if as_json:
|
|
114
|
+
groups = []
|
|
115
|
+
for (
|
|
116
|
+
fingerprint,
|
|
117
|
+
exc_type,
|
|
118
|
+
location,
|
|
119
|
+
last_seen,
|
|
120
|
+
last_sent,
|
|
121
|
+
total_count,
|
|
122
|
+
count_since_last_sent,
|
|
123
|
+
rate_limit_seconds,
|
|
124
|
+
muted,
|
|
125
|
+
backoff_multiplier,
|
|
126
|
+
) in rows:
|
|
127
|
+
limit = (
|
|
128
|
+
rate_limit_seconds
|
|
129
|
+
if rate_limit_seconds is not None
|
|
130
|
+
else _DEFAULT_RATE_LIMIT_SECONDS
|
|
131
|
+
) * backoff_multiplier
|
|
132
|
+
rate_limited = (
|
|
133
|
+
not muted and last_sent is not None and now - last_sent < limit
|
|
134
|
+
)
|
|
135
|
+
groups.append(
|
|
136
|
+
{
|
|
137
|
+
"fingerprint": fingerprint,
|
|
138
|
+
"exc_type": exc_type,
|
|
139
|
+
"location": location,
|
|
140
|
+
"last_seen": last_seen,
|
|
141
|
+
"last_sent": last_sent,
|
|
142
|
+
"total_count": total_count,
|
|
143
|
+
"count_since_last_sent": count_since_last_sent,
|
|
144
|
+
"rate_limited": rate_limited,
|
|
145
|
+
"muted": bool(muted),
|
|
146
|
+
"backoff_multiplier": backoff_multiplier,
|
|
147
|
+
}
|
|
148
|
+
)
|
|
149
|
+
print(json.dumps(groups))
|
|
150
|
+
return 0
|
|
151
|
+
|
|
152
|
+
unicode_ok = _supports_unicode()
|
|
153
|
+
sep_char = "─" if unicode_ok else "-"
|
|
154
|
+
dot_char = "●" if unicode_ok else "*"
|
|
155
|
+
ellipsis = "…" if unicode_ok else "..."
|
|
156
|
+
|
|
157
|
+
style = _Style(_color_enabled())
|
|
158
|
+
type_width = max(len("TYPE"), *(len(r[1]) for r in rows))
|
|
159
|
+
|
|
160
|
+
header = (
|
|
161
|
+
f" {'ID':<8} {'TYPE':<{type_width}} {'LOCATION':<{_LOCATION_WIDTH}} "
|
|
162
|
+
f"{'LAST SEEN':<10}{'TOTAL':>6} STATUS"
|
|
163
|
+
)
|
|
164
|
+
print(style.bold(header))
|
|
165
|
+
print(" " + sep_char * (len(header) - 2))
|
|
166
|
+
|
|
167
|
+
limited_count = 0
|
|
168
|
+
muted_count = 0
|
|
169
|
+
for (
|
|
170
|
+
fingerprint,
|
|
171
|
+
exc_type,
|
|
172
|
+
location,
|
|
173
|
+
last_seen,
|
|
174
|
+
last_sent,
|
|
175
|
+
total_count,
|
|
176
|
+
_count_since_last_sent,
|
|
177
|
+
rate_limit_seconds,
|
|
178
|
+
muted,
|
|
179
|
+
backoff_multiplier,
|
|
180
|
+
) in rows:
|
|
181
|
+
backoff_suffix = f" ×{backoff_multiplier}" if backoff_multiplier > 1 else ""
|
|
182
|
+
if muted:
|
|
183
|
+
muted_count += 1
|
|
184
|
+
status = style.dim(f"{dot_char} muted")
|
|
185
|
+
else:
|
|
186
|
+
limit = (
|
|
187
|
+
rate_limit_seconds
|
|
188
|
+
if rate_limit_seconds is not None
|
|
189
|
+
else _DEFAULT_RATE_LIMIT_SECONDS
|
|
190
|
+
) * backoff_multiplier
|
|
191
|
+
in_window = last_sent is not None and now - last_sent < limit
|
|
192
|
+
if in_window:
|
|
193
|
+
limited_count += 1
|
|
194
|
+
status = style.red(f"{dot_char} limited{backoff_suffix}")
|
|
195
|
+
else:
|
|
196
|
+
status = style.green(f"{dot_char} sending{backoff_suffix}")
|
|
197
|
+
|
|
198
|
+
row = (
|
|
199
|
+
f" {style.dim(f'{fingerprint[:8]:<8}')} "
|
|
200
|
+
f"{exc_type:<{type_width}} "
|
|
201
|
+
f"{_truncate(location, _LOCATION_WIDTH, ellipsis):<{_LOCATION_WIDTH}} "
|
|
202
|
+
f"{_relative_time(last_seen, now):<10}"
|
|
203
|
+
f"{total_count:>6} {status}"
|
|
204
|
+
)
|
|
205
|
+
print(row)
|
|
206
|
+
|
|
207
|
+
plural = "" if len(rows) == 1 else "s"
|
|
208
|
+
print(
|
|
209
|
+
f"\n{len(rows)} error group{plural}, {limited_count} currently rate-limited, "
|
|
210
|
+
f"{muted_count} muted."
|
|
211
|
+
)
|
|
212
|
+
return 0
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _resolve_fingerprint(prefix: str) -> str | None:
|
|
216
|
+
matches = _match_fingerprints(prefix)
|
|
217
|
+
if not matches:
|
|
218
|
+
print(f"No error group matches '{prefix}'.", file=sys.stderr)
|
|
219
|
+
return None
|
|
220
|
+
if len(matches) > 1:
|
|
221
|
+
print(
|
|
222
|
+
f"'{prefix}' matches {len(matches)} error groups, be more specific.",
|
|
223
|
+
file=sys.stderr,
|
|
224
|
+
)
|
|
225
|
+
return None
|
|
226
|
+
return matches[0]
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _mute_command(prefix: str, muted: bool) -> int:
|
|
230
|
+
fingerprint = _resolve_fingerprint(prefix)
|
|
231
|
+
if fingerprint is None:
|
|
232
|
+
return 1
|
|
233
|
+
_set_muted(fingerprint, muted)
|
|
234
|
+
print(f"{'Muted' if muted else 'Unmuted'} {fingerprint[:8]}.")
|
|
235
|
+
return 0
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _clear_command(prefix: str | None, clear_all: bool) -> int:
|
|
239
|
+
if clear_all:
|
|
240
|
+
_clear_all()
|
|
241
|
+
print("Cleared all error groups.")
|
|
242
|
+
return 0
|
|
243
|
+
# argparse's mutually exclusive group guarantees exactly one is set.
|
|
244
|
+
assert prefix is not None
|
|
245
|
+
fingerprint = _resolve_fingerprint(prefix)
|
|
246
|
+
if fingerprint is None:
|
|
247
|
+
return 1
|
|
248
|
+
_clear(fingerprint)
|
|
249
|
+
print(f"Cleared {fingerprint[:8]}.")
|
|
250
|
+
return 0
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _test(bot_token: str, chat_id: str) -> int:
|
|
254
|
+
ok = _send_telegram_message(
|
|
255
|
+
bot_token,
|
|
256
|
+
chat_id,
|
|
257
|
+
"✅ devalerts test message -- bot_token and chat_id are wired up correctly.",
|
|
258
|
+
)
|
|
259
|
+
if not ok:
|
|
260
|
+
print("Failed to send test message (see error above).", file=sys.stderr)
|
|
261
|
+
return 1
|
|
262
|
+
print("Test message sent -- check your Telegram chat.")
|
|
263
|
+
return 0
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def main(argv: list[str] | None = None) -> int:
|
|
267
|
+
parser = argparse.ArgumentParser(prog="devalerts")
|
|
268
|
+
parser.add_argument(
|
|
269
|
+
"--version", action="version", version=f"devalerts {_pkg_version('devalerts')}"
|
|
270
|
+
)
|
|
271
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
272
|
+
dashboard_parser = subparsers.add_parser(
|
|
273
|
+
"dashboard", help="Show grouped/rate-limited errors"
|
|
274
|
+
)
|
|
275
|
+
dashboard_parser.add_argument(
|
|
276
|
+
"--json", action="store_true", help="Output as JSON instead of a table"
|
|
277
|
+
)
|
|
278
|
+
test_parser = subparsers.add_parser(
|
|
279
|
+
"test", help="Send a test message to verify bot_token/chat_id"
|
|
280
|
+
)
|
|
281
|
+
test_parser.add_argument("--bot-token", required=True)
|
|
282
|
+
test_parser.add_argument("--chat-id", required=True)
|
|
283
|
+
mute_parser = subparsers.add_parser("mute", help="Silence a specific error group")
|
|
284
|
+
mute_parser.add_argument(
|
|
285
|
+
"fingerprint", help="Fingerprint or unique prefix (dashboard ID column)"
|
|
286
|
+
)
|
|
287
|
+
unmute_parser = subparsers.add_parser(
|
|
288
|
+
"unmute", help="Re-enable alerts for a muted error group"
|
|
289
|
+
)
|
|
290
|
+
unmute_parser.add_argument("fingerprint", help="Fingerprint or unique prefix")
|
|
291
|
+
clear_parser = subparsers.add_parser(
|
|
292
|
+
"clear", help="Delete error group(s) from local state"
|
|
293
|
+
)
|
|
294
|
+
clear_target = clear_parser.add_mutually_exclusive_group(required=True)
|
|
295
|
+
clear_target.add_argument(
|
|
296
|
+
"fingerprint", nargs="?", help="Fingerprint or unique prefix"
|
|
297
|
+
)
|
|
298
|
+
clear_target.add_argument(
|
|
299
|
+
"--all", action="store_true", help="Delete all error groups"
|
|
300
|
+
)
|
|
301
|
+
args = parser.parse_args(argv)
|
|
302
|
+
|
|
303
|
+
if args.command == "dashboard":
|
|
304
|
+
return _dashboard(as_json=args.json)
|
|
305
|
+
if args.command == "test":
|
|
306
|
+
return _test(args.bot_token, args.chat_id)
|
|
307
|
+
if args.command == "mute":
|
|
308
|
+
return _mute_command(args.fingerprint, muted=True)
|
|
309
|
+
if args.command == "unmute":
|
|
310
|
+
return _mute_command(args.fingerprint, muted=False)
|
|
311
|
+
if args.command == "clear":
|
|
312
|
+
return _clear_command(args.fingerprint, args.all)
|
|
313
|
+
return 1
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
if __name__ == "__main__":
|
|
317
|
+
sys.exit(main())
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
"""Local SQLite state: fingerprint-based grouping, dedup, and rate limiting."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import hashlib
|
|
6
|
-
import sqlite3
|
|
7
|
-
import sys
|
|
8
|
-
import time
|
|
9
|
-
import traceback
|
|
10
|
-
from pathlib import Path
|
|
11
|
-
|
|
12
|
-
_DB_PATH = Path.home() / ".devalerts" / "state.db"
|
|
13
|
-
_RETENTION_SECONDS = 7 * 24 * 3600
|
|
14
|
-
_DEFAULT_RATE_LIMIT_SECONDS = 300
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def _fingerprint(exc_type, tb) -> tuple[str, str]:
|
|
18
|
-
frames = traceback.extract_tb(tb)
|
|
19
|
-
location = f"{frames[-1].filename}:{frames[-1].lineno}" if frames else "unknown"
|
|
20
|
-
raw = f"{exc_type.__name__}:{location}"
|
|
21
|
-
return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16], location
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def _get_connection() -> sqlite3.Connection:
|
|
25
|
-
_DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
26
|
-
conn = sqlite3.connect(_DB_PATH, timeout=5)
|
|
27
|
-
conn.execute("PRAGMA journal_mode=WAL")
|
|
28
|
-
conn.execute(
|
|
29
|
-
"""
|
|
30
|
-
CREATE TABLE IF NOT EXISTS error_groups (
|
|
31
|
-
fingerprint TEXT PRIMARY KEY,
|
|
32
|
-
exc_type TEXT NOT NULL,
|
|
33
|
-
location TEXT NOT NULL,
|
|
34
|
-
first_seen REAL NOT NULL,
|
|
35
|
-
last_seen REAL NOT NULL,
|
|
36
|
-
last_sent REAL,
|
|
37
|
-
count_since_last_sent INTEGER NOT NULL DEFAULT 0,
|
|
38
|
-
total_count INTEGER NOT NULL DEFAULT 0
|
|
39
|
-
)
|
|
40
|
-
"""
|
|
41
|
-
)
|
|
42
|
-
return conn
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def _should_send(
|
|
46
|
-
fingerprint: str, exc_type_name: str, location: str, rate_limit_seconds: int
|
|
47
|
-
) -> tuple[bool, int]:
|
|
48
|
-
now = time.time()
|
|
49
|
-
try:
|
|
50
|
-
conn = _get_connection()
|
|
51
|
-
try:
|
|
52
|
-
with conn:
|
|
53
|
-
row = conn.execute(
|
|
54
|
-
"SELECT last_sent, count_since_last_sent FROM error_groups WHERE fingerprint = ?",
|
|
55
|
-
(fingerprint,),
|
|
56
|
-
).fetchone()
|
|
57
|
-
if row is None or row[0] is None or now - row[0] >= rate_limit_seconds:
|
|
58
|
-
send, skipped = True, (row[1] if row else 0)
|
|
59
|
-
conn.execute(
|
|
60
|
-
"""
|
|
61
|
-
INSERT INTO error_groups
|
|
62
|
-
(fingerprint, exc_type, location, first_seen, last_seen,
|
|
63
|
-
last_sent, count_since_last_sent, total_count)
|
|
64
|
-
VALUES (?, ?, ?, ?, ?, ?, 0, 1)
|
|
65
|
-
ON CONFLICT(fingerprint) DO UPDATE SET
|
|
66
|
-
last_seen = excluded.last_seen,
|
|
67
|
-
last_sent = excluded.last_sent,
|
|
68
|
-
count_since_last_sent = 0,
|
|
69
|
-
total_count = total_count + 1
|
|
70
|
-
""",
|
|
71
|
-
(fingerprint, exc_type_name, location, now, now, now),
|
|
72
|
-
)
|
|
73
|
-
else:
|
|
74
|
-
send, skipped = False, 0
|
|
75
|
-
conn.execute(
|
|
76
|
-
"""
|
|
77
|
-
UPDATE error_groups
|
|
78
|
-
SET last_seen = ?, count_since_last_sent = count_since_last_sent + 1,
|
|
79
|
-
total_count = total_count + 1
|
|
80
|
-
WHERE fingerprint = ?
|
|
81
|
-
""",
|
|
82
|
-
(now, fingerprint),
|
|
83
|
-
)
|
|
84
|
-
conn.execute(
|
|
85
|
-
"DELETE FROM error_groups WHERE last_seen < ?",
|
|
86
|
-
(now - _RETENTION_SECONDS,),
|
|
87
|
-
)
|
|
88
|
-
return send, skipped
|
|
89
|
-
finally:
|
|
90
|
-
conn.close()
|
|
91
|
-
except (sqlite3.Error, OSError) as error:
|
|
92
|
-
# ponytail: dedup/rate-limit state must never block an alert -- fail
|
|
93
|
-
# open (send, as if this were the first occurrence) on any DB error.
|
|
94
|
-
print(
|
|
95
|
-
f"devalerts: dedup/rate-limit state error, sending anyway: {error}",
|
|
96
|
-
file=sys.stderr,
|
|
97
|
-
)
|
|
98
|
-
return True, 0
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
"""Telegram Bot API delivery -- never raises on network failure."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import json
|
|
6
|
-
import sys
|
|
7
|
-
import urllib.error
|
|
8
|
-
import urllib.request
|
|
9
|
-
|
|
10
|
-
_TIMEOUT_SECONDS = 5
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def _send_telegram_message(bot_token: str, chat_id: int | str, text: str) -> None:
|
|
14
|
-
url = f"https://api.telegram.org/bot{bot_token}/sendMessage"
|
|
15
|
-
payload = json.dumps({"chat_id": chat_id, "text": text}).encode("utf-8")
|
|
16
|
-
request = urllib.request.Request(
|
|
17
|
-
url,
|
|
18
|
-
data=payload,
|
|
19
|
-
headers={"Content-Type": "application/json"},
|
|
20
|
-
method="POST",
|
|
21
|
-
)
|
|
22
|
-
try:
|
|
23
|
-
urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS)
|
|
24
|
-
except (urllib.error.URLError, OSError, ValueError) as error:
|
|
25
|
-
print(f"devalerts: failed to send Telegram alert: {error}", file=sys.stderr)
|
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
"""CLI: `devalerts dashboard` reports grouped/rate-limited errors from the local state DB."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import argparse
|
|
6
|
-
import os
|
|
7
|
-
import sqlite3
|
|
8
|
-
import sys
|
|
9
|
-
import time
|
|
10
|
-
|
|
11
|
-
from ._store import _DB_PATH, _DEFAULT_RATE_LIMIT_SECONDS
|
|
12
|
-
|
|
13
|
-
_LOCATION_WIDTH = 34
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def _supports_unicode() -> bool:
|
|
17
|
-
encoding = getattr(sys.stdout, "encoding", None) or ""
|
|
18
|
-
try:
|
|
19
|
-
"─●…".encode(encoding)
|
|
20
|
-
return True
|
|
21
|
-
except (LookupError, UnicodeEncodeError, TypeError):
|
|
22
|
-
return False
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def _truncate(text: str, width: int, ellipsis: str) -> str:
|
|
26
|
-
"""Keep the tail (file:line matters more than leading directories)."""
|
|
27
|
-
if len(text) <= width:
|
|
28
|
-
return text
|
|
29
|
-
keep = width - len(ellipsis)
|
|
30
|
-
return ellipsis + text[-keep:] if keep > 0 else ellipsis[:width]
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def _relative_time(ts: float, now: float) -> str:
|
|
34
|
-
delta = now - ts
|
|
35
|
-
if delta < 60:
|
|
36
|
-
return "just now"
|
|
37
|
-
if delta < 3600:
|
|
38
|
-
return f"{int(delta // 60)}m ago"
|
|
39
|
-
if delta < 86400:
|
|
40
|
-
return f"{int(delta // 3600)}h ago"
|
|
41
|
-
return f"{int(delta // 86400)}d ago"
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def _color_enabled() -> bool:
|
|
45
|
-
if os.environ.get("NO_COLOR") is not None or not sys.stdout.isatty():
|
|
46
|
-
return False
|
|
47
|
-
if sys.platform == "win32":
|
|
48
|
-
# ponytail: enable VT100 processing for legacy conhost.exe -- Windows
|
|
49
|
-
# Terminal / PowerShell 7 already support ANSI without this.
|
|
50
|
-
import ctypes
|
|
51
|
-
|
|
52
|
-
kernel32 = ctypes.windll.kernel32
|
|
53
|
-
handle = kernel32.GetStdHandle(-11)
|
|
54
|
-
mode = ctypes.c_uint32()
|
|
55
|
-
if not kernel32.GetConsoleMode(handle, ctypes.byref(mode)):
|
|
56
|
-
return False
|
|
57
|
-
kernel32.SetConsoleMode(handle, mode.value | 0x0004)
|
|
58
|
-
return True
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
class _Style:
|
|
62
|
-
def __init__(self, enabled: bool) -> None:
|
|
63
|
-
self._enabled = enabled
|
|
64
|
-
|
|
65
|
-
def _wrap(self, code: str, text: str) -> str:
|
|
66
|
-
return f"\033[{code}m{text}\033[0m" if self._enabled else text
|
|
67
|
-
|
|
68
|
-
def bold(self, text: str) -> str:
|
|
69
|
-
return self._wrap("1", text)
|
|
70
|
-
|
|
71
|
-
def dim(self, text: str) -> str:
|
|
72
|
-
return self._wrap("2", text)
|
|
73
|
-
|
|
74
|
-
def red(self, text: str) -> str:
|
|
75
|
-
return self._wrap("31", text)
|
|
76
|
-
|
|
77
|
-
def green(self, text: str) -> str:
|
|
78
|
-
return self._wrap("32", text)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
def _dashboard() -> int:
|
|
82
|
-
if not _DB_PATH.exists():
|
|
83
|
-
print("No errors recorded yet.")
|
|
84
|
-
return 0
|
|
85
|
-
|
|
86
|
-
conn = sqlite3.connect(_DB_PATH)
|
|
87
|
-
try:
|
|
88
|
-
rows = conn.execute(
|
|
89
|
-
"SELECT fingerprint, exc_type, location, last_seen, last_sent, total_count "
|
|
90
|
-
"FROM error_groups ORDER BY last_seen DESC"
|
|
91
|
-
).fetchall()
|
|
92
|
-
finally:
|
|
93
|
-
conn.close()
|
|
94
|
-
|
|
95
|
-
if not rows:
|
|
96
|
-
print("No errors recorded yet.")
|
|
97
|
-
return 0
|
|
98
|
-
|
|
99
|
-
unicode_ok = _supports_unicode()
|
|
100
|
-
sep_char = "─" if unicode_ok else "-"
|
|
101
|
-
dot_char = "●" if unicode_ok else "*"
|
|
102
|
-
ellipsis = "…" if unicode_ok else "..."
|
|
103
|
-
|
|
104
|
-
style = _Style(_color_enabled())
|
|
105
|
-
now = time.time()
|
|
106
|
-
type_width = max(len("TYPE"), *(len(r[1]) for r in rows))
|
|
107
|
-
|
|
108
|
-
header = (
|
|
109
|
-
f" {'ID':<8} {'TYPE':<{type_width}} {'LOCATION':<{_LOCATION_WIDTH}} "
|
|
110
|
-
f"{'LAST SEEN':<10}{'TOTAL':>6} STATUS"
|
|
111
|
-
)
|
|
112
|
-
print(style.bold(header))
|
|
113
|
-
print(" " + sep_char * (len(header) - 2))
|
|
114
|
-
|
|
115
|
-
limited_count = 0
|
|
116
|
-
for fingerprint, exc_type, location, last_seen, last_sent, total_count in rows:
|
|
117
|
-
# ponytail: dashboard runs in a separate process from init(), so it
|
|
118
|
-
# doesn't know the app's actual rate_limit_seconds -- uses the
|
|
119
|
-
# library default. Upgrade to persisting the configured value if
|
|
120
|
-
# apps commonly override it.
|
|
121
|
-
in_window = (
|
|
122
|
-
last_sent is not None and now - last_sent < _DEFAULT_RATE_LIMIT_SECONDS
|
|
123
|
-
)
|
|
124
|
-
if in_window:
|
|
125
|
-
limited_count += 1
|
|
126
|
-
status = style.red(f"{dot_char} limited")
|
|
127
|
-
else:
|
|
128
|
-
status = style.green(f"{dot_char} sending")
|
|
129
|
-
|
|
130
|
-
row = (
|
|
131
|
-
f" {style.dim(f'{fingerprint[:8]:<8}')} "
|
|
132
|
-
f"{exc_type:<{type_width}} "
|
|
133
|
-
f"{_truncate(location, _LOCATION_WIDTH, ellipsis):<{_LOCATION_WIDTH}} "
|
|
134
|
-
f"{_relative_time(last_seen, now):<10}"
|
|
135
|
-
f"{total_count:>6} {status}"
|
|
136
|
-
)
|
|
137
|
-
print(row)
|
|
138
|
-
|
|
139
|
-
plural = "" if len(rows) == 1 else "s"
|
|
140
|
-
print(f"\n{len(rows)} error group{plural}, {limited_count} currently rate-limited.")
|
|
141
|
-
return 0
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
def main(argv: list[str] | None = None) -> int:
|
|
145
|
-
parser = argparse.ArgumentParser(prog="devalerts")
|
|
146
|
-
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
147
|
-
subparsers.add_parser("dashboard", help="Show grouped/rate-limited errors")
|
|
148
|
-
args = parser.parse_args(argv)
|
|
149
|
-
|
|
150
|
-
if args.command == "dashboard":
|
|
151
|
-
return _dashboard()
|
|
152
|
-
return 1
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
if __name__ == "__main__":
|
|
156
|
-
sys.exit(main())
|
|
File without changes
|
|
File without changes
|