expr-tracker 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/.gitignore +5 -0
- expr_tracker-0.2.2/LICENSE +21 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/PKG-INFO +31 -2
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/README.md +9 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/guide/alerts.md +63 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/reference/configuration.md +2 -1
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/pyproject.toml +34 -1
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/backends/__init__.py +5 -1
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/backends/base.py +70 -1
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/engine.py +6 -1
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/models.py +3 -0
- expr_tracker-0.2.2/src/expr_tracker/py.typed +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_alert_backends.py +179 -1
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_rule_lifecycle.py +31 -0
- expr_tracker-0.2.1/outputs/hello/test/metrics.jsonl +0 -5
- expr_tracker-0.2.1/outputs/hello/test/metrics.meta.json +0 -1
- expr_tracker-0.2.1/outputs/hello/test/summary.json +0 -6
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/.github/workflows/docs.yaml +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/.github/workflows/release.yaml +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/architecture.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/design.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/getting-started.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/guide/artifacts.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/guide/backends.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/guide/cli.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/guide/distributed.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/guide/history.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/guide/logging.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/index.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/reference/api.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/docs/reference/expressions.md +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/mkdocs.yml +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/__init__.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/_compat.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/__init__.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/dispatch.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/expr/__init__.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/expr/eval.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/expr/functions.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/expr/lexer.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/expr/nodes.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/expr/parser.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/alerts/expr/rule.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/artifacts.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/cli.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/encoders.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/history/__init__.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/history/codec.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/history/frame.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/history/reader.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/history/series.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/history/store.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/history/writer.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/run.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/summary.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/tracker.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/src/expr_tracker/types.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/conftest.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_alert_delivery.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_alert_dispatch.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_alert_engine.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_alert_models.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_alert_routing.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_artifacts.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_benchmark.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_cache.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_cli.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_correctness.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_distributed.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_e2e.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_expr_builder.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_expr_eval.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_expr_functions.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_expr_parser.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_expr_properties.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_failure_modes.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_features.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_frame_codec_summary.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_history.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_hot_paths.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_integration.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_lark_live.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_perf.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_public_surfaces.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_review_regressions.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_run_backends.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_scenarios.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_step_commit.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_stress.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_trackio.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_value_encoding.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_wandb.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_writer_buffer.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/tests/test_writer_durability.py +0 -0
- {expr_tracker-0.2.1 → expr_tracker-0.2.2}/uv.lock +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hangxing Wei
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,8 +1,28 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: expr_tracker
|
|
3
|
-
Version: 0.2.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Local-first experiment tracking with queryable history and expression-based alerts on your training metrics
|
|
5
|
+
Project-URL: Homepage, https://hspk.github.io/expr_tracker/
|
|
6
|
+
Project-URL: Documentation, https://hspk.github.io/expr_tracker/
|
|
7
|
+
Project-URL: Repository, https://github.com/HSPK/expr_tracker
|
|
8
|
+
Project-URL: Issues, https://github.com/HSPK/expr_tracker/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/HSPK/expr_tracker/releases
|
|
5
10
|
Author-email: HSPK <whxway@whu.edu.cn>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: alerting,deep-learning,experiment-tracking,machine-learning,mlops,monitoring,training,wandb
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: System :: Monitoring
|
|
25
|
+
Classifier: Typing :: Typed
|
|
6
26
|
Requires-Python: >=3.10
|
|
7
27
|
Requires-Dist: click>=8.1.0
|
|
8
28
|
Requires-Dist: loguru>=0.7.3
|
|
@@ -27,6 +47,11 @@ Description-Content-Type: text/markdown
|
|
|
27
47
|
|
|
28
48
|
# Experiment Tracker
|
|
29
49
|
|
|
50
|
+
[](https://pypi.org/project/expr-tracker/)
|
|
51
|
+
[](https://pypi.org/project/expr-tracker/)
|
|
52
|
+
[](LICENSE)
|
|
53
|
+
[](https://hspk.github.io/expr_tracker/)
|
|
54
|
+
|
|
30
55
|
A local-first experiment tracker. Metrics land in a JSONL file you own, stay
|
|
31
56
|
queryable while the run is live, and can trigger alerts from an expression language.
|
|
32
57
|
`wandb` and `trackio` are optional mirrors, not requirements.
|
|
@@ -141,3 +166,7 @@ Published to GitHub Pages by `.github/workflows/docs.yaml` on every push to `mai
|
|
|
141
166
|
Internals: [`docs/design.md`](docs/design.md) (data model and key invariants) and
|
|
142
167
|
[`docs/architecture.md`](docs/architecture.md) (module map, read/write paths,
|
|
143
168
|
concurrency model).
|
|
169
|
+
|
|
170
|
+
## License
|
|
171
|
+
|
|
172
|
+
[MIT](LICENSE)
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# Experiment Tracker
|
|
2
2
|
|
|
3
|
+
[](https://pypi.org/project/expr-tracker/)
|
|
4
|
+
[](https://pypi.org/project/expr-tracker/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](https://hspk.github.io/expr_tracker/)
|
|
7
|
+
|
|
3
8
|
A local-first experiment tracker. Metrics land in a JSONL file you own, stay
|
|
4
9
|
queryable while the run is live, and can trigger alerts from an expression language.
|
|
5
10
|
`wandb` and `trackio` are optional mirrors, not requirements.
|
|
@@ -114,3 +119,7 @@ Published to GitHub Pages by `.github/workflows/docs.yaml` on every push to `mai
|
|
|
114
119
|
Internals: [`docs/design.md`](docs/design.md) (data model and key invariants) and
|
|
115
120
|
[`docs/architecture.md`](docs/architecture.md) (module map, read/write paths,
|
|
116
121
|
concurrency model).
|
|
122
|
+
|
|
123
|
+
## License
|
|
124
|
+
|
|
125
|
+
[MIT](LICENSE)
|
|
@@ -93,6 +93,69 @@ Built-in types: `lark`, `slack`, `dingtalk`, `wecom`, `webhook` (a generic JSON
|
|
|
93
93
|
template), `email`, `callable`. All but `lark` use only the standard library. Add
|
|
94
94
|
your own with `register_backend()`.
|
|
95
95
|
|
|
96
|
+
### Email
|
|
97
|
+
|
|
98
|
+
Email needs an SMTP server to send *from*, even when the recipient is Gmail.
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
et.init(
|
|
102
|
+
project="demo",
|
|
103
|
+
alert={
|
|
104
|
+
"channels": [
|
|
105
|
+
{
|
|
106
|
+
"type": "email",
|
|
107
|
+
"name": "inbox",
|
|
108
|
+
"options": {
|
|
109
|
+
"host": "smtp.gmail.com",
|
|
110
|
+
"port": 587,
|
|
111
|
+
"tls": True,
|
|
112
|
+
"user": "you@gmail.com",
|
|
113
|
+
"password": os.environ["SMTP_PASSWORD"],
|
|
114
|
+
"sender": "you@gmail.com",
|
|
115
|
+
"to": ["you@gmail.com", "teammate@example.com"],
|
|
116
|
+
},
|
|
117
|
+
"min_level": "error",
|
|
118
|
+
}
|
|
119
|
+
]
|
|
120
|
+
},
|
|
121
|
+
alert_rules=["isnan(loss) => critical: loss diverged"],
|
|
122
|
+
)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
| Option | Default | Meaning |
|
|
126
|
+
| --- | --- | --- |
|
|
127
|
+
| `host` | — | SMTP server, **required** |
|
|
128
|
+
| `to` | — | one address or a list, **required** |
|
|
129
|
+
| `port` | 465 with `ssl`, else 25 | |
|
|
130
|
+
| `tls` | `false` | STARTTLS on a plain connection (port 587) |
|
|
131
|
+
| `ssl` | `false` | implicit TLS from the start (port 465) |
|
|
132
|
+
| `user` / `password` | — | omit both for an unauthenticated relay |
|
|
133
|
+
| `sender` | `user`, else `expr-tracker` | the `From` address |
|
|
134
|
+
| `html` | `true` | send the HTML part as well as the text |
|
|
135
|
+
|
|
136
|
+
Use `tls` **or** `ssl`, not both: `tls` upgrades a plain connection, `ssl` starts
|
|
137
|
+
encrypted.
|
|
138
|
+
|
|
139
|
+
!!! warning "Keep the password out of your code"
|
|
140
|
+
Read it from the environment, as above. Gmail additionally rejects account
|
|
141
|
+
passwords for SMTP — turn on 2-step verification and create an
|
|
142
|
+
[app password](https://myaccount.google.com/apppasswords), which you can
|
|
143
|
+
revoke independently of your account.
|
|
144
|
+
|
|
145
|
+
Mail is sent as `multipart/alternative`: a severity-coloured HTML card with the
|
|
146
|
+
fields as a table, plus the plain text as a fallback for clients that will not
|
|
147
|
+
render HTML. Set `html: false` for text only.
|
|
148
|
+
|
|
149
|
+
Common servers:
|
|
150
|
+
|
|
151
|
+
| Provider | host | port | setting |
|
|
152
|
+
| --- | --- | --- | --- |
|
|
153
|
+
| Gmail | `smtp.gmail.com` | 587 | `tls: true` (app password required) |
|
|
154
|
+
| Outlook / Office 365 | `smtp.office365.com` | 587 | `tls: true` |
|
|
155
|
+
| QQ / 163 | `smtp.qq.com`, `smtp.163.com` | 465 | `ssl: true` (authorisation code) |
|
|
156
|
+
| SendGrid | `smtp.sendgrid.net` | 587 | `tls: true`, user `apikey` |
|
|
157
|
+
| Internal relay | your host | 25 | often no `user`/`password` |
|
|
158
|
+
|
|
96
159
|
### Routing
|
|
97
160
|
|
|
98
161
|
Each channel filters independently:
|
|
@@ -79,6 +79,7 @@ alert:
|
|
|
79
79
|
| `levels` | an exact allowlist, overriding `min_level` |
|
|
80
80
|
| `tags` | deliver only if the message tags intersect |
|
|
81
81
|
| `options` | backend-specific settings |
|
|
82
|
+
| `options.html` | email only: send an HTML part alongside the text (default `true`) |
|
|
82
83
|
| `policy` | overrides the default delivery policy |
|
|
83
84
|
|
|
84
85
|
### Delivery policy
|
|
@@ -104,7 +105,7 @@ alert:
|
|
|
104
105
|
| `condition` | — | the expression (alias: `expr`) |
|
|
105
106
|
| `name` | derived | identifies the rule (alias: `alert`) |
|
|
106
107
|
| `level` | `warning` | `debug`, `info`, `warning`, `error`, `critical` |
|
|
107
|
-
| `title` / `message` | derived | supports templates |
|
|
108
|
+
| `title` / `message` | derived | supports templates; the title defaults to `name` when you set one, else to the condition |
|
|
108
109
|
| `mode` | `"edge"` | `edge` fires on transition, `level` keeps firing |
|
|
109
110
|
| `for_steps` | 1 | must hold this many consecutive steps (alias: `for`) |
|
|
110
111
|
| `cooldown` | 300.0 | seconds between repeats in level mode |
|
|
@@ -1,9 +1,35 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "expr_tracker"
|
|
3
|
-
description = "
|
|
3
|
+
description = "Local-first experiment tracking with queryable history and expression-based alerts on your training metrics"
|
|
4
4
|
readme = "README.md"
|
|
5
|
+
license = "MIT"
|
|
6
|
+
license-files = ["LICENSE"]
|
|
5
7
|
authors = [{ name = "HSPK", email = "whxway@whu.edu.cn" }]
|
|
6
8
|
requires-python = ">=3.10"
|
|
9
|
+
keywords = [
|
|
10
|
+
"machine-learning",
|
|
11
|
+
"experiment-tracking",
|
|
12
|
+
"mlops",
|
|
13
|
+
"monitoring",
|
|
14
|
+
"alerting",
|
|
15
|
+
"training",
|
|
16
|
+
"deep-learning",
|
|
17
|
+
"wandb",
|
|
18
|
+
]
|
|
19
|
+
classifiers = [
|
|
20
|
+
"Development Status :: 4 - Beta",
|
|
21
|
+
"Intended Audience :: Developers",
|
|
22
|
+
"Intended Audience :: Science/Research",
|
|
23
|
+
"Operating System :: OS Independent",
|
|
24
|
+
"Programming Language :: Python :: 3",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Programming Language :: Python :: 3.13",
|
|
29
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
30
|
+
"Topic :: System :: Monitoring",
|
|
31
|
+
"Typing :: Typed",
|
|
32
|
+
]
|
|
7
33
|
dependencies = [
|
|
8
34
|
"click>=8.1.0",
|
|
9
35
|
"loguru>=0.7.3",
|
|
@@ -27,6 +53,13 @@ all = [
|
|
|
27
53
|
"polars>=0.20",
|
|
28
54
|
]
|
|
29
55
|
|
|
56
|
+
[project.urls]
|
|
57
|
+
Homepage = "https://hspk.github.io/expr_tracker/"
|
|
58
|
+
Documentation = "https://hspk.github.io/expr_tracker/"
|
|
59
|
+
Repository = "https://github.com/HSPK/expr_tracker"
|
|
60
|
+
Issues = "https://github.com/HSPK/expr_tracker/issues"
|
|
61
|
+
Changelog = "https://github.com/HSPK/expr_tracker/releases"
|
|
62
|
+
|
|
30
63
|
[project.scripts]
|
|
31
64
|
et = "expr_tracker.cli:main"
|
|
32
65
|
|
|
@@ -13,6 +13,7 @@ from .base import (
|
|
|
13
13
|
create_backend,
|
|
14
14
|
post_json,
|
|
15
15
|
register_backend,
|
|
16
|
+
render_html,
|
|
16
17
|
render_text,
|
|
17
18
|
)
|
|
18
19
|
|
|
@@ -189,7 +190,10 @@ class EmailBackend(AlertBackend):
|
|
|
189
190
|
mail["Subject"] = f"[{message.level.value}] {message.title}"
|
|
190
191
|
mail["From"] = options.get("sender") or options.get("user") or "expr-tracker"
|
|
191
192
|
mail["To"] = ", ".join(recipients)
|
|
192
|
-
|
|
193
|
+
# multipart/alternative: rich clients show the card, the rest see the text
|
|
194
|
+
mail.set_content(f"{message.title}\n\n{render_text(message)}")
|
|
195
|
+
if options.get("html", True):
|
|
196
|
+
mail.add_alternative(render_html(message), subtype="html")
|
|
193
197
|
policy_timeout = self.config.policy.timeout if self.config.policy else 10.0
|
|
194
198
|
try:
|
|
195
199
|
factory = smtplib.SMTP_SSL if options.get("ssl") else smtplib.SMTP
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import html
|
|
5
6
|
import json
|
|
6
7
|
import urllib.error
|
|
7
8
|
import urllib.parse
|
|
@@ -85,7 +86,7 @@ def post_json(
|
|
|
85
86
|
|
|
86
87
|
def render_text(message: AlertMessage, include_fields: bool = True) -> str:
|
|
87
88
|
"""Render a message as plain text for channels without rich cards."""
|
|
88
|
-
parts = [message.text]
|
|
89
|
+
parts = [message.subtitle, message.text] if message.subtitle else [message.text]
|
|
89
90
|
if include_fields and message.fields:
|
|
90
91
|
parts.append("")
|
|
91
92
|
parts.extend(f"{key}: {value}" for key, value in message.fields.items())
|
|
@@ -97,6 +98,74 @@ def render_text(message: AlertMessage, include_fields: bool = True) -> str:
|
|
|
97
98
|
return "\n".join(parts)
|
|
98
99
|
|
|
99
100
|
|
|
101
|
+
LEVEL_COLORS = {
|
|
102
|
+
"debug": "#616161",
|
|
103
|
+
"info": "#0288d1",
|
|
104
|
+
"warning": "#ed6c02",
|
|
105
|
+
"error": "#d32f2f",
|
|
106
|
+
"critical": "#b71c1c",
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def render_html(message: AlertMessage) -> str:
|
|
111
|
+
"""Render a message as self-contained HTML for email.
|
|
112
|
+
|
|
113
|
+
Everything is inline-styled and table-based: mail clients strip stylesheets,
|
|
114
|
+
and several of them ignore ``div`` layout entirely.
|
|
115
|
+
"""
|
|
116
|
+
color = LEVEL_COLORS.get(message.level.value, "#616161")
|
|
117
|
+
esc = html.escape
|
|
118
|
+
rows = "".join(
|
|
119
|
+
f'<tr><td style="padding:4px 12px 4px 0;color:#666;'
|
|
120
|
+
f'white-space:nowrap;vertical-align:top">{esc(str(key))}</td>'
|
|
121
|
+
f'<td style="padding:4px 0;color:#111">{esc(str(value))}</td></tr>'
|
|
122
|
+
for key, value in message.fields.items()
|
|
123
|
+
)
|
|
124
|
+
blocks = [
|
|
125
|
+
f'<tr><td style="background:{color};padding:14px 20px;color:#fff">'
|
|
126
|
+
f'<div style="font-size:17px;font-weight:600">{esc(message.title)}</div>'
|
|
127
|
+
f'<div style="font-size:12px;opacity:.85;text-transform:uppercase;'
|
|
128
|
+
f'letter-spacing:.05em">{esc(message.level.value)}'
|
|
129
|
+
+ (f" · {esc(message.subtitle)}" if message.subtitle else "")
|
|
130
|
+
+ "</div></td></tr>"
|
|
131
|
+
]
|
|
132
|
+
if message.text:
|
|
133
|
+
blocks.append(
|
|
134
|
+
'<tr><td style="padding:18px 20px 0;font-size:14px;line-height:1.5;'
|
|
135
|
+
f'color:#111">{esc(message.text)}</td></tr>'
|
|
136
|
+
)
|
|
137
|
+
if rows:
|
|
138
|
+
blocks.append(
|
|
139
|
+
'<tr><td style="padding:16px 20px 0"><table cellpadding="0" '
|
|
140
|
+
f'cellspacing="0" style="font-size:13px">{rows}</table></td></tr>'
|
|
141
|
+
)
|
|
142
|
+
if message.traceback:
|
|
143
|
+
blocks.append(
|
|
144
|
+
'<tr><td style="padding:16px 20px 0"><pre style="margin:0;padding:12px;'
|
|
145
|
+
"background:#f6f6f6;border-radius:4px;font-size:12px;overflow:auto;"
|
|
146
|
+
f'white-space:pre-wrap">{esc(message.traceback)}</pre></td></tr>'
|
|
147
|
+
)
|
|
148
|
+
if message.link:
|
|
149
|
+
link = esc(message.link)
|
|
150
|
+
blocks.append(
|
|
151
|
+
f'<tr><td style="padding:18px 20px 0"><a href="{link}" '
|
|
152
|
+
f'style="display:inline-block;padding:8px 16px;background:{color};'
|
|
153
|
+
'color:#fff;text-decoration:none;border-radius:4px;font-size:13px">'
|
|
154
|
+
"Open run</a></td></tr>"
|
|
155
|
+
)
|
|
156
|
+
blocks.append('<tr><td style="padding:20px"></td></tr>')
|
|
157
|
+
return (
|
|
158
|
+
'<html><body style="margin:0;background:#f4f4f4;'
|
|
159
|
+
'font-family:-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif">'
|
|
160
|
+
'<table cellpadding="0" cellspacing="0" width="100%" '
|
|
161
|
+
'style="background:#f4f4f4;padding:24px 0"><tr><td align="center">'
|
|
162
|
+
'<table cellpadding="0" cellspacing="0" width="520" '
|
|
163
|
+
'style="background:#fff;border-radius:6px;overflow:hidden;max-width:520px">'
|
|
164
|
+
+ "".join(blocks)
|
|
165
|
+
+ "</table></td></tr></table></body></html>"
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
100
169
|
def _parse_retry_after(value) -> float | None:
|
|
101
170
|
try:
|
|
102
171
|
return float(value)
|
|
@@ -209,7 +209,12 @@ class AlertEngine:
|
|
|
209
209
|
except Exception as e: # pragma: no cover - format_map already tolerates gaps
|
|
210
210
|
logger.warning(f"Failed to render message for rule {rule.name!r}: {e}")
|
|
211
211
|
rendered = rule.condition
|
|
212
|
-
|
|
212
|
+
if rule.title:
|
|
213
|
+
title = _format(rule.title, values)
|
|
214
|
+
else:
|
|
215
|
+
# An auto-generated name is a hash; the condition says far more,
|
|
216
|
+
# especially as an email subject line
|
|
217
|
+
title = rule.condition if getattr(rule, "auto_named", False) else rule.name
|
|
213
218
|
level = AlertLevel.INFO if recovered else rule.level
|
|
214
219
|
message = AlertMessage(
|
|
215
220
|
title=f"[recovered] {title}" if recovered else title,
|
|
@@ -230,6 +230,9 @@ class AlertRule:
|
|
|
230
230
|
f"Unknown alert rule mode {self.mode!r}; use 'edge' or 'level'."
|
|
231
231
|
)
|
|
232
232
|
self.for_steps = max(1, int(self.for_steps))
|
|
233
|
+
# A generated name identifies the rule but reads badly as a headline, so
|
|
234
|
+
# remember whether the user chose it. Not a field: it stays out of to_dict().
|
|
235
|
+
self.auto_named = not self.name
|
|
233
236
|
if not self.name:
|
|
234
237
|
self.name = f"rule_{self._auto_digest()}"
|
|
235
238
|
if not self.message:
|
|
File without changes
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Channel payload construction and HTTP error mapping."""
|
|
2
2
|
|
|
3
|
+
import json
|
|
3
4
|
import urllib.error
|
|
4
5
|
import urllib.request
|
|
5
6
|
|
|
@@ -8,7 +9,7 @@ import pytest
|
|
|
8
9
|
from expr_tracker.alerts import backends as backend_module
|
|
9
10
|
from expr_tracker.alerts.backends import create_backend
|
|
10
11
|
from expr_tracker.alerts.backends.base import SendError, post_json
|
|
11
|
-
from expr_tracker.alerts.models import AlertMessage, ChannelConfig
|
|
12
|
+
from expr_tracker.alerts.models import AlertLevel, AlertMessage, ChannelConfig
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
@pytest.fixture
|
|
@@ -179,3 +180,180 @@ def test_email_backend_sends(monkeypatch):
|
|
|
179
180
|
assert mail["Subject"] == "[error] Title"
|
|
180
181
|
assert mail["To"] == "a@b.c"
|
|
181
182
|
assert "tls" in sent
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------------------- subtitle
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def test_the_subtitle_reaches_the_rendered_text():
|
|
189
|
+
"""Every channel but Lark renders through render_text, which used to drop it."""
|
|
190
|
+
from expr_tracker.alerts.backends.base import render_text
|
|
191
|
+
|
|
192
|
+
message = AlertMessage(
|
|
193
|
+
title="T", text="body", subtitle="run-42", level=AlertLevel.INFO
|
|
194
|
+
)
|
|
195
|
+
rendered = render_text(message)
|
|
196
|
+
assert "run-42" in rendered
|
|
197
|
+
assert rendered.startswith("run-42") # context first, then the body
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_a_message_without_a_subtitle_is_unchanged():
|
|
201
|
+
from expr_tracker.alerts.backends.base import render_text
|
|
202
|
+
|
|
203
|
+
assert render_text(AlertMessage(title="T", text="body")) == "body"
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@pytest.mark.parametrize("backend_type", ["slack", "dingtalk", "wecom"])
|
|
207
|
+
def test_text_channels_carry_the_subtitle(captured, backend_type):
|
|
208
|
+
backend = create_backend(
|
|
209
|
+
ChannelConfig(type=backend_type, name="c", url="https://hook.test/x")
|
|
210
|
+
)
|
|
211
|
+
backend.send(
|
|
212
|
+
AlertMessage(title="T", text="body", subtitle="run-42", level=AlertLevel.ERROR)
|
|
213
|
+
)
|
|
214
|
+
assert "run-42" in json.dumps(captured[-1])
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# ---------------------------------------------------------------------- html mail
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def sent_mail(monkeypatch, message, **options):
|
|
221
|
+
"""Send through the real EmailBackend with SMTP stubbed out."""
|
|
222
|
+
import smtplib
|
|
223
|
+
|
|
224
|
+
box = {}
|
|
225
|
+
|
|
226
|
+
class FakeSMTP:
|
|
227
|
+
def __init__(self, *args, **kwargs):
|
|
228
|
+
pass
|
|
229
|
+
|
|
230
|
+
def __enter__(self):
|
|
231
|
+
return self
|
|
232
|
+
|
|
233
|
+
def __exit__(self, *args):
|
|
234
|
+
return False
|
|
235
|
+
|
|
236
|
+
def starttls(self):
|
|
237
|
+
box["tls"] = True
|
|
238
|
+
|
|
239
|
+
def login(self, user, password):
|
|
240
|
+
box["login"] = (user, password)
|
|
241
|
+
|
|
242
|
+
def send_message(self, mail):
|
|
243
|
+
box["mail"] = mail
|
|
244
|
+
|
|
245
|
+
monkeypatch.setattr(smtplib, "SMTP", FakeSMTP)
|
|
246
|
+
monkeypatch.setattr(smtplib, "SMTP_SSL", FakeSMTP)
|
|
247
|
+
backend = create_backend(
|
|
248
|
+
ChannelConfig(
|
|
249
|
+
type="email",
|
|
250
|
+
name="m",
|
|
251
|
+
options={"host": "smtp.test", "to": "a@b.c", **options},
|
|
252
|
+
)
|
|
253
|
+
)
|
|
254
|
+
backend.send(message)
|
|
255
|
+
return box
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def test_email_is_multipart_with_a_text_fallback(monkeypatch):
|
|
259
|
+
box = sent_mail(monkeypatch, AlertMessage(title="T", text="body"))
|
|
260
|
+
mail = box["mail"]
|
|
261
|
+
assert mail.is_multipart()
|
|
262
|
+
assert [p.get_content_type() for p in mail.walk()] == [
|
|
263
|
+
"multipart/alternative",
|
|
264
|
+
"text/plain",
|
|
265
|
+
"text/html",
|
|
266
|
+
]
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def test_the_plain_part_carries_the_title_and_body(monkeypatch):
|
|
270
|
+
message = AlertMessage(
|
|
271
|
+
title="Loss spike", text="diverged", subtitle="run-42", fields={"step": 3}
|
|
272
|
+
)
|
|
273
|
+
box = sent_mail(monkeypatch, message)
|
|
274
|
+
text = box["mail"].get_body(("plain",)).get_content()
|
|
275
|
+
assert "Loss spike" in text and "diverged" in text
|
|
276
|
+
assert "run-42" in text and "step: 3" in text
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def test_the_html_part_carries_everything(monkeypatch):
|
|
280
|
+
message = AlertMessage(
|
|
281
|
+
title="Loss spike",
|
|
282
|
+
text="diverged",
|
|
283
|
+
subtitle="run-42",
|
|
284
|
+
level=AlertLevel.ERROR,
|
|
285
|
+
fields={"step": 3, "loss": 8.42},
|
|
286
|
+
link="https://example.test/run",
|
|
287
|
+
traceback="Traceback: demo",
|
|
288
|
+
)
|
|
289
|
+
box = sent_mail(monkeypatch, message)
|
|
290
|
+
body = box["mail"].get_body(("html",)).get_content()
|
|
291
|
+
for fragment in (
|
|
292
|
+
"Loss spike",
|
|
293
|
+
"run-42",
|
|
294
|
+
"diverged",
|
|
295
|
+
"step",
|
|
296
|
+
"8.42",
|
|
297
|
+
"https://example.test/run",
|
|
298
|
+
"Traceback: demo",
|
|
299
|
+
"error",
|
|
300
|
+
):
|
|
301
|
+
assert fragment in body, fragment
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
@pytest.mark.parametrize(
|
|
305
|
+
("level", "color"),
|
|
306
|
+
[
|
|
307
|
+
("info", "#0288d1"),
|
|
308
|
+
("warning", "#ed6c02"),
|
|
309
|
+
("error", "#d32f2f"),
|
|
310
|
+
("critical", "#b71c1c"),
|
|
311
|
+
],
|
|
312
|
+
)
|
|
313
|
+
def test_the_html_header_is_coloured_by_severity(monkeypatch, level, color):
|
|
314
|
+
box = sent_mail(
|
|
315
|
+
monkeypatch, AlertMessage(title="T", text="b", level=AlertLevel.parse(level))
|
|
316
|
+
)
|
|
317
|
+
assert color in box["mail"].get_body(("html",)).get_content()
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def test_html_content_is_escaped(monkeypatch):
|
|
321
|
+
"""A metric name or message must never inject markup into the mail."""
|
|
322
|
+
message = AlertMessage(
|
|
323
|
+
title="<script>alert(1)</script>",
|
|
324
|
+
text="a & b < c",
|
|
325
|
+
fields={"<k>": "<v>"},
|
|
326
|
+
)
|
|
327
|
+
box = sent_mail(monkeypatch, message)
|
|
328
|
+
body = box["mail"].get_body(("html",)).get_content()
|
|
329
|
+
assert "<script>" not in body
|
|
330
|
+
assert "<script>" in body and "a & b < c" in body
|
|
331
|
+
assert "<k>" in body and "<v>" in body
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def test_html_can_be_switched_off(monkeypatch):
|
|
335
|
+
box = sent_mail(monkeypatch, AlertMessage(title="T", text="b"), html=False)
|
|
336
|
+
mail = box["mail"]
|
|
337
|
+
assert not mail.is_multipart()
|
|
338
|
+
assert mail.get_content_type() == "text/plain"
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def test_the_smtp_session_uses_the_configured_options(monkeypatch):
|
|
342
|
+
box = sent_mail(
|
|
343
|
+
monkeypatch,
|
|
344
|
+
AlertMessage(title="T", text="b"),
|
|
345
|
+
tls=True,
|
|
346
|
+
user="me@x.y",
|
|
347
|
+
password="secret",
|
|
348
|
+
sender="from@x.y",
|
|
349
|
+
)
|
|
350
|
+
assert box["tls"] is True
|
|
351
|
+
assert box["login"] == ("me@x.y", "secret")
|
|
352
|
+
assert box["mail"]["From"] == "from@x.y"
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def test_several_recipients(monkeypatch):
|
|
356
|
+
box = sent_mail(
|
|
357
|
+
monkeypatch, AlertMessage(title="T", text="b"), to=["a@x.y", "b@x.y"]
|
|
358
|
+
)
|
|
359
|
+
assert box["mail"]["To"] == "a@x.y, b@x.y"
|
|
@@ -451,3 +451,34 @@ def test_a_metric_that_appears_late_is_not_a_typo(run):
|
|
|
451
451
|
assert not next(iter(instance.info()["alerts"]["rules"].values()))[
|
|
452
452
|
"unresolved_metrics"
|
|
453
453
|
]
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
# ------------------------------------------------------------------ headlines
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def test_an_unnamed_rule_is_headlined_by_its_condition(run):
|
|
460
|
+
"""A generated name is a hash, which reads badly as an email subject."""
|
|
461
|
+
instance, received = run(["isnan(loss) => critical: diverged"])
|
|
462
|
+
instance.log({"loss": float("nan")})
|
|
463
|
+
assert received[0].title == "isnan(loss)"
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def test_a_named_rule_is_headlined_by_its_name(run):
|
|
467
|
+
instance, received = run([{"condition": "loss > 1", "name": "loss_high"}])
|
|
468
|
+
instance.log({"loss": 5.0})
|
|
469
|
+
assert received[0].title == "loss_high"
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def test_an_explicit_title_wins_and_is_templated(run):
|
|
473
|
+
instance, received = run([{"condition": "loss > 1", "title": "Spike @ {step}"}])
|
|
474
|
+
instance.log({"loss": 5.0}, step=7, commit=True)
|
|
475
|
+
assert received[0].title == "Spike @ 7"
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def test_the_auto_named_flag_is_not_serialised():
|
|
479
|
+
import dataclasses
|
|
480
|
+
|
|
481
|
+
rule = AlertRule(condition="loss > 1")
|
|
482
|
+
assert rule.auto_named is True
|
|
483
|
+
assert "auto_named" not in dataclasses.asdict(rule)
|
|
484
|
+
assert AlertRule(condition="loss > 1", name="mine").auto_named is False
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
{"_step": 0, "_time": 1785984521.3747878, "val/loss": 10, "train/loss": 20}
|
|
2
|
-
{"_step": 1, "_time": 1785984578.8073661, "val/loss": 10, "train/loss": 25}
|
|
3
|
-
{"_step": 2, "_time": 1785984751.5897822, "val/loss": 10, "train/loss": 25}
|
|
4
|
-
{"_step": 3, "_time": 1785984760.5739543, "val/loss": 10, "train/loss": 25}
|
|
5
|
-
{"_step": 4, "_time": 1785984823.973783, "train": 30, "loss": 1}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"schema": 2, "size": 369, "lines": 5, "last_step": 4, "max_step": 4, "sorted": true, "has_duplicate_steps": false, "index": [[0, 0, 0]], "run": {"project": "hello", "name": "test", "started_at": 1785984737.2126129}}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|