expr_tracker 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,7 +6,7 @@ import json
6
6
  import smtplib
7
7
  from email.message import EmailMessage
8
8
 
9
- from ..models import AlertLevel, AlertMessage, ChannelConfig
9
+ from ..models import AlertLevel, AlertMessage
10
10
  from .base import (
11
11
  AlertBackend,
12
12
  SendError,
@@ -16,6 +16,7 @@ from .base import (
16
16
  render_html,
17
17
  render_text,
18
18
  )
19
+ from .cards import build_card, card_payload
19
20
 
20
21
  LEVEL_EMOJI = {
21
22
  AlertLevel.DEBUG: "🔍",
@@ -56,13 +57,22 @@ class UrlBackend(AlertBackend):
56
57
  self.url, payload, self.timeout, self.config.options.get("headers")
57
58
  )
58
59
 
59
- def post_checked(self, payload: dict):
60
- """DingTalk and WeCom report failures via errcode inside an HTTP 200 body."""
61
- body = self.post(payload)
60
+ def post_reply(self, payload: dict) -> dict:
61
+ """Post, and decode the reply these APIs put their verdict in.
62
+
63
+ An unreadable body is not a failure: the POST itself succeeded, and some
64
+ proxies answer 200 with nothing at all.
65
+ """
66
+ body = self.post(payload) # outside the try: a failed POST must raise
62
67
  try:
63
68
  result = json.loads(body)
64
69
  except Exception:
65
- return
70
+ return {}
71
+ return result if isinstance(result, dict) else {}
72
+
73
+ def post_checked(self, payload: dict):
74
+ """DingTalk and WeCom report failures via errcode inside an HTTP 200 body."""
75
+ result = self.post_reply(payload)
66
76
  code = result.get("errcode")
67
77
  if code:
68
78
  raise SendError(
@@ -73,45 +83,31 @@ class UrlBackend(AlertBackend):
73
83
 
74
84
 
75
85
  class LarkBackend(UrlBackend):
76
- type = "lark"
77
-
78
- def __init__(self, config: ChannelConfig):
79
- super().__init__(config)
80
- self._client = None
86
+ """Feishu/Lark bot webhook, as an interactive card."""
81
87
 
82
- def _lark(self):
83
- if self._client is None:
84
- from slark import Lark
88
+ type = "lark"
85
89
 
86
- options = {k: v for k, v in self.config.options.items() if k != "headers"}
87
- self._client = Lark(webhook=self.url, **options)
88
- return self._client
90
+ # Lark answers HTTP 200 and puts the verdict in the body
91
+ RETRYABLE_CODES = frozenset({9499, 11232, 19024})
89
92
 
90
93
  def send(self, message: AlertMessage):
91
94
  title = f"{LEVEL_EMOJI.get(message.level, '')} {message.title}".strip()
92
- text = render_text(message)
93
- subtitle = message.subtitle
94
- try:
95
- client = self._lark()
96
- except ImportError as e: # pragma: no cover - optional extra
95
+ card = build_card(
96
+ render_text(message),
97
+ title,
98
+ message.subtitle,
99
+ message.traceback,
100
+ failed=message.level >= AlertLevel.ERROR,
101
+ )
102
+ result = self.post_reply(card_payload(card))
103
+ # Success is code 0, or StatusCode 0 from the older bot endpoint
104
+ code = result.get("code") or result.get("StatusCode") or 0
105
+ if code:
106
+ detail = result.get("msg") or result.get("StatusMessage")
97
107
  raise SendError(
98
- f'The lark channel needs slark: pip install "expr_tracker[lark]" ({e})',
99
- retryable=False,
100
- ) from e
101
- try:
102
- if message.level >= AlertLevel.ERROR:
103
- client.webhook.post_error_card(
104
- msg=text,
105
- traceback=message.traceback or "",
106
- title=title,
107
- subtitle=subtitle,
108
- )
109
- else:
110
- client.webhook.post_success_card(
111
- msg=text, title=title, subtitle=subtitle
112
- )
113
- except Exception as e:
114
- raise SendError(f"Lark webhook failed: {e}") from e
108
+ f"Lark rejected the message: code={code} msg={detail!r}",
109
+ retryable=code in self.RETRYABLE_CODES,
110
+ )
115
111
 
116
112
 
117
113
  class SlackBackend(UrlBackend):
@@ -0,0 +1,102 @@
1
+ """Lark (Feishu) interactive message cards.
2
+
3
+ The card layout is the one ``slark`` builds, reproduced here as plain dicts so
4
+ the Lark channel needs no third-party client: a webhook post is one JSON body
5
+ over stdlib HTTP, which is what every other channel already does.
6
+
7
+ Reference: https://open.feishu.cn/document/server-docs/im-v1/message-card
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import time
13
+ from typing import Any
14
+
15
+ GREEN = "green"
16
+ RED = "red"
17
+ YES_ICON = "yes_filled"
18
+ ERROR_ICON = "error_filled"
19
+
20
+
21
+ def _heading(content: str) -> dict[str, Any]:
22
+ """A section heading. Lark has no heading element, hence the wrapping."""
23
+ return {
24
+ "tag": "column_set",
25
+ "flex_mode": "none",
26
+ "horizontal_spacing": "default",
27
+ "horizontal_align": "left",
28
+ "background_style": "default",
29
+ "columns": [
30
+ {
31
+ "tag": "column",
32
+ "background_style": "default",
33
+ "elements": [
34
+ {
35
+ "tag": "div",
36
+ "text": {
37
+ "tag": "plain_text",
38
+ "content": content,
39
+ "text_size": "heading",
40
+ "text_align": "left",
41
+ "text_color": "default",
42
+ },
43
+ }
44
+ ],
45
+ "width": "auto",
46
+ "weight": 1,
47
+ "vertical_align": "top",
48
+ "vertical_spacing": "default",
49
+ }
50
+ ],
51
+ }
52
+
53
+
54
+ def _code_block(content: str, language: str = "txt") -> dict[str, Any]:
55
+ return {
56
+ "tag": "markdown",
57
+ "content": f"```{language}\n{content}\n```",
58
+ "text_align": "left",
59
+ "text_size": "normal",
60
+ }
61
+
62
+
63
+ def _header(title: str, subtitle: str, template: str, icon: str) -> dict[str, Any]:
64
+ return {
65
+ "title": {"tag": "plain_text", "content": title},
66
+ "subtitle": {"tag": "plain_text", "content": subtitle},
67
+ "template": template,
68
+ "ud_icon": {"token": icon},
69
+ }
70
+
71
+
72
+ def build_card(
73
+ message: str,
74
+ title: str,
75
+ subtitle: str | None = None,
76
+ traceback: str | None = None,
77
+ *,
78
+ failed: bool = False,
79
+ ) -> dict[str, Any]:
80
+ """One card: a coloured header, the message, and a traceback if there is one."""
81
+ elements: list[dict[str, Any]] = [
82
+ _heading("Message"),
83
+ _code_block(message),
84
+ ]
85
+ if traceback and traceback.strip():
86
+ # Skipped when absent: most alerts are metric conditions, not crashes,
87
+ # and an empty "Traceback" code block on every one of them is noise
88
+ elements += [_heading("Traceback"), _code_block(traceback.strip(), "")]
89
+ return {
90
+ "header": _header(
91
+ title,
92
+ subtitle if subtitle is not None else time.strftime("%Y-%m-%d %H:%M:%S"),
93
+ RED if failed else GREEN,
94
+ ERROR_ICON if failed else YES_ICON,
95
+ ),
96
+ "elements": elements,
97
+ }
98
+
99
+
100
+ def card_payload(card: dict[str, Any]) -> dict[str, Any]:
101
+ """The webhook envelope a card has to travel in."""
102
+ return {"msg_type": "interactive", "card": card}
expr_tracker/cli.py CHANGED
@@ -36,16 +36,29 @@ def alert(msg: str, title: str, level: str, channels: tuple[str, ...]):
36
36
  @click.option("-n", default=20, help="Number of steps (-1 for all)")
37
37
  @click.option("--metrics", default=None, help="Comma separated metric names")
38
38
  @click.option("--step-range", default=None, help="start:end (end exclusive)")
39
+ @click.option(
40
+ "--stream",
41
+ default=None,
42
+ help="Which stream to read; the unnamed one by default.",
43
+ )
39
44
  @click.option(
40
45
  "--format", "fmt", type=click.Choice(["table", "json", "csv"]), default="table"
41
46
  )
42
- def history(run: str, n: int, metrics: str | None, step_range: str | None, fmt: str):
47
+ def history(
48
+ run: str,
49
+ n: int,
50
+ metrics: str | None,
51
+ step_range: str | None,
52
+ stream: str | None,
53
+ fmt: str,
54
+ ):
43
55
  """Print the recorded history of a run."""
44
56
  rows = read_history(
45
57
  run,
46
58
  n,
47
59
  metrics=metrics.split(",") if metrics else None,
48
60
  step_range=_parse_range(step_range),
61
+ stream=None if stream in (None, "default", "") else stream,
49
62
  )
50
63
  if fmt == "json":
51
64
  click.echo(json.dumps(rows, ensure_ascii=False, indent=2))
expr_tracker/run.py CHANGED
@@ -73,6 +73,7 @@ class Run:
73
73
  alert: Any = None,
74
74
  alert_rules: Sequence[Any] = (),
75
75
  alert_on_rank: int | None = 0,
76
+ backend_on_rank: int | None = 0,
76
77
  stream: str | None = None,
77
78
  **history_kwargs,
78
79
  ):
@@ -115,10 +116,18 @@ class Run:
115
116
  root=Path(dir or "./tracker/jsonl") / project / "artifacts"
116
117
  )
117
118
 
119
+ self.rank = current_rank()
118
120
  self.backends: dict[str, Any] = {}
119
121
  self.alerts = None
122
+ # Every rank would otherwise open the same remote run and interleave its
123
+ # steps into it, which is what the local rankN shards exist to prevent
124
+ remote = backend_on_rank is None or self.rank == backend_on_rank
125
+ # Rank 0 belongs in the group too when the other ranks are reporting
126
+ self._grouped = bool(stream) or bool(self.rank) or backend_on_rank is None
127
+ if not remote:
128
+ logger.debug(f"Rank {self.rank} keeps its history local, no backends")
120
129
  try:
121
- for backend in dict.fromkeys(backends):
130
+ for backend in dict.fromkeys(backends) if remote else ():
122
131
  if backend == "jsonl":
123
132
  continue
124
133
  try:
@@ -146,7 +155,6 @@ class Run:
146
155
  kwargs=backend_kwargs.get(name, {}),
147
156
  )
148
157
  self._closers.append(instance.finish)
149
- self.rank = current_rank()
150
158
  # Every rank would otherwise raise the same alert N times
151
159
  if alert_on_rank is not None and self.rank != alert_on_rank:
152
160
  alert, alert_rules = {"enabled": False}, ()
@@ -158,18 +166,24 @@ class Run:
158
166
 
159
167
  @property
160
168
  def backend_run_name(self) -> str:
161
- """A stream is an independent producer, so it gets its own backend run.
169
+ """A stream or a rank is an independent producer of steps, so each gets
170
+ its own backend run.
162
171
 
163
172
  Grouping ties them back together: both wandb and trackio understand
164
173
  ``group``, and neither can merge two step axes into one run.
165
174
  """
166
- return f"{self.name}-{self.stream}" if self.stream else self.name
175
+ parts = [self.name]
176
+ if self.stream:
177
+ parts.append(self.stream)
178
+ if self.rank:
179
+ parts.append(f"rank{self.rank}")
180
+ return "-".join(parts)
167
181
 
168
182
  def _init_backend(
169
183
  self, name, instance, *, entity, dir, notes, tags, resume, kwargs
170
184
  ):
171
- # A grouped run keeps the streams together in the backend UI
172
- grouping = {"group": self.name} if self.stream else {}
185
+ # A grouped run keeps the streams and ranks together in the backend UI
186
+ grouping = {"group": self.name} if self._grouped else {}
173
187
  try:
174
188
  if name == "trackio":
175
189
  # trackio.init has no entity/notes/tags/dir/id, so those ride along
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: expr_tracker
3
- Version: 0.2.4
3
+ Version: 0.2.6
4
4
  Summary: Local-first experiment tracking with queryable history and expression-based alerts on your training metrics
5
5
  Project-URL: Homepage, https://hspk.github.io/expr_tracker/
6
6
  Project-URL: Documentation, https://hspk.github.io/expr_tracker/
@@ -31,13 +31,10 @@ Provides-Extra: all
31
31
  Requires-Dist: nvidia-ml-py>=12.0; extra == 'all'
32
32
  Requires-Dist: pandas>=1.5; extra == 'all'
33
33
  Requires-Dist: polars>=0.20; extra == 'all'
34
- Requires-Dist: slark>=0.1.28; extra == 'all'
35
34
  Requires-Dist: trackio>=0.4.0; extra == 'all'
36
35
  Requires-Dist: wandb>=0.21.0; extra == 'all'
37
36
  Provides-Extra: gpu
38
37
  Requires-Dist: nvidia-ml-py>=12.0; extra == 'gpu'
39
- Provides-Extra: lark
40
- Requires-Dist: slark>=0.1.28; extra == 'lark'
41
38
  Provides-Extra: pandas
42
39
  Requires-Dist: pandas>=1.5; extra == 'pandas'
43
40
  Provides-Extra: polars
@@ -92,7 +89,6 @@ et.history(-1, output_type="pd") # everything, as a DataFrame
92
89
  uv add expr_tracker # local-first: click, loguru, pydantic only
93
90
  uv add "expr_tracker[wandb]" # mirror to Weights & Biases
94
91
  uv add "expr_tracker[trackio]" # mirror to trackio
95
- uv add "expr_tracker[lark]" # Feishu/Lark alert channel
96
92
  uv add "expr_tracker[pandas]" # history(output_type="pandas")
97
93
  uv add "expr_tracker[all]" # everything
98
94
  ```
@@ -112,7 +108,26 @@ install command; it never crashes a run.
112
108
  | [Spans](https://hspk.github.io/expr_tracker/guide/spans/) | Time the parts of a step, and their parts. Each duration becomes a metric, so alerts and queries work on it unchanged; `et trace` exports the timeline for Perfetto. `print_fn` prints the tree live, and plugins attach CPU and GPU cost to each region. |
113
109
  | [Streams](https://hspk.github.io/expr_tracker/guide/streams/) | Independent producers — a data worker and a training loop — each with their own step cursor and file inside one run. |
114
110
  | [Distributed](https://hspk.github.io/expr_tracker/guide/distributed/) | Per-rank shards so concurrent appends cannot corrupt step order; only rank 0 alerts by default. |
115
- | [CLI](https://hspk.github.io/expr_tracker/guide/cli/) | `et history`, `et rules explain`, `et rules test`, `et alert`. |
111
+ | [CLI](https://hspk.github.io/expr_tracker/guide/cli/) | `et history`, `et trace`, `et rules explain`, `et rules test`, `et alert`. |
112
+
113
+ ## Examples
114
+
115
+ Every one runs offline, with no account and no network.
116
+
117
+ | | |
118
+ | --- | --- |
119
+ | [`quickstart.py`](examples/quickstart.py) | The sixty-second tour: log a run, merge eval into the training step, read the history back while it is open and again afterwards. |
120
+ | [`alert_rules.py`](examples/alert_rules.py) | Four rules against four faults — a loss spike, a non-finite loss, a stalled curve and a regression that must persist. `--fault none` fires nothing, which is the point. |
121
+ | [`profile_step.py`](examples/profile_step.py) | Where a step goes. Nested spans become metrics, a plugin adds CPU cost, and the tree exports to Perfetto. |
122
+ | [`early_stopping.py`](examples/early_stopping.py) | Querying your own history mid-run to decay the learning rate on a plateau and stop when it stops paying. |
123
+ | [`checkpoints.py`](examples/checkpoints.py) | Checkpoints as versioned artifacts, deduplicated by content and fetched later by alias. |
124
+ | [`multiprocess_pipeline.py`](examples/multiprocess_pipeline.py) | Four data producers and four trainers as eight processes in one run, with bounded staleness. Each worker gets its own stream and its own lane in the trace, and the blocking spans show which side is the bottleneck. |
125
+
126
+ ```bash
127
+ uv run python examples/quickstart.py
128
+ uv run python examples/alert_rules.py --fault spike
129
+ uv run python examples/multiprocess_pipeline.py --produce-ms 10 --train-ms 40
130
+ ```
116
131
 
117
132
  ## wandb compatibility
118
133
 
@@ -155,12 +170,13 @@ uv run ruff format src tests
155
170
  | `test_expr_properties.py` | DSL properties: render round-trip stability, precedence, the whole `M` builder |
156
171
  | `test_trace.py` | Chrome Trace export: lane layout, stream and step selection, the CLI |
157
172
  | `test_spans.py` | nesting, aggregation, decorator and async forms, errors, thread and task isolation |
173
+ | `test_examples.py` | the shipped examples run, and their backpressure claims hold |
158
174
  | `test_span_plugins.py` | `print_fn` output and indentation, the plugin protocol, failure isolation, CPU/GPU built-ins |
159
175
  | `test_streams.py` | stream naming and validation, isolation, resolution order, backend grouping, two-process runs |
160
176
  | `test_distributed.py` | rank shards, `alert_on_rank`, real multi-process runs |
161
177
  | `test_wandb.py` | real wandb in offline mode: parameter mapping, step alignment, artifacts |
162
178
  | `test_trackio.py` | trackio contract, resume mapping, real end-to-end |
163
- | `test_lark_live.py` | Lark channel; real delivery when `ET_LARK_TEST_WEBHOOK` is set |
179
+ | `test_lark.py` | Lark card construction; real delivery when `ET_LARK_TEST_WEBHOOK` is set |
164
180
  | `test_stress.py` (`slow`) | 100k-row writes, concurrency, cache thrash, write-failure recovery |
165
181
  | `test_benchmark.py` (`benchmark`) | throughput, tail latency, query cost, memory stability |
166
182
 
@@ -1,11 +1,11 @@
1
1
  expr_tracker/__init__.py,sha256=AdY6R3bKLlZFhNtJPBAA3UHhO9nZ7YpojLrkW9OevlM,1442
2
2
  expr_tracker/_compat.py,sha256=qf8oYIK_lkoLzcPp6wPCYLJtVatCl7ROYRNPwc5F1Pg,981
3
3
  expr_tracker/artifacts.py,sha256=yRVXmuqdZUxiZa0c_CwZi4rBxaItwhu9bZ1x21yP1S0,12560
4
- expr_tracker/cli.py,sha256=dtp0YsNtGg7T7M-elA511YX8NU_-OJRAwNbs8I1353o,7255
4
+ expr_tracker/cli.py,sha256=ZgtO6dZWiWPL3yUBfY0ATX6-GAtT-TWrI5m013nH9nI,7483
5
5
  expr_tracker/encoders.py,sha256=X5ySQasSrYPsdYCdk7nhpJFGFB9f_8HWQuGOQjTvh9E,11991
6
6
  expr_tracker/plugins.py,sha256=-ako71QBr5ARYS3aFUueHx3SMESk5ii2wj_yiz2_Mzc,9592
7
7
  expr_tracker/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
- expr_tracker/run.py,sha256=nuXhNmepG688X86cI0FMM_hqjyygD1Oi8Ck8-lyHsPA,16664
8
+ expr_tracker/run.py,sha256=kT0fDPI7KCDvd04NunnCPgd4QWS-3p8S462XHKc3Zmw,17373
9
9
  expr_tracker/spans.py,sha256=sYj7BoFuNXEd4Ic11W_cqhLEjkh3ViBfTnW4vOvTTkg,12144
10
10
  expr_tracker/summary.py,sha256=lpHSh3pci2Ym6wJhVZwNUJbUp2A64EN_rC7Ym2d7PPI,3150
11
11
  expr_tracker/trace.py,sha256=stOIuEzuFLHZrrqbaTWndOOnhSXEJzHF91zPGDpU5PA,7019
@@ -15,8 +15,9 @@ expr_tracker/alerts/__init__.py,sha256=dl96OYZh75kV3NLBSA35zNvTFcL6XFMuPoYvRoz7h
15
15
  expr_tracker/alerts/dispatch.py,sha256=k2pDZVJy6ajlQ7kUdvaBBZ_AmdtZ8opaQWjADn_i8uI,11652
16
16
  expr_tracker/alerts/engine.py,sha256=Lcfc3VchFegmbAwHwTE6mWuG4gUYAlbsAoq5gDd6cr8,13774
17
17
  expr_tracker/alerts/models.py,sha256=LjPGBKXroxGP1qYm456shNQAbkm1PEVga0vBy4nxRZI,11946
18
- expr_tracker/alerts/backends/__init__.py,sha256=6j2wyfG1mMuWkdI5i99gOUlxOTwfuMOeg78dYs_ciT0,8620
18
+ expr_tracker/alerts/backends/__init__.py,sha256=sdq19EWo0Rc3iqMjQC3ceHncKOKR0Lswic8OeTaDCnY,8629
19
19
  expr_tracker/alerts/backends/base.py,sha256=tILaNVDQ4rt9WoxO-KXHfWkIza_lX-eFO70LsSf2aTo,6479
20
+ expr_tracker/alerts/backends/cards.py,sha256=Wn3hmVK6BFdSlBthrQy6FeHZiEzEGrrbi7HXfYRXrhM,3164
20
21
  expr_tracker/alerts/expr/__init__.py,sha256=qXBt6ih1pMAplUozsDzYY_ZWHlLABX4NoshbjJO_oJk,616
21
22
  expr_tracker/alerts/expr/eval.py,sha256=UccIwg1r-A_yzUf16TGyx3MO8e0kegCsrRHDyHirsNg,11609
22
23
  expr_tracker/alerts/expr/functions.py,sha256=wuuFKplta_5KkFHIsvEmpryFtPCdfHV4wjqnx3t2g9A,7043
@@ -32,8 +33,8 @@ expr_tracker/history/reader.py,sha256=5PY5nbNfX-Cl34jcyYTq2qm8nNgNDTqv7U5Xnb9Rgx
32
33
  expr_tracker/history/series.py,sha256=cFdUO1FRs5KW0jYsNPzda3tcu645NH1wwfXJitQfXFM,4042
33
34
  expr_tracker/history/store.py,sha256=uSbHcGdLXFVFbYQKm6IaQgpdCrNvvb9Axpxi9emvwBo,36228
34
35
  expr_tracker/history/writer.py,sha256=q5kNANcZCq5nykvOK5jZ2xc6Z1B4oZxEn3uSXzDCtlY,16096
35
- expr_tracker-0.2.4.dist-info/METADATA,sha256=aSjFfY1k1dJwzBRHWvm3aqisY6O0Gi6S7HvMAp37Fs4,9447
36
- expr_tracker-0.2.4.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
37
- expr_tracker-0.2.4.dist-info/entry_points.txt,sha256=GfZCAG2JFsd5CxTjRsGJAhYInpBCuLjrU3csmUVBDSs,45
38
- expr_tracker-0.2.4.dist-info/licenses/LICENSE,sha256=ORsYTnF-2SLUaM8FgXlJ-uXuN4ceGACAAU3JZiS8pvs,1069
39
- expr_tracker-0.2.4.dist-info/RECORD,,
36
+ expr_tracker-0.2.6.dist-info/METADATA,sha256=H4E6G-Ksu2tD0qCnaQeBV-hNxcos5XaLprmuohGY4RA,10784
37
+ expr_tracker-0.2.6.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
38
+ expr_tracker-0.2.6.dist-info/entry_points.txt,sha256=GfZCAG2JFsd5CxTjRsGJAhYInpBCuLjrU3csmUVBDSs,45
39
+ expr_tracker-0.2.6.dist-info/licenses/LICENSE,sha256=ORsYTnF-2SLUaM8FgXlJ-uXuN4ceGACAAU3JZiS8pvs,1069
40
+ expr_tracker-0.2.6.dist-info/RECORD,,