expr_tracker 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- expr_tracker/cli.py +14 -1
- expr_tracker/run.py +20 -6
- {expr_tracker-0.2.4.dist-info → expr_tracker-0.2.5.dist-info}/METADATA +22 -2
- {expr_tracker-0.2.4.dist-info → expr_tracker-0.2.5.dist-info}/RECORD +7 -7
- {expr_tracker-0.2.4.dist-info → expr_tracker-0.2.5.dist-info}/WHEEL +0 -0
- {expr_tracker-0.2.4.dist-info → expr_tracker-0.2.5.dist-info}/entry_points.txt +0 -0
- {expr_tracker-0.2.4.dist-info → expr_tracker-0.2.5.dist-info}/licenses/LICENSE +0 -0
expr_tracker/cli.py
CHANGED
|
@@ -36,16 +36,29 @@ def alert(msg: str, title: str, level: str, channels: tuple[str, ...]):
|
|
|
36
36
|
@click.option("-n", default=20, help="Number of steps (-1 for all)")
|
|
37
37
|
@click.option("--metrics", default=None, help="Comma separated metric names")
|
|
38
38
|
@click.option("--step-range", default=None, help="start:end (end exclusive)")
|
|
39
|
+
@click.option(
|
|
40
|
+
"--stream",
|
|
41
|
+
default=None,
|
|
42
|
+
help="Which stream to read; the unnamed one by default.",
|
|
43
|
+
)
|
|
39
44
|
@click.option(
|
|
40
45
|
"--format", "fmt", type=click.Choice(["table", "json", "csv"]), default="table"
|
|
41
46
|
)
|
|
42
|
-
def history(
|
|
47
|
+
def history(
|
|
48
|
+
run: str,
|
|
49
|
+
n: int,
|
|
50
|
+
metrics: str | None,
|
|
51
|
+
step_range: str | None,
|
|
52
|
+
stream: str | None,
|
|
53
|
+
fmt: str,
|
|
54
|
+
):
|
|
43
55
|
"""Print the recorded history of a run."""
|
|
44
56
|
rows = read_history(
|
|
45
57
|
run,
|
|
46
58
|
n,
|
|
47
59
|
metrics=metrics.split(",") if metrics else None,
|
|
48
60
|
step_range=_parse_range(step_range),
|
|
61
|
+
stream=None if stream in (None, "default", "") else stream,
|
|
49
62
|
)
|
|
50
63
|
if fmt == "json":
|
|
51
64
|
click.echo(json.dumps(rows, ensure_ascii=False, indent=2))
|
expr_tracker/run.py
CHANGED
|
@@ -73,6 +73,7 @@ class Run:
|
|
|
73
73
|
alert: Any = None,
|
|
74
74
|
alert_rules: Sequence[Any] = (),
|
|
75
75
|
alert_on_rank: int | None = 0,
|
|
76
|
+
backend_on_rank: int | None = 0,
|
|
76
77
|
stream: str | None = None,
|
|
77
78
|
**history_kwargs,
|
|
78
79
|
):
|
|
@@ -115,10 +116,18 @@ class Run:
|
|
|
115
116
|
root=Path(dir or "./tracker/jsonl") / project / "artifacts"
|
|
116
117
|
)
|
|
117
118
|
|
|
119
|
+
self.rank = current_rank()
|
|
118
120
|
self.backends: dict[str, Any] = {}
|
|
119
121
|
self.alerts = None
|
|
122
|
+
# Every rank would otherwise open the same remote run and interleave its
|
|
123
|
+
# steps into it, which is what the local rankN shards exist to prevent
|
|
124
|
+
remote = backend_on_rank is None or self.rank == backend_on_rank
|
|
125
|
+
# Rank 0 belongs in the group too when the other ranks are reporting
|
|
126
|
+
self._grouped = bool(stream) or bool(self.rank) or backend_on_rank is None
|
|
127
|
+
if not remote:
|
|
128
|
+
logger.debug(f"Rank {self.rank} keeps its history local, no backends")
|
|
120
129
|
try:
|
|
121
|
-
for backend in dict.fromkeys(backends):
|
|
130
|
+
for backend in dict.fromkeys(backends) if remote else ():
|
|
122
131
|
if backend == "jsonl":
|
|
123
132
|
continue
|
|
124
133
|
try:
|
|
@@ -146,7 +155,6 @@ class Run:
|
|
|
146
155
|
kwargs=backend_kwargs.get(name, {}),
|
|
147
156
|
)
|
|
148
157
|
self._closers.append(instance.finish)
|
|
149
|
-
self.rank = current_rank()
|
|
150
158
|
# Every rank would otherwise raise the same alert N times
|
|
151
159
|
if alert_on_rank is not None and self.rank != alert_on_rank:
|
|
152
160
|
alert, alert_rules = {"enabled": False}, ()
|
|
@@ -158,18 +166,24 @@ class Run:
|
|
|
158
166
|
|
|
159
167
|
@property
|
|
160
168
|
def backend_run_name(self) -> str:
|
|
161
|
-
"""A stream is an independent producer, so
|
|
169
|
+
"""A stream or a rank is an independent producer of steps, so each gets
|
|
170
|
+
its own backend run.
|
|
162
171
|
|
|
163
172
|
Grouping ties them back together: both wandb and trackio understand
|
|
164
173
|
``group``, and neither can merge two step axes into one run.
|
|
165
174
|
"""
|
|
166
|
-
|
|
175
|
+
parts = [self.name]
|
|
176
|
+
if self.stream:
|
|
177
|
+
parts.append(self.stream)
|
|
178
|
+
if self.rank:
|
|
179
|
+
parts.append(f"rank{self.rank}")
|
|
180
|
+
return "-".join(parts)
|
|
167
181
|
|
|
168
182
|
def _init_backend(
|
|
169
183
|
self, name, instance, *, entity, dir, notes, tags, resume, kwargs
|
|
170
184
|
):
|
|
171
|
-
# A grouped run keeps the streams together in the backend UI
|
|
172
|
-
grouping = {"group": self.name} if self.
|
|
185
|
+
# A grouped run keeps the streams and ranks together in the backend UI
|
|
186
|
+
grouping = {"group": self.name} if self._grouped else {}
|
|
173
187
|
try:
|
|
174
188
|
if name == "trackio":
|
|
175
189
|
# trackio.init has no entity/notes/tags/dir/id, so those ride along
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: expr_tracker
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Local-first experiment tracking with queryable history and expression-based alerts on your training metrics
|
|
5
5
|
Project-URL: Homepage, https://hspk.github.io/expr_tracker/
|
|
6
6
|
Project-URL: Documentation, https://hspk.github.io/expr_tracker/
|
|
@@ -112,7 +112,26 @@ install command; it never crashes a run.
|
|
|
112
112
|
| [Spans](https://hspk.github.io/expr_tracker/guide/spans/) | Time the parts of a step, and their parts. Each duration becomes a metric, so alerts and queries work on it unchanged; `et trace` exports the timeline for Perfetto. `print_fn` prints the tree live, and plugins attach CPU and GPU cost to each region. |
|
|
113
113
|
| [Streams](https://hspk.github.io/expr_tracker/guide/streams/) | Independent producers — a data worker and a training loop — each with their own step cursor and file inside one run. |
|
|
114
114
|
| [Distributed](https://hspk.github.io/expr_tracker/guide/distributed/) | Per-rank shards so concurrent appends cannot corrupt step order; only rank 0 alerts by default. |
|
|
115
|
-
| [CLI](https://hspk.github.io/expr_tracker/guide/cli/) | `et history`, `et rules explain`, `et rules test`, `et alert`. |
|
|
115
|
+
| [CLI](https://hspk.github.io/expr_tracker/guide/cli/) | `et history`, `et trace`, `et rules explain`, `et rules test`, `et alert`. |
|
|
116
|
+
|
|
117
|
+
## Examples
|
|
118
|
+
|
|
119
|
+
Every one runs offline, with no account and no network.
|
|
120
|
+
|
|
121
|
+
| | |
|
|
122
|
+
| --- | --- |
|
|
123
|
+
| [`quickstart.py`](examples/quickstart.py) | The sixty-second tour: log a run, merge eval into the training step, read the history back while it is open and again afterwards. |
|
|
124
|
+
| [`alert_rules.py`](examples/alert_rules.py) | Four rules against four faults — a loss spike, a non-finite loss, a stalled curve and a regression that must persist. `--fault none` fires nothing, which is the point. |
|
|
125
|
+
| [`profile_step.py`](examples/profile_step.py) | Where a step goes. Nested spans become metrics, a plugin adds CPU cost, and the tree exports to Perfetto. |
|
|
126
|
+
| [`early_stopping.py`](examples/early_stopping.py) | Querying your own history mid-run to decay the learning rate on a plateau and stop when it stops paying. |
|
|
127
|
+
| [`checkpoints.py`](examples/checkpoints.py) | Checkpoints as versioned artifacts, deduplicated by content and fetched later by alias. |
|
|
128
|
+
| [`multiprocess_pipeline.py`](examples/multiprocess_pipeline.py) | Four data producers and four trainers as eight processes in one run, with bounded staleness. Each worker gets its own stream and its own lane in the trace, and the blocking spans show which side is the bottleneck. |
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
uv run python examples/quickstart.py
|
|
132
|
+
uv run python examples/alert_rules.py --fault spike
|
|
133
|
+
uv run python examples/multiprocess_pipeline.py --produce-ms 10 --train-ms 40
|
|
134
|
+
```
|
|
116
135
|
|
|
117
136
|
## wandb compatibility
|
|
118
137
|
|
|
@@ -155,6 +174,7 @@ uv run ruff format src tests
|
|
|
155
174
|
| `test_expr_properties.py` | DSL properties: render round-trip stability, precedence, the whole `M` builder |
|
|
156
175
|
| `test_trace.py` | Chrome Trace export: lane layout, stream and step selection, the CLI |
|
|
157
176
|
| `test_spans.py` | nesting, aggregation, decorator and async forms, errors, thread and task isolation |
|
|
177
|
+
| `test_examples.py` | the shipped examples run, and their backpressure claims hold |
|
|
158
178
|
| `test_span_plugins.py` | `print_fn` output and indentation, the plugin protocol, failure isolation, CPU/GPU built-ins |
|
|
159
179
|
| `test_streams.py` | stream naming and validation, isolation, resolution order, backend grouping, two-process runs |
|
|
160
180
|
| `test_distributed.py` | rank shards, `alert_on_rank`, real multi-process runs |
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
expr_tracker/__init__.py,sha256=AdY6R3bKLlZFhNtJPBAA3UHhO9nZ7YpojLrkW9OevlM,1442
|
|
2
2
|
expr_tracker/_compat.py,sha256=qf8oYIK_lkoLzcPp6wPCYLJtVatCl7ROYRNPwc5F1Pg,981
|
|
3
3
|
expr_tracker/artifacts.py,sha256=yRVXmuqdZUxiZa0c_CwZi4rBxaItwhu9bZ1x21yP1S0,12560
|
|
4
|
-
expr_tracker/cli.py,sha256=
|
|
4
|
+
expr_tracker/cli.py,sha256=ZgtO6dZWiWPL3yUBfY0ATX6-GAtT-TWrI5m013nH9nI,7483
|
|
5
5
|
expr_tracker/encoders.py,sha256=X5ySQasSrYPsdYCdk7nhpJFGFB9f_8HWQuGOQjTvh9E,11991
|
|
6
6
|
expr_tracker/plugins.py,sha256=-ako71QBr5ARYS3aFUueHx3SMESk5ii2wj_yiz2_Mzc,9592
|
|
7
7
|
expr_tracker/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
expr_tracker/run.py,sha256=
|
|
8
|
+
expr_tracker/run.py,sha256=kT0fDPI7KCDvd04NunnCPgd4QWS-3p8S462XHKc3Zmw,17373
|
|
9
9
|
expr_tracker/spans.py,sha256=sYj7BoFuNXEd4Ic11W_cqhLEjkh3ViBfTnW4vOvTTkg,12144
|
|
10
10
|
expr_tracker/summary.py,sha256=lpHSh3pci2Ym6wJhVZwNUJbUp2A64EN_rC7Ym2d7PPI,3150
|
|
11
11
|
expr_tracker/trace.py,sha256=stOIuEzuFLHZrrqbaTWndOOnhSXEJzHF91zPGDpU5PA,7019
|
|
@@ -32,8 +32,8 @@ expr_tracker/history/reader.py,sha256=5PY5nbNfX-Cl34jcyYTq2qm8nNgNDTqv7U5Xnb9Rgx
|
|
|
32
32
|
expr_tracker/history/series.py,sha256=cFdUO1FRs5KW0jYsNPzda3tcu645NH1wwfXJitQfXFM,4042
|
|
33
33
|
expr_tracker/history/store.py,sha256=uSbHcGdLXFVFbYQKm6IaQgpdCrNvvb9Axpxi9emvwBo,36228
|
|
34
34
|
expr_tracker/history/writer.py,sha256=q5kNANcZCq5nykvOK5jZ2xc6Z1B4oZxEn3uSXzDCtlY,16096
|
|
35
|
-
expr_tracker-0.2.
|
|
36
|
-
expr_tracker-0.2.
|
|
37
|
-
expr_tracker-0.2.
|
|
38
|
-
expr_tracker-0.2.
|
|
39
|
-
expr_tracker-0.2.
|
|
35
|
+
expr_tracker-0.2.5.dist-info/METADATA,sha256=F9swdSmR6JGdnCxll_RKzerh5euOuD8he2WEJU1INtU,10955
|
|
36
|
+
expr_tracker-0.2.5.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
37
|
+
expr_tracker-0.2.5.dist-info/entry_points.txt,sha256=GfZCAG2JFsd5CxTjRsGJAhYInpBCuLjrU3csmUVBDSs,45
|
|
38
|
+
expr_tracker-0.2.5.dist-info/licenses/LICENSE,sha256=ORsYTnF-2SLUaM8FgXlJ-uXuN4ceGACAAU3JZiS8pvs,1069
|
|
39
|
+
expr_tracker-0.2.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|