alag 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alag-0.1.0/.github/workflows/ci.yml +62 -0
- alag-0.1.0/.gitignore +7 -0
- alag-0.1.0/Cargo.lock +7 -0
- alag-0.1.0/Cargo.toml +18 -0
- alag-0.1.0/LICENSE +21 -0
- alag-0.1.0/PKG-INFO +201 -0
- alag-0.1.0/README.md +179 -0
- alag-0.1.0/pyproject.toml +35 -0
- alag-0.1.0/python/alag/__init__.py +6 -0
- alag-0.1.0/python/alag/logger.py +336 -0
- alag-0.1.0/scripts/build-wheels.sh +27 -0
- alag-0.1.0/src/index.html +1717 -0
- alag-0.1.0/src/main.rs +633 -0
- alag-0.1.0/tests/test_logger.py +177 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
tags: ["v*"]
|
|
7
|
+
pull_request:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
15
|
+
python: ["3.9", "3.14"]
|
|
16
|
+
runs-on: ${{ matrix.os }}
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: ${{ matrix.python }}
|
|
22
|
+
- run: cargo test
|
|
23
|
+
- run: pip install pytest psutil matplotlib
|
|
24
|
+
- run: python -m pytest tests
|
|
25
|
+
env:
|
|
26
|
+
PYTHONPATH: python
|
|
27
|
+
|
|
28
|
+
wheels:
|
|
29
|
+
strategy:
|
|
30
|
+
fail-fast: false
|
|
31
|
+
matrix:
|
|
32
|
+
include:
|
|
33
|
+
- { os: ubuntu-latest, target: x86_64, manylinux: "2014" }
|
|
34
|
+
- { os: ubuntu-latest, target: aarch64, manylinux: "2014" }
|
|
35
|
+
- { os: macos-latest, target: x86_64 }
|
|
36
|
+
- { os: macos-latest, target: aarch64 }
|
|
37
|
+
- { os: windows-latest, target: x64 }
|
|
38
|
+
runs-on: ${{ matrix.os }}
|
|
39
|
+
steps:
|
|
40
|
+
- uses: actions/checkout@v4
|
|
41
|
+
- uses: PyO3/maturin-action@v1
|
|
42
|
+
with:
|
|
43
|
+
target: ${{ matrix.target }}
|
|
44
|
+
manylinux: ${{ matrix.manylinux || 'auto' }}
|
|
45
|
+
args: --release --out dist
|
|
46
|
+
- uses: actions/upload-artifact@v4
|
|
47
|
+
with:
|
|
48
|
+
name: wheel-${{ matrix.os }}-${{ matrix.target }}
|
|
49
|
+
path: dist
|
|
50
|
+
|
|
51
|
+
sdist:
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
steps:
|
|
54
|
+
- uses: actions/checkout@v4
|
|
55
|
+
- uses: PyO3/maturin-action@v1
|
|
56
|
+
with:
|
|
57
|
+
command: sdist
|
|
58
|
+
args: --out dist
|
|
59
|
+
- uses: actions/upload-artifact@v4
|
|
60
|
+
with:
|
|
61
|
+
name: sdist
|
|
62
|
+
path: dist
|
alag-0.1.0/.gitignore
ADDED
alag-0.1.0/Cargo.lock
ADDED
alag-0.1.0/Cargo.toml
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "alag"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
description = "A simple logging and graphing tool for researchers"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
repository = "https://github.com/SandyTheAdventurer/alag"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
|
|
10
|
+
[[bin]]
|
|
11
|
+
name = "alag"
|
|
12
|
+
path = "src/main.rs"
|
|
13
|
+
|
|
14
|
+
[profile.release]
|
|
15
|
+
opt-level = "s"
|
|
16
|
+
lto = true
|
|
17
|
+
strip = true
|
|
18
|
+
panic = "abort"
|
alag-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sandeep S
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
alag-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: alag
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Classifier: Development Status :: 3 - Alpha
|
|
5
|
+
Classifier: Intended Audience :: Science/Research
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Programming Language :: Rust
|
|
8
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
9
|
+
Classifier: Topic :: Scientific/Engineering :: Visualization
|
|
10
|
+
Requires-Dist: psutil ; extra == 'system'
|
|
11
|
+
Provides-Extra: system
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Summary: A simple logging and graphing tool for researchers
|
|
14
|
+
Keywords: experiment tracking,logging,machine learning,plotting
|
|
15
|
+
Author: Sandeep S
|
|
16
|
+
License-Expression: MIT
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
19
|
+
Project-URL: Issues, https://github.com/SandyTheAdventurer/alag/issues
|
|
20
|
+
Project-URL: Repository, https://github.com/SandyTheAdventurer/alag
|
|
21
|
+
|
|
22
|
+
# ALAG
|
|
23
|
+
|
|
24
|
+
**A** simple **L**ogging **A**nd **G**raphing tool, for researchers.
|
|
25
|
+
|
|
26
|
+
## Why
|
|
27
|
+
|
|
28
|
+
Existing experiment trackers ask for more than a researcher who wants to log a run and look at the results should have to give:
|
|
29
|
+
|
|
30
|
+
- **Weights & Biases** requires an account and other setup.
|
|
31
|
+
- **MLflow** needs a tracking server that is heavy for this job, and comparing runs in it feels awkward.
|
|
32
|
+
- **TensorBoard** has visualizations that leave a lot to be desired, and makes comparing multiple runs hard.
|
|
33
|
+
|
|
34
|
+
ALAG aims to do two things well, logging and graphing, without the overhead. Comparing multiple runs is a core feature.
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```sh
|
|
39
|
+
pip install alag
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
This gives you the `ALAG` class for your training code and the `alag` command for viewing results.
|
|
43
|
+
|
|
44
|
+
Wheels are built for Linux (x86-64 and ARM, glibc 2.17 or newer) and Windows. On macOS, pip builds ALAG from source, which needs [Rust](https://rustup.rs).
|
|
45
|
+
|
|
46
|
+
## Usage
|
|
47
|
+
|
|
48
|
+
Log from your training code:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from alag import ALAG
|
|
52
|
+
|
|
53
|
+
logger = ALAG(group="my-group", experiment="my-experiment", config={"lr": 3e-4})
|
|
54
|
+
|
|
55
|
+
logger.log({"train/loss": 0.42}, step=100)
|
|
56
|
+
logger.log_image("confusion-matrix", "confusion.png", step=100) # from a file
|
|
57
|
+
logger.log_image("loss-curve", fig, step=100) # from a matplotlib figure
|
|
58
|
+
|
|
59
|
+
logger.tag("baseline", "ablation")
|
|
60
|
+
logger.note("Loss spikes after the LR warmup ends")
|
|
61
|
+
logger.flag("val/loss", below=0.5, by_step=1000)
|
|
62
|
+
logger.section("validation", "val/loss", "bleu")
|
|
63
|
+
logger.pin("val/loss")
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`ALAG` also takes `tags=[...]`, `note="..."` and `system_metrics=True` directly.
|
|
67
|
+
|
|
68
|
+
Then view your experiments:
|
|
69
|
+
|
|
70
|
+
```sh
|
|
71
|
+
alag # serves ./logs at http://localhost:6100
|
|
72
|
+
alag path/to/logs --port 8000
|
|
73
|
+
alag --export report.html # one HTML file to send to someone
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
If an experiment with that name already exists in the group, ALAG creates a new one with a number on the end, such as `my-experiment-2`, so a re-run never mixes with earlier results.
|
|
77
|
+
|
|
78
|
+
To view experiments on a remote machine, run `alag` there and forward the port with `ssh -L 6100:localhost:6100 my-server`.
|
|
79
|
+
|
|
80
|
+
## Speed
|
|
81
|
+
|
|
82
|
+
`logger.log` is built to stay out of your training loop. It only puts the values on an in-memory queue and returns. A background thread formats them and writes to disk about once a second.
|
|
83
|
+
|
|
84
|
+
You can pass GPU tensors straight to `logger.log`. The background thread reads them, so your training loop doesn't wait for the GPU the way it does with `loss.item()`.
|
|
85
|
+
|
|
86
|
+
On an RTX 3050 laptop GPU, a `logger.log` call with a GPU tensor took about 15 µs. A 6 ms training step was no slower with logging than without it, with or without system metrics. Any difference was smaller than the variation between runs.
|
|
87
|
+
|
|
88
|
+
`logger.log_image` does its work immediately, because matplotlib isn't thread-safe. Log images occasionally, not on every step.
|
|
89
|
+
|
|
90
|
+
Queued values are written when your program exits, including after an exception. If the process is killed outright, you can lose about the last second of values.
|
|
91
|
+
|
|
92
|
+
## What it logs
|
|
93
|
+
|
|
94
|
+
- **Scalars**, such as loss and accuracy.
|
|
95
|
+
- **Images**, such as custom graphs you make yourself.
|
|
96
|
+
- **Hyperparameters**, from the `config` you pass to `ALAG`.
|
|
97
|
+
- **System metrics**, if you turn them on.
|
|
98
|
+
|
|
99
|
+
### System metrics
|
|
100
|
+
|
|
101
|
+
These are off by default. To turn them on, install the extra and pass `system_metrics=True`:
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
pip install 'alag[system]'
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
logger = ALAG(group="my-group", experiment="my-experiment", system_metrics=True)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Every 10 seconds, a background thread logs CPU use, system RAM and your process's RAM under `system/`. If `nvidia-smi` is available, it also logs utilization, memory, temperature and power for each GPU. GPU memory is the total in use on that card, including other programs.
|
|
112
|
+
|
|
113
|
+
Each sample uses your latest training step. Before your first `logger.log`, or during a long step, several samples share a step, so set **X axis** to **Time** to read these charts.
|
|
114
|
+
|
|
115
|
+
## Storage
|
|
116
|
+
|
|
117
|
+
Experiments are organized into groups. Each experiment is a plain folder inside its group's folder. There is no database or account.
|
|
118
|
+
|
|
119
|
+
```
|
|
120
|
+
logs/
|
|
121
|
+
└── my-group/
|
|
122
|
+
├── flags.json # targets shared by the group
|
|
123
|
+
├── layout.json # chart sections and pins shared by the group
|
|
124
|
+
└── my-experiment/
|
|
125
|
+
├── config.json # the experiment's settings
|
|
126
|
+
├── metrics.csv # every scalar, appended as it is logged
|
|
127
|
+
├── meta.json # tags and note
|
|
128
|
+
└── images/ # logged images
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
`metrics.csv` has one row per logged value, so an experiment can add new metrics at any point and metrics logged at different intervals don't leave empty cells:
|
|
132
|
+
|
|
133
|
+
```csv
|
|
134
|
+
step,time,metric,value
|
|
135
|
+
100,1762444800.1,train/loss,0.42
|
|
136
|
+
100,1762444800.1,train/acc,0.81
|
|
137
|
+
200,1762444812.6,train/loss,0.37
|
|
138
|
+
200,1762444812.7,val/loss,0.45
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Viewing experiments
|
|
142
|
+
|
|
143
|
+
The `alag` command is a small compiled Rust program that generates an HTML page and serves it on localhost. The page updates automatically while your experiments are training.
|
|
144
|
+
|
|
145
|
+
A panel on the left lists your groups and the experiments inside them. Select experiments in the panel to compare them: each metric gets one chart, with a line for every selected experiment.
|
|
146
|
+
|
|
147
|
+
### Hyperparameters
|
|
148
|
+
|
|
149
|
+
The **Hyperparameters** tab has a row for each selected experiment and a column for each setting in its `config`. Settings that differ come first, in bold. Click a column to sort by it, or tick **Only settings that differ** to hide the rest.
|
|
150
|
+
|
|
151
|
+
### Sections and pins
|
|
152
|
+
|
|
153
|
+
Charts are grouped into sections by the part of the metric name before the first slash, so `val/loss` goes under `val` and `system/cpu_percent` goes under `system`.
|
|
154
|
+
|
|
155
|
+
To choose a section yourself, use **Section** on a chart or `logger.section`. **Pin** on a chart or image, or `logger.pin`, moves it to the top of its tab.
|
|
156
|
+
|
|
157
|
+
Sections and pins apply to the whole group, so they stay in place when you compare its experiments.
|
|
158
|
+
|
|
159
|
+
### Mean and std over seeds
|
|
160
|
+
|
|
161
|
+
Tick **Mean ± std over seeds** to merge selected experiments into one line with a shaded band. Experiments merge when they are in the same group and their configs differ only by a key named `seed`. Experiments with no `seed` in their config stay separate.
|
|
162
|
+
|
|
163
|
+
The band is one sample standard deviation at each step. Smoothing is applied to each seed before merging.
|
|
164
|
+
|
|
165
|
+
### Flags
|
|
166
|
+
|
|
167
|
+
A flag sets a target, such as "`val/loss` should go below 0.5 by step 1,000". Set one with `logger.flag`, or with **Flag** on a chart, where clicking the chart fills in the values.
|
|
168
|
+
|
|
169
|
+
The flag is drawn on the chart, and each experiment in the group is listed as reached, missed or not yet. Verdicts use the line as plotted, so they follow your smoothing and seed settings.
|
|
170
|
+
|
|
171
|
+
### Notes and tags
|
|
172
|
+
|
|
173
|
+
Add these in the **Notes** tab or from Python. Tags appear in the left panel, and the filter box searches names, tags and notes.
|
|
174
|
+
|
|
175
|
+
### Exporting charts
|
|
176
|
+
|
|
177
|
+
Each chart has **SVG**, **PNG**, **PDF**, **CSV** and **LaTeX** buttons. Exports use a white background and match the current view, including every selected experiment, smoothing, zoom, log scale and flags.
|
|
178
|
+
|
|
179
|
+
**LaTeX** gives you a `pgfplots` figure to copy and paste. The data is included, so there's no file to upload, but long runs are thinned to keep it small enough for pdflatex. Your document needs:
|
|
180
|
+
|
|
181
|
+
```latex
|
|
182
|
+
\usepackage{pgfplots}
|
|
183
|
+
\pgfplotsset{compat=1.18}
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
### Sharing a report
|
|
187
|
+
|
|
188
|
+
`alag --export report.html` writes one HTML file that opens in any browser, with no server or install. It opens on the most recently updated group, with its experiments selected.
|
|
189
|
+
|
|
190
|
+
You can still select experiments, zoom, smooth, merge seeds and export charts. It's a read-only snapshot, so it doesn't update, and notes, tags, flags, sections and pins can't be changed.
|
|
191
|
+
|
|
192
|
+
The file contains everything in the log folder, including every group, config, note and image. To share only part of it, export a folder with just those groups. Runs with many images can make the file large.
|
|
193
|
+
|
|
194
|
+
## Building wheels
|
|
195
|
+
|
|
196
|
+
`scripts/build-wheels.sh` builds the source package and every wheel into `dist/`. It needs only Docker.
|
|
197
|
+
|
|
198
|
+
## License
|
|
199
|
+
|
|
200
|
+
MIT
|
|
201
|
+
|
alag-0.1.0/README.md
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# ALAG
|
|
2
|
+
|
|
3
|
+
**A** simple **L**ogging **A**nd **G**raphing tool, for researchers.
|
|
4
|
+
|
|
5
|
+
## Why
|
|
6
|
+
|
|
7
|
+
Existing experiment trackers ask for more than a researcher who wants to log a run and look at the results should have to give:
|
|
8
|
+
|
|
9
|
+
- **Weights & Biases** requires an account and other setup.
|
|
10
|
+
- **MLflow** needs a tracking server that is heavy for this job, and comparing runs in it feels awkward.
|
|
11
|
+
- **TensorBoard** has visualizations that leave a lot to be desired, and makes comparing multiple runs hard.
|
|
12
|
+
|
|
13
|
+
ALAG aims to do two things well, logging and graphing, without the overhead. Comparing multiple runs is a core feature.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
pip install alag
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
This gives you the `ALAG` class for your training code and the `alag` command for viewing results.
|
|
22
|
+
|
|
23
|
+
Wheels are built for Linux (x86-64 and ARM, glibc 2.17 or newer) and Windows. On macOS, pip builds ALAG from source, which needs [Rust](https://rustup.rs).
|
|
24
|
+
|
|
25
|
+
## Usage
|
|
26
|
+
|
|
27
|
+
Log from your training code:
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from alag import ALAG
|
|
31
|
+
|
|
32
|
+
logger = ALAG(group="my-group", experiment="my-experiment", config={"lr": 3e-4})
|
|
33
|
+
|
|
34
|
+
logger.log({"train/loss": 0.42}, step=100)
|
|
35
|
+
logger.log_image("confusion-matrix", "confusion.png", step=100) # from a file
|
|
36
|
+
logger.log_image("loss-curve", fig, step=100) # from a matplotlib figure
|
|
37
|
+
|
|
38
|
+
logger.tag("baseline", "ablation")
|
|
39
|
+
logger.note("Loss spikes after the LR warmup ends")
|
|
40
|
+
logger.flag("val/loss", below=0.5, by_step=1000)
|
|
41
|
+
logger.section("validation", "val/loss", "bleu")
|
|
42
|
+
logger.pin("val/loss")
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`ALAG` also takes `tags=[...]`, `note="..."` and `system_metrics=True` directly.
|
|
46
|
+
|
|
47
|
+
Then view your experiments:
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
alag # serves ./logs at http://localhost:6100
|
|
51
|
+
alag path/to/logs --port 8000
|
|
52
|
+
alag --export report.html # one HTML file to send to someone
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
If an experiment with that name already exists in the group, ALAG creates a new one with a number on the end, such as `my-experiment-2`, so a re-run never mixes with earlier results.
|
|
56
|
+
|
|
57
|
+
To view experiments on a remote machine, run `alag` there and forward the port with `ssh -L 6100:localhost:6100 my-server`.
|
|
58
|
+
|
|
59
|
+
## Speed
|
|
60
|
+
|
|
61
|
+
`logger.log` is built to stay out of your training loop. It only puts the values on an in-memory queue and returns. A background thread formats them and writes to disk about once a second.
|
|
62
|
+
|
|
63
|
+
You can pass GPU tensors straight to `logger.log`. The background thread reads them, so your training loop doesn't wait for the GPU the way it does with `loss.item()`.
|
|
64
|
+
|
|
65
|
+
On an RTX 3050 laptop GPU, a `logger.log` call with a GPU tensor took about 15 µs. A 6 ms training step was no slower with logging than without it, with or without system metrics. Any difference was smaller than the variation between runs.
|
|
66
|
+
|
|
67
|
+
`logger.log_image` does its work immediately, because matplotlib isn't thread-safe. Log images occasionally, not on every step.
|
|
68
|
+
|
|
69
|
+
Queued values are written when your program exits, including after an exception. If the process is killed outright, you can lose about the last second of values.
|
|
70
|
+
|
|
71
|
+
## What it logs
|
|
72
|
+
|
|
73
|
+
- **Scalars**, such as loss and accuracy.
|
|
74
|
+
- **Images**, such as custom graphs you make yourself.
|
|
75
|
+
- **Hyperparameters**, from the `config` you pass to `ALAG`.
|
|
76
|
+
- **System metrics**, if you turn them on.
|
|
77
|
+
|
|
78
|
+
### System metrics
|
|
79
|
+
|
|
80
|
+
These are off by default. To turn them on, install the extra and pass `system_metrics=True`:
|
|
81
|
+
|
|
82
|
+
```sh
|
|
83
|
+
pip install 'alag[system]'
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
logger = ALAG(group="my-group", experiment="my-experiment", system_metrics=True)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Every 10 seconds, a background thread logs CPU use, system RAM and your process's RAM under `system/`. If `nvidia-smi` is available, it also logs utilization, memory, temperature and power for each GPU. GPU memory is the total in use on that card, including other programs.
|
|
91
|
+
|
|
92
|
+
Each sample uses your latest training step. Before your first `logger.log`, or during a long step, several samples share a step, so set **X axis** to **Time** to read these charts.
|
|
93
|
+
|
|
94
|
+
## Storage
|
|
95
|
+
|
|
96
|
+
Experiments are organized into groups. Each experiment is a plain folder inside its group's folder. There is no database or account.
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
logs/
|
|
100
|
+
└── my-group/
|
|
101
|
+
├── flags.json # targets shared by the group
|
|
102
|
+
├── layout.json # chart sections and pins shared by the group
|
|
103
|
+
└── my-experiment/
|
|
104
|
+
├── config.json # the experiment's settings
|
|
105
|
+
├── metrics.csv # every scalar, appended as it is logged
|
|
106
|
+
├── meta.json # tags and note
|
|
107
|
+
└── images/ # logged images
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
`metrics.csv` has one row per logged value, so an experiment can add new metrics at any point and metrics logged at different intervals don't leave empty cells:
|
|
111
|
+
|
|
112
|
+
```csv
|
|
113
|
+
step,time,metric,value
|
|
114
|
+
100,1762444800.1,train/loss,0.42
|
|
115
|
+
100,1762444800.1,train/acc,0.81
|
|
116
|
+
200,1762444812.6,train/loss,0.37
|
|
117
|
+
200,1762444812.7,val/loss,0.45
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Viewing experiments
|
|
121
|
+
|
|
122
|
+
The `alag` command is a small compiled Rust program that generates an HTML page and serves it on localhost. The page updates automatically while your experiments are training.
|
|
123
|
+
|
|
124
|
+
A panel on the left lists your groups and the experiments inside them. Select experiments in the panel to compare them: each metric gets one chart, with a line for every selected experiment.
|
|
125
|
+
|
|
126
|
+
### Hyperparameters
|
|
127
|
+
|
|
128
|
+
The **Hyperparameters** tab has a row for each selected experiment and a column for each setting in its `config`. Settings that differ come first, in bold. Click a column to sort by it, or tick **Only settings that differ** to hide the rest.
|
|
129
|
+
|
|
130
|
+
### Sections and pins
|
|
131
|
+
|
|
132
|
+
Charts are grouped into sections by the part of the metric name before the first slash, so `val/loss` goes under `val` and `system/cpu_percent` goes under `system`.
|
|
133
|
+
|
|
134
|
+
To choose a section yourself, use **Section** on a chart or `logger.section`. **Pin** on a chart or image, or `logger.pin`, moves it to the top of its tab.
|
|
135
|
+
|
|
136
|
+
Sections and pins apply to the whole group, so they stay in place when you compare its experiments.
|
|
137
|
+
|
|
138
|
+
### Mean and std over seeds
|
|
139
|
+
|
|
140
|
+
Tick **Mean ± std over seeds** to merge selected experiments into one line with a shaded band. Experiments merge when they are in the same group and their configs differ only by a key named `seed`. Experiments with no `seed` in their config stay separate.
|
|
141
|
+
|
|
142
|
+
The band is one sample standard deviation at each step. Smoothing is applied to each seed before merging.
|
|
143
|
+
|
|
144
|
+
### Flags
|
|
145
|
+
|
|
146
|
+
A flag sets a target, such as "`val/loss` should go below 0.5 by step 1,000". Set one with `logger.flag`, or with **Flag** on a chart, where clicking the chart fills in the values.
|
|
147
|
+
|
|
148
|
+
The flag is drawn on the chart, and each experiment in the group is listed as reached, missed or not yet. Verdicts use the line as plotted, so they follow your smoothing and seed settings.
|
|
149
|
+
|
|
150
|
+
### Notes and tags
|
|
151
|
+
|
|
152
|
+
Add these in the **Notes** tab or from Python. Tags appear in the left panel, and the filter box searches names, tags and notes.
|
|
153
|
+
|
|
154
|
+
### Exporting charts
|
|
155
|
+
|
|
156
|
+
Each chart has **SVG**, **PNG**, **PDF**, **CSV** and **LaTeX** buttons. Exports use a white background and match the current view, including every selected experiment, smoothing, zoom, log scale and flags.
|
|
157
|
+
|
|
158
|
+
**LaTeX** gives you a `pgfplots` figure to copy and paste. The data is included, so there's no file to upload, but long runs are thinned to keep it small enough for pdflatex. Your document needs:
|
|
159
|
+
|
|
160
|
+
```latex
|
|
161
|
+
\usepackage{pgfplots}
|
|
162
|
+
\pgfplotsset{compat=1.18}
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
### Sharing a report
|
|
166
|
+
|
|
167
|
+
`alag --export report.html` writes one HTML file that opens in any browser, with no server or install. It opens on the most recently updated group, with its experiments selected.
|
|
168
|
+
|
|
169
|
+
You can still select experiments, zoom, smooth, merge seeds and export charts. It's a read-only snapshot, so it doesn't update, and notes, tags, flags, sections and pins can't be changed.
|
|
170
|
+
|
|
171
|
+
The file contains everything in the log folder, including every group, config, note and image. To share only part of it, export a folder with just those groups. Runs with many images can make the file large.
|
|
172
|
+
|
|
173
|
+
## Building wheels
|
|
174
|
+
|
|
175
|
+
`scripts/build-wheels.sh` builds the source package and every wheel into `dist/`. It needs only Docker.
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
MIT
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.8,<2"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "alag"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A simple logging and graphing tool for researchers"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Sandeep S" }]
|
|
13
|
+
requires-python = ">=3.9"
|
|
14
|
+
dependencies = []
|
|
15
|
+
keywords = ["experiment tracking", "logging", "machine learning", "plotting"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Rust",
|
|
21
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
22
|
+
"Topic :: Scientific/Engineering :: Visualization",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Repository = "https://github.com/SandyTheAdventurer/alag"
|
|
27
|
+
Issues = "https://github.com/SandyTheAdventurer/alag/issues"
|
|
28
|
+
|
|
29
|
+
[project.optional-dependencies]
|
|
30
|
+
system = ["psutil"]
|
|
31
|
+
|
|
32
|
+
[tool.maturin]
|
|
33
|
+
bindings = "bin"
|
|
34
|
+
python-source = "python"
|
|
35
|
+
module-name = "alag"
|