epochclock 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- epochclock-0.1.0/LICENSE +21 -0
- epochclock-0.1.0/MANIFEST.in +3 -0
- epochclock-0.1.0/PKG-INFO +142 -0
- epochclock-0.1.0/README.md +119 -0
- epochclock-0.1.0/examples/pi05_example.py +34 -0
- epochclock-0.1.0/examples/pi05_libero_goal.csv +66 -0
- epochclock-0.1.0/pyproject.toml +34 -0
- epochclock-0.1.0/setup.cfg +4 -0
- epochclock-0.1.0/src/epochclock/__init__.py +38 -0
- epochclock-0.1.0/src/epochclock/cli.py +118 -0
- epochclock-0.1.0/src/epochclock/core.py +283 -0
- epochclock-0.1.0/src/epochclock/monitor.py +135 -0
- epochclock-0.1.0/src/epochclock/warmup.py +52 -0
- epochclock-0.1.0/src/epochclock.egg-info/PKG-INFO +142 -0
- epochclock-0.1.0/src/epochclock.egg-info/SOURCES.txt +18 -0
- epochclock-0.1.0/src/epochclock.egg-info/dependency_links.txt +1 -0
- epochclock-0.1.0/src/epochclock.egg-info/entry_points.txt +2 -0
- epochclock-0.1.0/src/epochclock.egg-info/requires.txt +7 -0
- epochclock-0.1.0/src/epochclock.egg-info/top_level.txt +1 -0
- epochclock-0.1.0/tests/test_epochclock.py +99 -0
epochclock-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The epochclock authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: epochclock
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Measure fine-tuning budgets on a small pilot and transfer them in epochs, not steps.
|
|
5
|
+
Author-email: Letian Cheng <cccltray@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/CCCLTRAY/epochclock
|
|
8
|
+
Project-URL: Repository, https://github.com/CCCLTRAY/epochclock
|
|
9
|
+
Project-URL: Issues, https://github.com/CCCLTRAY/epochclock/issues
|
|
10
|
+
Keywords: fine-tuning,training budget,early stopping,scaling laws,robot learning,LLM
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: numpy>=1.20
|
|
18
|
+
Provides-Extra: hf
|
|
19
|
+
Requires-Dist: transformers; extra == "hf"
|
|
20
|
+
Provides-Extra: test
|
|
21
|
+
Requires-Dist: pytest; extra == "test"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# epochclock
|
|
25
|
+
|
|
26
|
+
**How long should you fine-tune when you get more data?** Measure the budget on a
|
|
27
|
+
small pilot, then keep the number of *epochs*, not the number of steps.
|
|
28
|
+
|
|
29
|
+
`epochclock` is the companion tool for the paper
|
|
30
|
+
*Train in Epochs, Not Steps: How Fine-Tuning Compute Should Scale with Data*.
|
|
31
|
+
It reads a learning curve, finds the point after which another doubling of
|
|
32
|
+
training stops paying off, and transfers that budget to a larger dataset.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install epochclock
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Why
|
|
39
|
+
|
|
40
|
+
Fine-tuning recipes inherit a training length from a public config, such as
|
|
41
|
+
"30k updates" for a robot policy or "3 epochs" for an LLM. These two conventions
|
|
42
|
+
agree at one dataset size and disagree everywhere else. For the pretrained
|
|
43
|
+
models in the paper (π0.5 on LIBERO, SmolLM2 on math fine-tuning), the useful budget grows in proportion to data: the number of epochs stays
|
|
44
|
+
roughly constant. Reusing a step count undertrains larger datasets and can make
|
|
45
|
+
extra data look useless. Step-based warmup and randomly initialized components
|
|
46
|
+
(for example a new head) move training toward a step clock; this tool helps you
|
|
47
|
+
detect that from two pilots.
|
|
48
|
+
|
|
49
|
+
## Quick start
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from epochclock import budget, transfer
|
|
53
|
+
|
|
54
|
+
# One learning curve from a small pilot: steps and validation error.
|
|
55
|
+
r = budget(steps, val_error, eps=0.03, updates_per_epoch=193)
|
|
56
|
+
print(r) # Budget(eps=0.03): S*=1,750 updates, K=9.07 epochs
|
|
57
|
+
|
|
58
|
+
# Predict the budget for a 4x larger dataset (787 updates per epoch).
|
|
59
|
+
transfer(r.steps, 193, 787, rule="epoch") # -> 7,134 updates
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
With two pilot sizes, measure the timing exponent instead of assuming it:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from epochclock import fit_theta, transfer
|
|
66
|
+
theta = fit_theta([5, 10], [r5.steps, r10.steps]) # ~1: epoch clock, ~0: step clock
|
|
67
|
+
transfer(r5.steps, 193, 787, rule="theta", theta=theta)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
For curves with a minimum (e.g. LLM validation NLL), use `minimum`:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from epochclock import minimum
|
|
74
|
+
minimum(steps, val_nll, updates_per_epoch=50) # log-quadratic refinement of the minimum
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Warmup
|
|
78
|
+
|
|
79
|
+
A warmup fixed in steps covers many epochs of a small dataset and few of a large
|
|
80
|
+
one, which delays small-data optima:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from epochclock import budget_with_warmup, warmup_steps_for
|
|
84
|
+
budget_with_warmup(epochs=9.1, updates_per_epoch=193, warmup=1000) # -> 2,256 (vs 1,756)
|
|
85
|
+
warmup_steps_for(0.5, updates_per_epoch=787) # specify warmup in epochs instead
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### During training
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from epochclock import DoublingGainMonitor
|
|
92
|
+
mon = DoublingGainMonitor(eps=0.03, updates_per_epoch=193)
|
|
93
|
+
for step in eval_steps: # evaluate on a log-spaced schedule
|
|
94
|
+
gain = mon.update(step, val_error) # gain of the last completed doubling
|
|
95
|
+
if mon.suggest_stop:
|
|
96
|
+
break
|
|
97
|
+
print(mon.budget()) # full criterion on the recorded curve
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Hugging Face `Trainer` users can add `EpochClockCallback(eps=0.03, stop=False)`
|
|
101
|
+
(requires `pip install epochclock[hf]`); it logs `epochclock/doubling_gain`
|
|
102
|
+
after each evaluation.
|
|
103
|
+
|
|
104
|
+
### Command line
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
epochclock budget curves.csv --value-col arm_rmse --group-col demos_per_task --upe-col updates_per_epoch
|
|
108
|
+
epochclock transfer --pilot-steps 1750 --pilot-upe 193 --target-upe 787
|
|
109
|
+
epochclock warmup --epochs 9.1 --upe 193 --warmup 1000
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`examples/pi05_example.py` reproduces the paper's π0.5 budgets
|
|
113
|
+
(1,750 / 4,146 / 8,342 updates; θ = 1.13) from the included curves.
|
|
114
|
+
|
|
115
|
+
## The criterion
|
|
116
|
+
|
|
117
|
+
For a learning curve E(S), checkpoints are averaged within a factor of 1.2 in
|
|
118
|
+
steps, and the return from doubling is g(S) = 1 − E(2S)/E(S), computed at
|
|
119
|
+
checkpoints whose doubling was also measured. The budget S\*(ε) is the earliest
|
|
120
|
+
S such that g(S), g(2S), g(4S), … all stay below ε inside the measured range.
|
|
121
|
+
The result reports a status:
|
|
122
|
+
|
|
123
|
+
| status | meaning |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `ok` | budget found |
|
|
126
|
+
| `left_censored` | already flat at the first doubling; save earlier checkpoints |
|
|
127
|
+
| `not_reached` | still improving; train longer |
|
|
128
|
+
| `too_short` | fewer than two doublings measured |
|
|
129
|
+
|
|
130
|
+
Practical notes: save checkpoints on a log-spaced grid with exact doublings
|
|
131
|
+
(e.g. 1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4, … epochs), and evaluate every
|
|
132
|
+
checkpoint on the same fixed validation set. The threshold chooses how far
|
|
133
|
+
down the slow tail you go; a stricter ε buys more improvement for more compute,
|
|
134
|
+
and the epoch count at that point transfers in the same way. Check the task
|
|
135
|
+
metric near the transferred budget: validation loss and task success can peak
|
|
136
|
+
at different times.
|
|
137
|
+
|
|
138
|
+
## Citation
|
|
139
|
+
|
|
140
|
+
A paper link and citation will be added when the manuscript is available.
|
|
141
|
+
|
|
142
|
+
MIT License.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# epochclock
|
|
2
|
+
|
|
3
|
+
**How long should you fine-tune when you get more data?** Measure the budget on a
|
|
4
|
+
small pilot, then keep the number of *epochs*, not the number of steps.
|
|
5
|
+
|
|
6
|
+
`epochclock` is the companion tool for the paper
|
|
7
|
+
*Train in Epochs, Not Steps: How Fine-Tuning Compute Should Scale with Data*.
|
|
8
|
+
It reads a learning curve, finds the point after which another doubling of
|
|
9
|
+
training stops paying off, and transfers that budget to a larger dataset.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install epochclock
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Why
|
|
16
|
+
|
|
17
|
+
Fine-tuning recipes inherit a training length from a public config, such as
|
|
18
|
+
"30k updates" for a robot policy or "3 epochs" for an LLM. These two conventions
|
|
19
|
+
agree at one dataset size and disagree everywhere else. For the pretrained
|
|
20
|
+
models in the paper (π0.5 on LIBERO, SmolLM2 on math fine-tuning), the useful budget grows in proportion to data: the number of epochs stays
|
|
21
|
+
roughly constant. Reusing a step count undertrains larger datasets and can make
|
|
22
|
+
extra data look useless. Step-based warmup and randomly initialized components
|
|
23
|
+
(for example a new head) move training toward a step clock; this tool helps you
|
|
24
|
+
detect that from two pilots.
|
|
25
|
+
|
|
26
|
+
## Quick start
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from epochclock import budget, transfer
|
|
30
|
+
|
|
31
|
+
# One learning curve from a small pilot: steps and validation error.
|
|
32
|
+
r = budget(steps, val_error, eps=0.03, updates_per_epoch=193)
|
|
33
|
+
print(r) # Budget(eps=0.03): S*=1,750 updates, K=9.07 epochs
|
|
34
|
+
|
|
35
|
+
# Predict the budget for a 4x larger dataset (787 updates per epoch).
|
|
36
|
+
transfer(r.steps, 193, 787, rule="epoch") # -> 7,134 updates
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
With two pilot sizes, measure the timing exponent instead of assuming it:
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from epochclock import fit_theta, transfer
|
|
43
|
+
theta = fit_theta([5, 10], [r5.steps, r10.steps]) # ~1: epoch clock, ~0: step clock
|
|
44
|
+
transfer(r5.steps, 193, 787, rule="theta", theta=theta)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
For curves with a minimum (e.g. LLM validation NLL), use `minimum`:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from epochclock import minimum
|
|
51
|
+
minimum(steps, val_nll, updates_per_epoch=50) # log-quadratic refinement of the minimum
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### Warmup
|
|
55
|
+
|
|
56
|
+
A warmup fixed in steps covers many epochs of a small dataset and few of a large
|
|
57
|
+
one, which delays small-data optima:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from epochclock import budget_with_warmup, warmup_steps_for
|
|
61
|
+
budget_with_warmup(epochs=9.1, updates_per_epoch=193, warmup=1000) # -> 2,256 (vs 1,756)
|
|
62
|
+
warmup_steps_for(0.5, updates_per_epoch=787) # specify warmup in epochs instead
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### During training
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from epochclock import DoublingGainMonitor
|
|
69
|
+
mon = DoublingGainMonitor(eps=0.03, updates_per_epoch=193)
|
|
70
|
+
for step in eval_steps: # evaluate on a log-spaced schedule
|
|
71
|
+
gain = mon.update(step, val_error) # gain of the last completed doubling
|
|
72
|
+
if mon.suggest_stop:
|
|
73
|
+
break
|
|
74
|
+
print(mon.budget()) # full criterion on the recorded curve
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Hugging Face `Trainer` users can add `EpochClockCallback(eps=0.03, stop=False)`
|
|
78
|
+
(requires `pip install epochclock[hf]`); it logs `epochclock/doubling_gain`
|
|
79
|
+
after each evaluation.
|
|
80
|
+
|
|
81
|
+
### Command line
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
epochclock budget curves.csv --value-col arm_rmse --group-col demos_per_task --upe-col updates_per_epoch
|
|
85
|
+
epochclock transfer --pilot-steps 1750 --pilot-upe 193 --target-upe 787
|
|
86
|
+
epochclock warmup --epochs 9.1 --upe 193 --warmup 1000
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`examples/pi05_example.py` reproduces the paper's π0.5 budgets
|
|
90
|
+
(1,750 / 4,146 / 8,342 updates; θ = 1.13) from the included curves.
|
|
91
|
+
|
|
92
|
+
## The criterion
|
|
93
|
+
|
|
94
|
+
For a learning curve E(S), checkpoints are averaged within a factor of 1.2 in
|
|
95
|
+
steps, and the return from doubling is g(S) = 1 − E(2S)/E(S), computed at
|
|
96
|
+
checkpoints whose doubling was also measured. The budget S\*(ε) is the earliest
|
|
97
|
+
S such that g(S), g(2S), g(4S), … all stay below ε inside the measured range.
|
|
98
|
+
The result reports a status:
|
|
99
|
+
|
|
100
|
+
| status | meaning |
|
|
101
|
+
|---|---|
|
|
102
|
+
| `ok` | budget found |
|
|
103
|
+
| `left_censored` | already flat at the first doubling; save earlier checkpoints |
|
|
104
|
+
| `not_reached` | still improving; train longer |
|
|
105
|
+
| `too_short` | fewer than two doublings measured |
|
|
106
|
+
|
|
107
|
+
Practical notes: save checkpoints on a log-spaced grid with exact doublings
|
|
108
|
+
(e.g. 1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4, … epochs), and evaluate every
|
|
109
|
+
checkpoint on the same fixed validation set. The threshold chooses how far
|
|
110
|
+
down the slow tail you go; a stricter ε buys more improvement for more compute,
|
|
111
|
+
and the epoch count at that point transfers in the same way. Check the task
|
|
112
|
+
metric near the transferred budget: validation loss and task success can peak
|
|
113
|
+
at different times.
|
|
114
|
+
|
|
115
|
+
## Citation
|
|
116
|
+
|
|
117
|
+
A paper link and citation will be added when the manuscript is available.
|
|
118
|
+
|
|
119
|
+
MIT License.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Reproduce the pi0.5 LIBERO-Goal budgets from the paper (no-warmup grid, seed 42).
|
|
2
|
+
|
|
3
|
+
Expected: S* ~ 1,751 / 4,149 / 8,342 updates (9.1 / 10.6 / 10.6 epochs), theta ~ 1.13.
|
|
4
|
+
"""
|
|
5
|
+
import csv
|
|
6
|
+
from collections import defaultdict
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from epochclock import budget, fit_theta, transfer
|
|
10
|
+
|
|
11
|
+
rows = defaultdict(lambda: {"step": [], "rmse": [], "upe": None})
|
|
12
|
+
with open(Path(__file__).with_name("pi05_libero_goal.csv")) as f:
|
|
13
|
+
for r in csv.DictReader(f):
|
|
14
|
+
d = rows[int(float(r["demos_per_task"]))]
|
|
15
|
+
d["step"].append(float(r["step"]))
|
|
16
|
+
d["rmse"].append(float(r["arm_rmse"]))
|
|
17
|
+
d["upe"] = float(r["updates_per_epoch"])
|
|
18
|
+
|
|
19
|
+
results = {}
|
|
20
|
+
for demos, d in sorted(rows.items()):
|
|
21
|
+
res = budget(d["step"], d["rmse"], eps=0.03, updates_per_epoch=d["upe"])
|
|
22
|
+
results[demos] = (res, d["upe"])
|
|
23
|
+
print(f"D{demos:<3} {res}")
|
|
24
|
+
|
|
25
|
+
sizes = list(results)
|
|
26
|
+
print(f"\ntiming exponent theta = {fit_theta(sizes, [r.steps for r, _ in results.values()]):.2f}")
|
|
27
|
+
|
|
28
|
+
# Calibrate on the smallest dataset and predict the others.
|
|
29
|
+
pilot, q0 = results[5]
|
|
30
|
+
for demos in (10, 20):
|
|
31
|
+
q = results[demos][1]
|
|
32
|
+
pred = {rule: transfer(pilot.steps, q0, q, rule) for rule in ("epoch", "sqrt", "fixed")}
|
|
33
|
+
print(f"D{demos}: measured {results[demos][0].steps:,.0f} | "
|
|
34
|
+
+ " | ".join(f"{k} {v:,.0f}" for k, v in pred.items()))
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
demos_per_task,updates_per_epoch,step,arm_rmse
|
|
2
|
+
5,193.0,193,0.2042767863558137
|
|
3
|
+
5,193.0,386,0.1809431029663854
|
|
4
|
+
5,193.0,579,0.1641131791011565
|
|
5
|
+
5,193.0,772,0.155113141012549
|
|
6
|
+
5,193.0,965,0.147363238701602
|
|
7
|
+
5,193.0,1158,0.1463144050003368
|
|
8
|
+
5,193.0,1544,0.1420423582848829
|
|
9
|
+
5,193.0,1930,0.1403852431166273
|
|
10
|
+
5,193.0,2316,0.1370664654315984
|
|
11
|
+
5,193.0,2895,0.135198529807239
|
|
12
|
+
5,193.0,3474,0.1345399598102758
|
|
13
|
+
5,193.0,3860,0.1362540377090003
|
|
14
|
+
5,193.0,4825,0.1328416185134516
|
|
15
|
+
5,193.0,5790,0.1304922931042895
|
|
16
|
+
5,193.0,6755,0.1329181427736751
|
|
17
|
+
5,193.0,7720,0.1337503475348346
|
|
18
|
+
5,193.0,8685,0.1320682881444739
|
|
19
|
+
5,193.0,9650,0.1319887075212458
|
|
20
|
+
5,193.0,10615,0.1302776636195059
|
|
21
|
+
5,193.0,11580,0.1326504629133904
|
|
22
|
+
5,193.0,12545,0.1347036545920829
|
|
23
|
+
5,193.0,13510,0.1306795453663392
|
|
24
|
+
5,193.0,14475,0.1299598651393015
|
|
25
|
+
5,193.0,15440,0.1304603095109405
|
|
26
|
+
5,193.0,16405,0.1280288942841006
|
|
27
|
+
5,193.0,17370,0.1300657118960184
|
|
28
|
+
5,193.0,18335,0.1305962961643433
|
|
29
|
+
5,193.0,19300,0.1299745351190462
|
|
30
|
+
10,392.0,392,0.1787066066508155
|
|
31
|
+
10,392.0,784,0.1601658580954767
|
|
32
|
+
10,392.0,1176,0.1552480263941947
|
|
33
|
+
10,392.0,1568,0.1458124557125602
|
|
34
|
+
10,392.0,1960,0.1480797828860019
|
|
35
|
+
10,392.0,2352,0.1368746416065159
|
|
36
|
+
10,392.0,3136,0.1410051461747792
|
|
37
|
+
10,392.0,3920,0.1341070180271543
|
|
38
|
+
10,392.0,4704,0.1294270193493298
|
|
39
|
+
10,392.0,5880,0.1308368432535637
|
|
40
|
+
10,392.0,7056,0.1275244964206509
|
|
41
|
+
10,392.0,7840,0.1272012858833218
|
|
42
|
+
10,392.0,9800,0.1274890145350163
|
|
43
|
+
10,392.0,11760,0.1288658007113103
|
|
44
|
+
10,392.0,13720,0.1278291285245062
|
|
45
|
+
10,392.0,15680,0.1292641054726498
|
|
46
|
+
10,392.0,17640,0.1275269367563987
|
|
47
|
+
10,392.0,19600,0.1252138902319325
|
|
48
|
+
10,392.0,21560,0.1259372509772568
|
|
49
|
+
10,392.0,23520,0.1239882259980583
|
|
50
|
+
20,787.0,787,0.1648325878108779
|
|
51
|
+
20,787.0,1574,0.1455773892958117
|
|
52
|
+
20,787.0,2361,0.1404125829060106
|
|
53
|
+
20,787.0,3148,0.1340854305152559
|
|
54
|
+
20,787.0,3935,0.1316944303752955
|
|
55
|
+
20,787.0,4722,0.1286163993976851
|
|
56
|
+
20,787.0,6296,0.1243449510887657
|
|
57
|
+
20,787.0,7870,0.1228367432109372
|
|
58
|
+
20,787.0,9444,0.122226273928476
|
|
59
|
+
20,787.0,11805,0.1178079356619939
|
|
60
|
+
20,787.0,14166,0.1188796615192821
|
|
61
|
+
20,787.0,15740,0.1180733479023664
|
|
62
|
+
20,787.0,19675,0.1189757620170178
|
|
63
|
+
20,787.0,23610,0.1172247604173143
|
|
64
|
+
20,787.0,27545,0.1147344416168542
|
|
65
|
+
20,787.0,31480,0.1157501032450829
|
|
66
|
+
20,787.0,39350,0.1138448642335008
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "epochclock"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Measure fine-tuning budgets on a small pilot and transfer them in epochs, not steps."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
authors = [{ name = "Letian Cheng", email = "cccltray@gmail.com" }]
|
|
12
|
+
requires-python = ">=3.8"
|
|
13
|
+
dependencies = ["numpy>=1.20"]
|
|
14
|
+
keywords = ["fine-tuning", "training budget", "early stopping", "scaling laws", "robot learning", "LLM"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
hf = ["transformers"]
|
|
23
|
+
test = ["pytest"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/CCCLTRAY/epochclock"
|
|
27
|
+
Repository = "https://github.com/CCCLTRAY/epochclock"
|
|
28
|
+
Issues = "https://github.com/CCCLTRAY/epochclock/issues"
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
epochclock = "epochclock.cli:main"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
where = ["src"]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""epochclock: measure fine-tuning budgets on a pilot and transfer them in epochs.
|
|
2
|
+
|
|
3
|
+
Companion code for "Train in Epochs, Not Steps: How Fine-Tuning Compute Should
|
|
4
|
+
Scale with Data".
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from .core import (
|
|
8
|
+
BudgetResult,
|
|
9
|
+
MinimumResult,
|
|
10
|
+
budget,
|
|
11
|
+
doubling_gains,
|
|
12
|
+
fit_theta,
|
|
13
|
+
minimum,
|
|
14
|
+
smooth,
|
|
15
|
+
transfer,
|
|
16
|
+
)
|
|
17
|
+
from .monitor import DoublingGainMonitor, EpochClockCallback
|
|
18
|
+
from .warmup import budget_with_warmup, lr_exposure, warmup_in_epochs, warmup_steps_for
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"BudgetResult",
|
|
24
|
+
"MinimumResult",
|
|
25
|
+
"budget",
|
|
26
|
+
"doubling_gains",
|
|
27
|
+
"fit_theta",
|
|
28
|
+
"minimum",
|
|
29
|
+
"smooth",
|
|
30
|
+
"transfer",
|
|
31
|
+
"DoublingGainMonitor",
|
|
32
|
+
"EpochClockCallback",
|
|
33
|
+
"budget_with_warmup",
|
|
34
|
+
"lr_exposure",
|
|
35
|
+
"warmup_in_epochs",
|
|
36
|
+
"warmup_steps_for",
|
|
37
|
+
"__version__",
|
|
38
|
+
]
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Command-line interface.
|
|
2
|
+
|
|
3
|
+
epochclock budget curves.csv --eps 0.03 --updates-per-epoch 193
|
|
4
|
+
epochclock budget curves.csv --group-col data_size --upe-col updates_per_epoch
|
|
5
|
+
epochclock transfer --pilot-steps 1750 --pilot-upe 193 --target-upe 787
|
|
6
|
+
epochclock warmup --epochs 9.1 --upe 193 --warmup 1000
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import csv
|
|
13
|
+
import sys
|
|
14
|
+
from collections import OrderedDict
|
|
15
|
+
|
|
16
|
+
from .core import budget, fit_theta, minimum, transfer
|
|
17
|
+
from .warmup import budget_with_warmup, warmup_in_epochs
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _read(path, step_col, value_col, group_col, upe_col):
|
|
21
|
+
groups = OrderedDict()
|
|
22
|
+
with open(path, newline="", encoding="utf-8-sig") as f:
|
|
23
|
+
for row in csv.DictReader(f):
|
|
24
|
+
try:
|
|
25
|
+
s, v = float(row[step_col]), float(row[value_col])
|
|
26
|
+
except (KeyError, ValueError):
|
|
27
|
+
continue
|
|
28
|
+
key = row.get(group_col, "all") if group_col else "all"
|
|
29
|
+
g = groups.setdefault(key, {"steps": [], "values": [], "upe": None})
|
|
30
|
+
g["steps"].append(s)
|
|
31
|
+
g["values"].append(v)
|
|
32
|
+
if upe_col and row.get(upe_col):
|
|
33
|
+
try:
|
|
34
|
+
g["upe"] = float(row[upe_col])
|
|
35
|
+
except ValueError:
|
|
36
|
+
pass
|
|
37
|
+
return groups
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _cmd_budget(a):
|
|
41
|
+
groups = _read(a.file, a.step_col, a.value_col, a.group_col, a.upe_col)
|
|
42
|
+
if not groups:
|
|
43
|
+
sys.exit("no rows read; check column names")
|
|
44
|
+
sizes, budgets = [], []
|
|
45
|
+
for key, g in groups.items():
|
|
46
|
+
upe = g["upe"] or a.updates_per_epoch
|
|
47
|
+
if a.minimum:
|
|
48
|
+
r = minimum(g["steps"], g["values"], upe)
|
|
49
|
+
steps, status = r.steps, r.status
|
|
50
|
+
else:
|
|
51
|
+
r = budget(g["steps"], g["values"], a.eps, upe, lower_is_better=not a.higher_is_better)
|
|
52
|
+
steps, status = r.steps, r.status
|
|
53
|
+
ep = r.epochs
|
|
54
|
+
label = f"{a.group_col}={key}" if a.group_col else "curve"
|
|
55
|
+
s = "—" if steps is None else f"{steps:,.0f}"
|
|
56
|
+
e = "" if ep is None else f" ({ep:.2f} epochs)"
|
|
57
|
+
print(f"{label:<24} S* = {s} updates{e} [{status}]")
|
|
58
|
+
if steps is not None and status in ("ok", "interpolated", "interior"):
|
|
59
|
+
try:
|
|
60
|
+
sizes.append(float(key))
|
|
61
|
+
budgets.append(steps)
|
|
62
|
+
except ValueError:
|
|
63
|
+
pass
|
|
64
|
+
if len(sizes) >= 2:
|
|
65
|
+
print(f"\ntiming exponent theta = {fit_theta(sizes, budgets):.3f} over {len(sizes)} sizes")
|
|
66
|
+
print(" theta ~ 1: epoch clock (keep epochs fixed) theta ~ 0: step clock")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _cmd_transfer(a):
|
|
70
|
+
for rule in ("epoch", "sqrt", "fixed") if a.rule == "all" else (a.rule,):
|
|
71
|
+
s = transfer(a.pilot_steps, a.pilot_upe, a.target_upe, rule, a.theta)
|
|
72
|
+
print(f"{rule:<6} -> {s:,.0f} updates ({s / a.target_upe:.2f} epochs)")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _cmd_warmup(a):
|
|
76
|
+
s = budget_with_warmup(a.epochs, a.upe, a.warmup)
|
|
77
|
+
print(f"warmup of {a.warmup:,.0f} updates = {warmup_in_epochs(a.warmup, a.upe):.2f} epochs of this dataset")
|
|
78
|
+
print(f"budget for {a.epochs:g} full-rate epochs: {s:,.0f} updates "
|
|
79
|
+
f"(vs {a.epochs * a.upe:,.0f} without warmup)")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def main(argv=None):
|
|
83
|
+
p = argparse.ArgumentParser(prog="epochclock", description=__doc__,
|
|
84
|
+
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
85
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
86
|
+
|
|
87
|
+
b = sub.add_parser("budget", help="measure S* from learning curves in a CSV")
|
|
88
|
+
b.add_argument("file")
|
|
89
|
+
b.add_argument("--step-col", default="step")
|
|
90
|
+
b.add_argument("--value-col", default="value")
|
|
91
|
+
b.add_argument("--group-col", default=None, help="column with data size (one curve per value)")
|
|
92
|
+
b.add_argument("--upe-col", default=None, help="column with updates per epoch")
|
|
93
|
+
b.add_argument("--updates-per-epoch", type=float, default=None)
|
|
94
|
+
b.add_argument("--eps", type=float, default=0.03)
|
|
95
|
+
b.add_argument("--higher-is-better", action="store_true")
|
|
96
|
+
b.add_argument("--minimum", action="store_true", help="use the validation minimum (e.g. NLL)")
|
|
97
|
+
b.set_defaults(func=_cmd_budget)
|
|
98
|
+
|
|
99
|
+
t = sub.add_parser("transfer", help="predict a target budget from a pilot")
|
|
100
|
+
t.add_argument("--pilot-steps", type=float, required=True)
|
|
101
|
+
t.add_argument("--pilot-upe", type=float, required=True)
|
|
102
|
+
t.add_argument("--target-upe", type=float, required=True)
|
|
103
|
+
t.add_argument("--rule", default="all", choices=["all", "epoch", "sqrt", "fixed", "theta"])
|
|
104
|
+
t.add_argument("--theta", type=float, default=None)
|
|
105
|
+
t.set_defaults(func=_cmd_transfer)
|
|
106
|
+
|
|
107
|
+
w = sub.add_parser("warmup", help="effect of a step-based warmup on a budget")
|
|
108
|
+
w.add_argument("--epochs", type=float, required=True, help="full-rate epochs needed (K)")
|
|
109
|
+
w.add_argument("--upe", type=float, required=True, help="updates per epoch")
|
|
110
|
+
w.add_argument("--warmup", type=float, required=True, help="warmup updates")
|
|
111
|
+
w.set_defaults(func=_cmd_warmup)
|
|
112
|
+
|
|
113
|
+
a = p.parse_args(argv)
|
|
114
|
+
a.func(a)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
if __name__ == "__main__":
|
|
118
|
+
main()
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
"""Core measurements: the doubling-gain budget, validation minima, and transfer rules.
|
|
2
|
+
|
|
3
|
+
All step quantities are optimizer updates. "Updates per epoch" is the number of
|
|
4
|
+
optimizer updates in one pass over the training set (e.g. floor(N / batch)).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from typing import Optional, Sequence
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"BudgetResult",
|
|
17
|
+
"MinimumResult",
|
|
18
|
+
"smooth",
|
|
19
|
+
"doubling_gains",
|
|
20
|
+
"budget",
|
|
21
|
+
"minimum",
|
|
22
|
+
"transfer",
|
|
23
|
+
"fit_theta",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
WINDOW = 1.2 # centered multiplicative smoothing window, as in the paper
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
# helpers
|
|
31
|
+
# ---------------------------------------------------------------------------
|
|
32
|
+
def _clean(steps: Sequence[float], values: Sequence[float]):
|
|
33
|
+
s = np.asarray(steps, dtype=float)
|
|
34
|
+
v = np.asarray(values, dtype=float)
|
|
35
|
+
if s.shape != v.shape:
|
|
36
|
+
raise ValueError("steps and values must have the same length")
|
|
37
|
+
keep = np.isfinite(s) & np.isfinite(v) & (s > 0)
|
|
38
|
+
s, v = s[keep], v[keep]
|
|
39
|
+
order = np.argsort(s)
|
|
40
|
+
s, v = s[order], v[order]
|
|
41
|
+
# average duplicated steps
|
|
42
|
+
if len(np.unique(s)) != len(s):
|
|
43
|
+
us = np.unique(s)
|
|
44
|
+
v = np.array([v[s == u].mean() for u in us])
|
|
45
|
+
s = us
|
|
46
|
+
if len(s) < 3:
|
|
47
|
+
raise ValueError("need at least three checkpoints with step > 0")
|
|
48
|
+
return s, v
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def smooth(steps: Sequence[float], values: Sequence[float], window: float = WINDOW) -> np.ndarray:
|
|
52
|
+
"""Average every checkpoint with all checkpoints within a factor `window` in steps."""
|
|
53
|
+
s, v = _clean(steps, values)
|
|
54
|
+
ls = np.log(s)
|
|
55
|
+
half = math.log(window) + 1e-9
|
|
56
|
+
return np.array([v[np.abs(ls - x) <= half].mean() for x in ls])
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def doubling_gains(
|
|
60
|
+
steps: Sequence[float],
|
|
61
|
+
values: Sequence[float],
|
|
62
|
+
lower_is_better: bool = True,
|
|
63
|
+
window: float = WINDOW,
|
|
64
|
+
anchors: str = "auto",
|
|
65
|
+
tol: float = 0.02,
|
|
66
|
+
):
|
|
67
|
+
"""Relative improvement from doubling training, g(S) = 1 - E(2S)/E(S).
|
|
68
|
+
|
|
69
|
+
Both E(S) and E(2S) are read from the smoothed curve (checkpoints averaged
|
|
70
|
+
within a factor `window`).
|
|
71
|
+
|
|
72
|
+
anchors="exact" use only measured S whose doubling 2S is also a
|
|
73
|
+
measured checkpoint (within relative `tol`). This is
|
|
74
|
+
the procedure used in the paper.
|
|
75
|
+
anchors="interpolate" use every measured S with 2S inside the range and
|
|
76
|
+
read E(2S) by linear interpolation in log steps.
|
|
77
|
+
anchors="auto" "exact" if it yields at least two anchors, else
|
|
78
|
+
"interpolate".
|
|
79
|
+
|
|
80
|
+
Returns arrays (anchor_steps, gains).
|
|
81
|
+
"""
|
|
82
|
+
s, v = _clean(steps, values)
|
|
83
|
+
sm = smooth(s, v, window)
|
|
84
|
+
sign = 1.0 if lower_is_better else -1.0
|
|
85
|
+
|
|
86
|
+
def _exact():
|
|
87
|
+
a, g = [], []
|
|
88
|
+
for i, si in enumerate(s):
|
|
89
|
+
j = np.where(np.abs(s / (2 * si) - 1) <= tol)[0]
|
|
90
|
+
if len(j):
|
|
91
|
+
a.append(si)
|
|
92
|
+
g.append(sign * (1.0 - sm[j[0]] / sm[i]))
|
|
93
|
+
return np.array(a), np.array(g)
|
|
94
|
+
|
|
95
|
+
def _interp():
|
|
96
|
+
ls = np.log(s)
|
|
97
|
+
a, g = [], []
|
|
98
|
+
for si, ei in zip(s, sm):
|
|
99
|
+
if 2 * si > s[-1] * (1 + 1e-9):
|
|
100
|
+
break
|
|
101
|
+
e2 = float(np.interp(math.log(2 * si), ls, sm))
|
|
102
|
+
a.append(si)
|
|
103
|
+
g.append(sign * (1.0 - e2 / ei))
|
|
104
|
+
return np.array(a), np.array(g)
|
|
105
|
+
|
|
106
|
+
if anchors == "exact":
|
|
107
|
+
return _exact()
|
|
108
|
+
if anchors == "interpolate":
|
|
109
|
+
return _interp()
|
|
110
|
+
if anchors == "auto":
|
|
111
|
+
a, g = _exact()
|
|
112
|
+
return (a, g) if len(a) >= 2 else _interp()
|
|
113
|
+
raise ValueError(f"unknown anchors mode {anchors!r}")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# ---------------------------------------------------------------------------
|
|
117
|
+
# results
|
|
118
|
+
# ---------------------------------------------------------------------------
|
|
119
|
+
@dataclass
|
|
120
|
+
class BudgetResult:
|
|
121
|
+
"""Budget after which every later observed doubling improves by < eps."""
|
|
122
|
+
|
|
123
|
+
steps: Optional[float]
|
|
124
|
+
eps: float
|
|
125
|
+
updates_per_epoch: Optional[float] = None
|
|
126
|
+
status: str = "ok" # "ok", "left_censored", "not_reached", "too_short"
|
|
127
|
+
anchors: np.ndarray = field(default_factory=lambda: np.array([]), repr=False)
|
|
128
|
+
gains: np.ndarray = field(default_factory=lambda: np.array([]), repr=False)
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def epochs(self) -> Optional[float]:
|
|
132
|
+
if self.steps is None or not self.updates_per_epoch:
|
|
133
|
+
return None
|
|
134
|
+
return self.steps / self.updates_per_epoch
|
|
135
|
+
|
|
136
|
+
def __str__(self) -> str:
|
|
137
|
+
if self.steps is None:
|
|
138
|
+
return f"Budget(eps={self.eps:.3g}): {self.status}"
|
|
139
|
+
ep = f", K={self.epochs:.2f} epochs" if self.epochs is not None else ""
|
|
140
|
+
flag = "" if self.status == "ok" else f" [{self.status}]"
|
|
141
|
+
return f"Budget(eps={self.eps:.3g}): S*={self.steps:,.0f} updates{ep}{flag}"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass
|
|
145
|
+
class MinimumResult:
|
|
146
|
+
"""Location of a validation minimum (e.g. NLL), with optional log-quadratic refinement."""
|
|
147
|
+
|
|
148
|
+
steps: float
|
|
149
|
+
value: float
|
|
150
|
+
updates_per_epoch: Optional[float] = None
|
|
151
|
+
status: str = "interior" # "interior", "interpolated", "left_edge", "right_edge"
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def epochs(self) -> Optional[float]:
|
|
155
|
+
if not self.updates_per_epoch:
|
|
156
|
+
return None
|
|
157
|
+
return self.steps / self.updates_per_epoch
|
|
158
|
+
|
|
159
|
+
def __str__(self) -> str:
|
|
160
|
+
ep = f", {self.epochs:.2f} epochs" if self.epochs is not None else ""
|
|
161
|
+
return f"Minimum: {self.steps:,.0f} updates{ep} (value={self.value:.5g}, {self.status})"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# ---------------------------------------------------------------------------
|
|
165
|
+
# measurements
|
|
166
|
+
# ---------------------------------------------------------------------------
|
|
167
|
+
def budget(
|
|
168
|
+
steps: Sequence[float],
|
|
169
|
+
values: Sequence[float],
|
|
170
|
+
eps: float = 0.03,
|
|
171
|
+
updates_per_epoch: Optional[float] = None,
|
|
172
|
+
lower_is_better: bool = True,
|
|
173
|
+
window: float = WINDOW,
|
|
174
|
+
anchors: str = "auto",
|
|
175
|
+
grid: int = 4000,
|
|
176
|
+
) -> BudgetResult:
|
|
177
|
+
"""Measure the training budget S*(eps) from one learning curve.
|
|
178
|
+
|
|
179
|
+
The doubling gain g(S) is computed at anchor checkpoints and interpolated
|
|
180
|
+
linearly in log steps. S*(eps) is the earliest S such that g(S), g(2S),
|
|
181
|
+
g(4S), ... are all below `eps` for every dyadic doubling inside the
|
|
182
|
+
supported range. Requiring later doublings to agree prevents a temporary
|
|
183
|
+
flat region from being taken as the end of the rapid decline.
|
|
184
|
+
|
|
185
|
+
Status values:
|
|
186
|
+
ok a crossing was found inside the supported range
|
|
187
|
+
left_censored the criterion already holds at the first anchor
|
|
188
|
+
not_reached no supported S satisfies the criterion (train longer)
|
|
189
|
+
too_short fewer than two doubling anchors are available
|
|
190
|
+
"""
|
|
191
|
+
a, g = doubling_gains(steps, values, lower_is_better, window, anchors)
|
|
192
|
+
if len(a) < 2:
|
|
193
|
+
return BudgetResult(None, eps, updates_per_epoch, "too_short", a, g)
|
|
194
|
+
la = np.log(a)
|
|
195
|
+
xs = np.linspace(la[0], la[-1], grid)
|
|
196
|
+
ln2 = math.log(2.0)
|
|
197
|
+
for idx, x in enumerate(xs):
|
|
198
|
+
n = int((la[-1] - x) / ln2 + 1e-9)
|
|
199
|
+
chain = x + ln2 * np.arange(n + 1)
|
|
200
|
+
if np.all(np.interp(chain, la, g) < eps):
|
|
201
|
+
if idx == 0:
|
|
202
|
+
return BudgetResult(float(math.exp(x)), eps, updates_per_epoch, "left_censored", a, g)
|
|
203
|
+
# refine the switch point between the previous and current grid points
|
|
204
|
+
lo, hi = xs[idx - 1], x
|
|
205
|
+
for _ in range(50):
|
|
206
|
+
mid = 0.5 * (lo + hi)
|
|
207
|
+
n = int((la[-1] - mid) / ln2 + 1e-9)
|
|
208
|
+
if np.all(np.interp(mid + ln2 * np.arange(n + 1), la, g) < eps):
|
|
209
|
+
hi = mid
|
|
210
|
+
else:
|
|
211
|
+
lo = mid
|
|
212
|
+
return BudgetResult(float(math.exp(hi)), eps, updates_per_epoch, "ok", a, g)
|
|
213
|
+
return BudgetResult(None, eps, updates_per_epoch, "not_reached", a, g)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def minimum(
|
|
217
|
+
steps: Sequence[float],
|
|
218
|
+
values: Sequence[float],
|
|
219
|
+
updates_per_epoch: Optional[float] = None,
|
|
220
|
+
interpolate: bool = True,
|
|
221
|
+
) -> MinimumResult:
|
|
222
|
+
"""Locate the minimum of a validation curve (e.g. NLL).
|
|
223
|
+
|
|
224
|
+
With `interpolate=True`, an interior minimum is refined by a quadratic in
|
|
225
|
+
log steps through the minimum and its two neighbours; the vertex is kept
|
|
226
|
+
only if the curvature is positive and it lies inside that bracket.
|
|
227
|
+
"""
|
|
228
|
+
s, v = _clean(steps, values)
|
|
229
|
+
i = int(np.argmin(v))
|
|
230
|
+
if i == 0:
|
|
231
|
+
return MinimumResult(float(s[0]), float(v[0]), updates_per_epoch, "left_edge")
|
|
232
|
+
if i == len(s) - 1:
|
|
233
|
+
return MinimumResult(float(s[-1]), float(v[-1]), updates_per_epoch, "right_edge")
|
|
234
|
+
if interpolate:
|
|
235
|
+
x = np.log(s[i - 1 : i + 2])
|
|
236
|
+
c2, c1, c0 = np.polyfit(x, v[i - 1 : i + 2], 2)
|
|
237
|
+
if c2 > 0:
|
|
238
|
+
xv = -c1 / (2 * c2)
|
|
239
|
+
if x[0] < xv < x[2]:
|
|
240
|
+
return MinimumResult(
|
|
241
|
+
float(math.exp(xv)), float(c0 + c1 * xv + c2 * xv * xv), updates_per_epoch, "interpolated"
|
|
242
|
+
)
|
|
243
|
+
return MinimumResult(float(s[i]), float(v[i]), updates_per_epoch, "interior")
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
# ---------------------------------------------------------------------------
|
|
247
|
+
# transfer
|
|
248
|
+
# ---------------------------------------------------------------------------
|
|
249
|
+
def transfer(
|
|
250
|
+
pilot_steps: float,
|
|
251
|
+
pilot_updates_per_epoch: float,
|
|
252
|
+
target_updates_per_epoch: float,
|
|
253
|
+
rule: str = "epoch",
|
|
254
|
+
theta: Optional[float] = None,
|
|
255
|
+
) -> float:
|
|
256
|
+
"""Predict the target budget from a pilot budget.
|
|
257
|
+
|
|
258
|
+
rule="epoch" keep the epoch count S = S0 * (q/q0)
|
|
259
|
+
rule="sqrt" square-root scaling S = S0 * sqrt(q/q0)
|
|
260
|
+
rule="fixed" reuse the pilot's step count S = S0
|
|
261
|
+
rule="theta" measured exponent S = S0 * (q/q0)**theta
|
|
262
|
+
"""
|
|
263
|
+
ratio = target_updates_per_epoch / pilot_updates_per_epoch
|
|
264
|
+
if rule == "epoch":
|
|
265
|
+
return pilot_steps * ratio
|
|
266
|
+
if rule == "sqrt":
|
|
267
|
+
return pilot_steps * math.sqrt(ratio)
|
|
268
|
+
if rule == "fixed":
|
|
269
|
+
return float(pilot_steps)
|
|
270
|
+
if rule == "theta":
|
|
271
|
+
if theta is None:
|
|
272
|
+
raise ValueError("rule='theta' requires theta (see fit_theta)")
|
|
273
|
+
return pilot_steps * ratio**theta
|
|
274
|
+
raise ValueError(f"unknown rule {rule!r}")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def fit_theta(data_sizes: Sequence[float], budgets: Sequence[float]) -> float:
|
|
278
|
+
"""Timing exponent: least-squares slope of log(budget) on log(data size)."""
|
|
279
|
+
x = np.log(np.asarray(data_sizes, dtype=float))
|
|
280
|
+
y = np.log(np.asarray(budgets, dtype=float))
|
|
281
|
+
if len(x) < 2:
|
|
282
|
+
raise ValueError("need at least two data sizes")
|
|
283
|
+
return float(np.polyfit(x, y, 1)[0])
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Online monitoring of the doubling gain during training.
|
|
2
|
+
|
|
3
|
+
`DoublingGainMonitor` is framework-agnostic: call `update(step, value)` after
|
|
4
|
+
each evaluation. `EpochClockCallback` wraps it for the Hugging Face Trainer.
|
|
5
|
+
|
|
6
|
+
Online, only past doublings are observable, so the monitor reports the gain of
|
|
7
|
+
the most recent completed doubling, 1 - E(S)/E(S/2). Use `budget()` after
|
|
8
|
+
training (or on a pilot) for the full criterion, which also requires later
|
|
9
|
+
doublings to stay below the threshold.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import math
|
|
15
|
+
from typing import List, Optional
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
19
|
+
from .core import WINDOW, BudgetResult, budget
|
|
20
|
+
|
|
21
|
+
__all__ = ["DoublingGainMonitor", "EpochClockCallback"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class DoublingGainMonitor:
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
eps: float = 0.03,
|
|
28
|
+
updates_per_epoch: Optional[float] = None,
|
|
29
|
+
lower_is_better: bool = True,
|
|
30
|
+
patience: int = 2,
|
|
31
|
+
window: float = WINDOW,
|
|
32
|
+
):
|
|
33
|
+
self.eps = eps
|
|
34
|
+
self.updates_per_epoch = updates_per_epoch
|
|
35
|
+
self.lower_is_better = lower_is_better
|
|
36
|
+
self.patience = patience
|
|
37
|
+
self.window = window
|
|
38
|
+
self.steps: List[float] = []
|
|
39
|
+
self.values: List[float] = []
|
|
40
|
+
self._below = 0
|
|
41
|
+
|
|
42
|
+
def _smoothed_at(self, x: float) -> Optional[float]:
|
|
43
|
+
s = np.asarray(self.steps, dtype=float)
|
|
44
|
+
v = np.asarray(self.values, dtype=float)
|
|
45
|
+
if len(s) == 0 or x < s.min() / self.window or x > s.max() * self.window:
|
|
46
|
+
return None
|
|
47
|
+
near = np.abs(np.log(s) - math.log(x)) <= math.log(self.window) + 1e-9
|
|
48
|
+
if near.any():
|
|
49
|
+
return float(v[near].mean())
|
|
50
|
+
order = np.argsort(s)
|
|
51
|
+
return float(np.interp(math.log(x), np.log(s[order]), v[order]))
|
|
52
|
+
|
|
53
|
+
def update(self, step: float, value: float) -> Optional[float]:
|
|
54
|
+
"""Record an evaluation; return the gain of the last completed doubling."""
|
|
55
|
+
if step <= 0 or not math.isfinite(value):
|
|
56
|
+
return None
|
|
57
|
+
self.steps.append(float(step))
|
|
58
|
+
self.values.append(float(value))
|
|
59
|
+
g = self.latest_gain
|
|
60
|
+
if g is not None:
|
|
61
|
+
self._below = self._below + 1 if g < self.eps else 0
|
|
62
|
+
return g
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def latest_gain(self) -> Optional[float]:
|
|
66
|
+
if len(self.steps) < 2:
|
|
67
|
+
return None
|
|
68
|
+
s = max(self.steps)
|
|
69
|
+
if s / 2 < min(self.steps):
|
|
70
|
+
return None
|
|
71
|
+
now, half = self._smoothed_at(s), self._smoothed_at(s / 2)
|
|
72
|
+
if now is None or half is None or half == 0:
|
|
73
|
+
return None
|
|
74
|
+
g = 1.0 - now / half
|
|
75
|
+
return g if self.lower_is_better else -g
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def suggest_stop(self) -> bool:
|
|
79
|
+
"""True after `patience` consecutive evaluations whose last doubling gained < eps."""
|
|
80
|
+
return self._below >= self.patience
|
|
81
|
+
|
|
82
|
+
def budget(self, eps: Optional[float] = None) -> BudgetResult:
|
|
83
|
+
"""Apply the full (post-hoc) criterion to the recorded curve."""
|
|
84
|
+
return budget(
|
|
85
|
+
self.steps,
|
|
86
|
+
self.values,
|
|
87
|
+
eps=self.eps if eps is None else eps,
|
|
88
|
+
updates_per_epoch=self.updates_per_epoch,
|
|
89
|
+
lower_is_better=self.lower_is_better,
|
|
90
|
+
window=self.window,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
try: # optional Hugging Face integration
|
|
95
|
+
from transformers import TrainerCallback # type: ignore
|
|
96
|
+
except Exception: # pragma: no cover - transformers is optional
|
|
97
|
+
TrainerCallback = object # type: ignore
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class EpochClockCallback(TrainerCallback): # type: ignore[misc]
|
|
101
|
+
"""Hugging Face Trainer callback.
|
|
102
|
+
|
|
103
|
+
Logs `epochclock/doubling_gain` after each evaluation and, if `stop=True`,
|
|
104
|
+
stops training once the last completed doubling has gained less than `eps`
|
|
105
|
+
for `patience` consecutive evaluations. Evaluate on a log-spaced schedule
|
|
106
|
+
(or densely) so that doublings are observed.
|
|
107
|
+
|
|
108
|
+
trainer = Trainer(..., callbacks=[EpochClockCallback(eps=0.03,
|
|
109
|
+
updates_per_epoch=len(train_dataloader))])
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
def __init__(
|
|
113
|
+
self,
|
|
114
|
+
eps: float = 0.03,
|
|
115
|
+
metric: str = "eval_loss",
|
|
116
|
+
updates_per_epoch: Optional[float] = None,
|
|
117
|
+
lower_is_better: bool = True,
|
|
118
|
+
patience: int = 2,
|
|
119
|
+
stop: bool = False,
|
|
120
|
+
):
|
|
121
|
+
if TrainerCallback is object:
|
|
122
|
+
raise ImportError("EpochClockCallback requires `transformers`")
|
|
123
|
+
self.metric = metric
|
|
124
|
+
self.stop = stop
|
|
125
|
+
self.monitor = DoublingGainMonitor(eps, updates_per_epoch, lower_is_better, patience)
|
|
126
|
+
|
|
127
|
+
def on_evaluate(self, args, state, control, metrics=None, **kwargs):
|
|
128
|
+
if not metrics or self.metric not in metrics:
|
|
129
|
+
return control
|
|
130
|
+
g = self.monitor.update(state.global_step, float(metrics[self.metric]))
|
|
131
|
+
if g is not None:
|
|
132
|
+
state.log_history.append({"step": state.global_step, "epochclock/doubling_gain": g})
|
|
133
|
+
if self.stop and self.monitor.suggest_stop:
|
|
134
|
+
control.should_training_stop = True
|
|
135
|
+
return control
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Warmup in exposure units.
|
|
2
|
+
|
|
3
|
+
A warmup fixed in updates covers many epochs of a small dataset and few of a
|
|
4
|
+
large one. These helpers convert between the two and predict how a step-based
|
|
5
|
+
warmup delays a budget that needs K full-rate passes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
|
|
12
|
+
__all__ = ["lr_exposure", "budget_with_warmup", "warmup_in_epochs", "warmup_steps_for"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def lr_exposure(steps: float, warmup: float) -> float:
|
|
16
|
+
"""Cumulative learning rate relative to its peak after `steps` updates.
|
|
17
|
+
|
|
18
|
+
For linear warmup over `warmup` updates followed by a constant rate:
|
|
19
|
+
X_W(S) = S^2 / (2W) if S < W
|
|
20
|
+
= S - W/2 if S >= W
|
|
21
|
+
"""
|
|
22
|
+
if warmup <= 0:
|
|
23
|
+
return float(steps)
|
|
24
|
+
if steps < warmup:
|
|
25
|
+
return steps * steps / (2.0 * warmup)
|
|
26
|
+
return steps - warmup / 2.0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def budget_with_warmup(epochs: float, updates_per_epoch: float, warmup: float) -> float:
|
|
30
|
+
"""Updates needed to deliver `epochs` full-rate passes under a linear warmup.
|
|
31
|
+
|
|
32
|
+
Solves X_W(S) = epochs * updates_per_epoch:
|
|
33
|
+
S = sqrt(2 W K q) if K q < W/2 (the run ends inside the warmup)
|
|
34
|
+
S = K q + W/2 otherwise
|
|
35
|
+
Small datasets are delayed most; large datasets pay only W/2.
|
|
36
|
+
"""
|
|
37
|
+
target = epochs * updates_per_epoch
|
|
38
|
+
if warmup <= 0:
|
|
39
|
+
return float(target)
|
|
40
|
+
if target < warmup / 2.0:
|
|
41
|
+
return math.sqrt(2.0 * warmup * target)
|
|
42
|
+
return target + warmup / 2.0
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def warmup_in_epochs(warmup_steps: float, updates_per_epoch: float) -> float:
|
|
46
|
+
"""How many passes over the data a step-based warmup occupies."""
|
|
47
|
+
return warmup_steps / updates_per_epoch
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def warmup_steps_for(warmup_epochs: float, updates_per_epoch: float) -> int:
|
|
51
|
+
"""Warmup length in updates for a warmup specified in epochs."""
|
|
52
|
+
return max(0, int(round(warmup_epochs * updates_per_epoch)))
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: epochclock
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Measure fine-tuning budgets on a small pilot and transfer them in epochs, not steps.
|
|
5
|
+
Author-email: Letian Cheng <cccltray@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/CCCLTRAY/epochclock
|
|
8
|
+
Project-URL: Repository, https://github.com/CCCLTRAY/epochclock
|
|
9
|
+
Project-URL: Issues, https://github.com/CCCLTRAY/epochclock/issues
|
|
10
|
+
Keywords: fine-tuning,training budget,early stopping,scaling laws,robot learning,LLM
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: numpy>=1.20
|
|
18
|
+
Provides-Extra: hf
|
|
19
|
+
Requires-Dist: transformers; extra == "hf"
|
|
20
|
+
Provides-Extra: test
|
|
21
|
+
Requires-Dist: pytest; extra == "test"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# epochclock
|
|
25
|
+
|
|
26
|
+
**How long should you fine-tune when you get more data?** Measure the budget on a
|
|
27
|
+
small pilot, then keep the number of *epochs*, not the number of steps.
|
|
28
|
+
|
|
29
|
+
`epochclock` is the companion tool for the paper
|
|
30
|
+
*Train in Epochs, Not Steps: How Fine-Tuning Compute Should Scale with Data*.
|
|
31
|
+
It reads a learning curve, finds the point after which another doubling of
|
|
32
|
+
training stops paying off, and transfers that budget to a larger dataset.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install epochclock
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Why
|
|
39
|
+
|
|
40
|
+
Fine-tuning recipes inherit a training length from a public config, such as
|
|
41
|
+
"30k updates" for a robot policy or "3 epochs" for an LLM. These two conventions
|
|
42
|
+
agree at one dataset size and disagree everywhere else. For the pretrained
|
|
43
|
+
models in the paper (π0.5 on LIBERO, SmolLM2 on math fine-tuning), the useful budget grows in proportion to data: the number of epochs stays
|
|
44
|
+
roughly constant. Reusing a step count undertrains larger datasets and can make
|
|
45
|
+
extra data look useless. Step-based warmup and randomly initialized components
|
|
46
|
+
(for example a new head) move training toward a step clock; this tool helps you
|
|
47
|
+
detect that from two pilots.
|
|
48
|
+
|
|
49
|
+
## Quick start
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from epochclock import budget, transfer
|
|
53
|
+
|
|
54
|
+
# One learning curve from a small pilot: steps and validation error.
|
|
55
|
+
r = budget(steps, val_error, eps=0.03, updates_per_epoch=193)
|
|
56
|
+
print(r) # Budget(eps=0.03): S*=1,750 updates, K=9.07 epochs
|
|
57
|
+
|
|
58
|
+
# Predict the budget for a 4x larger dataset (787 updates per epoch).
|
|
59
|
+
transfer(r.steps, 193, 787, rule="epoch") # -> 7,134 updates
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
With two pilot sizes, measure the timing exponent instead of assuming it:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from epochclock import fit_theta, transfer
|
|
66
|
+
theta = fit_theta([5, 10], [r5.steps, r10.steps]) # ~1: epoch clock, ~0: step clock
|
|
67
|
+
transfer(r5.steps, 193, 787, rule="theta", theta=theta)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
For curves with a minimum (e.g. LLM validation NLL), use `minimum`:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from epochclock import minimum
|
|
74
|
+
minimum(steps, val_nll, updates_per_epoch=50) # log-quadratic refinement of the minimum
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Warmup
|
|
78
|
+
|
|
79
|
+
A warmup fixed in steps covers many epochs of a small dataset and few of a large
|
|
80
|
+
one, which delays small-data optima:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from epochclock import budget_with_warmup, warmup_steps_for
|
|
84
|
+
budget_with_warmup(epochs=9.1, updates_per_epoch=193, warmup=1000) # -> 2,256 (vs 1,756)
|
|
85
|
+
warmup_steps_for(0.5, updates_per_epoch=787) # specify warmup in epochs instead
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### During training
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from epochclock import DoublingGainMonitor
|
|
92
|
+
mon = DoublingGainMonitor(eps=0.03, updates_per_epoch=193)
|
|
93
|
+
for step in eval_steps: # evaluate on a log-spaced schedule
|
|
94
|
+
gain = mon.update(step, val_error) # gain of the last completed doubling
|
|
95
|
+
if mon.suggest_stop:
|
|
96
|
+
break
|
|
97
|
+
print(mon.budget()) # full criterion on the recorded curve
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Hugging Face `Trainer` users can add `EpochClockCallback(eps=0.03, stop=False)`
|
|
101
|
+
(requires `pip install epochclock[hf]`); it logs `epochclock/doubling_gain`
|
|
102
|
+
after each evaluation.
|
|
103
|
+
|
|
104
|
+
### Command line
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
epochclock budget curves.csv --value-col arm_rmse --group-col demos_per_task --upe-col updates_per_epoch
|
|
108
|
+
epochclock transfer --pilot-steps 1750 --pilot-upe 193 --target-upe 787
|
|
109
|
+
epochclock warmup --epochs 9.1 --upe 193 --warmup 1000
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`examples/pi05_example.py` reproduces the paper's π0.5 budgets
|
|
113
|
+
(1,750 / 4,146 / 8,342 updates; θ = 1.13) from the included curves.
|
|
114
|
+
|
|
115
|
+
## The criterion
|
|
116
|
+
|
|
117
|
+
For a learning curve E(S), checkpoints are averaged within a factor of 1.2 in
|
|
118
|
+
steps, and the return from doubling is g(S) = 1 − E(2S)/E(S), computed at
|
|
119
|
+
checkpoints whose doubling was also measured. The budget S\*(ε) is the earliest
|
|
120
|
+
S such that g(S), g(2S), g(4S), … all stay below ε inside the measured range.
|
|
121
|
+
The result reports a status:
|
|
122
|
+
|
|
123
|
+
| status | meaning |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `ok` | budget found |
|
|
126
|
+
| `left_censored` | already flat at the first doubling; save earlier checkpoints |
|
|
127
|
+
| `not_reached` | still improving; train longer |
|
|
128
|
+
| `too_short` | fewer than two doublings measured |
|
|
129
|
+
|
|
130
|
+
Practical notes: save checkpoints on a log-spaced grid with exact doublings
|
|
131
|
+
(e.g. 1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4, … epochs), and evaluate every
|
|
132
|
+
checkpoint on the same fixed validation set. The threshold chooses how far
|
|
133
|
+
down the slow tail you go; a stricter ε buys more improvement for more compute,
|
|
134
|
+
and the epoch count at that point transfers in the same way. Check the task
|
|
135
|
+
metric near the transferred budget: validation loss and task success can peak
|
|
136
|
+
at different times.
|
|
137
|
+
|
|
138
|
+
## Citation
|
|
139
|
+
|
|
140
|
+
A paper link and citation will be added when the manuscript is available.
|
|
141
|
+
|
|
142
|
+
MIT License.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
examples/pi05_example.py
|
|
6
|
+
examples/pi05_libero_goal.csv
|
|
7
|
+
src/epochclock/__init__.py
|
|
8
|
+
src/epochclock/cli.py
|
|
9
|
+
src/epochclock/core.py
|
|
10
|
+
src/epochclock/monitor.py
|
|
11
|
+
src/epochclock/warmup.py
|
|
12
|
+
src/epochclock.egg-info/PKG-INFO
|
|
13
|
+
src/epochclock.egg-info/SOURCES.txt
|
|
14
|
+
src/epochclock.egg-info/dependency_links.txt
|
|
15
|
+
src/epochclock.egg-info/entry_points.txt
|
|
16
|
+
src/epochclock.egg-info/requires.txt
|
|
17
|
+
src/epochclock.egg-info/top_level.txt
|
|
18
|
+
tests/test_epochclock.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
epochclock
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import math
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from epochclock import (
|
|
7
|
+
DoublingGainMonitor,
|
|
8
|
+
budget,
|
|
9
|
+
budget_with_warmup,
|
|
10
|
+
fit_theta,
|
|
11
|
+
lr_exposure,
|
|
12
|
+
minimum,
|
|
13
|
+
transfer,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def grid(n_max, per_octave=4, start=8):
|
|
18
|
+
k = np.arange(0, int(per_octave * math.log2(n_max / start)) + 1)
|
|
19
|
+
return np.unique(np.round(start * 2 ** (k / per_octave)))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def curve(steps, clock, floor=0.1, amp=1.0, gamma=1.0):
|
|
23
|
+
return floor + amp * (steps / clock) ** (-gamma)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_epoch_clock_gives_theta_one():
|
|
27
|
+
sizes = [1000, 2000, 4000, 8000]
|
|
28
|
+
budgets = []
|
|
29
|
+
for n in sizes:
|
|
30
|
+
q = n / 32
|
|
31
|
+
s = grid(q * 400)
|
|
32
|
+
budgets.append(budget(s, curve(s, q), eps=0.03, updates_per_epoch=q).steps)
|
|
33
|
+
assert fit_theta(sizes, budgets) == pytest.approx(1.0, abs=0.03)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_step_clock_gives_theta_zero():
|
|
37
|
+
sizes = [1000, 2000, 4000, 8000]
|
|
38
|
+
budgets = []
|
|
39
|
+
for _ in sizes:
|
|
40
|
+
s = grid(1e5)
|
|
41
|
+
budgets.append(budget(s, curve(s, 50.0)).steps)
|
|
42
|
+
assert fit_theta(sizes, budgets) == pytest.approx(0.0, abs=0.03)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_threshold_moves_budget_later():
|
|
46
|
+
s = grid(1e5)
|
|
47
|
+
v = curve(s, 50.0)
|
|
48
|
+
assert budget(s, v, eps=0.02).steps > budget(s, v, eps=0.05).steps
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_status_flags():
|
|
52
|
+
s = grid(64)
|
|
53
|
+
v = curve(s, 50.0) # still improving fast at the end
|
|
54
|
+
assert budget(s, v, eps=0.03).status == "not_reached"
|
|
55
|
+
flat = np.full_like(s, 0.5, dtype=float)
|
|
56
|
+
assert budget(s, flat, eps=0.03).status == "left_censored"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_higher_is_better():
|
|
60
|
+
s = grid(1e5)
|
|
61
|
+
acc = 1.0 - curve(s, 50.0)
|
|
62
|
+
r = budget(s, acc, eps=0.01, lower_is_better=False)
|
|
63
|
+
assert r.status in ("ok", "left_censored")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_minimum_interpolates():
|
|
67
|
+
s = np.array([10, 20, 40, 80, 160, 320], dtype=float)
|
|
68
|
+
x = np.log(s)
|
|
69
|
+
v = (x - math.log(60)) ** 2 + 1
|
|
70
|
+
r = minimum(s, v, updates_per_epoch=30)
|
|
71
|
+
assert r.status == "interpolated"
|
|
72
|
+
assert r.steps == pytest.approx(60, rel=1e-6)
|
|
73
|
+
assert r.epochs == pytest.approx(2.0, rel=1e-6)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_transfer_rules():
|
|
77
|
+
assert transfer(1000, 100, 400, "epoch") == pytest.approx(4000)
|
|
78
|
+
assert transfer(1000, 100, 400, "sqrt") == pytest.approx(2000)
|
|
79
|
+
assert transfer(1000, 100, 400, "fixed") == pytest.approx(1000)
|
|
80
|
+
assert transfer(1000, 100, 400, "theta", theta=0.5) == pytest.approx(2000)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_warmup_formula():
|
|
84
|
+
assert lr_exposure(50, 100) == pytest.approx(12.5)
|
|
85
|
+
assert lr_exposure(300, 100) == pytest.approx(250)
|
|
86
|
+
# ends inside warmup
|
|
87
|
+
s = budget_with_warmup(epochs=1, updates_per_epoch=20, warmup=100)
|
|
88
|
+
assert lr_exposure(s, 100) == pytest.approx(20)
|
|
89
|
+
# ends after warmup
|
|
90
|
+
s = budget_with_warmup(epochs=10, updates_per_epoch=100, warmup=100)
|
|
91
|
+
assert s == pytest.approx(1050)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_monitor():
|
|
95
|
+
m = DoublingGainMonitor(eps=0.03, patience=2)
|
|
96
|
+
for s in grid(1e5):
|
|
97
|
+
m.update(s, curve(s, 50.0))
|
|
98
|
+
assert m.suggest_stop
|
|
99
|
+
assert m.budget().status == "ok"
|