epochclock 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 The epochclock authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,3 @@
1
+ include README.md LICENSE
2
+ recursive-include examples *.py *.csv
3
+ recursive-include tests *.py
@@ -0,0 +1,142 @@
1
+ Metadata-Version: 2.4
2
+ Name: epochclock
3
+ Version: 0.1.0
4
+ Summary: Measure fine-tuning budgets on a small pilot and transfer them in epochs, not steps.
5
+ Author-email: Letian Cheng <cccltray@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/CCCLTRAY/epochclock
8
+ Project-URL: Repository, https://github.com/CCCLTRAY/epochclock
9
+ Project-URL: Issues, https://github.com/CCCLTRAY/epochclock/issues
10
+ Keywords: fine-tuning,training budget,early stopping,scaling laws,robot learning,LLM
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
14
+ Requires-Python: >=3.8
15
+ Description-Content-Type: text/markdown
16
+ License-File: LICENSE
17
+ Requires-Dist: numpy>=1.20
18
+ Provides-Extra: hf
19
+ Requires-Dist: transformers; extra == "hf"
20
+ Provides-Extra: test
21
+ Requires-Dist: pytest; extra == "test"
22
+ Dynamic: license-file
23
+
24
+ # epochclock
25
+
26
+ **How long should you fine-tune when you get more data?** Measure the budget on a
27
+ small pilot, then keep the number of *epochs*, not the number of steps.
28
+
29
+ `epochclock` is the companion tool for the paper
30
+ *Train in Epochs, Not Steps: How Fine-Tuning Compute Should Scale with Data*.
31
+ It reads a learning curve, finds the point after which another doubling of
32
+ training stops paying off, and transfers that budget to a larger dataset.
33
+
34
+ ```bash
35
+ pip install epochclock
36
+ ```
37
+
38
+ ## Why
39
+
40
+ Fine-tuning recipes inherit a training length from a public config, such as
41
+ "30k updates" for a robot policy or "3 epochs" for an LLM. These two conventions
42
+ agree at one dataset size and disagree everywhere else. For the pretrained
43
+ models in the paper (π0.5 on LIBERO, SmolLM2 on math fine-tuning), the useful budget grows in proportion to data: the number of epochs stays
44
+ roughly constant. Reusing a step count undertrains larger datasets and can make
45
+ extra data look useless. Step-based warmup and randomly initialized components
46
+ (for example a new head) move training toward a step clock; this tool helps you
47
+ detect that from two pilots.
48
+
49
+ ## Quick start
50
+
51
+ ```python
52
+ from epochclock import budget, transfer
53
+
54
+ # One learning curve from a small pilot: steps and validation error.
55
+ r = budget(steps, val_error, eps=0.03, updates_per_epoch=193)
56
+ print(r) # Budget(eps=0.03): S*=1,750 updates, K=9.07 epochs
57
+
58
+ # Predict the budget for a 4x larger dataset (787 updates per epoch).
59
+ transfer(r.steps, 193, 787, rule="epoch") # -> 7,134 updates
60
+ ```
61
+
62
+ With two pilot sizes, measure the timing exponent instead of assuming it:
63
+
64
+ ```python
65
+ from epochclock import fit_theta, transfer
66
+ theta = fit_theta([5, 10], [r5.steps, r10.steps]) # ~1: epoch clock, ~0: step clock
67
+ transfer(r5.steps, 193, 787, rule="theta", theta=theta)
68
+ ```
69
+
70
+ For curves with a minimum (e.g. LLM validation NLL), use `minimum`:
71
+
72
+ ```python
73
+ from epochclock import minimum
74
+ minimum(steps, val_nll, updates_per_epoch=50) # log-quadratic refinement of the minimum
75
+ ```
76
+
77
+ ### Warmup
78
+
79
+ A warmup fixed in steps covers many epochs of a small dataset and few of a large
80
+ one, which delays small-data optima:
81
+
82
+ ```python
83
+ from epochclock import budget_with_warmup, warmup_steps_for
84
+ budget_with_warmup(epochs=9.1, updates_per_epoch=193, warmup=1000) # -> 2,256 (vs 1,756)
85
+ warmup_steps_for(0.5, updates_per_epoch=787) # specify warmup in epochs instead
86
+ ```
87
+
88
+ ### During training
89
+
90
+ ```python
91
+ from epochclock import DoublingGainMonitor
92
+ mon = DoublingGainMonitor(eps=0.03, updates_per_epoch=193)
93
+ for step in eval_steps: # evaluate on a log-spaced schedule
94
+ gain = mon.update(step, val_error) # gain of the last completed doubling
95
+ if mon.suggest_stop:
96
+ break
97
+ print(mon.budget()) # full criterion on the recorded curve
98
+ ```
99
+
100
+ Hugging Face `Trainer` users can add `EpochClockCallback(eps=0.03, stop=False)`
101
+ (requires `pip install epochclock[hf]`); it logs `epochclock/doubling_gain`
102
+ after each evaluation.
103
+
104
+ ### Command line
105
+
106
+ ```bash
107
+ epochclock budget curves.csv --value-col arm_rmse --group-col demos_per_task --upe-col updates_per_epoch
108
+ epochclock transfer --pilot-steps 1750 --pilot-upe 193 --target-upe 787
109
+ epochclock warmup --epochs 9.1 --upe 193 --warmup 1000
110
+ ```
111
+
112
+ `examples/pi05_example.py` reproduces the paper's π0.5 budgets
113
+ (1,750 / 4,146 / 8,342 updates; θ = 1.13) from the included curves.
114
+
115
+ ## The criterion
116
+
117
+ For a learning curve E(S), checkpoints are averaged within a factor of 1.2 in
118
+ steps, and the return from doubling is g(S) = 1 − E(2S)/E(S), computed at
119
+ checkpoints whose doubling was also measured. The budget S\*(ε) is the earliest
120
+ S such that g(S), g(2S), g(4S), … all stay below ε inside the measured range.
121
+ The result reports a status:
122
+
123
+ | status | meaning |
124
+ |---|---|
125
+ | `ok` | budget found |
126
+ | `left_censored` | already flat at the first doubling; save earlier checkpoints |
127
+ | `not_reached` | still improving; train longer |
128
+ | `too_short` | fewer than two doublings measured |
129
+
130
+ Practical notes: save checkpoints on a log-spaced grid with exact doublings
131
+ (e.g. 1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4, … epochs), and evaluate every
132
+ checkpoint on the same fixed validation set. The threshold chooses how far
133
+ down the slow tail you go; a stricter ε buys more improvement for more compute,
134
+ and the epoch count at that point transfers in the same way. Check the task
135
+ metric near the transferred budget: validation loss and task success can peak
136
+ at different times.
137
+
138
+ ## Citation
139
+
140
+ A paper link and citation will be added when the manuscript is available.
141
+
142
+ MIT License.
@@ -0,0 +1,119 @@
1
+ # epochclock
2
+
3
+ **How long should you fine-tune when you get more data?** Measure the budget on a
4
+ small pilot, then keep the number of *epochs*, not the number of steps.
5
+
6
+ `epochclock` is the companion tool for the paper
7
+ *Train in Epochs, Not Steps: How Fine-Tuning Compute Should Scale with Data*.
8
+ It reads a learning curve, finds the point after which another doubling of
9
+ training stops paying off, and transfers that budget to a larger dataset.
10
+
11
+ ```bash
12
+ pip install epochclock
13
+ ```
14
+
15
+ ## Why
16
+
17
+ Fine-tuning recipes inherit a training length from a public config, such as
18
+ "30k updates" for a robot policy or "3 epochs" for an LLM. These two conventions
19
+ agree at one dataset size and disagree everywhere else. For the pretrained
20
+ models in the paper (π0.5 on LIBERO, SmolLM2 on math fine-tuning), the useful budget grows in proportion to data: the number of epochs stays
21
+ roughly constant. Reusing a step count undertrains larger datasets and can make
22
+ extra data look useless. Step-based warmup and randomly initialized components
23
+ (for example a new head) move training toward a step clock; this tool helps you
24
+ detect that from two pilots.
25
+
26
+ ## Quick start
27
+
28
+ ```python
29
+ from epochclock import budget, transfer
30
+
31
+ # One learning curve from a small pilot: steps and validation error.
32
+ r = budget(steps, val_error, eps=0.03, updates_per_epoch=193)
33
+ print(r) # Budget(eps=0.03): S*=1,750 updates, K=9.07 epochs
34
+
35
+ # Predict the budget for a 4x larger dataset (787 updates per epoch).
36
+ transfer(r.steps, 193, 787, rule="epoch") # -> 7,134 updates
37
+ ```
38
+
39
+ With two pilot sizes, measure the timing exponent instead of assuming it:
40
+
41
+ ```python
42
+ from epochclock import fit_theta, transfer
43
+ theta = fit_theta([5, 10], [r5.steps, r10.steps]) # ~1: epoch clock, ~0: step clock
44
+ transfer(r5.steps, 193, 787, rule="theta", theta=theta)
45
+ ```
46
+
47
+ For curves with a minimum (e.g. LLM validation NLL), use `minimum`:
48
+
49
+ ```python
50
+ from epochclock import minimum
51
+ minimum(steps, val_nll, updates_per_epoch=50) # log-quadratic refinement of the minimum
52
+ ```
53
+
54
+ ### Warmup
55
+
56
+ A warmup fixed in steps covers many epochs of a small dataset and few of a large
57
+ one, which delays small-data optima:
58
+
59
+ ```python
60
+ from epochclock import budget_with_warmup, warmup_steps_for
61
+ budget_with_warmup(epochs=9.1, updates_per_epoch=193, warmup=1000) # -> 2,256 (vs 1,756)
62
+ warmup_steps_for(0.5, updates_per_epoch=787) # specify warmup in epochs instead
63
+ ```
64
+
65
+ ### During training
66
+
67
+ ```python
68
+ from epochclock import DoublingGainMonitor
69
+ mon = DoublingGainMonitor(eps=0.03, updates_per_epoch=193)
70
+ for step in eval_steps: # evaluate on a log-spaced schedule
71
+ gain = mon.update(step, val_error) # gain of the last completed doubling
72
+ if mon.suggest_stop:
73
+ break
74
+ print(mon.budget()) # full criterion on the recorded curve
75
+ ```
76
+
77
+ Hugging Face `Trainer` users can add `EpochClockCallback(eps=0.03, stop=False)`
78
+ (requires `pip install epochclock[hf]`); it logs `epochclock/doubling_gain`
79
+ after each evaluation.
80
+
81
+ ### Command line
82
+
83
+ ```bash
84
+ epochclock budget curves.csv --value-col arm_rmse --group-col demos_per_task --upe-col updates_per_epoch
85
+ epochclock transfer --pilot-steps 1750 --pilot-upe 193 --target-upe 787
86
+ epochclock warmup --epochs 9.1 --upe 193 --warmup 1000
87
+ ```
88
+
89
+ `examples/pi05_example.py` reproduces the paper's π0.5 budgets
90
+ (1,750 / 4,146 / 8,342 updates; θ = 1.13) from the included curves.
91
+
92
+ ## The criterion
93
+
94
+ For a learning curve E(S), checkpoints are averaged within a factor of 1.2 in
95
+ steps, and the return from doubling is g(S) = 1 − E(2S)/E(S), computed at
96
+ checkpoints whose doubling was also measured. The budget S\*(ε) is the earliest
97
+ S such that g(S), g(2S), g(4S), … all stay below ε inside the measured range.
98
+ The result reports a status:
99
+
100
+ | status | meaning |
101
+ |---|---|
102
+ | `ok` | budget found |
103
+ | `left_censored` | already flat at the first doubling; save earlier checkpoints |
104
+ | `not_reached` | still improving; train longer |
105
+ | `too_short` | fewer than two doublings measured |
106
+
107
+ Practical notes: save checkpoints on a log-spaced grid with exact doublings
108
+ (e.g. 1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4, … epochs), and evaluate every
109
+ checkpoint on the same fixed validation set. The threshold chooses how far
110
+ down the slow tail you go; a stricter ε buys more improvement for more compute,
111
+ and the epoch count at that point transfers in the same way. Check the task
112
+ metric near the transferred budget: validation loss and task success can peak
113
+ at different times.
114
+
115
+ ## Citation
116
+
117
+ A paper link and citation will be added when the manuscript is available.
118
+
119
+ MIT License.
@@ -0,0 +1,34 @@
1
+ """Reproduce the pi0.5 LIBERO-Goal budgets from the paper (no-warmup grid, seed 42).
2
+
3
+ Expected: S* ~ 1,751 / 4,149 / 8,342 updates (9.1 / 10.6 / 10.6 epochs), theta ~ 1.13.
4
+ """
5
+ import csv
6
+ from collections import defaultdict
7
+ from pathlib import Path
8
+
9
+ from epochclock import budget, fit_theta, transfer
10
+
11
+ rows = defaultdict(lambda: {"step": [], "rmse": [], "upe": None})
12
+ with open(Path(__file__).with_name("pi05_libero_goal.csv")) as f:
13
+ for r in csv.DictReader(f):
14
+ d = rows[int(float(r["demos_per_task"]))]
15
+ d["step"].append(float(r["step"]))
16
+ d["rmse"].append(float(r["arm_rmse"]))
17
+ d["upe"] = float(r["updates_per_epoch"])
18
+
19
+ results = {}
20
+ for demos, d in sorted(rows.items()):
21
+ res = budget(d["step"], d["rmse"], eps=0.03, updates_per_epoch=d["upe"])
22
+ results[demos] = (res, d["upe"])
23
+ print(f"D{demos:<3} {res}")
24
+
25
+ sizes = list(results)
26
+ print(f"\ntiming exponent theta = {fit_theta(sizes, [r.steps for r, _ in results.values()]):.2f}")
27
+
28
+ # Calibrate on the smallest dataset and predict the others.
29
+ pilot, q0 = results[5]
30
+ for demos in (10, 20):
31
+ q = results[demos][1]
32
+ pred = {rule: transfer(pilot.steps, q0, q, rule) for rule in ("epoch", "sqrt", "fixed")}
33
+ print(f"D{demos}: measured {results[demos][0].steps:,.0f} | "
34
+ + " | ".join(f"{k} {v:,.0f}" for k, v in pred.items()))
@@ -0,0 +1,66 @@
1
+ demos_per_task,updates_per_epoch,step,arm_rmse
2
+ 5,193.0,193,0.2042767863558137
3
+ 5,193.0,386,0.1809431029663854
4
+ 5,193.0,579,0.1641131791011565
5
+ 5,193.0,772,0.155113141012549
6
+ 5,193.0,965,0.147363238701602
7
+ 5,193.0,1158,0.1463144050003368
8
+ 5,193.0,1544,0.1420423582848829
9
+ 5,193.0,1930,0.1403852431166273
10
+ 5,193.0,2316,0.1370664654315984
11
+ 5,193.0,2895,0.135198529807239
12
+ 5,193.0,3474,0.1345399598102758
13
+ 5,193.0,3860,0.1362540377090003
14
+ 5,193.0,4825,0.1328416185134516
15
+ 5,193.0,5790,0.1304922931042895
16
+ 5,193.0,6755,0.1329181427736751
17
+ 5,193.0,7720,0.1337503475348346
18
+ 5,193.0,8685,0.1320682881444739
19
+ 5,193.0,9650,0.1319887075212458
20
+ 5,193.0,10615,0.1302776636195059
21
+ 5,193.0,11580,0.1326504629133904
22
+ 5,193.0,12545,0.1347036545920829
23
+ 5,193.0,13510,0.1306795453663392
24
+ 5,193.0,14475,0.1299598651393015
25
+ 5,193.0,15440,0.1304603095109405
26
+ 5,193.0,16405,0.1280288942841006
27
+ 5,193.0,17370,0.1300657118960184
28
+ 5,193.0,18335,0.1305962961643433
29
+ 5,193.0,19300,0.1299745351190462
30
+ 10,392.0,392,0.1787066066508155
31
+ 10,392.0,784,0.1601658580954767
32
+ 10,392.0,1176,0.1552480263941947
33
+ 10,392.0,1568,0.1458124557125602
34
+ 10,392.0,1960,0.1480797828860019
35
+ 10,392.0,2352,0.1368746416065159
36
+ 10,392.0,3136,0.1410051461747792
37
+ 10,392.0,3920,0.1341070180271543
38
+ 10,392.0,4704,0.1294270193493298
39
+ 10,392.0,5880,0.1308368432535637
40
+ 10,392.0,7056,0.1275244964206509
41
+ 10,392.0,7840,0.1272012858833218
42
+ 10,392.0,9800,0.1274890145350163
43
+ 10,392.0,11760,0.1288658007113103
44
+ 10,392.0,13720,0.1278291285245062
45
+ 10,392.0,15680,0.1292641054726498
46
+ 10,392.0,17640,0.1275269367563987
47
+ 10,392.0,19600,0.1252138902319325
48
+ 10,392.0,21560,0.1259372509772568
49
+ 10,392.0,23520,0.1239882259980583
50
+ 20,787.0,787,0.1648325878108779
51
+ 20,787.0,1574,0.1455773892958117
52
+ 20,787.0,2361,0.1404125829060106
53
+ 20,787.0,3148,0.1340854305152559
54
+ 20,787.0,3935,0.1316944303752955
55
+ 20,787.0,4722,0.1286163993976851
56
+ 20,787.0,6296,0.1243449510887657
57
+ 20,787.0,7870,0.1228367432109372
58
+ 20,787.0,9444,0.122226273928476
59
+ 20,787.0,11805,0.1178079356619939
60
+ 20,787.0,14166,0.1188796615192821
61
+ 20,787.0,15740,0.1180733479023664
62
+ 20,787.0,19675,0.1189757620170178
63
+ 20,787.0,23610,0.1172247604173143
64
+ 20,787.0,27545,0.1147344416168542
65
+ 20,787.0,31480,0.1157501032450829
66
+ 20,787.0,39350,0.1138448642335008
@@ -0,0 +1,34 @@
1
+ [build-system]
2
+ requires = ["setuptools>=64", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "epochclock"
7
+ version = "0.1.0"
8
+ description = "Measure fine-tuning budgets on a small pilot and transfer them in epochs, not steps."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ authors = [{ name = "Letian Cheng", email = "cccltray@gmail.com" }]
12
+ requires-python = ">=3.8"
13
+ dependencies = ["numpy>=1.20"]
14
+ keywords = ["fine-tuning", "training budget", "early stopping", "scaling laws", "robot learning", "LLM"]
15
+ classifiers = [
16
+ "Programming Language :: Python :: 3",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
19
+ ]
20
+
21
+ [project.optional-dependencies]
22
+ hf = ["transformers"]
23
+ test = ["pytest"]
24
+
25
+ [project.urls]
26
+ Homepage = "https://github.com/CCCLTRAY/epochclock"
27
+ Repository = "https://github.com/CCCLTRAY/epochclock"
28
+ Issues = "https://github.com/CCCLTRAY/epochclock/issues"
29
+
30
+ [project.scripts]
31
+ epochclock = "epochclock.cli:main"
32
+
33
+ [tool.setuptools.packages.find]
34
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,38 @@
1
+ """epochclock: measure fine-tuning budgets on a pilot and transfer them in epochs.
2
+
3
+ Companion code for "Train in Epochs, Not Steps: How Fine-Tuning Compute Should
4
+ Scale with Data".
5
+ """
6
+
7
+ from .core import (
8
+ BudgetResult,
9
+ MinimumResult,
10
+ budget,
11
+ doubling_gains,
12
+ fit_theta,
13
+ minimum,
14
+ smooth,
15
+ transfer,
16
+ )
17
+ from .monitor import DoublingGainMonitor, EpochClockCallback
18
+ from .warmup import budget_with_warmup, lr_exposure, warmup_in_epochs, warmup_steps_for
19
+
20
+ __version__ = "0.1.0"
21
+
22
+ __all__ = [
23
+ "BudgetResult",
24
+ "MinimumResult",
25
+ "budget",
26
+ "doubling_gains",
27
+ "fit_theta",
28
+ "minimum",
29
+ "smooth",
30
+ "transfer",
31
+ "DoublingGainMonitor",
32
+ "EpochClockCallback",
33
+ "budget_with_warmup",
34
+ "lr_exposure",
35
+ "warmup_in_epochs",
36
+ "warmup_steps_for",
37
+ "__version__",
38
+ ]
@@ -0,0 +1,118 @@
1
+ """Command-line interface.
2
+
3
+ epochclock budget curves.csv --eps 0.03 --updates-per-epoch 193
4
+ epochclock budget curves.csv --group-col data_size --upe-col updates_per_epoch
5
+ epochclock transfer --pilot-steps 1750 --pilot-upe 193 --target-upe 787
6
+ epochclock warmup --epochs 9.1 --upe 193 --warmup 1000
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import csv
13
+ import sys
14
+ from collections import OrderedDict
15
+
16
+ from .core import budget, fit_theta, minimum, transfer
17
+ from .warmup import budget_with_warmup, warmup_in_epochs
18
+
19
+
20
+ def _read(path, step_col, value_col, group_col, upe_col):
21
+ groups = OrderedDict()
22
+ with open(path, newline="", encoding="utf-8-sig") as f:
23
+ for row in csv.DictReader(f):
24
+ try:
25
+ s, v = float(row[step_col]), float(row[value_col])
26
+ except (KeyError, ValueError):
27
+ continue
28
+ key = row.get(group_col, "all") if group_col else "all"
29
+ g = groups.setdefault(key, {"steps": [], "values": [], "upe": None})
30
+ g["steps"].append(s)
31
+ g["values"].append(v)
32
+ if upe_col and row.get(upe_col):
33
+ try:
34
+ g["upe"] = float(row[upe_col])
35
+ except ValueError:
36
+ pass
37
+ return groups
38
+
39
+
40
+ def _cmd_budget(a):
41
+ groups = _read(a.file, a.step_col, a.value_col, a.group_col, a.upe_col)
42
+ if not groups:
43
+ sys.exit("no rows read; check column names")
44
+ sizes, budgets = [], []
45
+ for key, g in groups.items():
46
+ upe = g["upe"] or a.updates_per_epoch
47
+ if a.minimum:
48
+ r = minimum(g["steps"], g["values"], upe)
49
+ steps, status = r.steps, r.status
50
+ else:
51
+ r = budget(g["steps"], g["values"], a.eps, upe, lower_is_better=not a.higher_is_better)
52
+ steps, status = r.steps, r.status
53
+ ep = r.epochs
54
+ label = f"{a.group_col}={key}" if a.group_col else "curve"
55
+ s = "—" if steps is None else f"{steps:,.0f}"
56
+ e = "" if ep is None else f" ({ep:.2f} epochs)"
57
+ print(f"{label:<24} S* = {s} updates{e} [{status}]")
58
+ if steps is not None and status in ("ok", "interpolated", "interior"):
59
+ try:
60
+ sizes.append(float(key))
61
+ budgets.append(steps)
62
+ except ValueError:
63
+ pass
64
+ if len(sizes) >= 2:
65
+ print(f"\ntiming exponent theta = {fit_theta(sizes, budgets):.3f} over {len(sizes)} sizes")
66
+ print(" theta ~ 1: epoch clock (keep epochs fixed) theta ~ 0: step clock")
67
+
68
+
69
+ def _cmd_transfer(a):
70
+ for rule in ("epoch", "sqrt", "fixed") if a.rule == "all" else (a.rule,):
71
+ s = transfer(a.pilot_steps, a.pilot_upe, a.target_upe, rule, a.theta)
72
+ print(f"{rule:<6} -> {s:,.0f} updates ({s / a.target_upe:.2f} epochs)")
73
+
74
+
75
+ def _cmd_warmup(a):
76
+ s = budget_with_warmup(a.epochs, a.upe, a.warmup)
77
+ print(f"warmup of {a.warmup:,.0f} updates = {warmup_in_epochs(a.warmup, a.upe):.2f} epochs of this dataset")
78
+ print(f"budget for {a.epochs:g} full-rate epochs: {s:,.0f} updates "
79
+ f"(vs {a.epochs * a.upe:,.0f} without warmup)")
80
+
81
+
82
+ def main(argv=None):
83
+ p = argparse.ArgumentParser(prog="epochclock", description=__doc__,
84
+ formatter_class=argparse.RawDescriptionHelpFormatter)
85
+ sub = p.add_subparsers(dest="cmd", required=True)
86
+
87
+ b = sub.add_parser("budget", help="measure S* from learning curves in a CSV")
88
+ b.add_argument("file")
89
+ b.add_argument("--step-col", default="step")
90
+ b.add_argument("--value-col", default="value")
91
+ b.add_argument("--group-col", default=None, help="column with data size (one curve per value)")
92
+ b.add_argument("--upe-col", default=None, help="column with updates per epoch")
93
+ b.add_argument("--updates-per-epoch", type=float, default=None)
94
+ b.add_argument("--eps", type=float, default=0.03)
95
+ b.add_argument("--higher-is-better", action="store_true")
96
+ b.add_argument("--minimum", action="store_true", help="use the validation minimum (e.g. NLL)")
97
+ b.set_defaults(func=_cmd_budget)
98
+
99
+ t = sub.add_parser("transfer", help="predict a target budget from a pilot")
100
+ t.add_argument("--pilot-steps", type=float, required=True)
101
+ t.add_argument("--pilot-upe", type=float, required=True)
102
+ t.add_argument("--target-upe", type=float, required=True)
103
+ t.add_argument("--rule", default="all", choices=["all", "epoch", "sqrt", "fixed", "theta"])
104
+ t.add_argument("--theta", type=float, default=None)
105
+ t.set_defaults(func=_cmd_transfer)
106
+
107
+ w = sub.add_parser("warmup", help="effect of a step-based warmup on a budget")
108
+ w.add_argument("--epochs", type=float, required=True, help="full-rate epochs needed (K)")
109
+ w.add_argument("--upe", type=float, required=True, help="updates per epoch")
110
+ w.add_argument("--warmup", type=float, required=True, help="warmup updates")
111
+ w.set_defaults(func=_cmd_warmup)
112
+
113
+ a = p.parse_args(argv)
114
+ a.func(a)
115
+
116
+
117
+ if __name__ == "__main__":
118
+ main()
@@ -0,0 +1,283 @@
1
+ """Core measurements: the doubling-gain budget, validation minima, and transfer rules.
2
+
3
+ All step quantities are optimizer updates. "Updates per epoch" is the number of
4
+ optimizer updates in one pass over the training set (e.g. floor(N / batch)).
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import math
10
+ from dataclasses import dataclass, field
11
+ from typing import Optional, Sequence
12
+
13
+ import numpy as np
14
+
15
+ __all__ = [
16
+ "BudgetResult",
17
+ "MinimumResult",
18
+ "smooth",
19
+ "doubling_gains",
20
+ "budget",
21
+ "minimum",
22
+ "transfer",
23
+ "fit_theta",
24
+ ]
25
+
26
+ WINDOW = 1.2 # centered multiplicative smoothing window, as in the paper
27
+
28
+
29
+ # ---------------------------------------------------------------------------
30
+ # helpers
31
+ # ---------------------------------------------------------------------------
32
+ def _clean(steps: Sequence[float], values: Sequence[float]):
33
+ s = np.asarray(steps, dtype=float)
34
+ v = np.asarray(values, dtype=float)
35
+ if s.shape != v.shape:
36
+ raise ValueError("steps and values must have the same length")
37
+ keep = np.isfinite(s) & np.isfinite(v) & (s > 0)
38
+ s, v = s[keep], v[keep]
39
+ order = np.argsort(s)
40
+ s, v = s[order], v[order]
41
+ # average duplicated steps
42
+ if len(np.unique(s)) != len(s):
43
+ us = np.unique(s)
44
+ v = np.array([v[s == u].mean() for u in us])
45
+ s = us
46
+ if len(s) < 3:
47
+ raise ValueError("need at least three checkpoints with step > 0")
48
+ return s, v
49
+
50
+
51
+ def smooth(steps: Sequence[float], values: Sequence[float], window: float = WINDOW) -> np.ndarray:
52
+ """Average every checkpoint with all checkpoints within a factor `window` in steps."""
53
+ s, v = _clean(steps, values)
54
+ ls = np.log(s)
55
+ half = math.log(window) + 1e-9
56
+ return np.array([v[np.abs(ls - x) <= half].mean() for x in ls])
57
+
58
+
59
+ def doubling_gains(
60
+ steps: Sequence[float],
61
+ values: Sequence[float],
62
+ lower_is_better: bool = True,
63
+ window: float = WINDOW,
64
+ anchors: str = "auto",
65
+ tol: float = 0.02,
66
+ ):
67
+ """Relative improvement from doubling training, g(S) = 1 - E(2S)/E(S).
68
+
69
+ Both E(S) and E(2S) are read from the smoothed curve (checkpoints averaged
70
+ within a factor `window`).
71
+
72
+ anchors="exact" use only measured S whose doubling 2S is also a
73
+ measured checkpoint (within relative `tol`). This is
74
+ the procedure used in the paper.
75
+ anchors="interpolate" use every measured S with 2S inside the range and
76
+ read E(2S) by linear interpolation in log steps.
77
+ anchors="auto" "exact" if it yields at least two anchors, else
78
+ "interpolate".
79
+
80
+ Returns arrays (anchor_steps, gains).
81
+ """
82
+ s, v = _clean(steps, values)
83
+ sm = smooth(s, v, window)
84
+ sign = 1.0 if lower_is_better else -1.0
85
+
86
+ def _exact():
87
+ a, g = [], []
88
+ for i, si in enumerate(s):
89
+ j = np.where(np.abs(s / (2 * si) - 1) <= tol)[0]
90
+ if len(j):
91
+ a.append(si)
92
+ g.append(sign * (1.0 - sm[j[0]] / sm[i]))
93
+ return np.array(a), np.array(g)
94
+
95
+ def _interp():
96
+ ls = np.log(s)
97
+ a, g = [], []
98
+ for si, ei in zip(s, sm):
99
+ if 2 * si > s[-1] * (1 + 1e-9):
100
+ break
101
+ e2 = float(np.interp(math.log(2 * si), ls, sm))
102
+ a.append(si)
103
+ g.append(sign * (1.0 - e2 / ei))
104
+ return np.array(a), np.array(g)
105
+
106
+ if anchors == "exact":
107
+ return _exact()
108
+ if anchors == "interpolate":
109
+ return _interp()
110
+ if anchors == "auto":
111
+ a, g = _exact()
112
+ return (a, g) if len(a) >= 2 else _interp()
113
+ raise ValueError(f"unknown anchors mode {anchors!r}")
114
+
115
+
116
+ # ---------------------------------------------------------------------------
117
+ # results
118
+ # ---------------------------------------------------------------------------
119
+ @dataclass
120
+ class BudgetResult:
121
+ """Budget after which every later observed doubling improves by < eps."""
122
+
123
+ steps: Optional[float]
124
+ eps: float
125
+ updates_per_epoch: Optional[float] = None
126
+ status: str = "ok" # "ok", "left_censored", "not_reached", "too_short"
127
+ anchors: np.ndarray = field(default_factory=lambda: np.array([]), repr=False)
128
+ gains: np.ndarray = field(default_factory=lambda: np.array([]), repr=False)
129
+
130
+ @property
131
+ def epochs(self) -> Optional[float]:
132
+ if self.steps is None or not self.updates_per_epoch:
133
+ return None
134
+ return self.steps / self.updates_per_epoch
135
+
136
+ def __str__(self) -> str:
137
+ if self.steps is None:
138
+ return f"Budget(eps={self.eps:.3g}): {self.status}"
139
+ ep = f", K={self.epochs:.2f} epochs" if self.epochs is not None else ""
140
+ flag = "" if self.status == "ok" else f" [{self.status}]"
141
+ return f"Budget(eps={self.eps:.3g}): S*={self.steps:,.0f} updates{ep}{flag}"
142
+
143
+
144
+ @dataclass
145
+ class MinimumResult:
146
+ """Location of a validation minimum (e.g. NLL), with optional log-quadratic refinement."""
147
+
148
+ steps: float
149
+ value: float
150
+ updates_per_epoch: Optional[float] = None
151
+ status: str = "interior" # "interior", "interpolated", "left_edge", "right_edge"
152
+
153
+ @property
154
+ def epochs(self) -> Optional[float]:
155
+ if not self.updates_per_epoch:
156
+ return None
157
+ return self.steps / self.updates_per_epoch
158
+
159
+ def __str__(self) -> str:
160
+ ep = f", {self.epochs:.2f} epochs" if self.epochs is not None else ""
161
+ return f"Minimum: {self.steps:,.0f} updates{ep} (value={self.value:.5g}, {self.status})"
162
+
163
+
164
+ # ---------------------------------------------------------------------------
165
+ # measurements
166
+ # ---------------------------------------------------------------------------
167
+ def budget(
168
+ steps: Sequence[float],
169
+ values: Sequence[float],
170
+ eps: float = 0.03,
171
+ updates_per_epoch: Optional[float] = None,
172
+ lower_is_better: bool = True,
173
+ window: float = WINDOW,
174
+ anchors: str = "auto",
175
+ grid: int = 4000,
176
+ ) -> BudgetResult:
177
+ """Measure the training budget S*(eps) from one learning curve.
178
+
179
+ The doubling gain g(S) is computed at anchor checkpoints and interpolated
180
+ linearly in log steps. S*(eps) is the earliest S such that g(S), g(2S),
181
+ g(4S), ... are all below `eps` for every dyadic doubling inside the
182
+ supported range. Requiring later doublings to agree prevents a temporary
183
+ flat region from being taken as the end of the rapid decline.
184
+
185
+ Status values:
186
+ ok a crossing was found inside the supported range
187
+ left_censored the criterion already holds at the first anchor
188
+ not_reached no supported S satisfies the criterion (train longer)
189
+ too_short fewer than two doubling anchors are available
190
+ """
191
+ a, g = doubling_gains(steps, values, lower_is_better, window, anchors)
192
+ if len(a) < 2:
193
+ return BudgetResult(None, eps, updates_per_epoch, "too_short", a, g)
194
+ la = np.log(a)
195
+ xs = np.linspace(la[0], la[-1], grid)
196
+ ln2 = math.log(2.0)
197
+ for idx, x in enumerate(xs):
198
+ n = int((la[-1] - x) / ln2 + 1e-9)
199
+ chain = x + ln2 * np.arange(n + 1)
200
+ if np.all(np.interp(chain, la, g) < eps):
201
+ if idx == 0:
202
+ return BudgetResult(float(math.exp(x)), eps, updates_per_epoch, "left_censored", a, g)
203
+ # refine the switch point between the previous and current grid points
204
+ lo, hi = xs[idx - 1], x
205
+ for _ in range(50):
206
+ mid = 0.5 * (lo + hi)
207
+ n = int((la[-1] - mid) / ln2 + 1e-9)
208
+ if np.all(np.interp(mid + ln2 * np.arange(n + 1), la, g) < eps):
209
+ hi = mid
210
+ else:
211
+ lo = mid
212
+ return BudgetResult(float(math.exp(hi)), eps, updates_per_epoch, "ok", a, g)
213
+ return BudgetResult(None, eps, updates_per_epoch, "not_reached", a, g)
214
+
215
+
216
+ def minimum(
217
+ steps: Sequence[float],
218
+ values: Sequence[float],
219
+ updates_per_epoch: Optional[float] = None,
220
+ interpolate: bool = True,
221
+ ) -> MinimumResult:
222
+ """Locate the minimum of a validation curve (e.g. NLL).
223
+
224
+ With `interpolate=True`, an interior minimum is refined by a quadratic in
225
+ log steps through the minimum and its two neighbours; the vertex is kept
226
+ only if the curvature is positive and it lies inside that bracket.
227
+ """
228
+ s, v = _clean(steps, values)
229
+ i = int(np.argmin(v))
230
+ if i == 0:
231
+ return MinimumResult(float(s[0]), float(v[0]), updates_per_epoch, "left_edge")
232
+ if i == len(s) - 1:
233
+ return MinimumResult(float(s[-1]), float(v[-1]), updates_per_epoch, "right_edge")
234
+ if interpolate:
235
+ x = np.log(s[i - 1 : i + 2])
236
+ c2, c1, c0 = np.polyfit(x, v[i - 1 : i + 2], 2)
237
+ if c2 > 0:
238
+ xv = -c1 / (2 * c2)
239
+ if x[0] < xv < x[2]:
240
+ return MinimumResult(
241
+ float(math.exp(xv)), float(c0 + c1 * xv + c2 * xv * xv), updates_per_epoch, "interpolated"
242
+ )
243
+ return MinimumResult(float(s[i]), float(v[i]), updates_per_epoch, "interior")
244
+
245
+
246
+ # ---------------------------------------------------------------------------
247
+ # transfer
248
+ # ---------------------------------------------------------------------------
249
+ def transfer(
250
+ pilot_steps: float,
251
+ pilot_updates_per_epoch: float,
252
+ target_updates_per_epoch: float,
253
+ rule: str = "epoch",
254
+ theta: Optional[float] = None,
255
+ ) -> float:
256
+ """Predict the target budget from a pilot budget.
257
+
258
+ rule="epoch" keep the epoch count S = S0 * (q/q0)
259
+ rule="sqrt" square-root scaling S = S0 * sqrt(q/q0)
260
+ rule="fixed" reuse the pilot's step count S = S0
261
+ rule="theta" measured exponent S = S0 * (q/q0)**theta
262
+ """
263
+ ratio = target_updates_per_epoch / pilot_updates_per_epoch
264
+ if rule == "epoch":
265
+ return pilot_steps * ratio
266
+ if rule == "sqrt":
267
+ return pilot_steps * math.sqrt(ratio)
268
+ if rule == "fixed":
269
+ return float(pilot_steps)
270
+ if rule == "theta":
271
+ if theta is None:
272
+ raise ValueError("rule='theta' requires theta (see fit_theta)")
273
+ return pilot_steps * ratio**theta
274
+ raise ValueError(f"unknown rule {rule!r}")
275
+
276
+
277
+ def fit_theta(data_sizes: Sequence[float], budgets: Sequence[float]) -> float:
278
+ """Timing exponent: least-squares slope of log(budget) on log(data size)."""
279
+ x = np.log(np.asarray(data_sizes, dtype=float))
280
+ y = np.log(np.asarray(budgets, dtype=float))
281
+ if len(x) < 2:
282
+ raise ValueError("need at least two data sizes")
283
+ return float(np.polyfit(x, y, 1)[0])
@@ -0,0 +1,135 @@
1
+ """Online monitoring of the doubling gain during training.
2
+
3
+ `DoublingGainMonitor` is framework-agnostic: call `update(step, value)` after
4
+ each evaluation. `EpochClockCallback` wraps it for the Hugging Face Trainer.
5
+
6
+ Online, only past doublings are observable, so the monitor reports the gain of
7
+ the most recent completed doubling, 1 - E(S)/E(S/2). Use `budget()` after
8
+ training (or on a pilot) for the full criterion, which also requires later
9
+ doublings to stay below the threshold.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import math
15
+ from typing import List, Optional
16
+
17
+ import numpy as np
18
+
19
+ from .core import WINDOW, BudgetResult, budget
20
+
21
+ __all__ = ["DoublingGainMonitor", "EpochClockCallback"]
22
+
23
+
24
+ class DoublingGainMonitor:
25
+ def __init__(
26
+ self,
27
+ eps: float = 0.03,
28
+ updates_per_epoch: Optional[float] = None,
29
+ lower_is_better: bool = True,
30
+ patience: int = 2,
31
+ window: float = WINDOW,
32
+ ):
33
+ self.eps = eps
34
+ self.updates_per_epoch = updates_per_epoch
35
+ self.lower_is_better = lower_is_better
36
+ self.patience = patience
37
+ self.window = window
38
+ self.steps: List[float] = []
39
+ self.values: List[float] = []
40
+ self._below = 0
41
+
42
+ def _smoothed_at(self, x: float) -> Optional[float]:
43
+ s = np.asarray(self.steps, dtype=float)
44
+ v = np.asarray(self.values, dtype=float)
45
+ if len(s) == 0 or x < s.min() / self.window or x > s.max() * self.window:
46
+ return None
47
+ near = np.abs(np.log(s) - math.log(x)) <= math.log(self.window) + 1e-9
48
+ if near.any():
49
+ return float(v[near].mean())
50
+ order = np.argsort(s)
51
+ return float(np.interp(math.log(x), np.log(s[order]), v[order]))
52
+
53
+ def update(self, step: float, value: float) -> Optional[float]:
54
+ """Record an evaluation; return the gain of the last completed doubling."""
55
+ if step <= 0 or not math.isfinite(value):
56
+ return None
57
+ self.steps.append(float(step))
58
+ self.values.append(float(value))
59
+ g = self.latest_gain
60
+ if g is not None:
61
+ self._below = self._below + 1 if g < self.eps else 0
62
+ return g
63
+
64
+ @property
65
+ def latest_gain(self) -> Optional[float]:
66
+ if len(self.steps) < 2:
67
+ return None
68
+ s = max(self.steps)
69
+ if s / 2 < min(self.steps):
70
+ return None
71
+ now, half = self._smoothed_at(s), self._smoothed_at(s / 2)
72
+ if now is None or half is None or half == 0:
73
+ return None
74
+ g = 1.0 - now / half
75
+ return g if self.lower_is_better else -g
76
+
77
+ @property
78
+ def suggest_stop(self) -> bool:
79
+ """True after `patience` consecutive evaluations whose last doubling gained < eps."""
80
+ return self._below >= self.patience
81
+
82
+ def budget(self, eps: Optional[float] = None) -> BudgetResult:
83
+ """Apply the full (post-hoc) criterion to the recorded curve."""
84
+ return budget(
85
+ self.steps,
86
+ self.values,
87
+ eps=self.eps if eps is None else eps,
88
+ updates_per_epoch=self.updates_per_epoch,
89
+ lower_is_better=self.lower_is_better,
90
+ window=self.window,
91
+ )
92
+
93
+
94
+ try: # optional Hugging Face integration
95
+ from transformers import TrainerCallback # type: ignore
96
+ except Exception: # pragma: no cover - transformers is optional
97
+ TrainerCallback = object # type: ignore
98
+
99
+
100
+ class EpochClockCallback(TrainerCallback): # type: ignore[misc]
101
+ """Hugging Face Trainer callback.
102
+
103
+ Logs `epochclock/doubling_gain` after each evaluation and, if `stop=True`,
104
+ stops training once the last completed doubling has gained less than `eps`
105
+ for `patience` consecutive evaluations. Evaluate on a log-spaced schedule
106
+ (or densely) so that doublings are observed.
107
+
108
+ trainer = Trainer(..., callbacks=[EpochClockCallback(eps=0.03,
109
+ updates_per_epoch=len(train_dataloader))])
110
+ """
111
+
112
+ def __init__(
113
+ self,
114
+ eps: float = 0.03,
115
+ metric: str = "eval_loss",
116
+ updates_per_epoch: Optional[float] = None,
117
+ lower_is_better: bool = True,
118
+ patience: int = 2,
119
+ stop: bool = False,
120
+ ):
121
+ if TrainerCallback is object:
122
+ raise ImportError("EpochClockCallback requires `transformers`")
123
+ self.metric = metric
124
+ self.stop = stop
125
+ self.monitor = DoublingGainMonitor(eps, updates_per_epoch, lower_is_better, patience)
126
+
127
+ def on_evaluate(self, args, state, control, metrics=None, **kwargs):
128
+ if not metrics or self.metric not in metrics:
129
+ return control
130
+ g = self.monitor.update(state.global_step, float(metrics[self.metric]))
131
+ if g is not None:
132
+ state.log_history.append({"step": state.global_step, "epochclock/doubling_gain": g})
133
+ if self.stop and self.monitor.suggest_stop:
134
+ control.should_training_stop = True
135
+ return control
@@ -0,0 +1,52 @@
1
+ """Warmup in exposure units.
2
+
3
+ A warmup fixed in updates covers many epochs of a small dataset and few of a
4
+ large one. These helpers convert between the two and predict how a step-based
5
+ warmup delays a budget that needs K full-rate passes.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import math
11
+
12
+ __all__ = ["lr_exposure", "budget_with_warmup", "warmup_in_epochs", "warmup_steps_for"]
13
+
14
+
15
+ def lr_exposure(steps: float, warmup: float) -> float:
16
+ """Cumulative learning rate relative to its peak after `steps` updates.
17
+
18
+ For linear warmup over `warmup` updates followed by a constant rate:
19
+ X_W(S) = S^2 / (2W) if S < W
20
+ = S - W/2 if S >= W
21
+ """
22
+ if warmup <= 0:
23
+ return float(steps)
24
+ if steps < warmup:
25
+ return steps * steps / (2.0 * warmup)
26
+ return steps - warmup / 2.0
27
+
28
+
29
+ def budget_with_warmup(epochs: float, updates_per_epoch: float, warmup: float) -> float:
30
+ """Updates needed to deliver `epochs` full-rate passes under a linear warmup.
31
+
32
+ Solves X_W(S) = epochs * updates_per_epoch:
33
+ S = sqrt(2 W K q) if K q < W/2 (the run ends inside the warmup)
34
+ S = K q + W/2 otherwise
35
+ Small datasets are delayed most; large datasets pay only W/2.
36
+ """
37
+ target = epochs * updates_per_epoch
38
+ if warmup <= 0:
39
+ return float(target)
40
+ if target < warmup / 2.0:
41
+ return math.sqrt(2.0 * warmup * target)
42
+ return target + warmup / 2.0
43
+
44
+
45
+ def warmup_in_epochs(warmup_steps: float, updates_per_epoch: float) -> float:
46
+ """How many passes over the data a step-based warmup occupies."""
47
+ return warmup_steps / updates_per_epoch
48
+
49
+
50
+ def warmup_steps_for(warmup_epochs: float, updates_per_epoch: float) -> int:
51
+ """Warmup length in updates for a warmup specified in epochs."""
52
+ return max(0, int(round(warmup_epochs * updates_per_epoch)))
@@ -0,0 +1,142 @@
1
+ Metadata-Version: 2.4
2
+ Name: epochclock
3
+ Version: 0.1.0
4
+ Summary: Measure fine-tuning budgets on a small pilot and transfer them in epochs, not steps.
5
+ Author-email: Letian Cheng <cccltray@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/CCCLTRAY/epochclock
8
+ Project-URL: Repository, https://github.com/CCCLTRAY/epochclock
9
+ Project-URL: Issues, https://github.com/CCCLTRAY/epochclock/issues
10
+ Keywords: fine-tuning,training budget,early stopping,scaling laws,robot learning,LLM
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
14
+ Requires-Python: >=3.8
15
+ Description-Content-Type: text/markdown
16
+ License-File: LICENSE
17
+ Requires-Dist: numpy>=1.20
18
+ Provides-Extra: hf
19
+ Requires-Dist: transformers; extra == "hf"
20
+ Provides-Extra: test
21
+ Requires-Dist: pytest; extra == "test"
22
+ Dynamic: license-file
23
+
24
+ # epochclock
25
+
26
+ **How long should you fine-tune when you get more data?** Measure the budget on a
27
+ small pilot, then keep the number of *epochs*, not the number of steps.
28
+
29
+ `epochclock` is the companion tool for the paper
30
+ *Train in Epochs, Not Steps: How Fine-Tuning Compute Should Scale with Data*.
31
+ It reads a learning curve, finds the point after which another doubling of
32
+ training stops paying off, and transfers that budget to a larger dataset.
33
+
34
+ ```bash
35
+ pip install epochclock
36
+ ```
37
+
38
+ ## Why
39
+
40
+ Fine-tuning recipes inherit a training length from a public config, such as
41
+ "30k updates" for a robot policy or "3 epochs" for an LLM. These two conventions
42
+ agree at one dataset size and disagree everywhere else. For the pretrained
43
+ models in the paper (π0.5 on LIBERO, SmolLM2 on math fine-tuning), the useful budget grows in proportion to data: the number of epochs stays
44
+ roughly constant. Reusing a step count undertrains larger datasets and can make
45
+ extra data look useless. Step-based warmup and randomly initialized components
46
+ (for example a new head) move training toward a step clock; this tool helps you
47
+ detect that from two pilots.
48
+
49
+ ## Quick start
50
+
51
+ ```python
52
+ from epochclock import budget, transfer
53
+
54
+ # One learning curve from a small pilot: steps and validation error.
55
+ r = budget(steps, val_error, eps=0.03, updates_per_epoch=193)
56
+ print(r) # Budget(eps=0.03): S*=1,750 updates, K=9.07 epochs
57
+
58
+ # Predict the budget for a 4x larger dataset (787 updates per epoch).
59
+ transfer(r.steps, 193, 787, rule="epoch") # -> 7,134 updates
60
+ ```
61
+
62
+ With two pilot sizes, measure the timing exponent instead of assuming it:
63
+
64
+ ```python
65
+ from epochclock import fit_theta, transfer
66
+ theta = fit_theta([5, 10], [r5.steps, r10.steps]) # ~1: epoch clock, ~0: step clock
67
+ transfer(r5.steps, 193, 787, rule="theta", theta=theta)
68
+ ```
69
+
70
+ For curves with a minimum (e.g. LLM validation NLL), use `minimum`:
71
+
72
+ ```python
73
+ from epochclock import minimum
74
+ minimum(steps, val_nll, updates_per_epoch=50) # log-quadratic refinement of the minimum
75
+ ```
76
+
77
+ ### Warmup
78
+
79
+ A warmup fixed in steps covers many epochs of a small dataset and few of a large
80
+ one, which delays small-data optima:
81
+
82
+ ```python
83
+ from epochclock import budget_with_warmup, warmup_steps_for
84
+ budget_with_warmup(epochs=9.1, updates_per_epoch=193, warmup=1000) # -> 2,256 (vs 1,756)
85
+ warmup_steps_for(0.5, updates_per_epoch=787) # specify warmup in epochs instead
86
+ ```
87
+
88
+ ### During training
89
+
90
+ ```python
91
+ from epochclock import DoublingGainMonitor
92
+ mon = DoublingGainMonitor(eps=0.03, updates_per_epoch=193)
93
+ for step in eval_steps: # evaluate on a log-spaced schedule
94
+ gain = mon.update(step, val_error) # gain of the last completed doubling
95
+ if mon.suggest_stop:
96
+ break
97
+ print(mon.budget()) # full criterion on the recorded curve
98
+ ```
99
+
100
+ Hugging Face `Trainer` users can add `EpochClockCallback(eps=0.03, stop=False)`
101
+ (requires `pip install epochclock[hf]`); it logs `epochclock/doubling_gain`
102
+ after each evaluation.
103
+
104
+ ### Command line
105
+
106
+ ```bash
107
+ epochclock budget curves.csv --value-col arm_rmse --group-col demos_per_task --upe-col updates_per_epoch
108
+ epochclock transfer --pilot-steps 1750 --pilot-upe 193 --target-upe 787
109
+ epochclock warmup --epochs 9.1 --upe 193 --warmup 1000
110
+ ```
111
+
112
+ `examples/pi05_example.py` reproduces the paper's π0.5 budgets
113
+ (1,750 / 4,146 / 8,342 updates; θ = 1.13) from the included curves.
114
+
115
+ ## The criterion
116
+
117
+ For a learning curve E(S), checkpoints are averaged within a factor of 1.2 in
118
+ steps, and the return from doubling is g(S) = 1 − E(2S)/E(S), computed at
119
+ checkpoints whose doubling was also measured. The budget S\*(ε) is the earliest
120
+ S such that g(S), g(2S), g(4S), … all stay below ε inside the measured range.
121
+ The result reports a status:
122
+
123
+ | status | meaning |
124
+ |---|---|
125
+ | `ok` | budget found |
126
+ | `left_censored` | already flat at the first doubling; save earlier checkpoints |
127
+ | `not_reached` | still improving; train longer |
128
+ | `too_short` | fewer than two doublings measured |
129
+
130
+ Practical notes: save checkpoints on a log-spaced grid with exact doublings
131
+ (e.g. 1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4, … epochs), and evaluate every
132
+ checkpoint on the same fixed validation set. The threshold chooses how far
133
+ down the slow tail you go; a stricter ε buys more improvement for more compute,
134
+ and the epoch count at that point transfers in the same way. Check the task
135
+ metric near the transferred budget: validation loss and task success can peak
136
+ at different times.
137
+
138
+ ## Citation
139
+
140
+ A paper link and citation will be added when the manuscript is available.
141
+
142
+ MIT License.
@@ -0,0 +1,18 @@
1
+ LICENSE
2
+ MANIFEST.in
3
+ README.md
4
+ pyproject.toml
5
+ examples/pi05_example.py
6
+ examples/pi05_libero_goal.csv
7
+ src/epochclock/__init__.py
8
+ src/epochclock/cli.py
9
+ src/epochclock/core.py
10
+ src/epochclock/monitor.py
11
+ src/epochclock/warmup.py
12
+ src/epochclock.egg-info/PKG-INFO
13
+ src/epochclock.egg-info/SOURCES.txt
14
+ src/epochclock.egg-info/dependency_links.txt
15
+ src/epochclock.egg-info/entry_points.txt
16
+ src/epochclock.egg-info/requires.txt
17
+ src/epochclock.egg-info/top_level.txt
18
+ tests/test_epochclock.py
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ epochclock = epochclock.cli:main
@@ -0,0 +1,7 @@
1
+ numpy>=1.20
2
+
3
+ [hf]
4
+ transformers
5
+
6
+ [test]
7
+ pytest
@@ -0,0 +1 @@
1
+ epochclock
@@ -0,0 +1,99 @@
1
+ import math
2
+
3
+ import numpy as np
4
+ import pytest
5
+
6
+ from epochclock import (
7
+ DoublingGainMonitor,
8
+ budget,
9
+ budget_with_warmup,
10
+ fit_theta,
11
+ lr_exposure,
12
+ minimum,
13
+ transfer,
14
+ )
15
+
16
+
17
+ def grid(n_max, per_octave=4, start=8):
18
+ k = np.arange(0, int(per_octave * math.log2(n_max / start)) + 1)
19
+ return np.unique(np.round(start * 2 ** (k / per_octave)))
20
+
21
+
22
+ def curve(steps, clock, floor=0.1, amp=1.0, gamma=1.0):
23
+ return floor + amp * (steps / clock) ** (-gamma)
24
+
25
+
26
+ def test_epoch_clock_gives_theta_one():
27
+ sizes = [1000, 2000, 4000, 8000]
28
+ budgets = []
29
+ for n in sizes:
30
+ q = n / 32
31
+ s = grid(q * 400)
32
+ budgets.append(budget(s, curve(s, q), eps=0.03, updates_per_epoch=q).steps)
33
+ assert fit_theta(sizes, budgets) == pytest.approx(1.0, abs=0.03)
34
+
35
+
36
+ def test_step_clock_gives_theta_zero():
37
+ sizes = [1000, 2000, 4000, 8000]
38
+ budgets = []
39
+ for _ in sizes:
40
+ s = grid(1e5)
41
+ budgets.append(budget(s, curve(s, 50.0)).steps)
42
+ assert fit_theta(sizes, budgets) == pytest.approx(0.0, abs=0.03)
43
+
44
+
45
+ def test_threshold_moves_budget_later():
46
+ s = grid(1e5)
47
+ v = curve(s, 50.0)
48
+ assert budget(s, v, eps=0.02).steps > budget(s, v, eps=0.05).steps
49
+
50
+
51
+ def test_status_flags():
52
+ s = grid(64)
53
+ v = curve(s, 50.0) # still improving fast at the end
54
+ assert budget(s, v, eps=0.03).status == "not_reached"
55
+ flat = np.full_like(s, 0.5, dtype=float)
56
+ assert budget(s, flat, eps=0.03).status == "left_censored"
57
+
58
+
59
+ def test_higher_is_better():
60
+ s = grid(1e5)
61
+ acc = 1.0 - curve(s, 50.0)
62
+ r = budget(s, acc, eps=0.01, lower_is_better=False)
63
+ assert r.status in ("ok", "left_censored")
64
+
65
+
66
+ def test_minimum_interpolates():
67
+ s = np.array([10, 20, 40, 80, 160, 320], dtype=float)
68
+ x = np.log(s)
69
+ v = (x - math.log(60)) ** 2 + 1
70
+ r = minimum(s, v, updates_per_epoch=30)
71
+ assert r.status == "interpolated"
72
+ assert r.steps == pytest.approx(60, rel=1e-6)
73
+ assert r.epochs == pytest.approx(2.0, rel=1e-6)
74
+
75
+
76
+ def test_transfer_rules():
77
+ assert transfer(1000, 100, 400, "epoch") == pytest.approx(4000)
78
+ assert transfer(1000, 100, 400, "sqrt") == pytest.approx(2000)
79
+ assert transfer(1000, 100, 400, "fixed") == pytest.approx(1000)
80
+ assert transfer(1000, 100, 400, "theta", theta=0.5) == pytest.approx(2000)
81
+
82
+
83
+ def test_warmup_formula():
84
+ assert lr_exposure(50, 100) == pytest.approx(12.5)
85
+ assert lr_exposure(300, 100) == pytest.approx(250)
86
+ # ends inside warmup
87
+ s = budget_with_warmup(epochs=1, updates_per_epoch=20, warmup=100)
88
+ assert lr_exposure(s, 100) == pytest.approx(20)
89
+ # ends after warmup
90
+ s = budget_with_warmup(epochs=10, updates_per_epoch=100, warmup=100)
91
+ assert s == pytest.approx(1050)
92
+
93
+
94
+ def test_monitor():
95
+ m = DoublingGainMonitor(eps=0.03, patience=2)
96
+ for s in grid(1e5):
97
+ m.update(s, curve(s, 50.0))
98
+ assert m.suggest_stop
99
+ assert m.budget().status == "ok"