useml 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {useml-0.3.0 → useml-0.3.2}/PKG-INFO +1 -1
- {useml-0.3.0 → useml-0.3.2}/pyproject.toml +1 -1
- {useml-0.3.0 → useml-0.3.2}/tests/test_usecases.py +361 -1
- {useml-0.3.0 → useml-0.3.2}/useml/__init__.py +16 -1
- {useml-0.3.0 → useml-0.3.2}/useml/template/trainer.py +39 -32
- {useml-0.3.0 → useml-0.3.2}/useml/vault/code_extractor.py +5 -2
- {useml-0.3.0 → useml-0.3.2}/useml/vault/runs.py +343 -29
- {useml-0.3.0 → useml-0.3.2}/useml.egg-info/PKG-INFO +1 -1
- {useml-0.3.0 → useml-0.3.2}/README.md +0 -0
- {useml-0.3.0 → useml-0.3.2}/setup.cfg +0 -0
- {useml-0.3.0 → useml-0.3.2}/tests/test_bundle.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/tests/test_level2.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/tests/test_session.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/tests/test_train_level0.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/tests/test_vault.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/dataset/__init__.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/dataset/bundle.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/dataset/loaders.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/errors.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/imports.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/session/__init__.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/session/component.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/session/manager.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/template/__init__.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/template/config.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/template/loss.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/template/model.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/vault/__init__.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/vault/core.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/vault/model_registry.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/vault/project.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/vault/snapshot.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/workdir/__init__.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml/workdir/_hook.py +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml.egg-info/SOURCES.txt +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml.egg-info/dependency_links.txt +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml.egg-info/requires.txt +0 -0
- {useml-0.3.0 → useml-0.3.2}/useml.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: useml
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Minimalist ML experiment versioning for PyTorch — checkpointing, source archiving, and model isolation.
|
|
5
5
|
Author-email: Enzo-py <85191552+Enzo-py@users.noreply.github.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/Enzo-py/useml
|
|
@@ -12,7 +12,7 @@ import torch.nn.functional as F
|
|
|
12
12
|
from torch.utils.data import TensorDataset, DataLoader
|
|
13
13
|
|
|
14
14
|
import useml
|
|
15
|
-
from useml import Config
|
|
15
|
+
from useml import Config, ModelInstantiationError
|
|
16
16
|
from useml.session.manager import _session
|
|
17
17
|
from useml.dataset import load_dataset
|
|
18
18
|
from useml.template.trainer import Trainer
|
|
@@ -60,6 +60,15 @@ class TinyReg(nn.Module):
|
|
|
60
60
|
def forward(self, x): return self.fc(x).squeeze(-1)
|
|
61
61
|
|
|
62
62
|
|
|
63
|
+
class _ModelNeedsEncoder(nn.Module):
|
|
64
|
+
"""Used by TestRunLoadModes — requires an object arg not storable in Config."""
|
|
65
|
+
def __init__(self, encoder):
|
|
66
|
+
super().__init__()
|
|
67
|
+
self.enc = encoder
|
|
68
|
+
self.fc = nn.Linear(4, 2)
|
|
69
|
+
def forward(self, x): return self.fc(x)
|
|
70
|
+
|
|
71
|
+
|
|
63
72
|
class TinyAE(nn.Module):
|
|
64
73
|
def __init__(self):
|
|
65
74
|
super().__init__()
|
|
@@ -887,3 +896,354 @@ class TestSideTrackANewAPI:
|
|
|
887
896
|
run2 = proj.runs.new(TinyClf, config=cfg)
|
|
888
897
|
run2.resume("\\latest")
|
|
889
898
|
assert run2._epoch >= 1 # picked up at least one epoch
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
902
|
+
# History & plot
|
|
903
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
904
|
+
|
|
905
|
+
def _train_with_history(proj, epochs=3):
|
|
906
|
+
"""Helper: train a small run and return its RunRecord."""
|
|
907
|
+
ds = _clf_ds()
|
|
908
|
+
loader = DataLoader(ds, batch_size=32)
|
|
909
|
+
cfg = _cfg(epochs=epochs, checkpoint_strategy="last")
|
|
910
|
+
run = proj.runs.new(TinyClf, config=cfg)
|
|
911
|
+
run.train(loader, loader)
|
|
912
|
+
return proj.runs.latest
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
class TestRunHistory:
|
|
916
|
+
"""history.yaml written at checkpoint; RunRecord.history / history_df / plot."""
|
|
917
|
+
|
|
918
|
+
def test_history_written_after_train(self):
|
|
919
|
+
proj = useml.focus("test")
|
|
920
|
+
record = _train_with_history(proj, epochs=3)
|
|
921
|
+
h_path = record._snapshot.path / "history.yaml"
|
|
922
|
+
assert h_path.exists(), "history.yaml must be created alongside weights"
|
|
923
|
+
|
|
924
|
+
def test_history_keys_present(self):
|
|
925
|
+
proj = useml.focus("test")
|
|
926
|
+
record = _train_with_history(proj, epochs=3)
|
|
927
|
+
h = record.history
|
|
928
|
+
assert "train_loss" in h
|
|
929
|
+
assert "val_loss" in h
|
|
930
|
+
|
|
931
|
+
def test_history_length_matches_epochs(self):
|
|
932
|
+
proj = useml.focus("test")
|
|
933
|
+
record = _train_with_history(proj, epochs=4)
|
|
934
|
+
h = record.history
|
|
935
|
+
assert len(h["train_loss"]) == 4
|
|
936
|
+
assert len(h["val_loss"]) == 4
|
|
937
|
+
|
|
938
|
+
def test_history_values_are_floats(self):
|
|
939
|
+
proj = useml.focus("test")
|
|
940
|
+
record = _train_with_history(proj, epochs=2)
|
|
941
|
+
h = record.history
|
|
942
|
+
assert all(isinstance(v, float) for v in h["train_loss"])
|
|
943
|
+
|
|
944
|
+
def test_history_missing_returns_empty_dict(self):
|
|
945
|
+
"""Snapshots created without history.yaml return {}."""
|
|
946
|
+
proj = useml.focus("test")
|
|
947
|
+
record = _train_with_history(proj, epochs=2)
|
|
948
|
+
(record._snapshot.path / "history.yaml").unlink()
|
|
949
|
+
assert record.history == {}
|
|
950
|
+
|
|
951
|
+
def test_history_df_shape(self):
|
|
952
|
+
pytest.importorskip("pandas")
|
|
953
|
+
proj = useml.focus("test")
|
|
954
|
+
record = _train_with_history(proj, epochs=3)
|
|
955
|
+
df = record.history_df()
|
|
956
|
+
assert df.shape[0] == 3 # 3 epochs
|
|
957
|
+
assert "train_loss" in df.columns
|
|
958
|
+
assert df.index[0] == 1 # 1-based
|
|
959
|
+
|
|
960
|
+
def test_history_df_no_pandas_raises(self, monkeypatch):
|
|
961
|
+
import builtins
|
|
962
|
+
real_import = builtins.__import__
|
|
963
|
+
|
|
964
|
+
def mock_import(name, *args, **kwargs):
|
|
965
|
+
if name == "pandas":
|
|
966
|
+
raise ImportError("mocked")
|
|
967
|
+
return real_import(name, *args, **kwargs)
|
|
968
|
+
|
|
969
|
+
proj = useml.focus("test")
|
|
970
|
+
record = _train_with_history(proj, epochs=2)
|
|
971
|
+
monkeypatch.setattr(builtins, "__import__", mock_import)
|
|
972
|
+
with pytest.raises(ImportError, match="pandas"):
|
|
973
|
+
record.history_df()
|
|
974
|
+
|
|
975
|
+
def test_plot_returns_fig_ax(self):
|
|
976
|
+
pytest.importorskip("matplotlib")
|
|
977
|
+
import matplotlib
|
|
978
|
+
matplotlib.use("Agg")
|
|
979
|
+
proj = useml.focus("test")
|
|
980
|
+
record = _train_with_history(proj, epochs=3)
|
|
981
|
+
fig, ax = record.plot()
|
|
982
|
+
assert fig is not None
|
|
983
|
+
assert ax is not None
|
|
984
|
+
import matplotlib.pyplot as plt
|
|
985
|
+
plt.close("all")
|
|
986
|
+
|
|
987
|
+
def test_plot_specific_metric(self):
|
|
988
|
+
pytest.importorskip("matplotlib")
|
|
989
|
+
import matplotlib
|
|
990
|
+
matplotlib.use("Agg")
|
|
991
|
+
proj = useml.focus("test")
|
|
992
|
+
record = _train_with_history(proj, epochs=3)
|
|
993
|
+
fig, ax = record.plot("val_loss")
|
|
994
|
+
lines = ax.get_lines()
|
|
995
|
+
assert len(lines) == 1
|
|
996
|
+
import matplotlib.pyplot as plt
|
|
997
|
+
plt.close("all")
|
|
998
|
+
|
|
999
|
+
def test_plot_no_history_raises(self):
|
|
1000
|
+
pytest.importorskip("matplotlib")
|
|
1001
|
+
import matplotlib
|
|
1002
|
+
matplotlib.use("Agg")
|
|
1003
|
+
proj = useml.focus("test")
|
|
1004
|
+
record = _train_with_history(proj, epochs=2)
|
|
1005
|
+
(record._snapshot.path / "history.yaml").unlink()
|
|
1006
|
+
with pytest.raises(ValueError, match="No history"):
|
|
1007
|
+
record.plot()
|
|
1008
|
+
|
|
1009
|
+
def test_plot_unknown_metric_raises(self):
|
|
1010
|
+
pytest.importorskip("matplotlib")
|
|
1011
|
+
import matplotlib
|
|
1012
|
+
matplotlib.use("Agg")
|
|
1013
|
+
proj = useml.focus("test")
|
|
1014
|
+
record = _train_with_history(proj, epochs=2)
|
|
1015
|
+
with pytest.raises(ValueError, match="No matching metrics"):
|
|
1016
|
+
record.plot("nonexistent_metric")
|
|
1017
|
+
|
|
1018
|
+
def test_runs_view_plot(self):
|
|
1019
|
+
pytest.importorskip("matplotlib")
|
|
1020
|
+
import matplotlib
|
|
1021
|
+
matplotlib.use("Agg")
|
|
1022
|
+
proj = useml.focus("test")
|
|
1023
|
+
_train_with_history(proj, epochs=2)
|
|
1024
|
+
_train_with_history(proj, epochs=2)
|
|
1025
|
+
fig, ax = proj.runs.plot("val_loss")
|
|
1026
|
+
assert len(ax.get_lines()) == 2
|
|
1027
|
+
import matplotlib.pyplot as plt
|
|
1028
|
+
plt.close("all")
|
|
1029
|
+
|
|
1030
|
+
def test_runs_view_plot_top(self):
|
|
1031
|
+
pytest.importorskip("matplotlib")
|
|
1032
|
+
import matplotlib
|
|
1033
|
+
matplotlib.use("Agg")
|
|
1034
|
+
proj = useml.focus("test")
|
|
1035
|
+
for _ in range(4):
|
|
1036
|
+
_train_with_history(proj, epochs=2)
|
|
1037
|
+
fig, ax = proj.runs.plot("val_loss", top=2)
|
|
1038
|
+
assert len(ax.get_lines()) == 2
|
|
1039
|
+
import matplotlib.pyplot as plt
|
|
1040
|
+
plt.close("all")
|
|
1041
|
+
|
|
1042
|
+
class TestRunLoadModes:
|
|
1043
|
+
"""state_dict / instantiate / load — three ways to restore a model."""
|
|
1044
|
+
|
|
1045
|
+
def _make_run(self, proj):
|
|
1046
|
+
ds = _clf_ds()
|
|
1047
|
+
loader = DataLoader(ds, batch_size=32)
|
|
1048
|
+
cfg = _cfg(epochs=2, checkpoint_strategy="last")
|
|
1049
|
+
run = proj.runs.new(TinyClf, config=cfg)
|
|
1050
|
+
run.train(loader, loader)
|
|
1051
|
+
return proj.runs.latest
|
|
1052
|
+
|
|
1053
|
+
def test_state_dict_returns_ordereddict(self):
|
|
1054
|
+
proj = useml.focus("test")
|
|
1055
|
+
record = self._make_run(proj)
|
|
1056
|
+
sd = record.state_dict()
|
|
1057
|
+
assert isinstance(sd, dict)
|
|
1058
|
+
assert any("fc" in k for k in sd)
|
|
1059
|
+
|
|
1060
|
+
def test_state_dict_explicit_name(self):
|
|
1061
|
+
proj = useml.focus("test")
|
|
1062
|
+
record = self._make_run(proj)
|
|
1063
|
+
sd = record.state_dict("tinyclf")
|
|
1064
|
+
assert isinstance(sd, dict)
|
|
1065
|
+
|
|
1066
|
+
def test_state_dict_wrong_name_raises(self):
|
|
1067
|
+
proj = useml.focus("test")
|
|
1068
|
+
record = self._make_run(proj)
|
|
1069
|
+
with pytest.raises(KeyError):
|
|
1070
|
+
record.state_dict("nonexistent")
|
|
1071
|
+
|
|
1072
|
+
def test_instantiate_returns_correct_type(self):
|
|
1073
|
+
proj = useml.focus("test")
|
|
1074
|
+
record = self._make_run(proj)
|
|
1075
|
+
model = record.instantiate()
|
|
1076
|
+
assert isinstance(model, TinyClf)
|
|
1077
|
+
|
|
1078
|
+
def test_instantiate_has_no_trained_weights(self):
|
|
1079
|
+
"""instantiate() gives a fresh (random) model, not the trained one."""
|
|
1080
|
+
proj = useml.focus("test")
|
|
1081
|
+
record = self._make_run(proj)
|
|
1082
|
+
fresh = record.instantiate()
|
|
1083
|
+
trained = record.load()
|
|
1084
|
+
fresh_params = list(fresh.parameters())
|
|
1085
|
+
trained_params = list(trained.parameters())
|
|
1086
|
+
# They should not all be equal (weights differ)
|
|
1087
|
+
any_different = any(
|
|
1088
|
+
not torch.equal(f, t) for f, t in zip(fresh_params, trained_params)
|
|
1089
|
+
)
|
|
1090
|
+
assert any_different
|
|
1091
|
+
|
|
1092
|
+
def test_instantiate_model_with_missing_required_arg_raises(self):
|
|
1093
|
+
"""Model with required object arg not in config → clear error message."""
|
|
1094
|
+
proj = useml.focus("test")
|
|
1095
|
+
ds = _clf_ds()
|
|
1096
|
+
loader = DataLoader(ds, batch_size=32)
|
|
1097
|
+
cfg = _cfg(epochs=1, checkpoint_strategy="last")
|
|
1098
|
+
model = _ModelNeedsEncoder(encoder=nn.Linear(4, 4))
|
|
1099
|
+
run = proj.runs.new(model, config=cfg)
|
|
1100
|
+
run.train(loader, loader)
|
|
1101
|
+
record = proj.runs.latest
|
|
1102
|
+
|
|
1103
|
+
with pytest.raises(ModelInstantiationError, match="encoder"):
|
|
1104
|
+
record.instantiate("_modelneedsencoder")
|
|
1105
|
+
|
|
1106
|
+
def test_load_with_provided_instance(self):
|
|
1107
|
+
proj = useml.focus("test")
|
|
1108
|
+
record = self._make_run(proj)
|
|
1109
|
+
model = record.load(model=TinyClf())
|
|
1110
|
+
assert isinstance(model, TinyClf)
|
|
1111
|
+
|
|
1112
|
+
def test_load_weights_via_state_dict_then_manual(self):
|
|
1113
|
+
proj = useml.focus("test")
|
|
1114
|
+
record = self._make_run(proj)
|
|
1115
|
+
sd = record.state_dict()
|
|
1116
|
+
model = TinyClf()
|
|
1117
|
+
model.load_state_dict(sd)
|
|
1118
|
+
# Loaded weights should match record.load()
|
|
1119
|
+
direct = record.load(model=TinyClf())
|
|
1120
|
+
for p1, p2 in zip(model.parameters(), direct.parameters()):
|
|
1121
|
+
assert torch.equal(p1, p2)
|
|
1122
|
+
|
|
1123
|
+
def test_error_message_suggests_all_three_alternatives(self):
|
|
1124
|
+
"""Error from missing required arg mentions state_dict, load(model=), and Config."""
|
|
1125
|
+
proj = useml.focus("test")
|
|
1126
|
+
ds = _clf_ds()
|
|
1127
|
+
loader = DataLoader(ds, batch_size=32)
|
|
1128
|
+
cfg = _cfg(epochs=1, checkpoint_strategy="last")
|
|
1129
|
+
run = proj.runs.new(_ModelNeedsEncoder(nn.Linear(4, 4)), config=cfg)
|
|
1130
|
+
run.train(loader, loader)
|
|
1131
|
+
record = proj.runs.latest
|
|
1132
|
+
|
|
1133
|
+
with pytest.raises(ModelInstantiationError) as exc_info:
|
|
1134
|
+
record.load("_modelneedsencoder")
|
|
1135
|
+
msg = str(exc_info.value)
|
|
1136
|
+
assert "encoder" in msg
|
|
1137
|
+
assert "state_dict" in msg
|
|
1138
|
+
assert "record.load" in msg
|
|
1139
|
+
|
|
1140
|
+
def test_pattern_b_history_written(self):
|
|
1141
|
+
"""Pattern B with epoch_end also writes history."""
|
|
1142
|
+
proj = useml.focus("test")
|
|
1143
|
+
ds = _clf_ds()
|
|
1144
|
+
loader = DataLoader(ds, batch_size=32)
|
|
1145
|
+
cfg = _cfg(epochs=3, checkpoint_strategy="last")
|
|
1146
|
+
run = proj.runs.new(config=cfg)
|
|
1147
|
+
run.register("tinyclf", TinyClf())
|
|
1148
|
+
for epoch in run.epochs(cfg.epochs):
|
|
1149
|
+
for batch in loader:
|
|
1150
|
+
x, y = batch
|
|
1151
|
+
loss = F.cross_entropy(run._models["tinyclf"](x), y)
|
|
1152
|
+
run._optimizers["tinyclf"].zero_grad()
|
|
1153
|
+
loss.backward()
|
|
1154
|
+
run._optimizers["tinyclf"].step()
|
|
1155
|
+
run.update(n=len(y), loss=loss.item())
|
|
1156
|
+
run.update_val(n=1, loss=0.5)
|
|
1157
|
+
run.epoch_end(epoch)
|
|
1158
|
+
h = proj.runs.latest.history
|
|
1159
|
+
assert "train_loss" in h
|
|
1160
|
+
assert len(h["train_loss"]) == 3
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
class TestKeyboardInterrupt:
|
|
1164
|
+
"""Interrupting a run must not crash — model state and history are preserved."""
|
|
1165
|
+
|
|
1166
|
+
def test_pattern_a_interrupt_saves_model(self):
|
|
1167
|
+
"""KeyboardInterrupt during Pattern A training → emergency checkpoint created."""
|
|
1168
|
+
proj = useml.focus("test")
|
|
1169
|
+
ds = _clf_ds()
|
|
1170
|
+
loader = DataLoader(ds, batch_size=32)
|
|
1171
|
+
|
|
1172
|
+
call_count = [0]
|
|
1173
|
+
original_epoch_end = None
|
|
1174
|
+
|
|
1175
|
+
cfg = _cfg(epochs=10, checkpoint_strategy="last")
|
|
1176
|
+
run = proj.runs.new(TinyClf, config=cfg)
|
|
1177
|
+
|
|
1178
|
+
# Simulate interrupt after epoch 2 by monkey-patching epoch_end
|
|
1179
|
+
real_epoch_end = run.epoch_end
|
|
1180
|
+
def patched_epoch_end(epoch, *args, **kwargs):
|
|
1181
|
+
call_count[0] += 1
|
|
1182
|
+
real_epoch_end(epoch, *args, **kwargs)
|
|
1183
|
+
if call_count[0] >= 2:
|
|
1184
|
+
raise KeyboardInterrupt()
|
|
1185
|
+
run.epoch_end = patched_epoch_end
|
|
1186
|
+
|
|
1187
|
+
history = run.train(loader, loader)
|
|
1188
|
+
|
|
1189
|
+
# Must return history (no exception escapes)
|
|
1190
|
+
assert isinstance(history, dict)
|
|
1191
|
+
assert "train_loss" in history
|
|
1192
|
+
assert len(history["train_loss"]) == 2
|
|
1193
|
+
|
|
1194
|
+
# Emergency checkpoint must have been created
|
|
1195
|
+
assert len(proj.runs) >= 1
|
|
1196
|
+
|
|
1197
|
+
def test_pattern_a_interrupt_returns_partial_history(self):
|
|
1198
|
+
"""run.train() returns the partial history accumulated before interrupt."""
|
|
1199
|
+
proj = useml.focus("test")
|
|
1200
|
+
ds = _clf_ds()
|
|
1201
|
+
loader = DataLoader(ds, batch_size=32)
|
|
1202
|
+
cfg = _cfg(epochs=5, checkpoint_strategy="last")
|
|
1203
|
+
run = proj.runs.new(TinyClf, config=cfg)
|
|
1204
|
+
|
|
1205
|
+
real_ee = run.epoch_end
|
|
1206
|
+
interrupted_at = [0]
|
|
1207
|
+
def patched(epoch, *args, **kwargs):
|
|
1208
|
+
real_ee(epoch, *args, **kwargs)
|
|
1209
|
+
interrupted_at[0] = epoch
|
|
1210
|
+
if epoch == 3:
|
|
1211
|
+
raise KeyboardInterrupt()
|
|
1212
|
+
run.epoch_end = patched
|
|
1213
|
+
|
|
1214
|
+
history = run.train(loader, loader)
|
|
1215
|
+
assert len(history["train_loss"]) == interrupted_at[0]
|
|
1216
|
+
|
|
1217
|
+
def test_pattern_b_interrupt_history_on_object(self):
|
|
1218
|
+
"""KeyboardInterrupt inside a Pattern B loop propagates cleanly.
|
|
1219
|
+
|
|
1220
|
+
In Pattern B the user controls the loop — we can't intercept a
|
|
1221
|
+
KeyboardInterrupt raised in the caller's body. We guarantee:
|
|
1222
|
+
- the exception propagates normally (no kernel crash)
|
|
1223
|
+
- the accumulated history is accessible on run.history
|
|
1224
|
+
"""
|
|
1225
|
+
proj = useml.focus("test")
|
|
1226
|
+
ds = _clf_ds()
|
|
1227
|
+
loader = DataLoader(ds, batch_size=32)
|
|
1228
|
+
cfg = _cfg(epochs=5, checkpoint_strategy="last")
|
|
1229
|
+
run = proj.runs.new(config=cfg)
|
|
1230
|
+
run.register("tinyclf", TinyClf())
|
|
1231
|
+
|
|
1232
|
+
with pytest.raises(KeyboardInterrupt):
|
|
1233
|
+
for epoch in run.epochs(cfg.epochs):
|
|
1234
|
+
for batch in loader:
|
|
1235
|
+
x, y = batch
|
|
1236
|
+
loss = F.cross_entropy(run._models["tinyclf"](x), y)
|
|
1237
|
+
run._optimizers["tinyclf"].zero_grad()
|
|
1238
|
+
loss.backward()
|
|
1239
|
+
run._optimizers["tinyclf"].step()
|
|
1240
|
+
run.update(n=len(y), loss=loss.item())
|
|
1241
|
+
run.update_val(n=1, loss=0.5)
|
|
1242
|
+
run.epoch_end(epoch)
|
|
1243
|
+
if epoch == 2:
|
|
1244
|
+
raise KeyboardInterrupt()
|
|
1245
|
+
|
|
1246
|
+
# History is available on the run object even without a persisted snapshot
|
|
1247
|
+
h = run.history
|
|
1248
|
+
assert "train_loss" in h
|
|
1249
|
+
assert len(h["train_loss"]) == 2
|
|
@@ -9,7 +9,7 @@ except PackageNotFoundError:
|
|
|
9
9
|
|
|
10
10
|
from .session.manager import _session
|
|
11
11
|
from .vault.project import Project
|
|
12
|
-
from .errors import ProjectAlreadyExistsError, ModelInstantiationError, UseMlError
|
|
12
|
+
from .errors import ProjectAlreadyExistsError, ModelInstantiationError, UseMlError, NotConnectedError
|
|
13
13
|
from .template import Config, Loss, Model
|
|
14
14
|
from .dataset import DataBundle
|
|
15
15
|
|
|
@@ -25,6 +25,14 @@ def init(vault_path: str = "vault") -> None:
|
|
|
25
25
|
_session.connect(vault_path)
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
def _require_vault(fn_name: str) -> None:
|
|
29
|
+
if _session.vault is None:
|
|
30
|
+
raise NotConnectedError(
|
|
31
|
+
f"useml.{fn_name}() called before useml.init(). "
|
|
32
|
+
f"Run useml.init('path/to/vault') first."
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
28
36
|
def new(project_name: str, force: bool = False) -> Project:
|
|
29
37
|
"""Create a new project in the vault and return it.
|
|
30
38
|
|
|
@@ -37,8 +45,10 @@ def new(project_name: str, force: bool = False) -> Project:
|
|
|
37
45
|
the entry point for ``project.runs``, ``project.models``, etc.
|
|
38
46
|
|
|
39
47
|
Raises:
|
|
48
|
+
NotConnectedError: If :func:`init` has not been called.
|
|
40
49
|
ProjectAlreadyExistsError: If a project with that name already exists.
|
|
41
50
|
"""
|
|
51
|
+
_require_vault("new")
|
|
42
52
|
if _session.vault.exists(project_name):
|
|
43
53
|
raise ProjectAlreadyExistsError(
|
|
44
54
|
f"Project '{project_name}' already exists. "
|
|
@@ -57,7 +67,11 @@ def focus(project_name: str, force: bool = False) -> Project:
|
|
|
57
67
|
|
|
58
68
|
Returns:
|
|
59
69
|
The focused :class:`~useml.vault.project.Project` instance.
|
|
70
|
+
|
|
71
|
+
Raises:
|
|
72
|
+
NotConnectedError: If :func:`init` has not been called.
|
|
60
73
|
"""
|
|
74
|
+
_require_vault("focus")
|
|
61
75
|
_session.set_focus(project_name, force=force)
|
|
62
76
|
return _session._project
|
|
63
77
|
|
|
@@ -75,5 +89,6 @@ __all__ = [
|
|
|
75
89
|
# Errors
|
|
76
90
|
"ProjectAlreadyExistsError",
|
|
77
91
|
"ModelInstantiationError",
|
|
92
|
+
"NotConnectedError",
|
|
78
93
|
"UseMlError",
|
|
79
94
|
]
|
|
@@ -513,45 +513,52 @@ class Trainer:
|
|
|
513
513
|
|
|
514
514
|
primary = self.model
|
|
515
515
|
opt = self.optimizer
|
|
516
|
+
self._was_interrupted = False
|
|
516
517
|
|
|
517
|
-
|
|
518
|
-
|
|
518
|
+
try:
|
|
519
|
+
for epoch in self.epochs(self.config.epochs):
|
|
520
|
+
t0 = time.time()
|
|
519
521
|
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
loss = self.step(batch)
|
|
525
|
-
loss.backward()
|
|
526
|
-
opt.step()
|
|
527
|
-
if self.scheduler is not None and self.scheduler_per_batch:
|
|
528
|
-
self.scheduler.step()
|
|
529
|
-
bs = _batch_size(batch)
|
|
530
|
-
batch_m: Dict[str, float] = {"loss": loss.item()}
|
|
531
|
-
if self.metrics:
|
|
532
|
-
with torch.no_grad():
|
|
533
|
-
for name, fn in self.metrics.items():
|
|
534
|
-
batch_m[name] = float(fn(primary, batch, self.config.device))
|
|
535
|
-
self.update(n=bs, **batch_m)
|
|
536
|
-
|
|
537
|
-
# ── val phase ──────────────────────────────────────────────
|
|
538
|
-
primary.eval()
|
|
539
|
-
with torch.no_grad():
|
|
540
|
-
for batch in val_loader:
|
|
522
|
+
# ── train phase ────────────────────────────────────────────
|
|
523
|
+
primary.train()
|
|
524
|
+
for batch in train_loader:
|
|
525
|
+
opt.zero_grad()
|
|
541
526
|
loss = self.step(batch)
|
|
527
|
+
loss.backward()
|
|
528
|
+
opt.step()
|
|
529
|
+
if self.scheduler is not None and self.scheduler_per_batch:
|
|
530
|
+
self.scheduler.step()
|
|
542
531
|
bs = _batch_size(batch)
|
|
543
|
-
|
|
532
|
+
batch_m: Dict[str, float] = {"loss": loss.item()}
|
|
544
533
|
if self.metrics:
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
534
|
+
with torch.no_grad():
|
|
535
|
+
for name, fn in self.metrics.items():
|
|
536
|
+
batch_m[name] = float(fn(primary, batch, self.config.device))
|
|
537
|
+
self.update(n=bs, **batch_m)
|
|
538
|
+
|
|
539
|
+
# ── val phase ──────────────────────────────────────────────
|
|
540
|
+
primary.eval()
|
|
541
|
+
with torch.no_grad():
|
|
542
|
+
for batch in val_loader:
|
|
543
|
+
loss = self.step(batch)
|
|
544
|
+
bs = _batch_size(batch)
|
|
545
|
+
batch_v: Dict[str, float] = {"loss": loss.item()}
|
|
546
|
+
if self.metrics:
|
|
547
|
+
for name, fn in self.metrics.items():
|
|
548
|
+
batch_v[name] = float(fn(primary, batch, self.config.device))
|
|
549
|
+
self.update_val(n=bs, **batch_v)
|
|
550
|
+
|
|
551
|
+
self.epoch_end(epoch, elapsed=time.time() - t0)
|
|
552
|
+
|
|
553
|
+
except KeyboardInterrupt:
|
|
554
|
+
self._was_interrupted = True
|
|
555
|
+
print(f"\n ⏸ Training interrupted at epoch {self._epoch}.")
|
|
550
556
|
|
|
551
557
|
best_val = min(self._history.get("val_loss", [float("inf")]))
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
558
|
+
if not self._was_interrupted:
|
|
559
|
+
print(f"{'=' * 55}")
|
|
560
|
+
print(f" Best val_loss: {best_val:.4f}")
|
|
561
|
+
print(f"{'=' * 55}\n")
|
|
555
562
|
|
|
556
563
|
return self.history
|
|
557
564
|
|
|
@@ -20,8 +20,11 @@ def _is_local_file(path: str, project_root: str) -> bool:
|
|
|
20
20
|
|
|
21
21
|
def _resolve_import(module_name: str, project_root: str) -> str | None:
|
|
22
22
|
try:
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
import importlib.util
|
|
24
|
+
spec = importlib.util.find_spec(module_name)
|
|
25
|
+
if spec is None or spec.origin is None:
|
|
26
|
+
return None
|
|
27
|
+
return spec.origin
|
|
25
28
|
except Exception:
|
|
26
29
|
return None
|
|
27
30
|
|
|
@@ -64,36 +64,210 @@ class RunRecord:
|
|
|
64
64
|
"""Names of the model components saved in this run."""
|
|
65
65
|
return list(self._snapshot.components.keys())
|
|
66
66
|
|
|
67
|
-
|
|
67
|
+
@property
|
|
68
|
+
def history(self) -> dict:
|
|
69
|
+
"""Per-epoch metric history: ``{"train_loss": [...], "val_loss": [...]}``.
|
|
70
|
+
|
|
71
|
+
Returns an empty dict if no history was recorded (old snapshots).
|
|
72
|
+
"""
|
|
73
|
+
h_path = self._snapshot.path / "history.yaml"
|
|
74
|
+
if not h_path.exists():
|
|
75
|
+
return {}
|
|
76
|
+
import yaml
|
|
77
|
+
with open(h_path, encoding="utf-8") as f:
|
|
78
|
+
return yaml.safe_load(f) or {}
|
|
79
|
+
|
|
80
|
+
def history_df(self):
|
|
81
|
+
"""Return :attr:`history` as a pandas ``DataFrame`` (requires pandas).
|
|
82
|
+
|
|
83
|
+
Each column is a metric; the index is 1-based epoch numbers.
|
|
84
|
+
|
|
85
|
+
Raises:
|
|
86
|
+
ImportError: If pandas is not installed.
|
|
87
|
+
"""
|
|
88
|
+
try:
|
|
89
|
+
import pandas as pd
|
|
90
|
+
except ImportError:
|
|
91
|
+
raise ImportError(
|
|
92
|
+
"pandas is required for history_df(). Run: pip install pandas"
|
|
93
|
+
)
|
|
94
|
+
h = self.history
|
|
95
|
+
if not h:
|
|
96
|
+
return pd.DataFrame()
|
|
97
|
+
df = pd.DataFrame(h)
|
|
98
|
+
df.index = range(1, len(df) + 1)
|
|
99
|
+
df.index.name = "epoch"
|
|
100
|
+
return df
|
|
101
|
+
|
|
102
|
+
def plot(self, metrics=None, *, ax=None, label_prefix: str = "", title: str = None):
|
|
103
|
+
"""Plot training curves for this run.
|
|
68
104
|
|
|
69
|
-
|
|
70
|
-
|
|
105
|
+
Train and val variants of the same base metric share a color;
|
|
106
|
+
train uses a solid line and val uses a dashed line.
|
|
71
107
|
|
|
72
108
|
Args:
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
109
|
+
metrics: Metric name(s) to plot. ``None`` plots all available.
|
|
110
|
+
ax: Existing :class:`matplotlib.axes.Axes` to draw on.
|
|
111
|
+
When ``None`` (default), a new figure is created.
|
|
112
|
+
label_prefix: String prepended to each legend label (useful when
|
|
113
|
+
combining several runs on the same axes).
|
|
114
|
+
title: Figure title. Defaults to the run id.
|
|
115
|
+
|
|
116
|
+
Returns:
|
|
117
|
+
``(fig, ax)`` where *fig* is ``None`` when *ax* was supplied.
|
|
77
118
|
|
|
78
119
|
Raises:
|
|
79
|
-
|
|
80
|
-
|
|
120
|
+
ImportError: If matplotlib is not installed.
|
|
121
|
+
ValueError: If no matching metrics are found.
|
|
81
122
|
"""
|
|
123
|
+
try:
|
|
124
|
+
import matplotlib.pyplot as plt
|
|
125
|
+
except ImportError:
|
|
126
|
+
raise ImportError(
|
|
127
|
+
"matplotlib is required for plot(). Run: pip install matplotlib"
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
h = self.history
|
|
131
|
+
if not h:
|
|
132
|
+
raise ValueError(f"No history data found for run '{self.id}'.")
|
|
133
|
+
|
|
134
|
+
if metrics is None:
|
|
135
|
+
keys = list(h.keys())
|
|
136
|
+
elif isinstance(metrics, str):
|
|
137
|
+
keys = [m for m in [metrics] if m in h]
|
|
138
|
+
else:
|
|
139
|
+
keys = [m for m in metrics if m in h]
|
|
140
|
+
|
|
141
|
+
if not keys:
|
|
142
|
+
raise ValueError(
|
|
143
|
+
f"No matching metrics in history. Available: {list(h.keys())}"
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
created_fig = ax is None
|
|
147
|
+
if created_fig:
|
|
148
|
+
fig, ax = plt.subplots(figsize=(9, 4))
|
|
149
|
+
else:
|
|
150
|
+
fig = None
|
|
151
|
+
|
|
152
|
+
color_cycle = plt.rcParams["axes.prop_cycle"].by_key()["color"]
|
|
153
|
+
base_to_color: dict = {}
|
|
154
|
+
color_idx = 0
|
|
155
|
+
|
|
156
|
+
for k in keys:
|
|
157
|
+
if k.startswith("train_"):
|
|
158
|
+
base = k[6:]
|
|
159
|
+
elif k.startswith("val_"):
|
|
160
|
+
base = k[4:]
|
|
161
|
+
else:
|
|
162
|
+
base = k
|
|
163
|
+
if base not in base_to_color:
|
|
164
|
+
base_to_color[base] = color_cycle[color_idx % len(color_cycle)]
|
|
165
|
+
color_idx += 1
|
|
166
|
+
|
|
167
|
+
vals = h[k]
|
|
168
|
+
epochs = list(range(1, len(vals) + 1))
|
|
169
|
+
linestyle = "--" if k.startswith("val_") else "-"
|
|
170
|
+
label = f"{label_prefix}{k}" if label_prefix else k
|
|
171
|
+
ax.plot(epochs, vals, linestyle=linestyle,
|
|
172
|
+
color=base_to_color[base], label=label)
|
|
173
|
+
|
|
174
|
+
ax.set_xlabel("Epoch")
|
|
175
|
+
ax.legend()
|
|
176
|
+
ax.grid(True, alpha=0.3)
|
|
177
|
+
|
|
178
|
+
if created_fig:
|
|
179
|
+
ax.set_title(title or self.id[:30])
|
|
180
|
+
fig.tight_layout()
|
|
181
|
+
|
|
182
|
+
return fig, ax
|
|
183
|
+
|
|
184
|
+
# ── Actions ────────────────────────────────────────────────────────────
|
|
185
|
+
|
|
186
|
+
def _resolve_name(self, name: Optional[str]) -> str:
|
|
187
|
+
"""Resolve component name, defaulting to the only one when there is one."""
|
|
82
188
|
names = self.models
|
|
83
189
|
if name is None:
|
|
84
|
-
if
|
|
190
|
+
if not names:
|
|
85
191
|
raise KeyError(f"No models saved in run '{self.id}'.")
|
|
86
192
|
if len(names) > 1:
|
|
87
193
|
raise ValueError(
|
|
88
194
|
f"Run '{self.id}' has multiple models ({names}); "
|
|
89
|
-
f"specify which one
|
|
195
|
+
f"specify which one."
|
|
90
196
|
)
|
|
91
|
-
|
|
197
|
+
return names[0]
|
|
92
198
|
if name not in names:
|
|
93
199
|
raise KeyError(
|
|
94
200
|
f"Model '{name}' not in run '{self.id}'. Available: {names}"
|
|
95
201
|
)
|
|
202
|
+
return name
|
|
203
|
+
|
|
204
|
+
def state_dict(self, name: Optional[str] = None) -> dict:
|
|
205
|
+
"""Return the raw weights dict for a component — no model needed.
|
|
206
|
+
|
|
207
|
+
Useful when the model requires complex constructor arguments that
|
|
208
|
+
cannot be auto-inferred from the snapshot config.
|
|
209
|
+
|
|
210
|
+
Args:
|
|
211
|
+
name: Component name. Omit when the run has only one model.
|
|
212
|
+
|
|
213
|
+
Returns:
|
|
214
|
+
``OrderedDict`` as returned by ``torch.load``.
|
|
215
|
+
"""
|
|
216
|
+
name = self._resolve_name(name)
|
|
217
|
+
weights_path = self._snapshot.path / "weights" / f"{name}.pth"
|
|
218
|
+
if not weights_path.exists():
|
|
219
|
+
raise FileNotFoundError(f"Weights file missing: {weights_path}")
|
|
220
|
+
return torch.load(weights_path, map_location="cpu", weights_only=True)
|
|
221
|
+
|
|
222
|
+
def instantiate(self, name: Optional[str] = None) -> nn.Module:
|
|
223
|
+
"""Instantiate the model class from the archived config — without loading weights.
|
|
224
|
+
|
|
225
|
+
Useful to inspect the architecture or create a fresh copy at the
|
|
226
|
+
same hyperparameters without restoring the trained weights.
|
|
227
|
+
|
|
228
|
+
Args:
|
|
229
|
+
name: Component name. Omit when the run has only one model.
|
|
230
|
+
|
|
231
|
+
Returns:
|
|
232
|
+
A newly constructed (untrained) model instance.
|
|
233
|
+
|
|
234
|
+
Raises:
|
|
235
|
+
ModelInstantiationError: If the class cannot be imported or its
|
|
236
|
+
constructor requires arguments not stored in the config YAML.
|
|
237
|
+
In that case pass a pre-built instance to :meth:`load` instead.
|
|
238
|
+
"""
|
|
239
|
+
name = self._resolve_name(name)
|
|
240
|
+
return self._try_instantiate(name)
|
|
96
241
|
|
|
242
|
+
def load(self, name: Optional[str] = None, model: Optional[nn.Module] = None) -> nn.Module:
|
|
243
|
+
"""Load trained weights into a model.
|
|
244
|
+
|
|
245
|
+
Three usage patterns:
|
|
246
|
+
|
|
247
|
+
.. code-block:: python
|
|
248
|
+
|
|
249
|
+
# 1. Auto-instantiate + load weights (works when all constructor
|
|
250
|
+
# args are stored as Config custom fields)
|
|
251
|
+
model = record.load("encoder")
|
|
252
|
+
|
|
253
|
+
# 2. Provide your own instance — always works
|
|
254
|
+
model = record.load("encoder", model=Encoder(hidden=256))
|
|
255
|
+
|
|
256
|
+
# 3. Just the weights dict (no model class needed)
|
|
257
|
+
sd = record.state_dict("encoder")
|
|
258
|
+
model.load_state_dict(sd)
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
name: Component name. Omit when the run has only one model.
|
|
262
|
+
model: Pre-instantiated model. When ``None`` useml tries to
|
|
263
|
+
import and construct the class from the archived config YAML.
|
|
264
|
+
|
|
265
|
+
Raises:
|
|
266
|
+
ModelInstantiationError: When ``model`` is ``None`` and the class
|
|
267
|
+
cannot be auto-instantiated (e.g. requires object-type args).
|
|
268
|
+
Solution: pass ``model=YourClass(<args>)`` explicitly.
|
|
269
|
+
"""
|
|
270
|
+
name = self._resolve_name(name)
|
|
97
271
|
weights_path = self._snapshot.path / "weights" / f"{name}.pth"
|
|
98
272
|
if not weights_path.exists():
|
|
99
273
|
raise FileNotFoundError(f"Weights file missing: {weights_path}")
|
|
@@ -106,14 +280,21 @@ class RunRecord:
|
|
|
106
280
|
|
|
107
281
|
# ── Private ────────────────────────────────────────────────────────────
|
|
108
282
|
|
|
109
|
-
def _try_instantiate(self, name: str):
|
|
110
|
-
"""
|
|
283
|
+
def _try_instantiate(self, name: str) -> nn.Module:
|
|
284
|
+
"""Import the model class and construct it from the archived config.
|
|
285
|
+
|
|
286
|
+
Raises ``ModelInstantiationError`` with an actionable message when
|
|
287
|
+
the class cannot be imported or has required constructor arguments
|
|
288
|
+
that were not stored in the config YAML.
|
|
289
|
+
"""
|
|
111
290
|
import importlib
|
|
112
291
|
import inspect
|
|
113
292
|
|
|
114
293
|
comp = self._snapshot.components.get(name, {})
|
|
115
294
|
module_path = comp.get("module_path", "")
|
|
116
295
|
class_name = comp.get("class_name", "")
|
|
296
|
+
|
|
297
|
+
# ── 1. Import the class ───────────────────────────────────────────
|
|
117
298
|
try:
|
|
118
299
|
if module_path == "__main__":
|
|
119
300
|
import __main__ as module
|
|
@@ -122,25 +303,53 @@ class RunRecord:
|
|
|
122
303
|
cls = getattr(module, class_name)
|
|
123
304
|
except Exception as exc:
|
|
124
305
|
raise ModelInstantiationError(
|
|
125
|
-
f"Cannot import {class_name} from '{module_path}'. "
|
|
126
|
-
f"Pass
|
|
306
|
+
f"Cannot import {class_name!r} from module '{module_path}'. "
|
|
307
|
+
f"Pass a pre-built instance: record.load(model={class_name}()). "
|
|
308
|
+
f"[{exc}]"
|
|
127
309
|
) from exc
|
|
128
310
|
|
|
311
|
+
# ── 2. Read config YAML ───────────────────────────────────────────
|
|
312
|
+
cfg_path = self._snapshot.path / "configs" / f"{name}.yaml"
|
|
313
|
+
cfg_dict: dict = {}
|
|
314
|
+
if cfg_path.exists():
|
|
315
|
+
import yaml
|
|
316
|
+
with open(cfg_path, encoding="utf-8") as f:
|
|
317
|
+
cfg_dict = yaml.safe_load(f) or {}
|
|
318
|
+
|
|
319
|
+
# ── 3. Identify which config keys match constructor params ─────────
|
|
320
|
+
sig = inspect.signature(cls.__init__)
|
|
321
|
+
all_params = list(sig.parameters.values())[1:] # skip 'self'
|
|
322
|
+
required = [
|
|
323
|
+
p.name for p in all_params
|
|
324
|
+
if p.default is inspect.Parameter.empty
|
|
325
|
+
and p.kind not in (
|
|
326
|
+
inspect.Parameter.VAR_POSITIONAL,
|
|
327
|
+
inspect.Parameter.VAR_KEYWORD,
|
|
328
|
+
)
|
|
329
|
+
]
|
|
330
|
+
matched_kwargs = {
|
|
331
|
+
p.name: cfg_dict[p.name]
|
|
332
|
+
for p in all_params
|
|
333
|
+
if p.name in cfg_dict
|
|
334
|
+
}
|
|
335
|
+
missing = [r for r in required if r not in matched_kwargs]
|
|
336
|
+
|
|
337
|
+
if missing:
|
|
338
|
+
raise ModelInstantiationError(
|
|
339
|
+
f"Cannot auto-instantiate '{class_name}': required argument(s) "
|
|
340
|
+
f"{missing} are not stored in the snapshot config.\n"
|
|
341
|
+
f" → Store them before training: Config(..., {missing[0]}=<value>)\n"
|
|
342
|
+
f" → Or provide an instance: record.load('{name}', model={class_name}(<args>))\n"
|
|
343
|
+
f" → Or load weights only: record.state_dict('{name}')"
|
|
344
|
+
)
|
|
345
|
+
|
|
129
346
|
try:
|
|
130
|
-
|
|
131
|
-
params = set(list(sig.parameters.keys())[1:])
|
|
132
|
-
cfg_path = self._snapshot.path / "configs" / f"{name}.yaml"
|
|
133
|
-
cfg_dict: dict = {}
|
|
134
|
-
if cfg_path.exists():
|
|
135
|
-
import yaml
|
|
136
|
-
with open(cfg_path, encoding="utf-8") as f:
|
|
137
|
-
cfg_dict = yaml.safe_load(f) or {}
|
|
138
|
-
kwargs = {k: v for k, v in cfg_dict.items() if k in params}
|
|
139
|
-
return cls(**kwargs) if kwargs else cls()
|
|
347
|
+
return cls(**matched_kwargs)
|
|
140
348
|
except TypeError as exc:
|
|
141
349
|
raise ModelInstantiationError(
|
|
142
|
-
f"Cannot instantiate '{class_name}'
|
|
143
|
-
f"
|
|
350
|
+
f"Cannot instantiate '{class_name}' even with config kwargs "
|
|
351
|
+
f"{list(matched_kwargs)}: {exc}.\n"
|
|
352
|
+
f" → Pass an instance: record.load('{name}', model={class_name}(<args>))"
|
|
144
353
|
) from exc
|
|
145
354
|
|
|
146
355
|
def __repr__(self) -> str:
|
|
@@ -253,6 +462,67 @@ class RunsView:
|
|
|
253
462
|
)
|
|
254
463
|
return "\n".join(lines)
|
|
255
464
|
|
|
465
|
+
def plot(
|
|
466
|
+
self,
|
|
467
|
+
metric: str,
|
|
468
|
+
*,
|
|
469
|
+
mode: str = "min",
|
|
470
|
+
top: Optional[int] = None,
|
|
471
|
+
title: Optional[str] = None,
|
|
472
|
+
):
|
|
473
|
+
"""Plot one metric across all (or top N) runs on a single figure.
|
|
474
|
+
|
|
475
|
+
Args:
|
|
476
|
+
metric: Metric to compare, e.g. ``"val_loss"`` or ``"train_accuracy"``.
|
|
477
|
+
mode: ``"min"`` or ``"max"`` — controls ranking when *top* is set.
|
|
478
|
+
top: If set, show only the top N runs (ranked by final metric value).
|
|
479
|
+
title: Figure title. Defaults to ``"<metric> across runs"``.
|
|
480
|
+
|
|
481
|
+
Returns:
|
|
482
|
+
``(fig, ax)`` tuple.
|
|
483
|
+
|
|
484
|
+
Raises:
|
|
485
|
+
ImportError: If matplotlib is not installed.
|
|
486
|
+
KeyError: If no run has history for *metric*.
|
|
487
|
+
"""
|
|
488
|
+
try:
|
|
489
|
+
import matplotlib.pyplot as plt
|
|
490
|
+
except ImportError:
|
|
491
|
+
raise ImportError(
|
|
492
|
+
"matplotlib is required for plot(). Run: pip install matplotlib"
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
candidates = []
|
|
496
|
+
for r in self:
|
|
497
|
+
h = r.history
|
|
498
|
+
if metric in h and h[metric]:
|
|
499
|
+
candidates.append((h[metric][-1], r, h[metric]))
|
|
500
|
+
|
|
501
|
+
if not candidates:
|
|
502
|
+
available = sorted({k for r in self for k in r.history})
|
|
503
|
+
raise KeyError(
|
|
504
|
+
f"No runs have history for metric '{metric}'. "
|
|
505
|
+
f"Available: {available}"
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
candidates.sort(key=lambda x: x[0], reverse=(mode == "max"))
|
|
509
|
+
if top is not None:
|
|
510
|
+
candidates = candidates[:top]
|
|
511
|
+
|
|
512
|
+
fig, ax = plt.subplots(figsize=(9, 4))
|
|
513
|
+
for final_val, r, vals in candidates:
|
|
514
|
+
epochs = list(range(1, len(vals) + 1))
|
|
515
|
+
label = f"{r.id[5:21]} (final {final_val:.4f})"
|
|
516
|
+
ax.plot(epochs, vals, label=label)
|
|
517
|
+
|
|
518
|
+
ax.set_xlabel("Epoch")
|
|
519
|
+
ax.set_ylabel(metric)
|
|
520
|
+
ax.set_title(title or f"{metric} across runs")
|
|
521
|
+
ax.legend(fontsize="small")
|
|
522
|
+
ax.grid(True, alpha=0.3)
|
|
523
|
+
fig.tight_layout()
|
|
524
|
+
return fig, ax
|
|
525
|
+
|
|
256
526
|
# ── Write side ─────────────────────────────────────────────────────────
|
|
257
527
|
|
|
258
528
|
def new(
|
|
@@ -419,7 +689,36 @@ class Run(Trainer):
|
|
|
419
689
|
else scheduler
|
|
420
690
|
)
|
|
421
691
|
|
|
422
|
-
|
|
692
|
+
result = self.run(train_loader, val_loader, resume_from=resume_from)
|
|
693
|
+
|
|
694
|
+
if getattr(self, "_was_interrupted", False) and self._epoch > 0:
|
|
695
|
+
monitor = self._last_val_metrics or self._last_train_metrics
|
|
696
|
+
if monitor:
|
|
697
|
+
print(" 💾 Saving current model state...")
|
|
698
|
+
try:
|
|
699
|
+
self._do_checkpoint(self._epoch, monitor)
|
|
700
|
+
except Exception as exc:
|
|
701
|
+
print(f" ⚠ Emergency save failed: {exc}")
|
|
702
|
+
|
|
703
|
+
self._write_history_to_latest()
|
|
704
|
+
return result
|
|
705
|
+
|
|
706
|
+
def epochs(self, n=None):
|
|
707
|
+
"""Epoch iterator — writes complete history when the loop ends or is interrupted."""
|
|
708
|
+
try:
|
|
709
|
+
yield from super().epochs(n)
|
|
710
|
+
except KeyboardInterrupt:
|
|
711
|
+
print(f"\n ⏸ Training interrupted at epoch {self._epoch}.")
|
|
712
|
+
monitor = self._last_val_metrics or self._last_train_metrics
|
|
713
|
+
if monitor and self._epoch > 0:
|
|
714
|
+
print(" 💾 Saving current model state...")
|
|
715
|
+
try:
|
|
716
|
+
self._do_checkpoint(self._epoch, monitor)
|
|
717
|
+
except Exception as exc:
|
|
718
|
+
print(f" ⚠ Emergency save failed: {exc}")
|
|
719
|
+
raise # let Jupyter show "KeyboardInterrupt" (cell stops, kernel lives)
|
|
720
|
+
finally:
|
|
721
|
+
self._write_history_to_latest()
|
|
423
722
|
|
|
424
723
|
# ── Override Trainer's session-based checkpoint with project-based ────
|
|
425
724
|
|
|
@@ -456,6 +755,21 @@ class Run(Trainer):
|
|
|
456
755
|
**metric_kwargs,
|
|
457
756
|
)
|
|
458
757
|
|
|
758
|
+
def _write_history_to_latest(self) -> None:
|
|
759
|
+
"""Write the full per-epoch history to the most recent snapshot.
|
|
760
|
+
|
|
761
|
+
Called once at the end of training — not at each checkpoint — so the
|
|
762
|
+
latest snapshot always holds the complete epoch-by-epoch record.
|
|
763
|
+
"""
|
|
764
|
+
import yaml as _yaml
|
|
765
|
+
|
|
766
|
+
snaps = self._project_obj.log()
|
|
767
|
+
if not snaps or not self._history:
|
|
768
|
+
return
|
|
769
|
+
history_path = snaps[0].path / "history.yaml"
|
|
770
|
+
with open(history_path, "w", encoding="utf-8") as f:
|
|
771
|
+
_yaml.safe_dump(dict(self._history), f, default_flow_style=False)
|
|
772
|
+
|
|
459
773
|
# ── resume() override (Trainer's version uses session._resolve_…) ─────
|
|
460
774
|
|
|
461
775
|
def resume(self, snapshot_tag: str = "\\latest") -> int:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: useml
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Minimalist ML experiment versioning for PyTorch — checkpointing, source archiving, and model isolation.
|
|
5
5
|
Author-email: Enzo-py <85191552+Enzo-py@users.noreply.github.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/Enzo-py/useml
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|