useml 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {useml-0.1.0 → useml-0.2.0}/PKG-INFO +109 -3
- {useml-0.1.0 → useml-0.2.0}/README.md +108 -2
- {useml-0.1.0 → useml-0.2.0}/pyproject.toml +1 -1
- useml-0.2.0/tests/test_bundle.py +360 -0
- {useml-0.1.0 → useml-0.2.0}/tests/test_train_level0.py +1 -1
- useml-0.2.0/tests/test_usecases.py +308 -0
- {useml-0.1.0 → useml-0.2.0}/useml/__init__.py +39 -28
- useml-0.2.0/useml/dataset/__init__.py +4 -0
- useml-0.2.0/useml/dataset/bundle.py +288 -0
- useml-0.1.0/useml/template/dataset.py → useml-0.2.0/useml/dataset/loaders.py +10 -2
- {useml-0.1.0 → useml-0.2.0}/useml/errors.py +13 -0
- {useml-0.1.0 → useml-0.2.0}/useml/session/manager.py +54 -3
- useml-0.2.0/useml/template/config.py +133 -0
- useml-0.2.0/useml/template/trainer.py +296 -0
- {useml-0.1.0 → useml-0.2.0}/useml/vault/project.py +8 -1
- {useml-0.1.0 → useml-0.2.0}/useml.egg-info/PKG-INFO +109 -3
- {useml-0.1.0 → useml-0.2.0}/useml.egg-info/SOURCES.txt +5 -1
- useml-0.1.0/useml/template/config.py +0 -100
- useml-0.1.0/useml/template/trainer.py +0 -228
- {useml-0.1.0 → useml-0.2.0}/setup.cfg +0 -0
- {useml-0.1.0 → useml-0.2.0}/tests/test_session.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/tests/test_vault.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/imports.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/session/__init__.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/session/component.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/template/__init__.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/template/loss.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/template/model.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/vault/__init__.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/vault/code_extractor.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/vault/core.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/vault/snapshot.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/workdir/__init__.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml/workdir/_hook.py +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml.egg-info/dependency_links.txt +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml.egg-info/requires.txt +0 -0
- {useml-0.1.0 → useml-0.2.0}/useml.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: useml
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Minimalist ML experiment versioning for PyTorch — checkpointing, source archiving, and model isolation.
|
|
5
5
|
Author-email: Enzo-py <85191552+Enzo-py@users.noreply.github.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/Enzo-py/useml
|
|
@@ -105,7 +105,7 @@ useml.show()
|
|
|
105
105
|
# Latest: baseline [snap_20240501_...]
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
-
## Level-0
|
|
108
|
+
## Level-0 — train in two lines
|
|
109
109
|
|
|
110
110
|
```python
|
|
111
111
|
import useml
|
|
@@ -128,6 +128,101 @@ history = useml.train(Net, "mnist", config=Config(epochs=10))
|
|
|
128
128
|
# Custom: any torch.utils.data.Dataset
|
|
129
129
|
```
|
|
130
130
|
|
|
131
|
+
### Custom step function
|
|
132
|
+
|
|
133
|
+
Override the default `criterion(model(x), y)` without subclassing:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
def ae_step(model, batch, device):
|
|
137
|
+
x, _ = batch
|
|
138
|
+
x = x.to(device)
|
|
139
|
+
return F.mse_loss(model(x), x) # target = input
|
|
140
|
+
|
|
141
|
+
useml.train(AutoEncoder, "mnist", config=config, step_fn=ae_step)
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### Config custom fields
|
|
145
|
+
|
|
146
|
+
`Config` accepts any extra keyword argument as a custom field. Custom fields
|
|
147
|
+
are serialised to the snapshot YAML and forwarded to the model constructor
|
|
148
|
+
by `useml.load()`.
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
config = Config(epochs=10, lr=1e-3, latent_dim=16, kl_weight=1e-3)
|
|
152
|
+
config.latent_dim # 16
|
|
153
|
+
config.to_dict() # includes latent_dim and kl_weight in snapshot YAML
|
|
154
|
+
|
|
155
|
+
# On load, useml calls VAE(latent_dim=16) automatically
|
|
156
|
+
model = useml.load("vae")
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## Level-1 — DataBundle (versioned data contracts)
|
|
160
|
+
|
|
161
|
+
`DataBundle` pairs a raw dataset with a preprocessing transform and hashes
|
|
162
|
+
both at every snapshot. The raw data is stored once; only the transform code
|
|
163
|
+
and three hashes are embedded in the snapshot.
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
from useml import DataBundle
|
|
167
|
+
|
|
168
|
+
bundle = DataBundle(
|
|
169
|
+
name="polar_clouds",
|
|
170
|
+
source=CartesianDataset(), # raw data — any torch Dataset or built-in name
|
|
171
|
+
transform=gridify, # per-sample callable: (sample) -> sample
|
|
172
|
+
version="v1", # optional label
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
history = useml.train(MyCNN, bundle, config=config)
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Three hashes are recorded in `metadata.yaml` for every snapshot that uses the bundle:
|
|
179
|
+
|
|
180
|
+
| Hash | What it tracks |
|
|
181
|
+
|---|---|
|
|
182
|
+
| `source_hash` | Fingerprint of the raw dataset |
|
|
183
|
+
| `transform_hash` | sha256 of the transform function's source code + name |
|
|
184
|
+
| `cache_key` | sha256(source_hash + transform_hash) — used for disk cache |
|
|
185
|
+
|
|
186
|
+
The transform source code is also archived verbatim in `source/_bundle_<name>_<fn>.py`
|
|
187
|
+
inside the snapshot, so the exact preprocessing of any checkpoint is always recoverable.
|
|
188
|
+
|
|
189
|
+
### Factory transforms
|
|
190
|
+
|
|
191
|
+
When a transform is produced by a factory (different params, same algorithm),
|
|
192
|
+
set a descriptive `__name__` so the hashes and archived file names are distinct:
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
def make_gridify(n_r_bins, n_theta_bins):
|
|
196
|
+
def gridify(sample):
|
|
197
|
+
...
|
|
198
|
+
gridify.__name__ = f"gridify_{n_r_bins}r_{n_theta_bins}t"
|
|
199
|
+
return gridify
|
|
200
|
+
|
|
201
|
+
bundle_fine = DataBundle("ds", raw_ds, transform=make_gridify(16, 32))
|
|
202
|
+
bundle_coarse = DataBundle("ds", raw_ds, transform=make_gridify(8, 16))
|
|
203
|
+
# → different transform_hash, different archived .py file
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
### Optional disk cache
|
|
207
|
+
|
|
208
|
+
For expensive transforms, set `cache=True` to materialise the processed dataset
|
|
209
|
+
on first run and reload it on subsequent runs:
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
bundle = DataBundle("ds", raw_ds, transform=heavy_preprocess, cache=True)
|
|
213
|
+
# Saved to: <data_dir>/.bundle_cache/<cache_key>.pt
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
### Manual commit with bundle metadata
|
|
217
|
+
|
|
218
|
+
When calling `useml.train()`, bundle metadata flows into the snapshot automatically.
|
|
219
|
+
For manual workflows, pass it explicitly:
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
useml.track("model", model, config=config)
|
|
223
|
+
useml.commit("run 1", bundle_meta=bundle.to_meta_dict())
|
|
224
|
+
```
|
|
225
|
+
|
|
131
226
|
## Loading models
|
|
132
227
|
|
|
133
228
|
```python
|
|
@@ -207,5 +302,16 @@ All exceptions raised by useml carry a `[UML-NNN]` code. See [ERRORS.md](ERRORS.
|
|
|
207
302
|
| `useml.show()` | Print session dashboard |
|
|
208
303
|
| `useml.stash()` | Park current project state in RAM |
|
|
209
304
|
| `useml.projects()` | List all projects in the vault |
|
|
210
|
-
| `useml.train(model_cls, dataset, config?)` | Level-0 training entry point |
|
|
305
|
+
| `useml.train(model_cls, dataset, config?, step_fn?)` | Level-0 training entry point |
|
|
211
306
|
| `useml.debug_imports()` | Inspect workdir importable modules |
|
|
307
|
+
|
|
308
|
+
**Level-1**
|
|
309
|
+
|
|
310
|
+
| Class | Description |
|
|
311
|
+
|---|---|
|
|
312
|
+
| `useml.DataBundle(name, source, transform?, version?, cache?)` | Versioned data contract with hash tracking |
|
|
313
|
+
| `bundle.source_hash()` | Fingerprint of the raw dataset |
|
|
314
|
+
| `bundle.transform_hash()` | sha256 of the transform source + name |
|
|
315
|
+
| `bundle.cache_key()` | Combined hash used for disk cache |
|
|
316
|
+
| `bundle.to_meta_dict()` | Dict embedded in snapshot `metadata.yaml` under `data:` |
|
|
317
|
+
| `bundle.transform_source()` | Source code of the transform (for archival) |
|
|
@@ -88,7 +88,7 @@ useml.show()
|
|
|
88
88
|
# Latest: baseline [snap_20240501_...]
|
|
89
89
|
```
|
|
90
90
|
|
|
91
|
-
## Level-0
|
|
91
|
+
## Level-0 — train in two lines
|
|
92
92
|
|
|
93
93
|
```python
|
|
94
94
|
import useml
|
|
@@ -111,6 +111,101 @@ history = useml.train(Net, "mnist", config=Config(epochs=10))
|
|
|
111
111
|
# Custom: any torch.utils.data.Dataset
|
|
112
112
|
```
|
|
113
113
|
|
|
114
|
+
### Custom step function
|
|
115
|
+
|
|
116
|
+
Override the default `criterion(model(x), y)` without subclassing:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
def ae_step(model, batch, device):
|
|
120
|
+
x, _ = batch
|
|
121
|
+
x = x.to(device)
|
|
122
|
+
return F.mse_loss(model(x), x) # target = input
|
|
123
|
+
|
|
124
|
+
useml.train(AutoEncoder, "mnist", config=config, step_fn=ae_step)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Config custom fields
|
|
128
|
+
|
|
129
|
+
`Config` accepts any extra keyword argument as a custom field. Custom fields
|
|
130
|
+
are serialised to the snapshot YAML and forwarded to the model constructor
|
|
131
|
+
by `useml.load()`.
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
config = Config(epochs=10, lr=1e-3, latent_dim=16, kl_weight=1e-3)
|
|
135
|
+
config.latent_dim # 16
|
|
136
|
+
config.to_dict() # includes latent_dim and kl_weight in snapshot YAML
|
|
137
|
+
|
|
138
|
+
# On load, useml calls VAE(latent_dim=16) automatically
|
|
139
|
+
model = useml.load("vae")
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Level-1 — DataBundle (versioned data contracts)
|
|
143
|
+
|
|
144
|
+
`DataBundle` pairs a raw dataset with a preprocessing transform and hashes
|
|
145
|
+
both at every snapshot. The raw data is stored once; only the transform code
|
|
146
|
+
and three hashes are embedded in the snapshot.
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
from useml import DataBundle
|
|
150
|
+
|
|
151
|
+
bundle = DataBundle(
|
|
152
|
+
name="polar_clouds",
|
|
153
|
+
source=CartesianDataset(), # raw data — any torch Dataset or built-in name
|
|
154
|
+
transform=gridify, # per-sample callable: (sample) -> sample
|
|
155
|
+
version="v1", # optional label
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
history = useml.train(MyCNN, bundle, config=config)
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Three hashes are recorded in `metadata.yaml` for every snapshot that uses the bundle:
|
|
162
|
+
|
|
163
|
+
| Hash | What it tracks |
|
|
164
|
+
|---|---|
|
|
165
|
+
| `source_hash` | Fingerprint of the raw dataset |
|
|
166
|
+
| `transform_hash` | sha256 of the transform function's source code + name |
|
|
167
|
+
| `cache_key` | sha256(source_hash + transform_hash) — used for disk cache |
|
|
168
|
+
|
|
169
|
+
The transform source code is also archived verbatim in `source/_bundle_<name>_<fn>.py`
|
|
170
|
+
inside the snapshot, so the exact preprocessing of any checkpoint is always recoverable.
|
|
171
|
+
|
|
172
|
+
### Factory transforms
|
|
173
|
+
|
|
174
|
+
When a transform is produced by a factory (different params, same algorithm),
|
|
175
|
+
set a descriptive `__name__` so the hashes and archived file names are distinct:
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
def make_gridify(n_r_bins, n_theta_bins):
|
|
179
|
+
def gridify(sample):
|
|
180
|
+
...
|
|
181
|
+
gridify.__name__ = f"gridify_{n_r_bins}r_{n_theta_bins}t"
|
|
182
|
+
return gridify
|
|
183
|
+
|
|
184
|
+
bundle_fine = DataBundle("ds", raw_ds, transform=make_gridify(16, 32))
|
|
185
|
+
bundle_coarse = DataBundle("ds", raw_ds, transform=make_gridify(8, 16))
|
|
186
|
+
# → different transform_hash, different archived .py file
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### Optional disk cache
|
|
190
|
+
|
|
191
|
+
For expensive transforms, set `cache=True` to materialise the processed dataset
|
|
192
|
+
on first run and reload it on subsequent runs:
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
bundle = DataBundle("ds", raw_ds, transform=heavy_preprocess, cache=True)
|
|
196
|
+
# Saved to: <data_dir>/.bundle_cache/<cache_key>.pt
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
### Manual commit with bundle metadata
|
|
200
|
+
|
|
201
|
+
When calling `useml.train()`, bundle metadata flows into the snapshot automatically.
|
|
202
|
+
For manual workflows, pass it explicitly:
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
useml.track("model", model, config=config)
|
|
206
|
+
useml.commit("run 1", bundle_meta=bundle.to_meta_dict())
|
|
207
|
+
```
|
|
208
|
+
|
|
114
209
|
## Loading models
|
|
115
210
|
|
|
116
211
|
```python
|
|
@@ -190,5 +285,16 @@ All exceptions raised by useml carry a `[UML-NNN]` code. See [ERRORS.md](ERRORS.
|
|
|
190
285
|
| `useml.show()` | Print session dashboard |
|
|
191
286
|
| `useml.stash()` | Park current project state in RAM |
|
|
192
287
|
| `useml.projects()` | List all projects in the vault |
|
|
193
|
-
| `useml.train(model_cls, dataset, config?)` | Level-0 training entry point |
|
|
288
|
+
| `useml.train(model_cls, dataset, config?, step_fn?)` | Level-0 training entry point |
|
|
194
289
|
| `useml.debug_imports()` | Inspect workdir importable modules |
|
|
290
|
+
|
|
291
|
+
**Level-1**
|
|
292
|
+
|
|
293
|
+
| Class | Description |
|
|
294
|
+
|---|---|
|
|
295
|
+
| `useml.DataBundle(name, source, transform?, version?, cache?)` | Versioned data contract with hash tracking |
|
|
296
|
+
| `bundle.source_hash()` | Fingerprint of the raw dataset |
|
|
297
|
+
| `bundle.transform_hash()` | sha256 of the transform source + name |
|
|
298
|
+
| `bundle.cache_key()` | Combined hash used for disk cache |
|
|
299
|
+
| `bundle.to_meta_dict()` | Dict embedded in snapshot `metadata.yaml` under `data:` |
|
|
300
|
+
| `bundle.transform_source()` | Source code of the transform (for archival) |
|
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""Tests for useml.DataBundle (Level-1 data contract)."""
|
|
2
|
+
import shutil
|
|
3
|
+
import tempfile
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
import torch
|
|
7
|
+
import torch.nn as nn
|
|
8
|
+
|
|
9
|
+
import useml
|
|
10
|
+
from useml.dataset.bundle import _TransformWrapper, _MaterializedDataset, _sha256
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# ---------------------------------------------------------------------------
|
|
14
|
+
# Fixtures & helpers
|
|
15
|
+
# ---------------------------------------------------------------------------
|
|
16
|
+
|
|
17
|
+
@pytest.fixture
|
|
18
|
+
def tmp_vault(tmp_path):
|
|
19
|
+
vault = tmp_path / "vault"
|
|
20
|
+
useml.init(str(vault))
|
|
21
|
+
useml.new("bundle-project")
|
|
22
|
+
yield tmp_path
|
|
23
|
+
useml._session.vault = None
|
|
24
|
+
useml._session._project = None
|
|
25
|
+
useml._session.components = {}
|
|
26
|
+
useml._session._is_dirty = False
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _TinyDS(torch.utils.data.Dataset):
|
|
30
|
+
"""20-sample in-memory dataset of random (x4, y_binary) pairs."""
|
|
31
|
+
def __init__(self):
|
|
32
|
+
torch.manual_seed(0)
|
|
33
|
+
self.x = torch.randn(20, 4)
|
|
34
|
+
self.y = torch.randint(0, 2, (20,))
|
|
35
|
+
|
|
36
|
+
def __len__(self):
|
|
37
|
+
return 20
|
|
38
|
+
|
|
39
|
+
def __getitem__(self, i):
|
|
40
|
+
return self.x[i], self.y[i]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class _TinyNet(nn.Module):
|
|
44
|
+
def __init__(self):
|
|
45
|
+
super().__init__()
|
|
46
|
+
self.fc = nn.Linear(4, 2)
|
|
47
|
+
|
|
48
|
+
def forward(self, x):
|
|
49
|
+
return self.fc(x)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _scale(sample):
|
|
53
|
+
x, y = sample
|
|
54
|
+
return x * 0.5, y
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _cfg(**kw):
|
|
58
|
+
defaults = dict(epochs=1, checkpoint_every=9999, loss="cross_entropy", batch_size=4)
|
|
59
|
+
defaults.update(kw)
|
|
60
|
+
return useml.Config(**defaults)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ---------------------------------------------------------------------------
|
|
64
|
+
# 1. Hash stability
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
|
|
67
|
+
class TestHashes:
|
|
68
|
+
def test_builtin_source_hash_is_deterministic(self):
|
|
69
|
+
b1 = useml.DataBundle("a", source="mnist")
|
|
70
|
+
b2 = useml.DataBundle("b", source="mnist")
|
|
71
|
+
assert b1.source_hash() == b2.source_hash()
|
|
72
|
+
|
|
73
|
+
def test_builtin_source_hash_differs_by_name(self):
|
|
74
|
+
h_mnist = useml.DataBundle("a", source="mnist").source_hash()
|
|
75
|
+
h_cifar = useml.DataBundle("b", source="cifar10").source_hash()
|
|
76
|
+
assert h_mnist != h_cifar
|
|
77
|
+
|
|
78
|
+
def test_custom_dataset_source_hash_is_deterministic(self):
|
|
79
|
+
ds = _TinyDS()
|
|
80
|
+
b1 = useml.DataBundle("a", source=ds)
|
|
81
|
+
b2 = useml.DataBundle("b", source=ds)
|
|
82
|
+
assert b1.source_hash() == b2.source_hash()
|
|
83
|
+
|
|
84
|
+
def test_transform_hash_none_is_literal_none(self):
|
|
85
|
+
b = useml.DataBundle("a", source="mnist", transform=None)
|
|
86
|
+
assert b.transform_hash() == "none"
|
|
87
|
+
|
|
88
|
+
def test_transform_hash_is_deterministic(self):
|
|
89
|
+
b1 = useml.DataBundle("a", source="mnist", transform=_scale)
|
|
90
|
+
b2 = useml.DataBundle("b", source="mnist", transform=_scale)
|
|
91
|
+
assert b1.transform_hash() == b2.transform_hash()
|
|
92
|
+
|
|
93
|
+
def test_transform_hash_differs_by_function(self):
|
|
94
|
+
def double(s):
|
|
95
|
+
x, y = s; return x * 2, y
|
|
96
|
+
|
|
97
|
+
h_scale = useml.DataBundle("a", source="mnist", transform=_scale).transform_hash()
|
|
98
|
+
h_double = useml.DataBundle("b", source="mnist", transform=double).transform_hash()
|
|
99
|
+
assert h_scale != h_double
|
|
100
|
+
|
|
101
|
+
def test_cache_key_combines_both_hashes(self):
|
|
102
|
+
b = useml.DataBundle("a", source="mnist", transform=_scale)
|
|
103
|
+
expected = _sha256(b.source_hash() + b.transform_hash())[:16]
|
|
104
|
+
assert b.cache_key() == expected
|
|
105
|
+
|
|
106
|
+
def test_cache_key_differs_when_transform_changes(self):
|
|
107
|
+
def other(s):
|
|
108
|
+
x, y = s; return x + 1, y
|
|
109
|
+
|
|
110
|
+
k1 = useml.DataBundle("a", source="mnist", transform=_scale).cache_key()
|
|
111
|
+
k2 = useml.DataBundle("b", source="mnist", transform=other).cache_key()
|
|
112
|
+
assert k1 != k2
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# ---------------------------------------------------------------------------
|
|
116
|
+
# 2. Transform source archival
|
|
117
|
+
# ---------------------------------------------------------------------------
|
|
118
|
+
|
|
119
|
+
class TestTransformSource:
|
|
120
|
+
def test_no_transform_returns_none(self):
|
|
121
|
+
b = useml.DataBundle("a", source="mnist")
|
|
122
|
+
assert b.transform_source() is None
|
|
123
|
+
|
|
124
|
+
def test_module_level_function_source_captured(self):
|
|
125
|
+
b = useml.DataBundle("a", source="mnist", transform=_scale)
|
|
126
|
+
src = b.transform_source()
|
|
127
|
+
assert src is not None
|
|
128
|
+
assert "def _scale" in src
|
|
129
|
+
|
|
130
|
+
def test_inline_source_key_includes_bundle_name_and_fn_name(self):
|
|
131
|
+
b = useml.DataBundle("my_ds", source="mnist", transform=_scale)
|
|
132
|
+
key = b.inline_source_key()
|
|
133
|
+
assert "my_ds" in key
|
|
134
|
+
assert "_scale" in key
|
|
135
|
+
assert key.endswith(".py")
|
|
136
|
+
|
|
137
|
+
def test_inline_source_key_with_no_transform(self):
|
|
138
|
+
b = useml.DataBundle("my_ds", source="mnist")
|
|
139
|
+
key = b.inline_source_key()
|
|
140
|
+
assert key.endswith(".py")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# ---------------------------------------------------------------------------
|
|
144
|
+
# 3. to_meta_dict
|
|
145
|
+
# ---------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
class TestMetaDict:
|
|
148
|
+
def test_required_keys_present(self):
|
|
149
|
+
b = useml.DataBundle("ds", source="mnist", transform=_scale, version="v1")
|
|
150
|
+
d = b.to_meta_dict()
|
|
151
|
+
for key in ("name", "source", "source_hash", "transform", "transform_hash",
|
|
152
|
+
"cache_key", "version"):
|
|
153
|
+
assert key in d, f"missing key: {key}"
|
|
154
|
+
|
|
155
|
+
def test_version_absent_when_none(self):
|
|
156
|
+
b = useml.DataBundle("ds", source="mnist")
|
|
157
|
+
assert "version" not in b.to_meta_dict()
|
|
158
|
+
|
|
159
|
+
def test_transform_none_recorded(self):
|
|
160
|
+
b = useml.DataBundle("ds", source="mnist")
|
|
161
|
+
assert b.to_meta_dict()["transform"] is None
|
|
162
|
+
|
|
163
|
+
def test_source_custom_dataset_uses_class_name(self):
|
|
164
|
+
b = useml.DataBundle("ds", source=_TinyDS())
|
|
165
|
+
assert b.to_meta_dict()["source"] == "_TinyDS"
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# ---------------------------------------------------------------------------
|
|
169
|
+
# 4. _TransformWrapper
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
class TestTransformWrapper:
|
|
173
|
+
def test_len_preserved(self):
|
|
174
|
+
ds = _TinyDS()
|
|
175
|
+
wrapped = _TransformWrapper(ds, _scale)
|
|
176
|
+
assert len(wrapped) == len(ds)
|
|
177
|
+
|
|
178
|
+
def test_transform_applied(self):
|
|
179
|
+
ds = _TinyDS()
|
|
180
|
+
wrapped = _TransformWrapper(ds, _scale)
|
|
181
|
+
x_orig, _ = ds[0]
|
|
182
|
+
x_wrap, _ = wrapped[0]
|
|
183
|
+
assert torch.allclose(x_wrap, x_orig * 0.5)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# ---------------------------------------------------------------------------
|
|
187
|
+
# 5. _MaterializedDataset
|
|
188
|
+
# ---------------------------------------------------------------------------
|
|
189
|
+
|
|
190
|
+
class TestMaterializedDataset:
|
|
191
|
+
def test_all_samples_materialised(self):
|
|
192
|
+
ds = _TinyDS()
|
|
193
|
+
mat = _MaterializedDataset(ds)
|
|
194
|
+
assert len(mat) == len(ds)
|
|
195
|
+
|
|
196
|
+
def test_samples_equal(self):
|
|
197
|
+
ds = _TinyDS()
|
|
198
|
+
mat = _MaterializedDataset(ds)
|
|
199
|
+
for i in range(len(ds)):
|
|
200
|
+
x_orig, y_orig = ds[i]
|
|
201
|
+
x_mat, y_mat = mat[i]
|
|
202
|
+
assert torch.equal(x_orig, x_mat)
|
|
203
|
+
assert torch.equal(y_orig, y_mat)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# ---------------------------------------------------------------------------
|
|
207
|
+
# 6. loaders()
|
|
208
|
+
# ---------------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
class TestLoaders:
|
|
211
|
+
def test_returns_two_dataloaders(self):
|
|
212
|
+
ds = _TinyDS()
|
|
213
|
+
bundle = useml.DataBundle("ds", source=ds, transform=_scale)
|
|
214
|
+
cfg = _cfg()
|
|
215
|
+
train_loader, val_loader = bundle.loaders(cfg)
|
|
216
|
+
assert hasattr(train_loader, "__iter__")
|
|
217
|
+
assert hasattr(val_loader, "__iter__")
|
|
218
|
+
|
|
219
|
+
def test_transform_applied_in_loader(self):
|
|
220
|
+
ds = _TinyDS()
|
|
221
|
+
bundle_with = useml.DataBundle("ds", source=ds, transform=_scale)
|
|
222
|
+
bundle_without = useml.DataBundle("ds_raw", source=ds)
|
|
223
|
+
cfg = _cfg()
|
|
224
|
+
loader_with, _ = bundle_with.loaders(cfg)
|
|
225
|
+
loader_without, _ = bundle_without.loaders(cfg)
|
|
226
|
+
# _scale multiplies x by 0.5, so max-abs of transformed batch < raw batch
|
|
227
|
+
batch_raw, _ = next(iter(loader_without))
|
|
228
|
+
batch_scaled, _ = next(iter(loader_with))
|
|
229
|
+
assert batch_scaled.abs().max() < batch_raw.abs().max()
|
|
230
|
+
|
|
231
|
+
def test_no_transform_still_produces_loaders(self):
|
|
232
|
+
ds = _TinyDS()
|
|
233
|
+
bundle = useml.DataBundle("ds", source=ds)
|
|
234
|
+
cfg = _cfg()
|
|
235
|
+
train_loader, val_loader = bundle.loaders(cfg)
|
|
236
|
+
assert sum(len(b[0]) for b in train_loader) > 0
|
|
237
|
+
|
|
238
|
+
def test_cache_creates_file(self, tmp_path):
|
|
239
|
+
ds = _TinyDS()
|
|
240
|
+
bundle = useml.DataBundle("ds", source=ds, transform=_scale, cache=True)
|
|
241
|
+
cfg = useml.Config(
|
|
242
|
+
epochs=1, loss="cross_entropy", batch_size=4,
|
|
243
|
+
data_dir=str(tmp_path), checkpoint_every=9999,
|
|
244
|
+
)
|
|
245
|
+
import warnings
|
|
246
|
+
with warnings.catch_warnings():
|
|
247
|
+
warnings.simplefilter("ignore")
|
|
248
|
+
bundle.loaders(cfg)
|
|
249
|
+
|
|
250
|
+
cache_dir = tmp_path / ".bundle_cache"
|
|
251
|
+
assert cache_dir.exists()
|
|
252
|
+
cache_files = list(cache_dir.iterdir())
|
|
253
|
+
assert len(cache_files) == 1
|
|
254
|
+
assert cache_files[0].name == f"{bundle.cache_key()}.pt"
|
|
255
|
+
|
|
256
|
+
def test_cache_reused_on_second_call(self, tmp_path):
|
|
257
|
+
ds = _TinyDS()
|
|
258
|
+
bundle = useml.DataBundle("ds", source=ds, transform=_scale, cache=True)
|
|
259
|
+
cfg = useml.Config(
|
|
260
|
+
epochs=1, loss="cross_entropy", batch_size=4,
|
|
261
|
+
data_dir=str(tmp_path), checkpoint_every=9999,
|
|
262
|
+
)
|
|
263
|
+
import warnings
|
|
264
|
+
with warnings.catch_warnings():
|
|
265
|
+
warnings.simplefilter("ignore")
|
|
266
|
+
bundle.loaders(cfg) # first call — builds cache
|
|
267
|
+
|
|
268
|
+
# Second call should produce no warnings (pure cache hit)
|
|
269
|
+
with warnings.catch_warnings(record=True) as record:
|
|
270
|
+
warnings.simplefilter("always")
|
|
271
|
+
bundle.loaders(cfg)
|
|
272
|
+
assert len(record) == 0
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
# ---------------------------------------------------------------------------
|
|
276
|
+
# 7. Integration with useml.train() → snapshot metadata
|
|
277
|
+
# ---------------------------------------------------------------------------
|
|
278
|
+
|
|
279
|
+
class TestSnapshotIntegration:
|
|
280
|
+
def test_metadata_yaml_contains_data_key(self, tmp_vault):
|
|
281
|
+
import yaml
|
|
282
|
+
|
|
283
|
+
ds = _TinyDS()
|
|
284
|
+
bundle = useml.DataBundle("tiny", source=ds, transform=_scale, version="v1")
|
|
285
|
+
cfg = _cfg(checkpoint_every=1)
|
|
286
|
+
|
|
287
|
+
useml.train(_TinyNet, bundle, config=cfg)
|
|
288
|
+
|
|
289
|
+
snap = useml.focus("bundle-project")[0]
|
|
290
|
+
meta = yaml.safe_load((snap.path / "metadata.yaml").read_text())
|
|
291
|
+
|
|
292
|
+
assert "data" in meta
|
|
293
|
+
d = meta["data"]
|
|
294
|
+
assert d["name"] == "tiny"
|
|
295
|
+
assert d["version"] == "v1"
|
|
296
|
+
assert d["transform"] == "_scale"
|
|
297
|
+
assert "source_hash" in d
|
|
298
|
+
assert "transform_hash" in d
|
|
299
|
+
assert "cache_key" in d
|
|
300
|
+
|
|
301
|
+
def test_transform_source_archived_in_snapshot(self, tmp_vault):
|
|
302
|
+
ds = _TinyDS()
|
|
303
|
+
bundle = useml.DataBundle("tiny", source=ds, transform=_scale)
|
|
304
|
+
cfg = _cfg(checkpoint_every=1)
|
|
305
|
+
|
|
306
|
+
useml.train(_TinyNet, bundle, config=cfg)
|
|
307
|
+
|
|
308
|
+
snap = useml.focus("bundle-project")[0]
|
|
309
|
+
transform_file = snap.path / "source" / bundle.inline_source_key()
|
|
310
|
+
assert transform_file.exists()
|
|
311
|
+
assert "def _scale" in transform_file.read_text()
|
|
312
|
+
|
|
313
|
+
def test_no_data_key_without_bundle(self, tmp_vault):
|
|
314
|
+
import yaml
|
|
315
|
+
|
|
316
|
+
ds = _TinyDS()
|
|
317
|
+
cfg = _cfg(checkpoint_every=1)
|
|
318
|
+
useml.train(_TinyNet, ds, config=cfg)
|
|
319
|
+
|
|
320
|
+
snap = useml.focus("bundle-project")[0]
|
|
321
|
+
meta = yaml.safe_load((snap.path / "metadata.yaml").read_text())
|
|
322
|
+
assert "data" not in meta
|
|
323
|
+
|
|
324
|
+
def test_hashes_are_stable_across_runs(self, tmp_vault):
|
|
325
|
+
import yaml
|
|
326
|
+
|
|
327
|
+
ds = _TinyDS()
|
|
328
|
+
bundle1 = useml.DataBundle("tiny", source=ds, transform=_scale)
|
|
329
|
+
bundle2 = useml.DataBundle("tiny", source=ds, transform=_scale)
|
|
330
|
+
cfg = _cfg(checkpoint_every=1)
|
|
331
|
+
|
|
332
|
+
useml.train(_TinyNet, bundle1, config=cfg)
|
|
333
|
+
useml.train(_TinyNet, bundle2, config=cfg)
|
|
334
|
+
|
|
335
|
+
snaps = useml.focus("bundle-project").log()
|
|
336
|
+
metas = [
|
|
337
|
+
yaml.safe_load((s.path / "metadata.yaml").read_text())["data"]
|
|
338
|
+
for s in snaps
|
|
339
|
+
]
|
|
340
|
+
assert metas[0]["source_hash"] == metas[1]["source_hash"]
|
|
341
|
+
assert metas[0]["transform_hash"] == metas[1]["transform_hash"]
|
|
342
|
+
assert metas[0]["cache_key"] == metas[1]["cache_key"]
|
|
343
|
+
|
|
344
|
+
def test_manual_commit_passes_bundle_meta(self, tmp_vault):
|
|
345
|
+
import yaml
|
|
346
|
+
|
|
347
|
+
ds = _TinyDS()
|
|
348
|
+
bundle = useml.DataBundle("tiny", source=ds, transform=_scale)
|
|
349
|
+
model = _TinyNet()
|
|
350
|
+
|
|
351
|
+
useml.track("net", model)
|
|
352
|
+
useml.commit(
|
|
353
|
+
"manual with bundle",
|
|
354
|
+
bundle_meta=bundle.to_meta_dict(),
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
snap = useml.focus("bundle-project")[0]
|
|
358
|
+
meta = yaml.safe_load((snap.path / "metadata.yaml").read_text())
|
|
359
|
+
assert "data" in meta
|
|
360
|
+
assert meta["data"]["name"] == "tiny"
|