useml 0.3.2__tar.gz → 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- useml-1.0.1/.gitignore +33 -0
- useml-1.0.1/LICENSE +21 -0
- useml-1.0.1/PKG-INFO +242 -0
- useml-1.0.1/README.md +204 -0
- useml-1.0.1/pyproject.toml +56 -0
- useml-1.0.1/src/useml/__init__.py +42 -0
- useml-1.0.1/src/useml/__main__.py +5 -0
- useml-1.0.1/src/useml/_progress.py +90 -0
- useml-1.0.1/src/useml/_project.py +45 -0
- useml-1.0.1/src/useml/_setup.py +468 -0
- useml-1.0.1/src/useml/cli.py +474 -0
- useml-1.0.1/src/useml/config/__init__.py +127 -0
- useml-1.0.1/src/useml/config/_config.py +201 -0
- useml-1.0.1/src/useml/config/_git.py +81 -0
- useml-1.0.1/src/useml/config/_library.py +357 -0
- useml-1.0.1/src/useml/config/_types.py +111 -0
- useml-1.0.1/src/useml/config/_write.py +248 -0
- useml-1.0.1/src/useml/data/__init__.py +39 -0
- useml-1.0.1/src/useml/data/_catalog.py +148 -0
- useml-1.0.1/src/useml/data/_jobs.py +227 -0
- useml-1.0.1/src/useml/data/_keys.py +53 -0
- useml-1.0.1/src/useml/data/_processor.py +151 -0
- useml-1.0.1/src/useml/data/_run.py +240 -0
- useml-1.0.1/src/useml/data/_sources.py +218 -0
- useml-1.0.1/src/useml/data/_table.py +288 -0
- useml-1.0.1/src/useml/data/_view.py +138 -0
- useml-1.0.1/src/useml/exp/__init__.py +28 -0
- useml-1.0.1/src/useml/exp/_bundle.py +421 -0
- useml-1.0.1/src/useml/exp/_experience.py +472 -0
- useml-1.0.1/src/useml/exp/_launch.py +222 -0
- useml-1.0.1/src/useml/exp/_outputs.py +162 -0
- useml-1.0.1/src/useml/exp/_provenance.py +159 -0
- useml-1.0.1/src/useml/exp/_refs.py +242 -0
- useml-1.0.1/src/useml/exp/_runtime.py +303 -0
- useml-1.0.1/src/useml/exp/_supervise.py +252 -0
- useml-1.0.1/src/useml/hub/__init__.py +23 -0
- useml-1.0.1/src/useml/hub/_fetch.py +321 -0
- useml-1.0.1/src/useml/hub/_policy.py +377 -0
- useml-1.0.1/src/useml/hub/_registry.py +115 -0
- useml-1.0.1/src/useml/hub/_scan.py +107 -0
- useml-1.0.1/src/useml/hub/_watch.py +157 -0
- useml-1.0.1/src/useml/paths.py +331 -0
- useml-1.0.1/src/useml/store/__init__.py +65 -0
- useml-1.0.1/src/useml/store/_maintenance.py +253 -0
- useml-1.0.1/src/useml/store/_spaces.py +67 -0
- useml-1.0.1/src/useml/store/_store.py +676 -0
- useml-1.0.1/tests/conftest.py +47 -0
- useml-1.0.1/tests/data_procs.py +54 -0
- useml-1.0.1/tests/exp_steps.py +117 -0
- useml-1.0.1/tests/test_config.py +389 -0
- useml-1.0.1/tests/test_data.py +302 -0
- useml-1.0.1/tests/test_exp.py +228 -0
- useml-1.0.1/tests/test_exp_robust.py +276 -0
- useml-1.0.1/tests/test_fetch.py +179 -0
- useml-1.0.1/tests/test_paths.py +97 -0
- useml-1.0.1/tests/test_policy.py +200 -0
- useml-1.0.1/tests/test_setup.py +165 -0
- useml-1.0.1/tests/test_store.py +342 -0
- useml-0.3.2/PKG-INFO +0 -471
- useml-0.3.2/README.md +0 -454
- useml-0.3.2/pyproject.toml +0 -30
- useml-0.3.2/setup.cfg +0 -4
- useml-0.3.2/tests/test_bundle.py +0 -366
- useml-0.3.2/tests/test_level2.py +0 -244
- useml-0.3.2/tests/test_session.py +0 -208
- useml-0.3.2/tests/test_train_level0.py +0 -234
- useml-0.3.2/tests/test_usecases.py +0 -1249
- useml-0.3.2/tests/test_vault.py +0 -243
- useml-0.3.2/useml/__init__.py +0 -94
- useml-0.3.2/useml/dataset/__init__.py +0 -4
- useml-0.3.2/useml/dataset/bundle.py +0 -288
- useml-0.3.2/useml/dataset/loaders.py +0 -168
- useml-0.3.2/useml/errors.py +0 -377
- useml-0.3.2/useml/imports.py +0 -215
- useml-0.3.2/useml/session/__init__.py +0 -4
- useml-0.3.2/useml/session/component.py +0 -59
- useml-0.3.2/useml/session/manager.py +0 -482
- useml-0.3.2/useml/template/__init__.py +0 -6
- useml-0.3.2/useml/template/config.py +0 -140
- useml-0.3.2/useml/template/loss.py +0 -28
- useml-0.3.2/useml/template/model.py +0 -28
- useml-0.3.2/useml/template/trainer.py +0 -758
- useml-0.3.2/useml/vault/__init__.py +0 -5
- useml-0.3.2/useml/vault/code_extractor.py +0 -261
- useml-0.3.2/useml/vault/core.py +0 -63
- useml-0.3.2/useml/vault/model_registry.py +0 -267
- useml-0.3.2/useml/vault/project.py +0 -309
- useml-0.3.2/useml/vault/runs.py +0 -814
- useml-0.3.2/useml/vault/snapshot.py +0 -170
- useml-0.3.2/useml/workdir/__init__.py +0 -14
- useml-0.3.2/useml/workdir/_hook.py +0 -79
- useml-0.3.2/useml.egg-info/PKG-INFO +0 -471
- useml-0.3.2/useml.egg-info/SOURCES.txt +0 -36
- useml-0.3.2/useml.egg-info/dependency_links.txt +0 -1
- useml-0.3.2/useml.egg-info/requires.txt +0 -8
- useml-0.3.2/useml.egg-info/top_level.txt +0 -1
useml-1.0.1/.gitignore
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# --- Python Build & Distribution ---
|
|
2
|
+
dist/
|
|
3
|
+
build/
|
|
4
|
+
*.egg-info/
|
|
5
|
+
egg-info/
|
|
6
|
+
nosetests.xml
|
|
7
|
+
pip-wheel-metadata/
|
|
8
|
+
.pytest_cache
|
|
9
|
+
|
|
10
|
+
# --- Virtual Environments ---
|
|
11
|
+
venv/
|
|
12
|
+
.venv/
|
|
13
|
+
env/
|
|
14
|
+
|
|
15
|
+
# --- Compiled Files ---
|
|
16
|
+
__pycache__/
|
|
17
|
+
*.py[cod]
|
|
18
|
+
*$py.class
|
|
19
|
+
|
|
20
|
+
# --- Machine Learning specifics (Vault/Data) ---
|
|
21
|
+
*.pt
|
|
22
|
+
*.pth
|
|
23
|
+
*.ckpt
|
|
24
|
+
|
|
25
|
+
# IDE / OS
|
|
26
|
+
.DS_Store
|
|
27
|
+
.claude/
|
|
28
|
+
|
|
29
|
+
# testing
|
|
30
|
+
playground
|
|
31
|
+
|
|
32
|
+
# --- useml v1 ---
|
|
33
|
+
bundles/
|
useml-1.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Enzo-py
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
useml-1.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: useml
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: One root for every ML cache, one gate for every download, data processed once.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Enzo-py/useml
|
|
6
|
+
Project-URL: Bug Tracker, https://github.com/Enzo-py/useml/issues
|
|
7
|
+
Author-email: Enzo-py <85191552+Enzo-py@users.noreply.github.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: cache,datasets,deep learning,experiments,huggingface,machine learning,reproducibility
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: MacOS
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: numpy>=1.22
|
|
22
|
+
Requires-Dist: pyyaml>=6
|
|
23
|
+
Requires-Dist: tomli>=1.1; python_version < '3.11'
|
|
24
|
+
Requires-Dist: tqdm>=4.60
|
|
25
|
+
Provides-Extra: all
|
|
26
|
+
Requires-Dist: gdown>=4.6; extra == 'all'
|
|
27
|
+
Requires-Dist: huggingface-hub>=0.20; extra == 'all'
|
|
28
|
+
Requires-Dist: psutil>=5.9; extra == 'all'
|
|
29
|
+
Provides-Extra: gdrive
|
|
30
|
+
Requires-Dist: gdown>=4.6; extra == 'gdrive'
|
|
31
|
+
Provides-Extra: hf
|
|
32
|
+
Requires-Dist: huggingface-hub>=0.20; extra == 'hf'
|
|
33
|
+
Provides-Extra: test
|
|
34
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
35
|
+
Provides-Extra: watch
|
|
36
|
+
Requires-Dist: psutil>=5.9; extra == 'watch'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# useml
|
|
40
|
+
|
|
41
|
+
One root for every ML cache, one gate for every download.
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install "useml[all]"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Quick start
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install "useml[all]"
|
|
51
|
+
cd my-project
|
|
52
|
+
useml init # shows what it would set up (adds, never overwrites); confirm to write
|
|
53
|
+
useml check # checks that everything resolves
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## One root per machine
|
|
57
|
+
|
|
58
|
+
```toml
|
|
59
|
+
# ~/.config/useml/config.toml
|
|
60
|
+
root = "/data/me/useml"
|
|
61
|
+
|
|
62
|
+
[hub]
|
|
63
|
+
parallel = false # default: one download thread at a time
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`import useml` (first, before torch / transformers) points `HF_HOME`, `TORCH_HOME`,
|
|
67
|
+
`TMPDIR`, pip, triton… under the root. A cache already placed outside the home is never
|
|
68
|
+
moved. `useml paths` shows where everything goes.
|
|
69
|
+
|
|
70
|
+
## One gate for downloads
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
import useml
|
|
74
|
+
|
|
75
|
+
useml.hub.get("hf://openai/clip-vit-large-patch14")
|
|
76
|
+
useml.hub.get("hf://datasets/org/name@main/file.json")
|
|
77
|
+
useml.hub.get("https://example.org/annotations.zip", extract_archive=True)
|
|
78
|
+
useml.hub.get("gdrive://<file id>")
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Already downloaded → answered from disk, no network. Every download is recorded:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
useml hub ls # what is on disk, how big, last used
|
|
85
|
+
useml hub scan # caches left in the home (~/.cache/huggingface…)
|
|
86
|
+
useml hub scan --move --apply
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## The network policy
|
|
90
|
+
|
|
91
|
+
By default, only one thread may download at a time, in every library:
|
|
92
|
+
|
|
93
|
+
- Hugging Face `xet` (Rust, its own parallel connections) is disabled;
|
|
94
|
+
- the thread pools of `huggingface_hub` and `datasets` become sequential;
|
|
95
|
+
- `socket.connect` refuses an outbound connection from a second thread
|
|
96
|
+
(`ParallelDownloadError`), and logs it;
|
|
97
|
+
- `useml hub watch -- python train.py` watches the whole process tree from outside
|
|
98
|
+
and kills it on a sustained violation.
|
|
99
|
+
|
|
100
|
+
`useml.hub.allow_parallel()` lifts it, explicitly.
|
|
101
|
+
|
|
102
|
+
## One cache store
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
store = useml.store.open() # <root>/store
|
|
106
|
+
masks = store.declare("processing/masks", cost="expensive")
|
|
107
|
+
masks.put({"image": digest, "model": "sam-h"}, {"masks": array})
|
|
108
|
+
masks.get({"image": digest, "model": "sam-h"})
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
One SQLite index (WAL, safe across processes), large values in files written atomically,
|
|
112
|
+
one write per key. Spaces are declared with a cost (`cheap`, `expensive`, `precious`)
|
|
113
|
+
kept in the index, so cleaning needs no project code:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
useml store spaces
|
|
117
|
+
useml store gc --max-size 450GB # dry run: cheapest to recompute first
|
|
118
|
+
useml store gc --max-size 450GB --apply
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`[store] max_size = "450GB"` in the config sets the default budget. Nothing is ever
|
|
122
|
+
evicted automatically, pinned entries and `precious` spaces are kept.
|
|
123
|
+
|
|
124
|
+
## Data processed once
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
from useml import data
|
|
128
|
+
|
|
129
|
+
@data.processor("clip-image", version=1, batch_size=64, dtype="float16")
|
|
130
|
+
class ClipImage:
|
|
131
|
+
def __init__(self, model="openai/clip-vit-base-patch32", device="cuda"): ...
|
|
132
|
+
def load(self, path): ... # per item, in worker threads
|
|
133
|
+
def __call__(self, images): ... # a batch → (B, D) array, or a dict of arrays
|
|
134
|
+
|
|
135
|
+
data.process(data.images("/data/photos"), ClipImage, name="photos-clip")
|
|
136
|
+
feats = data.load("photos-clip") # one line; nothing is read until used
|
|
137
|
+
feats[0], feats[100:200], feats.numpy()
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
- One table per (processor, version, parameters, dtype), shared by every dataset: rows
|
|
141
|
+
are found by a hash of their CONTENT, so an image two datasets share is computed once.
|
|
142
|
+
- Only missing rows are computed; when nothing is missing, the model is never loaded.
|
|
143
|
+
- Storage `float16` (default), `float32` or `int8` (one byte per number + a scale per row).
|
|
144
|
+
- Memory-mapped, append-only shards; one writer per table (a file lock); an interrupted
|
|
145
|
+
run keeps its rows and resumes.
|
|
146
|
+
- The size to write is checked against the free disk and `[data] max_size` before writing.
|
|
147
|
+
- `process(..., detach=True)` runs in its own process: `job.follow()`, `useml data jobs`.
|
|
148
|
+
- Anything longer than ~2 s shows a tqdm bar or a "⏳ … / ✓ … (12.3s)" line.
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
useml data ls # named datasets
|
|
152
|
+
useml data tables # tables, size, who uses them
|
|
153
|
+
useml data rm photos-clip --table
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
## Configs in YAML, objects in one line
|
|
157
|
+
|
|
158
|
+
```toml
|
|
159
|
+
# pyproject.toml of your project
|
|
160
|
+
[tool.useml.config]
|
|
161
|
+
root = "configs" # configs/models/*.yaml, configs/scorers/*.yaml
|
|
162
|
+
packages = ["mypkg.models"] # where `type:` names are looked up (no imports by string)
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
```yaml
|
|
166
|
+
# configs/models/resnet_ft.yaml
|
|
167
|
+
meta: {display_name: ResNet-50 (fine-tuned)}
|
|
168
|
+
type: Classifier
|
|
169
|
+
backbone: {ref: resnet50} # another config, built once and shared
|
|
170
|
+
optimizer: {type: SGD, lr: 0.01}
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
from useml import config
|
|
175
|
+
model = config.load("resnet_ft") # memoised by name
|
|
176
|
+
model = config.load("resnet_ft", optimizer={"lr": 0.1}) # one setting, merged
|
|
177
|
+
model.config.hash, model.config.provenance() # cite what ran
|
|
178
|
+
config.create("variant", model); config.diff("resnet_ft", model) # and back to a file
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Misspelled keys and types raise with the closest name and the place in the file.
|
|
182
|
+
|
|
183
|
+
## Experiences, and their outputs in one line
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
from useml import exp, config, data
|
|
187
|
+
|
|
188
|
+
with exp.experience("vision/baseline") as e: # nothing runs in the block
|
|
189
|
+
trained = e.step(train, data.load("photos-clip"), config.load("classifier"),
|
|
190
|
+
epochs=10, name="model")
|
|
191
|
+
e.step(report, trained, name="report")
|
|
192
|
+
# block end: declaration written, run detached, followed with a progress bar
|
|
193
|
+
|
|
194
|
+
model = exp.get("vision/baseline/model") # later, anywhere: the trained model
|
|
195
|
+
exp.open("vision/baseline").results() # every step's numbers
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
- A step is an importable function. Arguments are recipes: outputs of earlier steps,
|
|
199
|
+
`config.load(...)` objects, `data.load(name)` datasets, outputs of other experiences.
|
|
200
|
+
- Fingerprints chain: running the block again skips what is done; changing a line reruns
|
|
201
|
+
only what depends on it. An interrupted run resumes.
|
|
202
|
+
- Outputs are saved by type: a model built from a config is kept as config + weights and
|
|
203
|
+
comes back rebuilt; arrays as `.npz`; figures as `.png`; numbers as JSON.
|
|
204
|
+
- A step with a `ctx` parameter gets its folder, `ctx.progress(i, n)`, `ctx.checkpoints`
|
|
205
|
+
and `ctx.should_stop()`.
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
useml exp ls
|
|
209
|
+
useml exp show vision/baseline
|
|
210
|
+
useml exp follow vision/baseline
|
|
211
|
+
useml exp stop vision/baseline
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### Runs that survive, crashes that explain themselves
|
|
215
|
+
|
|
216
|
+
A detached run lives in a tmux session (`useml exp attach <name>`), or in its own session
|
|
217
|
+
without tmux: closing the laptop, the SSH connection or the notebook does not stop it.
|
|
218
|
+
A small supervisor process is its parent and records how it ended — exit code, signal
|
|
219
|
+
(SIGKILL ≈ out of RAM, SIGSEGV = native crash), peak RAM, kernel OOM lines, traceback,
|
|
220
|
+
last log lines:
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
useml exp why vision/baseline # where it is, or why it died
|
|
224
|
+
useml exp stack vision/baseline # the Python stack of a run that seems stuck
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
```toml
|
|
228
|
+
# ~/.config/useml/config.toml — optional: be told when a run ends or crashes
|
|
229
|
+
[exp]
|
|
230
|
+
notify = "your-command $USEML_EXP_NAME $USEML_EXP_STATUS"
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
### Bundles: archive the version that worked
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
useml exp bundle vision/baseline -o archive/baseline-final [--archive]
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
A standalone folder: the code exactly as it ran (commit + uncommitted diff + untracked
|
|
240
|
+
sources, recorded at every launch), a copy of useml, the configs, every step's output,
|
|
241
|
+
the rows of the datasets it read, `packages.txt`, and sha256 checksums. `python load.py`
|
|
242
|
+
/ `from load import get` reads an output; `python run.py --from-scratch` recomputes it.
|
useml-1.0.1/README.md
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# useml
|
|
2
|
+
|
|
3
|
+
One root for every ML cache, one gate for every download.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install "useml[all]"
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Quick start
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install "useml[all]"
|
|
13
|
+
cd my-project
|
|
14
|
+
useml init # shows what it would set up (adds, never overwrites); confirm to write
|
|
15
|
+
useml check # checks that everything resolves
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## One root per machine
|
|
19
|
+
|
|
20
|
+
```toml
|
|
21
|
+
# ~/.config/useml/config.toml
|
|
22
|
+
root = "/data/me/useml"
|
|
23
|
+
|
|
24
|
+
[hub]
|
|
25
|
+
parallel = false # default: one download thread at a time
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
`import useml` (first, before torch / transformers) points `HF_HOME`, `TORCH_HOME`,
|
|
29
|
+
`TMPDIR`, pip, triton… under the root. A cache already placed outside the home is never
|
|
30
|
+
moved. `useml paths` shows where everything goes.
|
|
31
|
+
|
|
32
|
+
## One gate for downloads
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
import useml
|
|
36
|
+
|
|
37
|
+
useml.hub.get("hf://openai/clip-vit-large-patch14")
|
|
38
|
+
useml.hub.get("hf://datasets/org/name@main/file.json")
|
|
39
|
+
useml.hub.get("https://example.org/annotations.zip", extract_archive=True)
|
|
40
|
+
useml.hub.get("gdrive://<file id>")
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Already downloaded → answered from disk, no network. Every download is recorded:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
useml hub ls # what is on disk, how big, last used
|
|
47
|
+
useml hub scan # caches left in the home (~/.cache/huggingface…)
|
|
48
|
+
useml hub scan --move --apply
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## The network policy
|
|
52
|
+
|
|
53
|
+
By default, only one thread may download at a time, in every library:
|
|
54
|
+
|
|
55
|
+
- Hugging Face `xet` (Rust, its own parallel connections) is disabled;
|
|
56
|
+
- the thread pools of `huggingface_hub` and `datasets` become sequential;
|
|
57
|
+
- `socket.connect` refuses an outbound connection from a second thread
|
|
58
|
+
(`ParallelDownloadError`), and logs it;
|
|
59
|
+
- `useml hub watch -- python train.py` watches the whole process tree from outside
|
|
60
|
+
and kills it on a sustained violation.
|
|
61
|
+
|
|
62
|
+
`useml.hub.allow_parallel()` lifts it, explicitly.
|
|
63
|
+
|
|
64
|
+
## One cache store
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
store = useml.store.open() # <root>/store
|
|
68
|
+
masks = store.declare("processing/masks", cost="expensive")
|
|
69
|
+
masks.put({"image": digest, "model": "sam-h"}, {"masks": array})
|
|
70
|
+
masks.get({"image": digest, "model": "sam-h"})
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
One SQLite index (WAL, safe across processes), large values in files written atomically,
|
|
74
|
+
one write per key. Spaces are declared with a cost (`cheap`, `expensive`, `precious`)
|
|
75
|
+
kept in the index, so cleaning needs no project code:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
useml store spaces
|
|
79
|
+
useml store gc --max-size 450GB # dry run: cheapest to recompute first
|
|
80
|
+
useml store gc --max-size 450GB --apply
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`[store] max_size = "450GB"` in the config sets the default budget. Nothing is ever
|
|
84
|
+
evicted automatically, pinned entries and `precious` spaces are kept.
|
|
85
|
+
|
|
86
|
+
## Data processed once
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from useml import data
|
|
90
|
+
|
|
91
|
+
@data.processor("clip-image", version=1, batch_size=64, dtype="float16")
|
|
92
|
+
class ClipImage:
|
|
93
|
+
def __init__(self, model="openai/clip-vit-base-patch32", device="cuda"): ...
|
|
94
|
+
def load(self, path): ... # per item, in worker threads
|
|
95
|
+
def __call__(self, images): ... # a batch → (B, D) array, or a dict of arrays
|
|
96
|
+
|
|
97
|
+
data.process(data.images("/data/photos"), ClipImage, name="photos-clip")
|
|
98
|
+
feats = data.load("photos-clip") # one line; nothing is read until used
|
|
99
|
+
feats[0], feats[100:200], feats.numpy()
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
- One table per (processor, version, parameters, dtype), shared by every dataset: rows
|
|
103
|
+
are found by a hash of their CONTENT, so an image two datasets share is computed once.
|
|
104
|
+
- Only missing rows are computed; when nothing is missing, the model is never loaded.
|
|
105
|
+
- Storage `float16` (default), `float32` or `int8` (one byte per number + a scale per row).
|
|
106
|
+
- Memory-mapped, append-only shards; one writer per table (a file lock); an interrupted
|
|
107
|
+
run keeps its rows and resumes.
|
|
108
|
+
- The size to write is checked against the free disk and `[data] max_size` before writing.
|
|
109
|
+
- `process(..., detach=True)` runs in its own process: `job.follow()`, `useml data jobs`.
|
|
110
|
+
- Anything longer than ~2 s shows a tqdm bar or a "⏳ … / ✓ … (12.3s)" line.
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
useml data ls # named datasets
|
|
114
|
+
useml data tables # tables, size, who uses them
|
|
115
|
+
useml data rm photos-clip --table
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Configs in YAML, objects in one line
|
|
119
|
+
|
|
120
|
+
```toml
|
|
121
|
+
# pyproject.toml of your project
|
|
122
|
+
[tool.useml.config]
|
|
123
|
+
root = "configs" # configs/models/*.yaml, configs/scorers/*.yaml
|
|
124
|
+
packages = ["mypkg.models"] # where `type:` names are looked up (no imports by string)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
```yaml
|
|
128
|
+
# configs/models/resnet_ft.yaml
|
|
129
|
+
meta: {display_name: ResNet-50 (fine-tuned)}
|
|
130
|
+
type: Classifier
|
|
131
|
+
backbone: {ref: resnet50} # another config, built once and shared
|
|
132
|
+
optimizer: {type: SGD, lr: 0.01}
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from useml import config
|
|
137
|
+
model = config.load("resnet_ft") # memoised by name
|
|
138
|
+
model = config.load("resnet_ft", optimizer={"lr": 0.1}) # one setting, merged
|
|
139
|
+
model.config.hash, model.config.provenance() # cite what ran
|
|
140
|
+
config.create("variant", model); config.diff("resnet_ft", model) # and back to a file
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Misspelled keys and types raise with the closest name and the place in the file.
|
|
144
|
+
|
|
145
|
+
## Experiences, and their outputs in one line
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from useml import exp, config, data
|
|
149
|
+
|
|
150
|
+
with exp.experience("vision/baseline") as e: # nothing runs in the block
|
|
151
|
+
trained = e.step(train, data.load("photos-clip"), config.load("classifier"),
|
|
152
|
+
epochs=10, name="model")
|
|
153
|
+
e.step(report, trained, name="report")
|
|
154
|
+
# block end: declaration written, run detached, followed with a progress bar
|
|
155
|
+
|
|
156
|
+
model = exp.get("vision/baseline/model") # later, anywhere: the trained model
|
|
157
|
+
exp.open("vision/baseline").results() # every step's numbers
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
- A step is an importable function. Arguments are recipes: outputs of earlier steps,
|
|
161
|
+
`config.load(...)` objects, `data.load(name)` datasets, outputs of other experiences.
|
|
162
|
+
- Fingerprints chain: running the block again skips what is done; changing a line reruns
|
|
163
|
+
only what depends on it. An interrupted run resumes.
|
|
164
|
+
- Outputs are saved by type: a model built from a config is kept as config + weights and
|
|
165
|
+
comes back rebuilt; arrays as `.npz`; figures as `.png`; numbers as JSON.
|
|
166
|
+
- A step with a `ctx` parameter gets its folder, `ctx.progress(i, n)`, `ctx.checkpoints`
|
|
167
|
+
and `ctx.should_stop()`.
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
useml exp ls
|
|
171
|
+
useml exp show vision/baseline
|
|
172
|
+
useml exp follow vision/baseline
|
|
173
|
+
useml exp stop vision/baseline
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
### Runs that survive, crashes that explain themselves
|
|
177
|
+
|
|
178
|
+
A detached run lives in a tmux session (`useml exp attach <name>`), or in its own session
|
|
179
|
+
without tmux: closing the laptop, the SSH connection or the notebook does not stop it.
|
|
180
|
+
A small supervisor process is its parent and records how it ended — exit code, signal
|
|
181
|
+
(SIGKILL ≈ out of RAM, SIGSEGV = native crash), peak RAM, kernel OOM lines, traceback,
|
|
182
|
+
last log lines:
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
useml exp why vision/baseline # where it is, or why it died
|
|
186
|
+
useml exp stack vision/baseline # the Python stack of a run that seems stuck
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
```toml
|
|
190
|
+
# ~/.config/useml/config.toml — optional: be told when a run ends or crashes
|
|
191
|
+
[exp]
|
|
192
|
+
notify = "your-command $USEML_EXP_NAME $USEML_EXP_STATUS"
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
### Bundles: archive the version that worked
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
useml exp bundle vision/baseline -o archive/baseline-final [--archive]
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
A standalone folder: the code exactly as it ran (commit + uncommitted diff + untracked
|
|
202
|
+
sources, recorded at every launch), a copy of useml, the configs, every step's output,
|
|
203
|
+
the rows of the datasets it read, `packages.txt`, and sha256 checksums. `python load.py`
|
|
204
|
+
/ `from load import get` reads an output; `python run.py --from-scratch` recomputes it.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.18"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "useml"
|
|
7
|
+
version = "1.0.1"
|
|
8
|
+
description = "One root for every ML cache, one gate for every download, data processed once."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Enzo-py", email = "85191552+Enzo-py@users.noreply.github.com" }]
|
|
14
|
+
keywords = ["machine learning", "deep learning", "cache", "experiments", "datasets",
|
|
15
|
+
"huggingface", "reproducibility"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Operating System :: POSIX :: Linux",
|
|
20
|
+
"Operating System :: MacOS",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
26
|
+
]
|
|
27
|
+
dependencies = [
|
|
28
|
+
"numpy>=1.22",
|
|
29
|
+
"tqdm>=4.60",
|
|
30
|
+
"pyyaml>=6",
|
|
31
|
+
"tomli>=1.1; python_version < '3.11'",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/Enzo-py/useml"
|
|
36
|
+
"Bug Tracker" = "https://github.com/Enzo-py/useml/issues"
|
|
37
|
+
|
|
38
|
+
[project.optional-dependencies]
|
|
39
|
+
hf = ["huggingface_hub>=0.20"]
|
|
40
|
+
gdrive = ["gdown>=4.6"]
|
|
41
|
+
watch = ["psutil>=5.9"]
|
|
42
|
+
all = ["huggingface_hub>=0.20", "gdown>=4.6", "psutil>=5.9"]
|
|
43
|
+
test = ["pytest>=7"]
|
|
44
|
+
|
|
45
|
+
[project.scripts]
|
|
46
|
+
useml = "useml.cli:main"
|
|
47
|
+
|
|
48
|
+
[tool.hatch.build.targets.wheel]
|
|
49
|
+
packages = ["src/useml"]
|
|
50
|
+
|
|
51
|
+
# Only what users need: never local notes.
|
|
52
|
+
[tool.hatch.build.targets.sdist]
|
|
53
|
+
include = ["src/useml", "tests", "README.md", "LICENSE", "pyproject.toml"]
|
|
54
|
+
|
|
55
|
+
[tool.pytest.ini_options]
|
|
56
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""useml — one root for every ML cache, one gate for every download.
|
|
2
|
+
|
|
3
|
+
IMPORTING useml HAS TWO EFFECTS, on purpose, and only these two:
|
|
4
|
+
|
|
5
|
+
1. `useml.paths.apply()` — if a root is configured, every library cache variable
|
|
6
|
+
(HF_HOME, TORCH_HOME, TMPDIR…) points under it. Without a root, nothing changes.
|
|
7
|
+
2. `useml.hub.apply_configured_policy()` — downloads become sequential (one thread
|
|
8
|
+
at a time), unless the machine config says `[hub] parallel = true`.
|
|
9
|
+
|
|
10
|
+
Import useml FIRST, before torch / transformers / open_clip, so they read the right paths.
|
|
11
|
+
If they were imported earlier, `paths.apply()` realigns huggingface_hub's paths anyway.
|
|
12
|
+
|
|
13
|
+
Set `USEML_NO_AUTO=1` to import without either effect (then call them yourself).
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import os as _os
|
|
18
|
+
|
|
19
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
__version__ = version("useml")
|
|
23
|
+
except PackageNotFoundError: # pragma: no cover - source checkout
|
|
24
|
+
__version__ = "dev"
|
|
25
|
+
|
|
26
|
+
from . import paths, hub, store
|
|
27
|
+
|
|
28
|
+
if _os.environ.get("USEML_NO_AUTO", "").strip() not in ("1", "true", "yes"):
|
|
29
|
+
paths.apply()
|
|
30
|
+
hub.apply_configured_policy()
|
|
31
|
+
|
|
32
|
+
__all__ = ["paths", "hub", "store", "data", "config", "exp", "__version__"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def __getattr__(name):
|
|
36
|
+
"""`useml.data` and `useml.config` are imported on first use: they need numpy / PyYAML,
|
|
37
|
+
`import useml` does not."""
|
|
38
|
+
if name in ("data", "config", "exp"):
|
|
39
|
+
import importlib
|
|
40
|
+
|
|
41
|
+
return importlib.import_module(f".{name}", __name__)
|
|
42
|
+
raise AttributeError(f"module 'useml' has no attribute {name!r}")
|