useml 0.3.2__tar.gz → 1.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. useml-1.0.2/.gitignore +33 -0
  2. useml-1.0.2/LICENSE +21 -0
  3. useml-1.0.2/PKG-INFO +256 -0
  4. useml-1.0.2/README.md +218 -0
  5. useml-1.0.2/pyproject.toml +56 -0
  6. useml-1.0.2/src/useml/__init__.py +42 -0
  7. useml-1.0.2/src/useml/__main__.py +5 -0
  8. useml-1.0.2/src/useml/_progress.py +90 -0
  9. useml-1.0.2/src/useml/_project.py +45 -0
  10. useml-1.0.2/src/useml/_setup.py +468 -0
  11. useml-1.0.2/src/useml/cli.py +515 -0
  12. useml-1.0.2/src/useml/config/__init__.py +127 -0
  13. useml-1.0.2/src/useml/config/_config.py +201 -0
  14. useml-1.0.2/src/useml/config/_git.py +81 -0
  15. useml-1.0.2/src/useml/config/_library.py +357 -0
  16. useml-1.0.2/src/useml/config/_types.py +111 -0
  17. useml-1.0.2/src/useml/config/_write.py +248 -0
  18. useml-1.0.2/src/useml/data/__init__.py +39 -0
  19. useml-1.0.2/src/useml/data/_catalog.py +148 -0
  20. useml-1.0.2/src/useml/data/_jobs.py +227 -0
  21. useml-1.0.2/src/useml/data/_keys.py +53 -0
  22. useml-1.0.2/src/useml/data/_processor.py +151 -0
  23. useml-1.0.2/src/useml/data/_run.py +240 -0
  24. useml-1.0.2/src/useml/data/_sources.py +218 -0
  25. useml-1.0.2/src/useml/data/_table.py +288 -0
  26. useml-1.0.2/src/useml/data/_view.py +138 -0
  27. useml-1.0.2/src/useml/docs/SKILL.md +63 -0
  28. useml-1.0.2/src/useml/docs/adopt.md +95 -0
  29. useml-1.0.2/src/useml/docs/config.md +93 -0
  30. useml-1.0.2/src/useml/docs/data.md +125 -0
  31. useml-1.0.2/src/useml/docs/exp.md +172 -0
  32. useml-1.0.2/src/useml/docs/faq.md +53 -0
  33. useml-1.0.2/src/useml/docs/index.md +36 -0
  34. useml-1.0.2/src/useml/docs/paths-and-hub.md +89 -0
  35. useml-1.0.2/src/useml/docs/quickstart.md +71 -0
  36. useml-1.0.2/src/useml/docs/reference.md +116 -0
  37. useml-1.0.2/src/useml/docs/store.md +88 -0
  38. useml-1.0.2/src/useml/docs/troubleshooting.md +89 -0
  39. useml-1.0.2/src/useml/exp/__init__.py +28 -0
  40. useml-1.0.2/src/useml/exp/_bundle.py +421 -0
  41. useml-1.0.2/src/useml/exp/_experience.py +472 -0
  42. useml-1.0.2/src/useml/exp/_launch.py +222 -0
  43. useml-1.0.2/src/useml/exp/_outputs.py +162 -0
  44. useml-1.0.2/src/useml/exp/_provenance.py +159 -0
  45. useml-1.0.2/src/useml/exp/_refs.py +242 -0
  46. useml-1.0.2/src/useml/exp/_runtime.py +303 -0
  47. useml-1.0.2/src/useml/exp/_supervise.py +252 -0
  48. useml-1.0.2/src/useml/hub/__init__.py +23 -0
  49. useml-1.0.2/src/useml/hub/_fetch.py +321 -0
  50. useml-1.0.2/src/useml/hub/_policy.py +377 -0
  51. useml-1.0.2/src/useml/hub/_registry.py +115 -0
  52. useml-1.0.2/src/useml/hub/_scan.py +107 -0
  53. useml-1.0.2/src/useml/hub/_watch.py +157 -0
  54. useml-1.0.2/src/useml/paths.py +331 -0
  55. useml-1.0.2/src/useml/store/__init__.py +65 -0
  56. useml-1.0.2/src/useml/store/_maintenance.py +253 -0
  57. useml-1.0.2/src/useml/store/_spaces.py +67 -0
  58. useml-1.0.2/src/useml/store/_store.py +676 -0
  59. useml-1.0.2/tests/conftest.py +47 -0
  60. useml-1.0.2/tests/data_procs.py +54 -0
  61. useml-1.0.2/tests/exp_steps.py +117 -0
  62. useml-1.0.2/tests/test_config.py +389 -0
  63. useml-1.0.2/tests/test_data.py +302 -0
  64. useml-1.0.2/tests/test_docs.py +95 -0
  65. useml-1.0.2/tests/test_exp.py +228 -0
  66. useml-1.0.2/tests/test_exp_robust.py +276 -0
  67. useml-1.0.2/tests/test_fetch.py +179 -0
  68. useml-1.0.2/tests/test_paths.py +97 -0
  69. useml-1.0.2/tests/test_policy.py +200 -0
  70. useml-1.0.2/tests/test_setup.py +165 -0
  71. useml-1.0.2/tests/test_store.py +342 -0
  72. useml-0.3.2/PKG-INFO +0 -471
  73. useml-0.3.2/README.md +0 -454
  74. useml-0.3.2/pyproject.toml +0 -30
  75. useml-0.3.2/setup.cfg +0 -4
  76. useml-0.3.2/tests/test_bundle.py +0 -366
  77. useml-0.3.2/tests/test_level2.py +0 -244
  78. useml-0.3.2/tests/test_session.py +0 -208
  79. useml-0.3.2/tests/test_train_level0.py +0 -234
  80. useml-0.3.2/tests/test_usecases.py +0 -1249
  81. useml-0.3.2/tests/test_vault.py +0 -243
  82. useml-0.3.2/useml/__init__.py +0 -94
  83. useml-0.3.2/useml/dataset/__init__.py +0 -4
  84. useml-0.3.2/useml/dataset/bundle.py +0 -288
  85. useml-0.3.2/useml/dataset/loaders.py +0 -168
  86. useml-0.3.2/useml/errors.py +0 -377
  87. useml-0.3.2/useml/imports.py +0 -215
  88. useml-0.3.2/useml/session/__init__.py +0 -4
  89. useml-0.3.2/useml/session/component.py +0 -59
  90. useml-0.3.2/useml/session/manager.py +0 -482
  91. useml-0.3.2/useml/template/__init__.py +0 -6
  92. useml-0.3.2/useml/template/config.py +0 -140
  93. useml-0.3.2/useml/template/loss.py +0 -28
  94. useml-0.3.2/useml/template/model.py +0 -28
  95. useml-0.3.2/useml/template/trainer.py +0 -758
  96. useml-0.3.2/useml/vault/__init__.py +0 -5
  97. useml-0.3.2/useml/vault/code_extractor.py +0 -261
  98. useml-0.3.2/useml/vault/core.py +0 -63
  99. useml-0.3.2/useml/vault/model_registry.py +0 -267
  100. useml-0.3.2/useml/vault/project.py +0 -309
  101. useml-0.3.2/useml/vault/runs.py +0 -814
  102. useml-0.3.2/useml/vault/snapshot.py +0 -170
  103. useml-0.3.2/useml/workdir/__init__.py +0 -14
  104. useml-0.3.2/useml/workdir/_hook.py +0 -79
  105. useml-0.3.2/useml.egg-info/PKG-INFO +0 -471
  106. useml-0.3.2/useml.egg-info/SOURCES.txt +0 -36
  107. useml-0.3.2/useml.egg-info/dependency_links.txt +0 -1
  108. useml-0.3.2/useml.egg-info/requires.txt +0 -8
  109. useml-0.3.2/useml.egg-info/top_level.txt +0 -1
useml-1.0.2/.gitignore ADDED
@@ -0,0 +1,33 @@
1
+ # --- Python Build & Distribution ---
2
+ dist/
3
+ build/
4
+ *.egg-info/
5
+ egg-info/
6
+ nosetests.xml
7
+ pip-wheel-metadata/
8
+ .pytest_cache
9
+
10
+ # --- Virtual Environments ---
11
+ venv/
12
+ .venv/
13
+ env/
14
+
15
+ # --- Compiled Files ---
16
+ __pycache__/
17
+ *.py[cod]
18
+ *$py.class
19
+
20
+ # --- Machine Learning specifics (Vault/Data) ---
21
+ *.pt
22
+ *.pth
23
+ *.ckpt
24
+
25
+ # IDE / OS
26
+ .DS_Store
27
+ .claude/
28
+
29
+ # testing
30
+ playground
31
+
32
+ # --- useml v1 ---
33
+ bundles/
useml-1.0.2/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Enzo-py
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
useml-1.0.2/PKG-INFO ADDED
@@ -0,0 +1,256 @@
1
+ Metadata-Version: 2.5
2
+ Name: useml
3
+ Version: 1.0.2
4
+ Summary: One root for every ML cache, one gate for every download, data processed once.
5
+ Project-URL: Homepage, https://github.com/Enzo-py/useml
6
+ Project-URL: Bug Tracker, https://github.com/Enzo-py/useml/issues
7
+ Author-email: Enzo-py <85191552+Enzo-py@users.noreply.github.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: cache,datasets,deep learning,experiments,huggingface,machine learning,reproducibility
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: MacOS
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
20
+ Requires-Python: >=3.10
21
+ Requires-Dist: numpy>=1.22
22
+ Requires-Dist: pyyaml>=6
23
+ Requires-Dist: tomli>=1.1; python_version < '3.11'
24
+ Requires-Dist: tqdm>=4.60
25
+ Provides-Extra: all
26
+ Requires-Dist: gdown>=4.6; extra == 'all'
27
+ Requires-Dist: huggingface-hub>=0.20; extra == 'all'
28
+ Requires-Dist: psutil>=5.9; extra == 'all'
29
+ Provides-Extra: gdrive
30
+ Requires-Dist: gdown>=4.6; extra == 'gdrive'
31
+ Provides-Extra: hf
32
+ Requires-Dist: huggingface-hub>=0.20; extra == 'hf'
33
+ Provides-Extra: test
34
+ Requires-Dist: pytest>=7; extra == 'test'
35
+ Provides-Extra: watch
36
+ Requires-Dist: psutil>=5.9; extra == 'watch'
37
+ Description-Content-Type: text/markdown
38
+
39
+ # useml
40
+
41
+ A toolkit for machine-learning projects on big machines: one root for every cache, one
42
+ gate for every download, datasets processed once and loaded by name, YAML configs, and
43
+ experiments that run in the background, survive a closed laptop, and say why they crashed.
44
+
45
+ ## Quick start
46
+
47
+ ```bash
48
+ pip install "useml[all]"
49
+ cd my-project
50
+ useml init # shows what it would set up (adds, never overwrites); confirm to write
51
+ useml check # checks that everything resolves
52
+ useml docs # the full documentation, installed with the package
53
+ ```
54
+
55
+ ## Documentation
56
+
57
+ The guides are in [`src/useml/docs/`](src/useml/docs/index.md) and installed with the
58
+ package (`useml docs <topic>`):
59
+
60
+ - [Quickstart](src/useml/docs/quickstart.md) · [Adopt useml in a project](src/useml/docs/adopt.md)
61
+ - [Caches and downloads](src/useml/docs/paths-and-hub.md) · [Cache store](src/useml/docs/store.md)
62
+ · [Processed data](src/useml/docs/data.md) · [Configs](src/useml/docs/config.md)
63
+ · [Experiences](src/useml/docs/exp.md)
64
+ - [Troubleshooting](src/useml/docs/troubleshooting.md) · [Reference](src/useml/docs/reference.md)
65
+ · [FAQ](src/useml/docs/faq.md)
66
+ - [Instructions for AI coding assistants](src/useml/docs/SKILL.md)
67
+
68
+ The sections below are a tour.
69
+
70
+ ## One root per machine
71
+
72
+ ```toml
73
+ # ~/.config/useml/config.toml
74
+ root = "/data/me/useml"
75
+
76
+ [hub]
77
+ parallel = false # default: one download thread at a time
78
+ ```
79
+
80
+ `import useml` (first, before torch / transformers) points `HF_HOME`, `TORCH_HOME`,
81
+ `TMPDIR`, pip, triton… under the root. A cache already placed outside the home is never
82
+ moved. `useml paths` shows where everything goes.
83
+
84
+ ## One gate for downloads
85
+
86
+ ```python
87
+ import useml
88
+
89
+ useml.hub.get("hf://openai/clip-vit-large-patch14")
90
+ useml.hub.get("hf://datasets/org/name@main/file.json")
91
+ useml.hub.get("https://example.org/annotations.zip", extract_archive=True)
92
+ useml.hub.get("gdrive://<file id>")
93
+ ```
94
+
95
+ Already downloaded → answered from disk, no network. Every download is recorded:
96
+
97
+ ```bash
98
+ useml hub ls # what is on disk, how big, last used
99
+ useml hub scan # caches left in the home (~/.cache/huggingface…)
100
+ useml hub scan --move --apply
101
+ ```
102
+
103
+ ## The network policy
104
+
105
+ By default, only one thread may download at a time, in every library:
106
+
107
+ - Hugging Face `xet` (Rust, its own parallel connections) is disabled;
108
+ - the thread pools of `huggingface_hub` and `datasets` become sequential;
109
+ - `socket.connect` refuses an outbound connection from a second thread
110
+ (`ParallelDownloadError`), and logs it;
111
+ - `useml hub watch -- python train.py` watches the whole process tree from outside
112
+ and kills it on a sustained violation.
113
+
114
+ `useml.hub.allow_parallel()` lifts it, explicitly.
115
+
116
+ ## One cache store
117
+
118
+ ```python
119
+ store = useml.store.open() # <root>/store
120
+ masks = store.declare("processing/masks", cost="expensive")
121
+ masks.put({"image": digest, "model": "sam-h"}, {"masks": array})
122
+ masks.get({"image": digest, "model": "sam-h"})
123
+ ```
124
+
125
+ One SQLite index (WAL, safe across processes), large values in files written atomically,
126
+ one write per key. Spaces are declared with a cost (`cheap`, `expensive`, `precious`)
127
+ kept in the index, so cleaning needs no project code:
128
+
129
+ ```bash
130
+ useml store spaces
131
+ useml store gc --max-size 450GB # dry run: cheapest to recompute first
132
+ useml store gc --max-size 450GB --apply
133
+ ```
134
+
135
+ `[store] max_size = "450GB"` in the config sets the default budget. Nothing is ever
136
+ evicted automatically, pinned entries and `precious` spaces are kept.
137
+
138
+ ## Data processed once
139
+
140
+ ```python
141
+ from useml import data
142
+
143
+ @data.processor("clip-image", version=1, batch_size=64, dtype="float16")
144
+ class ClipImage:
145
+ def __init__(self, model="openai/clip-vit-base-patch32", device="cuda"): ...
146
+ def load(self, path): ... # per item, in worker threads
147
+ def __call__(self, images): ... # a batch → (B, D) array, or a dict of arrays
148
+
149
+ data.process(data.images("/data/photos"), ClipImage, name="photos-clip")
150
+ feats = data.load("photos-clip") # one line; nothing is read until used
151
+ feats[0], feats[100:200], feats.numpy()
152
+ ```
153
+
154
+ - One table per (processor, version, parameters, dtype), shared by every dataset: rows
155
+ are found by a hash of their CONTENT, so an image two datasets share is computed once.
156
+ - Only missing rows are computed; when nothing is missing, the model is never loaded.
157
+ - Storage `float16` (default), `float32` or `int8` (one byte per number + a scale per row).
158
+ - Memory-mapped, append-only shards; one writer per table (a file lock); an interrupted
159
+ run keeps its rows and resumes.
160
+ - The size to write is checked against the free disk and `[data] max_size` before writing.
161
+ - `process(..., detach=True)` runs in its own process: `job.follow()`, `useml data jobs`.
162
+ - Anything longer than ~2 s shows a tqdm bar or a "⏳ … / ✓ … (12.3s)" line.
163
+
164
+ ```bash
165
+ useml data ls # named datasets
166
+ useml data tables # tables, size, who uses them
167
+ useml data rm photos-clip --table
168
+ ```
169
+
170
+ ## Configs in YAML, objects in one line
171
+
172
+ ```toml
173
+ # pyproject.toml of your project
174
+ [tool.useml.config]
175
+ root = "configs" # configs/models/*.yaml, configs/scorers/*.yaml
176
+ packages = ["mypkg.models"] # where `type:` names are looked up (no imports by string)
177
+ ```
178
+
179
+ ```yaml
180
+ # configs/models/resnet_ft.yaml
181
+ meta: {display_name: ResNet-50 (fine-tuned)}
182
+ type: Classifier
183
+ backbone: {ref: resnet50} # another config, built once and shared
184
+ optimizer: {type: SGD, lr: 0.01}
185
+ ```
186
+
187
+ ```python
188
+ from useml import config
189
+ model = config.load("resnet_ft") # memoised by name
190
+ model = config.load("resnet_ft", optimizer={"lr": 0.1}) # one setting, merged
191
+ model.config.hash, model.config.provenance() # cite what ran
192
+ config.create("variant", model); config.diff("resnet_ft", model) # and back to a file
193
+ ```
194
+
195
+ Misspelled keys and types raise with the closest name and the place in the file.
196
+
197
+ ## Experiences, and their outputs in one line
198
+
199
+ ```python
200
+ from useml import exp, config, data
201
+
202
+ with exp.experience("vision/baseline") as e: # nothing runs in the block
203
+ trained = e.step(train, data.load("photos-clip"), config.load("classifier"),
204
+ epochs=10, name="model")
205
+ e.step(report, trained, name="report")
206
+ # block end: declaration written, run detached, followed with a progress bar
207
+
208
+ model = exp.get("vision/baseline/model") # later, anywhere: the trained model
209
+ exp.open("vision/baseline").results() # every step's numbers
210
+ ```
211
+
212
+ - A step is an importable function. Arguments are recipes: outputs of earlier steps,
213
+ `config.load(...)` objects, `data.load(name)` datasets, outputs of other experiences.
214
+ - Fingerprints chain: running the block again skips what is done; changing a line reruns
215
+ only what depends on it. An interrupted run resumes.
216
+ - Outputs are saved by type: a model built from a config is kept as config + weights and
217
+ comes back rebuilt; arrays as `.npz`; figures as `.png`; numbers as JSON.
218
+ - A step with a `ctx` parameter gets its folder, `ctx.progress(i, n)`, `ctx.checkpoints`
219
+ and `ctx.should_stop()`.
220
+
221
+ ```bash
222
+ useml exp ls
223
+ useml exp show vision/baseline
224
+ useml exp follow vision/baseline
225
+ useml exp stop vision/baseline
226
+ ```
227
+
228
+ ### Runs that survive, crashes that explain themselves
229
+
230
+ A detached run lives in a tmux session (`useml exp attach <name>`), or in its own session
231
+ without tmux: closing the laptop, the SSH connection or the notebook does not stop it.
232
+ A small supervisor process is its parent and records how it ended — exit code, signal
233
+ (SIGKILL ≈ out of RAM, SIGSEGV = native crash), peak RAM, kernel OOM lines, traceback,
234
+ last log lines:
235
+
236
+ ```bash
237
+ useml exp why vision/baseline # where it is, or why it died
238
+ useml exp stack vision/baseline # the Python stack of a run that seems stuck
239
+ ```
240
+
241
+ ```toml
242
+ # ~/.config/useml/config.toml — optional: be told when a run ends or crashes
243
+ [exp]
244
+ notify = "your-command $USEML_EXP_NAME $USEML_EXP_STATUS"
245
+ ```
246
+
247
+ ### Bundles: archive the version that worked
248
+
249
+ ```bash
250
+ useml exp bundle vision/baseline -o archive/baseline-final [--archive]
251
+ ```
252
+
253
+ A standalone folder: the code exactly as it ran (commit + uncommitted diff + untracked
254
+ sources, recorded at every launch), a copy of useml, the configs, every step's output,
255
+ the rows of the datasets it read, `packages.txt`, and sha256 checksums. `python load.py`
256
+ / `from load import get` reads an output; `python run.py --from-scratch` recomputes it.
useml-1.0.2/README.md ADDED
@@ -0,0 +1,218 @@
1
+ # useml
2
+
3
+ A toolkit for machine-learning projects on big machines: one root for every cache, one
4
+ gate for every download, datasets processed once and loaded by name, YAML configs, and
5
+ experiments that run in the background, survive a closed laptop, and say why they crashed.
6
+
7
+ ## Quick start
8
+
9
+ ```bash
10
+ pip install "useml[all]"
11
+ cd my-project
12
+ useml init # shows what it would set up (adds, never overwrites); confirm to write
13
+ useml check # checks that everything resolves
14
+ useml docs # the full documentation, installed with the package
15
+ ```
16
+
17
+ ## Documentation
18
+
19
+ The guides are in [`src/useml/docs/`](src/useml/docs/index.md) and installed with the
20
+ package (`useml docs <topic>`):
21
+
22
+ - [Quickstart](src/useml/docs/quickstart.md) · [Adopt useml in a project](src/useml/docs/adopt.md)
23
+ - [Caches and downloads](src/useml/docs/paths-and-hub.md) · [Cache store](src/useml/docs/store.md)
24
+ · [Processed data](src/useml/docs/data.md) · [Configs](src/useml/docs/config.md)
25
+ · [Experiences](src/useml/docs/exp.md)
26
+ - [Troubleshooting](src/useml/docs/troubleshooting.md) · [Reference](src/useml/docs/reference.md)
27
+ · [FAQ](src/useml/docs/faq.md)
28
+ - [Instructions for AI coding assistants](src/useml/docs/SKILL.md)
29
+
30
+ The sections below are a tour.
31
+
32
+ ## One root per machine
33
+
34
+ ```toml
35
+ # ~/.config/useml/config.toml
36
+ root = "/data/me/useml"
37
+
38
+ [hub]
39
+ parallel = false # default: one download thread at a time
40
+ ```
41
+
42
+ `import useml` (first, before torch / transformers) points `HF_HOME`, `TORCH_HOME`,
43
+ `TMPDIR`, pip, triton… under the root. A cache already placed outside the home is never
44
+ moved. `useml paths` shows where everything goes.
45
+
46
+ ## One gate for downloads
47
+
48
+ ```python
49
+ import useml
50
+
51
+ useml.hub.get("hf://openai/clip-vit-large-patch14")
52
+ useml.hub.get("hf://datasets/org/name@main/file.json")
53
+ useml.hub.get("https://example.org/annotations.zip", extract_archive=True)
54
+ useml.hub.get("gdrive://<file id>")
55
+ ```
56
+
57
+ Already downloaded → answered from disk, no network. Every download is recorded:
58
+
59
+ ```bash
60
+ useml hub ls # what is on disk, how big, last used
61
+ useml hub scan # caches left in the home (~/.cache/huggingface…)
62
+ useml hub scan --move --apply
63
+ ```
64
+
65
+ ## The network policy
66
+
67
+ By default, only one thread may download at a time, in every library:
68
+
69
+ - Hugging Face `xet` (Rust, its own parallel connections) is disabled;
70
+ - the thread pools of `huggingface_hub` and `datasets` become sequential;
71
+ - `socket.connect` refuses an outbound connection from a second thread
72
+ (`ParallelDownloadError`), and logs it;
73
+ - `useml hub watch -- python train.py` watches the whole process tree from outside
74
+ and kills it on a sustained violation.
75
+
76
+ `useml.hub.allow_parallel()` lifts it, explicitly.
77
+
78
+ ## One cache store
79
+
80
+ ```python
81
+ store = useml.store.open() # <root>/store
82
+ masks = store.declare("processing/masks", cost="expensive")
83
+ masks.put({"image": digest, "model": "sam-h"}, {"masks": array})
84
+ masks.get({"image": digest, "model": "sam-h"})
85
+ ```
86
+
87
+ One SQLite index (WAL, safe across processes), large values in files written atomically,
88
+ one write per key. Spaces are declared with a cost (`cheap`, `expensive`, `precious`)
89
+ kept in the index, so cleaning needs no project code:
90
+
91
+ ```bash
92
+ useml store spaces
93
+ useml store gc --max-size 450GB # dry run: cheapest to recompute first
94
+ useml store gc --max-size 450GB --apply
95
+ ```
96
+
97
+ `[store] max_size = "450GB"` in the config sets the default budget. Nothing is ever
98
+ evicted automatically, pinned entries and `precious` spaces are kept.
99
+
100
+ ## Data processed once
101
+
102
+ ```python
103
+ from useml import data
104
+
105
+ @data.processor("clip-image", version=1, batch_size=64, dtype="float16")
106
+ class ClipImage:
107
+ def __init__(self, model="openai/clip-vit-base-patch32", device="cuda"): ...
108
+ def load(self, path): ... # per item, in worker threads
109
+ def __call__(self, images): ... # a batch → (B, D) array, or a dict of arrays
110
+
111
+ data.process(data.images("/data/photos"), ClipImage, name="photos-clip")
112
+ feats = data.load("photos-clip") # one line; nothing is read until used
113
+ feats[0], feats[100:200], feats.numpy()
114
+ ```
115
+
116
+ - One table per (processor, version, parameters, dtype), shared by every dataset: rows
117
+ are found by a hash of their CONTENT, so an image two datasets share is computed once.
118
+ - Only missing rows are computed; when nothing is missing, the model is never loaded.
119
+ - Storage `float16` (default), `float32` or `int8` (one byte per number + a scale per row).
120
+ - Memory-mapped, append-only shards; one writer per table (a file lock); an interrupted
121
+ run keeps its rows and resumes.
122
+ - The size to write is checked against the free disk and `[data] max_size` before writing.
123
+ - `process(..., detach=True)` runs in its own process: `job.follow()`, `useml data jobs`.
124
+ - Anything longer than ~2 s shows a tqdm bar or a "⏳ … / ✓ … (12.3s)" line.
125
+
126
+ ```bash
127
+ useml data ls # named datasets
128
+ useml data tables # tables, size, who uses them
129
+ useml data rm photos-clip --table
130
+ ```
131
+
132
+ ## Configs in YAML, objects in one line
133
+
134
+ ```toml
135
+ # pyproject.toml of your project
136
+ [tool.useml.config]
137
+ root = "configs" # configs/models/*.yaml, configs/scorers/*.yaml
138
+ packages = ["mypkg.models"] # where `type:` names are looked up (no imports by string)
139
+ ```
140
+
141
+ ```yaml
142
+ # configs/models/resnet_ft.yaml
143
+ meta: {display_name: ResNet-50 (fine-tuned)}
144
+ type: Classifier
145
+ backbone: {ref: resnet50} # another config, built once and shared
146
+ optimizer: {type: SGD, lr: 0.01}
147
+ ```
148
+
149
+ ```python
150
+ from useml import config
151
+ model = config.load("resnet_ft") # memoised by name
152
+ model = config.load("resnet_ft", optimizer={"lr": 0.1}) # one setting, merged
153
+ model.config.hash, model.config.provenance() # cite what ran
154
+ config.create("variant", model); config.diff("resnet_ft", model) # and back to a file
155
+ ```
156
+
157
+ Misspelled keys and types raise with the closest name and the place in the file.
158
+
159
+ ## Experiences, and their outputs in one line
160
+
161
+ ```python
162
+ from useml import exp, config, data
163
+
164
+ with exp.experience("vision/baseline") as e: # nothing runs in the block
165
+ trained = e.step(train, data.load("photos-clip"), config.load("classifier"),
166
+ epochs=10, name="model")
167
+ e.step(report, trained, name="report")
168
+ # block end: declaration written, run detached, followed with a progress bar
169
+
170
+ model = exp.get("vision/baseline/model") # later, anywhere: the trained model
171
+ exp.open("vision/baseline").results() # every step's numbers
172
+ ```
173
+
174
+ - A step is an importable function. Arguments are recipes: outputs of earlier steps,
175
+ `config.load(...)` objects, `data.load(name)` datasets, outputs of other experiences.
176
+ - Fingerprints chain: running the block again skips what is done; changing a line reruns
177
+ only what depends on it. An interrupted run resumes.
178
+ - Outputs are saved by type: a model built from a config is kept as config + weights and
179
+ comes back rebuilt; arrays as `.npz`; figures as `.png`; numbers as JSON.
180
+ - A step with a `ctx` parameter gets its folder, `ctx.progress(i, n)`, `ctx.checkpoints`
181
+ and `ctx.should_stop()`.
182
+
183
+ ```bash
184
+ useml exp ls
185
+ useml exp show vision/baseline
186
+ useml exp follow vision/baseline
187
+ useml exp stop vision/baseline
188
+ ```
189
+
190
+ ### Runs that survive, crashes that explain themselves
191
+
192
+ A detached run lives in a tmux session (`useml exp attach <name>`), or in its own session
193
+ without tmux: closing the laptop, the SSH connection or the notebook does not stop it.
194
+ A small supervisor process is its parent and records how it ended — exit code, signal
195
+ (SIGKILL ≈ out of RAM, SIGSEGV = native crash), peak RAM, kernel OOM lines, traceback,
196
+ last log lines:
197
+
198
+ ```bash
199
+ useml exp why vision/baseline # where it is, or why it died
200
+ useml exp stack vision/baseline # the Python stack of a run that seems stuck
201
+ ```
202
+
203
+ ```toml
204
+ # ~/.config/useml/config.toml — optional: be told when a run ends or crashes
205
+ [exp]
206
+ notify = "your-command $USEML_EXP_NAME $USEML_EXP_STATUS"
207
+ ```
208
+
209
+ ### Bundles: archive the version that worked
210
+
211
+ ```bash
212
+ useml exp bundle vision/baseline -o archive/baseline-final [--archive]
213
+ ```
214
+
215
+ A standalone folder: the code exactly as it ran (commit + uncommitted diff + untracked
216
+ sources, recorded at every launch), a copy of useml, the configs, every step's output,
217
+ the rows of the datasets it read, `packages.txt`, and sha256 checksums. `python load.py`
218
+ / `from load import get` reads an output; `python run.py --from-scratch` recomputes it.
@@ -0,0 +1,56 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.18"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "useml"
7
+ version = "1.0.2"
8
+ description = "One root for every ML cache, one gate for every download, data processed once."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Enzo-py", email = "85191552+Enzo-py@users.noreply.github.com" }]
14
+ keywords = ["machine learning", "deep learning", "cache", "experiments", "datasets",
15
+ "huggingface", "reproducibility"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Science/Research",
19
+ "Operating System :: POSIX :: Linux",
20
+ "Operating System :: MacOS",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3.10",
23
+ "Programming Language :: Python :: 3.11",
24
+ "Programming Language :: Python :: 3.12",
25
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
26
+ ]
27
+ dependencies = [
28
+ "numpy>=1.22",
29
+ "tqdm>=4.60",
30
+ "pyyaml>=6",
31
+ "tomli>=1.1; python_version < '3.11'",
32
+ ]
33
+
34
+ [project.urls]
35
+ Homepage = "https://github.com/Enzo-py/useml"
36
+ "Bug Tracker" = "https://github.com/Enzo-py/useml/issues"
37
+
38
+ [project.optional-dependencies]
39
+ hf = ["huggingface_hub>=0.20"]
40
+ gdrive = ["gdown>=4.6"]
41
+ watch = ["psutil>=5.9"]
42
+ all = ["huggingface_hub>=0.20", "gdown>=4.6", "psutil>=5.9"]
43
+ test = ["pytest>=7"]
44
+
45
+ [project.scripts]
46
+ useml = "useml.cli:main"
47
+
48
+ [tool.hatch.build.targets.wheel]
49
+ packages = ["src/useml"]
50
+
51
+ # Only what users need: never local notes.
52
+ [tool.hatch.build.targets.sdist]
53
+ include = ["src/useml", "tests", "README.md", "LICENSE", "pyproject.toml"]
54
+
55
+ [tool.pytest.ini_options]
56
+ testpaths = ["tests"]
@@ -0,0 +1,42 @@
1
+ """useml — one root for every ML cache, one gate for every download.
2
+
3
+ IMPORTING useml HAS TWO EFFECTS, on purpose, and only these two:
4
+
5
+ 1. `useml.paths.apply()` — if a root is configured, every library cache variable
6
+ (HF_HOME, TORCH_HOME, TMPDIR…) points under it. Without a root, nothing changes.
7
+ 2. `useml.hub.apply_configured_policy()` — downloads become sequential (one thread
8
+ at a time), unless the machine config says `[hub] parallel = true`.
9
+
10
+ Import useml FIRST, before torch / transformers / open_clip, so they read the right paths.
11
+ If they were imported earlier, `paths.apply()` realigns huggingface_hub's paths anyway.
12
+
13
+ Set `USEML_NO_AUTO=1` to import without either effect (then call them yourself).
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import os as _os
18
+
19
+ from importlib.metadata import PackageNotFoundError, version
20
+
21
+ try:
22
+ __version__ = version("useml")
23
+ except PackageNotFoundError: # pragma: no cover - source checkout
24
+ __version__ = "dev"
25
+
26
+ from . import paths, hub, store
27
+
28
+ if _os.environ.get("USEML_NO_AUTO", "").strip() not in ("1", "true", "yes"):
29
+ paths.apply()
30
+ hub.apply_configured_policy()
31
+
32
+ __all__ = ["paths", "hub", "store", "data", "config", "exp", "__version__"]
33
+
34
+
35
+ def __getattr__(name):
36
+ """`useml.data` and `useml.config` are imported on first use: they need numpy / PyYAML,
37
+ `import useml` does not."""
38
+ if name in ("data", "config", "exp"):
39
+ import importlib
40
+
41
+ return importlib.import_module(f".{name}", __name__)
42
+ raise AttributeError(f"module 'useml' has no attribute {name!r}")
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())