useml 0.3.2__tar.gz → 1.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. useml-1.0.1/.gitignore +33 -0
  2. useml-1.0.1/LICENSE +21 -0
  3. useml-1.0.1/PKG-INFO +242 -0
  4. useml-1.0.1/README.md +204 -0
  5. useml-1.0.1/pyproject.toml +56 -0
  6. useml-1.0.1/src/useml/__init__.py +42 -0
  7. useml-1.0.1/src/useml/__main__.py +5 -0
  8. useml-1.0.1/src/useml/_progress.py +90 -0
  9. useml-1.0.1/src/useml/_project.py +45 -0
  10. useml-1.0.1/src/useml/_setup.py +468 -0
  11. useml-1.0.1/src/useml/cli.py +474 -0
  12. useml-1.0.1/src/useml/config/__init__.py +127 -0
  13. useml-1.0.1/src/useml/config/_config.py +201 -0
  14. useml-1.0.1/src/useml/config/_git.py +81 -0
  15. useml-1.0.1/src/useml/config/_library.py +357 -0
  16. useml-1.0.1/src/useml/config/_types.py +111 -0
  17. useml-1.0.1/src/useml/config/_write.py +248 -0
  18. useml-1.0.1/src/useml/data/__init__.py +39 -0
  19. useml-1.0.1/src/useml/data/_catalog.py +148 -0
  20. useml-1.0.1/src/useml/data/_jobs.py +227 -0
  21. useml-1.0.1/src/useml/data/_keys.py +53 -0
  22. useml-1.0.1/src/useml/data/_processor.py +151 -0
  23. useml-1.0.1/src/useml/data/_run.py +240 -0
  24. useml-1.0.1/src/useml/data/_sources.py +218 -0
  25. useml-1.0.1/src/useml/data/_table.py +288 -0
  26. useml-1.0.1/src/useml/data/_view.py +138 -0
  27. useml-1.0.1/src/useml/exp/__init__.py +28 -0
  28. useml-1.0.1/src/useml/exp/_bundle.py +421 -0
  29. useml-1.0.1/src/useml/exp/_experience.py +472 -0
  30. useml-1.0.1/src/useml/exp/_launch.py +222 -0
  31. useml-1.0.1/src/useml/exp/_outputs.py +162 -0
  32. useml-1.0.1/src/useml/exp/_provenance.py +159 -0
  33. useml-1.0.1/src/useml/exp/_refs.py +242 -0
  34. useml-1.0.1/src/useml/exp/_runtime.py +303 -0
  35. useml-1.0.1/src/useml/exp/_supervise.py +252 -0
  36. useml-1.0.1/src/useml/hub/__init__.py +23 -0
  37. useml-1.0.1/src/useml/hub/_fetch.py +321 -0
  38. useml-1.0.1/src/useml/hub/_policy.py +377 -0
  39. useml-1.0.1/src/useml/hub/_registry.py +115 -0
  40. useml-1.0.1/src/useml/hub/_scan.py +107 -0
  41. useml-1.0.1/src/useml/hub/_watch.py +157 -0
  42. useml-1.0.1/src/useml/paths.py +331 -0
  43. useml-1.0.1/src/useml/store/__init__.py +65 -0
  44. useml-1.0.1/src/useml/store/_maintenance.py +253 -0
  45. useml-1.0.1/src/useml/store/_spaces.py +67 -0
  46. useml-1.0.1/src/useml/store/_store.py +676 -0
  47. useml-1.0.1/tests/conftest.py +47 -0
  48. useml-1.0.1/tests/data_procs.py +54 -0
  49. useml-1.0.1/tests/exp_steps.py +117 -0
  50. useml-1.0.1/tests/test_config.py +389 -0
  51. useml-1.0.1/tests/test_data.py +302 -0
  52. useml-1.0.1/tests/test_exp.py +228 -0
  53. useml-1.0.1/tests/test_exp_robust.py +276 -0
  54. useml-1.0.1/tests/test_fetch.py +179 -0
  55. useml-1.0.1/tests/test_paths.py +97 -0
  56. useml-1.0.1/tests/test_policy.py +200 -0
  57. useml-1.0.1/tests/test_setup.py +165 -0
  58. useml-1.0.1/tests/test_store.py +342 -0
  59. useml-0.3.2/PKG-INFO +0 -471
  60. useml-0.3.2/README.md +0 -454
  61. useml-0.3.2/pyproject.toml +0 -30
  62. useml-0.3.2/setup.cfg +0 -4
  63. useml-0.3.2/tests/test_bundle.py +0 -366
  64. useml-0.3.2/tests/test_level2.py +0 -244
  65. useml-0.3.2/tests/test_session.py +0 -208
  66. useml-0.3.2/tests/test_train_level0.py +0 -234
  67. useml-0.3.2/tests/test_usecases.py +0 -1249
  68. useml-0.3.2/tests/test_vault.py +0 -243
  69. useml-0.3.2/useml/__init__.py +0 -94
  70. useml-0.3.2/useml/dataset/__init__.py +0 -4
  71. useml-0.3.2/useml/dataset/bundle.py +0 -288
  72. useml-0.3.2/useml/dataset/loaders.py +0 -168
  73. useml-0.3.2/useml/errors.py +0 -377
  74. useml-0.3.2/useml/imports.py +0 -215
  75. useml-0.3.2/useml/session/__init__.py +0 -4
  76. useml-0.3.2/useml/session/component.py +0 -59
  77. useml-0.3.2/useml/session/manager.py +0 -482
  78. useml-0.3.2/useml/template/__init__.py +0 -6
  79. useml-0.3.2/useml/template/config.py +0 -140
  80. useml-0.3.2/useml/template/loss.py +0 -28
  81. useml-0.3.2/useml/template/model.py +0 -28
  82. useml-0.3.2/useml/template/trainer.py +0 -758
  83. useml-0.3.2/useml/vault/__init__.py +0 -5
  84. useml-0.3.2/useml/vault/code_extractor.py +0 -261
  85. useml-0.3.2/useml/vault/core.py +0 -63
  86. useml-0.3.2/useml/vault/model_registry.py +0 -267
  87. useml-0.3.2/useml/vault/project.py +0 -309
  88. useml-0.3.2/useml/vault/runs.py +0 -814
  89. useml-0.3.2/useml/vault/snapshot.py +0 -170
  90. useml-0.3.2/useml/workdir/__init__.py +0 -14
  91. useml-0.3.2/useml/workdir/_hook.py +0 -79
  92. useml-0.3.2/useml.egg-info/PKG-INFO +0 -471
  93. useml-0.3.2/useml.egg-info/SOURCES.txt +0 -36
  94. useml-0.3.2/useml.egg-info/dependency_links.txt +0 -1
  95. useml-0.3.2/useml.egg-info/requires.txt +0 -8
  96. useml-0.3.2/useml.egg-info/top_level.txt +0 -1
useml-1.0.1/.gitignore ADDED
@@ -0,0 +1,33 @@
1
+ # --- Python Build & Distribution ---
2
+ dist/
3
+ build/
4
+ *.egg-info/
5
+ egg-info/
6
+ nosetests.xml
7
+ pip-wheel-metadata/
8
+ .pytest_cache
9
+
10
+ # --- Virtual Environments ---
11
+ venv/
12
+ .venv/
13
+ env/
14
+
15
+ # --- Compiled Files ---
16
+ __pycache__/
17
+ *.py[cod]
18
+ *$py.class
19
+
20
+ # --- Machine Learning specifics (Vault/Data) ---
21
+ *.pt
22
+ *.pth
23
+ *.ckpt
24
+
25
+ # IDE / OS
26
+ .DS_Store
27
+ .claude/
28
+
29
+ # testing
30
+ playground
31
+
32
+ # --- useml v1 ---
33
+ bundles/
useml-1.0.1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Enzo-py
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
useml-1.0.1/PKG-INFO ADDED
@@ -0,0 +1,242 @@
1
+ Metadata-Version: 2.5
2
+ Name: useml
3
+ Version: 1.0.1
4
+ Summary: One root for every ML cache, one gate for every download, data processed once.
5
+ Project-URL: Homepage, https://github.com/Enzo-py/useml
6
+ Project-URL: Bug Tracker, https://github.com/Enzo-py/useml/issues
7
+ Author-email: Enzo-py <85191552+Enzo-py@users.noreply.github.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: cache,datasets,deep learning,experiments,huggingface,machine learning,reproducibility
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: MacOS
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
20
+ Requires-Python: >=3.10
21
+ Requires-Dist: numpy>=1.22
22
+ Requires-Dist: pyyaml>=6
23
+ Requires-Dist: tomli>=1.1; python_version < '3.11'
24
+ Requires-Dist: tqdm>=4.60
25
+ Provides-Extra: all
26
+ Requires-Dist: gdown>=4.6; extra == 'all'
27
+ Requires-Dist: huggingface-hub>=0.20; extra == 'all'
28
+ Requires-Dist: psutil>=5.9; extra == 'all'
29
+ Provides-Extra: gdrive
30
+ Requires-Dist: gdown>=4.6; extra == 'gdrive'
31
+ Provides-Extra: hf
32
+ Requires-Dist: huggingface-hub>=0.20; extra == 'hf'
33
+ Provides-Extra: test
34
+ Requires-Dist: pytest>=7; extra == 'test'
35
+ Provides-Extra: watch
36
+ Requires-Dist: psutil>=5.9; extra == 'watch'
37
+ Description-Content-Type: text/markdown
38
+
39
+ # useml
40
+
41
+ One root for every ML cache, one gate for every download.
42
+
43
+ ```bash
44
+ pip install "useml[all]"
45
+ ```
46
+
47
+ ## Quick start
48
+
49
+ ```bash
50
+ pip install "useml[all]"
51
+ cd my-project
52
+ useml init # shows what it would set up (adds, never overwrites); confirm to write
53
+ useml check # checks that everything resolves
54
+ ```
55
+
56
+ ## One root per machine
57
+
58
+ ```toml
59
+ # ~/.config/useml/config.toml
60
+ root = "/data/me/useml"
61
+
62
+ [hub]
63
+ parallel = false # default: one download thread at a time
64
+ ```
65
+
66
+ `import useml` (first, before torch / transformers) points `HF_HOME`, `TORCH_HOME`,
67
+ `TMPDIR`, pip, triton… under the root. A cache already placed outside the home is never
68
+ moved. `useml paths` shows where everything goes.
69
+
70
+ ## One gate for downloads
71
+
72
+ ```python
73
+ import useml
74
+
75
+ useml.hub.get("hf://openai/clip-vit-large-patch14")
76
+ useml.hub.get("hf://datasets/org/name@main/file.json")
77
+ useml.hub.get("https://example.org/annotations.zip", extract_archive=True)
78
+ useml.hub.get("gdrive://<file id>")
79
+ ```
80
+
81
+ Already downloaded → answered from disk, no network. Every download is recorded:
82
+
83
+ ```bash
84
+ useml hub ls # what is on disk, how big, last used
85
+ useml hub scan # caches left in the home (~/.cache/huggingface…)
86
+ useml hub scan --move --apply
87
+ ```
88
+
89
+ ## The network policy
90
+
91
+ By default, only one thread may download at a time, in every library:
92
+
93
+ - Hugging Face `xet` (Rust, its own parallel connections) is disabled;
94
+ - the thread pools of `huggingface_hub` and `datasets` become sequential;
95
+ - `socket.connect` refuses an outbound connection from a second thread
96
+ (`ParallelDownloadError`), and logs it;
97
+ - `useml hub watch -- python train.py` watches the whole process tree from outside
98
+ and kills it on a sustained violation.
99
+
100
+ `useml.hub.allow_parallel()` lifts it, explicitly.
101
+
102
+ ## One cache store
103
+
104
+ ```python
105
+ store = useml.store.open() # <root>/store
106
+ masks = store.declare("processing/masks", cost="expensive")
107
+ masks.put({"image": digest, "model": "sam-h"}, {"masks": array})
108
+ masks.get({"image": digest, "model": "sam-h"})
109
+ ```
110
+
111
+ One SQLite index (WAL, safe across processes), large values in files written atomically,
112
+ one write per key. Spaces are declared with a cost (`cheap`, `expensive`, `precious`)
113
+ kept in the index, so cleaning needs no project code:
114
+
115
+ ```bash
116
+ useml store spaces
117
+ useml store gc --max-size 450GB # dry run: cheapest to recompute first
118
+ useml store gc --max-size 450GB --apply
119
+ ```
120
+
121
+ `[store] max_size = "450GB"` in the config sets the default budget. Nothing is ever
122
+ evicted automatically, pinned entries and `precious` spaces are kept.
123
+
124
+ ## Data processed once
125
+
126
+ ```python
127
+ from useml import data
128
+
129
+ @data.processor("clip-image", version=1, batch_size=64, dtype="float16")
130
+ class ClipImage:
131
+ def __init__(self, model="openai/clip-vit-base-patch32", device="cuda"): ...
132
+ def load(self, path): ... # per item, in worker threads
133
+ def __call__(self, images): ... # a batch → (B, D) array, or a dict of arrays
134
+
135
+ data.process(data.images("/data/photos"), ClipImage, name="photos-clip")
136
+ feats = data.load("photos-clip") # one line; nothing is read until used
137
+ feats[0], feats[100:200], feats.numpy()
138
+ ```
139
+
140
+ - One table per (processor, version, parameters, dtype), shared by every dataset: rows
141
+ are found by a hash of their CONTENT, so an image two datasets share is computed once.
142
+ - Only missing rows are computed; when nothing is missing, the model is never loaded.
143
+ - Storage `float16` (default), `float32` or `int8` (one byte per number + a scale per row).
144
+ - Memory-mapped, append-only shards; one writer per table (a file lock); an interrupted
145
+ run keeps its rows and resumes.
146
+ - The size to write is checked against the free disk and `[data] max_size` before writing.
147
+ - `process(..., detach=True)` runs in its own process: `job.follow()`, `useml data jobs`.
148
+ - Anything longer than ~2 s shows a tqdm bar or a "⏳ … / ✓ … (12.3s)" line.
149
+
150
+ ```bash
151
+ useml data ls # named datasets
152
+ useml data tables # tables, size, who uses them
153
+ useml data rm photos-clip --table
154
+ ```
155
+
156
+ ## Configs in YAML, objects in one line
157
+
158
+ ```toml
159
+ # pyproject.toml of your project
160
+ [tool.useml.config]
161
+ root = "configs" # configs/models/*.yaml, configs/scorers/*.yaml
162
+ packages = ["mypkg.models"] # where `type:` names are looked up (no imports by string)
163
+ ```
164
+
165
+ ```yaml
166
+ # configs/models/resnet_ft.yaml
167
+ meta: {display_name: ResNet-50 (fine-tuned)}
168
+ type: Classifier
169
+ backbone: {ref: resnet50} # another config, built once and shared
170
+ optimizer: {type: SGD, lr: 0.01}
171
+ ```
172
+
173
+ ```python
174
+ from useml import config
175
+ model = config.load("resnet_ft") # memoised by name
176
+ model = config.load("resnet_ft", optimizer={"lr": 0.1}) # one setting, merged
177
+ model.config.hash, model.config.provenance() # cite what ran
178
+ config.create("variant", model); config.diff("resnet_ft", model) # and back to a file
179
+ ```
180
+
181
+ Misspelled keys and types raise with the closest name and the place in the file.
182
+
183
+ ## Experiences, and their outputs in one line
184
+
185
+ ```python
186
+ from useml import exp, config, data
187
+
188
+ with exp.experience("vision/baseline") as e: # nothing runs in the block
189
+ trained = e.step(train, data.load("photos-clip"), config.load("classifier"),
190
+ epochs=10, name="model")
191
+ e.step(report, trained, name="report")
192
+ # block end: declaration written, run detached, followed with a progress bar
193
+
194
+ model = exp.get("vision/baseline/model") # later, anywhere: the trained model
195
+ exp.open("vision/baseline").results() # every step's numbers
196
+ ```
197
+
198
+ - A step is an importable function. Arguments are recipes: outputs of earlier steps,
199
+ `config.load(...)` objects, `data.load(name)` datasets, outputs of other experiences.
200
+ - Fingerprints chain: running the block again skips what is done; changing a line reruns
201
+ only what depends on it. An interrupted run resumes.
202
+ - Outputs are saved by type: a model built from a config is kept as config + weights and
203
+ comes back rebuilt; arrays as `.npz`; figures as `.png`; numbers as JSON.
204
+ - A step with a `ctx` parameter gets its folder, `ctx.progress(i, n)`, `ctx.checkpoints`
205
+ and `ctx.should_stop()`.
206
+
207
+ ```bash
208
+ useml exp ls
209
+ useml exp show vision/baseline
210
+ useml exp follow vision/baseline
211
+ useml exp stop vision/baseline
212
+ ```
213
+
214
+ ### Runs that survive, crashes that explain themselves
215
+
216
+ A detached run lives in a tmux session (`useml exp attach <name>`), or in its own session
217
+ without tmux: closing the laptop, the SSH connection or the notebook does not stop it.
218
+ A small supervisor process is its parent and records how it ended — exit code, signal
219
+ (SIGKILL ≈ out of RAM, SIGSEGV = native crash), peak RAM, kernel OOM lines, traceback,
220
+ last log lines:
221
+
222
+ ```bash
223
+ useml exp why vision/baseline # where it is, or why it died
224
+ useml exp stack vision/baseline # the Python stack of a run that seems stuck
225
+ ```
226
+
227
+ ```toml
228
+ # ~/.config/useml/config.toml — optional: be told when a run ends or crashes
229
+ [exp]
230
+ notify = "your-command $USEML_EXP_NAME $USEML_EXP_STATUS"
231
+ ```
232
+
233
+ ### Bundles: archive the version that worked
234
+
235
+ ```bash
236
+ useml exp bundle vision/baseline -o archive/baseline-final [--archive]
237
+ ```
238
+
239
+ A standalone folder: the code exactly as it ran (commit + uncommitted diff + untracked
240
+ sources, recorded at every launch), a copy of useml, the configs, every step's output,
241
+ the rows of the datasets it read, `packages.txt`, and sha256 checksums. `python load.py`
242
+ / `from load import get` reads an output; `python run.py --from-scratch` recomputes it.
useml-1.0.1/README.md ADDED
@@ -0,0 +1,204 @@
1
+ # useml
2
+
3
+ One root for every ML cache, one gate for every download.
4
+
5
+ ```bash
6
+ pip install "useml[all]"
7
+ ```
8
+
9
+ ## Quick start
10
+
11
+ ```bash
12
+ pip install "useml[all]"
13
+ cd my-project
14
+ useml init # shows what it would set up (adds, never overwrites); confirm to write
15
+ useml check # checks that everything resolves
16
+ ```
17
+
18
+ ## One root per machine
19
+
20
+ ```toml
21
+ # ~/.config/useml/config.toml
22
+ root = "/data/me/useml"
23
+
24
+ [hub]
25
+ parallel = false # default: one download thread at a time
26
+ ```
27
+
28
+ `import useml` (first, before torch / transformers) points `HF_HOME`, `TORCH_HOME`,
29
+ `TMPDIR`, pip, triton… under the root. A cache already placed outside the home is never
30
+ moved. `useml paths` shows where everything goes.
31
+
32
+ ## One gate for downloads
33
+
34
+ ```python
35
+ import useml
36
+
37
+ useml.hub.get("hf://openai/clip-vit-large-patch14")
38
+ useml.hub.get("hf://datasets/org/name@main/file.json")
39
+ useml.hub.get("https://example.org/annotations.zip", extract_archive=True)
40
+ useml.hub.get("gdrive://<file id>")
41
+ ```
42
+
43
+ Already downloaded → answered from disk, no network. Every download is recorded:
44
+
45
+ ```bash
46
+ useml hub ls # what is on disk, how big, last used
47
+ useml hub scan # caches left in the home (~/.cache/huggingface…)
48
+ useml hub scan --move --apply
49
+ ```
50
+
51
+ ## The network policy
52
+
53
+ By default, only one thread may download at a time, in every library:
54
+
55
+ - Hugging Face `xet` (Rust, its own parallel connections) is disabled;
56
+ - the thread pools of `huggingface_hub` and `datasets` become sequential;
57
+ - `socket.connect` refuses an outbound connection from a second thread
58
+ (`ParallelDownloadError`), and logs it;
59
+ - `useml hub watch -- python train.py` watches the whole process tree from outside
60
+ and kills it on a sustained violation.
61
+
62
+ `useml.hub.allow_parallel()` lifts it, explicitly.
63
+
64
+ ## One cache store
65
+
66
+ ```python
67
+ store = useml.store.open() # <root>/store
68
+ masks = store.declare("processing/masks", cost="expensive")
69
+ masks.put({"image": digest, "model": "sam-h"}, {"masks": array})
70
+ masks.get({"image": digest, "model": "sam-h"})
71
+ ```
72
+
73
+ One SQLite index (WAL, safe across processes), large values in files written atomically,
74
+ one write per key. Spaces are declared with a cost (`cheap`, `expensive`, `precious`)
75
+ kept in the index, so cleaning needs no project code:
76
+
77
+ ```bash
78
+ useml store spaces
79
+ useml store gc --max-size 450GB # dry run: cheapest to recompute first
80
+ useml store gc --max-size 450GB --apply
81
+ ```
82
+
83
+ `[store] max_size = "450GB"` in the config sets the default budget. Nothing is ever
84
+ evicted automatically, pinned entries and `precious` spaces are kept.
85
+
86
+ ## Data processed once
87
+
88
+ ```python
89
+ from useml import data
90
+
91
+ @data.processor("clip-image", version=1, batch_size=64, dtype="float16")
92
+ class ClipImage:
93
+ def __init__(self, model="openai/clip-vit-base-patch32", device="cuda"): ...
94
+ def load(self, path): ... # per item, in worker threads
95
+ def __call__(self, images): ... # a batch → (B, D) array, or a dict of arrays
96
+
97
+ data.process(data.images("/data/photos"), ClipImage, name="photos-clip")
98
+ feats = data.load("photos-clip") # one line; nothing is read until used
99
+ feats[0], feats[100:200], feats.numpy()
100
+ ```
101
+
102
+ - One table per (processor, version, parameters, dtype), shared by every dataset: rows
103
+ are found by a hash of their CONTENT, so an image two datasets share is computed once.
104
+ - Only missing rows are computed; when nothing is missing, the model is never loaded.
105
+ - Storage `float16` (default), `float32` or `int8` (one byte per number + a scale per row).
106
+ - Memory-mapped, append-only shards; one writer per table (a file lock); an interrupted
107
+ run keeps its rows and resumes.
108
+ - The size to write is checked against the free disk and `[data] max_size` before writing.
109
+ - `process(..., detach=True)` runs in its own process: `job.follow()`, `useml data jobs`.
110
+ - Anything longer than ~2 s shows a tqdm bar or a "⏳ … / ✓ … (12.3s)" line.
111
+
112
+ ```bash
113
+ useml data ls # named datasets
114
+ useml data tables # tables, size, who uses them
115
+ useml data rm photos-clip --table
116
+ ```
117
+
118
+ ## Configs in YAML, objects in one line
119
+
120
+ ```toml
121
+ # pyproject.toml of your project
122
+ [tool.useml.config]
123
+ root = "configs" # configs/models/*.yaml, configs/scorers/*.yaml
124
+ packages = ["mypkg.models"] # where `type:` names are looked up (no imports by string)
125
+ ```
126
+
127
+ ```yaml
128
+ # configs/models/resnet_ft.yaml
129
+ meta: {display_name: ResNet-50 (fine-tuned)}
130
+ type: Classifier
131
+ backbone: {ref: resnet50} # another config, built once and shared
132
+ optimizer: {type: SGD, lr: 0.01}
133
+ ```
134
+
135
+ ```python
136
+ from useml import config
137
+ model = config.load("resnet_ft") # memoised by name
138
+ model = config.load("resnet_ft", optimizer={"lr": 0.1}) # one setting, merged
139
+ model.config.hash, model.config.provenance() # cite what ran
140
+ config.create("variant", model); config.diff("resnet_ft", model) # and back to a file
141
+ ```
142
+
143
+ Misspelled keys and types raise with the closest name and the place in the file.
144
+
145
+ ## Experiences, and their outputs in one line
146
+
147
+ ```python
148
+ from useml import exp, config, data
149
+
150
+ with exp.experience("vision/baseline") as e: # nothing runs in the block
151
+ trained = e.step(train, data.load("photos-clip"), config.load("classifier"),
152
+ epochs=10, name="model")
153
+ e.step(report, trained, name="report")
154
+ # block end: declaration written, run detached, followed with a progress bar
155
+
156
+ model = exp.get("vision/baseline/model") # later, anywhere: the trained model
157
+ exp.open("vision/baseline").results() # every step's numbers
158
+ ```
159
+
160
+ - A step is an importable function. Arguments are recipes: outputs of earlier steps,
161
+ `config.load(...)` objects, `data.load(name)` datasets, outputs of other experiences.
162
+ - Fingerprints chain: running the block again skips what is done; changing a line reruns
163
+ only what depends on it. An interrupted run resumes.
164
+ - Outputs are saved by type: a model built from a config is kept as config + weights and
165
+ comes back rebuilt; arrays as `.npz`; figures as `.png`; numbers as JSON.
166
+ - A step with a `ctx` parameter gets its folder, `ctx.progress(i, n)`, `ctx.checkpoints`
167
+ and `ctx.should_stop()`.
168
+
169
+ ```bash
170
+ useml exp ls
171
+ useml exp show vision/baseline
172
+ useml exp follow vision/baseline
173
+ useml exp stop vision/baseline
174
+ ```
175
+
176
+ ### Runs that survive, crashes that explain themselves
177
+
178
+ A detached run lives in a tmux session (`useml exp attach <name>`), or in its own session
179
+ without tmux: closing the laptop, the SSH connection or the notebook does not stop it.
180
+ A small supervisor process is its parent and records how it ended — exit code, signal
181
+ (SIGKILL ≈ out of RAM, SIGSEGV = native crash), peak RAM, kernel OOM lines, traceback,
182
+ last log lines:
183
+
184
+ ```bash
185
+ useml exp why vision/baseline # where it is, or why it died
186
+ useml exp stack vision/baseline # the Python stack of a run that seems stuck
187
+ ```
188
+
189
+ ```toml
190
+ # ~/.config/useml/config.toml — optional: be told when a run ends or crashes
191
+ [exp]
192
+ notify = "your-command $USEML_EXP_NAME $USEML_EXP_STATUS"
193
+ ```
194
+
195
+ ### Bundles: archive the version that worked
196
+
197
+ ```bash
198
+ useml exp bundle vision/baseline -o archive/baseline-final [--archive]
199
+ ```
200
+
201
+ A standalone folder: the code exactly as it ran (commit + uncommitted diff + untracked
202
+ sources, recorded at every launch), a copy of useml, the configs, every step's output,
203
+ the rows of the datasets it read, `packages.txt`, and sha256 checksums. `python load.py`
204
+ / `from load import get` reads an output; `python run.py --from-scratch` recomputes it.
@@ -0,0 +1,56 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.18"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "useml"
7
+ version = "1.0.1"
8
+ description = "One root for every ML cache, one gate for every download, data processed once."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Enzo-py", email = "85191552+Enzo-py@users.noreply.github.com" }]
14
+ keywords = ["machine learning", "deep learning", "cache", "experiments", "datasets",
15
+ "huggingface", "reproducibility"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Science/Research",
19
+ "Operating System :: POSIX :: Linux",
20
+ "Operating System :: MacOS",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3.10",
23
+ "Programming Language :: Python :: 3.11",
24
+ "Programming Language :: Python :: 3.12",
25
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
26
+ ]
27
+ dependencies = [
28
+ "numpy>=1.22",
29
+ "tqdm>=4.60",
30
+ "pyyaml>=6",
31
+ "tomli>=1.1; python_version < '3.11'",
32
+ ]
33
+
34
+ [project.urls]
35
+ Homepage = "https://github.com/Enzo-py/useml"
36
+ "Bug Tracker" = "https://github.com/Enzo-py/useml/issues"
37
+
38
+ [project.optional-dependencies]
39
+ hf = ["huggingface_hub>=0.20"]
40
+ gdrive = ["gdown>=4.6"]
41
+ watch = ["psutil>=5.9"]
42
+ all = ["huggingface_hub>=0.20", "gdown>=4.6", "psutil>=5.9"]
43
+ test = ["pytest>=7"]
44
+
45
+ [project.scripts]
46
+ useml = "useml.cli:main"
47
+
48
+ [tool.hatch.build.targets.wheel]
49
+ packages = ["src/useml"]
50
+
51
+ # Only what users need: never local notes.
52
+ [tool.hatch.build.targets.sdist]
53
+ include = ["src/useml", "tests", "README.md", "LICENSE", "pyproject.toml"]
54
+
55
+ [tool.pytest.ini_options]
56
+ testpaths = ["tests"]
@@ -0,0 +1,42 @@
1
+ """useml — one root for every ML cache, one gate for every download.
2
+
3
+ IMPORTING useml HAS TWO EFFECTS, on purpose, and only these two:
4
+
5
+ 1. `useml.paths.apply()` — if a root is configured, every library cache variable
6
+ (HF_HOME, TORCH_HOME, TMPDIR…) points under it. Without a root, nothing changes.
7
+ 2. `useml.hub.apply_configured_policy()` — downloads become sequential (one thread
8
+ at a time), unless the machine config says `[hub] parallel = true`.
9
+
10
+ Import useml FIRST, before torch / transformers / open_clip, so they read the right paths.
11
+ If they were imported earlier, `paths.apply()` realigns huggingface_hub's paths anyway.
12
+
13
+ Set `USEML_NO_AUTO=1` to import without either effect (then call them yourself).
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import os as _os
18
+
19
+ from importlib.metadata import PackageNotFoundError, version
20
+
21
+ try:
22
+ __version__ = version("useml")
23
+ except PackageNotFoundError: # pragma: no cover - source checkout
24
+ __version__ = "dev"
25
+
26
+ from . import paths, hub, store
27
+
28
+ if _os.environ.get("USEML_NO_AUTO", "").strip() not in ("1", "true", "yes"):
29
+ paths.apply()
30
+ hub.apply_configured_policy()
31
+
32
+ __all__ = ["paths", "hub", "store", "data", "config", "exp", "__version__"]
33
+
34
+
35
+ def __getattr__(name):
36
+ """`useml.data` and `useml.config` are imported on first use: they need numpy / PyYAML,
37
+ `import useml` does not."""
38
+ if name in ("data", "config", "exp"):
39
+ import importlib
40
+
41
+ return importlib.import_module(f".{name}", __name__)
42
+ raise AttributeError(f"module 'useml' has no attribute {name!r}")
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())