mlcflow 1.3.6__tar.gz → 1.3.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mlcflow-1.3.6 → mlcflow-1.3.8}/PKG-INFO +10 -2
- {mlcflow-1.3.6 → mlcflow-1.3.8}/README.md +9 -1
- mlcflow-1.3.8/VERSION +1 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/apptainer.py +68 -23
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/meta_schema.py +18 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/module.py +16 -1
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/action.py +8 -1
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/index.py +40 -1
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/main.py +22 -14
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/meta_schema.py +18 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/repo_action.py +680 -187
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/script_action.py +220 -20
- mlcflow-1.3.8/mlc/typo_mixin.py +127 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/utils.py +56 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/PKG-INFO +10 -2
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/SOURCES.txt +6 -0
- mlcflow-1.3.8/tests/test_apptainer_docker_unification.py +286 -0
- mlcflow-1.3.8/tests/test_build_wheels_workflow.py +142 -0
- mlcflow-1.3.8/tests/test_pull_repo_thread_safety.py +1036 -0
- mlcflow-1.3.8/tests/test_release_version.py +101 -0
- mlcflow-1.3.8/tests/test_typo_mixin.py +294 -0
- mlcflow-1.3.6/VERSION +0 -1
- {mlcflow-1.3.6 → mlcflow-1.3.8}/LICENSE.md +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/COPYRIGHT.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/__init__.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/README.md +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/__init__.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/cache_utils.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/compat.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/doc.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/docker.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/docker_utils.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/experiment.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/help.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/lint.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/meta.json +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/remote_run.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/script_utils.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/slurm_run.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/validate.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/utils.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/__init__.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/__main__.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/action_factory.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/cache_action.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/cfg_action.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/error_codes.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/experiment_action.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/item.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/logger.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/repo.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/dependency_links.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/entry_points.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/requires.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/top_level.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/pyproject.toml +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/setup.cfg +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_action_invalid_meta_entries.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_cache_mark_tmp.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_convert_env_to_script.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_installer_venv_deactivation.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_mlcflow_unix_installer.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_repo_action_extra_git_args.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_script_action_apptainer.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_slurm_run.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_thread_safety.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_unix_installer_venv.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mlcflow
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.8
|
|
4
4
|
Summary: An automation interface tailored for CPU/GPU benchmarking
|
|
5
5
|
Author-email: MLCommons <systems@mlcommons.org>
|
|
6
6
|
License:
|
|
@@ -214,13 +214,21 @@ Dynamic: license-file
|
|
|
214
214
|
|
|
215
215
|
MLCFlow is a versatile CLI and Python interface developed by MLCommons in collaboration with a dedicated team of volunteers (see [Contributors](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md)). It serves as a streamlined replacement for the [CMind](https://github.com/mlcommons/ck/tree/master/cm) tool, designed to drive the automation workflows of MLPerf benchmarks more efficiently. You can use this tool for any of your workflow automation tasks.
|
|
216
216
|
|
|
217
|
-
The concept behind CMind originated from **Grigori Fursin
|
|
217
|
+
The concept behind CMind originated from [**Grigori Fursin**](https://github.com/gfursin), while the **MLPerf Automations** project was created by [**Grigori Fursin**](https://github.com/gfursin) and [**Arjun Suresh**](https://github.com/arjunsuresh), whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by [**Arjun Suresh**](https://github.com/arjunsuresh) and [**Anandhu Sooraj**](https://github.com/anandhu-eng), with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the developers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
|
|
218
218
|
|
|
219
219
|
On February 9, 2025, MLCFlow released its first stable version, 1.0.0.
|
|
220
220
|
|
|
221
221
|
### Key Features
|
|
222
222
|
Building upon the core idea of CMind—wrapping native scripts with Python wrappers and YAML metadata—MLCFlow focuses exclusively on key automation components: **Scripts**, along with its complementary modules: **Cache**, **Docker**, and **Experiments**. This targeted design simplifies both implementation and interface, enabling a more user-friendly experience.
|
|
223
223
|
|
|
224
|
+
- **Typo detection** — Mistyped actions or targets produce a "Did you mean …?" hint before the error message, so mistakes are quick to correct:
|
|
225
|
+
```
|
|
226
|
+
$ mlc rune script
|
|
227
|
+
Did you mean one of: 'prune', 'run'?
|
|
228
|
+
mlc: error: argument command: invalid choice: 'rune' …
|
|
229
|
+
```
|
|
230
|
+
See [docs/typo_detection.md](docs/typo_detection.md) for details.
|
|
231
|
+
|
|
224
232
|
---
|
|
225
233
|
|
|
226
234
|
### Status
|
|
@@ -12,13 +12,21 @@
|
|
|
12
12
|
|
|
13
13
|
MLCFlow is a versatile CLI and Python interface developed by MLCommons in collaboration with a dedicated team of volunteers (see [Contributors](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md)). It serves as a streamlined replacement for the [CMind](https://github.com/mlcommons/ck/tree/master/cm) tool, designed to drive the automation workflows of MLPerf benchmarks more efficiently. You can use this tool for any of your workflow automation tasks.
|
|
14
14
|
|
|
15
|
-
The concept behind CMind originated from **Grigori Fursin
|
|
15
|
+
The concept behind CMind originated from [**Grigori Fursin**](https://github.com/gfursin), while the **MLPerf Automations** project was created by [**Grigori Fursin**](https://github.com/gfursin) and [**Arjun Suresh**](https://github.com/arjunsuresh), whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by [**Arjun Suresh**](https://github.com/arjunsuresh) and [**Anandhu Sooraj**](https://github.com/anandhu-eng), with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the developers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
|
|
16
16
|
|
|
17
17
|
On February 9, 2025, MLCFlow released its first stable version, 1.0.0.
|
|
18
18
|
|
|
19
19
|
### Key Features
|
|
20
20
|
Building upon the core idea of CMind—wrapping native scripts with Python wrappers and YAML metadata—MLCFlow focuses exclusively on key automation components: **Scripts**, along with its complementary modules: **Cache**, **Docker**, and **Experiments**. This targeted design simplifies both implementation and interface, enabling a more user-friendly experience.
|
|
21
21
|
|
|
22
|
+
- **Typo detection** — Mistyped actions or targets produce a "Did you mean …?" hint before the error message, so mistakes are quick to correct:
|
|
23
|
+
```
|
|
24
|
+
$ mlc rune script
|
|
25
|
+
Did you mean one of: 'prune', 'run'?
|
|
26
|
+
mlc: error: argument command: invalid choice: 'rune' …
|
|
27
|
+
```
|
|
28
|
+
See [docs/typo_detection.md](docs/typo_detection.md) for details.
|
|
29
|
+
|
|
22
30
|
---
|
|
23
31
|
|
|
24
32
|
### Status
|
mlcflow-1.3.8/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
1.3.8
|
|
@@ -8,9 +8,10 @@ import copy
|
|
|
8
8
|
|
|
9
9
|
def apptainerfile(self_module, input_params):
|
|
10
10
|
|
|
11
|
-
# Step 1: Prune and prepare input
|
|
11
|
+
# Step 1: Prune and prepare input (remove both apptainer_ and docker_
|
|
12
|
+
# prefixed keys)
|
|
12
13
|
prune_result = prune_input(
|
|
13
|
-
{'input': input_params, 'extra_keys_starts_with': ['apptainer_']})
|
|
14
|
+
{'input': input_params, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
|
|
14
15
|
if prune_result['return'] > 0:
|
|
15
16
|
return prune_result
|
|
16
17
|
|
|
@@ -56,13 +57,15 @@ def apptainerfile(self_module, input_params):
|
|
|
56
57
|
|
|
57
58
|
run_state = self_module.run_state
|
|
58
59
|
|
|
59
|
-
apptainer_settings = run_state.get('docker', {})
|
|
60
|
+
apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
|
|
61
|
+
utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
|
|
62
|
+
'append_lists': True, 'append_unique': True})
|
|
60
63
|
apptainer_settings_default_env = apptainer_settings.get('default_env', {})
|
|
61
64
|
for key in apptainer_settings_default_env:
|
|
62
65
|
env.setdefault(key, apptainer_settings_default_env[key])
|
|
63
66
|
|
|
64
67
|
if not apptainer_settings.get('run', True) and not input_params.get(
|
|
65
|
-
'apptainer_run_override', False):
|
|
68
|
+
'apptainer_run_override', input_params.get('docker_run_override', False)):
|
|
66
69
|
logger.info("Apptainer 'run' is set to False in meta.yaml")
|
|
67
70
|
return {'return': 0,
|
|
68
71
|
'warning': 'Apptainer run is set to false in script meta'}
|
|
@@ -86,7 +89,9 @@ def apptainerfile(self_module, input_params):
|
|
|
86
89
|
if update_state_result['return'] > 0:
|
|
87
90
|
return update_state_result
|
|
88
91
|
|
|
89
|
-
apptainer_settings = run_state.get('docker', {})
|
|
92
|
+
apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
|
|
93
|
+
utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
|
|
94
|
+
'append_lists': True, 'append_unique': True})
|
|
90
95
|
|
|
91
96
|
# Prune temporary environment variables
|
|
92
97
|
run_command = copy.deepcopy(run_command_arc)
|
|
@@ -102,7 +107,7 @@ def apptainerfile(self_module, input_params):
|
|
|
102
107
|
'tags': script_tags,
|
|
103
108
|
'fake_run': True,
|
|
104
109
|
'docker_settings': apptainer_settings,
|
|
105
|
-
'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', ''))
|
|
110
|
+
'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', input_params.get('docker_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', '')))
|
|
106
111
|
})
|
|
107
112
|
if regenerate_result['return'] > 0:
|
|
108
113
|
return regenerate_result
|
|
@@ -152,13 +157,16 @@ def apptainerfile(self_module, input_params):
|
|
|
152
157
|
if apptainer_inputs.get('mlc_repo_path', '') != '':
|
|
153
158
|
mlc_apptainer_input['mlc_repo_path'] = apptainer_inputs['mlc_repo_path']
|
|
154
159
|
|
|
160
|
+
if is_true(get_apptainer_input(input_params, 'host_mlc_repos', '')):
|
|
161
|
+
mlc_apptainer_input['host_mlc_repos'] = 'yes'
|
|
162
|
+
|
|
155
163
|
apptainer_v = False
|
|
156
164
|
apptainer_s = False
|
|
157
165
|
if is_true(input_params.get(
|
|
158
|
-
'apptainer_v', input_params.get('apptainer_verbose', False))):
|
|
166
|
+
'apptainer_v', input_params.get('apptainer_verbose', input_params.get('docker_v', input_params.get('docker_verbose', False))))):
|
|
159
167
|
apptainer_v = True
|
|
160
168
|
if is_true(input_params.get(
|
|
161
|
-
'apptainer_s', input_params.get('apptainer_silent', False))):
|
|
169
|
+
'apptainer_s', input_params.get('apptainer_silent', input_params.get('docker_s', input_params.get('docker_silent', False))))):
|
|
162
170
|
apptainer_s = True
|
|
163
171
|
|
|
164
172
|
if apptainer_s and apptainer_v:
|
|
@@ -213,11 +221,16 @@ def apptainer_run(self_module, i):
|
|
|
213
221
|
if quiet:
|
|
214
222
|
env['MLC_QUIET'] = 'yes'
|
|
215
223
|
|
|
216
|
-
regenerate_def_file = not i.get(
|
|
217
|
-
|
|
224
|
+
regenerate_def_file = not i.get(
|
|
225
|
+
'apptainer_noregenerate', i.get(
|
|
226
|
+
'docker_noregenerate', False))
|
|
227
|
+
rebuild_apptainer_image = i.get(
|
|
228
|
+
'apptainer_rebuild', i.get(
|
|
229
|
+
'docker_rebuild', False))
|
|
218
230
|
|
|
219
|
-
# Prune unnecessary Apptainer-related input keys
|
|
220
|
-
r = prune_input(
|
|
231
|
+
# Prune unnecessary Apptainer- and Docker-related input keys
|
|
232
|
+
r = prune_input(
|
|
233
|
+
{'input': i, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
|
|
221
234
|
f_run_cmd = r['new_input']
|
|
222
235
|
|
|
223
236
|
# Save current directory and prepare to search for scripts
|
|
@@ -259,7 +272,7 @@ def apptainer_run(self_module, i):
|
|
|
259
272
|
mounts = copy.deepcopy(
|
|
260
273
|
i.get(
|
|
261
274
|
'apptainer_mounts',
|
|
262
|
-
[]))
|
|
275
|
+
i.get('docker_mounts', [])))
|
|
263
276
|
variations = meta.get('variations', {})
|
|
264
277
|
|
|
265
278
|
if not hasattr(self_module, 'run_state'):
|
|
@@ -273,7 +286,9 @@ def apptainer_run(self_module, i):
|
|
|
273
286
|
|
|
274
287
|
run_state = self_module.run_state
|
|
275
288
|
|
|
276
|
-
apptainer_settings = run_state.get('docker', {})
|
|
289
|
+
apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
|
|
290
|
+
utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
|
|
291
|
+
'append_lists': True, 'append_unique': True})
|
|
277
292
|
|
|
278
293
|
apptainer_settings_default_env = apptainer_settings.get('default_env', {})
|
|
279
294
|
for key in apptainer_settings_default_env:
|
|
@@ -297,7 +312,7 @@ def apptainer_run(self_module, i):
|
|
|
297
312
|
|
|
298
313
|
# Skip scripts marked as non-runnable
|
|
299
314
|
if not apptainer_settings.get('run', True) and not i.get(
|
|
300
|
-
'apptainer_run_override', False):
|
|
315
|
+
'apptainer_run_override', i.get('docker_run_override', False)):
|
|
301
316
|
logger.info("apptainer.run set to False in meta.yaml")
|
|
302
317
|
return {'return': 0,
|
|
303
318
|
'warning': 'Apptainer run is set to false in script meta'}
|
|
@@ -379,6 +394,20 @@ def apptainer_run(self_module, i):
|
|
|
379
394
|
'quiet': True, 'real_run': True,
|
|
380
395
|
'add_deps_recursive': {'build-apptainer-image': {'def_file': def_file_path}}
|
|
381
396
|
}
|
|
397
|
+
|
|
398
|
+
# Detached mode (mirrors --docker_dt): run as a background instance.
|
|
399
|
+
if is_true(get_apptainer_input(
|
|
400
|
+
i, 'dt', get_apptainer_input(i, 'detached', ''))):
|
|
401
|
+
mlc_apptainer_input['detached'] = 'yes'
|
|
402
|
+
# Drop into an interactive shell after the run (mirrors --docker_it).
|
|
403
|
+
elif is_true(get_apptainer_input(
|
|
404
|
+
i, 'it', get_apptainer_input(i, 'interactive', ''))):
|
|
405
|
+
mlc_apptainer_input['interactive'] = 'yes'
|
|
406
|
+
|
|
407
|
+
# Tell the run step host repos were baked in, so it points MLC_REPOS at
|
|
408
|
+
# /opt/mlc_host_repos (matters under fakeroot, which otherwise overrides it).
|
|
409
|
+
if is_true(get_apptainer_input(i, 'host_mlc_repos', '')):
|
|
410
|
+
mlc_apptainer_input['host_mlc_repos'] = 'yes'
|
|
382
411
|
# Forward build options to build-apptainer-image
|
|
383
412
|
build_image_extras = {}
|
|
384
413
|
if apptainer_inputs.get('ignore_fakeroot_cmd'):
|
|
@@ -443,14 +472,15 @@ def prepare_apptainer_inputs(input_params, apptainer_settings,
|
|
|
443
472
|
"network", "security_opt"
|
|
444
473
|
]
|
|
445
474
|
|
|
446
|
-
# Collect inputs
|
|
447
|
-
apptainer_inputs = {
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
475
|
+
# Collect inputs: apptainer_X > docker_X > meta settings > default
|
|
476
|
+
apptainer_inputs = {}
|
|
477
|
+
for key in keys:
|
|
478
|
+
value = input_params.get(
|
|
479
|
+
f"apptainer_{key}",
|
|
480
|
+
input_params.get(
|
|
481
|
+
f"docker_{key}", apptainer_settings.get(key, get_apptainer_default(key))))
|
|
482
|
+
if value is not None:
|
|
483
|
+
apptainer_inputs[key] = value
|
|
454
484
|
|
|
455
485
|
# Convert boolean values to 'yes'/'no' strings for MLC input mapping
|
|
456
486
|
for key in list(apptainer_inputs.keys()):
|
|
@@ -544,6 +574,21 @@ def process_apptainer_mounts(mounts, env, apptainer_settings, run_state):
|
|
|
544
574
|
'container_env_string': container_env_string}
|
|
545
575
|
|
|
546
576
|
|
|
577
|
+
def get_apptainer_input(input_params, key, default=None):
|
|
578
|
+
"""Return apptainer_<key> if provided, else fall back to docker_<key>.
|
|
579
|
+
|
|
580
|
+
Lets docker_* options passed to `mlca` take effect unless the matching
|
|
581
|
+
apptainer_* option overrides them.
|
|
582
|
+
"""
|
|
583
|
+
apptainer_key = f"apptainer_{key}"
|
|
584
|
+
if input_params.get(apptainer_key) is not None:
|
|
585
|
+
return input_params[apptainer_key]
|
|
586
|
+
docker_key = f"docker_{key}"
|
|
587
|
+
if input_params.get(docker_key) is not None:
|
|
588
|
+
return input_params[docker_key]
|
|
589
|
+
return default
|
|
590
|
+
|
|
591
|
+
|
|
547
592
|
def get_apptainer_default(key):
|
|
548
593
|
defaults = {
|
|
549
594
|
"mlc_repo": "mlcommons@mlperf-automations",
|
|
@@ -86,6 +86,7 @@ TOP_LEVEL_SCHEMA = {
|
|
|
86
86
|
|
|
87
87
|
# Docker
|
|
88
88
|
"docker": DICT, # dict - see DOCKER_SCHEMA
|
|
89
|
+
"apptainer": DICT, # dict - apptainer overrides; merges with docker settings
|
|
89
90
|
|
|
90
91
|
# Output / debugging
|
|
91
92
|
"print_env_at_the_end": DICT, # dict[str, list[str]]
|
|
@@ -164,6 +165,7 @@ VARIATION_ENTRY_SCHEMA = {
|
|
|
164
165
|
"state": DICT,
|
|
165
166
|
"const": DICT,
|
|
166
167
|
"docker": DICT,
|
|
168
|
+
"apptainer": DICT,
|
|
167
169
|
"alias": STR,
|
|
168
170
|
"default_version": STR_OR_FLOAT,
|
|
169
171
|
"required_disk_space": INT,
|
|
@@ -248,6 +250,7 @@ UPDATE_META_IF_ENV_SCHEMA = {
|
|
|
248
250
|
"default_env": DICT,
|
|
249
251
|
"default_variations": DICT,
|
|
250
252
|
"docker": DICT,
|
|
253
|
+
"apptainer": DICT,
|
|
251
254
|
"adr": DICT,
|
|
252
255
|
"ad": DICT,
|
|
253
256
|
}
|
|
@@ -403,6 +406,21 @@ def validate_meta(data, file_path=""):
|
|
|
403
406
|
errors.append(
|
|
404
407
|
f"{prefix}docker.{dk} has type '{actual}', expected {allowed}")
|
|
405
408
|
|
|
409
|
+
# Validate apptainer section (same schema as docker; apptainer overrides
|
|
410
|
+
# docker)
|
|
411
|
+
apptainer = data.get("apptainer")
|
|
412
|
+
if isinstance(apptainer, dict):
|
|
413
|
+
for ak, av in apptainer.items():
|
|
414
|
+
if ak not in DOCKER_SCHEMA:
|
|
415
|
+
warnings.append(
|
|
416
|
+
f"{prefix}apptainer: unknown key '{ak}'")
|
|
417
|
+
continue
|
|
418
|
+
actual = type(av).__name__
|
|
419
|
+
allowed = DOCKER_SCHEMA[ak]
|
|
420
|
+
if actual not in allowed:
|
|
421
|
+
errors.append(
|
|
422
|
+
f"{prefix}apptainer.{ak} has type '{actual}', expected {allowed}")
|
|
423
|
+
|
|
406
424
|
# Validate tests section
|
|
407
425
|
tests = data.get("tests")
|
|
408
426
|
if isinstance(tests, dict):
|
|
@@ -151,7 +151,7 @@ class ScriptAutomation(Automation):
|
|
|
151
151
|
for f in ['fake_deps', 'cache']:
|
|
152
152
|
run_state.setdefault(f, False)
|
|
153
153
|
|
|
154
|
-
for d in ['input_mapping', 'docker', 'remote_run']:
|
|
154
|
+
for d in ['input_mapping', 'docker', 'remote_run', 'apptainer']:
|
|
155
155
|
run_state.setdefault(d, {})
|
|
156
156
|
|
|
157
157
|
for l in ['deps', 'post_deps', 'prehook_deps', 'posthook_deps',
|
|
@@ -5768,6 +5768,14 @@ def _apply_conditional_meta_updates(update_meta_if_env, default_env, env, const,
|
|
|
5768
5768
|
'append_lists': True,
|
|
5769
5769
|
'append_unique': True})
|
|
5770
5770
|
|
|
5771
|
+
if c_meta.get('apptainer', {}):
|
|
5772
|
+
if not run_state.get('apptainer', {}):
|
|
5773
|
+
run_state['apptainer'] = {}
|
|
5774
|
+
utils.merge_dicts({'dict1': run_state['apptainer'],
|
|
5775
|
+
'dict2': c_meta['apptainer'],
|
|
5776
|
+
'append_lists': True,
|
|
5777
|
+
'append_unique': True})
|
|
5778
|
+
|
|
5771
5779
|
c_add_deps_info = c_meta.get('ad', {})
|
|
5772
5780
|
if not c_add_deps_info:
|
|
5773
5781
|
c_add_deps_info = c_meta.get('add_deps', {})
|
|
@@ -5975,6 +5983,13 @@ def update_state_from_meta(meta, env, state, const, const_state, run_state, i):
|
|
|
5975
5983
|
if remote_run_settings.get('deps', []):
|
|
5976
5984
|
update_deps(remote_run_settings['deps'], add_deps_info, False, env)
|
|
5977
5985
|
|
|
5986
|
+
new_apptainer_settings = meta.get('apptainer')
|
|
5987
|
+
if new_apptainer_settings:
|
|
5988
|
+
utils.merge_dicts({'dict1': run_state['apptainer'],
|
|
5989
|
+
'dict2': new_apptainer_settings,
|
|
5990
|
+
'append_lists': True,
|
|
5991
|
+
'append_unique': True})
|
|
5992
|
+
|
|
5978
5993
|
new_env_keys_from_meta = meta.get('new_env_keys', [])
|
|
5979
5994
|
if new_env_keys_from_meta:
|
|
5980
5995
|
run_state['new_env_keys'] += new_env_keys_from_meta
|
|
@@ -448,6 +448,7 @@ class Action:
|
|
|
448
448
|
force_remove = True
|
|
449
449
|
|
|
450
450
|
results = res['list']
|
|
451
|
+
removed_any = False
|
|
451
452
|
|
|
452
453
|
for result in results:
|
|
453
454
|
item_path = result.path
|
|
@@ -468,7 +469,13 @@ class Action:
|
|
|
468
469
|
logger.warning(
|
|
469
470
|
f"{item_path} was already removed by another process.")
|
|
470
471
|
|
|
471
|
-
|
|
472
|
+
# Batch the index write: de-index in memory, persist once after
|
|
473
|
+
# the loop instead of rewriting all index files per item.
|
|
474
|
+
self.get_index().rm(item_meta, target_name, item_path, save=False)
|
|
475
|
+
removed_any = True
|
|
476
|
+
|
|
477
|
+
if removed_any:
|
|
478
|
+
self.get_index().save()
|
|
472
479
|
|
|
473
480
|
return {
|
|
474
481
|
"return": 0,
|
|
@@ -192,7 +192,7 @@ class Index:
|
|
|
192
192
|
}
|
|
193
193
|
self._save_indices()
|
|
194
194
|
|
|
195
|
-
def rm(self, meta, folder_type, path):
|
|
195
|
+
def rm(self, meta, folder_type, path, save=True):
|
|
196
196
|
uid = meta['uid']
|
|
197
197
|
index = self.get_index(folder_type, uid)
|
|
198
198
|
if index == -1:
|
|
@@ -200,6 +200,11 @@ class Index:
|
|
|
200
200
|
f"Index is not having the {folder_type} item {path}")
|
|
201
201
|
else:
|
|
202
202
|
del (self.indices[folder_type][index])
|
|
203
|
+
if save:
|
|
204
|
+
self._save_indices()
|
|
205
|
+
|
|
206
|
+
def save(self):
|
|
207
|
+
"""Persist all in-memory indices to disk (for batched callers)."""
|
|
203
208
|
self._save_indices()
|
|
204
209
|
|
|
205
210
|
def get_item_mtime(self, file):
|
|
@@ -349,6 +354,40 @@ class Index:
|
|
|
349
354
|
logger.debug(
|
|
350
355
|
f"Deleted keys removed from modified times and indices: {deleted_keys}")
|
|
351
356
|
|
|
357
|
+
# Reconcile against the filesystem. Entries added via add()/update()
|
|
358
|
+
# (e.g. caches written during a script run) are never recorded in
|
|
359
|
+
# modified_times, so the deleted-key check above cannot catch them
|
|
360
|
+
# once their folder is removed out-of-band. Drop any index entry
|
|
361
|
+
# whose folder no longer exists, scoped to repos currently present
|
|
362
|
+
# so an empty/unavailable repo list can never wipe a valid index.
|
|
363
|
+
repo_prefixes = [
|
|
364
|
+
os.path.normpath(r.path) for r in self.repos
|
|
365
|
+
if getattr(r, 'path', None)
|
|
366
|
+
]
|
|
367
|
+
if repo_prefixes:
|
|
368
|
+
for ft in self.indices:
|
|
369
|
+
kept = []
|
|
370
|
+
for item in self.indices[ft]:
|
|
371
|
+
path = item.get('path', '')
|
|
372
|
+
norm = os.path.normpath(path) if path else ''
|
|
373
|
+
under_known_repo = any(
|
|
374
|
+
norm == rp or norm.startswith(rp + os.sep)
|
|
375
|
+
for rp in repo_prefixes
|
|
376
|
+
)
|
|
377
|
+
if under_known_repo and not os.path.exists(path):
|
|
378
|
+
logger.warning(
|
|
379
|
+
f"Removing stale {ft} index entry whose folder "
|
|
380
|
+
f"no longer exists: {path}")
|
|
381
|
+
for mt_key in [
|
|
382
|
+
k for k in self.modified_times
|
|
383
|
+
if os.path.normpath(k) == norm
|
|
384
|
+
or os.path.normpath(k).startswith(norm + os.sep)]:
|
|
385
|
+
del self.modified_times[mt_key]
|
|
386
|
+
changed = True
|
|
387
|
+
continue
|
|
388
|
+
kept.append(item)
|
|
389
|
+
self.indices[ft] = kept
|
|
390
|
+
|
|
352
391
|
if force_rebuild or changed:
|
|
353
392
|
logger.debug(
|
|
354
393
|
"Changes detected, saving updated index and modified times.")
|
|
@@ -13,12 +13,17 @@ from .script_action import ScriptAction
|
|
|
13
13
|
from .cache_action import CacheAction
|
|
14
14
|
from .cfg_action import CfgAction
|
|
15
15
|
from .experiment_action import ExperimentAction
|
|
16
|
+
from .typo_mixin import TypoMixin
|
|
16
17
|
|
|
17
18
|
from .item import Item
|
|
18
19
|
from .action_factory import get_action
|
|
19
20
|
from .logger import logger, logging
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
class TypoArgumentParser(TypoMixin, argparse.ArgumentParser):
|
|
24
|
+
"""ArgumentParser that suggests close matches when the user mistype a command or target."""
|
|
25
|
+
|
|
26
|
+
|
|
22
27
|
class Automation:
|
|
23
28
|
action_object = None
|
|
24
29
|
automation_type = None
|
|
@@ -316,10 +321,6 @@ def process_console_output(res, target, action, run_args):
|
|
|
316
321
|
f"Warning code: {warning['code']}, Discription: {warning['description']}")
|
|
317
322
|
|
|
318
323
|
|
|
319
|
-
if default_parent is None:
|
|
320
|
-
default_parent = Action()
|
|
321
|
-
|
|
322
|
-
|
|
323
324
|
log_flag_aliases = {'-v': '--verbose', '-s': '--silent'}
|
|
324
325
|
log_levels = {'--verbose': logging.DEBUG, '--silent': logging.WARNING}
|
|
325
326
|
|
|
@@ -340,21 +341,15 @@ def convert_hyphen_to_underscore_in_args():
|
|
|
340
341
|
|
|
341
342
|
|
|
342
343
|
def build_pre_parser():
|
|
343
|
-
pre_parser =
|
|
344
|
+
pre_parser = TypoArgumentParser(add_help=False)
|
|
344
345
|
pre_parser.add_argument(
|
|
345
346
|
"action",
|
|
346
347
|
nargs="?",
|
|
347
348
|
help="Top-level action (run, build, help, etc.)")
|
|
349
|
+
# No choices here: valid targets depend on the action, so validation is
|
|
350
|
+
# left to the per-action subparsers in build_parser().
|
|
348
351
|
pre_parser.add_argument(
|
|
349
352
|
"target",
|
|
350
|
-
choices=[
|
|
351
|
-
'run',
|
|
352
|
-
'script',
|
|
353
|
-
'cache',
|
|
354
|
-
'repo',
|
|
355
|
-
'repos',
|
|
356
|
-
'experiment',
|
|
357
|
-
'all'],
|
|
358
353
|
nargs="?",
|
|
359
354
|
help="Target (repo, script, cache, ...)")
|
|
360
355
|
pre_parser.add_argument("-h", "--help", action="store_true")
|
|
@@ -362,7 +357,7 @@ def build_pre_parser():
|
|
|
362
357
|
|
|
363
358
|
|
|
364
359
|
def build_parser(pre_args):
|
|
365
|
-
parser =
|
|
360
|
+
parser = TypoArgumentParser(
|
|
366
361
|
prog="mlc",
|
|
367
362
|
description="Manage repos, scripts, and caches.",
|
|
368
363
|
add_help=False)
|
|
@@ -424,6 +419,11 @@ def build_parser(pre_args):
|
|
|
424
419
|
# Load cfg
|
|
425
420
|
load_parser = subparsers.add_parser("load", add_help=False)
|
|
426
421
|
load_parser.add_argument("target", choices=["cfg"])
|
|
422
|
+
load_parser.add_argument(
|
|
423
|
+
'details',
|
|
424
|
+
nargs='?',
|
|
425
|
+
help='Details or identifier (optional)')
|
|
426
|
+
load_parser.add_argument('extra', nargs=argparse.REMAINDER)
|
|
427
427
|
return parser
|
|
428
428
|
|
|
429
429
|
|
|
@@ -480,6 +480,14 @@ def build_run_args(args):
|
|
|
480
480
|
if hasattr(args, 'target') and args.target:
|
|
481
481
|
run_args['reindex_target'] = args.target
|
|
482
482
|
|
|
483
|
+
# Merge bulk options from a YAML/JSON file (--mlc_input_file=run.yaml).
|
|
484
|
+
# Done last so explicit CLI args (including positional tags) win.
|
|
485
|
+
r = utils.load_cli_input_file(run_args)
|
|
486
|
+
if r['return'] > 0:
|
|
487
|
+
logger.error(r['error'])
|
|
488
|
+
sys.exit(1)
|
|
489
|
+
run_args = r['args_dict']
|
|
490
|
+
|
|
483
491
|
# Check for path-only flag (for script-friendly output)
|
|
484
492
|
if run_args.get('path_only') or run_args.get('p'):
|
|
485
493
|
run_args['path_only'] = True
|
|
@@ -86,6 +86,7 @@ TOP_LEVEL_SCHEMA = {
|
|
|
86
86
|
|
|
87
87
|
# Docker
|
|
88
88
|
"docker": DICT, # dict - see DOCKER_SCHEMA
|
|
89
|
+
"apptainer": DICT, # dict - apptainer overrides; merges with docker settings
|
|
89
90
|
|
|
90
91
|
# Output / debugging
|
|
91
92
|
"print_env_at_the_end": DICT, # dict[str, list[str]]
|
|
@@ -164,6 +165,7 @@ VARIATION_ENTRY_SCHEMA = {
|
|
|
164
165
|
"state": DICT,
|
|
165
166
|
"const": DICT,
|
|
166
167
|
"docker": DICT,
|
|
168
|
+
"apptainer": DICT,
|
|
167
169
|
"alias": STR,
|
|
168
170
|
"default_version": STR_OR_FLOAT,
|
|
169
171
|
"required_disk_space": INT,
|
|
@@ -248,6 +250,7 @@ UPDATE_META_IF_ENV_SCHEMA = {
|
|
|
248
250
|
"default_env": DICT,
|
|
249
251
|
"default_variations": DICT,
|
|
250
252
|
"docker": DICT,
|
|
253
|
+
"apptainer": DICT,
|
|
251
254
|
"adr": DICT,
|
|
252
255
|
"ad": DICT,
|
|
253
256
|
}
|
|
@@ -403,6 +406,21 @@ def validate_meta(data, file_path=""):
|
|
|
403
406
|
errors.append(
|
|
404
407
|
f"{prefix}docker.{dk} has type '{actual}', expected {allowed}")
|
|
405
408
|
|
|
409
|
+
# Validate apptainer section (same schema as docker; apptainer overrides
|
|
410
|
+
# docker)
|
|
411
|
+
apptainer = data.get("apptainer")
|
|
412
|
+
if isinstance(apptainer, dict):
|
|
413
|
+
for ak, av in apptainer.items():
|
|
414
|
+
if ak not in DOCKER_SCHEMA:
|
|
415
|
+
warnings.append(
|
|
416
|
+
f"{prefix}apptainer: unknown key '{ak}'")
|
|
417
|
+
continue
|
|
418
|
+
actual = type(av).__name__
|
|
419
|
+
allowed = DOCKER_SCHEMA[ak]
|
|
420
|
+
if actual not in allowed:
|
|
421
|
+
errors.append(
|
|
422
|
+
f"{prefix}apptainer.{ak} has type '{actual}', expected {allowed}")
|
|
423
|
+
|
|
406
424
|
# Validate tests section
|
|
407
425
|
tests = data.get("tests")
|
|
408
426
|
if isinstance(tests, dict):
|