mlcflow 1.3.6__tar.gz → 1.3.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mlcflow-1.3.6 → mlcflow-1.3.9}/PKG-INFO +10 -2
- {mlcflow-1.3.6 → mlcflow-1.3.9}/README.md +9 -1
- mlcflow-1.3.9/VERSION +1 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/apptainer.py +117 -30
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/docker_utils.py +6 -2
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/meta_schema.py +18 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/module.py +35 -6
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/slurm_run.py +52 -14
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/utils.py +9 -9
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/action.py +8 -1
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/index.py +58 -3
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/main.py +66 -43
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/meta_schema.py +18 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/repo_action.py +684 -188
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/script_action.py +368 -92
- mlcflow-1.3.9/mlc/typo_mixin.py +127 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/utils.py +56 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/PKG-INFO +10 -2
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/SOURCES.txt +11 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/entry_points.txt +2 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/pyproject.toml +2 -0
- mlcflow-1.3.9/tests/test_apptainer_docker_unification.py +286 -0
- mlcflow-1.3.9/tests/test_build_wheels_workflow.py +142 -0
- mlcflow-1.3.9/tests/test_cli_input_file.py +132 -0
- mlcflow-1.3.9/tests/test_container_cmd_quoting.py +189 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_convert_env_to_script.py +33 -0
- mlcflow-1.3.9/tests/test_help_coverage.py +109 -0
- mlcflow-1.3.9/tests/test_index_reconcile.py +113 -0
- mlcflow-1.3.9/tests/test_pull_repo_thread_safety.py +1036 -0
- mlcflow-1.3.9/tests/test_release_version.py +101 -0
- mlcflow-1.3.9/tests/test_run_target_dispatch.py +41 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_script_action_apptainer.py +1 -1
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_slurm_run.py +81 -0
- mlcflow-1.3.9/tests/test_typo_mixin.py +294 -0
- mlcflow-1.3.6/VERSION +0 -1
- {mlcflow-1.3.6 → mlcflow-1.3.9}/LICENSE.md +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/COPYRIGHT.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/__init__.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/README.md +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/__init__.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/cache_utils.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/compat.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/doc.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/docker.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/experiment.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/help.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/lint.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/meta.json +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/remote_run.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/script_utils.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/validate.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/__init__.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/__main__.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/action_factory.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/cache_action.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/cfg_action.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/error_codes.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/experiment_action.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/item.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/logger.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/repo.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/dependency_links.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/requires.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/top_level.txt +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/setup.cfg +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_action_invalid_meta_entries.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_cache_mark_tmp.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_installer_venv_deactivation.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_mlcflow_unix_installer.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_repo_action_extra_git_args.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_thread_safety.py +0 -0
- {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_unix_installer_venv.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mlcflow
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.9
|
|
4
4
|
Summary: An automation interface tailored for CPU/GPU benchmarking
|
|
5
5
|
Author-email: MLCommons <systems@mlcommons.org>
|
|
6
6
|
License:
|
|
@@ -214,13 +214,21 @@ Dynamic: license-file
|
|
|
214
214
|
|
|
215
215
|
MLCFlow is a versatile CLI and Python interface developed by MLCommons in collaboration with a dedicated team of volunteers (see [Contributors](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md)). It serves as a streamlined replacement for the [CMind](https://github.com/mlcommons/ck/tree/master/cm) tool, designed to drive the automation workflows of MLPerf benchmarks more efficiently. You can use this tool for any of your workflow automation tasks.
|
|
216
216
|
|
|
217
|
-
The concept behind CMind originated from **Grigori Fursin
|
|
217
|
+
The concept behind CMind originated from [**Grigori Fursin**](https://github.com/gfursin), while the **MLPerf Automations** project was created by [**Grigori Fursin**](https://github.com/gfursin) and [**Arjun Suresh**](https://github.com/arjunsuresh), whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by [**Arjun Suresh**](https://github.com/arjunsuresh) and [**Anandhu Sooraj**](https://github.com/anandhu-eng), with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the developers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
|
|
218
218
|
|
|
219
219
|
On February 9, 2025, MLCFlow released its first stable version, 1.0.0.
|
|
220
220
|
|
|
221
221
|
### Key Features
|
|
222
222
|
Building upon the core idea of CMind—wrapping native scripts with Python wrappers and YAML metadata—MLCFlow focuses exclusively on key automation components: **Scripts**, along with its complementary modules: **Cache**, **Docker**, and **Experiments**. This targeted design simplifies both implementation and interface, enabling a more user-friendly experience.
|
|
223
223
|
|
|
224
|
+
- **Typo detection** — Mistyped actions or targets produce a "Did you mean …?" hint before the error message, so mistakes are quick to correct:
|
|
225
|
+
```
|
|
226
|
+
$ mlc rune script
|
|
227
|
+
Did you mean one of: 'prune', 'run'?
|
|
228
|
+
mlc: error: argument command: invalid choice: 'rune' …
|
|
229
|
+
```
|
|
230
|
+
See [docs/typo_detection.md](docs/typo_detection.md) for details.
|
|
231
|
+
|
|
224
232
|
---
|
|
225
233
|
|
|
226
234
|
### Status
|
|
@@ -12,13 +12,21 @@
|
|
|
12
12
|
|
|
13
13
|
MLCFlow is a versatile CLI and Python interface developed by MLCommons in collaboration with a dedicated team of volunteers (see [Contributors](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md)). It serves as a streamlined replacement for the [CMind](https://github.com/mlcommons/ck/tree/master/cm) tool, designed to drive the automation workflows of MLPerf benchmarks more efficiently. You can use this tool for any of your workflow automation tasks.
|
|
14
14
|
|
|
15
|
-
The concept behind CMind originated from **Grigori Fursin
|
|
15
|
+
The concept behind CMind originated from [**Grigori Fursin**](https://github.com/gfursin), while the **MLPerf Automations** project was created by [**Grigori Fursin**](https://github.com/gfursin) and [**Arjun Suresh**](https://github.com/arjunsuresh), whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by [**Arjun Suresh**](https://github.com/arjunsuresh) and [**Anandhu Sooraj**](https://github.com/anandhu-eng), with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the developers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
|
|
16
16
|
|
|
17
17
|
On February 9, 2025, MLCFlow released its first stable version, 1.0.0.
|
|
18
18
|
|
|
19
19
|
### Key Features
|
|
20
20
|
Building upon the core idea of CMind—wrapping native scripts with Python wrappers and YAML metadata—MLCFlow focuses exclusively on key automation components: **Scripts**, along with its complementary modules: **Cache**, **Docker**, and **Experiments**. This targeted design simplifies both implementation and interface, enabling a more user-friendly experience.
|
|
21
21
|
|
|
22
|
+
- **Typo detection** — Mistyped actions or targets produce a "Did you mean …?" hint before the error message, so mistakes are quick to correct:
|
|
23
|
+
```
|
|
24
|
+
$ mlc rune script
|
|
25
|
+
Did you mean one of: 'prune', 'run'?
|
|
26
|
+
mlc: error: argument command: invalid choice: 'rune' …
|
|
27
|
+
```
|
|
28
|
+
See [docs/typo_detection.md](docs/typo_detection.md) for details.
|
|
29
|
+
|
|
22
30
|
---
|
|
23
31
|
|
|
24
32
|
### Status
|
mlcflow-1.3.9/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
1.3.9
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import shlex
|
|
2
3
|
from mlc import utils
|
|
3
4
|
from utils import *
|
|
4
5
|
import logging
|
|
@@ -8,9 +9,10 @@ import copy
|
|
|
8
9
|
|
|
9
10
|
def apptainerfile(self_module, input_params):
|
|
10
11
|
|
|
11
|
-
# Step 1: Prune and prepare input
|
|
12
|
+
# Step 1: Prune and prepare input (remove both apptainer_ and docker_
|
|
13
|
+
# prefixed keys)
|
|
12
14
|
prune_result = prune_input(
|
|
13
|
-
{'input': input_params, 'extra_keys_starts_with': ['apptainer_']})
|
|
15
|
+
{'input': input_params, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
|
|
14
16
|
if prune_result['return'] > 0:
|
|
15
17
|
return prune_result
|
|
16
18
|
|
|
@@ -56,13 +58,15 @@ def apptainerfile(self_module, input_params):
|
|
|
56
58
|
|
|
57
59
|
run_state = self_module.run_state
|
|
58
60
|
|
|
59
|
-
apptainer_settings = run_state.get('docker', {})
|
|
61
|
+
apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
|
|
62
|
+
utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
|
|
63
|
+
'append_lists': True, 'append_unique': True})
|
|
60
64
|
apptainer_settings_default_env = apptainer_settings.get('default_env', {})
|
|
61
65
|
for key in apptainer_settings_default_env:
|
|
62
66
|
env.setdefault(key, apptainer_settings_default_env[key])
|
|
63
67
|
|
|
64
68
|
if not apptainer_settings.get('run', True) and not input_params.get(
|
|
65
|
-
'apptainer_run_override', False):
|
|
69
|
+
'apptainer_run_override', input_params.get('docker_run_override', False)):
|
|
66
70
|
logger.info("Apptainer 'run' is set to False in meta.yaml")
|
|
67
71
|
return {'return': 0,
|
|
68
72
|
'warning': 'Apptainer run is set to false in script meta'}
|
|
@@ -86,7 +90,9 @@ def apptainerfile(self_module, input_params):
|
|
|
86
90
|
if update_state_result['return'] > 0:
|
|
87
91
|
return update_state_result
|
|
88
92
|
|
|
89
|
-
apptainer_settings = run_state.get('docker', {})
|
|
93
|
+
apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
|
|
94
|
+
utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
|
|
95
|
+
'append_lists': True, 'append_unique': True})
|
|
90
96
|
|
|
91
97
|
# Prune temporary environment variables
|
|
92
98
|
run_command = copy.deepcopy(run_command_arc)
|
|
@@ -102,7 +108,7 @@ def apptainerfile(self_module, input_params):
|
|
|
102
108
|
'tags': script_tags,
|
|
103
109
|
'fake_run': True,
|
|
104
110
|
'docker_settings': apptainer_settings,
|
|
105
|
-
'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', ''))
|
|
111
|
+
'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', input_params.get('docker_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', '')))
|
|
106
112
|
})
|
|
107
113
|
if regenerate_result['return'] > 0:
|
|
108
114
|
return regenerate_result
|
|
@@ -152,13 +158,23 @@ def apptainerfile(self_module, input_params):
|
|
|
152
158
|
if apptainer_inputs.get('mlc_repo_path', '') != '':
|
|
153
159
|
mlc_apptainer_input['mlc_repo_path'] = apptainer_inputs['mlc_repo_path']
|
|
154
160
|
|
|
161
|
+
if is_true(get_apptainer_input(input_params, 'host_mlc_repos', '')):
|
|
162
|
+
mlc_apptainer_input['host_mlc_repos'] = 'yes'
|
|
163
|
+
|
|
164
|
+
# Bind destinations must exist in the image rootfs, otherwise the
|
|
165
|
+
# build-time fake run fails when the binds are applied.
|
|
166
|
+
build_binds = get_apptainer_bind_mounts(
|
|
167
|
+
input_params, env, apptainer_settings, run_state)
|
|
168
|
+
if build_binds:
|
|
169
|
+
mlc_apptainer_input['bind'] = build_binds
|
|
170
|
+
|
|
155
171
|
apptainer_v = False
|
|
156
172
|
apptainer_s = False
|
|
157
173
|
if is_true(input_params.get(
|
|
158
|
-
'apptainer_v', input_params.get('apptainer_verbose', False))):
|
|
174
|
+
'apptainer_v', input_params.get('apptainer_verbose', input_params.get('docker_v', input_params.get('docker_verbose', False))))):
|
|
159
175
|
apptainer_v = True
|
|
160
176
|
if is_true(input_params.get(
|
|
161
|
-
'apptainer_s', input_params.get('apptainer_silent', False))):
|
|
177
|
+
'apptainer_s', input_params.get('apptainer_silent', input_params.get('docker_s', input_params.get('docker_silent', False))))):
|
|
162
178
|
apptainer_s = True
|
|
163
179
|
|
|
164
180
|
if apptainer_s and apptainer_v:
|
|
@@ -213,11 +229,16 @@ def apptainer_run(self_module, i):
|
|
|
213
229
|
if quiet:
|
|
214
230
|
env['MLC_QUIET'] = 'yes'
|
|
215
231
|
|
|
216
|
-
regenerate_def_file = not i.get(
|
|
217
|
-
|
|
232
|
+
regenerate_def_file = not i.get(
|
|
233
|
+
'apptainer_noregenerate', i.get(
|
|
234
|
+
'docker_noregenerate', False))
|
|
235
|
+
rebuild_apptainer_image = i.get(
|
|
236
|
+
'apptainer_rebuild', i.get(
|
|
237
|
+
'docker_rebuild', False))
|
|
218
238
|
|
|
219
|
-
# Prune unnecessary Apptainer-related input keys
|
|
220
|
-
r = prune_input(
|
|
239
|
+
# Prune unnecessary Apptainer- and Docker-related input keys
|
|
240
|
+
r = prune_input(
|
|
241
|
+
{'input': i, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
|
|
221
242
|
f_run_cmd = r['new_input']
|
|
222
243
|
|
|
223
244
|
# Save current directory and prepare to search for scripts
|
|
@@ -259,7 +280,7 @@ def apptainer_run(self_module, i):
|
|
|
259
280
|
mounts = copy.deepcopy(
|
|
260
281
|
i.get(
|
|
261
282
|
'apptainer_mounts',
|
|
262
|
-
[]))
|
|
283
|
+
i.get('docker_mounts', [])))
|
|
263
284
|
variations = meta.get('variations', {})
|
|
264
285
|
|
|
265
286
|
if not hasattr(self_module, 'run_state'):
|
|
@@ -273,7 +294,9 @@ def apptainer_run(self_module, i):
|
|
|
273
294
|
|
|
274
295
|
run_state = self_module.run_state
|
|
275
296
|
|
|
276
|
-
apptainer_settings = run_state.get('docker', {})
|
|
297
|
+
apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
|
|
298
|
+
utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
|
|
299
|
+
'append_lists': True, 'append_unique': True})
|
|
277
300
|
|
|
278
301
|
apptainer_settings_default_env = apptainer_settings.get('default_env', {})
|
|
279
302
|
for key in apptainer_settings_default_env:
|
|
@@ -297,7 +320,7 @@ def apptainer_run(self_module, i):
|
|
|
297
320
|
|
|
298
321
|
# Skip scripts marked as non-runnable
|
|
299
322
|
if not apptainer_settings.get('run', True) and not i.get(
|
|
300
|
-
'apptainer_run_override', False):
|
|
323
|
+
'apptainer_run_override', i.get('docker_run_override', False)):
|
|
301
324
|
logger.info("apptainer.run set to False in meta.yaml")
|
|
302
325
|
return {'return': 0,
|
|
303
326
|
'warning': 'Apptainer run is set to false in script meta'}
|
|
@@ -337,12 +360,13 @@ def apptainer_run(self_module, i):
|
|
|
337
360
|
mounts, env, apptainer_settings, run_state)
|
|
338
361
|
container_env_string = bind_mounts.get('container_env_string', '')
|
|
339
362
|
|
|
340
|
-
# Forward input-mapped env vars to the container's mlcr command
|
|
363
|
+
# Forward input-mapped env vars to the container's mlcr command.
|
|
364
|
+
# Unresolved default_env placeholders (<<<VAR>>>) are dropped.
|
|
341
365
|
input_mapping = meta.get('input_mapping', {})
|
|
342
366
|
for input_key, env_key in input_mapping.items():
|
|
343
367
|
val = env.get(env_key, '')
|
|
344
|
-
if val and isinstance(val, str):
|
|
345
|
-
container_env_string += f' --env.{env_key}={val}'
|
|
368
|
+
if val and isinstance(val, str) and '<<<' not in val:
|
|
369
|
+
container_env_string += f' --env.{env_key}={shlex.quote(val)}'
|
|
346
370
|
|
|
347
371
|
# Forward framework-level env keys (MLC_OUTDIRNAME, MLC_INPUT, MLC_OUTPUT, etc.)
|
|
348
372
|
# These are skipped in regenerate_script_cmd (docker_utils.py) and must be
|
|
@@ -352,14 +376,14 @@ def apptainer_run(self_module, i):
|
|
|
352
376
|
for input_key in framework_input_mapping:
|
|
353
377
|
env_key = 'MLC_' + input_key.upper().replace('-', '_').replace(' ', '_')
|
|
354
378
|
val = env.get(env_key, '')
|
|
355
|
-
if val and isinstance(val, str):
|
|
356
|
-
container_env_string += f' --env.{env_key}={val}'
|
|
379
|
+
if val and isinstance(val, str) and '<<<' not in val:
|
|
380
|
+
container_env_string += f' --env.{env_key}={shlex.quote(val)}'
|
|
357
381
|
|
|
358
382
|
# Forward proxy env vars (matching Docker behavior)
|
|
359
383
|
for proxy_key in self_module.host_env_keys:
|
|
360
384
|
proxy_value = os.environ.get(proxy_key)
|
|
361
385
|
if proxy_value:
|
|
362
|
-
container_env_string += f' --env.{proxy_key}={proxy_value}'
|
|
386
|
+
container_env_string += f' --env.{proxy_key}={shlex.quote(proxy_value)}'
|
|
363
387
|
|
|
364
388
|
# Generate the run command
|
|
365
389
|
r = regenerate_script_cmd({'script_uid': script_uid,
|
|
@@ -379,6 +403,20 @@ def apptainer_run(self_module, i):
|
|
|
379
403
|
'quiet': True, 'real_run': True,
|
|
380
404
|
'add_deps_recursive': {'build-apptainer-image': {'def_file': def_file_path}}
|
|
381
405
|
}
|
|
406
|
+
|
|
407
|
+
# Detached mode (mirrors --docker_dt): run as a background instance.
|
|
408
|
+
if is_true(get_apptainer_input(
|
|
409
|
+
i, 'dt', get_apptainer_input(i, 'detached', ''))):
|
|
410
|
+
mlc_apptainer_input['detached'] = 'yes'
|
|
411
|
+
# Drop into an interactive shell after the run (mirrors --docker_it).
|
|
412
|
+
elif is_true(get_apptainer_input(
|
|
413
|
+
i, 'it', get_apptainer_input(i, 'interactive', ''))):
|
|
414
|
+
mlc_apptainer_input['interactive'] = 'yes'
|
|
415
|
+
|
|
416
|
+
# Tell the run step host repos were baked in, so it points MLC_REPOS at
|
|
417
|
+
# /opt/mlc_host_repos (matters under fakeroot, which otherwise overrides it).
|
|
418
|
+
if is_true(get_apptainer_input(i, 'host_mlc_repos', '')):
|
|
419
|
+
mlc_apptainer_input['host_mlc_repos'] = 'yes'
|
|
382
420
|
# Forward build options to build-apptainer-image
|
|
383
421
|
build_image_extras = {}
|
|
384
422
|
if apptainer_inputs.get('ignore_fakeroot_cmd'):
|
|
@@ -390,6 +428,10 @@ def apptainer_run(self_module, i):
|
|
|
390
428
|
build_image_extras)
|
|
391
429
|
# Wire resolved docker.mounts to apptainer bind mounts
|
|
392
430
|
resolved_mounts = bind_mounts.get('mounts', [])
|
|
431
|
+
mlc_local_bind = get_mlc_local_bind(i)
|
|
432
|
+
if mlc_local_bind:
|
|
433
|
+
os.makedirs(mlc_local_bind.split(':', 1)[0], exist_ok=True)
|
|
434
|
+
resolved_mounts.append(mlc_local_bind)
|
|
393
435
|
if resolved_mounts:
|
|
394
436
|
mlc_apptainer_input['bind'] = resolved_mounts
|
|
395
437
|
|
|
@@ -443,14 +485,15 @@ def prepare_apptainer_inputs(input_params, apptainer_settings,
|
|
|
443
485
|
"network", "security_opt"
|
|
444
486
|
]
|
|
445
487
|
|
|
446
|
-
# Collect inputs
|
|
447
|
-
apptainer_inputs = {
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
488
|
+
# Collect inputs: apptainer_X > docker_X > meta settings > default
|
|
489
|
+
apptainer_inputs = {}
|
|
490
|
+
for key in keys:
|
|
491
|
+
value = input_params.get(
|
|
492
|
+
f"apptainer_{key}",
|
|
493
|
+
input_params.get(
|
|
494
|
+
f"docker_{key}", apptainer_settings.get(key, get_apptainer_default(key))))
|
|
495
|
+
if value is not None:
|
|
496
|
+
apptainer_inputs[key] = value
|
|
454
497
|
|
|
455
498
|
# Convert boolean values to 'yes'/'no' strings for MLC input mapping
|
|
456
499
|
for key in list(apptainer_inputs.keys()):
|
|
@@ -544,10 +587,54 @@ def process_apptainer_mounts(mounts, env, apptainer_settings, run_state):
|
|
|
544
587
|
'container_env_string': container_env_string}
|
|
545
588
|
|
|
546
589
|
|
|
590
|
+
def get_mlc_local_bind(input_params):
|
|
591
|
+
"""Bind for --apptainer_mlc_local_dir: a writable host directory over the
|
|
592
|
+
container's MLC local dir (cache), used with --apptainer_host_mlc_repos so
|
|
593
|
+
caches do not land in the small, discarded --writable-tmpfs overlay."""
|
|
594
|
+
local_dir = get_apptainer_input(input_params, 'mlc_local_dir', '')
|
|
595
|
+
if not local_dir or not is_true(
|
|
596
|
+
get_apptainer_input(input_params, 'host_mlc_repos', '')):
|
|
597
|
+
return ''
|
|
598
|
+
return f"{os.path.abspath(local_dir)}:/opt/mlc_host_repos/local"
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def get_apptainer_bind_mounts(input_params, env, apptainer_settings,
|
|
602
|
+
run_state):
|
|
603
|
+
"""All bind mounts (user binds, resolved docker.mounts, MLC local dir)
|
|
604
|
+
known before the image is built."""
|
|
605
|
+
binds = get_apptainer_input(input_params, 'bind',
|
|
606
|
+
apptainer_settings.get('bind')) or []
|
|
607
|
+
if isinstance(binds, str):
|
|
608
|
+
binds = [binds]
|
|
609
|
+
binds = list(binds)
|
|
610
|
+
mounts = copy.deepcopy(get_apptainer_input(input_params, 'mounts', []))
|
|
611
|
+
binds += process_apptainer_mounts(
|
|
612
|
+
mounts, env, apptainer_settings, run_state)['mounts']
|
|
613
|
+
mlc_local_bind = get_mlc_local_bind(input_params)
|
|
614
|
+
if mlc_local_bind:
|
|
615
|
+
binds.append(mlc_local_bind)
|
|
616
|
+
return binds
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def get_apptainer_input(input_params, key, default=None):
|
|
620
|
+
"""Return apptainer_<key> if provided, else fall back to docker_<key>.
|
|
621
|
+
|
|
622
|
+
Lets docker_* options passed to `mlca` take effect unless the matching
|
|
623
|
+
apptainer_* option overrides them.
|
|
624
|
+
"""
|
|
625
|
+
apptainer_key = f"apptainer_{key}"
|
|
626
|
+
if input_params.get(apptainer_key) is not None:
|
|
627
|
+
return input_params[apptainer_key]
|
|
628
|
+
docker_key = f"docker_{key}"
|
|
629
|
+
if input_params.get(docker_key) is not None:
|
|
630
|
+
return input_params[docker_key]
|
|
631
|
+
return default
|
|
632
|
+
|
|
633
|
+
|
|
547
634
|
def get_apptainer_default(key):
|
|
548
635
|
defaults = {
|
|
549
636
|
"mlc_repo": "mlcommons@mlperf-automations",
|
|
550
|
-
"mlc_repo_branch": "
|
|
637
|
+
"mlc_repo_branch": "main",
|
|
551
638
|
"os": "ubuntu",
|
|
552
639
|
"os_version": "",
|
|
553
640
|
"fake_run_deps": False,
|
|
@@ -328,7 +328,7 @@ def regenerate_script_cmd(i):
|
|
|
328
328
|
if isinstance(val, str) and (
|
|
329
329
|
os.path.join("local", "cache", "") in val or
|
|
330
330
|
os.path.join("MLC", "repos", "") in val or
|
|
331
|
-
"<<<" in
|
|
331
|
+
"<<<" in val
|
|
332
332
|
)
|
|
333
333
|
]
|
|
334
334
|
|
|
@@ -373,6 +373,10 @@ def regenerate_script_cmd(i):
|
|
|
373
373
|
for key in keys:
|
|
374
374
|
if key in ["input", "output", "outdirname"]:
|
|
375
375
|
continue # We have the corresponding env keys in container env string
|
|
376
|
+
# Provenance only; nesting an already shell-joined command breaks
|
|
377
|
+
# quoting in the container command and tmp-run.sh.
|
|
378
|
+
if key == "mlc_run_cmd" and not prefix:
|
|
379
|
+
continue
|
|
376
380
|
# Construct the full key with the prefix.
|
|
377
381
|
full_key = f"{prefix}.{key}" if prefix else key
|
|
378
382
|
|
|
@@ -422,7 +426,7 @@ def regenerate_script_cmd(i):
|
|
|
422
426
|
def get_docker_default(key):
|
|
423
427
|
defaults = {
|
|
424
428
|
"mlc_repo": "mlcommons@mlperf-automations",
|
|
425
|
-
"mlc_repo_branch": "
|
|
429
|
+
"mlc_repo_branch": "main",
|
|
426
430
|
"os": "ubuntu",
|
|
427
431
|
"os_version": "",
|
|
428
432
|
"fake_run_deps": False,
|
|
@@ -86,6 +86,7 @@ TOP_LEVEL_SCHEMA = {
|
|
|
86
86
|
|
|
87
87
|
# Docker
|
|
88
88
|
"docker": DICT, # dict - see DOCKER_SCHEMA
|
|
89
|
+
"apptainer": DICT, # dict - apptainer overrides; merges with docker settings
|
|
89
90
|
|
|
90
91
|
# Output / debugging
|
|
91
92
|
"print_env_at_the_end": DICT, # dict[str, list[str]]
|
|
@@ -164,6 +165,7 @@ VARIATION_ENTRY_SCHEMA = {
|
|
|
164
165
|
"state": DICT,
|
|
165
166
|
"const": DICT,
|
|
166
167
|
"docker": DICT,
|
|
168
|
+
"apptainer": DICT,
|
|
167
169
|
"alias": STR,
|
|
168
170
|
"default_version": STR_OR_FLOAT,
|
|
169
171
|
"required_disk_space": INT,
|
|
@@ -248,6 +250,7 @@ UPDATE_META_IF_ENV_SCHEMA = {
|
|
|
248
250
|
"default_env": DICT,
|
|
249
251
|
"default_variations": DICT,
|
|
250
252
|
"docker": DICT,
|
|
253
|
+
"apptainer": DICT,
|
|
251
254
|
"adr": DICT,
|
|
252
255
|
"ad": DICT,
|
|
253
256
|
}
|
|
@@ -403,6 +406,21 @@ def validate_meta(data, file_path=""):
|
|
|
403
406
|
errors.append(
|
|
404
407
|
f"{prefix}docker.{dk} has type '{actual}', expected {allowed}")
|
|
405
408
|
|
|
409
|
+
# Validate apptainer section (same schema as docker; apptainer overrides
|
|
410
|
+
# docker)
|
|
411
|
+
apptainer = data.get("apptainer")
|
|
412
|
+
if isinstance(apptainer, dict):
|
|
413
|
+
for ak, av in apptainer.items():
|
|
414
|
+
if ak not in DOCKER_SCHEMA:
|
|
415
|
+
warnings.append(
|
|
416
|
+
f"{prefix}apptainer: unknown key '{ak}'")
|
|
417
|
+
continue
|
|
418
|
+
actual = type(av).__name__
|
|
419
|
+
allowed = DOCKER_SCHEMA[ak]
|
|
420
|
+
if actual not in allowed:
|
|
421
|
+
errors.append(
|
|
422
|
+
f"{prefix}apptainer.{ak} has type '{actual}', expected {allowed}")
|
|
423
|
+
|
|
406
424
|
# Validate tests section
|
|
407
425
|
tests = data.get("tests")
|
|
408
426
|
if isinstance(tests, dict):
|
|
@@ -151,7 +151,7 @@ class ScriptAutomation(Automation):
|
|
|
151
151
|
for f in ['fake_deps', 'cache']:
|
|
152
152
|
run_state.setdefault(f, False)
|
|
153
153
|
|
|
154
|
-
for d in ['input_mapping', 'docker', 'remote_run']:
|
|
154
|
+
for d in ['input_mapping', 'docker', 'remote_run', 'apptainer']:
|
|
155
155
|
run_state.setdefault(d, {})
|
|
156
156
|
|
|
157
157
|
for l in ['deps', 'post_deps', 'prehook_deps', 'posthook_deps',
|
|
@@ -4558,6 +4558,18 @@ pip install mlcflow
|
|
|
4558
4558
|
i['remote_action'] = 'slurm-experiment'
|
|
4559
4559
|
return remote_run(self, i)
|
|
4560
4560
|
|
|
4561
|
+
############################################################
|
|
4562
|
+
def remote_slurm_docker(self, i):
|
|
4563
|
+
from script.remote_run import remote_run
|
|
4564
|
+
i['remote_action'] = 'slurm-docker'
|
|
4565
|
+
return remote_run(self, i)
|
|
4566
|
+
|
|
4567
|
+
############################################################
|
|
4568
|
+
def remote_slurm_apptainer(self, i):
|
|
4569
|
+
from script.remote_run import remote_run
|
|
4570
|
+
i['remote_action'] = 'slurm-apptainer'
|
|
4571
|
+
return remote_run(self, i)
|
|
4572
|
+
|
|
4561
4573
|
##########################################################################
|
|
4562
4574
|
|
|
4563
4575
|
def _available_variations(self, i):
|
|
@@ -5428,12 +5440,14 @@ def convert_env_to_script(env, os_info, start_script=None):
|
|
|
5428
5440
|
env_separator = os_info.get('env_separator', ':')
|
|
5429
5441
|
env_value = env_separator.join(str(v) for v in env_value)
|
|
5430
5442
|
|
|
5431
|
-
|
|
5432
|
-
#
|
|
5433
|
-
#
|
|
5443
|
+
env_value = str(env_value)
|
|
5444
|
+
# cmd.exe `set` takes the value literally (backslash is not an escape
|
|
5445
|
+
# there), so only the POSIX double-quoted export needs escaping.
|
|
5446
|
+
if not is_windows:
|
|
5447
|
+
env_quote = os_info.get('env_quote', '"')
|
|
5448
|
+
env_value = env_value.replace(env_quote, f"""\\{env_quote}""")
|
|
5434
5449
|
env_command = os_info['set_env'].replace(
|
|
5435
|
-
'${key}', key).replace(
|
|
5436
|
-
'${value}', str(env_value).replace(env_quote, f"""\\{env_quote}"""))
|
|
5450
|
+
'${key}', key).replace('${value}', env_value)
|
|
5437
5451
|
script.append(env_command)
|
|
5438
5452
|
|
|
5439
5453
|
return script
|
|
@@ -5768,6 +5782,14 @@ def _apply_conditional_meta_updates(update_meta_if_env, default_env, env, const,
|
|
|
5768
5782
|
'append_lists': True,
|
|
5769
5783
|
'append_unique': True})
|
|
5770
5784
|
|
|
5785
|
+
if c_meta.get('apptainer', {}):
|
|
5786
|
+
if not run_state.get('apptainer', {}):
|
|
5787
|
+
run_state['apptainer'] = {}
|
|
5788
|
+
utils.merge_dicts({'dict1': run_state['apptainer'],
|
|
5789
|
+
'dict2': c_meta['apptainer'],
|
|
5790
|
+
'append_lists': True,
|
|
5791
|
+
'append_unique': True})
|
|
5792
|
+
|
|
5771
5793
|
c_add_deps_info = c_meta.get('ad', {})
|
|
5772
5794
|
if not c_add_deps_info:
|
|
5773
5795
|
c_add_deps_info = c_meta.get('add_deps', {})
|
|
@@ -5975,6 +5997,13 @@ def update_state_from_meta(meta, env, state, const, const_state, run_state, i):
|
|
|
5975
5997
|
if remote_run_settings.get('deps', []):
|
|
5976
5998
|
update_deps(remote_run_settings['deps'], add_deps_info, False, env)
|
|
5977
5999
|
|
|
6000
|
+
new_apptainer_settings = meta.get('apptainer')
|
|
6001
|
+
if new_apptainer_settings:
|
|
6002
|
+
utils.merge_dicts({'dict1': run_state['apptainer'],
|
|
6003
|
+
'dict2': new_apptainer_settings,
|
|
6004
|
+
'append_lists': True,
|
|
6005
|
+
'append_unique': True})
|
|
6006
|
+
|
|
5978
6007
|
new_env_keys_from_meta = meta.get('new_env_keys', [])
|
|
5979
6008
|
if new_env_keys_from_meta:
|
|
5980
6009
|
run_state['new_env_keys'] += new_env_keys_from_meta
|
|
@@ -5,6 +5,7 @@ import subprocess
|
|
|
5
5
|
import copy
|
|
6
6
|
from utils import is_true, prune_input
|
|
7
7
|
from script.script_utils import *
|
|
8
|
+
from script.apptainer import get_apptainer_input
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
def slurm_run(self_module, i, slurm_action='run'):
|
|
@@ -113,6 +114,11 @@ def slurm_run(self_module, i, slurm_action='run'):
|
|
|
113
114
|
|
|
114
115
|
run_input = prune_result['new_input']
|
|
115
116
|
|
|
117
|
+
# Flag SLURM execution as an MLC env var (like docker's MLC_RUN_STATE_DOCKER)
|
|
118
|
+
# so scripts can enable deps via enable_if_env: MLC_RUN_STATE_SLURM.
|
|
119
|
+
if isinstance(run_input, dict):
|
|
120
|
+
run_input.setdefault('env', {})['MLC_RUN_STATE_SLURM'] = 'yes'
|
|
121
|
+
|
|
116
122
|
cur_dir = os.getcwd()
|
|
117
123
|
|
|
118
124
|
r = self_module._select_script(i)
|
|
@@ -150,6 +156,15 @@ def slurm_run(self_module, i, slurm_action='run'):
|
|
|
150
156
|
if quiet:
|
|
151
157
|
script_run_cmd += ' --quiet'
|
|
152
158
|
|
|
159
|
+
# With host repos baked into the image, MLC_REPOS points inside the
|
|
160
|
+
# container and caches would land in the small, discarded writable-tmpfs
|
|
161
|
+
# overlay; bind the isolated workspace's local dir so copy-back sees them.
|
|
162
|
+
if (slurm_isolated and slurm_action == 'apptainer'
|
|
163
|
+
and not get_apptainer_input(i, 'mlc_local_dir')
|
|
164
|
+
and is_true(get_apptainer_input(i, 'host_mlc_repos', False))):
|
|
165
|
+
script_run_cmd += \
|
|
166
|
+
' --apptainer_mlc_local_dir="$MLC_REPOS/local"'
|
|
167
|
+
|
|
153
168
|
# Build the commands to run inside srun
|
|
154
169
|
run_cmds = []
|
|
155
170
|
|
|
@@ -211,22 +226,28 @@ def slurm_run(self_module, i, slurm_action='run'):
|
|
|
211
226
|
if slurm_copy_back_mlc_cache:
|
|
212
227
|
local_cache = slurm_copy_back_mlc_cache_path or get_local_mlc_cache_path(
|
|
213
228
|
self_module)
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
.replace('\\', '\\\\')
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
)
|
|
226
|
-
|
|
229
|
+
|
|
230
|
+
def _dq(path):
|
|
231
|
+
return (str(path).replace('\\', '\\\\').replace('"', '\\"')
|
|
232
|
+
.replace('$', '\\$').replace('`', '\\`'))
|
|
233
|
+
|
|
234
|
+
safe_local_cache = _dq(local_cache)
|
|
235
|
+
# An explicit container local dir (bound by apptainer.py only with
|
|
236
|
+
# host repos) is where the container's cache is written.
|
|
237
|
+
container_local_dir = ''
|
|
238
|
+
if slurm_action == 'apptainer' and is_true(
|
|
239
|
+
get_apptainer_input(i, 'host_mlc_repos', False)):
|
|
240
|
+
container_local_dir = get_apptainer_input(i, 'mlc_local_dir', '')
|
|
241
|
+
if container_local_dir:
|
|
242
|
+
source_cache = f'"{_dq(container_local_dir)}/cache/"'
|
|
243
|
+
elif slurm_isolated:
|
|
244
|
+
source_cache = '"$MLC_REPOS/local/cache/"'
|
|
245
|
+
else:
|
|
246
|
+
source_cache = '~/MLC/repos/local/cache/'
|
|
247
|
+
if slurm_isolated or slurm_copy_back_mlc_cache_path or container_local_dir:
|
|
227
248
|
run_cmds.append(
|
|
228
249
|
f'mkdir -p "{safe_local_cache}" && '
|
|
229
|
-
f'rsync -a
|
|
250
|
+
f'rsync -a {source_cache} "{safe_local_cache}/"'
|
|
230
251
|
)
|
|
231
252
|
|
|
232
253
|
# Join all commands with && so failure stops execution
|
|
@@ -303,6 +324,23 @@ def regenerate_script_cmd(i):
|
|
|
303
324
|
i_run_cmd = i['run_cmd']
|
|
304
325
|
slurm_action = i.get('slurm_action', 'run')
|
|
305
326
|
|
|
327
|
+
# Provenance only; the node-side mlc computes its own.
|
|
328
|
+
i_run_cmd.pop('mlc_run_cmd', None)
|
|
329
|
+
|
|
330
|
+
# Drop unresolved default_env placeholders (<<<VAR>>>) so they are not
|
|
331
|
+
# forwarded as explicit inputs to the node.
|
|
332
|
+
env = i_run_cmd.get('env')
|
|
333
|
+
if isinstance(env, dict):
|
|
334
|
+
for key in list(env):
|
|
335
|
+
value = env[key]
|
|
336
|
+
if isinstance(value, str) and '<<<' in value:
|
|
337
|
+
del env[key]
|
|
338
|
+
elif isinstance(value, list):
|
|
339
|
+
env[key] = [v for v in value
|
|
340
|
+
if not (isinstance(v, str) and '<<<' in v)]
|
|
341
|
+
if not env[key]:
|
|
342
|
+
del env[key]
|
|
343
|
+
|
|
306
344
|
action_to_cmd = {
|
|
307
345
|
'run': 'mlcr',
|
|
308
346
|
'docker': 'mlcd',
|
|
@@ -1207,18 +1207,18 @@ def parse_expiration(user_input: str) -> float:
|
|
|
1207
1207
|
return time.time() + seconds
|
|
1208
1208
|
|
|
1209
1209
|
|
|
1210
|
-
def quote_if_needed(val: str, quote: str) -> str:
|
|
1210
|
+
def quote_if_needed(val: str, quote: str = '') -> str:
|
|
1211
1211
|
"""
|
|
1212
|
-
Return the value
|
|
1213
|
-
|
|
1212
|
+
Return the value quoted for a POSIX shell when it contains anything other
|
|
1213
|
+
than shell-safe characters. Single quotes are used, so $VAR, $(...),
|
|
1214
|
+
backticks and backslashes stay literal. quote='"' forces quoting.
|
|
1214
1215
|
"""
|
|
1216
|
+
import shlex
|
|
1215
1217
|
s = str(val)
|
|
1216
|
-
|
|
1217
|
-
if
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
return f'"{s}"'
|
|
1221
|
-
return s
|
|
1218
|
+
quoted = shlex.quote(s)
|
|
1219
|
+
if quoted == s and quote == '"':
|
|
1220
|
+
return f"'{s}'"
|
|
1221
|
+
return quoted
|
|
1222
1222
|
|
|
1223
1223
|
|
|
1224
1224
|
def has_function_in_file(py_file, func_name):
|
|
@@ -448,6 +448,7 @@ class Action:
|
|
|
448
448
|
force_remove = True
|
|
449
449
|
|
|
450
450
|
results = res['list']
|
|
451
|
+
removed_any = False
|
|
451
452
|
|
|
452
453
|
for result in results:
|
|
453
454
|
item_path = result.path
|
|
@@ -468,7 +469,13 @@ class Action:
|
|
|
468
469
|
logger.warning(
|
|
469
470
|
f"{item_path} was already removed by another process.")
|
|
470
471
|
|
|
471
|
-
|
|
472
|
+
# Batch the index write: de-index in memory, persist once after
|
|
473
|
+
# the loop instead of rewriting all index files per item.
|
|
474
|
+
self.get_index().rm(item_meta, target_name, item_path, save=False)
|
|
475
|
+
removed_any = True
|
|
476
|
+
|
|
477
|
+
if removed_any:
|
|
478
|
+
self.get_index().save()
|
|
472
479
|
|
|
473
480
|
return {
|
|
474
481
|
"return": 0,
|