mlcflow 1.3.6__tar.gz → 1.3.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {mlcflow-1.3.6 → mlcflow-1.3.9}/PKG-INFO +10 -2
  2. {mlcflow-1.3.6 → mlcflow-1.3.9}/README.md +9 -1
  3. mlcflow-1.3.9/VERSION +1 -0
  4. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/apptainer.py +117 -30
  5. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/docker_utils.py +6 -2
  6. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/meta_schema.py +18 -0
  7. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/module.py +35 -6
  8. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/slurm_run.py +52 -14
  9. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/utils.py +9 -9
  10. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/action.py +8 -1
  11. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/index.py +58 -3
  12. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/main.py +66 -43
  13. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/meta_schema.py +18 -0
  14. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/repo_action.py +684 -188
  15. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/script_action.py +368 -92
  16. mlcflow-1.3.9/mlc/typo_mixin.py +127 -0
  17. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/utils.py +56 -0
  18. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/PKG-INFO +10 -2
  19. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/SOURCES.txt +11 -0
  20. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/entry_points.txt +2 -0
  21. {mlcflow-1.3.6 → mlcflow-1.3.9}/pyproject.toml +2 -0
  22. mlcflow-1.3.9/tests/test_apptainer_docker_unification.py +286 -0
  23. mlcflow-1.3.9/tests/test_build_wheels_workflow.py +142 -0
  24. mlcflow-1.3.9/tests/test_cli_input_file.py +132 -0
  25. mlcflow-1.3.9/tests/test_container_cmd_quoting.py +189 -0
  26. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_convert_env_to_script.py +33 -0
  27. mlcflow-1.3.9/tests/test_help_coverage.py +109 -0
  28. mlcflow-1.3.9/tests/test_index_reconcile.py +113 -0
  29. mlcflow-1.3.9/tests/test_pull_repo_thread_safety.py +1036 -0
  30. mlcflow-1.3.9/tests/test_release_version.py +101 -0
  31. mlcflow-1.3.9/tests/test_run_target_dispatch.py +41 -0
  32. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_script_action_apptainer.py +1 -1
  33. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_slurm_run.py +81 -0
  34. mlcflow-1.3.9/tests/test_typo_mixin.py +294 -0
  35. mlcflow-1.3.6/VERSION +0 -1
  36. {mlcflow-1.3.6 → mlcflow-1.3.9}/LICENSE.md +0 -0
  37. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/COPYRIGHT.txt +0 -0
  38. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/__init__.py +0 -0
  39. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/README.md +0 -0
  40. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/__init__.py +0 -0
  41. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/cache_utils.py +0 -0
  42. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/compat.py +0 -0
  43. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/doc.py +0 -0
  44. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/docker.py +0 -0
  45. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/experiment.py +0 -0
  46. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/help.py +0 -0
  47. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/lint.py +0 -0
  48. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/meta.json +0 -0
  49. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/remote_run.py +0 -0
  50. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/script_utils.py +0 -0
  51. {mlcflow-1.3.6 → mlcflow-1.3.9}/automation/script/validate.py +0 -0
  52. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/__init__.py +0 -0
  53. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/__main__.py +0 -0
  54. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/action_factory.py +0 -0
  55. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/cache_action.py +0 -0
  56. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/cfg_action.py +0 -0
  57. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/error_codes.py +0 -0
  58. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/experiment_action.py +0 -0
  59. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/item.py +0 -0
  60. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/logger.py +0 -0
  61. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlc/repo.py +0 -0
  62. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/dependency_links.txt +0 -0
  63. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/requires.txt +0 -0
  64. {mlcflow-1.3.6 → mlcflow-1.3.9}/mlcflow.egg-info/top_level.txt +0 -0
  65. {mlcflow-1.3.6 → mlcflow-1.3.9}/setup.cfg +0 -0
  66. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_action_invalid_meta_entries.py +0 -0
  67. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_cache_mark_tmp.py +0 -0
  68. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_installer_venv_deactivation.py +0 -0
  69. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_mlcflow_unix_installer.py +0 -0
  70. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_repo_action_extra_git_args.py +0 -0
  71. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_thread_safety.py +0 -0
  72. {mlcflow-1.3.6 → mlcflow-1.3.9}/tests/test_unix_installer_venv.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mlcflow
3
- Version: 1.3.6
3
+ Version: 1.3.9
4
4
  Summary: An automation interface tailored for CPU/GPU benchmarking
5
5
  Author-email: MLCommons <systems@mlcommons.org>
6
6
  License:
@@ -214,13 +214,21 @@ Dynamic: license-file
214
214
 
215
215
  MLCFlow is a versatile CLI and Python interface developed by MLCommons in collaboration with a dedicated team of volunteers (see [Contributors](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md)). It serves as a streamlined replacement for the [CMind](https://github.com/mlcommons/ck/tree/master/cm) tool, designed to drive the automation workflows of MLPerf benchmarks more efficiently. You can use this tool for any of your workflow automation tasks.
216
216
 
217
- The concept behind CMind originated from **Grigori Fursin**, while the **MLPerf Automations** project was created by **Grigori Fursin** and **Arjun Suresh**, whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by **Arjun Suresh** and **Anandhu Sooraj**, with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the devlopers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
217
+ The concept behind CMind originated from [**Grigori Fursin**](https://github.com/gfursin), while the **MLPerf Automations** project was created by [**Grigori Fursin**](https://github.com/gfursin) and [**Arjun Suresh**](https://github.com/arjunsuresh), whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by [**Arjun Suresh**](https://github.com/arjunsuresh) and [**Anandhu Sooraj**](https://github.com/anandhu-eng), with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the developers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
218
218
 
219
219
  On February 9, 2025, MLCFlow released its first stable version, 1.0.0.
220
220
 
221
221
  ### Key Features
222
222
  Building upon the core idea of CMind—wrapping native scripts with Python wrappers and YAML metadata—MLCFlow focuses exclusively on key automation components: **Scripts**, along with its complementary modules: **Cache**, **Docker**, and **Experiments**. This targeted design simplifies both implementation and interface, enabling a more user-friendly experience.
223
223
 
224
+ - **Typo detection** — Mistyped actions or targets produce a "Did you mean …?" hint before the error message, so mistakes are quick to correct:
225
+ ```
226
+ $ mlc rune script
227
+ Did you mean one of: 'prune', 'run'?
228
+ mlc: error: argument command: invalid choice: 'rune' …
229
+ ```
230
+ See [docs/typo_detection.md](docs/typo_detection.md) for details.
231
+
224
232
  ---
225
233
 
226
234
  ### Status
@@ -12,13 +12,21 @@
12
12
 
13
13
  MLCFlow is a versatile CLI and Python interface developed by MLCommons in collaboration with a dedicated team of volunteers (see [Contributors](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md)). It serves as a streamlined replacement for the [CMind](https://github.com/mlcommons/ck/tree/master/cm) tool, designed to drive the automation workflows of MLPerf benchmarks more efficiently. You can use this tool for any of your workflow automation tasks.
14
14
 
15
- The concept behind CMind originated from **Grigori Fursin**, while the **MLPerf Automations** project was created by **Grigori Fursin** and **Arjun Suresh**, whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by **Arjun Suresh** and **Anandhu Sooraj**, with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the devlopers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
15
+ The concept behind CMind originated from [**Grigori Fursin**](https://github.com/gfursin), while the **MLPerf Automations** project was created by [**Grigori Fursin**](https://github.com/gfursin) and [**Arjun Suresh**](https://github.com/arjunsuresh), whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by [**Arjun Suresh**](https://github.com/arjunsuresh) and [**Anandhu Sooraj**](https://github.com/anandhu-eng), with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the developers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
16
16
 
17
17
  On February 9, 2025, MLCFlow released its first stable version, 1.0.0.
18
18
 
19
19
  ### Key Features
20
20
  Building upon the core idea of CMind—wrapping native scripts with Python wrappers and YAML metadata—MLCFlow focuses exclusively on key automation components: **Scripts**, along with its complementary modules: **Cache**, **Docker**, and **Experiments**. This targeted design simplifies both implementation and interface, enabling a more user-friendly experience.
21
21
 
22
+ - **Typo detection** — Mistyped actions or targets produce a "Did you mean …?" hint before the error message, so mistakes are quick to correct:
23
+ ```
24
+ $ mlc rune script
25
+ Did you mean one of: 'prune', 'run'?
26
+ mlc: error: argument command: invalid choice: 'rune' …
27
+ ```
28
+ See [docs/typo_detection.md](docs/typo_detection.md) for details.
29
+
22
30
  ---
23
31
 
24
32
  ### Status
mlcflow-1.3.9/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.3.9
@@ -1,4 +1,5 @@
1
1
  import os
2
+ import shlex
2
3
  from mlc import utils
3
4
  from utils import *
4
5
  import logging
@@ -8,9 +9,10 @@ import copy
8
9
 
9
10
  def apptainerfile(self_module, input_params):
10
11
 
11
- # Step 1: Prune and prepare input
12
+ # Step 1: Prune and prepare input (remove both apptainer_ and docker_
13
+ # prefixed keys)
12
14
  prune_result = prune_input(
13
- {'input': input_params, 'extra_keys_starts_with': ['apptainer_']})
15
+ {'input': input_params, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
14
16
  if prune_result['return'] > 0:
15
17
  return prune_result
16
18
 
@@ -56,13 +58,15 @@ def apptainerfile(self_module, input_params):
56
58
 
57
59
  run_state = self_module.run_state
58
60
 
59
- apptainer_settings = run_state.get('docker', {})
61
+ apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
62
+ utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
63
+ 'append_lists': True, 'append_unique': True})
60
64
  apptainer_settings_default_env = apptainer_settings.get('default_env', {})
61
65
  for key in apptainer_settings_default_env:
62
66
  env.setdefault(key, apptainer_settings_default_env[key])
63
67
 
64
68
  if not apptainer_settings.get('run', True) and not input_params.get(
65
- 'apptainer_run_override', False):
69
+ 'apptainer_run_override', input_params.get('docker_run_override', False)):
66
70
  logger.info("Apptainer 'run' is set to False in meta.yaml")
67
71
  return {'return': 0,
68
72
  'warning': 'Apptainer run is set to false in script meta'}
@@ -86,7 +90,9 @@ def apptainerfile(self_module, input_params):
86
90
  if update_state_result['return'] > 0:
87
91
  return update_state_result
88
92
 
89
- apptainer_settings = run_state.get('docker', {})
93
+ apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
94
+ utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
95
+ 'append_lists': True, 'append_unique': True})
90
96
 
91
97
  # Prune temporary environment variables
92
98
  run_command = copy.deepcopy(run_command_arc)
@@ -102,7 +108,7 @@ def apptainerfile(self_module, input_params):
102
108
  'tags': script_tags,
103
109
  'fake_run': True,
104
110
  'docker_settings': apptainer_settings,
105
- 'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', ''))
111
+ 'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', input_params.get('docker_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', '')))
106
112
  })
107
113
  if regenerate_result['return'] > 0:
108
114
  return regenerate_result
@@ -152,13 +158,23 @@ def apptainerfile(self_module, input_params):
152
158
  if apptainer_inputs.get('mlc_repo_path', '') != '':
153
159
  mlc_apptainer_input['mlc_repo_path'] = apptainer_inputs['mlc_repo_path']
154
160
 
161
+ if is_true(get_apptainer_input(input_params, 'host_mlc_repos', '')):
162
+ mlc_apptainer_input['host_mlc_repos'] = 'yes'
163
+
164
+ # Bind destinations must exist in the image rootfs, otherwise the
165
+ # build-time fake run fails when the binds are applied.
166
+ build_binds = get_apptainer_bind_mounts(
167
+ input_params, env, apptainer_settings, run_state)
168
+ if build_binds:
169
+ mlc_apptainer_input['bind'] = build_binds
170
+
155
171
  apptainer_v = False
156
172
  apptainer_s = False
157
173
  if is_true(input_params.get(
158
- 'apptainer_v', input_params.get('apptainer_verbose', False))):
174
+ 'apptainer_v', input_params.get('apptainer_verbose', input_params.get('docker_v', input_params.get('docker_verbose', False))))):
159
175
  apptainer_v = True
160
176
  if is_true(input_params.get(
161
- 'apptainer_s', input_params.get('apptainer_silent', False))):
177
+ 'apptainer_s', input_params.get('apptainer_silent', input_params.get('docker_s', input_params.get('docker_silent', False))))):
162
178
  apptainer_s = True
163
179
 
164
180
  if apptainer_s and apptainer_v:
@@ -213,11 +229,16 @@ def apptainer_run(self_module, i):
213
229
  if quiet:
214
230
  env['MLC_QUIET'] = 'yes'
215
231
 
216
- regenerate_def_file = not i.get('apptainer_noregenerate', False)
217
- rebuild_apptainer_image = i.get('apptainer_rebuild', False)
232
+ regenerate_def_file = not i.get(
233
+ 'apptainer_noregenerate', i.get(
234
+ 'docker_noregenerate', False))
235
+ rebuild_apptainer_image = i.get(
236
+ 'apptainer_rebuild', i.get(
237
+ 'docker_rebuild', False))
218
238
 
219
- # Prune unnecessary Apptainer-related input keys
220
- r = prune_input({'input': i, 'extra_keys_starts_with': ['apptainer_']})
239
+ # Prune unnecessary Apptainer- and Docker-related input keys
240
+ r = prune_input(
241
+ {'input': i, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
221
242
  f_run_cmd = r['new_input']
222
243
 
223
244
  # Save current directory and prepare to search for scripts
@@ -259,7 +280,7 @@ def apptainer_run(self_module, i):
259
280
  mounts = copy.deepcopy(
260
281
  i.get(
261
282
  'apptainer_mounts',
262
- []))
283
+ i.get('docker_mounts', [])))
263
284
  variations = meta.get('variations', {})
264
285
 
265
286
  if not hasattr(self_module, 'run_state'):
@@ -273,7 +294,9 @@ def apptainer_run(self_module, i):
273
294
 
274
295
  run_state = self_module.run_state
275
296
 
276
- apptainer_settings = run_state.get('docker', {})
297
+ apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
298
+ utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
299
+ 'append_lists': True, 'append_unique': True})
277
300
 
278
301
  apptainer_settings_default_env = apptainer_settings.get('default_env', {})
279
302
  for key in apptainer_settings_default_env:
@@ -297,7 +320,7 @@ def apptainer_run(self_module, i):
297
320
 
298
321
  # Skip scripts marked as non-runnable
299
322
  if not apptainer_settings.get('run', True) and not i.get(
300
- 'apptainer_run_override', False):
323
+ 'apptainer_run_override', i.get('docker_run_override', False)):
301
324
  logger.info("apptainer.run set to False in meta.yaml")
302
325
  return {'return': 0,
303
326
  'warning': 'Apptainer run is set to false in script meta'}
@@ -337,12 +360,13 @@ def apptainer_run(self_module, i):
337
360
  mounts, env, apptainer_settings, run_state)
338
361
  container_env_string = bind_mounts.get('container_env_string', '')
339
362
 
340
- # Forward input-mapped env vars to the container's mlcr command
363
+ # Forward input-mapped env vars to the container's mlcr command.
364
+ # Unresolved default_env placeholders (<<<VAR>>>) are dropped.
341
365
  input_mapping = meta.get('input_mapping', {})
342
366
  for input_key, env_key in input_mapping.items():
343
367
  val = env.get(env_key, '')
344
- if val and isinstance(val, str):
345
- container_env_string += f' --env.{env_key}={val}'
368
+ if val and isinstance(val, str) and '<<<' not in val:
369
+ container_env_string += f' --env.{env_key}={shlex.quote(val)}'
346
370
 
347
371
  # Forward framework-level env keys (MLC_OUTDIRNAME, MLC_INPUT, MLC_OUTPUT, etc.)
348
372
  # These are skipped in regenerate_script_cmd (docker_utils.py) and must be
@@ -352,14 +376,14 @@ def apptainer_run(self_module, i):
352
376
  for input_key in framework_input_mapping:
353
377
  env_key = 'MLC_' + input_key.upper().replace('-', '_').replace(' ', '_')
354
378
  val = env.get(env_key, '')
355
- if val and isinstance(val, str):
356
- container_env_string += f' --env.{env_key}={val}'
379
+ if val and isinstance(val, str) and '<<<' not in val:
380
+ container_env_string += f' --env.{env_key}={shlex.quote(val)}'
357
381
 
358
382
  # Forward proxy env vars (matching Docker behavior)
359
383
  for proxy_key in self_module.host_env_keys:
360
384
  proxy_value = os.environ.get(proxy_key)
361
385
  if proxy_value:
362
- container_env_string += f' --env.{proxy_key}={proxy_value}'
386
+ container_env_string += f' --env.{proxy_key}={shlex.quote(proxy_value)}'
363
387
 
364
388
  # Generate the run command
365
389
  r = regenerate_script_cmd({'script_uid': script_uid,
@@ -379,6 +403,20 @@ def apptainer_run(self_module, i):
379
403
  'quiet': True, 'real_run': True,
380
404
  'add_deps_recursive': {'build-apptainer-image': {'def_file': def_file_path}}
381
405
  }
406
+
407
+ # Detached mode (mirrors --docker_dt): run as a background instance.
408
+ if is_true(get_apptainer_input(
409
+ i, 'dt', get_apptainer_input(i, 'detached', ''))):
410
+ mlc_apptainer_input['detached'] = 'yes'
411
+ # Drop into an interactive shell after the run (mirrors --docker_it).
412
+ elif is_true(get_apptainer_input(
413
+ i, 'it', get_apptainer_input(i, 'interactive', ''))):
414
+ mlc_apptainer_input['interactive'] = 'yes'
415
+
416
+ # Tell the run step host repos were baked in, so it points MLC_REPOS at
417
+ # /opt/mlc_host_repos (matters under fakeroot, which otherwise overrides it).
418
+ if is_true(get_apptainer_input(i, 'host_mlc_repos', '')):
419
+ mlc_apptainer_input['host_mlc_repos'] = 'yes'
382
420
  # Forward build options to build-apptainer-image
383
421
  build_image_extras = {}
384
422
  if apptainer_inputs.get('ignore_fakeroot_cmd'):
@@ -390,6 +428,10 @@ def apptainer_run(self_module, i):
390
428
  build_image_extras)
391
429
  # Wire resolved docker.mounts to apptainer bind mounts
392
430
  resolved_mounts = bind_mounts.get('mounts', [])
431
+ mlc_local_bind = get_mlc_local_bind(i)
432
+ if mlc_local_bind:
433
+ os.makedirs(mlc_local_bind.split(':', 1)[0], exist_ok=True)
434
+ resolved_mounts.append(mlc_local_bind)
393
435
  if resolved_mounts:
394
436
  mlc_apptainer_input['bind'] = resolved_mounts
395
437
 
@@ -443,14 +485,15 @@ def prepare_apptainer_inputs(input_params, apptainer_settings,
443
485
  "network", "security_opt"
444
486
  ]
445
487
 
446
- # Collect inputs
447
- apptainer_inputs = {
448
- key: input_params.get(
449
- f"apptainer_{key}", apptainer_settings.get(
450
- key, get_apptainer_default(key)))
451
- for key in keys
452
- if (value := input_params.get(f"apptainer_{key}", apptainer_settings.get(key, get_apptainer_default(key)))) is not None
453
- }
488
+ # Collect inputs: apptainer_X > docker_X > meta settings > default
489
+ apptainer_inputs = {}
490
+ for key in keys:
491
+ value = input_params.get(
492
+ f"apptainer_{key}",
493
+ input_params.get(
494
+ f"docker_{key}", apptainer_settings.get(key, get_apptainer_default(key))))
495
+ if value is not None:
496
+ apptainer_inputs[key] = value
454
497
 
455
498
  # Convert boolean values to 'yes'/'no' strings for MLC input mapping
456
499
  for key in list(apptainer_inputs.keys()):
@@ -544,10 +587,54 @@ def process_apptainer_mounts(mounts, env, apptainer_settings, run_state):
544
587
  'container_env_string': container_env_string}
545
588
 
546
589
 
590
+ def get_mlc_local_bind(input_params):
591
+ """Bind for --apptainer_mlc_local_dir: a writable host directory over the
592
+ container's MLC local dir (cache), used with --apptainer_host_mlc_repos so
593
+ caches do not land in the small, discarded --writable-tmpfs overlay."""
594
+ local_dir = get_apptainer_input(input_params, 'mlc_local_dir', '')
595
+ if not local_dir or not is_true(
596
+ get_apptainer_input(input_params, 'host_mlc_repos', '')):
597
+ return ''
598
+ return f"{os.path.abspath(local_dir)}:/opt/mlc_host_repos/local"
599
+
600
+
601
+ def get_apptainer_bind_mounts(input_params, env, apptainer_settings,
602
+ run_state):
603
+ """All bind mounts (user binds, resolved docker.mounts, MLC local dir)
604
+ known before the image is built."""
605
+ binds = get_apptainer_input(input_params, 'bind',
606
+ apptainer_settings.get('bind')) or []
607
+ if isinstance(binds, str):
608
+ binds = [binds]
609
+ binds = list(binds)
610
+ mounts = copy.deepcopy(get_apptainer_input(input_params, 'mounts', []))
611
+ binds += process_apptainer_mounts(
612
+ mounts, env, apptainer_settings, run_state)['mounts']
613
+ mlc_local_bind = get_mlc_local_bind(input_params)
614
+ if mlc_local_bind:
615
+ binds.append(mlc_local_bind)
616
+ return binds
617
+
618
+
619
+ def get_apptainer_input(input_params, key, default=None):
620
+ """Return apptainer_<key> if provided, else fall back to docker_<key>.
621
+
622
+ Lets docker_* options passed to `mlca` take effect unless the matching
623
+ apptainer_* option overrides them.
624
+ """
625
+ apptainer_key = f"apptainer_{key}"
626
+ if input_params.get(apptainer_key) is not None:
627
+ return input_params[apptainer_key]
628
+ docker_key = f"docker_{key}"
629
+ if input_params.get(docker_key) is not None:
630
+ return input_params[docker_key]
631
+ return default
632
+
633
+
547
634
  def get_apptainer_default(key):
548
635
  defaults = {
549
636
  "mlc_repo": "mlcommons@mlperf-automations",
550
- "mlc_repo_branch": "dev",
637
+ "mlc_repo_branch": "main",
551
638
  "os": "ubuntu",
552
639
  "os_version": "",
553
640
  "fake_run_deps": False,
@@ -328,7 +328,7 @@ def regenerate_script_cmd(i):
328
328
  if isinstance(val, str) and (
329
329
  os.path.join("local", "cache", "") in val or
330
330
  os.path.join("MLC", "repos", "") in val or
331
- "<<<" in value
331
+ "<<<" in val
332
332
  )
333
333
  ]
334
334
 
@@ -373,6 +373,10 @@ def regenerate_script_cmd(i):
373
373
  for key in keys:
374
374
  if key in ["input", "output", "outdirname"]:
375
375
  continue # We have the corresponding env keys in container env string
376
+ # Provenance only; nesting an already shell-joined command breaks
377
+ # quoting in the container command and tmp-run.sh.
378
+ if key == "mlc_run_cmd" and not prefix:
379
+ continue
376
380
  # Construct the full key with the prefix.
377
381
  full_key = f"{prefix}.{key}" if prefix else key
378
382
 
@@ -422,7 +426,7 @@ def regenerate_script_cmd(i):
422
426
  def get_docker_default(key):
423
427
  defaults = {
424
428
  "mlc_repo": "mlcommons@mlperf-automations",
425
- "mlc_repo_branch": "dev",
429
+ "mlc_repo_branch": "main",
426
430
  "os": "ubuntu",
427
431
  "os_version": "",
428
432
  "fake_run_deps": False,
@@ -86,6 +86,7 @@ TOP_LEVEL_SCHEMA = {
86
86
 
87
87
  # Docker
88
88
  "docker": DICT, # dict - see DOCKER_SCHEMA
89
+ "apptainer": DICT, # dict - apptainer overrides; merges with docker settings
89
90
 
90
91
  # Output / debugging
91
92
  "print_env_at_the_end": DICT, # dict[str, list[str]]
@@ -164,6 +165,7 @@ VARIATION_ENTRY_SCHEMA = {
164
165
  "state": DICT,
165
166
  "const": DICT,
166
167
  "docker": DICT,
168
+ "apptainer": DICT,
167
169
  "alias": STR,
168
170
  "default_version": STR_OR_FLOAT,
169
171
  "required_disk_space": INT,
@@ -248,6 +250,7 @@ UPDATE_META_IF_ENV_SCHEMA = {
248
250
  "default_env": DICT,
249
251
  "default_variations": DICT,
250
252
  "docker": DICT,
253
+ "apptainer": DICT,
251
254
  "adr": DICT,
252
255
  "ad": DICT,
253
256
  }
@@ -403,6 +406,21 @@ def validate_meta(data, file_path=""):
403
406
  errors.append(
404
407
  f"{prefix}docker.{dk} has type '{actual}', expected {allowed}")
405
408
 
409
+ # Validate apptainer section (same schema as docker; apptainer overrides
410
+ # docker)
411
+ apptainer = data.get("apptainer")
412
+ if isinstance(apptainer, dict):
413
+ for ak, av in apptainer.items():
414
+ if ak not in DOCKER_SCHEMA:
415
+ warnings.append(
416
+ f"{prefix}apptainer: unknown key '{ak}'")
417
+ continue
418
+ actual = type(av).__name__
419
+ allowed = DOCKER_SCHEMA[ak]
420
+ if actual not in allowed:
421
+ errors.append(
422
+ f"{prefix}apptainer.{ak} has type '{actual}', expected {allowed}")
423
+
406
424
  # Validate tests section
407
425
  tests = data.get("tests")
408
426
  if isinstance(tests, dict):
@@ -151,7 +151,7 @@ class ScriptAutomation(Automation):
151
151
  for f in ['fake_deps', 'cache']:
152
152
  run_state.setdefault(f, False)
153
153
 
154
- for d in ['input_mapping', 'docker', 'remote_run']:
154
+ for d in ['input_mapping', 'docker', 'remote_run', 'apptainer']:
155
155
  run_state.setdefault(d, {})
156
156
 
157
157
  for l in ['deps', 'post_deps', 'prehook_deps', 'posthook_deps',
@@ -4558,6 +4558,18 @@ pip install mlcflow
4558
4558
  i['remote_action'] = 'slurm-experiment'
4559
4559
  return remote_run(self, i)
4560
4560
 
4561
+ ############################################################
4562
+ def remote_slurm_docker(self, i):
4563
+ from script.remote_run import remote_run
4564
+ i['remote_action'] = 'slurm-docker'
4565
+ return remote_run(self, i)
4566
+
4567
+ ############################################################
4568
+ def remote_slurm_apptainer(self, i):
4569
+ from script.remote_run import remote_run
4570
+ i['remote_action'] = 'slurm-apptainer'
4571
+ return remote_run(self, i)
4572
+
4561
4573
  ##########################################################################
4562
4574
 
4563
4575
  def _available_variations(self, i):
@@ -5428,12 +5440,14 @@ def convert_env_to_script(env, os_info, start_script=None):
5428
5440
  env_separator = os_info.get('env_separator', ':')
5429
5441
  env_value = env_separator.join(str(v) for v in env_value)
5430
5442
 
5431
- env_quote = os_info.get('env_quote', '"')
5432
- # Replace placeholders in the platform-specific environment command
5433
- # and escapes any quote in the env value
5443
+ env_value = str(env_value)
5444
+ # cmd.exe `set` takes the value literally (backslash is not an escape
5445
+ # there), so only the POSIX double-quoted export needs escaping.
5446
+ if not is_windows:
5447
+ env_quote = os_info.get('env_quote', '"')
5448
+ env_value = env_value.replace(env_quote, f"""\\{env_quote}""")
5434
5449
  env_command = os_info['set_env'].replace(
5435
- '${key}', key).replace(
5436
- '${value}', str(env_value).replace(env_quote, f"""\\{env_quote}"""))
5450
+ '${key}', key).replace('${value}', env_value)
5437
5451
  script.append(env_command)
5438
5452
 
5439
5453
  return script
@@ -5768,6 +5782,14 @@ def _apply_conditional_meta_updates(update_meta_if_env, default_env, env, const,
5768
5782
  'append_lists': True,
5769
5783
  'append_unique': True})
5770
5784
 
5785
+ if c_meta.get('apptainer', {}):
5786
+ if not run_state.get('apptainer', {}):
5787
+ run_state['apptainer'] = {}
5788
+ utils.merge_dicts({'dict1': run_state['apptainer'],
5789
+ 'dict2': c_meta['apptainer'],
5790
+ 'append_lists': True,
5791
+ 'append_unique': True})
5792
+
5771
5793
  c_add_deps_info = c_meta.get('ad', {})
5772
5794
  if not c_add_deps_info:
5773
5795
  c_add_deps_info = c_meta.get('add_deps', {})
@@ -5975,6 +5997,13 @@ def update_state_from_meta(meta, env, state, const, const_state, run_state, i):
5975
5997
  if remote_run_settings.get('deps', []):
5976
5998
  update_deps(remote_run_settings['deps'], add_deps_info, False, env)
5977
5999
 
6000
+ new_apptainer_settings = meta.get('apptainer')
6001
+ if new_apptainer_settings:
6002
+ utils.merge_dicts({'dict1': run_state['apptainer'],
6003
+ 'dict2': new_apptainer_settings,
6004
+ 'append_lists': True,
6005
+ 'append_unique': True})
6006
+
5978
6007
  new_env_keys_from_meta = meta.get('new_env_keys', [])
5979
6008
  if new_env_keys_from_meta:
5980
6009
  run_state['new_env_keys'] += new_env_keys_from_meta
@@ -5,6 +5,7 @@ import subprocess
5
5
  import copy
6
6
  from utils import is_true, prune_input
7
7
  from script.script_utils import *
8
+ from script.apptainer import get_apptainer_input
8
9
 
9
10
 
10
11
  def slurm_run(self_module, i, slurm_action='run'):
@@ -113,6 +114,11 @@ def slurm_run(self_module, i, slurm_action='run'):
113
114
 
114
115
  run_input = prune_result['new_input']
115
116
 
117
+ # Flag SLURM execution as an MLC env var (like docker's MLC_RUN_STATE_DOCKER)
118
+ # so scripts can enable deps via enable_if_env: MLC_RUN_STATE_SLURM.
119
+ if isinstance(run_input, dict):
120
+ run_input.setdefault('env', {})['MLC_RUN_STATE_SLURM'] = 'yes'
121
+
116
122
  cur_dir = os.getcwd()
117
123
 
118
124
  r = self_module._select_script(i)
@@ -150,6 +156,15 @@ def slurm_run(self_module, i, slurm_action='run'):
150
156
  if quiet:
151
157
  script_run_cmd += ' --quiet'
152
158
 
159
+ # With host repos baked into the image, MLC_REPOS points inside the
160
+ # container and caches would land in the small, discarded writable-tmpfs
161
+ # overlay; bind the isolated workspace's local dir so copy-back sees them.
162
+ if (slurm_isolated and slurm_action == 'apptainer'
163
+ and not get_apptainer_input(i, 'mlc_local_dir')
164
+ and is_true(get_apptainer_input(i, 'host_mlc_repos', False))):
165
+ script_run_cmd += \
166
+ ' --apptainer_mlc_local_dir="$MLC_REPOS/local"'
167
+
153
168
  # Build the commands to run inside srun
154
169
  run_cmds = []
155
170
 
@@ -211,22 +226,28 @@ def slurm_run(self_module, i, slurm_action='run'):
211
226
  if slurm_copy_back_mlc_cache:
212
227
  local_cache = slurm_copy_back_mlc_cache_path or get_local_mlc_cache_path(
213
228
  self_module)
214
- safe_local_cache = (
215
- str(local_cache)
216
- .replace('\\', '\\\\')
217
- .replace('"', '\\"')
218
- .replace('$', '\\$')
219
- .replace('`', '\\`')
220
- )
221
- if slurm_isolated:
222
- run_cmds.append(
223
- f'mkdir -p "{safe_local_cache}" && '
224
- f'rsync -a "$MLC_REPOS/local/cache/" "{safe_local_cache}/"'
225
- )
226
- elif slurm_copy_back_mlc_cache_path:
229
+
230
+ def _dq(path):
231
+ return (str(path).replace('\\', '\\\\').replace('"', '\\"')
232
+ .replace('$', '\\$').replace('`', '\\`'))
233
+
234
+ safe_local_cache = _dq(local_cache)
235
+ # An explicit container local dir (bound by apptainer.py only with
236
+ # host repos) is where the container's cache is written.
237
+ container_local_dir = ''
238
+ if slurm_action == 'apptainer' and is_true(
239
+ get_apptainer_input(i, 'host_mlc_repos', False)):
240
+ container_local_dir = get_apptainer_input(i, 'mlc_local_dir', '')
241
+ if container_local_dir:
242
+ source_cache = f'"{_dq(container_local_dir)}/cache/"'
243
+ elif slurm_isolated:
244
+ source_cache = '"$MLC_REPOS/local/cache/"'
245
+ else:
246
+ source_cache = '~/MLC/repos/local/cache/'
247
+ if slurm_isolated or slurm_copy_back_mlc_cache_path or container_local_dir:
227
248
  run_cmds.append(
228
249
  f'mkdir -p "{safe_local_cache}" && '
229
- f'rsync -a ~/MLC/repos/local/cache/ "{safe_local_cache}/"'
250
+ f'rsync -a {source_cache} "{safe_local_cache}/"'
230
251
  )
231
252
 
232
253
  # Join all commands with && so failure stops execution
@@ -303,6 +324,23 @@ def regenerate_script_cmd(i):
303
324
  i_run_cmd = i['run_cmd']
304
325
  slurm_action = i.get('slurm_action', 'run')
305
326
 
327
+ # Provenance only; the node-side mlc computes its own.
328
+ i_run_cmd.pop('mlc_run_cmd', None)
329
+
330
+ # Drop unresolved default_env placeholders (<<<VAR>>>) so they are not
331
+ # forwarded as explicit inputs to the node.
332
+ env = i_run_cmd.get('env')
333
+ if isinstance(env, dict):
334
+ for key in list(env):
335
+ value = env[key]
336
+ if isinstance(value, str) and '<<<' in value:
337
+ del env[key]
338
+ elif isinstance(value, list):
339
+ env[key] = [v for v in value
340
+ if not (isinstance(v, str) and '<<<' in v)]
341
+ if not env[key]:
342
+ del env[key]
343
+
306
344
  action_to_cmd = {
307
345
  'run': 'mlcr',
308
346
  'docker': 'mlcd',
@@ -1207,18 +1207,18 @@ def parse_expiration(user_input: str) -> float:
1207
1207
  return time.time() + seconds
1208
1208
 
1209
1209
 
1210
- def quote_if_needed(val: str, quote: str) -> str:
1210
+ def quote_if_needed(val: str, quote: str = '') -> str:
1211
1211
  """
1212
- Return the value, quoted with escaped quotes if it contains spaces
1213
- or quotes.
1212
+ Return the value quoted for a POSIX shell when it contains anything other
1213
+ than shell-safe characters. Single quotes are used, so $VAR, $(...),
1214
+ backticks and backslashes stay literal. quote='"' forces quoting.
1214
1215
  """
1216
+ import shlex
1215
1217
  s = str(val)
1216
-
1217
- if " " in s or '"' in s or quote == '"':
1218
- # Escape all existing double quotes
1219
- s = s.replace('"', r'\"')
1220
- return f'"{s}"'
1221
- return s
1218
+ quoted = shlex.quote(s)
1219
+ if quoted == s and quote == '"':
1220
+ return f"'{s}'"
1221
+ return quoted
1222
1222
 
1223
1223
 
1224
1224
  def has_function_in_file(py_file, func_name):
@@ -448,6 +448,7 @@ class Action:
448
448
  force_remove = True
449
449
 
450
450
  results = res['list']
451
+ removed_any = False
451
452
 
452
453
  for result in results:
453
454
  item_path = result.path
@@ -468,7 +469,13 @@ class Action:
468
469
  logger.warning(
469
470
  f"{item_path} was already removed by another process.")
470
471
 
471
- self.get_index().rm(item_meta, target_name, item_path)
472
+ # Batch the index write: de-index in memory, persist once after
473
+ # the loop instead of rewriting all index files per item.
474
+ self.get_index().rm(item_meta, target_name, item_path, save=False)
475
+ removed_any = True
476
+
477
+ if removed_any:
478
+ self.get_index().save()
472
479
 
473
480
  return {
474
481
  "return": 0,