mlcflow 1.3.6__tar.gz → 1.3.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {mlcflow-1.3.6 → mlcflow-1.3.8}/PKG-INFO +10 -2
  2. {mlcflow-1.3.6 → mlcflow-1.3.8}/README.md +9 -1
  3. mlcflow-1.3.8/VERSION +1 -0
  4. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/apptainer.py +68 -23
  5. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/meta_schema.py +18 -0
  6. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/module.py +16 -1
  7. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/action.py +8 -1
  8. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/index.py +40 -1
  9. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/main.py +22 -14
  10. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/meta_schema.py +18 -0
  11. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/repo_action.py +680 -187
  12. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/script_action.py +220 -20
  13. mlcflow-1.3.8/mlc/typo_mixin.py +127 -0
  14. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/utils.py +56 -0
  15. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/PKG-INFO +10 -2
  16. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/SOURCES.txt +6 -0
  17. mlcflow-1.3.8/tests/test_apptainer_docker_unification.py +286 -0
  18. mlcflow-1.3.8/tests/test_build_wheels_workflow.py +142 -0
  19. mlcflow-1.3.8/tests/test_pull_repo_thread_safety.py +1036 -0
  20. mlcflow-1.3.8/tests/test_release_version.py +101 -0
  21. mlcflow-1.3.8/tests/test_typo_mixin.py +294 -0
  22. mlcflow-1.3.6/VERSION +0 -1
  23. {mlcflow-1.3.6 → mlcflow-1.3.8}/LICENSE.md +0 -0
  24. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/COPYRIGHT.txt +0 -0
  25. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/__init__.py +0 -0
  26. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/README.md +0 -0
  27. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/__init__.py +0 -0
  28. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/cache_utils.py +0 -0
  29. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/compat.py +0 -0
  30. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/doc.py +0 -0
  31. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/docker.py +0 -0
  32. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/docker_utils.py +0 -0
  33. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/experiment.py +0 -0
  34. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/help.py +0 -0
  35. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/lint.py +0 -0
  36. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/meta.json +0 -0
  37. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/remote_run.py +0 -0
  38. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/script_utils.py +0 -0
  39. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/slurm_run.py +0 -0
  40. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/script/validate.py +0 -0
  41. {mlcflow-1.3.6 → mlcflow-1.3.8}/automation/utils.py +0 -0
  42. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/__init__.py +0 -0
  43. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/__main__.py +0 -0
  44. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/action_factory.py +0 -0
  45. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/cache_action.py +0 -0
  46. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/cfg_action.py +0 -0
  47. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/error_codes.py +0 -0
  48. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/experiment_action.py +0 -0
  49. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/item.py +0 -0
  50. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/logger.py +0 -0
  51. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlc/repo.py +0 -0
  52. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/dependency_links.txt +0 -0
  53. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/entry_points.txt +0 -0
  54. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/requires.txt +0 -0
  55. {mlcflow-1.3.6 → mlcflow-1.3.8}/mlcflow.egg-info/top_level.txt +0 -0
  56. {mlcflow-1.3.6 → mlcflow-1.3.8}/pyproject.toml +0 -0
  57. {mlcflow-1.3.6 → mlcflow-1.3.8}/setup.cfg +0 -0
  58. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_action_invalid_meta_entries.py +0 -0
  59. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_cache_mark_tmp.py +0 -0
  60. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_convert_env_to_script.py +0 -0
  61. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_installer_venv_deactivation.py +0 -0
  62. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_mlcflow_unix_installer.py +0 -0
  63. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_repo_action_extra_git_args.py +0 -0
  64. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_script_action_apptainer.py +0 -0
  65. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_slurm_run.py +0 -0
  66. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_thread_safety.py +0 -0
  67. {mlcflow-1.3.6 → mlcflow-1.3.8}/tests/test_unix_installer_venv.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mlcflow
3
- Version: 1.3.6
3
+ Version: 1.3.8
4
4
  Summary: An automation interface tailored for CPU/GPU benchmarking
5
5
  Author-email: MLCommons <systems@mlcommons.org>
6
6
  License:
@@ -214,13 +214,21 @@ Dynamic: license-file
214
214
 
215
215
  MLCFlow is a versatile CLI and Python interface developed by MLCommons in collaboration with a dedicated team of volunteers (see [Contributors](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md)). It serves as a streamlined replacement for the [CMind](https://github.com/mlcommons/ck/tree/master/cm) tool, designed to drive the automation workflows of MLPerf benchmarks more efficiently. You can use this tool for any of your workflow automation tasks.
216
216
 
217
- The concept behind CMind originated from **Grigori Fursin**, while the **MLPerf Automations** project was created by **Grigori Fursin** and **Arjun Suresh**, whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by **Arjun Suresh** and **Anandhu Sooraj**, with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the devlopers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
217
+ The concept behind CMind originated from [**Grigori Fursin**](https://github.com/gfursin), while the **MLPerf Automations** project was created by [**Grigori Fursin**](https://github.com/gfursin) and [**Arjun Suresh**](https://github.com/arjunsuresh), whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by [**Arjun Suresh**](https://github.com/arjunsuresh) and [**Anandhu Sooraj**](https://github.com/anandhu-eng), with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the developers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
218
218
 
219
219
  On February 9, 2025, MLCFlow released its first stable version, 1.0.0.
220
220
 
221
221
  ### Key Features
222
222
  Building upon the core idea of CMind—wrapping native scripts with Python wrappers and YAML metadata—MLCFlow focuses exclusively on key automation components: **Scripts**, along with its complementary modules: **Cache**, **Docker**, and **Experiments**. This targeted design simplifies both implementation and interface, enabling a more user-friendly experience.
223
223
 
224
+ - **Typo detection** — Mistyped actions or targets produce a "Did you mean …?" hint before the error message, so mistakes are quick to correct:
225
+ ```
226
+ $ mlc rune script
227
+ Did you mean one of: 'prune', 'run'?
228
+ mlc: error: argument command: invalid choice: 'rune' …
229
+ ```
230
+ See [docs/typo_detection.md](docs/typo_detection.md) for details.
231
+
224
232
  ---
225
233
 
226
234
  ### Status
@@ -12,13 +12,21 @@
12
12
 
13
13
  MLCFlow is a versatile CLI and Python interface developed by MLCommons in collaboration with a dedicated team of volunteers (see [Contributors](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md)). It serves as a streamlined replacement for the [CMind](https://github.com/mlcommons/ck/tree/master/cm) tool, designed to drive the automation workflows of MLPerf benchmarks more efficiently. You can use this tool for any of your workflow automation tasks.
14
14
 
15
- The concept behind CMind originated from **Grigori Fursin**, while the **MLPerf Automations** project was created by **Grigori Fursin** and **Arjun Suresh**, whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by **Arjun Suresh** and **Anandhu Sooraj**, with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the devlopers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
15
+ The concept behind CMind originated from [**Grigori Fursin**](https://github.com/gfursin), while the **MLPerf Automations** project was created by [**Grigori Fursin**](https://github.com/gfursin) and [**Arjun Suresh**](https://github.com/arjunsuresh), whose collective contributions laid the foundation for modernizing MLPerf benchmarking tools. MLCFlow is a complete replacement for the CMind package with an entirely new implementation led by [**Arjun Suresh**](https://github.com/arjunsuresh) and [**Anandhu Sooraj**](https://github.com/anandhu-eng), with support from the [MLCFlow community](https://github.com/mlcommons/mlcflow/blob/main/CONTRIBUTORS.md). The script automation used in MLCFlow is inherited from the CM project but is now modified to support MLC and provides a simpler interface to the developers. Please see [the documentation](https://docs.mlcommons.org/mlcflow/) for more details.
16
16
 
17
17
  On February 9, 2025, MLCFlow released its first stable version, 1.0.0.
18
18
 
19
19
  ### Key Features
20
20
  Building upon the core idea of CMind—wrapping native scripts with Python wrappers and YAML metadata—MLCFlow focuses exclusively on key automation components: **Scripts**, along with its complementary modules: **Cache**, **Docker**, and **Experiments**. This targeted design simplifies both implementation and interface, enabling a more user-friendly experience.
21
21
 
22
+ - **Typo detection** — Mistyped actions or targets produce a "Did you mean …?" hint before the error message, so mistakes are quick to correct:
23
+ ```
24
+ $ mlc rune script
25
+ Did you mean one of: 'prune', 'run'?
26
+ mlc: error: argument command: invalid choice: 'rune' …
27
+ ```
28
+ See [docs/typo_detection.md](docs/typo_detection.md) for details.
29
+
22
30
  ---
23
31
 
24
32
  ### Status
mlcflow-1.3.8/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.3.8
@@ -8,9 +8,10 @@ import copy
8
8
 
9
9
  def apptainerfile(self_module, input_params):
10
10
 
11
- # Step 1: Prune and prepare input
11
+ # Step 1: Prune and prepare input (remove both apptainer_ and docker_
12
+ # prefixed keys)
12
13
  prune_result = prune_input(
13
- {'input': input_params, 'extra_keys_starts_with': ['apptainer_']})
14
+ {'input': input_params, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
14
15
  if prune_result['return'] > 0:
15
16
  return prune_result
16
17
 
@@ -56,13 +57,15 @@ def apptainerfile(self_module, input_params):
56
57
 
57
58
  run_state = self_module.run_state
58
59
 
59
- apptainer_settings = run_state.get('docker', {})
60
+ apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
61
+ utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
62
+ 'append_lists': True, 'append_unique': True})
60
63
  apptainer_settings_default_env = apptainer_settings.get('default_env', {})
61
64
  for key in apptainer_settings_default_env:
62
65
  env.setdefault(key, apptainer_settings_default_env[key])
63
66
 
64
67
  if not apptainer_settings.get('run', True) and not input_params.get(
65
- 'apptainer_run_override', False):
68
+ 'apptainer_run_override', input_params.get('docker_run_override', False)):
66
69
  logger.info("Apptainer 'run' is set to False in meta.yaml")
67
70
  return {'return': 0,
68
71
  'warning': 'Apptainer run is set to false in script meta'}
@@ -86,7 +89,9 @@ def apptainerfile(self_module, input_params):
86
89
  if update_state_result['return'] > 0:
87
90
  return update_state_result
88
91
 
89
- apptainer_settings = run_state.get('docker', {})
92
+ apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
93
+ utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
94
+ 'append_lists': True, 'append_unique': True})
90
95
 
91
96
  # Prune temporary environment variables
92
97
  run_command = copy.deepcopy(run_command_arc)
@@ -102,7 +107,7 @@ def apptainerfile(self_module, input_params):
102
107
  'tags': script_tags,
103
108
  'fake_run': True,
104
109
  'docker_settings': apptainer_settings,
105
- 'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', ''))
110
+ 'docker_run_cmd_prefix': input_params.get('apptainer_run_cmd_prefix', input_params.get('docker_run_cmd_prefix', apptainer_settings.get('run_cmd_prefix', '')))
106
111
  })
107
112
  if regenerate_result['return'] > 0:
108
113
  return regenerate_result
@@ -152,13 +157,16 @@ def apptainerfile(self_module, input_params):
152
157
  if apptainer_inputs.get('mlc_repo_path', '') != '':
153
158
  mlc_apptainer_input['mlc_repo_path'] = apptainer_inputs['mlc_repo_path']
154
159
 
160
+ if is_true(get_apptainer_input(input_params, 'host_mlc_repos', '')):
161
+ mlc_apptainer_input['host_mlc_repos'] = 'yes'
162
+
155
163
  apptainer_v = False
156
164
  apptainer_s = False
157
165
  if is_true(input_params.get(
158
- 'apptainer_v', input_params.get('apptainer_verbose', False))):
166
+ 'apptainer_v', input_params.get('apptainer_verbose', input_params.get('docker_v', input_params.get('docker_verbose', False))))):
159
167
  apptainer_v = True
160
168
  if is_true(input_params.get(
161
- 'apptainer_s', input_params.get('apptainer_silent', False))):
169
+ 'apptainer_s', input_params.get('apptainer_silent', input_params.get('docker_s', input_params.get('docker_silent', False))))):
162
170
  apptainer_s = True
163
171
 
164
172
  if apptainer_s and apptainer_v:
@@ -213,11 +221,16 @@ def apptainer_run(self_module, i):
213
221
  if quiet:
214
222
  env['MLC_QUIET'] = 'yes'
215
223
 
216
- regenerate_def_file = not i.get('apptainer_noregenerate', False)
217
- rebuild_apptainer_image = i.get('apptainer_rebuild', False)
224
+ regenerate_def_file = not i.get(
225
+ 'apptainer_noregenerate', i.get(
226
+ 'docker_noregenerate', False))
227
+ rebuild_apptainer_image = i.get(
228
+ 'apptainer_rebuild', i.get(
229
+ 'docker_rebuild', False))
218
230
 
219
- # Prune unnecessary Apptainer-related input keys
220
- r = prune_input({'input': i, 'extra_keys_starts_with': ['apptainer_']})
231
+ # Prune unnecessary Apptainer- and Docker-related input keys
232
+ r = prune_input(
233
+ {'input': i, 'extra_keys_starts_with': ['apptainer_', 'docker_']})
221
234
  f_run_cmd = r['new_input']
222
235
 
223
236
  # Save current directory and prepare to search for scripts
@@ -259,7 +272,7 @@ def apptainer_run(self_module, i):
259
272
  mounts = copy.deepcopy(
260
273
  i.get(
261
274
  'apptainer_mounts',
262
- []))
275
+ i.get('docker_mounts', [])))
263
276
  variations = meta.get('variations', {})
264
277
 
265
278
  if not hasattr(self_module, 'run_state'):
@@ -273,7 +286,9 @@ def apptainer_run(self_module, i):
273
286
 
274
287
  run_state = self_module.run_state
275
288
 
276
- apptainer_settings = run_state.get('docker', {})
289
+ apptainer_settings = copy.deepcopy(run_state.get('docker', {}))
290
+ utils.merge_dicts({'dict1': apptainer_settings, 'dict2': run_state.get('apptainer', {}),
291
+ 'append_lists': True, 'append_unique': True})
277
292
 
278
293
  apptainer_settings_default_env = apptainer_settings.get('default_env', {})
279
294
  for key in apptainer_settings_default_env:
@@ -297,7 +312,7 @@ def apptainer_run(self_module, i):
297
312
 
298
313
  # Skip scripts marked as non-runnable
299
314
  if not apptainer_settings.get('run', True) and not i.get(
300
- 'apptainer_run_override', False):
315
+ 'apptainer_run_override', i.get('docker_run_override', False)):
301
316
  logger.info("apptainer.run set to False in meta.yaml")
302
317
  return {'return': 0,
303
318
  'warning': 'Apptainer run is set to false in script meta'}
@@ -379,6 +394,20 @@ def apptainer_run(self_module, i):
379
394
  'quiet': True, 'real_run': True,
380
395
  'add_deps_recursive': {'build-apptainer-image': {'def_file': def_file_path}}
381
396
  }
397
+
398
+ # Detached mode (mirrors --docker_dt): run as a background instance.
399
+ if is_true(get_apptainer_input(
400
+ i, 'dt', get_apptainer_input(i, 'detached', ''))):
401
+ mlc_apptainer_input['detached'] = 'yes'
402
+ # Drop into an interactive shell after the run (mirrors --docker_it).
403
+ elif is_true(get_apptainer_input(
404
+ i, 'it', get_apptainer_input(i, 'interactive', ''))):
405
+ mlc_apptainer_input['interactive'] = 'yes'
406
+
407
+ # Tell the run step host repos were baked in, so it points MLC_REPOS at
408
+ # /opt/mlc_host_repos (matters under fakeroot, which otherwise overrides it).
409
+ if is_true(get_apptainer_input(i, 'host_mlc_repos', '')):
410
+ mlc_apptainer_input['host_mlc_repos'] = 'yes'
382
411
  # Forward build options to build-apptainer-image
383
412
  build_image_extras = {}
384
413
  if apptainer_inputs.get('ignore_fakeroot_cmd'):
@@ -443,14 +472,15 @@ def prepare_apptainer_inputs(input_params, apptainer_settings,
443
472
  "network", "security_opt"
444
473
  ]
445
474
 
446
- # Collect inputs
447
- apptainer_inputs = {
448
- key: input_params.get(
449
- f"apptainer_{key}", apptainer_settings.get(
450
- key, get_apptainer_default(key)))
451
- for key in keys
452
- if (value := input_params.get(f"apptainer_{key}", apptainer_settings.get(key, get_apptainer_default(key)))) is not None
453
- }
475
+ # Collect inputs: apptainer_X > docker_X > meta settings > default
476
+ apptainer_inputs = {}
477
+ for key in keys:
478
+ value = input_params.get(
479
+ f"apptainer_{key}",
480
+ input_params.get(
481
+ f"docker_{key}", apptainer_settings.get(key, get_apptainer_default(key))))
482
+ if value is not None:
483
+ apptainer_inputs[key] = value
454
484
 
455
485
  # Convert boolean values to 'yes'/'no' strings for MLC input mapping
456
486
  for key in list(apptainer_inputs.keys()):
@@ -544,6 +574,21 @@ def process_apptainer_mounts(mounts, env, apptainer_settings, run_state):
544
574
  'container_env_string': container_env_string}
545
575
 
546
576
 
577
+ def get_apptainer_input(input_params, key, default=None):
578
+ """Return apptainer_<key> if provided, else fall back to docker_<key>.
579
+
580
+ Lets docker_* options passed to `mlca` take effect unless the matching
581
+ apptainer_* option overrides them.
582
+ """
583
+ apptainer_key = f"apptainer_{key}"
584
+ if input_params.get(apptainer_key) is not None:
585
+ return input_params[apptainer_key]
586
+ docker_key = f"docker_{key}"
587
+ if input_params.get(docker_key) is not None:
588
+ return input_params[docker_key]
589
+ return default
590
+
591
+
547
592
  def get_apptainer_default(key):
548
593
  defaults = {
549
594
  "mlc_repo": "mlcommons@mlperf-automations",
@@ -86,6 +86,7 @@ TOP_LEVEL_SCHEMA = {
86
86
 
87
87
  # Docker
88
88
  "docker": DICT, # dict - see DOCKER_SCHEMA
89
+ "apptainer": DICT, # dict - apptainer overrides; merges with docker settings
89
90
 
90
91
  # Output / debugging
91
92
  "print_env_at_the_end": DICT, # dict[str, list[str]]
@@ -164,6 +165,7 @@ VARIATION_ENTRY_SCHEMA = {
164
165
  "state": DICT,
165
166
  "const": DICT,
166
167
  "docker": DICT,
168
+ "apptainer": DICT,
167
169
  "alias": STR,
168
170
  "default_version": STR_OR_FLOAT,
169
171
  "required_disk_space": INT,
@@ -248,6 +250,7 @@ UPDATE_META_IF_ENV_SCHEMA = {
248
250
  "default_env": DICT,
249
251
  "default_variations": DICT,
250
252
  "docker": DICT,
253
+ "apptainer": DICT,
251
254
  "adr": DICT,
252
255
  "ad": DICT,
253
256
  }
@@ -403,6 +406,21 @@ def validate_meta(data, file_path=""):
403
406
  errors.append(
404
407
  f"{prefix}docker.{dk} has type '{actual}', expected {allowed}")
405
408
 
409
+ # Validate apptainer section (same schema as docker; apptainer overrides
410
+ # docker)
411
+ apptainer = data.get("apptainer")
412
+ if isinstance(apptainer, dict):
413
+ for ak, av in apptainer.items():
414
+ if ak not in DOCKER_SCHEMA:
415
+ warnings.append(
416
+ f"{prefix}apptainer: unknown key '{ak}'")
417
+ continue
418
+ actual = type(av).__name__
419
+ allowed = DOCKER_SCHEMA[ak]
420
+ if actual not in allowed:
421
+ errors.append(
422
+ f"{prefix}apptainer.{ak} has type '{actual}', expected {allowed}")
423
+
406
424
  # Validate tests section
407
425
  tests = data.get("tests")
408
426
  if isinstance(tests, dict):
@@ -151,7 +151,7 @@ class ScriptAutomation(Automation):
151
151
  for f in ['fake_deps', 'cache']:
152
152
  run_state.setdefault(f, False)
153
153
 
154
- for d in ['input_mapping', 'docker', 'remote_run']:
154
+ for d in ['input_mapping', 'docker', 'remote_run', 'apptainer']:
155
155
  run_state.setdefault(d, {})
156
156
 
157
157
  for l in ['deps', 'post_deps', 'prehook_deps', 'posthook_deps',
@@ -5768,6 +5768,14 @@ def _apply_conditional_meta_updates(update_meta_if_env, default_env, env, const,
5768
5768
  'append_lists': True,
5769
5769
  'append_unique': True})
5770
5770
 
5771
+ if c_meta.get('apptainer', {}):
5772
+ if not run_state.get('apptainer', {}):
5773
+ run_state['apptainer'] = {}
5774
+ utils.merge_dicts({'dict1': run_state['apptainer'],
5775
+ 'dict2': c_meta['apptainer'],
5776
+ 'append_lists': True,
5777
+ 'append_unique': True})
5778
+
5771
5779
  c_add_deps_info = c_meta.get('ad', {})
5772
5780
  if not c_add_deps_info:
5773
5781
  c_add_deps_info = c_meta.get('add_deps', {})
@@ -5975,6 +5983,13 @@ def update_state_from_meta(meta, env, state, const, const_state, run_state, i):
5975
5983
  if remote_run_settings.get('deps', []):
5976
5984
  update_deps(remote_run_settings['deps'], add_deps_info, False, env)
5977
5985
 
5986
+ new_apptainer_settings = meta.get('apptainer')
5987
+ if new_apptainer_settings:
5988
+ utils.merge_dicts({'dict1': run_state['apptainer'],
5989
+ 'dict2': new_apptainer_settings,
5990
+ 'append_lists': True,
5991
+ 'append_unique': True})
5992
+
5978
5993
  new_env_keys_from_meta = meta.get('new_env_keys', [])
5979
5994
  if new_env_keys_from_meta:
5980
5995
  run_state['new_env_keys'] += new_env_keys_from_meta
@@ -448,6 +448,7 @@ class Action:
448
448
  force_remove = True
449
449
 
450
450
  results = res['list']
451
+ removed_any = False
451
452
 
452
453
  for result in results:
453
454
  item_path = result.path
@@ -468,7 +469,13 @@ class Action:
468
469
  logger.warning(
469
470
  f"{item_path} was already removed by another process.")
470
471
 
471
- self.get_index().rm(item_meta, target_name, item_path)
472
+ # Batch the index write: de-index in memory, persist once after
473
+ # the loop instead of rewriting all index files per item.
474
+ self.get_index().rm(item_meta, target_name, item_path, save=False)
475
+ removed_any = True
476
+
477
+ if removed_any:
478
+ self.get_index().save()
472
479
 
473
480
  return {
474
481
  "return": 0,
@@ -192,7 +192,7 @@ class Index:
192
192
  }
193
193
  self._save_indices()
194
194
 
195
- def rm(self, meta, folder_type, path):
195
+ def rm(self, meta, folder_type, path, save=True):
196
196
  uid = meta['uid']
197
197
  index = self.get_index(folder_type, uid)
198
198
  if index == -1:
@@ -200,6 +200,11 @@ class Index:
200
200
  f"Index is not having the {folder_type} item {path}")
201
201
  else:
202
202
  del (self.indices[folder_type][index])
203
+ if save:
204
+ self._save_indices()
205
+
206
+ def save(self):
207
+ """Persist all in-memory indices to disk (for batched callers)."""
203
208
  self._save_indices()
204
209
 
205
210
  def get_item_mtime(self, file):
@@ -349,6 +354,40 @@ class Index:
349
354
  logger.debug(
350
355
  f"Deleted keys removed from modified times and indices: {deleted_keys}")
351
356
 
357
+ # Reconcile against the filesystem. Entries added via add()/update()
358
+ # (e.g. caches written during a script run) are never recorded in
359
+ # modified_times, so the deleted-key check above cannot catch them
360
+ # once their folder is removed out-of-band. Drop any index entry
361
+ # whose folder no longer exists, scoped to repos currently present
362
+ # so an empty/unavailable repo list can never wipe a valid index.
363
+ repo_prefixes = [
364
+ os.path.normpath(r.path) for r in self.repos
365
+ if getattr(r, 'path', None)
366
+ ]
367
+ if repo_prefixes:
368
+ for ft in self.indices:
369
+ kept = []
370
+ for item in self.indices[ft]:
371
+ path = item.get('path', '')
372
+ norm = os.path.normpath(path) if path else ''
373
+ under_known_repo = any(
374
+ norm == rp or norm.startswith(rp + os.sep)
375
+ for rp in repo_prefixes
376
+ )
377
+ if under_known_repo and not os.path.exists(path):
378
+ logger.warning(
379
+ f"Removing stale {ft} index entry whose folder "
380
+ f"no longer exists: {path}")
381
+ for mt_key in [
382
+ k for k in self.modified_times
383
+ if os.path.normpath(k) == norm
384
+ or os.path.normpath(k).startswith(norm + os.sep)]:
385
+ del self.modified_times[mt_key]
386
+ changed = True
387
+ continue
388
+ kept.append(item)
389
+ self.indices[ft] = kept
390
+
352
391
  if force_rebuild or changed:
353
392
  logger.debug(
354
393
  "Changes detected, saving updated index and modified times.")
@@ -13,12 +13,17 @@ from .script_action import ScriptAction
13
13
  from .cache_action import CacheAction
14
14
  from .cfg_action import CfgAction
15
15
  from .experiment_action import ExperimentAction
16
+ from .typo_mixin import TypoMixin
16
17
 
17
18
  from .item import Item
18
19
  from .action_factory import get_action
19
20
  from .logger import logger, logging
20
21
 
21
22
 
23
+ class TypoArgumentParser(TypoMixin, argparse.ArgumentParser):
24
+ """ArgumentParser that suggests close matches when the user mistype a command or target."""
25
+
26
+
22
27
  class Automation:
23
28
  action_object = None
24
29
  automation_type = None
@@ -316,10 +321,6 @@ def process_console_output(res, target, action, run_args):
316
321
  f"Warning code: {warning['code']}, Discription: {warning['description']}")
317
322
 
318
323
 
319
- if default_parent is None:
320
- default_parent = Action()
321
-
322
-
323
324
  log_flag_aliases = {'-v': '--verbose', '-s': '--silent'}
324
325
  log_levels = {'--verbose': logging.DEBUG, '--silent': logging.WARNING}
325
326
 
@@ -340,21 +341,15 @@ def convert_hyphen_to_underscore_in_args():
340
341
 
341
342
 
342
343
  def build_pre_parser():
343
- pre_parser = argparse.ArgumentParser(add_help=False)
344
+ pre_parser = TypoArgumentParser(add_help=False)
344
345
  pre_parser.add_argument(
345
346
  "action",
346
347
  nargs="?",
347
348
  help="Top-level action (run, build, help, etc.)")
349
+ # No choices here: valid targets depend on the action, so validation is
350
+ # left to the per-action subparsers in build_parser().
348
351
  pre_parser.add_argument(
349
352
  "target",
350
- choices=[
351
- 'run',
352
- 'script',
353
- 'cache',
354
- 'repo',
355
- 'repos',
356
- 'experiment',
357
- 'all'],
358
353
  nargs="?",
359
354
  help="Target (repo, script, cache, ...)")
360
355
  pre_parser.add_argument("-h", "--help", action="store_true")
@@ -362,7 +357,7 @@ def build_pre_parser():
362
357
 
363
358
 
364
359
  def build_parser(pre_args):
365
- parser = argparse.ArgumentParser(
360
+ parser = TypoArgumentParser(
366
361
  prog="mlc",
367
362
  description="Manage repos, scripts, and caches.",
368
363
  add_help=False)
@@ -424,6 +419,11 @@ def build_parser(pre_args):
424
419
  # Load cfg
425
420
  load_parser = subparsers.add_parser("load", add_help=False)
426
421
  load_parser.add_argument("target", choices=["cfg"])
422
+ load_parser.add_argument(
423
+ 'details',
424
+ nargs='?',
425
+ help='Details or identifier (optional)')
426
+ load_parser.add_argument('extra', nargs=argparse.REMAINDER)
427
427
  return parser
428
428
 
429
429
 
@@ -480,6 +480,14 @@ def build_run_args(args):
480
480
  if hasattr(args, 'target') and args.target:
481
481
  run_args['reindex_target'] = args.target
482
482
 
483
+ # Merge bulk options from a YAML/JSON file (--mlc_input_file=run.yaml).
484
+ # Done last so explicit CLI args (including positional tags) win.
485
+ r = utils.load_cli_input_file(run_args)
486
+ if r['return'] > 0:
487
+ logger.error(r['error'])
488
+ sys.exit(1)
489
+ run_args = r['args_dict']
490
+
483
491
  # Check for path-only flag (for script-friendly output)
484
492
  if run_args.get('path_only') or run_args.get('p'):
485
493
  run_args['path_only'] = True
@@ -86,6 +86,7 @@ TOP_LEVEL_SCHEMA = {
86
86
 
87
87
  # Docker
88
88
  "docker": DICT, # dict - see DOCKER_SCHEMA
89
+ "apptainer": DICT, # dict - apptainer overrides; merges with docker settings
89
90
 
90
91
  # Output / debugging
91
92
  "print_env_at_the_end": DICT, # dict[str, list[str]]
@@ -164,6 +165,7 @@ VARIATION_ENTRY_SCHEMA = {
164
165
  "state": DICT,
165
166
  "const": DICT,
166
167
  "docker": DICT,
168
+ "apptainer": DICT,
167
169
  "alias": STR,
168
170
  "default_version": STR_OR_FLOAT,
169
171
  "required_disk_space": INT,
@@ -248,6 +250,7 @@ UPDATE_META_IF_ENV_SCHEMA = {
248
250
  "default_env": DICT,
249
251
  "default_variations": DICT,
250
252
  "docker": DICT,
253
+ "apptainer": DICT,
251
254
  "adr": DICT,
252
255
  "ad": DICT,
253
256
  }
@@ -403,6 +406,21 @@ def validate_meta(data, file_path=""):
403
406
  errors.append(
404
407
  f"{prefix}docker.{dk} has type '{actual}', expected {allowed}")
405
408
 
409
+ # Validate apptainer section (same schema as docker; apptainer overrides
410
+ # docker)
411
+ apptainer = data.get("apptainer")
412
+ if isinstance(apptainer, dict):
413
+ for ak, av in apptainer.items():
414
+ if ak not in DOCKER_SCHEMA:
415
+ warnings.append(
416
+ f"{prefix}apptainer: unknown key '{ak}'")
417
+ continue
418
+ actual = type(av).__name__
419
+ allowed = DOCKER_SCHEMA[ak]
420
+ if actual not in allowed:
421
+ errors.append(
422
+ f"{prefix}apptainer.{ak} has type '{actual}', expected {allowed}")
423
+
406
424
  # Validate tests section
407
425
  tests = data.get("tests")
408
426
  if isinstance(tests, dict):