mlcflow 1.3.8__tar.gz → 1.3.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {mlcflow-1.3.8 → mlcflow-1.3.9}/PKG-INFO +1 -1
  2. mlcflow-1.3.9/VERSION +1 -0
  3. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/apptainer.py +49 -7
  4. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/docker_utils.py +6 -2
  5. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/module.py +19 -5
  6. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/slurm_run.py +52 -14
  7. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/utils.py +9 -9
  8. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/index.py +18 -2
  9. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/main.py +44 -29
  10. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/repo_action.py +4 -1
  11. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/script_action.py +150 -74
  12. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlcflow.egg-info/PKG-INFO +1 -1
  13. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlcflow.egg-info/SOURCES.txt +5 -0
  14. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlcflow.egg-info/entry_points.txt +2 -0
  15. {mlcflow-1.3.8 → mlcflow-1.3.9}/pyproject.toml +2 -0
  16. mlcflow-1.3.9/tests/test_cli_input_file.py +132 -0
  17. mlcflow-1.3.9/tests/test_container_cmd_quoting.py +189 -0
  18. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_convert_env_to_script.py +33 -0
  19. mlcflow-1.3.9/tests/test_help_coverage.py +109 -0
  20. mlcflow-1.3.9/tests/test_index_reconcile.py +113 -0
  21. mlcflow-1.3.9/tests/test_run_target_dispatch.py +41 -0
  22. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_script_action_apptainer.py +1 -1
  23. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_slurm_run.py +81 -0
  24. mlcflow-1.3.8/VERSION +0 -1
  25. {mlcflow-1.3.8 → mlcflow-1.3.9}/LICENSE.md +0 -0
  26. {mlcflow-1.3.8 → mlcflow-1.3.9}/README.md +0 -0
  27. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/COPYRIGHT.txt +0 -0
  28. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/__init__.py +0 -0
  29. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/README.md +0 -0
  30. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/__init__.py +0 -0
  31. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/cache_utils.py +0 -0
  32. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/compat.py +0 -0
  33. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/doc.py +0 -0
  34. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/docker.py +0 -0
  35. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/experiment.py +0 -0
  36. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/help.py +0 -0
  37. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/lint.py +0 -0
  38. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/meta.json +0 -0
  39. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/meta_schema.py +0 -0
  40. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/remote_run.py +0 -0
  41. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/script_utils.py +0 -0
  42. {mlcflow-1.3.8 → mlcflow-1.3.9}/automation/script/validate.py +0 -0
  43. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/__init__.py +0 -0
  44. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/__main__.py +0 -0
  45. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/action.py +0 -0
  46. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/action_factory.py +0 -0
  47. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/cache_action.py +0 -0
  48. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/cfg_action.py +0 -0
  49. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/error_codes.py +0 -0
  50. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/experiment_action.py +0 -0
  51. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/item.py +0 -0
  52. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/logger.py +0 -0
  53. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/meta_schema.py +0 -0
  54. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/repo.py +0 -0
  55. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/typo_mixin.py +0 -0
  56. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlc/utils.py +0 -0
  57. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlcflow.egg-info/dependency_links.txt +0 -0
  58. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlcflow.egg-info/requires.txt +0 -0
  59. {mlcflow-1.3.8 → mlcflow-1.3.9}/mlcflow.egg-info/top_level.txt +0 -0
  60. {mlcflow-1.3.8 → mlcflow-1.3.9}/setup.cfg +0 -0
  61. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_action_invalid_meta_entries.py +0 -0
  62. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_apptainer_docker_unification.py +0 -0
  63. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_build_wheels_workflow.py +0 -0
  64. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_cache_mark_tmp.py +0 -0
  65. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_installer_venv_deactivation.py +0 -0
  66. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_mlcflow_unix_installer.py +0 -0
  67. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_pull_repo_thread_safety.py +0 -0
  68. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_release_version.py +0 -0
  69. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_repo_action_extra_git_args.py +0 -0
  70. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_thread_safety.py +0 -0
  71. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_typo_mixin.py +0 -0
  72. {mlcflow-1.3.8 → mlcflow-1.3.9}/tests/test_unix_installer_venv.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mlcflow
3
- Version: 1.3.8
3
+ Version: 1.3.9
4
4
  Summary: An automation interface tailored for CPU/GPU benchmarking
5
5
  Author-email: MLCommons <systems@mlcommons.org>
6
6
  License:
mlcflow-1.3.9/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.3.9
@@ -1,4 +1,5 @@
1
1
  import os
2
+ import shlex
2
3
  from mlc import utils
3
4
  from utils import *
4
5
  import logging
@@ -160,6 +161,13 @@ def apptainerfile(self_module, input_params):
160
161
  if is_true(get_apptainer_input(input_params, 'host_mlc_repos', '')):
161
162
  mlc_apptainer_input['host_mlc_repos'] = 'yes'
162
163
 
164
+ # Bind destinations must exist in the image rootfs, otherwise the
165
+ # build-time fake run fails when the binds are applied.
166
+ build_binds = get_apptainer_bind_mounts(
167
+ input_params, env, apptainer_settings, run_state)
168
+ if build_binds:
169
+ mlc_apptainer_input['bind'] = build_binds
170
+
163
171
  apptainer_v = False
164
172
  apptainer_s = False
165
173
  if is_true(input_params.get(
@@ -352,12 +360,13 @@ def apptainer_run(self_module, i):
352
360
  mounts, env, apptainer_settings, run_state)
353
361
  container_env_string = bind_mounts.get('container_env_string', '')
354
362
 
355
- # Forward input-mapped env vars to the container's mlcr command
363
+ # Forward input-mapped env vars to the container's mlcr command.
364
+ # Unresolved default_env placeholders (<<<VAR>>>) are dropped.
356
365
  input_mapping = meta.get('input_mapping', {})
357
366
  for input_key, env_key in input_mapping.items():
358
367
  val = env.get(env_key, '')
359
- if val and isinstance(val, str):
360
- container_env_string += f' --env.{env_key}={val}'
368
+ if val and isinstance(val, str) and '<<<' not in val:
369
+ container_env_string += f' --env.{env_key}={shlex.quote(val)}'
361
370
 
362
371
  # Forward framework-level env keys (MLC_OUTDIRNAME, MLC_INPUT, MLC_OUTPUT, etc.)
363
372
  # These are skipped in regenerate_script_cmd (docker_utils.py) and must be
@@ -367,14 +376,14 @@ def apptainer_run(self_module, i):
367
376
  for input_key in framework_input_mapping:
368
377
  env_key = 'MLC_' + input_key.upper().replace('-', '_').replace(' ', '_')
369
378
  val = env.get(env_key, '')
370
- if val and isinstance(val, str):
371
- container_env_string += f' --env.{env_key}={val}'
379
+ if val and isinstance(val, str) and '<<<' not in val:
380
+ container_env_string += f' --env.{env_key}={shlex.quote(val)}'
372
381
 
373
382
  # Forward proxy env vars (matching Docker behavior)
374
383
  for proxy_key in self_module.host_env_keys:
375
384
  proxy_value = os.environ.get(proxy_key)
376
385
  if proxy_value:
377
- container_env_string += f' --env.{proxy_key}={proxy_value}'
386
+ container_env_string += f' --env.{proxy_key}={shlex.quote(proxy_value)}'
378
387
 
379
388
  # Generate the run command
380
389
  r = regenerate_script_cmd({'script_uid': script_uid,
@@ -419,6 +428,10 @@ def apptainer_run(self_module, i):
419
428
  build_image_extras)
420
429
  # Wire resolved docker.mounts to apptainer bind mounts
421
430
  resolved_mounts = bind_mounts.get('mounts', [])
431
+ mlc_local_bind = get_mlc_local_bind(i)
432
+ if mlc_local_bind:
433
+ os.makedirs(mlc_local_bind.split(':', 1)[0], exist_ok=True)
434
+ resolved_mounts.append(mlc_local_bind)
422
435
  if resolved_mounts:
423
436
  mlc_apptainer_input['bind'] = resolved_mounts
424
437
 
@@ -574,6 +587,35 @@ def process_apptainer_mounts(mounts, env, apptainer_settings, run_state):
574
587
  'container_env_string': container_env_string}
575
588
 
576
589
 
590
+ def get_mlc_local_bind(input_params):
591
+ """Bind for --apptainer_mlc_local_dir: a writable host directory over the
592
+ container's MLC local dir (cache), used with --apptainer_host_mlc_repos so
593
+ caches do not land in the small, discarded --writable-tmpfs overlay."""
594
+ local_dir = get_apptainer_input(input_params, 'mlc_local_dir', '')
595
+ if not local_dir or not is_true(
596
+ get_apptainer_input(input_params, 'host_mlc_repos', '')):
597
+ return ''
598
+ return f"{os.path.abspath(local_dir)}:/opt/mlc_host_repos/local"
599
+
600
+
601
+ def get_apptainer_bind_mounts(input_params, env, apptainer_settings,
602
+ run_state):
603
+ """All bind mounts (user binds, resolved docker.mounts, MLC local dir)
604
+ known before the image is built."""
605
+ binds = get_apptainer_input(input_params, 'bind',
606
+ apptainer_settings.get('bind')) or []
607
+ if isinstance(binds, str):
608
+ binds = [binds]
609
+ binds = list(binds)
610
+ mounts = copy.deepcopy(get_apptainer_input(input_params, 'mounts', []))
611
+ binds += process_apptainer_mounts(
612
+ mounts, env, apptainer_settings, run_state)['mounts']
613
+ mlc_local_bind = get_mlc_local_bind(input_params)
614
+ if mlc_local_bind:
615
+ binds.append(mlc_local_bind)
616
+ return binds
617
+
618
+
577
619
  def get_apptainer_input(input_params, key, default=None):
578
620
  """Return apptainer_<key> if provided, else fall back to docker_<key>.
579
621
 
@@ -592,7 +634,7 @@ def get_apptainer_input(input_params, key, default=None):
592
634
  def get_apptainer_default(key):
593
635
  defaults = {
594
636
  "mlc_repo": "mlcommons@mlperf-automations",
595
- "mlc_repo_branch": "dev",
637
+ "mlc_repo_branch": "main",
596
638
  "os": "ubuntu",
597
639
  "os_version": "",
598
640
  "fake_run_deps": False,
@@ -328,7 +328,7 @@ def regenerate_script_cmd(i):
328
328
  if isinstance(val, str) and (
329
329
  os.path.join("local", "cache", "") in val or
330
330
  os.path.join("MLC", "repos", "") in val or
331
- "<<<" in value
331
+ "<<<" in val
332
332
  )
333
333
  ]
334
334
 
@@ -373,6 +373,10 @@ def regenerate_script_cmd(i):
373
373
  for key in keys:
374
374
  if key in ["input", "output", "outdirname"]:
375
375
  continue # We have the corresponding env keys in container env string
376
+ # Provenance only; nesting an already shell-joined command breaks
377
+ # quoting in the container command and tmp-run.sh.
378
+ if key == "mlc_run_cmd" and not prefix:
379
+ continue
376
380
  # Construct the full key with the prefix.
377
381
  full_key = f"{prefix}.{key}" if prefix else key
378
382
 
@@ -422,7 +426,7 @@ def regenerate_script_cmd(i):
422
426
  def get_docker_default(key):
423
427
  defaults = {
424
428
  "mlc_repo": "mlcommons@mlperf-automations",
425
- "mlc_repo_branch": "dev",
429
+ "mlc_repo_branch": "main",
426
430
  "os": "ubuntu",
427
431
  "os_version": "",
428
432
  "fake_run_deps": False,
@@ -4558,6 +4558,18 @@ pip install mlcflow
4558
4558
  i['remote_action'] = 'slurm-experiment'
4559
4559
  return remote_run(self, i)
4560
4560
 
4561
+ ############################################################
4562
+ def remote_slurm_docker(self, i):
4563
+ from script.remote_run import remote_run
4564
+ i['remote_action'] = 'slurm-docker'
4565
+ return remote_run(self, i)
4566
+
4567
+ ############################################################
4568
+ def remote_slurm_apptainer(self, i):
4569
+ from script.remote_run import remote_run
4570
+ i['remote_action'] = 'slurm-apptainer'
4571
+ return remote_run(self, i)
4572
+
4561
4573
  ##########################################################################
4562
4574
 
4563
4575
  def _available_variations(self, i):
@@ -5428,12 +5440,14 @@ def convert_env_to_script(env, os_info, start_script=None):
5428
5440
  env_separator = os_info.get('env_separator', ':')
5429
5441
  env_value = env_separator.join(str(v) for v in env_value)
5430
5442
 
5431
- env_quote = os_info.get('env_quote', '"')
5432
- # Replace placeholders in the platform-specific environment command
5433
- # and escapes any quote in the env value
5443
+ env_value = str(env_value)
5444
+ # cmd.exe `set` takes the value literally (backslash is not an escape
5445
+ # there), so only the POSIX double-quoted export needs escaping.
5446
+ if not is_windows:
5447
+ env_quote = os_info.get('env_quote', '"')
5448
+ env_value = env_value.replace(env_quote, f"""\\{env_quote}""")
5434
5449
  env_command = os_info['set_env'].replace(
5435
- '${key}', key).replace(
5436
- '${value}', str(env_value).replace(env_quote, f"""\\{env_quote}"""))
5450
+ '${key}', key).replace('${value}', env_value)
5437
5451
  script.append(env_command)
5438
5452
 
5439
5453
  return script
@@ -5,6 +5,7 @@ import subprocess
5
5
  import copy
6
6
  from utils import is_true, prune_input
7
7
  from script.script_utils import *
8
+ from script.apptainer import get_apptainer_input
8
9
 
9
10
 
10
11
  def slurm_run(self_module, i, slurm_action='run'):
@@ -113,6 +114,11 @@ def slurm_run(self_module, i, slurm_action='run'):
113
114
 
114
115
  run_input = prune_result['new_input']
115
116
 
117
+ # Flag SLURM execution as an MLC env var (like docker's MLC_RUN_STATE_DOCKER)
118
+ # so scripts can enable deps via enable_if_env: MLC_RUN_STATE_SLURM.
119
+ if isinstance(run_input, dict):
120
+ run_input.setdefault('env', {})['MLC_RUN_STATE_SLURM'] = 'yes'
121
+
116
122
  cur_dir = os.getcwd()
117
123
 
118
124
  r = self_module._select_script(i)
@@ -150,6 +156,15 @@ def slurm_run(self_module, i, slurm_action='run'):
150
156
  if quiet:
151
157
  script_run_cmd += ' --quiet'
152
158
 
159
+ # With host repos baked into the image, MLC_REPOS points inside the
160
+ # container and caches would land in the small, discarded writable-tmpfs
161
+ # overlay; bind the isolated workspace's local dir so copy-back sees them.
162
+ if (slurm_isolated and slurm_action == 'apptainer'
163
+ and not get_apptainer_input(i, 'mlc_local_dir')
164
+ and is_true(get_apptainer_input(i, 'host_mlc_repos', False))):
165
+ script_run_cmd += \
166
+ ' --apptainer_mlc_local_dir="$MLC_REPOS/local"'
167
+
153
168
  # Build the commands to run inside srun
154
169
  run_cmds = []
155
170
 
@@ -211,22 +226,28 @@ def slurm_run(self_module, i, slurm_action='run'):
211
226
  if slurm_copy_back_mlc_cache:
212
227
  local_cache = slurm_copy_back_mlc_cache_path or get_local_mlc_cache_path(
213
228
  self_module)
214
- safe_local_cache = (
215
- str(local_cache)
216
- .replace('\\', '\\\\')
217
- .replace('"', '\\"')
218
- .replace('$', '\\$')
219
- .replace('`', '\\`')
220
- )
221
- if slurm_isolated:
222
- run_cmds.append(
223
- f'mkdir -p "{safe_local_cache}" && '
224
- f'rsync -a "$MLC_REPOS/local/cache/" "{safe_local_cache}/"'
225
- )
226
- elif slurm_copy_back_mlc_cache_path:
229
+
230
+ def _dq(path):
231
+ return (str(path).replace('\\', '\\\\').replace('"', '\\"')
232
+ .replace('$', '\\$').replace('`', '\\`'))
233
+
234
+ safe_local_cache = _dq(local_cache)
235
+ # An explicit container local dir (bound by apptainer.py only with
236
+ # host repos) is where the container's cache is written.
237
+ container_local_dir = ''
238
+ if slurm_action == 'apptainer' and is_true(
239
+ get_apptainer_input(i, 'host_mlc_repos', False)):
240
+ container_local_dir = get_apptainer_input(i, 'mlc_local_dir', '')
241
+ if container_local_dir:
242
+ source_cache = f'"{_dq(container_local_dir)}/cache/"'
243
+ elif slurm_isolated:
244
+ source_cache = '"$MLC_REPOS/local/cache/"'
245
+ else:
246
+ source_cache = '~/MLC/repos/local/cache/'
247
+ if slurm_isolated or slurm_copy_back_mlc_cache_path or container_local_dir:
227
248
  run_cmds.append(
228
249
  f'mkdir -p "{safe_local_cache}" && '
229
- f'rsync -a ~/MLC/repos/local/cache/ "{safe_local_cache}/"'
250
+ f'rsync -a {source_cache} "{safe_local_cache}/"'
230
251
  )
231
252
 
232
253
  # Join all commands with && so failure stops execution
@@ -303,6 +324,23 @@ def regenerate_script_cmd(i):
303
324
  i_run_cmd = i['run_cmd']
304
325
  slurm_action = i.get('slurm_action', 'run')
305
326
 
327
+ # Provenance only; the node-side mlc computes its own.
328
+ i_run_cmd.pop('mlc_run_cmd', None)
329
+
330
+ # Drop unresolved default_env placeholders (<<<VAR>>>) so they are not
331
+ # forwarded as explicit inputs to the node.
332
+ env = i_run_cmd.get('env')
333
+ if isinstance(env, dict):
334
+ for key in list(env):
335
+ value = env[key]
336
+ if isinstance(value, str) and '<<<' in value:
337
+ del env[key]
338
+ elif isinstance(value, list):
339
+ env[key] = [v for v in value
340
+ if not (isinstance(v, str) and '<<<' in v)]
341
+ if not env[key]:
342
+ del env[key]
343
+
306
344
  action_to_cmd = {
307
345
  'run': 'mlcr',
308
346
  'docker': 'mlcd',
@@ -1207,18 +1207,18 @@ def parse_expiration(user_input: str) -> float:
1207
1207
  return time.time() + seconds
1208
1208
 
1209
1209
 
1210
- def quote_if_needed(val: str, quote: str) -> str:
1210
+ def quote_if_needed(val: str, quote: str = '') -> str:
1211
1211
  """
1212
- Return the value, quoted with escaped quotes if it contains spaces
1213
- or quotes.
1212
+ Return the value quoted for a POSIX shell when it contains anything other
1213
+ than shell-safe characters. Single quotes are used, so $VAR, $(...),
1214
+ backticks and backslashes stay literal. quote='"' forces quoting.
1214
1215
  """
1216
+ import shlex
1215
1217
  s = str(val)
1216
-
1217
- if " " in s or '"' in s or quote == '"':
1218
- # Escape all existing double quotes
1219
- s = s.replace('"', r'\"')
1220
- return f'"{s}"'
1221
- return s
1218
+ quoted = shlex.quote(s)
1219
+ if quoted == s and quote == '"':
1220
+ return f"'{s}'"
1221
+ return quoted
1222
1222
 
1223
1223
 
1224
1224
  def has_function_in_file(py_file, func_name):
@@ -227,6 +227,12 @@ class Index:
227
227
  if not os.path.isdir(folder_path):
228
228
  continue
229
229
 
230
+ indexed_paths = {
231
+ os.path.normpath(item.get("path", ""))
232
+ for item in self.indices[folder_type]}
233
+ indexed_uids = {
234
+ item.get("uid") for item in self.indices[folder_type]}
235
+
230
236
  for automation_dir in os.listdir(folder_path):
231
237
  automation_path = os.path.join(folder_path, automation_dir)
232
238
  if not os.path.isdir(automation_path):
@@ -277,9 +283,19 @@ class Index:
277
283
  mtime = self.get_item_mtime(config_path)
278
284
  old_mtime = self._get_stored_mtime(config_path)
279
285
 
280
- # skip if unchanged
286
+ # Skip if unchanged, unless the entry is missing from the
287
+ # index (e.g. a concurrent mlc process saved the index
288
+ # without it). A missing entry whose uid is indexed under
289
+ # another path is a duplicate uid and stays skipped.
281
290
  if old_mtime == mtime and not repos_changed:
282
- continue
291
+ if os.path.normpath(automation_path) in indexed_paths:
292
+ continue
293
+ if not isinstance(data, dict) or not data.get(
294
+ "uid") or data.get("uid") in indexed_uids:
295
+ continue
296
+ logger.debug(
297
+ f"Re-indexing {folder_type} missing from index: "
298
+ f"{automation_path}")
283
299
 
284
300
  # Validate script meta against schema during indexing if meta
285
301
  # changed or repos changed
@@ -83,6 +83,17 @@ class Automation:
83
83
 
84
84
 
85
85
  mlc_run_cmd = None
86
+
87
+ # Actions that only apply to scripts (and accept `run` as their target), in
88
+ # the normalized underscore form that main() compares against.
89
+ SCRIPT_ONLY_ACTIONS = (
90
+ 'docker', 'docker_run', 'apptainer', 'apptainer_run',
91
+ 'experiment', 'remote_run', 'remote_experiment',
92
+ 'remote_docker', 'remote_slurm', 'remote_slurm_experiment',
93
+ 'remote_slurm_docker', 'remote_slurm_apptainer',
94
+ 'slurm_run', 'slurm_docker', 'slurm_apptainer', 'slurm_experiment',
95
+ 'doc', 'lint',
96
+ )
86
97
  _current_target = None
87
98
 
88
99
 
@@ -273,6 +284,14 @@ def mlcrse():
273
284
  mlc_expand_short("remote-slurm-experiment")
274
285
 
275
286
 
287
+ def mlcrsd():
288
+ mlc_expand_short("remote-slurm-docker")
289
+
290
+
291
+ def mlcrsa():
292
+ mlc_expand_short("remote-slurm-apptainer")
293
+
294
+
276
295
  def mlce():
277
296
  mlc_expand_short("experiment")
278
297
 
@@ -402,13 +421,8 @@ def build_parser(pre_args):
402
421
  reindex_parser.add_argument('extra', nargs=argparse.REMAINDER)
403
422
 
404
423
  # Script-only
405
- for action in ['docker', 'docker-run', 'apptainer', 'apptainer-run',
406
- 'experiment', 'remote-run', 'remote-experiment',
407
- 'remote-docker', 'remote-slurm', 'remote-slurm-experiment',
408
- 'slurm-run',
409
- 'slurm-docker', 'slurm-apptainer', 'slurm-experiment',
410
- 'doc', 'lint']:
411
- p = subparsers.add_parser(action, add_help=False)
424
+ for action in SCRIPT_ONLY_ACTIONS:
425
+ p = subparsers.add_parser(action.replace('_', '-'), add_help=False)
412
426
  p.add_argument('target', choices=['script', 'run'])
413
427
  p.add_argument(
414
428
  'details',
@@ -452,12 +466,7 @@ def build_run_args(args):
452
466
  if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo":
453
467
  run_args['repo'] = args.details
454
468
 
455
- if args.command in ['docker', 'docker-run', 'apptainer', 'apptainer-run',
456
- 'experiment', 'remote-run', 'remote-experiment',
457
- 'remote-docker', 'remote-slurm', 'remote-slurm-experiment',
458
- 'slurm-run',
459
- 'slurm-docker', 'slurm-apptainer', 'slurm-experiment',
460
- 'doc', 'lint'] and args.target == "run":
469
+ if args.command in SCRIPT_ONLY_ACTIONS and args.target == "run":
461
470
  # run_args['target'] = 'script' #dont modify this as script might have
462
471
  # target as in input
463
472
  args.target = "script"
@@ -590,8 +599,7 @@ def main():
590
599
  if pre_args.help and not "tags" in run_args:
591
600
  help_text = ""
592
601
  if pre_args.target == "run":
593
- if pre_args.action.startswith(
594
- "docker") or pre_args.action in ("apptainer", "apptainer-run"):
602
+ if pre_args.action in SCRIPT_ONLY_ACTIONS:
595
603
  pre_args.target = "script"
596
604
  else:
597
605
  logger.error(
@@ -601,17 +609,7 @@ def main():
601
609
  if not pre_args.action and not pre_args.target:
602
610
  help_text += main.__doc__
603
611
  elif pre_args.action and not pre_args.target:
604
- script_only_actions = {
605
- a.replace("-", "_")
606
- for a in [
607
- 'docker', 'docker-run', 'apptainer', 'apptainer-run',
608
- 'experiment', 'remote-run', 'remote-experiment',
609
- 'remote-docker', 'remote-slurm', 'remote-slurm-experiment',
610
- 'slurm-run', 'slurm-docker', 'slurm-apptainer', 'slurm-experiment',
611
- 'doc', 'lint',
612
- ]
613
- }
614
- if pre_args.action in script_only_actions:
612
+ if pre_args.action in SCRIPT_ONLY_ACTIONS:
615
613
  pre_args.target = 'script'
616
614
  elif pre_args.action not in ['script', 'cache', 'repo']:
617
615
  logger.error(f"Invalid target {pre_args.action}")
@@ -630,10 +628,11 @@ def main():
630
628
  actions = get_action(pre_args.target, default_parent)
631
629
  action_name = pre_args.action.replace("-", "_")
632
630
  try:
633
- method = getattr(actions, action_name)
631
+ getattr(actions, action_name)
634
632
  help_text += actions.__doc__ or ""
635
- if method.__doc__:
636
- help_text += method.__doc__
633
+ for name in getattr(actions, 'HELP_COMPONENTS', {}).get(
634
+ action_name, [action_name]):
635
+ help_text += getattr(actions, name).__doc__ or ""
637
636
  except AttributeError:
638
637
  logger.error(
639
638
  f"Error: '{pre_args.action}' is not supported for {pre_args.target}.")
@@ -641,6 +640,22 @@ def main():
641
640
  print(help_text)
642
641
  sys.exit(0)
643
642
 
643
+ # Script-specific help (`mlcsr <tags> --help`): show the options of the
644
+ # wrapping actions, then the script's own inputs, without running it.
645
+ if pre_args.help and getattr(args, 'target', None) in ('script', 'run'):
646
+ actions = get_action('script', default_parent)
647
+ components = [
648
+ c for c in getattr(actions, 'HELP_COMPONENTS', {}).get(
649
+ args.command, []) if c != 'run']
650
+ if components:
651
+ print(''.join(getattr(actions, c).__doc__ or ''
652
+ for c in components))
653
+ res = actions.help(run_args)
654
+ if res['return'] > 0:
655
+ logging.error(res.get('error', 'Error displaying help'))
656
+ sys.exit(1)
657
+ sys.exit(0)
658
+
644
659
  if hasattr(args, 'target') and args.target == "repos":
645
660
  args.target = "repo"
646
661
 
@@ -522,7 +522,7 @@ class RepoAction(Action):
522
522
  "error": "The repository to be added is not specified"}
523
523
 
524
524
  i_repo_path = run_args['repo'] # can be a path, forder_name or URL
525
- repo_folder_name = os.path.basename(i_repo_path.rstrip('/'))
525
+ repo_folder_name = os.path.basename(i_repo_path.rstrip('/\\'))
526
526
 
527
527
  repo_path = os.path.join(self.repos_path, repo_folder_name)
528
528
 
@@ -547,6 +547,9 @@ class RepoAction(Action):
547
547
  return res
548
548
  repo_folder_name = res['value']
549
549
  repo_path = os.path.join(self.repos_path, repo_folder_name)
550
+ elif os.path.isabs(i_repo_path):
551
+ return {'return': 1,
552
+ "error": f"Path {i_repo_path} does not exist or is not accessible"}
550
553
 
551
554
  os.makedirs(repo_path)
552
555
  else: