vaultlayer 0.1.76__tar.gz → 0.1.78__tar.gz

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vaultlayer
-Version: 0.1.76
+Version: 0.1.78
 Summary: AI compute arbitrage CLI — move GPU training jobs between clouds automatically
 Author: VaultLayer
 License: MIT

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "vaultlayer"
 
-version = "0.1.76"
+version = "0.1.78"
 
 description = "AI compute arbitrage CLI — move GPU training jobs between clouds automatically"
 readme = "README.md"

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/agents/broker/startup_scripts.py

@@ -1082,11 +1082,77 @@ def _checkpoint_helper_lines(job: "Job", dest_dir_expr: str) -> list[str]:
 
 _FRAMEWORK_TRAINER_IMPORTS = ("transformers", "trl", "pytorch_lightning", "lightning")
 
+# Launchers that start an HF/TRL/Lightning Trainer in a SUBPROCESS. The trainer
+# import never appears in the script we scan (`axolotl train cfg.yml` reaches
+# transformers via `accelerate launch` -> `python -m axolotl.cli.train`), so
+# detected_imports and the inline-script scan both miss it, no hook is shipped,
+# and resume silently restarts at step 0 with checkpoints sitting in R2. The
+# import scan alone is only correct when the user's script IS the trainer.
+_FRAMEWORK_TRAINER_LAUNCHERS = frozenset({
+    "axolotl", "accelerate", "deepspeed", "torchrun", "llamafactory-cli",
+})
+
+# `-m` modules whose ROOT package is too broad to match on. `torchrun` is a thin
+# console-script over `torch.distributed.run`, and both spellings are common, but
+# matching the root "torch" would ship the hook for every `python -m torch.*`
+# (e.g. torch.utils.bottleneck). Matched in FULL, never by root.
+_FRAMEWORK_TRAINER_LAUNCH_MODULES = frozenset({
+    "torch.distributed.run",
+    "torch.distributed.launch",   # deprecated alias, still widely used
+})
+
+
+def _command_uses_framework_launcher(command: str) -> bool:
+    """True if ``command`` invokes a known subprocess trainer launcher.
+
+    Matches on the executable BASENAME (so ``/usr/local/bin/accelerate`` counts),
+    on the ROOT of a ``-m`` module (``python -m axolotl.cli.train`` -> axolotl),
+    and on a FULL ``-m`` module path for launchers whose root is too broad to
+    match (``python -m torch.distributed.run``). Flags are skipped, so
+    ``python train.py --deepspeed ds.json`` does NOT match on the flag, and a
+    script merely NAMED after a launcher (``python axolotl_test.py``) does not
+    match either -- both would be false positives that ship the ~20KB hook to
+    jobs that cannot use it.
+    """
+    # Guard non-str BEFORE shlex: shlex.split() treats a non-str as a file-like
+    # stream, reads chars off it, and dies deep inside on `nextchar in whitespace`
+    # (TypeError, not ValueError). job.command is "" on older rows and a Mock in
+    # tests that build scripts from stubbed jobs.
+    if not isinstance(command, str) or not command:
+        return False
+    try:
+        tokens = shlex.split(command)
+    except ValueError:              # unbalanced quotes -- degrade, never raise
+        tokens = command.split()
+    expect_module = False
+    for tok in tokens:
+        if expect_module:
+            expect_module = False
+            if tok in _FRAMEWORK_TRAINER_LAUNCH_MODULES:
+                return True
+            if tok.split(".", 1)[0] in _FRAMEWORK_TRAINER_LAUNCHERS:
+                return True
+            continue
+        if tok == "-m":
+            expect_module = True
+            continue
+        if tok.startswith("-"):
+            continue
+        # Node paths are posix; rsplit keeps this correct when the broker runs
+        # on Windows (os.path.basename would not split "/" there).
+        name = tok.replace("\\", "/").rsplit("/", 1)[-1]
+        if name in _FRAMEWORK_TRAINER_LAUNCHERS:
+            return True
+    return False
+
 
 def _job_uses_framework_trainer(job: "Job") -> bool:
-    """True if the job's script uses an HF/TRL/Lightning Trainer (so it needs the
-    resume hook materialized). Driven by runtime_spec.detected_imports; falls back
-    to an inline-script scan when no runtime_spec is present."""
+    """True if the job runs an HF/TRL/Lightning Trainer (so it needs the resume
+    hook materialized). Checks the launcher command FIRST -- a subprocess launcher
+    imports the trainer out of view of both scans below -- then falls back to
+    runtime_spec.detected_imports, then to an inline-script scan."""
+    if _command_uses_framework_launcher(getattr(job, "command", "") or ""):
+        return True
     spec = _extract_runtime_spec(job)
     if spec is not None:
         detected = set(spec.detected_imports or [])
@@ -2851,7 +2917,7 @@ def build_vm_startup_script(broker, job: "Job", *, launch_provider: str = "") ->
         "touch $_VL_LOG",
         "echo '[VL] VM bootstrap started' | tee -a $_VL_LOG",
         # Ubuntu 24.04 ships only python3; ensure 'python' resolves so users can
-        # run `vaultlayer run python train.py` without getting exit code 127.
+        # run `vaultlayer run train.py` without getting exit code 127.
         "python --version >/dev/null 2>&1 || "
         "ln -sf \"$(command -v python3)\" /usr/local/bin/python 2>/dev/null || true",
     ]

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/agents/tester/agent.py

@@ -10,7 +10,7 @@ PROBLEM_STATEMENT = """
 VaultLayer wraps AI training commands and provides:
 1. Cost savings: 60-70% cheaper than AWS On-Demand by routing to Lambda Labs/CoreWeave
 2. Reliability: 99.9% job completion — detect Spot terminations, checkpoint, auto-resume
-3. Zero code changes: user just runs 'vaultlayer run python train.py'
+3. Zero code changes: user just runs 'vaultlayer run train.py'
 4. Autonomous: an Orchestration Agent (Claude-powered) makes all decisions
 5. Checkpoints: stored in Cloudflare R2, restorable on any provider
 """

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/api/email.py

@@ -81,7 +81,7 @@ def _build_welcome_bodies() -> tuple[str, str, str]:
         f"   Add them at {dash}/dashboard/billing (or run `vl credits buy`).\n"
         "\n"
         "5. Run your first job\n"
-        "   vl run python train.py\n"
+        "   vl run train.py\n"
         "\n"
         f"Open {dash}/dashboard to watch your jobs, GPUs, and balance.\n"
         "\n"
@@ -112,9 +112,9 @@ def _build_welcome_bodies() -> tuple[str, str, str]:
         <a href="{dash}/dashboard/billing">billing</a> page (the CLI equivalent is
         <code>vl credits buy</code>).</p>
 
-        <p><strong>5. Run your first job</strong> — just put <code>vl run</code> in
-        front of your normal training command:</p>
-        <pre style="{_code}">vl run python train.py</pre>
+        <p><strong>5. Run your first job</strong> — just point <code>vl run</code>
+        at your training script:</p>
+        <pre style="{_code}">vl run train.py</pre>
 
         <p><a href="{dash}/dashboard">Open your dashboard</a> to watch your jobs,
         GPUs, and balance.</p>
@@ -285,7 +285,14 @@ async def send_topup_confirmation(
     package: str,
 ):
     """Sent after a successful Stripe payment."""
-    usd_paid = {"starter": 10, "growth": 50, "scale": 200}.get(package, 0)
+    # "Amount paid": fixed packages have a set price — and it can't be back-derived
+    # from the credit count because package credits INCLUDE a bonus (growth $50→5,500,
+    # scale $200→24,000). A custom top-up has no bonus, so its dollars are exactly
+    # credits / CREDITS_PER_USD. Deriving from the package NAME alone returned $0 for
+    # every 'custom' purchase — i.e. every dashboard buy, the launch path.
+    usd_paid = {"starter": 10.0, "growth": 50.0, "scale": 200.0}.get(
+        package, credits_added / CREDITS_PER_USD
+    )
     await _send(
         to=email,
         subject=f"VaultLayer — {credits_added:,} credits added",
@@ -298,7 +305,7 @@ async def send_topup_confirmation(
             </tr>
             <tr>
                 <td style="padding:6px 16px 6px 0;color:#666;">Amount paid</td>
-                <td style="padding:6px 0;"><strong>${usd_paid}</strong></td>
+                <td style="padding:6px 0;"><strong>${usd_paid:,.2f}</strong></td>
             </tr>
             <tr>
                 <td style="padding:6px 16px 6px 0;color:#666;">Credits added</td>

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/api/job_routes.py

@@ -460,7 +460,19 @@ def _resolve_prior_submit_state(job_id: str) -> dict:
         if result.data:
             qs = result.data[0].get("queue_state")
             if isinstance(qs, dict):
-                return qs
+                # Durable queue_state persists credential material encrypted at rest:
+                # `environment` collapses to a single {"__vlenc_env__": "VLENC1:..."}
+                # entry, and registry_password to a "VLENC1:" string
+                # (shared.cred_at_rest.encrypt_queue_state_secrets, called on the
+                # write path). Returning it RAW made the environment inherit adopt
+                # that sentinel as if it were the user's env -- reproducing the exact
+                # resume failure the inherit exists to prevent, but only for
+                # DB-backed ancestors, i.e. after an API restart when _active_jobs no
+                # longer holds the prior leg. Idempotent on legacy plaintext rows;
+                # fails closed to {} on a decrypt error, which callers already treat
+                # as "not set".
+                from shared.cred_at_rest import decrypt_queue_state_secrets
+                return decrypt_queue_state_secrets(qs)
     except Exception:
         pass
     return {}
@@ -517,6 +529,85 @@ def _coerce_inherited_value(field: str, value):
         return None
 
 
+def _inherit_environment_from_chain(body, resume_chain: list[str]) -> bool:
+    """Carry the original leg's `--env` values onto a resume submit.
+
+    Neither carry layer covered `environment`, so every resume leg started with an
+    EMPTY env. A script reading `os.environ.get("MODEL_ID", <default>)` silently
+    fell back to its default, loaded a DIFFERENT model, then resumed from the prior
+    model's checkpoint -- surfacing as
+    `ValueError: loaded state dict contains a parameter group that doesn't match`.
+    Two models with compatible param groups would have resumed with NO error at all.
+    It also dropped HF_TOKEN (gated models 401 on every resume) and any
+    hyperparameters passed via --env.
+
+    Server-side is the ONLY possible carry path: `environment` is deliberately
+    excluded from _SAFE_JOB_RESPONSE_FIELDS (it holds HF_TOKEN), so `vl restart`
+    never sees it and cannot resend it. Inheriting here keeps the secret server-side.
+
+    NOT gated on `inherit_constraints`: `--no-carry` exists to drop stale
+    ROUTING/BUDGET constraints. Dropping the environment would strip MODEL_ID and
+    HF_TOKEN and break the run outright -- an escape hatch that bricks the job.
+
+    MERGED per key, NOT all-or-nothing. The original version bailed out whenever the
+    body carried ANY environment ("the caller specified one, honour it"), mirroring
+    the is_unset predicate _inherit_constraints_from_chain uses for scalars. That was
+    wrong, and it made this whole carry a no-op in production: cli/_remote.py scrapes
+    the user's SHELL on every submit, including `vl restart` --
+
+        _FORWARD_PREFIXES = ("VL_USER_", "WANDB_", "HF_", "HUGGING_FACE_")
+        _FORWARD_EXACT = {"MAX_STEPS", ..., "MODEL_ID"}
+
+    -- so a developer with HF_TOKEN exported sends environment={"HF_TOKEN": ...} on a
+    restart, the guard short-circuits, and MODEL_ID/MAX_STEPS never carry. Observed
+    live: a resumed leg loaded the script's default model and printed
+    "save_steps: 10 (from args) != 20 (from trainer_state.json)".
+
+    body-wins is right for a SCALAR constraint (max_cost_usd); it is wrong for a MAP.
+    One unrelated forwarded var must not discard every other key. So: inherited keys
+    fill gaps, and any key the body actually sent wins.
+    """
+    # The key under `environment` that holds the encrypted blob when queue_state was
+    # persisted with credentials encrypted at rest. Imported here (not module scope)
+    # to keep cred_at_rest off the import path of callers that never resume.
+    from shared.cred_at_rest import _ENV_SENTINEL
+
+    _incoming = getattr(body, "environment", None)
+    incoming = dict(_incoming) if isinstance(_incoming, dict) else {}
+    for prior in resume_chain:
+        state = _resolve_prior_submit_state(prior)
+        if state.get("byoc_cleanup_pending"):
+            # Its stored env was REPLACED with AWS termination-confirmation creds
+            # only (job_worker._byoc_cleanup_queue_state_for_unconfirmed_aws). It is
+            # non-empty but carries none of the user's --env, so stopping here would
+            # inherit a stripped snapshot instead of walking to a complete ancestor.
+            continue
+        env = state.get("environment")
+        if not isinstance(env, dict) or not env:
+            continue
+        # Defense in depth. _resolve_prior_submit_state decrypts the DB fallback, so
+        # this should not trigger -- but the encrypted shape is str->str and would
+        # sail through the sanitizer below, handing the resumed leg
+        # {"__vlenc_env__": "VLENC1:..."} instead of MODEL_ID/HF_TOKEN. If decryption
+        # is ever unavailable (key rotated away, legacy row), skip the leg and keep
+        # walking rather than inherit a sentinel.
+        if _ENV_SENTINEL in env:
+            continue
+        # A legacy or hand-edited queue_state could hold a shape no live submit can
+        # produce. Keep str->str only; never 500 on inherited data.
+        clean = {k: v for k, v in env.items()
+                 if isinstance(k, str) and k and isinstance(v, str)}
+        if not clean:
+            continue
+        if not any(k not in incoming for k in clean):
+            return False        # ancestor found, but it adds nothing new
+        # Body wins per key: a var the caller explicitly sent (or that the CLI
+        # forwarded from the shell) overrides the stored value for that key only.
+        body.environment = {**clean, **incoming}
+        return True
+    return False
+
+
 def _inherit_constraints_from_chain(body, resume_chain: list[str]) -> list[str]:
     """Fill unset routing/budget constraints on a resume submit from the chain.
 
@@ -1546,6 +1637,19 @@ async def submit_job(body: RunJobRequest, user: dict = Depends(require_auth)):
     # `vl restart --no-carry` sets inherit_constraints=False to resume with
     # fresh (unconstrained) routing — the escape hatch when a carried
     # constraint no longer fits.
+    #
+    # The environment carry is NOT gated on inherit_constraints: --no-carry drops
+    # stale ROUTING/BUDGET constraints, but dropping the env would strip MODEL_ID
+    # and HF_TOKEN and break the run outright. Without this, a resume leg ran with
+    # an EMPTY env, so a script's `os.environ.get("MODEL_ID", <default>)` silently
+    # loaded a different model and resumed from the prior model's checkpoint.
+    if body.resume_from_job_id and _resume_chain:
+        if _inherit_environment_from_chain(body, _resume_chain):
+            logger.info(
+                "[submit] resume leg merged environment from prior leg(s) %s "
+                "(%d vars total; keys only, never values -- HF_TOKEN lives here)",
+                _resume_chain, len(body.environment),
+            )
     if body.resume_from_job_id and _resume_chain and body.inherit_constraints:
         _inherited_constraints = _inherit_constraints_from_chain(body, _resume_chain)
         if "preferred_providers" in _inherited_constraints:

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/cli/_preflight.py

@@ -210,6 +210,143 @@ _HF_TRAINER_CLASSES = frozenset({"Trainer", "Seq2SeqTrainer"}) | _TRL_TRAINER_CL
 _TRAINER_MODULES = ("transformers", "trl")
 
 
+# Launcher-driven frameworks: the user hands over a config and the launcher starts
+# the real trainer in a SUBPROCESS. There is no user training loop to instrument,
+# so the AST injector correctly refuses -- but it then tells the user to "add the
+# manual 3-line integration", which is impossible: an Axolotl YAML has nowhere to
+# put restore()/checkpoint(). Mirrors the broker-side launcher set that decides
+# whether to ship the resume hook (agents/broker/startup_scripts.py).
+_FRAMEWORK_LAUNCHERS = frozenset({
+    "axolotl", "accelerate", "deepspeed", "torchrun", "llamafactory-cli",
+})
+
+# Of the launchers above, ONLY these own the training loop themselves: the user
+# supplies a YAML/config and the framework runs (and checkpoints) the Trainer. A
+# zero_code verdict is safe for them because there is no user loop to instrument.
+#
+# torchrun / accelerate / deepspeed are GENERIC process launchers. They own
+# nothing -- `torchrun train.py` may front a raw PyTorch loop that still needs the
+# injector. Classifying those zero_code told the user "checkpointing handled by
+# VaultLayer" and skipped injection, so NO checkpoints were ever written and a
+# preemption lost the whole run. The _has_injectable_loop() guard did not save it:
+# a loop whose optimizer.step() sits in a helper function has no .step()/.backward()
+# in the for-body, so the guard reads False. A generic launcher must fall through to
+# _detect_hf_trl_trainer / Lightning (which already classify `accelerate launch
+# train.py` correctly when train.py uses HF Trainer) and otherwise stay "missing",
+# where the user is at least WARNED that integration is absent.
+#
+# NOTE: the broker's launcher set (agents/broker/startup_scripts.py) deliberately
+# stays wider -- shipping the resume hook to a generic launcher is a harmless
+# superset. The two sets are not interchangeable; do not "consolidate" them.
+_LAUNCHER_OWNS_TRAINING_LOOP = frozenset({"axolotl", "llamafactory-cli"})
+# `-m` modules whose ROOT package is too broad to match on ("torch").
+_FRAMEWORK_LAUNCH_MODULES = {
+    "torch.distributed.run": "torchrun",
+    "torch.distributed.launch": "torchrun",
+}
+_LAUNCHER_DISPLAY = {
+    "axolotl": "Axolotl", "accelerate": "Accelerate", "deepspeed": "DeepSpeed",
+    "torchrun": "torchrun", "llamafactory-cli": "LLaMA-Factory",
+}
+
+
+def _command_launcher_name(command) -> str:
+    """Launcher name if ``command`` invokes one, else "". Matches the executable
+    basename, a ``-m`` module root, and the full ``-m`` path for torch.distributed.*.
+    Flags are skipped, so ``python train.py --deepspeed ds.json`` does not match,
+    and a script merely NAMED after a launcher (``python axolotl_test.py``) does not
+    either -- it is an argument, not the executable."""
+    expect_module = False
+    for tok in command or ():
+        if not isinstance(tok, str):
+            continue
+        if expect_module:
+            expect_module = False
+            if tok in _FRAMEWORK_LAUNCH_MODULES:
+                return _FRAMEWORK_LAUNCH_MODULES[tok]
+            root = tok.split(".", 1)[0]
+            if root in _FRAMEWORK_LAUNCHERS:
+                return root
+            continue
+        if tok == "-m":
+            expect_module = True
+            continue
+        if tok.startswith("-"):
+            continue
+        name = tok.replace("\\", "/").rsplit("/", 1)[-1]
+        if name in _FRAMEWORK_LAUNCHERS:
+            return name
+    return ""
+
+
+def _script_subprocess_launcher(src: str) -> str:
+    """Launcher name if the script shells out to one (``subprocess.call(["axolotl",
+    "train", cfg])``), else "". Covers the wrapper pattern, where the command is a
+    plain ``python wrapper.py`` and the launcher only appears at runtime.
+
+    Delegates the whole literal argv to _command_launcher_name rather than testing
+    argv[0]: the launcher is not always the executable. `subprocess.call(["python",
+    "-m", "torch.distributed.run", ...])` puts it in the `-m` module, and
+    `[sys.executable, "-m", ...]` hides argv[0] behind a non-literal entirely.
+    Non-literal elements are dropped, so a `-m` whose module is a variable simply
+    does not match.
+
+    Both subprocess arg shapes are handled: a literal argv list, and the string form
+    used with shell=True (`subprocess.run("axolotl train cfg.yml", shell=True)`),
+    which is shlex-split first -- otherwise it arrives as one unsplit token whose
+    "basename" is the entire command line and can never match a launcher.
+    """
+    import ast
+    import shlex
+    try:
+        tree = ast.parse(src)
+    except SyntaxError:
+        return ""
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call) or not node.args:
+            continue
+        fn = node.func
+        if not (isinstance(fn, ast.Attribute) and isinstance(fn.value, ast.Name)
+                and fn.value.id == "subprocess"):
+            continue
+        arg0 = node.args[0]
+        if isinstance(arg0, ast.Constant) and isinstance(arg0.value, str):
+            try:
+                argv = shlex.split(arg0.value)
+            except ValueError:          # unbalanced quotes -- degrade, never raise
+                argv = arg0.value.split()
+        else:
+            elts = arg0.elts if isinstance(arg0, (ast.List, ast.Tuple)) else [arg0]
+            argv = [e.value for e in elts
+                    if isinstance(e, ast.Constant) and isinstance(e.value, str)]
+        name = _command_launcher_name(argv)
+        if name:
+            return name
+    return ""
+
+
+def _has_injectable_loop(src: str) -> bool:
+    """True if the script owns a training loop the AST injector can instrument.
+
+    Load-bearing guard: `vl run torchrun train.py` names a launcher but train.py may
+    be a raw-PyTorch loop that still NEEDS injection. Classifying it zero_code would
+    silently strip its checkpointing -- worse than a misleading message.
+    """
+    import ast
+    try:
+        tree = ast.parse(src)
+    except SyntaxError:
+        return False
+    for node in ast.walk(tree):
+        if not isinstance(node, (ast.For, ast.While)):
+            continue
+        for sub in ast.walk(node):
+            if (isinstance(sub, ast.Call) and isinstance(sub.func, ast.Attribute)
+                    and sub.func.attr in ("step", "backward")):
+                return True
+    return False
+
+
 def _detect_hf_trl_trainer(src: str) -> "tuple[bool, bool]":
     """``(uses_trainer, is_trl)`` from AST — REAL Trainer imports/instantiation.
 
@@ -461,7 +598,16 @@ def check_script_integration(command: list[str]) -> dict:
             script_path = pathlib.Path(token)
             break
 
+    # `axolotl train cfg.yml` carries no .py for us to scan, but the framework CLI
+    # itself proves a framework owns the loop and its checkpointing. A GENERIC
+    # launcher proves nothing (see _LAUNCHER_OWNS_TRAINING_LOOP) -- `torchrun
+    # train.py` with an unresolvable path stays "unknown", as it did before.
+    _cmd_launcher = _command_launcher_name(command)
+
     if script_path is None or not script_path.exists():
+        if _cmd_launcher in _LAUNCHER_OWNS_TRAINING_LOOP:
+            result["status"] = "zero_code"
+            result["framework"] = _LAUNCHER_DISPLAY.get(_cmd_launcher, _cmd_launcher)
         return result
 
     result["script"] = script_path.name
@@ -532,6 +678,17 @@ def check_script_integration(command: list[str]) -> dict:
         )
         return result
 
+    # Framework-CLI driven: either the command IS one, or the script shells out to
+    # one. Restricted to _LAUNCHER_OWNS_TRAINING_LOOP -- a generic launcher over a
+    # raw PyTorch loop must reach the injector (or its warning), never a silent
+    # zero_code. _has_injectable_loop stays as defense in depth: if a script somehow
+    # owns a loop AND shells out to a framework CLI, prefer instrumenting the loop.
+    _launcher = _cmd_launcher or _script_subprocess_launcher(src)
+    if _launcher in _LAUNCHER_OWNS_TRAINING_LOOP and not _has_injectable_loop(src):
+        result["status"] = "zero_code"
+        result["framework"] = _LAUNCHER_DISPLAY.get(_launcher, _launcher)
+        return result
+
     if "import jax" in lines or "import flax" in lines or "from flax" in lines:
         result["status"] = "missing"
         result["framework"] = "JAX/Flax"
@@ -754,14 +911,29 @@ def print_preflight(run_cmd: "RunCommand", yes: bool = False, train_mode: str =
                 (G, Y, R, B, BO),
             )
         elif _hint:
-            # Checkpoint saving is automatic; resume requires one line in the script.
-            _example = integration.get("resume_example") or (
-                f"os.environ.get('{_hint}')"
-            )
-            integ_line = (
-                f"{G}{_fw} — checkpoints auto-saved ✓{R}  "
-                f"{Y}(for resume: {_example}){R}"
-            )
+            # The framework (HF Trainer / Lightning) already saves checkpoints — VaultLayer
+            # syncs them to R2 and, on single-node, auto-resumes from the last one with NO
+            # code change (the runtime hook resolves the checkpoint from output_dir even when
+            # VAULTLAYER_RESUME_CHECKPOINT is unset). Do NOT tell the user to add a resume
+            # line there — the old "(for resume: trainer.train(resume_from_checkpoint=os.
+            # environ.get(...)))" hint was a footgun: on a restart/local leg that env var is
+            # None, which trained fresh AND suppressed the auto-resume hook. Multi-node does
+            # NOT arm the hook (it is _SINGLE_NODE-gated), so there the explicit hand-off IS
+            # still required — surface it only in that case.
+            _nnodes = int(getattr(run_cmd, "world_nnodes", 1) or 1)
+            if _nnodes > 1:
+                _example = integration.get("resume_example") or (
+                    f"os.environ.get('{_hint}')"
+                )
+                integ_line = (
+                    f"{G}{_fw} — checkpoints auto-saved ✓{R}  "
+                    f"{Y}(multi-node resume: {_example}){R}"
+                )
+            else:
+                integ_line = (
+                    f"{G}{_fw} — checkpoints auto-saved ✓{R}  "
+                    f"{Y}(VaultLayer resumes from your last saved checkpoint automatically){R}"
+                )
         else:
             integ_line = f"{G}{_fw} — zero-code ✓{R}"
     elif _st == "missing":

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/cli/_remote.py

@@ -221,13 +221,19 @@ _ALLOWED_REMOTE_MODULE_LAUNCHERS = {
 
 
 def _normalize_bare_python_script_command(command: list[str]) -> list[str]:
-    """Run bare local Python scripts through the interpreter on remote nodes.
-
-    Users naturally type `vl run train.py`. The GPU node receives the script as
-    a regular uploaded file, not an executable with a guaranteed shebang/chmod,
-    so the remote command must be `python train.py`. Explicit launchers such as
-    `python -u train.py`, `torchrun ... train.py`, and `accelerate launch ...`
-    are already intentional and must remain unchanged.
+    """Run bare local Python scripts through the interpreter.
+
+    Users naturally type `vl run train.py`. Neither end of the run can execute
+    that directly: the GPU node receives the script as a regular uploaded file,
+    not an executable with a guaranteed shebang/chmod, and the local-execution
+    path spawns argv via create_subprocess_exec. Both need `python train.py`.
+    Explicit launchers such as `python -u train.py`, `torchrun ... train.py`,
+    and `accelerate launch ...` are already intentional and must remain
+    unchanged.
+
+    Applied at RunCommand construction (cli/run.py), so callers downstream
+    already see a normalized command. Idempotent — re-normalizing a command
+    that already starts with an interpreter is a no-op.
     """
     if not command:
         return []
@@ -478,11 +484,11 @@ async def run_remote(
             raise SystemExit(
                 f"vaultlayer run: script file not found: {py_args[0]}\n"
                 "Run from the directory containing the script, or pass the path, "
-                f"for example: vaultlayer run python path/to/{Path(py_args[0]).name}"
+                f"for example: vaultlayer run path/to/{Path(py_args[0]).name}"
             )
         raise SystemExit(
             "vaultlayer run: managed jobs require one local .py training script, "
-            "for example: vaultlayer run python train.py"
+            "for example: vaultlayer run train.py"
         )
     _workspace_paths = _local_paths_requiring_workspace(command, script_arg_index)
     if _workspace_paths:
@@ -1173,8 +1179,8 @@ async def run_remote(
             print(f"\n  ⚠  No {gpu_str} found after 15 minutes.")
             print(f"  Options:")
             print(f"    • Press Ctrl+C to cancel (no charge) and retry with a different GPU:")
-            print(f"        vaultlayer run --gpu A100_40GB python train.py ...")
-            print(f"        vaultlayer run --gpu A100_80GB python train.py ...")
+            print(f"        vaultlayer run --gpu A100_40GB train.py ...")
+            print(f"        vaultlayer run --gpu A100_80GB train.py ...")
             print(f"    • Keep waiting — job will start automatically if capacity opens\n")
             _queue_warned = True
             sys.stdout.flush()

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/cli/connect.py

@@ -696,7 +696,7 @@ def connect_aws(role_arn, external_id, access_key, secret_key, region):
         _append_to_config_env(config_env, "AWS_REGION", region or "us-east-1")
 
     click.echo("  AWS connected.")
-    click.echo("  Run: vaultlayer run python train.py")
+    click.echo("  Run: vaultlayer run train.py")
 
 
 @connect.command(name="gcp")
@@ -809,7 +809,7 @@ def connect_gcp(service_account_json, project_id, region):
     _append_to_config_env(config_env, "GCP_REGION", region)
 
     click.echo("  GCP connected.")
-    click.echo("  Run: vaultlayer run python train.py")
+    click.echo("  Run: vaultlayer run train.py")
 
 
 @connect.command(name="azure")
@@ -926,7 +926,7 @@ def connect_azure(subscription_id, tenant_id, client_id, client_secret, resource
     _append_to_config_env(config_env, "AZURE_LOCATION", location)
 
     click.echo("  Azure connected.")
-    click.echo("  Run: vaultlayer run python train.py")
+    click.echo("  Run: vaultlayer run train.py")
 
 
 @connect.command(name="storage")
@@ -995,7 +995,7 @@ def connect_storage(provider):
         _append_to_config_env(config_env, "VL_S3_BUCKET", bucket)
 
         click.echo("  S3 storage connected.")
-        click.echo("  Run: vaultlayer run python train.py")
+        click.echo("  Run: vaultlayer run train.py")
 
     elif provider == "gcs":
         click.echo("")

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/cli/data.py

@@ -94,7 +94,7 @@ def datasets(ctx, delete):
     Upload a dataset:
         vaultlayer sync ./data --dataset-id my-dataset
     Use in training:
-        vaultlayer run --data r2://my-dataset python train.py
+        vaultlayer run --data r2://my-dataset train.py
     Delete a dataset:
         vaultlayer datasets delete <dataset-id>
     """
@@ -228,9 +228,9 @@ def regions_list_all():
         click.echo(f"  {code:<20} {note}")
 
     click.echo("\n  To restrict your job to specific regions:")
-    click.echo("    vaultlayer run --regions eu-central-1,eu-west-1 python train.py")
+    click.echo("    vaultlayer run --regions eu-central-1,eu-west-1 train.py")
     click.echo("  To exclude a region:")
-    click.echo("    vaultlayer run --excluded-regions cn-north-1 python train.py\n")
+    click.echo("    vaultlayer run --excluded-regions cn-north-1 train.py\n")
 
 
 @regions.command(name="current")

{vaultlayer-0.1.76 → vaultlayer-0.1.78}/src/cli/estimate.py

@@ -405,7 +405,7 @@ def estimate(script, gpu, hours, gpus, data, data_gb, checkpoint_gb, model_param
                 else:
                     click.echo(f"  (band spans model sizes — larger models run longer; pass --model-params for a size-specific band)")
 
-    click.echo(f"  Run:  vaultlayer run --gpu {gpu} python train.py")
+    click.echo(f"  Run:  vaultlayer run --gpu {gpu} train.py")
     click.echo("")