freesolo-flash-dev 1.2.132__tar.gz → 1.2.133__tar.gz

{freesolo_flash_dev-1.2.132 → freesolo_flash_dev-1.2.133}/.github/workflows/ci.yml

@@ -60,8 +60,7 @@ jobs:
       - name: Function size check
         run: uv run python scripts/check_function_size.py
 
-      # Keep the source hierarchy reviewable: no directory may exceed ten tracked files, and package
-      # roots may mix subdirectories only with the three required marker files.
+      # Package roots may mix subdirectories only with semantic package marker files.
       - name: Source layout check
         run: uv run python scripts/check_source_layout.py
 

{freesolo_flash_dev-1.2.132 → freesolo_flash_dev-1.2.133}/.github/workflows/deploy-modal-dev.yml

@@ -114,7 +114,7 @@ jobs:
           # pinned to the exact client pyproject declares; see deploy-modal.yml for why an
           # unbounded range is unsafe for the deploy path.
           # exact pins keep the deploy client bootstrap reproducible across runner image changes.
-          uv pip install --system "modal==1.5.4" "python-dotenv==1.2.3"
+          uv pip install --system "modal==1.6.1" "python-dotenv==1.2.3"
 
       - name: Verify development deployment inputs
         if: steps.gate.outputs.deploy == 'true'

{freesolo_flash_dev-1.2.132 → freesolo_flash_dev-1.2.133}/.github/workflows/deploy-modal.yml

@@ -11,6 +11,7 @@ name: Deploy serving Modal app
 #   FREESOLO_INTERNAL_KEY                - shared platform internal key (backend + serving)
 #   PLATFORM_BACKEND_URL                 - platform backend base URL
 #   SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY - serving startup hydration (durable adapter registry)
+# required Actions variable: SERVING_CANARY_ORG_ID - existing internal organization billed for probes
 # modal_app.py reads these from the environment (load_dotenv only supplements os.environ), so no
 # .env file is written in CI. the "Verify required runtime secrets" step below hard-fails the deploy
 # if a required secret is absent, preventing a broken production config with external chat failures,
@@ -40,13 +41,13 @@ jobs:
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     # the job's own bounded waits now sum well past the old 30: two 300s readiness polls, the remote
-    # image build, a 420s canary that pays a GPU cold start, a 180s accounting deadline, and on the
-    # failure path a SECOND `modal deploy` plus a 300s restore poll. 60 minutes clears that worst
+    # image build, a 3360s canary that pays a GPU cold start, a 180s accounting deadline, and on the
+    # failure path a SECOND `modal deploy` plus a 300s restore poll. 90 minutes clears that worst
     # case with room while keeping a wedged deploy from holding the runner for GitHub's 6-hour
     # default. the headroom is specifically for the rollback: a job killed partway through the
     # restore leaves the broken release live with no second attempt, which is the one outcome this
     # workflow exists to prevent.
-    timeout-minutes: 60
+    timeout-minutes: 90
     # serialize serving deploys across runs because each deploy mutates the live app. github can still
     # replace a pending run, so the gate compares against the sha reported by the deployed app rather
     # than assuming every push predecessor was deployed.
@@ -75,6 +76,7 @@ jobs:
       # (empty => default *.modal.run url), so non-prod/fork/local workspaces stay deployable. This
       # domain is verified in the production Modal workspace, so prod keeps serving on serve.freesolo.co.
       SERVING_CUSTOM_DOMAIN: serve.freesolo.co
+      SERVING_CANARY_ORG_ID: ${{ vars.SERVING_CANARY_ORG_ID }}
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -168,7 +170,7 @@ jobs:
           # pinned to the exact client pyproject declares and the provisioning path was validated
           # against: deploy behaviour (app lifecycle, tags, lookup) is client-version sensitive, so
           # an unbounded range lets a future release change deploys with no diff here.
-          python -m pip install "modal==1.5.4" "python-dotenv==1.2.3"
+          python -m pip install "modal==1.6.1" "python-dotenv==1.2.3"
 
       - name: Verify Modal auth is configured
         if: steps.gate.outputs.deploy == 'true'
@@ -198,7 +200,7 @@ jobs:
           if [ -z "$SUPABASE_SERVICE_ROLE_KEY" ]; then
             missing="$missing SUPABASE_SERVICE_ROLE_KEY"
           fi
-          required="FREESOLO_INTERNAL_KEY PLATFORM_BACKEND_URL"
+          required="FREESOLO_INTERNAL_KEY PLATFORM_BACKEND_URL SERVING_CANARY_ORG_ID"
           for name in $required; do
             # indirect expansion: the secrets are exported into the env via the job-level `env:` block.
             if [ -z "${!name}" ]; then
@@ -206,10 +208,21 @@ jobs:
             fi
           done
           if [ -n "$missing" ]; then
-            echo "::error::Refusing to deploy serving to prod: missing required secret(s):${missing}. Set them in repo Settings -> Secrets and variables -> Actions."
+            echo "::error::Refusing to deploy serving to prod: missing required setting(s):${missing}. Set them in repo Settings -> Secrets and variables -> Actions."
             exit 1
           fi
-          echo "All required serving runtime secrets are present."
+          # reject malformed attribution before replacing the live app.
+          python - <<'PY'
+          import os
+          import sys
+          import uuid
+
+          try:
+              uuid.UUID(os.environ["SERVING_CANARY_ORG_ID"].strip())
+          except ValueError:
+              sys.exit("::error::SERVING_CANARY_ORG_ID must be an existing internal organization UUID.")
+          PY
+          echo "All required serving runtime secrets and canary settings are present."
 
       - name: Wait for production adapter schema readiness
         if: steps.gate.outputs.deploy == 'true'
@@ -411,6 +424,7 @@ jobs:
           # durably. this step is the only thing standing between "the process booted" and "the
           # release can actually serve", so a failure here must not be promoted.
           SERVING_BASE_URL: https://serve.freesolo.co
+          SERVING_CANARY_ORG_ID: ${{ vars.SERVING_CANARY_ORG_ID }}
           FREESOLO_INTERNAL_KEY: ${{ secrets.FREESOLO_INTERNAL_KEY }}
           SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
           SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
@@ -456,10 +470,20 @@ jobs:
           echo "Promotion failed; restoring ${PREVIOUS_SHA}."
           git checkout --quiet --force "$PREVIOUS_SHA"
 
+          # resolve the entrypoint in the restored tree, including releases before the source move.
+          if [ -f flash/serving/app/modal_app.py ]; then
+            rollback_app=flash/serving/app/modal_app.py
+          elif [ -f flash/serving/modal_app.py ]; then
+            rollback_app=flash/serving/modal_app.py
+          else
+            echo "::error::Previous release has no supported serving entrypoint. Restore production manually."
+            exit 1
+          fi
+
           # the restored app must report the RESTORED sha, deployed from the restored tree.
           FREESOLO_DEPLOYMENT_SHA="$PREVIOUS_SHA" \
           FREESOLO_DEPLOYMENT_ID="$ROLLBACK_DEPLOYMENT_ID" \
-            modal deploy flash/serving/app/modal_app.py
+            modal deploy "$rollback_app"
 
           response_file="$(mktemp)"
           trap 'rm -f "$response_file"' EXIT

{freesolo_flash_dev-1.2.132 → freesolo_flash_dev-1.2.133}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.132
+Version: 1.2.133
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

freesolo_flash_dev-1.2.133/flash/adapters/config.py

@@ -0,0 +1,136 @@
+"""one strict reader for the bytes of an ``adapter_config.json``.
+
+Three boundaries admit an adapter by reading this one file, and all three are gated on the same
+question: do these exact bytes describe a LoRA adapter that the engine about to load them can
+actually serve? Answering it separately let them drift -- customer-owned resolution rejected a
+config that hosted admission accepted, so the same artifact was refused before provisioning on one
+path and refused inside the paid container on the other.
+
+Every rule here is decidable from the config alone. Nothing in this module reads the hub, the
+filesystem, or a provider, so a verdict costs a caller nothing beyond the bytes it already holds.
+That is why it sits beside the rank and target rules rather than under a deployment path: the
+GPU-side materializer revalidates a cache entry it already holds, and it needs the same verdict
+without importing anything that resolves or deploys.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from flash.adapters.lora_rank import rank_from_adapter_config
+from flash.serve.contract.protocol import reject_non_finite_json_constant
+
+
+class AdapterConfigError(ValueError):
+    """these ``adapter_config.json`` bytes cannot describe a servable LoRA adapter."""
+
+
+@dataclass(frozen=True)
+class DeclaredAdapterConfig:
+    """what an adapter stamps about itself, after every deterministic rule has passed."""
+
+    config: dict[str, Any]
+    lora_rank: int
+    base_model: str
+    base_revision: str | None
+
+
+class _DuplicateConfigKey(AdapterConfigError):
+    """raised through ``json.loads``, so it stays a ValueError for any caller expecting one."""
+
+
+def parse_declared_adapter_config(raw: bytes, *, source: str) -> DeclaredAdapterConfig:
+    """read one adapter's declared provenance, or refuse the bytes.
+
+    ``source`` names the file in every message, so a caller that fetched the config from the hub and
+    one that read it off disk both report a location the reader can go look at.
+    """
+
+    config = _strict_object(raw, source=source)
+    try:
+        rank = rank_from_adapter_config(config, source=source)
+    except ValueError as exc:
+        raise AdapterConfigError(f"{source} declares no usable lora rank: {exc}") from exc
+
+    peft_type = config.get("peft_type")
+    if peft_type != "LORA":
+        raise AdapterConfigError(f"{source} peft_type must be LORA, not {peft_type!r}")
+    task_type = config.get("task_type")
+    if task_type not in {None, "CAUSAL_LM"}:
+        raise AdapterConfigError(
+            f"{source} task_type must be absent or CAUSAL_LM, not {task_type!r}"
+        )
+    modules_to_save = config.get("modules_to_save")
+    if modules_to_save is not None and modules_to_save != []:
+        raise AdapterConfigError(f"{source} modules_to_save adapters are not supported")
+
+    # the engine compares `base_model_name_or_path` for equality, so an absent, empty, or
+    # non-string one can never match and the deployment is already doomed. skipping the check
+    # instead deferred a certain failure until after a provider had allocated and started billing.
+    base_model = _text(config, "base_model_name_or_path", source=source)
+    if base_model is None:
+        raise AdapterConfigError(f"{source} declares no base_model_name_or_path")
+    revision = config.get("revision")
+    if revision is not None and not isinstance(revision, str):
+        raise AdapterConfigError(f"{source} revision must be a string when present")
+    return DeclaredAdapterConfig(
+        config=config,
+        lora_rank=rank,
+        base_model=base_model,
+        base_revision=_text(config, "revision", source=source),
+    )
+
+
+def _strict_object(raw: bytes, *, source: str) -> dict[str, Any]:
+    """decode exactly the way the gpu-side materializer does, from the same bytes.
+
+    handing the bytes straight to ``json.load`` instead lets it auto-detect utf-16 and accept a bom
+    (rfc 4627), so a config the container refuses outright parsed cleanly at admission -- and the
+    deployment failed only after the provider resources had been allocated. duplicate keys are the
+    same case: plain decoding takes the last value, so a config declaring ``r`` twice would be
+    admitted against one rank and then rejected inside the container this check exists to avoid
+    paying for.
+    """
+
+    try:
+        config = json.loads(
+            raw.decode("utf-8"),
+            object_pairs_hook=_reject_duplicate_keys,
+            parse_constant=reject_non_finite_json_constant,
+        )
+    except _DuplicateConfigKey as exc:
+        raise AdapterConfigError(f"{source} contains a duplicate key") from exc
+    except (UnicodeDecodeError, ValueError) as exc:
+        raise AdapterConfigError(f"{source} is not readable json") from exc
+    if not isinstance(config, dict):
+        raise AdapterConfigError(f"{source} must be a json object")
+    return config
+
+
+def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in result:
+            raise _DuplicateConfigKey(f"{key!r} appears twice")
+        result[key] = value
+    return result
+
+
+def _text(config: dict[str, Any], key: str, *, source: str) -> str | None:
+    """read a config string exactly as the engine will compare it.
+
+    the engine compares these raw bytes for equality, so a value that matches only after stripping
+    is admitted here and then rejected inside the paid container -- the outcome this reader exists
+    to prevent. normalizing the padding away instead would be worse for ``revision``, which the
+    resolver *adopts* into the immutable manifest: that would launder a padded string into the record
+    rather than surface it. rejecting keeps the admission-time and container-time verdicts identical.
+    """
+
+    value = config.get(key)
+    if not isinstance(value, str):
+        return None
+    if value != value.strip():
+        raise AdapterConfigError(f"{source} {key} has surrounding whitespace: {value!r}")
+    return value or None

{freesolo_flash_dev-1.2.132 → freesolo_flash_dev-1.2.133}/flash/engine/profiling/dataset_profile.py

@@ -189,96 +189,64 @@ def _selected_dataset_path(base_dir: Path, params: dict[str, Any]) -> Path:
     )
 
 
-def _read_jsonl_rows(path: Path, *, max_examples: int) -> tuple[int, list[dict[str, Any]]]:
-    source_examples = 0
-    values = []
-    missing = []
-    with path.open(encoding="utf-8") as handle:
-        for line in handle:
-            if not line.strip():
-                continue
-            row = json.loads(line)
-            if not isinstance(row, dict):
-                raise TypeError("dataset row is not an object")
-            if _CANONICAL_INPUT_KEY not in row and len(missing) < 10:
-                missing.append(source_examples)
-            source_examples += 1
-            if max_examples <= 0 or len(values) < max_examples:
-                values.append(row)
-    if missing:
-        raise ValueError(
-            "Freesolo dataset records must contain an input field; missing at row indexes "
-            f"{missing}"
-        )
-    return source_examples, values
-
-
 def _validate_dataset_rows(
     values: object,
     *,
     invalid_message: str,
-    empty_message: str,
     max_examples: int,
+    empty_message: str | None = None,
 ) -> tuple[int, list[dict[str, Any]]]:
     if not isinstance(values, list) or not all(isinstance(row, dict) for row in values):
         raise PackagedDatasetUnavailable(invalid_message)
-    source_examples = len(values)
-    missing = [index for index, row in enumerate(values) if _CANONICAL_INPUT_KEY not in row]
+    rows = values
+    source_examples = len(rows)
+    missing = [
+        index
+        for index, row in enumerate(rows)
+        if _CANONICAL_INPUT_KEY not in row or row.get(_CANONICAL_INPUT_KEY) is None
+    ]
     if missing:
         raise ValueError(
             "Freesolo dataset records must contain an input field; missing at row indexes "
             f"{missing[:10]}"
         )
-    selected = values[:max_examples] if max_examples > 0 else values
-    if not selected:
+    selected = rows[:max_examples] if max_examples > 0 else rows
+    if not selected and empty_message is not None:
         raise PackagedDatasetUnavailable(empty_message)
     return source_examples, [dict(row) for row in selected]
 
 
 def _read_dataset_rows(path: Path, *, max_examples: int) -> tuple[int, list[dict[str, Any]]]:
+    from freesolo.datasets.records import load_records
+
+    invalid_message = (
+        f"environment dataset file {path.name!r} must contain JSON object or string rows. Add a "
+        "valid dataset/train.jsonl to the environment package."
+    )
     try:
         if path.stat().st_size > _MAX_PROFILE_DATASET_BYTES:
             raise PackagedDatasetUnavailable(
                 f"environment dataset file {path.name!r} exceeds the 32 MiB control-plane profiling "
                 "limit. Reduce the packaged training dataset."
             )
-        if path.suffix.lower() == ".jsonl":
-            source_examples, values = _read_jsonl_rows(path, max_examples=max_examples)
-        elif path.suffix.lower() == ".json":
-            loaded = json.loads(path.read_text(encoding="utf-8"))
-            values = loaded.get("records") if isinstance(loaded, dict) else loaded
-            source_examples, values = _validate_dataset_rows(
-                values,
-                invalid_message=(
-                    f"environment dataset file {path.name!r} must contain JSON object rows. Add a "
-                    "valid dataset/train.jsonl to the environment package."
-                ),
-                empty_message=(
-                    f"environment dataset file {path.name!r} contains no rows. Add training rows to "
-                    "dataset/train.jsonl in the environment package."
-                ),
-                max_examples=max_examples,
-            )
-        else:
-            raise ValueError("dataset file must end in .jsonl or .json")
+        if path.suffix.lower() not in {".jsonl", ".json"}:
+            raise PackagedDatasetUnavailable("dataset file must end in .jsonl or .json")
+        # use the worker's decoder; it materializes all rows before profiling selects a prefix.
+        rows = load_records(path)
     except PackagedDatasetUnavailable:
         raise
-    except TypeError as exc:
-        raise PackagedDatasetUnavailable(
-            f"environment dataset file {path.name!r} must contain JSON object rows. Add a valid "
-            "dataset/train.jsonl to the environment package."
-        ) from exc
     except (OSError, UnicodeError, json.JSONDecodeError) as exc:
         raise PackagedDatasetUnavailable(
             f"environment dataset file {path.name!r} is not readable JSON. Add a valid "
             "dataset/train.jsonl to the environment package."
         ) from exc
-    if not values:
-        raise PackagedDatasetUnavailable(
-            f"environment dataset file {path.name!r} contains no rows. Add training rows to "
-            "dataset/train.jsonl in the environment package."
-        )
-    return source_examples, [dict(row) for row in values]
+    except (TypeError, ValueError) as exc:
+        raise PackagedDatasetUnavailable(invalid_message) from exc
+    return _validate_dataset_rows(
+        rows,
+        invalid_message=invalid_message,
+        max_examples=max_examples,
+    )
 
 
 def profile_packaged_sft_dataset(

freesolo_flash_dev-1.2.133/flash/providers/_lifecycle/bootstrapping/attempt.py

@@ -0,0 +1,92 @@
+"""Serialize bootstrap re-entry and retain the first terminal result on the worker."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import tempfile
+from pathlib import Path
+
+
+def _create_once(path: Path, value: dict) -> None:
+    """Publish complete json without replacing a concurrent writer's result."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=".result-")
+    try:
+        with os.fdopen(fd, "w") as stream:
+            json.dump(value, stream, sort_keys=True)
+            stream.flush()
+            os.fsync(stream.fileno())
+        with contextlib.suppress(FileExistsError):
+            os.link(temporary, path)
+    finally:
+        os.unlink(temporary)
+
+
+class AttemptState:
+    """Local state survives a provider restarting the same attempt's container command."""
+
+    def __init__(self, code_dir: str, payload: dict):
+        self.directory = Path(code_dir + ".bootstrap")
+        self.record_path = self.directory / "result.json"
+        self.marker_path = self.directory / "marker.json"
+        self.identity = {
+            key: payload.get(key)
+            for key in (
+                "run_id",
+                "attempt",
+                "flash_arm",
+                "phase",
+                "hf_repo",
+                "hf_prefix",
+                "source_snapshot",
+            )
+        }
+
+    @contextlib.contextmanager
+    def locked(self):
+        import fcntl
+
+        self.directory.mkdir(parents=True, exist_ok=True)
+        with (self.directory / "lock").open("a") as lock:
+            # wait for the owner to finish; returning early lets the launch script destroy its gpu.
+            try:
+                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                print("bootstrap: duplicate start waiting for attempt owner", flush=True)
+                fcntl.flock(lock, fcntl.LOCK_EX)
+            try:
+                yield
+            finally:
+                fcntl.flock(lock, fcntl.LOCK_UN)
+
+    def read(self) -> dict | None:
+        try:
+            record = json.loads(self.record_path.read_text())
+        except FileNotFoundError:
+            return None
+        if not isinstance(record, dict) or record.get("identity") != self.identity:
+            raise RuntimeError("persisted bootstrap result belongs to a different attempt payload")
+        marker = record.get("marker")
+        if (
+            not isinstance(marker, dict)
+            or type(marker.get("ok")) is not bool
+            or marker.get("run_id") != self.identity["run_id"]
+            or marker.get("attempt") != self.identity["attempt"]
+        ):
+            raise RuntimeError("persisted bootstrap result is invalid")
+        return marker
+
+    def save(self, marker: dict) -> dict:
+        # persist before network i/o, so a restart can retry publication without rerunning training.
+        _create_once(self.record_path, {"identity": self.identity, "marker": marker})
+        saved = self.read()
+        assert saved is not None
+        return saved
+
+    def publication_path(self, marker: dict) -> str:
+        _create_once(self.marker_path, marker)
+        if json.loads(self.marker_path.read_text()) != marker:
+            raise RuntimeError("persisted bootstrap marker does not match its terminal result")
+        return str(self.marker_path)

{freesolo_flash_dev-1.2.132 → freesolo_flash_dev-1.2.133}/flash/providers/_lifecycle/bootstrapping/bootstrap.py

@@ -23,6 +23,8 @@ if __package__:
     from flash.providers._lifecycle.bootstrapping import console as _bootstrap_console
     from flash.providers._lifecycle.bootstrapping import pip as bootstrap_pip
     from flash.providers._lifecycle.bootstrapping import processes as _bootstrap_processes
+    from flash.providers._lifecycle.bootstrapping.attempt import AttemptState
+    from flash.providers._lifecycle.bootstrapping.preload import run_preload
     from flash.providers._lifecycle.bootstrapping.secrets import (
         _payload_secrets,
         _read_console_tail,
@@ -36,6 +38,8 @@ else:
     import bootstrap_console as _bootstrap_console  # type: ignore[no-redef]
     import bootstrap_pip  # type: ignore[no-redef]
     import bootstrap_processes as _bootstrap_processes  # type: ignore[no-redef]
+    from bootstrap_attempt import AttemptState  # type: ignore[no-redef]
+    from bootstrap_preload import run_preload  # type: ignore[no-redef]
     from bootstrap_secrets import (  # type: ignore[no-redef]
         _payload_secrets,
         _read_console_tail,
@@ -800,9 +804,14 @@ def write_attempt_marker(payload: dict, ok: bool, error: str = "", retriable: bo
             run_id=run_id,
             attempt=attempt,
         )
-    p = "/tmp/attempt_marker.json"
-    with open(p, "w") as f:
-        json.dump(marker, f)
+    if payload.get("mode") == "preload":
+        p = "/tmp/attempt_marker.json"
+        with open(p, "w") as f:
+            json.dump(marker, f)
+    else:
+        state = AttemptState(_code_dir(payload), payload)
+        marker = state.save(marker)
+        p = state.publication_path(marker)
     hf_upload(
         payload,
         p,
@@ -832,68 +841,28 @@ def _arm_preload_wall_cap(payload: dict) -> tuple[threading.Timer, threading.Eve
     return timer, done
 
 
-def run_preload(payload: dict) -> dict:
-    """Download models into the bind-mounted weight cache. Fails if cache isn't mounted to avoid warming ephemeral disk."""
-    env = payload.get("env") or {}
-    cache_dir = env.get("FLASH_WEIGHT_CACHE_DIR") or ""
-    token = env.get("HF_TOKEN")
-    mount = os.path.dirname(os.path.dirname(cache_dir.rstrip("/"))) if cache_dir else ""
-    if not cache_dir or not mount or not os.path.isdir(mount):
-        return {
-            "preloaded": [],
-            "already_cached": [],
-            "failed": {},
-            "error": f"weight-cache not mounted (FLASH_WEIGHT_CACHE_DIR={cache_dir!r}); refusing to warm ephemeral disk",
-        }
-    # Sentinel written by cloud-init only onto a real mount; absent sentinel means Docker silently
-    # auto-created an empty host dir (isdir passes) — we'd warm ephemeral disk. Must check.
-    if payload.get("cache_mount_marker"):
-        marker = os.path.join(mount, payload["cache_mount_marker"])
-        if not os.path.exists(marker):
-            return {
-                "preloaded": [],
-                "already_cached": [],
-                "failed": {},
-                "error": (
-                    f"weight-cache NFS filesystem not mounted (no sentinel at {marker}); "
-                    "refusing to warm ephemeral disk"
-                ),
-            }
-    from huggingface_hub import snapshot_download
-
-    ignore_patterns = ["*.pth", "*.gguf", "original/*", "*.onnx", "*.msgpack", "*.h5"]
-    done, already, failed = [], [], {}
-    for repo_id in payload.get("models") or []:
-        try:
-            # Probe with local_files_only first (HF's own resolution, not a dir-name guess).
-            try:
-                snapshot_download(
-                    repo_id=repo_id,
-                    token=token,
-                    cache_dir=cache_dir,
-                    ignore_patterns=ignore_patterns,
-                    local_files_only=True,
-                )
-                already.append(repo_id)
-                print(f"preload: {repo_id} -> {cache_dir} (cached)", flush=True)
-                continue
-            except Exception:
-                pass
-            snapshot_download(
-                repo_id=repo_id, token=token, cache_dir=cache_dir, ignore_patterns=ignore_patterns
-            )
-            done.append(repo_id)
-            print(f"preload: {repo_id} -> {cache_dir} (downloaded)", flush=True)
-        except Exception as exc:
-            failed[repo_id] = _safe_detail(exc, secrets=_payload_secrets(payload))
-            print(f"preload FAILED {repo_id}: {failed[repo_id]}", flush=True)
-    return {"preloaded": done, "already_cached": already, "failed": failed}
-
-
 def main() -> int:
     # SIGTERM → sys.exit so the finally block still uploads the terminal marker.
     signal.signal(signal.SIGTERM, lambda *a: sys.exit(1))
     payload = load_payload()
+    if payload.get("mode") == "preload":
+        return _run_attempt(payload)
+    state = AttemptState(_code_dir(payload), payload)
+    with state.locked():
+        marker = state.read()
+        if marker is not None:
+            print("bootstrap: replaying the persisted terminal result", flush=True)
+            hf_upload(
+                payload,
+                state.publication_path(marker),
+                f"{_arm(payload)}_attempt{payload['attempt']}.json",
+                enforce_deadline=False,
+            )
+            return 0 if marker["ok"] else 1
+        return _run_attempt(payload)
+
+
+def _run_attempt(payload: dict) -> int:
     ok = False
     error = ""
     retriable = False