freesolo-flash-dev 1.2.68__tar.gz → 1.2.69__tar.gz

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/CHANGELOG.md

@@ -21,6 +21,7 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Changed
 
+- Managed attempts now pin, verify, and report one immutable Flash source archive across retries and recovery.
 - Multi-turn GRPO now clones each sibling rollout from the task state that produced its prepared
   prompt. Environments that record randomized episode choices on the task during `start_episode`
   are stepped and scored against the same episode the model saw, without sharing mutable state

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/Dockerfile.worker

@@ -1,5 +1,5 @@
 # Flash prebuilt WORKER image — the full training stack baked in for RunPod workers.
-# The worker still fetches the flash `code/**` from HF at runtime, so only DEPS live here.
+# the worker fetches one pinned source archive at runtime; the trusted verifier is baked here.
 #
 # this image name is defined in flash/providers/_lifecycle/worker.py. it must be
 # published to GHCR and the package set PUBLIC (no registry login is configured) so RunPod can
@@ -460,7 +460,8 @@ print('  gdn boundary resets: fla', M.version('flash-linear-attention'), 'and ca
 # from the real _train_body source in providers/runpod/serverless/endpoints.py (single source of truth;
 # _train_body is self-contained — every name is imported inside its body) and run it as the CMD so the
 # image is a self-contained RunPod Serverless worker. Build context is the repo root.
-COPY flash/providers/runpod/serverless/endpoints.py docker/make_rp_handler.py /tmp/rpgen/
-RUN python /tmp/rpgen/make_rp_handler.py /tmp/rpgen/endpoints.py /rp_handler.py \
+COPY flash/providers/runpod/serverless/endpoints.py flash/source_snapshot.py docker/make_rp_handler.py /tmp/rpgen/
+RUN cp /tmp/rpgen/source_snapshot.py /source_snapshot.py \
+    && python /tmp/rpgen/make_rp_handler.py /tmp/rpgen/endpoints.py /tmp/rpgen/source_snapshot.py /rp_handler.py \
     && python -c "import ast; ast.parse(open('/rp_handler.py').read()); print('rp_handler OK')"
 CMD ["python", "-u", "/rp_handler.py"]

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.68
+Version: 1.2.69
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/docker/kernel_fingerprint.py

@@ -236,6 +236,7 @@ def collect_inputs(
             root / "flash" / "providers" / "runpod" / "serverless" / "endpoints.py"
         ),
         "make_rp_handler_sha256": _sha256_file(root / "docker" / "make_rp_handler.py"),
+        "source_snapshot_sha256": _sha256_file(root / "flash" / "source_snapshot.py"),
     }
     return cache_inputs, base_inputs_partial
 

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/docker/make_rp_handler.py

@@ -5,11 +5,10 @@ This makes the worker image a *self-contained RunPod Serverless worker* (Pattern
 Flash deploys it as a custom image (client mode), the baked CMD runs this handler, which reads the
 job payload from job["input"] and runs the training via `_train_body`.
 
-`_train_body` is fully self-contained (every name it uses is imported inside its body and it fetches
-the flash `code/**` from HF at runtime), so extracting just its source needs no flash import.
-Single source of truth: it's generated from the real `_train_body`, so it can't drift.
+`_train_body` imports its trusted verifier from the sibling module baked into the image, then fetches
+the pinned Flash source archive from HF.
 
-Usage: python make_rp_handler.py <path-to-endpoints.py> <out-path>
+Usage: python make_rp_handler.py <path-to-endpoints.py> <path-to-source-snapshot.py> <out-path>
 """
 
 import ast
@@ -17,9 +16,11 @@ import sys
 
 
 def main() -> int:
-    if len(sys.argv) != 3:
-        raise SystemExit(f"usage: {sys.argv[0]} <path-to-endpoints.py> <out-path>")
-    src_path, out_path = sys.argv[1], sys.argv[2]
+    if len(sys.argv) != 4:
+        raise SystemExit(
+            f"usage: {sys.argv[0]} <path-to-endpoints.py> <path-to-source-snapshot.py> <out-path>"
+        )
+    src_path, verifier_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
     with open(src_path, encoding="utf-8") as f:
         src = f.read()
     fn_src = None
@@ -29,12 +30,20 @@ def main() -> int:
             break
     if not fn_src:
         raise SystemExit("could not find _train_body in " + src_path)
+    with open(verifier_path, "rb") as f:
+        verifier = f.read()
+    ast.parse(verifier, filename=verifier_path)
     out = (
         "# AUTO-GENERATED by flash/docker/make_rp_handler.py from providers/runpod/train/endpoints.py.\n"
         "# Do not edit; the worker image regenerates it at build time.\n"
         "import runpod\n\n" + fn_src + "\n\ndef handler(job):\n"
         '    """RunPod Serverless entry: the job payload is the control plane\'s train input."""\n'
-        '    return _train_body(job["input"])\n\n'
+        "    try:\n"
+        '        return _train_body(job["input"])\n'
+        "    except Exception as exc:\n"
+        '        if getattr(exc, "flash_retriable", False):\n'
+        '            return {"error": str(exc), "_flash_retriable": True}\n'
+        "        raise\n\n"
         'runpod.serverless.start({"handler": handler})\n'
     )
     # Validate it parses before baking.

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/bootstrap.py

@@ -20,6 +20,7 @@ from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
 
 if __package__:
+    from flash import source_snapshot as _source_snapshot
     from flash.providers._lifecycle import bootstrap_console as _bootstrap_console
     from flash.providers._lifecycle import bootstrap_pip
     from flash.providers._lifecycle.bootstrap_secrets import (
@@ -32,6 +33,7 @@ else:
     # same directory, and the script directory leads sys.path.
     import bootstrap_console as _bootstrap_console  # type: ignore[no-redef]
     import bootstrap_pip  # type: ignore[no-redef]
+    import source_snapshot as _source_snapshot  # type: ignore[no-redef]
     from bootstrap_secrets import (  # type: ignore[no-redef]
         _payload_secrets,
         _read_console_tail,
@@ -139,38 +141,22 @@ def _arm(payload: dict) -> str:
     return str(payload.get("flash_arm") or "instance")
 
 
-def _code_prefix(payload: dict) -> str:
-    raw = payload.get("code_prefix")
-    if not isinstance(raw, str) or not raw.strip():
-        raise ValueError("missing code_prefix")
-    prefix = raw.strip().strip("/")
-    parts = prefix.split("/")
-    digest = parts[1] if len(parts) == 3 else ""
-    if (
-        len(parts) != 3
-        or parts[0] != "code"
-        or parts[2] != "flash"
-        or len(digest) != 32
-        or any(c not in "0123456789abcdef" for c in digest)
-    ):
-        raise ValueError(f"invalid code_prefix: {prefix!r}")
-    return prefix
+def _source_descriptor(payload: dict):
+    return _source_snapshot.parse_descriptor(payload.get("source_snapshot"))
 
 
 def _code_dir(payload: dict) -> str:
-    raw = payload.get("code_prefix")
-    if not isinstance(raw, str) or not raw.strip():
-        return os.path.join(CODE_ROOT, "code")
-    return os.path.join(CODE_ROOT, os.path.dirname(_code_prefix(payload)))
+    return str(
+        _source_snapshot.attempt_materialization_path(
+            CODE_ROOT,
+            payload.get("run_id"),
+            payload.get("attempt"),
+        )
+    )
 
 
 def _hf_status_code(exc: BaseException) -> int | None:
-    response = getattr(exc, "response", None)
-    code = getattr(response, "status_code", None)
-    try:
-        return int(code)
-    except (TypeError, ValueError):
-        return None
+    return _source_snapshot.response_status_code(exc)
 
 
 def _hf_retry_after(exc: BaseException) -> float | None:
@@ -208,7 +194,7 @@ def _hf_call(call, label: str, *, deadline_at: float | None = None, secrets: dic
         try:
             return call()
         except Exception as exc:
-            if _hf_status_code(exc) not in _HF_TRANSIENT_STATUS_CODES or attempt >= len(
+            if not _source_snapshot.is_transient_fetch_error(exc) or attempt >= len(
                 _HF_RETRY_DELAYS_S
             ):
                 raise
@@ -580,45 +566,36 @@ def install_extra_pip(payload: dict) -> None:
 
 
 def fetch_code(payload: dict) -> None:
-    from huggingface_hub import HfApi, hf_hub_download
+    from huggingface_hub import hf_hub_download
 
     deadline_at = require_deadline_at(payload)
-    prefix = _code_prefix(payload)
+    descriptor = _source_descriptor(payload)
     token = (payload.get("env") or {}).get("HF_TOKEN")
-    api = HfApi(token=token)
-    files = [
-        entry.path
-        for entry in _hf_call(
-            lambda: list(
-                api.list_repo_tree(
-                    repo_id=payload["hf_repo"],
-                    repo_type="dataset",
-                    path_in_repo=prefix,
-                    recursive=True,
-                    token=token,
-                )
-            ),
-            f"list flash code under {payload['hf_repo']}:{prefix}",
-            deadline_at=deadline_at,
-            secrets=_payload_secrets(payload),
-        )
-        if getattr(entry, "path", None) and getattr(entry, "size", None) is not None
-    ]
-    if not files:
-        raise RuntimeError(f"no flash code files found under {payload['hf_repo']}:{prefix}")
-    for filename in files:
-        _hf_call(
-            lambda filename=filename: hf_hub_download(
+    try:
+        archive_path = _hf_call(
+            lambda: hf_hub_download(
                 repo_id=payload["hf_repo"],
                 repo_type="dataset",
-                filename=filename,
-                local_dir=CODE_ROOT,
+                filename=descriptor.archive_path,
+                revision=descriptor.revision,
                 token=token,
             ),
-            f"download flash code file {payload['hf_repo']}:{filename}",
+            "download pinned flash source snapshot",
             deadline_at=deadline_at,
             secrets=_payload_secrets(payload),
         )
+    except Exception as exc:
+        error_type = (
+            RetriableBootstrapError
+            if _source_snapshot.is_transient_fetch_error(exc)
+            else RuntimeError
+        )
+        raise error_type("failed to fetch the pinned flash source snapshot") from None
+    _source_snapshot.materialize_verified_archive_file(
+        archive_path,
+        descriptor,
+        _code_dir(payload),
+    )
 
 
 def run_mode(payload: dict, env: dict, mode: str, deadline_ts: float) -> int:
@@ -780,6 +757,12 @@ def write_attempt_marker(payload: dict, ok: bool, error: str = "", retriable: bo
         "run_id": run_id,
         "ts": now,
     }
+    if ok:
+        marker["source_attestation"] = _source_snapshot.source_attestation(
+            _source_descriptor(payload),
+            run_id=run_id,
+            attempt=attempt,
+        )
     p = "/tmp/attempt_marker.json"
     with open(p, "w") as f:
         json.dump(marker, f)
@@ -924,13 +907,9 @@ def main() -> int:
             error = "model preload failed" if not ok else ""
             return 0 if ok else 1
         deadline_watchdog = arm_deadline_watchdog(deadline, payload)
+        # fetch and verify the pinned archive before pip, downloaded imports, path mutation, or child.
+        fetch_code(payload)
         install_extra_pip(payload)
-        # Pre-worker HF fetch of the run's own code (control plane uploaded it before submit), same
-        # infra-shaped class as fetch_spec_from_hf above: a transient HF blip must retry, not fail.
-        try:
-            fetch_code(payload)
-        except Exception:
-            raise RetriableBootstrapError("failed to fetch run code from HF") from None
         env = build_worker_env(payload)
         phase = payload["phase"]
         for stale in ("/tmp/train_meta.json", "/tmp/metrics.json"):

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/instance.py

@@ -191,7 +191,7 @@ def build_payload(
     cache_host_mount: str | None = None,
     mode: str | None = None,
     models: list | None = None,
-    code_prefix: str | None = None,
+    source_snapshot: dict | None = None,
     deadline_at: float | None = None,
 ) -> dict:
     """The bootstrap's input — field-compatible with the RunPod ``_train_body`` payload, plus the
@@ -203,7 +203,7 @@ def build_payload(
         build_worker_env,
         strip_runpod_volume_env,
     )
-    from flash.runner import flash_code_prefix
+    from flash.source_snapshot import parse_descriptor
 
     canonical_seed = require_matching_seed(spec, seed)
     # strip the runpod-only volume redirect; point base-model prefetch at this provider's cache unless the user overrode it.
@@ -233,12 +233,13 @@ def build_payload(
         # [environment] pip is appended to the worker requirement, never substituted for it.
         "extra_pip": worker_pip_with_extras(spec.environment.id, spec.environment.pip),
         "hf_prefix": f"{spec.phase}/{spec.run_id}",
-        "code_prefix": code_prefix or flash_code_prefix(),
         "deadline_at": absolute_deadline,
         "run_created_at": absolute_deadline - max_wall_seconds,
         "run_max_wall_seconds": max_wall_seconds,
         "attempt": attempt_id,
     }
+    if mode != "preload":
+        payload["source_snapshot"] = parse_descriptor(source_snapshot).to_dict()
     if cache_host_mount:
         payload["cache_host_mount"] = cache_host_mount
         # Carry the mount sentinel filename so the bootstrap's mount-check reads it from one constant.

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/provider.py

@@ -66,7 +66,7 @@ class InstanceProvider(abc.ABC):
         on_handle: Any,
         attempt: int,
         runtime_secrets: dict[str, str] | None,
-        code_prefix: str | None,
+        source_snapshot: dict | None,
         deadline_at: float | None,
     ) -> PollResult: ...
 
@@ -122,7 +122,7 @@ class InstanceProvider(abc.ABC):
         attempt: int = 0,
         runtime_secrets: dict[str, str] | None = None,
         on_last_gpu: bool = False,
-        code_prefix: str | None = None,
+        source_snapshot: dict | None = None,
         _deadline_at: float | None = None,
     ) -> PollResult:
         from flash.core.spec import require_matching_seed
@@ -136,7 +136,7 @@ class InstanceProvider(abc.ABC):
             on_handle=on_handle,
             attempt=attempt,
             runtime_secrets=runtime_secrets,
-            code_prefix=code_prefix,
+            source_snapshot=source_snapshot,
             deadline_at=_deadline_at,
         )
 

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/terminal_artifacts.py

@@ -156,19 +156,22 @@ def decode_terminal_marker(
     the caller allows. A marker that cannot be verified must never settle an attempt.
     """
     marker = json.loads(raw)
-    if not isinstance(marker, dict) or set(marker) != {
-        "attempt",
-        "error",
-        "ok",
-        "retriable",
-        "run_id",
-        "ts",
+    base_fields = {"attempt", "error", "ok", "retriable", "run_id", "ts"}
+    if not isinstance(marker, dict) or set(marker) not in {
+        frozenset(base_fields),
+        frozenset(base_fields | {"source_attestation"}),
     }:
         raise ValueError("invalid terminal marker schema")
     marker_attempt = marker["attempt"]
     ts = marker["ts"]
     error = marker["error"]
     now = time.time()
+    if "source_attestation" in marker:
+        from flash.source_snapshot import parse_attestation
+
+        attestation = parse_attestation(marker["source_attestation"])
+        if attestation["run_id"] != run_id or attestation["attempt"] != attempt:
+            raise ValueError("invalid terminal marker source identity")
     if (
         isinstance(launch_floor, bool)
         or not isinstance(launch_floor, (int, float))
@@ -258,4 +261,8 @@ def resolve_terminal_artifacts(
             marker=marker,
             metrics_unparseable=raw_metrics is not None,
         )
+    if "source_attestation" in marker:
+        from flash.source_snapshot import TERMINAL_ATTESTATION_KEY
+
+        metrics = {**metrics, TERMINAL_ATTESTATION_KEY: marker["source_attestation"]}
     return TerminalResolution(TerminalKind.SUCCESS, metrics=metrics, marker=marker)

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/worker.py

@@ -1,10 +1,11 @@
-"""Provider-neutral worker packaging: the deps/image/env every rent-a-box or serverless worker ships, plus upload_code for the HF code snapshot. Shared kernel — no provider package imports another for this."""
+"""Provider-neutral worker packaging and immutable managed source publication."""
 
 from __future__ import annotations
 
 import os
 import time
 from io import BytesIO
+from pathlib import Path
 
 from flash._internal.diagnostics import SECRET_ENV_KEYS_ENV
 from flash._internal.logging import get_logger
@@ -205,9 +206,6 @@ def build_worker_env(
     return env
 
 
-_CODE_SNAPSHOT_COMPLETE = ".flash-code-snapshot-complete"
-
-
 def _hf_call(call, label: str, *, deadline_at: float | None = None):
     return hf_call(
         call,
@@ -246,53 +244,120 @@ def _ensure_private_artifact_repo(
     )
 
 
-def upload_code(
+def _repo_revision(api, repo: str, *, deadline_at: float | None) -> str:
+    info = _hf_call(
+        lambda: api.repo_info(repo_id=repo, repo_type="dataset"),
+        f"read artifact repo revision {repo}",
+        deadline_at=deadline_at,
+    )
+    revision = str(getattr(info, "sha", None) or getattr(info, "oid", None) or "").strip()
+    if not revision:
+        raise RuntimeError("artifact repository revision is unavailable")
+    return revision
+
+
+def _download_source_archive(
+    repo: str,
+    archive_path: str,
+    revision: str,
+    token: str | None,
+    *,
+    deadline_at: float | None,
+) -> bytes | None:
+    from huggingface_hub import hf_hub_download
+
+    try:
+        local_path = _hf_call(
+            lambda: hf_hub_download(
+                repo_id=repo,
+                repo_type="dataset",
+                filename=archive_path,
+                revision=revision,
+                token=token,
+            ),
+            f"download source snapshot {repo}:{archive_path}@{revision}",
+            deadline_at=deadline_at,
+        )
+    except Exception as exc:
+        if hf_status_code(exc) == 404 or exc.__class__.__name__ in {
+            "EntryNotFoundError",
+            "LocalEntryNotFoundError",
+        }:
+            return None
+        raise
+    with open(local_path, "rb") as source:
+        return source.read()
+
+
+def publish_source_snapshot(
     repo: str | None = None,
     *,
-    code_prefix: str | None = None,
     deadline_at: float | None = None,
-) -> str:
-    """Upload the ``flash`` package to its content-addressed HF artifact prefix."""
+) -> dict:
+    """Publish one deterministic source archive and return its immutable descriptor."""
     from huggingface_hub import HfApi
 
     import flash
-    from flash.runner import flash_code_prefix
+    from flash.source_snapshot import (
+        build_source_archive,
+        canonical_archive_path,
+        descriptor_for_archive,
+        read_verified_archive,
+        sha256_bytes,
+    )
 
     if not repo:
         raise RuntimeError(
             "hf_repo must be set (the run's [train] hf_repo: HF dataset repo for code + artifacts)"
         )
     token = os.environ.get("HF_TOKEN")
-    pkg_dir = os.path.realpath(os.path.dirname(os.path.abspath(flash.__file__)))
     api = HfApi(token=token)
     _ensure_private_artifact_repo(api, repo, deadline_at=deadline_at)
-    code_prefix = code_prefix or flash_code_prefix()
-    code_marker = f"{code_prefix}/{_CODE_SNAPSHOT_COMPLETE}"
-    if _hf_call(
-        lambda: api.file_exists(repo_id=repo, filename=code_marker, repo_type="dataset"),
-        f"check flash code snapshot {repo}:{code_marker}",
-        deadline_at=deadline_at,
-    ):
-        return repo
-    _hf_call(
-        lambda: api.upload_folder(
-            folder_path=pkg_dir,
-            path_in_repo=code_prefix,
-            repo_id=repo,
-            repo_type="dataset",
-            ignore_patterns=["__pycache__/*", "*.pyc", "*.pyo"],
-        ),
-        f"upload flash code to {repo}:{code_prefix}",
-        deadline_at=deadline_at,
-    )
-    _hf_call(
-        lambda: api.upload_file(
-            path_or_fileobj=BytesIO(b"complete\n"),
-            path_in_repo=code_marker,
-            repo_id=repo,
-            repo_type="dataset",
-        ),
-        f"mark flash code snapshot complete {repo}:{code_marker}",
-        deadline_at=deadline_at,
-    )
-    return repo
+    package_dir = os.path.realpath(os.path.dirname(os.path.abspath(flash.__file__)))
+    archive = build_source_archive(package_dir=Path(package_dir))
+    digest = sha256_bytes(archive)
+    archive_path = canonical_archive_path(digest)
+
+    def verified_at(revision: str) -> dict | None:
+        candidate = _download_source_archive(
+            repo,
+            archive_path,
+            revision,
+            token,
+            deadline_at=deadline_at,
+        )
+        if candidate is None:
+            return None
+        descriptor = descriptor_for_archive(archive, revision)
+        read_verified_archive(candidate, descriptor)
+        return descriptor.to_dict()
+
+    head = _repo_revision(api, repo, deadline_at=deadline_at)
+    existing = verified_at(head)
+    if existing is not None:
+        return existing
+    try:
+        commit = _hf_call(
+            lambda: api.upload_file(
+                path_or_fileobj=BytesIO(archive),
+                path_in_repo=archive_path,
+                repo_id=repo,
+                repo_type="dataset",
+                commit_message=f"publish flash source {digest}",
+            ),
+            f"publish source snapshot {repo}:{archive_path}",
+            deadline_at=deadline_at,
+        )
+        revision = str(getattr(commit, "oid", None) or "").strip()
+        if not revision:
+            raise RuntimeError("source snapshot publication returned no immutable revision")
+        published = verified_at(revision)
+        if published is None:
+            raise RuntimeError("published source snapshot is not readable at its commit revision")
+        return published
+    except Exception:
+        winner_revision = _repo_revision(api, repo, deadline_at=deadline_at)
+        winner = verified_at(winner_revision)
+        if winner is not None:
+            return winner
+        raise

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/flash/providers/base.py

@@ -903,7 +903,7 @@ class Provider(Protocol):
         attempt: int = 0,
         runtime_secrets: dict[str, str] | None = None,
         on_last_gpu: bool = False,
-        code_prefix: str | None = None,
+        source_snapshot: dict | None = None,
         _deadline_at: float | None = None,
     ) -> PollResult:
         """Deploy/rent -> submit -> persist handle (via ``on_handle``) -> poll to terminal.

{freesolo_flash_dev-1.2.68 → freesolo_flash_dev-1.2.69}/flash/providers/lambda_/__init__.py

@@ -55,7 +55,7 @@ class LambdaProvider(InstanceProvider):
         on_handle: Any,
         attempt: int,
         runtime_secrets: dict[str, str] | None,
-        code_prefix: str | None,
+        source_snapshot: dict | None,
         deadline_at: float | None,
     ) -> PollResult:
         from flash.providers.lambda_.jobs import submit_run_lambda
@@ -67,7 +67,7 @@ class LambdaProvider(InstanceProvider):
             on_handle=on_handle,
             attempt=attempt,
             runtime_secrets=runtime_secrets,
-            code_prefix=code_prefix,
+            source_snapshot=source_snapshot,
             deadline_at=deadline_at,
         )