freesolo-flash-dev 1.2.67__tar.gz → 1.2.69__tar.gz

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/CHANGELOG.md

@@ -21,6 +21,7 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Changed
 
+- Managed attempts now pin, verify, and report one immutable Flash source archive across retries and recovery.
 - Multi-turn GRPO now clones each sibling rollout from the task state that produced its prepared
   prompt. Environments that record randomized episode choices on the task during `start_episode`
   are stepped and scored against the same episode the model saw, without sharing mutable state

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/Dockerfile.worker

@@ -1,5 +1,5 @@
 # Flash prebuilt WORKER image — the full training stack baked in for RunPod workers.
-# The worker still fetches the flash `code/**` from HF at runtime, so only DEPS live here.
+# the worker fetches one pinned source archive at runtime; the trusted verifier is baked here.
 #
 # this image name is defined in flash/providers/_lifecycle/worker.py. it must be
 # published to GHCR and the package set PUBLIC (no registry login is configured) so RunPod can
@@ -460,7 +460,8 @@ print('  gdn boundary resets: fla', M.version('flash-linear-attention'), 'and ca
 # from the real _train_body source in providers/runpod/serverless/endpoints.py (single source of truth;
 # _train_body is self-contained — every name is imported inside its body) and run it as the CMD so the
 # image is a self-contained RunPod Serverless worker. Build context is the repo root.
-COPY flash/providers/runpod/serverless/endpoints.py docker/make_rp_handler.py /tmp/rpgen/
-RUN python /tmp/rpgen/make_rp_handler.py /tmp/rpgen/endpoints.py /rp_handler.py \
+COPY flash/providers/runpod/serverless/endpoints.py flash/source_snapshot.py docker/make_rp_handler.py /tmp/rpgen/
+RUN cp /tmp/rpgen/source_snapshot.py /source_snapshot.py \
+    && python /tmp/rpgen/make_rp_handler.py /tmp/rpgen/endpoints.py /tmp/rpgen/source_snapshot.py /rp_handler.py \
     && python -c "import ast; ast.parse(open('/rp_handler.py').read()); print('rp_handler OK')"
 CMD ["python", "-u", "/rp_handler.py"]

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.67
+Version: 1.2.69
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/docker/kernel_fingerprint.py

@@ -236,6 +236,7 @@ def collect_inputs(
             root / "flash" / "providers" / "runpod" / "serverless" / "endpoints.py"
         ),
         "make_rp_handler_sha256": _sha256_file(root / "docker" / "make_rp_handler.py"),
+        "source_snapshot_sha256": _sha256_file(root / "flash" / "source_snapshot.py"),
     }
     return cache_inputs, base_inputs_partial
 

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/docker/make_rp_handler.py

@@ -5,11 +5,10 @@ This makes the worker image a *self-contained RunPod Serverless worker* (Pattern
 Flash deploys it as a custom image (client mode), the baked CMD runs this handler, which reads the
 job payload from job["input"] and runs the training via `_train_body`.
 
-`_train_body` is fully self-contained (every name it uses is imported inside its body and it fetches
-the flash `code/**` from HF at runtime), so extracting just its source needs no flash import.
-Single source of truth: it's generated from the real `_train_body`, so it can't drift.
+`_train_body` imports its trusted verifier from the sibling module baked into the image, then fetches
+the pinned Flash source archive from HF.
 
-Usage: python make_rp_handler.py <path-to-endpoints.py> <out-path>
+Usage: python make_rp_handler.py <path-to-endpoints.py> <path-to-source-snapshot.py> <out-path>
 """
 
 import ast
@@ -17,9 +16,11 @@ import sys
 
 
 def main() -> int:
-    if len(sys.argv) != 3:
-        raise SystemExit(f"usage: {sys.argv[0]} <path-to-endpoints.py> <out-path>")
-    src_path, out_path = sys.argv[1], sys.argv[2]
+    if len(sys.argv) != 4:
+        raise SystemExit(
+            f"usage: {sys.argv[0]} <path-to-endpoints.py> <path-to-source-snapshot.py> <out-path>"
+        )
+    src_path, verifier_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
     with open(src_path, encoding="utf-8") as f:
         src = f.read()
     fn_src = None
@@ -29,12 +30,20 @@ def main() -> int:
             break
     if not fn_src:
         raise SystemExit("could not find _train_body in " + src_path)
+    with open(verifier_path, "rb") as f:
+        verifier = f.read()
+    ast.parse(verifier, filename=verifier_path)
     out = (
         "# AUTO-GENERATED by flash/docker/make_rp_handler.py from providers/runpod/train/endpoints.py.\n"
         "# Do not edit; the worker image regenerates it at build time.\n"
         "import runpod\n\n" + fn_src + "\n\ndef handler(job):\n"
         '    """RunPod Serverless entry: the job payload is the control plane\'s train input."""\n'
-        '    return _train_body(job["input"])\n\n'
+        "    try:\n"
+        '        return _train_body(job["input"])\n'
+        "    except Exception as exc:\n"
+        '        if getattr(exc, "flash_retriable", False):\n'
+        '            return {"error": str(exc), "_flash_retriable": True}\n'
+        "        raise\n\n"
         'runpod.serverless.start({"handler": handler})\n'
     )
     # Validate it parses before baking.

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/flash/cli/commands/env/push.py

@@ -716,7 +716,20 @@ def cmd_env_push(args) -> int:
         readme = pkg / "README.md"
         if not readme.exists():
             readme.write_text(f"# {env_name}\n\nFlash Freesolo environment.\n")
-        # One progress widget spans both phases the user otherwise waits through silently:
+        from flash.envs.package.direct_tokens import (
+            DirectTokenScanError,
+            package_contains_direct_token,
+        )
+
+        try:
+            contains_direct_token = package_contains_direct_token(pkg)
+        except DirectTokenScanError:
+            return _err("environment package could not be scanned safely")
+        if contains_direct_token:
+            return _err(
+                "environment package contains a direct access token; remove it before publishing"
+            )
+        # one progress widget spans both phases the user otherwise waits through silently:
         # packaging (walk + gzip, slow for large datasets) and the upload itself.
         bar = _UploadProgress(env_name)
         bar.status("packaging environment")

freesolo_flash_dev-1.2.69/flash/envs/package/direct_tokens.py

@@ -0,0 +1,136 @@
+"""Bounded raw-byte scanning for direct environment package tokens."""
+
+from __future__ import annotations
+
+import os
+import re
+import stat
+from collections.abc import Iterator
+from pathlib import Path
+
+_CHUNK_SIZE = 64 * 1024
+_URLSAFE_CLASS = rb"A-Za-z0-9_-"
+_ALPHANUMERIC_CLASS = rb"A-Za-z0-9"
+_TOKEN_RULES = (
+    (b"fslo_", _URLSAFE_CLASS, 45),
+    (b"hf_", _ALPHANUMERIC_CLASS, 34),
+    (b"pit_", _ALPHANUMERIC_CLASS, 64),
+)
+_TOKEN_PATTERNS = tuple(
+    re.compile(
+        rb"(?<!["
+        + _URLSAFE_CLASS
+        + rb"])(?:"
+        + re.escape(prefix)
+        + rb")(["
+        + body_class
+        + rb"]{"
+        + str(body_length).encode()
+        + rb"})(?!["
+        + _URLSAFE_CLASS
+        + rb"])"
+    )
+    for prefix, body_class, body_length in _TOKEN_RULES
+)
+_MAX_TOKEN_SIZE = max(len(prefix) + body_length for prefix, _, body_length in _TOKEN_RULES)
+_OVERLAP = _MAX_TOKEN_SIZE + 1
+_PLACEHOLDER_BODIES = frozenset(
+    {
+        b"exampletoken",
+        b"notarealtoken",
+        b"placeholdertoken",
+        b"replacemewithtoken",
+        b"testtoken",
+        b"yourapitokenhere",
+        b"yourtokenhere",
+    }
+)
+
+
+class DirectTokenScanError(Exception):
+    """A package could not be scanned without following or skipping a member."""
+
+    def __init__(self) -> None:
+        super().__init__("package scan failed")
+
+
+def _is_obvious_placeholder(body: bytes) -> bool:
+    normalized = body.lower().translate(None, b"_-")
+    if normalized in _PLACEHOLDER_BODIES:
+        return True
+    return bool(normalized) and normalized[:1] in {b"0", b"x"} and len(set(normalized)) == 1
+
+
+def _buffer_contains_direct_token(data: bytes, *, start_at: int, finalized_start_end: int) -> bool:
+    for pattern in _TOKEN_PATTERNS:
+        for match in pattern.finditer(data):
+            if match.start() < start_at:
+                continue
+            if match.start() >= finalized_start_end:
+                break
+            if not _is_obvious_placeholder(match.group(1)):
+                return True
+    return False
+
+
+def _file_contains_direct_token(path: Path) -> bool:
+    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
+    fd: int | None = os.open(path, flags)
+    try:
+        if not stat.S_ISREG(os.fstat(fd).st_mode):
+            raise DirectTokenScanError
+        with os.fdopen(fd, "rb") as stream:
+            fd = None
+            data = b""
+            buffer_start = 0
+            finalized_start = 0
+            while chunk := stream.read(_CHUNK_SIZE):
+                data += chunk
+                start_at = finalized_start - buffer_start
+                finalized_start_end = max(start_at, len(data) - _MAX_TOKEN_SIZE)
+                if _buffer_contains_direct_token(
+                    data,
+                    start_at=start_at,
+                    finalized_start_end=finalized_start_end,
+                ):
+                    return True
+                finalized_start = buffer_start + finalized_start_end
+                keep_from = max(0, len(data) - _OVERLAP)
+                data = data[keep_from:]
+                buffer_start += keep_from
+            return _buffer_contains_direct_token(
+                data,
+                start_at=finalized_start - buffer_start,
+                finalized_start_end=len(data),
+            )
+    finally:
+        if fd is not None:
+            os.close(fd)
+
+
+def _regular_files(directory: Path) -> Iterator[Path]:
+    with os.scandir(directory) as listing:
+        entries = sorted(listing, key=lambda entry: entry.name)
+    for entry in entries:
+        if entry.is_symlink():
+            raise DirectTokenScanError
+        if entry.is_dir(follow_symlinks=False):
+            yield from _regular_files(Path(entry.path))
+        elif entry.is_file(follow_symlinks=False):
+            yield Path(entry.path)
+        else:
+            raise DirectTokenScanError
+
+
+def package_contains_direct_token(root: Path) -> bool:
+    """Return whether a package tree contains a supported direct issued-token form."""
+    try:
+        root_stat = os.stat(root, follow_symlinks=False)
+        if not stat.S_ISDIR(root_stat.st_mode):
+            raise DirectTokenScanError
+        return any(_file_contains_direct_token(path) for path in _regular_files(root))
+    except DirectTokenScanError:
+        raise
+    except OSError:
+        pass
+    raise DirectTokenScanError

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/bootstrap.py

@@ -20,6 +20,7 @@ from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
 
 if __package__:
+    from flash import source_snapshot as _source_snapshot
     from flash.providers._lifecycle import bootstrap_console as _bootstrap_console
     from flash.providers._lifecycle import bootstrap_pip
     from flash.providers._lifecycle.bootstrap_secrets import (
@@ -32,6 +33,7 @@ else:
     # same directory, and the script directory leads sys.path.
     import bootstrap_console as _bootstrap_console  # type: ignore[no-redef]
     import bootstrap_pip  # type: ignore[no-redef]
+    import source_snapshot as _source_snapshot  # type: ignore[no-redef]
     from bootstrap_secrets import (  # type: ignore[no-redef]
         _payload_secrets,
         _read_console_tail,
@@ -139,38 +141,22 @@ def _arm(payload: dict) -> str:
     return str(payload.get("flash_arm") or "instance")
 
 
-def _code_prefix(payload: dict) -> str:
-    raw = payload.get("code_prefix")
-    if not isinstance(raw, str) or not raw.strip():
-        raise ValueError("missing code_prefix")
-    prefix = raw.strip().strip("/")
-    parts = prefix.split("/")
-    digest = parts[1] if len(parts) == 3 else ""
-    if (
-        len(parts) != 3
-        or parts[0] != "code"
-        or parts[2] != "flash"
-        or len(digest) != 32
-        or any(c not in "0123456789abcdef" for c in digest)
-    ):
-        raise ValueError(f"invalid code_prefix: {prefix!r}")
-    return prefix
+def _source_descriptor(payload: dict):
+    return _source_snapshot.parse_descriptor(payload.get("source_snapshot"))
 
 
 def _code_dir(payload: dict) -> str:
-    raw = payload.get("code_prefix")
-    if not isinstance(raw, str) or not raw.strip():
-        return os.path.join(CODE_ROOT, "code")
-    return os.path.join(CODE_ROOT, os.path.dirname(_code_prefix(payload)))
+    return str(
+        _source_snapshot.attempt_materialization_path(
+            CODE_ROOT,
+            payload.get("run_id"),
+            payload.get("attempt"),
+        )
+    )
 
 
 def _hf_status_code(exc: BaseException) -> int | None:
-    response = getattr(exc, "response", None)
-    code = getattr(response, "status_code", None)
-    try:
-        return int(code)
-    except (TypeError, ValueError):
-        return None
+    return _source_snapshot.response_status_code(exc)
 
 
 def _hf_retry_after(exc: BaseException) -> float | None:
@@ -208,7 +194,7 @@ def _hf_call(call, label: str, *, deadline_at: float | None = None, secrets: dic
         try:
             return call()
         except Exception as exc:
-            if _hf_status_code(exc) not in _HF_TRANSIENT_STATUS_CODES or attempt >= len(
+            if not _source_snapshot.is_transient_fetch_error(exc) or attempt >= len(
                 _HF_RETRY_DELAYS_S
             ):
                 raise
@@ -580,45 +566,36 @@ def install_extra_pip(payload: dict) -> None:
 
 
 def fetch_code(payload: dict) -> None:
-    from huggingface_hub import HfApi, hf_hub_download
+    from huggingface_hub import hf_hub_download
 
     deadline_at = require_deadline_at(payload)
-    prefix = _code_prefix(payload)
+    descriptor = _source_descriptor(payload)
     token = (payload.get("env") or {}).get("HF_TOKEN")
-    api = HfApi(token=token)
-    files = [
-        entry.path
-        for entry in _hf_call(
-            lambda: list(
-                api.list_repo_tree(
-                    repo_id=payload["hf_repo"],
-                    repo_type="dataset",
-                    path_in_repo=prefix,
-                    recursive=True,
-                    token=token,
-                )
-            ),
-            f"list flash code under {payload['hf_repo']}:{prefix}",
-            deadline_at=deadline_at,
-            secrets=_payload_secrets(payload),
-        )
-        if getattr(entry, "path", None) and getattr(entry, "size", None) is not None
-    ]
-    if not files:
-        raise RuntimeError(f"no flash code files found under {payload['hf_repo']}:{prefix}")
-    for filename in files:
-        _hf_call(
-            lambda filename=filename: hf_hub_download(
+    try:
+        archive_path = _hf_call(
+            lambda: hf_hub_download(
                 repo_id=payload["hf_repo"],
                 repo_type="dataset",
-                filename=filename,
-                local_dir=CODE_ROOT,
+                filename=descriptor.archive_path,
+                revision=descriptor.revision,
                 token=token,
             ),
-            f"download flash code file {payload['hf_repo']}:{filename}",
+            "download pinned flash source snapshot",
             deadline_at=deadline_at,
             secrets=_payload_secrets(payload),
         )
+    except Exception as exc:
+        error_type = (
+            RetriableBootstrapError
+            if _source_snapshot.is_transient_fetch_error(exc)
+            else RuntimeError
+        )
+        raise error_type("failed to fetch the pinned flash source snapshot") from None
+    _source_snapshot.materialize_verified_archive_file(
+        archive_path,
+        descriptor,
+        _code_dir(payload),
+    )
 
 
 def run_mode(payload: dict, env: dict, mode: str, deadline_ts: float) -> int:
@@ -780,6 +757,12 @@ def write_attempt_marker(payload: dict, ok: bool, error: str = "", retriable: bo
         "run_id": run_id,
         "ts": now,
     }
+    if ok:
+        marker["source_attestation"] = _source_snapshot.source_attestation(
+            _source_descriptor(payload),
+            run_id=run_id,
+            attempt=attempt,
+        )
     p = "/tmp/attempt_marker.json"
     with open(p, "w") as f:
         json.dump(marker, f)
@@ -924,13 +907,9 @@ def main() -> int:
             error = "model preload failed" if not ok else ""
             return 0 if ok else 1
         deadline_watchdog = arm_deadline_watchdog(deadline, payload)
+        # fetch and verify the pinned archive before pip, downloaded imports, path mutation, or child.
+        fetch_code(payload)
         install_extra_pip(payload)
-        # Pre-worker HF fetch of the run's own code (control plane uploaded it before submit), same
-        # infra-shaped class as fetch_spec_from_hf above: a transient HF blip must retry, not fail.
-        try:
-            fetch_code(payload)
-        except Exception:
-            raise RetriableBootstrapError("failed to fetch run code from HF") from None
         env = build_worker_env(payload)
         phase = payload["phase"]
         for stale in ("/tmp/train_meta.json", "/tmp/metrics.json"):

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/instance.py

@@ -191,7 +191,7 @@ def build_payload(
     cache_host_mount: str | None = None,
     mode: str | None = None,
     models: list | None = None,
-    code_prefix: str | None = None,
+    source_snapshot: dict | None = None,
     deadline_at: float | None = None,
 ) -> dict:
     """The bootstrap's input — field-compatible with the RunPod ``_train_body`` payload, plus the
@@ -203,7 +203,7 @@ def build_payload(
         build_worker_env,
         strip_runpod_volume_env,
     )
-    from flash.runner import flash_code_prefix
+    from flash.source_snapshot import parse_descriptor
 
     canonical_seed = require_matching_seed(spec, seed)
     # strip the runpod-only volume redirect; point base-model prefetch at this provider's cache unless the user overrode it.
@@ -233,12 +233,13 @@ def build_payload(
         # [environment] pip is appended to the worker requirement, never substituted for it.
         "extra_pip": worker_pip_with_extras(spec.environment.id, spec.environment.pip),
         "hf_prefix": f"{spec.phase}/{spec.run_id}",
-        "code_prefix": code_prefix or flash_code_prefix(),
         "deadline_at": absolute_deadline,
         "run_created_at": absolute_deadline - max_wall_seconds,
         "run_max_wall_seconds": max_wall_seconds,
         "attempt": attempt_id,
     }
+    if mode != "preload":
+        payload["source_snapshot"] = parse_descriptor(source_snapshot).to_dict()
     if cache_host_mount:
         payload["cache_host_mount"] = cache_host_mount
         # Carry the mount sentinel filename so the bootstrap's mount-check reads it from one constant.

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/provider.py

@@ -66,7 +66,7 @@ class InstanceProvider(abc.ABC):
         on_handle: Any,
         attempt: int,
         runtime_secrets: dict[str, str] | None,
-        code_prefix: str | None,
+        source_snapshot: dict | None,
         deadline_at: float | None,
     ) -> PollResult: ...
 
@@ -122,7 +122,7 @@ class InstanceProvider(abc.ABC):
         attempt: int = 0,
         runtime_secrets: dict[str, str] | None = None,
         on_last_gpu: bool = False,
-        code_prefix: str | None = None,
+        source_snapshot: dict | None = None,
         _deadline_at: float | None = None,
     ) -> PollResult:
         from flash.core.spec import require_matching_seed
@@ -136,7 +136,7 @@ class InstanceProvider(abc.ABC):
             on_handle=on_handle,
             attempt=attempt,
             runtime_secrets=runtime_secrets,
-            code_prefix=code_prefix,
+            source_snapshot=source_snapshot,
             deadline_at=_deadline_at,
         )
 

{freesolo_flash_dev-1.2.67 → freesolo_flash_dev-1.2.69}/flash/providers/_lifecycle/terminal_artifacts.py

@@ -156,19 +156,22 @@ def decode_terminal_marker(
     the caller allows. A marker that cannot be verified must never settle an attempt.
     """
     marker = json.loads(raw)
-    if not isinstance(marker, dict) or set(marker) != {
-        "attempt",
-        "error",
-        "ok",
-        "retriable",
-        "run_id",
-        "ts",
+    base_fields = {"attempt", "error", "ok", "retriable", "run_id", "ts"}
+    if not isinstance(marker, dict) or set(marker) not in {
+        frozenset(base_fields),
+        frozenset(base_fields | {"source_attestation"}),
     }:
         raise ValueError("invalid terminal marker schema")
     marker_attempt = marker["attempt"]
     ts = marker["ts"]
     error = marker["error"]
     now = time.time()
+    if "source_attestation" in marker:
+        from flash.source_snapshot import parse_attestation
+
+        attestation = parse_attestation(marker["source_attestation"])
+        if attestation["run_id"] != run_id or attestation["attempt"] != attempt:
+            raise ValueError("invalid terminal marker source identity")
     if (
         isinstance(launch_floor, bool)
         or not isinstance(launch_floor, (int, float))
@@ -258,4 +261,8 @@ def resolve_terminal_artifacts(
             marker=marker,
             metrics_unparseable=raw_metrics is not None,
         )
+    if "source_attestation" in marker:
+        from flash.source_snapshot import TERMINAL_ATTESTATION_KEY
+
+        metrics = {**metrics, TERMINAL_ATTESTATION_KEY: marker["source_attestation"]}
     return TerminalResolution(TerminalKind.SUCCESS, metrics=metrics, marker=marker)