freesolo-flash-dev 1.1.68__tar.gz → 1.1.70__tar.gz

{freesolo_flash_dev-1.1.68 → freesolo_flash_dev-1.1.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.1.68
+Version: 1.1.70
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

freesolo_flash_dev-1.1.70/flash/adapters/artifacts.py

@@ -0,0 +1,141 @@
+"""Artifact filenames the worker writes and the control plane reads.
+
+Every name here crosses a machine boundary: the worker uploads under it, the control plane fetches
+by it, and the two never share a call stack to check. Spelling one on each side makes them agree by
+coincidence, so they are defined once, here.
+"""
+
+# The PEFT adapter weights file a deployable adapter must carry. Modern saves use safetensors,
+# but older PEFT/default settings may emit adapter_model.bin.
+ADAPTER_WEIGHT_FILES = ("adapter_model.safetensors", "adapter_model.bin")
+
+# A PEFT save past its shard size splits the weights across `adapter_model-0000N-of-0000M.<ext>`
+# beside an `adapter_model.<ext>.index.json` mapping every tensor key to the shard holding it.
+ADAPTER_SHARD_PREFIX = "adapter_model-"
+ADAPTER_WEIGHT_SUFFIXES = (".safetensors", ".bin")
+ADAPTER_WEIGHT_INDEX_FILES = tuple(f"{name}.index.json" for name in ADAPTER_WEIGHT_FILES)
+
+
+def is_adapter_weight_filename(filename: str) -> bool:
+    """True for every adapter weight file shape the pipeline accepts, single-file or sharded.
+
+    One predicate for the same reason the filenames above are defined once: serving validation and
+    the exporter never share a call stack, and each spelling the shapes for itself makes them agree
+    by coincidence. A disagreement fails silently, because validation accepts a shape the exporter
+    skips and the export then ships weights peft loads as a no-op.
+
+    Filename shape only. Whether a SET of these files is loadable is a different question, because a
+    lone shard is not: see :func:`has_loadable_adapter_weights`.
+    """
+    name = filename.rsplit("/", 1)[-1]
+    if name in ADAPTER_WEIGHT_FILES:
+        return True
+    return name.startswith(ADAPTER_SHARD_PREFIX) and name.endswith(ADAPTER_WEIGHT_SUFFIXES)
+
+
+def loadable_adapter_weight_files(filenames) -> list[str]:
+    """The files peft would load out of ``filenames``, or ``[]`` when it could load none.
+
+    The set, not the filename, is what decides this, and the difference is the whole point. peft
+    binds one representation per suffix -- the single ``adapter_model.<ext>`` when present, otherwise
+    the shards -- and it discovers the sharded form ONLY through ``adapter_model.<ext>.index.json``.
+    A directory holding ``adapter_model-00001-of-00002.safetensors`` and no index therefore has no
+    loadable representation at all: nothing tells peft the sharded form exists, or which shards
+    complete it.
+
+    Accepting that orphan is what let an incomplete upload pass validation. Export then falls back to
+    shipping every matching shard when it finds no index, so deployment and export both "succeeded"
+    while producing weights peft loads as a no-op -- the silent-disagreement failure this module
+    exists to prevent, arriving one level up at the set instead of at the name.
+
+    A partial shard set is rejected for the same reason: the index names the complete set, so a
+    listing missing one of its members cannot be loaded either. Residue carrying a second shard total
+    also fails closed. Filenames cannot reveal which total the index references, and guessing risks
+    binding stale weights; cleaning the residue or re-uploading makes the representation decidable.
+
+    Returned in peft's own precedence order, so the caller that reads tensor keys reads exactly the
+    files the serving engine will.
+    """
+    names = {str(name).rsplit("/", 1)[-1] for name in filenames}
+    for suffix in ADAPTER_WEIGHT_SUFFIXES:
+        single = f"adapter_model{suffix}"
+        if single in names:
+            return [single]
+        referenced = _index_shard_names(names, suffix)
+        if referenced:
+            return sorted(referenced)
+    return []
+
+
+def has_loadable_adapter_weights(filenames) -> bool:
+    """True when ``filenames`` holds weights peft can actually load."""
+    return bool(loadable_adapter_weight_files(filenames))
+
+
+def _index_shard_names(names: set[str], suffix: str) -> set[str]:
+    """The shard names an index for ``suffix`` would have to be accompanied by, from names alone.
+
+    Derived from the ``adapter_model-{k}-of-{n}.<ext>`` convention rather than by reading the index
+    file, because every caller here works from a LISTING: serving validates against a remote repo
+    listing and the worker against a directory listing, and neither should have to download and
+    parse json to answer "is this set complete". The exporter, which already has the files locally,
+    still reads the real ``weight_map`` -- that is the authority on which shards are live.
+
+    Empty when no index is present, the sole candidate shard set is incomplete, or more than one
+    candidate total appears. The last case is genuinely undecidable from filenames alone: the index
+    names exactly one set but the listing cannot say which, so rejecting loudly is safer than silently
+    serving stale weights.
+    """
+    if f"adapter_model{suffix}.index.json" not in names:
+        return set()
+    totals = set()
+    oversized_total = False
+    for name in names:
+        if not name.startswith(ADAPTER_SHARD_PREFIX) or not name.endswith(suffix):
+            continue
+        stem = name[len(ADAPTER_SHARD_PREFIX) : -len(suffix)]
+        shard, sep, total = stem.partition("-of-")
+        if not (sep and shard.isdigit() and total.isdigit()):
+            continue
+        parsed_total = int(total)
+        if parsed_total <= 0:
+            continue
+        if parsed_total > len(names):
+            # this total cannot be complete, but it still proves the listing contains residue from a
+            # different save. keep that ambiguity evidence without materializing an unbounded set.
+            oversized_total = True
+            continue
+        totals.add((total, len(shard)))
+    if oversized_total or len(totals) != 1:
+        return set()
+    total, width = next(iter(totals))
+    candidate = {
+        f"{ADAPTER_SHARD_PREFIX}{index:0{width}d}-of-{total}{suffix}"
+        for index in range(1, int(total) + 1)
+    }
+    return candidate if candidate <= names else set()
+
+
+# The largest attempt identity any artifact name may carry. An attempt number reaches a filename and
+# an HF path, so it is bounded rather than merely nonnegative: an unbounded int would still format,
+# and the resulting name is the one thing a later reader has to match exactly.
+MAX_ATTEMPT_ID = (1 << 63) - 1
+
+
+def attempt_scoped_artifact_name(kind: str, phase: str, attempt: int) -> str:
+    """The name of one per-phase, per-attempt worker text artifact.
+
+    The worker uploads these and the control plane reads them back, so the format is a contract
+    between two processes that never share a call stack. It lives here, beside the adapter filenames,
+    because a reader that spells the name for itself agrees with the writer only by coincidence --
+    and the failure is silent: a missing artifact reads as "the worker uploaded nothing", which is
+    exactly what it looks like when the worker crashed. That is the case these files exist to explain.
+
+    Attempt-scoped because ``hf_prefix()`` is per-RUN, not per-attempt: without the suffix a retry
+    would overwrite the attempt that actually reproduced the failure.
+    """
+    if isinstance(attempt, bool) or not isinstance(attempt, int):
+        raise ValueError("attempt must be a nonnegative integer")
+    if attempt < 0 or attempt > MAX_ATTEMPT_ID:
+        raise ValueError("attempt must be a bounded nonnegative integer")
+    return f"{kind}_{phase}_attempt{attempt}.txt"

{freesolo_flash_dev-1.1.68 → freesolo_flash_dev-1.1.70}/flash/adapters/lora_rank.py

@@ -189,8 +189,17 @@ def adapter_artifact_identity(
     token: str | None = None,
     revision: str | None = None,
 ) -> AdapterArtifactIdentity:
-    """Bind adapter config semantics and one required weight object's immutable metadata."""
-    from flash.adapters.artifacts import ADAPTER_WEIGHT_FILES
+    """Bind adapter config semantics and the required weight objects' immutable metadata.
+
+    Binds whatever peft would load, which for a save past its shard size is the complete
+    index-referenced shard set rather than a single file. Asking for the two single-file names by
+    path could not see a sharded adapter at all, so warm-starting from one failed here -- at
+    submission, before the worker that would have loaded it ever ran.
+
+    A single-file adapter's digest is unchanged by that: it still binds exactly its one name and
+    identity, so an in-flight run's stored identity still compares equal.
+    """
+    from flash.adapters.artifacts import loadable_adapter_weight_files
 
     resolved = resolve_adapter_ref(adapter_ref)
     if resolved is None:
@@ -198,26 +207,37 @@ def adapter_artifact_identity(
     repo, prefix = resolved
     config_bytes = _canonical_config_bytes(config)
     config_sha256 = hashlib.sha256(config_bytes).hexdigest()
-    paths = [f"{prefix}/adapter/{name}" for name in ADAPTER_WEIGHT_FILES]
+    folder = f"{prefix}/adapter"
     try:
         from huggingface_hub import HfApi
 
-        infos = HfApi(token=token).get_paths_info(
-            repo, paths=paths, repo_type="dataset", revision=revision
+        infos = list(
+            HfApi(token=token).list_repo_tree(
+                repo_id=repo,
+                path_in_repo=folder,
+                repo_type="dataset",
+                recursive=False,
+                revision=revision,
+            )
         )
     except Exception as exc:
         raise ValueError("could not verify source adapter weight identity") from exc
     by_name = {str(getattr(info, "path", "")).rsplit("/", 1)[-1]: info for info in infos}
-    selected = next((name for name in ADAPTER_WEIGHT_FILES if name in by_name), None)
-    if selected is None:
+    selected = loadable_adapter_weight_files(by_name)
+    if not selected:
         raise ValueError("source adapter has no required weight file")
-    weight_identity = _file_identity(by_name[selected])
-    payload = f"v1\0{config_sha256}\0{selected}\0{weight_identity}".encode()
+    identities = [_file_identity(by_name[name]) for name in selected]
+    # every selected object, because a sharded adapter's content is the union of its shards: binding
+    # only the first would let a later attempt rewrite the rest and still compare equal.
+    bound = "\0".join(
+        f"{name}\0{identity}" for name, identity in zip(selected, identities, strict=True)
+    )
+    payload = f"v1\0{config_sha256}\0{bound}".encode()
     return AdapterArtifactIdentity(
         digest=hashlib.sha256(payload).hexdigest(),
         config_sha256=config_sha256,
-        weight_filename=selected,
-        weight_identity=weight_identity,
+        weight_filename=",".join(selected),
+        weight_identity=",".join(identities),
     )
 
 

{freesolo_flash_dev-1.1.68 → freesolo_flash_dev-1.1.70}/flash/cli/commands/env/eval.py

@@ -2,8 +2,6 @@
 
 from __future__ import annotations
 
-import argparse
-import math
 import queue
 import tempfile
 import threading
@@ -12,6 +10,12 @@ from datetime import UTC, datetime
 from pathlib import Path
 
 from flash._internal.channel import CLI_NAME
+from flash.cli.commands.env.eval_args import (
+    _MAX_CONCURRENCY,
+    bounded_concurrency,
+    finite_float,
+    positive_int,
+)
 from flash.cli.commands.env.push import _err
 from flash.cli.commands.env.test import _env_params, _evaluation_example
 from flash.cli.ui import render
@@ -25,36 +29,6 @@ from flash.envs.evaluations import (
     validate_evaluation_cases,
 )
 
-_MAX_CONCURRENCY = 32
-
-
-def positive_int(value: str) -> int:
-    parsed = int(value)
-    if parsed < 1:
-        raise argparse.ArgumentTypeError("must be at least 1")
-    return parsed
-
-
-def finite_float(value: str) -> float:
-    """A temperature the chat route will accept.
-
-    Reject non-finite or negative values before they become one paid rejection per case.
-    The nonnegative floor matches `flash/schema/__init__.py`.
-    """
-    parsed = float(value)
-    if not math.isfinite(parsed):
-        raise argparse.ArgumentTypeError(f"must be a finite number, got {value}")
-    if parsed < 0.0:
-        raise argparse.ArgumentTypeError(f"must be at least 0.0, got {value}")
-    return parsed
-
-
-def bounded_concurrency(value: str) -> int:
-    parsed = positive_int(value)
-    if parsed > _MAX_CONCURRENCY:
-        raise argparse.ArgumentTypeError(f"must be at most {_MAX_CONCURRENCY}")
-    return parsed
-
 
 def _generation_error(case_id: str, message: str) -> EvalResult:
     return EvalResult(
@@ -159,13 +133,17 @@ def _scored_response(response: str, *, thinking: bool) -> str:
     """The response as a scorer should see it, matching what training grades.
 
     Strip reasoning only for `thinking` runs; raw reasoning mis-grades them, while unconditional
-    stripping can truncate non-thinking answers that mention `<think>`.
+    stripping can truncate non-thinking answers that mention `<think>`. An already-structured value
+    passes through: re-deriving it from its own answer-only string destroys the raw and thinking
+    views only its producer had the generation to build.
     """
     if not thinking:
         return response
     from flash.content.thinking import strip_think, thinking_text
     from flash.envs.adapter import _ScoredResponseText
 
+    if isinstance(response, _ScoredResponseText):
+        return response
     answer = strip_think(response)
     return _ScoredResponseText(
         answer if isinstance(answer, str) else response,
@@ -194,22 +172,28 @@ def _score_case(
     `state_keyword` says which way that suite's signature can take it -- `**kwargs` and
     keyword-only scorers reject a third positional, `*args` scorers reject the keyword -- so the
     caller decides, having read the signature.
+
+    The scorer grades the stripped answer; the RESULT records the raw generation. Stripping for the
+    scorer must not shorten what is recorded, on the error path least of all -- a failed case is
+    when the reasoning that explains it matters most.
     """
+    recorded_response = response
     try:
         scored_text = _scored_response(response, thinking=thinking)
+        recorded_response = getattr(scored_text, "raw", response)
         if state is None:
             scored = suite.score(case, scored_text)
         elif state_keyword:
             scored = suite.score(case, scored_text, state=state)
         else:
             scored = suite.score(case, scored_text, state)
-        return normalize_eval_result(case, response, scored, case_id=case_id)
+        return normalize_eval_result(case, recorded_response, scored, case_id=case_id)
     except (Exception, SystemExit) as exc:
         return EvalResult(
             case_id=case_id,
             passed=False,
             score=0.0,
-            response=response,
+            response=recorded_response,
             error=f"scoring failed: {str(exc) or exc.__class__.__name__}",
         )
 
@@ -992,4 +976,11 @@ def cmd_env_eval(args) -> int:
         )
 
 
-__all__ = ["_MAX_CONCURRENCY", "bounded_concurrency", "cmd_env_eval", "positive_int"]
+# re-exported from eval_args so `flash.cli` keeps importing the eval flag validators from here.
+__all__ = [
+    "_MAX_CONCURRENCY",
+    "bounded_concurrency",
+    "cmd_env_eval",
+    "finite_float",
+    "positive_int",
+]

freesolo_flash_dev-1.1.70/flash/cli/commands/env/eval_args.py

@@ -0,0 +1,43 @@
+"""argparse type validators for `flash env eval`.
+
+Separated from `eval.py` so the evaluation driver stays under the file-size limit. These reject a
+bad flag value at parse time, before the command spends anything: a temperature the chat route will
+refuse costs one paid rejection per case if it is discovered case by case instead.
+"""
+
+from __future__ import annotations
+
+import argparse
+import math
+
+# ceiling on parallel model requests: enough to keep a deployment busy without opening so many
+# connections that the suite is rate limited into slower wall-clock than a smaller fan-out.
+_MAX_CONCURRENCY = 32
+
+
+def positive_int(value: str) -> int:
+    parsed = int(value)
+    if parsed < 1:
+        raise argparse.ArgumentTypeError("must be at least 1")
+    return parsed
+
+
+def finite_float(value: str) -> float:
+    """A temperature the chat route will accept.
+
+    Reject non-finite or negative values before they become one paid rejection per case.
+    The nonnegative floor matches `flash/schema/__init__.py`.
+    """
+    parsed = float(value)
+    if not math.isfinite(parsed):
+        raise argparse.ArgumentTypeError(f"must be a finite number, got {value}")
+    if parsed < 0.0:
+        raise argparse.ArgumentTypeError(f"must be at least 0.0, got {value}")
+    return parsed
+
+
+def bounded_concurrency(value: str) -> int:
+    parsed = positive_int(value)
+    if parsed > _MAX_CONCURRENCY:
+        raise argparse.ArgumentTypeError(f"must be at most {_MAX_CONCURRENCY}")
+    return parsed

{freesolo_flash_dev-1.1.68 → freesolo_flash_dev-1.1.70}/flash/cli/commands/env/retained.py

@@ -11,6 +11,8 @@ import tomllib
 from collections.abc import Callable
 from pathlib import Path
 
+from flash._internal.channel import CLI_NAME
+
 
 def _text(path: Path) -> str:
     """The file's text, or "" if it cannot be decoded as UTF-8.
@@ -40,8 +42,15 @@ def _warn_if_retained_starter_files_describe_another_plane(
     Deliberately project-INDEPENDENT, unlike `_render_starter`, which interpolates the real uuid: a
     directory scaffolded under one project and rerun under another still holds the OLD uuid, so an
     interpolated marker would miss it. Match the project-invariant prefix instead.
+
+    CLI-name-independent for the same reason: the guidance names the executable the operator invoked
+    (`flash`, `flash-cli`, `python -m flash.cli`), so a scaffold written under the alias carries a
+    different command than one written under `flash`. Anchoring on `flash` would silently stop
+    recognising those files as hosted scaffolds -- the retained-file warning is the only thing that
+    tells the operator their guidance now describes the wrong plane, so it must not depend on which
+    entry point happened to write it.
     """
-    push_marker = "`flash env push --project "
+    push_marker = "env push --project "
     # `_render_starter` writes DIFFERENT guidance into each file, so the reverse direction needs
     # both markers: matching only environment.py's would warn about that file while silently
     # keeping a stale evaluations.py beside it.
@@ -63,11 +72,12 @@ def _warn_if_retained_starter_files_describe_another_plane(
         warn(
             f"existing {names} document a self-hosted plane, but this one publishes to the managed "
             "hub; keeping the files unchanged. Their guidance to commit the environment to your own "
-            "git repo does not apply here -- run `flash env push` and use the returned id instead"
+            f"git repo does not apply here -- run `{CLI_NAME} env push` and use the returned id "
+            "instead"
         )
         return
     warn(
-        f"existing {names} still tell you to run `flash env push`, which this plane cannot do; "
+        f"existing {names} still tell you to run `{CLI_NAME} env push`, which this plane cannot do; "
         "keeping the files unchanged. Commit the environment to a git repo your plane can read and "
         "name it in [environment] id"
     )

{freesolo_flash_dev-1.1.68 → freesolo_flash_dev-1.1.70}/flash/cli/commands/env/setup.py

@@ -396,19 +396,23 @@ def _require_setup_project(args) -> str:
 # Guidance the starter .py docstrings carry, per plane kind. The templates hold placeholders
 # rather than one plane's wording plus a rewrite pass: matching prose back out after rendering
 # breaks silently the moment a template is reworded.
+# `CLI_NAME`, not a literal `flash`: the executable is `flash-cli` (or `python -m flash.cli`) when
+# `flash` on PATH belongs to RunPod, and the terminal-facing output below already renders it that
+# way. A generated file that says `flash env push` sends the user to the shadowing binary, which
+# exits 0 without publishing anything -- so the instructions look followed and nothing happened.
 _HOSTED_GUIDANCE = {
     "ENVIRONMENT_GUIDANCE": (
         "upload with\n"
-        "`flash env push --project PROJECT_UUID --name my-env .`.\n"
+        f"`{CLI_NAME} env push --project PROJECT_UUID --name my-env .`.\n"
         "\n"
         "A managed run should use the returned [environment] id from\n"
-        "`flash env push --project PROJECT_UUID --name my-env .`."
+        f"`{CLI_NAME} env push --project PROJECT_UUID --name my-env .`."
     ),
     "EVALUATIONS_GUIDANCE": (
         "Publish this file beside environment.py with\n"
-        "`flash env push --project PROJECT_UUID --name my-env .`, then run the suites against a\n"
-        "model trained on it with `flash env eval TARGET`. The run names the published\n"
-        "environment, so `env eval` takes no local path."
+        f"`{CLI_NAME} env push --project PROJECT_UUID --name my-env .`, then run the suites\n"
+        f"against a model trained on it with `{CLI_NAME} env eval TARGET`. The run names the\n"
+        "published environment, so `env eval` takes no local path."
     ),
 }
 

{freesolo_flash_dev-1.1.68 → freesolo_flash_dev-1.1.70}/flash/engine/worker/__init__.py

@@ -91,7 +91,7 @@ from flash.engine.worker.train.rl.config import (
     grpo_overrides,
     resolve_grpo_prompts_per_step,
 )
-from flash.envs.adapter import GitHubRateLimitError
+from flash.envs.adapter import GitHubTransientError
 from flash.envs.base import load_environment
 from flash.teacher.retry_contract import OPD_RESUME_REVISION_ENV
 
@@ -213,7 +213,10 @@ def require_active_env():
 
 
 def _worker_failure_flags(exc: BaseException) -> dict[str, bool]:
-    retriable = isinstance(exc, (RetriableInfraError, GitHubRateLimitError))
+    # GitHubTransientError covers both the quota case and GitHub being unreachable. The worker's
+    # answer to either is the same -- reschedule -- so it catches the base; only the control plane,
+    # which has a status code to report, distinguishes them.
+    retriable = isinstance(exc, (RetriableInfraError, GitHubTransientError))
     return {"retriable": retriable, "oom": (not retriable and is_cuda_oom(exc))}
 
 

{freesolo_flash_dev-1.1.68 → freesolo_flash_dev-1.1.70}/flash/engine/worker/io/hf.py

@@ -14,7 +14,7 @@ import time
 from collections.abc import Callable
 
 from flash._internal.diagnostics import sanitize_diagnostic
-from flash.adapters.artifacts import ADAPTER_WEIGHT_FILES, attempt_scoped_artifact_name
+from flash.adapters.artifacts import attempt_scoped_artifact_name, has_loadable_adapter_weights
 from flash.engine.profiling.tokenizer import (  # noqa: F401
     load_tokenizer,
     model_revision_kwargs,
@@ -324,10 +324,20 @@ _CHECKPOINT_TRAINER_STATE = (
 
 
 def _has_deployable_adapter(ckpt_dir: str) -> bool:
-    """Return True if ckpt_dir has a loadable LoRA adapter (config + weights)."""
-    return os.path.isfile(os.path.join(ckpt_dir, "adapter_config.json")) and any(
-        os.path.isfile(os.path.join(ckpt_dir, w)) for w in ADAPTER_WEIGHT_FILES
-    )
+    """Return True if ckpt_dir has a loadable LoRA adapter (config + weights).
+
+    Weights via the shared rule rather than the two single-file names, because a save past peft's
+    shard size writes ``adapter_model-0000N-of-0000M.<ext>`` plus an index instead. Spelling only the
+    single-file names here made this the strict side of a disagreement: serving and export accept the
+    sharded save, so a sharded per-step adapter was silently skipped -- or failed a required save --
+    for an artifact the rest of the pipeline would have deployed.
+    """
+    if not os.path.isfile(os.path.join(ckpt_dir, "adapter_config.json")):
+        return False
+    try:
+        return has_loadable_adapter_weights(os.listdir(ckpt_dir))
+    except OSError:
+        return False
 
 
 def _write_deployable_provenance(ckpt_dir: str) -> None:

freesolo_flash_dev-1.1.70/flash/engine/worker/model/lora.py

@@ -0,0 +1,118 @@
+"""read adapter tensor keys from a downloaded adapter directory, without loading tensor data.
+
+helpers must not import ``flash.engine.worker``. heavy dependencies remain lazy so this leaf
+module has no package cycle or eager gpu stack import.
+"""
+
+from __future__ import annotations
+
+# A safetensors header is small even for huge models (a few hundred KB at most); 100 MB is a wildly
+# generous ceiling that still refuses a corrupt/hostile file declaring a multi-GB header length
+# before we allocate/read it.
+_MAX_SAFETENSORS_HEADER_BYTES = 100 * 1024 * 1024
+
+
+def _read_adapter_tensor_keys(adir: str) -> list[str] | None:
+    """return tensor key names from adapter weights without loading tensor data.
+
+    read only safetensors json headers or torch weights-only state-dict keys. return none if absent.
+
+    reads whatever peft would bind, which is not always one file: a save past peft's shard size
+    writes ``adapter_model-0000N-of-0000M.<ext>`` beside an index, and the keys then live spread
+    across the shards. reading only the two single-file names reported such an adapter as having no
+    keys at all, so ``_warmstart_adapter_is_loadable`` rejected a warm-start source that serving and
+    export both accept.
+    """
+    import os
+
+    from flash.adapters.artifacts import loadable_adapter_weight_files
+
+    try:
+        selected = loadable_adapter_weight_files(os.listdir(adir))
+    except OSError:
+        return None
+    if not selected:
+        return None
+    keys: list[str] = []
+    for name in selected:
+        path = os.path.join(adir, name)
+        if name.endswith(".safetensors"):
+            keys.extend(_read_safetensors_tensor_keys(path))
+        else:
+            keys.extend(_read_bin_tensor_keys(path))
+    return keys
+
+
+def _read_safetensors_tensor_keys(st_path: str) -> list[str]:
+    """the tensor keys declared by one safetensors file's header."""
+    import json
+    import os
+    import struct
+
+    # safetensors layout: 8-byte LE header length, then the JSON header, then the tensor data.
+    # Bound the DECLARED header length against the real file size (and an absolute ceiling)
+    # BEFORE reading it, so a corrupt/hostile file can't trigger a huge allocation / long read.
+    file_size = os.path.getsize(st_path)
+    with open(st_path, "rb") as f:
+        len_bytes = f.read(8)
+        if len(len_bytes) < 8:
+            raise ValueError(f"{st_path}: too small to be a safetensors file")
+        (hdr_len,) = struct.unpack("<Q", len_bytes)
+        if hdr_len > file_size - 8 or hdr_len > _MAX_SAFETENSORS_HEADER_BYTES:
+            raise ValueError(
+                f"{st_path}: declared safetensors header length {hdr_len} is implausible "
+                f"(file is {file_size} bytes) — refusing to read a corrupt/oversized header"
+            )
+        header_bytes = f.read(hdr_len)
+        if len(header_bytes) < hdr_len:
+            raise ValueError(f"{st_path}: truncated safetensors header")
+        try:
+            header = json.loads(header_bytes)
+        except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+            # A bare JSONDecodeError ("Expecting value: line 1 column 1") — or a
+            # UnicodeDecodeError from non-UTF8 header bytes — gives no clue WHICH adapter is
+            # corrupt. Re-raise with the file path so a bad download is diagnosable.
+            raise ValueError(
+                f"{st_path}: safetensors header is not valid JSON "
+                f"(corrupt or not a safetensors file): {exc}"
+            ) from exc
+    # The safetensors header MUST be a JSON object keyed by tensor name. A corrupt/hostile file
+    # could decode to a list/int/str, which would later blow up with a confusing TypeError
+    # downstream (substring search on a non-str). (JSON object keys are always str, so only the
+    # container type needs checking.) Reject a non-object header early with a clear message.
+    if not isinstance(header, dict):
+        raise ValueError(
+            f"{st_path}: safetensors header is not a JSON object "
+            "(corrupt or not a safetensors file)"
+        )
+    data_size = file_size - 8 - hdr_len
+    for key, tensor_info in header.items():
+        if key == "__metadata__":
+            continue
+        offsets = tensor_info.get("data_offsets") if isinstance(tensor_info, dict) else None
+        if (
+            not isinstance(offsets, list)
+            or len(offsets) != 2
+            or any(not isinstance(offset, int) or isinstance(offset, bool) for offset in offsets)
+            or offsets[0] < 0
+            or offsets[0] > offsets[1]
+            or offsets[1] > data_size
+        ):
+            raise ValueError(f"{st_path}: invalid or truncated tensor data for {key!r}")
+    return [k for k in header if k != "__metadata__"]
+
+
+def _read_bin_tensor_keys(bin_path: str) -> list[str]:
+    """the tensor keys in one torch ``.bin`` adapter state dict, without loading tensor data."""
+    import torch
+
+    state = torch.load(bin_path, map_location="cpu", weights_only=True)
+    if not isinstance(state, dict):
+        raise ValueError(f"{bin_path}: expected a tensor state dict, got {type(state).__name__}")
+    bad = [k for k, v in state.items() if not isinstance(k, str) or not isinstance(v, torch.Tensor)]
+    if bad:
+        raise ValueError(
+            f"{bin_path}: contains non-tensor entries (e.g. {bad[:4]}); "
+            "expected a plain PEFT adapter state dict"
+        )
+    return list(state)

{freesolo_flash_dev-1.1.68 → freesolo_flash_dev-1.1.70}/flash/engine/worker/sft_train.py

@@ -448,7 +448,15 @@ def _write_sft_result(options, data, model, child, progress, verified, outputs)
             # share of the batch, so a reader reconstructing the token budget off the request would
             # believe each rank held rows it never received.
             "per_device_train_batch_size": child.micro_batch,
-            "gradient_accumulation_steps": math.ceil(model.train_batch_size / child.micro_batch),
+            # over one RANK'S share of the batch, not the global batch. `micro_batch` is already
+            # capped to `train_batch_size // world_size` (see `_resolve_sft_width_and_micro_batch`),
+            # so dividing the global batch by it was the sequence-parallel formula, where every rank
+            # sees the whole batch. under data parallelism it over-counts by the world size, and a
+            # reader reconstructing tokens as micro-batch x grad-accum x DP size lands world_size
+            # times too high.
+            "gradient_accumulation_steps": math.ceil(
+                (model.train_batch_size / max(1, child.world_size)) / child.micro_batch
+            ),
             # verl concatenates either way; the profile's mode records whether more than one
             # example was allowed to share a concatenated batch, which is what a reader of these
             # metrics needs in order to compare a run's step count against its row count.