freesolo-flash-dev 1.2.95__tar.gz → 1.2.97__tar.gz

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.95
+Version: 1.2.97
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/flash/cli/commands/__init__.py

@@ -7,6 +7,7 @@ import os
 import sys
 import time
 import uuid
+from typing import NamedTuple
 
 from flash import __version__
 from flash._internal.channel import BRAND_NAME, CLI_NAME
@@ -424,6 +425,8 @@ def cmd_train(args) -> int:
             )
         else:
             print(json.dumps(status, indent=2))
+        if spec.algorithm == "sft":
+            _print_published_sft_environment_note(status, spec)
         _print_unpacked_batch_warning(status, spec)  # after the payload, so stdout stays parseable
         print_status_prompt_budget_warning(status)
         _print_reasoning_loss_warning(status)
@@ -599,12 +602,17 @@ def _log_follow_metric_rows(status: dict | None, seen_steps: set) -> list[str]:
     return rows
 
 
-def _poll_logs(client: ApiClient, run_id: str, interval: float) -> tuple[str, bool]:
-    """Stream offset-paged logs until the run reaches a terminal state.
+class _LogPollResult(NamedTuple):
+    state: str
+    printed_any: bool
+    live_attempt: int | None
 
-    Returns (terminal state, whether any log bytes were printed)."""
+
+def _poll_logs(client: ApiClient, run_id: str, interval: float) -> _LogPollResult:
+    """Stream logs until terminal and return the final state, output, and attempt snapshot."""
     offset = 0
     printed_any = False
+    attempt: int | None = None
     last_progress: str | None = None
     seen_metric_steps: set = set()
     spinner = _LogFollowSpinner(run_id)
@@ -620,6 +628,7 @@ def _poll_logs(client: ApiClient, run_id: str, interval: float) -> tuple[str, bo
             # from the run status endpoint. The log page's embedded state is only a fallback for
             # older servers or test doubles.
             status = client.get_run(run_id)
+            attempt = render.live_attempt(status) if isinstance(status, dict) else attempt
             state, progress = _log_follow_progress(status, str(page.get("state") or ""))
             metric_rows = _log_follow_metric_rows(status, seen_metric_steps)
             if metric_rows:
@@ -628,7 +637,7 @@ def _poll_logs(client: ApiClient, run_id: str, interval: float) -> tuple[str, bo
                     print(row, file=sys.stderr, flush=True)
             if state in _CLI_DONE_STATES:
                 spinner.clear()
-                return state, printed_any
+                return _LogPollResult(state, printed_any, attempt)
             if not spinner.enabled and progress != last_progress:
                 print(f"status: {progress}", file=sys.stderr, flush=True)
                 last_progress = progress
@@ -677,12 +686,12 @@ def _print_detached_note(run_id: str) -> None:
 def _follow_run(client: ApiClient, run_id: str) -> int:
     """Poll logs until the run reaches a terminal state, then print the final status."""
     try:
-        state, _ = _poll_logs(client, run_id, interval=2.0)
+        result = _poll_logs(client, run_id, interval=2.0)
     except KeyboardInterrupt:
         _print_detached_note(run_id)
         return 130
     print(_render_status(client.get_run(run_id)))
-    return 0 if state in _OK_STATES else 1
+    return 0 if result.state in _OK_STATES else 1
 
 
 def _follow_status(
@@ -710,16 +719,23 @@ def cmd_log(args) -> int:
     client = client_from_config()
     if getattr(args, "follow", False):
         try:
-            state, printed_any = _poll_logs(client, args.run_id, interval=2.0)
+            result = _poll_logs(client, args.run_id, interval=2.0)
         except KeyboardInterrupt:
             _print_detached_note(args.run_id)
             return 130
-        _print_worker_output(client, args.run_id, printed_any=printed_any)
-        return 0 if state in _OK_STATES else 1
+        sections = _worker_sections(client, args.run_id)
+        _print_worker_output(
+            sections,
+            printed_any=result.printed_any,
+            current_attempt=result.live_attempt,
+        )
+        return 0 if result.state in _OK_STATES else 1
     text = str(client.get_logs(args.run_id, offset=0).get("logs") or "")
     if text:
         print(text, end="" if text.endswith("\n") else "\n")
-    _print_worker_output(client, args.run_id, printed_any=bool(text))
+    sections = _worker_sections(client, args.run_id)
+    attempt = _snapshot_live_attempt(client, args.run_id) if sections else None
+    _print_worker_output(sections, printed_any=bool(text), current_attempt=attempt)
     return 0
 
 
@@ -953,6 +969,7 @@ from flash.cli.commands.train_cost import (  # noqa: E402,F401
     _cmd_train_cost,
     _cmd_train_cost_offline,
     _cmd_train_cost_sft,
+    _print_published_sft_environment_note,
     _print_reasoning_loss_warning,
     _print_sft_cost,
     _print_train_schema_compatibility,
@@ -961,9 +978,11 @@ from flash.cli.commands.train_cost import (  # noqa: E402,F401
     _warn_if_wandb_requested_without_key,
 )
 
-# re-exported so `commands._print_worker_output(...)` keeps resolving for `cmd_log` and the CLI tests.
+# re-exported because cmd_log and focused cli tests address these through the command package.
 from flash.cli.commands.worker_output import (  # noqa: E402,F401
     _artifact_attempt,
     _print_worker_output,
+    _snapshot_live_attempt,
     _worker_section_name,
+    _worker_sections,
 )

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/flash/cli/commands/train_cost.py

@@ -114,8 +114,19 @@ def _client_train_schema(authored_train_keys: frozenset[str]) -> dict:
     }
 
 
+def _has_inline_records(spec) -> bool:
+    """Whether this config supplied its SFT rows inline rather than from the resolved package.
+
+    One predicate for every surface that attributes the counts -- the cost rows and the provenance
+    note sit within a few lines of each other, so disagreeing about where the data came from would
+    print two contradictory answers in one quote.
+    """
+    params = getattr(getattr(spec, "environment", None), "params", None)
+    return isinstance(params, dict) and params.get("records") is not None
+
+
 def _sft_cost_rows(spec, profile: dict) -> list[tuple[str, str | None]]:
-    """Rows describing the packaged-dataset estimate behind an SFT quote.
+    """Rows describing the published packaged-dataset estimate behind an SFT quote.
 
     Only aggregates the server actually returned are shown. A field that is absent is dropped
     rather than defaulted, so the panel never reports a count the profile did not compute.
@@ -125,19 +136,34 @@ def _sft_cost_rows(spec, profile: dict) -> list[tuple[str, str | None]]:
         value = profile.get(key)
         return value if isinstance(value, int) and not isinstance(value, bool) else None
 
+    def text(key: str) -> str:
+        value = profile.get(key)
+        return value.strip() if isinstance(value, str) else ""
+
     steps = count("authoritative_steps")
+    source = count("source_examples")
     retained, selected = count("retained_examples"), count("selected_examples")
     dropped = count("dropped_examples")
     compute, supervised = (
         count("authoritative_compute_tokens"),
         count("authoritative_supervised_tokens"),
     )
-    packing = str(profile.get("packing_mode") or "")
-    architecture = str(profile.get("architecture_mode") or "")
-    digest = str(profile.get("content_digest") or "")
+    packing = text("packing_mode")
+    architecture = text("architecture_mode")
+    environment_id = text("environment_id")
+    environment_revision = text("environment_revision")
+    digest = text("content_digest")
+    # inline `[environment.params] records` supply the rows from the request body, so the resolved
+    # package contributed the environment but not the dataset. labelling those counts "published
+    # copy" names a source they did not come from. one adjective decides all three rows, so they
+    # cannot describe the same quote two different ways.
+    inline_records = _has_inline_records(spec)
+    origin = "inline records" if inline_records else "published copy"
+    resolved = "resolved" if inline_records else "published"
     examples = None
     if retained is not None and selected is not None:
-        examples = f"{retained:,} trained of {selected:,} selected"
+        examples = f"{retained:,} trained of {selected:,} selected from "
+        examples += f"{source:,} source rows in {origin}" if source is not None else origin
         if dropped:
             examples += f" ({dropped:,} dropped)"
     tokens = None
@@ -147,10 +173,15 @@ def _sft_cost_rows(spec, profile: dict) -> list[tuple[str, str | None]]:
             tokens += f", {supervised:,} supervised"
     return [
         ("run", f"{spec.model}  [SFT{f', {steps} steps' if steps is not None else ''}]"),
+        ("env", f"{resolved} environment {environment_id}" if environment_id else None),
+        (
+            "revision",
+            f"{environment_revision[:12]} ({resolved} commit)" if environment_revision else None,
+        ),
         ("workload", f"{packing} ({architecture})" if packing and architecture else None),
         ("examples", examples),
         ("tokens", tokens),
-        ("profile", digest[:12] or None),
+        ("digest", digest[:12] or None),
     ]
 
 
@@ -218,6 +249,71 @@ def _print_reasoning_loss_warning(status: object) -> None:
     print(render.warn(message) if render.styled() else f"warning: {message}", file=sys.stderr)
 
 
+def _print_published_sft_environment_note(status: object, spec) -> None:
+    """Warn when SFT counts came from a published managed or GitHub environment."""
+    if _has_inline_records(spec):
+        return
+
+    profile = status.get("workload_profile") if isinstance(status, dict) else None
+    if not isinstance(profile, dict):
+        return
+    environment_id = profile.get("environment_id")
+    revision = profile.get("environment_revision")
+    if not isinstance(environment_id, str) or not environment_id.strip():
+        return
+    if not isinstance(revision, str) or not revision.strip():
+        return
+
+    environment_id = environment_id.strip()
+    advice = _republish_advice(environment_id)
+    if advice is None:
+        return
+
+    source = profile.get("source_examples")
+    source_suffix = (
+        f" ({source:,} source rows)"
+        if isinstance(source, int) and not isinstance(source, bool)
+        else ""
+    )
+    message = (
+        f"published environment: {environment_id} @ {revision.strip()[:12]}{source_suffix}. "
+        "SFT dataset counts come from this resolved published copy, not local files. "
+        f"{advice}"
+    )
+    print(render.note(message) if render.styled() else message, file=sys.stderr)
+
+
+def _republish_advice(environment_id: str) -> str | None:
+    """Return conditional remediation for a managed or GitHub environment id."""
+    from flash.envs.identity import (
+        canonical_managed_environment_slug,
+        github_environment_ref_is_pinned,
+        is_github_environment_ref,
+    )
+
+    prefix = "If you expected local dataset edits to be included, "
+    try:
+        managed_slug = canonical_managed_environment_slug(environment_id)
+    except ValueError:
+        return None
+    if managed_slug is not None:
+        return (
+            f"{prefix}run `{_commands().CLI_NAME} env push --name NAME "
+            "--project PROJECT_UUID [path]` again for this managed environment."
+        )
+    if not is_github_environment_ref(environment_id):
+        return None
+    if github_environment_ref_is_pinned(environment_id):
+        return (
+            f"{prefix}publish the new commit, then update [environment] id to the full new commit "
+            "SHA; the existing immutable SHA cannot move."
+        )
+    return (
+        f"{prefix}make the named remote branch or tag point at the new commit. A tag may instead "
+        "need a new tag and an updated [environment] id rather than moving the existing tag."
+    )
+
+
 def _print_sft_cost(status: dict, spec) -> None:
     total = status.get("estimated_cost_usd") if isinstance(status, dict) else None
     if not isinstance(total, (int, float)) or isinstance(total, bool):
@@ -234,6 +330,7 @@ def _print_sft_cost(status: dict, spec) -> None:
             if value is not None:
                 print(f"{key.ljust(8)}: {value}")
         print(f"{'TOTAL'.ljust(8)}: ${float(total):.2f}")
+    _print_published_sft_environment_note(status, spec)
     print(
         "tokens, retained rows, truncation, and optimizer steps are estimated from packaged "
         "input/output fields plus contract_text, contract_path, or TRAINING_CONTRACT.md. other "

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/flash/cli/commands/worker_output.py

@@ -17,8 +17,11 @@ import re
 from typing import TYPE_CHECKING
 
 from flash.cli.ui import render
+from flash.client import ClientError
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping
+
     from flash.client import ApiClient
 
 
@@ -42,18 +45,24 @@ def _worker_section_name(name: str, current_attempt: int | None) -> str:
     return f"{name} (attempt={attempt}, current attempt)"
 
 
-def _print_worker_output(client: ApiClient, run_id: str, *, printed_any: bool = False) -> bool:
-    worker_output = client.get_worker_output(run_id) or {}
-    if not worker_output:
-        return printed_any
+def _worker_sections(client: ApiClient, run_id: str) -> dict[str, str]:
+    """Fetch only worker artifacts with printable text."""
+    return {name: text for name, text in (client.get_worker_output(run_id) or {}).items() if text}
+
+
+def _snapshot_live_attempt(client: ApiClient, run_id: str) -> int | None:
+    """Read the live attempt without letting a heading lookup hide diagnostic artifacts."""
     try:
-        current_attempt = render.live_attempt(client.get_run(run_id) or {})
-    except Exception:
-        # artifact text is best-effort diagnostic output; a status refresh failure must not hide it.
-        current_attempt = None
-    for name, text in worker_output.items():
-        if not text:
-            continue
+        return render.live_attempt(client.get_run(run_id) or {})
+    except ClientError:
+        return None
+
+
+def _print_worker_output(
+    sections: Mapping[str, str], *, printed_any: bool = False, current_attempt: int | None = None
+) -> bool:
+    """Print worker artifacts under headings naming the attempt that produced them."""
+    for name, text in sections.items():
         # label provenance from the filename rather than letting the final section, which is merely
         # the last one appended, read as the live attempt's output.
         section_name = _worker_section_name(name, current_attempt)

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/flash/engine/worker/__init__.py

@@ -145,6 +145,9 @@ _HB_LAST_PROGRESS_TS = 0.0
 # per-step heartbeat past the provider's stall window, killing a healthy run.
 _HB_PROGRESS_SEQ = 0
 _HB_PROGRESS_UPLOADED_SEQ = 0
+# a replace-in-place failure heartbeat can be overwritten before the provider polls it. retain the
+# failed checkpoint identity and cause so every terminal heartbeat carries the same structured state.
+_HB_PENDING_CHECKPOINT_FAILURE: dict[str, int | str] | None = None
 # Environment-scoped artifact repos are shared by many runs. Keep heartbeat commits below the HF
 # per-repo commit cap while staying under the provider poller's training stall window
 # (STALL_AFTER_S=1500s in flash/providers/_lifecycle/poll.py).

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/flash/engine/worker/io/heartbeat.py

@@ -78,15 +78,27 @@ _HB_THROTTLED_STAGES = _HB_TIGHT_LIVENESS_STAGES | frozenset({"rl_step", "sft_st
 # right after a download that has been pinging throughout, so the throttle would almost always
 # swallow it. the wrap that follows re-emits with liveness=True and stays throttled.
 _HB_MODEL_LOAD_STAGES = frozenset({"sft_model_load", "opd_model_load"})
-_HB_TERMINAL_STAGES = frozenset({"done", "already_done"})
+_HB_SUCCESS_TERMINAL_STAGES = frozenset({"done", "already_done"})
 # 600s -> ~6 commits/hr; keeps stall detector alive without hitting the HF commit cap.
 _HB_TERMINAL_ONLY_INTERVAL_S = 600.0
 
 
+def _is_terminal_stage(stage: str) -> bool:
+    """return whether this stage ends the worker attempt."""
+    return stage in _HB_SUCCESS_TERMINAL_STAGES or stage.startswith("error_")
+
+
 def _is_critical_stage(stage: str) -> bool:
     """A terminal transition or an error is CRITICAL: never throttled (the commit must land) and
-    given the longer upload-lock timeout, because no later heartbeat can repair a missed one."""
-    return stage in _HB_TERMINAL_STAGES or stage.startswith("error_")
+    given the longer upload-lock timeout, because no later heartbeat can repair a missed one.
+
+    `_failed` earns this as much as the `error_` prefix does: a failure heartbeat is emitted once, at
+    the moment the failure is known, and nothing later restates it. `checkpoint_upload_failed` is
+    raised from inside a long HF upload, which is precisely when `_HB_UPLOAD_IN_FLIGHT` is set and an
+    unforced ping is dropped -- so the one report of the failure would be discarded exactly in the
+    case that produces it.
+    """
+    return _is_terminal_stage(stage) or stage.endswith("_failed")
 
 
 # Guards throttle bookkeeping; slow HF commit runs outside this lock so heartbeat and liveness
@@ -210,6 +222,17 @@ def heartbeat(
 ):
     genuine_progress = not liveness
     with _HB_LOCK:
+        if stage == "checkpoint_upload_failed":
+            failure = kw.get("checkpoint_failure")
+            if isinstance(failure, dict):
+                _w._HB_PENDING_CHECKPOINT_FAILURE = dict(failure)
+        elif stage == "checkpoint_uploaded":
+            # a later full resume checkpoint landed, so the earlier failure no longer describes the
+            # run's outcome. deployable or final adapter publication cannot clear this because neither
+            # restores the missing full resume state.
+            _w._HB_PENDING_CHECKPOINT_FAILURE = None
+        elif _is_terminal_stage(stage) and _w._HB_PENDING_CHECKPOINT_FAILURE:
+            kw.setdefault("checkpoint_failure", dict(_w._HB_PENDING_CHECKPOINT_FAILURE))
         ts = time.time()
         if genuine_progress:
             _w._HB_LAST_PROGRESS_TS = ts

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/flash/engine/worker/io/hf.py

@@ -553,7 +553,17 @@ def upload_resume_checkpoint(
                         )
                         if emit_heartbeat:
                             with contextlib.suppress(Exception):
-                                _w.heartbeat("checkpoint_upload_failed", step=step)
+                                # worker stdout is not part of control-plane run logs, so the stage
+                                # name alone would otherwise be the entire failure report.
+                                _w.heartbeat(
+                                    "checkpoint_upload_failed",
+                                    step=step,
+                                    checkpoint_failure={
+                                        "step": step,
+                                        "operation": failure_stage,
+                                        "error": sanitize_diagnostic(e, limit=300),
+                                    },
+                                )
                         if failure_stage in {"before", "after"}:
                             raise
     return False

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/flash/envs/identity.py

@@ -255,3 +255,13 @@ def _is_safe_github_path_parts(parts: list[str] | tuple[str, ...]) -> bool:
 def is_commit_sha(value: str) -> bool:
     """True when value is a full 40-hex-char git commit id (an immutable ref)."""
     return _COMMIT_SHA_RE.fullmatch(value) is not None
+
+
+def github_environment_ref_is_pinned(value: str) -> bool:
+    """True when a ``github:`` environment id names an immutable commit.
+
+    Only the SHA case is decidable from the id. Branches and tags share one string shape, so callers
+    must give symbolic-ref advice that is valid for either rather than assuming the ref is movable.
+    """
+    parsed = _parse_github_environment_ref(value)
+    return parsed is not None and is_commit_sha(parsed.ref)

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/flash/providers/_lifecycle/poll.py

@@ -262,7 +262,25 @@ def _format_heartbeat(hb: dict) -> str:
         for name, value in reward_metrics.items():
             if isinstance(value, (int, float)):
                 msg += f" {name}={value:.3f}"
+    error = hb.get("error")
+    if isinstance(error, str) and error.strip():
+        msg += f" error={error.strip()}"
+    checkpoint_failure = hb.get("checkpoint_failure")
+    if isinstance(checkpoint_failure, dict):
+        failure_step = checkpoint_failure.get("step")
+        if isinstance(failure_step, (int, float)) and not isinstance(failure_step, bool):
+            msg += f" checkpoint_failure_step={int(failure_step)}"
+        operation = checkpoint_failure.get("operation")
+        if isinstance(operation, str) and operation.strip():
+            msg += f" checkpoint_failure_stage={operation.strip()}"
+        checkpoint_error = checkpoint_failure.get("error")
+        if isinstance(checkpoint_error, str) and checkpoint_error.strip():
+            msg += f" checkpoint_error={checkpoint_error.strip()}"
     msg += format_gpu_status(hb.get("gpu"))
+    # everything above is one summary record, including untrusted stage, error, and gpu fallback text.
+    # sanitize it once at that boundary so no field can forge a second worker record. the deliberately
+    # multiline sample and child-output sections are appended below and retain their line breaks.
+    msg = neutralize_control_chars(msg).replace("\n", "\\n")
     sample_lines: list[str] = []
     rendered_samples = 0
     samples = hb.get("sampled_completions")

{freesolo_flash_dev-1.2.95 → freesolo_flash_dev-1.2.97}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.95"
+version = "1.2.97"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -137,7 +137,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.95"
+version = "1.2.97"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]