freesolo-flash-dev 0.2.58__tar.gz → 1.0.0__tar.gz

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 0.2.58
+Version: 1.0.0
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.0}/flash/cost/analytical.py

@@ -40,6 +40,22 @@ MFU_TRAIN = 0.35  # GRPO policy/reference update
 MFU_SFT_TRAIN = 0.25  # SFT fwd/bwd (smaller effective batch, long sequences)
 MFU_DECODE = 0.12  # batched vLLM rollout (decode is memory-bandwidth-bound)
 
+# --- MoE (mixture-of-experts) per-step correction ----------------------------------------------
+# For an MoE model (active params << total, e.g. Qwen3.6-35B-A3B: ~3B active / 35B total) the wall
+# time per step is NOT the tiny active-param FLOPs the dense model predicts. Routing, all-expert
+# coordination, and grouped-GEMM under-utilization at small batch make the step scale with TOTAL
+# params. Pricing 35B-A3B on active params under-quoted real RunPod runs by 13-27x (SFT ~1.2s vs
+# ~42s realized/step; GRPO ~1.4s vs ~24s). So an MoE prices on TOTAL params at a reduced effective
+# MFU + a per-step overhead + a one-time compile. DENSE models (active == total) are unaffected --
+# they keep the original active-param path and the MFUs above.
+MFU_SFT_TRAIN_MOE = 0.10  # MoE SFT fwd/bwd priced on total params
+MFU_TRAIN_MOE = 0.09  # MoE GRPO/OPD policy update priced on total params
+MOE_STEP_OVERHEAD_S = 2.0  # routing/dispatch/kernel-launch overhead an MoE pays every step
+# One-time kernel/graph compile amortized into the first training step (fused Triton kernels;
+# + vLLM cudagraph capture for rollout methods). MoE-only; the prior model omitted it entirely.
+COMPILE_MOE_SFT_S = 35.0
+COMPILE_MOE_ROLLOUT_S = 48.0  # GRPO / OPD (adds vLLM cudagraph capture)
+
 # Reward grading is CONCURRENT: a step's completions score in parallel slots, so the reward
 # wall is ceil(completions / slots) waves x latency, not completions x latency.
 REWARD_CONCURRENCY = 16.0
@@ -116,13 +132,32 @@ def required_save_overhead_seconds(config: RunConfig) -> float:
     return len(n.save_at_steps) * per_save
 
 
+def _is_moe(model_id: str) -> bool:
+    """True when the model routes each token through a subset of experts (active < total params)."""
+    return active_params_b(model_id) < total_params_b(model_id)
+
+
+def compile_seconds(config: RunConfig, gpu: str) -> float:
+    """One-time kernel/graph compile folded into the first training step. MoE-only; 0 for dense
+    (whose original timing did not model it). Larger for rollout methods (add vLLM cudagraph
+    capture)."""
+    _ = gpu
+    if not _is_moe(config.model_id):
+        return 0.0
+    return COMPILE_MOE_SFT_S if config.method == "sft" else COMPILE_MOE_ROLLOUT_S
+
+
 def seconds_per_step(config: RunConfig, gpu: str) -> float:
     """Steady-state wall time for one optimizer step on ``gpu``."""
     n = config.normalized()
-    # Per-token FLOPs scale with the ACTIVE params (an MoE token routes through only a subset of
-    # experts); for a dense model this equals the total. Memory/size terms below keep total_params_b.
-    params = active_params_b(n.model_id) * 1e9
     peak = gpu_tflops(gpu) * 1e12  # FLOP/s
+    # An MoE's per-step wall scales with TOTAL params (routing + all-expert coordination + grouped
+    # GEMM under-utilization), not the tiny active-param FLOPs; dense models keep active (== total).
+    moe = _is_moe(n.model_id)
+    params = (total_params_b(n.model_id) if moe else active_params_b(n.model_id)) * 1e9
+    overhead = MOE_STEP_OVERHEAD_S if moe else 0.0
+    sft_mfu = MFU_SFT_TRAIN_MOE if moe else MFU_SFT_TRAIN
+    update_mfu = MFU_TRAIN_MOE if moe else MFU_TRAIN
 
     if n.is_opd:
         # OPD step = on-policy student rollout (like GRPO) + remote teacher scoring (CONCURRENT
@@ -131,7 +166,7 @@ def seconds_per_step(config: RunConfig, gpu: str) -> float:
         # sequence (see _opd_step_shape), not completion-only, or long-prompt opd is underquoted.
         completions, seq_tokens = _opd_step_shape(n)
         gen_s = (GRPO_GEN_FLOPS_PER_TOKEN_PER_PARAM * params * seq_tokens) / (peak * MFU_DECODE)
-        update_s = (OPD_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * seq_tokens) / (peak * MFU_TRAIN)
+        update_s = (OPD_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * seq_tokens) / (peak * update_mfu)
         teacher_lat = teacher_seconds_per_completion()
         # run_opd's primary path scores a step's completions CONCURRENTLY over Fireworks with a fan-out
         # cap of the step's OWN completion count (prompts_per_step * group_size, opd.py Phase 2), so
@@ -139,30 +174,33 @@ def seconds_per_step(config: RunConfig, gpu: str) -> float:
         # latency, NOT the full serial sum (that describes only the CPU-test fallback that can't
         # batch-generate). The teacher endpoint's rate limit is the real ceiling on this fan-out.
         teacher_s = teacher_lat
-        return gen_s + teacher_s + update_s
+        return overhead + gen_s + teacher_s + update_s
 
     if not n.is_grpo:
         flops = SFT_FLOPS_PER_TOKEN_PER_PARAM * params * (n.batch_size * n.seq_len)
-        return flops / (peak * MFU_SFT_TRAIN)
+        return overhead + flops / (peak * sft_mfu)
 
     # GRPO step = rollout (G completions/prompt) + concurrent reward grading + policy/ref update.
     completions = n.batch_size * n.group_size
     gen_tokens = completions * n.completion_len
     gen_s = (GRPO_GEN_FLOPS_PER_TOKEN_PER_PARAM * params * gen_tokens) / (peak * MFU_DECODE)
-    update_s = (GRPO_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * gen_tokens) / (peak * MFU_TRAIN)
+    update_s = (GRPO_UPDATE_FLOPS_PER_TOKEN_PER_PARAM * params * gen_tokens) / (peak * update_mfu)
     latency = reward_seconds_per_completion(n.reward_seconds_per_completion)
     reward_s = (
         math.ceil(completions / REWARD_CONCURRENCY) * latency
     )  # ceil: a partial wave still costs one latency
-    return gen_s + reward_s + update_s
+    return overhead + gen_s + reward_s + update_s
 
 def sft_seconds_for_tokens(config: RunConfig, gpu: str, train_tokens: float) -> float:
     """SFT steady-state wall time for an actual token count on ``gpu``."""
     n = config.normalized()
-    params = active_params_b(n.model_id) * 1e9
+    # MoE prices on total params at a reduced MFU (see seconds_per_step); dense keeps active.
+    moe = _is_moe(n.model_id)
+    params = (total_params_b(n.model_id) if moe else active_params_b(n.model_id)) * 1e9
+    mfu = MFU_SFT_TRAIN_MOE if moe else MFU_SFT_TRAIN
     peak = gpu_tflops(gpu) * 1e12
     flops = SFT_FLOPS_PER_TOKEN_PER_PARAM * params * train_tokens
-    return flops / (peak * MFU_SFT_TRAIN)
+    return flops / (peak * mfu)
 
 
 def select_gpu(config: RunConfig, *, max_wall_seconds: float = 0.0) -> tuple[str, int]:
@@ -221,6 +259,11 @@ def _notes(
             f"{len(n.save_at_steps)} synchronous required save(s) add "
             f"~{_fmt_duration(required_save_s)}"
         )
+    if _is_moe(n.model_id):
+        notes.append(
+            "MoE model: per-step time priced on total params (routing + all-expert coordination), "
+            "not just active-param FLOPs, plus a one-time kernel compile"
+        )
     notes.append(f"GPU sized with {vram_headroom() - 1:.0%} VRAM headroom; static GPU $/hr")
     if wall_capped:
         notes.append(
@@ -283,9 +326,12 @@ def estimate_cost(config: RunConfig, *, wall_cap_s: float = DEFAULT_WALL_CAP_S)
     setup = setup_seconds(config)
     sps = seconds_per_step(config, gpu)
     required_save_s = required_save_overhead_seconds(config)
-    raw_train = config.steps * sps + required_save_s
+    # A one-time kernel/graph compile is paid once on the first step (MoE-only; 0 for dense). It is
+    # training GPU time, so it belongs in the (billed) train term, not setup.
+    compile_s = compile_seconds(config, gpu)
+    raw_train = compile_s + config.steps * sps + required_save_s
     if not config.is_grpo and config.train_tokens is not None:
-        raw_train = sft_seconds_for_tokens(config, gpu, config.train_tokens) + required_save_s
+        raw_train = compile_s + sft_seconds_for_tokens(config, gpu, config.train_tokens) + required_save_s
     sps = raw_train / config.steps
 
     # The cap is on total elapsed wall; setup is reported but not billed, so only training

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.0}/flash/envs/loader.py

@@ -36,7 +36,7 @@ from flash.envs.archive_policy import (
 _DEFAULT_GITHUB_REF = "main"
 _DEFAULT_ENVIRONMENT_PATH = "environment.py"
 _DEFAULT_MANAGED_ENV_REPO = "freesolo-co/environment-hub"
-_CACHE_ROOT = Path(os.environ.get("FLASH_ENV_CACHE_DIR", "/tmp/flash-env-cache"))
+_CACHE_ROOT = Path("/tmp/flash-env-cache")
 # bound the on-disk env cache so it cannot grow without limit (one subdir per env
 # content-sha, ~30-80 MB each). evicted LRU by dir mtime, which we bump on cache hit.
 _CACHE_MAX_ENTRIES = 32

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.0}/flash/serve/deploy.py

@@ -26,6 +26,7 @@ READBACK_DELAY_SECONDS = 2.0
 REVISION_READY_BUDGET_SECONDS = 5 * 60.0
 ACTIVATION_READBACK_ATTEMPTS = 3
 THINKING_STRUCTURED_OUTPUTS_CAPABILITY = "thinking_structured_outputs_deferred_v1"
+REVISION_PROVENANCE_CAPABILITY = "revision_provenance"
 _RETRYABLE_SMOKE_503_CODES = frozenset({"adapter_loading", "engine_unavailable"})
 
 
@@ -382,7 +383,10 @@ def deploy_adapter(
     dep.adapter_revision = revision
     checkpoint = f"{run_id}/step-{checkpoint_step}" if checkpoint_step is not None else run_id
     so_default = parse_structured_outputs(structured_outputs) if structured_outputs else None
-    _require_serving_capabilities(thinking_structured_outputs=thinking and so_default is not None)
+    advertised = _require_serving_capabilities(
+        thinking_structured_outputs=thinking and so_default is not None
+    )
+    require_provenance = REVISION_PROVENANCE_CAPABILITY in advertised
 
     body = {
         "adapter_id": revision,
@@ -391,7 +395,9 @@ def deploy_adapter(
         "subfolder": subfolder,
         "repo_type": "dataset",
         "checkpoint": checkpoint,
-        "status": "ready",
+        # NOTE: do NOT send a client-chosen "status" -- the serving backend sets the record status
+        # itself on registration and rejects an incoming "status" as an extra field (HTTP 422
+        # extra_forbidden). Sending it blocked every deploy against the deployed serving backend.
         "metadata": {
             "record_type": "revision",
             "run_id": run_id,
@@ -419,14 +425,18 @@ def deploy_adapter(
             record = _registered_adapter(revision)
         except ServingError as read_exc:
             raise exc from read_exc
-        if record is None or not _matches_revision_identity(record, body):
+        if record is None or not _matches_revision_identity(
+            record, body, require_provenance=require_provenance
+        ):
             raise exc
         logger.warning(
             "adapter registration response was ambiguous; revision %s exists with matching identity",
             revision,
         )
 
-    _wait_revision_ready(revision, subfolder, expected_identity=body)
+    _wait_revision_ready(
+        revision, subfolder, expected_identity=body, require_provenance=require_provenance
+    )
     if before_activate is not None:
         before_activate(revision, checkpoint)
     activation = _activate_revision(
@@ -469,9 +479,9 @@ def _registered_adapter(adapter_id: str, *, timeout_s: float | None = None) -> d
     return record
 
 
-def _matches_revision_identity(record: dict, expected: dict) -> bool:
-    metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
-    expected_metadata = expected["metadata"]
+def _matches_revision_identity(
+    record: dict, expected: dict, *, require_provenance: bool = True
+) -> bool:
     scalar_fields = (
         "adapter_id",
         "repo_id",
@@ -487,19 +497,35 @@ def _matches_revision_identity(record: dict, expected: dict) -> bool:
         return False
     if (record.get("structured_outputs") or None) != (expected.get("structured_outputs") or None):
         return False
+    if not require_provenance:
+        # backends without revision_provenance do not echo provenance metadata; the immutable
+        # adapter_id already pins the artifact, so this cross-check is best effort here.
+        return True
+    metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
+    expected_metadata = expected["metadata"]
     return all(
         metadata.get(field) == expected_metadata.get(field)
         for field in ("record_type", "run_id", "checkpoint_step", "hf_revision")
     )
 
 
-def _require_serving_capabilities(*, thinking_structured_outputs: bool = False) -> None:
+def _require_serving_capabilities(*, thinking_structured_outputs: bool = False) -> set[str]:
+    # SAFETY-CRITICAL capabilities the deploy correctness contract genuinely depends on: immutable
+    # revisions (a revision id always maps to one artifact) and an atomic alias compare-and-swap
+    # (the alias flip can't race). These are hard-required.
     required = {
         "immutable_adapter_revisions",
         "alias_compare_and_swap",
-        "revision_provenance",
     }
+    # PREFERRED (not required): `revision_provenance` only lets the serving backend echo back the
+    # run/checkpoint/hf_revision metadata, which is used ONLY on the rare 5xx-during-registration
+    # recovery path (`_matches_revision_identity`). Hard-requiring it blocked EVERY deploy org-wide
+    # whenever the serving build lagged on advertising it, even though the happy path never uses it.
+    # So its absence is a logged warning, not a deploy-blocking error.
+    preferred = {REVISION_PROVENANCE_CAPABILITY}
     if thinking_structured_outputs:
+        # Genuinely required for this run: thinking + structured outputs needs the serving backend's
+        # deferred-constraint support (grammar applied after </think>) or served output is invalid.
         required.add(THINKING_STRUCTURED_OUTPUTS_CAPABILITY)
     url = f"{serving_base_url()}/healthz"
     response = _serving_request("GET", url)
@@ -524,12 +550,24 @@ def _require_serving_capabilities(*, thinking_structured_outputs: bool = False)
             f"serving_contract_unsupported: serving health check at {url} capabilities must be "
             "strings"
         )
-    missing = sorted(required - set(capabilities))
+    advertised = set(capabilities)
+    missing = sorted(required - advertised)
     if missing:
         raise ServingError(
             "serving_contract_unsupported: serving is missing required capabilities "
             + ", ".join(missing)
         )
+    missing_preferred = sorted(preferred - advertised)
+    if missing_preferred:
+        # Not fatal: the happy-path deploy does not use these; only the ambiguous-registration
+        # recovery path degrades (best-effort identity check). Surface it so an operator can ship
+        # the serving build that advertises them, without blocking customers meanwhile.
+        logger.warning(
+            "serving backend does not advertise preferred capabilities %s; deploying anyway "
+            "(ambiguous-registration recovery will be best-effort)",
+            ", ".join(missing_preferred),
+        )
+    return advertised
 
 
 def _wait_revision_ready(
@@ -537,6 +575,7 @@ def _wait_revision_ready(
     subfolder: str,
     *,
     expected_identity: dict | None = None,
+    require_provenance: bool = True,
     budget_s: float = REVISION_READY_BUDGET_SECONDS,
 ) -> dict:
     deadline = time.monotonic() + max(0.0, float(budget_s))
@@ -566,7 +605,7 @@ def _wait_revision_ready(
             continue
         last_read_error = None
         if expected_identity is not None and not _matches_revision_identity(
-            record, expected_identity
+            record, expected_identity, require_provenance=require_provenance
         ):
             raise ServingError(
                 f"adapter revision {revision} resolved to a different immutable identity"

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "0.2.58"
+version = "1.0.0"
 description = "Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -113,7 +113,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "0.2.58"
+version = "1.0.0"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

freesolo_flash_dev-1.0.0/tests/test_accounting_coverage.py

@@ -0,0 +1,76 @@
+"""Hermetic coverage for worker-metric sanitization fallbacks and persistence."""
+
+from __future__ import annotations
+
+import json
+
+import flash.engine.accounting as accounting
+
+
+def test_sanitizer_falls_back_safely_when_nested_adapter_parser_raises(monkeypatch) -> None:
+    """Nested adapter fields must still redact colon-bearing refs when the canonical parser fails."""
+    import flash.schema as schema
+
+    monkeypatch.setattr(
+        schema,
+        "parse_adapter_storage_ref",
+        lambda value: (_ for _ in ()).throw(RuntimeError("parser failed")),
+    )
+
+    assert accounting.sanitize_worker_metrics(
+        {"init_from_adapter": "owner/repo:rl/run", "plain": "visible"}
+    ) == {
+        "init_from_adapter": accounting._PRIVATE_SOURCE_MARKER,
+        "plain": "visible",
+    }
+
+
+def test_sanitizer_recurses_through_lists_and_tuples() -> None:
+    """Container normalization must sanitize every nested element and convert tuples to JSON arrays."""
+    value = [
+        {"hf_repo": "private/repo"},
+        ({"init_from_adapter_revision": "a" * 40}, "plain"),
+    ]
+
+    assert accounting.sanitize_worker_metrics(value) == [
+        {"hf_repo": accounting._PRIVATE_SOURCE_MARKER},
+        [{"init_from_adapter_revision": accounting._PRIVATE_SOURCE_MARKER}, "plain"],
+    ]
+
+
+def test_sanitizer_redacts_direct_adapter_ref_strings() -> None:
+    """Direct strings that parse as storage refs must not escape outside named adapter fields."""
+    assert (
+        accounting.sanitize_worker_metrics("owner/repo:rl/run")
+        == accounting._PRIVATE_SOURCE_MARKER
+    )
+
+
+def test_sanitizer_preserves_direct_strings_when_parser_raises(monkeypatch) -> None:
+    """Ambiguous direct strings must remain unchanged when the canonical parser itself is unavailable."""
+    import flash.schema as schema
+
+    monkeypatch.setattr(
+        schema,
+        "parse_adapter_storage_ref",
+        lambda value: (_ for _ in ()).throw(RuntimeError("parser failed")),
+    )
+
+    assert accounting.sanitize_worker_metrics("owner/repo:rl/run") == "owner/repo:rl/run"
+
+
+def test_run_metrics_save_writes_sanitized_json(tmp_path) -> None:
+    """Metrics persistence must write valid sanitized JSON through the public save boundary."""
+    path = tmp_path / "metrics.json"
+    metrics = accounting.RunMetrics(
+        phase="sft",
+        train_tokens=12,
+        notes={"hf_repo": "private/repo"},
+    )
+
+    metrics.save(str(path))
+
+    saved = json.loads(path.read_text())
+    assert saved["phase"] == "sft"
+    assert saved["train_tokens"] == 12
+    assert saved["notes"]["hf_repo"] == accounting._PRIVATE_SOURCE_MARKER

freesolo_flash_dev-1.0.0/tests/test_cli_commands_coverage.py

@@ -0,0 +1,213 @@
+"""Hermetic branch coverage for CLI rendering and terminal-state helpers."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import flash.cli.commands as commands
+
+
+class _Client:
+    def __init__(self, *, checkpoints=None, deployments=None, run=None, worker_output=None) -> None:
+        self._checkpoints = checkpoints or []
+        self._deployments = deployments or []
+        self._run = run or {"run_id": "flash-1", "state": "done"}
+        self._worker_output = worker_output or {}
+
+    def checkpoints(self, run_id: str):
+        assert run_id == "flash-1"
+        return self._checkpoints
+
+    def deployments(self):
+        return self._deployments
+
+    def get_run(self, run_id: str):
+        assert run_id == "flash-1"
+        return self._run
+
+    def get_worker_output(self, run_id: str):
+        assert run_id == "flash-1"
+        return self._worker_output
+
+
+def test_env_list_filters_non_environments_and_uses_styled_renderer(
+    tmp_path, monkeypatch, capsys
+) -> None:
+    """Environment listing must include only loadable sources before handing them to the renderer."""
+    (tmp_path / "environment.py").write_text("# root\n")
+    environments = tmp_path / "environments"
+    environments.mkdir()
+    (environments / "standalone.py").write_text("# standalone\n")
+    (environments / "ignore.txt").write_text("not python\n")
+    (environments / "__private.py").write_text("# private\n")
+    canonical = environments / "canonical-name"
+    canonical.mkdir()
+    (canonical / "environment.py").write_text("# canonical\n")
+    module = environments / "module-name"
+    module.mkdir()
+    (module / "module_name.py").write_text("# module\n")
+    (environments / "empty").mkdir()
+    monkeypatch.chdir(tmp_path)
+    seen = []
+    monkeypatch.setattr(commands.render, "styled", lambda: True)
+    monkeypatch.setattr(commands.render, "env_list", lambda paths: seen.extend(paths) or "styled-envs")
+
+    assert commands.cmd_env_list(SimpleNamespace()) == 0
+
+    assert capsys.readouterr().out == "styled-envs\n"
+    assert seen == [
+        ".",
+        "environments/canonical-name",
+        "environments/module-name",
+        "environments/standalone.py",
+    ]
+
+
+def test_follow_run_returns_success_and_failure_codes(monkeypatch, capsys) -> None:
+    """Following a run must print its final state and preserve terminal success semantics."""
+    client = _Client(run={"run_id": "flash-1", "state": "done"})
+    monkeypatch.setattr(commands, "_render_status", lambda status: f"final:{status['state']}")
+    monkeypatch.setattr(commands, "_poll_logs", lambda *args, **kwargs: ("done", False))
+    assert commands._follow_run(client, "flash-1") == 0
+    assert capsys.readouterr().out == "final:done\n"
+
+    monkeypatch.setattr(commands, "_poll_logs", lambda *args, **kwargs: ("failed", False))
+    assert commands._follow_run(client, "flash-1") == 1
+    assert capsys.readouterr().out == "final:done\n"
+
+
+def test_print_worker_output_skips_empty_sections_and_styles_nonempty_output(
+    monkeypatch, capsys
+) -> None:
+    """Worker output must omit empty artifacts while retaining a styled section heading."""
+    client = _Client(worker_output={"empty.txt": "", "console.txt": "worker line"})
+    monkeypatch.setattr(commands.render, "styled", lambda: True)
+    monkeypatch.setattr(commands.render, "log_section", lambda name: f"section:{name}")
+
+    assert commands._print_worker_output(client, "flash-1") is True
+
+    assert capsys.readouterr().out == "section:console.txt\nworker line\n"
+
+
+def test_log_follow_progress_preserves_nonnumeric_realized_cost() -> None:
+    """Malformed realized cost values must remain visible instead of crashing log following."""
+    state, progress = commands._log_follow_progress(
+        {"state": "running", "realized_cost_usd": "pending"}, "unknown"
+    )
+
+    assert state == "running"
+    assert progress == "running realized_cost=pending"
+
+
+def test_checkpoints_empty_and_styled_paths(monkeypatch, capsys) -> None:
+    """Checkpoint listing must render both empty and populated styled states without plain leakage."""
+    client = _Client()
+    monkeypatch.setattr(commands, "client_from_config", lambda: client)
+    monkeypatch.setattr(commands.render, "styled", lambda: True)
+    monkeypatch.setattr(commands.render, "empty", lambda *args: "empty-checkpoints")
+    assert commands.cmd_checkpoints(SimpleNamespace(run_id="flash-1")) == 0
+    assert capsys.readouterr().out == "empty-checkpoints\n"
+
+    client._checkpoints = [{"step": 12}]
+    monkeypatch.setattr(commands.render, "checkpoints_table", lambda run_id, rows: "checkpoint-table")
+    assert commands.cmd_checkpoints(SimpleNamespace(run_id="flash-1")) == 0
+    assert capsys.readouterr().out == "checkpoint-table\n"
+
+
+def test_checkpoints_plain_path_prints_canonical_refs(monkeypatch, capsys) -> None:
+    """Plain checkpoint output must stay grep-friendly and include the deployment hint on stderr."""
+    client = _Client(checkpoints=[{"step": 12}, {"step": 24}])
+    monkeypatch.setattr(commands, "client_from_config", lambda: client)
+    monkeypatch.setattr(commands.render, "styled", lambda: False)
+
+    assert commands.cmd_checkpoints(SimpleNamespace(run_id="flash-1")) == 0
+
+    captured = capsys.readouterr()
+    assert captured.out.splitlines() == [
+        "step 12 flash-1/step-12",
+        "step 24 flash-1/step-24",
+    ]
+    assert "deploy one with" in captured.err
+
+
+def test_deployments_empty_and_styled_paths(monkeypatch, capsys) -> None:
+    """Deployment listing must distinguish empty and populated styled states."""
+    client = _Client()
+    monkeypatch.setattr(commands, "client_from_config", lambda: client)
+    monkeypatch.setattr(commands.render, "styled", lambda: True)
+    monkeypatch.setattr(commands.render, "empty", lambda *args: "empty-deployments")
+    assert commands.cmd_deployments(SimpleNamespace(json=False)) == 0
+    assert capsys.readouterr().out == "empty-deployments\n"
+
+    client._deployments = [{"run_id": "flash-1", "deployment": {"state": "ready"}}]
+    monkeypatch.setattr(commands.render, "deployments_table", lambda rows: "deployment-table")
+    assert commands.cmd_deployments(SimpleNamespace(json=False)) == 0
+    assert capsys.readouterr().out == "deployment-table\n"
+
+
+def test_deployments_plain_path_handles_final_and_detailed_rows(monkeypatch, capsys) -> None:
+    """Plain deployment output must expose immutable revision, final step, model, and failure detail."""
+    rows = [
+        {
+            "run_id": "fallback-run",
+            "deployment": {
+                "run_id": "flash-1",
+                "checkpoint_step": None,
+                "adapter_revision": "a" * 40,
+                "state": "failed",
+                "verified_at": 123,
+                "openai_model": "model-1",
+                "error": "broken",
+            },
+        }
+    ]
+    monkeypatch.setattr(commands, "client_from_config", lambda: _Client(deployments=rows))
+    monkeypatch.setattr(commands.render, "styled", lambda: False)
+
+    assert commands.cmd_deployments(SimpleNamespace(json=False)) == 0
+
+    output = capsys.readouterr().out
+    assert "flash-1" in output
+    assert "final" in output
+    assert "model-1" in output
+    assert "broken" in output
+
+
+def test_chat_rejects_invalid_target_without_constructing_a_client(monkeypatch, capsys) -> None:
+    """Invalid chat targets must fail locally before any API client or network boundary is touched."""
+    monkeypatch.setattr(
+        commands,
+        "client_from_config",
+        lambda: (_ for _ in ()).throw(AssertionError("client must not be constructed")),
+    )
+
+    result = commands.cmd_chat(SimpleNamespace(run_id="bad target", message="hello", system=None))
+
+    assert result == 1
+    assert "invalid chat target" in capsys.readouterr().err
+
+
+def test_chat_prints_styled_label_before_streaming(monkeypatch, capsys) -> None:
+    """Styled chat must label assistant output while preserving streamed content."""
+    class ChatClient:
+        def chat_stream(self, run_id, messages, **kwargs):
+            assert run_id == "flash-1"
+            assert messages == [{"role": "user", "content": "hello"}]
+            yield "hi"
+
+    monkeypatch.setattr(commands, "client_from_config", ChatClient)
+    monkeypatch.setattr(commands.render, "styled", lambda: True)
+    monkeypatch.setattr(commands.render, "chat_label", lambda: "assistant-label")
+
+    result = commands.cmd_chat(
+        SimpleNamespace(
+            run_id="flash-1",
+            message="hello",
+            system=None,
+            temperature=0.0,
+            max_tokens=32,
+        )
+    )
+
+    assert result == 0
+    assert capsys.readouterr().out.startswith("assistant-label\nhi")

freesolo_flash_dev-1.0.0/tests/test_cli_main_coverage.py

@@ -0,0 +1,19 @@
+"""Hermetic execution coverage for the module CLI entrypoint."""
+
+from __future__ import annotations
+
+import runpy
+
+import pytest
+
+import flash.cli
+
+
+def test_cli_module_exits_with_main_return_code(monkeypatch) -> None:
+    """Executing `python -m flash.cli` must pass the CLI result directly to SystemExit."""
+    monkeypatch.setattr(flash.cli, "main", lambda: 7)
+
+    with pytest.raises(SystemExit) as exc_info:
+        runpy.run_module("flash.cli.__main__", run_name="__main__")
+
+    assert exc_info.value.code == 7

{freesolo_flash_dev-0.2.58 → freesolo_flash_dev-1.0.0}/tests/test_env_cache_evict.py

@@ -11,14 +11,22 @@ from pathlib import Path
 from flash.envs import loader as adapter
 
 
-def test_cache_limits_ignore_ambient_overrides():
+def test_cache_config_ignores_ambient_overrides():
+    # the on-disk env cache location and bounds are hardcoded, not env-tunable: ambient
+    # FLASH_ENV_CACHE_* vars (a stray shell or CI export) must never change them.
     script = (
+        "from pathlib import Path; "
         "from flash.envs import loader; "
+        "assert loader._CACHE_ROOT == Path('/tmp/flash-env-cache'); "
         "assert loader._CACHE_MAX_ENTRIES == 32; "
         "assert loader._CACHE_MAX_BYTES == 4 * 1024 * 1024 * 1024"
     )
-    for max_entries, max_bytes in (("1", "2"), ("invalid", "also-invalid")):
+    for cache_dir, max_entries, max_bytes in (
+        ("/some/other/dir", "1", "2"),
+        ("also-bogus", "invalid", "also-invalid"),
+    ):
         env = os.environ.copy()
+        env["FLASH_ENV_CACHE_DIR"] = cache_dir
         env["FLASH_ENV_CACHE_MAX_ENTRIES"] = max_entries
         env["FLASH_ENV_CACHE_MAX_BYTES"] = max_bytes
         subprocess.run(