freesolo-flash-dev 1.2.33__tar.gz → 1.2.37__tar.gz

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.33
+Version: 1.2.37
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/__init__.py

@@ -738,11 +738,11 @@ def _add_deployment_commands(models_sub: argparse._SubParsersAction) -> None:
         "--wait",
         nargs="?",
         type=_wait_seconds,
-        const=1800.0,
+        const=2400.0,
         default=None,
         metavar="SECONDS",
         help=(
-            "block until the requested revision is ready or failed (default timeout 1800s). "
+            "block until the requested revision is ready or failed (default timeout 2400s). "
             "without it, deploy returns while the revision is still queued"
         ),
     )

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/commands/__init__.py

@@ -399,6 +399,7 @@ def cmd_train(args) -> int:
         # equally not a verification -- so treat anything but an explicit True as unverified.
         affordability_verified = status.pop("affordability_verified", None) is True
         cost = "and cost" if affordability_verified else "but NOT cost"
+        budget_validated = prompt_budget_validation_suffix(status)
         environment = (
             "it did NOT import or run your environment.py. packaged input/output fields and the "
             "statically readable training contract were tokenized together; tokens, retention, "
@@ -411,8 +412,8 @@ def cmd_train(args) -> int:
         )
         print(
             "dry-run validated: config/schema, model+algorithm compatibility, lora rank, "
-            f"runtime-secret presence, warm-start source, serving context cap, {cost}. "
-            f"{environment}",
+            f"runtime-secret presence, warm-start source, serving context cap{budget_validated}, "
+            f"{cost}. {environment}",
             file=sys.stderr,
         )
         if not affordability_verified:
@@ -430,8 +431,10 @@ def cmd_train(args) -> int:
         else:
             print(json.dumps(status, indent=2))
         _print_unpacked_batch_warning(status, spec)  # after the payload, so stdout stays parseable
+        print_status_prompt_budget_warning(status)
         _print_reasoning_loss_warning(status)
         return 0
+    local_budget = warn_before_paid_submit(client, spec)
     status = client.create_run(
         payload,
         runtime_secrets=runtime_secrets,
@@ -440,6 +443,7 @@ def cmd_train(args) -> int:
     run_id = status["run_id"]
     _print_unpacked_batch_warning(status, spec)  # a real submit overrides batch_size the same way
     _print_reasoning_loss_warning(status)  # and trains on whatever reasoning the template kept
+    print_warmstart_context_supplement(local_budget, status)
     logger.info(
         "submitted run %s: model=%s algorithm=%s gpu=%s",
         run_id,
@@ -963,9 +967,13 @@ from flash.cli.commands.deploy import (  # noqa: E402,F401
     cmd_undeploy,
 )
 
-# re-exported at the bottom rather than imported at the top: `train_cost` resolves names back
-# through this package, so a top import would be circular. `cmd_train` below calls these, and the
-# estimate tests import `_warn_if_wandb_requested_without_key` from this package by name.
+# imported at the bottom because the train helpers resolve patched command-package names lazily.
+from flash.cli.commands.prompt_budget import (  # noqa: E402
+    print_status_prompt_budget_warning,
+    print_warmstart_context_supplement,
+    prompt_budget_validation_suffix,
+    warn_before_paid_submit,
+)
 from flash.cli.commands.train_cost import (  # noqa: E402,F401
     _client_train_schema,
     _cmd_train_cost,

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/commands/env/episode.py

@@ -13,9 +13,11 @@ from __future__ import annotations
 
 import contextlib
 import inspect
+import sys
 
 from flash.cli.commands.env.test_evaluations import _evaluation_example
-from flash.envs.evaluations import EvalCase, EvalResult
+from flash.cli.ui import render
+from flash.envs.evaluations import EvalCase, EvalResult, _episode_grading_enabled
 
 
 def _eval_module():
@@ -34,7 +36,7 @@ def _grades_episodes(suite) -> bool:
     Keying off the environment scored the wrong turn there and turned a well-formed first action
     into an error, so the suite has to say so itself.
     """
-    return bool(getattr(suite, "grades_episodes", False))
+    return _episode_grading_enabled(suite)
 
 
 def _drive_episode(client, target: str, environment, case: EvalCase, args) -> dict | str:
@@ -123,7 +125,14 @@ def _effective_turn_cap(environment, state: dict) -> int:
 
 
 def _score_episode_case(
-    suite, case: EvalCase, case_id: str, state: dict, *, thinking: bool = False
+    suite,
+    case: EvalCase,
+    case_id: str,
+    state: dict,
+    *,
+    scorer,
+    state_style: str | None,
+    thinking: bool = False,
 ) -> EvalResult:
     """Grade a finished episode through the suite's own scorer.
 
@@ -134,18 +143,19 @@ def _score_episode_case(
     `reward(completion, example, state)`, which is what reaches the SDK's `score_episodes`.
 
     `EvalSuite.score(case, response)` is the published two-argument contract, so state is offered
-    only to a suite that accepts it, detected by signature rather than by try/except: a TypeError
-    raised INSIDE a scorer that does accept state would otherwise be silently retried as a
-    two-argument call and graded on the wrong text.
+    only to a suite that accepts it. The caller resolves one scorer and signature style for the
+    whole suite rather than retrying on TypeError: an error raised INSIDE a state-aware scorer must
+    not be retried as a two-argument call and graded on the wrong text.
     """
     response = state.get("response_text")
     if not isinstance(response, str):
         turns = state.get("turns") or []
         response = str(turns[-1]) if turns else ""
     eval_module = _eval_module()
-    style = _state_argument(getattr(suite, "score", None))
-    if style is None:
-        return eval_module._score_case(suite, case, case_id, response, thinking=thinking)
+    if state_style is None:
+        return eval_module._score_case(
+            suite, case, case_id, response, thinking=thinking, scorer=scorer
+        )
     return eval_module._score_case(
         suite,
         case,
@@ -153,7 +163,8 @@ def _score_episode_case(
         response,
         thinking=thinking,
         state=state,
-        state_keyword=style == "keyword",
+        state_keyword=state_style == "keyword",
+        scorer=scorer,
     )
 
 
@@ -173,27 +184,45 @@ def _state_argument(score) -> str | None:
     except (TypeError, ValueError):
         # builtins and C callables expose no signature; treat them as the published contract.
         return None
-    named_state = next((p for p in parameters if p.name == "state"), None)
-    if named_state is not None and named_state.kind in (
-        inspect.Parameter.KEYWORD_ONLY,
-        inspect.Parameter.POSITIONAL_OR_KEYWORD,
-    ):
-        # By keyword even where positional would also bind: it cannot land on the wrong parameter.
-        return "keyword"
-    # Checked before `*args`, since a scorer with both can take `state=` but one with only
-    # `*args` cannot.
-    if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters):
-        return "keyword"
-    if any(p.kind is inspect.Parameter.VAR_POSITIONAL for p in parameters):
-        return "positional"
     positional = [
-        p
-        for p in parameters
-        if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+        parameter
+        for parameter in parameters
+        if parameter.kind
+        in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
     ]
-    # (case, response) is the published contract; a third positional is the episode state. This
-    # also catches a positional-only `state`, which the named check above deliberately skips.
-    return "positional" if len(positional) >= 3 else None
+    named_state = next((parameter for parameter in parameters if parameter.name == "state"), None)
+    if named_state is not None:
+        if named_state.kind is inspect.Parameter.POSITIONAL_ONLY:
+            return "positional" if positional.index(named_state) == 2 else None
+        if named_state.kind in (
+            inspect.Parameter.KEYWORD_ONLY,
+            inspect.Parameter.POSITIONAL_OR_KEYWORD,
+        ):
+            # by keyword even where positional would also bind: it cannot land on the wrong parameter.
+            return "keyword"
+    # checked before `*args`, since a scorer with both can take `state=` but one with only
+    # `*args` cannot.
+    if any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters):
+        return "keyword"
+    if any(parameter.kind is inspect.Parameter.VAR_POSITIONAL for parameter in parameters):
+        # scorer(case, response, state) reaches *args when the signature has no fixed third slot.
+        # zero, one, or two regular positional parameters can all bind that call.
+        return "positional" if len(positional) <= 2 else None
+    return None
+
+
+def _warn_if_episode_state_is_hidden(suite, state_style: str | None) -> None:
+    """Warn once when full-episode work feeds a scorer that cannot see the transcript."""
+    if state_style is not None:
+        return
+    message = (
+        f"suite {getattr(suite, 'name', 'evaluation')!r} sets grades_episodes = True; "
+        "each episode will still be played out with one generation per turn, but the scorer "
+        "will receive only the episode's final response text, not the transcript. Add a third "
+        "`state` argument to "
+        "`score(case, response, state)` to grade the transcript."
+    )
+    print(render.warn(message) if render.styled() else f"warning: {message}", file=sys.stderr)
 
 
 def _run_episode_cases(
@@ -220,6 +249,9 @@ def _run_episode_cases(
             )
             for case_id in case_ids
         )
+    scorer = getattr(suite, "score", None)
+    state_style = _state_argument(scorer)
+    _warn_if_episode_state_is_hidden(suite, state_style)
     results = []
     # The adapter defaults to `thinking = False` (flash/envs/adapter.py), and with it off
     # `_scored_turn_text` returns the turn unstripped -- so `state["response_text"]` keeps its
@@ -237,7 +269,17 @@ def _run_episode_cases(
             if isinstance(outcome, str):
                 results.append(eval_module._generation_error(case_id, outcome))
                 continue
-            results.append(_score_episode_case(suite, case, case_id, outcome, thinking=thinking))
+            results.append(
+                _score_episode_case(
+                    suite,
+                    case,
+                    case_id,
+                    outcome,
+                    scorer=scorer,
+                    state_style=state_style,
+                    thinking=thinking,
+                )
+            )
     return tuple(results)
 
 

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/commands/env/eval.py

@@ -162,6 +162,7 @@ def _score_case(
     thinking: bool = False,
     state=None,
     state_keyword: bool = False,
+    scorer=None,
 ) -> EvalResult:
     """Grade one response on the caller's thread.
 
@@ -172,7 +173,8 @@ def _score_case(
     `episode._score_episode_case`); single-turn scoring keeps the two-argument contract.
     `state_keyword` says which way that suite's signature can take it -- `**kwargs` and
     keyword-only scorers reject a third positional, `*args` scorers reject the keyword -- so the
-    caller decides, having read the signature.
+    caller decides, having read the signature. Episode scoring also passes the exact callable it
+    inspected, so a descriptor cannot change shape between capability detection and invocation.
 
     The scorer grades the stripped answer; the RESULT records the raw generation. Stripping for the
     scorer must not shorten what is recorded, on the error path least of all -- a failed case is
@@ -182,12 +184,13 @@ def _score_case(
     try:
         scored_text = _scored_response(response, thinking=thinking)
         recorded_response = getattr(scored_text, "raw", response)
+        score = suite.score if scorer is None else scorer
         if state is None:
-            scored = suite.score(case, scored_text)
+            scored = score(case, scored_text)
         elif state_keyword:
-            scored = suite.score(case, scored_text, state=state)
+            scored = score(case, scored_text, state=state)
         else:
-            scored = suite.score(case, scored_text, state)
+            scored = score(case, scored_text, state)
         return normalize_eval_result(case, recorded_response, scored, case_id=case_id)
     except (Exception, SystemExit) as exc:
         return EvalResult(

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/commands/env/setup.py

@@ -138,12 +138,14 @@ Given the opening prompt, does the model emit the single integer `step_episode`
 to advance the game? That is a real held-out check of the action format the episode
 depends on, and it is the honest scope of a single-shot evaluation.
 
-To grade a finished transcript instead, set `grades_episodes = True` on the suite.
-`env eval` then plays each case out against the deployed model -- generating, stepping
-the environment, and repeating to `max_episode_turns` -- and scores the resulting
-transcript. Such a suite needs cases the environment can actually advance (the cases
-below carry no `output`, so `step_episode` has no secret to compare against), and each
-case then costs one generation per turn rather than one in total.
+To grade a finished transcript instead, set `grades_episodes = True` on the suite
+and define `score(self, case, response, state)`. `env eval` then plays each case out
+against the deployed model -- generating, stepping the environment, and repeating to
+`max_episode_turns` -- and passes the resulting transcript in `state`; without that
+argument the scorer receives only the episode's final response text. Such a suite
+needs cases the environment can actually advance (the cases below carry no `output`,
+so `step_episode` has no secret to compare against), and each case then costs one
+generation per turn rather than one in total.
 
 Do NOT grade these by calling `environment.reward(response, example)`: with no episode
 state that call scores an empty transcript, so an unrelated answer can score 1.0 and
@@ -289,7 +291,7 @@ class StarterMultiTurnEnv(EnvironmentMultiTurn):
         # Advance the world after one assistant action. Return done=True to end the
         # episode, or append an observation message and keep going.
         # `messages` already ends with this action: messages[-1]["content"] is
-        # `assistant_response`. If you rebuild state by replaying the transcript, replay
+        # `assistant_response`. if you rebuild state by replaying the transcript, replay
         # `messages[:-1]` and apply `assistant_response` once, or you apply it twice.
         try:
             guess = int(assistant_response.strip().split()[0])

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/commands/env/test.py

@@ -9,7 +9,10 @@ from pathlib import Path
 
 from flash.cli.commands.env.episode import _effective_turn_cap
 from flash.cli.commands.env.push import _err, _resolve_local_env_entrypoint
-from flash.cli.commands.env.test_evaluations import _check_evaluation_suites
+from flash.cli.commands.env.test_evaluations import (
+    _check_evaluation_suites,
+    _normalize_prompt_images,
+)
 from flash.cli.commands.env.test_params import _env_params
 from flash.cli.commands.env.test_warnings import (
     _warn_on_low_replay_reward,
@@ -265,9 +268,20 @@ def _drive_single_turn(env, example: dict, record: dict, *, force_echo: bool = F
     )
 
 
-def _drive_multi_turn(env, example: dict, record: dict, *, force_echo: bool = False) -> None:
+def _new_multi_turn_replay_state(env, example: dict, record: dict) -> dict:
     state = env.new_rollout_state(example)
     record["prompt"] = _check_messages(state.get("prompt") or state.get("messages"), "prompt")
+    _normalize_prompt_images(env, example, record["prompt"])
+    return state
+
+
+def _validate_multi_turn_reply(env, example: dict, state: dict, messages: object) -> None:
+    _check_messages(messages, "env_reply")
+    _normalize_prompt_images(env, example, [dict(message) for message in state["messages"]])
+
+
+def _drive_multi_turn(env, example: dict, record: dict, *, force_echo: bool = False) -> None:
+    state = _new_multi_turn_replay_state(env, example, record)
     completion = _gold_completion(env, example)
     record["completion"] = completion
     reference_turns = _reference_turns(completion)
@@ -326,18 +340,19 @@ def _drive_multi_turn(env, example: dict, record: dict, *, force_echo: bool = Fa
             break
         env_msgs = env.env_reply(state["messages"], state)
         env_step_pending = False
-        if not env_msgs:
-            break
-        # the env's own reply messages feed the chat template for the next turn in the real
-        # rollout, so validate their envelope here too: a malformed reply that would break
-        # remotely must fail the episode instead of slipping through on a finite reward.
-        _check_messages(env_msgs, "env_reply")
         if gold_finished is None and turns >= len(reference_turns):
             # the gold answer has just run out and its last turn is applied: this is the only
             # moment the reference can be judged on its own, before junk padding touches the state.
             gold_finished = _episode_completed(env, state, hard_cap)
+        if not env_msgs:
+            break
         if env.rollout_done(state, max_turns=hard_cap):
             break
+        # the env's own reply messages feed the chat template for the next turn in the real
+        # rollout, so validate their envelope and accumulated images here too: a malformed reply
+        # that would break remotely must fail the episode instead of slipping through on a finite
+        # reward. normalize copies so the check cannot replace authoritative rollout messages.
+        _validate_multi_turn_reply(env, example, state, env_msgs)
 
     # the driver-side exits above stop before the inter-turn env_reply, so the last replayed turn
     # is still unapplied. a stateful env would then score a board or transcript missing the last
@@ -351,18 +366,9 @@ def _drive_multi_turn(env, example: dict, record: dict, *, force_echo: bool = Fa
     # env that replied with nothing, a natural finish) -- stepping again there would be a spurious
     # extra move. rollout_done covers the env having declared the episode over.
     if env_step_pending and not env.rollout_done(state, hard_cap):
-        # validated exactly like the in-loop replies above: a malformed reply breaks the chat
-        # template on the paid run whether it was the last one or not. this call is the ONLY
-        # env_reply for an episode stopped at the ceiling, and a per-example `max_episode_turns`
-        # makes that the common case rather than a rare one -- leaving it unchecked let an env whose
-        # final reply is malformed reach `overall: PASS`.
-        #
-        # an EMPTY reply is allowed first, exactly as the in-loop path does (`if not env_msgs:
-        # break` precedes its own `_check_messages`): an env with nothing further to observe
-        # legitimately returns nothing, and `_check_messages` rejects an empty list.
-        final_msgs = env.env_reply(state["messages"], state)
-        if final_msgs:
-            _check_messages(final_msgs, "env_reply")
+        # apply the final action's side effects, but do not validate its reply as a future prompt:
+        # no later model generation consumes it in the online path.
+        env.env_reply(state["messages"], state)
     if gold_finished is None:
         # the gold answer never ran out inside the loop -- it covered the whole episode, so the
         # break came first and the mid-loop sample never fired. NOW is its moment: the deferred

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/commands/env/test_evaluations.py

@@ -1,12 +1,13 @@
 """Offline contract checks for an environment's evaluation sidecar.
 
-Split from `test.py` to keep that file under the 1000-line gate. This group is cohesive and has one
-entry point, `check_evaluation_suites`: everything else here exists to build one held-out case's
-prompt and replayed response the same way `flash env eval` would.
+Split from `test.py` to keep that file under the 1000-line gate. Single-turn cases validate prompt
+construction and reference replay. Episode cases validate initial rollout and prompt construction,
+then bind the scorer signature without executing it against unfinished state.
 """
 
 from __future__ import annotations
 
+import inspect
 from pathlib import Path
 
 from flash.cli.commands.env.push import _err
@@ -76,7 +77,41 @@ def _evaluation_response(env, case) -> tuple[str, str]:
     return policy, response
 
 
+def _check_episode_evaluation_prompt(env, case) -> None:
+    """Validate the held-out case's initial rollout state and prompt without advancing it."""
+    # imported here rather than at module scope: `test.py` imports this module.
+    from flash.cli.commands.env.test import _new_multi_turn_replay_state, _new_record
+
+    _new_multi_turn_replay_state(env, _evaluation_example(case), _new_record())
+
+
+def _validate_episode_scorer_signature(scorer, case, state_style: str | None) -> None:
+    """Bind the online scorer call shape without executing it against unfinished state."""
+    if not callable(scorer):
+        raise TypeError("evaluation suite score must be callable")
+    try:
+        signature = inspect.signature(scorer)
+    except (TypeError, ValueError):
+        # some builtins and extension callables expose no signature, so keep the permissive runtime
+        # behavior without calling them during an offline check.
+        return
+
+    response = "response"
+    state = {}
+    if state_style is None:
+        signature.bind(case, response)
+    elif state_style == "keyword":
+        signature.bind(case, response, state=state)
+    else:
+        signature.bind(case, response, state)
+
+
 def _check_evaluation_suites(entrypoint: Path, env) -> bool:
+    from flash.cli.commands.env.episode import (
+        _grades_episodes,
+        _state_argument,
+        _warn_if_episode_state_is_hidden,
+    )
     from flash.envs.evaluations import (
         _DEFAULT_EVALUATIONS_PATH,
         EvalSuiteReport,
@@ -107,9 +142,28 @@ def _check_evaluation_suites(entrypoint: Path, env) -> bool:
                     f"evaluation suite {suite.name} failed contract checks: suite produced no cases"
                 )
                 continue
+            episode_suite = _grades_episodes(suite)
+            if episode_suite and not getattr(env, "multi_turn", False):
+                raise TypeError(
+                    "suite sets grades_episodes = True, but this environment is single-turn "
+                    "and has no episode to play"
+                )
+            scorer = getattr(suite, "score", None)
+            if episode_suite:
+                state_style = _state_argument(scorer)
+                _warn_if_episode_state_is_hidden(suite, state_style)
+                for case in cases:
+                    _check_episode_evaluation_prompt(env, case)
+                _validate_episode_scorer_signature(scorer, cases[0], state_style)
+                print(
+                    f"evaluation suite {suite.name}: {len(cases)}/{len(cases)} cases "
+                    "passed contract checks"
+                )
+                continue
+
             for index, case in enumerate(cases, start=1):
                 _policy, response = _evaluation_response(env, case)
-                scored = suite.score(case, response)
+                scored = scorer(case, response)
                 result = normalize_eval_result(
                     case,
                     response,

freesolo_flash_dev-1.2.37/flash/cli/commands/prompt_budget.py

@@ -0,0 +1,146 @@
+"""prompt-budget warnings and warm-start context lookup for training commands."""
+
+from __future__ import annotations
+
+import sys
+from typing import TypeGuard, cast
+
+from flash import __version__
+from flash.cli.ui import render
+from flash.engine.plan.prompt_budget import PromptBudget, rl_prompt_budget
+
+
+def _positive_int(value: object) -> TypeGuard[int]:
+    return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+
+def _budget_from_status(status: object) -> PromptBudget | None:
+    budget = status.get("prompt_budget") if isinstance(status, dict) else None
+    if not isinstance(budget, dict):
+        return None
+    if budget.get("algorithm") not in {"grpo", "opd", "rl"}:
+        return None
+    if budget.get("context_source") not in {"authored", "recipe_default"}:
+        return None
+    if budget.get("prompt_budget_is_upper_bound") is not True:
+        return None
+    engine_len = budget.get("engine_len")
+    max_completion = budget.get("max_completion")
+    prompt_budget = budget.get("prompt_budget")
+    if not _positive_int(engine_len):
+        return None
+    if not _positive_int(max_completion):
+        return None
+    if not _positive_int(prompt_budget):
+        return None
+    if prompt_budget != engine_len - max_completion:
+        return None
+    source_context = budget.get("warm_start_context")
+    if source_context is not None and not _positive_int(source_context):
+        return None
+    return cast("PromptBudget", budget)
+
+
+def prompt_budget_validation_suffix(status: object) -> str:
+    """qualify a dry-run summary only when the server returned a budget."""
+    return ", prompt budget (upper bound)" if _budget_from_status(status) is not None else ""
+
+
+def warmstart_source_context(client, spec) -> int | None:
+    """read a warm-start source's authored context through the cli's authenticated client."""
+    from flash.core.catalog import samples_on_policy
+    from flash.schema import parse_checkpoint_ref
+
+    ref = getattr(spec.train, "init_from_adapter", "")
+    if not ref or not samples_on_policy(spec.algorithm):
+        return None
+    parsed = parse_checkpoint_ref(ref)
+    if parsed is None:
+        return None
+    try:
+        source = client.get_run(parsed[0])
+    # reporting is best-effort and must never block a valid submission
+    except Exception:
+        return None
+    train = (source or {}).get("spec", {}).get("train", {}) if isinstance(source, dict) else {}
+    context = train.get("max_context_tokens") if isinstance(train, dict) else None
+    if isinstance(context, bool) or not isinstance(context, int) or context <= 0:
+        return None
+    return context
+
+
+def _warmstart_context_sentence(context: object) -> str | None:
+    if isinstance(context, bool) or not isinstance(context, int) or context <= 0:
+        return None
+    return (
+        f"The warm-start source was configured with max_context_tokens={context}; "
+        "it is NOT inherited."
+    )
+
+
+def prompt_budget_warning(
+    budget: object,
+    *,
+    derived_locally: bool = False,
+    include_warm_start_context: bool = True,
+) -> str | None:
+    """render the defaulted-budget warning for one prompt-budget descriptor."""
+    if not isinstance(budget, dict) or budget.get("context_source") != "recipe_default":
+        return None
+    try:
+        prompt_budget = int(budget["prompt_budget"])
+        max_completion = int(budget["max_completion"])
+    except (KeyError, TypeError, ValueError):
+        return None
+    algorithm = str(budget.get("algorithm") or "").upper() or "RL"
+    message = (
+        f"train.max_context_tokens is unset, so {algorithm} derives a prompt budget of at most "
+        f"{prompt_budget} tokens from the recipe default (engine {budget.get('engine_len')} minus "
+        f"max_completion_tokens {max_completion}), not from the model's context. The worker clamps "
+        "the engine length to the model architecture first, so its budget can be smaller than "
+        "this. Prompts over the budget are DROPPED, not truncated, so the run trains on a "
+        "shorter, biased subset."
+    )
+    if derived_locally:
+        message += (
+            f" (Derived locally by this CLI {__version__}; if the control plane is on a "
+            "different release its recipe default may differ.)"
+        )
+    if include_warm_start_context:
+        source = _warmstart_context_sentence(budget.get("warm_start_context"))
+        if source:
+            message += f" {source}"
+    return message + " Set train.max_context_tokens explicitly to choose the budget."
+
+
+def _print_warning(message: str | None) -> None:
+    if message:
+        print(render.warn(message) if render.styled() else f"warning: {message}", file=sys.stderr)
+
+
+def print_status_prompt_budget_warning(status: object) -> None:
+    """print the server-derived warning carried by a submit or dry-run response."""
+    _print_warning(prompt_budget_warning(_budget_from_status(status)))
+
+
+def warn_before_paid_submit(client, spec) -> PromptBudget | None:
+    """print the locally derived warning before create_run can start provisioning."""
+    budget = rl_prompt_budget(spec, warm_start_context=warmstart_source_context(client, spec))
+    _print_warning(prompt_budget_warning(budget, derived_locally=True))
+    return budget
+
+
+def print_warmstart_context_supplement(local_budget: PromptBudget | None, status: object) -> None:
+    """print only context the server resolved after the generic pre-submit warning."""
+    served_budget = _budget_from_status(status)
+    if served_budget is None or (
+        local_budget is not None and local_budget.get("warm_start_context")
+    ):
+        return
+    _print_warning(_warmstart_context_sentence(served_budget.get("warm_start_context")))
+
+
+def warn_cost_prompt_budget(spec) -> None:
+    """print the local prompt-budget warning for the offline ``--cost`` path."""
+    budget = rl_prompt_budget(spec)
+    _print_warning(prompt_budget_warning(budget, derived_locally=True))

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/commands/train_cost.py

@@ -91,6 +91,9 @@ def _cmd_train_cost_offline(spec) -> int:
         print(render.cost_panel(estimate))
     else:
         print(estimate.breakdown())
+    from flash.cli.commands.prompt_budget import warn_cost_prompt_budget
+
+    warn_cost_prompt_budget(spec)
     return 0
 
 

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.37}/flash/cli/scaffold/TRAINING.md

@@ -224,6 +224,11 @@ int, `strict=true` as a bool, and anything that does not parse as a TOML scalar
 `flash env eval` takes the same two flags, for the same reason: a held-out suite scored
 against a differently-configured environment is not measuring your run.
 
+For a multi-turn environment, `flash env eval` still defaults to one prompt and one reply, which is
+a valid first-action check. Set `grades_episodes = True` on a suite to play one generation per turn.
+A scorer with the recommended `score(case, response, state)` signature receives the finished
+transcript state; otherwise the scorer receives only the episode's final response text.
+
 **It is also a three-example smoke test, not a dataset audit.** `flash env test` runs the
 contract checks against the **first 3 rows only** (`_DEFAULT_EPISODES = 3`, and it iterates
 `dataset[:episode_count]`), so a malformed prompt, a scorer that raises, or a bad episode