freesolo-flash-dev 1.2.33__tar.gz → 1.2.36__tar.gz

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.33
+Version: 1.2.36
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/cli/__init__.py

@@ -738,11 +738,11 @@ def _add_deployment_commands(models_sub: argparse._SubParsersAction) -> None:
         "--wait",
         nargs="?",
         type=_wait_seconds,
-        const=1800.0,
+        const=2400.0,
         default=None,
         metavar="SECONDS",
         help=(
-            "block until the requested revision is ready or failed (default timeout 1800s). "
+            "block until the requested revision is ready or failed (default timeout 2400s). "
             "without it, deploy returns while the revision is still queued"
         ),
     )

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/cli/commands/env/episode.py

@@ -13,9 +13,11 @@ from __future__ import annotations
 
 import contextlib
 import inspect
+import sys
 
 from flash.cli.commands.env.test_evaluations import _evaluation_example
-from flash.envs.evaluations import EvalCase, EvalResult
+from flash.cli.ui import render
+from flash.envs.evaluations import EvalCase, EvalResult, _episode_grading_enabled
 
 
 def _eval_module():
@@ -34,7 +36,7 @@ def _grades_episodes(suite) -> bool:
     Keying off the environment scored the wrong turn there and turned a well-formed first action
     into an error, so the suite has to say so itself.
     """
-    return bool(getattr(suite, "grades_episodes", False))
+    return _episode_grading_enabled(suite)
 
 
 def _drive_episode(client, target: str, environment, case: EvalCase, args) -> dict | str:
@@ -123,7 +125,14 @@ def _effective_turn_cap(environment, state: dict) -> int:
 
 
 def _score_episode_case(
-    suite, case: EvalCase, case_id: str, state: dict, *, thinking: bool = False
+    suite,
+    case: EvalCase,
+    case_id: str,
+    state: dict,
+    *,
+    scorer,
+    state_style: str | None,
+    thinking: bool = False,
 ) -> EvalResult:
     """Grade a finished episode through the suite's own scorer.
 
@@ -134,18 +143,19 @@ def _score_episode_case(
     `reward(completion, example, state)`, which is what reaches the SDK's `score_episodes`.
 
     `EvalSuite.score(case, response)` is the published two-argument contract, so state is offered
-    only to a suite that accepts it, detected by signature rather than by try/except: a TypeError
-    raised INSIDE a scorer that does accept state would otherwise be silently retried as a
-    two-argument call and graded on the wrong text.
+    only to a suite that accepts it. The caller resolves one scorer and signature style for the
+    whole suite rather than retrying on TypeError: an error raised INSIDE a state-aware scorer must
+    not be retried as a two-argument call and graded on the wrong text.
     """
     response = state.get("response_text")
     if not isinstance(response, str):
         turns = state.get("turns") or []
         response = str(turns[-1]) if turns else ""
     eval_module = _eval_module()
-    style = _state_argument(getattr(suite, "score", None))
-    if style is None:
-        return eval_module._score_case(suite, case, case_id, response, thinking=thinking)
+    if state_style is None:
+        return eval_module._score_case(
+            suite, case, case_id, response, thinking=thinking, scorer=scorer
+        )
     return eval_module._score_case(
         suite,
         case,
@@ -153,7 +163,8 @@ def _score_episode_case(
         response,
         thinking=thinking,
         state=state,
-        state_keyword=style == "keyword",
+        state_keyword=state_style == "keyword",
+        scorer=scorer,
     )
 
 
@@ -173,27 +184,45 @@ def _state_argument(score) -> str | None:
     except (TypeError, ValueError):
         # builtins and C callables expose no signature; treat them as the published contract.
         return None
-    named_state = next((p for p in parameters if p.name == "state"), None)
-    if named_state is not None and named_state.kind in (
-        inspect.Parameter.KEYWORD_ONLY,
-        inspect.Parameter.POSITIONAL_OR_KEYWORD,
-    ):
-        # By keyword even where positional would also bind: it cannot land on the wrong parameter.
-        return "keyword"
-    # Checked before `*args`, since a scorer with both can take `state=` but one with only
-    # `*args` cannot.
-    if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters):
-        return "keyword"
-    if any(p.kind is inspect.Parameter.VAR_POSITIONAL for p in parameters):
-        return "positional"
     positional = [
-        p
-        for p in parameters
-        if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+        parameter
+        for parameter in parameters
+        if parameter.kind
+        in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
     ]
-    # (case, response) is the published contract; a third positional is the episode state. This
-    # also catches a positional-only `state`, which the named check above deliberately skips.
-    return "positional" if len(positional) >= 3 else None
+    named_state = next((parameter for parameter in parameters if parameter.name == "state"), None)
+    if named_state is not None:
+        if named_state.kind is inspect.Parameter.POSITIONAL_ONLY:
+            return "positional" if positional.index(named_state) == 2 else None
+        if named_state.kind in (
+            inspect.Parameter.KEYWORD_ONLY,
+            inspect.Parameter.POSITIONAL_OR_KEYWORD,
+        ):
+            # by keyword even where positional would also bind: it cannot land on the wrong parameter.
+            return "keyword"
+    # checked before `*args`, since a scorer with both can take `state=` but one with only
+    # `*args` cannot.
+    if any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters):
+        return "keyword"
+    if any(parameter.kind is inspect.Parameter.VAR_POSITIONAL for parameter in parameters):
+        # scorer(case, response, state) reaches *args when the signature has no fixed third slot.
+        # zero, one, or two regular positional parameters can all bind that call.
+        return "positional" if len(positional) <= 2 else None
+    return None
+
+
+def _warn_if_episode_state_is_hidden(suite, state_style: str | None) -> None:
+    """Warn once when full-episode work feeds a scorer that cannot see the transcript."""
+    if state_style is not None:
+        return
+    message = (
+        f"suite {getattr(suite, 'name', 'evaluation')!r} sets grades_episodes = True; "
+        "each episode will still be played out with one generation per turn, but the scorer "
+        "will receive only the episode's final response text, not the transcript. Add a third "
+        "`state` argument to "
+        "`score(case, response, state)` to grade the transcript."
+    )
+    print(render.warn(message) if render.styled() else f"warning: {message}", file=sys.stderr)
 
 
 def _run_episode_cases(
@@ -220,6 +249,9 @@ def _run_episode_cases(
             )
             for case_id in case_ids
         )
+    scorer = getattr(suite, "score", None)
+    state_style = _state_argument(scorer)
+    _warn_if_episode_state_is_hidden(suite, state_style)
     results = []
     # The adapter defaults to `thinking = False` (flash/envs/adapter.py), and with it off
     # `_scored_turn_text` returns the turn unstripped -- so `state["response_text"]` keeps its
@@ -237,7 +269,17 @@ def _run_episode_cases(
             if isinstance(outcome, str):
                 results.append(eval_module._generation_error(case_id, outcome))
                 continue
-            results.append(_score_episode_case(suite, case, case_id, outcome, thinking=thinking))
+            results.append(
+                _score_episode_case(
+                    suite,
+                    case,
+                    case_id,
+                    outcome,
+                    scorer=scorer,
+                    state_style=state_style,
+                    thinking=thinking,
+                )
+            )
     return tuple(results)
 
 

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/cli/commands/env/eval.py

@@ -162,6 +162,7 @@ def _score_case(
     thinking: bool = False,
     state=None,
     state_keyword: bool = False,
+    scorer=None,
 ) -> EvalResult:
     """Grade one response on the caller's thread.
 
@@ -172,7 +173,8 @@ def _score_case(
     `episode._score_episode_case`); single-turn scoring keeps the two-argument contract.
     `state_keyword` says which way that suite's signature can take it -- `**kwargs` and
     keyword-only scorers reject a third positional, `*args` scorers reject the keyword -- so the
-    caller decides, having read the signature.
+    caller decides, having read the signature. Episode scoring also passes the exact callable it
+    inspected, so a descriptor cannot change shape between capability detection and invocation.
 
     The scorer grades the stripped answer; the RESULT records the raw generation. Stripping for the
     scorer must not shorten what is recorded, on the error path least of all -- a failed case is
@@ -182,12 +184,13 @@ def _score_case(
     try:
         scored_text = _scored_response(response, thinking=thinking)
         recorded_response = getattr(scored_text, "raw", response)
+        score = suite.score if scorer is None else scorer
         if state is None:
-            scored = suite.score(case, scored_text)
+            scored = score(case, scored_text)
         elif state_keyword:
-            scored = suite.score(case, scored_text, state=state)
+            scored = score(case, scored_text, state=state)
         else:
-            scored = suite.score(case, scored_text, state)
+            scored = score(case, scored_text, state)
         return normalize_eval_result(case, recorded_response, scored, case_id=case_id)
     except (Exception, SystemExit) as exc:
         return EvalResult(

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/cli/commands/env/setup.py

@@ -138,12 +138,14 @@ Given the opening prompt, does the model emit the single integer `step_episode`
 to advance the game? That is a real held-out check of the action format the episode
 depends on, and it is the honest scope of a single-shot evaluation.
 
-To grade a finished transcript instead, set `grades_episodes = True` on the suite.
-`env eval` then plays each case out against the deployed model -- generating, stepping
-the environment, and repeating to `max_episode_turns` -- and scores the resulting
-transcript. Such a suite needs cases the environment can actually advance (the cases
-below carry no `output`, so `step_episode` has no secret to compare against), and each
-case then costs one generation per turn rather than one in total.
+To grade a finished transcript instead, set `grades_episodes = True` on the suite
+and define `score(self, case, response, state)`. `env eval` then plays each case out
+against the deployed model -- generating, stepping the environment, and repeating to
+`max_episode_turns` -- and passes the resulting transcript in `state`; without that
+argument the scorer receives only the episode's final response text. Such a suite
+needs cases the environment can actually advance (the cases below carry no `output`,
+so `step_episode` has no secret to compare against), and each case then costs one
+generation per turn rather than one in total.
 
 Do NOT grade these by calling `environment.reward(response, example)`: with no episode
 state that call scores an empty transcript, so an unrelated answer can score 1.0 and
@@ -289,7 +291,7 @@ class StarterMultiTurnEnv(EnvironmentMultiTurn):
         # Advance the world after one assistant action. Return done=True to end the
         # episode, or append an observation message and keep going.
         # `messages` already ends with this action: messages[-1]["content"] is
-        # `assistant_response`. If you rebuild state by replaying the transcript, replay
+        # `assistant_response`. if you rebuild state by replaying the transcript, replay
         # `messages[:-1]` and apply `assistant_response` once, or you apply it twice.
         try:
             guess = int(assistant_response.strip().split()[0])

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/cli/commands/env/test.py

@@ -9,7 +9,10 @@ from pathlib import Path
 
 from flash.cli.commands.env.episode import _effective_turn_cap
 from flash.cli.commands.env.push import _err, _resolve_local_env_entrypoint
-from flash.cli.commands.env.test_evaluations import _check_evaluation_suites
+from flash.cli.commands.env.test_evaluations import (
+    _check_evaluation_suites,
+    _normalize_prompt_images,
+)
 from flash.cli.commands.env.test_params import _env_params
 from flash.cli.commands.env.test_warnings import (
     _warn_on_low_replay_reward,
@@ -265,9 +268,20 @@ def _drive_single_turn(env, example: dict, record: dict, *, force_echo: bool = F
     )
 
 
-def _drive_multi_turn(env, example: dict, record: dict, *, force_echo: bool = False) -> None:
+def _new_multi_turn_replay_state(env, example: dict, record: dict) -> dict:
     state = env.new_rollout_state(example)
     record["prompt"] = _check_messages(state.get("prompt") or state.get("messages"), "prompt")
+    _normalize_prompt_images(env, example, record["prompt"])
+    return state
+
+
+def _validate_multi_turn_reply(env, example: dict, state: dict, messages: object) -> None:
+    _check_messages(messages, "env_reply")
+    _normalize_prompt_images(env, example, [dict(message) for message in state["messages"]])
+
+
+def _drive_multi_turn(env, example: dict, record: dict, *, force_echo: bool = False) -> None:
+    state = _new_multi_turn_replay_state(env, example, record)
     completion = _gold_completion(env, example)
     record["completion"] = completion
     reference_turns = _reference_turns(completion)
@@ -326,18 +340,19 @@ def _drive_multi_turn(env, example: dict, record: dict, *, force_echo: bool = Fa
             break
         env_msgs = env.env_reply(state["messages"], state)
         env_step_pending = False
-        if not env_msgs:
-            break
-        # the env's own reply messages feed the chat template for the next turn in the real
-        # rollout, so validate their envelope here too: a malformed reply that would break
-        # remotely must fail the episode instead of slipping through on a finite reward.
-        _check_messages(env_msgs, "env_reply")
         if gold_finished is None and turns >= len(reference_turns):
             # the gold answer has just run out and its last turn is applied: this is the only
             # moment the reference can be judged on its own, before junk padding touches the state.
             gold_finished = _episode_completed(env, state, hard_cap)
+        if not env_msgs:
+            break
         if env.rollout_done(state, max_turns=hard_cap):
             break
+        # the env's own reply messages feed the chat template for the next turn in the real
+        # rollout, so validate their envelope and accumulated images here too: a malformed reply
+        # that would break remotely must fail the episode instead of slipping through on a finite
+        # reward. normalize copies so the check cannot replace authoritative rollout messages.
+        _validate_multi_turn_reply(env, example, state, env_msgs)
 
     # the driver-side exits above stop before the inter-turn env_reply, so the last replayed turn
     # is still unapplied. a stateful env would then score a board or transcript missing the last
@@ -351,18 +366,9 @@ def _drive_multi_turn(env, example: dict, record: dict, *, force_echo: bool = Fa
     # env that replied with nothing, a natural finish) -- stepping again there would be a spurious
     # extra move. rollout_done covers the env having declared the episode over.
     if env_step_pending and not env.rollout_done(state, hard_cap):
-        # validated exactly like the in-loop replies above: a malformed reply breaks the chat
-        # template on the paid run whether it was the last one or not. this call is the ONLY
-        # env_reply for an episode stopped at the ceiling, and a per-example `max_episode_turns`
-        # makes that the common case rather than a rare one -- leaving it unchecked let an env whose
-        # final reply is malformed reach `overall: PASS`.
-        #
-        # an EMPTY reply is allowed first, exactly as the in-loop path does (`if not env_msgs:
-        # break` precedes its own `_check_messages`): an env with nothing further to observe
-        # legitimately returns nothing, and `_check_messages` rejects an empty list.
-        final_msgs = env.env_reply(state["messages"], state)
-        if final_msgs:
-            _check_messages(final_msgs, "env_reply")
+        # apply the final action's side effects, but do not validate its reply as a future prompt:
+        # no later model generation consumes it in the online path.
+        env.env_reply(state["messages"], state)
     if gold_finished is None:
         # the gold answer never ran out inside the loop -- it covered the whole episode, so the
         # break came first and the mid-loop sample never fired. NOW is its moment: the deferred

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/cli/commands/env/test_evaluations.py

@@ -1,12 +1,13 @@
 """Offline contract checks for an environment's evaluation sidecar.
 
-Split from `test.py` to keep that file under the 1000-line gate. This group is cohesive and has one
-entry point, `check_evaluation_suites`: everything else here exists to build one held-out case's
-prompt and replayed response the same way `flash env eval` would.
+Split from `test.py` to keep that file under the 1000-line gate. Single-turn cases validate prompt
+construction and reference replay. Episode cases validate initial rollout and prompt construction,
+then bind the scorer signature without executing it against unfinished state.
 """
 
 from __future__ import annotations
 
+import inspect
 from pathlib import Path
 
 from flash.cli.commands.env.push import _err
@@ -76,7 +77,41 @@ def _evaluation_response(env, case) -> tuple[str, str]:
     return policy, response
 
 
+def _check_episode_evaluation_prompt(env, case) -> None:
+    """Validate the held-out case's initial rollout state and prompt without advancing it."""
+    # imported here rather than at module scope: `test.py` imports this module.
+    from flash.cli.commands.env.test import _new_multi_turn_replay_state, _new_record
+
+    _new_multi_turn_replay_state(env, _evaluation_example(case), _new_record())
+
+
+def _validate_episode_scorer_signature(scorer, case, state_style: str | None) -> None:
+    """Bind the online scorer call shape without executing it against unfinished state."""
+    if not callable(scorer):
+        raise TypeError("evaluation suite score must be callable")
+    try:
+        signature = inspect.signature(scorer)
+    except (TypeError, ValueError):
+        # some builtins and extension callables expose no signature, so keep the permissive runtime
+        # behavior without calling them during an offline check.
+        return
+
+    response = "response"
+    state = {}
+    if state_style is None:
+        signature.bind(case, response)
+    elif state_style == "keyword":
+        signature.bind(case, response, state=state)
+    else:
+        signature.bind(case, response, state)
+
+
 def _check_evaluation_suites(entrypoint: Path, env) -> bool:
+    from flash.cli.commands.env.episode import (
+        _grades_episodes,
+        _state_argument,
+        _warn_if_episode_state_is_hidden,
+    )
     from flash.envs.evaluations import (
         _DEFAULT_EVALUATIONS_PATH,
         EvalSuiteReport,
@@ -107,9 +142,28 @@ def _check_evaluation_suites(entrypoint: Path, env) -> bool:
                     f"evaluation suite {suite.name} failed contract checks: suite produced no cases"
                 )
                 continue
+            episode_suite = _grades_episodes(suite)
+            if episode_suite and not getattr(env, "multi_turn", False):
+                raise TypeError(
+                    "suite sets grades_episodes = True, but this environment is single-turn "
+                    "and has no episode to play"
+                )
+            scorer = getattr(suite, "score", None)
+            if episode_suite:
+                state_style = _state_argument(scorer)
+                _warn_if_episode_state_is_hidden(suite, state_style)
+                for case in cases:
+                    _check_episode_evaluation_prompt(env, case)
+                _validate_episode_scorer_signature(scorer, cases[0], state_style)
+                print(
+                    f"evaluation suite {suite.name}: {len(cases)}/{len(cases)} cases "
+                    "passed contract checks"
+                )
+                continue
+
             for index, case in enumerate(cases, start=1):
                 _policy, response = _evaluation_response(env, case)
-                scored = suite.score(case, response)
+                scored = scorer(case, response)
                 result = normalize_eval_result(
                     case,
                     response,

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/cli/scaffold/TRAINING.md

@@ -224,6 +224,11 @@ int, `strict=true` as a bool, and anything that does not parse as a TOML scalar
 `flash env eval` takes the same two flags, for the same reason: a held-out suite scored
 against a differently-configured environment is not measuring your run.
 
+For a multi-turn environment, `flash env eval` still defaults to one prompt and one reply, which is
+a valid first-action check. Set `grades_episodes = True` on a suite to play one generation per turn.
+A scorer with the recommended `score(case, response, state)` signature receives the finished
+transcript state; otherwise the scorer receives only the episode's final response text.
+
 **It is also a three-example smoke test, not a dataset audit.** `flash env test` runs the
 contract checks against the **first 3 rows only** (`_DEFAULT_EPISODES = 3`, and it iterates
 `dataset[:episode_count]`), so a malformed prompt, a scorer that raises, or a bad episode

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/envs/evaluations.py

@@ -124,15 +124,35 @@ class EvalSuiteReport:
 
 @runtime_checkable
 class EvalSuite(Protocol):
-    """Authoring contract for one named held-out evaluation suite."""
+    """Authoring contract for a suite that grades one model response."""
 
-    name: str
+    @property
+    def name(self) -> str: ...
 
     def cases(self) -> list[EvalCase]: ...
 
     def score(self, case: EvalCase, response: str) -> EvalResult | float | bool: ...
 
 
+class EpisodeEvalSuite(Protocol):
+    """Suite contract for full-episode play at one generation per turn.
+
+    ``grades_episodes`` must be true at runtime. The finished transcript is passed in ``state``.
+    """
+
+    @property
+    def name(self) -> str: ...
+
+    grades_episodes: bool
+
+    def cases(self) -> list[EvalCase]: ...
+
+    def score(self, case: EvalCase, response: str, state: dict) -> EvalResult | float | bool: ...
+
+
+EvaluationSuite = EvalSuite | EpisodeEvalSuite
+
+
 class BaseEvalSuite:
     """Convenience base with expected-answer substring scoring."""
 
@@ -194,8 +214,20 @@ def normalize_eval_result(
     )
 
 
-def validate_evaluation_cases(suite: EvalSuite, *, source: str | Path) -> list[EvalCase]:
+def _episode_grading_enabled(suite: EvaluationSuite) -> bool:
+    """Return the validated full-episode grading opt-in."""
+    value = getattr(suite, "grades_episodes", False)
+    if not isinstance(value, bool):
+        name = getattr(suite, "name", "evaluation")
+        raise TypeError(
+            f"suite {name!r} grades_episodes must be a bool, got {type(value).__name__}"
+        )
+    return value
+
+
+def validate_evaluation_cases(suite: EvaluationSuite, *, source: str | Path) -> list[EvalCase]:
     """Load and validate one suite's cases with source-aware errors."""
+    _episode_grading_enabled(suite)
     try:
         cases = suite.cases()
     except Exception as exc:
@@ -398,7 +430,7 @@ class _ScopedSuite:
 
     __slots__ = ("_module_dir", "_suite")
 
-    def __init__(self, suite: EvalSuite, module_dir: str) -> None:
+    def __init__(self, suite: EvaluationSuite, module_dir: str) -> None:
         self._suite = suite
         self._module_dir = module_dir
 
@@ -492,7 +524,7 @@ def _call_factory(factory, kwargs: dict[str, object]) -> object:
     return factory(*positional, **filtered_kwargs)
 
 
-def _validate_suites(value: object, *, source: Path) -> list[EvalSuite]:
+def _validate_suites(value: object, *, source: Path) -> list[EvaluationSuite]:
     if not isinstance(value, (list, tuple)):
         raise TypeError(
             f"{source}: expected {_DEFAULT_EVALUATIONS_FACTORY}() or EVALUATIONS "
@@ -503,7 +535,7 @@ def _validate_suites(value: object, *, source: Path) -> list[EvalSuite]:
             f"{source}: expected at least one evaluation suite from "
             f"{_DEFAULT_EVALUATIONS_FACTORY}() or EVALUATIONS"
         )
-    suites: list[EvalSuite] = []
+    suites: list[EvaluationSuite] = []
     names: set[str] = set()
     for index, suite in enumerate(value, start=1):
         name = getattr(suite, "name", None)
@@ -526,7 +558,7 @@ def _validate_suites(value: object, *, source: Path) -> list[EvalSuite]:
 
 def load_evaluation_suites(
     env_ref_or_path: str | Path, *, environment: object | None = None
-) -> list[EvalSuite]:
+) -> list[EvaluationSuite]:
     """Load evaluations.py beside a resolved environment entrypoint."""
     source = _evaluation_path(env_ref_or_path)
     if not source.is_file():
@@ -574,10 +606,12 @@ def load_evaluation_suites(
 __all__ = [
     "_DEFAULT_EVALUATIONS_PATH",
     "BaseEvalSuite",
+    "EpisodeEvalSuite",
     "EvalCase",
     "EvalResult",
     "EvalSuite",
     "EvalSuiteReport",
+    "EvaluationSuite",
     "_evaluation_path",
     "has_evaluations",
     "load_evaluation_suites",

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/runner/supervise/transitions.py

@@ -195,6 +195,25 @@ def mark_deployment_pending(
         return status
 
 
+def _confirmed_active_failed_predecessor(deployment: object, run_id: str) -> bool:
+    if not isinstance(deployment, dict):
+        return False
+    revision = deployment.get("adapter_revision")
+    parsed = parse_adapter_revision(revision) if isinstance(revision, str) else None
+    return (
+        deployment.get("state") == "failed"
+        and deployment.get("alias_activation_confirmed") is True
+        and parsed is not None
+        and parsed[0] == run_id
+    )
+
+
+def _restorable_deployment_predecessor(deployment: object, run_id: str) -> bool:
+    return (
+        isinstance(deployment, dict) and deployment.get("state") in _RESTORABLE_DEPLOYMENT_STATES
+    ) or _confirmed_active_failed_predecessor(deployment, run_id)
+
+
 def mark_deployment_failed(run_id: str, deployment: dict) -> RunStatus:
     """Record a failed deployment attempt while preserving the run lifecycle state."""
     from flash.runner import _save_status_unlocked, _status_guard, get_status
@@ -212,10 +231,12 @@ def mark_deployment_failed(run_id: str, deployment: dict) -> RunStatus:
         ):
             return status
         previous = deployment.get("previous_deployment")
-        if (
+        if _restorable_deployment_predecessor(previous, run_id) and (
             not deployment.get("activation_outcome_unknown")
-            and isinstance(previous, dict)
-            and previous.get("state") in _RESTORABLE_DEPLOYMENT_STATES
+            or (
+                deployment.get("state") == "failed"
+                and _confirmed_active_failed_predecessor(previous, run_id)
+            )
         ):
             status.deployment = {
                 **previous,

{freesolo_flash_dev-1.2.33 → freesolo_flash_dev-1.2.36}/flash/serve/deploy.py

@@ -23,6 +23,7 @@ from flash.serve.errors import (  # noqa: F401 -- re-exported: callers import th
     ActivationOutcomeUnknown,
     AdapterConfigMissing,
     AdapterTensorMissing,
+    AliasThinkingSilent,
     RetryableServingUnavailable,
     ServingError,
 )
@@ -67,14 +68,15 @@ READBACK_MAX_DELAY_SECONDS = 2.0
 # with margin. the floor is doubled to 10 and the per-B term covers a bigger base on top.
 #
 # the cap is the real constraint, and readiness is only one leg of the attempt. the same deploy also
-# spends time BEFORE this wait (resolving the hub revision, downloading the adapter config to read
-# its rank, the capability check, registration) and AFTER it (`_SMOKE_BUDGET_SECONDS` = 600s of
-# smoke, then activation). the whole attempt must finish inside BOTH `_DEPLOYMENT_STALE_SECONDS`
-# (1800s, when the plane declares an in-flight attempt abandoned) and the CLI's 1800s default
-# `--wait`, or a deploy that is still progressing is reaped or reported as failed.
+# spends time before this wait (resolving the hub revision, downloading the adapter config to read
+# its rank, the capability check, registration) and after it (600s of immutable smoke, activation,
+# then 600s of alias smoke). the whole attempt must finish inside both
+# `_DEPLOYMENT_STALE_SECONDS` (2400s, when the plane declares an in-flight attempt abandoned) and
+# the CLI's 2400s default `--wait`, or a deploy that is still progressing is reaped or reported as
+# failed.
 #
-# so the cap leaves real slack rather than just clearing smoke: 900 + 600 = 1500, keeping 300s for
-# the surrounding hub reads, registration, activation, and poll latency, none of which share a
+# so the cap leaves real slack rather than just clearing smoke: 900 + 600 + 600 = 2100, keeping 300s
+# for the surrounding hub reads, registration, activation, and poll latency, none of which share a
 # wall-clock bound with this one.
 #
 # a longer budget costs little: an adapter serving REJECTS raises as soon as the revision reports
@@ -818,6 +820,7 @@ def _retryable_smoke_unavailable(
     response: httpx.Response,
     *,
     requested_model: str,
+    expected_adapter_revision: str,
 ) -> RetryableServingUnavailable | None:
     if response.status_code != 503:
         return None
@@ -834,7 +837,7 @@ def _retryable_smoke_unavailable(
         or error.get("retryable") is not True
         or code not in _RETRYABLE_SMOKE_503_CODES
         or error.get("requested_model") != requested_model
-        or error.get("adapter_revision") != requested_model
+        or error.get("adapter_revision") != expected_adapter_revision
     ):
         return None
     raw_delay = response.headers.get("Retry-After") or error.get("retry_after_seconds")
@@ -854,6 +857,7 @@ def chat(
     max_tokens: int = 512,
     thinking: bool = False,
     expected_checkpoint: str | None = None,
+    expected_adapter_revision: str | None = None,
     timeout_s: float | None = None,
     retry_unavailable: bool = False,
     stop: list[str] | None = None,
@@ -887,7 +891,11 @@ def chat(
     with client_context as client:
         resp = client.post(f"{base}/chat/completions", json=body, headers=headers, timeout=timeout)
         if retry_unavailable:
-            retryable_error = _retryable_smoke_unavailable(resp, requested_model=run_id)
+            retryable_error = _retryable_smoke_unavailable(
+                resp,
+                requested_model=run_id,
+                expected_adapter_revision=expected_adapter_revision or run_id,
+            )
             if retryable_error is not None:
                 raise retryable_error
         resp.raise_for_status()