freesolo-flash-dev 1.2.128__tar.gz → 1.2.129__tar.gz

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/.env.example

@@ -33,6 +33,9 @@ RUNPOD_API_KEY=
 LAMBDA_API_KEY=
 # Vast.ai API key.
 VAST_API_KEY=
+# exact https origins Vast may return for asynchronous result retrieval. configured values replace
+# the default https://s3.amazonaws.com allowlist. this belongs only on the control-plane host.
+# FLASH_VAST_RESULT_ORIGINS=https://s3.amazonaws.com
 
 # --- required: artifacts ---
 # HuggingFace token with write access to the dataset repos flash creates for artifacts

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/.github/workflows/deploy-modal.yml

@@ -39,10 +39,14 @@ jobs:
     # this only constrains manual dispatches; a non-main dispatch skips without running `modal deploy`.
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
-    # the job's own bounded waits are two 300s readiness polls plus the remote image build; 30
-    # minutes clears that with room while keeping a wedged deploy from holding the runner for
-    # GitHub's 6-hour default.
-    timeout-minutes: 30
+    # the job's own bounded waits now sum well past the old 30: two 300s readiness polls, the remote
+    # image build, a 420s canary that pays a GPU cold start, a 180s accounting deadline, and on the
+    # failure path a SECOND `modal deploy` plus a 300s restore poll. 60 minutes clears that worst
+    # case with room while keeping a wedged deploy from holding the runner for GitHub's 6-hour
+    # default. the headroom is specifically for the rollback: a job killed partway through the
+    # restore leaves the broken release live with no second attempt, which is the one outcome this
+    # workflow exists to prevent.
+    timeout-minutes: 60
     # serialize serving deploys across runs because each deploy mutates the live app. github can still
     # replace a pending run, so the gate compares against the sha reported by the deployed app rather
     # than assuming every push predecessor was deployed.
@@ -87,9 +91,14 @@ jobs:
           DEPLOYED_HEALTH_URL: https://serve.freesolo.co/healthz
         run: |
           deploy=false
-          if [ "$EVENT" = "workflow_dispatch" ]; then
-            deploy=true
-          elif ! last_deployed_sha="$(python3 - <<'PY'
+          # the sha reported by the LIVE app is read ONCE, before any decision, because it serves
+          # two purposes: it bounds the undeployed diff below, and it is the rollback target for a
+          # failed promotion. `modal deploy` overwrites the app in place, so after that point there
+          # is no way left to ask what the previous release was. reading it on workflow_dispatch
+          # too -- where the deploy decision does not need it -- is what makes a dispatched deploy
+          # rollback-capable rather than one-way.
+          read_ok=true
+          last_deployed_sha="$(python3 - <<'PY'
           import json
           import os
           import re
@@ -107,7 +116,17 @@ jobs:
               raise ValueError("deployed app returned no valid source revision")
           print(revision.lower())
           PY
-          )"; then
+          )" || read_ok=false
+          if [ "$read_ok" != "true" ]; then
+            last_deployed_sha=""
+          fi
+          # an empty previous sha means rollback is IMPOSSIBLE for this run. the rollback step below
+          # says so out loud rather than reporting a restore it never performed.
+          echo "previous_sha=$last_deployed_sha" >> "$GITHUB_OUTPUT"
+
+          if [ "$EVENT" = "workflow_dispatch" ]; then
+            deploy=true
+          elif [ "$read_ok" != "true" ]; then
             # a missing deployment or any health, network, or parsing failure must deploy safely.
             deploy=true
           elif ! git merge-base --is-ancestor "$last_deployed_sha" "$CURRENT_SHA" 2>/dev/null; then
@@ -282,6 +301,7 @@ jobs:
           done
 
       - name: Deploy serving
+        id: deploy_serving
         if: steps.gate.outputs.deploy == 'true'
         env:
           FREESOLO_DEPLOYMENT_SHA: ${{ github.sha }}
@@ -371,6 +391,124 @@ jobs:
             sleep "$sleep_seconds"
           done
 
+      - name: Install promotion gate dependencies
+        if: steps.gate.outputs.deploy == 'true'
+        working-directory: .
+        run: |
+          # the gate imports flash.serving.promotion, which needs httpx for the streaming canary and
+          # pydantic-settings for the accounting client. this job deliberately does not `uv sync`
+          # (see "Install Modal client"), so install exactly what the gate imports and nothing more.
+          # these are the exact versions uv.lock resolves, so the gate runs against the same client
+          # behaviour the serving tests were validated on rather than whatever pypi ships that day.
+          python -m pip install "httpx==0.28.1" "pydantic==2.13.4" "pydantic-settings==2.15.0"
+
+      - name: Prove the promotion with a real streaming canary
+        if: steps.gate.outputs.deploy == 'true'
+        working-directory: .
+        env:
+          # a healthy /healthz proves a ROUTER answered with the expected identity. it does not prove
+          # a gpu engine started, that a generation ran, that streaming works, or that usage settled
+          # durably. this step is the only thing standing between "the process booted" and "the
+          # release can actually serve", so a failure here must not be promoted.
+          SERVING_BASE_URL: https://serve.freesolo.co
+          FREESOLO_INTERNAL_KEY: ${{ secrets.FREESOLO_INTERNAL_KEY }}
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
+          GITHUB_SHA: ${{ github.sha }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+        # credentials reach the gate ONLY through the environment above. passing any of them as an
+        # argument would put them in the process table and in the step's own command echo.
+        run: python -m flash.serving.promotion.gate
+
+      - name: Restore the previous release
+        # `failure()` alone is too broad. It also fires for the steps that run BEFORE any deploy --
+        # installing Modal, verifying auth, verifying the runtime secrets, waiting for the adapter
+        # schema. Restoring there would `modal deploy` the previous SHA over a live app this run never
+        # replaced, using the very secret set whose verification just failed: a healthy release
+        # overwritten by a rollback that had nothing to roll back. Gating on the deploy step's own
+        # outcome keeps the case the previous comment was right about -- a deploy that half-succeeded
+        # and then failed readiness is `failure`, not skipped, so it still restores. Any outcome other
+        # than an attempted deploy (empty, `skipped`) means production is untouched and must stay so.
+        if: >-
+          failure() && steps.gate.outputs.deploy == 'true'
+          && (steps.deploy_serving.outcome == 'success' || steps.deploy_serving.outcome == 'failure')
+        working-directory: .
+        env:
+          PREVIOUS_SHA: ${{ steps.gate.outputs.previous_sha }}
+          CURRENT_SHA: ${{ github.sha }}
+          # a distinct attempt id so /healthz cannot report the FORWARD deploy's identity and be
+          # mistaken for a completed rollback.
+          ROLLBACK_DEPLOYMENT_ID: ${{ github.run_id }}-${{ github.run_attempt }}-rollback
+          HEALTH_URL: https://serve.freesolo.co/healthz
+        run: |
+          if [ -z "$PREVIOUS_SHA" ]; then
+            echo "::error::Promotion failed and NO previous release was recorded, so the broken release is still live. Restore production manually."
+            exit 1
+          fi
+          if ! git merge-base --is-ancestor "$PREVIOUS_SHA" "$CURRENT_SHA" 2>/dev/null; then
+            # refusing here is the safe branch: redeploying a sha that is not an ancestor of this
+            # head would deploy an unknown tree rather than the release that was actually live.
+            echo "::error::Recorded previous release ${PREVIOUS_SHA} is not an ancestor of ${CURRENT_SHA}; refusing to redeploy an unverified tree. Restore production manually."
+            exit 1
+          fi
+
+          echo "Promotion failed; restoring ${PREVIOUS_SHA}."
+          git checkout --quiet --force "$PREVIOUS_SHA"
+
+          # the restored app must report the RESTORED sha, deployed from the restored tree.
+          FREESOLO_DEPLOYMENT_SHA="$PREVIOUS_SHA" \
+          FREESOLO_DEPLOYMENT_ID="$ROLLBACK_DEPLOYMENT_ID" \
+            modal deploy flash/serving/app/modal_app.py
+
+          response_file="$(mktemp)"
+          trap 'rm -f "$response_file"' EXIT
+          deadline=$((SECONDS + 300))
+          restored=false
+          while (( SECONDS < deadline )); do
+            if curl --silent --show-error --output "$response_file" --fail \
+              --connect-timeout 5 --max-time 10 "$HEALTH_URL"; then
+              if python3 - "$response_file" "$PREVIOUS_SHA" "$ROLLBACK_DEPLOYMENT_ID" <<'PY'
+          import json
+          import sys
+
+
+          def reject_constant(_value: str) -> None:
+              raise ValueError
+
+
+          try:
+              with open(sys.argv[1], encoding="utf-8") as response:
+                  payload = json.load(response, parse_constant=reject_constant)
+          except (OSError, UnicodeDecodeError, ValueError):
+              sys.exit(1)
+
+          restored = (
+              isinstance(payload, dict)
+              and payload.get("ok") is True
+              and payload.get("deployment_sha") == sys.argv[2]
+              and payload.get("deployment_id") == sys.argv[3]
+          )
+          sys.exit(0 if restored else 1)
+          PY
+              then
+                restored=true
+                break
+              fi
+            fi
+            sleep 10
+          done
+
+          if [ "$restored" != "true" ]; then
+            echo "::error::Rollback to ${PREVIOUS_SHA} was deployed but production never reported the restored release. Production may be serving a broken release; intervene manually."
+            exit 1
+          fi
+          # exit nonzero regardless: a restored predecessor means production is serving again, NOT
+          # that this promotion succeeded. reporting green here would mark an unpromotable commit as
+          # deployed and let the next push build on a release that never passed its gate.
+          echo "::error::Promotion failed; production was restored to ${PREVIOUS_SHA}. This commit is NOT deployed."
+          exit 1
+
       - name: Nothing to deploy
         if: steps.gate.outputs.deploy != 'true'
         run: echo "No serving changes in this push; skipping deploy."

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/CHANGELOG.md

@@ -13,6 +13,11 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Added
 
+- OpenAI chat serving now implements and offline-validates strict function tool calling for the exact
+  Qwen3.5 packaged and hosted paths, including validated tool history, buffered and raw SSE
+  `tool_calls`, and managed forwarding. Flash parses schema-valid calls but does not execute them;
+  malformed automatic calls remain exact assistant text. Tool calling is not live-qualified pending
+  exact model testing.
 - OpenAI chat serving now preserves mirror sampling controls across packaged, hosted, and managed
   entry points: up to four indexed choices, signed seeds, frequency and presence penalties, and
   OpenAI token logprobs. Buffered and raw SSE responses retain independent choice terminals and

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.128
+Version: 1.2.129
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/SELF_HOSTING.md

@@ -235,6 +235,12 @@ placeholder value to set for the ones you skip.
 | Lambda   | `LAMBDA_API_KEY` |                                                                 |
 | Vast     | `VAST_API_KEY`   |                                                                 |
 
+Vast log retrieval accepts signed result URLs only from exact HTTPS origins in
+`FLASH_VAST_RESULT_ORIGINS`. The variable is control-plane-only and comma-separated. Leaving it
+unset or blank allows only `https://s3.amazonaws.com`; setting it replaces that default. Origins
+must be canonical HTTPS origins without credentials, explicit ports, paths, queries, fragments,
+wildcards, spaces, or control characters.
+
 The allocator ranks candidate GPU classes only on the substrates you configured, so a
 plane with just `VAST_API_KEY` set allocates on Vast and never proposes a class it cannot
 provision. Startup fails only when **all three** are missing.

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/docs/serving-contract.md

@@ -75,6 +75,58 @@ The packaged, hosted, and managed OpenAI entry points retain the shared strict s
 - `logprobs` is a strict boolean and `top_logprobs` is an integer from 0 through 20.
 - thinking-enabled adapters reject logprobs.
 
+Function tools are implemented and offline-validated only for the exact qualified Qwen3.5 packaged
+and hosted profile. This path is not live-qualified pending exact model testing. The parser identity
+is `qwen3_coder`; Qwen3.6 and unqualified engines reject tools after authorization and checkpoint
+resolution, before generation. Declarations must be closed function objects with unique nonempty
+names. Parameters use a bounded root-object JSON Schema profile with `properties`, `required`,
+`additionalProperties: false`, recursive scalar, object, and array types, descriptions, enums, and
+array `items`. Numeric enum members must be JSON integers; decimal and exponent numeric enum lexemes
+are rejected because Flash cannot preserve them exactly across every ingress and template boundary.
+Generated and historical JSON numeric literals support at most 1024 significand digits and exponent
+magnitude at most 1,000,000; generated candidates outside either bound remain exact text, while
+history outside either bound is rejected. A historical value within those bounds that no native
+template number carries faithfully, because it
+would overflow, underflow, or lose digits, is rendered as its exact compact text instead of being
+converted or rejected, and a container holding such a value is rendered exactly as a whole. A
+top-level boolean or null argument is likewise pre-rendered as `true`, `false`, or `null`, because the
+template spells a scalar with `string` and would otherwise show the model Python's `True` and `None`;
+inside a container those values stay native, since the template's `tojson` already spells them
+correctly. A response carries at most 408 tool calls, and a longer run of candidates stays exact text. That is
+the most any continuation could carry, not a promise that this batch replays. Replay cost depends on
+the follow-up request as a whole: the prior conversation shares the same message-complexity budget,
+and each result costs four nodes as a plain string, five with the optional `name`, and seven for a
+single text block plus three for each additional block. A long enough history leaves no room for
+even one call. Clients that intend to continue the tool lifecycle must therefore not assume any
+positive call count replays under every history: a follow-up that exceeds the budget is rejected
+with `messages exceed the supported complexity`. The budget counts structure rather than text, so
+the remedies are to send fewer messages or fewer content blocks, drop optional result `name` fields,
+use plain string results rather than text blocks, or request fewer calls; shortening the text inside
+a message does not help, because a string is one node however long it is. These local
+serving-contract bounds do not depend on Python's process-wide integer conversion limit. Unsupported
+keywords and `strict: true` are rejected.
+
+`tool_choice` defaults to `auto` and accepts only `auto` or `none`. `parallel_tool_calls` defaults to
+`true` and accepts only exact `true`; either control requires `tools`. Tools cannot be combined with
+thinking, logprobs, structured outputs, a non-text `response_format`, or image messages. Historical
+assistant calls require unique IDs, function type, valid names, and JSON-string object arguments.
+Each immediately following tool-result turn must resolve every declared call exactly once before a
+non-tool turn. External messages remain unchanged; only a detached template copy converts argument
+strings to objects.
+
+In `auto` mode, complete schema-valid Qwen3 Coder XML candidates become OpenAI `tool_calls` with
+independent call indexes and `finish_reason: "tool_calls"`. Parsed responses contain no raw Qwen
+tags. Ordinary text and malformed or incomplete candidates remain exact assistant text with the
+native finish reason. Candidate regions are buffered in raw SSE because argument deltas cannot be
+retracted safely. Native prompt, completion, cached, and reasoning token accounting remains
+authoritative and is never recomputed from serialized arguments. Flash parses calls but does not
+execute or resubmit them.
+
+The decoded convenience `chat_stream` iterator remains text-only and single-choice. It rejects
+`n != 1`, logprobs, and tools before opening transport; use buffered JSON or raw SSE for those
+features. Raw `/generate` and `/adapters/{id}/generate` remain tool-free. Buffered hosted responses
+and raw hosted SSE are authoritative; managed serving forwards them and never reparses tool calls.
+
 ## Public provenance
 
 Successful run-backed responses identify only the permanent checkpoint:

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/flash/client/http.py

@@ -33,6 +33,7 @@ from flash.client.streaming import (
 )
 from flash.core.spec import require_project_id
 from flash.serve.contract.urls import is_freesolo_hosted_url
+from flash.serve.request.tool_calls import validate_tool_control_presence
 
 
 class ClientError(RuntimeError):
@@ -213,9 +214,18 @@ def _prepare_chat_request(
     max_tokens: int,
     *,
     stream: bool = False,
+    tools: list[dict[str, Any]] | None = None,
+    tool_choice: str | None = None,
+    parallel_tool_calls: bool | None = None,
 ) -> tuple[str, dict[str, Any]]:
     base_run_id, checkpoint_id = _parse_chat_target(target)
     _validate_chat_messages(messages)
+    validate_tool_control_presence(
+        tools,
+        tool_choice,
+        parallel_tool_calls,
+        error_type=ClientError,
+    )
     body: dict[str, Any] = {
         "checkpoint_id": checkpoint_id,
         "messages": messages,
@@ -224,6 +234,10 @@ def _prepare_chat_request(
     }
     if stream:
         body["stream"] = True
+    if tools is not None:
+        body["tools"] = tools
+        body["tool_choice"] = "auto" if tool_choice is None else tool_choice
+        body["parallel_tool_calls"] = True if parallel_tool_calls is None else parallel_tool_calls
     return base_run_id, body
 
 
@@ -767,12 +781,19 @@ class ApiClient:
         temperature: float = 0.0,
         max_tokens: int = 512,
         timeout: float | None = None,
+        *,
+        tools: list[dict[str, Any]] | None = None,
+        tool_choice: str | None = None,
+        parallel_tool_calls: bool | None = None,
     ) -> dict:
         base_run_id, body = _prepare_chat_request(
             run_id,
             messages,
             temperature,
             max_tokens,
+            tools=tools,
+            tool_choice=tool_choice,
+            parallel_tool_calls=parallel_tool_calls,
         )
         return self._request(
             "POST",
@@ -787,7 +808,11 @@ class ApiClient:
         messages: list[dict],
         temperature: float = 0.0,
         max_tokens: int = 512,
+        *,
+        tools: list[dict[str, Any]] | None = None,
     ) -> Iterator[str]:
+        if tools is not None:
+            raise ValueError("decoded chat_stream does not support tools")
         base_run_id, body = _prepare_chat_request(
             run_id,
             messages,

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/flash/engine/worker/train/entry/score_batcher.py

@@ -18,9 +18,9 @@ Callers differ in three ways, each a constructor argument rather than a fork in
   ``TeacherError`` so the caller does not retry a batcher that is gone.
 * ``wrap_batch_error`` re-shapes an exception raised by ``score_batch`` before it reaches the
   waiters. GRPO propagates as-is; OPD narrows to its own error type, preserving permanence.
-* ``recheck_closed_after_wait`` decides whether a batch assembled during the flush window is still
-  scored once ``close`` has been called. Leaving it off scores that final partial batch; turning it
-  on abandons it to the stranded-waiter path.
+* ``cancel_undispatched_on_close`` decides whether ``close`` immediately cancels work that has not
+  yet won the condition-protected dispatch claim. Leaving it off gives the consumer the bounded
+  shutdown window to claim the final partial batch; turning it on fails that work without scoring it.
 """
 
 from __future__ import annotations
@@ -87,7 +87,7 @@ class ScoreBatcher:
         thread_name: str,
         make_error: Callable[[str], Exception] = RuntimeError,
         wrap_batch_error: Callable[[Exception], Exception] | None = None,
-        recheck_closed_after_wait: bool = False,
+        cancel_undispatched_on_close: bool = False,
     ) -> None:
         if max_batch_size <= 0:
             raise ValueError(f"{label} batch size must be positive")
@@ -100,7 +100,7 @@ class ScoreBatcher:
         self.thread_name = thread_name
         self._make_error = make_error
         self._wrap_batch_error = wrap_batch_error
-        self._recheck_closed_after_wait = bool(recheck_closed_after_wait)
+        self._cancel_undispatched_on_close = bool(cancel_undispatched_on_close)
         self._condition = threading.Condition()
         self._pending: list[_Waiter] = []
         self._in_flight: list[_Waiter] = []
@@ -114,9 +114,9 @@ class ScoreBatcher:
                 raise self._make_error(f"{self.label} shut down")
             if self._thread is not None:
                 return
-            self._thread = threading.Thread(target=self._run, name=self.thread_name, daemon=True)
-            thread = self._thread
-        thread.start()
+            thread = threading.Thread(target=self._run, name=self.thread_name, daemon=True)
+            thread.start()
+            self._thread = thread
 
     def submit(self, request: Any) -> Any:
         """Enqueue one request and block until its batch has been scored."""
@@ -129,7 +129,8 @@ class ScoreBatcher:
             self._condition.notify_all()
         return waiter.wait(self._make_error)
 
-    def _take_batch(self) -> list[_Waiter] | None:
+    def _claim_batch(self) -> list[_Waiter] | None:
+        """Wait for work, then atomically cancel it or claim it for dispatch."""
         with self._condition:
             while not self._pending:
                 if self._closed:
@@ -141,17 +142,20 @@ class ScoreBatcher:
                 if remaining <= 0:
                     break
                 self._condition.wait(remaining)
-            if self._recheck_closed_after_wait and self._closed:
-                return None
             batch = self._pending[: self.max_batch_size]
+            if not batch:
+                # `close` cancelled the queue while this held no claim: the only way the queue can
+                # empty between the outer loop and here. dispatching now would score an empty batch.
+                return None
             del self._pending[: len(batch)]
             self._in_flight = batch
+            self._condition.notify_all()
             return batch
 
     def _run(self) -> None:
         try:
             while True:
-                batch = self._take_batch()
+                batch = self._claim_batch()
                 if batch is None:
                     return
                 try:
@@ -163,10 +167,9 @@ class ScoreBatcher:
                     wrapped = self._wrap_batch_error(error) if self._wrap_batch_error else error
                     for waiter in batch:
                         waiter.complete(error=wrapped)
-                finally:
-                    with self._condition:
-                        self._in_flight = []
-                        self._condition.notify_all()
+                with self._condition:
+                    self._in_flight = []
+                    self._condition.notify_all()
         finally:
             # the thread is leaving for good, so anything still queued or in flight will never be
             # answered. fail them here rather than leaving their callers blocked forever.
@@ -181,11 +184,17 @@ class ScoreBatcher:
                 waiter.complete(error=error)
 
     def close(self, timeout_s: float) -> None:
-        """Stop accepting requests, fail what is queued, and wait out the batch in flight.
+        """Stop accepting requests and settle queued or claimed work within the shutdown bound.
+
+        There is exactly one cancellation point, and ``_pending`` is what it cancels: work that has
+        not won the condition-protected claim in ``_claim_batch``. The flag only decides whether the
+        consumer is given the bounded window to win that claim first, so an unbillable dispatch is
+        prevented by never yielding the condition rather than by a second racing check.
 
-        The join is bounded because the scorer is a network call that can hang. A batch that lands
-        within the bound scatters its real results -- ``complete`` is first-writer-wins, so the
-        drain below cannot overwrite them. One that does not is failed so its callers unblock.
+        ``_in_flight`` is the claimed batch, so shutdown retains it through the join and the final
+        drain. A batch that lands within the bound scatters its real results; ``complete`` is
+        first-writer-wins, so the drain below cannot overwrite them. One that does not is failed so
+        its callers unblock.
 
         The two callers this replaced spelled the final drain differently -- one gated it on
         ``thread.is_alive()``, the other ran it unconditionally -- but the forms are equivalent, so
@@ -193,16 +202,30 @@ class ScoreBatcher:
         its own stranded waiters in ``_run``'s ``finally``, leaving nothing for the drain to find.
         """
         error = self._make_error(f"{self.label} shut down")
+        deadline = time.monotonic() + max(0.0, timeout_s)
         with self._condition:
             self._closed = True
-            pending = list(self._pending)
-            self._pending.clear()
             self._condition.notify_all()
             thread = self._thread
-        for waiter in pending:
+            # the consumer cannot run while this holds the condition, so skipping the window below
+            # means it never gets to claim -- which is exactly what an unbillable dispatch needs.
+            while (
+                not self._cancel_undispatched_on_close
+                and self._pending
+                and thread is not None
+                and thread.is_alive()
+            ):
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    break
+                self._condition.wait(remaining)
+            cancelled = list(self._pending)
+            self._pending.clear()
+            self._condition.notify_all()
+        for waiter in cancelled:
             waiter.complete(error=error)
         if thread is not None:
-            thread.join(timeout=max(0.0, timeout_s))
+            thread.join(timeout=max(0.0, deadline - time.monotonic()))
         # unconditional: a thread that exited already emptied `_in_flight` and settled its own
         # stranded waiters, so this is a no-op in that case rather than a second verdict.
         with self._condition:

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/flash/engine/worker/train/opd/bridging/batching.py

@@ -136,8 +136,8 @@ class _TextTeacherBatcher(ScoreBatcher):
     temperature), and the teacher charges per scored item. Sending the pair once and marking the
     repeats unbilled keeps the answer identical while paying for it once.
 
-    ``recheck_closed_after_wait`` is on: a batch assembled during the flush window after ``close``
-    must NOT be sent, because a teacher call bills real money and the run is already shutting down.
+    ``cancel_undispatched_on_close`` is on: ``close`` cancels any batch that has not yet won the
+    dispatch claim, because a teacher call bills real money and the run is already shutting down.
     """
 
     def __init__(
@@ -158,7 +158,7 @@ class _TextTeacherBatcher(ScoreBatcher):
             thread_name="flash-opd-text-teacher-batcher",
             make_error=_permanent_teacher_error,
             wrap_batch_error=_teacher_batch_error,
-            recheck_closed_after_wait=True,
+            cancel_undispatched_on_close=True,
         )
         self.teacher = teacher
         self.on_scored = on_scored

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/flash/engine/worker/train/opd/bridging/bridge.py

@@ -328,9 +328,9 @@ class _TeacherAlignmentBridge(TeacherFailureRecording):
         if self.max_turns <= 0:
             raise ValueError("flash OPD bridge multi-turn limit is invalid")
 
-    def _env_call(self, method: str, *args):
+    def _env_call(self, method: str, *args, **kwargs):
         with self.parent_work.busy():
-            return getattr(self.active_env, method)(*args)
+            return getattr(self.active_env, method)(*args, **kwargs)
 
     @staticmethod
     def _validate_session_id(session_id: str) -> str:
@@ -426,12 +426,26 @@ class _TeacherAlignmentBridge(TeacherFailureRecording):
                 existing["lease_deadline"] = now + self.session_lease_s
                 return {"max_turns": existing["turn_limit"]}
             with self._env_lock:
-                state = self._env_call("new_rollout_state", prompt.example)
-                initial_messages, fresh_descriptors = normalize_initial_prompt(
-                    prompt,
-                    state,
-                    self.processor,
+                state = self._env_call(
+                    "new_rollout_state",
+                    prompt.example,
+                    prepared_prompt=prompt.student_messages,
                 )
+                # the prepared prompt and its descriptors were frozen together before the child
+                # authenticated them. normalized image blocks intentionally carry no source, so an
+                # exact prepared state must reuse those frozen descriptors instead of rehydrating the
+                # prompt and rerunning environment preparation.
+                if state.get("prompt") == prompt.student_messages:
+                    initial_messages = validate_structured_messages(
+                        state["prompt"], source="environment initial prompt"
+                    )
+                    fresh_descriptors = tuple(prompt.image_descriptors)
+                else:
+                    initial_messages, fresh_descriptors = normalize_initial_prompt(
+                        prompt,
+                        state,
+                        self.processor,
+                    )
             if initial_messages != frozen_prompt or fresh_descriptors != prompt.image_descriptors:
                 raise ValueError(
                     "multi-turn environment initial prompt changed after prompt freezing"

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/flash/providers/core/preflight.py

@@ -127,6 +127,13 @@ def require_operator_config() -> None:
     problems: list[str] = []
 
     configured = available_providers()
+    if "vast" in configured or "FLASH_VAST_RESULT_ORIGINS" in os.environ:
+        from flash.providers.vast.client.result import configured_result_origins
+
+        try:
+            configured_result_origins()
+        except ValueError as exc:
+            problems.append(f"  - {exc}")
     if not configured:
         problems.append(
             "  - no GPU provider is configured. Set at least ONE of:\n"

{freesolo_flash_dev-1.2.128 → freesolo_flash_dev-1.2.129}/flash/providers/vast/client/api.py

@@ -12,7 +12,6 @@ import json
 import time
 import urllib.error
 import urllib.parse
-import urllib.request
 from typing import Any
 
 from flash._internal.logging import get_logger
@@ -22,6 +21,7 @@ from flash.providers._lifecycle.net.deadline import (
     require_deadline_at,
 )
 from flash.providers._lifecycle.net.http import RestClient, is_not_found
+from flash.providers.vast.client.result import VastResultError, fetch_result
 
 logger = get_logger(__name__)
 
@@ -404,13 +404,13 @@ def instance_logs(instance_id: int, *, deadline_at: float | None = None) -> str
             if remaining <= 0:
                 break
             try:
-                with urllib.request.urlopen(url, timeout=min(15.0, remaining)) as resp:
-                    body = resp.read().decode(errors="replace")
+                result = fetch_result(url, timeout=min(15.0, remaining))
+            except VastResultError:
+                return None
+            if result is not None:
+                body = result.decode(errors="replace")
                 if body.strip():
                     return body
-            except urllib.error.HTTPError as e:
-                if e.code != 404:  # 404 = not materialized yet
-                    return None
             sleep_for = min(2.0, remaining_seconds(poll_deadline))
             if sleep_for <= 0:
                 break