freesolo-flash-dev 1.2.127__tar.gz → 1.2.129__tar.gz

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/.env.example

@@ -33,6 +33,9 @@ RUNPOD_API_KEY=
 LAMBDA_API_KEY=
 # Vast.ai API key.
 VAST_API_KEY=
+# exact https origins Vast may return for asynchronous result retrieval. configured values replace
+# the default https://s3.amazonaws.com allowlist. this belongs only on the control-plane host.
+# FLASH_VAST_RESULT_ORIGINS=https://s3.amazonaws.com
 
 # --- required: artifacts ---
 # HuggingFace token with write access to the dataset repos flash creates for artifacts

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/.github/workflows/deploy-modal.yml

@@ -39,10 +39,14 @@ jobs:
     # this only constrains manual dispatches; a non-main dispatch skips without running `modal deploy`.
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
-    # the job's own bounded waits are two 300s readiness polls plus the remote image build; 30
-    # minutes clears that with room while keeping a wedged deploy from holding the runner for
-    # GitHub's 6-hour default.
-    timeout-minutes: 30
+    # the job's own bounded waits now sum well past the old 30: two 300s readiness polls, the remote
+    # image build, a 420s canary that pays a GPU cold start, a 180s accounting deadline, and on the
+    # failure path a SECOND `modal deploy` plus a 300s restore poll. 60 minutes clears that worst
+    # case with room while keeping a wedged deploy from holding the runner for GitHub's 6-hour
+    # default. the headroom is specifically for the rollback: a job killed partway through the
+    # restore leaves the broken release live with no second attempt, which is the one outcome this
+    # workflow exists to prevent.
+    timeout-minutes: 60
     # serialize serving deploys across runs because each deploy mutates the live app. github can still
     # replace a pending run, so the gate compares against the sha reported by the deployed app rather
     # than assuming every push predecessor was deployed.
@@ -87,9 +91,14 @@ jobs:
           DEPLOYED_HEALTH_URL: https://serve.freesolo.co/healthz
         run: |
           deploy=false
-          if [ "$EVENT" = "workflow_dispatch" ]; then
-            deploy=true
-          elif ! last_deployed_sha="$(python3 - <<'PY'
+          # the sha reported by the LIVE app is read ONCE, before any decision, because it serves
+          # two purposes: it bounds the undeployed diff below, and it is the rollback target for a
+          # failed promotion. `modal deploy` overwrites the app in place, so after that point there
+          # is no way left to ask what the previous release was. reading it on workflow_dispatch
+          # too -- where the deploy decision does not need it -- is what makes a dispatched deploy
+          # rollback-capable rather than one-way.
+          read_ok=true
+          last_deployed_sha="$(python3 - <<'PY'
           import json
           import os
           import re
@@ -107,7 +116,17 @@ jobs:
               raise ValueError("deployed app returned no valid source revision")
           print(revision.lower())
           PY
-          )"; then
+          )" || read_ok=false
+          if [ "$read_ok" != "true" ]; then
+            last_deployed_sha=""
+          fi
+          # an empty previous sha means rollback is IMPOSSIBLE for this run. the rollback step below
+          # says so out loud rather than reporting a restore it never performed.
+          echo "previous_sha=$last_deployed_sha" >> "$GITHUB_OUTPUT"
+
+          if [ "$EVENT" = "workflow_dispatch" ]; then
+            deploy=true
+          elif [ "$read_ok" != "true" ]; then
             # a missing deployment or any health, network, or parsing failure must deploy safely.
             deploy=true
           elif ! git merge-base --is-ancestor "$last_deployed_sha" "$CURRENT_SHA" 2>/dev/null; then
@@ -282,6 +301,7 @@ jobs:
           done
 
       - name: Deploy serving
+        id: deploy_serving
         if: steps.gate.outputs.deploy == 'true'
         env:
           FREESOLO_DEPLOYMENT_SHA: ${{ github.sha }}
@@ -371,6 +391,124 @@ jobs:
             sleep "$sleep_seconds"
           done
 
+      - name: Install promotion gate dependencies
+        if: steps.gate.outputs.deploy == 'true'
+        working-directory: .
+        run: |
+          # the gate imports flash.serving.promotion, which needs httpx for the streaming canary and
+          # pydantic-settings for the accounting client. this job deliberately does not `uv sync`
+          # (see "Install Modal client"), so install exactly what the gate imports and nothing more.
+          # these are the exact versions uv.lock resolves, so the gate runs against the same client
+          # behaviour the serving tests were validated on rather than whatever pypi ships that day.
+          python -m pip install "httpx==0.28.1" "pydantic==2.13.4" "pydantic-settings==2.15.0"
+
+      - name: Prove the promotion with a real streaming canary
+        if: steps.gate.outputs.deploy == 'true'
+        working-directory: .
+        env:
+          # a healthy /healthz proves a ROUTER answered with the expected identity. it does not prove
+          # a gpu engine started, that a generation ran, that streaming works, or that usage settled
+          # durably. this step is the only thing standing between "the process booted" and "the
+          # release can actually serve", so a failure here must not be promoted.
+          SERVING_BASE_URL: https://serve.freesolo.co
+          FREESOLO_INTERNAL_KEY: ${{ secrets.FREESOLO_INTERNAL_KEY }}
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
+          GITHUB_SHA: ${{ github.sha }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+        # credentials reach the gate ONLY through the environment above. passing any of them as an
+        # argument would put them in the process table and in the step's own command echo.
+        run: python -m flash.serving.promotion.gate
+
+      - name: Restore the previous release
+        # `failure()` alone is too broad. It also fires for the steps that run BEFORE any deploy --
+        # installing Modal, verifying auth, verifying the runtime secrets, waiting for the adapter
+        # schema. Restoring there would `modal deploy` the previous SHA over a live app this run never
+        # replaced, using the very secret set whose verification just failed: a healthy release
+        # overwritten by a rollback that had nothing to roll back. Gating on the deploy step's own
+        # outcome keeps the case the previous comment was right about -- a deploy that half-succeeded
+        # and then failed readiness is `failure`, not skipped, so it still restores. Any outcome other
+        # than an attempted deploy (empty, `skipped`) means production is untouched and must stay so.
+        if: >-
+          failure() && steps.gate.outputs.deploy == 'true'
+          && (steps.deploy_serving.outcome == 'success' || steps.deploy_serving.outcome == 'failure')
+        working-directory: .
+        env:
+          PREVIOUS_SHA: ${{ steps.gate.outputs.previous_sha }}
+          CURRENT_SHA: ${{ github.sha }}
+          # a distinct attempt id so /healthz cannot report the FORWARD deploy's identity and be
+          # mistaken for a completed rollback.
+          ROLLBACK_DEPLOYMENT_ID: ${{ github.run_id }}-${{ github.run_attempt }}-rollback
+          HEALTH_URL: https://serve.freesolo.co/healthz
+        run: |
+          if [ -z "$PREVIOUS_SHA" ]; then
+            echo "::error::Promotion failed and NO previous release was recorded, so the broken release is still live. Restore production manually."
+            exit 1
+          fi
+          if ! git merge-base --is-ancestor "$PREVIOUS_SHA" "$CURRENT_SHA" 2>/dev/null; then
+            # refusing here is the safe branch: redeploying a sha that is not an ancestor of this
+            # head would deploy an unknown tree rather than the release that was actually live.
+            echo "::error::Recorded previous release ${PREVIOUS_SHA} is not an ancestor of ${CURRENT_SHA}; refusing to redeploy an unverified tree. Restore production manually."
+            exit 1
+          fi
+
+          echo "Promotion failed; restoring ${PREVIOUS_SHA}."
+          git checkout --quiet --force "$PREVIOUS_SHA"
+
+          # the restored app must report the RESTORED sha, deployed from the restored tree.
+          FREESOLO_DEPLOYMENT_SHA="$PREVIOUS_SHA" \
+          FREESOLO_DEPLOYMENT_ID="$ROLLBACK_DEPLOYMENT_ID" \
+            modal deploy flash/serving/app/modal_app.py
+
+          response_file="$(mktemp)"
+          trap 'rm -f "$response_file"' EXIT
+          deadline=$((SECONDS + 300))
+          restored=false
+          while (( SECONDS < deadline )); do
+            if curl --silent --show-error --output "$response_file" --fail \
+              --connect-timeout 5 --max-time 10 "$HEALTH_URL"; then
+              if python3 - "$response_file" "$PREVIOUS_SHA" "$ROLLBACK_DEPLOYMENT_ID" <<'PY'
+          import json
+          import sys
+
+
+          def reject_constant(_value: str) -> None:
+              raise ValueError
+
+
+          try:
+              with open(sys.argv[1], encoding="utf-8") as response:
+                  payload = json.load(response, parse_constant=reject_constant)
+          except (OSError, UnicodeDecodeError, ValueError):
+              sys.exit(1)
+
+          restored = (
+              isinstance(payload, dict)
+              and payload.get("ok") is True
+              and payload.get("deployment_sha") == sys.argv[2]
+              and payload.get("deployment_id") == sys.argv[3]
+          )
+          sys.exit(0 if restored else 1)
+          PY
+              then
+                restored=true
+                break
+              fi
+            fi
+            sleep 10
+          done
+
+          if [ "$restored" != "true" ]; then
+            echo "::error::Rollback to ${PREVIOUS_SHA} was deployed but production never reported the restored release. Production may be serving a broken release; intervene manually."
+            exit 1
+          fi
+          # exit nonzero regardless: a restored predecessor means production is serving again, NOT
+          # that this promotion succeeded. reporting green here would mark an unpromotable commit as
+          # deployed and let the next push build on a release that never passed its gate.
+          echo "::error::Promotion failed; production was restored to ${PREVIOUS_SHA}. This commit is NOT deployed."
+          exit 1
+
       - name: Nothing to deploy
         if: steps.gate.outputs.deploy != 'true'
         run: echo "No serving changes in this push; skipping deploy."

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/CHANGELOG.md

@@ -13,6 +13,11 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Added
 
+- OpenAI chat serving now implements and offline-validates strict function tool calling for the exact
+  Qwen3.5 packaged and hosted paths, including validated tool history, buffered and raw SSE
+  `tool_calls`, and managed forwarding. Flash parses schema-valid calls but does not execute them;
+  malformed automatic calls remain exact assistant text. Tool calling is not live-qualified pending
+  exact model testing.
 - OpenAI chat serving now preserves mirror sampling controls across packaged, hosted, and managed
   entry points: up to four indexed choices, signed seeds, frequency and presence penalties, and
   OpenAI token logprobs. Buffered and raw SSE responses retain independent choice terminals and
@@ -25,6 +30,16 @@ This file starts at 1.1.35. Earlier releases are not reconstructed here; use
 
 ### Changed
 
+- A Flash run is one seed and may execute several attempts. The provider seed argument threaded
+  beside `spec.seed` is gone, so the spec is the only seed channel and no caller can supply a
+  second one to disagree with it; `Provider.submit_run`/`poll` are now `submit_attempt`/
+  `poll_attempt`; and `runner/supervise/seed_submission.py` is `attempt_supervision.py`. Instance
+  labels drop the redundant seed field (`<run>-s<seed>-a<n>` becomes `<run>-a<n>`), and every
+  RunPod endpoint carries an explicit `-a<n>` ordinal including attempt zero, replacing the `r<n>`
+  retry suffix that made a retry ordinal an identity and left the base name ambiguous between a run
+  and its first attempt. Names that exceed the provider budget now raise rather than truncate,
+  since a clipped ordinal is one two attempts of a run can collide on.
+
 - Run-backed adapters now use only explicit permanent checkpoint identities, `<run_id>/final` and
   `<run_id>/step-N`. Bare run aliases, composite source-revision identities, activation, implicit
   checkpoint fallback, and public artifact provenance have been removed across hosted, managed, and

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.127
+Version: 1.2.129
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/SELF_HOSTING.md

@@ -235,6 +235,12 @@ placeholder value to set for the ones you skip.
 | Lambda   | `LAMBDA_API_KEY` |                                                                 |
 | Vast     | `VAST_API_KEY`   |                                                                 |
 
+Vast log retrieval accepts signed result URLs only from exact HTTPS origins in
+`FLASH_VAST_RESULT_ORIGINS`. The variable is control-plane-only and comma-separated. Leaving it
+unset or blank allows only `https://s3.amazonaws.com`; setting it replaces that default. Origins
+must be canonical HTTPS origins without credentials, explicit ports, paths, queries, fragments,
+wildcards, spaces, or control characters.
+
 The allocator ranks candidate GPU classes only on the substrates you configured, so a
 plane with just `VAST_API_KEY` set allocates on Vast and never proposes a class it cannot
 provision. Startup fails only when **all three** are missing.

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/docs/serving-contract.md

@@ -75,6 +75,58 @@ The packaged, hosted, and managed OpenAI entry points retain the shared strict s
 - `logprobs` is a strict boolean and `top_logprobs` is an integer from 0 through 20.
 - thinking-enabled adapters reject logprobs.
 
+Function tools are implemented and offline-validated only for the exact qualified Qwen3.5 packaged
+and hosted profile. This path is not live-qualified pending exact model testing. The parser identity
+is `qwen3_coder`; Qwen3.6 and unqualified engines reject tools after authorization and checkpoint
+resolution, before generation. Declarations must be closed function objects with unique nonempty
+names. Parameters use a bounded root-object JSON Schema profile with `properties`, `required`,
+`additionalProperties: false`, recursive scalar, object, and array types, descriptions, enums, and
+array `items`. Numeric enum members must be JSON integers; decimal and exponent numeric enum lexemes
+are rejected because Flash cannot preserve them exactly across every ingress and template boundary.
+Generated and historical JSON numeric literals support at most 1024 significand digits and exponent
+magnitude at most 1,000,000; generated candidates outside either bound remain exact text, while
+history outside either bound is rejected. A historical value within those bounds that no native
+template number carries faithfully, because it
+would overflow, underflow, or lose digits, is rendered as its exact compact text instead of being
+converted or rejected, and a container holding such a value is rendered exactly as a whole. A
+top-level boolean or null argument is likewise pre-rendered as `true`, `false`, or `null`, because the
+template spells a scalar with `string` and would otherwise show the model Python's `True` and `None`;
+inside a container those values stay native, since the template's `tojson` already spells them
+correctly. A response carries at most 408 tool calls, and a longer run of candidates stays exact text. That is
+the most any continuation could carry, not a promise that this batch replays. Replay cost depends on
+the follow-up request as a whole: the prior conversation shares the same message-complexity budget,
+and each result costs four nodes as a plain string, five with the optional `name`, and seven for a
+single text block plus three for each additional block. A long enough history leaves no room for
+even one call. Clients that intend to continue the tool lifecycle must therefore not assume any
+positive call count replays under every history: a follow-up that exceeds the budget is rejected
+with `messages exceed the supported complexity`. The budget counts structure rather than text, so
+the remedies are to send fewer messages or fewer content blocks, drop optional result `name` fields,
+use plain string results rather than text blocks, or request fewer calls; shortening the text inside
+a message does not help, because a string is one node however long it is. These local
+serving-contract bounds do not depend on Python's process-wide integer conversion limit. Unsupported
+keywords and `strict: true` are rejected.
+
+`tool_choice` defaults to `auto` and accepts only `auto` or `none`. `parallel_tool_calls` defaults to
+`true` and accepts only exact `true`; either control requires `tools`. Tools cannot be combined with
+thinking, logprobs, structured outputs, a non-text `response_format`, or image messages. Historical
+assistant calls require unique IDs, function type, valid names, and JSON-string object arguments.
+Each immediately following tool-result turn must resolve every declared call exactly once before a
+non-tool turn. External messages remain unchanged; only a detached template copy converts argument
+strings to objects.
+
+In `auto` mode, complete schema-valid Qwen3 Coder XML candidates become OpenAI `tool_calls` with
+independent call indexes and `finish_reason: "tool_calls"`. Parsed responses contain no raw Qwen
+tags. Ordinary text and malformed or incomplete candidates remain exact assistant text with the
+native finish reason. Candidate regions are buffered in raw SSE because argument deltas cannot be
+retracted safely. Native prompt, completion, cached, and reasoning token accounting remains
+authoritative and is never recomputed from serialized arguments. Flash parses calls but does not
+execute or resubmit them.
+
+The decoded convenience `chat_stream` iterator remains text-only and single-choice. It rejects
+`n != 1`, logprobs, and tools before opening transport; use buffered JSON or raw SSE for those
+features. Raw `/generate` and `/adapters/{id}/generate` remain tool-free. Buffered hosted responses
+and raw hosted SSE are authoritative; managed serving forwards them and never reparses tool calls.
+
 ## Public provenance
 
 Successful run-backed responses identify only the permanent checkpoint:

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/flash/cli/commands/env/ops/imports.py

@@ -17,6 +17,7 @@ from pathlib import Path
 
 from flash.cli.commands.env.ops.push import (
     _ENV_EVALUATIONS_SIDECAR,
+    _decode_env_entrypoint_source,
     _ignore_env_push_path,
     _raise_walk_error,
 )
@@ -149,6 +150,18 @@ def _imported_module_names(tree, *, relative_names_are_siblings: bool = True) ->
     return names
 
 
+def _read_env_module_source(path: Path) -> str:
+    """Read one packaged Python file under the interpreter's own encoding contract.
+
+    the publish path decodes the entrypoint with `tokenize.detect_encoding`, so a file carrying a
+    non-utf-8 encoding cookie is publishable. reading it as utf-8 here raises instead, and every
+    caller below treats that as "no imports" -- so the sibling helpers never enter the archive and
+    `env push` exits 0 with an environment that only fails once a gpu has been rented. one decoder
+    for both paths is what keeps "publishable" and "walked for imports" the same set of files.
+    """
+    return _decode_env_entrypoint_source(path.read_bytes())[0]
+
+
 def _helper_imports(helper: Path, *, env_root: Path) -> set[str]:
     """Return imports needed by a packaged helper.
 
@@ -160,7 +173,7 @@ def _helper_imports(helper: Path, *, env_root: Path) -> set[str]:
         return set()
     try:
         return _imported_module_names(
-            ast.parse(helper.read_text(encoding="utf-8"), filename=str(helper)),
+            ast.parse(_read_env_module_source(helper), filename=str(helper)),
             relative_names_are_siblings=helper.parent == env_root,
         )
     except (OSError, SyntaxError, UnicodeDecodeError):
@@ -295,7 +308,7 @@ def _iter_env_sidecar_files(
         # GPU has been rented and the worker imports the module. parsed with the same reader as
         # every other module here, so a lazily imported helper (inside load_environment) counts.
         try:
-            entry_tree = ast.parse(entrypoint.read_text(encoding="utf-8"), filename=str(entrypoint))
+            entry_tree = ast.parse(_read_env_module_source(entrypoint), filename=str(entrypoint))
         except (OSError, SyntaxError, UnicodeDecodeError):
             # the entrypoint is validated for syntax on its own path, and a push that cannot read
             # it fails there with a better message. an unreadable file here just means no closure.
@@ -325,7 +338,7 @@ def _iter_env_sidecar_files(
                 yielded.add(sidecar)
                 yield sidecar, sidecar.relative_to(env_root)
             try:
-                tree = ast.parse(sidecar.read_text(encoding="utf-8"), filename=str(sidecar))
+                tree = ast.parse(_read_env_module_source(sidecar), filename=str(sidecar))
             except SyntaxError as exc:
                 raise ValueError(
                     f"{sidecar}: invalid evaluation sidecar syntax: {exc.msg}"

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/flash/cli/commands/env/ops/push.py

@@ -382,15 +382,40 @@ def _env_name_error(raw: str) -> str:
     return f"env name invalid: {text!r} has no usable characters"
 
 
+def _validate_env_entrypoint_source(source: bytes) -> None:
+    """Reject exact Python bytes that cannot execute as an environment module."""
+    compile(source, _ENV_ENTRYPOINT, "exec")
+
+
+def _decode_env_entrypoint_source(source: bytes) -> tuple[str, str]:
+    """Decode Python bytes with the interpreter's encoding-cookie and BOM contract."""
+    import io
+    import tokenize
+
+    encoding, _ = tokenize.detect_encoding(io.BytesIO(source).readline)
+    return source.decode(encoding), encoding
+
+
+def _python_preamble_lines(lines: list[str]) -> int:
+    """Return the shebang and encoding-cookie lines that must remain first."""
+    if not lines:
+        return 0
+    encoding_cookie = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*[-\w.]+", re.ASCII)
+    blank_or_comment = re.compile(r"^[ \t\f]*(?:#|[\r\n]|$)", re.ASCII)
+    if encoding_cookie.match(lines[0]):
+        return 1
+    if len(lines) > 1 and blank_or_comment.match(lines[0]) and encoding_cookie.match(lines[1]):
+        return 2
+    return 1 if lines[0].startswith("#!") else 0
+
+
 def _with_syspath_bootstrap(env_source: str) -> str:
-    """Prepend a sys.path bootstrap so a published env can resolve shipped sibling helpers."""
+    """Insert the sys.path bootstrap after the Python module preamble."""
     import ast
 
-    try:
-        tree = ast.parse(env_source)
-    except SyntaxError:
-        return _ENV_SYSPATH_BOOTSTRAP + env_source
-    insert_after = 0
+    tree = ast.parse(env_source, filename=_ENV_ENTRYPOINT)
+    lines = env_source.splitlines(keepends=True)
+    insert_after = _python_preamble_lines(lines)
     body = tree.body
     i = 0
     if (
@@ -399,13 +424,27 @@ def _with_syspath_bootstrap(env_source: str) -> str:
         and isinstance(getattr(body[0], "value", None), ast.Constant)
         and isinstance(body[0].value.value, str)
     ):
-        insert_after = body[0].end_lineno or 0
+        insert_after = max(insert_after, body[0].end_lineno or 0)
         i = 1
     while i < len(body) and isinstance(body[i], ast.ImportFrom) and body[i].module == "__future__":
-        insert_after = body[i].end_lineno or insert_after
+        insert_after = max(insert_after, body[i].end_lineno or 0)
         i += 1
-    lines = env_source.splitlines(keepends=True)
-    return "".join(lines[:insert_after]) + _ENV_SYSPATH_BOOTSTRAP + "".join(lines[insert_after:])
+    prefix = "".join(lines[:insert_after])
+    if prefix and not prefix.endswith(("\n", "\r")):
+        prefix += "\n"
+    published_source = prefix + _ENV_SYSPATH_BOOTSTRAP + "".join(lines[insert_after:])
+    if not env_source.endswith(("\n", "\r")):
+        published_source = published_source.removesuffix("\n")
+    return published_source
+
+
+def _prepare_env_entrypoint_source(env_source: bytes) -> bytes:
+    """Validate the authored bytes and return the exact validated archive bytes."""
+    _validate_env_entrypoint_source(env_source)
+    decoded_source, encoding = _decode_env_entrypoint_source(env_source)
+    published_source = _with_syspath_bootstrap(decoded_source).encode(encoding)
+    _validate_env_entrypoint_source(published_source)
+    return published_source
 
 
 def _raise_walk_error(error: OSError) -> None:
@@ -674,6 +713,18 @@ def cmd_env_push(args) -> int:
     except ValueError as exc:
         return _err(str(exc))
 
+    try:
+        published_source = _prepare_env_entrypoint_source(entrypoint.read_bytes())
+    except (SyntaxError, UnicodeError, ValueError) as exc:
+        location = (
+            f"line {exc.lineno}" if getattr(exc, "lineno", None) is not None else "unknown line"
+        )
+        return _err(
+            f"cannot publish {src}: environment entrypoint has invalid Python syntax ({location})"
+        )
+    except OSError:
+        return _err(f"cannot publish {src}: environment entrypoint could not be read")
+
     try:
         _check_env_push_limits(
             env_root,
@@ -686,8 +737,7 @@ def cmd_env_push(args) -> int:
 
     with tempfile.TemporaryDirectory(prefix="flash-env-push-") as tmp:
         pkg = Path(tmp)
-        module_source = entrypoint.read_text()
-        (pkg / _ENV_ENTRYPOINT).write_text(_with_syspath_bootstrap(module_source))
+        (pkg / _ENV_ENTRYPOINT).write_bytes(published_source)
         _write_entrypoint_alias(pkg, entrypoint=entrypoint)
         _copy_env_sidecars(
             env_root,

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/flash/cli/scaffold/TRAINING.md

@@ -255,7 +255,7 @@ model = "Qwen/Qwen3.5-9B"   # see `flash models list`
 project = "PROJECT_UUID"  # required UUID from `flash projects create`
 algorithm = "sft"           # "sft" (supervised), "grpo" (RL), or "opd" (on-policy distillation)
 # thinking = true           # opt-in reasoning mode, for models that support it
-# seed = 42                 # reproducible per-run seed; omitted defaults to 42
+# seed = 42                 # the run's one seed; omitted defaults to 42
 
 [environment]
 id = "your-org/your-project/my-env"      # the id printed by `flash env push`
@@ -364,7 +364,9 @@ remove that failover, and cannot be combined with `providers`. An unknown name i
 time and the error lists the names your plane accepts. `[gpu] type` pins one exact active
 validated GPU class. Run artifacts are stored in a
 private environment-scoped repo with content-addressed Flash code snapshots. Set `seed` only at the
-top level. Compose or tweak configs without editing files: `--config extra.toml` (deep-merge) and
+top level: a run has exactly one seed, and if a host fails Flash executes another attempt of that
+same run rather than a new one, so every attempt trains on the seed you set here.
+Compose or tweak configs without editing files: `--config extra.toml` (deep-merge) and
 `--set key=value` (e.g. `--set train.epochs=3`). `--gpus N` is
 shorthand for `--set gpu.count=N`; both land in one override list, so repeats resolve left to
 right.

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/flash/cli/ui/heartbeat.py

@@ -69,7 +69,7 @@ def live_attempt(obj: dict) -> int | None:
 def heartbeat_is_current_attempt(obj: dict, heartbeat: dict) -> bool:
     """False only when the heartbeat provably belongs to a superseded retry attempt.
 
-    Retries reuse the seed's heartbeat path, so a recovered run flips back to ``running`` for the
+    Every attempt of a run shares one heartbeat path, so a recovered run flips back to ``running`` for the
     replacement worker while ``last_heartbeat`` can still be the previous attempt's setup ping until
     the new worker publishes one. ``remote.attempt`` is the live attempt; ``last_heartbeat.attempt``
     is the one that produced the ping. When the live attempt is known, keep the reassurance only for a

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/flash/client/http.py

@@ -33,6 +33,7 @@ from flash.client.streaming import (
 )
 from flash.core.spec import require_project_id
 from flash.serve.contract.urls import is_freesolo_hosted_url
+from flash.serve.request.tool_calls import validate_tool_control_presence
 
 
 class ClientError(RuntimeError):
@@ -213,9 +214,18 @@ def _prepare_chat_request(
     max_tokens: int,
     *,
     stream: bool = False,
+    tools: list[dict[str, Any]] | None = None,
+    tool_choice: str | None = None,
+    parallel_tool_calls: bool | None = None,
 ) -> tuple[str, dict[str, Any]]:
     base_run_id, checkpoint_id = _parse_chat_target(target)
     _validate_chat_messages(messages)
+    validate_tool_control_presence(
+        tools,
+        tool_choice,
+        parallel_tool_calls,
+        error_type=ClientError,
+    )
     body: dict[str, Any] = {
         "checkpoint_id": checkpoint_id,
         "messages": messages,
@@ -224,6 +234,10 @@ def _prepare_chat_request(
     }
     if stream:
         body["stream"] = True
+    if tools is not None:
+        body["tools"] = tools
+        body["tool_choice"] = "auto" if tool_choice is None else tool_choice
+        body["parallel_tool_calls"] = True if parallel_tool_calls is None else parallel_tool_calls
     return base_run_id, body
 
 
@@ -767,12 +781,19 @@ class ApiClient:
         temperature: float = 0.0,
         max_tokens: int = 512,
         timeout: float | None = None,
+        *,
+        tools: list[dict[str, Any]] | None = None,
+        tool_choice: str | None = None,
+        parallel_tool_calls: bool | None = None,
     ) -> dict:
         base_run_id, body = _prepare_chat_request(
             run_id,
             messages,
             temperature,
             max_tokens,
+            tools=tools,
+            tool_choice=tool_choice,
+            parallel_tool_calls=parallel_tool_calls,
         )
         return self._request(
             "POST",
@@ -787,7 +808,11 @@ class ApiClient:
         messages: list[dict],
         temperature: float = 0.0,
         max_tokens: int = 512,
+        *,
+        tools: list[dict[str, Any]] | None = None,
     ) -> Iterator[str]:
+        if tools is not None:
+            raise ValueError("decoded chat_stream does not support tools")
         base_run_id, body = _prepare_chat_request(
             run_id,
             messages,

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/flash/core/spec.py

@@ -268,18 +268,6 @@ CONTROL_PLANE_OWNED_ENV_KEYS = frozenset(
 TRAINER_BACKEND = "verl"
 
 
-def require_matching_seed(spec: JobSpec, seed: Any) -> int:
-    """Require the retained provider seed argument to match the authoritative JobSpec seed."""
-    provided = parse_seed(seed)
-    canonical = parse_seed(spec.seed)
-    if provided != canonical:
-        raise ValueError(
-            f"provider seed {provided} does not match JobSpec.seed {canonical}; "
-            "use spec.seed as the provider seed"
-        )
-    return canonical
-
-
 def _model_revision(value: Any) -> str:
     """parse an optional exact model-repository revision."""
     if not isinstance(value, str):

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/flash/cost/spec.py

@@ -134,7 +134,7 @@ def _rollout_batch_for_quote(spec) -> int:
 
 
 def spec_steps(spec) -> int:
-    """Per-seed optimizer steps implied by a train spec (mirrors the worker).
+    """Per-run optimizer steps implied by a train spec (mirrors the worker).
 
     sft reads the horizon its workload profile measured: the profile already resolved epochs,
     retained rows, realized batch, and ``max_steps`` against the exact tokenized dataset, so

{freesolo_flash_dev-1.2.127 → freesolo_flash_dev-1.2.129}/flash/engine/worker/entry/worker.py

@@ -17,6 +17,7 @@ import flash.engine.worker.runtime.kernel_warmup as kernel_warmup
 import flash.engine.worker.train.entry.backend_common as backend_common
 from flash._internal.diagnostics import sanitize_diagnostic
 from flash.core.spec import gpu_count_of
+from flash.engine.support.huggingface import hub_error_transience
 from flash.engine.worker.runtime import state
 from flash.envs.loading.staged import (
     StagedEnvironmentTransientError,
@@ -67,6 +68,7 @@ def _run_worker_mode() -> None:
     # Idempotency: check DONE before any env-mutating pip install (fla fast path).
     if state.HF_REPO:
         from huggingface_hub import hf_hub_download
+        from huggingface_hub.errors import RemoteEntryNotFoundError
 
         try:
             hf_hub_download(
@@ -76,8 +78,15 @@ def _run_worker_mode() -> None:
                 token=os.environ.get("HF_TOKEN"),
             )
             done = True
-        except Exception:
-            done = False
+        except Exception as exc:
+            if type(exc) is RemoteEntryNotFoundError:
+                done = False
+            elif hub_error_transience(exc) is True:
+                raise worker_perf.RetriableInfraError(
+                    f"DONE marker read failed transiently ({type(exc).__name__})"
+                ) from exc
+            else:
+                raise
         remaining = state._remaining_worker_wall_seconds()
         if remaining is not None and remaining <= 0:
             raise RuntimeError("worker run wall deadline exceeded")