freesolo-flash-dev 1.2.110__tar.gz → 1.2.112__tar.gz

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/.github/workflows/bake-kernel-cache.yml

@@ -197,6 +197,8 @@ jobs:
         if: steps.gate.outputs.run == 'true'
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           # no cache: the post-run cache-prune calls `uv`, but the "Free disk space" step deletes
           # /opt/hostedtoolcache (where setup-uv put uv), so the post step fails ("uv not found") and
           # marks the whole job red even though the bake + push already succeeded.

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/.github/workflows/ci.yml

@@ -35,10 +35,12 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           python-version: ${{ matrix.python-version }}
 
       - name: Sync (server extra + dev)
-        run: uv sync --extra server --extra serving --dev
+        run: uv sync --extra server --extra serving --dev --locked
 
       - name: Ruff check
         run: uv run ruff check .
@@ -158,10 +160,12 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           python-version: "3.11"
 
       - name: Sync (server extra + dev)
-        run: uv sync --extra server --extra serving --dev
+        run: uv sync --extra server --extra serving --dev --locked
 
       - name: mypy
         run: |

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/.github/workflows/deploy-modal-dev.yml

@@ -99,6 +99,9 @@ jobs:
 
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         if: steps.gate.outputs.deploy == 'true'
+        with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
 
       - name: Install Modal client
         if: steps.gate.outputs.deploy == 'true'
@@ -109,7 +112,8 @@ jobs:
           # app file, so running from the repo root is what resolves `from flash.serving.src...`.
           # pinned to the exact client pyproject declares; see deploy-modal.yml for why an
           # unbounded range is unsafe for the deploy path.
-          uv pip install --system "modal==1.5.4" "python-dotenv>=1.0.0"
+          # exact pins keep the deploy client bootstrap reproducible across runner image changes.
+          uv pip install --system "modal==1.5.4" "python-dotenv==1.2.3"
 
       - name: Verify development deployment inputs
         if: steps.gate.outputs.deploy == 'true'

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/.github/workflows/deploy-modal.yml

@@ -143,11 +143,12 @@ jobs:
           # `working-directory` default) is what makes `from flash.serving.src...` resolve at deploy
           # time. Installing flash here would instead risk importing a DIFFERENT copy than the one
           # `add_local_python_source("flash")` ships to the container.
-          python -m pip install --upgrade pip
+          # exact pins keep the deploy client bootstrap reproducible across runner image changes.
+          python -m pip install --upgrade "pip==26.2.1"
           # pinned to the exact client pyproject declares and the provisioning path was validated
           # against: deploy behaviour (app lifecycle, tags, lookup) is client-version sensitive, so
           # an unbounded range lets a future release change deploys with no diff here.
-          python -m pip install "modal==1.5.4" "python-dotenv>=1.0.0"
+          python -m pip install "modal==1.5.4" "python-dotenv==1.2.3"
 
       - name: Verify Modal auth is configured
         if: steps.gate.outputs.deploy == 'true'

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/.github/workflows/publish-dev.yml

@@ -138,6 +138,8 @@ jobs:
         if: steps.decide.outputs.publish == 'true'
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           python-version: "3.11"
 
       - name: Build dev-channel distribution

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/.github/workflows/publish.yml

@@ -204,6 +204,8 @@ jobs:
         if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.package_changed == 'true'
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           python-version: "3.11"
 
       - name: Build distributions

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/Dockerfile

@@ -14,7 +14,8 @@
 # also mount its own volume at the new location — otherwise state lands on the container's
 # writable layer and disappears when the container is replaced.
 
-FROM python:3.12-slim
+# digest-pinned so control-plane rebuilds reuse the exact reviewed base image bytes.
+FROM python:3.12-slim@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
 
 WORKDIR /app
 COPY . .

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/Dockerfile.serve

@@ -1,6 +1,7 @@
 # immutable flash serving runtime. the cuda major version is set by the serve-runtime vllm pin,
 # whose compiled extension links libcudart.so.13 -- see the assertion below.
-FROM pytorch/pytorch:2.11.0-cuda13.0-cudnn9-devel
+# digest-pinned so serving rebuilds reuse the exact cuda and torch base image bytes.
+FROM pytorch/pytorch:2.11.0-cuda13.0-cudnn9-devel@sha256:6e8a7a6dedf900096f90190f66f988e7e658cda4f1e6cbc7c17e3a38980a4f89
 
 # PIP_BREAK_SYSTEM_PACKAGES matches Dockerfile.worker, which shares this base image. The base
 # ships a Debian-style PEP 668 marker (/usr/lib/python3.12/EXTERNALLY-MANAGED), so the pinned-uv

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/Dockerfile.worker

@@ -11,9 +11,10 @@
 #   # then make the GHCR package public (org packages settings) once.
 #
 # Base is torch 2.10 / cu12.8 / cudnn9 (-devel = has nvcc for any source builds); it ships the
-# CUDA 12.8 runtime libs the PyPI wheels link against. The pinned stack mirrors
-# flash/providers/_lifecycle/worker.py:WORKER_DEPS.
-FROM pytorch/pytorch:2.10.0-cuda12.8-cudnn9-devel
+# CUDA 12.8 runtime libs the PyPI wheels link against. The exact install block below is the deployed
+# worker stack source of truth; the fallback verl venv is kept in lockstep in capabilities.py.
+# digest-pinned so worker rebuilds reuse the exact torch and cuda base image bytes.
+FROM pytorch/pytorch:2.10.0-cuda12.8-cudnn9-devel@sha256:b574d4ccf6d8856a5d87dcadc667aa4f95dc18d337ef3a28d02b7b01897d7081
 
 ENV DEBIAN_FRONTEND=noninteractive \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
@@ -47,23 +48,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential
     && rm -rf /var/lib/apt/lists/*
 
 # The pinned worker stack (torch already in the base image, version-matched so pip skips it).
+# exact pins preserve the already-resolved worker environment across image rebuilds.
 RUN pip install --no-cache-dir \
-    "transformers>=5.6,<5.13" \
-    "peft>=0.19" \
+    "transformers==5.12.1" \
+    "peft==0.20.0" \
     "vllm==0.19.1" \
     "flashinfer-python==0.6.6" \
-    "bitsandbytes>=0.49" \
-    "datasets>=4.7,<6" \
-    "huggingface_hub>=1.2.0" \
-    "accelerate>=1.4" \
-    "wandb>=0.17" \
+    "bitsandbytes==0.50.1" \
+    "datasets==5.0.1" \
+    "huggingface_hub==1.28.0" \
+    "accelerate==1.14.0" \
+    "wandb==0.28.2" \
     "flash-linear-attention @ git+https://github.com/fla-org/flash-linear-attention.git@9c8e42e762fce087c27b673af4922795d9edb85e" \
     "flash-qla==0.1.2" \
     "tilelang==0.1.9" \
     "apache-tvm-ffi==0.1.9" \
-    "hf_transfer" \
-    "freesolo>=0.4.1" \
-    "runpod"
+    "hf_transfer==0.1.9" \
+    "freesolo==0.4.1" \
+    "runpod==1.12.0"
 # Gated-DeltaNet (Qwen3.5/3.6) runs on flash-linear-attention's Triton kernels. On consumer cards
 # (Ampere/Ada/Blackwell) the Triton path is correct. On Hopper (sm90) fla's GDN chunk_bwd is
 # miscomputed with Triton>=3.4 (fla #640) — the fix is fla's TILELANG backend (correct there), so
@@ -310,22 +312,22 @@ print('worker stack OK | torch', torch.__version__, '| cudnn', cudnn, '| triton'
 # fallback protects nobody and a vllm respin breaks launch on the default image alone. Keep them in
 # lockstep with verl/capabilities.py's install list.
 #
-# transformers is pinned to the SAME range as the main interpreter (>=5.6,<5.13) and in the override
-# file, not just the direct list -- verl and vllm both depend on transformers, so without the
-# override a transitive requirement re-widens it. The venv is the interpreter that TRAINS, and
-# transformers owns the gdn modelling code the boundary-reset shim patches (chunk_gated_delta_rule,
-# the causal conv, and the kwargs they read), so leaving it unbounded means the training path rides
-# a transformers line nothing validated. The main interpreter carrying the pin protects nothing in
-# here (/opt/verl-venv is built without --system-site-packages). Raising this ceiling means
-# re-validating the gdn shim against the new line, not just bumping the number.
+# transformers is exact-pinned in both the direct list and override file to the version this image
+# already resolved. verl and vllm both depend on transformers, so without the exact override a
+# transitive requirement can re-widen even an exact direct pin. the
+# venv is the interpreter that trains, and transformers owns the gdn modelling code the boundary-
+# reset shim patches (chunk_gated_delta_rule, the causal conv, and the kwargs they read). the main
+# interpreter carrying an exact pin protects nothing in here (/opt/verl-venv is built without
+# --system-site-packages). changing this exact pin means re-validating the gdn shim against the new
+# version, not just bumping the number.
 #
 # This pin is NOT what makes the sanity block below pass. is_flash_linear_attention_available() and
 # is_causal_conv1d_available() are byte-identical in 5.12.1 and 5.14.1, and both begin with
 # is_torch_cuda_available() -- they answer False on this cpu builder at every version. An earlier
 # revision of this comment blamed a 5.13 import-path move; no such move exists.
-ARG VERL_SPEC=verl[vllm]@git+https://github.com/freesolo-co/verl@32d6200de81dc484893baf8b9cf30297ebe7fa49
+ARG VERL_SPEC=verl[vllm]@git+https://github.com/freesolo-co/verl@f71a02ddb32a9c6a6915f7519bda6dede92e9dd0
 ENV FLASH_VERL_PYTHON=/opt/verl-venv/bin/python
-RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" "transformers>=5.6,<5.13" > /tmp/verl-overrides.txt \
+RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" "transformers==5.12.1" > /tmp/verl-overrides.txt \
     && uv venv --seed /opt/verl-venv \
     && VIRTUAL_ENV=/opt/verl-venv uv pip install --python /opt/verl-venv/bin/python \
         --override /tmp/verl-overrides.txt \
@@ -333,12 +335,13 @@ RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" "transformers
         "vllm==0.19.1" \
         "xgrammar==0.1.25" \
         "numpy==2.2.6" \
-        "transformers>=5.6,<5.13" \
-        "datasets>=4.7,<6" \
+        "transformers==5.12.1" \
+        "datasets==5.0.1" \
         "TransferQueue==0.1.7" \
-        "bitsandbytes>=0.49" qwen-vl-utils \
-        "wandb>=0.17" \
-        cachetools uvicorn fastapi torchvision tqdm pyarrow \
+        "bitsandbytes==0.50.1" "qwen-vl-utils==0.0.14" \
+        "wandb==0.28.2" \
+        "cachetools==7.1.7" "uvicorn==0.52.4" "fastapi==0.141.1" \
+        "torchvision==0.25.0" "tqdm==4.70.0" "pyarrow==25.0.1" \
         "flash-linear-attention @ git+https://github.com/fla-org/flash-linear-attention.git@9c8e42e762fce087c27b673af4922795d9edb85e" \
         "flash-qla==0.1.2" \
         "tilelang==0.1.9" \

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.110
+Version: 1.2.112
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash
@@ -18,22 +18,23 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: <3.13,>=3.11
 Provides-Extra: dev
 Requires-Dist: datasets>=2.19; extra == 'dev'
-Requires-Dist: fastapi; extra == 'dev'
+Requires-Dist: fastapi<1,>=0.136; extra == 'dev'
 Requires-Dist: freesolo>=0.4.1; extra == 'dev'
 Requires-Dist: httpx>=0.27; extra == 'dev'
 Requires-Dist: huggingface-hub>=1.2.0; extra == 'dev'
 Requires-Dist: jsonschema>=4.23; extra == 'dev'
 Requires-Dist: mypy>=1.13.0; extra == 'dev'
+Requires-Dist: packaging<27,>=24; extra == 'dev'
 Requires-Dist: pillow>=11; extra == 'dev'
 Requires-Dist: pytest>=9.0.3; extra == 'dev'
 Requires-Dist: referencing>=0.28.4; extra == 'dev'
 Requires-Dist: regex>=2024.11.6; extra == 'dev'
 Requires-Dist: ruff<0.17,>=0.6; extra == 'dev'
-Requires-Dist: runpod-flash; extra == 'dev'
+Requires-Dist: runpod-flash<2,>=1.19; extra == 'dev'
 Requires-Dist: tiktoken>=0.12; extra == 'dev'
 Requires-Dist: tokenizers>=0.22; extra == 'dev'
 Requires-Dist: transformers<5.13,>=5.6; extra == 'dev'
-Requires-Dist: uvicorn; extra == 'dev'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'dev'
 Provides-Extra: gpu
 Requires-Dist: accelerate>=1.4; extra == 'gpu'
 Requires-Dist: bitsandbytes>=0.49; extra == 'gpu'
@@ -48,16 +49,16 @@ Requires-Dist: transformers<5.13,>=5.6; extra == 'gpu'
 Requires-Dist: uv>=0.5; extra == 'gpu'
 Requires-Dist: vllm==0.19.1; extra == 'gpu'
 Provides-Extra: serve-runtime
-Requires-Dist: fastapi; extra == 'serve-runtime'
+Requires-Dist: fastapi<0.137,>=0.136; extra == 'serve-runtime'
 Requires-Dist: huggingface-hub>=1.2.0; extra == 'serve-runtime'
 Requires-Dist: pillow>=11.0.0; extra == 'serve-runtime'
-Requires-Dist: safetensors; extra == 'serve-runtime'
+Requires-Dist: safetensors<1,>=0.8; extra == 'serve-runtime'
 Requires-Dist: transformers<5.13,>=5.6; extra == 'serve-runtime'
-Requires-Dist: uvicorn; extra == 'serve-runtime'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'serve-runtime'
 Requires-Dist: vllm==0.23.0; extra == 'serve-runtime'
 Provides-Extra: server
 Requires-Dist: datasets>=2.19; extra == 'server'
-Requires-Dist: fastapi; extra == 'server'
+Requires-Dist: fastapi<1,>=0.136; extra == 'server'
 Requires-Dist: freesolo>=0.4.1; extra == 'server'
 Requires-Dist: httpx>=0.27; extra == 'server'
 Requires-Dist: huggingface-hub>=1.2.0; extra == 'server'
@@ -66,11 +67,11 @@ Requires-Dist: modal==1.5.4; extra == 'server'
 Requires-Dist: pillow>=11; extra == 'server'
 Requires-Dist: referencing>=0.28.4; extra == 'server'
 Requires-Dist: regex>=2024.11.6; extra == 'server'
-Requires-Dist: runpod-flash; extra == 'server'
+Requires-Dist: runpod-flash<2,>=1.19; extra == 'server'
 Requires-Dist: transformers<5.13,>=5.6; extra == 'server'
-Requires-Dist: uvicorn; extra == 'server'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'server'
 Provides-Extra: serving
-Requires-Dist: fastapi; extra == 'serving'
+Requires-Dist: fastapi<1,>=0.136; extra == 'serving'
 Requires-Dist: httpx>=0.27.0; extra == 'serving'
 Requires-Dist: modal==1.5.4; extra == 'serving'
 Requires-Dist: orjson>=3.10.0; extra == 'serving'

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/flash/cli/commands/worker_output.py

@@ -55,7 +55,7 @@ def _worker_section_name(name: str, current_attempt: int | str | None) -> str:
 _HEARTBEAT_LINE_RE = re.compile(r"(?m)^(HEARTBEAT )(?!\[superseded)")
 
 
-def _mark_superseded_heartbeats(text: str, attempt: int, current_attempt: int | str) -> str:
+def _mark_superseded_heartbeats(text: str, attempt: int | None, current_attempt: int | str) -> str:
     """Tag every HEARTBEAT line in a dead attempt's console dump.
 
     The section heading already says which attempt produced this text, but a heading is only read
@@ -71,13 +71,19 @@ def _mark_superseded_heartbeats(text: str, attempt: int, current_attempt: int |
     Tagging the line itself means the provenance survives the pipe. A consumer that filters on the
     tag gets only live heartbeats; one that does not at least sees the attempt in the line it
     printed rather than silently trusting a dead worker's last words.
+
+    ``attempt`` is ``None`` for the canonical ``console_<phase>.txt``, which encodes no attempt in
+    its name. That is reported as ``attempt=unknown`` rather than guessed: the file is written at
+    teardown, so on a retry whose terminal upload never ran it belongs to an OLDER attempt, and
+    naming a number the filename does not carry would be a second wrong answer rather than a fix.
     """
     current = (
         "worker torn down"
         if current_attempt == _NO_LIVE_WORKER
         else f"current attempt={current_attempt}"
     )
-    return _HEARTBEAT_LINE_RE.sub(f"\\1[superseded attempt={attempt}; {current}] ", text)
+    which = "unknown" if attempt is None else attempt
+    return _HEARTBEAT_LINE_RE.sub(f"\\1[superseded attempt={which}; {current}] ", text)
 
 
 def _worker_sections(client: ApiClient, run_id: str) -> dict[str, str]:
@@ -133,11 +139,15 @@ def _print_worker_output(
         # heartbeats carry their own provenance into the pipe. `_NO_LIVE_WORKER` tags every
         # attempt: during teardown the plane has PROVEN no worker is live, so every heartbeat on
         # screen is a dead one and none of them may reach a monitor unmarked.
+        # an artifact whose name encodes NO attempt (the canonical `console_<phase>.txt`) is tagged
+        # too. it is fetched alongside the scoped snapshot and appended last, so it is what a pipe
+        # sees -- and being written at teardown it can belong to an older attempt on a retry.
+        # unknown provenance is not evidence of liveness, so it does not earn an untagged pass.
         attempt = _artifact_attempt(name)
         supersedes = current_attempt is not None and (
             current_attempt == _NO_LIVE_WORKER or attempt != current_attempt
         )
-        if attempt is not None and supersedes:
+        if supersedes:
             text = _mark_superseded_heartbeats(text, attempt, current_attempt)
         sep = "\n" if printed_any else ""
         if render.styled():

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/flash/cli/scaffold/TRAINING.md

@@ -1104,7 +1104,7 @@ in a sensible value, so only override with a reason.
 
 | Knob                           | Convention                                                                                                                                                                                                                                                                                                                                                                                                        |
 | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `group_size`                   | Completions sampled per prompt. GRPO accepts exactly `2`, `4`, or `8` and defaults to `8`; Flash never changes the configured value. More completions can improve the within-group comparison but multiply rollout cost.                                                                                                                                                                                          |
+| `group_size`                   | Completions sampled per prompt. GRPO accepts exactly `2`, `4`, `8`, or `16` and defaults to `8`; Flash never changes the configured value. More completions can improve the within-group comparison but multiply rollout cost.                                                                                                                                                                                    |
 | `max_completion_tokens`        | Completion cap for each model turn, not the whole episode. In multi-turn runs every turn gets the full cap again, while the whole transcript (all model turns plus environment replies) must fit inside `max_context_tokens`. Size `max_context_tokens` for the whole episode, not just one turn; an undersized turn cap silently truncates good answers and poisons the reward.                                  |
 | `temperature`                  | Rollout sampling temperature. Keep it near 1.0 for GRPO — too low collapses diversity (and the model can collapse within a few steps); raise it to widen exploration against uniform-reward groups.                                                                                                                                                                                                               |
 | `kl_penalty_coef`              | Keeps the trained model from drifting too far from the base. Raise it to anchor against entropy collapse; lower it for more freedom to move.                                                                                                                                                                                                                                                                      |

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/flash/core/grpo.py

@@ -9,7 +9,7 @@ from typing import Any
 
 DEFAULT_GRPO_PROMPTS_PER_STEP = 64
 DEFAULT_GRPO_GROUP_SIZE = 8
-SUPPORTED_GRPO_GROUP_SIZES = (2, 4, 8)
+SUPPORTED_GRPO_GROUP_SIZES = (2, 4, 8, 16)
 MAX_GRPO_COMPLETIONS_PER_STEP = 512
 
 GRPO_NATIVE_THREAD_ENV: Mapping[str, str] = MappingProxyType(

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/flash/engine/worker/backend_common.py

@@ -25,13 +25,14 @@ from concurrent.futures import thread as _thread_module
 from http.server import ThreadingHTTPServer
 from typing import Self
 
-# verl 0.8.0 exactly, plus the truncation-mask, 3d position-id, and ulysses fused-label fixes.
+# verl 0.8.0 exactly, plus the truncation-mask, 3d position-id, and ulysses fused-label fixes, the
+# whole-adapter lora bucket sync, and the fused-linear grad-flag preservation.
 # it must stay on the 0.8.0 base: the opd plugin patches 0.8.0 internals and imports
 # verl.trainer.main_ppo_sync, which verl deleted after 0.8.0, and opd's exact-version gate reads the
 # version file this branch pins to the release value.
 VERL_REQUIREMENT_NAME = "verl"
 VERL_REQUIREMENT_URL = (
-    "git+https://github.com/freesolo-co/verl@32d6200de81dc484893baf8b9cf30297ebe7fa49"
+    "git+https://github.com/freesolo-co/verl@f71a02ddb32a9c6a6915f7519bda6dede92e9dd0"
 )
 # the pin, as the venv stamp records it. the provisioning install asks for the [vllm] extra of this
 # same commit; the stamp stays extra-free so it identifies the verl a venv holds, not how it was
@@ -92,11 +93,13 @@ CAUSAL_CONV1D_REQUIREMENT = "causal-conv1d==1.6.2.post1"
 # installed version. An earlier revision of this comment claimed a 5.13 import-path move made them
 # answer False; that mechanism does not exist. See Dockerfile.worker's sanity block.
 TRANSFORMERS_REQUIREMENT = "transformers>=5.6,<5.13"
+# exact deployed install and override pin; the range above remains public package metadata.
+TRANSFORMERS_INSTALL_REQUIREMENT = "transformers==5.12.1"
 
 # the stamp must identify every separately installed package the venv holds. omitting flash-attn,
-# fla, causal_conv1d, or the transformers range lets an older partial venv match forever; conv1d
+# fla, causal_conv1d, or the exact transformers pin lets an older partial venv match forever; conv1d
 # leaves GRPO/OPD failing ``require_gdn_boundary_resets`` with no rebuild path, and a stale venv
-# resolved before the transformers pin keeps training on an out-of-range transformers indefinitely.
+# resolved before the transformers pin keeps training on a different transformers indefinitely.
 # bump when a repair applied at build time changes what a fully provisioned venv looks like. the
 # libcudart repair runs only on the rebuild path, so a venv stamped by a release that predates it
 # matches, is reused, and keeps tilelang's original stub -- vLLM then aborts its import in the child
@@ -110,7 +113,8 @@ VERL_VENV_BUILD_REPAIRS = "libcudart-stub-neutralized-v1"
 # digest instead of letting it match forever and reuse an install that was never re-verified.
 VERL_VENV_STAMP = (
     f"{VERL_REQUIREMENT}\n{FLASH_ATTN_INSTALL_SPEC}\n{FLA_REQUIREMENT}\n{FLASH_QLA_REQUIREMENT}\n"
-    f"{CAUSAL_CONV1D_REQUIREMENT}\n{TRANSFORMERS_REQUIREMENT}\n{VERL_VENV_BUILD_REPAIRS}"
+    f"{CAUSAL_CONV1D_REQUIREMENT}\n{TRANSFORMERS_INSTALL_REQUIREMENT}\n"
+    f"{VERL_VENV_BUILD_REPAIRS}"
 )
 
 
@@ -927,6 +931,7 @@ from flash.engine.worker.verl.capabilities import (  # noqa: E402,F401
     _CAPABILITIES_UNAVAILABLE,
     _CAPABILITY_PROBE,
     _CAPABILITY_PROBE_TIMEOUT_S,
+    actor_fsdp_strategy_overrides,
     gdn_probe_module,
     gdn_reset_arch_from_caps,
     probe_verl_capabilities,
@@ -936,6 +941,7 @@ from flash.engine.worker.verl.capabilities import (  # noqa: E402,F401
     resolve_verl_loggers,
     resolve_verl_python,
     rollout_fp8_kv,
+    rollout_layered_summon_overrides,
     rollout_mm_processor_cache_overrides,
     rollout_resident_overrides,
     rollout_sleep_unsupported,

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/flash/engine/worker/runtime/kernel_warmup.py

@@ -16,29 +16,35 @@ DEFAULT_CACHE_DIR = "/opt/flash/kernelcache"
 MEGA_CACHE_FILENAME = "mega_cache.bin"
 MEGA_CACHE_META_FILENAME = "mega_cache.json"
 
+# every env var that addresses the baked kernel cache, mapped to its subdirectory of the cache root.
+# Dockerfile.worker sets these image-wide to the same layout; this module re-forces them so a stale
+# image or a prior run cannot defeat the redirect. it is also the allowlist the verl child inherits
+# (see sft_train._CHILD_ENV_EXACT) -- the child is the interpreter that actually trains, so a cache
+# only the parent can see is no cache at all. one definition, so a var added here cannot be added
+# without the child getting it too.
+KERNEL_CACHE_ENV_SUBDIRS = {
+    "TRITON_CACHE_DIR": "triton",
+    "TORCHINDUCTOR_CACHE_DIR": "inductor",
+    # flashinfer downloads b200 attention cubins and builds a host wrapper on first use. redirect
+    # both to distinct persistent-cache subdirs so cold workers do not depend on nvidia artifactory
+    # or pay the first-step build repeatedly. it reads FLASHINFER_CUBIN_DIR for cubins and
+    # FLASHINFER_WORKSPACE_BASE for the 0.6.x jit wrapper; FLASHINFER_CACHE_DIR is
+    # forward-compatible only, and is set anyway so a future version cannot silently miss the cache.
+    "FLASHINFER_CUBIN_DIR": "flashinfer_cubin",
+    "FLASHINFER_CACHE_DIR": "flashinfer",
+    "FLASHINFER_WORKSPACE_BASE": "flashinfer",
+}
+
 
 def _log(msg: str) -> None:
     print(f"[kernel-warmup] {msg}", flush=True)
 
 
 def _point_backends_at(cache_dir: str) -> None:
-    os.makedirs(os.path.join(cache_dir, "triton"), exist_ok=True)
-    os.makedirs(os.path.join(cache_dir, "inductor"), exist_ok=True)
-    os.environ["TRITON_CACHE_DIR"] = os.path.join(cache_dir, "triton")
-    os.environ["TORCHINDUCTOR_CACHE_DIR"] = os.path.join(cache_dir, "inductor")
-    # flashinfer downloads b200 attention cubins and builds a host wrapper on first use. redirect both
-    # to distinct persistent-cache subdirs so cold workers do not depend on nvidia artifactory or pay
-    # the first-step build repeatedly.
-    fi_cubin = os.path.join(cache_dir, "flashinfer_cubin")
-    fi_cache = os.path.join(cache_dir, "flashinfer")
-    os.makedirs(fi_cubin, exist_ok=True)
-    os.makedirs(fi_cache, exist_ok=True)
-    # flashinfer reads FLASHINFER_CUBIN_DIR for cubins and FLASHINFER_WORKSPACE_BASE for the 0.6.x
-    # jit wrapper; FLASHINFER_CACHE_DIR is forward-compatible only. force all three so stale image or
-    # prior-run env values cannot defeat this workspace redirect.
-    os.environ["FLASHINFER_CUBIN_DIR"] = fi_cubin
-    os.environ["FLASHINFER_CACHE_DIR"] = fi_cache
-    os.environ["FLASHINFER_WORKSPACE_BASE"] = fi_cache
+    for var, subdir in KERNEL_CACHE_ENV_SUBDIRS.items():
+        path = os.path.join(cache_dir, subdir)
+        os.makedirs(path, exist_ok=True)
+        os.environ[var] = path
 
 
 def _torch_sm(torch) -> str:

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/flash/engine/worker/sft_train.py

@@ -16,6 +16,7 @@ import time
 
 from flash._internal.diagnostics import SECRET_ENV_KEYS_ENV
 from flash.adapters.lora_rank import alpha_from_adapter_config, rank_from_adapter_config
+from flash.engine.worker.runtime.kernel_warmup import KERNEL_CACHE_ENV_SUBDIRS
 from flash.engine.worker.runtime.pkg_proxy import W as _w
 from flash.engine.worker.verl.checkpoints import resume_checkpoint_is_loadable
 from flash.engine.worker.verl.parallelism import ULYSSES_SEQUENCE_PARALLEL_SIZE
@@ -209,6 +210,16 @@ _CHILD_ENV_EXACT = frozenset(
         "TOKENIZERS_PARALLELISM",
         "FLASH_VERL_PYTHON",
     }
+    # the baked per-sm kernel cache is addressed ONLY through these vars, and the child is the
+    # interpreter that actually trains: a cache the child never sees is no cache at all, so without
+    # them it falls back to $HOME/.cache and re-JITs the FA2/GDN triton kernels the bake already paid
+    # ~124s to build. taken from kernel_warmup rather than restated so the two cannot drift -- and
+    # they must be named EXACTly, because TORCHINDUCTOR_CACHE_DIR does NOT match the "TORCH_" prefix
+    # below (its sixth character is "I"), which is how it was dropped despite looking like it passes.
+    # safe to share: /opt/verl-venv resolves the SAME torch as the parent (both 2.10.0+cu128 on the
+    # published cu128-sm90 image), and both are image-local, so this shares nothing across tenants --
+    # see providers/_lifecycle/worker.py, which keeps JIT caches OFF the multi-tenant volume.
+    | KERNEL_CACHE_ENV_SUBDIRS.keys()
 )
 _CHILD_ENV_PREFIXES = (
     "CUDA_",

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/flash/engine/worker/train/opd/overrides.py

@@ -15,8 +15,10 @@ import os
 
 from flash.content.structured_outputs import reasoning_parser_for
 from flash.engine.worker.backend_common import (
+    actor_fsdp_strategy_overrides,
     agent_loop_workers,
     ray_num_cpus,
+    rollout_layered_summon_overrides,
     rollout_mm_processor_cache_overrides,
     rollout_resident_overrides,
     shim_marker_file,
@@ -130,7 +132,6 @@ def _actor_rollout_overrides(config: dict, *, max_tokens: int) -> list[str]:
             else []
         ),
         f"actor_rollout_ref.model.lora_adapter_path={_hydra_val(config.get('lora_adapter_path'))}",
-        "actor_rollout_ref.actor.strategy=fsdp",
         "actor_rollout_ref.actor.use_kl_loss=false",
         "actor_rollout_ref.actor.loss_agg_mode=seq-mean-token-mean",
         "actor_rollout_ref.actor.use_dynamic_bsz=true",
@@ -148,6 +149,9 @@ def _actor_rollout_overrides(config: dict, *, max_tokens: int) -> list[str]:
         # absent -> True, verl's own default: a config built without the gate must render the
         # lower-memory strategy, never fall into zero-2 by omission.
         f"actor_rollout_ref.actor.fsdp_config.reshard_after_forward={_hydra_val(bool(config.get('reshard_after_forward', True)))}",
+        # fused-expert lora is a parametrization on a named tensor, which only fsdp2 can host.
+        # writes `actor.strategy` for both cases, so the dense default lands here too.
+        *actor_fsdp_strategy_overrides(config.get("target_parameters")),
         # store the frozen base in bf16, not verl's fp32 yaml default. shared with the rl driver.
         *trainer_dtype_overrides(),
         "actor_rollout_ref.rollout.name=vllm",
@@ -181,6 +185,9 @@ def _actor_rollout_overrides(config: dict, *, max_tokens: int) -> list[str]:
             if config.get("mm_encoder_attn_backend")
             else []
         ),
+        # fused-expert lora models gather the weight sync one submodule at a time, as the grpo path
+        # does; opd shares verl's collect_lora_params. requires load_format=safetensors above.
+        *rollout_layered_summon_overrides(config.get("target_parameters")),
         *rollout_mm_processor_cache_overrides(),
         # keep the rollout engine RESIDENT for models whose vLLM wake/reload HANGS (catalog
         # sleep_unsupported), exactly as the grpo path does. opd is NOT exempt: main_ppo_sync calls

{freesolo_flash_dev-1.2.110 → freesolo_flash_dev-1.2.112}/flash/engine/worker/train/rl/verl_config.py

@@ -17,8 +17,10 @@ from flash.adapters.targets import resolve_lora_targeting
 from flash.content.multimodal import messages_with_decoded_images
 from flash.content.structured_outputs import reasoning_parser_for
 from flash.engine.worker.backend_common import (
+    actor_fsdp_strategy_overrides,
     agent_loop_workers,
     ray_num_cpus,
+    rollout_layered_summon_overrides,
     rollout_mm_processor_cache_overrides,
     rollout_resident_overrides,
     rollout_sleep_unsupported,
@@ -299,6 +301,9 @@ def _actor_overrides(cfg: dict) -> list[str]:
         # absent -> True, verl's own default: a cfg built without the gate must render the
         # lower-memory strategy, never fall into zero-2 by omission.
         f"actor_rollout_ref.actor.fsdp_config.reshard_after_forward={_hydra_val(bool(cfg.get('reshard_after_forward', True)))}",
+        # fused-expert lora is a parametrization on a named tensor, which only fsdp2 can host.
+        # shared with the opd driver; sft has always been on fsdp2 for the same reason.
+        *actor_fsdp_strategy_overrides(cfg.get("target_parameters")),
         # store the frozen base in bf16, not verl's fp32 yaml default. shared with the opd driver.
         *trainer_dtype_overrides(),
     ]
@@ -350,6 +355,10 @@ def _rollout_overrides(cfg: dict) -> list[str]:
         ),
         # safetensors load format is required for lora rollout on vllm.
         "actor_rollout_ref.rollout.load_format=safetensors",
+        # fused-expert lora models gather the weight sync one submodule at a time; the whole-model
+        # summon materializes every layer's expert stack at once. requires load_format=safetensors
+        # above, which verl checks as base_sync_done before allowing the layered walk.
+        *rollout_layered_summon_overrides(cfg.get("target_parameters")),
         *rollout_mm_processor_cache_overrides(),
         # keep the rollout engine RESIDENT for models whose vLLM wake/reload HANGS (catalog
         # sleep_unsupported). shared with the opd driver, which runs the same verl sleep path.