freesolo-flash-dev 1.2.111__tar.gz → 1.2.112__tar.gz

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/.github/workflows/bake-kernel-cache.yml

@@ -197,6 +197,8 @@ jobs:
         if: steps.gate.outputs.run == 'true'
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           # no cache: the post-run cache-prune calls `uv`, but the "Free disk space" step deletes
           # /opt/hostedtoolcache (where setup-uv put uv), so the post step fails ("uv not found") and
           # marks the whole job red even though the bake + push already succeeded.

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/.github/workflows/ci.yml

@@ -35,10 +35,12 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           python-version: ${{ matrix.python-version }}
 
       - name: Sync (server extra + dev)
-        run: uv sync --extra server --extra serving --dev
+        run: uv sync --extra server --extra serving --dev --locked
 
       - name: Ruff check
         run: uv run ruff check .
@@ -158,10 +160,12 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           python-version: "3.11"
 
       - name: Sync (server extra + dev)
-        run: uv sync --extra server --extra serving --dev
+        run: uv sync --extra server --extra serving --dev --locked
 
       - name: mypy
         run: |

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/.github/workflows/deploy-modal-dev.yml

@@ -99,6 +99,9 @@ jobs:
 
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         if: steps.gate.outputs.deploy == 'true'
+        with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
 
       - name: Install Modal client
         if: steps.gate.outputs.deploy == 'true'
@@ -109,7 +112,8 @@ jobs:
           # app file, so running from the repo root is what resolves `from flash.serving.src...`.
           # pinned to the exact client pyproject declares; see deploy-modal.yml for why an
           # unbounded range is unsafe for the deploy path.
-          uv pip install --system "modal==1.5.4" "python-dotenv>=1.0.0"
+          # exact pins keep the deploy client bootstrap reproducible across runner image changes.
+          uv pip install --system "modal==1.5.4" "python-dotenv==1.2.3"
 
       - name: Verify development deployment inputs
         if: steps.gate.outputs.deploy == 'true'

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/.github/workflows/deploy-modal.yml

@@ -143,11 +143,12 @@ jobs:
           # `working-directory` default) is what makes `from flash.serving.src...` resolve at deploy
           # time. Installing flash here would instead risk importing a DIFFERENT copy than the one
           # `add_local_python_source("flash")` ships to the container.
-          python -m pip install --upgrade pip
+          # exact pins keep the deploy client bootstrap reproducible across runner image changes.
+          python -m pip install --upgrade "pip==26.2.1"
           # pinned to the exact client pyproject declares and the provisioning path was validated
           # against: deploy behaviour (app lifecycle, tags, lookup) is client-version sensitive, so
           # an unbounded range lets a future release change deploys with no diff here.
-          python -m pip install "modal==1.5.4" "python-dotenv>=1.0.0"
+          python -m pip install "modal==1.5.4" "python-dotenv==1.2.3"
 
       - name: Verify Modal auth is configured
         if: steps.gate.outputs.deploy == 'true'

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/.github/workflows/publish-dev.yml

@@ -138,6 +138,8 @@ jobs:
         if: steps.decide.outputs.publish == 'true'
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           python-version: "3.11"
 
       - name: Build dev-channel distribution

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/.github/workflows/publish.yml

@@ -204,6 +204,8 @@ jobs:
         if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.package_changed == 'true'
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
           python-version: "3.11"
 
       - name: Build distributions

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/Dockerfile

@@ -14,7 +14,8 @@
 # also mount its own volume at the new location — otherwise state lands on the container's
 # writable layer and disappears when the container is replaced.
 
-FROM python:3.12-slim
+# digest-pinned so control-plane rebuilds reuse the exact reviewed base image bytes.
+FROM python:3.12-slim@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
 
 WORKDIR /app
 COPY . .

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/Dockerfile.serve

@@ -1,6 +1,7 @@
 # immutable flash serving runtime. the cuda major version is set by the serve-runtime vllm pin,
 # whose compiled extension links libcudart.so.13 -- see the assertion below.
-FROM pytorch/pytorch:2.11.0-cuda13.0-cudnn9-devel
+# digest-pinned so serving rebuilds reuse the exact cuda and torch base image bytes.
+FROM pytorch/pytorch:2.11.0-cuda13.0-cudnn9-devel@sha256:6e8a7a6dedf900096f90190f66f988e7e658cda4f1e6cbc7c17e3a38980a4f89
 
 # PIP_BREAK_SYSTEM_PACKAGES matches Dockerfile.worker, which shares this base image. The base
 # ships a Debian-style PEP 668 marker (/usr/lib/python3.12/EXTERNALLY-MANAGED), so the pinned-uv

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/Dockerfile.worker

@@ -11,9 +11,10 @@
 #   # then make the GHCR package public (org packages settings) once.
 #
 # Base is torch 2.10 / cu12.8 / cudnn9 (-devel = has nvcc for any source builds); it ships the
-# CUDA 12.8 runtime libs the PyPI wheels link against. The pinned stack mirrors
-# flash/providers/_lifecycle/worker.py:WORKER_DEPS.
-FROM pytorch/pytorch:2.10.0-cuda12.8-cudnn9-devel
+# CUDA 12.8 runtime libs the PyPI wheels link against. The exact install block below is the deployed
+# worker stack source of truth; the fallback verl venv is kept in lockstep in capabilities.py.
+# digest-pinned so worker rebuilds reuse the exact torch and cuda base image bytes.
+FROM pytorch/pytorch:2.10.0-cuda12.8-cudnn9-devel@sha256:b574d4ccf6d8856a5d87dcadc667aa4f95dc18d337ef3a28d02b7b01897d7081
 
 ENV DEBIAN_FRONTEND=noninteractive \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
@@ -47,23 +48,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends build-essential
     && rm -rf /var/lib/apt/lists/*
 
 # The pinned worker stack (torch already in the base image, version-matched so pip skips it).
+# exact pins preserve the already-resolved worker environment across image rebuilds.
 RUN pip install --no-cache-dir \
-    "transformers>=5.6,<5.13" \
-    "peft>=0.19" \
+    "transformers==5.12.1" \
+    "peft==0.20.0" \
     "vllm==0.19.1" \
     "flashinfer-python==0.6.6" \
-    "bitsandbytes>=0.49" \
-    "datasets>=4.7,<6" \
-    "huggingface_hub>=1.2.0" \
-    "accelerate>=1.4" \
-    "wandb>=0.17" \
+    "bitsandbytes==0.50.1" \
+    "datasets==5.0.1" \
+    "huggingface_hub==1.28.0" \
+    "accelerate==1.14.0" \
+    "wandb==0.28.2" \
     "flash-linear-attention @ git+https://github.com/fla-org/flash-linear-attention.git@9c8e42e762fce087c27b673af4922795d9edb85e" \
     "flash-qla==0.1.2" \
     "tilelang==0.1.9" \
     "apache-tvm-ffi==0.1.9" \
-    "hf_transfer" \
-    "freesolo>=0.4.1" \
-    "runpod"
+    "hf_transfer==0.1.9" \
+    "freesolo==0.4.1" \
+    "runpod==1.12.0"
 # Gated-DeltaNet (Qwen3.5/3.6) runs on flash-linear-attention's Triton kernels. On consumer cards
 # (Ampere/Ada/Blackwell) the Triton path is correct. On Hopper (sm90) fla's GDN chunk_bwd is
 # miscomputed with Triton>=3.4 (fla #640) — the fix is fla's TILELANG backend (correct there), so
@@ -310,22 +312,22 @@ print('worker stack OK | torch', torch.__version__, '| cudnn', cudnn, '| triton'
 # fallback protects nobody and a vllm respin breaks launch on the default image alone. Keep them in
 # lockstep with verl/capabilities.py's install list.
 #
-# transformers is pinned to the SAME range as the main interpreter (>=5.6,<5.13) and in the override
-# file, not just the direct list -- verl and vllm both depend on transformers, so without the
-# override a transitive requirement re-widens it. The venv is the interpreter that TRAINS, and
-# transformers owns the gdn modelling code the boundary-reset shim patches (chunk_gated_delta_rule,
-# the causal conv, and the kwargs they read), so leaving it unbounded means the training path rides
-# a transformers line nothing validated. The main interpreter carrying the pin protects nothing in
-# here (/opt/verl-venv is built without --system-site-packages). Raising this ceiling means
-# re-validating the gdn shim against the new line, not just bumping the number.
+# transformers is exact-pinned in both the direct list and override file to the version this image
+# already resolved. verl and vllm both depend on transformers, so without the exact override a
+# transitive requirement can re-widen even an exact direct pin. the
+# venv is the interpreter that trains, and transformers owns the gdn modelling code the boundary-
+# reset shim patches (chunk_gated_delta_rule, the causal conv, and the kwargs they read). the main
+# interpreter carrying an exact pin protects nothing in here (/opt/verl-venv is built without
+# --system-site-packages). changing this exact pin means re-validating the gdn shim against the new
+# version, not just bumping the number.
 #
 # This pin is NOT what makes the sanity block below pass. is_flash_linear_attention_available() and
 # is_causal_conv1d_available() are byte-identical in 5.12.1 and 5.14.1, and both begin with
 # is_torch_cuda_available() -- they answer False on this cpu builder at every version. An earlier
 # revision of this comment blamed a 5.13 import-path move; no such move exists.
-ARG VERL_SPEC=verl[vllm]@git+https://github.com/freesolo-co/verl@32d6200de81dc484893baf8b9cf30297ebe7fa49
+ARG VERL_SPEC=verl[vllm]@git+https://github.com/freesolo-co/verl@f71a02ddb32a9c6a6915f7519bda6dede92e9dd0
 ENV FLASH_VERL_PYTHON=/opt/verl-venv/bin/python
-RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" "transformers>=5.6,<5.13" > /tmp/verl-overrides.txt \
+RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" "transformers==5.12.1" > /tmp/verl-overrides.txt \
     && uv venv --seed /opt/verl-venv \
     && VIRTUAL_ENV=/opt/verl-venv uv pip install --python /opt/verl-venv/bin/python \
         --override /tmp/verl-overrides.txt \
@@ -333,12 +335,13 @@ RUN printf '%s\n' "numpy==2.2.6" "xgrammar==0.1.25" "vllm==0.19.1" "transformers
         "vllm==0.19.1" \
         "xgrammar==0.1.25" \
         "numpy==2.2.6" \
-        "transformers>=5.6,<5.13" \
-        "datasets>=4.7,<6" \
+        "transformers==5.12.1" \
+        "datasets==5.0.1" \
         "TransferQueue==0.1.7" \
-        "bitsandbytes>=0.49" qwen-vl-utils \
-        "wandb>=0.17" \
-        cachetools uvicorn fastapi torchvision tqdm pyarrow \
+        "bitsandbytes==0.50.1" "qwen-vl-utils==0.0.14" \
+        "wandb==0.28.2" \
+        "cachetools==7.1.7" "uvicorn==0.52.4" "fastapi==0.141.1" \
+        "torchvision==0.25.0" "tqdm==4.70.0" "pyarrow==25.0.1" \
         "flash-linear-attention @ git+https://github.com/fla-org/flash-linear-attention.git@9c8e42e762fce087c27b673af4922795d9edb85e" \
         "flash-qla==0.1.2" \
         "tilelang==0.1.9" \

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.111
+Version: 1.2.112
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash
@@ -18,22 +18,23 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: <3.13,>=3.11
 Provides-Extra: dev
 Requires-Dist: datasets>=2.19; extra == 'dev'
-Requires-Dist: fastapi; extra == 'dev'
+Requires-Dist: fastapi<1,>=0.136; extra == 'dev'
 Requires-Dist: freesolo>=0.4.1; extra == 'dev'
 Requires-Dist: httpx>=0.27; extra == 'dev'
 Requires-Dist: huggingface-hub>=1.2.0; extra == 'dev'
 Requires-Dist: jsonschema>=4.23; extra == 'dev'
 Requires-Dist: mypy>=1.13.0; extra == 'dev'
+Requires-Dist: packaging<27,>=24; extra == 'dev'
 Requires-Dist: pillow>=11; extra == 'dev'
 Requires-Dist: pytest>=9.0.3; extra == 'dev'
 Requires-Dist: referencing>=0.28.4; extra == 'dev'
 Requires-Dist: regex>=2024.11.6; extra == 'dev'
 Requires-Dist: ruff<0.17,>=0.6; extra == 'dev'
-Requires-Dist: runpod-flash; extra == 'dev'
+Requires-Dist: runpod-flash<2,>=1.19; extra == 'dev'
 Requires-Dist: tiktoken>=0.12; extra == 'dev'
 Requires-Dist: tokenizers>=0.22; extra == 'dev'
 Requires-Dist: transformers<5.13,>=5.6; extra == 'dev'
-Requires-Dist: uvicorn; extra == 'dev'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'dev'
 Provides-Extra: gpu
 Requires-Dist: accelerate>=1.4; extra == 'gpu'
 Requires-Dist: bitsandbytes>=0.49; extra == 'gpu'
@@ -48,16 +49,16 @@ Requires-Dist: transformers<5.13,>=5.6; extra == 'gpu'
 Requires-Dist: uv>=0.5; extra == 'gpu'
 Requires-Dist: vllm==0.19.1; extra == 'gpu'
 Provides-Extra: serve-runtime
-Requires-Dist: fastapi; extra == 'serve-runtime'
+Requires-Dist: fastapi<0.137,>=0.136; extra == 'serve-runtime'
 Requires-Dist: huggingface-hub>=1.2.0; extra == 'serve-runtime'
 Requires-Dist: pillow>=11.0.0; extra == 'serve-runtime'
-Requires-Dist: safetensors; extra == 'serve-runtime'
+Requires-Dist: safetensors<1,>=0.8; extra == 'serve-runtime'
 Requires-Dist: transformers<5.13,>=5.6; extra == 'serve-runtime'
-Requires-Dist: uvicorn; extra == 'serve-runtime'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'serve-runtime'
 Requires-Dist: vllm==0.23.0; extra == 'serve-runtime'
 Provides-Extra: server
 Requires-Dist: datasets>=2.19; extra == 'server'
-Requires-Dist: fastapi; extra == 'server'
+Requires-Dist: fastapi<1,>=0.136; extra == 'server'
 Requires-Dist: freesolo>=0.4.1; extra == 'server'
 Requires-Dist: httpx>=0.27; extra == 'server'
 Requires-Dist: huggingface-hub>=1.2.0; extra == 'server'
@@ -66,11 +67,11 @@ Requires-Dist: modal==1.5.4; extra == 'server'
 Requires-Dist: pillow>=11; extra == 'server'
 Requires-Dist: referencing>=0.28.4; extra == 'server'
 Requires-Dist: regex>=2024.11.6; extra == 'server'
-Requires-Dist: runpod-flash; extra == 'server'
+Requires-Dist: runpod-flash<2,>=1.19; extra == 'server'
 Requires-Dist: transformers<5.13,>=5.6; extra == 'server'
-Requires-Dist: uvicorn; extra == 'server'
+Requires-Dist: uvicorn<1,>=0.52; extra == 'server'
 Provides-Extra: serving
-Requires-Dist: fastapi; extra == 'serving'
+Requires-Dist: fastapi<1,>=0.136; extra == 'serving'
 Requires-Dist: httpx>=0.27.0; extra == 'serving'
 Requires-Dist: modal==1.5.4; extra == 'serving'
 Requires-Dist: orjson>=3.10.0; extra == 'serving'

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/flash/cli/commands/worker_output.py

@@ -55,7 +55,7 @@ def _worker_section_name(name: str, current_attempt: int | str | None) -> str:
 _HEARTBEAT_LINE_RE = re.compile(r"(?m)^(HEARTBEAT )(?!\[superseded)")
 
 
-def _mark_superseded_heartbeats(text: str, attempt: int, current_attempt: int | str) -> str:
+def _mark_superseded_heartbeats(text: str, attempt: int | None, current_attempt: int | str) -> str:
     """Tag every HEARTBEAT line in a dead attempt's console dump.
 
     The section heading already says which attempt produced this text, but a heading is only read
@@ -71,13 +71,19 @@ def _mark_superseded_heartbeats(text: str, attempt: int, current_attempt: int |
     Tagging the line itself means the provenance survives the pipe. A consumer that filters on the
     tag gets only live heartbeats; one that does not at least sees the attempt in the line it
     printed rather than silently trusting a dead worker's last words.
+
+    ``attempt`` is ``None`` for the canonical ``console_<phase>.txt``, which encodes no attempt in
+    its name. That is reported as ``attempt=unknown`` rather than guessed: the file is written at
+    teardown, so on a retry whose terminal upload never ran it belongs to an OLDER attempt, and
+    naming a number the filename does not carry would be a second wrong answer rather than a fix.
     """
     current = (
         "worker torn down"
         if current_attempt == _NO_LIVE_WORKER
         else f"current attempt={current_attempt}"
     )
-    return _HEARTBEAT_LINE_RE.sub(f"\\1[superseded attempt={attempt}; {current}] ", text)
+    which = "unknown" if attempt is None else attempt
+    return _HEARTBEAT_LINE_RE.sub(f"\\1[superseded attempt={which}; {current}] ", text)
 
 
 def _worker_sections(client: ApiClient, run_id: str) -> dict[str, str]:
@@ -133,11 +139,15 @@ def _print_worker_output(
         # heartbeats carry their own provenance into the pipe. `_NO_LIVE_WORKER` tags every
         # attempt: during teardown the plane has PROVEN no worker is live, so every heartbeat on
         # screen is a dead one and none of them may reach a monitor unmarked.
+        # an artifact whose name encodes NO attempt (the canonical `console_<phase>.txt`) is tagged
+        # too. it is fetched alongside the scoped snapshot and appended last, so it is what a pipe
+        # sees -- and being written at teardown it can belong to an older attempt on a retry.
+        # unknown provenance is not evidence of liveness, so it does not earn an untagged pass.
         attempt = _artifact_attempt(name)
         supersedes = current_attempt is not None and (
             current_attempt == _NO_LIVE_WORKER or attempt != current_attempt
         )
-        if attempt is not None and supersedes:
+        if supersedes:
             text = _mark_superseded_heartbeats(text, attempt, current_attempt)
         sep = "\n" if printed_any else ""
         if render.styled():

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/flash/engine/worker/backend_common.py

@@ -25,13 +25,14 @@ from concurrent.futures import thread as _thread_module
 from http.server import ThreadingHTTPServer
 from typing import Self
 
-# verl 0.8.0 exactly, plus the truncation-mask, 3d position-id, and ulysses fused-label fixes.
+# verl 0.8.0 exactly, plus the truncation-mask, 3d position-id, and ulysses fused-label fixes, the
+# whole-adapter lora bucket sync, and the fused-linear grad-flag preservation.
 # it must stay on the 0.8.0 base: the opd plugin patches 0.8.0 internals and imports
 # verl.trainer.main_ppo_sync, which verl deleted after 0.8.0, and opd's exact-version gate reads the
 # version file this branch pins to the release value.
 VERL_REQUIREMENT_NAME = "verl"
 VERL_REQUIREMENT_URL = (
-    "git+https://github.com/freesolo-co/verl@32d6200de81dc484893baf8b9cf30297ebe7fa49"
+    "git+https://github.com/freesolo-co/verl@f71a02ddb32a9c6a6915f7519bda6dede92e9dd0"
 )
 # the pin, as the venv stamp records it. the provisioning install asks for the [vllm] extra of this
 # same commit; the stamp stays extra-free so it identifies the verl a venv holds, not how it was
@@ -92,11 +93,13 @@ CAUSAL_CONV1D_REQUIREMENT = "causal-conv1d==1.6.2.post1"
 # installed version. An earlier revision of this comment claimed a 5.13 import-path move made them
 # answer False; that mechanism does not exist. See Dockerfile.worker's sanity block.
 TRANSFORMERS_REQUIREMENT = "transformers>=5.6,<5.13"
+# exact deployed install and override pin; the range above remains public package metadata.
+TRANSFORMERS_INSTALL_REQUIREMENT = "transformers==5.12.1"
 
 # the stamp must identify every separately installed package the venv holds. omitting flash-attn,
-# fla, causal_conv1d, or the transformers range lets an older partial venv match forever; conv1d
+# fla, causal_conv1d, or the exact transformers pin lets an older partial venv match forever; conv1d
 # leaves GRPO/OPD failing ``require_gdn_boundary_resets`` with no rebuild path, and a stale venv
-# resolved before the transformers pin keeps training on an out-of-range transformers indefinitely.
+# resolved before the transformers pin keeps training on a different transformers indefinitely.
 # bump when a repair applied at build time changes what a fully provisioned venv looks like. the
 # libcudart repair runs only on the rebuild path, so a venv stamped by a release that predates it
 # matches, is reused, and keeps tilelang's original stub -- vLLM then aborts its import in the child
@@ -110,7 +113,8 @@ VERL_VENV_BUILD_REPAIRS = "libcudart-stub-neutralized-v1"
 # digest instead of letting it match forever and reuse an install that was never re-verified.
 VERL_VENV_STAMP = (
     f"{VERL_REQUIREMENT}\n{FLASH_ATTN_INSTALL_SPEC}\n{FLA_REQUIREMENT}\n{FLASH_QLA_REQUIREMENT}\n"
-    f"{CAUSAL_CONV1D_REQUIREMENT}\n{TRANSFORMERS_REQUIREMENT}\n{VERL_VENV_BUILD_REPAIRS}"
+    f"{CAUSAL_CONV1D_REQUIREMENT}\n{TRANSFORMERS_INSTALL_REQUIREMENT}\n"
+    f"{VERL_VENV_BUILD_REPAIRS}"
 )
 
 
@@ -927,6 +931,7 @@ from flash.engine.worker.verl.capabilities import (  # noqa: E402,F401
     _CAPABILITIES_UNAVAILABLE,
     _CAPABILITY_PROBE,
     _CAPABILITY_PROBE_TIMEOUT_S,
+    actor_fsdp_strategy_overrides,
     gdn_probe_module,
     gdn_reset_arch_from_caps,
     probe_verl_capabilities,

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/flash/engine/worker/runtime/kernel_warmup.py

@@ -16,29 +16,35 @@ DEFAULT_CACHE_DIR = "/opt/flash/kernelcache"
 MEGA_CACHE_FILENAME = "mega_cache.bin"
 MEGA_CACHE_META_FILENAME = "mega_cache.json"
 
+# every env var that addresses the baked kernel cache, mapped to its subdirectory of the cache root.
+# Dockerfile.worker sets these image-wide to the same layout; this module re-forces them so a stale
+# image or a prior run cannot defeat the redirect. it is also the allowlist the verl child inherits
+# (see sft_train._CHILD_ENV_EXACT) -- the child is the interpreter that actually trains, so a cache
+# only the parent can see is no cache at all. one definition, so a var added here cannot be added
+# without the child getting it too.
+KERNEL_CACHE_ENV_SUBDIRS = {
+    "TRITON_CACHE_DIR": "triton",
+    "TORCHINDUCTOR_CACHE_DIR": "inductor",
+    # flashinfer downloads b200 attention cubins and builds a host wrapper on first use. redirect
+    # both to distinct persistent-cache subdirs so cold workers do not depend on nvidia artifactory
+    # or pay the first-step build repeatedly. it reads FLASHINFER_CUBIN_DIR for cubins and
+    # FLASHINFER_WORKSPACE_BASE for the 0.6.x jit wrapper; FLASHINFER_CACHE_DIR is
+    # forward-compatible only, and is set anyway so a future version cannot silently miss the cache.
+    "FLASHINFER_CUBIN_DIR": "flashinfer_cubin",
+    "FLASHINFER_CACHE_DIR": "flashinfer",
+    "FLASHINFER_WORKSPACE_BASE": "flashinfer",
+}
+
 
 def _log(msg: str) -> None:
     print(f"[kernel-warmup] {msg}", flush=True)
 
 
 def _point_backends_at(cache_dir: str) -> None:
-    os.makedirs(os.path.join(cache_dir, "triton"), exist_ok=True)
-    os.makedirs(os.path.join(cache_dir, "inductor"), exist_ok=True)
-    os.environ["TRITON_CACHE_DIR"] = os.path.join(cache_dir, "triton")
-    os.environ["TORCHINDUCTOR_CACHE_DIR"] = os.path.join(cache_dir, "inductor")
-    # flashinfer downloads b200 attention cubins and builds a host wrapper on first use. redirect both
-    # to distinct persistent-cache subdirs so cold workers do not depend on nvidia artifactory or pay
-    # the first-step build repeatedly.
-    fi_cubin = os.path.join(cache_dir, "flashinfer_cubin")
-    fi_cache = os.path.join(cache_dir, "flashinfer")
-    os.makedirs(fi_cubin, exist_ok=True)
-    os.makedirs(fi_cache, exist_ok=True)
-    # flashinfer reads FLASHINFER_CUBIN_DIR for cubins and FLASHINFER_WORKSPACE_BASE for the 0.6.x
-    # jit wrapper; FLASHINFER_CACHE_DIR is forward-compatible only. force all three so stale image or
-    # prior-run env values cannot defeat this workspace redirect.
-    os.environ["FLASHINFER_CUBIN_DIR"] = fi_cubin
-    os.environ["FLASHINFER_CACHE_DIR"] = fi_cache
-    os.environ["FLASHINFER_WORKSPACE_BASE"] = fi_cache
+    for var, subdir in KERNEL_CACHE_ENV_SUBDIRS.items():
+        path = os.path.join(cache_dir, subdir)
+        os.makedirs(path, exist_ok=True)
+        os.environ[var] = path
 
 
 def _torch_sm(torch) -> str:

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/flash/engine/worker/sft_train.py

@@ -16,6 +16,7 @@ import time
 
 from flash._internal.diagnostics import SECRET_ENV_KEYS_ENV
 from flash.adapters.lora_rank import alpha_from_adapter_config, rank_from_adapter_config
+from flash.engine.worker.runtime.kernel_warmup import KERNEL_CACHE_ENV_SUBDIRS
 from flash.engine.worker.runtime.pkg_proxy import W as _w
 from flash.engine.worker.verl.checkpoints import resume_checkpoint_is_loadable
 from flash.engine.worker.verl.parallelism import ULYSSES_SEQUENCE_PARALLEL_SIZE
@@ -209,6 +210,16 @@ _CHILD_ENV_EXACT = frozenset(
         "TOKENIZERS_PARALLELISM",
         "FLASH_VERL_PYTHON",
     }
+    # the baked per-sm kernel cache is addressed ONLY through these vars, and the child is the
+    # interpreter that actually trains: a cache the child never sees is no cache at all, so without
+    # them it falls back to $HOME/.cache and re-JITs the FA2/GDN triton kernels the bake already paid
+    # ~124s to build. taken from kernel_warmup rather than restated so the two cannot drift -- and
+    # they must be named EXACTly, because TORCHINDUCTOR_CACHE_DIR does NOT match the "TORCH_" prefix
+    # below (its sixth character is "I"), which is how it was dropped despite looking like it passes.
+    # safe to share: /opt/verl-venv resolves the SAME torch as the parent (both 2.10.0+cu128 on the
+    # published cu128-sm90 image), and both are image-local, so this shares nothing across tenants --
+    # see providers/_lifecycle/worker.py, which keeps JIT caches OFF the multi-tenant volume.
+    | KERNEL_CACHE_ENV_SUBDIRS.keys()
 )
 _CHILD_ENV_PREFIXES = (
     "CUDA_",

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/flash/engine/worker/train/opd/overrides.py

@@ -15,6 +15,7 @@ import os
 
 from flash.content.structured_outputs import reasoning_parser_for
 from flash.engine.worker.backend_common import (
+    actor_fsdp_strategy_overrides,
     agent_loop_workers,
     ray_num_cpus,
     rollout_layered_summon_overrides,
@@ -131,7 +132,6 @@ def _actor_rollout_overrides(config: dict, *, max_tokens: int) -> list[str]:
             else []
         ),
         f"actor_rollout_ref.model.lora_adapter_path={_hydra_val(config.get('lora_adapter_path'))}",
-        "actor_rollout_ref.actor.strategy=fsdp",
         "actor_rollout_ref.actor.use_kl_loss=false",
         "actor_rollout_ref.actor.loss_agg_mode=seq-mean-token-mean",
         "actor_rollout_ref.actor.use_dynamic_bsz=true",
@@ -149,6 +149,9 @@ def _actor_rollout_overrides(config: dict, *, max_tokens: int) -> list[str]:
         # absent -> True, verl's own default: a config built without the gate must render the
         # lower-memory strategy, never fall into zero-2 by omission.
         f"actor_rollout_ref.actor.fsdp_config.reshard_after_forward={_hydra_val(bool(config.get('reshard_after_forward', True)))}",
+        # fused-expert lora is a parametrization on a named tensor, which only fsdp2 can host.
+        # writes `actor.strategy` for both cases, so the dense default lands here too.
+        *actor_fsdp_strategy_overrides(config.get("target_parameters")),
         # store the frozen base in bf16, not verl's fp32 yaml default. shared with the rl driver.
         *trainer_dtype_overrides(),
         "actor_rollout_ref.rollout.name=vllm",

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/flash/engine/worker/train/rl/verl_config.py

@@ -17,6 +17,7 @@ from flash.adapters.targets import resolve_lora_targeting
 from flash.content.multimodal import messages_with_decoded_images
 from flash.content.structured_outputs import reasoning_parser_for
 from flash.engine.worker.backend_common import (
+    actor_fsdp_strategy_overrides,
     agent_loop_workers,
     ray_num_cpus,
     rollout_layered_summon_overrides,
@@ -300,6 +301,9 @@ def _actor_overrides(cfg: dict) -> list[str]:
         # absent -> True, verl's own default: a cfg built without the gate must render the
         # lower-memory strategy, never fall into zero-2 by omission.
         f"actor_rollout_ref.actor.fsdp_config.reshard_after_forward={_hydra_val(bool(cfg.get('reshard_after_forward', True)))}",
+        # fused-expert lora is a parametrization on a named tensor, which only fsdp2 can host.
+        # shared with the opd driver; sft has always been on fsdp2 for the same reason.
+        *actor_fsdp_strategy_overrides(cfg.get("target_parameters")),
         # store the frozen base in bf16, not verl's fp32 yaml default. shared with the opd driver.
         *trainer_dtype_overrides(),
     ]

{freesolo_flash_dev-1.2.111 → freesolo_flash_dev-1.2.112}/flash/engine/worker/verl/capabilities.py

@@ -21,7 +21,7 @@ from flash.engine.worker.backend_common import (
     CAUSAL_CONV1D_REQUIREMENT,
     FLA_REQUIREMENT,
     FLASH_QLA_REQUIREMENT,
-    TRANSFORMERS_REQUIREMENT,
+    TRANSFORMERS_INSTALL_REQUIREMENT,
     VERL_REQUIREMENT_NAME,
     VERL_REQUIREMENT_URL,
     VERL_VENV_PYTHON,
@@ -438,7 +438,10 @@ def resolve_verl_python(workdir: str, *, install_wandb: bool = False) -> str:
         # and transformers owns the gdn modelling code the boundary-reset shim patches.
         overrides = os.path.join(workdir, "verl-overrides.txt")
         with open(overrides, "w") as f:
-            f.write(f"numpy==2.2.6\nxgrammar==0.1.25\nvllm==0.19.1\n{TRANSFORMERS_REQUIREMENT}\n")
+            f.write(
+                f"numpy==2.2.6\nxgrammar==0.1.25\nvllm==0.19.1\n"
+                f"{TRANSFORMERS_INSTALL_REQUIREMENT}\n"
+            )
         subprocess.run(
             [
                 "uv",
@@ -454,25 +457,25 @@ def resolve_verl_python(workdir: str, *, install_wandb: bool = False) -> str:
                 f"{VERL_REQUIREMENT_NAME}[vllm] @ {VERL_REQUIREMENT_URL}",
                 "vllm==0.19.1",
                 "numpy==2.2.6",
-                TRANSFORMERS_REQUIREMENT,
-                # NOT transitively guaranteed: verl imports these at MODULE level on the launch path
+                TRANSFORMERS_INSTALL_REQUIREMENT,
+                # not transitively guaranteed: verl imports these at module level on the launch path
                 # (main_ppo -> ppo.ray_trainer -> rollout.llm_server imports cachetools, and
                 # rollout.utils imports uvicorn + fastapi), yet declares none of them. vllm happens
                 # to pull cachetools and fastapi today, but that is vllm's dependency choice, not a
-                # contract verl states -- name them so a vllm respin cannot silently break launch.
-                "cachetools",
-                "uvicorn",
-                "fastapi",
+                # contract verl states -- exact pins mirror the worker dockerfile's resolved venv.
+                "cachetools==7.1.7",
+                "uvicorn==0.52.4",
+                "fastapi==0.141.1",
                 # opd's entrypoint calls tq.init()/tq.close(); absent from verl's setup.py.
                 "TransferQueue==0.1.7",
                 # older raises AttributeError on PyArrow PyExtensionType.
-                "datasets>=4.7,<6",
-                "bitsandbytes>=0.49",
-                "qwen-vl-utils",
-                "torchvision",
+                "datasets==5.0.1",
+                "bitsandbytes==0.50.1",
+                "qwen-vl-utils==0.0.14",
+                "torchvision==0.25.0",
                 "xgrammar==0.1.25",
-                "tqdm",
-                "pyarrow",
+                "tqdm==4.70.0",
+                "pyarrow==25.0.1",
                 # gated-deltanet kernels, in LOCKSTEP with Dockerfile.worker's
                 # verl-venv layer (same fla sha, same tilelang pins). the model
                 # trains in THIS interpreter, so fla being installed in the
@@ -536,7 +539,8 @@ def resolve_verl_python(workdir: str, *, install_wandb: bool = False) -> str:
         # keep wandb outside the stamped rebuild branch. wandb is absent from the stamp, so reused venvs
         # and transient failures must retry it on later runs; installing an already satisfied package
         # is a cheap no-op.
-        subprocess.run(["uv", "pip", "install", "--python", py, "wandb"], check=False)
+        # exact-pinned to the same version baked into Dockerfile.worker's verl venv.
+        subprocess.run(["uv", "pip", "install", "--python", py, "wandb==0.28.2"], check=False)
     return py
 
 
@@ -860,6 +864,38 @@ def rollout_layered_summon_overrides(target_parameters: Sequence[str] | None) ->
     return ["actor_rollout_ref.rollout.layered_summon=true"]
 
 
+def actor_fsdp_strategy_overrides(target_parameters: list[str] | None) -> list[str]:
+    """select the actor's fsdp wrapper: fused-expert lora needs fsdp2, dense keeps fsdp1.
+
+    PEFT implements ``target_parameters`` by registering a torch parametrization onto a NAMED
+    TENSOR at forward time (``peft/tuners/lora/layer.py:2463``), unlike ``target_modules``, which
+    swaps in a wrapper layer once at injection. ``register_parametrization`` calls
+    ``delattr(module, name)`` and then asserts the name is gone
+    (``torch/nn/utils/parametrize.py:387``). fsdp1 cannot satisfy that on either setting:
+    ``use_orig_params=False`` (verl's default, ``workers/config/engine.py:254``) flattens the
+    tensor away so the name never resolves (``parametrize.py:639``), and ``use_orig_params=True``
+    leaves ``FSDP.__getattr__`` resolving it from ``_fsdp_wrapped_module``, so ``hasattr`` stays
+    True after ``delattr`` and the bare assert fires (``parametrize.py:634`` -> ``:387``).
+
+    no config value satisfies that assert while fsdp1 owns the module, so the strategy itself has
+    to change. fsdp2's ``fully_shard`` keeps parameters under their own names with no wrapper
+    indirection. flash's sft driver has always pinned ``engine.strategy=fsdp2``
+    (``train/sft/config.py:138``) and trains this same model with these same ``target_parameters``.
+
+    verl supports it here: ``strategy`` is declared in ``actor/dp_actor.yaml:26`` so it takes a
+    bare override, ``workers/config/engine.py:265`` accepts ``fsdp2``, the ref policy interpolates
+    the actor's value (``ref/ref.yaml:5``) so the two cannot drift, and ``layered_summon_lora_params``
+    carries an explicit fsdp2 prefix block (``utils/fsdp_utils.py:646-649``).
+
+    dense models stay on fsdp1: their lora lives on wrapper modules swapped in at injection, so no
+    parametrization is ever registered. the key is written for BOTH cases rather than omitted for
+    dense, so it has exactly one writer per config and the rl and opd drivers cannot disagree about
+    the dense default or inherit a future change to verl's yaml.
+    """
+    strategy = "fsdp2" if target_parameters else "fsdp"
+    return [f"actor_rollout_ref.actor.strategy={strategy}"]
+
+
 def rollout_mm_processor_cache_overrides() -> list[str]:
     """disable vllm's split multimodal processor cache for every verl rollout.