freesolo-flash-dev 1.4.8__tar.gz → 1.4.10__tar.gz

freesolo_flash_dev-1.4.10/.github/scripts/ci_sharding.py

@@ -0,0 +1,59 @@
+"""partition collected tests by whole file without changing pytest discovery."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+import pytest
+
+# slow-file weights come from offline junit timings; unmeasured files still participate.
+FILE_SECONDS = {
+    "tests/test_client.py": 23,
+    "tests/test_instance_bootstrap_coverage.py": 6,
+    "tests/test_repo_cleanup.py": 24,
+    "tests/test_runmgmt.py": 16,
+    "tests/test_server_api.py": 39,
+    "tests/test_server_envs_coverage.py": 15,
+    "tests/test_weight_cache.py": 7,
+    "tests/test_worker_init_heartbeat.py": 4,
+}
+
+
+def assign_files(paths: Iterable[str], count: int) -> dict[str, int]:
+    loads = [0.0] * count
+    assignments = {}
+    for path in sorted(paths, key=lambda path: (-FILE_SECONDS.get(path, 1.0), path)):
+        shard = min(range(count), key=lambda index: (loads[index], index))
+        assignments[path] = shard
+        loads[shard] += FILE_SECONDS.get(path, 1.0)
+    return assignments
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("ci sharding")
+    group.addoption("--ci-shard", type=int, default=None)
+    group.addoption("--ci-shard-count", type=int, default=2)
+
+
+def pytest_configure(config):
+    shard = config.getoption("--ci-shard")
+    count = config.getoption("--ci-shard-count")
+    if shard is None or count < 1 or not 0 <= shard < count:
+        raise pytest.UsageError("ci shard must satisfy 0 <= --ci-shard < --ci-shard-count")
+
+
+@pytest.hookimpl(wrapper=True)
+def pytest_collection_modifyitems(config, items):
+    # partition after marker selection so every worker assigns the same eligible file set.
+    result = yield
+    count = config.getoption("--ci-shard-count")
+    shard = config.getoption("--ci-shard")
+    paths = {item.path.relative_to(config.rootpath).as_posix() for item in items}
+    assignments = assign_files(paths, count)
+    kept, removed = [], []
+    for item in items:
+        path = item.path.relative_to(config.rootpath).as_posix()
+        (kept if assignments[path] == shard else removed).append(item)
+    config.hook.pytest_deselected(items=removed)
+    items[:] = kept
+    return result

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/.github/workflows/ci.yml

@@ -16,19 +16,10 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: lint + offline tests (py${{ matrix.python-version }})
+  lint:
+    name: lint
     runs-on: ubuntu-latest
-    # The suite is CPU-only and finishes well inside this; the cap is here so a wedged run
-    # releases its runner instead of holding it for the 6-hour default.
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        # pyproject declares `requires-python = ">=3.11,<3.13"`, so both supported interpreters
-        # are tested. Testing only 3.11 let a 3.12-only failure reach a user the project claims
-        # to support.
-        python-version: ["3.11", "3.12"]
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
@@ -37,7 +28,7 @@ jobs:
         with:
           # pin the installed binary as well as the action implementation.
           version: "0.12.5"
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.11"
 
       - name: Sync (server extra + dev)
         run: uv sync --extra server --extra serving --dev --locked
@@ -64,69 +55,68 @@ jobs:
       - name: Source layout check
         run: uv run python scripts/check_source_layout.py
 
-      # The job has always been NAMED offline (tests/conftest.py stubs the network boundaries in
-      # Python), but nothing enforced it, so a test that reached hf.co passed on a connected
-      # runner and would go red on a blip. Setting these makes the name true: an unstubbed call
-      # fails locally and deterministically instead of intermittently in CI.
-      #
-      # Sharding: this step is ~300s of the job's ~305s, and almost all of it is waiting rather
-      # than computing (the offline tests exercise retry/backoff/watchdog paths, so 125 sleep call
-      # sites sit idle holding a core). One process left the runner's second core idle for five
-      # minutes; `-n 2` cuts the step roughly in half.
-      #
-      # `loadfile` (not the default `load`, and not `loadscope`) is required, not a preference:
-      # tests in the same FILE share module-level import state -- several re-import
-      # `flash.engine.worker` to re-run its import scope, and others patch dotted targets under it.
-      # Splitting a file across workers breaks that sharing; `loadscope` splits at class/module
-      # granularity and fails tests/test_warmstart_cross_repo.py for exactly this reason. `loadfile`
-      # keeps every file whole on one worker, which is the boundary the suite already assumes.
-      #
-      # `-n 2` is pinned deliberately instead of `-n auto`. `auto` does NOT mean "the cores this
-      # process may use": xdist prefers psutil.cpu_count() when psutil is importable (it is, via a
-      # transitive dep) and that ignores CPU affinity and cgroup limits. So `auto` silently couples
-      # the worker count to a transitive dependency and to whatever tier the runner pool hands out.
-      # A standard GitHub runner is 2-core, which is what this suite was measured against; pinning
-      # keeps the sharding identical on every run instead of changing under us.
+      # The offline suite can only compare the pin against a hardcoded copy of itself, so a
+      # mis-transcribed sha passes every pin test while naming a commit that was never pushed --
+      # which then fails EVERY worker install at provisioning time. This is the one pin check that
+      # asks upstream instead of asking the repo about itself, so it needs the network.
       #
-      # Oversubscription is not free here: at 48 workers on 2 cores, three timing-sensitive tests in
-      # tests/test_server_envs_coverage.py fail, because they assert on a schema-validation timeout
-      # budget that CPU starvation blows through. Raise this only alongside the runner's core count.
+      # Offline environment variables are scoped to pytest steps; this check needs the network.
+      - name: verl pin exists upstream
+        run: uv run python scripts/check_verl_pin_exists.py
+
+  test:
+    name: offline tests (py${{ matrix.python-version }}, shard ${{ matrix.shard }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+        shard: [0, 1]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+        with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Sync (server extra + dev)
+        run: uv sync --extra server --extra serving --dev --locked
+
+      # Keep each file whole and each shard single-process for shared module state.
       - name: Tests (CPU, offline)
         env:
           HF_HUB_OFFLINE: "1"
           TRANSFORMERS_OFFLINE: "1"
           HF_HUB_DISABLE_TELEMETRY: "1"
-        run: uv run pytest -q -n 2 --dist loadfile -m "not wallclock"
-
-      # The `wallclock` tests assert on real elapsed time with tens-of-milliseconds margins, so a
-      # second worker competing for the core fails them for reasons unrelated to the code. They are
-      # deselected above and run here alone, single-process: ~5s, which is why isolating them is
-      # cheaper than the scaled-up timing budgets that would be needed to survive contention.
-      - name: Tests (CPU, offline, wall-clock sensitive)
+          PYTHONPATH: .github/scripts
+        run: >-
+          uv run pytest -q -p ci_sharding --ci-shard ${{ matrix.shard }}
+          --ci-shard-count 2 -m "not wallclock" --durations=20
+
+      # Run timing-sensitive tests separately and exactly once per Python version.
+      # Shard zero also retains the real CPU tensor regressions after dependency installation.
+      - if: matrix.shard == 0
+        name: Tests (CPU, offline, wall-clock sensitive)
         env:
           HF_HUB_OFFLINE: "1"
           TRANSFORMERS_OFFLINE: "1"
           HF_HUB_DISABLE_TELEMETRY: "1"
         run: uv run pytest -q -m wallclock
 
-      # The offline suite can only compare the pin against a hardcoded copy of itself, so a
-      # mis-transcribed sha passes every pin test while naming a commit that was never pushed --
-      # which then fails EVERY worker install at provisioning time. This is the one pin check that
-      # asks upstream instead of asking the repo about itself, so it needs the network.
-      #
-      # Deliberately NOT covered by the offline env above: those vars are scoped to the pytest step,
-      # so this step still reaches github as it must.
-      - name: verl pin exists upstream
-        run: uv run python scripts/check_verl_pin_exists.py
-
       # The ordinary offline environment omits torch/PEFT, so its adapter tests skip.
       # Exercise real meta tensors and frozen references with the worker's versions.
-      - name: Install CPU adapter-test dependencies
+      - if: matrix.shard == 0
+        name: Install CPU adapter-test dependencies
         run: |
           uv pip install torch==2.10.0 --index-url https://download.pytorch.org/whl/cpu
           uv pip install peft==0.20.0 transformers==5.12.1
 
-      - name: Warm-start KL reference regression (real torch and PEFT)
+      - if: matrix.shard == 0
+        name: Warm-start KL reference regression (real torch and PEFT)
         env:
           HF_HUB_OFFLINE: "1"
           TRANSFORMERS_OFFLINE: "1"
@@ -135,32 +125,13 @@ jobs:
           .venv/bin/python -c "import torch, peft"
           .venv/bin/pytest -q tests/test_rl_train.py -k kl_ref_adapter
 
-      - name: Custom training gradient and session regressions (real CPU torch)
+      - if: matrix.shard == 0
+        name: Custom training gradient and session regressions (real CPU torch)
         env:
           HF_HUB_OFFLINE: "1"
           TRANSFORMERS_OFFLINE: "1"
         run: .venv/bin/pytest -q tests/test_custom_gradients.py tests/test_custom_session.py tests/test_custom_client.py tests/test_custom_managed.py
 
-  # `lint + offline tests` is the required status check, and the matrix above turned that one
-  # check into two per-version checks. This aggregate keeps a single stable name for branch
-  # protection to require, and fails if either matrix leg failed or was cancelled.
-  test-result:
-    name: lint + offline tests
-    runs-on: ubuntu-latest
-    needs: test
-    if: always()
-    timeout-minutes: 5
-    steps:
-      - name: Require every matrix leg to have succeeded
-        env:
-          RESULT: ${{ needs.test.result }}
-        run: |
-          if [ "$RESULT" != "success" ]; then
-            echo "::error::matrix job 'test' finished with result '$RESULT'."
-            exit 1
-          fi
-          echo "All matrix legs passed."
-
   typecheck:
     name: mypy (advisory)
     runs-on: ubuntu-latest
@@ -222,3 +193,23 @@ jobs:
             } >> "$GITHUB_STEP_SUMMARY"
             echo "::warning::mypy stopped early; the reported error count is not the real total."
           fi
+
+  test-result:
+    name: lint + offline tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    needs: [lint, test]
+    if: always()
+    steps:
+      - name: Require every test group to have succeeded
+        env:
+          LINT: ${{ needs.lint.result }}
+          TEST: ${{ needs.test.result }}
+        run: |
+          for result in "$LINT" "$TEST"; do
+            if [ "$result" != "success" ]; then
+              echo "::error::required ci group finished with result '$result'."
+              exit 1
+            fi
+          done
+          echo "all required ci groups passed"

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/Dockerfile

@@ -18,11 +18,9 @@
 FROM python:3.12-slim@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
 
 WORKDIR /app
-COPY . .
 RUN apt-get update \
     && apt-get install -y --no-install-recommends ca-certificates git curl \
     && rm -rf /var/lib/apt/lists/*
-RUN pip install --no-cache-dir ".[server]"
 
 # Optional Infisical CLI, for deployments that pull secrets from a vault at container start
 # instead of receiving them as environment variables. OFF by default: a plain `docker build .`
@@ -67,6 +65,10 @@ RUN if [ "${INSTALL_INFISICAL}" = "true" ]; then \
         infisical --version; \
     fi
 
+# Source changes reuse the operating-system and optional Infisical layers.
+COPY . .
+RUN pip install --no-cache-dir ".[server]"
+
 VOLUME /root/.flash
 EXPOSE 8080
 

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.4.8
+Version: 1.4.10
 Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/flash/custom/backend.py

@@ -308,6 +308,7 @@ def load_backend(
     seed=42,
     checkpoint_dir,
     max_tokens=2048,
+    init_adapter=None,
 ):
     import torch
     from peft import LoraConfig, get_peft_model
@@ -335,6 +336,12 @@ def load_backend(
             "exclude_modules": surface.exclude_modules,
             "target_parameters": surface.target_parameters,
         }
+    if init_adapter and _uses_language_model_layout(init_adapter):
+        # SFT and GRPO adapt Qwen3.5 text modules under model.language_model even for text-only
+        # runs; load that same layout so the source's module paths and exclusions resolve.
+        from transformers import AutoModelForImageTextToText
+
+        model_class = AutoModelForImageTextToText
     model = model_class.from_pretrained(
         model_id,
         revision=revision,
@@ -342,17 +349,56 @@ def load_backend(
         torch_dtype=torch.bfloat16,
         attn_implementation="sdpa",
     ).to("cuda")
-    model = get_peft_model(
-        model,
-        LoraConfig(
-            r=lora_rank,
-            lora_alpha=lora_alpha or 2 * lora_rank,
-            lora_dropout=0,
-            target_modules="all-linear",
-            task_type="CAUSAL_LM",
-            revision=revision,
-            **targeting,
-        ),
-    )
+    if init_adapter:
+        model = _warm_start(model, init_adapter, lora_rank, lora_alpha)
+    else:
+        model = get_peft_model(
+            model,
+            LoraConfig(
+                r=lora_rank,
+                lora_alpha=lora_alpha or 2 * lora_rank,
+                lora_dropout=0,
+                target_modules="all-linear",
+                task_type="CAUSAL_LM",
+                revision=revision,
+                **targeting,
+            ),
+        )
     model.config._commit_hash = revision
     return TorchBackend(model, tokenizer, checkpoint_dir=checkpoint_dir, max_tokens=max_tokens)
+
+
+def _uses_language_model_layout(adapter_dir):
+    """Whether a source adapter's tensors address the multimodal class's language_model."""
+    from safetensors import safe_open
+
+    with safe_open(str(Path(adapter_dir) / "adapter_model.safetensors"), "np") as weights:
+        names = weights.keys()  # a safetensors handle, not a dict
+    return any(".language_model." in name for name in names)
+
+
+def _warm_start(model, adapter_dir, lora_rank, lora_alpha):
+    """Continue a verified source LoRA: its exact modules and weights, with fresh optimizer state.
+
+    The source config keeps its target modules; dropout is disabled like every custom adapter so
+    gradient checkpointing stays exact. Every source tensor must land on a trainable parameter:
+    peft only warns about unmatched keys, which would silently restart from zero.
+    """
+    from peft import LoraConfig, get_peft_model, load_peft_weights, set_peft_model_state_dict
+
+    config = LoraConfig.from_pretrained(adapter_dir)
+    if (config.r, config.lora_alpha) != (lora_rank, lora_alpha or 2 * lora_rank):
+        raise ValueError("warm-start adapter rank or alpha differs from the prepared run")
+    config.lora_dropout = 0
+    config.inference_mode = False
+    model = get_peft_model(model, config)
+    weights = load_peft_weights(adapter_dir, device="cuda")
+    result = set_peft_model_state_dict(model, weights)
+    trainable = [name for name, p in model.named_parameters() if p.requires_grad]
+    if result.unexpected_keys or len(weights) != len(trainable):
+        raise ValueError(
+            "warm-start adapter weights do not match the model's LoRA parameters "
+            f"({len(weights)} source tensors, {len(trainable)} trainable, "
+            f"{len(result.unexpected_keys)} unmatched)"
+        )
+    return model

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/flash/custom/worker.py

@@ -13,10 +13,34 @@ from flash.engine.worker.io import hf
 from flash.engine.worker.io.heartbeat import heartbeat, liveness_heartbeat
 
 
+def _custom_targeting(model_id):
+    """Custom runs adapt only text modules; a warm-start source must have the same modality."""
+    from flash.core.catalog import supports_image_training
+
+    if not supports_image_training(model_id):
+        return None
+    from flash.adapters.targets import resolve_lora_targeting
+
+    return resolve_lora_targeting(model_id, algorithm="custom", multimodal=False)
+
+
 def run(*, context):
     spec = context.config.spec
     heartbeat("custom_model_load", context=context)
     with liveness_heartbeat("custom_model_load", context=context):
+        init_adapter = None
+        if spec.train.init_from_adapter:
+            from flash.engine.worker.train.entry.sft_train import _warmstart_adapter_path
+
+            # downloads the prepared source and checks its rank, alpha, modules and base model.
+            init_adapter = _warmstart_adapter_path(
+                spec.model,
+                spec.model_revision,
+                spec.train.lora_rank,
+                spec.train.lora_alpha,
+                _custom_targeting(spec.model),
+                context=context,
+            )
         backend = load_backend(
             spec.model,
             revision=spec.model_revision,
@@ -25,6 +49,7 @@ def run(*, context):
             seed=spec.seed,
             checkpoint_dir="/tmp/flash-custom-checkpoints",
             max_tokens=spec.train.max_context_tokens,
+            init_adapter=init_adapter,
         )
     checkpoints = RunCheckpoints(context)
     backend.checkpoints = checkpoints

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/flash/runner/lifecycle/preparation.py

@@ -344,6 +344,15 @@ def _preparation_digest(
             and not stored_train.get("init_from_adapter")
         ):
             public_payload["train"].pop("lora_alpha", None)
+        # a custom run's public `train` is filtered to `flash.schema.custom.TRAIN_KEYS`, so adding a
+        # key there (`init_from_adapter` in 1.4.9) makes `to_dict()` emit it, empty, for every
+        # custom run prepared before -- and their digests then fail. replay the omission the same
+        # way: drop a key only when the STORED spec lacks it and its value is empty, so a set value
+        # stays bound and a run prepared with the key keeps hashing it.
+        if public_spec.algorithm == "custom" and isinstance(stored_train, Mapping):
+            for key in [k for k, v in public_payload["train"].items() if k not in stored_train]:
+                if not public_payload["train"][key]:
+                    public_payload["train"].pop(key)
     worker_payload = worker_spec.to_internal_dict()
     # NORMALIZATION, not legacy replay: this is the canonical shape `dev` hashes TODAY (1.2.88), and
     # it is unconditional -- it reads nothing off the stored record and applies to every run this

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/flash/schema/__init__.py

@@ -412,7 +412,11 @@ def _validate_train_section(raw: dict[str, Any], algorithm: str) -> dict[str, An
 
         if set(train_raw) - TRAIN_KEYS:
             raise ConfigError(f"[train] custom mode accepts only: {', '.join(sorted(TRAIN_KEYS))}")
-        return {**TRAIN_DEFAULTS, **train_raw}
+        defaults = dict(TRAIN_DEFAULTS)
+        if train_raw.get("init_from_adapter"):
+            # a warm start inherits the source adapter's rank and alpha.
+            defaults.pop("lora_rank")
+        return {**defaults, **train_raw}
     validate_train_keys_for_algorithm(train_raw, algorithm)
     return train_raw
 

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/flash/schema/custom.py

@@ -10,6 +10,7 @@ TRAIN_KEYS = {
     "lora_alpha",
     "save_every",
     "save_at_steps",
+    "init_from_adapter",
 }
 TRAIN_DEFAULTS = {"max_steps": 1000, "max_context_tokens": 2048, "lora_rank": 16, "save_every": 10}
 CUSTOM_KEYS = {"file", "resume_from", "idle_seconds", "max_wall_seconds", "controller_id"}
@@ -70,3 +71,8 @@ def validate_custom_spec(spec):
         integer(spec.train.save_every, "train.save_every", 1, 1_000_000)
     except ValueError as exc:
         raise ConfigError(str(exc)) from exc
+    if spec.train.init_from_adapter and spec.custom.get("resume_from"):
+        raise ConfigError(
+            "train.init_from_adapter starts fresh optimizer state from any adapter; "
+            "custom.resume_from continues a custom checkpoint. Set only one"
+        )

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/flash/serving/app/modal_app.py

@@ -711,7 +711,11 @@ def _build_chat_authorizer(settings: Any) -> Any:
     import time
 
     import httpx
-    from fastapi import HTTPException, status
+
+    from flash.serving.src.http.authorization_transport import (
+        TransientAuthorizationError,
+        post_authorization,
+    )
 
     _client = httpx.AsyncClient(timeout=10.0, headers={"Authorization": f"Bearer {key}"})
 
@@ -720,43 +724,44 @@ def _build_chat_authorizer(settings: Any) -> Any:
     _cache: dict[tuple[str, str, str], tuple[float, AuthorizedTraffic]] = {}
     # In-flight single-flight tasks so concurrent identical misses share ONE backend call.
     _inflight: dict[tuple[str, str, str], asyncio.Task[AuthorizedTraffic]] = {}
+    _refresh_after: dict[tuple[str, str, str], float] = {}
 
     async def _authorize_backend(
         api_key: str, model_id: str, scope: "dict[str, str]"
     ) -> "AuthorizedTraffic":
-        try:
-            resp = await _client.post(
-                url,
-                json={"apiKey": api_key, "modelId": model_id},
-                # A training container authenticates with the platform internal key rather
-                # than a customer key, so the backend cannot resolve it via
-                # authenticate_api_key. It identifies the calling job from these headers and
-                # re-checks them against the live job row. Dropping them here is
-                # indistinguishable, to the backend, from a request that never had them: it
-                # refuses with missing_training_context and catalog sampling fails closed.
-                headers=scope or None,
-            )
-        except Exception as exc:  # backend unreachable -> fail closed, never serve unauthorized
-            raise HTTPException(
-                status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth backend unreachable"
-            ) from exc
+        resp = await post_authorization(
+            _client, url, api_key=api_key, model_id=model_id, scope=scope
+        )
         return _authorize_response(resp, model_id, require_api_key_id=api_key.startswith("fslo_"))
 
     def _prune(now: float) -> None:
         for expired in [k for k, (exp, _) in _cache.items() if exp <= now]:
             _cache.pop(expired, None)
+            _refresh_after.pop(expired, None)
         if len(_cache) > _AUTH_CACHE_MAX_ENTRIES:
             n_drop = len(_cache) - _AUTH_CACHE_MAX_ENTRIES
             for k in sorted(_cache, key=lambda c: _cache[c][0])[:n_drop]:
                 _cache.pop(k, None)
+                _refresh_after.pop(k, None)
 
     async def _authorize_and_cache(
         ck: tuple[str, str, str], api_key: str, model_id: str, scope: "dict[str, str]"
     ) -> "AuthorizedTraffic":
         try:
             authorization = await _authorize_backend(api_key, model_id, scope)
+        except TransientAuthorizationError:
+            # Keep the original approval only until its existing expiry; never renew it on error.
+            cached = _cache.get(ck)
+            now = time.monotonic()
+            if cached is not None and cached[0] > now:
+                _refresh_after[ck] = min(cached[0], now + 1.0)
+            else:
+                _cache.pop(ck, None)
+                _refresh_after.pop(ck, None)
+            raise
         except BaseException:
             _cache.pop(ck, None)
+            _refresh_after.pop(ck, None)
             raise
         finally:
             # Drop the single-flight slot regardless of outcome so a failed lookup re-checks next
@@ -764,6 +769,7 @@ def _build_chat_authorizer(settings: Any) -> Any:
             _inflight.pop(ck, None)
         now = time.monotonic()
         # Cache the ALLOW only — reached iff _authorize_backend did not raise above.
+        _refresh_after.pop(ck, None)
         _cache[ck] = (now + _AUTH_CACHE_TTL_SECONDS, authorization)
         _prune(now)
         return authorization
@@ -799,11 +805,14 @@ def _build_chat_authorizer(settings: Any) -> Any:
         if cached is not None:
             if cached[0] > now:
                 # Refresh active keys before expiry, but never serve past the original TTL
-                # unless the backend has approved them again. Failures evict the old ALLOW.
-                if cached[0] - now <= _AUTH_CACHE_REFRESH_SECONDS:
+                # unless approved again. Denials evict immediately; transient failures do not.
+                if cached[0] - now <= _AUTH_CACHE_REFRESH_SECONDS and now >= _refresh_after.get(
+                    ck, 0
+                ):
                     refresh(ck, api_key, model_id, scope)
                 return cached[1]
             _cache.pop(ck, None)
+            _refresh_after.pop(ck, None)
         return await asyncio.shield(refresh(ck, api_key, model_id, scope))
 
     async def close():

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/flash/serving/promotion/evidence.py

@@ -17,7 +17,7 @@ from __future__ import annotations
 
 import math
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 _SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$")
@@ -76,6 +76,9 @@ ACCOUNTING_UNSETTLED = "accounting_canary_unsettled"
 class PromotionVerdict:
     ok: bool
     reason: str = ""
+    # diagnostic only (e.g. the error behind an unreadable accounting snapshot); never compared, so
+    # verdicts stay equal by outcome and reason.
+    detail: str = field(default="", compare=False)
 
 
 PASS = PromotionVerdict(ok=True)

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.10}/flash/serving/promotion/gate.py

@@ -199,6 +199,22 @@ async def _await_health(
     return verdict
 
 
+def _error_detail(error: Exception) -> str:
+    """name the failure behind an unreadable snapshot without request headers or credentials.
+
+    an RPC failure (for example a postgres statement timeout, SQLSTATE 57014) otherwise reads as
+    schema drift, because both surface as accounting_malformed.
+    """
+    response = getattr(error, "response", None)
+    if response is not None and getattr(response, "status_code", None) is not None:
+        try:
+            body = response.text
+        except Exception:
+            body = ""
+        return f"{type(error).__name__}: HTTP {response.status_code} {body[:300]}".strip()
+    return f"{type(error).__name__}: {str(error)[:300]}".rstrip(": ")
+
+
 async def _await_accounting(
     loader: AccountingLoader,
     *,
@@ -215,8 +231,10 @@ async def _await_accounting(
             snapshot = await asyncio.wait_for(loader(), timeout=remaining)
         except TimeoutError:
             return verdict
-        except Exception:
-            verdict = PromotionVerdict(ok=False, reason=ACCOUNTING_MALFORMED)
+        except Exception as error:
+            verdict = PromotionVerdict(
+                ok=False, reason=ACCOUNTING_MALFORMED, detail=_error_detail(error)
+            )
         else:
             # even a successful response arriving at the deadline cannot promote the release.
             if clock() >= deadline:
@@ -385,7 +403,8 @@ def main(argv: list[str] | None = None) -> int:
     if verdict.ok:
         print("promotion gate passed")
         return 0
-    print(f"promotion gate failed: {verdict.reason}", file=sys.stderr)
+    detail = f" ({verdict.detail})" if verdict.detail else ""
+    print(f"promotion gate failed: {verdict.reason}{detail}", file=sys.stderr)
     return 1