freesolo-flash-dev 1.4.8__tar.gz → 1.4.9__tar.gz

freesolo_flash_dev-1.4.9/.github/scripts/ci_sharding.py

@@ -0,0 +1,59 @@
+"""partition collected tests by whole file without changing pytest discovery."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+import pytest
+
+# slow-file weights come from offline junit timings; unmeasured files still participate.
+FILE_SECONDS = {
+    "tests/test_client.py": 23,
+    "tests/test_instance_bootstrap_coverage.py": 6,
+    "tests/test_repo_cleanup.py": 24,
+    "tests/test_runmgmt.py": 16,
+    "tests/test_server_api.py": 39,
+    "tests/test_server_envs_coverage.py": 15,
+    "tests/test_weight_cache.py": 7,
+    "tests/test_worker_init_heartbeat.py": 4,
+}
+
+
+def assign_files(paths: Iterable[str], count: int) -> dict[str, int]:
+    loads = [0.0] * count
+    assignments = {}
+    for path in sorted(paths, key=lambda path: (-FILE_SECONDS.get(path, 1.0), path)):
+        shard = min(range(count), key=lambda index: (loads[index], index))
+        assignments[path] = shard
+        loads[shard] += FILE_SECONDS.get(path, 1.0)
+    return assignments
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("ci sharding")
+    group.addoption("--ci-shard", type=int, default=None)
+    group.addoption("--ci-shard-count", type=int, default=2)
+
+
+def pytest_configure(config):
+    shard = config.getoption("--ci-shard")
+    count = config.getoption("--ci-shard-count")
+    if shard is None or count < 1 or not 0 <= shard < count:
+        raise pytest.UsageError("ci shard must satisfy 0 <= --ci-shard < --ci-shard-count")
+
+
+@pytest.hookimpl(wrapper=True)
+def pytest_collection_modifyitems(config, items):
+    # partition after marker selection so every worker assigns the same eligible file set.
+    result = yield
+    count = config.getoption("--ci-shard-count")
+    shard = config.getoption("--ci-shard")
+    paths = {item.path.relative_to(config.rootpath).as_posix() for item in items}
+    assignments = assign_files(paths, count)
+    kept, removed = [], []
+    for item in items:
+        path = item.path.relative_to(config.rootpath).as_posix()
+        (kept if assignments[path] == shard else removed).append(item)
+    config.hook.pytest_deselected(items=removed)
+    items[:] = kept
+    return result

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/.github/workflows/ci.yml

@@ -16,19 +16,10 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: lint + offline tests (py${{ matrix.python-version }})
+  lint:
+    name: lint
     runs-on: ubuntu-latest
-    # The suite is CPU-only and finishes well inside this; the cap is here so a wedged run
-    # releases its runner instead of holding it for the 6-hour default.
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        # pyproject declares `requires-python = ">=3.11,<3.13"`, so both supported interpreters
-        # are tested. Testing only 3.11 let a 3.12-only failure reach a user the project claims
-        # to support.
-        python-version: ["3.11", "3.12"]
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
@@ -37,7 +28,7 @@ jobs:
         with:
           # pin the installed binary as well as the action implementation.
           version: "0.12.5"
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.11"
 
       - name: Sync (server extra + dev)
         run: uv sync --extra server --extra serving --dev --locked
@@ -64,69 +55,68 @@ jobs:
       - name: Source layout check
         run: uv run python scripts/check_source_layout.py
 
-      # The job has always been NAMED offline (tests/conftest.py stubs the network boundaries in
-      # Python), but nothing enforced it, so a test that reached hf.co passed on a connected
-      # runner and would go red on a blip. Setting these makes the name true: an unstubbed call
-      # fails locally and deterministically instead of intermittently in CI.
-      #
-      # Sharding: this step is ~300s of the job's ~305s, and almost all of it is waiting rather
-      # than computing (the offline tests exercise retry/backoff/watchdog paths, so 125 sleep call
-      # sites sit idle holding a core). One process left the runner's second core idle for five
-      # minutes; `-n 2` cuts the step roughly in half.
-      #
-      # `loadfile` (not the default `load`, and not `loadscope`) is required, not a preference:
-      # tests in the same FILE share module-level import state -- several re-import
-      # `flash.engine.worker` to re-run its import scope, and others patch dotted targets under it.
-      # Splitting a file across workers breaks that sharing; `loadscope` splits at class/module
-      # granularity and fails tests/test_warmstart_cross_repo.py for exactly this reason. `loadfile`
-      # keeps every file whole on one worker, which is the boundary the suite already assumes.
-      #
-      # `-n 2` is pinned deliberately instead of `-n auto`. `auto` does NOT mean "the cores this
-      # process may use": xdist prefers psutil.cpu_count() when psutil is importable (it is, via a
-      # transitive dep) and that ignores CPU affinity and cgroup limits. So `auto` silently couples
-      # the worker count to a transitive dependency and to whatever tier the runner pool hands out.
-      # A standard GitHub runner is 2-core, which is what this suite was measured against; pinning
-      # keeps the sharding identical on every run instead of changing under us.
+      # The offline suite can only compare the pin against a hardcoded copy of itself, so a
+      # mis-transcribed sha passes every pin test while naming a commit that was never pushed --
+      # which then fails EVERY worker install at provisioning time. This is the one pin check that
+      # asks upstream instead of asking the repo about itself, so it needs the network.
       #
-      # Oversubscription is not free here: at 48 workers on 2 cores, three timing-sensitive tests in
-      # tests/test_server_envs_coverage.py fail, because they assert on a schema-validation timeout
-      # budget that CPU starvation blows through. Raise this only alongside the runner's core count.
+      # Offline environment variables are scoped to pytest steps; this check needs the network.
+      - name: verl pin exists upstream
+        run: uv run python scripts/check_verl_pin_exists.py
+
+  test:
+    name: offline tests (py${{ matrix.python-version }}, shard ${{ matrix.shard }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+        shard: [0, 1]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+        with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Sync (server extra + dev)
+        run: uv sync --extra server --extra serving --dev --locked
+
+      # Keep each file whole and each shard single-process for shared module state.
       - name: Tests (CPU, offline)
         env:
           HF_HUB_OFFLINE: "1"
           TRANSFORMERS_OFFLINE: "1"
           HF_HUB_DISABLE_TELEMETRY: "1"
-        run: uv run pytest -q -n 2 --dist loadfile -m "not wallclock"
-
-      # The `wallclock` tests assert on real elapsed time with tens-of-milliseconds margins, so a
-      # second worker competing for the core fails them for reasons unrelated to the code. They are
-      # deselected above and run here alone, single-process: ~5s, which is why isolating them is
-      # cheaper than the scaled-up timing budgets that would be needed to survive contention.
-      - name: Tests (CPU, offline, wall-clock sensitive)
+          PYTHONPATH: .github/scripts
+        run: >-
+          uv run pytest -q -p ci_sharding --ci-shard ${{ matrix.shard }}
+          --ci-shard-count 2 -m "not wallclock" --durations=20
+
+      # Run timing-sensitive tests separately and exactly once per Python version.
+      # Shard zero also retains the real CPU tensor regressions after dependency installation.
+      - if: matrix.shard == 0
+        name: Tests (CPU, offline, wall-clock sensitive)
         env:
           HF_HUB_OFFLINE: "1"
           TRANSFORMERS_OFFLINE: "1"
           HF_HUB_DISABLE_TELEMETRY: "1"
         run: uv run pytest -q -m wallclock
 
-      # The offline suite can only compare the pin against a hardcoded copy of itself, so a
-      # mis-transcribed sha passes every pin test while naming a commit that was never pushed --
-      # which then fails EVERY worker install at provisioning time. This is the one pin check that
-      # asks upstream instead of asking the repo about itself, so it needs the network.
-      #
-      # Deliberately NOT covered by the offline env above: those vars are scoped to the pytest step,
-      # so this step still reaches github as it must.
-      - name: verl pin exists upstream
-        run: uv run python scripts/check_verl_pin_exists.py
-
       # The ordinary offline environment omits torch/PEFT, so its adapter tests skip.
       # Exercise real meta tensors and frozen references with the worker's versions.
-      - name: Install CPU adapter-test dependencies
+      - if: matrix.shard == 0
+        name: Install CPU adapter-test dependencies
         run: |
           uv pip install torch==2.10.0 --index-url https://download.pytorch.org/whl/cpu
           uv pip install peft==0.20.0 transformers==5.12.1
 
-      - name: Warm-start KL reference regression (real torch and PEFT)
+      - if: matrix.shard == 0
+        name: Warm-start KL reference regression (real torch and PEFT)
         env:
           HF_HUB_OFFLINE: "1"
           TRANSFORMERS_OFFLINE: "1"
@@ -135,32 +125,13 @@ jobs:
           .venv/bin/python -c "import torch, peft"
           .venv/bin/pytest -q tests/test_rl_train.py -k kl_ref_adapter
 
-      - name: Custom training gradient and session regressions (real CPU torch)
+      - if: matrix.shard == 0
+        name: Custom training gradient and session regressions (real CPU torch)
         env:
           HF_HUB_OFFLINE: "1"
           TRANSFORMERS_OFFLINE: "1"
         run: .venv/bin/pytest -q tests/test_custom_gradients.py tests/test_custom_session.py tests/test_custom_client.py tests/test_custom_managed.py
 
-  # `lint + offline tests` is the required status check, and the matrix above turned that one
-  # check into two per-version checks. This aggregate keeps a single stable name for branch
-  # protection to require, and fails if either matrix leg failed or was cancelled.
-  test-result:
-    name: lint + offline tests
-    runs-on: ubuntu-latest
-    needs: test
-    if: always()
-    timeout-minutes: 5
-    steps:
-      - name: Require every matrix leg to have succeeded
-        env:
-          RESULT: ${{ needs.test.result }}
-        run: |
-          if [ "$RESULT" != "success" ]; then
-            echo "::error::matrix job 'test' finished with result '$RESULT'."
-            exit 1
-          fi
-          echo "All matrix legs passed."
-
   typecheck:
     name: mypy (advisory)
     runs-on: ubuntu-latest
@@ -222,3 +193,23 @@ jobs:
             } >> "$GITHUB_STEP_SUMMARY"
             echo "::warning::mypy stopped early; the reported error count is not the real total."
           fi
+
+  test-result:
+    name: lint + offline tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    needs: [lint, test]
+    if: always()
+    steps:
+      - name: Require every test group to have succeeded
+        env:
+          LINT: ${{ needs.lint.result }}
+          TEST: ${{ needs.test.result }}
+        run: |
+          for result in "$LINT" "$TEST"; do
+            if [ "$result" != "success" ]; then
+              echo "::error::required ci group finished with result '$result'."
+              exit 1
+            fi
+          done
+          echo "all required ci groups passed"

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/Dockerfile

@@ -18,11 +18,9 @@
 FROM python:3.12-slim@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
 
 WORKDIR /app
-COPY . .
 RUN apt-get update \
     && apt-get install -y --no-install-recommends ca-certificates git curl \
     && rm -rf /var/lib/apt/lists/*
-RUN pip install --no-cache-dir ".[server]"
 
 # Optional Infisical CLI, for deployments that pull secrets from a vault at container start
 # instead of receiving them as environment variables. OFF by default: a plain `docker build .`
@@ -67,6 +65,10 @@ RUN if [ "${INSTALL_INFISICAL}" = "true" ]; then \
         infisical --version; \
     fi
 
+# Source changes reuse the operating-system and optional Infisical layers.
+COPY . .
+RUN pip install --no-cache-dir ".[server]"
+
 VOLUME /root/.flash
 EXPOSE 8080
 

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.4.8
+Version: 1.4.9
 Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/flash/custom/backend.py

@@ -308,6 +308,7 @@ def load_backend(
     seed=42,
     checkpoint_dir,
     max_tokens=2048,
+    init_adapter=None,
 ):
     import torch
     from peft import LoraConfig, get_peft_model
@@ -342,17 +343,47 @@ def load_backend(
         torch_dtype=torch.bfloat16,
         attn_implementation="sdpa",
     ).to("cuda")
-    model = get_peft_model(
-        model,
-        LoraConfig(
-            r=lora_rank,
-            lora_alpha=lora_alpha or 2 * lora_rank,
-            lora_dropout=0,
-            target_modules="all-linear",
-            task_type="CAUSAL_LM",
-            revision=revision,
-            **targeting,
-        ),
-    )
+    if init_adapter:
+        model = _warm_start(model, init_adapter, lora_rank, lora_alpha)
+    else:
+        model = get_peft_model(
+            model,
+            LoraConfig(
+                r=lora_rank,
+                lora_alpha=lora_alpha or 2 * lora_rank,
+                lora_dropout=0,
+                target_modules="all-linear",
+                task_type="CAUSAL_LM",
+                revision=revision,
+                **targeting,
+            ),
+        )
     model.config._commit_hash = revision
     return TorchBackend(model, tokenizer, checkpoint_dir=checkpoint_dir, max_tokens=max_tokens)
+
+
+def _warm_start(model, adapter_dir, lora_rank, lora_alpha):
+    """Continue a verified source LoRA: its exact modules and weights, with fresh optimizer state.
+
+    The source config keeps its target modules; dropout is disabled like every custom adapter so
+    gradient checkpointing stays exact. Every source tensor must land on a trainable parameter:
+    peft only warns about unmatched keys, which would silently restart from zero.
+    """
+    from peft import LoraConfig, get_peft_model, load_peft_weights, set_peft_model_state_dict
+
+    config = LoraConfig.from_pretrained(adapter_dir)
+    if (config.r, config.lora_alpha) != (lora_rank, lora_alpha or 2 * lora_rank):
+        raise ValueError("warm-start adapter rank or alpha differs from the prepared run")
+    config.lora_dropout = 0
+    config.inference_mode = False
+    model = get_peft_model(model, config)
+    weights = load_peft_weights(adapter_dir, device="cuda")
+    result = set_peft_model_state_dict(model, weights)
+    trainable = [name for name, p in model.named_parameters() if p.requires_grad]
+    if result.unexpected_keys or len(weights) != len(trainable):
+        raise ValueError(
+            "warm-start adapter weights do not match the model's LoRA parameters "
+            f"({len(weights)} source tensors, {len(trainable)} trainable, "
+            f"{len(result.unexpected_keys)} unmatched)"
+        )
+    return model

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/flash/custom/worker.py

@@ -13,10 +13,34 @@ from flash.engine.worker.io import hf
 from flash.engine.worker.io.heartbeat import heartbeat, liveness_heartbeat
 
 
+def _custom_targeting(model_id):
+    """Custom runs adapt only text modules; a warm-start source must have the same modality."""
+    from flash.core.catalog import supports_image_training
+
+    if not supports_image_training(model_id):
+        return None
+    from flash.adapters.targets import resolve_lora_targeting
+
+    return resolve_lora_targeting(model_id, algorithm="custom", multimodal=False)
+
+
 def run(*, context):
     spec = context.config.spec
     heartbeat("custom_model_load", context=context)
     with liveness_heartbeat("custom_model_load", context=context):
+        init_adapter = None
+        if spec.train.init_from_adapter:
+            from flash.engine.worker.train.entry.sft_train import _warmstart_adapter_path
+
+            # downloads the prepared source and checks its rank, alpha, modules and base model.
+            init_adapter = _warmstart_adapter_path(
+                spec.model,
+                spec.model_revision,
+                spec.train.lora_rank,
+                spec.train.lora_alpha,
+                _custom_targeting(spec.model),
+                context=context,
+            )
         backend = load_backend(
             spec.model,
             revision=spec.model_revision,
@@ -25,6 +49,7 @@ def run(*, context):
             seed=spec.seed,
             checkpoint_dir="/tmp/flash-custom-checkpoints",
             max_tokens=spec.train.max_context_tokens,
+            init_adapter=init_adapter,
         )
     checkpoints = RunCheckpoints(context)
     backend.checkpoints = checkpoints

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/flash/schema/__init__.py

@@ -412,7 +412,11 @@ def _validate_train_section(raw: dict[str, Any], algorithm: str) -> dict[str, An
 
         if set(train_raw) - TRAIN_KEYS:
             raise ConfigError(f"[train] custom mode accepts only: {', '.join(sorted(TRAIN_KEYS))}")
-        return {**TRAIN_DEFAULTS, **train_raw}
+        defaults = dict(TRAIN_DEFAULTS)
+        if train_raw.get("init_from_adapter"):
+            # a warm start inherits the source adapter's rank and alpha.
+            defaults.pop("lora_rank")
+        return {**defaults, **train_raw}
     validate_train_keys_for_algorithm(train_raw, algorithm)
     return train_raw
 

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/flash/schema/custom.py

@@ -10,6 +10,7 @@ TRAIN_KEYS = {
     "lora_alpha",
     "save_every",
     "save_at_steps",
+    "init_from_adapter",
 }
 TRAIN_DEFAULTS = {"max_steps": 1000, "max_context_tokens": 2048, "lora_rank": 16, "save_every": 10}
 CUSTOM_KEYS = {"file", "resume_from", "idle_seconds", "max_wall_seconds", "controller_id"}
@@ -70,3 +71,8 @@ def validate_custom_spec(spec):
         integer(spec.train.save_every, "train.save_every", 1, 1_000_000)
     except ValueError as exc:
         raise ConfigError(str(exc)) from exc
+    if spec.train.init_from_adapter and spec.custom.get("resume_from"):
+        raise ConfigError(
+            "train.init_from_adapter starts fresh optimizer state from any adapter; "
+            "custom.resume_from continues a custom checkpoint. Set only one"
+        )

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/flash/serving/app/modal_app.py

@@ -711,7 +711,11 @@ def _build_chat_authorizer(settings: Any) -> Any:
     import time
 
     import httpx
-    from fastapi import HTTPException, status
+
+    from flash.serving.src.http.authorization_transport import (
+        TransientAuthorizationError,
+        post_authorization,
+    )
 
     _client = httpx.AsyncClient(timeout=10.0, headers={"Authorization": f"Bearer {key}"})
 
@@ -720,43 +724,44 @@ def _build_chat_authorizer(settings: Any) -> Any:
     _cache: dict[tuple[str, str, str], tuple[float, AuthorizedTraffic]] = {}
     # In-flight single-flight tasks so concurrent identical misses share ONE backend call.
     _inflight: dict[tuple[str, str, str], asyncio.Task[AuthorizedTraffic]] = {}
+    _refresh_after: dict[tuple[str, str, str], float] = {}
 
     async def _authorize_backend(
         api_key: str, model_id: str, scope: "dict[str, str]"
     ) -> "AuthorizedTraffic":
-        try:
-            resp = await _client.post(
-                url,
-                json={"apiKey": api_key, "modelId": model_id},
-                # A training container authenticates with the platform internal key rather
-                # than a customer key, so the backend cannot resolve it via
-                # authenticate_api_key. It identifies the calling job from these headers and
-                # re-checks them against the live job row. Dropping them here is
-                # indistinguishable, to the backend, from a request that never had them: it
-                # refuses with missing_training_context and catalog sampling fails closed.
-                headers=scope or None,
-            )
-        except Exception as exc:  # backend unreachable -> fail closed, never serve unauthorized
-            raise HTTPException(
-                status.HTTP_503_SERVICE_UNAVAILABLE, "serving auth backend unreachable"
-            ) from exc
+        resp = await post_authorization(
+            _client, url, api_key=api_key, model_id=model_id, scope=scope
+        )
         return _authorize_response(resp, model_id, require_api_key_id=api_key.startswith("fslo_"))
 
     def _prune(now: float) -> None:
         for expired in [k for k, (exp, _) in _cache.items() if exp <= now]:
             _cache.pop(expired, None)
+            _refresh_after.pop(expired, None)
         if len(_cache) > _AUTH_CACHE_MAX_ENTRIES:
             n_drop = len(_cache) - _AUTH_CACHE_MAX_ENTRIES
             for k in sorted(_cache, key=lambda c: _cache[c][0])[:n_drop]:
                 _cache.pop(k, None)
+                _refresh_after.pop(k, None)
 
     async def _authorize_and_cache(
         ck: tuple[str, str, str], api_key: str, model_id: str, scope: "dict[str, str]"
     ) -> "AuthorizedTraffic":
         try:
             authorization = await _authorize_backend(api_key, model_id, scope)
+        except TransientAuthorizationError:
+            # Keep the original approval only until its existing expiry; never renew it on error.
+            cached = _cache.get(ck)
+            now = time.monotonic()
+            if cached is not None and cached[0] > now:
+                _refresh_after[ck] = min(cached[0], now + 1.0)
+            else:
+                _cache.pop(ck, None)
+                _refresh_after.pop(ck, None)
+            raise
         except BaseException:
             _cache.pop(ck, None)
+            _refresh_after.pop(ck, None)
             raise
         finally:
             # Drop the single-flight slot regardless of outcome so a failed lookup re-checks next
@@ -764,6 +769,7 @@ def _build_chat_authorizer(settings: Any) -> Any:
             _inflight.pop(ck, None)
         now = time.monotonic()
         # Cache the ALLOW only — reached iff _authorize_backend did not raise above.
+        _refresh_after.pop(ck, None)
         _cache[ck] = (now + _AUTH_CACHE_TTL_SECONDS, authorization)
         _prune(now)
         return authorization
@@ -799,11 +805,14 @@ def _build_chat_authorizer(settings: Any) -> Any:
         if cached is not None:
             if cached[0] > now:
                 # Refresh active keys before expiry, but never serve past the original TTL
-                # unless the backend has approved them again. Failures evict the old ALLOW.
-                if cached[0] - now <= _AUTH_CACHE_REFRESH_SECONDS:
+                # unless approved again. Denials evict immediately; transient failures do not.
+                if cached[0] - now <= _AUTH_CACHE_REFRESH_SECONDS and now >= _refresh_after.get(
+                    ck, 0
+                ):
                     refresh(ck, api_key, model_id, scope)
                 return cached[1]
             _cache.pop(ck, None)
+            _refresh_after.pop(ck, None)
         return await asyncio.shield(refresh(ck, api_key, model_id, scope))
 
     async def close():

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/flash/serving/src/engine/lora_engine.py

@@ -362,6 +362,28 @@ class _LoraEngineImpl(TextPromptTokenizer):
     def _cached_lora_request_locked(self, record: Any, path: Path) -> Any:
         return cached_lora_request(self, record, path)
 
+    def _loaded_lora_request_locked(self, record: Any) -> Any | None:
+        """Steady-state hit: vLLM already holds exactly this adapter source.
+
+        Every checkpoint request runs under the per-adapter lock, and the full path re-verifies the
+        adapter directory with stat/iterdir calls on a network-backed Modal Volume. Repeating that
+        scan per request serialized concurrent requests for one checkpoint behind filesystem
+        latency (~3.5 s per queued request under load), while base-model requests never take this
+        lock. Once the entry is loaded from the same source and the registry's local path is still
+        current, the scan cannot change the answer. Any other state, including a changed source,
+        an unconfirmed removal, or a stale or missing local path, takes the full path.
+        """
+        entry = self._entries().get(record_key(record))
+        if (
+            entry is None
+            or entry.state != "loaded"
+            or entry.source_ident != _adapter_source_ident(record)
+            or self.registry.local_path_is_stale(record)
+            or self.registry.local_path(record) is None
+        ):
+            return None
+        return entry.lora_request
+
     async def _lora_request(
         self, model_id: str, record_dict: dict[str, Any] | None = None
     ) -> tuple[Any, Any]:
@@ -392,6 +414,9 @@ class _LoraEngineImpl(TextPromptTokenizer):
                 raise ValueError(f"Unknown adapter id on {self.base_model}: {model_id}")
             if not record.serve_base_model and not record.is_checkpoint:
                 raise ValueError(f"Unknown adapter id on {self.base_model}: {model_id}")
+            loaded = self._loaded_lora_request_locked(record)
+            if loaded is not None:
+                return loaded, record
             path = await self._ensure_adapter_local_locked(record)
             lora_request = self._cached_lora_request_locked(record, path)
             # first-use loading must enter the same lifecycle as explicit registration so

freesolo_flash_dev-1.4.9/flash/serving/src/http/authorization_transport.py

@@ -0,0 +1,59 @@
+"""Bounded retries for read-only authorization lookups, without logging credentials."""
+
+import asyncio
+import logging
+import random
+import time
+
+import httpx
+from fastapi import HTTPException
+
+logger = logging.getLogger(__name__)
+AUTHORIZATION_BUDGET_SECONDS = 20.0
+_ATTEMPTS = 2
+_TRANSIENT_STATUS = {500, 502, 503, 504}
+_TRANSIENT_ERRORS = (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError)
+
+
+class TransientAuthorizationError(HTTPException):
+    """The backend gave no authorization decision because its transport was unavailable."""
+
+    def __init__(self, detail: str) -> None:
+        super().__init__(503, detail, headers={"Retry-After": "1"})
+
+
+async def post_authorization(client, url, *, api_key, model_id, scope):
+    started = time.monotonic()
+    detail = "serving auth backend unreachable"
+    try:
+        async with asyncio.timeout(AUTHORIZATION_BUDGET_SECONDS):
+            for attempt in range(1, _ATTEMPTS + 1):
+                try:
+                    response = await client.post(
+                        url, json={"apiKey": api_key, "modelId": model_id}, headers=scope or None
+                    )
+                except _TRANSIENT_ERRORS as error:
+                    # Exception messages/URLs can contain credentials. Log only the type.
+                    kind = type(error).__name__
+                    detail = "serving auth backend unreachable"
+                except Exception as error:
+                    logger.warning("serving_auth_transport_error kind=%s", type(error).__name__)
+                    raise HTTPException(503, "serving auth backend unreachable") from error
+                else:
+                    if response.status_code not in _TRANSIENT_STATUS:
+                        return response
+                    kind = f"http_{response.status_code}"
+                    detail = "serving auth backend unavailable"
+                logger.warning(
+                    "serving_auth_transient_failure kind=%s attempt=%d elapsed_ms=%d",
+                    kind,
+                    attempt,
+                    int((time.monotonic() - started) * 1000),
+                )
+                if attempt < _ATTEMPTS:
+                    await asyncio.sleep(random.uniform(0.2, 0.4))
+    except TimeoutError:
+        logger.warning(
+            "serving_auth_deadline_exceeded budget_seconds=%s", AUTHORIZATION_BUDGET_SECONDS
+        )
+    raise TransientAuthorizationError(detail)

{freesolo_flash_dev-1.4.8 → freesolo_flash_dev-1.4.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.4.8"
+version = "1.4.9"
 description ="Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # The training and serving stack is validated on Python 3.11 and 3.12.
@@ -209,7 +209,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.4.8"
+version = "1.4.9"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]