freesolo-flash-dev 1.4.25__tar.gz → 1.4.26__tar.gz

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/.github/workflows/auto-rebake.yml

@@ -73,7 +73,7 @@ jobs:
           set -euo pipefail
           ver=v0.20.2
           sha=c14340087103ba9dadf61d45acd20675490fd0ccbd56ac7901fc1b502137f44b
-          curl -fsSL -o /tmp/crane.tar.gz \
+          curl -fsSL --retry 4 --retry-delay 2 --connect-timeout 15 --max-time 120 -o /tmp/crane.tar.gz \
             "https://github.com/google/go-containerregistry/releases/download/${ver}/go-containerregistry_Linux_x86_64.tar.gz"
           echo "${sha}  /tmp/crane.tar.gz" | sha256sum -c -
           sudo tar -xz -C /usr/local/bin -f /tmp/crane.tar.gz crane

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/.github/workflows/bake-kernel-cache.yml

@@ -154,7 +154,7 @@ jobs:
           set -euo pipefail
           ver=v0.20.2
           sha=c14340087103ba9dadf61d45acd20675490fd0ccbd56ac7901fc1b502137f44b
-          curl -fsSL -o /tmp/crane.tar.gz \
+          curl -fsSL --retry 4 --retry-delay 2 --connect-timeout 15 --max-time 120 -o /tmp/crane.tar.gz \
             "https://github.com/google/go-containerregistry/releases/download/${ver}/go-containerregistry_Linux_x86_64.tar.gz"
           echo "${sha}  /tmp/crane.tar.gz" | sha256sum -c -
           sudo tar -xz -C /usr/local/bin -f /tmp/crane.tar.gz crane

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/.github/workflows/deploy-modal-dev.yml

@@ -27,6 +27,7 @@ jobs:
         # free. (in freesolo the app WAS the root of its own tree and used cwd-relative imports.)
         working-directory: .
     env:
+      LAMBDA_API_KEY: ${{ secrets.LAMBDA_API_KEY }}
       MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
       MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
@@ -118,7 +119,7 @@ jobs:
           # pinned to the exact client pyproject declares; see deploy-modal.yml for why an
           # unbounded range is unsafe for the deploy path.
           # exact pins keep the deploy client bootstrap reproducible across runner image changes.
-          uv pip install --system "modal==1.6.2" "python-dotenv==1.2.3"
+          uv pip install --system "modal==1.6.2" "python-dotenv==1.2.3" "httpx==0.28.1"
 
       - name: Verify development deployment inputs
         if: steps.gate.outputs.deploy == 'true'
@@ -230,6 +231,26 @@ jobs:
           fi
           echo "Development Supabase flash.hosted_lora_adapters is queryable."
 
+      # This independent app keeps orphan VM cleanup running across serving releases.
+      - name: Validate inference provider
+        id: provider
+        if: steps.gate.outputs.deploy == 'true'
+        run: |
+          python3 - <<'PYCODE'
+          import os
+          import runpy
+          policy = runpy.run_path("flash/serving/app/inference_provider.py")
+          enabled = policy["INFERENCE_PROVIDER"] != "modal"
+          if enabled and not os.environ.get("LAMBDA_API_KEY"):
+              raise SystemExit("LAMBDA_API_KEY is required for lambda/all inference")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write(f"lambda_enabled={str(enabled).lower()}\n")
+          PYCODE
+
+      - name: Deploy Lambda fleet control
+        if: steps.gate.outputs.deploy == 'true' && steps.provider.outputs.lambda_enabled == 'true'
+        run: modal deploy --strategy rolling --env dev -m flash.serving.lambda_cloud.control_app
+
       - name: Deploy development serving
         if: steps.gate.outputs.deploy == 'true'
         run: modal deploy --strategy rolling --env dev flash/serving/app/modal_app.py

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/.github/workflows/deploy-modal.yml

@@ -41,9 +41,8 @@ jobs:
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     # Include the full router drain window before the canary, its GPU cold start,
-    # accounting settlement, image builds, and a rollback with its own readiness
-    # poll. The workflow contract test sums these budgets so rollback is never
-    # cut off by the job timeout.
+    # accounting settlement and image builds. The workflow contract test checks
+    # these budgets against the job timeout.
     timeout-minutes: 300
     # serialize serving deploys across runs because each deploy mutates the live app. github can still
     # replace a pending run, so the gate compares against the sha reported by the deployed app rather
@@ -61,6 +60,7 @@ jobs:
         working-directory: .
     env:
       # Modal CLI auth (read from the environment by `modal`).
+      LAMBDA_API_KEY: ${{ secrets.LAMBDA_API_KEY }}
       MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
       MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
       # runtime secrets forwarded into the serving app's Modal secret at deploy time.
@@ -97,12 +97,7 @@ jobs:
           DEPLOYED_HEALTH_URL: https://serve.freesolo.co/healthz
         run: |
           deploy=false
-          # the sha reported by the LIVE app is read ONCE, before any decision, because it serves
-          # two purposes: it bounds the undeployed diff below, and it is the rollback target for a
-          # failed promotion. `modal deploy` overwrites the app in place, so after that point there
-          # is no way left to ask what the previous release was. reading it on workflow_dispatch
-          # too -- where the deploy decision does not need it -- is what makes a dispatched deploy
-          # rollback-capable rather than one-way.
+          # Read the live revision to bound the undeployed diff.
           read_ok=true
           last_deployed_sha="$(python3 - <<'PY'
           import json
@@ -126,11 +121,8 @@ jobs:
           if [ "$read_ok" != "true" ]; then
             last_deployed_sha=""
           fi
-          # an empty previous sha means rollback is IMPOSSIBLE for this run. the rollback step below
-          # says so out loud rather than reporting a restore it never performed.
-          echo "previous_sha=$last_deployed_sha" >> "$GITHUB_OUTPUT"
 
-          # A failed promotion may leave this SHA live after refusing rollback.
+          # A failed promotion may leave this SHA live; an explicit rerun must reverify it.
           # An explicit rerun must re-prove it, not pass by skipping the canary.
           if [ "$EVENT" = "workflow_dispatch" ] || [ "${GITHUB_RUN_ATTEMPT:-1}" -gt 1 ]; then
             deploy=true
@@ -176,7 +168,7 @@ jobs:
           # pinned to the exact client pyproject declares and the provisioning path was validated
           # against: deploy behaviour (app lifecycle, tags, lookup) is client-version sensitive, so
           # an unbounded range lets a future release change deploys with no diff here.
-          python -m pip install "modal==1.6.2" "python-dotenv==1.2.3"
+          python -m pip install "modal==1.6.2" "python-dotenv==1.2.3" "httpx==0.28.1"
 
       - name: Verify Modal auth is configured
         if: steps.gate.outputs.deploy == 'true'
@@ -319,6 +311,26 @@ jobs:
             sleep "$sleep_seconds"
           done
 
+      # This independent app keeps orphan VM cleanup running across serving releases.
+      - name: Validate inference provider
+        id: provider
+        if: steps.gate.outputs.deploy == 'true'
+        run: |
+          python3 - <<'PYCODE'
+          import os
+          import runpy
+          policy = runpy.run_path("flash/serving/app/inference_provider.py")
+          enabled = policy["INFERENCE_PROVIDER"] != "modal"
+          if enabled and not os.environ.get("LAMBDA_API_KEY"):
+              raise SystemExit("LAMBDA_API_KEY is required for lambda/all inference")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write(f"lambda_enabled={str(enabled).lower()}\n")
+          PYCODE
+
+      - name: Deploy Lambda fleet control
+        if: steps.gate.outputs.deploy == 'true' && steps.provider.outputs.lambda_enabled == 'true'
+        run: modal deploy --strategy rolling --env main -m flash.serving.lambda_cloud.control_app
+
       - name: Deploy serving
         id: deploy_serving
         if: steps.gate.outputs.deploy == 'true'
@@ -442,143 +454,6 @@ jobs:
         # argument would put them in the process table and in the step's own command echo.
         run: python -m flash.serving.promotion.gate
 
-      - name: Restore the previous release
-        # `failure()` alone is too broad. It also fires for the steps that run BEFORE any deploy --
-        # installing Modal, verifying auth, verifying the runtime secrets, waiting for the adapter
-        # schema. Restoring there would `modal deploy` the previous SHA over a live app this run never
-        # replaced, using the very secret set whose verification just failed: a healthy release
-        # overwritten by a rollback that had nothing to roll back. Gating on the deploy step's own
-        # outcome keeps the case the previous comment was right about -- a deploy that half-succeeded
-        # and then failed readiness is `failure`, not skipped, so it still restores. Any outcome other
-        # than an attempted deploy (empty, `skipped`) means production is untouched and must stay so.
-        if: >-
-          failure() && steps.gate.outputs.deploy == 'true'
-          && (steps.deploy_serving.outcome == 'success' || steps.deploy_serving.outcome == 'failure')
-        working-directory: .
-        env:
-          PREVIOUS_SHA: ${{ steps.gate.outputs.previous_sha }}
-          CURRENT_SHA: ${{ github.sha }}
-          # a distinct attempt id so /healthz cannot report the FORWARD deploy's identity and be
-          # mistaken for a completed rollback.
-          ROLLBACK_DEPLOYMENT_ID: ${{ github.run_id }}-${{ github.run_attempt }}-rollback
-          HEALTH_URL: https://serve.freesolo.co/healthz
-        run: |
-          set -euo pipefail
-          if [ -z "$PREVIOUS_SHA" ]; then
-            echo "::error::Promotion failed and NO previous release was recorded, so the broken release is still live. Restore production manually."
-            exit 1
-          fi
-          if ! git merge-base --is-ancestor "$PREVIOUS_SHA" "$CURRENT_SHA" 2>/dev/null; then
-            # refusing here is the safe branch: redeploying a sha that is not an ancestor of this
-            # head would deploy an unknown tree rather than the release that was actually live.
-            echo "::error::Recorded previous release ${PREVIOUS_SHA} is not an ancestor of ${CURRENT_SHA}; refusing to redeploy an unverified tree. Restore production manually."
-            exit 1
-          fi
-
-          # Capture the forward dispatch contract before checkout removes its code.
-          # A bare predecessor deploy deletes new engine handles held by live routers.
-          rollback_bundle="$(mktemp -d)"
-          trap 'rm -rf "$rollback_bundle"' EXIT
-          if ! python -m flash.serving.app.rollback snapshot \
-            --source-sha "$CURRENT_SHA" --directory "$rollback_bundle"; then
-            echo "::error::Cannot preserve forward engine identities; refusing unsafe rollback."
-            exit 1
-          fi
-          echo "Promotion failed; restoring ${PREVIOUS_SHA}."
-          git checkout --quiet --force "$PREVIOUS_SHA"
-
-          # resolve the entrypoint in the restored tree, including releases before the source move.
-          if [ -f flash/serving/app/modal_app.py ]; then
-            rollback_app=flash/serving/app/modal_app.py
-          elif [ -f flash/serving/modal_app.py ]; then
-            rollback_app=flash/serving/modal_app.py
-          else
-            echo "::error::Previous release has no supported serving entrypoint. Restore production manually."
-            exit 1
-          fi
-
-          # A source ancestor can be incompatible after an irreversible schema cutover.
-          # Probe its own column contract before replacing the schema-compatible app.
-          columns_file=flash/serving/src/store/persisted_columns.py
-          if [ ! -f "$columns_file" ]; then
-            columns_file=flash/serving/src/persisted_columns.py
-          fi
-          if ! columns="$(python -c 'import runpy, sys; print(runpy.run_path(sys.argv[1])["PERSISTED_COLUMNS"])' "$columns_file")" || [ -z "$columns" ]; then
-            echo "::error::Cannot verify the previous release's schema contract; refusing rollback. Restore production manually."
-            exit 1
-          fi
-          if ! schema_status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
-            --connect-timeout 5 --max-time 15 \
-            --header "apikey: $SUPABASE_SERVICE_ROLE_KEY" --header "Accept-Profile: flash" \
-            "${SUPABASE_URL%/}/rest/v1/hosted_lora_adapters?select=${columns}&limit=1")" || [ "$schema_status" != "200" ]; then
-            echo "::error::Previous release is not compatible with the current database schema or its schema could not be verified; refusing rollback. Restore production manually."
-            exit 1
-          fi
-
-          # Build a compatibility artifact from the predecessor: retain forward handles,
-          # restore the predecessor's engine implementation, and gate router startup on
-          # active restored pools. Failure occurs before Modal mutates the deployment.
-          if ! python "$rollback_bundle/rollback.py" prepare \
-            --manifest "$rollback_bundle/manifest.json" --entrypoint "$rollback_app"; then
-            echo "::error::Previous release cannot retain active engine handles; refusing unsafe rollback."
-            exit 1
-          fi
-
-          # The source behavior is the predecessor; the distinct attempt identifies
-          # this rollback artifact and its forward-handle compatibility overlay.
-          FREESOLO_DEPLOYMENT_SHA="$PREVIOUS_SHA" \
-          FREESOLO_DEPLOYMENT_ID="$ROLLBACK_DEPLOYMENT_ID" \
-            modal deploy --strategy rolling "$rollback_app"
-
-          response_file="$(mktemp)"
-          trap 'rm -f "$response_file"; rm -rf "$rollback_bundle"' EXIT
-          # Match router startup: its full engine budget plus 60 seconds of headroom.
-          deadline=$((SECONDS + 3660))
-          restored=false
-          while (( SECONDS < deadline )); do
-            if curl --silent --show-error --output "$response_file" --fail \
-              --connect-timeout 5 --max-time 10 "$HEALTH_URL"; then
-              if python3 - "$response_file" "$PREVIOUS_SHA" "$ROLLBACK_DEPLOYMENT_ID" <<'PY'
-          import json
-          import sys
-
-
-          def reject_constant(_value: str) -> None:
-              raise ValueError
-
-
-          try:
-              with open(sys.argv[1], encoding="utf-8") as response:
-                  payload = json.load(response, parse_constant=reject_constant)
-          except (OSError, UnicodeDecodeError, ValueError):
-              sys.exit(1)
-
-          restored = (
-              isinstance(payload, dict)
-              and payload.get("ok") is True
-              and payload.get("deployment_sha") == sys.argv[2]
-              and payload.get("deployment_id") == sys.argv[3]
-          )
-          sys.exit(0 if restored else 1)
-          PY
-              then
-                restored=true
-                break
-              fi
-            fi
-            sleep 10
-          done
-
-          if [ "$restored" != "true" ]; then
-            echo "::error::Rollback to ${PREVIOUS_SHA} was deployed but production never reported the restored release. Production may be serving a broken release; intervene manually."
-            exit 1
-          fi
-          # exit nonzero regardless: a restored predecessor means production is serving again, NOT
-          # that this promotion succeeded. reporting green here would mark an unpromotable commit as
-          # deployed and let the next push build on a release that never passed its gate.
-          echo "::error::Promotion failed; production was restored to ${PREVIOUS_SHA}. This commit is NOT deployed."
-          exit 1
-
       - name: Nothing to deploy
         if: steps.gate.outputs.deploy != 'true'
         run: echo "No serving changes in this push; skipping deploy."

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/.github/workflows/dev-kernel-cache.yml

@@ -95,7 +95,7 @@ jobs:
           set -euo pipefail
           ver=v0.20.2
           sha=c14340087103ba9dadf61d45acd20675490fd0ccbd56ac7901fc1b502137f44b
-          curl -fsSL -o /tmp/crane.tar.gz \
+          curl -fsSL --retry 4 --retry-delay 2 --connect-timeout 15 --max-time 120 -o /tmp/crane.tar.gz \
             "https://github.com/google/go-containerregistry/releases/download/${ver}/go-containerregistry_Linux_x86_64.tar.gz"
           echo "${sha}  /tmp/crane.tar.gz" | sha256sum -c -
           sudo tar -xz -C /usr/local/bin -f /tmp/crane.tar.gz crane
@@ -189,7 +189,7 @@ jobs:
           set -euo pipefail
           ver=v0.20.2
           sha=c14340087103ba9dadf61d45acd20675490fd0ccbd56ac7901fc1b502137f44b
-          curl -fsSL -o /tmp/crane.tar.gz \
+          curl -fsSL --retry 4 --retry-delay 2 --connect-timeout 15 --max-time 120 -o /tmp/crane.tar.gz \
             "https://github.com/google/go-containerregistry/releases/download/${ver}/go-containerregistry_Linux_x86_64.tar.gz"
           echo "${sha}  /tmp/crane.tar.gz" | sha256sum -c -
           sudo tar -xz -C /usr/local/bin -f /tmp/crane.tar.gz crane

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/.github/workflows/production-kernel-cache-ready.yml

@@ -35,7 +35,7 @@ jobs:
           set -euo pipefail
           ver=v0.20.2
           sha=c14340087103ba9dadf61d45acd20675490fd0ccbd56ac7901fc1b502137f44b
-          curl -fsSL -o /tmp/crane.tar.gz \
+          curl -fsSL --retry 4 --retry-delay 2 --connect-timeout 15 --max-time 120 -o /tmp/crane.tar.gz \
             "https://github.com/google/go-containerregistry/releases/download/${ver}/go-containerregistry_Linux_x86_64.tar.gz"
           echo "${sha}  /tmp/crane.tar.gz" | sha256sum -c -
           sudo tar -xz -C /usr/local/bin -f /tmp/crane.tar.gz crane

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/Dockerfile

@@ -54,7 +54,7 @@ RUN if [ "${INSTALL_INFISICAL}" = "true" ]; then \
             *) echo "no pinned infisical build for dpkg arch: ${arch}" >&2; exit 1 ;; \
         esac; \
         deb="/tmp/infisical.deb"; \
-        curl -fsSL -o "${deb}" \
+        curl -fsSL --retry 4 --retry-delay 2 --connect-timeout 15 --max-time 120 -o "${deb}" \
             "https://github.com/Infisical/cli/releases/download/v${INFISICAL_VERSION}/infisical_${INFISICAL_VERSION}_linux_${arch}.deb"; \
         # Verify BEFORE dpkg touches it: a mismatch must fail the build, never install.
         echo "${sha}  ${deb}" | sha256sum -c -; \

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.4.25
+Version: 1.4.26
 Summary: Flash — managed LoRA post-training (SFT/GRPO) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/flash/runner/supervise/retry_decision.py

@@ -294,12 +294,11 @@ def _finish(
 def _spot_preemption_plan(state: RetryState, chosen) -> tuple[RetryState, RetryPlan]:
     if chosen is None or chosen.provider != "spot":
         return state, RetryPlan(False, "not retrying")
-    if state.infra_used >= state.infra_retries:
-        return state, RetryPlan(False, "not retrying: infrastructure retry budget exhausted")
-    ordinal = state.infra_used + 1
-    # Preemption says nothing about memory requirements: keep the same shape eligible.
-    return replace(state, infra_used=ordinal), RetryPlan(
-        True, "retrying Spot allocation (resume from last checkpoint)", ordinal
+    # Spot eviction is expected, including when the running price cap is hit.
+    # Fresh prices gate the replacement worker; cancellation and the run deadline
+    # bound the wait rather than the failure budget. Keep the same shape eligible.
+    return replace(state, cache_retry_shape=None), RetryPlan(
+        True, "requeuing spot capacity (resume from last checkpoint)"
     )
 
 

freesolo_flash_dev-1.4.26/flash/serving/app/inference_provider.py

@@ -0,0 +1,32 @@
+"""Deployment-time provider policy for hosted Qwen; DeepSeek is independent."""
+
+INFERENCE_PROVIDER = "lambda"  # "modal", "lambda", or "all"; redeploy after changing.
+
+
+def validate_provider(provider):
+    if provider not in {"modal", "lambda", "all"}:
+        raise ValueError("INFERENCE_PROVIDER must be modal, lambda, or all")
+    return provider
+
+
+def replica_limit(provider):
+    return 1 if validate_provider(provider) == "all" else 2
+
+
+def modal_gpus_for(model):
+    from flash.serving.src.engine.model_config import base_models
+
+    if model not in base_models():
+        raise ValueError("Provider policy is only defined for hosted Qwen")
+    if model == "Qwen/Qwen3.6-35B-A3B":
+        return ["H200"]
+    if model == "Qwen/Qwen3.5-0.8B":
+        return ["A10G", "A100-80GB"]
+    if model == "Qwen/Qwen3.5-4B":
+        return ["A10G", "L40S", "A100-80GB"]
+    if model in {"Qwen/Qwen3.5-9B", "Qwen/Qwen3.5-9B-Base"}:
+        return ["L40S", "A100-80GB"]
+    return ["A100-80GB"]
+
+
+validate_provider(INFERENCE_PROVIDER)

{freesolo_flash_dev-1.4.25 → freesolo_flash_dev-1.4.26}/flash/serving/app/modal_app.py

@@ -1,6 +1,7 @@
-"""Modal app: multi-LoRA serving with one GPU container per base model. ``LoraEngine`` is a
-Modal class parametrized by ``base_model`` (one GPU per base model, sharing its adapters via
-``enable_lora``); ``router`` is the CPU front door that dispatches to the right engine.
+"""Hosted serving: Modal CPU admission controllers own Lambda GPU workers.
+
+Model-scoped identities preserve admission, scale-to-zero and rolling handoffs.
+The public router and the separate DeepSeek deployment remain on Modal.
 """
 
 import asyncio
@@ -123,6 +124,8 @@ SCALEDOWN_WINDOW_SECONDS_BY_GPU: dict[str, int] = {
     "L40S": 2100,
     "H100": 2100,
     "H200": 2100,
+    "A10G": 2100,
+    "A100-80GB": 2100,
 }
 # The tiers this app is allowed to run an engine on. `gpu` in the serving catalog is a plain string,
 # so a typo ("b200", "B2OO") or an unvalidated new card used to fall through `dict.get` to a default
@@ -316,12 +319,31 @@ app = modal.App(APP_NAME, image=image)
 hf_cache_volume = modal.Volume.from_name(HF_CACHE_VOLUME_NAME, create_if_missing=True)
 
 
-# ``_LoraEngineImpl`` lives in ``flash.serving.src.engine.lora_engine`` (kept free of any ``modal``
-# import so it registers nothing, and inside the ``flash`` package so the image's
-# ``add_local_python_source("flash")`` ships it to the remote container under the same import path
-# it has here), re-exported so ``_build_engine`` can subclass it.
-from flash.serving.src.engine.lora_engine import _LoraEngineImpl  # noqa: E402
-
+# Retain the GPU implementation, registering only the providers selected below.
+from flash.serving.app.inference_provider import (  # noqa: E402
+    INFERENCE_PROVIDER,
+    modal_gpus_for,
+    replica_limit,
+    validate_provider,
+)
+from flash.serving.app.modal_engine import HostedModalEngine  # noqa: E402
+from flash.serving.lambda_cloud.proxy import LambdaEngineProxy  # noqa: E402
+from flash.serving.src.engine.lora_engine import _LoraEngineImpl  # noqa: E402, F401
+
+lambda_controller_image = (
+    modal.Image.debian_slim(python_version="3.12")
+    .apt_install("openssl")
+    .pip_install("httpx>=0.27", "modal==1.6.2")
+    .add_local_python_source("flash")
+)
+lambda_proxy_secrets = [
+    modal.Secret.from_dict(
+        {
+            **{name: os.environ[name] for name in _ENGINE_SECRET_NAMES if os.environ.get(name)},
+            "FREESOLO_DEPLOYMENT_SHA": os.environ.get("FREESOLO_DEPLOYMENT_SHA", ""),
+        }
+    )
+]
 # ---- One Modal LoraEngine class per GPU tier -----------------------------------------------------
 # model_config is a pure-stdlib module (no heavy deps), so importing it at module scope is safe for
 # `modal deploy` (which imports modal_app.py locally) — unlike the vllm/transformers imports, which
@@ -351,25 +373,19 @@ def _engine_concurrency(base_model: str) -> tuple[int, int]:
 
 
 def _build_engine(
-    gpu: str,
+    gpu: str | list[str],
     class_name: str,
     max_inputs: int,
     target_inputs: int,
+    provider: str = "lambda",
 ) -> Any:
-    """Register one Modal ``@app.cls`` LoraEngine pinned to ``gpu``.
-
-    Modal fixes a class's GPU at decoration time, so the H200 35B model and the L4 models need
-    distinct classes. Each model has a stable class. The Modal entrypoints are defined
-    fresh here (so each class owns its own method objects) and forward to the shared ``_``-prefixed impl on ``_LoraEngineImpl``.
+    """Register a Modal GPU engine or Lambda CPU proxy under a stable model name.
 
-    Class identity (name/qualname + module-global binding) is fixed BEFORE the class-level decorators
-    run. With the real Modal SDK ``@modal.concurrent`` returns a wrapper that holds the user class
-    separately, so renaming the bound name AFTER the decorator would rename the wrapper, not the class
-    Modal registers — every tier would then register under ``_Engine`` and the ``<locals>`` qualname
-    would fail Modal's global-scope validation. So we apply ``modal.concurrent``/``app.cls`` in call
-    form on an already-renamed, already-module-global class."""
+    Set the class identity before decorators so Modal can re-import it remotely.
+    """
+    implementation = HostedModalEngine if provider == "modal" else LambdaEngineProxy
 
-    class _Engine(_LoraEngineImpl):
+    class _Engine(implementation):
         base_model: str = modal.parameter()
 
         @modal.enter()
@@ -436,43 +452,71 @@ def _build_engine(
             return await self._unregister(org_id, model_id, expected_generation)
 
         @modal.method()
-        def health(self) -> dict[str, Any]:
-            return self._health()
-
-    # Record the GPU this class was actually pinned to (a plain class attribute, NOT a
-    # modal.parameter() field). _health() reports THIS rather than re-deriving the tier from the base
-    # model name, so a base model accidentally routed onto the wrong tier's class surfaces in health
-    # instead of being masked by the expected-tier lookup.
-    _Engine.pinned_gpu = gpu
-    # Give the REAL class its distinct, module-level identity BEFORE decorating. A clean (no-``<locals>``)
-    # __qualname__ satisfies Modal's global-scope validation, and binding it as a module attribute under
-    # that name makes ``getattr(module, class_name)`` resolve the class Modal re-imports in the container.
+        async def health(self) -> dict[str, Any]:
+            result = self._health()
+            return await result if hasattr(result, "__await__") else result
+
+    # Lambda reports actual hardware remotely; Modal is pinned by its decorator.
+    _Engine.pinned_gpu = gpu if provider == "modal" else None
+    # Modal must see the final module-level identity before decoration.
     _Engine.__name__ = class_name
     _Engine.__qualname__ = class_name
     globals()[class_name] = _Engine
+    resources = (
+        {
+            "image": image,
+            "gpu": gpu,
+            "secrets": engine_secrets,
+            "volumes": {HOSTING_CACHE_MOUNT: hf_cache_volume},
+        }
+        if provider == "modal"
+        else {
+            "image": lambda_controller_image,
+            "cpu": 1,
+            "memory": 2048,
+            "secrets": lambda_proxy_secrets,
+        }
+    )
     engine = app.cls(
-        gpu="H100!" if gpu == "H100" else gpu,
+        **resources,
+        nonpreemptible=True,
         region="us",
-        secrets=engine_secrets,
-        volumes={HOSTING_CACHE_MOUNT: hf_cache_volume},
-        scaledown_window=scaledown_window_for(gpu),
+        scaledown_window=scaledown_window_for(gpu[0] if isinstance(gpu, list) else gpu),
         startup_timeout=STARTUP_TIMEOUT_SECONDS,
         timeout=REQUEST_TIMEOUT_SECONDS,
         min_containers=MIN_CONTAINERS,
-        max_containers=MAX_CONTAINERS,
+        max_containers=replica_limit(INFERENCE_PROVIDER),
         buffer_containers=ENGINE_BUFFER_CONTAINERS,
     )(modal.concurrent(max_inputs=max_inputs, target_inputs=target_inputs)(_Engine))
-    # Rebind the module name to the decorated handle, matching the normal ``@app.cls class X`` pattern
-    # where the module attribute ends up referring to the decorated class.
     globals()[class_name] = engine
     return engine
 
 
-# GPU and concurrency may change without deleting the identity old routers call.
+# Both pools are registered only in all mode; each is then capped at one replica.
+validate_provider(INFERENCE_PROVIDER)
+_PRIMARY_PROVIDER = "modal" if INFERENCE_PROVIDER == "modal" else "lambda"
 ENGINE_BY_MODEL: dict[str, Any] = {
-    bm: _build_engine(gpu_for(bm), model_engine_name(bm), *_engine_concurrency(bm))
+    bm: _build_engine(
+        modal_gpus_for(bm) if _PRIMARY_PROVIDER == "modal" else gpu_for(bm),
+        model_engine_name(bm),
+        *_engine_concurrency(bm),
+        provider=_PRIMARY_PROVIDER,
+    )
     for bm in base_models()
 }
+MODAL_FALLBACK_BY_MODEL: dict[str, Any] = (
+    {
+        bm: _build_engine(
+            modal_gpus_for(bm),
+            model_engine_name(bm) + "_ModalGPU",
+            *_engine_concurrency(bm),
+            provider="modal",
+        )
+        for bm in base_models()
+    }
+    if INFERENCE_PROVIDER == "all"
+    else {}
+)
 
 
 def _engine_cls_for(base_model: str) -> Any:
@@ -488,11 +532,25 @@ class _ModalEnginePool:
         from flash.serving.deepinfer.app.client import HostedDeepInfer
 
         self._deepinfer = HostedDeepInfer()
+        self._provider_selection = None
+        if INFERENCE_PROVIDER == "all":
+            from flash.serving.app.provider_selection import ProviderSelection
+            from flash.serving.lambda_cloud.config import APP_NAME as fleet_app
+
+            fleet = modal.Cls.from_name(fleet_app, "LambdaFleet")()
+            self._provider_selection = ProviderSelection(fleet.has_capacity.remote.aio)
+
+    async def _engine_for(self, model):
+        if self._provider_selection and await self._provider_selection.choose(model) == "modal":
+            return MODAL_FALLBACK_BY_MODEL[model](base_model=model)
+        return _engine_cls_for(model)(base_model=model)
 
     async def start(self):
-        await warm_active_models(
-            {bm: cls(base_model=bm) for bm, cls in ENGINE_BY_MODEL.items()},
-        )
+        await warm_active_models({bm: cls(base_model=bm) for bm, cls in ENGINE_BY_MODEL.items()})
+        if MODAL_FALLBACK_BY_MODEL:
+            await warm_active_models(
+                {bm: cls(base_model=bm) for bm, cls in MODAL_FALLBACK_BY_MODEL.items()}
+            )
 
     async def aclose(self):
         await self._deepinfer.aclose()
@@ -522,7 +580,7 @@ class _ModalEnginePool:
         generation_id = payload.generation_id
         if not generation_id:
             raise RuntimeError("generation id is required before modal dispatch")
-        engine = _engine_cls_for(base_model)(base_model=base_model)
+        engine = await self._engine_for(base_model)
         return await engine.generate.remote.aio(
             internal_generation_payload(payload),
             self._record_payload(record),
@@ -555,7 +613,7 @@ class _ModalEnginePool:
         generation_id = payload.generation_id
         if not generation_id:
             raise RuntimeError("generation id is required before modal dispatch")
-        engine = _engine_cls_for(base_model)(base_model=base_model)
+        engine = await self._engine_for(base_model)
         from flash.serving.src.stream_channel.client import CancellableStreamChannel
 
         remote_stream = CancellableStreamChannel(
@@ -576,7 +634,7 @@ class _ModalEnginePool:
                 await close()
 
     async def register(self, base_model: str, record: Any) -> None:
-        engine = _engine_cls_for(base_model)(base_model=base_model)
+        engine = await self._engine_for(base_model)
         await engine.register.remote.aio(
             self._record_payload(record),
             getattr(record, "deployment_generation", None),
@@ -589,8 +647,17 @@ class _ModalEnginePool:
         model_id: str,
         expected_generation: str | None = None,
     ) -> None:
-        engine = _engine_cls_for(base_model)(base_model=base_model)
-        await engine.unregister.remote.aio(org_id, model_id, expected_generation)
+        if INFERENCE_PROVIDER == "all":
+            from flash.serving.app.provider_selection import unregister_active
+
+            engines = [
+                ENGINE_BY_MODEL[base_model](base_model=base_model),
+                MODAL_FALLBACK_BY_MODEL[base_model](base_model=base_model),
+            ]
+            await unregister_active(engines, org_id, model_id, expected_generation)
+        else:
+            engine = await self._engine_for(base_model)
+            await engine.unregister.remote.aio(org_id, model_id, expected_generation)
 
 
 def _build_usage_outbox(settings: Any) -> Any: