freesolo-flash-dev 1.2.116__tar.gz → 1.2.118__tar.gz

freesolo_flash_dev-1.2.118/.env.example

@@ -0,0 +1,130 @@
+# flash control plane (operator-side). copy to .env and fill in.
+# provider credentials live only here; clients never see them and authenticate
+# with their api key.
+#
+# everything under "required" is checked by `flash/providers/core/preflight.py` at
+# startup; `flash-server` refuses to serve if any of them is missing. settings
+# that only degrade the plane (a single-account runpod pool, no `github_token`)
+# warn at startup instead of blocking it.
+#
+# self-hosting? read the self-hosting guide first - the short version is the two lines
+# below, and it is a much smaller list than this file suggests:
+#
+#   FLASH_STANDALONE=1
+#   FREESOLO_INTERNAL_KEY=$(openssl rand -hex 32)
+#
+# IMPORTANT: `flash-server` does NOT read this file. It reads the process
+# environment, so copying it to .env is not by itself enough — load it:
+#
+#   set -a && . ./.env && set +a && flash-server     # shell
+#   docker run --env-file .env ...                   # docker
+
+# --- required: GPU substrate (AT LEAST ONE of the three) ---
+# Flash allocates across RunPod, Lambda, and Vast. Configure the ones you have an
+# account with; a provider whose key is unset is simply never considered, and the
+# allocator ranks candidates only on the substrates you configured. One is enough
+# to run. The preflight fails only when ALL THREE are missing.
+#
+# RunPod: one key, or several comma-separated for multi-account failover. A
+# single-account pool works but cannot fail over or reap across accounts, so the
+# preflight warns.
+RUNPOD_API_KEY=
+# Lambda Cloud API key.
+LAMBDA_API_KEY=
+# Vast.ai API key.
+VAST_API_KEY=
+
+# --- required: artifacts ---
+# HuggingFace token with write access to the dataset repos flash creates for artifacts
+# (code upload + streamed checkpoints/adapters). The repo is assigned by the control
+# plane and scoped to the ENVIRONMENT, so runs sharing an environment share a repo and
+# each run lands under its own prefix. It is not a user-authored field: the TOML parser
+# rejects a [train] hf_repo set by hand.
+HF_TOKEN=
+# The HuggingFace user or org those artifact repos are created under. Defaults to
+# Freesolo's namespace, which a self-hoster's token cannot write to - so on a
+# self-hosted plane SET THIS to a namespace your HF_TOKEN owns, or every run fails at
+# the code upload before training starts.
+# FLASH_HF_NAMESPACE=your-hf-username
+
+# --- required: control-plane auth ---
+# The plane's own auth key. On a self-hosted (FLASH_STANDALONE=1) deployment this is
+# the ONLY credential the plane accepts and is therefore the entire trust boundary -
+# generate a high-entropy value and treat it like a root password:
+#
+#   export FREESOLO_INTERNAL_KEY=$(openssl rand -hex 32)
+#
+# On a managed deployment it is the shared internal key that maps to a single service
+# identity without a network call, alongside per-user keys verified against
+# {FREESOLO_BASE_URL}/api/auth/verify.
+FREESOLO_INTERNAL_KEY=
+
+# --- optional: self-hosting ---
+# Run without a Freesolo backend. Off by default. With it set, the plane stops calling
+# out for project/environment/billing validation and trusts FREESOLO_INTERNAL_KEY as a
+# single-tenant operator credential; external bearer tokens are REJECTED rather than
+# accepted unverified. Single tenant - see SELF_HOSTING.md before enabling.
+# FLASH_STANDALONE=1
+
+# --- optional: deployment topology ---
+# Where Flash keeps its local state: the saved login, the SQLite database, run records, and
+# results. Defaults to ~/.flash (= /root/.flash for the container's default user, which is the
+# path the Dockerfile declares as a VOLUME). Set this for a rootless container, a Kubernetes
+# PVC, a read-only home directory, or a systemd unit with ProtectHome. Everything moves
+# together, so point it at ONE durable location and mount a volume there.
+# FLASH_DATA_DIR=/var/lib/flash
+# Log verbosity for `flash-server`: DEBUG, INFO, WARNING, ERROR, CRITICAL. The server defaults
+# to INFO -- provider resolution, capability revocation, reaper startup, and degraded-config
+# warnings are all reported at INFO/WARNING. An unrecognized value falls back to the default
+# rather than blocking startup.
+# FLASH_LOG_LEVEL=INFO
+# `text` (default) or `json`. JSON emits one object per line for a structured log sink. The
+# same information is logged either way; only the punctuation differs.
+# FLASH_LOG_FORMAT=text
+# Bind address for `flash-server`. Defaults to 127.0.0.1; the container image passes 0.0.0.0
+# explicitly. The --host and --port flags win over both of these.
+# FLASH_SERVER_HOST=127.0.0.1
+# Port for `flash-server`, default 8080. Must be 1-65535: an unusable value stops startup
+# rather than silently binding the default, since a plane listening on the wrong port looks
+# healthy and is unreachable.
+# FLASH_SERVER_PORT=8080
+# How long (seconds) to wait for a SQLite lock held by another writer, default 30. Raise it on
+# networked storage, where a lock can be held far longer than on a local volume. This changes
+# only patience: journal mode, synchronous, and foreign keys are correctness settings and are
+# deliberately not configurable.
+# FLASH_SQLITE_BUSY_TIMEOUT_SECONDS=30
+
+# --- optional ---
+# Server token for the environment repository. Without it, environments hosted in a
+# private GitHub repo cannot be fetched; environments in public GitHub repos still work,
+# so this warns at startup rather than blocking it. (A local directory is never a valid
+# environment source - the GPU worker fetches the environment itself.)
+GITHUB_TOKEN=
+# the managed opd teacher credential stays on the control plane. submissions without it
+# fail before gpu allocation; workers receive only an attempt-scoped broker capability.
+PARASAIL_API_KEY=
+# this plane's own public https origin, where gpu workers reach the teacher broker route. it is
+# not the client's FLASH_API_URL / --api-url, which may be a private address (a tunnel, a vpn,
+# http://your-plane:8080) that no rented worker can resolve. required for opd; a value workers
+# cannot reach only surfaces after the gpu is allocated.
+FLASH_PUBLIC_URL=https://flash.example.com
+# Where to verify user keys. Defaults to https://api.freesolo.co when unset. Ignored
+# under FLASH_STANDALONE=1, which verifies nothing over the network.
+# FREESOLO_BASE_URL=
+# Adapter serving is delegated to a multi-LoRA serving app (a Modal app that serves
+# every adapter on one GPU per base model, scaling to zero when idle).
+# `flash deploy`/`undeploy`/`chat` register/deregister/chat against it. Defaults to the
+# hosted URL when unset; point it at your own deployment to self-host serving.
+#
+# Under FLASH_STANDALONE=1 there is NO default: those three commands error out until this
+# is set to a backend YOU operate. Every serving request carries FREESOLO_INTERNAL_KEY,
+# which on a self-hosted plane is the credential controlling that plane, so pointing it at
+# a service you do not run would hand that credential over. Training never uses this.
+#
+# A Freesolo-hosted value is refused the same way an unset one is -- assigning the hosted
+# URL here does not get you past the guard, it just produces the same error. Set it to
+# YOUR serving deployment; the managed plane gets the hosted URL from the code default.
+# FREESOLO_SERVING_URL=https://serving.example.internal
+# The worker container image is not configurable: every provider runs the published
+# ghcr.io/freesolo-co/flash-worker tags (per-GPU-arch kernel-cache variants, falling back to the
+# base :cu128), selected by GPU class in flash/providers/_lifecycle/net/worker.py.

freesolo_flash_dev-1.2.118/.github/workflows/auto-rebake.yml

@@ -0,0 +1,284 @@
+name: auto-rebake per-arch kernel cache
+
+# Keeps the per-arch ghcr.io/freesolo-co/flash-worker:cu128-sm* images fresh whenever the :cu128 base
+# is rebuilt. Each cu128-smXX is physically `FROM :cu128` + a baked mega-cache, so a new base leaves
+# the per-arch tags pointing at the OLD base until rebaked. "Out of date" has two flavors, keyed off
+# the fp_cache / fp_base labels worker-image.yml stamps on :cu128 (docker/kernel_fingerprint.py):
+#
+#   * fp_cache changed -> promote a matching dev-ready cache by free re-layer. a real gpu re-warm is
+#     only the fail-safe when the required candidate is absent.
+#   * fp_cache same but fp_base changed -> the cache is still valid, the base just moved (FA wheel /
+#     causal-conv1d / rp_handler / a non-kernel dep) -> a FREE re-layer grafts the existing cache onto
+#     the fresh base (docker/Dockerfile.kernelcache.relayer), no GPU.
+#   * both same -> skip (this is the pure-flash-code case the old reminder over-fired on).
+#
+# The gate READS the published labels (authoritative for what was actually built, incl. a manual FA2
+# override) rather than recomputing from the checkout, and aborts unless the base's revision label ==
+# the build's head_sha (so a superseded base build or a `tag=...` dispatch that never touched :cu128
+# can't trigger a stale rebake).
+
+on:
+  workflow_run:
+    workflows: ["flash worker image"]
+    types: [completed]
+
+permissions:
+  contents: read
+  packages: write
+  actions: read
+  issues: write
+
+# Scope the group to the triggering base build's commit so a no-op completion (e.g. a tag=cu128-test
+# worker-image run, whose head_sha differs) can't cancel an in-flight real bake. The revision guard
+# already neutralizes superseded runs, so a global cancel-in-progress isn't needed.
+concurrency:
+  group: auto-rebake-${{ github.event.workflow_run.head_sha }}
+  cancel-in-progress: true
+
+env:
+  BASE: ghcr.io/freesolo-co/flash-worker:cu128
+
+jobs:
+  gate:
+    # Upstream only -- see publish.yml. Guarded at the GATE rather than on each downstream job so
+    # one condition covers the whole chain: `gpu-warm` calls bake-kernel-cache.yml, which spends
+    # real GPU money, and `relayer` wants the org's larger runner. A fork can never reach either if
+    # the gate does not run.
+    if: >-
+      github.repository == 'freesolo-co/flash'
+      && github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    # Registry metadata reads only (crane digest/config per arch); every recorded gate run
+    # finishes inside a minute. The cap bounds a hung GHCR call, not real work.
+    timeout-minutes: 15
+    outputs:
+      proceed: ${{ steps.base.outputs.proceed }}
+      relayer_matrix: ${{ steps.classify.outputs.relayer_matrix }}
+      gpu_sms: ${{ steps.classify.outputs.gpu_sms }}
+      fp_cache: ${{ steps.base.outputs.fp_cache }}
+      fp_base: ${{ steps.base.outputs.fp_base }}
+      base_ref: ${{ steps.base.outputs.base_ref }}
+      base_revision: ${{ steps.base.outputs.revision }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      # Downloaded to disk and checksum-verified before extraction, rather than piped straight
+      # into tar: a piped stream is extracted as it arrives, so a corrupted or substituted
+      # artifact is already on PATH by the time anything could notice. Bump the version and the
+      # digest together (the vendor publishes checksums.txt alongside each release).
+      - name: Install crane
+        run: |
+          set -euo pipefail
+          ver=v0.20.2
+          sha=c14340087103ba9dadf61d45acd20675490fd0ccbd56ac7901fc1b502137f44b
+          curl -fsSL -o /tmp/crane.tar.gz \
+            "https://github.com/google/go-containerregistry/releases/download/${ver}/go-containerregistry_Linux_x86_64.tar.gz"
+          echo "${sha}  /tmp/crane.tar.gz" | sha256sum -c -
+          sudo tar -xz -C /usr/local/bin -f /tmp/crane.tar.gz crane
+          rm -f /tmp/crane.tar.gz
+          crane version
+
+      - name: Log in to GHCR
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Read base labels
+        id: base
+        env:
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          # canonical digest ref (repo@sha256, no tag) so every tool (crane/docker/runpod) accepts it.
+          digest=$(crane digest "$BASE")
+          repo=${BASE%:*}
+          cfg=$(crane config "$repo@$digest")
+          fpc=$(echo "$cfg" | jq -r '.config.Labels["co.freesolo.flash.kernelcache.fp_cache"] // ""')
+          fpb=$(echo "$cfg" | jq -r '.config.Labels["co.freesolo.flash.kernelcache.fp_base"] // ""')
+          rev=$(echo "$cfg" | jq -r '.config.Labels["org.opencontainers.image.revision"] // ""')
+          echo "base_ref=$repo@$digest" >> "$GITHUB_OUTPUT"
+          echo "fp_cache=$fpc" >> "$GITHUB_OUTPUT"
+          echo "fp_base=$fpb" >> "$GITHUB_OUTPUT"
+          echo "revision=$rev" >> "$GITHUB_OUTPUT"
+          # Only proceed if this base IS the build that triggered us: its revision label must match the
+          # build's head_sha. Guards (a) a newer base that already superseded this run, and (b) a
+          # `tag=cu128-test` dispatch that never updated :cu128 (the workflow_run payload can't reveal
+          # the dispatch tag). An unlabeled base (pre-rollout) also no-ops here.
+          if [ -n "$fpc" ] && [ "$rev" = "$HEAD_SHA" ]; then
+            echo "proceed=true" >> "$GITHUB_OUTPUT"
+            echo "base $BASE@$digest is rev $rev (== head_sha) fp_cache=$fpc fp_base=$fpb"
+          else
+            echo "proceed=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::auto-rebake no-op: base rev '$rev' != head_sha '$HEAD_SHA' (or base unlabeled)"
+          fi
+
+      - name: Classify arches
+        id: classify
+        if: steps.base.outputs.proceed == 'true'
+        env:
+          FP_CACHE: ${{ steps.base.outputs.fp_cache }}
+          FP_BASE: ${{ steps.base.outputs.fp_base }}
+          REV: ${{ steps.base.outputs.revision }}
+        run: |
+          # flash/providers/_lifecycle/net/worker.py is the single source of truth for the arch list; the bake
+          # matrix mirrors it. read the canonical assignment statically without importing flash.
+          arches=$(python3 docker/kernel_fingerprint.py --print-baked-arches)
+          relayer=(); gpu=()
+          summary="| arch | published fp_cache | published fp_base | action |\n|---|---|---|---|\n"
+          for sm in $arches; do
+            img="ghcr.io/freesolo-co/flash-worker:cu128-$sm"
+            if cfg=$(crane config "$img" 2>/dev/null); then
+              pc=$(echo "$cfg" | jq -r '.config.Labels["co.freesolo.flash.kernelcache.fp_cache"] // ""')
+              pb=$(echo "$cfg" | jq -r '.config.Labels["co.freesolo.flash.kernelcache.fp_base"] // ""')
+            else
+              pc="(missing)"; pb="(missing)"
+            fi
+            if [ "$pc" != "$FP_CACHE" ]; then
+              candidate="ghcr.io/freesolo-co/flash-worker:cu128-dev-ready-$sm"
+              candidate_pc=""; candidate_ref=""
+              if candidate_digest=$(crane digest "$candidate" 2>/dev/null); then
+                candidate_ref="${candidate%:*}@$candidate_digest"
+              fi
+              if [ -n "$candidate_ref" ] && candidate_cfg=$(crane config "$candidate_ref" 2>/dev/null); then
+                candidate_pc=$(echo "$candidate_cfg" | jq -r '.config.Labels["co.freesolo.flash.kernelcache.fp_cache"] // ""')
+              fi
+              if [ "$candidate_pc" = "$FP_CACHE" ]; then
+                relayer+=("$(jq -cn --arg sm "$sm" --arg source "$candidate_ref" '{sm:$sm, source:$source}')")
+                action="promote dev cache (free)"
+              else
+                gpu+=("$sm"); action="gpu re-warm (candidate missing)"
+              fi
+            elif [ "$pb" != "$FP_BASE" ]; then
+              relayer+=("$(jq -cn --arg sm "$sm" --arg source "$img" '{sm:$sm, source:$source}')")
+              action="re-layer (free)"
+            else
+              action="up to date"
+            fi
+            echo "$sm: pc=$pc pb=$pb -> $action"
+            summary="$summary| $sm | \`$pc\` | \`$pb\` | $action |\n"
+          done
+          if [ ${#relayer[@]} -gt 0 ]; then
+            relayer_json=$(printf '%s\n' "${relayer[@]}" | jq -cs .)
+          else
+            relayer_json='[]'
+          fi
+          gpu_csv=$(IFS=,; echo "${gpu[*]}")
+          echo "relayer_matrix=$relayer_json" >> "$GITHUB_OUTPUT"
+          echo "gpu_sms=$gpu_csv" >> "$GITHUB_OUTPUT"
+          {
+            echo "### auto-rebake plan (base rev \`$REV\`, fp_cache \`$FP_CACHE\` fp_base \`$FP_BASE\`)"
+            echo ""
+            echo -e "$summary"
+            echo ""
+            echo "re-layer (free): \`$relayer_json\` | GPU re-warm (paid, auto-started): \`${gpu_csv:-none}\`"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      # FYI notification (not an action item -- the bake auto-runs). Keeps a heads-up + cost trail of
+      # which arches a paid GPU re-warm fired for.
+      - name: Notify GPU re-bake auto-started
+        if: steps.base.outputs.proceed == 'true' && steps.classify.outputs.gpu_sms != ''
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GPU_SMS: ${{ steps.classify.outputs.gpu_sms }}
+          REV: ${{ steps.base.outputs.revision }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          title="kernel-cache: per-arch GPU re-bake auto-started"
+          body=$(printf 'The kernel toolchain changed (fp_cache), so a PAID GPU re-warm auto-started for these per-arch images (no approval needed):\n\n- arches: `%s`\n- base rev: `%s`\n\nTrack it in the run: %s' "$GPU_SMS" "$REV" "$RUN_URL")
+          # best-effort: a labeling/permission hiccup must never fail the gate. self-heal the label so
+          # no manual setup is needed, then reuse one open issue (comment) instead of spamming new ones.
+          gh label create kernel-rebake-pending --color B60205 --description "per-arch kernel cache GPU re-bake auto-started" 2>/dev/null || true
+          existing=$(gh issue list --label kernel-rebake-pending --state open --json number -q '.[0].number' 2>/dev/null || true)
+          if [ -n "${existing:-}" ]; then
+            gh issue comment "$existing" --body "$body" || true
+          else
+            gh issue create --title "$title" --body "$body" --label kernel-rebake-pending \
+              || gh issue create --title "$title" --body "$body" || true
+          fi
+
+  # FREE path: graft the existing valid cache onto the fresh base, no GPU. Runs automatically.
+  relayer:
+    needs: gate
+    if: needs.gate.outputs.proceed == 'true' && needs.gate.outputs.relayer_matrix != '[]' && needs.gate.outputs.relayer_matrix != ''
+    runs-on: ubuntu-24.04-8core
+    # The graft pulls two ~20GB images and pushes one. 21min is the slowest auto-rebake run on
+    # record; 60 covers a slow registry without holding the org's larger runner indefinitely.
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.gate.outputs.relayer_matrix || '[]') }}
+    env:
+      BASE_REF: ${{ needs.gate.outputs.base_ref }}
+      FP_CACHE: ${{ needs.gate.outputs.fp_cache }}
+      FP_BASE: ${{ needs.gate.outputs.fp_base }}
+      BASE_REVISION: ${{ needs.gate.outputs.base_revision }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      # The graft pulls two ~20GB images (old per-arch tag + fresh base); reclaim runner disk first.
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/.ghcup \
+            /usr/local/lib/android /opt/hostedtoolcache /usr/share/swift \
+            /usr/local/share/boost /usr/local/lib/node_modules \
+            /usr/local/share/powershell /usr/share/miniconda || true
+          sudo apt-get clean || true
+          sudo docker image prune -af || true
+          df -h /
+
+      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+
+      - name: Log in to GHCR
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Pin OLDCACHE by digest so a concurrent overwrite of the source tag can't graft a wrong cache.
+      - name: Resolve old per-arch cache digest
+        id: old
+        run: |
+          img="${{ matrix.source }}"
+          if [[ "$img" == *@sha256:* ]]; then
+            echo "ref=$img" >> "$GITHUB_OUTPUT"
+          else
+            digest=$(docker buildx imagetools inspect "$img" --format '{{.Manifest.Digest}}')
+            # canonical digest ref (repo@sha256, no tag) for the FROM in the relayer build.
+            echo "ref=${img%:*}@$digest" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Re-layer cache onto the fresh base (no GPU)
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: docker/Dockerfile.kernelcache.relayer
+          push: true
+          tags: ghcr.io/freesolo-co/flash-worker:cu128-${{ matrix.sm }}
+          provenance: false
+          build-args: |
+            BASE=${{ env.BASE_REF }}
+            OLDCACHE=${{ steps.old.outputs.ref }}
+            FP_CACHE=${{ env.FP_CACHE }}
+            FP_BASE=${{ env.FP_BASE }}
+            BASE_REVISION=${{ env.BASE_REVISION }}
+
+  # PAID path: re-warm the cache on a real GPU per arch. Runs automatically -- the fingerprint gate
+  # above is the only thing that decides it fires (only when fp_cache changed), so no manual approval.
+  gpu-warm:
+    needs: gate
+    if: needs.gate.outputs.proceed == 'true' && needs.gate.outputs.gpu_sms != ''
+    uses: ./.github/workflows/bake-kernel-cache.yml
+    with:
+      sms: ${{ needs.gate.outputs.gpu_sms }}
+      fp_cache: ${{ needs.gate.outputs.fp_cache }}
+      fp_base: ${{ needs.gate.outputs.fp_base }}
+      base_ref: ${{ needs.gate.outputs.base_ref }}
+      base_revision: ${{ needs.gate.outputs.base_revision }}
+    secrets: inherit

freesolo_flash_dev-1.2.118/.github/workflows/bake-kernel-cache.yml

@@ -0,0 +1,259 @@
+name: flash worker kernel-cache (per-arch)
+
+# Builds per-SM worker images with the compiled-kernel mega-cache baked in, killing the ~10-15 min
+# first-use JIT on a cold worker (the regression #194 reintroduced). Each matrix leg:
+#   1. offloads the warmup to a RunPod GPU of that arch (GH runners have none) -> build/kernel_cache,
+#   2. docker build --build-arg BUILD_KERNEL_CACHE=true -> ghcr.io/freesolo-co/flash-worker:cu128-<sm>,
+#   3. pushes it. The allocator then selects the cu128-<sm> tag per GPU class for baked arches
+#      (see flash/providers/_lifecycle/net/worker.py).
+#
+# Depends on the base ghcr.io/freesolo-co/flash-worker:cu128 image existing (worker-image.yml) — the
+# warmup runs INSIDE it so the cache matches the image's pinned torch/triton/fla/liger toolchain.
+#
+# Secrets: RUNPOD_API_KEY (the warmup GPU pod) + HF_TOKEN (the temp dataset that ferries code + the
+# cache artifact) are native repo Actions secrets, synced from Infisical (prod:/flash-ci) by the
+# github-ci-flash Secret Sync; the GHCR push uses the default GITHUB_TOKEN. Heavy + paid,
+# but UNGATED: the fingerprint detection in auto-rebake.yml decides WHEN this runs (only on a real
+# kernel-toolchain change), so there is no manual approval step. Invoked by hand (workflow_dispatch)
+# OR automatically by auto-rebake.yml (workflow_call) when the kernel toolchain changed.
+
+on:
+  workflow_dispatch:
+    inputs:
+      sms:
+        description: "comma-separated sm list to bake; default is all validated arches"
+        default: "sm80,sm86,sm89,sm90,sm120,sm100"
+        required: false
+      target_tag_prefix:
+        description: "target image tag before the -sm suffix"
+        default: "cu128"
+        required: false
+  # auto-rebake.yml calls this for the EXPENSIVE path (the cache is invalid and must be re-warmed on a
+  # real GPU). It passes the arch subset, the fingerprints to stamp, and a digest-pinned base so the
+  # warm runs inside the exact :cu128 the per-arch tag will sit on.
+  workflow_call:
+    inputs:
+      sms:
+        type: string
+        required: true
+      fp_cache:
+        type: string
+        required: false
+        default: ""
+      fp_base:
+        type: string
+        required: false
+        default: ""
+      base_ref:
+        type: string
+        required: false
+        default: "ghcr.io/freesolo-co/flash-worker:cu128"
+      base_revision:
+        type: string
+        required: false
+        default: ""
+      target_tag_prefix:
+        type: string
+        required: false
+        default: "cu128"
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  # Ungated: the PAID RunPod bake runs as soon as it's called/dispatched. The decision of WHEN to run
+  # lives in auto-rebake.yml's fingerprint gate (it only invokes this when fp_cache changed), so a
+  # second manual approval here would be redundant. Re-add an `approve` job with `environment: gpu-bake`
+  # (and `needs: approve`) if you ever want a human checkpoint back.
+  bake:
+    # Upstream only -- see publish.yml. "Ungated" above means no human APPROVAL step; it never meant
+    # any repository may start it. This workflow is directly dispatchable (`workflow_dispatch`), so
+    # a fork reaches it without passing auto-rebake's fingerprint gate, and it both requests the
+    # org-only `ubuntu-24.04-8core` runner (a label a fork has no runner for, so the job QUEUES
+    # instead of failing) and creates paid RunPod resources. `workflow_call` from auto-rebake still
+    # satisfies this, because a called workflow evaluates `github.repository` as the CALLER's repo.
+    if: github.repository == 'freesolo-co/flash'
+    runs-on: ubuntu-24.04-8core
+    # 47min is the slowest run on record (the rest sit at 12-23min); 120 absorbs a slow provider
+    # allocation without letting a hung bake hold the runner for the 6-hour default.
+    #
+    # This bounds the RUNNER, and only indirectly the spend. The rented pod is released by the
+    # `finally` in docker/bake_kernel_cache.py, so the pod outlives the timeout in exactly the case
+    # the timeout exists for: if the controller is wedged inside a blocking call, GitHub kills it
+    # and that `finally` may never complete, leaving a billed pod with nothing left to terminate it.
+    # Closing that needs a server-side pod lifetime or an out-of-band reaper -- a change to the bake
+    # tooling rather than to this workflow, so it is deliberately NOT bundled here. Until then, read
+    # this cap as "the runner stops waiting", not "the bill stops".
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {
+              sm: sm80,
+              arch: "8.0",
+              allowed_cuda: "",
+            }
+          - {
+              sm: sm86,
+              arch: "8.6",
+              allowed_cuda: "",
+            }
+          # the controller walks ordered same-sm secure-cloud gpu types from its per-sm map.
+          # preferred datacenter cards stay first, but one empty sku cannot strand this architecture
+          # on a stale image and send ordinary starts through jit.
+          - {
+              sm: sm89,
+              arch: "8.9",
+              allowed_cuda: "",
+            }
+          - {
+              sm: sm90,
+              arch: "9.0",
+              allowed_cuda: "",
+            }
+          # blackwell needs cuda-13 hosts to jit its ptx (matches min_cuda_for in the provider).
+          - {
+              sm: sm120,
+              arch: "12.0",
+              allowed_cuda: "13.0",
+            }
+          # datacenter blackwell is distinct from sm120. b200 is the only validated sm100 type.
+          - {
+              sm: sm100,
+              arch: "10.0",
+              allowed_cuda: "13.0",
+            }
+    steps:
+      # Gate FIRST (no checkout needed) so skipped arches do no crane / checkout / build work.
+      - name: Should bake ${{ matrix.sm }}?
+        id: gate
+        run: |
+          req="${{ inputs.sms || 'sm80,sm86,sm89,sm90,sm120,sm100' }}"
+          req="${req// /}"   # tolerate spaces, e.g. "sm80, sm86"
+          case ",$req," in
+            *,${{ matrix.sm }},*) echo "run=true" >> "$GITHUB_OUTPUT" ;;
+            *) echo "run=false" >> "$GITHUB_OUTPUT"; echo "skipping ${{ matrix.sm }}" ;;
+          esac
+
+      # Downloaded to disk and checksum-verified before extraction, rather than piped straight
+      # into tar: a piped stream is extracted as it arrives, so a corrupted or substituted
+      # artifact is already on PATH by the time anything could notice. Bump the version and the
+      # digest together (the vendor publishes checksums.txt alongside each release).
+      - name: Install crane
+        if: steps.gate.outputs.run == 'true'
+        run: |
+          set -euo pipefail
+          ver=v0.20.2
+          sha=c14340087103ba9dadf61d45acd20675490fd0ccbd56ac7901fc1b502137f44b
+          curl -fsSL -o /tmp/crane.tar.gz \
+            "https://github.com/google/go-containerregistry/releases/download/${ver}/go-containerregistry_Linux_x86_64.tar.gz"
+          echo "${sha}  /tmp/crane.tar.gz" | sha256sum -c -
+          sudo tar -xz -C /usr/local/bin -f /tmp/crane.tar.gz crane
+          rm -f /tmp/crane.tar.gz
+          crane version
+
+      - name: Log in to GHCR
+        if: steps.gate.outputs.run == 'true'
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Resolve the base + fingerprints BEFORE checkout, so the checkout (hence the warmed flash code)
+      # is pinned to the EXACT commit the base was built from. auto-rebake passes everything (fast path);
+      # a manual dispatch READS the labels off the published base, so the per-arch image always matches
+      # the base it extends (same source of truth as the gate) instead of recomputing from the checkout.
+      - name: Resolve base + fingerprints
+        id: base
+        if: steps.gate.outputs.run == 'true'
+        run: |
+          if [ -n "${{ inputs.fp_cache }}" ]; then
+            echo "ref=${{ inputs.base_ref }}" >> "$GITHUB_OUTPUT"
+            echo "fp_cache=${{ inputs.fp_cache }}" >> "$GITHUB_OUTPUT"
+            echo "fp_base=${{ inputs.fp_base }}" >> "$GITHUB_OUTPUT"
+            echo "revision=${{ inputs.base_revision }}" >> "$GITHUB_OUTPUT"
+          else
+            base="${{ inputs.base_ref || 'ghcr.io/freesolo-co/flash-worker:cu128' }}"
+            digest=$(crane digest "$base"); repo=${base%:*}
+            cfg=$(crane config "$repo@$digest")
+            echo "ref=$repo@$digest" >> "$GITHUB_OUTPUT"
+            echo "fp_cache=$(echo "$cfg" | jq -r '.config.Labels["co.freesolo.flash.kernelcache.fp_cache"] // ""')" >> "$GITHUB_OUTPUT"
+            echo "fp_base=$(echo "$cfg" | jq -r '.config.Labels["co.freesolo.flash.kernelcache.fp_base"] // ""')" >> "$GITHUB_OUTPUT"
+            echo "revision=$(echo "$cfg" | jq -r '.config.Labels["org.opencontainers.image.revision"] // ""')" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Pin the checkout to the base's source commit so the warmed flash code matches the base exactly
+      # (the cache is content-keyed by the code that compiled it). Falls back to the event sha if the
+      # base predates the revision label.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        if: steps.gate.outputs.run == 'true'
+        with:
+          ref: ${{ steps.base.outputs.revision || github.sha }}
+
+      - name: Install uv
+        if: steps.gate.outputs.run == 'true'
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+        with:
+          # pin the installed binary as well as the action implementation.
+          version: "0.12.5"
+          # no cache: the post-run cache-prune calls `uv`, but the "Free disk space" step deletes
+          # /opt/hostedtoolcache (where setup-uv put uv), so the post step fails ("uv not found") and
+          # marks the whole job red even though the bake + push already succeeded.
+          enable-cache: false
+
+      - name: Sync deps (flash + runpod + hf)
+        if: steps.gate.outputs.run == 'true'
+        run: uv sync --extra server
+
+      # RUNPOD_API_KEY (the warmup GPU pod) + HF_TOKEN (the temp dataset that ferries code/cache) are
+      # native repo Actions secrets, synced from Infisical (prod:/flash-ci) by the github-ci-flash
+      # Secret Sync. No Infisical CLI/login at job time -- Infisical stays the source of truth and
+      # rotations propagate via the sync. BASE_IMAGE
+      # is digest-pinned so the warm runs inside the EXACT base the per-arch tag will sit on.
+      - name: Warm kernels on a RunPod ${{ matrix.sm }} GPU -> build/kernel_cache
+        if: steps.gate.outputs.run == 'true'
+        env:
+          RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          BASE_IMAGE: ${{ steps.base.outputs.ref }}
+        run: |
+          uv run python docker/bake_kernel_cache.py \
+            --arch "${{ matrix.arch }}" --sm "${{ matrix.sm }}" \
+            --allowed-cuda "${{ matrix.allowed_cuda }}" \
+            --image "$BASE_IMAGE" \
+            --out build/kernel_cache
+
+      - name: Free disk space
+        if: steps.gate.outputs.run == 'true'
+        run: |
+          sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/.ghcup \
+            /usr/local/lib/android /opt/hostedtoolcache /usr/share/swift \
+            /usr/local/share/boost /usr/local/lib/node_modules \
+            /usr/local/share/powershell /usr/share/miniconda || true
+          sudo apt-get clean || true
+          sudo docker image prune -af || true
+          df -h /
+
+      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+        if: steps.gate.outputs.run == 'true'
+
+      # thin layer FROM the SAME base the warmup ran inside -> the cache toolchain always matches the
+      # shipped image (rebuilding the full Dockerfile.worker from the checkout could drift from the
+      # published base and bake a cache that silently won't load). GHCR auth from the login step above.
+      - name: Build + push baked per-sm image (thin layer on the warmed base)
+        if: steps.gate.outputs.run == 'true'
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: docker/Dockerfile.kernelcache
+          push: true
+          tags: ghcr.io/freesolo-co/flash-worker:${{ inputs.target_tag_prefix || 'cu128' }}-${{ matrix.sm }}
+          provenance: false
+          build-args: |
+            BASE=${{ steps.base.outputs.ref }}
+            FP_CACHE=${{ steps.base.outputs.fp_cache }}
+            FP_BASE=${{ steps.base.outputs.fp_base }}
+            BASE_REVISION=${{ steps.base.outputs.revision }}