rsc-universal 0.1.1

package/skills/data-scraper/scripts/verify.sh

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+#
+# verify.sh — compliance/resilience linter for the `data-scraper` skill.
+#
+# WHAT IT DOES (read-only; never edits a file)
+#   Statically lints a candidate scraper (a file or directory of source) for the
+#   four properties the skill insists on before a scrape ships. It greps source
+#   for signal patterns — it does not run the scraper or hit any network.
+#
+#   Per-file scan of *.py *.js *.ts *.mjs *.cjs (the languages the starters use).
+#   A file with no scraping signal at all is skipped, so this never false-fails
+#   on an unrelated repo (and an empty/clean target exits 0).
+#
+#   Checks (a FAILURE is only ever an unbounded scrape or a circumvention call —
+#   those are not judgement calls):
+#     1. Compliance preflight present  -> robots.txt / ToS / ai.txt / robotparser
+#        / ROBOTSTXT_OBEY reference somewhere in the target.   (missing -> warn)
+#     2. Per-host pacing present       -> a delay/sleep/throttle/concurrency/
+#        rate-limit/Crawl-delay/AUTOTHROTTLE signal.           (missing -> FAIL)
+#     3. Retry / backoff present       -> backoff/retry/Retry-After/max_retries
+#        /exponential signal.                                  (missing -> warn)
+#     4. No circumvention              -> a CAPTCHA-solver call (2captcha,
+#        twocaptcha, anticaptcha, capsolver, deathbycaptcha, solve_captcha) or an obvious
+#        auth-bypass marker.                                   (present -> FAIL)
+#
+#   "Scraping signal" = the file imports/uses one of: requests, httpx, curl_cffi,
+#   playwright, crawlee, scrapy, beautifulsoup/bs4, selectolax, puppeteer, cheerio.
+#
+# HOW TO RUN (inside YOUR project, not the skills repo)
+#   ./verify.sh                 # scan ./ for scraper source
+#   ./verify.sh --path src      # scan a subdirectory or a single file
+#   ./verify.sh --strict        # treat any warning as a failure (exit 1)
+#
+# EXIT CODES
+#   0  clean, or warnings only without --strict (also: nothing to check)
+#   1  an unbounded scrape (no pacing) or a circumvention call, or --strict + warn
+#   2  bad usage
+#
+# Runs on stock macOS bash 3.2 — no mapfile, no associative arrays.
+
+set -euo pipefail
+
+if [ -t 1 ]; then
+  RED=$'\033[31m'; GREEN=$'\033[32m'; YELLOW=$'\033[33m'; NC=$'\033[0m'
+else
+  RED=''; GREEN=''; YELLOW=''; NC=''
+fi
+
+ok_count=0; skip_count=0; warn_count=0; fail_count=0
+ok()   { printf '%s[ ok ]%s %s\n' "$GREEN"  "$NC" "$*"; ok_count=$((ok_count + 1)); }
+skip() { printf '%s[skip]%s %s\n' "$YELLOW" "$NC" "$*"; skip_count=$((skip_count + 1)); }
+warn() { printf '%s[warn]%s %s\n' "$YELLOW" "$NC" "$*"; warn_count=$((warn_count + 1)); }
+fail() { printf '%s[fail]%s %s\n' "$RED"    "$NC" "$*"; fail_count=$((fail_count + 1)); }
+
+usage() { sed -n '2,46p' "$0" | sed 's/^# \{0,1\}//'; }
+
+SCAN_PATH="."
+STRICT=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --path)    SCAN_PATH="${2:?--path needs a value}"; shift 2 ;;
+    --strict)  STRICT=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) printf '%sUnknown argument: %s%s\n\n' "$RED" "$1" "$NC"; usage; exit 2 ;;
+  esac
+done
+
+if [ ! -e "$SCAN_PATH" ]; then
+  printf '%sPath not found: %s%s\n' "$RED" "$SCAN_PATH" "$NC"; exit 2
+fi
+
+# --- collect candidate source files -----------------------------------------
+TMPDIR_V="$(mktemp -d 2>/dev/null || printf '/tmp/dscrape-verify.%s' "$$")"
+mkdir -p "$TMPDIR_V" 2>/dev/null || true
+cleanup() { rm -rf "$TMPDIR_V" 2>/dev/null || true; }
+trap cleanup EXIT
+ALL="$TMPDIR_V/all"
+TARGETS="$TMPDIR_V/targets"
+
+find "$SCAN_PATH" -type f \
+  \( -name '*.py' -o -name '*.js' -o -name '*.ts' -o -name '*.mjs' -o -name '*.cjs' \) \
+  2>/dev/null > "$ALL" || true
+
+if [ ! -s "$ALL" ]; then
+  skip "no source files (*.py *.js *.ts *.mjs *.cjs) under: $SCAN_PATH"
+  printf '\nok=%d skip=%d warn=%d fail=%d\n' "$ok_count" "$skip_count" "$warn_count" "$fail_count"
+  exit 0
+fi
+
+# Keep only files that look like scrapers (import/use an HTTP/scrape library).
+SCRAPE_LIB='requests|httpx|curl_cffi|playwright|crawlee|scrapy|bs4|beautifulsoup|selectolax|puppeteer|cheerio'
+: > "$TARGETS"
+while IFS= read -r f; do
+  [ -z "$f" ] && continue
+  if grep -iEq "$SCRAPE_LIB" "$f" 2>/dev/null; then
+    printf '%s\n' "$f" >> "$TARGETS"
+  fi
+done < "$ALL"
+
+if [ ! -s "$TARGETS" ]; then
+  skip "no scraper source detected (no requests/httpx/playwright/crawlee/scrapy/... usage)"
+  printf '\nok=%d skip=%d warn=%d fail=%d\n' "$ok_count" "$skip_count" "$warn_count" "$fail_count"
+  exit 0
+fi
+
+n_targets="$(wc -l < "$TARGETS" | tr -dc '0-9')"
+ok "scraper source detected in $n_targets file(s)"
+
+# Helper: does ANY target file match a (case-insensitive) ERE pattern?
+any_match() {
+  pattern="$1"
+  while IFS= read -r f; do
+    [ -z "$f" ] && continue
+    if grep -iEq "$pattern" "$f" 2>/dev/null; then return 0; fi
+  done < "$TARGETS"
+  return 1
+}
+
+# Helper: list target files that match (for FAIL detail).
+list_match() {
+  pattern="$1"
+  while IFS= read -r f; do
+    [ -z "$f" ] && continue
+    if grep -iEq "$pattern" "$f" 2>/dev/null; then printf '  %s\n' "$f"; fi
+  done < "$TARGETS"
+}
+
+# --- 1. compliance preflight -------------------------------------------------
+PREFLIGHT='robots\.txt|robotstxt_obey|robotfileparser|robotparser|can_fetch|ai\.txt|terms of service|tos preflight|compliance preflight'
+if any_match "$PREFLIGHT"; then
+  ok "compliance preflight present (robots.txt / ToS / robotparser reference)"
+else
+  warn "no robots.txt / ToS / compliance preflight found — add the legal gate before requests"
+fi
+
+# --- 2. per-host pacing (the hard one) ---------------------------------------
+PACING='download_delay|crawl[_-]?delay|autothrottle|rate[_-]?limit|ratelimit|max_concurrency|concurrent_requests|max_requests_per|throttle|time\.sleep|asyncio\.sleep|setTimeout|sleep\('
+if any_match "$PACING"; then
+  ok "per-host pacing / concurrency cap present (delay / throttle / concurrency)"
+else
+  fail "no rate limit, delay, or concurrency cap found — unbounded scrape will get blocked/banned:"
+  list_match "$SCRAPE_LIB"
+fi
+
+# --- 3. retry / backoff ------------------------------------------------------
+BACKOFF='backoff|retry|retries|retry-after|retry_after|max_request_retries|exponential|tenacity'
+if any_match "$BACKOFF"; then
+  ok "retry / backoff present"
+else
+  warn "no retry/backoff found — transient errors will drop records; honor 429/Retry-After"
+fi
+
+# --- 4. no circumvention (the other hard one) --------------------------------
+CIRCUMVENT='2captcha|twocaptcha|anti-?captcha|anticaptcha|capsolver|deathbycaptcha|captcha[_-]?solver|solve[_-]?captcha|bypass[_-]?(auth|login|captcha|block)'
+if any_match "$CIRCUMVENT"; then
+  fail "circumvention call detected (CAPTCHA solver / auth-or-block bypass) — this is the legal line you do not cross:"
+  list_match "$CIRCUMVENT"
+else
+  ok "no CAPTCHA-solver / auth-bypass call detected"
+fi
+
+printf '\nok=%d skip=%d warn=%d fail=%d\n' "$ok_count" "$skip_count" "$warn_count" "$fail_count"
+
+if [ "$fail_count" -gt 0 ]; then exit 1; fi
+if [ "$STRICT" -eq 1 ] && [ "$warn_count" -gt 0 ]; then exit 1; fi
+exit 0

package/skills/db-migrations/SKILL.md

@@ -0,0 +1,254 @@
+---
+name: db-migrations
+description: "Use when planning a schema change that must ship without downtime — adding a NOT NULL column, renaming/splitting/merging a column or table, changing a type, or backfilling millions of rows on a live table — and you need the expand-contract sequence plus the lock/timeout/batching discipline that keeps each step from freezing prod. Triggers: 'zero-downtime migration', 'add NOT NULL column to a huge table', 'rename a column safely', 'our deploy keeps deadlocking when we add an index', 'backfill 80M rows without replication lag', 'expand contract', 'gh-ost vs pt-osc', 'migración sin downtime', 'migració sense tallar el servei'. NOT Postgres lock internals/EXPLAIN/RLS (that is postgresdb), NOT drizzle-kit runner mechanics (that is drizzle-orm), NOT restore drills/PITR (that is backups)."
+tags: [migrations, zero-downtime, expand-contract, schema, backfill, ddl]
+recommends: [postgresdb, drizzle-orm, backups, sql, planetscale]
+origin: risco
+---
+
+# Database migrations without downtime
+
+A production migration is a **sequence**, not a statement. The moment you treat `ALTER TABLE ... NOT NULL`
+as one atomic change, you have already lost: the lock blocks every writer, the deploy that depends on the
+new column races the deploy that still reads the old one, and a rollback means a second outage. Your job
+is to decompose one risky change into a chain of small steps where **every intermediate state is
+independently deployable** — the old app keeps working, the new app keeps working, and you can stop at any
+point without an outage.
+
+This skill is engine-neutral. It owns the *choreography* — what order to run things in and how to keep each
+DDL from freezing the table — across Postgres, MySQL, and serverless variants, whatever runner you use
+(Flyway, Alembic, golang-migrate, drizzle-kit, raw SQL). It does not teach one ORM's CLI and it is not a
+Postgres tuning manual.
+
+## When to use / When NOT to use
+
+**Use when:**
+
+- Sequencing a multi-step zero-downtime change: NOT NULL, rename, split/merge, type change, drop, add FK.
+- Backfilling a new column/table over millions of rows without blocking writes or blowing up replica lag.
+- Making one DDL statement safe: `lock_timeout`, `CONCURRENTLY`, running outside a transaction.
+- Coordinating deploy order so app version N and N+1 are both schema-compatible at every step.
+- Deciding rollback strategy (forward-fix vs reversible) and choosing an OSC tool for a big MySQL ALTER.
+
+**Route elsewhere:**
+
+- Postgres lock internals, EXPLAIN/ANALYZE, indexing strategy, RLS, PgBouncer → `../postgresdb/SKILL.md`.
+- Writing schema and running drizzle-kit specifically → `../drizzle-orm/SKILL.md`.
+- PlanetScale deploy-request / branch cutover (Vitess online DDL) → `../planetscale/SKILL.md`.
+- Creating/verifying backups, restore drills, PITR → `../backups/SKILL.md` (a migration *requires* a backup; making it lives there).
+- Pure query logic — joins, windows, CTEs → `../sql/SKILL.md` (backfill batching is here; query correctness is there).
+
+## Step 1 — Is this change online-safe in one step?
+
+Most outages come from running a multi-step change as a single statement. Decide first:
+
+| Change | One step? | Why |
+|---|---|---|
+| Add nullable column, no volatile default | Yes | Metadata-only on PG 11+/modern MySQL; no rewrite, no long lock. |
+| Add column with a **constant** default | Yes (PG 11+) | Stored as a catalog default, no table rewrite. |
+| Add column with a **volatile/expression** default | No | Rewrites every row under lock → expand-contract. |
+| Add `NOT NULL` to existing/new column | No | Full-table validation scan under lock → add-nullable → backfill → validate. |
+| Rename column / table | No | App reads the old name; needs dual-name window → expand-contract. |
+| Change column type (non-trivial) | No | Rewrites rows, may invalidate the running app → new column + backfill. |
+| Split / merge columns | No | Two-sided data move → expand-contract. |
+| Drop column / table | No | Running app may still reference it → contract only after a cooling period. |
+| Add foreign key | No | `ADD CONSTRAINT` validates all rows under lock → add `NOT VALID` then `VALIDATE`. |
+| Create index | No, but cheap | Plain `CREATE INDEX` locks writes → use `CONCURRENTLY`. |
+
+If the row says **No**, you are doing expand-contract. Go to Step 2.
+
+## Step 2 — The expand-contract loop
+
+Three phases, each tied to a **distinct deploy**. Worked example: rename `users.email` → `users.email_address`.
+
+**Expand** — add the new structure, start dual-writing, leave the old one fully working.
+
+```sql
+-- migration 1 (deploy A): additive only, old app untouched
+SET lock_timeout = '1s';
+ALTER TABLE users ADD COLUMN email_address text;  -- nullable, no default → metadata-only
+```
+
+```text
+App deploy A: write BOTH columns on every insert/update; keep READING email.
+```
+
+**Backfill** — copy historical data in small background batches (Step 4), then verify counts/checksums.
+
+**Switch reads** — once backfilled and verified, deploy the app that reads `email_address`.
+
+```text
+App deploy B: read email_address; still dual-write both.  ← cooling period starts here
+```
+
+**Contract** — only after a cooling period (hours to days) where the new app has run clean and you can
+still roll back the app without touching schema, stop writing the old column and drop it.
+
+```sql
+-- migration 2 (deploy C): runs days later, after cooling period
+SET lock_timeout = '1s';
+ALTER TABLE users DROP COLUMN email;
+```
+
+The cooling period is the point. It is what lets you roll back app deploy B to app deploy A with **zero
+schema change** — because the old column still exists and is still being written. Contracting early throws
+that safety away.
+
+## Step 3 — The N/N-1 rule
+
+At every moment, two app versions can be live: the one running and the one rolling out. **The schema must
+be compatible with both, and each app must tolerate the other's schema.** That is the entire reason
+dual-write exists — it makes the schema a superset that satisfies app N and app N+1 simultaneously.
+
+```text
+schema:  +email_address ......................... -email
+apps:    [N reads email] [N+1 reads email_address] [N+2 ignores email]
+              ^ both alive during rollout ^         ^ safe to drop now ^
+```
+
+If you ever have a single instant where one deployed app version cannot run against the current schema,
+your migration is not zero-downtime — it is a timed outage.
+
+## Step 4 — Make each DDL statement safe
+
+**Always set a short `lock_timeout` first.** A DDL waiting on a lock builds a *queue* — every query behind
+it stalls too, and one slow ALTER freezes the whole table. A short timeout makes the migration fail fast so
+traffic keeps flowing; you retry instead of taking the site down.
+
+```sql
+-- Bad: unbounded wait; if a lock is held, this queues every reader/writer behind it
+ALTER TABLE orders ADD COLUMN region text;
+
+-- Good: fail fast, retry
+SET lock_timeout = '1s';
+ALTER TABLE orders ADD COLUMN region text;
+```
+
+**Create indexes `CONCURRENTLY`, outside a transaction.** Plain `CREATE INDEX` takes a SHARE lock and
+blocks all writes for its whole duration — that is the "our deploy deadlocks when we add an index" symptom.
+`CONCURRENTLY` takes SHARE UPDATE EXCLUSIVE and does not block writes, but **cannot run inside a transaction
+block**. Runners (Flyway, Alembic, Django, Rails) wrap each migration in a transaction by default, so you
+must opt that step out — see `references/tools-and-runners.md`.
+
+```sql
+-- Bad: locks out every writer for the full build
+CREATE INDEX idx_orders_region ON orders (region);
+
+-- Good: non-blocking; this statement must NOT be inside BEGIN/COMMIT
+CREATE INDEX CONCURRENTLY idx_orders_region ON orders (region);
+```
+
+> A failed `CONCURRENTLY` build leaves an INVALID index behind. Drop it (`DROP INDEX CONCURRENTLY`) before
+> retrying. For PG-specific lock-level detail, see `../postgresdb/SKILL.md`.
+
+**Add NOT NULL in four steps, never in one.** A single `ALTER ... SET NOT NULL` (or `ADD COLUMN ... NOT
+NULL DEFAULT <volatile>`) scans/rewrites the whole table under lock.
+
+```sql
+-- Bad: full-table validation/rewrite under an exclusive lock
+ALTER TABLE users ADD COLUMN status text NOT NULL DEFAULT 'active';  -- on a 40M-row table = outage
+
+-- Good: add nullable, backfill in batches (Step 5), then add a constraint without a blocking scan
+SET lock_timeout = '1s';
+ALTER TABLE users ADD COLUMN status text;                              -- 1. nullable
+-- 2. backfill in batches (Step 5), app dual-writes the new column
+ALTER TABLE users ADD CONSTRAINT users_status_nn
+  CHECK (status IS NOT NULL) NOT VALID;                                -- 3. instant, no scan
+ALTER TABLE users VALIDATE CONSTRAINT users_status_nn;                 -- 4. scans WITHOUT blocking writes
+-- PG 12+: a validated CHECK (col IS NOT NULL) lets SET NOT NULL skip its own scan
+ALTER TABLE users ALTER COLUMN status SET NOT NULL;
+```
+
+Same `NOT VALID` → `VALIDATE` pattern for adding a foreign key. The full per-change checklists live in
+`references/expand-contract-playbook.md`.
+
+## Step 5 — Backfill at scale
+
+**Never one giant `UPDATE`.** A single statement over millions of rows holds locks, bloats WAL/undo, and
+pins replicas behind a wall of lag. Loop bounded chunks, commit each, throttle on lag, then verify.
+
+```sql
+-- Bad: one statement locks rows, balloons WAL, spikes replica lag for minutes
+UPDATE users SET email_address = email WHERE email_address IS NULL;
+```
+
+```sql
+-- Good: bounded PK-range batches, committed individually, throttled between batches
+-- pseudo-loop (driven by your runner/script): a few thousand rows per batch
+UPDATE users
+SET email_address = email
+WHERE id BETWEEN :lo AND :hi
+  AND email_address IS NULL;
+-- commit; check replica lag; sleep if lag/error-rate is high; advance :lo/:hi; repeat
+```
+
+Watch replication lag and the app error rate between batches; back off when either climbs. **Verify before
+contracting** — row counts and a checksum of old-vs-new must match. For very large sets, snapshot + CDC
+instead of an online loop. Postgres and MySQL backfill scripts, the lag-throttle, and checksum queries are
+in `references/backfill-and-batching.md`.
+
+## Step 6 — Rollback reality
+
+**Forward-only is the prevailing 2025-2026 production stance.** Reliable `down()` migrations are rarely
+worth the effort — whether a rollback is even safe depends on the migration type and how much time has
+passed (data has changed under you). Treat applied migrations as immutable and **fix forward**.
+
+Your real safety net is not a `down()` script — it is the expand-contract structure itself: because the old
+column/table still exists during the cooling period, "rolling back" is just redeploying the previous app.
+A reversible down-migration is only worth writing when the change is **additive-only and pre-traffic**
+(e.g. a brand-new table no app version reads yet). For anything that touched live data, a down-migration is
+a trap that looks like a seatbelt.
+
+## Step 7 — Pick a runner / OSC tool
+
+Versioned runners replay ordered scripts; declarative tools diff your target schema. Pick by stack:
+
+| Stack / need | Tool (2026) | Note |
+|---|---|---|
+| JVM, SQL-first, 50+ DBs | Flyway 12.6.x | Version-based; Redgate dropped the Teams tier in 2025. |
+| Max format flexibility | Liquibase | SQL/XML/YAML/JSON changelogs; most verbose. |
+| Python / SQLAlchemy | Alembic 1.18.x | Requires Python ≥3.10; plugin system since 1.18.0. |
+| Go | golang-migrate | The Go standard. |
+| TypeScript / Drizzle | drizzle-kit | Mechanics in `../drizzle-orm/SKILL.md`; safety sequence here. |
+| Schema-as-code, planned diffs | Atlas | Declarative — you give the target, it plans the diff. |
+| Big MySQL ALTER, native online DDL can't keep it online | gh-ost or pt-osc | See below. |
+
+**gh-ost vs pt-osc** for a large MySQL ALTER: **gh-ost** is triggerless — it copies in chunks and replays
+the binlog, throttles on replica lag, and gives easier pause/cutover control; prefer it on a very busy
+master. **pt-osc** is trigger-based — strictly consistent, but the triggers add write overhead and can fail
+to acquire the metadata lock under highly concurrent or long-transaction load. Decision detail and
+per-runner transaction opt-out in `references/tools-and-runners.md`.
+
+## Step 8 — Gate migrations in CI
+
+Add a static linter for migration SQL **before merge**, so a bare `CREATE INDEX` or a one-shot `NOT NULL`
+never reaches review. **Squawk** is the standard for Postgres migration SQL — it flags adding NOT NULL
+columns, non-concurrent index creation, blocking constraint adds, and column drops.
+
+```yaml
+# CI step: fail the PR on dangerous migration DDL
+- name: Lint migrations
+  run: npx squawk@latest migrations/*.sql
+```
+
+Config snippet and rule notes in `references/tools-and-runners.md`. `scripts/verify.sh` in this skill runs
+the same static checks against the example migrations shipped in `references/`.
+
+## Anti-patterns
+
+| Anti-pattern | Why it hurts | Do instead |
+|---|---|---|
+| `ALTER TABLE ... ADD COLUMN ... NOT NULL DEFAULT <expr>` on a big table | Full-table rewrite under lock = outage | add nullable → backfill → `NOT VALID` → `VALIDATE` (Step 4) |
+| Plain `CREATE INDEX` in prod | SHARE lock blocks all writes for the build → deploy "deadlock" | `CREATE INDEX CONCURRENTLY`, outside a transaction |
+| DDL with no `lock_timeout` | Builds a lock queue; one slow ALTER freezes the table | `SET lock_timeout='1s'` first, fail fast, retry |
+| One giant backfill `UPDATE` | Locks rows, bloats WAL, spikes replica lag | bounded batches, commit each, throttle on lag (Step 5) |
+| Renaming a column the running app still reads | Instant errors for app version N | dual-name window: add new, dual-write, switch reads, then drop |
+| Contracting before the cooling period | Kills your no-schema-change rollback path | wait until the new app has run clean; then drop |
+| Relying on `down()` in prod | Data already changed; rollback is unsafe or wrong | forward-only; expand-contract IS the rollback (Step 6) |
+| Coupling app + schema in one deploy | Breaks N/N-1; an instant where some app can't run | separate deploys; schema is a superset of both (Step 3) |
+
+## References
+
+- `references/expand-contract-playbook.md` — ordered deploy/migrate checklists per change (NOT NULL, rename, split, type change, drop, add FK).
+- `references/backfill-and-batching.md` — batched backfill scripts (Postgres + MySQL), replica-lag throttle, checksum verification, snapshot+CDC for huge tables.
+- `references/tools-and-runners.md` — 2026 tool matrix, gh-ost vs pt-osc, per-runner transaction opt-out, Squawk CI config.

package/skills/db-migrations/evals/README.md

@@ -0,0 +1,10 @@
+# Evals — db-migrations
+
+`cases.yaml` specifies routing and capability expectations, not live database runs. `should_trigger` /
+`should_not_trigger` are prompt-routing checks: feed each prompt to the router and confirm db-migrations
+fires (or that the named sibling — postgresdb, drizzle-orm, backups, sql, planetscale — wins instead).
+The `capability` case is a rubric: ask the model to plan the column rename, then score the response
+against the `must_include` bullets (each phase present, in order, with lock_timeout, batched/throttled
+backfill, checksum verify, N/N-1, cooling period, forward-only). There is no database connection here;
+the DDL-shape and self-consistency check lives in `../scripts/verify.sh`. Run these through the repo's
+standard eval harness.

package/skills/db-migrations/evals/cases.yaml

@@ -0,0 +1,46 @@
+skill: db-migrations
+
+should_trigger:
+  - prompt: "We need to add a NOT NULL column to a 40M-row users table without taking the app down. How?"
+    why: Core expand-contract — add-nullable, backfill, NOT VALID/VALIDATE, on a live large table.
+  - prompt: "Rename a heavily-written column on a live Postgres table without downtime."
+    why: Dual-name window with dual-write + batched backfill; the canonical sequencing case.
+  - prompt: "Our deploy keeps deadlocking every time the migration adds an index. What are we doing wrong?"
+    why: Non-obvious — symptom names a deadlock, not a migration; the fix is CONCURRENTLY + lock_timeout.
+  - prompt: "How do we backfill 80M rows into a new column without blowing up replication lag?"
+    why: Batched backfill with lag throttle and verify-before-contract; not query logic.
+  - prompt: "Necesito hacer una migración sin downtime para cambiar el tipo de una columna en producción."
+    why: Spanish phrasing for a type change on a live table — new column + backfill expand-contract.
+  - prompt: "Big ALTER on a busy MySQL table — should we use gh-ost or pt-osc?"
+    why: Non-obvious OSC-tool choice; engine-neutral choreography layer owns this decision.
+
+should_not_trigger:
+  - prompt: "This query is slow — can you read the EXPLAIN ANALYZE plan and tell me which index to add?"
+    route_to: postgresdb
+    why: Query plan reading and Postgres indexing strategy are postgresdb internals, not migration sequencing.
+  - prompt: "How do I generate and run a migration with drizzle-kit?"
+    route_to: drizzle-orm
+    why: That is the runner's CLI mechanics; the safety sequence is here but the tool usage lives in drizzle-orm.
+  - prompt: "Set up a PITR restore drill and verify our nightly backups actually restore."
+    route_to: backups
+    why: Creating and verifying backups/restores is backups; a migration only requires one as a precondition.
+  - prompt: "Write a window-function query to rank orders per customer by date."
+    route_to: sql
+    why: Pure query logic (window functions) is sql, not schema-change choreography.
+  - prompt: "Cut over a PlanetScale branch and merge the deploy request."
+    route_to: planetscale
+    why: Vitess online DDL and deploy-request cutover mechanics belong to planetscale.
+
+capability:
+  - scenario: >
+      Plan, end to end, a zero-downtime rename of a heavily-written column users.email →
+      users.email_address on Postgres, across app deploys and migrations.
+    must_include:
+      - "Expand: add email_address as a NULLABLE column, metadata-only, with lock_timeout set first"
+      - "App deploy that dual-writes both email and email_address while still reading email"
+      - "Batched backfill (bounded PK-range chunks, commit per batch) with replica-lag throttle"
+      - "Verify counts and old-vs-new checksum before contracting"
+      - "N/N-1 note: schema must satisfy both the running app and the next app at every step"
+      - "Switch reads to email_address in a separate deploy, then a cooling period"
+      - "Contract: DROP COLUMN email as a separate migration AFTER the cooling period"
+      - "Forward-only stance: rollback is redeploying the prior app, not a down() script"

package/skills/db-migrations/references/backfill-and-batching.md

@@ -0,0 +1,105 @@
+# Backfill and batching
+
+Filling historical rows is where a "zero-downtime" plan most often turns into an outage: one giant
+`UPDATE` locks rows, balloons WAL/undo, and pins replicas behind minutes of lag. The fix is always the
+same shape — **bounded chunks, commit per chunk, throttle on lag, verify before you contract.**
+
+## The loop, in words
+
+1. Pick a chunk key. A monotonic PK is best; range over `id BETWEEN :lo AND :hi`, not `OFFSET`
+   (OFFSET re-scans and gets slower every batch).
+2. Update one chunk, touching only rows that still need it (`WHERE new_col IS NULL`), then **commit**.
+3. Read replica lag and the app error rate. If either climbs past your threshold, sleep / back off.
+4. Advance the window and repeat until the table is covered.
+5. Verify counts and a checksum of old-vs-new. Only then schedule the contract step.
+
+A few thousand rows per batch is a sane start; tune by watching lag. Smaller batches = lower lag, more
+total wall-clock time.
+
+## Postgres — server-side batched backfill
+
+```sql
+-- Run once per batch from your runner/script, advancing :lo/:hi between calls.
+-- Touch only unfilled rows so re-runs are cheap and the loop is idempotent.
+UPDATE users
+SET    email_address = email
+WHERE  id BETWEEN :lo AND :hi
+  AND  email_address IS NULL;
+```
+
+```sql
+-- A self-contained PL/pgSQL loop that commits each batch (PG 11+ supports COMMIT in a procedure).
+-- Run as a PROCEDURE via CALL, not inside an outer transaction.
+CREATE OR REPLACE PROCEDURE backfill_email_address(batch_size int DEFAULT 5000)
+LANGUAGE plpgsql AS $$
+DECLARE
+  lo bigint := 0;
+  hi bigint;
+  max_id bigint;
+BEGIN
+  SELECT max(id) INTO max_id FROM users;
+  WHILE lo <= max_id LOOP
+    hi := lo + batch_size;
+    UPDATE users
+    SET    email_address = email
+    WHERE  id >= lo AND id < hi
+      AND  email_address IS NULL;
+    COMMIT;                       -- release locks + flush WAL each batch
+    PERFORM pg_sleep(0.1);        -- throttle; raise if replica lag climbs
+    lo := hi;
+  END LOOP;
+END;
+$$;
+```
+
+Throttle by replica lag instead of a fixed sleep when you can — query
+`SELECT max(replay_lag) FROM pg_stat_replication;` between batches and back off while it is high.
+(For the meaning of those replication views, see `../../postgresdb/SKILL.md`.)
+
+## MySQL — batched backfill
+
+MySQL has no in-loop `COMMIT` inside a single statement, so drive the loop from your script and run one
+bounded `UPDATE` per iteration with `autocommit` on:
+
+```sql
+-- One batch; the script advances @lo/@hi and sleeps between iterations.
+UPDATE users
+SET    email_address = email
+WHERE  id >= @lo AND id < @hi
+  AND  email_address IS NULL
+LIMIT  5000;
+```
+
+Watch `SHOW REPLICA STATUS` → `Seconds_Behind_Source` between batches; pause while it grows. For a
+single very large `ALTER` (not a data backfill), use an online-schema-change tool instead — see
+`tools-and-runners.md` for gh-ost vs pt-osc.
+
+## Verify before you contract
+
+Counts and a checksum must agree before any column is dropped or marked `NOT NULL`.
+
+```sql
+-- 1. No rows left unfilled.
+SELECT count(*) AS remaining FROM users WHERE email_address IS NULL;  -- expect 0
+
+-- 2. Old and new agree, row for row.
+SELECT count(*) AS mismatches
+FROM   users
+WHERE  email_address IS DISTINCT FROM email;                          -- expect 0
+```
+
+A non-zero mismatch is a data-correctness bug, not a migration step — investigate the dual-write path
+before contracting. If the divergence is in the *source* data (bad/duplicate rows), that is
+`../../data-cleaning/SKILL.md` territory.
+
+## Very large tables — snapshot + CDC
+
+When an online loop would run for days, do it out of band:
+
+1. Snapshot the table into the new shape (a copy job or a logical dump).
+2. Stream changes that happened during the snapshot via change-data-capture (logical replication / binlog
+   readers) so the copy catches up to live.
+3. Cut over reads once the copy is caught up and verified.
+
+This is the same idea gh-ost and pt-osc automate for a single MySQL `ALTER`; for a multi-table or
+cross-engine move you build it explicitly. Either way the verify-before-contract rule does not change.