vanara 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/README.md +14 -0
  2. package/bin/vanara.js +66 -2
  3. package/catalog/agents/api-designer/AGENT.md +140 -0
  4. package/catalog/agents/api-designer/examples/openapi-snippet.yaml +130 -0
  5. package/catalog/agents/api-designer/examples/review-notes.md +61 -0
  6. package/catalog/agents/api-designer/references/contract-and-openapi.md +82 -0
  7. package/catalog/agents/api-designer/references/design-checklist.md +68 -0
  8. package/catalog/agents/api-designer/references/versioning-and-evolution.md +63 -0
  9. package/catalog/agents/api-designer/scripts/lint-openapi.mjs +180 -0
  10. package/catalog/agents/code-reviewer/AGENT.md +168 -0
  11. package/catalog/agents/code-reviewer/examples/pr-comment-template.md +48 -0
  12. package/catalog/agents/code-reviewer/examples/sample-review-output.md +38 -0
  13. package/catalog/agents/code-reviewer/references/review-checklist.md +46 -0
  14. package/catalog/agents/code-reviewer/references/security-review.md +62 -0
  15. package/catalog/agents/code-reviewer/references/severity-rubric.md +63 -0
  16. package/catalog/agents/code-reviewer/scripts/review-guard.mjs +102 -0
  17. package/catalog/agents/debugger/AGENT.md +112 -0
  18. package/catalog/agents/debugger/examples/postmortem-template.md +59 -0
  19. package/catalog/agents/debugger/examples/repro-template.md +50 -0
  20. package/catalog/agents/debugger/references/bisection.md +63 -0
  21. package/catalog/agents/debugger/references/debugging-method.md +57 -0
  22. package/catalog/agents/debugger/references/observability.md +61 -0
  23. package/catalog/agents/debugger/scripts/parse-stacktrace.mjs +109 -0
  24. package/catalog/agents/pr-summarizer/AGENT.md +156 -0
  25. package/catalog/agents/pr-summarizer/examples/sample-pr-summary.md +46 -0
  26. package/catalog/agents/pr-summarizer/examples/weak-vs-strong-summary.md +68 -0
  27. package/catalog/agents/pr-summarizer/references/identifying-risk.md +68 -0
  28. package/catalog/agents/pr-summarizer/references/reading-the-diff.md +68 -0
  29. package/catalog/agents/pr-summarizer/references/test-plan-and-missing-tests.md +64 -0
  30. package/catalog/agents/pr-summarizer/references/writing-the-summary.md +63 -0
  31. package/catalog/agents/pr-summarizer/scripts/diff-risk.mjs +111 -0
  32. package/catalog/agents/refactoring-specialist/AGENT.md +185 -0
  33. package/catalog/agents/refactoring-specialist/examples/before-after.md +91 -0
  34. package/catalog/agents/refactoring-specialist/examples/refactor-plan.md +97 -0
  35. package/catalog/agents/refactoring-specialist/references/large-scale-refactor.md +50 -0
  36. package/catalog/agents/refactoring-specialist/references/safe-workflow.md +75 -0
  37. package/catalog/agents/refactoring-specialist/references/smell-catalog.md +76 -0
  38. package/catalog/agents/refactoring-specialist/scripts/detect-smells.mjs +150 -0
  39. package/catalog/agents/security-auditor/AGENT.md +168 -0
  40. package/catalog/agents/security-auditor/examples/audit-report.md +68 -0
  41. package/catalog/agents/security-auditor/examples/finding-template.md +31 -0
  42. package/catalog/agents/security-auditor/references/audit-workflow.md +116 -0
  43. package/catalog/agents/security-auditor/references/severity-and-reporting.md +56 -0
  44. package/catalog/agents/security-auditor/references/vuln-classes.md +129 -0
  45. package/catalog/agents/security-auditor/scripts/scan-secrets.mjs +86 -0
  46. package/catalog/agents/technical-writer/AGENT.md +163 -0
  47. package/catalog/agents/technical-writer/examples/how-to-example.md +76 -0
  48. package/catalog/agents/technical-writer/examples/reference-example.md +75 -0
  49. package/catalog/agents/technical-writer/references/clarity-and-style.md +52 -0
  50. package/catalog/agents/technical-writer/references/diataxis-and-doc-types.md +61 -0
  51. package/catalog/agents/technical-writer/references/structure-and-maintenance.md +59 -0
  52. package/catalog/agents/technical-writer/scripts/readability.mjs +103 -0
  53. package/catalog/agents/test-author/AGENT.md +177 -0
  54. package/catalog/agents/test-author/examples/test-plan.md +54 -0
  55. package/catalog/agents/test-author/examples/unit-test-example.ts +55 -0
  56. package/catalog/agents/test-author/references/aaa-and-naming.md +75 -0
  57. package/catalog/agents/test-author/references/mocking-and-fixtures.md +94 -0
  58. package/catalog/agents/test-author/references/test-types-and-pyramid.md +60 -0
  59. package/catalog/agents/test-author/scripts/check-coverage.mjs +145 -0
  60. package/catalog/agents/threat-modeler/AGENT.md +163 -0
  61. package/catalog/agents/threat-modeler/examples/dfd-example.md +55 -0
  62. package/catalog/agents/threat-modeler/examples/threat-model.md +81 -0
  63. package/catalog/agents/threat-modeler/references/dfd-and-trust-boundaries.md +76 -0
  64. package/catalog/agents/threat-modeler/references/mitigation-catalog.md +77 -0
  65. package/catalog/agents/threat-modeler/references/stride-method.md +68 -0
  66. package/catalog/agents/threat-modeler/scripts/stride-checklist.mjs +183 -0
  67. package/catalog/agents/vuln-scanner/AGENT.md +146 -0
  68. package/catalog/agents/vuln-scanner/examples/finding-template.md +57 -0
  69. package/catalog/agents/vuln-scanner/examples/scan-report.md +49 -0
  70. package/catalog/agents/vuln-scanner/references/remediation-and-severity.md +68 -0
  71. package/catalog/agents/vuln-scanner/references/scan-types-and-tools.md +83 -0
  72. package/catalog/agents/vuln-scanner/references/triage-and-false-positives.md +73 -0
  73. package/catalog/agents/vuln-scanner/scripts/parse-scan-results.mjs +133 -0
  74. package/catalog/packs/code-review-pack/PACK.md +100 -0
  75. package/catalog/packs/code-review-pack/examples/pr-flow-example.md +50 -0
  76. package/catalog/packs/code-review-pack/references/rollout-guide.md +40 -0
  77. package/catalog/packs/security-pack/PACK.md +128 -0
  78. package/catalog/packs/security-pack/examples/owasp-vuln-triage-and-fix.md +66 -0
  79. package/catalog/packs/security-pack/examples/threat-model-then-audit.md +77 -0
  80. package/catalog/packs/security-pack/references/choosing-the-right-tool.md +37 -0
  81. package/catalog/packs/security-pack/references/threat-model-to-remediation.md +69 -0
  82. package/catalog/packs/security-pack/references/workflow.md +56 -0
  83. package/catalog/skills/api-pagination/SKILL.md +83 -0
  84. package/catalog/skills/api-pagination/examples/keyset-queries.sql +38 -0
  85. package/catalog/skills/api-pagination/examples/response-shapes.md +51 -0
  86. package/catalog/skills/api-pagination/references/consuming-pages.md +39 -0
  87. package/catalog/skills/api-pagination/references/counting-and-totals.md +41 -0
  88. package/catalog/skills/api-pagination/references/cursor-encoding.md +34 -0
  89. package/catalog/skills/api-pagination/scripts/check-cursor.mjs +51 -0
  90. package/catalog/skills/caching-strategies/SKILL.md +180 -0
  91. package/catalog/skills/caching-strategies/examples/http-cache-headers.md +82 -0
  92. package/catalog/skills/caching-strategies/examples/redis-cache-aside.js +110 -0
  93. package/catalog/skills/caching-strategies/references/cache-patterns.md +93 -0
  94. package/catalog/skills/caching-strategies/references/eviction-and-ttl.md +67 -0
  95. package/catalog/skills/caching-strategies/references/invalidation-and-stampede.md +99 -0
  96. package/catalog/skills/caching-strategies/scripts/ttl-jitter.mjs +99 -0
  97. package/catalog/skills/conventional-commits/SKILL.md +135 -0
  98. package/catalog/skills/conventional-commits/examples/commit-examples.md +60 -0
  99. package/catalog/skills/conventional-commits/examples/commitlint.config.js +31 -0
  100. package/catalog/skills/conventional-commits/references/breaking-changes-semver.md +50 -0
  101. package/catalog/skills/conventional-commits/references/scoping-commits.md +42 -0
  102. package/catalog/skills/conventional-commits/references/spec.md +78 -0
  103. package/catalog/skills/conventional-commits/scripts/lint-commit.mjs +97 -0
  104. package/catalog/skills/database-migrations/SKILL.md +137 -0
  105. package/catalog/skills/database-migrations/examples/add-column-safe.sql +34 -0
  106. package/catalog/skills/database-migrations/examples/migration-config.example.json +27 -0
  107. package/catalog/skills/database-migrations/references/expand-contract.md +74 -0
  108. package/catalog/skills/database-migrations/references/rollback-and-safety.md +71 -0
  109. package/catalog/skills/database-migrations/references/zero-downtime-changes.md +84 -0
  110. package/catalog/skills/database-migrations/scripts/check-migration-reversible.mjs +139 -0
  111. package/catalog/skills/error-handling-patterns/SKILL.md +136 -0
  112. package/catalog/skills/error-handling-patterns/examples/express-error-middleware.js +93 -0
  113. package/catalog/skills/error-handling-patterns/examples/result-pattern.ts +81 -0
  114. package/catalog/skills/error-handling-patterns/references/error-taxonomy.md +91 -0
  115. package/catalog/skills/error-handling-patterns/references/language-patterns.md +102 -0
  116. package/catalog/skills/error-handling-patterns/references/retry-and-backoff.md +76 -0
  117. package/catalog/skills/error-handling-patterns/scripts/lint-empty-catch.mjs +113 -0
  118. package/catalog/skills/git-collaboration-workflows/SKILL.md +104 -0
  119. package/catalog/skills/git-collaboration-workflows/examples/branch-protection.md +48 -0
  120. package/catalog/skills/git-collaboration-workflows/examples/pr-review-playbook.md +48 -0
  121. package/catalog/skills/git-collaboration-workflows/references/branching-models.md +52 -0
  122. package/catalog/skills/git-collaboration-workflows/references/history-hygiene.md +44 -0
  123. package/catalog/skills/git-collaboration-workflows/scripts/check-branch-hygiene.mjs +62 -0
  124. package/catalog/skills/owasp-top10/SKILL.md +112 -0
  125. package/catalog/skills/owasp-top10/examples/sql-injection-fix.md +33 -0
  126. package/catalog/skills/owasp-top10/examples/xss-fix.md +27 -0
  127. package/catalog/skills/owasp-top10/references/access-control.md +36 -0
  128. package/catalog/skills/owasp-top10/references/crypto-failures.md +29 -0
  129. package/catalog/skills/owasp-top10/references/injection.md +39 -0
  130. package/catalog/skills/owasp-top10/references/ssrf-and-design.md +35 -0
  131. package/catalog/skills/owasp-top10/scripts/scan-injection.mjs +73 -0
  132. package/catalog/skills/prompt-engineering/SKILL.md +100 -0
  133. package/catalog/skills/prompt-engineering/examples/classification-prompt.md +29 -0
  134. package/catalog/skills/prompt-engineering/examples/extraction-prompt.md +25 -0
  135. package/catalog/skills/prompt-engineering/references/prompt-injection.md +34 -0
  136. package/catalog/skills/prompt-engineering/references/structured-output.md +34 -0
  137. package/catalog/skills/prompt-engineering/references/techniques.md +33 -0
  138. package/catalog/skills/prompt-engineering/scripts/validate-output.mjs +71 -0
  139. package/catalog/skills/readme-writing/SKILL.md +176 -0
  140. package/catalog/skills/readme-writing/examples/README.template.md +73 -0
  141. package/catalog/skills/readme-writing/examples/good-readme-example.md +95 -0
  142. package/catalog/skills/readme-writing/references/anatomy.md +107 -0
  143. package/catalog/skills/readme-writing/references/maintenance.md +68 -0
  144. package/catalog/skills/readme-writing/references/writing-style.md +79 -0
  145. package/catalog/skills/readme-writing/scripts/lint-readme.mjs +117 -0
  146. package/catalog/skills/refactoring-patterns/SKILL.md +178 -0
  147. package/catalog/skills/refactoring-patterns/examples/extract-function.before-after.js +57 -0
  148. package/catalog/skills/refactoring-patterns/examples/replace-conditional-with-polymorphism.before-after.ts +63 -0
  149. package/catalog/skills/refactoring-patterns/references/code-smells.md +92 -0
  150. package/catalog/skills/refactoring-patterns/references/refactoring-catalog.md +78 -0
  151. package/catalog/skills/refactoring-patterns/references/safe-workflow.md +74 -0
  152. package/catalog/skills/refactoring-patterns/scripts/detect-smells.mjs +137 -0
  153. package/catalog/skills/rest-api-design/SKILL.md +147 -0
  154. package/catalog/skills/rest-api-design/examples/error-envelope.json +26 -0
  155. package/catalog/skills/rest-api-design/examples/orders-api.openapi.yaml +106 -0
  156. package/catalog/skills/rest-api-design/references/error-handling.md +43 -0
  157. package/catalog/skills/rest-api-design/references/pagination.md +41 -0
  158. package/catalog/skills/rest-api-design/references/status-codes.md +32 -0
  159. package/catalog/skills/rest-api-design/references/versioning.md +31 -0
  160. package/catalog/skills/rest-api-design/scripts/check-envelope.mjs +62 -0
  161. package/catalog/skills/secrets-management/SKILL.md +134 -0
  162. package/catalog/skills/secrets-management/examples/env-example.md +41 -0
  163. package/catalog/skills/secrets-management/examples/rotation-runbook.md +50 -0
  164. package/catalog/skills/secrets-management/references/leak-response.md +47 -0
  165. package/catalog/skills/secrets-management/references/rotation.md +46 -0
  166. package/catalog/skills/secrets-management/references/secret-managers.md +52 -0
  167. package/catalog/skills/secrets-management/scripts/detect-hardcoded.mjs +96 -0
  168. package/catalog/skills/secure-auth/SKILL.md +135 -0
  169. package/catalog/skills/secure-auth/examples/auth-flow.md +60 -0
  170. package/catalog/skills/secure-auth/examples/jwt-verify.ts +89 -0
  171. package/catalog/skills/secure-auth/references/auth-attacks.md +76 -0
  172. package/catalog/skills/secure-auth/references/oauth2-oidc.md +65 -0
  173. package/catalog/skills/secure-auth/references/password-hashing.md +70 -0
  174. package/catalog/skills/secure-auth/references/sessions-vs-jwt.md +69 -0
  175. package/catalog/skills/secure-auth/scripts/check-password-policy.mjs +114 -0
  176. package/catalog/skills/sql-index-tuning/SKILL.md +171 -0
  177. package/catalog/skills/sql-index-tuning/examples/explain-walkthrough.sql +91 -0
  178. package/catalog/skills/sql-index-tuning/examples/index-ddl.sql +67 -0
  179. package/catalog/skills/sql-index-tuning/references/btree-internals.md +58 -0
  180. package/catalog/skills/sql-index-tuning/references/composite-and-covering.md +60 -0
  181. package/catalog/skills/sql-index-tuning/references/reading-explain.md +54 -0
  182. package/catalog/skills/sql-index-tuning/references/selectivity-and-cardinality.md +62 -0
  183. package/catalog/skills/sql-index-tuning/scripts/suggest-index.mjs +213 -0
  184. package/catalog/skills/test-plan-design/SKILL.md +154 -0
  185. package/catalog/skills/test-plan-design/examples/test-case-template.md +45 -0
  186. package/catalog/skills/test-plan-design/examples/test-plan.md +57 -0
  187. package/catalog/skills/test-plan-design/references/case-design.md +79 -0
  188. package/catalog/skills/test-plan-design/references/risk-prioritization.md +58 -0
  189. package/catalog/skills/test-plan-design/references/test-pyramid.md +58 -0
  190. package/catalog/skills/test-plan-design/scripts/coverage-gaps.mjs +109 -0
  191. package/free-tier.json +15 -0
  192. package/package.json +15 -4
  193. package/premium-index.json +1 -0
  194. package/src/config.js +31 -1
  195. package/src/license.js +143 -0
@@ -0,0 +1,137 @@
1
+ ---
2
+ name: database-migrations
3
+ description: How to write safe, reversible, zero-downtime database schema migrations — additive-first changes, the expand/migrate/contract pattern, batched backfills, concurrent index builds, safe NOT NULL, rollbacks, and the locking pitfalls that cause outages. A deep reference with runnable checks.
4
+ type: skill
5
+ version: 2.0.0
6
+ updated: 2026-06-29
7
+ ---
8
+ # Database Migrations
9
+
10
+ A migration runs against **live data while old code may still be serving traffic**. The dangerous moment
11
+ is never the steady state before or after — it's the in-between, when the schema has changed but not every
12
+ app instance has. Design every change to be correct *during* that window. Heavy detail lives in
13
+ `references/`; copy-paste material in `examples/`; a runnable safety check in `scripts/`.
14
+
15
+ ## Mental model
16
+
17
+ Two things deploy on different clocks: your **schema** (one atomic change) and your **code** (rolled out
18
+ instance-by-instance over minutes). A migration is safe only if **both the old and new code work against
19
+ both the old and new schema** for the overlap window. That single rule explains almost every practice
20
+ below.
21
+
22
+ | Concern | Safe answer |
23
+ |---|---|
24
+ | What changed | the smallest possible step |
25
+ | When old code sees it | it must still work (backward-compatible) |
26
+ | Locks held | none long enough to block traffic |
27
+ | If it goes wrong | a tested, reversible path back |
28
+ | Big rename/retype | expand → migrate → contract, across deploys |
29
+
30
+ ## 1. Additive-first
31
+
32
+ Prefer **additive, backward-compatible** changes. Adding a nullable column, adding a table, or adding an
33
+ index never breaks code that doesn't know about it. Destructive changes (drop/rename column, change type,
34
+ add `NOT NULL`) break the old code still running mid-deploy, so they must be sequenced — see §3.
35
+
36
+ ```sql
37
+ -- SAFE: old code ignores the new column; new code can start using it.
38
+ ALTER TABLE users ADD COLUMN email_verified_at timestamptz NULL;
39
+
40
+ -- UNSAFE in one step: old code still INSERTs rows without this column.
41
+ ALTER TABLE users ADD COLUMN email_verified_at timestamptz NOT NULL;
42
+ ```
43
+
44
+ ## 2. Never hold a long lock
45
+
46
+ Most outages from migrations are **lock waits**, not data loss. A statement that rewrites a table or takes
47
+ an `ACCESS EXCLUSIVE` lock blocks every read/write behind it, and that queue backs up into your connection
48
+ pool within seconds.
49
+
50
+ - Add columns as **nullable** (or with a constant default — on modern Postgres a constant default is
51
+ metadata-only and does not rewrite the table; a *volatile* default does).
52
+ - Build indexes with `CREATE INDEX CONCURRENTLY` (no table rewrite, no write lock — but it can't run
53
+ inside a transaction).
54
+ - **Backfill in batches** with short transactions, not one giant `UPDATE` that locks every row.
55
+ - Set a `lock_timeout` so a migration that *can't* get its lock fails fast instead of stalling traffic.
56
+
57
+ ```sql
58
+ SET lock_timeout = '3s'; -- fail fast rather than queue behind traffic
59
+ CREATE INDEX CONCURRENTLY idx_orders_status ON orders (status);
60
+ ```
61
+
62
+ Full catalogue of which operations rewrite/lock and the safe alternative: `references/zero-downtime-changes.md`.
63
+
64
+ ## 3. Expand / migrate / contract
65
+
66
+ The core pattern for any breaking change (rename, retype, split, add `NOT NULL`) without downtime. It
67
+ spreads one logical change across **multiple deploys** so old and new code always overlap safely:
68
+
69
+ 1. **Expand** — add the new shape (nullable column / new table / new index). Backward-compatible.
70
+ 2. **Migrate** — deploy code that **dual-writes** old+new, then **backfill** existing rows in batches,
71
+ then switch **reads** to the new shape and verify.
72
+ 3. **Contract** — once nothing reads or writes the old shape, drop it in a *later* deploy.
73
+
74
+ Each step is independently deployable and independently reversible. The full worked walkthrough
75
+ (including a column rename and a non-null-ification) is in `references/expand-contract.md`.
76
+
77
+ ## 4. Reversibility & safety
78
+
79
+ Every migration should declare how to undo it. Some operations are **irreversible** in practice
80
+ (`DROP COLUMN`, `DROP TABLE`, `TRUNCATE` destroy data; a down-migration can recreate the *structure* but
81
+ not the *data*). Treat those specially:
82
+
83
+ - Provide a real `down` for reversible ops; for irreversible ones, document that recovery is **restore from
84
+ backup**, and take a verified backup/snapshot immediately before.
85
+ - Separate the destructive `contract` step into its own migration so you can ship the safe parts and pause.
86
+ - Dry-run on a production-like copy and **time it** — a 30-minute backfill on staging may be hours in prod.
87
+
88
+ Reversible-vs-irreversible rules, backup checklists, and dry-run guidance: `references/rollback-and-safety.md`.
89
+ Validate a `.sql` file with `scripts/check-migration-reversible.mjs` before shipping.
90
+
91
+ ## 5. Operational discipline
92
+
93
+ - **One concern per migration.** Mixing a schema change and a data backfill in one file makes rollback
94
+ ambiguous and the transaction huge.
95
+ - **Forward-only in production.** Prefer rolling *forward* with a fix over rolling back a migration that
96
+ already ran against live data; keep `down` for local/staging and emergencies.
97
+ - **Idempotent where possible** (`IF NOT EXISTS`, `IF EXISTS`) so a retried run after a partial failure
98
+ doesn't error.
99
+ - **Backfills run outside the schema transaction**, in batches with a sleep, so they don't hold locks or
100
+ bloat WAL/replication lag.
101
+
102
+ ## Common pitfalls (failure modes)
103
+
104
+ - **`NOT NULL` in one shot** — adds a column the still-running old code inserts NULLs into, *or* triggers a
105
+ full validating scan. Add nullable, backfill, then `SET NOT NULL` (validate via a `CHECK ... NOT VALID`
106
+ then `VALIDATE`). See the zero-downtime reference.
107
+ - **Plain `CREATE INDEX`** on a hot table — takes a write lock for the whole build. Use `CONCURRENTLY`.
108
+ - **One giant `UPDATE` backfill** — locks millions of rows, blocks writes, balloons WAL. Batch it.
109
+ - **Rename a column in one deploy** — old code references the old name and 500s during the rollout window.
110
+ Use expand/contract.
111
+ - **Volatile/changing default on add-column** — forces a full table rewrite under an exclusive lock.
112
+ - **No `lock_timeout`** — the migration silently queues behind a long transaction and takes the app down.
113
+ - **Destructive `down` that "reverses" a drop** — it recreates an empty column; the data is gone. Mark it
114
+ irreversible and rely on backups.
115
+
116
+ ## When NOT to use / trade-offs
117
+
118
+ - **Tiny project, maintenance window available.** If you can take 5 minutes of downtime at 3am and the
119
+ table is small, a single blocking `ALTER` is simpler and cheaper than a three-deploy expand/contract.
120
+ Zero-downtime machinery is overhead you only need at scale or with strict SLAs.
121
+ - **Throwaway / pre-launch schemas** with no real data — just recreate the schema; migrations add ceremony
122
+ you don't need yet.
123
+ - **NoSQL / schemaless stores** shift the work to the application (versioned documents, read-time
124
+ migration) rather than DDL — the expand/contract *idea* still applies, the SQL specifics don't.
125
+ - **Data migrations vs schema migrations** are different beasts; heavy data reshaping often belongs in a
126
+ background job, not a migration runner that blocks deploys.
127
+
128
+ ## Files in this package
129
+
130
+ - `references/expand-contract.md` — the expand/migrate/contract pattern, worked end-to-end
131
+ - `references/zero-downtime-changes.md` — per-operation safe recipes (add column, backfill, index, NOT NULL)
132
+ - `references/rollback-and-safety.md` — reversible vs irreversible ops, backups, dry-runs
133
+ - `examples/add-column-safe.sql` — a safe, batched, reversible migration
134
+ - `examples/migration-config.example.json` — a migration runner config with safety guardrails
135
+ - `scripts/check-migration-reversible.mjs` — Node check that flags irreversible SQL + missing down-migrations
136
+
137
+ Pairs with the `database-scaling` skill, the `data-modeling` skill, and the `database-reviewer` agent.
@@ -0,0 +1,34 @@
1
+ -- Safe, batched, reversible migration: add orders.priority with a backfill.
2
+ -- Demonstrates: fail-fast lock, additive nullable add, batched backfill outside
3
+ -- the schema transaction, NOT NULL via validated CHECK, and a real down-migration.
4
+ --
5
+ -- == UP =====================================================================
6
+
7
+ -- Fail fast instead of queueing behind live traffic.
8
+ SET lock_timeout = '3s';
9
+
10
+ -- Step 1 (Expand): additive, nullable, metadata-only. Old code is unaffected.
11
+ ALTER TABLE orders ADD COLUMN IF NOT EXISTS priority int NULL;
12
+
13
+ -- Step 2 (Migrate): backfill existing rows in bounded batches.
14
+ -- Run this loop in the app/runner, NOT in the schema transaction, committing
15
+ -- between batches. Repeat until 0 rows are affected. WHERE ... IS NULL makes it
16
+ -- idempotent and resumable after a partial failure.
17
+ -- UPDATE orders
18
+ -- SET priority = 0
19
+ -- WHERE id IN (
20
+ -- SELECT id FROM orders WHERE priority IS NULL ORDER BY id LIMIT 5000
21
+ -- );
22
+
23
+ -- Step 3: enforce NOT NULL without a blocking validating scan.
24
+ ALTER TABLE orders
25
+ ADD CONSTRAINT orders_priority_nn CHECK (priority IS NOT NULL) NOT VALID;
26
+ -- After the backfill confirms no NULLs remain:
27
+ ALTER TABLE orders VALIDATE CONSTRAINT orders_priority_nn;
28
+
29
+ -- == DOWN ===================================================================
30
+ -- Reversible: the column was newly added and the constraint is ours to drop.
31
+ -- No data loss, because nothing depended on this column before this migration.
32
+
33
+ ALTER TABLE orders DROP CONSTRAINT IF EXISTS orders_priority_nn;
34
+ ALTER TABLE orders DROP COLUMN IF EXISTS priority;
@@ -0,0 +1,27 @@
1
+ {
2
+ "$comment": "Example migration-runner config with safety guardrails. No secrets: the connection comes from an env var, never hardcoded.",
3
+ "databaseUrlEnv": "DATABASE_URL",
4
+ "migrationsDir": "./migrations",
5
+ "migrationsTable": "schema_migrations",
6
+ "direction": "up",
7
+ "safety": {
8
+ "lockTimeout": "3s",
9
+ "statementTimeout": "0",
10
+ "requireDownMigration": true,
11
+ "transactionPerMigration": true,
12
+ "disableTransactionForConcurrent": true,
13
+ "blockIrreversibleWithoutAck": true,
14
+ "irreversibleStatements": ["DROP COLUMN", "DROP TABLE", "TRUNCATE"]
15
+ },
16
+ "backfill": {
17
+ "runOutsideSchemaTransaction": true,
18
+ "batchSize": 5000,
19
+ "sleepMsBetweenBatches": 100,
20
+ "resumeWhere": "col IS NULL"
21
+ },
22
+ "verification": {
23
+ "dryRunOnCopy": true,
24
+ "requireBackupBeforeDestructive": true,
25
+ "checkScript": "node ../scripts/check-migration-reversible.mjs"
26
+ }
27
+ }
@@ -0,0 +1,74 @@
1
+ # Expand / Migrate / Contract
2
+
3
+ The pattern for making a breaking schema change with **zero downtime**. The insight: you cannot atomically
4
+ change schema *and* code together, so you split one logical change into a sequence of backward-compatible
5
+ deploys where old and new code always coexist safely.
6
+
7
+ ## The three phases
8
+
9
+ 1. **Expand** — additively introduce the new shape. The schema now supports *both* old and new. Old code
10
+ is untouched; new code can begin targeting the new shape. This phase is always backward-compatible.
11
+ 2. **Migrate** — get all data and all traffic onto the new shape:
12
+ - deploy code that **dual-writes** (writes both old and new),
13
+ - **backfill** existing rows in batches,
14
+ - switch **reads** to the new shape and verify metrics/parity,
15
+ - stop writing the old shape.
16
+ 3. **Contract** — once nothing reads or writes the old shape, **remove it** in a later, separate deploy.
17
+
18
+ Each step is independently shippable and reversible until the final destructive contract.
19
+
20
+ ## Worked example: rename `users.email` → `users.email_address`
21
+
22
+ A column rename in one statement breaks every still-running old instance the moment it lands. Spread it out:
23
+
24
+ ### Deploy 1 — Expand
25
+ ```sql
26
+ ALTER TABLE users ADD COLUMN email_address text NULL;
27
+ ```
28
+ Old code ignores the new column; nothing breaks.
29
+
30
+ ### Deploy 2 — Migrate
31
+ App code now writes **both** columns on every insert/update (dual-write). Backfill the old rows in batches
32
+ (see `zero-downtime-changes.md` for the batching loop):
33
+ ```sql
34
+ UPDATE users
35
+ SET email_address = email
36
+ WHERE email_address IS NULL
37
+ AND id BETWEEN :lo AND :hi; -- repeat across id ranges
38
+ ```
39
+ Then switch reads to `email_address`, verify parity (counts match, no NULLs where there shouldn't be), and
40
+ stop writing `email`.
41
+
42
+ ### Deploy 3 — Contract
43
+ Only after every instance has stopped using `email`:
44
+ ```sql
45
+ ALTER TABLE users DROP COLUMN email;
46
+ ```
47
+ This is the one irreversible step — gate it behind a verified backup.
48
+
49
+ ## Worked example: making a column `NOT NULL`
50
+
51
+ You cannot just `SET NOT NULL` on a populated table serving traffic — old code may still insert NULLs, and
52
+ the validating scan takes a lock.
53
+
54
+ 1. **Expand:** add a `CHECK (col IS NOT NULL) NOT VALID` constraint — this applies to *new* rows only and
55
+ takes no validating scan.
56
+ 2. **Migrate:** deploy code that always sets the column; backfill existing NULLs in batches; then
57
+ `ALTER TABLE t VALIDATE CONSTRAINT ...` (scans without an exclusive lock).
58
+ 3. **Contract:** optionally replace with a real `SET NOT NULL` now that the data is clean, or keep the
59
+ validated CHECK.
60
+
61
+ ## Why each phase is reversible
62
+
63
+ - After Expand, drop the new (empty) column — no data lost.
64
+ - During Migrate, revert reads to the old column — both are still populated by dual-write.
65
+ - Only Contract is destructive, which is exactly why it lives in its own deploy that you can delay.
66
+
67
+ ## Checklist
68
+
69
+ - [ ] New shape added nullable / non-blocking
70
+ - [ ] Dual-write deployed and confirmed on every instance
71
+ - [ ] Backfill complete and verified (no missing rows)
72
+ - [ ] Reads switched and validated against metrics
73
+ - [ ] Old-shape writes stopped everywhere
74
+ - [ ] Backup taken before the contract/drop
@@ -0,0 +1,71 @@
1
+ # Rollback & Safety
2
+
3
+ How to make migrations recoverable, and how to handle the changes that *can't* be cleanly undone.
4
+
5
+ ## Reversible vs irreversible
6
+
7
+ A migration's `down` should restore the prior state. But "reverse the DDL" is not the same as "restore the
8
+ data." Classify every change:
9
+
10
+ | Operation | Reversible? | Notes |
11
+ |---|---|---|
12
+ | ADD COLUMN | yes | `down` = DROP COLUMN (the column was empty/new) |
13
+ | CREATE TABLE / INDEX | yes | `down` = DROP it |
14
+ | ADD CONSTRAINT | yes | `down` = DROP CONSTRAINT |
15
+ | DROP COLUMN | **no** | `down` recreates *structure*, not the deleted data |
16
+ | DROP TABLE | **no** | data is gone; only a backup restores it |
17
+ | TRUNCATE | **no** | not a schema change at all; deletes all rows irrecoverably |
18
+ | Type change with data loss | **no** | narrowing/casting can lose precision |
19
+
20
+ For irreversible operations, a fake "reversing" `down` is *worse than none* — it gives false confidence
21
+ while the data is already lost. Mark them explicitly irreversible and rely on backups.
22
+
23
+ ## The backup rule
24
+
25
+ Before any **destructive** migration (drop/truncate/lossy type change):
26
+
27
+ 1. Take a verified backup or snapshot **immediately before** the migration (point-in-time recovery window
28
+ confirmed, or a fresh logical/physical dump).
29
+ 2. Confirm you can actually restore it (an untested backup is a hope, not a plan).
30
+ 3. Run the destructive step in its **own migration**, separate from non-destructive work, so a rollback of
31
+ the safe parts doesn't force you to touch the dangerous one.
32
+
33
+ ## Forward-fix over rollback
34
+
35
+ In production, rolling *back* a migration that already ran against live data is often more dangerous than
36
+ rolling *forward* with a corrective migration. Reasons:
37
+
38
+ - The `down` may itself be slow/locking on a now-larger table.
39
+ - Code already deployed may depend on the new schema.
40
+ - Reverting can discard data written since the migration ran.
41
+
42
+ Keep `down` migrations for local dev and staging; in prod, prefer a new forward migration that fixes the
43
+ problem. Design changes (via expand/contract) so a forward fix is always available.
44
+
45
+ ## Dry-run discipline
46
+
47
+ - Run the migration on a **production-like copy** (similar row counts, indexes, and load), not an empty dev
48
+ DB.
49
+ - **Time it.** A backfill that takes 30s on 10k staging rows can take hours on 50M prod rows.
50
+ - Watch lock waits and replication lag during the dry run; that's where surprises hide.
51
+ - For long backfills, prove the batch loop terminates and that a mid-run failure can be safely resumed
52
+ (idempotent: `WHERE col IS NULL`).
53
+
54
+ ## Idempotency & partial failure
55
+
56
+ A migration may die halfway (timeout, deploy abort). Make re-runs safe:
57
+
58
+ ```sql
59
+ ALTER TABLE orders ADD COLUMN IF NOT EXISTS priority int NULL;
60
+ DROP INDEX IF EXISTS idx_orders_old;
61
+ ```
62
+ Backfills keyed on `WHERE col IS NULL` naturally resume from where they stopped.
63
+
64
+ ## Pre-ship checklist
65
+
66
+ - [ ] Every step classified reversible or irreversible
67
+ - [ ] `down` provided for reversible steps; irreversible ones flagged, not faked
68
+ - [ ] Destructive steps isolated in their own migration
69
+ - [ ] Verified backup taken before destructive changes
70
+ - [ ] Dry-run on prod-like data, timed, lock/lag observed
71
+ - [ ] Re-run is idempotent / resumable
@@ -0,0 +1,84 @@
1
+ # Zero-Downtime Change Recipes
2
+
3
+ Per-operation guidance: which DDL is safe, which rewrites/locks the table, and the safe alternative.
4
+ Examples use PostgreSQL syntax; the *principles* (don't hold long locks, batch backfills, additive-first)
5
+ apply to MySQL/MariaDB too, though the exact lock behaviour differs by engine and version.
6
+
7
+ ## Add a column
8
+
9
+ ```sql
10
+ -- SAFE: nullable add is metadata-only.
11
+ ALTER TABLE orders ADD COLUMN priority int NULL;
12
+
13
+ -- SAFE on modern Postgres: a CONSTANT default is metadata-only (no rewrite).
14
+ ALTER TABLE orders ADD COLUMN status text NOT NULL DEFAULT 'open';
15
+
16
+ -- UNSAFE: a VOLATILE default forces a full table rewrite under an exclusive lock.
17
+ ALTER TABLE orders ADD COLUMN token uuid NOT NULL DEFAULT gen_random_uuid();
18
+ ```
19
+ For a volatile default: add the column nullable, backfill in batches, then add the default + NOT NULL.
20
+
21
+ ## Backfill in batches
22
+
23
+ Never `UPDATE` an entire large table in one statement — it locks every touched row, blocks writes, and
24
+ bloats WAL/replication lag. Loop in bounded chunks with short transactions:
25
+
26
+ ```sql
27
+ -- Run repeatedly (in the app or a job) until 0 rows are affected.
28
+ UPDATE orders
29
+ SET priority = 0
30
+ WHERE priority IS NULL
31
+ AND id IN (
32
+ SELECT id FROM orders WHERE priority IS NULL ORDER BY id LIMIT 5000
33
+ );
34
+ -- COMMIT between batches; optionally sleep to ease replica lag.
35
+ ```
36
+ Keep batches small enough that each transaction is sub-second. Backfills belong **outside** the schema
37
+ migration transaction.
38
+
39
+ ## Create an index
40
+
41
+ ```sql
42
+ -- UNSAFE: plain CREATE INDEX takes a write lock for the whole build.
43
+ CREATE INDEX idx_orders_status ON orders (status);
44
+
45
+ -- SAFE: CONCURRENTLY builds without blocking reads/writes.
46
+ -- Caveat: cannot run inside a transaction, and a failure leaves an INVALID index to drop & retry.
47
+ CREATE INDEX CONCURRENTLY idx_orders_status ON orders (status);
48
+ ```
49
+ After a failed concurrent build: `DROP INDEX CONCURRENTLY idx_orders_status;` then retry.
50
+
51
+ ## Add NOT NULL safely
52
+
53
+ ```sql
54
+ -- Step 1: validate new rows only, no scan, no exclusive lock.
55
+ ALTER TABLE users ADD CONSTRAINT users_email_nn CHECK (email IS NOT NULL) NOT VALID;
56
+ -- Step 2: backfill NULLs in batches (see above).
57
+ -- Step 3: validate existing rows without an exclusive lock.
58
+ ALTER TABLE users VALIDATE CONSTRAINT users_email_nn;
59
+ ```
60
+
61
+ ## Set lock_timeout
62
+
63
+ ```sql
64
+ -- Make a migration that can't acquire its lock fail fast instead of queueing behind traffic.
65
+ SET lock_timeout = '3s';
66
+ SET statement_timeout = '0'; -- but let a long CONCURRENTLY build run
67
+ ```
68
+
69
+ ## Drop / rename — never one-shot in production
70
+
71
+ `DROP COLUMN`, `RENAME COLUMN`, and type changes break still-running old code. Route them through
72
+ expand/contract (`expand-contract.md`). A drop is destructive — gate it behind a backup.
73
+
74
+ ## Quick reference
75
+
76
+ | Operation | One-shot safe? | Safe approach |
77
+ |---|---|---|
78
+ | Add nullable column | yes | direct |
79
+ | Add column w/ constant default | yes (modern PG) | direct |
80
+ | Add column w/ volatile default | no | nullable → backfill → default |
81
+ | Create index | no | `CONCURRENTLY` |
82
+ | Add NOT NULL | no | CHECK NOT VALID → backfill → VALIDATE |
83
+ | Rename / drop column | no | expand/contract |
84
+ | Change column type | no | new column + backfill + swap |
@@ -0,0 +1,139 @@
1
+ #!/usr/bin/env node
2
+ // Runnable check: scans a migration's SQL text for safety problems —
3
+ // 1. irreversible statements (DROP COLUMN/TABLE, TRUNCATE) that destroy data
4
+ // 2. missing a down-migration (no "-- DOWN" / "== DOWN" section)
5
+ // 3. risky operations that should be made non-blocking (plain CREATE INDEX)
6
+ //
7
+ // Usage:
8
+ // node check-migration-reversible.mjs path/to/migration.sql
9
+ // cat migration.sql | node check-migration-reversible.mjs
10
+ // node check-migration-reversible.mjs --selftest # built-in test cases
11
+ //
12
+ // Exit code: 0 = no blocking issues, 1 = blocking issue (or selftest failed).
13
+
14
+ import { readFileSync } from 'node:fs';
15
+
16
+ const IRREVERSIBLE = [
17
+ [/\bDROP\s+COLUMN\b/i, 'DROP COLUMN destroys data — irreversible; isolate it and back up first'],
18
+ [/\bDROP\s+TABLE\b/i, 'DROP TABLE destroys data — irreversible; back up before running'],
19
+ [/\bTRUNCATE\b/i, 'TRUNCATE deletes all rows irrecoverably — irreversible'],
20
+ ];
21
+
22
+ const WARNINGS = [
23
+ [/\bCREATE\s+INDEX\b(?!\s+CONCURRENTLY)(?![^\n]*CONCURRENTLY)/i,
24
+ 'plain CREATE INDEX locks the table — prefer CREATE INDEX CONCURRENTLY'],
25
+ ];
26
+
27
+ // Strip line/block comments so a statement mentioned in a comment isn't flagged,
28
+ // but keep section markers detectable separately (we test the raw text for those).
29
+ function stripComments(sql) {
30
+ return sql
31
+ .replace(/\/\*[\s\S]*?\*\//g, ' ') // block comments
32
+ .replace(/--[^\n]*/g, ' '); // line comments
33
+ }
34
+
35
+ const DOWN_MARKER = /(^|\n)[^\S\n]*(--+\s*)?(==+\s*)?DOWN\b/i;
36
+
37
+ function hasDownSection(rawSql) {
38
+ return DOWN_MARKER.test(rawSql) || /\bMIGRATE\s+DOWN\b/i.test(rawSql);
39
+ }
40
+
41
+ // The UP section is what runs against live data; a DROP COLUMN there is
42
+ // destructive. The DOWN section legitimately reverses an additive UP (e.g. an
43
+ // ADD COLUMN's down IS a DROP COLUMN), so only the UP portion is scanned for
44
+ // irreversible/risky statements.
45
+ function upSection(rawSql) {
46
+ const m = DOWN_MARKER.exec(rawSql);
47
+ return m ? rawSql.slice(0, m.index) : rawSql;
48
+ }
49
+
50
+ export function analyze(rawSql) {
51
+ const code = stripComments(upSection(rawSql));
52
+ const errors = [];
53
+ const warnings = [];
54
+
55
+ for (const [re, msg] of IRREVERSIBLE) {
56
+ if (re.test(code)) errors.push(`irreversible: ${msg}`);
57
+ }
58
+ const irreversible = errors.length > 0;
59
+
60
+ // A down-migration is required UNLESS the change is irreversible (a faked
61
+ // "down" for an irreversible op is worse than none — see rollback-and-safety).
62
+ if (!irreversible && !hasDownSection(rawSql)) {
63
+ errors.push('missing down-migration: add a "-- DOWN" section or mark reversibility');
64
+ }
65
+
66
+ for (const [re, msg] of WARNINGS) {
67
+ if (re.test(code)) warnings.push(`risky: ${msg}`);
68
+ }
69
+
70
+ return { ok: errors.length === 0, errors, warnings, irreversible };
71
+ }
72
+
73
+ function report(label, sql) {
74
+ const { ok, errors, warnings } = analyze(sql);
75
+ if (ok) console.log(`✓ ${label}: no blocking issues`);
76
+ else {
77
+ console.error(`✗ ${label}:`);
78
+ errors.forEach((e) => console.error(` - ${e}`));
79
+ }
80
+ warnings.forEach((w) => console.warn(` ! ${label}: ${w}`));
81
+ return ok;
82
+ }
83
+
84
+ function selftest() {
85
+ const cases = {
86
+ pass_add_column_reversible: {
87
+ sql: '-- UP\nALTER TABLE t ADD COLUMN x int NULL;\n-- DOWN\nALTER TABLE t DROP COLUMN x;',
88
+ shouldPass: true,
89
+ },
90
+ pass_concurrent_index: {
91
+ sql: '-- UP\nCREATE INDEX CONCURRENTLY i ON t (x);\n-- DOWN\nDROP INDEX CONCURRENTLY i;',
92
+ shouldPass: true,
93
+ },
94
+ pass_drop_in_comment_only: {
95
+ // "DROP TABLE" appears ONLY inside a comment; real statements are additive
96
+ // with a valid down. Comment-stripping must prevent a false positive.
97
+ sql: '-- NOTE: this does NOT drop table t\n-- UP\nALTER TABLE t ADD COLUMN y int NULL;\n-- DOWN\nALTER TABLE t ALTER COLUMN y DROP DEFAULT;',
98
+ shouldPass: true,
99
+ },
100
+ fail_drop_table: {
101
+ sql: '-- UP\nDROP TABLE legacy;',
102
+ shouldPass: false,
103
+ },
104
+ fail_truncate: {
105
+ sql: '-- UP\nTRUNCATE audit_log;',
106
+ shouldPass: false,
107
+ },
108
+ fail_missing_down: {
109
+ sql: '-- UP\nALTER TABLE t ADD COLUMN z int NULL;',
110
+ shouldPass: false,
111
+ },
112
+ };
113
+
114
+ let allOk = true;
115
+ for (const [name, { sql, shouldPass }] of Object.entries(cases)) {
116
+ const { ok } = analyze(sql);
117
+ const correct = ok === shouldPass;
118
+ allOk = allOk && correct;
119
+ console.log(
120
+ `${correct ? '✓' : '✗'} selftest ${name}: ${ok ? 'pass' : 'fail'} ` +
121
+ `(expected ${shouldPass ? 'pass' : 'fail'})`
122
+ );
123
+ }
124
+ process.exit(allOk ? 0 : 1);
125
+ }
126
+
127
+ const arg = process.argv[2];
128
+ if (arg === '--selftest') {
129
+ selftest();
130
+ } else if (arg) {
131
+ let sql;
132
+ try { sql = readFileSync(arg, 'utf8'); }
133
+ catch { console.error(`cannot read file: ${arg}`); process.exit(1); }
134
+ process.exit(report(arg, sql) ? 0 : 1);
135
+ } else {
136
+ let buf = '';
137
+ process.stdin.on('data', (c) => (buf += c));
138
+ process.stdin.on('end', () => process.exit(report('stdin', buf) ? 0 : 1));
139
+ }