splithub 0.1.9 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/chunk-3DUWIEHB.js +561 -0
  2. package/dist/chunk-3DUWIEHB.js.map +1 -0
  3. package/dist/chunk-3VACHXWE.js +319 -0
  4. package/dist/chunk-3VACHXWE.js.map +1 -0
  5. package/dist/chunk-3VY5CP3E.js +383 -0
  6. package/dist/chunk-3VY5CP3E.js.map +1 -0
  7. package/dist/chunk-74NI7BED.js +58 -0
  8. package/dist/chunk-74NI7BED.js.map +1 -0
  9. package/dist/chunk-A2RCSPG6.js +184 -0
  10. package/dist/chunk-A2RCSPG6.js.map +1 -0
  11. package/dist/chunk-AJO3C4BS.js +97 -0
  12. package/dist/chunk-AJO3C4BS.js.map +1 -0
  13. package/dist/chunk-C5S7RPYM.js +54 -0
  14. package/dist/chunk-C5S7RPYM.js.map +1 -0
  15. package/dist/chunk-EUF75FUC.js +160 -0
  16. package/dist/chunk-EUF75FUC.js.map +1 -0
  17. package/dist/chunk-EYYLDI5J.js +195 -0
  18. package/dist/chunk-EYYLDI5J.js.map +1 -0
  19. package/dist/chunk-GVARYUL3.js +28 -0
  20. package/dist/chunk-GVARYUL3.js.map +1 -0
  21. package/dist/chunk-HSH4ZI2V.js +72 -0
  22. package/dist/chunk-HSH4ZI2V.js.map +1 -0
  23. package/dist/chunk-PLXDG4B5.js +134 -0
  24. package/dist/chunk-PLXDG4B5.js.map +1 -0
  25. package/dist/chunk-PWSRKABE.js +113 -0
  26. package/dist/chunk-PWSRKABE.js.map +1 -0
  27. package/dist/chunk-QOXGIB3R.js +393 -0
  28. package/dist/chunk-QOXGIB3R.js.map +1 -0
  29. package/dist/chunk-UUTLLC54.js +158 -0
  30. package/dist/chunk-UUTLLC54.js.map +1 -0
  31. package/dist/chunk-V3DLUVOE.js +112 -0
  32. package/dist/chunk-V3DLUVOE.js.map +1 -0
  33. package/dist/chunk-YSXW2HN6.js +104 -0
  34. package/dist/chunk-YSXW2HN6.js.map +1 -0
  35. package/dist/ci-ZQZ6RBUJ.js +16 -0
  36. package/dist/ci-ZQZ6RBUJ.js.map +1 -0
  37. package/dist/create-K75OY6XK.js +18 -0
  38. package/dist/create-K75OY6XK.js.map +1 -0
  39. package/dist/dev-WQWFEA6I.js +11 -0
  40. package/dist/dev-WQWFEA6I.js.map +1 -0
  41. package/dist/doctor-MFYBZUKO.js +13 -0
  42. package/dist/doctor-MFYBZUKO.js.map +1 -0
  43. package/dist/holdout-PECSNGAD.js +44 -0
  44. package/dist/holdout-PECSNGAD.js.map +1 -0
  45. package/dist/index.js +84 -2239
  46. package/dist/index.js.map +1 -1
  47. package/dist/init-QAAM2GQ7.js +15 -0
  48. package/dist/init-QAAM2GQ7.js.map +1 -0
  49. package/dist/lifecycle-56HM7ITG.js +12 -0
  50. package/dist/lifecycle-56HM7ITG.js.map +1 -0
  51. package/dist/login-TTWSQ7XS.js +18 -0
  52. package/dist/login-TTWSQ7XS.js.map +1 -0
  53. package/dist/menu-UQX3K5QR.js +162 -0
  54. package/dist/menu-UQX3K5QR.js.map +1 -0
  55. package/dist/plan-BSFSMKTU.js +135 -0
  56. package/dist/plan-BSFSMKTU.js.map +1 -0
  57. package/dist/push-YVS25VVD.js +16 -0
  58. package/dist/push-YVS25VVD.js.map +1 -0
  59. package/dist/refs-ZDNH6NRK.js +44 -0
  60. package/dist/refs-ZDNH6NRK.js.map +1 -0
  61. package/dist/remove-KFQUK367.js +54 -0
  62. package/dist/remove-KFQUK367.js.map +1 -0
  63. package/dist/report-TF2BLJMC.js +64 -0
  64. package/dist/report-TF2BLJMC.js.map +1 -0
  65. package/dist/results-TLV5KJWO.js +17 -0
  66. package/dist/results-TLV5KJWO.js.map +1 -0
  67. package/dist/review-EIZDOGDC.js +34 -0
  68. package/dist/review-EIZDOGDC.js.map +1 -0
  69. package/dist/ship-4YX2WCBP.js +450 -0
  70. package/dist/ship-4YX2WCBP.js.map +1 -0
  71. package/dist/skills-HWQT3DTJ.js +13 -0
  72. package/dist/skills-HWQT3DTJ.js.map +1 -0
  73. package/dist/status-CWI7ZCYJ.js +11 -0
  74. package/dist/status-CWI7ZCYJ.js.map +1 -0
  75. package/package.json +22 -5
@@ -0,0 +1,561 @@
1
+ #!/usr/bin/env node
2
+
3
+ // src/skills-self-hosted.ts
4
+ var SKILL_SELF_HOSTED = `---
5
+ name: splithub-self-hosted
6
+ description: Keep SplitHub events in your own database (self mode): which tables, how to write a Store for Postgres, MySQL, Prisma, Drizzle, Supabase or anything else, where the ingest route lives, how the CLI reads results from it, and how to get cloud data into your own database for reports. Use when the user wants data in their infra or asks about tables, schema or reporting.
7
+ ---
8
+
9
+ # SplitHub with your own database
10
+
11
+ Two ways to own the data:
12
+
13
+ 1. **Self mode** \u2014 the app hosts the ingest route and the store; nothing leaves your infra.
14
+ \`splithub.config.ts\`: \`storage: { mode: 'self', endpoint: '/api/splithub' }\`.
15
+ 2. **Cloud + export** \u2014 events go to SplitHub Cloud; you pull summaries into your warehouse for reports.
16
+
17
+ ## Self mode: the pieces
18
+
19
+ - Route: \`app/api/splithub/[[...path]]/route.ts\` (Next.js) or any fetch-style handler from
20
+ \`createSplitHubHandler({ config, store, adminToken })\` in \`@splithub/node\`. It accepts
21
+ \`POST /events\` from the SDK and answers \`GET /status\`, \`GET /results/<id>\` for the CLI
22
+ (\`npx splithub status\` talks to the local dev server, or \`--url https://your.app/api/splithub\`).
23
+ - Store: an object with three methods (below). Ready-made: \`@splithub/adapter-sqlite\`
24
+ (Node 22.13+, \`node:sqlite\`, zero setup) and \`@splithub/adapter-d1\` (Cloudflare D1).
25
+ \`npx splithub init --mode self --db postgres|mysql|custom\` writes \`splithub/store.ts\` with a
26
+ working Postgres (pg) or MySQL (mysql2) store, or a documented stub for anything else.
27
+ - Admin token: set \`SPLITHUB_ADMIN_TOKEN\` in the app and \`--token\` / \`SPLITHUB_ADMIN_TOKEN\` for
28
+ the CLI, so results are not public.
29
+
30
+ ## The events table
31
+
32
+ One table, one row per event. Column names are yours; the templates use these:
33
+
34
+ | column | type | meaning |
35
+ |---|---|---|
36
+ | id | text, primary key | client-generated, unique; dedupe on it (retries resend batches) |
37
+ | type | text | \`exposure\` or \`goal\` |
38
+ | experiment, variant | text | for exposures |
39
+ | goal | text | for goals |
40
+ | visitor_id, session_id | text | \`sh_vid\` cookie / session |
41
+ | ts | bigint | client time, unix ms |
42
+ | url | text | page |
43
+ | props | json | goal payload, e.g. \`{ "revenue": 49 }\` |
44
+ | received_at | bigint | server time, unix ms |
45
+ | host, country, device, browser | text | request metadata (fill from headers if you want breakdowns) |
46
+
47
+ Indexes: \`(type, experiment, visitor_id, ts)\` and \`(type, goal, visitor_id, ts)\`.
48
+
49
+ ## The Store interface (\`@splithub/node\`)
50
+
51
+ \`\`\`ts
52
+ interface Store {
53
+ insertEvents(events: StoredEvent[]): Promise<number>; // skip existing ids, return inserted count
54
+ variantCounts(experiment: string, goal: string): Promise<VariantCounts[]>; // per variant: exposures, conversions, revenue
55
+ summaries(): Promise<ExperimentSummary[]>; // per experiment: exposures, visitors, first/last ts
56
+ close?(): Promise<void> | void;
57
+ }
58
+ \`\`\`
59
+
60
+ Attribution rule \u2014 keep it exactly, so self-hosted results equal the cloud's:
61
+ **a visitor converts for a variant when they sent the goal at or after their first exposure to
62
+ that variant; exposures and conversions count unique visitors.** The SQL in the generated
63
+ \`splithub/store.ts\` implements it (CTE \`first_exposure\`, then goals joined with \`ts >= first_ts\`).
64
+
65
+ Prisma / Drizzle: map the table in your schema and write the three methods with the ORM, or run the
66
+ one raw query for \`variantCounts\`. Supabase: the same SQL as a Postgres function called through
67
+ \`rpc\`. Mongo: \`$group\` by variant and visitor with \`$min ts\`, then \`$lookup\` goals.
68
+
69
+ ## Reports from the cloud into your database
70
+
71
+ - CSV: dashboard \u2192 experiment \u2192 Export (summary, daily timeline, raw events up to 50k rows).
72
+ - Webhook channel for alerts and the digest: POST JSON to your endpoint when something happens.
73
+ - Scheduled pull: a cron in your app calls \`GET /v1/status\` and \`GET /v1/results/<id>\` with the
74
+ API key (\`sk_\u2026\`) and stores the verdicts. Raw events live 30\u201390 days in the cloud (plan dependent),
75
+ aggregates a year: pull summaries, not raw rows, for long-term reporting.
76
+ - Need every raw event forever in your infra? Use self mode; one store per project.
77
+
78
+ ## Do not
79
+
80
+ - Put database credentials in \`splithub.config.ts\` (it ships to the browser). They belong in the route's env.
81
+ - Compute winners from raw rates in SQL; use \`npx splithub results\` (or \`@splithub/stats\`) for the verdict.
82
+ `;
83
+
84
+ // src/skills-ship.ts
85
+ var SKILL_SHIP = `---
86
+ name: splithub-ship
87
+ description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up; and whenever a test shows status finished and the user agreed on which variant to keep.
88
+ ---
89
+
90
+ # Shipping the winner
91
+
92
+ Goal: the app behaves exactly as the kept variant did, with no experiment left in the code or in the config.
93
+
94
+ ## 1. Decide
95
+
96
+ \`npx splithub results <id> --json\` (MCP: \`splithub_results\`). Act on \`verdict.status\`:
97
+ - \`winner_found\`: keep \`verdict.winner\`.
98
+ - \`no_significant_difference\`: keep the control, or the simpler variant if the user agrees.
99
+ - \`insufficient_sample\`: do not ship. Say when to look again: \`daysToVerdict\` days at the current traffic.
100
+ - \`srm_detected\`: do not ship. The split is broken; fix the setup first.
101
+ - \`guardrail_breached\`: do not ship the variant in \`verdict.guardrailBreached\`; it made a guardrail worse. Say what broke.
102
+ Ask the user which variant to keep before going on. Never pick from raw rates.
103
+
104
+ ## 2. Preview, then ship
105
+
106
+ \`\`\`bash
107
+ npx splithub ship <id> --keep <variant> --dry-run --json # diff + leftovers, nothing written
108
+ npx splithub ship <id> --keep <variant> --json # apply
109
+ \`\`\`
110
+ MCP: \`splithub_ship\` with \`dryRun: true\`, then without. It archives the test in the cloud (numbers stay readable), removes it from
111
+ \`splithub.config.ts\` (checked by reloading the file) and rewrites the code:
112
+ - \`<Experiment id>\` \u2192 the kept \`<Variant>\`'s children; the other variants are dropped.
113
+ - \`useValue('<id>')\` / \`sh.value('<id>')\` \u2192 the kept value as a literal.
114
+ - \`useVariant('<id>')\` comparisons, ternaries, \`&&\`, \`if\` and \`switch\` \u2192 the kept branch.
115
+ - Unused \`Experiment\`, \`Variant\`, \`useVariant\`, \`useValue\` imports go.
116
+
117
+ ## 3. Finish by hand
118
+
119
+ - \`leftovers[]\`: each \`{ file, line, text, reason }\` still mentions the experiment; rewrite those yourself.
120
+ - component kind: delete components and files only the losing variants used (e.g. \`splithub/<id>/\`).
121
+ - redirect kind: make the kept URL the real page (move the route or add a permanent redirect), delete the other page.
122
+ - Goal: keep \`track('<goal>')\` when another test measures it (\`next\` says so); otherwise ask before removing it.
123
+ - Cloud mode: \`npx splithub push\`. Then \`npx splithub doctor\` and the project's typecheck and tests.
124
+ - \`npx splithub refs <id> --json\` must return \`count: 0\` for the experiment.
125
+
126
+ Report what changed (files, the kept content) and the verdict in one line with its numbers. Never reuse the id.
127
+ `;
128
+ var SKILL_LOOP = `---
129
+ name: splithub-loop
130
+ description: The whole A/B test loop for an agent with SplitHub, step by step with the MCP tools and CLI --json fields: plan, create, implement, check, push, read results, finish, ship. Use when the user asks to run an A/B test end to end, or to pick up a test where it stands.
131
+ ---
132
+
133
+ # The full loop
134
+
135
+ Each step: MCP tool, or the CLI with \`--json\` (one line of JSON; errors are \`{ error, code, hint }\`, exit 1).
136
+
137
+ | step | MCP tool | CLI | read |
138
+ |---|---|---|---|
139
+ | 0. size it | \`splithub_plan\` | \`plan --baseline 0.05 --mde 0.1 [<id>]\` | \`perVariant\`, \`total\`, \`traffic.daysLeft\` |
140
+ | 1. create | \`splithub_create_experiment\` | \`create --kind \u2026 --id \u2026 --goal \u2026 --yes\` | \`snippet\`, \`files\`, \`nextStep\`, \`handoff\` |
141
+ | 2. implement | (edit code) | | wire \`snippet\`; \`track('<goal>')\` where it converts |
142
+ | 3. check | \`splithub_doctor\`, \`splithub_refs\` | \`doctor\`, \`refs <id>\` | \`ok\`, \`checks[]\`; \`refs[]\` shows the wiring |
143
+ | 4. push (cloud) | \`splithub_push\` | \`push\` (\`--dry-run\` to preview) | \`adopted\`, \`diff\` |
144
+ | 5. results | \`splithub_status\`, \`splithub_results\` | \`status\`, \`results <id>\` | \`verdict.status\`, \`verdict.message\`, \`verdict.notes[]\`, \`daysToVerdict\` |
145
+ | 6. finish | \`splithub_set_status\` | \`finish <id>\` | \`live\`, \`next\` |
146
+ | 7. ship | \`splithub_ship\` | \`ship <id> --keep <v> [--dry-run]\` | \`diff\`, \`leftovers[]\`, \`next\` |
147
+
148
+ Rules that keep the answer honest:
149
+ - Plan before creating: if \`traffic.daysLeft\` is months, test a bigger change or a busier page.
150
+ - \`create\` writes a running test; it goes live when the code is deployed (and pushed in cloud mode). Tell the user.
151
+ - Verdicts are sequential (always-valid): checking any day is safe. Quote \`verdict.message\` and the always-valid
152
+ \`comparisons[].sequential.pValue\`, not the fixed-horizon \`pValue\` (p = 0.01 there can still be not significant).
153
+ Relay \`verdict.notes[]\`. There is no verdict before the experiment's \`minDays\` (default 1 day, 0 = none; \`create --min-days\`).
154
+ - \`daysToVerdict\` is the re-check date: schedule (or tell the user) a look in that many days instead of polling.
155
+ - \`srm_detected\`: the split is broken; fix the setup, do not read the numbers.
156
+ - \`guardrail_breached\`: don't ship. A variant made a guardrail worse (\`verdict.guardrailBreached[]\`), e.g. "bold wins on
157
+ signups but raises refunds". \`verdict.metrics[]\` has the secondary metrics and guardrails, read like the main table.
158
+ - Guard what matters when creating: \`--secondary pricing_view --guardrail refunds:higher\` (MCP: \`secondary\`, \`guardrails\`).
159
+ Tests that could clash on one page go in one layer: \`--layer checkout:0-50\` and \`checkout:50-100\` never share a visitor.
160
+ - \`splithub holdout set 5\` keeps 5% of visitors out of every test; the \`__holdout\` row ("Holdout: long-term impact")
161
+ compares them with everyone else.
162
+ - Ask the user which variant to keep before \`ship\`; preview with \`dryRun\` and show them the diff.
163
+ - Error \`code\` \u2192 action: \`NOT_LOGGED_IN\` \u2192 \`splithub_login\`, show the link, wait, retry \xB7 \`INSUFFICIENT_SCOPE\` \u2192 the key
164
+ can only read; ask for a key with write scope \xB7 \`NO_CONFIG\` \u2192 \`splithub_init\` \xB7
165
+ \`EXPERIMENT_NOT_FOUND\` \u2192 \`splithub_status\` for ids \xB7 \`CONFIG_UNSAFE\` \u2192 edit the config by hand where the experiment is
166
+ defined \xB7 \`CONFIG_WRITE\` \u2192 the file was restored; edit by hand, then \`doctor\`.
167
+ `;
168
+
169
+ // src/skills.ts
170
+ var SKILL_SPLITHUB = `---
171
+ name: splithub
172
+ description: Add, change, run or read A/B experiments in an app that uses SplitHub (@splithub/react, splithub.config.ts, the splithub CLI, SplitHub Cloud). Use when the user asks to A/B test a component, headline, image, price, CTA or page, asks which variant is winning, or wants to start, pause or finish a test.
173
+ ---
174
+
175
+ # SplitHub for coding agents
176
+
177
+ Three pieces: \`splithub.config.ts\` (the experiments; source of truth when it exists),
178
+ \`@splithub/react\` (renders variants), the \`splithub\` CLI (creates, pushes, reads results).
179
+ Events go to the app's own database (self mode) or to SplitHub Cloud (cloud mode).
180
+
181
+ ## Where things live
182
+
183
+ - Config: \`splithub.config.ts\` in the project root. Experiments and goals. Ships to the browser: never put secrets in it.
184
+ - Credentials: \`npx splithub login\` (or \`init --mode cloud\`) prints a link; the human approves it in the browser; the CLI
185
+ stores a project API key for this folder. Never ask the user to paste keys. CI uses \`SPLITHUB_API_KEY\`.
186
+ - The public key (\`pk_\`) ends up in \`storage.key\` of the config from any of: \`login\` (saved credentials), the first
187
+ \`push\`, \`init --key\`, or \`SPLITHUB_API_KEY\` in the environment (init asks the cloud). Missing key = events go nowhere.
188
+ - Cloud: \`npx splithub push\` uploads the config; \`npx splithub pull\` writes dashboard-created experiments into the file.
189
+ - Results: \`npx splithub status --json\`, \`npx splithub results <id> --json\`. Sizing: \`npx splithub plan --baseline 0.05 --mde 0.1\`.
190
+ - Code that uses a test: \`npx splithub refs <id> --json\`. Keep a variant and remove the test: \`npx splithub ship <id> --keep <v>\`.
191
+ - \`--json\` prints one line; errors are \`{ error, code, hint }\` with exit code 1. The \`splithub-loop\` skill has the whole flow.
192
+ - Everything interactive: \`npx splithub\` (menu). Everything scriptable: subcommands + \`--json\`.
193
+
194
+ ## Pick the kind
195
+
196
+ | kind | use it when | create | read in code |
197
+ |---|---|---|---|
198
+ | \`value\` | only data changes: text, number, image URL, price, JSON. The default choice: no new components. | \`--variants control,bold --value control="Old text" --value bold="New text"\` | \`const text = useValue<string>('<id>')\`; server components: \`getValue\` from \`@splithub/next\`; no SDK: \`sh.value('<id>')\` |
199
+ | \`component\` | structure changes: a layout, a section, a flow, a different component per variant | \`--variants control,bold\` (the CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs) | \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`, or \`useVariant('<id>')\` in a switch |
200
+ | \`redirect\` | a whole page changes: another URL per variant | \`--match / --to control=/ --to v2=/home-v2\` | nothing: \`@splithub/next\` middleware or the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` sends visitors there |
201
+ | \`mvt\` | several things change at once (headline \xD7 button \xD7 image) and the user wants to know which of them matters; every combination is tested and each factor is read on its own | \`--factor headline=old:"Old text",new:"New text" --factor button=blue:#2563eb,green:#16a34a\` (no \`--variants\`; first level = control; at most 4 factors \xD7 4 levels, 16 combinations) | \`const { headline, button } = useValue<{ headline: string; button: string }>('<id>')\` |
202
+
203
+ mvt needs more visitors than an A/B test: every combination must reach the minimum on its own (the results show the
204
+ combinations and, below them, each factor summed over its combinations). With little traffic, test one thing with \`value\`.
205
+
206
+ ## Create an experiment
207
+
208
+ 1. \`npx splithub create --kind <value|component|redirect|mvt> --id <key> \u2026 --goal <goal> --yes --json\` (see the table).
209
+ 2. Goal: pick an existing key or add one to \`goals\`:
210
+ \`{ type: 'event' }\` + \`track('<goal>')\` where the conversion happens; \`{ type: 'pageview', path: '/thanks' }\`;
211
+ \`{ type: 'click', selector: '#buy' }\` (tracked automatically by the SDK and the script).
212
+ 3. Wire it in code. \`create\` writes \`status: 'running'\`: the test goes live for every visitor, first page view
213
+ included, as soon as this code is deployed. Use \`--status draft\` only when the user asks to hold it.
214
+ 4. Cloud mode: \`npx splithub push\` after any config change.
215
+ 5. Hand-off, always: end your reply by saying the test is **running in the config** and goes live when the user deploys
216
+ this code; nothing is recorded before that, and local development never sends events to the cloud. Tell them how to
217
+ check it locally (\`?sh_force=<id>:<variant>\`) and that Pause in the dashboard or \`npx splithub pause <id>\` holds it.
218
+ In cloud mode a status set in the dashboard wins over the file for every visitor, first page included.
219
+ For a draft (only if asked): say visitors do not see it and nothing is recorded until \`npx splithub start <id>\`.
220
+
221
+ A draft is invisible to visitors. To check one before starting: \`runDraftsInDev: true\` at the top level of the config
222
+ serves drafts like running tests in development only (\`NODE_ENV=development\`: \`next dev\`, \`vite\`);
223
+ \`?sh_force=<id>:<variant>\` shows one variant on any page, draft or not, without recording an exposure;
224
+ \`?sh_debug=1\` (or \`npx splithub dev\`) logs every decision and opens a panel to switch variants.
225
+ In development the SDK explains drafts, paused tests and held-back events in one line each. Vite apps: add \`splithub()\`
226
+ from \`@splithub/core/vite\` to the plugins in vite.config so these lines and a summary of every experiment appear in the
227
+ terminal that runs \`vite\`; Next.js server code prints them in its terminal anyway.
228
+
229
+ Keys: lowercase a-z 0-9 - _, max 64 chars. First variant = control. Weights are relative (50/50, 1/1, 90/10).
230
+ Targeting (country, device, UTM, cookie, time) is a \`flow\` graph on the experiment, edited in the dashboard; \`pull\` brings it into the file.
231
+
232
+ ## Read results
233
+
234
+ \`verdict.status\` \u2208 no_data | insufficient_sample | srm_detected | winner_found | no_significant_difference.
235
+ Report \`verdict.message\` verbatim and the numbers behind it (visitors per variant, lift, the always-valid
236
+ \`comparisons[].sequential.pValue\` and \`sequential.ci\`; the plain \`pValue\` is fixed-horizon, for display only) and relay
237
+ \`verdict.notes[]\`. Verdicts are sequential: safe to check daily, none before the experiment's \`minDays\` (default 1 day, 0 = none; set it with \`create --min-days\`). \`daysToVerdict\` says when to look again.
238
+ Never declare a winner from raw rates. \`srm_detected\` means the split is broken: fix the setup before anything else.
239
+
240
+ ## Finish, and what to do when a test is finished
241
+
242
+ \`npx splithub finish <id>\` stops the test (live within a minute in cloud mode). A finished test keeps its numbers for as long
243
+ as it stays in the config; the code still contains every variant until someone removes it.
244
+
245
+ Whenever you see a test with \`status: finished\` (in \`status\`, \`results\`, the MCP tools, or right after you finished it):
246
+ 1. Read \`npx splithub results <id> --json\` and tell the user the verdict in one line with its numbers.
247
+ 2. Ask two questions, do not assume: **which variant to keep** (recommend \`verdict.winner\`; with no significant
248
+ difference, the control or the simpler one) and **whether to remove the test from the code now**.
249
+ 3. On yes, follow the \`splithub-ship\` skill: \`npx splithub ship <id> --keep <variant> --dry-run\`, show the diff, then run it
250
+ without \`--dry-run\`; fix the \`leftovers\`, \`push\` in cloud mode, run \`doctor\` and the project's checks.
251
+ 4. On no, leave everything: a finished test costs nothing and the user may come back to it.
252
+
253
+ A \`paused\` test is not over: say so and ask whether to resume (\`start\`) or finish it.
254
+
255
+ ## Do not
256
+
257
+ - Nest an \`<Experiment>\` inside a \`<Variant>\` of the same id; call hooks conditionally.
258
+ - Change \`seed\`, weights or variants of a running experiment (reshuffles visitors). Create a new experiment instead.
259
+ - Edit in the dashboard what the config file owns: file experiments are read-only there (pause/finish still work).
260
+ - Run more than 2-3 experiments on one page of a low-traffic site.
261
+ - Put secrets, API keys or database credentials in the config or in the browser. \`pk_\` is public, \`sk_\` is not.
262
+ `;
263
+ var SKILL_AB_TESTING = `---
264
+ name: ab-testing
265
+ description: How to design, run and read an A/B test correctly: hypothesis, control, primary metric, sample size, significance, SRM, peeking, when to stop. Use before creating an experiment or when interpreting results, with any tool.
266
+ ---
267
+
268
+ # A/B testing, the parts that matter
269
+
270
+ An A/B test splits visitors at random into variants and compares one metric. It answers one question:
271
+ "does this change move this metric?" Everything below exists to keep that answer honest.
272
+
273
+ ## Before creating a test
274
+
275
+ 1. **One hypothesis, one primary metric.** "A shorter signup form raises signup completion." The metric is a goal
276
+ (event, page visit or click) that happens after the exposure. Pick it before looking at data.
277
+ 2. **Control first.** The first variant is what exists today. Change one thing per variant; several unrelated changes
278
+ in one variant cannot be attributed.
279
+ 3. **Traffic reality.** Rough rule: to detect a relative lift L on a baseline rate p you need about
280
+ \`16 \xB7 p(1-p) / (p\xB7L)\xB2\` visitors per variant (95% confidence, 80% power).
281
+ Examples: baseline 5%, lift 20% \u2192 ~7,600 per variant; baseline 2%, lift 10% \u2192 ~78,000 per variant.
282
+ If the site cannot reach that in 2-4 weeks, test a bigger change or a higher-traffic page.
283
+ 4. **Few tests per page.** Interactions between simultaneous tests on the same page muddy both.
284
+
285
+ ## While it runs
286
+
287
+ - **Do not peek and stop early.** Checking daily and stopping at the first "significant" moment inflates false
288
+ positives several times. Decide the sample or the duration up front; run whole weeks (weekday/weekend cycles).
289
+ - **SRM (sample ratio mismatch).** If the split is 50/50 and you see 60/40, the assignment or the tracking is broken
290
+ (bots, caching, redirect loops, a variant that errors). Fix first; results from an SRM run are meaningless.
291
+ - **Keep variants frozen.** Changing copy, weights or targeting mid-run reshuffles who saw what. Start a new test.
292
+ - **Watch the guardrails.** A variant that wins the primary metric but breaks revenue, errors or latency is not a win.
293
+
294
+ ## Reading results
295
+
296
+ - Significance (p < 0.05 after correction for several variants) says the difference is unlikely to be noise;
297
+ the confidence interval says how big it plausibly is. A "winner" with a CI that touches 0 is a weak win.
298
+ - "No significant difference" with a sample smaller than needed means **keep running**, not "no effect".
299
+ With enough sample it means the change does not matter: ship whichever is cheaper or cleaner.
300
+ - Relative lift is on the baseline rate: 5% \u2192 6% is +20% relative, +1 point absolute. Report both.
301
+ - Attribution: conversions count only after the visitor's first exposure, per unique visitor.
302
+ - Segments (country, device) are for hypotheses, not verdicts: slicing after the fact finds noise.
303
+
304
+ ## When to stop
305
+
306
+ - Winner found with the planned sample \u2192 ship it, finish the test, remove the losing code.
307
+ - Planned sample reached, no difference \u2192 ship the simpler variant, finish.
308
+ - SRM or broken tracking \u2192 stop, fix, restart as a new test.
309
+ - The business moved on \u2192 finish; do not leave tests running forever (they cost events and split traffic).
310
+
311
+ ## Vocabulary
312
+
313
+ exposure = a visitor saw a variant \xB7 conversion = the goal happened after exposure \xB7 lift = relative change in rate \xB7
314
+ p-value = chance of seeing this difference by luck \xB7 CI = range the true difference likely lies in \xB7
315
+ SRM = the split differs from the configured weights \xB7 guardrail = a metric that must not get worse.
316
+ `;
317
+ var SKILL_SPLITHUB_API = `---
318
+ name: splithub-api
319
+ description: Use the SplitHub Cloud HTTP API directly: public key event ingest, API-key endpoints for config, status, results and overrides, hosted experiments. Use when writing a server, a script, a non-JS client, or when the CLI is not available.
320
+ ---
321
+
322
+ # SplitHub Cloud API
323
+
324
+ Base URL: \`https://go.splithub.io\`. JSON in, JSON out (no envelope under \`/v1/*\`). Two credentials:
325
+
326
+ - **Public key \`pk_\u2026\`**: identifies the project to browsers and SDKs. Safe to ship. Can only send events and read
327
+ the public overrides JSON.
328
+ - **API key \`sk_\u2026\`**: project-scoped secret for servers, CI and the CLI (\`npx splithub login\` mints one through a
329
+ browser approval; or Settings \u2192 API keys in the dashboard). Never in client code.
330
+
331
+ ## Send events (public key)
332
+
333
+ \`\`\`http
334
+ POST /v1/events
335
+ Authorization: Bearer pk_live_\u2026
336
+ Content-Type: application/json
337
+
338
+ { "project": "my-app", "sdk": "custom/1.0", "events": [
339
+ { "id": "e_01h\u2026", "type": "exposure", "exp": "hero-cta", "var": "bold", "vid": "v_abc", "sid": "s_1", "ts": 1760000000000, "url": "/" },
340
+ { "id": "g_01h\u2026", "type": "goal", "goal": "signup", "vid": "v_abc", "sid": "s_1", "ts": 1760000010000, "props": { "revenue": 49 } }
341
+ ] }
342
+ \`\`\`
343
+
344
+ - \`id\`: unique per event (8-64 chars, [a-z0-9_-]); duplicates are ignored. \`vid\`: stable visitor id (cookie \`sh_vid\`).
345
+ - Reply \`202 { ok, received, inserted, dropped }\`. A project with nothing running or no config yet answers
346
+ \`202 { ok, inserted: 0, idle: 'no_running' | 'no_config' }\`: nothing stored, not an error. \`429\` with \`code\` \u2208
347
+ quota_exceeded | deleted: back off for \`Retry-After\` seconds; nothing was stored. \`403 origin_not_allowed\` when the project restricts domains.
348
+ - Exposures for experiments that are not running are dropped and not billed. Goals are never billed.
349
+
350
+ ## Live config for clients (public key)
351
+
352
+ \`GET /v1/overrides/<pk>.json\` \u2192 \`{ version, experiments: { <id>: { status?, traffic?, weights?, values? } }, added: [ \u2026hosted experiments\u2026 ], goals: [...], ctx: { country, region, city } }\`.
353
+ Poll it at most once a minute; the SDK caches it in localStorage. \`ctx\` is the visitor's geo for targeting.
354
+
355
+ ## Project endpoints (API key)
356
+
357
+ \`\`\`http
358
+ GET /v1/status every experiment: status, visitors, days, verdict headline
359
+ GET /v1/results/<experiment> full verdict: sample, comparisons (lift, p-value, CI, P(beat)), srm, winner
360
+ GET /v1/config the config stored in the cloud (file + dashboard experiments)
361
+ PUT /v1/config { config } push a normalized config (what defineConfig returns); answers cloudOnly experiments to adopt
362
+ GET /v1/overrides live overrides
363
+ PUT /v1/overrides/<experiment> { status | traffic | weights | values } pause / resume / reweight without a deploy
364
+ DELETE /v1/overrides/<experiment>
365
+ \`\`\`
366
+
367
+ \`Authorization: Bearer sk_live_\u2026\` on each. Status values: draft | running | paused | finished.
368
+
369
+ ## Dashboard-scope endpoints (session, same shapes the dashboard uses)
370
+
371
+ Under \`/v1/projects/<projectId>/\u2026\` with a user session: \`experiments\` (hosted experiments CRUD incl. \`flow\` targeting),
372
+ \`goals\`, \`timeline\`, \`breakdown\`, \`realtime\`, \`events\`, \`export\`, \`notes\`, \`domains\`, \`links\`, \`alerts\`, \`billing\`, \`ai\`.
373
+ Scripts should prefer the API-key endpoints above; the session ones exist for the UI.
374
+
375
+ ## Drop-in script (no SDK)
376
+
377
+ \`<script src="https://go.splithub.io/sh.js" data-key="pk_live_\u2026"></script>\` in \`<head>\`: redirect experiments, page-visit and
378
+ click goals from the cloud config, \`window.splithub.track('goal')\` for custom events,
379
+ \`<meta name="splithub-experiment" content="<id>">\` on a tested page for an instant decision.
380
+
381
+ ## Errors
382
+
383
+ \`400\` validation (message in \`error\`), \`401\` bad key, \`403\` forbidden, \`404\` unknown experiment/project, \`409\` state
384
+ (e.g. \`config_not_pushed\`, \`file_owned\`), \`429\` gate or rate limit. Bodies: \`{ error, code? }\`.
385
+ `;
386
+ var SKILL_SPLITHUB_MCP = `---
387
+ name: splithub-mcp
388
+ description: Use or set up the SplitHub MCP server (@splithub/mcp) so an agent can create experiments, read verdicts, start/pause/finish tests and push config through tools instead of shelling out. Use when the user asks to connect SplitHub to Claude Code, Cursor, Codex or another MCP client.
389
+ ---
390
+
391
+ # SplitHub MCP server
392
+
393
+ \`@splithub/mcp\` is a stdio MCP server that wraps the \`splithub\` CLI for the current project folder.
394
+ Same credentials and config file as the CLI; nothing extra to configure.
395
+
396
+ ## Register it
397
+
398
+ Claude Code (project scope, committed in \`.mcp.json\`):
399
+
400
+ \`\`\`json
401
+ { "mcpServers": { "splithub": { "command": "npx", "args": ["-y", "@splithub/mcp"] } } }
402
+ \`\`\`
403
+
404
+ or \`claude mcp add splithub -- npx -y @splithub/mcp\`. Cursor: the same object in \`.cursor/mcp.json\`.
405
+ Codex / other clients: command \`npx\`, args \`-y @splithub/mcp\`. \`npx splithub skills --mcp\` writes both files.
406
+ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory), \`SPLITHUB_API\` (staging or self-hosted cloud), \`SPLITHUB_API_KEY\` (CI).
407
+
408
+ ## Tools
409
+
410
+ | tool | does | notes |
411
+ |---|---|---|
412
+ | \`splithub_status\` | all experiments with verdict headline | start here |
413
+ | \`splithub_results\` | full verdict for one experiment | report \`verdict.message\` verbatim; \`daysToVerdict\` = when to check again |
414
+ | \`splithub_review\` | stored AI reviews (Pro): at start, on verdict changes, on finish, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |
415
+ | \`splithub_plan\` | visitors per variant for a baseline and a lift; days at an experiment's traffic | before creating |
416
+ | \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind value / component / redirect / mvt, id, variants (or factors for mvt), goal, values/to, weights, traffic, maxVisitors, start/end; running by default |
417
+ | \`splithub_refs\` | files and lines that use an experiment and its goal | check the wiring, find leftovers |
418
+ | \`splithub_ship\` | finish, remove from the config, rewrite the code to the kept variant | \`dryRun: true\` first; returns \`diff\` and \`leftovers\` |
419
+ | \`splithub_remove\` | removes an experiment from the config (finished in the cloud) | when the code is already clean |
420
+ | \`splithub_projects\` | every folder on this machine connected to a cloud project | |
421
+ | \`splithub_set_status\` | start / pause / finish | applies live in cloud mode; after \`finished\`, ask which variant to keep and whether to clean the code (splithub-ship skill) |
422
+ | \`splithub_push\` / \`splithub_pull\` | sync the config with the cloud | pull adopts dashboard-created experiments |
423
+ | \`splithub_doctor\` | checks the installation | run after wiring code |
424
+ | \`splithub_login\` | returns a sign-in link; the human approves it in the browser | show the link, then call \`splithub_whoami\` |
425
+ | \`splithub_whoami\` | connected project and keys | |
426
+
427
+ ## Working rules
428
+
429
+ - Create (running by default) \u2192 wire in code \u2192 \`doctor\` \u2192 \`push\` (cloud mode) \u2192 tell the user it goes live when they
430
+ deploy this code and that \`set_status paused\` or Pause in the dashboard holds it.
431
+ - Results: never call a winner from raw rates; quote the verdict. \`srm_detected\` = fix the setup first.
432
+ - Running experiments are frozen: no weight / variant / seed edits; create a new one.
433
+ - Errors carry a \`code\`. \`NOT_LOGGED_IN\`: call \`splithub_login\`, show the link to the user, wait for them, then retry.
434
+ - The whole loop (plan \u2192 create \u2192 implement \u2192 doctor \u2192 push \u2192 results \u2192 finish \u2192 ship) is in the \`splithub-loop\` skill.
435
+ - A \`finished\` experiment in \`splithub_status\` or \`splithub_results\` means the test is over but its code is still there: report
436
+ the verdict, ask which variant to keep and whether to remove the test from the code, then follow the \`splithub-ship\` skill.
437
+ `;
438
+ var SKILLS = [
439
+ { dir: "splithub", body: SKILL_SPLITHUB },
440
+ { dir: "ab-testing", body: SKILL_AB_TESTING },
441
+ { dir: "splithub-api", body: SKILL_SPLITHUB_API },
442
+ { dir: "splithub-mcp", body: SKILL_SPLITHUB_MCP },
443
+ { dir: "splithub-self-hosted", body: SKILL_SELF_HOSTED },
444
+ { dir: "splithub-ship", body: SKILL_SHIP },
445
+ { dir: "splithub-loop", body: SKILL_LOOP }
446
+ ];
447
+ var MCP_SERVER_ENTRY = { command: "npx", args: ["-y", "@splithub/mcp"] };
448
+
449
+ // src/templates.ts
450
+ function configTemplate(project, mode, key) {
451
+ const storage = mode === "self" ? `{ mode: 'self', endpoint: '/api/splithub' }` : `{ mode: 'cloud', key: ${key ? `'${key}'` : "process.env.NEXT_PUBLIC_SPLITHUB_KEY ?? ''"} }`;
452
+ return `import { defineConfig } from '@splithub/core';
453
+
454
+ // SplitHub experiments. This file is the source of truth: the SDK, the CLI
455
+ // and the dashboard all read it. Keys: lowercase a-z 0-9 - _. First variant = control.
456
+ export default defineConfig({
457
+ project: '${project}',
458
+ storage: ${storage},
459
+ // Drafts behave like running tests while you develop (next dev, vite); production ignores them.
460
+ runDraftsInDev: true,
461
+
462
+ goals: {
463
+ // call track('signup') where the conversion happens
464
+ signup: { type: 'event' },
465
+ },
466
+
467
+ experiments: {
468
+ // run \`npx splithub create\` or add entries here:
469
+ //
470
+ // 'hero-title': {
471
+ // kind: 'value',
472
+ // status: 'running',
473
+ // variants: {
474
+ // control: { weight: 50, value: 'Ship faster' },
475
+ // direct: { weight: 50, value: 'Know which version wins' },
476
+ // },
477
+ // goal: 'signup',
478
+ // },
479
+ },
480
+ });
481
+ `;
482
+ }
483
+ function nextMiddlewareTemplate(configImport) {
484
+ return `import { splithubMiddleware } from '@splithub/next';
485
+ import splithub from '${configImport}';
486
+
487
+ // Sets the visitor cookie, forwards it to Server Components and serves
488
+ // redirect experiments with an invisible rewrite.
489
+ export default splithubMiddleware(splithub);
490
+
491
+ export const config = {
492
+ matcher: ['/((?!_next|api|.*\\\\..*).*)'],
493
+ };
494
+ `;
495
+ }
496
+ function nextRouteTemplate(configImport, store = { imports: "import { sqliteStore } from '@splithub/adapter-sqlite';", factory: "sqliteStore({ file: process.env.SPLITHUB_DB ?? './splithub.db' })" }) {
497
+ return `import { createSplitHubHandler } from '@splithub/node';
498
+ ${store.imports}
499
+ import config from '${configImport}';
500
+
501
+ // Self-hosted ingest + results: events stay in your database. The CLI reads
502
+ // status / results from this route; set SPLITHUB_ADMIN_TOKEN to protect it.
503
+ const handler = createSplitHubHandler({
504
+ config,
505
+ store: ${store.factory},
506
+ adminToken: process.env.SPLITHUB_ADMIN_TOKEN,
507
+ });
508
+
509
+ export const POST = handler;
510
+ export const GET = handler;
511
+ export const runtime = 'nodejs';
512
+ `;
513
+ }
514
+ function componentVariantTemplate(name, experiment, variant) {
515
+ return `// Variant "${variant}" of experiment "${experiment}". Edit freely.
516
+ export function ${name}() {
517
+ return (
518
+ <section>
519
+ <h2>${experiment} \xB7 ${variant}</h2>
520
+ </section>
521
+ );
522
+ }
523
+ `;
524
+ }
525
+ var AGENTS_SECTION = `
526
+ ## A/B testing (SplitHub)
527
+
528
+ This project uses SplitHub (\`@splithub/react\`). Experiments are declared in
529
+ \`splithub.config.ts\` and rendered with \`<Experiment id>\` / \`useVariant(id)\` /
530
+ \`useValue(id)\`. Conversions: \`useTrack()\` then \`track('<goal>')\`.
531
+
532
+ - Not connected yet? \`npx splithub init --mode cloud\` (or \`npx splithub login\`) prints a link: show it to the user, they sign in and approve, the CLI gets the keys. Never ask the user to paste keys.
533
+ - New experiment: \`npx splithub create --kind value|component|redirect|mvt --id <key> --variants control,<other> --goal <goal> --yes\`
534
+ then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key, redirect \u2192 nothing,
535
+ mvt \u2192 \`--factor name=level:value,\u2026\` and \`const { a, b } = useValue(id)\`).
536
+ - Results: \`npx splithub results <id> --json\`; report \`verdict.message\` verbatim, never call a winner from raw rates.
537
+ - Lifecycle: \`npx splithub start|pause|finish <id>\`. Check setup: \`npx splithub doctor\`.
538
+ - A \`finished\` test still has all its variants in the code: report the verdict, ask which variant to keep and whether to
539
+ remove the test from the code, then follow .claude/skills/splithub-ship/SKILL.md. Never delete test code unasked.
540
+ - \`create\` writes \`status: 'running'\`: the test goes live when the code is deployed. Always tell the user that, and that
541
+ \`npx splithub pause <id>\` or Pause in the dashboard holds it. Never change \`seed\` of a running experiment.
542
+ - Config ships to the browser: no secrets in it. Full rules: .claude/skills/splithub/SKILL.md
543
+ `;
544
+ var CURSOR_RULE = `---
545
+ description: SplitHub A/B testing conventions
546
+ globs: ["splithub.config.ts", "splithub/**", "**/*.tsx"]
547
+ alwaysApply: false
548
+ ---
549
+ ${SKILL_SPLITHUB.replace(/^---[\s\S]*?---\n/, "")}`;
550
+
551
+ export {
552
+ SKILLS,
553
+ MCP_SERVER_ENTRY,
554
+ configTemplate,
555
+ nextMiddlewareTemplate,
556
+ nextRouteTemplate,
557
+ componentVariantTemplate,
558
+ AGENTS_SECTION,
559
+ CURSOR_RULE
560
+ };
561
+ //# sourceMappingURL=chunk-3DUWIEHB.js.map