splithub 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alerts-Q46GJETJ.js +28 -0
- package/dist/analytics-TRB4FPA5.js +18 -0
- package/dist/{chunk-ZND3T67D.js → chunk-4OQEIP33.js} +28 -3
- package/dist/chunk-4OQEIP33.js.map +1 -0
- package/dist/{chunk-7EB5WOYE.js → chunk-6GP534WA.js} +6 -6
- package/dist/{chunk-IGYK7N2F.js → chunk-6MWK7QV4.js} +12 -11
- package/dist/chunk-6MWK7QV4.js.map +1 -0
- package/dist/{chunk-D7KVM4L7.js → chunk-6T652N3X.js} +2 -2
- package/dist/{chunk-X5AV7YBR.js → chunk-BEQ74VTG.js} +8 -8
- package/dist/{chunk-X5AV7YBR.js.map → chunk-BEQ74VTG.js.map} +1 -1
- package/dist/chunk-CAFWOX3S.js +49 -0
- package/dist/chunk-CAFWOX3S.js.map +1 -0
- package/dist/{chunk-XVWJZJSF.js → chunk-EVTPXMLO.js} +22 -9
- package/dist/chunk-EVTPXMLO.js.map +1 -0
- package/dist/chunk-F5ESBFEN.js +237 -0
- package/dist/chunk-F5ESBFEN.js.map +1 -0
- package/dist/chunk-FDSNZRQQ.js +125 -0
- package/dist/chunk-FDSNZRQQ.js.map +1 -0
- package/dist/{chunk-LK3NRAHA.js → chunk-FOHCW2J3.js} +44 -17
- package/dist/chunk-FOHCW2J3.js.map +1 -0
- package/dist/{chunk-5VFCW4FI.js → chunk-GIYKIJKW.js} +25 -8
- package/dist/chunk-GIYKIJKW.js.map +1 -0
- package/dist/{chunk-IFCSJYXN.js → chunk-HFIBLDUA.js} +2 -2
- package/dist/{chunk-6OXCLYOI.js → chunk-ILEPMQNT.js} +56 -2
- package/dist/chunk-ILEPMQNT.js.map +1 -0
- package/dist/chunk-J3DANPLP.js +250 -0
- package/dist/chunk-J3DANPLP.js.map +1 -0
- package/dist/{chunk-ECV5QR6H.js → chunk-KV7D6SB7.js} +8 -8
- package/dist/{chunk-ECV5QR6H.js.map → chunk-KV7D6SB7.js.map} +1 -1
- package/dist/chunk-MIHX7XYV.js +819 -0
- package/dist/chunk-MIHX7XYV.js.map +1 -0
- package/dist/chunk-MSYL3XZF.js +243 -0
- package/dist/chunk-MSYL3XZF.js.map +1 -0
- package/dist/{chunk-PLZRB3VX.js → chunk-MUDDASN6.js} +2 -2
- package/dist/{chunk-BSWCWCYP.js → chunk-NFZZSF4M.js} +3 -3
- package/dist/{chunk-AZJEZFOB.js → chunk-QBQKQH6S.js} +42 -26
- package/dist/chunk-QBQKQH6S.js.map +1 -0
- package/dist/{chunk-VKLZJRYW.js → chunk-QCFQFDEB.js} +35 -13
- package/dist/chunk-QCFQFDEB.js.map +1 -0
- package/dist/{chunk-NTAIRQP6.js → chunk-RMWW4YMN.js} +4 -4
- package/dist/{chunk-NTAIRQP6.js.map → chunk-RMWW4YMN.js.map} +1 -1
- package/dist/chunk-SHKXD2C6.js +199 -0
- package/dist/chunk-SHKXD2C6.js.map +1 -0
- package/dist/chunk-SPXUP3WA.js +321 -0
- package/dist/chunk-SPXUP3WA.js.map +1 -0
- package/dist/chunk-SUUNAQAC.js +270 -0
- package/dist/chunk-SUUNAQAC.js.map +1 -0
- package/dist/{chunk-EJC4E3VD.js → chunk-ZISCJRR5.js} +4 -3
- package/dist/chunk-ZISCJRR5.js.map +1 -0
- package/dist/{ci-4OC23DTI.js → ci-4SY2X2CU.js} +4 -4
- package/dist/connect-TCBFW2NT.js +220 -0
- package/dist/connect-TCBFW2NT.js.map +1 -0
- package/dist/create-GM5H72LW.js +18 -0
- package/dist/dev-YNKPWJHX.js +11 -0
- package/dist/doctor-BKS7TUAT.js +14 -0
- package/dist/domains-CR34APOD.js +10 -0
- package/dist/experiment-reports-LL2TFQQN.js +49 -0
- package/dist/{holdout-OKHR2CKW.js → holdout-JJEP6673.js} +7 -6
- package/dist/holdout-JJEP6673.js.map +1 -0
- package/dist/index.js +130 -37
- package/dist/index.js.map +1 -1
- package/dist/init-WHK4EV37.js +15 -0
- package/dist/lifecycle-3GUIJ74P.js +12 -0
- package/dist/links-BQZLMGZJ.js +18 -0
- package/dist/{login-ZEIH7KIH.js → login-OT77MNKL.js} +4 -4
- package/dist/login-OT77MNKL.js.map +1 -0
- package/dist/menu-QYXFUWNE.js +726 -0
- package/dist/menu-QYXFUWNE.js.map +1 -0
- package/dist/notes-BASGWNNX.js +18 -0
- package/dist/notes-BASGWNNX.js.map +1 -0
- package/dist/{plan-QLYHRYCV.js → plan-44334RQS.js} +7 -7
- package/dist/plan-44334RQS.js.map +1 -0
- package/dist/push-UV4O66QR.js +16 -0
- package/dist/push-UV4O66QR.js.map +1 -0
- package/dist/{refs-IWZ5GP7Q.js → refs-5DTSKMXU.js} +5 -5
- package/dist/{remove-YDJ7JTPR.js → remove-PYYPRP3R.js} +8 -8
- package/dist/{report-JS7Z42FD.js → report-JVMKVZTS.js} +6 -6
- package/dist/results-KC24ZZTX.js +20 -0
- package/dist/results-KC24ZZTX.js.map +1 -0
- package/dist/{review-BGGNVL6L.js → review-GIYBPVP3.js} +3 -3
- package/dist/{ship-C22SGFUD.js → ship-3TDWAQIJ.js} +8 -8
- package/dist/skills-LVIFTOTI.js +18 -0
- package/dist/skills-LVIFTOTI.js.map +1 -0
- package/dist/status-SK5PPBPL.js +17 -0
- package/dist/status-SK5PPBPL.js.map +1 -0
- package/dist/{update-SAHMBHIZ.js → update-XF7HWJ3A.js} +3 -3
- package/package.json +3 -3
- package/dist/chunk-5VFCW4FI.js.map +0 -1
- package/dist/chunk-6OXCLYOI.js.map +0 -1
- package/dist/chunk-AZJEZFOB.js.map +0 -1
- package/dist/chunk-BQAWJPIK.js +0 -567
- package/dist/chunk-BQAWJPIK.js.map +0 -1
- package/dist/chunk-EJC4E3VD.js.map +0 -1
- package/dist/chunk-IGYK7N2F.js.map +0 -1
- package/dist/chunk-LK3NRAHA.js.map +0 -1
- package/dist/chunk-VKLZJRYW.js.map +0 -1
- package/dist/chunk-XVWJZJSF.js.map +0 -1
- package/dist/chunk-ZND3T67D.js.map +0 -1
- package/dist/create-NCBDS2YG.js +0 -18
- package/dist/dev-7FXVGFL6.js +0 -11
- package/dist/doctor-E4KLPZZI.js +0 -14
- package/dist/holdout-OKHR2CKW.js.map +0 -1
- package/dist/init-PFNPS6XO.js +0 -15
- package/dist/lifecycle-G7OAC4ZZ.js +0 -12
- package/dist/menu-4X5S37OK.js +0 -163
- package/dist/menu-4X5S37OK.js.map +0 -1
- package/dist/plan-QLYHRYCV.js.map +0 -1
- package/dist/push-3NDLL7US.js +0 -16
- package/dist/results-HCXQH7IY.js +0 -17
- package/dist/skills-UBPQOTDQ.js +0 -18
- package/dist/status-6IODQYHV.js +0 -12
- /package/dist/{ci-4OC23DTI.js.map → alerts-Q46GJETJ.js.map} +0 -0
- /package/dist/{create-NCBDS2YG.js.map → analytics-TRB4FPA5.js.map} +0 -0
- /package/dist/{chunk-7EB5WOYE.js.map → chunk-6GP534WA.js.map} +0 -0
- /package/dist/{chunk-D7KVM4L7.js.map → chunk-6T652N3X.js.map} +0 -0
- /package/dist/{chunk-IFCSJYXN.js.map → chunk-HFIBLDUA.js.map} +0 -0
- /package/dist/{chunk-PLZRB3VX.js.map → chunk-MUDDASN6.js.map} +0 -0
- /package/dist/{chunk-BSWCWCYP.js.map → chunk-NFZZSF4M.js.map} +0 -0
- /package/dist/{dev-7FXVGFL6.js.map → ci-4SY2X2CU.js.map} +0 -0
- /package/dist/{doctor-E4KLPZZI.js.map → create-GM5H72LW.js.map} +0 -0
- /package/dist/{init-PFNPS6XO.js.map → dev-YNKPWJHX.js.map} +0 -0
- /package/dist/{lifecycle-G7OAC4ZZ.js.map → doctor-BKS7TUAT.js.map} +0 -0
- /package/dist/{login-ZEIH7KIH.js.map → domains-CR34APOD.js.map} +0 -0
- /package/dist/{push-3NDLL7US.js.map → experiment-reports-LL2TFQQN.js.map} +0 -0
- /package/dist/{results-HCXQH7IY.js.map → init-WHK4EV37.js.map} +0 -0
- /package/dist/{skills-UBPQOTDQ.js.map → lifecycle-3GUIJ74P.js.map} +0 -0
- /package/dist/{status-6IODQYHV.js.map → links-BQZLMGZJ.js.map} +0 -0
- /package/dist/{refs-IWZ5GP7Q.js.map → refs-5DTSKMXU.js.map} +0 -0
- /package/dist/{remove-YDJ7JTPR.js.map → remove-PYYPRP3R.js.map} +0 -0
- /package/dist/{report-JS7Z42FD.js.map → report-JVMKVZTS.js.map} +0 -0
- /package/dist/{review-BGGNVL6L.js.map → review-GIYBPVP3.js.map} +0 -0
- /package/dist/{ship-C22SGFUD.js.map → ship-3TDWAQIJ.js.map} +0 -0
- /package/dist/{update-SAHMBHIZ.js.map → update-XF7HWJ3A.js.map} +0 -0
|
@@ -0,0 +1,819 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/skills-self-hosted.ts
|
|
4
|
+
var SKILL_SELF_HOSTED = `---
|
|
5
|
+
name: splithub-self-hosted
|
|
6
|
+
description: Keep SplitHub events in your own database (self mode): which tables, how to write a Store for Postgres, MySQL, Prisma, Drizzle, Supabase or anything else, where the ingest route lives, how the CLI reads results from it, and how to get cloud data into your own database for reports. Use when the user wants data in their infra or asks about tables, schema or reporting.
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# SplitHub with your own database
|
|
10
|
+
|
|
11
|
+
Two ways to own the data:
|
|
12
|
+
|
|
13
|
+
1. **Self mode** \u2014 the app hosts the ingest route and the store; nothing leaves your infra.
|
|
14
|
+
\`splithub.config.ts\`: \`storage: { mode: 'self', endpoint: '/api/splithub' }\`.
|
|
15
|
+
2. **Cloud + export** \u2014 events go to SplitHub Cloud; you pull summaries into your warehouse for reports.
|
|
16
|
+
|
|
17
|
+
## Self mode: the pieces
|
|
18
|
+
|
|
19
|
+
- Route: \`app/api/splithub/[[...path]]/route.ts\` (Next.js) or any fetch-style handler from
|
|
20
|
+
\`createSplitHubHandler({ config, store, adminToken })\` in \`@splithub/node\`. It accepts
|
|
21
|
+
\`POST /events\` from the SDK and answers \`GET /status\`, \`GET /results/<id>\` for the CLI
|
|
22
|
+
(\`npx splithub status\` talks to the local dev server, or \`--url https://your.app/api/splithub\`).
|
|
23
|
+
Self-hosted reads are raw numbers per variant (\`variants[]\`: visitors, conversions, rate, revenue)
|
|
24
|
+
with \`analysis: { available: false }\`. Verdicts, data-quality stars and recommendations come with SplitHub Cloud,
|
|
25
|
+
and so do slices, breakdowns, timelines and raw experiment events (\`results --from/--filter/--by\` answer \`NOT_CLOUD\` here).
|
|
26
|
+
With an analytics property the SDK sends \`track()\` as one \`event\` row and a page as one \`page\` row; the handler
|
|
27
|
+
stores the goal rows experiments need next to them (goals of the config's experiments only), so the store sees goals as before.
|
|
28
|
+
- Store: an object with three methods (below). Ready-made: \`@splithub/adapter-sqlite\`
|
|
29
|
+
(Node 22.13+, \`node:sqlite\`, zero setup) and \`@splithub/adapter-d1\` (Cloudflare D1).
|
|
30
|
+
\`npx splithub init --mode self --db postgres|mysql|custom\` writes \`splithub/store.ts\` with a
|
|
31
|
+
working Postgres (pg) or MySQL (mysql2) store, or a documented stub for anything else.
|
|
32
|
+
- Admin token: set \`SPLITHUB_ADMIN_TOKEN\` in the app and \`--token\` / \`SPLITHUB_ADMIN_TOKEN\` for
|
|
33
|
+
the CLI, so results are not public.
|
|
34
|
+
|
|
35
|
+
## The events table
|
|
36
|
+
|
|
37
|
+
One table, one row per event. Column names are yours; the templates use these:
|
|
38
|
+
|
|
39
|
+
| column | type | meaning |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| id | text, primary key | client-generated, unique; dedupe on it (retries resend batches) |
|
|
42
|
+
| type | text | \`exposure\` or \`goal\` |
|
|
43
|
+
| experiment, variant | text | for exposures |
|
|
44
|
+
| goal | text | for goals |
|
|
45
|
+
| visitor_id, session_id | text | \`sh_vid\` cookie / session |
|
|
46
|
+
| ts | bigint | client time, unix ms |
|
|
47
|
+
| url | text | page |
|
|
48
|
+
| props | json | goal payload, e.g. \`{ "revenue": 49 }\` |
|
|
49
|
+
| received_at | bigint | server time, unix ms |
|
|
50
|
+
| host, country, device, browser | text | request metadata (fill from headers if you want breakdowns) |
|
|
51
|
+
|
|
52
|
+
Indexes: \`(type, experiment, visitor_id, ts)\` and \`(type, goal, visitor_id, ts)\`.
|
|
53
|
+
|
|
54
|
+
## The Store interface (\`@splithub/node\`)
|
|
55
|
+
|
|
56
|
+
\`\`\`ts
|
|
57
|
+
interface Store {
|
|
58
|
+
insertEvents(events: StoredEvent[]): Promise<number>; // skip existing ids, return inserted count
|
|
59
|
+
variantCounts(experiment: string, goal: string): Promise<VariantCounts[]>; // per variant: exposures, conversions, revenue
|
|
60
|
+
summaries(): Promise<ExperimentSummary[]>; // per experiment: exposures, visitors, first/last ts
|
|
61
|
+
close?(): Promise<void> | void;
|
|
62
|
+
}
|
|
63
|
+
\`\`\`
|
|
64
|
+
|
|
65
|
+
Attribution rule \u2014 keep it exactly, so self-hosted results equal the cloud's:
|
|
66
|
+
**a visitor converts for a variant when they sent the goal at or after their first exposure to
|
|
67
|
+
that variant; exposures and conversions count unique visitors.** The SQL in the generated
|
|
68
|
+
\`splithub/store.ts\` implements it (CTE \`first_exposure\`, then goals joined with \`ts >= first_ts\`).
|
|
69
|
+
|
|
70
|
+
Prisma / Drizzle: map the table in your schema and write the three methods with the ORM, or run the
|
|
71
|
+
one raw query for \`variantCounts\`. Supabase: the same SQL as a Postgres function called through
|
|
72
|
+
\`rpc\`. Mongo: \`$group\` by variant and visitor with \`$min ts\`, then \`$lookup\` goals.
|
|
73
|
+
|
|
74
|
+
## Reports from the cloud into your database
|
|
75
|
+
|
|
76
|
+
- CSV: dashboard \u2192 experiment \u2192 Export (summary, daily timeline, raw events up to 50k rows).
|
|
77
|
+
- Webhook channel for alerts and the digest: POST JSON to your endpoint when something happens.
|
|
78
|
+
- Scheduled pull: a cron in your app calls \`GET /v1/status\` and \`GET /v1/results/<id>\` with the
|
|
79
|
+
API key (\`sk_\u2026\`) and stores the verdicts. Raw events live 30\u201390 days in the cloud (plan dependent),
|
|
80
|
+
aggregates a year: pull summaries, not raw rows, for long-term reporting.
|
|
81
|
+
- Need every raw event forever in your infra? Use self mode; one store per project.
|
|
82
|
+
|
|
83
|
+
## Do not
|
|
84
|
+
|
|
85
|
+
- Put database credentials in \`splithub.config.ts\` (it ships to the browser). They belong in the route's env.
|
|
86
|
+
- Call a winner from raw rates: a higher rate is often noise. Say the numbers as they are; verdicts come with SplitHub Cloud.
|
|
87
|
+
`;
|
|
88
|
+
|
|
89
|
+
// src/skills-ship.ts
|
|
90
|
+
var SKILL_SHIP = `---
|
|
91
|
+
name: splithub-ship
|
|
92
|
+
description: Ship the winner of an experiment and remove the experiment from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when an experiment has a verdict and the user says ship it, roll out the winner, remove the experiment, or clean up; and whenever an experiment shows status finished and the user agreed on which variant to keep.
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
# Shipping the winner
|
|
96
|
+
|
|
97
|
+
Goal: the app behaves exactly as the kept variant did, with no experiment left in the code or in the config.
|
|
98
|
+
|
|
99
|
+
## 1. Decide
|
|
100
|
+
|
|
101
|
+
\`npx splithub results <id> --json\` (MCP: \`splithub_results\`). Act on \`verdict.status\`:
|
|
102
|
+
- \`winner_found\`: keep \`verdict.winner\`.
|
|
103
|
+
- \`no_significant_difference\`: keep the control, or the simpler variant if the user agrees.
|
|
104
|
+
- \`insufficient_sample\`: do not ship. Say when to look again: \`daysToVerdict\` days at the current traffic.
|
|
105
|
+
- \`srm_detected\`: do not ship. The split is broken; fix the setup first.
|
|
106
|
+
- \`guardrail_breached\`: do not ship the variant in \`verdict.guardrailBreached\`; it made a guardrail worse. Say what broke.
|
|
107
|
+
Ask the user which variant to keep before going on. Never pick from raw rates.
|
|
108
|
+
|
|
109
|
+
## 2. Preview, then ship
|
|
110
|
+
|
|
111
|
+
\`\`\`bash
|
|
112
|
+
npx splithub ship <id> --keep <variant> --dry-run --json # diff + leftovers, nothing written
|
|
113
|
+
npx splithub ship <id> --keep <variant> --json # apply
|
|
114
|
+
\`\`\`
|
|
115
|
+
MCP: \`splithub_ship\` with \`dryRun: true\`, then without. It archives the experiment in the cloud (numbers stay readable), removes it from
|
|
116
|
+
\`splithub.config.ts\` (checked by reloading the file) and rewrites the code:
|
|
117
|
+
- \`<Experiment id>\` \u2192 the kept \`<Variant>\`'s children; the other variants are dropped.
|
|
118
|
+
- \`useValue('<id>')\` / \`sh.value('<id>')\` \u2192 the kept value as a literal.
|
|
119
|
+
- \`useVariant('<id>')\` comparisons, ternaries, \`&&\`, \`if\` and \`switch\` \u2192 the kept branch.
|
|
120
|
+
- Unused \`Experiment\`, \`Variant\`, \`useVariant\`, \`useValue\` imports go.
|
|
121
|
+
|
|
122
|
+
## 3. Finish by hand
|
|
123
|
+
|
|
124
|
+
- \`leftovers[]\`: each \`{ file, line, text, reason }\` still mentions the experiment; rewrite those yourself.
|
|
125
|
+
- component kind: delete components and files only the losing variants used (e.g. \`splithub/<id>/\`).
|
|
126
|
+
- redirect kind: make the kept URL the real page (move the route or add a permanent redirect), delete the other page.
|
|
127
|
+
- Goal: keep \`track('<goal>')\` when another experiment measures it (\`next\` says so); otherwise ask before removing it.
|
|
128
|
+
- Cloud mode: \`npx splithub push\`. Then \`npx splithub doctor\` and the project's typecheck and tests.
|
|
129
|
+
- \`npx splithub refs <id> --json\` must return \`count: 0\` for the experiment.
|
|
130
|
+
|
|
131
|
+
Report what changed (files, the kept content) and the verdict in one line with its numbers. Never reuse the id.
|
|
132
|
+
`;
|
|
133
|
+
var SKILL_LOOP = `---
|
|
134
|
+
name: splithub-loop
|
|
135
|
+
description: The whole experiment loop for an agent with SplitHub, step by step with the MCP tools and CLI --json fields: plan, create, implement, check, push, read results, finish, ship. Use when the user asks to run an experiment end to end, or to pick up an experiment where it stands.
|
|
136
|
+
---
|
|
137
|
+
|
|
138
|
+
# The full loop
|
|
139
|
+
|
|
140
|
+
Each step: MCP tool, or the CLI with \`--json\` (one line of JSON; errors are \`{ error, code, hint }\`, exit 1).
|
|
141
|
+
|
|
142
|
+
| step | MCP tool | CLI | read |
|
|
143
|
+
|---|---|---|---|
|
|
144
|
+
| 0. size it | \`splithub_plan\` | \`plan --baseline 0.05 --mde 0.1 [<id>]\` | \`perVariant\`, \`total\`, \`traffic.daysLeft\` |
|
|
145
|
+
| 1. create | \`splithub_experiment_create\` | \`create --kind \u2026 --id \u2026 --goal \u2026 --yes\` | \`snippet\`, \`files\`, \`nextStep\`, \`handoff\` |
|
|
146
|
+
| 2. implement | (edit code) | | wire \`snippet\`; \`track('<goal>')\` where it converts |
|
|
147
|
+
| 3. check | \`splithub_doctor\`, \`splithub_refs\` | \`doctor\`, \`refs <id>\` | \`ok\`, \`checks[]\`; \`refs[]\` shows the wiring |
|
|
148
|
+
| 4. push (cloud) | \`splithub_push\` | \`push\` (\`--dry-run\` to preview) | \`adopted\`, \`diff\` |
|
|
149
|
+
| 5. results | \`splithub_status\`, \`splithub_results\` | \`status\`, \`results <id>\` | \`verdict.status\`, \`verdict.message\`, \`verdict.notes[]\`, \`daysToVerdict\` |
|
|
150
|
+
| 6. finish | \`splithub_experiment_status\` | \`finish <id>\` | \`live\`, \`next\` |
|
|
151
|
+
| 7. ship | \`splithub_ship\` | \`ship <id> --keep <v> [--dry-run]\` | \`diff\`, \`leftovers[]\`, \`next\` |
|
|
152
|
+
|
|
153
|
+
Rules that keep the answer honest:
|
|
154
|
+
- Plan before creating: if \`traffic.daysLeft\` is months, try a bigger change or a busier page.
|
|
155
|
+
- \`create\` writes a running experiment; it goes live when the code is deployed (and pushed in cloud mode). Tell the user.
|
|
156
|
+
- Verdicts are sequential (always-valid): checking any day is safe. Quote \`verdict.message\` and the always-valid
|
|
157
|
+
\`comparisons[].sequential.pValue\`, not the fixed-horizon \`pValue\` (p = 0.01 there can still be not significant).
|
|
158
|
+
Relay \`verdict.notes[]\`. Verdicts depend on sample size, not days. Say \`verdict.sample.quality\` (small | ok | good;
|
|
159
|
+
good = every variant has its share of \`goodSample\`, default 5000 visitors, \`create --good-sample\`) next to the verdict.
|
|
160
|
+
- \`daysToVerdict\` is the re-check date: schedule (or tell the user) a look in that many days instead of polling.
|
|
161
|
+
- \`srm_detected\`: the split is broken; fix the setup, do not read the numbers.
|
|
162
|
+
- \`guardrail_breached\`: don't ship. A variant made a guardrail worse (\`verdict.guardrailBreached[]\`), e.g. "bold wins on
|
|
163
|
+
signups but raises refunds". \`verdict.metrics[]\` has the secondary metrics and guardrails, read like the main table.
|
|
164
|
+
- Guard what matters when creating: \`--secondary pricing_view --guardrail refunds:higher\` (MCP: \`secondary\`, \`guardrails\`).
|
|
165
|
+
Experiments that could clash on one page go in one layer: \`--layer checkout:0-50\` and \`checkout:50-100\` never share a visitor.
|
|
166
|
+
- \`splithub holdout set 5\` keeps 5% of visitors out of every experiment; the \`__holdout\` row ("Holdout: long-term impact")
|
|
167
|
+
compares them with everyone else.
|
|
168
|
+
- Ask the user which variant to keep before \`ship\`; preview with \`dryRun\` and show them the diff.
|
|
169
|
+
- Error \`code\` \u2192 action: \`NOT_LOGGED_IN\` \u2192 \`splithub_login\`, show the link, wait, retry \xB7 \`INSUFFICIENT_SCOPE\` \u2192 the key
|
|
170
|
+
can only read; ask for a key with write scope \xB7 \`NO_CONFIG\` \u2192 \`splithub_init\` \xB7
|
|
171
|
+
\`EXPERIMENT_NOT_FOUND\` \u2192 \`splithub_status\` for ids \xB7 \`CONFIG_UNSAFE\` \u2192 edit the config by hand where the experiment is
|
|
172
|
+
defined \xB7 \`CONFIG_WRITE\` \u2192 the file was restored; edit by hand, then \`doctor\`.
|
|
173
|
+
`;
|
|
174
|
+
|
|
175
|
+
// src/skills-analytics.ts
|
|
176
|
+
var SKILL_ANALYTICS = `---
|
|
177
|
+
name: splithub-analytics
|
|
178
|
+
description: Collect and read web analytics with SplitHub: analytics properties (pageviews, events with track(), identified users, sources, channels, pages, goals and conversions), setting one up in the app (analytics connect), funnels, paths, retention, realtime, event props, CSV export, segments, notes and alerts on traffic, sh.js or the SDK, allowed domains. Use when the user wants traffic numbers, page or source reports, event tracking, funnels, or asks to "add analytics" to an app that uses SplitHub or could.
|
|
179
|
+
---
|
|
180
|
+
|
|
181
|
+
# SplitHub analytics
|
|
182
|
+
|
|
183
|
+
An analytics property is a stream of pageviews and custom events, like a site in Google Analytics. It sits next to the
|
|
184
|
+
experiments: same config file, same CLI, same key, same storage choice (SplitHub Cloud or the app's own database).
|
|
185
|
+
There are no variants and no verdicts: reports are counts.
|
|
186
|
+
|
|
187
|
+
## Set it up
|
|
188
|
+
|
|
189
|
+
1. One step: \`npx splithub analytics connect --json\` (or Analytics \u2192 Set up in the \`npx splithub\` menu) picks the project's
|
|
190
|
+
property (or creates one: \`--name "Marketing site"\`; a given id is used, \`connect sh-x1h78w\`) and adds
|
|
191
|
+
\`property="sh-\u2026"\` to the one \`<SplitHubProvider>\` / \`<SplitHubNextProvider>\` in the code. It answers \`wired.status\`:
|
|
192
|
+
added | already | other (another property is named there: ask, then \`--force\`) | none | many (add \`snippets.provider\`
|
|
193
|
+
or \`snippets.script\` yourself), plus \`snippets.track\` and \`next\`.
|
|
194
|
+
By hand: \`npx splithub analytics create --name "Marketing site" --json\` writes \`properties: { 'sh-x1h78w': { name: \u2026 } }\`
|
|
195
|
+
into \`splithub.config.ts\` (self-hosted projects keep their properties there). Cloud mode: \`npx splithub push\` after
|
|
196
|
+
either. A property made in the dashboard needs no push; its id is on its page.
|
|
197
|
+
2. Send events. Events go only to a property named by its id, never to a guess:
|
|
198
|
+
- SDK: \`<SplitHubProvider config={config} property="sh-x1h78w">\`: a pageview on load and on every client-side route change.
|
|
199
|
+
- Events: one call, \`const track = useTrack(); track('Video Played', { id: 3 })\` (or \`client.track(...)\`). Names: letters,
|
|
200
|
+
digits, spaces and \`_ . : / -\`, up to 64 characters; props: a flat object up to 4 KB. The same call counts for
|
|
201
|
+
experiments whose goal is that event. Only what the app passes to \`track()\` is sent.
|
|
202
|
+
- Users: \`const identify = useIdentify(); identify(user.id, { plan: 'pro' })\` after sign-in (\`identify(null)\` on sign-out;
|
|
203
|
+
sh.js: \`splithub.identify(id, traits)\`) ties visits to the app's user; reports then count \`users\`. No emails or names in traits.
|
|
204
|
+
- No SDK: \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026" async></script>\` sends pageviews (SPA navigations
|
|
205
|
+
included) and \`window.splithub.track(name, props)\` once the tag has \`data-property="sh-x1h78w"\`.
|
|
206
|
+
- One install covers analytics and experiments: the provider (or sh.js) with \`property\` collects pageviews and runs
|
|
207
|
+
experiments. Each \`track()\` sends one event row and each page one page row; experiments whose goal is that event or
|
|
208
|
+
page count them from those rows, so nothing extra is sent or billed. Never add sh.js and the SDK for the same
|
|
209
|
+
property: when both run on a page, only the first collects pageviews.
|
|
210
|
+
- Servers and other languages: \`POST /v1/events\` with \`{ type: 'page', prop, url, vid, sid, ts, id, title?, ref? }\` or
|
|
211
|
+
\`{ type: 'event', prop, name, props?, \u2026 }\` (splithub-api skill).
|
|
212
|
+
3. Development never sends to the cloud (same rule as experiments): check a deploy or \`vite preview\`.
|
|
213
|
+
4. Events: any name is recorded, nothing to declare first. A goal is an event marked as one (\`goals\` in the config, the
|
|
214
|
+
goal switch on an event in the dashboard, or simply naming the event as an experiment's goal); goals count as
|
|
215
|
+
conversions in every property (a property's \`goals\` list narrows that). \`value\` is what one conversion is worth
|
|
216
|
+
when the event carries no \`revenue\` prop; \`event: 'Video Played'\` matches an event name that is not a valid key.
|
|
217
|
+
|
|
218
|
+
## Read it
|
|
219
|
+
|
|
220
|
+
- \`npx splithub analytics --json\`: every property with visitors, visits, pageviews and bounce for the last 30 days.
|
|
221
|
+
- \`npx splithub status --json\` (SplitHub Cloud) also lists every property under \`properties\`: the last 7 days against the 7 before
|
|
222
|
+
(visitors, \`change\`, goal conversions); \`npx splithub report\` adds them as an Analytics table. The digest (Pro, set in the
|
|
223
|
+
dashboard under Alerts) covers properties too: visitors, top sources and pages, goals and events against the period before.
|
|
224
|
+
- \`npx splithub analytics site --json\` (\`--days 7\`, or \`--from 2026-10-01 --to 2026-10-31\`): totals, \`users\` (identified),
|
|
225
|
+
goals (conversions = unique visitors who reached the goal in the range, completions, revenue), top pages, top sources, notes.
|
|
226
|
+
- \`--by page|source|channel|referrer|utm_source|utm_medium|utm_campaign|country|device|browser|os|entry|exit|event|goal\`; with a
|
|
227
|
+
visit dimension, \`--goal signup\` adds conversions per row; \`--by variant --experiment hero\` splits visits by the
|
|
228
|
+
variant each visitor saw, \`--by trait --key plan\` by an identify() trait. \`--filter country=DE,FR --filter source="(none)"\` narrows any report.
|
|
229
|
+
- \`npx splithub analytics events site --json\`: the latest raw pageviews and events, to check tracking right after wiring it.
|
|
230
|
+
- More reports (SplitHub Cloud), each with the same \`--days\` / \`--from --to\` / \`--filter\` flags:
|
|
231
|
+
- \`analytics funnel site --steps "page:/,event:Signup,goal:paid" [--within visit|7d|30d] [--by country]\`: visitors per step
|
|
232
|
+
in order, \`conversion\` (share of step 1) and \`dropoff\` (share lost since the step before).
|
|
233
|
+
- \`analytics paths site --page /pricing [--prev]\`: where visits went next (or came from).
|
|
234
|
+
- \`analytics retention site [--start visit|event:<name>|goal:<key>] [--return visit|event:<name>] [--period day|week]\`: cohorts
|
|
235
|
+
and how many came back per period.
|
|
236
|
+
- \`analytics realtime site\`: visitors in the last 5 minutes, top pages and sources.
|
|
237
|
+
- \`analytics props site --event Signup [--key plan]\`: an event's prop names, or one prop's values.
|
|
238
|
+
- \`analytics export site --by page|day [--raw] > report.csv\`: CSV to stdout.
|
|
239
|
+
- \`--compare\` on a report adds the period right before (\`previous\`), \`--compare year\` the same days a year earlier, \`--compare 2026-08-01..2026-08-31\` chosen days; \`--segment "Paid traffic"\` applies a saved segment
|
|
240
|
+
(\`analytics segments\`, \`--save <name> country=DE\`, \`--delete <id|name>\`).
|
|
241
|
+
- Notes mark a day on the charts: \`npx splithub notes add "Launched new pricing" [--kind win] [--property site] [--day 2026-10-05]\`,
|
|
242
|
+
\`notes --property site\` lists them; add one when you ship something that may move traffic.
|
|
243
|
+
- Alerts on a property (SplitHub Cloud, Pro): \`npx splithub alerts add traffic_drop --property site --percent 50\`, \`traffic_spike\`,
|
|
244
|
+
\`goal_silent --property site --goal signup --hours 24\`, \`event_count --property site --event Signup --count 1000 [--repeat]\`.
|
|
245
|
+
- Words: a visit (\`sessions\`) ends after 30 minutes without events; a bounce is a visit with one page and no event;
|
|
246
|
+
\`source\` is the UTM source or the referrer's site (Google, Facebook, \u2026), \`(none)\` = direct; \`channel\` groups them (Direct,
|
|
247
|
+
Organic Search, Paid Social, Email, Referral, \u2026); a conversion is a visitor reaching a goal (once per visitor in the range);
|
|
248
|
+
\`duration\` is in milliseconds. Bots are never counted.
|
|
249
|
+
- Self-hosted (\`mode: 'self'\`): the same commands give raw counts from the app's database (pageviews, visitors,
|
|
250
|
+
sessions, events by name) for \`--days\` or \`--from --to\` (UTC days). Filters, \`--by\`, \`--compare\` and reports with sources,
|
|
251
|
+
pages, visits and conversions come with SplitHub Cloud: say so.
|
|
252
|
+
|
|
253
|
+
## Manage it
|
|
254
|
+
|
|
255
|
+
- \`npx splithub analytics pause site\` / \`resume site\`: live in the cloud within a minute; paused events are dropped and not billed.
|
|
256
|
+
- \`npx splithub analytics archive site\` / \`restore site\` (SplitHub Cloud): like Archive in the dashboard, it stops collecting and leaves
|
|
257
|
+
the list (\`archived\` in it), numbers kept; restored comes back paused.
|
|
258
|
+
- \`npx splithub analytics remove site\`: out of the config (push next); a dashboard-made property is deleted with its numbers.
|
|
259
|
+
- Allowed domains are per experiment and per property: \`npx splithub domains set property site example.com\` (subdomains included),
|
|
260
|
+
\`npx splithub domains clear property site\` (any domain again), \`npx splithub domains\` lists them all. An empty list
|
|
261
|
+
accepts every origin. Add localhost too when testing from it.
|
|
262
|
+
- Billing: every pageview and custom event counts like one exposure against the project's allowance.
|
|
263
|
+
|
|
264
|
+
## Do not
|
|
265
|
+
|
|
266
|
+
- Put personal data in event props (emails, names, ids from other systems).
|
|
267
|
+
- Fire an event in a render loop or on every scroll: each one is stored and billed.
|
|
268
|
+
- Read analytics counts as an experiment result: use \`splithub results\` for experiments.
|
|
269
|
+
`;
|
|
270
|
+
|
|
271
|
+
// src/skills.ts
|
|
272
|
+
var SKILL_SPLITHUB = `---
|
|
273
|
+
name: splithub
|
|
274
|
+
description: Add, change, run or read experiments in an app that uses SplitHub (@splithub/react, splithub.config.ts, the splithub CLI, SplitHub Cloud). Use when the user asks to A/B test a component, headline, image, price, CTA or page, asks which variant is winning, or wants to start, pause or finish an experiment; also for notes on the charts, alert rules (events used, traffic, goals, winners), short links and allowed domains.
|
|
275
|
+
---
|
|
276
|
+
|
|
277
|
+
# SplitHub for coding agents
|
|
278
|
+
|
|
279
|
+
Three pieces: \`splithub.config.ts\` (the experiments; source of truth when it exists),
|
|
280
|
+
\`@splithub/react\` (renders variants), the \`splithub\` CLI (creates, pushes, reads results).
|
|
281
|
+
Events go to the app's own database (self mode) or to SplitHub Cloud (cloud mode).
|
|
282
|
+
|
|
283
|
+
## Where things live
|
|
284
|
+
|
|
285
|
+
- Config: \`splithub.config.ts\` in the project root. Experiments and goals. Ships to the browser: never put secrets in it.
|
|
286
|
+
- Credentials: \`npx splithub login\` (or \`init --mode cloud\`) prints a link; the human approves it in the browser; the CLI
|
|
287
|
+
stores a project API key for this folder. Never ask the user to paste keys. CI uses \`SPLITHUB_API_KEY\`.
|
|
288
|
+
- The public key (\`pk_\`) ends up in \`storage.key\` of the config from any of: \`login\` (saved credentials), the first
|
|
289
|
+
\`push\`, \`init --key\`, or \`SPLITHUB_API_KEY\` in the environment (init asks the cloud). Missing key = events go nowhere.
|
|
290
|
+
- Cloud: \`npx splithub push\` uploads the config; \`npx splithub pull\` writes dashboard-created experiments into the file.
|
|
291
|
+
- Results: \`npx splithub status --json\` (in the cloud it also has \`properties\`: each analytics property's last 7 days), \`npx splithub results <id> --json\`. Sizing: \`npx splithub plan --baseline 0.05 --mde 0.1\`.
|
|
292
|
+
- Closer reads (cloud): \`results <id> --from --to --filter country=DE\` (a slice), \`--by <dimension>\`, \`--timeline\`, \`--engagement\`,
|
|
293
|
+
\`--events\`, \`--export --by <dimension>\` or \`--export --events\` (CSV); \`npx splithub events --experiment <id>\` for the raw tail. See "Read results".
|
|
294
|
+
- Code that uses an experiment: \`npx splithub refs <id> --json\`. Keep a variant and remove the experiment: \`npx splithub ship <id> --keep <v>\`.
|
|
295
|
+
- \`--json\` prints one line; errors are \`{ error, code, hint }\` with exit code 1. \`INSUFFICIENT_SCOPE\` adds \`scope\`
|
|
296
|
+
(e.g. \`notes:write\`): the API key is limited to some areas; ask the user for a key with that scope. The \`splithub-loop\` skill has the whole flow.
|
|
297
|
+
- Everything interactive: \`npx splithub\`, one menu: Experiments, Analytics, Short links, Alerts, Notes, Allowed domains,
|
|
298
|
+
push / pull, local testing, doctor, agent skills. Everything scriptable: subcommands + \`--json\` (\`npx splithub --help\`).
|
|
299
|
+
- Web analytics (pageviews, events, sources, pages, funnels) live in analytics properties: \`npx splithub analytics connect\`
|
|
300
|
+
sets one up in the app; the \`splithub-analytics\` skill has the rest.
|
|
301
|
+
|
|
302
|
+
## Pick the kind
|
|
303
|
+
|
|
304
|
+
| kind | use it when | create | read in code |
|
|
305
|
+
|---|---|---|---|
|
|
306
|
+
| \`value\` | only data changes: text, number, image URL, price, JSON. The default choice: no new components. | \`--variants control,bold --value control="Old text" --value bold="New text"\` | \`const text = useValue<string>('<id>')\`; server components: \`getValue\` from \`@splithub/next\`; no SDK: \`sh.value('<id>')\` |
|
|
307
|
+
| \`component\` | structure changes: a layout, a section, a flow, a different component per variant | \`--variants control,bold\` (the CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs) | \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`, or \`useVariant('<id>')\` in a switch |
|
|
308
|
+
| \`redirect\` | a whole page changes: another URL per variant | \`--match / --to control=/ --to v2=/home-v2\` | nothing: \`@splithub/next\` middleware or the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` sends visitors there |
|
|
309
|
+
| \`mvt\` | several things change at once (headline \xD7 button \xD7 image) and the user wants to know which of them matters; every combination is tested and each factor is read on its own | \`--factor headline=old:"Old text",new:"New text" --factor button=blue:#2563eb,green:#16a34a\` (no \`--variants\`; first level = control; at most 4 factors \xD7 4 levels, 16 combinations) | \`const { headline, button } = useValue<{ headline: string; button: string }>('<id>')\` |
|
|
310
|
+
|
|
311
|
+
mvt needs more visitors than a two-variant experiment: every combination must reach the minimum on its own (the results show the
|
|
312
|
+
combinations and, below them, each factor summed over its combinations). With little traffic, change one thing with \`value\`.
|
|
313
|
+
|
|
314
|
+
## Create an experiment
|
|
315
|
+
|
|
316
|
+
1. \`npx splithub create --kind <value|component|redirect|mvt> --id <key> \u2026 --goal <event name> --yes --json\` (see the table).
|
|
317
|
+
The key is the experiment's id for good: never reuse one from an earlier experiment. \`--name "Signup button"\` adds a human
|
|
318
|
+
name (dashboard, CLI) that can change any time. \`push\` fails with \`KEY_CONFLICT\` when a key already belongs to
|
|
319
|
+
another experiment: ask the user, then rename the key, or push with \`--replace <key>\` (new experiment, old numbers cleared)
|
|
320
|
+
or \`--same <key>\` (same experiment). \`KEY_DELETED\`: a deleted experiment used the key; pick a new one.
|
|
321
|
+
2. Goal: any event the app sends with \`track('<name>', props?)\`, the one call for every event. Nothing to declare
|
|
322
|
+
first: a name not in \`goals\` becomes an event goal keyed by its slug (\`Video Played\` \u2192 \`video-played\`) and \`create\`
|
|
323
|
+
says so. A goal is just an event marked as one; page visits and clicks are goals declared in \`goals\`:
|
|
324
|
+
\`{ type: 'pageview', path: '/thanks' }\`, \`{ type: 'click', selector: '#buy' }\` (tracked automatically by the SDK and the script).
|
|
325
|
+
With an analytics property on the provider, the same \`track()\` and pageviews feed both: one install, one row per action.
|
|
326
|
+
3. Wire it in code. \`create\` writes \`status: 'running'\`: the experiment goes live for every visitor, first page view
|
|
327
|
+
included, as soon as this code is deployed. Use \`--status draft\` only when the user asks to hold it.
|
|
328
|
+
4. Cloud mode: \`npx splithub push\` after any config change.
|
|
329
|
+
5. Hand-off, always: end your reply by saying the experiment is **running in the config** and goes live when the user deploys
|
|
330
|
+
this code; nothing is recorded before that, and local development never sends events to the cloud. Tell them how to
|
|
331
|
+
check it locally (\`?sh_force=<id>:<variant>\`) and that Pause in the dashboard or \`npx splithub pause <id>\` holds it.
|
|
332
|
+
In cloud mode a status set in the dashboard wins over the file for every visitor, first page included.
|
|
333
|
+
For a draft (only if asked): say visitors do not see it and nothing is recorded until \`npx splithub start <id>\`.
|
|
334
|
+
|
|
335
|
+
A draft is invisible to visitors. To check one before starting: \`runDraftsInDev: true\` at the top level of the config
|
|
336
|
+
serves drafts like running experiments in development only (\`NODE_ENV=development\`: \`next dev\`, \`vite\`);
|
|
337
|
+
\`?sh_force=<id>:<variant>\` shows one variant on any page, draft or not, without recording an exposure;
|
|
338
|
+
\`?sh_debug=1\` (or \`npx splithub dev\`) logs every decision and opens a panel to switch variants.
|
|
339
|
+
In development the SDK explains drafts, paused experiments and held-back events in one line each. Vite apps: add \`splithub()\`
|
|
340
|
+
from \`@splithub/core/vite\` to the plugins in vite.config so these lines and a summary of every experiment appear in the
|
|
341
|
+
terminal that runs \`vite\`; Next.js server code prints them in its terminal anyway.
|
|
342
|
+
|
|
343
|
+
Keys: lowercase a-z 0-9 - _, max 64 chars. First variant = control. Weights are relative (50/50, 1/1, 90/10).
|
|
344
|
+
Targeting (country, device, UTM, cookie, time) is a \`flow\` graph on the experiment, edited in the dashboard; \`pull\` brings it into the file.
|
|
345
|
+
|
|
346
|
+
## Read results
|
|
347
|
+
|
|
348
|
+
\`verdict.status\` \u2208 no_data | insufficient_sample | srm_detected | winner_found | no_significant_difference.
|
|
349
|
+
Report \`verdict.message\` verbatim and the numbers behind it (visitors per variant, lift, the always-valid
|
|
350
|
+
\`comparisons[].sequential.pValue\` and \`sequential.ci\`; the plain \`pValue\` is fixed-horizon, for display only) and relay
|
|
351
|
+
\`verdict.notes[]\`. Verdicts are sequential and depend on sample size, not days: safe to check any time. \`verdict.sample.quality\` is
|
|
352
|
+
small (below 100 visitors per variant or 50 conversions), ok, or good (every variant has its share of \`goodSample\`, default 5000
|
|
353
|
+
visitors; \`create --good-sample\`); say it next to the verdict. \`verdict.sample.stars\` (0..5) scores data quality from
|
|
354
|
+
\`verdict.sample.checks[]\`: report the stars and the unmet labels; they never override the verdict. \`daysToVerdict\` says when to look again.
|
|
355
|
+
Never declare a winner from raw rates. \`srm_detected\` means the split is broken: fix the setup before anything else.
|
|
356
|
+
|
|
357
|
+
### Slices, breakdowns, timelines (SplitHub Cloud)
|
|
358
|
+
|
|
359
|
+
- A slice: \`npx splithub results <id> --from 2026-10-01 --to 2026-10-15 --filter country=DE --filter channel="Paid Search" --json\`.
|
|
360
|
+
\`from\` / \`to\` are first-exposure days (project time zone): the visitors who entered the experiment then, with all their
|
|
361
|
+
later conversions. Filters: country, region, city, device, os, browser, language, host, source, channel, utm_source,
|
|
362
|
+
utm_medium, utm_campaign, utm_term, utm_content, landing or page (the page of the first exposure); comma-separated values,
|
|
363
|
+
\`(none)\` matches empty. The answer carries \`scope\` and its verdict is computed on the slice only: say so and quote
|
|
364
|
+
\`scope.note\`. The decision stays with the unfiltered verdict; a slice carries no \`voice\` (no advice).
|
|
365
|
+
- One view at a time, same range and filters (\`--json\` adds \`view\`): \`--by country|device|source|channel|page|landing|utm_campaign|\u2026\`
|
|
366
|
+
gives \`segments[]\` with each variant's cells and its lift against the control inside the segment (\`significant\`, \`better\`,
|
|
367
|
+
\`small\` = too few visitors); \`--timeline\` gives visitors and conversions per day and variant (a day = visitors first seen
|
|
368
|
+
that day); \`--engagement\` gives bounce rate, pages per visit and visit length per variant from the analytics property on
|
|
369
|
+
the same visitors (\`property: null\` = none); \`--events\` gives \`rows[]\` for every goal (the experiment's goal, metrics and
|
|
370
|
+
guardrails first, \`measured\`) and event of these visitors, each with \`perVariant\` visitors and rate and \`comparisons[]\` against
|
|
371
|
+
the control (\`property: null\` = goals only: other events need analytics on the same site). \`--export --by <dimension>\` prints
|
|
372
|
+
the breakdown as CSV, \`--export --events\` the events table.
|
|
373
|
+
- \`npx splithub events --experiment <id> [--type exposure|goal] [--limit 50] [--before <nextBefore>]\` lists the raw exposures
|
|
374
|
+
and goals (same filters; 7 days, 30 on Pro). Use it to check the wiring.
|
|
375
|
+
- Segments and events are for hypotheses: never call a winner from one segment or one event. With several segments, one will look great by chance.
|
|
376
|
+
A self-hosted endpoint answers none of these (\`NOT_CLOUD\`); its results stay raw counts.
|
|
377
|
+
|
|
378
|
+
## Finish, and what to do when an experiment is finished
|
|
379
|
+
|
|
380
|
+
\`npx splithub finish <id>\` stops the experiment (live within a minute in cloud mode). A finished experiment keeps its numbers for as long
|
|
381
|
+
as it stays in the config; the code still contains every variant until someone removes it.
|
|
382
|
+
|
|
383
|
+
Whenever you see an experiment with \`status: finished\` (in \`status\`, \`results\`, the MCP tools, or right after you finished it):
|
|
384
|
+
1. Read \`npx splithub results <id> --json\` and tell the user the verdict in one line with its numbers.
|
|
385
|
+
2. Ask two questions, do not assume: **which variant to keep** (recommend \`verdict.winner\`; with no significant
|
|
386
|
+
difference, the control or the simpler one) and **whether to remove the experiment from the code now**.
|
|
387
|
+
3. On yes, follow the \`splithub-ship\` skill: \`npx splithub ship <id> --keep <variant> --dry-run\`, show the diff, then run it
|
|
388
|
+
without \`--dry-run\`; fix the \`leftovers\`, \`push\` in cloud mode, run \`doctor\` and the project's checks.
|
|
389
|
+
4. On no, leave everything: a finished experiment costs nothing and the user may come back to it.
|
|
390
|
+
|
|
391
|
+
A \`paused\` experiment is not over: say so and ask whether to resume (\`start\`) or finish it.
|
|
392
|
+
|
|
393
|
+
## Around experiments (SplitHub Cloud)
|
|
394
|
+
|
|
395
|
+
- **Notes**: what happened on a day, marked on experiment timelines and analytics charts. \`npx splithub notes add "Launched the
|
|
396
|
+
new pricing page" [--kind note|issue|win] [--experiment <key> | --property <id>] [--day 2026-10-05]\` (none = the whole
|
|
397
|
+
project); \`notes\` lists them (\`--experiment\`, \`--property\`, \`--kind\`, \`--days\`), \`notes edit <id> "<text>"\`, \`notes delete <id>\`.
|
|
398
|
+
Add one when you ship something that may move the numbers.
|
|
399
|
+
- **Alerts** (they fire on Pro; added on another plan they are saved switched off): \`npx splithub alerts\` lists rules, progress
|
|
400
|
+
and recent alerts; \`alerts add <condition>\`: experiments \`srm\`, \`winner\`, \`loser --lift 10\`, \`exposures --visitors N\`,
|
|
401
|
+
\`conversions --conversions N\`, \`duration --days N\`, \`no_events --hours 24\` (\`--experiment <key>\`, else every running one;
|
|
402
|
+
\`--pause\`, \`--finish\`, \`--drop-loser\` act on it); account \`quota --percent 80\`, \`usage --every 1000\` (each time this month's
|
|
403
|
+
events cross another 1,000); analytics \`traffic_drop|traffic_spike --property <id> --percent 50\`, \`goal_silent --property <id>
|
|
404
|
+
--goal <key> --hours 24\`, \`event_count --property <id> [--event <name>] --count 500 [--repeat]\`. \`alerts enable|disable|delete|test <id>\`
|
|
405
|
+
(\`test\` = what would fire now, nothing sent). Channels (email, Slack, Discord, webhook) are set in the dashboard.
|
|
406
|
+
- **Short links**: \`npx splithub links create https://example.com/spring [--slug spring] [--name "Spring"]\`; several URLs with
|
|
407
|
+
\`--weights 70,30\` split visitors between pages. \`links\` lists them with clicks, \`links <slug>\` shows one link's clicks,
|
|
408
|
+
\`links pause|resume|delete <slug>\`.
|
|
409
|
+
- **Allowed domains**: \`npx splithub domains set experiment <key> example.com\` (subdomains included), \`domains clear experiment <key>\`,
|
|
410
|
+
\`domains\` lists them. No list = any origin.
|
|
411
|
+
|
|
412
|
+
## Do not
|
|
413
|
+
|
|
414
|
+
- Nest an \`<Experiment>\` inside a \`<Variant>\` of the same id; call hooks conditionally.
|
|
415
|
+
- Change \`seed\`, weights or variants of a running experiment (reshuffles visitors). Create a new experiment instead.
|
|
416
|
+
- Edit in the dashboard what the config file owns: file experiments are read-only there (pause/finish still work).
|
|
417
|
+
- Run more than 2-3 experiments on one page of a low-traffic site.
|
|
418
|
+
- Put secrets, API keys or database credentials in the config or in the browser. \`pk_\` is public, \`sk_\` is not.
|
|
419
|
+
`;
|
|
420
|
+
var SKILL_AB_TESTING = `---
|
|
421
|
+
name: ab-testing
|
|
422
|
+
description: How to design, run and read an experiment correctly: hypothesis, control, primary metric, sample size, significance, SRM, peeking, when to stop. Use before creating an experiment or when interpreting results, with any tool.
|
|
423
|
+
---
|
|
424
|
+
|
|
425
|
+
# A/B testing, the parts that matter
|
|
426
|
+
|
|
427
|
+
An experiment splits visitors at random into variants and compares one metric. It answers one question:
|
|
428
|
+
"does this change move this metric?" Everything below exists to keep that answer honest.
|
|
429
|
+
|
|
430
|
+
## Before creating an experiment
|
|
431
|
+
|
|
432
|
+
1. **One hypothesis, one primary metric.** "A shorter signup form raises signup completion." The metric is a goal
|
|
433
|
+
(event, page visit or click) that happens after the exposure. Pick it before looking at data.
|
|
434
|
+
2. **Control first.** The first variant is what exists today. Change one thing per variant; several unrelated changes
|
|
435
|
+
in one variant cannot be attributed.
|
|
436
|
+
3. **Traffic reality.** Rough rule: to detect a relative lift L on a baseline rate p you need about
|
|
437
|
+
\`16 \xB7 p(1-p) / (p\xB7L)\xB2\` visitors per variant (95% confidence, 80% power).
|
|
438
|
+
Examples: baseline 5%, lift 20% \u2192 ~7,600 per variant; baseline 2%, lift 10% \u2192 ~78,000 per variant.
|
|
439
|
+
If the site cannot reach that in 2-4 weeks, try a bigger change or a higher-traffic page.
|
|
440
|
+
4. **Few experiments per page.** Interactions between simultaneous experiments on the same page muddy both.
|
|
441
|
+
|
|
442
|
+
## While it runs
|
|
443
|
+
|
|
444
|
+
- **Do not peek and stop early.** Checking daily and stopping at the first "significant" moment inflates false
|
|
445
|
+
positives several times. Decide the sample or the duration up front; run whole weeks (weekday/weekend cycles).
|
|
446
|
+
- **SRM (sample ratio mismatch).** If the split is 50/50 and you see 60/40, the assignment or the tracking is broken
|
|
447
|
+
(bots, caching, redirect loops, a variant that errors). Fix first; results from an SRM run are meaningless.
|
|
448
|
+
- **Keep variants frozen.** Changing copy, weights or targeting mid-run reshuffles who saw what. Start a new experiment.
|
|
449
|
+
- **Watch the guardrails.** A variant that wins the primary metric but breaks revenue, errors or latency is not a win.
|
|
450
|
+
|
|
451
|
+
## Reading results
|
|
452
|
+
|
|
453
|
+
- Significance (p < 0.05 after correction for several variants) says the difference is unlikely to be noise;
|
|
454
|
+
the confidence interval says how big it plausibly is. A "winner" with a CI that touches 0 is a weak win.
|
|
455
|
+
- "No significant difference" with a sample smaller than needed means **keep running**, not "no effect".
|
|
456
|
+
With enough sample it means the change does not matter: ship whichever is cheaper or cleaner.
|
|
457
|
+
- Relative lift is on the baseline rate: 5% \u2192 6% is +20% relative, +1 point absolute. Report both.
|
|
458
|
+
- Attribution: conversions count only after the visitor's first exposure, per unique visitor.
|
|
459
|
+
- Segments (country, device) are for hypotheses, not verdicts: slicing after the fact finds noise.
|
|
460
|
+
|
|
461
|
+
## When to stop
|
|
462
|
+
|
|
463
|
+
- Winner found with the planned sample \u2192 ship it, finish the experiment, remove the losing code.
|
|
464
|
+
- Planned sample reached, no difference \u2192 ship the simpler variant, finish.
|
|
465
|
+
- SRM or broken tracking \u2192 stop, fix, restart as a new experiment.
|
|
466
|
+
- The business moved on \u2192 finish; do not leave experiments running forever (they cost events and split traffic).
|
|
467
|
+
|
|
468
|
+
## Vocabulary
|
|
469
|
+
|
|
470
|
+
exposure = a visitor saw a variant \xB7 conversion = the goal happened after exposure \xB7 lift = relative change in rate \xB7
|
|
471
|
+
p-value = chance of seeing this difference by luck \xB7 CI = range the true difference likely lies in \xB7
|
|
472
|
+
SRM = the split differs from the configured weights \xB7 guardrail = a metric that must not get worse.
|
|
473
|
+
`;
|
|
474
|
+
var SKILL_SPLITHUB_API = `---
|
|
475
|
+
name: splithub-api
|
|
476
|
+
description: Use the SplitHub Cloud HTTP API directly: public key event ingest, API-key endpoints for config, status, results, overrides, analytics reports, notes, short links and alert rules. Use when writing a server, a script, a non-JS client, or when the CLI is not available.
|
|
477
|
+
---
|
|
478
|
+
|
|
479
|
+
# SplitHub Cloud API
|
|
480
|
+
|
|
481
|
+
Base URL: \`https://go.splithub.io\`. JSON in, JSON out (no envelope under \`/v1/*\`). Two credentials:
|
|
482
|
+
|
|
483
|
+
- **Public key \`pk_\u2026\`**: identifies the project to browsers and SDKs. Safe to ship. Can only send events and read
|
|
484
|
+
the public overrides JSON.
|
|
485
|
+
- **API key \`sk_\u2026\`**: project-scoped secret for servers, CI and the CLI (\`npx splithub login\` mints one through a
|
|
486
|
+
browser approval; or Settings \u2192 API keys in the dashboard). Never in client code. It can be limited to some areas:
|
|
487
|
+
see API keys and scopes below.
|
|
488
|
+
|
|
489
|
+
## Send events (public key)
|
|
490
|
+
|
|
491
|
+
\`\`\`http
|
|
492
|
+
POST /v1/events
|
|
493
|
+
Authorization: Bearer pk_live_\u2026
|
|
494
|
+
Content-Type: application/json
|
|
495
|
+
|
|
496
|
+
{ "project": "my-app", "sdk": "custom/1.0", "events": [
|
|
497
|
+
{ "id": "e_01h\u2026", "type": "exposure", "exp": "hero-cta", "var": "bold", "vid": "v_abc", "sid": "s_1", "ts": 1760000000000, "url": "/" },
|
|
498
|
+
{ "id": "g_01h\u2026", "type": "goal", "goal": "signup", "vid": "v_abc", "sid": "s_1", "ts": 1760000010000, "props": { "revenue": 49 } }
|
|
499
|
+
] }
|
|
500
|
+
\`\`\`
|
|
501
|
+
|
|
502
|
+
- Analytics properties take two more types: \`{ "type": "page", "prop": "site", "url": "/pricing", "title"?, "ref"?, \u2026 }\`
|
|
503
|
+
(\`ref\` = document.referrer) and \`{ "type": "event", "prop": "site", "name": "Video Played", "props"?: { \u2026 }, \u2026 }\`.
|
|
504
|
+
Both are billed like one exposure each; events of a property that is paused or unknown are dropped.
|
|
505
|
+
An \`event\` row also converts the experiments whose goal is that event, and a \`page\` row the page-visit goals its path
|
|
506
|
+
matches (not billed again): send one row per action, not a goal row next to it (one within 1 s is counted once).
|
|
507
|
+
\`{ "type": "identify", "prop": "site", "uid": "user_42", "traits"?: { "plan": "pro" }, \u2026 }\` ties the visitor to your user
|
|
508
|
+
(not billed). Requests from bots are dropped.
|
|
509
|
+
- \`id\`: unique per event (8-64 chars, [a-z0-9_-]); duplicates are ignored. \`vid\`: stable visitor id (cookie \`sh_vid\`).
|
|
510
|
+
- Reply \`202 { ok, received, inserted, dropped }\`. A project with nothing running or no config yet answers
|
|
511
|
+
\`202 { ok, inserted: 0, idle: 'no_running' | 'no_config' }\`: nothing stored, not an error. \`429\` with \`code\` \u2208
|
|
512
|
+
quota_exceeded | deleted: back off for \`Retry-After\` seconds; nothing was stored. Allowed domains are set per experiment and per
|
|
513
|
+
property: events from another origin are dropped, and \`403 origin_not_allowed\` comes back when nothing in the batch was allowed.
|
|
514
|
+
- Exposures for experiments that are not running are dropped and not billed. Goals are never billed.
|
|
515
|
+
|
|
516
|
+
## API keys and scopes
|
|
517
|
+
|
|
518
|
+
A key can do everything (\`*\`, the default) or only some areas, each with read or write (write includes read).
|
|
519
|
+
Pick them in Settings \u2192 API keys when creating the key.
|
|
520
|
+
|
|
521
|
+
| Area | Covers |
|
|
522
|
+
|---|---|
|
|
523
|
+
| \`experiments\` | status, results, report, config push, overrides, archive, reviews |
|
|
524
|
+
| \`analytics\` | properties, their reports, segments, the event catalog |
|
|
525
|
+
| \`events\` | read: raw events and \`export.csv?raw=1\`; write: send events from your server with this key |
|
|
526
|
+
| \`notes\`, \`links\`, \`alerts\`, \`domains\` | those endpoints |
|
|
527
|
+
|
|
528
|
+
Scope strings: \`<area>:read\` / \`<area>:write\` (e.g. \`notes:write\`, \`analytics:read\`), \`read\` / \`write\` for every area,
|
|
529
|
+
\`*\` for everything. Keys made before scopes existed have \`*\`.
|
|
530
|
+
|
|
531
|
+
Server-side sending: a key with \`events:write\` can post to \`POST /v1/events\` instead of the public key. It is trusted,
|
|
532
|
+
so there is no origin check; keep it on the server.
|
|
533
|
+
|
|
534
|
+
\`\`\`bash
|
|
535
|
+
curl -X POST https://go.splithub.io/v1/events \\
|
|
536
|
+
-H "Authorization: Bearer $SPLITHUB_API_KEY" -H "Content-Type: application/json" \\
|
|
537
|
+
-d '{ "sdk": "server/1.0", "events": [
|
|
538
|
+
{ "id": "g_order_1042", "type": "goal", "goal": "purchase", "vid": "v_abc", "ts": 1760000010000, "props": { "revenue": 49 } },
|
|
539
|
+
{ "id": "e_order_1042", "type": "event", "prop": "site", "name": "Order Paid", "vid": "v_abc", "ts": 1760000010000 }
|
|
540
|
+
] }'
|
|
541
|
+
\`\`\`
|
|
542
|
+
|
|
543
|
+
\`vid\` is the visitor's \`sh_vid\` cookie, so the goal lands on the variant they saw. A request the key may not make
|
|
544
|
+
answers \`403 { code: 'insufficient_scope', scope: 'notes:write', error, hint }\`: \`scope\` names the missing one, and the CLI
|
|
545
|
+
shows it (\`--json\` adds \`scope\`). Ask the user for a key with that scope; don't retry with another key.
|
|
546
|
+
|
|
547
|
+
## Live config for clients (public key)
|
|
548
|
+
|
|
549
|
+
\`GET /v1/overrides/<pk>.json\` \u2192 \`{ version, experiments: { <id>: { status?, traffic?, weights?, values? } }, added: [ \u2026hosted experiments\u2026 ], goals: [...], ctx: { country, region, city } }\`.
|
|
550
|
+
Poll it at most once a minute; the SDK caches it in localStorage. \`ctx\` is the visitor's geo for targeting.
|
|
551
|
+
|
|
552
|
+
## Project endpoints (API key)
|
|
553
|
+
|
|
554
|
+
\`\`\`http
|
|
555
|
+
GET /v1/status every experiment: status, visitors, days, verdict headline
|
|
556
|
+
GET /v1/results/<experiment> full verdict: sample, comparisons (lift, p-value, CI, P(beat)), srm, winner
|
|
557
|
+
?from=&to= (first-exposure days) &country=DE&channel=\u2026: the verdict on that slice, with scope { filtered, range, filters, note }
|
|
558
|
+
GET /v1/experiments/<experiment>/timeline ?from=&to=&<filters>: exposures[] and conversions[] per day and variant, notes
|
|
559
|
+
GET /v1/experiments/<experiment>/breakdown/<by> by country|region|city|device|os|browser|language|host|page (= url)|referrer|source|channel|landing|utm_*:
|
|
560
|
+
order[], segments[] { dim, cells, comparisons { lift, pValue, significant, better }, total, small }
|
|
561
|
+
GET /v1/experiments/<experiment>/engagement { property, variants[] { visitors, sessions, pagesPerVisit, bounceRate, avgVisit } } (&property=)
|
|
562
|
+
GET /v1/experiments/<experiment>/events-summary { property, variants[], visitors, rows[] { name, goalKey, isGoal, measured, perVariant, comparisons, small } } (&property=)
|
|
563
|
+
GET /v1/experiments/<experiment>/export.breakdown.csv?by= the breakdown as CSV, lift per segment
|
|
564
|
+
GET /v1/experiments/<experiment>/export.events-summary.csv the events table as CSV, lift per event
|
|
565
|
+
GET /v1/experiments/<experiment>/events ?type=exposure|goal&before=&limit=&<filters>: raw events, nextBefore (needs events:read)
|
|
566
|
+
GET /v1/config the config stored in the cloud (file + dashboard experiments)
|
|
567
|
+
PUT /v1/config { config } push a normalized config (what defineConfig returns); answers cloudOnly experiments to adopt
|
|
568
|
+
GET /v1/overrides live overrides
|
|
569
|
+
PUT /v1/overrides/<experiment> { status | traffic | weights | values } pause / resume / reweight without a deploy
|
|
570
|
+
DELETE /v1/overrides/<experiment>
|
|
571
|
+
GET /v1/properties analytics properties: live status, goals, domains, last 30 days
|
|
572
|
+
POST /v1/properties { key, name?, goals?, domains? } create one (dashboard-owned)
|
|
573
|
+
PATCH /v1/properties/<key> { status: running | paused, name?, goals? }
|
|
574
|
+
GET /v1/properties/<key>/summary ?from=&to= (YYYY-MM-DD) &<dimension>=a,b[&compare=previous|year|custom&cfrom=&cto=]: totals, days, goals, notes
|
|
575
|
+
GET /v1/properties/<key>/breakdown ?by=source|channel|page|event|goal|country|\u2026&goal=[&compare=\u2026]: rows
|
|
576
|
+
GET /v1/properties/<key>/funnel ?steps=page:/,event:Signup,goal:paid&within=visit|7d|30d[&by=]: steps with visitors, conversion, dropoff
|
|
577
|
+
GET /v1/properties/<key>/paths ?page=/pricing&direction=next|prev: rows { value, visits }
|
|
578
|
+
GET /v1/properties/<key>/retention ?start=visit|event:<n>|goal:<k>&return=visit|event:<n>&period=day|week: cohorts
|
|
579
|
+
GET /v1/properties/<key>/realtime | /events?type=page|event | /props?name=&key= | /export.csv?by=|day (&raw=1)
|
|
580
|
+
GET|POST /v1/segments { name, filters, propertyKey? } PATCH|DELETE /v1/segments/<id>
|
|
581
|
+
GET /v1/domains every allowed domain and where it applies, observed origins
|
|
582
|
+
PUT /v1/domains/<experiment|property>/<key> { hosts: [...] } replace one list ([] = any origin)
|
|
583
|
+
GET /v1/notes ?experiment=&property=&kind=&from=&to=[&exact=1] newest first; experiment / property also return project-wide notes unless exact=1
|
|
584
|
+
POST /v1/notes { text (\u2264 2000), day? (default today, project time zone), kind?: note|issue|win, experimentKey? | propertyKey? (none = whole project) }
|
|
585
|
+
PATCH|DELETE /v1/notes/<id> { text?, day?, kind? }
|
|
586
|
+
GET /v1/links short links: shortUrl, destinations [{ id, url, weight }], status, totalClicks
|
|
587
|
+
POST /v1/links { name, destinations: [{ url, weight? }], shortCode?, description?, clickLimit? }
|
|
588
|
+
PATCH|DELETE /v1/links/<id> { name?, shortCode?, status: active|paused, destinations?, description? } GET /v1/links/<id>/stats: byDay (30 days), byDestination, byCountry, byDevice, byReferrer
|
|
589
|
+
GET /v1/alerts rules, channels, digest, history, conditions, plan
|
|
590
|
+
POST /v1/alerts/rules { condition, params, actions?: { notify, pause, finish, dropLoser }, name?, experiment?, enabled?, cooldownHours? }
|
|
591
|
+
PATCH|DELETE /v1/alerts/rules/<id> POST /v1/alerts/rules/<id>/test (dry run: what would fire now)
|
|
592
|
+
\`\`\`
|
|
593
|
+
|
|
594
|
+
Alert conditions and params: experiments \`srm\`, \`winner\` { minVisitors? }, \`loser\` { lift, minVisitors? }, \`exposures\` { visitors },
|
|
595
|
+
\`conversions\` { conversions }, \`duration\` { days }, \`no_events\` { hours }; account \`quota\` { percent }, \`usage\` { every } (each time this
|
|
596
|
+
month's billed events cross another multiple); analytics \`traffic_drop\` / \`traffic_spike\` { property, percent }, \`goal_silent\`
|
|
597
|
+
{ property, goal, hours }, \`event_count\` { property, event? (any event when absent), count, repeat? }. \`params.progress\` { total, fired,
|
|
598
|
+
month? } is read-only. Enabled rules need Pro: \`402 pro_required\` otherwise (save it with \`enabled: false\`).
|
|
599
|
+
|
|
600
|
+
\`Authorization: Bearer sk_live_\u2026\` on each. Status values: draft | running | paused | finished.
|
|
601
|
+
|
|
602
|
+
## Dashboard-scope endpoints (session, same shapes the dashboard uses)
|
|
603
|
+
|
|
604
|
+
Under \`/v1/projects/<projectId>/\u2026\` with a user session: \`experiments\` (hosted experiments CRUD incl. \`flow\` targeting),
|
|
605
|
+
\`goals\`, \`timeline\`, \`breakdown\`, \`realtime\`, \`events\`, \`export\`, \`notes\`, \`domains\`, \`properties\`, \`links\`, \`alerts\`, \`billing\`, \`ai\`.
|
|
606
|
+
Scripts should prefer the API-key endpoints above; the session ones exist for the UI.
|
|
607
|
+
|
|
608
|
+
## Drop-in script (no SDK)
|
|
609
|
+
|
|
610
|
+
\`<script src="https://go.splithub.io/sh.js" data-key="pk_live_\u2026"></script>\` in \`<head>\`: redirect experiments, page-visit and
|
|
611
|
+
click goals from the cloud config, \`window.splithub.track(name, props)\` for every event (goals included), pageviews
|
|
612
|
+
for analytics properties (\`data-property="site"\`). An app with the SDK does not need it: one install per property,
|
|
613
|
+
or only the first of the two collects pageviews.
|
|
614
|
+
\`<meta name="splithub-experiment" content="<id>">\` on a tested page for an instant decision.
|
|
615
|
+
|
|
616
|
+
## Errors
|
|
617
|
+
|
|
618
|
+
\`400\` validation (message in \`error\`), \`401\` bad key, \`403\` forbidden (\`insufficient_scope\` names the missing \`scope\`), \`404\` unknown experiment/project, \`409\` state
|
|
619
|
+
(e.g. \`config_not_pushed\`, \`file_owned\`), \`429\` gate or rate limit. Bodies: \`{ error, code?, hint? }\`, auth errors included; \`hint\` is the next step.
|
|
620
|
+
`;
|
|
621
|
+
var SKILL_SPLITHUB_MCP = `---
|
|
622
|
+
name: splithub-mcp
|
|
623
|
+
description: Use or set up the SplitHub MCP server (@splithub/mcp) so an agent can create experiments, read verdicts, start/pause/finish experiments and push config through tools instead of shelling out, plus analytics, notes, alerts and short links. Use when the user asks to connect SplitHub to Claude Code, Cursor, Codex or another MCP client.
|
|
624
|
+
---
|
|
625
|
+
|
|
626
|
+
# SplitHub MCP server
|
|
627
|
+
|
|
628
|
+
\`@splithub/mcp\` is a stdio MCP server that wraps the \`splithub\` CLI for the current project folder.
|
|
629
|
+
Same credentials and config file as the CLI; nothing extra to configure.
|
|
630
|
+
|
|
631
|
+
## Register it
|
|
632
|
+
|
|
633
|
+
Claude Code (project scope, committed in \`.mcp.json\`):
|
|
634
|
+
|
|
635
|
+
\`\`\`json
|
|
636
|
+
{ "mcpServers": { "splithub": { "command": "npx", "args": ["-y", "@splithub/mcp"] } } }
|
|
637
|
+
\`\`\`
|
|
638
|
+
|
|
639
|
+
or \`claude mcp add splithub -- npx -y @splithub/mcp\`. Cursor: the same object in \`.cursor/mcp.json\`.
|
|
640
|
+
Codex / other clients: command \`npx\`, args \`-y @splithub/mcp\`. \`npx splithub skills --mcp\` writes both files.
|
|
641
|
+
Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory), \`SPLITHUB_API\` (staging or self-hosted cloud), \`SPLITHUB_API_KEY\` (CI).
|
|
642
|
+
|
|
643
|
+
## Tools
|
|
644
|
+
|
|
645
|
+
| tool | does | notes |
|
|
646
|
+
|---|---|---|
|
|
647
|
+
| \`splithub_status\` | all experiments with verdict headline | start here |
|
|
648
|
+
| \`splithub_results\` | full verdict for one experiment; from / to / filters for a slice, by / timeline / engagement / events for one view (cloud) | report \`verdict.message\` verbatim; \`daysToVerdict\` = when to check again; a slice's verdict is not the experiment's |
|
|
649
|
+
| \`splithub_experiment_events\` | the latest raw exposures and goals of an experiment (cloud) | check the wiring |
|
|
650
|
+
| \`splithub_review\` | stored AI reviews (Pro): at start, on verdict changes, on finish, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |
|
|
651
|
+
| \`splithub_plan\` | visitors per variant for a baseline and a lift; days at an experiment's traffic | before creating |
|
|
652
|
+
| \`splithub_experiment_create\` | adds an experiment to splithub.config.ts | kind value / component / redirect / mvt, id, variants (or factors for mvt), goal, values/to, weights, traffic, maxVisitors, start/end; running by default |
|
|
653
|
+
| \`splithub_refs\` | files and lines that use an experiment and its goal | check the wiring, find leftovers |
|
|
654
|
+
| \`splithub_ship\` | finish, remove from the config, rewrite the code to the kept variant | \`dryRun: true\` first; returns \`diff\` and \`leftovers\` |
|
|
655
|
+
| \`splithub_remove\` | removes an experiment from the config (archived in the cloud) | when the code is already clean |
|
|
656
|
+
| \`splithub_projects\` | every folder on this machine connected to a cloud project | |
|
|
657
|
+
| \`splithub_init\` | sets up the config, framework files and skills | once per project |
|
|
658
|
+
| \`splithub_experiment_status\` | start / pause / finish | applies live in cloud mode; after \`finished\`, ask which variant to keep and whether to clean the code (splithub-ship skill) |
|
|
659
|
+
| \`splithub_push\` / \`splithub_pull\` | sync the config with the cloud | pull adopts dashboard-created experiments |
|
|
660
|
+
| \`splithub_doctor\` | checks the installation | run after wiring code |
|
|
661
|
+
| \`splithub_login\` | returns a sign-in link; the human approves it in the browser | show the link, then call \`splithub_whoami\` |
|
|
662
|
+
| \`splithub_whoami\` | connected project and keys | |
|
|
663
|
+
| \`splithub_analytics\` | analytics properties; one property's totals, goals, top pages and sources; \`by\` a dimension | counts, never verdicts |
|
|
664
|
+
| \`splithub_analytics_events\` | the latest raw pageviews and events of a property | check tracking after wiring it |
|
|
665
|
+
| \`splithub_analytics_funnel\` / \`_paths\` / \`_retention\` / \`_realtime\` | funnels, next pages, cohorts, visitors now (cloud) | counts, never verdicts |
|
|
666
|
+
| \`splithub_analytics_connect\` | picks or creates a property and names it on the app's provider | \`wired.status\` none / many: add the snippet yourself |
|
|
667
|
+
| \`splithub_segments\` | saved report filters (cloud) | |
|
|
668
|
+
| \`splithub_notes\` | what happened on a day, per experiment, property or project: list / add / edit / delete (cloud) | add one when you ship |
|
|
669
|
+
| \`splithub_alerts\` | alert rules: list / add / enable / disable / delete / test (cloud, fire on Pro) | conditions in the splithub skill |
|
|
670
|
+
| \`splithub_links\` | short links: list / stats / create / edit / pause / resume / delete (cloud) | several URLs split by weight |
|
|
671
|
+
| \`splithub_property_create\` | adds an analytics property to splithub.config.ts | push in cloud mode |
|
|
672
|
+
| \`splithub_property_status\` | pause / resume, archive / restore (cloud) or remove a property | live in cloud mode |
|
|
673
|
+
| \`splithub_domains\` | allowed domains per experiment and property; set or clear one list | empty = any origin |
|
|
674
|
+
|
|
675
|
+
Old names still answer for one release: \`splithub_create_experiment\`, \`splithub_set_status\`, \`splithub_create_property\`,
|
|
676
|
+
\`splithub_set_property_status\`. Use the names above.
|
|
677
|
+
|
|
678
|
+
## Working rules
|
|
679
|
+
|
|
680
|
+
- Create (running by default) \u2192 wire in code \u2192 \`doctor\` \u2192 \`push\` (cloud mode) \u2192 tell the user it goes live when they
|
|
681
|
+
deploy this code and that \`set_status paused\` or Pause in the dashboard holds it.
|
|
682
|
+
- Results: never call a winner from raw rates; quote the verdict. \`srm_detected\` = fix the setup first.
|
|
683
|
+
- Running experiments are frozen: no weight / variant / seed edits; create a new one.
|
|
684
|
+
- Errors carry a \`code\`. \`NOT_LOGGED_IN\`: call \`splithub_login\`, show the link to the user, wait for them, then retry.
|
|
685
|
+
- The whole loop (plan \u2192 create \u2192 implement \u2192 doctor \u2192 push \u2192 results \u2192 finish \u2192 ship) is in the \`splithub-loop\` skill.
|
|
686
|
+
- A \`finished\` experiment in \`splithub_status\` or \`splithub_results\` means the experiment is over but its code is still there: report
|
|
687
|
+
the verdict, ask which variant to keep and whether to remove the experiment from the code, then follow the \`splithub-ship\` skill.
|
|
688
|
+
`;
|
|
689
|
+
var SKILLS = [
|
|
690
|
+
{ dir: "splithub", body: SKILL_SPLITHUB },
|
|
691
|
+
{ dir: "ab-testing", body: SKILL_AB_TESTING },
|
|
692
|
+
{ dir: "splithub-api", body: SKILL_SPLITHUB_API },
|
|
693
|
+
{ dir: "splithub-mcp", body: SKILL_SPLITHUB_MCP },
|
|
694
|
+
{ dir: "splithub-self-hosted", body: SKILL_SELF_HOSTED },
|
|
695
|
+
{ dir: "splithub-ship", body: SKILL_SHIP },
|
|
696
|
+
{ dir: "splithub-loop", body: SKILL_LOOP },
|
|
697
|
+
{ dir: "splithub-analytics", body: SKILL_ANALYTICS }
|
|
698
|
+
];
|
|
699
|
+
var MCP_SERVER_ENTRY = { command: "npx", args: ["-y", "@splithub/mcp"] };
|
|
700
|
+
|
|
701
|
+
// src/templates.ts
|
|
702
|
+
function configTemplate(project, mode, key) {
|
|
703
|
+
const storage = mode === "self" ? `{ mode: 'self', endpoint: '/api/splithub' }` : `{ mode: 'cloud', key: ${key ? `'${key}'` : "process.env.NEXT_PUBLIC_SPLITHUB_KEY ?? ''"} }`;
|
|
704
|
+
return `import { defineConfig } from '@splithub/core';
|
|
705
|
+
|
|
706
|
+
// SplitHub experiments. This file is the source of truth: the SDK, the CLI
|
|
707
|
+
// and the dashboard all read it. Keys: lowercase a-z 0-9 - _. First variant = control.
|
|
708
|
+
export default defineConfig({
|
|
709
|
+
project: '${project}',
|
|
710
|
+
storage: ${storage},
|
|
711
|
+
// Drafts behave like running experiments while you develop (next dev, vite); production ignores them.
|
|
712
|
+
runDraftsInDev: true,
|
|
713
|
+
|
|
714
|
+
goals: {
|
|
715
|
+
// events you mark as goals: call track('signup') where it happens.
|
|
716
|
+
// An experiment's goal may also be any other event name you track().
|
|
717
|
+
signup: { type: 'event' },
|
|
718
|
+
},
|
|
719
|
+
|
|
720
|
+
experiments: {
|
|
721
|
+
// run \`npx splithub create\` or add entries here:
|
|
722
|
+
//
|
|
723
|
+
// 'hero-title': {
|
|
724
|
+
// kind: 'value',
|
|
725
|
+
// status: 'running',
|
|
726
|
+
// variants: {
|
|
727
|
+
// control: { weight: 50, value: 'Ship faster' },
|
|
728
|
+
// direct: { weight: 50, value: 'Know which version wins' },
|
|
729
|
+
// },
|
|
730
|
+
// goal: 'signup',
|
|
731
|
+
// },
|
|
732
|
+
},
|
|
733
|
+
});
|
|
734
|
+
`;
|
|
735
|
+
}
|
|
736
|
+
function nextMiddlewareTemplate(configImport) {
|
|
737
|
+
return `import { splithubMiddleware } from '@splithub/next';
|
|
738
|
+
import splithub from '${configImport}';
|
|
739
|
+
|
|
740
|
+
// Sets the visitor cookie, forwards it to Server Components and serves
|
|
741
|
+
// redirect experiments with an invisible rewrite.
|
|
742
|
+
export default splithubMiddleware(splithub);
|
|
743
|
+
|
|
744
|
+
export const config = {
|
|
745
|
+
matcher: ['/((?!_next|api|.*\\\\..*).*)'],
|
|
746
|
+
};
|
|
747
|
+
`;
|
|
748
|
+
}
|
|
749
|
+
function nextRouteTemplate(configImport, store = { imports: "import { sqliteStore } from '@splithub/adapter-sqlite';", factory: "sqliteStore({ file: process.env.SPLITHUB_DB ?? './splithub.db' })" }) {
|
|
750
|
+
return `import { createSplitHubHandler } from '@splithub/node';
|
|
751
|
+
${store.imports}
|
|
752
|
+
import config from '${configImport}';
|
|
753
|
+
|
|
754
|
+
// Self-hosted ingest + results: events stay in your database. The CLI reads
|
|
755
|
+
// status / results from this route; set SPLITHUB_ADMIN_TOKEN to protect it.
|
|
756
|
+
const handler = createSplitHubHandler({
|
|
757
|
+
config,
|
|
758
|
+
store: ${store.factory},
|
|
759
|
+
adminToken: process.env.SPLITHUB_ADMIN_TOKEN,
|
|
760
|
+
});
|
|
761
|
+
|
|
762
|
+
export const POST = handler;
|
|
763
|
+
export const GET = handler;
|
|
764
|
+
export const runtime = 'nodejs';
|
|
765
|
+
`;
|
|
766
|
+
}
|
|
767
|
+
function componentVariantTemplate(name, experiment, variant) {
|
|
768
|
+
return `// Variant "${variant}" of experiment "${experiment}". Edit freely.
|
|
769
|
+
export function ${name}() {
|
|
770
|
+
return (
|
|
771
|
+
<section>
|
|
772
|
+
<h2>${experiment} \xB7 ${variant}</h2>
|
|
773
|
+
</section>
|
|
774
|
+
);
|
|
775
|
+
}
|
|
776
|
+
`;
|
|
777
|
+
}
|
|
778
|
+
var AGENTS_SECTION = `
|
|
779
|
+
## A/B testing (SplitHub)
|
|
780
|
+
|
|
781
|
+
This project uses SplitHub (\`@splithub/react\`). Experiments are declared in
|
|
782
|
+
\`splithub.config.ts\` and rendered with \`<Experiment id>\` / \`useVariant(id)\` /
|
|
783
|
+
\`useValue(id)\`. Events: \`useTrack()\` then \`track('<event name>')\`; an experiment's goal is any event name.
|
|
784
|
+
|
|
785
|
+
- Not connected yet? \`npx splithub init --mode cloud\` (or \`npx splithub login\`) prints a link: show it to the user, they sign in and approve, the CLI gets the keys. Never ask the user to paste keys.
|
|
786
|
+
- New experiment: \`npx splithub create --kind value|component|redirect|mvt --id <key> --variants control,<other> --goal <event name> --yes\`
|
|
787
|
+
then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key, redirect \u2192 nothing,
|
|
788
|
+
mvt \u2192 \`--factor name=level:value,\u2026\` and \`const { a, b } = useValue(id)\`).
|
|
789
|
+
- Results: \`npx splithub results <id> --json\`; report \`verdict.message\` verbatim, never call a winner from raw rates.
|
|
790
|
+
- Lifecycle: \`npx splithub start|pause|finish <id>\`. Check setup: \`npx splithub doctor\`.
|
|
791
|
+
- A \`finished\` experiment still has all its variants in the code: report the verdict, ask which variant to keep and whether to
|
|
792
|
+
remove the experiment from the code, then follow .claude/skills/splithub-ship/SKILL.md. Never delete experiment code unasked.
|
|
793
|
+
- \`create\` writes \`status: 'running'\`: the experiment goes live when the code is deployed. Always tell the user that, and that
|
|
794
|
+
\`npx splithub pause <id>\` or Pause in the dashboard holds it. Never change \`seed\` of a running experiment.
|
|
795
|
+
- Analytics: set it up with \`npx splithub analytics connect --json\` (names the property on the provider), read it with
|
|
796
|
+
\`npx splithub analytics <property> --json\`; also \`funnel\`, \`paths\`, \`retention\`, \`realtime\`, \`export\` (SplitHub Cloud).
|
|
797
|
+
Details: .claude/skills/splithub-analytics/SKILL.md
|
|
798
|
+
- Notes, alerts, links, domains (SplitHub Cloud): \`npx splithub notes add "<what shipped>"\` when you ship, \`npx splithub alerts\` (rules, e.g.
|
|
799
|
+
\`alerts add usage --every 1000\`), \`npx splithub links\` (short links), \`npx splithub domains\`. Humans: \`npx splithub\` opens the menu.
|
|
800
|
+
- Config ships to the browser: no secrets in it. Full rules: .claude/skills/splithub/SKILL.md
|
|
801
|
+
`;
|
|
802
|
+
var CURSOR_RULE = `---
|
|
803
|
+
description: SplitHub A/B testing conventions
|
|
804
|
+
globs: ["splithub.config.ts", "splithub/**", "**/*.tsx"]
|
|
805
|
+
alwaysApply: false
|
|
806
|
+
---
|
|
807
|
+
${SKILL_SPLITHUB.replace(/^---[\s\S]*?---\n/, "")}`;
|
|
808
|
+
|
|
809
|
+
export {
|
|
810
|
+
SKILLS,
|
|
811
|
+
MCP_SERVER_ENTRY,
|
|
812
|
+
configTemplate,
|
|
813
|
+
nextMiddlewareTemplate,
|
|
814
|
+
nextRouteTemplate,
|
|
815
|
+
componentVariantTemplate,
|
|
816
|
+
AGENTS_SECTION,
|
|
817
|
+
CURSOR_RULE
|
|
818
|
+
};
|
|
819
|
+
//# sourceMappingURL=chunk-MIHX7XYV.js.map
|