splithub 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -5,6 +5,7 @@ import { parseArgs } from "util";
5
5
 
6
6
  // src/commands/create.ts
7
7
  import * as p from "@clack/prompts";
8
+ import { mvtVariants } from "@splithub/core";
8
9
  import { existsSync as existsSync2, mkdirSync, writeFileSync as writeFileSync2 } from "fs";
9
10
  import { join as join2, relative } from "path";
10
11
 
@@ -78,7 +79,13 @@ function renderExperiment(e, indent = " ") {
78
79
  lines.push(`${i2}kind: ${q(e.kind)},`);
79
80
  lines.push(`${i2}status: ${q(e.status)},`);
80
81
  if (e.match) lines.push(`${i2}match: ${q(e.match)},`);
81
- if (e.kind === "component") {
82
+ if (e.kind === "mvt") {
83
+ lines.push(`${i2}factors: {`);
84
+ for (const [factor, levels] of Object.entries(e.factors ?? {})) {
85
+ lines.push(`${i3}${k(factor)}: { ${Object.entries(levels).map(([l, v]) => `${k(l)}: ${q(v)}`).join(", ")} },`);
86
+ }
87
+ lines.push(`${i2}},`);
88
+ } else if (e.kind === "component") {
82
89
  lines.push(`${i2}variants: { ${e.variants.map((v) => `${k(v.key)}: ${v.weight}`).join(", ")} },`);
83
90
  } else {
84
91
  lines.push(`${i2}variants: {`);
@@ -321,7 +328,7 @@ one raw query for \`variantCounts\`. Supabase: the same SQL as a Postgres functi
321
328
  // src/skills-ship.ts
322
329
  var SKILL_SHIP = `---
323
330
  name: splithub-ship
324
- description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up.
331
+ description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up; and whenever a test shows status finished and the user agreed on which variant to keep.
325
332
  ---
326
333
 
327
334
  # Shipping the winner
@@ -385,21 +392,32 @@ Events go to the app's own database (self mode) or to SplitHub Cloud (cloud mode
385
392
  - Results: \`npx splithub status --json\`, \`npx splithub results <id> --json\`.
386
393
  - Everything interactive: \`npx splithub\` (menu). Everything scriptable: subcommands + \`--json\`.
387
394
 
395
+ ## Pick the kind
396
+
397
+ | kind | use it when | create | read in code |
398
+ |---|---|---|---|
399
+ | \`value\` | only data changes: text, number, image URL, price, JSON. The default choice: no new components. | \`--variants control,bold --value control="Old text" --value bold="New text"\` | \`const text = useValue<string>('<id>')\`; server components: \`getValue\` from \`@splithub/next\`; no SDK: \`sh.value('<id>')\` |
400
+ | \`component\` | structure changes: a layout, a section, a flow, a different component per variant | \`--variants control,bold\` (the CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs) | \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`, or \`useVariant('<id>')\` in a switch |
401
+ | \`redirect\` | a whole page changes: another URL per variant | \`--match / --to control=/ --to v2=/home-v2\` | nothing: \`@splithub/next\` middleware or the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` sends visitors there |
402
+ | \`mvt\` | several things change at once (headline \xD7 button \xD7 image) and the user wants to know which of them matters; every combination is tested and each factor is read on its own | \`--factor headline=old:"Old text",new:"New text" --factor button=blue:#2563eb,green:#16a34a\` (no \`--variants\`; first level = control; at most 4 factors \xD7 4 levels, 16 combinations) | \`const { headline, button } = useValue<{ headline: string; button: string }>('<id>')\` |
403
+
404
+ mvt needs more visitors than an A/B test: every combination must reach the minimum on its own (the results show the
405
+ combinations and, below them, each factor summed over its combinations). With little traffic, test one thing with \`value\`.
406
+
388
407
  ## Create an experiment
389
408
 
390
- 1. \`npx splithub create --kind <value|component|redirect> --id <key> --variants control,<other> --goal <goal> --yes --json\`
391
- - \`value\`: only data changes (text, image URL, number, JSON). Prefer it: no new components.
392
- \`--value control="Old text" --value bold="New text"\`; read with \`useValue<string>('<id>')\`.
393
- - \`component\`: structure changes. \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`.
394
- The CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs.
395
- - \`redirect\`: a whole page changes. \`--match / --to control=/ --to v2=/home-v2\`. Served by \`@splithub/next\` middleware,
396
- or by the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` on sites without the SDK.
409
+ 1. \`npx splithub create --kind <value|component|redirect|mvt> --id <key> \u2026 --goal <goal> --yes --json\` (see the table).
397
410
  2. Goal: pick an existing key or add one to \`goals\`:
398
411
  \`{ type: 'event' }\` + \`track('<goal>')\` where the conversion happens; \`{ type: 'pageview', path: '/thanks' }\`;
399
412
  \`{ type: 'click', selector: '#buy' }\` (tracked automatically by the SDK and the script).
400
413
  3. Wire it in code, keep \`status: 'draft'\` until the user confirms, then \`npx splithub start <id>\`.
401
414
  4. Cloud mode: \`npx splithub push\` after any config change.
402
415
 
416
+ A draft is invisible to visitors. To check it before starting: \`runDraftsInDev: true\` at the top level of the config
417
+ serves drafts like running tests in development only (\`NODE_ENV=development\`: \`next dev\`, \`vite\`; nothing is billed);
418
+ \`?sh_force=<id>:<variant>\` shows one variant on any page, draft or not, without recording an exposure;
419
+ \`?sh_debug=1\` (or \`npx splithub dev\`) logs every decision and opens a panel to switch variants.
420
+
403
421
  Keys: lowercase a-z 0-9 - _, max 64 chars. First variant = control. Weights are relative (50/50, 1/1, 90/10).
404
422
  Targeting (country, device, UTM, cookie, time) is a \`flow\` graph on the experiment, edited in the dashboard; \`pull\` brings it into the file.
405
423
 
@@ -409,10 +427,20 @@ Targeting (country, device, UTM, cookie, time) is a \`flow\` graph on the experi
409
427
  Report \`verdict.message\` verbatim and the numbers behind it (visitors per variant, lift, p-value, CI).
410
428
  Never declare a winner from raw rates. \`srm_detected\` means the split is broken: fix the setup before anything else.
411
429
 
412
- ## Finish
430
+ ## Finish, and what to do when a test is finished
431
+
432
+ \`npx splithub finish <id>\` stops the test (live within a minute in cloud mode). A finished test keeps its numbers for as long
433
+ as it stays in the config; the code still contains every variant until someone removes it.
413
434
 
414
- \`npx splithub finish <id>\`; replace the \`<Experiment>\` / \`useValue\` call with the winning content; delete losing
415
- components; keep the config entry for history (or remove it after the user agrees).
435
+ Whenever you see a test with \`status: finished\` (in \`status\`, \`results\`, the MCP tools, or right after you finished it):
436
+ 1. Read \`npx splithub results <id> --json\` and tell the user the verdict in one line with its numbers.
437
+ 2. Ask two questions, do not assume: **which variant to keep** (recommend \`verdict.winner\`; with no significant
438
+ difference, the control or the simpler one) and **whether to remove the test from the code now**.
439
+ 3. On yes, follow the \`splithub-ship\` skill: keep only the chosen variant's content, delete the other variants' code and
440
+ files, remove the entry from \`splithub.config.ts\`, \`push\` in cloud mode, run \`doctor\` and the project's checks.
441
+ 4. On no, leave everything: a finished test costs nothing and the user may come back to it.
442
+
443
+ A \`paused\` test is not over: say so and ask whether to resume (\`start\`) or finish it.
416
444
 
417
445
  ## Do not
418
446
 
@@ -572,9 +600,9 @@ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory),
572
600
  |---|---|---|
573
601
  | \`splithub_status\` | all experiments with verdict headline | start here |
574
602
  | \`splithub_results\` | full verdict for one experiment | report \`verdict.message\` verbatim |
575
- | \`splithub_review\` | stored AI reviews (Pro): on finish, daily, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |
576
- | \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind, id, variants, goal, values/to; starts as draft |
577
- | \`splithub_set_status\` | start / pause / finish | applies live in cloud mode |
603
+ | \`splithub_review\` | stored AI reviews (Pro): at start, on verdict changes, on finish, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |
604
+ | \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind value / component / redirect / mvt, id, variants (or factors for mvt), goal, values/to; starts as draft |
605
+ | \`splithub_set_status\` | start / pause / finish | applies live in cloud mode; after \`finished\`, ask which variant to keep and whether to clean the code (splithub-ship skill) |
578
606
  | \`splithub_push\` / \`splithub_pull\` | sync the config with the cloud | pull adopts dashboard-created experiments |
579
607
  | \`splithub_doctor\` | checks the installation | run after wiring code |
580
608
  | \`splithub_login\` | returns a sign-in link; the human approves it in the browser | show the link, then call \`splithub_whoami\` |
@@ -586,6 +614,8 @@ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory),
586
614
  - Results: never call a winner from raw rates; quote the verdict. \`srm_detected\` = fix the setup first.
587
615
  - Running experiments are frozen: no weight / variant / seed edits; create a new one.
588
616
  - If a tool answers that the folder is not connected, call \`splithub_login\`, show the link to the user, wait for them, then retry.
617
+ - A \`finished\` experiment in \`splithub_status\` or \`splithub_results\` means the test is over but its code is still there: report
618
+ the verdict, ask which variant to keep and whether to remove the test from the code, then follow the \`splithub-ship\` skill.
589
619
  `;
590
620
  var SKILLS = [
591
621
  { dir: "splithub", body: SKILL_SPLITHUB },
@@ -607,6 +637,8 @@ function configTemplate(project, mode, key) {
607
637
  export default defineConfig({
608
638
  project: '${project}',
609
639
  storage: ${storage},
640
+ // Drafts behave like running tests while you develop (next dev, vite); production ignores them.
641
+ runDraftsInDev: true,
610
642
 
611
643
  goals: {
612
644
  // call track('signup') where the conversion happens
@@ -679,10 +711,13 @@ This project uses SplitHub (\`@splithub/react\`). Experiments are declared in
679
711
  \`useValue(id)\`. Conversions: \`useTrack()\` then \`track('<goal>')\`.
680
712
 
681
713
  - Not connected yet? \`npx splithub init --mode cloud\` (or \`npx splithub login\`) prints a link: show it to the user, they sign in and approve, the CLI gets the keys. Never ask the user to paste keys.
682
- - New experiment: \`npx splithub create --kind value|component|redirect --id <key> --variants control,<other> --goal <goal> --yes\`
683
- then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key).
714
+ - New experiment: \`npx splithub create --kind value|component|redirect|mvt --id <key> --variants control,<other> --goal <goal> --yes\`
715
+ then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key, redirect \u2192 nothing,
716
+ mvt \u2192 \`--factor name=level:value,\u2026\` and \`const { a, b } = useValue(id)\`).
684
717
  - Results: \`npx splithub results <id> --json\`; report \`verdict.message\` verbatim, never call a winner from raw rates.
685
718
  - Lifecycle: \`npx splithub start|pause|finish <id>\`. Check setup: \`npx splithub doctor\`.
719
+ - A \`finished\` test still has all its variants in the code: report the verdict, ask which variant to keep and whether to
720
+ remove the test from the code, then follow .claude/skills/splithub-ship/SKILL.md. Never delete test code unasked.
686
721
  - Keep \`status: 'draft'\` until the user confirms. Never change \`seed\` of a running experiment.
687
722
  - Config ships to the browser: no secrets in it. Full rules: .claude/skills/splithub/SKILL.md
688
723
  `;
@@ -804,14 +839,15 @@ async function create(flags) {
804
839
  if (interactive) p.intro(c.bold("splithub create"));
805
840
  let kind = str(flags.kind);
806
841
  if (!kind) {
807
- if (!interactive) throw new Error("--kind value|component|redirect is required with --yes");
842
+ if (!interactive) throw new Error("--kind value|component|redirect|mvt is required with --yes");
808
843
  kind = await ask(
809
844
  p.select({
810
845
  message: "What changes between variants?",
811
846
  options: [
812
847
  { value: "value", label: "Text, image URL, number", hint: "no new components" },
813
848
  { value: "component", label: "Layout / structure", hint: "one component per variant" },
814
- { value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" }
849
+ { value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" },
850
+ { value: "mvt", label: "Several things at once", hint: "multivariate: every combination, each factor read on its own" }
815
851
  ]
816
852
  })
817
853
  );
@@ -829,7 +865,47 @@ async function create(flags) {
829
865
  }
830
866
  if (!KEY.test(id)) throw new Error(`invalid id "${id}" (lowercase a-z 0-9 - _, max 64)`);
831
867
  if (hasExperiment(source, id)) throw new Error(`experiment "${id}" already exists`);
832
- let variantKeys = str(flags.variants)?.split(",").map((s) => s.trim()).filter(Boolean);
868
+ const factors = {};
869
+ if (kind === "mvt") {
870
+ for (const raw of list(flags.factor)) {
871
+ const eq = raw.indexOf("=");
872
+ if (eq <= 0) throw new Error(`--factor expects name=level:value,level:value, got "${raw}"`);
873
+ const name = raw.slice(0, eq).trim();
874
+ const levels = {};
875
+ for (const part of raw.slice(eq + 1).split(",")) {
876
+ const colon = part.indexOf(":");
877
+ const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
878
+ if (!level) continue;
879
+ levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
880
+ }
881
+ factors[name] = levels;
882
+ }
883
+ while (interactive && Object.keys(factors).length === 0) {
884
+ const name = await ask(p.text({ message: "Factor name (what changes), e.g. headline", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
885
+ const levelsRaw = await ask(p.text({ message: `Levels of "${name}" as key:value, comma separated (first = control)`, placeholder: "old:Old headline, new:New headline" }));
886
+ const levels = {};
887
+ for (const part of levelsRaw.split(",")) {
888
+ const colon = part.indexOf(":");
889
+ const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
890
+ if (level) levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
891
+ }
892
+ factors[name] = levels;
893
+ while (await ask(p.confirm({ message: "Add another factor?", initialValue: false }))) {
894
+ const more = await ask(p.text({ message: "Factor name", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
895
+ const moreRaw = await ask(p.text({ message: `Levels of "${more}" as key:value, comma separated`, placeholder: "blue:#2563eb, green:#16a34a" }));
896
+ const moreLevels = {};
897
+ for (const part of moreRaw.split(",")) {
898
+ const colon = part.indexOf(":");
899
+ const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
900
+ if (level) moreLevels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
901
+ }
902
+ factors[more] = moreLevels;
903
+ }
904
+ }
905
+ if (Object.keys(factors).length === 0) throw new Error("--factor name=level:value,level:value is required for mvt experiments (repeat it per factor)");
906
+ }
907
+ const mvt = kind === "mvt" ? mvtVariants(id, factors) : null;
908
+ let variantKeys = mvt ? mvt.variants.map((v) => v.key) : str(flags.variants)?.split(",").map((s) => s.trim()).filter(Boolean);
833
909
  if (!variantKeys || variantKeys.length < 2) {
834
910
  if (!interactive) variantKeys = ["control", "b"];
835
911
  else {
@@ -885,7 +961,8 @@ async function create(flags) {
885
961
  goal,
886
962
  match,
887
963
  description,
888
- variants: variantKeys.map((k2, i) => ({ key: k2, weight: weights[i], value: values[k2], to: tos[k2] })),
964
+ variants: mvt ? mvt.variants.map((v) => ({ key: v.key, weight: v.weight })) : variantKeys.map((k2, i) => ({ key: k2, weight: weights[i], value: values[k2], to: tos[k2] })),
965
+ ...mvt ? { factors } : {},
889
966
  // limits: --traffic 50 (percent), --max-visitors 10000, --start / --end ISO dates
890
967
  traffic: typeof flags.traffic === "string" && flags.traffic !== "" ? Math.min(1, Math.max(0, Number(flags.traffic) / 100)) : void 0,
891
968
  maxExposures: typeof flags["max-visitors"] === "string" && Number(flags["max-visitors"]) > 0 ? Math.floor(Number(flags["max-visitors"])) : void 0,
@@ -907,7 +984,7 @@ async function create(flags) {
907
984
  }
908
985
  }
909
986
  }
910
- const snippet = kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
987
+ const snippet = kind === "mvt" ? `const { ${Object.keys(factors).join(", ")} } = useValue<{ ${Object.keys(factors).map((f) => `${f}: string`).join("; ")} }>('${id}');` : kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
911
988
  ${variantKeys.map((k2) => ` <Variant name="${k2}"><${componentNames[k2]} /></Variant>`).join("\n")}
912
989
  </Experiment>` : `// handled by @splithub/next middleware on ${match}; no code needed`;
913
990
  const result = { experiment: id, kind, status: status2, variants: experiment.variants, goal: goal ?? null, configFile: relative(project.root, project.configFile), files, snippet };
@@ -920,6 +997,7 @@ ${variantKeys.map((k2) => ` <Variant name="${k2}"><${componentNames[k2]} /></Va
920
997
  if (files.length) p.note(files.join("\n"), "Scaffolded");
921
998
  const cloud = /mode\s*:\s*['"]cloud['"]/.test(readSource(project.configFile));
922
999
  if (cloud) p.log.warn(`Cloud mode: the cloud does not know about it until you run ${c.cyan("npx splithub push")}.`);
1000
+ if (status2 === "draft") p.log.info(`See it before you start it: ${c.cyan("runDraftsInDev: true")} in the config serves drafts in development, ${c.cyan(`?sh_force=${id}:<variant>`)} shows one variant anywhere.`);
923
1001
  p.outro(`Added ${c.bold(id)} (${status2}) to ${relative(project.root, project.configFile)}. ${status2 === "running" ? "" : `Start it with ${c.cyan(`npx splithub start ${id}`)}. `}${cloud ? `Then ${c.cyan("npx splithub push")}.` : ""}`);
924
1002
  }
925
1003
 
@@ -1299,9 +1377,14 @@ export function pgStore(pool: Pool = new Pool({ connectionString: process.env.DA
1299
1377
  },
1300
1378
  async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
1301
1379
  const r = await pool.query(
1302
- \`WITH first_exposure AS (
1303
- SELECT variant, visitor_id, MIN(ts) AS first_ts FROM splithub_events
1304
- WHERE type = 'exposure' AND experiment = $1 GROUP BY variant, visitor_id),
1380
+ \`WITH fv AS (
1381
+ SELECT visitor_id, MIN(ts) AS first_ts FROM splithub_events
1382
+ WHERE type = 'exposure' AND experiment = $1 GROUP BY visitor_id),
1383
+ first_exposure AS (
1384
+ SELECT MIN(e.variant) AS variant, fv.visitor_id, fv.first_ts
1385
+ FROM fv JOIN splithub_events e
1386
+ ON e.type = 'exposure' AND e.experiment = $1 AND e.visitor_id = fv.visitor_id AND e.ts = fv.first_ts
1387
+ GROUP BY fv.visitor_id, fv.first_ts),
1305
1388
  converted AS (
1306
1389
  SELECT fe.variant, fe.visitor_id, SUM(COALESCE((g.props->>'revenue')::numeric, 0)) AS revenue
1307
1390
  FROM first_exposure fe JOIN splithub_events g
@@ -1366,20 +1449,21 @@ export function mysqlStore(pool: Pool = mysql.createPool(process.env.DATABASE_UR
1366
1449
  },
1367
1450
  async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
1368
1451
  const [rows] = await pool.query(
1369
- \`SELECT fe.variant, COUNT(*) AS exposures,
1370
- (SELECT COUNT(*) FROM (
1371
- SELECT fe2.variant, fe2.visitor_id FROM (
1372
- SELECT variant, visitor_id, MIN(ts) AS first_ts FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY variant, visitor_id) fe2
1373
- JOIN splithub_events g ON g.type = 'goal' AND g.goal = ? AND g.visitor_id = fe2.visitor_id AND g.ts >= fe2.first_ts
1374
- GROUP BY fe2.variant, fe2.visitor_id) c WHERE c.variant = fe.variant) AS conversions,
1375
- (SELECT COALESCE(SUM(rev), 0) FROM (
1376
- SELECT fe3.variant, SUM(COALESCE(JSON_EXTRACT(g.props, '$.revenue'), 0)) AS rev FROM (
1377
- SELECT variant, visitor_id, MIN(ts) AS first_ts FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY variant, visitor_id) fe3
1378
- JOIN splithub_events g ON g.type = 'goal' AND g.goal = ? AND g.visitor_id = fe3.visitor_id AND g.ts >= fe3.first_ts
1379
- GROUP BY fe3.variant, fe3.visitor_id) rv WHERE rv.variant = fe.variant) AS revenue
1380
- FROM (SELECT variant, visitor_id FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY variant, visitor_id) fe
1381
- GROUP BY fe.variant\`,
1382
- [experiment, goal, experiment, goal, experiment]
1452
+ // Derived tables only (no CTEs, no window functions) so this runs on MySQL 5.7.
1453
+ \`SELECT v.variant, COUNT(*) AS exposures, SUM(v.converted) AS conversions, COALESCE(SUM(v.revenue), 0) AS revenue
1454
+ FROM (
1455
+ SELECT fe.variant, fe.visitor_id,
1456
+ CASE WHEN COUNT(g.id) > 0 THEN 1 ELSE 0 END AS converted,
1457
+ SUM(COALESCE(JSON_EXTRACT(g.props, '$.revenue'), 0)) AS revenue
1458
+ FROM (
1459
+ SELECT MIN(e.variant) AS variant, fv.visitor_id, fv.first_ts
1460
+ FROM (SELECT visitor_id, MIN(ts) AS first_ts FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY visitor_id) fv
1461
+ JOIN splithub_events e ON e.type = 'exposure' AND e.experiment = ? AND e.visitor_id = fv.visitor_id AND e.ts = fv.first_ts
1462
+ GROUP BY fv.visitor_id, fv.first_ts) fe
1463
+ LEFT JOIN splithub_events g ON g.type = 'goal' AND g.goal = ? AND g.visitor_id = fe.visitor_id AND g.ts >= fe.first_ts
1464
+ GROUP BY fe.variant, fe.visitor_id) v
1465
+ GROUP BY v.variant\`,
1466
+ [experiment, experiment, goal]
1383
1467
  );
1384
1468
  return (rows as Array<Record<string, unknown>>).map((x) => ({ variant: String(x.variant), exposures: Number(x.exposures), conversions: Number(x.conversions), revenue: Number(x.revenue) }));
1385
1469
  },
@@ -1400,8 +1484,9 @@ import type { VariantCounts } from '@splithub/stats';
1400
1484
  /**
1401
1485
  * Your own store (Prisma, Drizzle, Supabase, Mongo, anything). Three methods.
1402
1486
  * Attribution rule (keep it, so results match the dashboard):
1403
- * a visitor converts for a variant when they sent the goal at or after their
1404
- * first exposure to that variant; count unique visitors, not events.
1487
+ * a visitor counts once per experiment, in the variant of their earliest
1488
+ * exposure (same ts: smallest variant key), and converts when they sent the
1489
+ * goal at or after that exposure; count unique visitors, not events.
1405
1490
  * Columns worth keeping: id (unique, dedupe on it), type, experiment, variant,
1406
1491
  * goal, visitor_id, session_id, ts, url, props (json), received_at, host,
1407
1492
  * country, device, browser.
@@ -1413,8 +1498,8 @@ export function customStore(): Store {
1413
1498
  throw new Error('implement insertEvents ' + events.length);
1414
1499
  },
1415
1500
  async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
1416
- // For each variant: exposures = unique visitors first exposed to it;
1417
- // conversions = those visitors with a goal event (goal === goal) at ts >= their first exposure;
1501
+ // For each variant: exposures = visitors whose earliest exposure in this experiment was to it;
1502
+ // conversions = those visitors with a goal event (goal === goal) at ts >= that earliest exposure;
1418
1503
  // revenue = sum of props.revenue over those goal events.
1419
1504
  throw new Error('implement variantCounts ' + experiment + ' ' + goal);
1420
1505
  },
@@ -1728,11 +1813,13 @@ async function lifecycle(command, id, flags) {
1728
1813
  } catch (error) {
1729
1814
  live = `could not apply live: ${error instanceof Error ? error.message : String(error)}`;
1730
1815
  }
1731
- if (flags.json) console.log(JSON.stringify({ experiment: id, status: status2, configFile: file, live }));
1816
+ const next = status2 === "finished" ? `Read the verdict (npx splithub results ${id}), decide which variant to keep, then remove the test from the code and the config: the splithub-ship skill walks an agent through it.` : null;
1817
+ if (flags.json) console.log(JSON.stringify({ experiment: id, status: status2, configFile: file, live, ...next ? { next } : {} }));
1732
1818
  else {
1733
1819
  console.log(`${c.green("\u2714")} ${c.bold(id)} \u2192 ${status2} ${c.dim(`(${file})`)}`);
1734
1820
  if (live === "applied") console.log(`${c.green("\u2714")} live in the cloud within 60s, no deploy needed (the config change ships with your next deploy)`);
1735
1821
  else if (live !== "skipped") console.log(c.yellow(`! ${live}`));
1822
+ if (next) console.log(c.dim(`Next: ${next}`));
1736
1823
  }
1737
1824
  }
1738
1825
 
@@ -1742,6 +1829,18 @@ import { readFileSync as readFileSync7 } from "fs";
1742
1829
 
1743
1830
  // src/commands/push.ts
1744
1831
  import { relative as relative6 } from "path";
1832
+ function factorsOf(e) {
1833
+ const out = {};
1834
+ for (const f of e.factors ?? []) {
1835
+ out[f.key] = {};
1836
+ for (const level of f.levels) {
1837
+ const combo = e.variants.find((v) => v.levels?.[f.key] === level);
1838
+ const value = combo?.value?.[f.key];
1839
+ out[f.key][level] = typeof value === "string" ? value : value === void 0 ? level : JSON.stringify(value);
1840
+ }
1841
+ }
1842
+ return out;
1843
+ }
1745
1844
  function adoptCloudExperiments(file, cloudOnly) {
1746
1845
  if (!cloudOnly || cloudOnly.length === 0) return [];
1747
1846
  let source = readSource(file);
@@ -1760,7 +1859,8 @@ function adoptCloudExperiments(file, cloudOnly) {
1760
1859
  traffic: e.traffic,
1761
1860
  maxExposures: e.maxExposures ?? void 0,
1762
1861
  schedule: e.schedule ?? void 0,
1763
- variants: e.variants.map((v) => ({ key: v.key, weight: Math.round(v.weight / total * 100), value: typeof v.value === "string" ? v.value : v.value === void 0 ? void 0 : JSON.stringify(v.value), to: v.to }))
1862
+ variants: e.variants.map((v) => ({ key: v.key, weight: Math.round(v.weight / total * 100), value: typeof v.value === "string" ? v.value : v.value === void 0 ? void 0 : JSON.stringify(v.value), to: v.to })),
1863
+ ...e.kind === "mvt" && e.factors ? { factors: factorsOf(e) } : {}
1764
1864
  });
1765
1865
  written.push(e.key);
1766
1866
  }
@@ -1830,7 +1930,7 @@ async function results(key, flags) {
1830
1930
  const target = resolveTarget(flags, detectProject());
1831
1931
  const r = await fetchResults(target, key);
1832
1932
  if (flags.json) {
1833
- console.log(JSON.stringify(r, null, 2));
1933
+ console.log(JSON.stringify(r.status === "finished" ? { ...r, next: FINISHED_NEXT } : r, null, 2));
1834
1934
  return;
1835
1935
  }
1836
1936
  const v = r.verdict;
@@ -1886,7 +1986,24 @@ async function results(key, flags) {
1886
1986
  if (v.status === "insufficient_sample" && v.sample.neededPerVariant) {
1887
1987
  console.log(c.dim(`Estimated ${v.sample.neededPerVariant} visitors per variant needed to confirm the current lift.`));
1888
1988
  }
1989
+ if (r.factors?.length) {
1990
+ console.log();
1991
+ console.log(c.bold("Each factor on its own") + c.dim(" (a level sums every combination that uses it, compared with the first level)"));
1992
+ for (const f of r.factors) {
1993
+ console.log(
1994
+ table(
1995
+ [f.key.toUpperCase(), "VISITORS", "CONV.", "RATE", "LIFT", "P-VALUE"],
1996
+ f.levels.map((l) => [l.level + (f.best === l.level ? c.green(" \u2714") : ""), String(l.exposures), String(l.conversions), pct(l.rate), l.relativeLift === null ? "\u2013" : pct(l.relativeLift, true), l.pValue === null ? "\u2013" : l.pValue.toFixed(3)])
1997
+ )
1998
+ );
1999
+ }
2000
+ }
2001
+ if (r.status === "finished") {
2002
+ console.log();
2003
+ console.log(c.dim(FINISHED_NEXT));
2004
+ }
1889
2005
  }
2006
+ var FINISHED_NEXT = "This test is finished but its variants are still in the code. Decide which variant to keep, then remove the test from the code and the config (agents: the splithub-ship skill).";
1890
2007
  function label(status2) {
1891
2008
  const icon = verdictIcon(status2);
1892
2009
  switch (status2) {
@@ -1907,8 +2024,10 @@ function label(status2) {
1907
2024
  async function status(flags) {
1908
2025
  const target = resolveTarget(flags, detectProject());
1909
2026
  const data = await fetchStatus(target);
2027
+ const finished = data.experiments.filter((e) => e.status === "finished").map((e) => e.experiment);
2028
+ const next = finished.length ? `Finished but still in the code: ${finished.join(", ")}. Decide which variant to keep, then remove the test from the code and the config (agents: the splithub-ship skill).` : null;
1910
2029
  if (flags.json) {
1911
- console.log(JSON.stringify(data, null, 2));
2030
+ console.log(JSON.stringify(next ? { ...data, next } : data, null, 2));
1912
2031
  return;
1913
2032
  }
1914
2033
  console.log(`${c.bold("SplitHub")} ${c.dim("\xB7")} project ${c.cyan(data.project)} ${c.dim(`\xB7 ${target.url}`)}
@@ -1932,6 +2051,8 @@ async function status(flags) {
1932
2051
  ])
1933
2052
  )
1934
2053
  );
2054
+ if (next) console.log(`
2055
+ ${c.dim(next)}`);
1935
2056
  }
1936
2057
 
1937
2058
  // src/commands/menu.ts
@@ -2054,7 +2175,7 @@ async function review(key, flags) {
2054
2175
  return;
2055
2176
  }
2056
2177
  if (reviews.length === 0) {
2057
- console.log(c.dim(key ? `no reviews for ${key} yet (Pro writes one on finish and daily while running; or click "Review now" in the dashboard)` : "no reviews yet"));
2178
+ console.log(c.dim(key ? `no reviews for ${key} yet (Pro writes one at the start, when the verdict changes while running, and on finish; or click "Review now" in the dashboard)` : "no reviews yet"));
2058
2179
  return;
2059
2180
  }
2060
2181
  for (const r of reviews) {
@@ -2075,7 +2196,7 @@ Usage
2075
2196
  splithub start|pause|finish <id>
2076
2197
  splithub status all experiments with a verdict headline
2077
2198
  splithub results <id> full verdict for one experiment
2078
- splithub review [<id>] AI reviews stored in the cloud (Pro: on finish, daily, or on demand in the dashboard)
2199
+ splithub review [<id>] AI reviews stored in the cloud (Pro: at start, on verdict changes, on finish, or on demand)
2079
2200
  splithub dev [--url] [--hold] open the app with ?sh_debug=1 (console + panel: variants, reasons, events) and watch results
2080
2201
  splithub doctor check the installation
2081
2202
  splithub skills [--mcp] install / refresh agent skills (library, A/B testing, API, MCP, self-hosting, ship the winner); --mcp registers the MCP server
@@ -2128,6 +2249,7 @@ async function main() {
2128
2249
  value: { type: "string", multiple: true },
2129
2250
  match: { type: "string" },
2130
2251
  to: { type: "string", multiple: true },
2252
+ factor: { type: "string", multiple: true },
2131
2253
  status: { type: "string" },
2132
2254
  description: { type: "string" },
2133
2255
  traffic: { type: "string" },