splithub 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
- # splithub (CLI)
1
+ # splithub
2
2
 
3
- Interactive for humans, flags + `--json` for coding agents.
3
+ A/B testing for your app in one command. Run it without arguments and it walks you through; add flags and `--json` when a coding agent runs it.
4
4
 
5
5
  ```bash
6
6
  npx splithub # menu: set up, connect, create, results, start/pause, push — step by step
package/dist/index.js CHANGED
@@ -5,6 +5,7 @@ import { parseArgs } from "util";
5
5
 
6
6
  // src/commands/create.ts
7
7
  import * as p from "@clack/prompts";
8
+ import { mvtVariants } from "@splithub/core";
8
9
  import { existsSync as existsSync2, mkdirSync, writeFileSync as writeFileSync2 } from "fs";
9
10
  import { join as join2, relative } from "path";
10
11
 
@@ -78,7 +79,13 @@ function renderExperiment(e, indent = " ") {
78
79
  lines.push(`${i2}kind: ${q(e.kind)},`);
79
80
  lines.push(`${i2}status: ${q(e.status)},`);
80
81
  if (e.match) lines.push(`${i2}match: ${q(e.match)},`);
81
- if (e.kind === "component") {
82
+ if (e.kind === "mvt") {
83
+ lines.push(`${i2}factors: {`);
84
+ for (const [factor, levels] of Object.entries(e.factors ?? {})) {
85
+ lines.push(`${i3}${k(factor)}: { ${Object.entries(levels).map(([l, v]) => `${k(l)}: ${q(v)}`).join(", ")} },`);
86
+ }
87
+ lines.push(`${i2}},`);
88
+ } else if (e.kind === "component") {
82
89
  lines.push(`${i2}variants: { ${e.variants.map((v) => `${k(v.key)}: ${v.weight}`).join(", ")} },`);
83
90
  } else {
84
91
  lines.push(`${i2}variants: {`);
@@ -321,7 +328,7 @@ one raw query for \`variantCounts\`. Supabase: the same SQL as a Postgres functi
321
328
  // src/skills-ship.ts
322
329
  var SKILL_SHIP = `---
323
330
  name: splithub-ship
324
- description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up.
331
+ description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up; and whenever a test shows status finished and the user agreed on which variant to keep.
325
332
  ---
326
333
 
327
334
  # Shipping the winner
@@ -385,15 +392,21 @@ Events go to the app's own database (self mode) or to SplitHub Cloud (cloud mode
385
392
  - Results: \`npx splithub status --json\`, \`npx splithub results <id> --json\`.
386
393
  - Everything interactive: \`npx splithub\` (menu). Everything scriptable: subcommands + \`--json\`.
387
394
 
395
+ ## Pick the kind
396
+
397
+ | kind | use it when | create | read in code |
398
+ |---|---|---|---|
399
+ | \`value\` | only data changes: text, number, image URL, price, JSON. The default choice: no new components. | \`--variants control,bold --value control="Old text" --value bold="New text"\` | \`const text = useValue<string>('<id>')\`; server components: \`getValue\` from \`@splithub/next\`; no SDK: \`sh.value('<id>')\` |
400
+ | \`component\` | structure changes: a layout, a section, a flow, a different component per variant | \`--variants control,bold\` (the CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs) | \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`, or \`useVariant('<id>')\` in a switch |
401
+ | \`redirect\` | a whole page changes: another URL per variant | \`--match / --to control=/ --to v2=/home-v2\` | nothing: \`@splithub/next\` middleware or the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` sends visitors there |
402
+ | \`mvt\` | several things change at once (headline \xD7 button \xD7 image) and the user wants to know which of them matters; every combination is tested and each factor is read on its own | \`--factor headline=old:"Old text",new:"New text" --factor button=blue:#2563eb,green:#16a34a\` (no \`--variants\`; first level = control; at most 4 factors \xD7 4 levels, 16 combinations) | \`const { headline, button } = useValue<{ headline: string; button: string }>('<id>')\` |
403
+
404
+ mvt needs more visitors than an A/B test: every combination must reach the minimum on its own (the results show the
405
+ combinations and, below them, each factor summed over its combinations). With little traffic, test one thing with \`value\`.
406
+
388
407
  ## Create an experiment
389
408
 
390
- 1. \`npx splithub create --kind <value|component|redirect> --id <key> --variants control,<other> --goal <goal> --yes --json\`
391
- - \`value\`: only data changes (text, image URL, number, JSON). Prefer it: no new components.
392
- \`--value control="Old text" --value bold="New text"\`; read with \`useValue<string>('<id>')\`.
393
- - \`component\`: structure changes. \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`.
394
- The CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs.
395
- - \`redirect\`: a whole page changes. \`--match / --to control=/ --to v2=/home-v2\`. Served by \`@splithub/next\` middleware,
396
- or by the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` on sites without the SDK.
409
+ 1. \`npx splithub create --kind <value|component|redirect|mvt> --id <key> \u2026 --goal <goal> --yes --json\` (see the table).
397
410
  2. Goal: pick an existing key or add one to \`goals\`:
398
411
  \`{ type: 'event' }\` + \`track('<goal>')\` where the conversion happens; \`{ type: 'pageview', path: '/thanks' }\`;
399
412
  \`{ type: 'click', selector: '#buy' }\` (tracked automatically by the SDK and the script).
@@ -409,10 +422,20 @@ Targeting (country, device, UTM, cookie, time) is a \`flow\` graph on the experi
409
422
  Report \`verdict.message\` verbatim and the numbers behind it (visitors per variant, lift, p-value, CI).
410
423
  Never declare a winner from raw rates. \`srm_detected\` means the split is broken: fix the setup before anything else.
411
424
 
412
- ## Finish
425
+ ## Finish, and what to do when a test is finished
426
+
427
+ \`npx splithub finish <id>\` stops the test (live within a minute in cloud mode). A finished test keeps its numbers for as long
428
+ as it stays in the config; the code still contains every variant until someone removes it.
429
+
430
+ Whenever you see a test with \`status: finished\` (in \`status\`, \`results\`, the MCP tools, or right after you finished it):
431
+ 1. Read \`npx splithub results <id> --json\` and tell the user the verdict in one line with its numbers.
432
+ 2. Ask two questions, do not assume: **which variant to keep** (recommend \`verdict.winner\`; with no significant
433
+ difference, the control or the simpler one) and **whether to remove the test from the code now**.
434
+ 3. On yes, follow the \`splithub-ship\` skill: keep only the chosen variant's content, delete the other variants' code and
435
+ files, remove the entry from \`splithub.config.ts\`, \`push\` in cloud mode, run \`doctor\` and the project's checks.
436
+ 4. On no, leave everything: a finished test costs nothing and the user may come back to it.
413
437
 
414
- \`npx splithub finish <id>\`; replace the \`<Experiment>\` / \`useValue\` call with the winning content; delete losing
415
- components; keep the config entry for history (or remove it after the user agrees).
438
+ A \`paused\` test is not over: say so and ask whether to resume (\`start\`) or finish it.
416
439
 
417
440
  ## Do not
418
441
 
@@ -572,9 +595,9 @@ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory),
572
595
  |---|---|---|
573
596
  | \`splithub_status\` | all experiments with verdict headline | start here |
574
597
  | \`splithub_results\` | full verdict for one experiment | report \`verdict.message\` verbatim |
575
- | \`splithub_review\` | stored AI reviews (Pro): on finish, daily, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |
576
- | \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind, id, variants, goal, values/to; starts as draft |
577
- | \`splithub_set_status\` | start / pause / finish | applies live in cloud mode |
598
+ | \`splithub_review\` | stored AI reviews (Pro): at start, on verdict changes, on finish, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |
599
+ | \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind value / component / redirect / mvt, id, variants (or factors for mvt), goal, values/to; starts as draft |
600
+ | \`splithub_set_status\` | start / pause / finish | applies live in cloud mode; after \`finished\`, ask which variant to keep and whether to clean the code (splithub-ship skill) |
578
601
  | \`splithub_push\` / \`splithub_pull\` | sync the config with the cloud | pull adopts dashboard-created experiments |
579
602
  | \`splithub_doctor\` | checks the installation | run after wiring code |
580
603
  | \`splithub_login\` | returns a sign-in link; the human approves it in the browser | show the link, then call \`splithub_whoami\` |
@@ -586,6 +609,8 @@ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory),
586
609
  - Results: never call a winner from raw rates; quote the verdict. \`srm_detected\` = fix the setup first.
587
610
  - Running experiments are frozen: no weight / variant / seed edits; create a new one.
588
611
  - If a tool answers that the folder is not connected, call \`splithub_login\`, show the link to the user, wait for them, then retry.
612
+ - A \`finished\` experiment in \`splithub_status\` or \`splithub_results\` means the test is over but its code is still there: report
613
+ the verdict, ask which variant to keep and whether to remove the test from the code, then follow the \`splithub-ship\` skill.
589
614
  `;
590
615
  var SKILLS = [
591
616
  { dir: "splithub", body: SKILL_SPLITHUB },
@@ -679,10 +704,13 @@ This project uses SplitHub (\`@splithub/react\`). Experiments are declared in
679
704
  \`useValue(id)\`. Conversions: \`useTrack()\` then \`track('<goal>')\`.
680
705
 
681
706
  - Not connected yet? \`npx splithub init --mode cloud\` (or \`npx splithub login\`) prints a link: show it to the user, they sign in and approve, the CLI gets the keys. Never ask the user to paste keys.
682
- - New experiment: \`npx splithub create --kind value|component|redirect --id <key> --variants control,<other> --goal <goal> --yes\`
683
- then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key).
707
+ - New experiment: \`npx splithub create --kind value|component|redirect|mvt --id <key> --variants control,<other> --goal <goal> --yes\`
708
+ then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key, redirect \u2192 nothing,
709
+ mvt \u2192 \`--factor name=level:value,\u2026\` and \`const { a, b } = useValue(id)\`).
684
710
  - Results: \`npx splithub results <id> --json\`; report \`verdict.message\` verbatim, never call a winner from raw rates.
685
711
  - Lifecycle: \`npx splithub start|pause|finish <id>\`. Check setup: \`npx splithub doctor\`.
712
+ - A \`finished\` test still has all its variants in the code: report the verdict, ask which variant to keep and whether to
713
+ remove the test from the code, then follow .claude/skills/splithub-ship/SKILL.md. Never delete test code unasked.
686
714
  - Keep \`status: 'draft'\` until the user confirms. Never change \`seed\` of a running experiment.
687
715
  - Config ships to the browser: no secrets in it. Full rules: .claude/skills/splithub/SKILL.md
688
716
  `;
@@ -804,14 +832,15 @@ async function create(flags) {
804
832
  if (interactive) p.intro(c.bold("splithub create"));
805
833
  let kind = str(flags.kind);
806
834
  if (!kind) {
807
- if (!interactive) throw new Error("--kind value|component|redirect is required with --yes");
835
+ if (!interactive) throw new Error("--kind value|component|redirect|mvt is required with --yes");
808
836
  kind = await ask(
809
837
  p.select({
810
838
  message: "What changes between variants?",
811
839
  options: [
812
840
  { value: "value", label: "Text, image URL, number", hint: "no new components" },
813
841
  { value: "component", label: "Layout / structure", hint: "one component per variant" },
814
- { value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" }
842
+ { value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" },
843
+ { value: "mvt", label: "Several things at once", hint: "multivariate: every combination, each factor read on its own" }
815
844
  ]
816
845
  })
817
846
  );
@@ -829,7 +858,47 @@ async function create(flags) {
829
858
  }
830
859
  if (!KEY.test(id)) throw new Error(`invalid id "${id}" (lowercase a-z 0-9 - _, max 64)`);
831
860
  if (hasExperiment(source, id)) throw new Error(`experiment "${id}" already exists`);
832
- let variantKeys = str(flags.variants)?.split(",").map((s) => s.trim()).filter(Boolean);
861
+ const factors = {};
862
+ if (kind === "mvt") {
863
+ for (const raw of list(flags.factor)) {
864
+ const eq = raw.indexOf("=");
865
+ if (eq <= 0) throw new Error(`--factor expects name=level:value,level:value, got "${raw}"`);
866
+ const name = raw.slice(0, eq).trim();
867
+ const levels = {};
868
+ for (const part of raw.slice(eq + 1).split(",")) {
869
+ const colon = part.indexOf(":");
870
+ const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
871
+ if (!level) continue;
872
+ levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
873
+ }
874
+ factors[name] = levels;
875
+ }
876
+ while (interactive && Object.keys(factors).length === 0) {
877
+ const name = await ask(p.text({ message: "Factor name (what changes), e.g. headline", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
878
+ const levelsRaw = await ask(p.text({ message: `Levels of "${name}" as key:value, comma separated (first = control)`, placeholder: "old:Old headline, new:New headline" }));
879
+ const levels = {};
880
+ for (const part of levelsRaw.split(",")) {
881
+ const colon = part.indexOf(":");
882
+ const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
883
+ if (level) levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
884
+ }
885
+ factors[name] = levels;
886
+ while (await ask(p.confirm({ message: "Add another factor?", initialValue: false }))) {
887
+ const more = await ask(p.text({ message: "Factor name", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
888
+ const moreRaw = await ask(p.text({ message: `Levels of "${more}" as key:value, comma separated`, placeholder: "blue:#2563eb, green:#16a34a" }));
889
+ const moreLevels = {};
890
+ for (const part of moreRaw.split(",")) {
891
+ const colon = part.indexOf(":");
892
+ const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
893
+ if (level) moreLevels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
894
+ }
895
+ factors[more] = moreLevels;
896
+ }
897
+ }
898
+ if (Object.keys(factors).length === 0) throw new Error("--factor name=level:value,level:value is required for mvt experiments (repeat it per factor)");
899
+ }
900
+ const mvt = kind === "mvt" ? mvtVariants(id, factors) : null;
901
+ let variantKeys = mvt ? mvt.variants.map((v) => v.key) : str(flags.variants)?.split(",").map((s) => s.trim()).filter(Boolean);
833
902
  if (!variantKeys || variantKeys.length < 2) {
834
903
  if (!interactive) variantKeys = ["control", "b"];
835
904
  else {
@@ -885,7 +954,8 @@ async function create(flags) {
885
954
  goal,
886
955
  match,
887
956
  description,
888
- variants: variantKeys.map((k2, i) => ({ key: k2, weight: weights[i], value: values[k2], to: tos[k2] })),
957
+ variants: mvt ? mvt.variants.map((v) => ({ key: v.key, weight: v.weight })) : variantKeys.map((k2, i) => ({ key: k2, weight: weights[i], value: values[k2], to: tos[k2] })),
958
+ ...mvt ? { factors } : {},
889
959
  // limits: --traffic 50 (percent), --max-visitors 10000, --start / --end ISO dates
890
960
  traffic: typeof flags.traffic === "string" && flags.traffic !== "" ? Math.min(1, Math.max(0, Number(flags.traffic) / 100)) : void 0,
891
961
  maxExposures: typeof flags["max-visitors"] === "string" && Number(flags["max-visitors"]) > 0 ? Math.floor(Number(flags["max-visitors"])) : void 0,
@@ -907,7 +977,7 @@ async function create(flags) {
907
977
  }
908
978
  }
909
979
  }
910
- const snippet = kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
980
+ const snippet = kind === "mvt" ? `const { ${Object.keys(factors).join(", ")} } = useValue<{ ${Object.keys(factors).map((f) => `${f}: string`).join("; ")} }>('${id}');` : kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
911
981
  ${variantKeys.map((k2) => ` <Variant name="${k2}"><${componentNames[k2]} /></Variant>`).join("\n")}
912
982
  </Experiment>` : `// handled by @splithub/next middleware on ${match}; no code needed`;
913
983
  const result = { experiment: id, kind, status: status2, variants: experiment.variants, goal: goal ?? null, configFile: relative(project.root, project.configFile), files, snippet };
@@ -1299,9 +1369,14 @@ export function pgStore(pool: Pool = new Pool({ connectionString: process.env.DA
1299
1369
  },
1300
1370
  async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
1301
1371
  const r = await pool.query(
1302
- \`WITH first_exposure AS (
1303
- SELECT variant, visitor_id, MIN(ts) AS first_ts FROM splithub_events
1304
- WHERE type = 'exposure' AND experiment = $1 GROUP BY variant, visitor_id),
1372
+ \`WITH fv AS (
1373
+ SELECT visitor_id, MIN(ts) AS first_ts FROM splithub_events
1374
+ WHERE type = 'exposure' AND experiment = $1 GROUP BY visitor_id),
1375
+ first_exposure AS (
1376
+ SELECT MIN(e.variant) AS variant, fv.visitor_id, fv.first_ts
1377
+ FROM fv JOIN splithub_events e
1378
+ ON e.type = 'exposure' AND e.experiment = $1 AND e.visitor_id = fv.visitor_id AND e.ts = fv.first_ts
1379
+ GROUP BY fv.visitor_id, fv.first_ts),
1305
1380
  converted AS (
1306
1381
  SELECT fe.variant, fe.visitor_id, SUM(COALESCE((g.props->>'revenue')::numeric, 0)) AS revenue
1307
1382
  FROM first_exposure fe JOIN splithub_events g
@@ -1366,20 +1441,21 @@ export function mysqlStore(pool: Pool = mysql.createPool(process.env.DATABASE_UR
1366
1441
  },
1367
1442
  async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
1368
1443
  const [rows] = await pool.query(
1369
- \`SELECT fe.variant, COUNT(*) AS exposures,
1370
- (SELECT COUNT(*) FROM (
1371
- SELECT fe2.variant, fe2.visitor_id FROM (
1372
- SELECT variant, visitor_id, MIN(ts) AS first_ts FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY variant, visitor_id) fe2
1373
- JOIN splithub_events g ON g.type = 'goal' AND g.goal = ? AND g.visitor_id = fe2.visitor_id AND g.ts >= fe2.first_ts
1374
- GROUP BY fe2.variant, fe2.visitor_id) c WHERE c.variant = fe.variant) AS conversions,
1375
- (SELECT COALESCE(SUM(rev), 0) FROM (
1376
- SELECT fe3.variant, SUM(COALESCE(JSON_EXTRACT(g.props, '$.revenue'), 0)) AS rev FROM (
1377
- SELECT variant, visitor_id, MIN(ts) AS first_ts FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY variant, visitor_id) fe3
1378
- JOIN splithub_events g ON g.type = 'goal' AND g.goal = ? AND g.visitor_id = fe3.visitor_id AND g.ts >= fe3.first_ts
1379
- GROUP BY fe3.variant, fe3.visitor_id) rv WHERE rv.variant = fe.variant) AS revenue
1380
- FROM (SELECT variant, visitor_id FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY variant, visitor_id) fe
1381
- GROUP BY fe.variant\`,
1382
- [experiment, goal, experiment, goal, experiment]
1444
+ // Derived tables only (no CTEs, no window functions) so this runs on MySQL 5.7.
1445
+ \`SELECT v.variant, COUNT(*) AS exposures, SUM(v.converted) AS conversions, COALESCE(SUM(v.revenue), 0) AS revenue
1446
+ FROM (
1447
+ SELECT fe.variant, fe.visitor_id,
1448
+ CASE WHEN COUNT(g.id) > 0 THEN 1 ELSE 0 END AS converted,
1449
+ SUM(COALESCE(JSON_EXTRACT(g.props, '$.revenue'), 0)) AS revenue
1450
+ FROM (
1451
+ SELECT MIN(e.variant) AS variant, fv.visitor_id, fv.first_ts
1452
+ FROM (SELECT visitor_id, MIN(ts) AS first_ts FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY visitor_id) fv
1453
+ JOIN splithub_events e ON e.type = 'exposure' AND e.experiment = ? AND e.visitor_id = fv.visitor_id AND e.ts = fv.first_ts
1454
+ GROUP BY fv.visitor_id, fv.first_ts) fe
1455
+ LEFT JOIN splithub_events g ON g.type = 'goal' AND g.goal = ? AND g.visitor_id = fe.visitor_id AND g.ts >= fe.first_ts
1456
+ GROUP BY fe.variant, fe.visitor_id) v
1457
+ GROUP BY v.variant\`,
1458
+ [experiment, experiment, goal]
1383
1459
  );
1384
1460
  return (rows as Array<Record<string, unknown>>).map((x) => ({ variant: String(x.variant), exposures: Number(x.exposures), conversions: Number(x.conversions), revenue: Number(x.revenue) }));
1385
1461
  },
@@ -1400,8 +1476,9 @@ import type { VariantCounts } from '@splithub/stats';
1400
1476
  /**
1401
1477
  * Your own store (Prisma, Drizzle, Supabase, Mongo, anything). Three methods.
1402
1478
  * Attribution rule (keep it, so results match the dashboard):
1403
- * a visitor converts for a variant when they sent the goal at or after their
1404
- * first exposure to that variant; count unique visitors, not events.
1479
+ * a visitor counts once per experiment, in the variant of their earliest
1480
+ * exposure (same ts: smallest variant key), and converts when they sent the
1481
+ * goal at or after that exposure; count unique visitors, not events.
1405
1482
  * Columns worth keeping: id (unique, dedupe on it), type, experiment, variant,
1406
1483
  * goal, visitor_id, session_id, ts, url, props (json), received_at, host,
1407
1484
  * country, device, browser.
@@ -1413,8 +1490,8 @@ export function customStore(): Store {
1413
1490
  throw new Error('implement insertEvents ' + events.length);
1414
1491
  },
1415
1492
  async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
1416
- // For each variant: exposures = unique visitors first exposed to it;
1417
- // conversions = those visitors with a goal event (goal === goal) at ts >= their first exposure;
1493
+ // For each variant: exposures = visitors whose earliest exposure in this experiment was to it;
1494
+ // conversions = those visitors with a goal event (goal === goal) at ts >= that earliest exposure;
1418
1495
  // revenue = sum of props.revenue over those goal events.
1419
1496
  throw new Error('implement variantCounts ' + experiment + ' ' + goal);
1420
1497
  },
@@ -1728,11 +1805,13 @@ async function lifecycle(command, id, flags) {
1728
1805
  } catch (error) {
1729
1806
  live = `could not apply live: ${error instanceof Error ? error.message : String(error)}`;
1730
1807
  }
1731
- if (flags.json) console.log(JSON.stringify({ experiment: id, status: status2, configFile: file, live }));
1808
+ const next = status2 === "finished" ? `Read the verdict (npx splithub results ${id}), decide which variant to keep, then remove the test from the code and the config: the splithub-ship skill walks an agent through it.` : null;
1809
+ if (flags.json) console.log(JSON.stringify({ experiment: id, status: status2, configFile: file, live, ...next ? { next } : {} }));
1732
1810
  else {
1733
1811
  console.log(`${c.green("\u2714")} ${c.bold(id)} \u2192 ${status2} ${c.dim(`(${file})`)}`);
1734
1812
  if (live === "applied") console.log(`${c.green("\u2714")} live in the cloud within 60s, no deploy needed (the config change ships with your next deploy)`);
1735
1813
  else if (live !== "skipped") console.log(c.yellow(`! ${live}`));
1814
+ if (next) console.log(c.dim(`Next: ${next}`));
1736
1815
  }
1737
1816
  }
1738
1817
 
@@ -1742,6 +1821,18 @@ import { readFileSync as readFileSync7 } from "fs";
1742
1821
 
1743
1822
  // src/commands/push.ts
1744
1823
  import { relative as relative6 } from "path";
1824
+ function factorsOf(e) {
1825
+ const out = {};
1826
+ for (const f of e.factors ?? []) {
1827
+ out[f.key] = {};
1828
+ for (const level of f.levels) {
1829
+ const combo = e.variants.find((v) => v.levels?.[f.key] === level);
1830
+ const value = combo?.value?.[f.key];
1831
+ out[f.key][level] = typeof value === "string" ? value : value === void 0 ? level : JSON.stringify(value);
1832
+ }
1833
+ }
1834
+ return out;
1835
+ }
1745
1836
  function adoptCloudExperiments(file, cloudOnly) {
1746
1837
  if (!cloudOnly || cloudOnly.length === 0) return [];
1747
1838
  let source = readSource(file);
@@ -1760,7 +1851,8 @@ function adoptCloudExperiments(file, cloudOnly) {
1760
1851
  traffic: e.traffic,
1761
1852
  maxExposures: e.maxExposures ?? void 0,
1762
1853
  schedule: e.schedule ?? void 0,
1763
- variants: e.variants.map((v) => ({ key: v.key, weight: Math.round(v.weight / total * 100), value: typeof v.value === "string" ? v.value : v.value === void 0 ? void 0 : JSON.stringify(v.value), to: v.to }))
1854
+ variants: e.variants.map((v) => ({ key: v.key, weight: Math.round(v.weight / total * 100), value: typeof v.value === "string" ? v.value : v.value === void 0 ? void 0 : JSON.stringify(v.value), to: v.to })),
1855
+ ...e.kind === "mvt" && e.factors ? { factors: factorsOf(e) } : {}
1764
1856
  });
1765
1857
  written.push(e.key);
1766
1858
  }
@@ -1830,7 +1922,7 @@ async function results(key, flags) {
1830
1922
  const target = resolveTarget(flags, detectProject());
1831
1923
  const r = await fetchResults(target, key);
1832
1924
  if (flags.json) {
1833
- console.log(JSON.stringify(r, null, 2));
1925
+ console.log(JSON.stringify(r.status === "finished" ? { ...r, next: FINISHED_NEXT } : r, null, 2));
1834
1926
  return;
1835
1927
  }
1836
1928
  const v = r.verdict;
@@ -1886,7 +1978,24 @@ async function results(key, flags) {
1886
1978
  if (v.status === "insufficient_sample" && v.sample.neededPerVariant) {
1887
1979
  console.log(c.dim(`Estimated ${v.sample.neededPerVariant} visitors per variant needed to confirm the current lift.`));
1888
1980
  }
1981
+ if (r.factors?.length) {
1982
+ console.log();
1983
+ console.log(c.bold("Each factor on its own") + c.dim(" (a level sums every combination that uses it, compared with the first level)"));
1984
+ for (const f of r.factors) {
1985
+ console.log(
1986
+ table(
1987
+ [f.key.toUpperCase(), "VISITORS", "CONV.", "RATE", "LIFT", "P-VALUE"],
1988
+ f.levels.map((l) => [l.level + (f.best === l.level ? c.green(" \u2714") : ""), String(l.exposures), String(l.conversions), pct(l.rate), l.relativeLift === null ? "\u2013" : pct(l.relativeLift, true), l.pValue === null ? "\u2013" : l.pValue.toFixed(3)])
1989
+ )
1990
+ );
1991
+ }
1992
+ }
1993
+ if (r.status === "finished") {
1994
+ console.log();
1995
+ console.log(c.dim(FINISHED_NEXT));
1996
+ }
1889
1997
  }
1998
+ var FINISHED_NEXT = "This test is finished but its variants are still in the code. Decide which variant to keep, then remove the test from the code and the config (agents: the splithub-ship skill).";
1890
1999
  function label(status2) {
1891
2000
  const icon = verdictIcon(status2);
1892
2001
  switch (status2) {
@@ -1907,8 +2016,10 @@ function label(status2) {
1907
2016
  async function status(flags) {
1908
2017
  const target = resolveTarget(flags, detectProject());
1909
2018
  const data = await fetchStatus(target);
2019
+ const finished = data.experiments.filter((e) => e.status === "finished").map((e) => e.experiment);
2020
+ const next = finished.length ? `Finished but still in the code: ${finished.join(", ")}. Decide which variant to keep, then remove the test from the code and the config (agents: the splithub-ship skill).` : null;
1910
2021
  if (flags.json) {
1911
- console.log(JSON.stringify(data, null, 2));
2022
+ console.log(JSON.stringify(next ? { ...data, next } : data, null, 2));
1912
2023
  return;
1913
2024
  }
1914
2025
  console.log(`${c.bold("SplitHub")} ${c.dim("\xB7")} project ${c.cyan(data.project)} ${c.dim(`\xB7 ${target.url}`)}
@@ -1932,6 +2043,8 @@ async function status(flags) {
1932
2043
  ])
1933
2044
  )
1934
2045
  );
2046
+ if (next) console.log(`
2047
+ ${c.dim(next)}`);
1935
2048
  }
1936
2049
 
1937
2050
  // src/commands/menu.ts
@@ -2054,7 +2167,7 @@ async function review(key, flags) {
2054
2167
  return;
2055
2168
  }
2056
2169
  if (reviews.length === 0) {
2057
- console.log(c.dim(key ? `no reviews for ${key} yet (Pro writes one on finish and daily while running; or click "Review now" in the dashboard)` : "no reviews yet"));
2170
+ console.log(c.dim(key ? `no reviews for ${key} yet (Pro writes one at the start, when the verdict changes while running, and on finish; or click "Review now" in the dashboard)` : "no reviews yet"));
2058
2171
  return;
2059
2172
  }
2060
2173
  for (const r of reviews) {
@@ -2075,7 +2188,7 @@ Usage
2075
2188
  splithub start|pause|finish <id>
2076
2189
  splithub status all experiments with a verdict headline
2077
2190
  splithub results <id> full verdict for one experiment
2078
- splithub review [<id>] AI reviews stored in the cloud (Pro: on finish, daily, or on demand in the dashboard)
2191
+ splithub review [<id>] AI reviews stored in the cloud (Pro: at start, on verdict changes, on finish, or on demand)
2079
2192
  splithub dev [--url] [--hold] open the app with ?sh_debug=1 (console + panel: variants, reasons, events) and watch results
2080
2193
  splithub doctor check the installation
2081
2194
  splithub skills [--mcp] install / refresh agent skills (library, A/B testing, API, MCP, self-hosting, ship the winner); --mcp registers the MCP server
@@ -2128,6 +2241,7 @@ async function main() {
2128
2241
  value: { type: "string", multiple: true },
2129
2242
  match: { type: "string" },
2130
2243
  to: { type: "string", multiple: true },
2244
+ factor: { type: "string", multiple: true },
2131
2245
  status: { type: "string" },
2132
2246
  description: { type: "string" },
2133
2247
  traffic: { type: "string" },