splithub 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +171 -49
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import { parseArgs } from "util";
|
|
|
5
5
|
|
|
6
6
|
// src/commands/create.ts
|
|
7
7
|
import * as p from "@clack/prompts";
|
|
8
|
+
import { mvtVariants } from "@splithub/core";
|
|
8
9
|
import { existsSync as existsSync2, mkdirSync, writeFileSync as writeFileSync2 } from "fs";
|
|
9
10
|
import { join as join2, relative } from "path";
|
|
10
11
|
|
|
@@ -78,7 +79,13 @@ function renderExperiment(e, indent = " ") {
|
|
|
78
79
|
lines.push(`${i2}kind: ${q(e.kind)},`);
|
|
79
80
|
lines.push(`${i2}status: ${q(e.status)},`);
|
|
80
81
|
if (e.match) lines.push(`${i2}match: ${q(e.match)},`);
|
|
81
|
-
if (e.kind === "
|
|
82
|
+
if (e.kind === "mvt") {
|
|
83
|
+
lines.push(`${i2}factors: {`);
|
|
84
|
+
for (const [factor, levels] of Object.entries(e.factors ?? {})) {
|
|
85
|
+
lines.push(`${i3}${k(factor)}: { ${Object.entries(levels).map(([l, v]) => `${k(l)}: ${q(v)}`).join(", ")} },`);
|
|
86
|
+
}
|
|
87
|
+
lines.push(`${i2}},`);
|
|
88
|
+
} else if (e.kind === "component") {
|
|
82
89
|
lines.push(`${i2}variants: { ${e.variants.map((v) => `${k(v.key)}: ${v.weight}`).join(", ")} },`);
|
|
83
90
|
} else {
|
|
84
91
|
lines.push(`${i2}variants: {`);
|
|
@@ -321,7 +328,7 @@ one raw query for \`variantCounts\`. Supabase: the same SQL as a Postgres functi
|
|
|
321
328
|
// src/skills-ship.ts
|
|
322
329
|
var SKILL_SHIP = `---
|
|
323
330
|
name: splithub-ship
|
|
324
|
-
description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up.
|
|
331
|
+
description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up; and whenever a test shows status finished and the user agreed on which variant to keep.
|
|
325
332
|
---
|
|
326
333
|
|
|
327
334
|
# Shipping the winner
|
|
@@ -385,21 +392,32 @@ Events go to the app's own database (self mode) or to SplitHub Cloud (cloud mode
|
|
|
385
392
|
- Results: \`npx splithub status --json\`, \`npx splithub results <id> --json\`.
|
|
386
393
|
- Everything interactive: \`npx splithub\` (menu). Everything scriptable: subcommands + \`--json\`.
|
|
387
394
|
|
|
395
|
+
## Pick the kind
|
|
396
|
+
|
|
397
|
+
| kind | use it when | create | read in code |
|
|
398
|
+
|---|---|---|---|
|
|
399
|
+
| \`value\` | only data changes: text, number, image URL, price, JSON. The default choice: no new components. | \`--variants control,bold --value control="Old text" --value bold="New text"\` | \`const text = useValue<string>('<id>')\`; server components: \`getValue\` from \`@splithub/next\`; no SDK: \`sh.value('<id>')\` |
|
|
400
|
+
| \`component\` | structure changes: a layout, a section, a flow, a different component per variant | \`--variants control,bold\` (the CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs) | \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`, or \`useVariant('<id>')\` in a switch |
|
|
401
|
+
| \`redirect\` | a whole page changes: another URL per variant | \`--match / --to control=/ --to v2=/home-v2\` | nothing: \`@splithub/next\` middleware or the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` sends visitors there |
|
|
402
|
+
| \`mvt\` | several things change at once (headline \xD7 button \xD7 image) and the user wants to know which of them matters; every combination is tested and each factor is read on its own | \`--factor headline=old:"Old text",new:"New text" --factor button=blue:#2563eb,green:#16a34a\` (no \`--variants\`; first level = control; at most 4 factors \xD7 4 levels, 16 combinations) | \`const { headline, button } = useValue<{ headline: string; button: string }>('<id>')\` |
|
|
403
|
+
|
|
404
|
+
mvt needs more visitors than an A/B test: every combination must reach the minimum on its own (the results show the
|
|
405
|
+
combinations and, below them, each factor summed over its combinations). With little traffic, test one thing with \`value\`.
|
|
406
|
+
|
|
388
407
|
## Create an experiment
|
|
389
408
|
|
|
390
|
-
1. \`npx splithub create --kind <value|component|redirect> --id <key>
|
|
391
|
-
- \`value\`: only data changes (text, image URL, number, JSON). Prefer it: no new components.
|
|
392
|
-
\`--value control="Old text" --value bold="New text"\`; read with \`useValue<string>('<id>')\`.
|
|
393
|
-
- \`component\`: structure changes. \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`.
|
|
394
|
-
The CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs.
|
|
395
|
-
- \`redirect\`: a whole page changes. \`--match / --to control=/ --to v2=/home-v2\`. Served by \`@splithub/next\` middleware,
|
|
396
|
-
or by the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` on sites without the SDK.
|
|
409
|
+
1. \`npx splithub create --kind <value|component|redirect|mvt> --id <key> \u2026 --goal <goal> --yes --json\` (see the table).
|
|
397
410
|
2. Goal: pick an existing key or add one to \`goals\`:
|
|
398
411
|
\`{ type: 'event' }\` + \`track('<goal>')\` where the conversion happens; \`{ type: 'pageview', path: '/thanks' }\`;
|
|
399
412
|
\`{ type: 'click', selector: '#buy' }\` (tracked automatically by the SDK and the script).
|
|
400
413
|
3. Wire it in code, keep \`status: 'draft'\` until the user confirms, then \`npx splithub start <id>\`.
|
|
401
414
|
4. Cloud mode: \`npx splithub push\` after any config change.
|
|
402
415
|
|
|
416
|
+
A draft is invisible to visitors. To check it before starting: \`runDraftsInDev: true\` at the top level of the config
|
|
417
|
+
serves drafts like running tests in development only (\`NODE_ENV=development\`: \`next dev\`, \`vite\`; nothing is billed);
|
|
418
|
+
\`?sh_force=<id>:<variant>\` shows one variant on any page, draft or not, without recording an exposure;
|
|
419
|
+
\`?sh_debug=1\` (or \`npx splithub dev\`) logs every decision and opens a panel to switch variants.
|
|
420
|
+
|
|
403
421
|
Keys: lowercase a-z 0-9 - _, max 64 chars. First variant = control. Weights are relative (50/50, 1/1, 90/10).
|
|
404
422
|
Targeting (country, device, UTM, cookie, time) is a \`flow\` graph on the experiment, edited in the dashboard; \`pull\` brings it into the file.
|
|
405
423
|
|
|
@@ -409,10 +427,20 @@ Targeting (country, device, UTM, cookie, time) is a \`flow\` graph on the experi
|
|
|
409
427
|
Report \`verdict.message\` verbatim and the numbers behind it (visitors per variant, lift, p-value, CI).
|
|
410
428
|
Never declare a winner from raw rates. \`srm_detected\` means the split is broken: fix the setup before anything else.
|
|
411
429
|
|
|
412
|
-
## Finish
|
|
430
|
+
## Finish, and what to do when a test is finished
|
|
431
|
+
|
|
432
|
+
\`npx splithub finish <id>\` stops the test (live within a minute in cloud mode). A finished test keeps its numbers for as long
|
|
433
|
+
as it stays in the config; the code still contains every variant until someone removes it.
|
|
413
434
|
|
|
414
|
-
|
|
415
|
-
|
|
435
|
+
Whenever you see a test with \`status: finished\` (in \`status\`, \`results\`, the MCP tools, or right after you finished it):
|
|
436
|
+
1. Read \`npx splithub results <id> --json\` and tell the user the verdict in one line with its numbers.
|
|
437
|
+
2. Ask two questions, do not assume: **which variant to keep** (recommend \`verdict.winner\`; with no significant
|
|
438
|
+
difference, the control or the simpler one) and **whether to remove the test from the code now**.
|
|
439
|
+
3. On yes, follow the \`splithub-ship\` skill: keep only the chosen variant's content, delete the other variants' code and
|
|
440
|
+
files, remove the entry from \`splithub.config.ts\`, \`push\` in cloud mode, run \`doctor\` and the project's checks.
|
|
441
|
+
4. On no, leave everything: a finished test costs nothing and the user may come back to it.
|
|
442
|
+
|
|
443
|
+
A \`paused\` test is not over: say so and ask whether to resume (\`start\`) or finish it.
|
|
416
444
|
|
|
417
445
|
## Do not
|
|
418
446
|
|
|
@@ -572,9 +600,9 @@ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory),
|
|
|
572
600
|
|---|---|---|
|
|
573
601
|
| \`splithub_status\` | all experiments with verdict headline | start here |
|
|
574
602
|
| \`splithub_results\` | full verdict for one experiment | report \`verdict.message\` verbatim |
|
|
575
|
-
| \`splithub_review\` | stored AI reviews (Pro): on
|
|
576
|
-
| \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind, id, variants, goal, values/to; starts as draft |
|
|
577
|
-
| \`splithub_set_status\` | start / pause / finish | applies live in cloud mode |
|
|
603
|
+
| \`splithub_review\` | stored AI reviews (Pro): at start, on verdict changes, on finish, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |
|
|
604
|
+
| \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind value / component / redirect / mvt, id, variants (or factors for mvt), goal, values/to; starts as draft |
|
|
605
|
+
| \`splithub_set_status\` | start / pause / finish | applies live in cloud mode; after \`finished\`, ask which variant to keep and whether to clean the code (splithub-ship skill) |
|
|
578
606
|
| \`splithub_push\` / \`splithub_pull\` | sync the config with the cloud | pull adopts dashboard-created experiments |
|
|
579
607
|
| \`splithub_doctor\` | checks the installation | run after wiring code |
|
|
580
608
|
| \`splithub_login\` | returns a sign-in link; the human approves it in the browser | show the link, then call \`splithub_whoami\` |
|
|
@@ -586,6 +614,8 @@ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory),
|
|
|
586
614
|
- Results: never call a winner from raw rates; quote the verdict. \`srm_detected\` = fix the setup first.
|
|
587
615
|
- Running experiments are frozen: no weight / variant / seed edits; create a new one.
|
|
588
616
|
- If a tool answers that the folder is not connected, call \`splithub_login\`, show the link to the user, wait for them, then retry.
|
|
617
|
+
- A \`finished\` experiment in \`splithub_status\` or \`splithub_results\` means the test is over but its code is still there: report
|
|
618
|
+
the verdict, ask which variant to keep and whether to remove the test from the code, then follow the \`splithub-ship\` skill.
|
|
589
619
|
`;
|
|
590
620
|
var SKILLS = [
|
|
591
621
|
{ dir: "splithub", body: SKILL_SPLITHUB },
|
|
@@ -607,6 +637,8 @@ function configTemplate(project, mode, key) {
|
|
|
607
637
|
export default defineConfig({
|
|
608
638
|
project: '${project}',
|
|
609
639
|
storage: ${storage},
|
|
640
|
+
// Drafts behave like running tests while you develop (next dev, vite); production ignores them.
|
|
641
|
+
runDraftsInDev: true,
|
|
610
642
|
|
|
611
643
|
goals: {
|
|
612
644
|
// call track('signup') where the conversion happens
|
|
@@ -679,10 +711,13 @@ This project uses SplitHub (\`@splithub/react\`). Experiments are declared in
|
|
|
679
711
|
\`useValue(id)\`. Conversions: \`useTrack()\` then \`track('<goal>')\`.
|
|
680
712
|
|
|
681
713
|
- Not connected yet? \`npx splithub init --mode cloud\` (or \`npx splithub login\`) prints a link: show it to the user, they sign in and approve, the CLI gets the keys. Never ask the user to paste keys.
|
|
682
|
-
- New experiment: \`npx splithub create --kind value|component|redirect --id <key> --variants control,<other> --goal <goal> --yes\`
|
|
683
|
-
then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key
|
|
714
|
+
- New experiment: \`npx splithub create --kind value|component|redirect|mvt --id <key> --variants control,<other> --goal <goal> --yes\`
|
|
715
|
+
then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key, redirect \u2192 nothing,
|
|
716
|
+
mvt \u2192 \`--factor name=level:value,\u2026\` and \`const { a, b } = useValue(id)\`).
|
|
684
717
|
- Results: \`npx splithub results <id> --json\`; report \`verdict.message\` verbatim, never call a winner from raw rates.
|
|
685
718
|
- Lifecycle: \`npx splithub start|pause|finish <id>\`. Check setup: \`npx splithub doctor\`.
|
|
719
|
+
- A \`finished\` test still has all its variants in the code: report the verdict, ask which variant to keep and whether to
|
|
720
|
+
remove the test from the code, then follow .claude/skills/splithub-ship/SKILL.md. Never delete test code unasked.
|
|
686
721
|
- Keep \`status: 'draft'\` until the user confirms. Never change \`seed\` of a running experiment.
|
|
687
722
|
- Config ships to the browser: no secrets in it. Full rules: .claude/skills/splithub/SKILL.md
|
|
688
723
|
`;
|
|
@@ -804,14 +839,15 @@ async function create(flags) {
|
|
|
804
839
|
if (interactive) p.intro(c.bold("splithub create"));
|
|
805
840
|
let kind = str(flags.kind);
|
|
806
841
|
if (!kind) {
|
|
807
|
-
if (!interactive) throw new Error("--kind value|component|redirect is required with --yes");
|
|
842
|
+
if (!interactive) throw new Error("--kind value|component|redirect|mvt is required with --yes");
|
|
808
843
|
kind = await ask(
|
|
809
844
|
p.select({
|
|
810
845
|
message: "What changes between variants?",
|
|
811
846
|
options: [
|
|
812
847
|
{ value: "value", label: "Text, image URL, number", hint: "no new components" },
|
|
813
848
|
{ value: "component", label: "Layout / structure", hint: "one component per variant" },
|
|
814
|
-
{ value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" }
|
|
849
|
+
{ value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" },
|
|
850
|
+
{ value: "mvt", label: "Several things at once", hint: "multivariate: every combination, each factor read on its own" }
|
|
815
851
|
]
|
|
816
852
|
})
|
|
817
853
|
);
|
|
@@ -829,7 +865,47 @@ async function create(flags) {
|
|
|
829
865
|
}
|
|
830
866
|
if (!KEY.test(id)) throw new Error(`invalid id "${id}" (lowercase a-z 0-9 - _, max 64)`);
|
|
831
867
|
if (hasExperiment(source, id)) throw new Error(`experiment "${id}" already exists`);
|
|
832
|
-
|
|
868
|
+
const factors = {};
|
|
869
|
+
if (kind === "mvt") {
|
|
870
|
+
for (const raw of list(flags.factor)) {
|
|
871
|
+
const eq = raw.indexOf("=");
|
|
872
|
+
if (eq <= 0) throw new Error(`--factor expects name=level:value,level:value, got "${raw}"`);
|
|
873
|
+
const name = raw.slice(0, eq).trim();
|
|
874
|
+
const levels = {};
|
|
875
|
+
for (const part of raw.slice(eq + 1).split(",")) {
|
|
876
|
+
const colon = part.indexOf(":");
|
|
877
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
878
|
+
if (!level) continue;
|
|
879
|
+
levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
880
|
+
}
|
|
881
|
+
factors[name] = levels;
|
|
882
|
+
}
|
|
883
|
+
while (interactive && Object.keys(factors).length === 0) {
|
|
884
|
+
const name = await ask(p.text({ message: "Factor name (what changes), e.g. headline", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
|
|
885
|
+
const levelsRaw = await ask(p.text({ message: `Levels of "${name}" as key:value, comma separated (first = control)`, placeholder: "old:Old headline, new:New headline" }));
|
|
886
|
+
const levels = {};
|
|
887
|
+
for (const part of levelsRaw.split(",")) {
|
|
888
|
+
const colon = part.indexOf(":");
|
|
889
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
890
|
+
if (level) levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
891
|
+
}
|
|
892
|
+
factors[name] = levels;
|
|
893
|
+
while (await ask(p.confirm({ message: "Add another factor?", initialValue: false }))) {
|
|
894
|
+
const more = await ask(p.text({ message: "Factor name", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
|
|
895
|
+
const moreRaw = await ask(p.text({ message: `Levels of "${more}" as key:value, comma separated`, placeholder: "blue:#2563eb, green:#16a34a" }));
|
|
896
|
+
const moreLevels = {};
|
|
897
|
+
for (const part of moreRaw.split(",")) {
|
|
898
|
+
const colon = part.indexOf(":");
|
|
899
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
900
|
+
if (level) moreLevels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
901
|
+
}
|
|
902
|
+
factors[more] = moreLevels;
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
if (Object.keys(factors).length === 0) throw new Error("--factor name=level:value,level:value is required for mvt experiments (repeat it per factor)");
|
|
906
|
+
}
|
|
907
|
+
const mvt = kind === "mvt" ? mvtVariants(id, factors) : null;
|
|
908
|
+
let variantKeys = mvt ? mvt.variants.map((v) => v.key) : str(flags.variants)?.split(",").map((s) => s.trim()).filter(Boolean);
|
|
833
909
|
if (!variantKeys || variantKeys.length < 2) {
|
|
834
910
|
if (!interactive) variantKeys = ["control", "b"];
|
|
835
911
|
else {
|
|
@@ -885,7 +961,8 @@ async function create(flags) {
|
|
|
885
961
|
goal,
|
|
886
962
|
match,
|
|
887
963
|
description,
|
|
888
|
-
variants: variantKeys.map((k2, i) => ({ key: k2, weight: weights[i], value: values[k2], to: tos[k2] })),
|
|
964
|
+
variants: mvt ? mvt.variants.map((v) => ({ key: v.key, weight: v.weight })) : variantKeys.map((k2, i) => ({ key: k2, weight: weights[i], value: values[k2], to: tos[k2] })),
|
|
965
|
+
...mvt ? { factors } : {},
|
|
889
966
|
// limits: --traffic 50 (percent), --max-visitors 10000, --start / --end ISO dates
|
|
890
967
|
traffic: typeof flags.traffic === "string" && flags.traffic !== "" ? Math.min(1, Math.max(0, Number(flags.traffic) / 100)) : void 0,
|
|
891
968
|
maxExposures: typeof flags["max-visitors"] === "string" && Number(flags["max-visitors"]) > 0 ? Math.floor(Number(flags["max-visitors"])) : void 0,
|
|
@@ -907,7 +984,7 @@ async function create(flags) {
|
|
|
907
984
|
}
|
|
908
985
|
}
|
|
909
986
|
}
|
|
910
|
-
const snippet = kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
|
|
987
|
+
const snippet = kind === "mvt" ? `const { ${Object.keys(factors).join(", ")} } = useValue<{ ${Object.keys(factors).map((f) => `${f}: string`).join("; ")} }>('${id}');` : kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
|
|
911
988
|
${variantKeys.map((k2) => ` <Variant name="${k2}"><${componentNames[k2]} /></Variant>`).join("\n")}
|
|
912
989
|
</Experiment>` : `// handled by @splithub/next middleware on ${match}; no code needed`;
|
|
913
990
|
const result = { experiment: id, kind, status: status2, variants: experiment.variants, goal: goal ?? null, configFile: relative(project.root, project.configFile), files, snippet };
|
|
@@ -920,6 +997,7 @@ ${variantKeys.map((k2) => ` <Variant name="${k2}"><${componentNames[k2]} /></Va
|
|
|
920
997
|
if (files.length) p.note(files.join("\n"), "Scaffolded");
|
|
921
998
|
const cloud = /mode\s*:\s*['"]cloud['"]/.test(readSource(project.configFile));
|
|
922
999
|
if (cloud) p.log.warn(`Cloud mode: the cloud does not know about it until you run ${c.cyan("npx splithub push")}.`);
|
|
1000
|
+
if (status2 === "draft") p.log.info(`See it before you start it: ${c.cyan("runDraftsInDev: true")} in the config serves drafts in development, ${c.cyan(`?sh_force=${id}:<variant>`)} shows one variant anywhere.`);
|
|
923
1001
|
p.outro(`Added ${c.bold(id)} (${status2}) to ${relative(project.root, project.configFile)}. ${status2 === "running" ? "" : `Start it with ${c.cyan(`npx splithub start ${id}`)}. `}${cloud ? `Then ${c.cyan("npx splithub push")}.` : ""}`);
|
|
924
1002
|
}
|
|
925
1003
|
|
|
@@ -1299,9 +1377,14 @@ export function pgStore(pool: Pool = new Pool({ connectionString: process.env.DA
|
|
|
1299
1377
|
},
|
|
1300
1378
|
async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
|
|
1301
1379
|
const r = await pool.query(
|
|
1302
|
-
\`WITH
|
|
1303
|
-
SELECT
|
|
1304
|
-
WHERE type = 'exposure' AND experiment = $1 GROUP BY
|
|
1380
|
+
\`WITH fv AS (
|
|
1381
|
+
SELECT visitor_id, MIN(ts) AS first_ts FROM splithub_events
|
|
1382
|
+
WHERE type = 'exposure' AND experiment = $1 GROUP BY visitor_id),
|
|
1383
|
+
first_exposure AS (
|
|
1384
|
+
SELECT MIN(e.variant) AS variant, fv.visitor_id, fv.first_ts
|
|
1385
|
+
FROM fv JOIN splithub_events e
|
|
1386
|
+
ON e.type = 'exposure' AND e.experiment = $1 AND e.visitor_id = fv.visitor_id AND e.ts = fv.first_ts
|
|
1387
|
+
GROUP BY fv.visitor_id, fv.first_ts),
|
|
1305
1388
|
converted AS (
|
|
1306
1389
|
SELECT fe.variant, fe.visitor_id, SUM(COALESCE((g.props->>'revenue')::numeric, 0)) AS revenue
|
|
1307
1390
|
FROM first_exposure fe JOIN splithub_events g
|
|
@@ -1366,20 +1449,21 @@ export function mysqlStore(pool: Pool = mysql.createPool(process.env.DATABASE_UR
|
|
|
1366
1449
|
},
|
|
1367
1450
|
async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
|
|
1368
1451
|
const [rows] = await pool.query(
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1452
|
+
// Derived tables only (no CTEs, no window functions) so this runs on MySQL 5.7.
|
|
1453
|
+
\`SELECT v.variant, COUNT(*) AS exposures, SUM(v.converted) AS conversions, COALESCE(SUM(v.revenue), 0) AS revenue
|
|
1454
|
+
FROM (
|
|
1455
|
+
SELECT fe.variant, fe.visitor_id,
|
|
1456
|
+
CASE WHEN COUNT(g.id) > 0 THEN 1 ELSE 0 END AS converted,
|
|
1457
|
+
SUM(COALESCE(JSON_EXTRACT(g.props, '$.revenue'), 0)) AS revenue
|
|
1458
|
+
FROM (
|
|
1459
|
+
SELECT MIN(e.variant) AS variant, fv.visitor_id, fv.first_ts
|
|
1460
|
+
FROM (SELECT visitor_id, MIN(ts) AS first_ts FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY visitor_id) fv
|
|
1461
|
+
JOIN splithub_events e ON e.type = 'exposure' AND e.experiment = ? AND e.visitor_id = fv.visitor_id AND e.ts = fv.first_ts
|
|
1462
|
+
GROUP BY fv.visitor_id, fv.first_ts) fe
|
|
1463
|
+
LEFT JOIN splithub_events g ON g.type = 'goal' AND g.goal = ? AND g.visitor_id = fe.visitor_id AND g.ts >= fe.first_ts
|
|
1464
|
+
GROUP BY fe.variant, fe.visitor_id) v
|
|
1465
|
+
GROUP BY v.variant\`,
|
|
1466
|
+
[experiment, experiment, goal]
|
|
1383
1467
|
);
|
|
1384
1468
|
return (rows as Array<Record<string, unknown>>).map((x) => ({ variant: String(x.variant), exposures: Number(x.exposures), conversions: Number(x.conversions), revenue: Number(x.revenue) }));
|
|
1385
1469
|
},
|
|
@@ -1400,8 +1484,9 @@ import type { VariantCounts } from '@splithub/stats';
|
|
|
1400
1484
|
/**
|
|
1401
1485
|
* Your own store (Prisma, Drizzle, Supabase, Mongo, anything). Three methods.
|
|
1402
1486
|
* Attribution rule (keep it, so results match the dashboard):
|
|
1403
|
-
* a visitor
|
|
1404
|
-
*
|
|
1487
|
+
* a visitor counts once per experiment, in the variant of their earliest
|
|
1488
|
+
* exposure (same ts: smallest variant key), and converts when they sent the
|
|
1489
|
+
* goal at or after that exposure; count unique visitors, not events.
|
|
1405
1490
|
* Columns worth keeping: id (unique, dedupe on it), type, experiment, variant,
|
|
1406
1491
|
* goal, visitor_id, session_id, ts, url, props (json), received_at, host,
|
|
1407
1492
|
* country, device, browser.
|
|
@@ -1413,8 +1498,8 @@ export function customStore(): Store {
|
|
|
1413
1498
|
throw new Error('implement insertEvents ' + events.length);
|
|
1414
1499
|
},
|
|
1415
1500
|
async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
|
|
1416
|
-
// For each variant: exposures =
|
|
1417
|
-
// conversions = those visitors with a goal event (goal === goal) at ts >=
|
|
1501
|
+
// For each variant: exposures = visitors whose earliest exposure in this experiment was to it;
|
|
1502
|
+
// conversions = those visitors with a goal event (goal === goal) at ts >= that earliest exposure;
|
|
1418
1503
|
// revenue = sum of props.revenue over those goal events.
|
|
1419
1504
|
throw new Error('implement variantCounts ' + experiment + ' ' + goal);
|
|
1420
1505
|
},
|
|
@@ -1728,11 +1813,13 @@ async function lifecycle(command, id, flags) {
|
|
|
1728
1813
|
} catch (error) {
|
|
1729
1814
|
live = `could not apply live: ${error instanceof Error ? error.message : String(error)}`;
|
|
1730
1815
|
}
|
|
1731
|
-
|
|
1816
|
+
const next = status2 === "finished" ? `Read the verdict (npx splithub results ${id}), decide which variant to keep, then remove the test from the code and the config: the splithub-ship skill walks an agent through it.` : null;
|
|
1817
|
+
if (flags.json) console.log(JSON.stringify({ experiment: id, status: status2, configFile: file, live, ...next ? { next } : {} }));
|
|
1732
1818
|
else {
|
|
1733
1819
|
console.log(`${c.green("\u2714")} ${c.bold(id)} \u2192 ${status2} ${c.dim(`(${file})`)}`);
|
|
1734
1820
|
if (live === "applied") console.log(`${c.green("\u2714")} live in the cloud within 60s, no deploy needed (the config change ships with your next deploy)`);
|
|
1735
1821
|
else if (live !== "skipped") console.log(c.yellow(`! ${live}`));
|
|
1822
|
+
if (next) console.log(c.dim(`Next: ${next}`));
|
|
1736
1823
|
}
|
|
1737
1824
|
}
|
|
1738
1825
|
|
|
@@ -1742,6 +1829,18 @@ import { readFileSync as readFileSync7 } from "fs";
|
|
|
1742
1829
|
|
|
1743
1830
|
// src/commands/push.ts
|
|
1744
1831
|
import { relative as relative6 } from "path";
|
|
1832
|
+
function factorsOf(e) {
|
|
1833
|
+
const out = {};
|
|
1834
|
+
for (const f of e.factors ?? []) {
|
|
1835
|
+
out[f.key] = {};
|
|
1836
|
+
for (const level of f.levels) {
|
|
1837
|
+
const combo = e.variants.find((v) => v.levels?.[f.key] === level);
|
|
1838
|
+
const value = combo?.value?.[f.key];
|
|
1839
|
+
out[f.key][level] = typeof value === "string" ? value : value === void 0 ? level : JSON.stringify(value);
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
return out;
|
|
1843
|
+
}
|
|
1745
1844
|
function adoptCloudExperiments(file, cloudOnly) {
|
|
1746
1845
|
if (!cloudOnly || cloudOnly.length === 0) return [];
|
|
1747
1846
|
let source = readSource(file);
|
|
@@ -1760,7 +1859,8 @@ function adoptCloudExperiments(file, cloudOnly) {
|
|
|
1760
1859
|
traffic: e.traffic,
|
|
1761
1860
|
maxExposures: e.maxExposures ?? void 0,
|
|
1762
1861
|
schedule: e.schedule ?? void 0,
|
|
1763
|
-
variants: e.variants.map((v) => ({ key: v.key, weight: Math.round(v.weight / total * 100), value: typeof v.value === "string" ? v.value : v.value === void 0 ? void 0 : JSON.stringify(v.value), to: v.to }))
|
|
1862
|
+
variants: e.variants.map((v) => ({ key: v.key, weight: Math.round(v.weight / total * 100), value: typeof v.value === "string" ? v.value : v.value === void 0 ? void 0 : JSON.stringify(v.value), to: v.to })),
|
|
1863
|
+
...e.kind === "mvt" && e.factors ? { factors: factorsOf(e) } : {}
|
|
1764
1864
|
});
|
|
1765
1865
|
written.push(e.key);
|
|
1766
1866
|
}
|
|
@@ -1830,7 +1930,7 @@ async function results(key, flags) {
|
|
|
1830
1930
|
const target = resolveTarget(flags, detectProject());
|
|
1831
1931
|
const r = await fetchResults(target, key);
|
|
1832
1932
|
if (flags.json) {
|
|
1833
|
-
console.log(JSON.stringify(r, null, 2));
|
|
1933
|
+
console.log(JSON.stringify(r.status === "finished" ? { ...r, next: FINISHED_NEXT } : r, null, 2));
|
|
1834
1934
|
return;
|
|
1835
1935
|
}
|
|
1836
1936
|
const v = r.verdict;
|
|
@@ -1886,7 +1986,24 @@ async function results(key, flags) {
|
|
|
1886
1986
|
if (v.status === "insufficient_sample" && v.sample.neededPerVariant) {
|
|
1887
1987
|
console.log(c.dim(`Estimated ${v.sample.neededPerVariant} visitors per variant needed to confirm the current lift.`));
|
|
1888
1988
|
}
|
|
1989
|
+
if (r.factors?.length) {
|
|
1990
|
+
console.log();
|
|
1991
|
+
console.log(c.bold("Each factor on its own") + c.dim(" (a level sums every combination that uses it, compared with the first level)"));
|
|
1992
|
+
for (const f of r.factors) {
|
|
1993
|
+
console.log(
|
|
1994
|
+
table(
|
|
1995
|
+
[f.key.toUpperCase(), "VISITORS", "CONV.", "RATE", "LIFT", "P-VALUE"],
|
|
1996
|
+
f.levels.map((l) => [l.level + (f.best === l.level ? c.green(" \u2714") : ""), String(l.exposures), String(l.conversions), pct(l.rate), l.relativeLift === null ? "\u2013" : pct(l.relativeLift, true), l.pValue === null ? "\u2013" : l.pValue.toFixed(3)])
|
|
1997
|
+
)
|
|
1998
|
+
);
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
if (r.status === "finished") {
|
|
2002
|
+
console.log();
|
|
2003
|
+
console.log(c.dim(FINISHED_NEXT));
|
|
2004
|
+
}
|
|
1889
2005
|
}
|
|
2006
|
+
var FINISHED_NEXT = "This test is finished but its variants are still in the code. Decide which variant to keep, then remove the test from the code and the config (agents: the splithub-ship skill).";
|
|
1890
2007
|
function label(status2) {
|
|
1891
2008
|
const icon = verdictIcon(status2);
|
|
1892
2009
|
switch (status2) {
|
|
@@ -1907,8 +2024,10 @@ function label(status2) {
|
|
|
1907
2024
|
async function status(flags) {
|
|
1908
2025
|
const target = resolveTarget(flags, detectProject());
|
|
1909
2026
|
const data = await fetchStatus(target);
|
|
2027
|
+
const finished = data.experiments.filter((e) => e.status === "finished").map((e) => e.experiment);
|
|
2028
|
+
const next = finished.length ? `Finished but still in the code: ${finished.join(", ")}. Decide which variant to keep, then remove the test from the code and the config (agents: the splithub-ship skill).` : null;
|
|
1910
2029
|
if (flags.json) {
|
|
1911
|
-
console.log(JSON.stringify(data, null, 2));
|
|
2030
|
+
console.log(JSON.stringify(next ? { ...data, next } : data, null, 2));
|
|
1912
2031
|
return;
|
|
1913
2032
|
}
|
|
1914
2033
|
console.log(`${c.bold("SplitHub")} ${c.dim("\xB7")} project ${c.cyan(data.project)} ${c.dim(`\xB7 ${target.url}`)}
|
|
@@ -1932,6 +2051,8 @@ async function status(flags) {
|
|
|
1932
2051
|
])
|
|
1933
2052
|
)
|
|
1934
2053
|
);
|
|
2054
|
+
if (next) console.log(`
|
|
2055
|
+
${c.dim(next)}`);
|
|
1935
2056
|
}
|
|
1936
2057
|
|
|
1937
2058
|
// src/commands/menu.ts
|
|
@@ -2054,7 +2175,7 @@ async function review(key, flags) {
|
|
|
2054
2175
|
return;
|
|
2055
2176
|
}
|
|
2056
2177
|
if (reviews.length === 0) {
|
|
2057
|
-
console.log(c.dim(key ? `no reviews for ${key} yet (Pro writes one
|
|
2178
|
+
console.log(c.dim(key ? `no reviews for ${key} yet (Pro writes one at the start, when the verdict changes while running, and on finish; or click "Review now" in the dashboard)` : "no reviews yet"));
|
|
2058
2179
|
return;
|
|
2059
2180
|
}
|
|
2060
2181
|
for (const r of reviews) {
|
|
@@ -2075,7 +2196,7 @@ Usage
|
|
|
2075
2196
|
splithub start|pause|finish <id>
|
|
2076
2197
|
splithub status all experiments with a verdict headline
|
|
2077
2198
|
splithub results <id> full verdict for one experiment
|
|
2078
|
-
splithub review [<id>] AI reviews stored in the cloud (Pro: on
|
|
2199
|
+
splithub review [<id>] AI reviews stored in the cloud (Pro: at start, on verdict changes, on finish, or on demand)
|
|
2079
2200
|
splithub dev [--url] [--hold] open the app with ?sh_debug=1 (console + panel: variants, reasons, events) and watch results
|
|
2080
2201
|
splithub doctor check the installation
|
|
2081
2202
|
splithub skills [--mcp] install / refresh agent skills (library, A/B testing, API, MCP, self-hosting, ship the winner); --mcp registers the MCP server
|
|
@@ -2128,6 +2249,7 @@ async function main() {
|
|
|
2128
2249
|
value: { type: "string", multiple: true },
|
|
2129
2250
|
match: { type: "string" },
|
|
2130
2251
|
to: { type: "string", multiple: true },
|
|
2252
|
+
factor: { type: "string", multiple: true },
|
|
2131
2253
|
status: { type: "string" },
|
|
2132
2254
|
description: { type: "string" },
|
|
2133
2255
|
traffic: { type: "string" },
|