splithub 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +163 -49
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import { parseArgs } from "util";
|
|
|
5
5
|
|
|
6
6
|
// src/commands/create.ts
|
|
7
7
|
import * as p from "@clack/prompts";
|
|
8
|
+
import { mvtVariants } from "@splithub/core";
|
|
8
9
|
import { existsSync as existsSync2, mkdirSync, writeFileSync as writeFileSync2 } from "fs";
|
|
9
10
|
import { join as join2, relative } from "path";
|
|
10
11
|
|
|
@@ -78,7 +79,13 @@ function renderExperiment(e, indent = " ") {
|
|
|
78
79
|
lines.push(`${i2}kind: ${q(e.kind)},`);
|
|
79
80
|
lines.push(`${i2}status: ${q(e.status)},`);
|
|
80
81
|
if (e.match) lines.push(`${i2}match: ${q(e.match)},`);
|
|
81
|
-
if (e.kind === "
|
|
82
|
+
if (e.kind === "mvt") {
|
|
83
|
+
lines.push(`${i2}factors: {`);
|
|
84
|
+
for (const [factor, levels] of Object.entries(e.factors ?? {})) {
|
|
85
|
+
lines.push(`${i3}${k(factor)}: { ${Object.entries(levels).map(([l, v]) => `${k(l)}: ${q(v)}`).join(", ")} },`);
|
|
86
|
+
}
|
|
87
|
+
lines.push(`${i2}},`);
|
|
88
|
+
} else if (e.kind === "component") {
|
|
82
89
|
lines.push(`${i2}variants: { ${e.variants.map((v) => `${k(v.key)}: ${v.weight}`).join(", ")} },`);
|
|
83
90
|
} else {
|
|
84
91
|
lines.push(`${i2}variants: {`);
|
|
@@ -321,7 +328,7 @@ one raw query for \`variantCounts\`. Supabase: the same SQL as a Postgres functi
|
|
|
321
328
|
// src/skills-ship.ts
|
|
322
329
|
var SKILL_SHIP = `---
|
|
323
330
|
name: splithub-ship
|
|
324
|
-
description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up.
|
|
331
|
+
description: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up; and whenever a test shows status finished and the user agreed on which variant to keep.
|
|
325
332
|
---
|
|
326
333
|
|
|
327
334
|
# Shipping the winner
|
|
@@ -385,15 +392,21 @@ Events go to the app's own database (self mode) or to SplitHub Cloud (cloud mode
|
|
|
385
392
|
- Results: \`npx splithub status --json\`, \`npx splithub results <id> --json\`.
|
|
386
393
|
- Everything interactive: \`npx splithub\` (menu). Everything scriptable: subcommands + \`--json\`.
|
|
387
394
|
|
|
395
|
+
## Pick the kind
|
|
396
|
+
|
|
397
|
+
| kind | use it when | create | read in code |
|
|
398
|
+
|---|---|---|---|
|
|
399
|
+
| \`value\` | only data changes: text, number, image URL, price, JSON. The default choice: no new components. | \`--variants control,bold --value control="Old text" --value bold="New text"\` | \`const text = useValue<string>('<id>')\`; server components: \`getValue\` from \`@splithub/next\`; no SDK: \`sh.value('<id>')\` |
|
|
400
|
+
| \`component\` | structure changes: a layout, a section, a flow, a different component per variant | \`--variants control,bold\` (the CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs) | \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`, or \`useVariant('<id>')\` in a switch |
|
|
401
|
+
| \`redirect\` | a whole page changes: another URL per variant | \`--match / --to control=/ --to v2=/home-v2\` | nothing: \`@splithub/next\` middleware or the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` sends visitors there |
|
|
402
|
+
| \`mvt\` | several things change at once (headline \xD7 button \xD7 image) and the user wants to know which of them matters; every combination is tested and each factor is read on its own | \`--factor headline=old:"Old text",new:"New text" --factor button=blue:#2563eb,green:#16a34a\` (no \`--variants\`; first level = control; at most 4 factors \xD7 4 levels, 16 combinations) | \`const { headline, button } = useValue<{ headline: string; button: string }>('<id>')\` |
|
|
403
|
+
|
|
404
|
+
mvt needs more visitors than an A/B test: every combination must reach the minimum on its own (the results show the
|
|
405
|
+
combinations and, below them, each factor summed over its combinations). With little traffic, test one thing with \`value\`.
|
|
406
|
+
|
|
388
407
|
## Create an experiment
|
|
389
408
|
|
|
390
|
-
1. \`npx splithub create --kind <value|component|redirect> --id <key>
|
|
391
|
-
- \`value\`: only data changes (text, image URL, number, JSON). Prefer it: no new components.
|
|
392
|
-
\`--value control="Old text" --value bold="New text"\`; read with \`useValue<string>('<id>')\`.
|
|
393
|
-
- \`component\`: structure changes. \`<Experiment id="<id>"><Variant name="control">\u2026</Variant><Variant name="bold">\u2026</Variant></Experiment>\`.
|
|
394
|
-
The CLI scaffolds \`splithub/<id>/<Variant>.tsx\` stubs.
|
|
395
|
-
- \`redirect\`: a whole page changes. \`--match / --to control=/ --to v2=/home-v2\`. Served by \`@splithub/next\` middleware,
|
|
396
|
-
or by the drop-in script \`<script src="https://go.splithub.io/sh.js" data-key="pk_\u2026">\` on sites without the SDK.
|
|
409
|
+
1. \`npx splithub create --kind <value|component|redirect|mvt> --id <key> \u2026 --goal <goal> --yes --json\` (see the table).
|
|
397
410
|
2. Goal: pick an existing key or add one to \`goals\`:
|
|
398
411
|
\`{ type: 'event' }\` + \`track('<goal>')\` where the conversion happens; \`{ type: 'pageview', path: '/thanks' }\`;
|
|
399
412
|
\`{ type: 'click', selector: '#buy' }\` (tracked automatically by the SDK and the script).
|
|
@@ -409,10 +422,20 @@ Targeting (country, device, UTM, cookie, time) is a \`flow\` graph on the experi
|
|
|
409
422
|
Report \`verdict.message\` verbatim and the numbers behind it (visitors per variant, lift, p-value, CI).
|
|
410
423
|
Never declare a winner from raw rates. \`srm_detected\` means the split is broken: fix the setup before anything else.
|
|
411
424
|
|
|
412
|
-
## Finish
|
|
425
|
+
## Finish, and what to do when a test is finished
|
|
426
|
+
|
|
427
|
+
\`npx splithub finish <id>\` stops the test (live within a minute in cloud mode). A finished test keeps its numbers for as long
|
|
428
|
+
as it stays in the config; the code still contains every variant until someone removes it.
|
|
429
|
+
|
|
430
|
+
Whenever you see a test with \`status: finished\` (in \`status\`, \`results\`, the MCP tools, or right after you finished it):
|
|
431
|
+
1. Read \`npx splithub results <id> --json\` and tell the user the verdict in one line with its numbers.
|
|
432
|
+
2. Ask two questions, do not assume: **which variant to keep** (recommend \`verdict.winner\`; with no significant
|
|
433
|
+
difference, the control or the simpler one) and **whether to remove the test from the code now**.
|
|
434
|
+
3. On yes, follow the \`splithub-ship\` skill: keep only the chosen variant's content, delete the other variants' code and
|
|
435
|
+
files, remove the entry from \`splithub.config.ts\`, \`push\` in cloud mode, run \`doctor\` and the project's checks.
|
|
436
|
+
4. On no, leave everything: a finished test costs nothing and the user may come back to it.
|
|
413
437
|
|
|
414
|
-
\`
|
|
415
|
-
components; keep the config entry for history (or remove it after the user agrees).
|
|
438
|
+
A \`paused\` test is not over: say so and ask whether to resume (\`start\`) or finish it.
|
|
416
439
|
|
|
417
440
|
## Do not
|
|
418
441
|
|
|
@@ -572,9 +595,9 @@ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory),
|
|
|
572
595
|
|---|---|---|
|
|
573
596
|
| \`splithub_status\` | all experiments with verdict headline | start here |
|
|
574
597
|
| \`splithub_results\` | full verdict for one experiment | report \`verdict.message\` verbatim |
|
|
575
|
-
| \`splithub_review\` | stored AI reviews (Pro): on
|
|
576
|
-
| \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind, id, variants, goal, values/to; starts as draft |
|
|
577
|
-
| \`splithub_set_status\` | start / pause / finish | applies live in cloud mode |
|
|
598
|
+
| \`splithub_review\` | stored AI reviews (Pro): at start, on verdict changes, on finish, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |
|
|
599
|
+
| \`splithub_create_experiment\` | adds an experiment to splithub.config.ts | kind value / component / redirect / mvt, id, variants (or factors for mvt), goal, values/to; starts as draft |
|
|
600
|
+
| \`splithub_set_status\` | start / pause / finish | applies live in cloud mode; after \`finished\`, ask which variant to keep and whether to clean the code (splithub-ship skill) |
|
|
578
601
|
| \`splithub_push\` / \`splithub_pull\` | sync the config with the cloud | pull adopts dashboard-created experiments |
|
|
579
602
|
| \`splithub_doctor\` | checks the installation | run after wiring code |
|
|
580
603
|
| \`splithub_login\` | returns a sign-in link; the human approves it in the browser | show the link, then call \`splithub_whoami\` |
|
|
@@ -586,6 +609,8 @@ Env: \`SPLITHUB_PROJECT_DIR\` (defaults to the MCP client's working directory),
|
|
|
586
609
|
- Results: never call a winner from raw rates; quote the verdict. \`srm_detected\` = fix the setup first.
|
|
587
610
|
- Running experiments are frozen: no weight / variant / seed edits; create a new one.
|
|
588
611
|
- If a tool answers that the folder is not connected, call \`splithub_login\`, show the link to the user, wait for them, then retry.
|
|
612
|
+
- A \`finished\` experiment in \`splithub_status\` or \`splithub_results\` means the test is over but its code is still there: report
|
|
613
|
+
the verdict, ask which variant to keep and whether to remove the test from the code, then follow the \`splithub-ship\` skill.
|
|
589
614
|
`;
|
|
590
615
|
var SKILLS = [
|
|
591
616
|
{ dir: "splithub", body: SKILL_SPLITHUB },
|
|
@@ -679,10 +704,13 @@ This project uses SplitHub (\`@splithub/react\`). Experiments are declared in
|
|
|
679
704
|
\`useValue(id)\`. Conversions: \`useTrack()\` then \`track('<goal>')\`.
|
|
680
705
|
|
|
681
706
|
- Not connected yet? \`npx splithub init --mode cloud\` (or \`npx splithub login\`) prints a link: show it to the user, they sign in and approve, the CLI gets the keys. Never ask the user to paste keys.
|
|
682
|
-
- New experiment: \`npx splithub create --kind value|component|redirect --id <key> --variants control,<other> --goal <goal> --yes\`
|
|
683
|
-
then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key
|
|
707
|
+
- New experiment: \`npx splithub create --kind value|component|redirect|mvt --id <key> --variants control,<other> --goal <goal> --yes\`
|
|
708
|
+
then wire it in code (value \u2192 \`useValue\`, component \u2192 \`<Experiment>\` with one \`<Variant>\` per key, redirect \u2192 nothing,
|
|
709
|
+
mvt \u2192 \`--factor name=level:value,\u2026\` and \`const { a, b } = useValue(id)\`).
|
|
684
710
|
- Results: \`npx splithub results <id> --json\`; report \`verdict.message\` verbatim, never call a winner from raw rates.
|
|
685
711
|
- Lifecycle: \`npx splithub start|pause|finish <id>\`. Check setup: \`npx splithub doctor\`.
|
|
712
|
+
- A \`finished\` test still has all its variants in the code: report the verdict, ask which variant to keep and whether to
|
|
713
|
+
remove the test from the code, then follow .claude/skills/splithub-ship/SKILL.md. Never delete test code unasked.
|
|
686
714
|
- Keep \`status: 'draft'\` until the user confirms. Never change \`seed\` of a running experiment.
|
|
687
715
|
- Config ships to the browser: no secrets in it. Full rules: .claude/skills/splithub/SKILL.md
|
|
688
716
|
`;
|
|
@@ -804,14 +832,15 @@ async function create(flags) {
|
|
|
804
832
|
if (interactive) p.intro(c.bold("splithub create"));
|
|
805
833
|
let kind = str(flags.kind);
|
|
806
834
|
if (!kind) {
|
|
807
|
-
if (!interactive) throw new Error("--kind value|component|redirect is required with --yes");
|
|
835
|
+
if (!interactive) throw new Error("--kind value|component|redirect|mvt is required with --yes");
|
|
808
836
|
kind = await ask(
|
|
809
837
|
p.select({
|
|
810
838
|
message: "What changes between variants?",
|
|
811
839
|
options: [
|
|
812
840
|
{ value: "value", label: "Text, image URL, number", hint: "no new components" },
|
|
813
841
|
{ value: "component", label: "Layout / structure", hint: "one component per variant" },
|
|
814
|
-
{ value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" }
|
|
842
|
+
{ value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" },
|
|
843
|
+
{ value: "mvt", label: "Several things at once", hint: "multivariate: every combination, each factor read on its own" }
|
|
815
844
|
]
|
|
816
845
|
})
|
|
817
846
|
);
|
|
@@ -829,7 +858,47 @@ async function create(flags) {
|
|
|
829
858
|
}
|
|
830
859
|
if (!KEY.test(id)) throw new Error(`invalid id "${id}" (lowercase a-z 0-9 - _, max 64)`);
|
|
831
860
|
if (hasExperiment(source, id)) throw new Error(`experiment "${id}" already exists`);
|
|
832
|
-
|
|
861
|
+
const factors = {};
|
|
862
|
+
if (kind === "mvt") {
|
|
863
|
+
for (const raw of list(flags.factor)) {
|
|
864
|
+
const eq = raw.indexOf("=");
|
|
865
|
+
if (eq <= 0) throw new Error(`--factor expects name=level:value,level:value, got "${raw}"`);
|
|
866
|
+
const name = raw.slice(0, eq).trim();
|
|
867
|
+
const levels = {};
|
|
868
|
+
for (const part of raw.slice(eq + 1).split(",")) {
|
|
869
|
+
const colon = part.indexOf(":");
|
|
870
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
871
|
+
if (!level) continue;
|
|
872
|
+
levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
873
|
+
}
|
|
874
|
+
factors[name] = levels;
|
|
875
|
+
}
|
|
876
|
+
while (interactive && Object.keys(factors).length === 0) {
|
|
877
|
+
const name = await ask(p.text({ message: "Factor name (what changes), e.g. headline", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
|
|
878
|
+
const levelsRaw = await ask(p.text({ message: `Levels of "${name}" as key:value, comma separated (first = control)`, placeholder: "old:Old headline, new:New headline" }));
|
|
879
|
+
const levels = {};
|
|
880
|
+
for (const part of levelsRaw.split(",")) {
|
|
881
|
+
const colon = part.indexOf(":");
|
|
882
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
883
|
+
if (level) levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
884
|
+
}
|
|
885
|
+
factors[name] = levels;
|
|
886
|
+
while (await ask(p.confirm({ message: "Add another factor?", initialValue: false }))) {
|
|
887
|
+
const more = await ask(p.text({ message: "Factor name", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
|
|
888
|
+
const moreRaw = await ask(p.text({ message: `Levels of "${more}" as key:value, comma separated`, placeholder: "blue:#2563eb, green:#16a34a" }));
|
|
889
|
+
const moreLevels = {};
|
|
890
|
+
for (const part of moreRaw.split(",")) {
|
|
891
|
+
const colon = part.indexOf(":");
|
|
892
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
893
|
+
if (level) moreLevels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
894
|
+
}
|
|
895
|
+
factors[more] = moreLevels;
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
if (Object.keys(factors).length === 0) throw new Error("--factor name=level:value,level:value is required for mvt experiments (repeat it per factor)");
|
|
899
|
+
}
|
|
900
|
+
const mvt = kind === "mvt" ? mvtVariants(id, factors) : null;
|
|
901
|
+
let variantKeys = mvt ? mvt.variants.map((v) => v.key) : str(flags.variants)?.split(",").map((s) => s.trim()).filter(Boolean);
|
|
833
902
|
if (!variantKeys || variantKeys.length < 2) {
|
|
834
903
|
if (!interactive) variantKeys = ["control", "b"];
|
|
835
904
|
else {
|
|
@@ -885,7 +954,8 @@ async function create(flags) {
|
|
|
885
954
|
goal,
|
|
886
955
|
match,
|
|
887
956
|
description,
|
|
888
|
-
variants: variantKeys.map((k2, i) => ({ key: k2, weight: weights[i], value: values[k2], to: tos[k2] })),
|
|
957
|
+
variants: mvt ? mvt.variants.map((v) => ({ key: v.key, weight: v.weight })) : variantKeys.map((k2, i) => ({ key: k2, weight: weights[i], value: values[k2], to: tos[k2] })),
|
|
958
|
+
...mvt ? { factors } : {},
|
|
889
959
|
// limits: --traffic 50 (percent), --max-visitors 10000, --start / --end ISO dates
|
|
890
960
|
traffic: typeof flags.traffic === "string" && flags.traffic !== "" ? Math.min(1, Math.max(0, Number(flags.traffic) / 100)) : void 0,
|
|
891
961
|
maxExposures: typeof flags["max-visitors"] === "string" && Number(flags["max-visitors"]) > 0 ? Math.floor(Number(flags["max-visitors"])) : void 0,
|
|
@@ -907,7 +977,7 @@ async function create(flags) {
|
|
|
907
977
|
}
|
|
908
978
|
}
|
|
909
979
|
}
|
|
910
|
-
const snippet = kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
|
|
980
|
+
const snippet = kind === "mvt" ? `const { ${Object.keys(factors).join(", ")} } = useValue<{ ${Object.keys(factors).map((f) => `${f}: string`).join("; ")} }>('${id}');` : kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
|
|
911
981
|
${variantKeys.map((k2) => ` <Variant name="${k2}"><${componentNames[k2]} /></Variant>`).join("\n")}
|
|
912
982
|
</Experiment>` : `// handled by @splithub/next middleware on ${match}; no code needed`;
|
|
913
983
|
const result = { experiment: id, kind, status: status2, variants: experiment.variants, goal: goal ?? null, configFile: relative(project.root, project.configFile), files, snippet };
|
|
@@ -1299,9 +1369,14 @@ export function pgStore(pool: Pool = new Pool({ connectionString: process.env.DA
|
|
|
1299
1369
|
},
|
|
1300
1370
|
async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
|
|
1301
1371
|
const r = await pool.query(
|
|
1302
|
-
\`WITH
|
|
1303
|
-
SELECT
|
|
1304
|
-
WHERE type = 'exposure' AND experiment = $1 GROUP BY
|
|
1372
|
+
\`WITH fv AS (
|
|
1373
|
+
SELECT visitor_id, MIN(ts) AS first_ts FROM splithub_events
|
|
1374
|
+
WHERE type = 'exposure' AND experiment = $1 GROUP BY visitor_id),
|
|
1375
|
+
first_exposure AS (
|
|
1376
|
+
SELECT MIN(e.variant) AS variant, fv.visitor_id, fv.first_ts
|
|
1377
|
+
FROM fv JOIN splithub_events e
|
|
1378
|
+
ON e.type = 'exposure' AND e.experiment = $1 AND e.visitor_id = fv.visitor_id AND e.ts = fv.first_ts
|
|
1379
|
+
GROUP BY fv.visitor_id, fv.first_ts),
|
|
1305
1380
|
converted AS (
|
|
1306
1381
|
SELECT fe.variant, fe.visitor_id, SUM(COALESCE((g.props->>'revenue')::numeric, 0)) AS revenue
|
|
1307
1382
|
FROM first_exposure fe JOIN splithub_events g
|
|
@@ -1366,20 +1441,21 @@ export function mysqlStore(pool: Pool = mysql.createPool(process.env.DATABASE_UR
|
|
|
1366
1441
|
},
|
|
1367
1442
|
async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
|
|
1368
1443
|
const [rows] = await pool.query(
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1444
|
+
// Derived tables only (no CTEs, no window functions) so this runs on MySQL 5.7.
|
|
1445
|
+
\`SELECT v.variant, COUNT(*) AS exposures, SUM(v.converted) AS conversions, COALESCE(SUM(v.revenue), 0) AS revenue
|
|
1446
|
+
FROM (
|
|
1447
|
+
SELECT fe.variant, fe.visitor_id,
|
|
1448
|
+
CASE WHEN COUNT(g.id) > 0 THEN 1 ELSE 0 END AS converted,
|
|
1449
|
+
SUM(COALESCE(JSON_EXTRACT(g.props, '$.revenue'), 0)) AS revenue
|
|
1450
|
+
FROM (
|
|
1451
|
+
SELECT MIN(e.variant) AS variant, fv.visitor_id, fv.first_ts
|
|
1452
|
+
FROM (SELECT visitor_id, MIN(ts) AS first_ts FROM splithub_events WHERE type = 'exposure' AND experiment = ? GROUP BY visitor_id) fv
|
|
1453
|
+
JOIN splithub_events e ON e.type = 'exposure' AND e.experiment = ? AND e.visitor_id = fv.visitor_id AND e.ts = fv.first_ts
|
|
1454
|
+
GROUP BY fv.visitor_id, fv.first_ts) fe
|
|
1455
|
+
LEFT JOIN splithub_events g ON g.type = 'goal' AND g.goal = ? AND g.visitor_id = fe.visitor_id AND g.ts >= fe.first_ts
|
|
1456
|
+
GROUP BY fe.variant, fe.visitor_id) v
|
|
1457
|
+
GROUP BY v.variant\`,
|
|
1458
|
+
[experiment, experiment, goal]
|
|
1383
1459
|
);
|
|
1384
1460
|
return (rows as Array<Record<string, unknown>>).map((x) => ({ variant: String(x.variant), exposures: Number(x.exposures), conversions: Number(x.conversions), revenue: Number(x.revenue) }));
|
|
1385
1461
|
},
|
|
@@ -1400,8 +1476,9 @@ import type { VariantCounts } from '@splithub/stats';
|
|
|
1400
1476
|
/**
|
|
1401
1477
|
* Your own store (Prisma, Drizzle, Supabase, Mongo, anything). Three methods.
|
|
1402
1478
|
* Attribution rule (keep it, so results match the dashboard):
|
|
1403
|
-
* a visitor
|
|
1404
|
-
*
|
|
1479
|
+
* a visitor counts once per experiment, in the variant of their earliest
|
|
1480
|
+
* exposure (same ts: smallest variant key), and converts when they sent the
|
|
1481
|
+
* goal at or after that exposure; count unique visitors, not events.
|
|
1405
1482
|
* Columns worth keeping: id (unique, dedupe on it), type, experiment, variant,
|
|
1406
1483
|
* goal, visitor_id, session_id, ts, url, props (json), received_at, host,
|
|
1407
1484
|
* country, device, browser.
|
|
@@ -1413,8 +1490,8 @@ export function customStore(): Store {
|
|
|
1413
1490
|
throw new Error('implement insertEvents ' + events.length);
|
|
1414
1491
|
},
|
|
1415
1492
|
async variantCounts(experiment: string, goal: string): Promise<VariantCounts[]> {
|
|
1416
|
-
// For each variant: exposures =
|
|
1417
|
-
// conversions = those visitors with a goal event (goal === goal) at ts >=
|
|
1493
|
+
// For each variant: exposures = visitors whose earliest exposure in this experiment was to it;
|
|
1494
|
+
// conversions = those visitors with a goal event (goal === goal) at ts >= that earliest exposure;
|
|
1418
1495
|
// revenue = sum of props.revenue over those goal events.
|
|
1419
1496
|
throw new Error('implement variantCounts ' + experiment + ' ' + goal);
|
|
1420
1497
|
},
|
|
@@ -1728,11 +1805,13 @@ async function lifecycle(command, id, flags) {
|
|
|
1728
1805
|
} catch (error) {
|
|
1729
1806
|
live = `could not apply live: ${error instanceof Error ? error.message : String(error)}`;
|
|
1730
1807
|
}
|
|
1731
|
-
|
|
1808
|
+
const next = status2 === "finished" ? `Read the verdict (npx splithub results ${id}), decide which variant to keep, then remove the test from the code and the config: the splithub-ship skill walks an agent through it.` : null;
|
|
1809
|
+
if (flags.json) console.log(JSON.stringify({ experiment: id, status: status2, configFile: file, live, ...next ? { next } : {} }));
|
|
1732
1810
|
else {
|
|
1733
1811
|
console.log(`${c.green("\u2714")} ${c.bold(id)} \u2192 ${status2} ${c.dim(`(${file})`)}`);
|
|
1734
1812
|
if (live === "applied") console.log(`${c.green("\u2714")} live in the cloud within 60s, no deploy needed (the config change ships with your next deploy)`);
|
|
1735
1813
|
else if (live !== "skipped") console.log(c.yellow(`! ${live}`));
|
|
1814
|
+
if (next) console.log(c.dim(`Next: ${next}`));
|
|
1736
1815
|
}
|
|
1737
1816
|
}
|
|
1738
1817
|
|
|
@@ -1742,6 +1821,18 @@ import { readFileSync as readFileSync7 } from "fs";
|
|
|
1742
1821
|
|
|
1743
1822
|
// src/commands/push.ts
|
|
1744
1823
|
import { relative as relative6 } from "path";
|
|
1824
|
+
function factorsOf(e) {
|
|
1825
|
+
const out = {};
|
|
1826
|
+
for (const f of e.factors ?? []) {
|
|
1827
|
+
out[f.key] = {};
|
|
1828
|
+
for (const level of f.levels) {
|
|
1829
|
+
const combo = e.variants.find((v) => v.levels?.[f.key] === level);
|
|
1830
|
+
const value = combo?.value?.[f.key];
|
|
1831
|
+
out[f.key][level] = typeof value === "string" ? value : value === void 0 ? level : JSON.stringify(value);
|
|
1832
|
+
}
|
|
1833
|
+
}
|
|
1834
|
+
return out;
|
|
1835
|
+
}
|
|
1745
1836
|
function adoptCloudExperiments(file, cloudOnly) {
|
|
1746
1837
|
if (!cloudOnly || cloudOnly.length === 0) return [];
|
|
1747
1838
|
let source = readSource(file);
|
|
@@ -1760,7 +1851,8 @@ function adoptCloudExperiments(file, cloudOnly) {
|
|
|
1760
1851
|
traffic: e.traffic,
|
|
1761
1852
|
maxExposures: e.maxExposures ?? void 0,
|
|
1762
1853
|
schedule: e.schedule ?? void 0,
|
|
1763
|
-
variants: e.variants.map((v) => ({ key: v.key, weight: Math.round(v.weight / total * 100), value: typeof v.value === "string" ? v.value : v.value === void 0 ? void 0 : JSON.stringify(v.value), to: v.to }))
|
|
1854
|
+
variants: e.variants.map((v) => ({ key: v.key, weight: Math.round(v.weight / total * 100), value: typeof v.value === "string" ? v.value : v.value === void 0 ? void 0 : JSON.stringify(v.value), to: v.to })),
|
|
1855
|
+
...e.kind === "mvt" && e.factors ? { factors: factorsOf(e) } : {}
|
|
1764
1856
|
});
|
|
1765
1857
|
written.push(e.key);
|
|
1766
1858
|
}
|
|
@@ -1830,7 +1922,7 @@ async function results(key, flags) {
|
|
|
1830
1922
|
const target = resolveTarget(flags, detectProject());
|
|
1831
1923
|
const r = await fetchResults(target, key);
|
|
1832
1924
|
if (flags.json) {
|
|
1833
|
-
console.log(JSON.stringify(r, null, 2));
|
|
1925
|
+
console.log(JSON.stringify(r.status === "finished" ? { ...r, next: FINISHED_NEXT } : r, null, 2));
|
|
1834
1926
|
return;
|
|
1835
1927
|
}
|
|
1836
1928
|
const v = r.verdict;
|
|
@@ -1886,7 +1978,24 @@ async function results(key, flags) {
|
|
|
1886
1978
|
if (v.status === "insufficient_sample" && v.sample.neededPerVariant) {
|
|
1887
1979
|
console.log(c.dim(`Estimated ${v.sample.neededPerVariant} visitors per variant needed to confirm the current lift.`));
|
|
1888
1980
|
}
|
|
1981
|
+
if (r.factors?.length) {
|
|
1982
|
+
console.log();
|
|
1983
|
+
console.log(c.bold("Each factor on its own") + c.dim(" (a level sums every combination that uses it, compared with the first level)"));
|
|
1984
|
+
for (const f of r.factors) {
|
|
1985
|
+
console.log(
|
|
1986
|
+
table(
|
|
1987
|
+
[f.key.toUpperCase(), "VISITORS", "CONV.", "RATE", "LIFT", "P-VALUE"],
|
|
1988
|
+
f.levels.map((l) => [l.level + (f.best === l.level ? c.green(" \u2714") : ""), String(l.exposures), String(l.conversions), pct(l.rate), l.relativeLift === null ? "\u2013" : pct(l.relativeLift, true), l.pValue === null ? "\u2013" : l.pValue.toFixed(3)])
|
|
1989
|
+
)
|
|
1990
|
+
);
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
if (r.status === "finished") {
|
|
1994
|
+
console.log();
|
|
1995
|
+
console.log(c.dim(FINISHED_NEXT));
|
|
1996
|
+
}
|
|
1889
1997
|
}
|
|
1998
|
+
var FINISHED_NEXT = "This test is finished but its variants are still in the code. Decide which variant to keep, then remove the test from the code and the config (agents: the splithub-ship skill).";
|
|
1890
1999
|
function label(status2) {
|
|
1891
2000
|
const icon = verdictIcon(status2);
|
|
1892
2001
|
switch (status2) {
|
|
@@ -1907,8 +2016,10 @@ function label(status2) {
|
|
|
1907
2016
|
async function status(flags) {
|
|
1908
2017
|
const target = resolveTarget(flags, detectProject());
|
|
1909
2018
|
const data = await fetchStatus(target);
|
|
2019
|
+
const finished = data.experiments.filter((e) => e.status === "finished").map((e) => e.experiment);
|
|
2020
|
+
const next = finished.length ? `Finished but still in the code: ${finished.join(", ")}. Decide which variant to keep, then remove the test from the code and the config (agents: the splithub-ship skill).` : null;
|
|
1910
2021
|
if (flags.json) {
|
|
1911
|
-
console.log(JSON.stringify(data, null, 2));
|
|
2022
|
+
console.log(JSON.stringify(next ? { ...data, next } : data, null, 2));
|
|
1912
2023
|
return;
|
|
1913
2024
|
}
|
|
1914
2025
|
console.log(`${c.bold("SplitHub")} ${c.dim("\xB7")} project ${c.cyan(data.project)} ${c.dim(`\xB7 ${target.url}`)}
|
|
@@ -1932,6 +2043,8 @@ async function status(flags) {
|
|
|
1932
2043
|
])
|
|
1933
2044
|
)
|
|
1934
2045
|
);
|
|
2046
|
+
if (next) console.log(`
|
|
2047
|
+
${c.dim(next)}`);
|
|
1935
2048
|
}
|
|
1936
2049
|
|
|
1937
2050
|
// src/commands/menu.ts
|
|
@@ -2054,7 +2167,7 @@ async function review(key, flags) {
|
|
|
2054
2167
|
return;
|
|
2055
2168
|
}
|
|
2056
2169
|
if (reviews.length === 0) {
|
|
2057
|
-
console.log(c.dim(key ? `no reviews for ${key} yet (Pro writes one
|
|
2170
|
+
console.log(c.dim(key ? `no reviews for ${key} yet (Pro writes one at the start, when the verdict changes while running, and on finish; or click "Review now" in the dashboard)` : "no reviews yet"));
|
|
2058
2171
|
return;
|
|
2059
2172
|
}
|
|
2060
2173
|
for (const r of reviews) {
|
|
@@ -2075,7 +2188,7 @@ Usage
|
|
|
2075
2188
|
splithub start|pause|finish <id>
|
|
2076
2189
|
splithub status all experiments with a verdict headline
|
|
2077
2190
|
splithub results <id> full verdict for one experiment
|
|
2078
|
-
splithub review [<id>] AI reviews stored in the cloud (Pro: on
|
|
2191
|
+
splithub review [<id>] AI reviews stored in the cloud (Pro: at start, on verdict changes, on finish, or on demand)
|
|
2079
2192
|
splithub dev [--url] [--hold] open the app with ?sh_debug=1 (console + panel: variants, reasons, events) and watch results
|
|
2080
2193
|
splithub doctor check the installation
|
|
2081
2194
|
splithub skills [--mcp] install / refresh agent skills (library, A/B testing, API, MCP, self-hosting, ship the winner); --mcp registers the MCP server
|
|
@@ -2128,6 +2241,7 @@ async function main() {
|
|
|
2128
2241
|
value: { type: "string", multiple: true },
|
|
2129
2242
|
match: { type: "string" },
|
|
2130
2243
|
to: { type: "string", multiple: true },
|
|
2244
|
+
factor: { type: "string", multiple: true },
|
|
2131
2245
|
status: { type: "string" },
|
|
2132
2246
|
description: { type: "string" },
|
|
2133
2247
|
traffic: { type: "string" },
|