splithub 0.1.9 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/chunk-3RVNT2HA.js +54 -0
- package/dist/chunk-3RVNT2HA.js.map +1 -0
- package/dist/chunk-5C24F2ZN.js +113 -0
- package/dist/chunk-5C24F2ZN.js.map +1 -0
- package/dist/chunk-B6FITBWN.js +384 -0
- package/dist/chunk-B6FITBWN.js.map +1 -0
- package/dist/chunk-BJQJGLOB.js +160 -0
- package/dist/chunk-BJQJGLOB.js.map +1 -0
- package/dist/chunk-DDWYTOX4.js +565 -0
- package/dist/chunk-DDWYTOX4.js.map +1 -0
- package/dist/chunk-DUIJ5DTV.js +170 -0
- package/dist/chunk-DUIJ5DTV.js.map +1 -0
- package/dist/chunk-E5VH4BQX.js +326 -0
- package/dist/chunk-E5VH4BQX.js.map +1 -0
- package/dist/chunk-EYYLDI5J.js +195 -0
- package/dist/chunk-EYYLDI5J.js.map +1 -0
- package/dist/chunk-GVARYUL3.js +28 -0
- package/dist/chunk-GVARYUL3.js.map +1 -0
- package/dist/chunk-KMO5GGGC.js +185 -0
- package/dist/chunk-KMO5GGGC.js.map +1 -0
- package/dist/chunk-LT4JP3IF.js +119 -0
- package/dist/chunk-LT4JP3IF.js.map +1 -0
- package/dist/chunk-NMN5YGNP.js +72 -0
- package/dist/chunk-NMN5YGNP.js.map +1 -0
- package/dist/chunk-OHA2M7QR.js +264 -0
- package/dist/chunk-OHA2M7QR.js.map +1 -0
- package/dist/chunk-OYM64FMD.js +63 -0
- package/dist/chunk-OYM64FMD.js.map +1 -0
- package/dist/chunk-PLXDG4B5.js +134 -0
- package/dist/chunk-PLXDG4B5.js.map +1 -0
- package/dist/chunk-PNPOOAHF.js +393 -0
- package/dist/chunk-PNPOOAHF.js.map +1 -0
- package/dist/chunk-RYDA7PQF.js +97 -0
- package/dist/chunk-RYDA7PQF.js.map +1 -0
- package/dist/chunk-V3DLUVOE.js +112 -0
- package/dist/chunk-V3DLUVOE.js.map +1 -0
- package/dist/ci-ZQZ6RBUJ.js +16 -0
- package/dist/ci-ZQZ6RBUJ.js.map +1 -0
- package/dist/create-AEJVXN25.js +18 -0
- package/dist/create-AEJVXN25.js.map +1 -0
- package/dist/dev-W2JGUV2A.js +11 -0
- package/dist/dev-W2JGUV2A.js.map +1 -0
- package/dist/doctor-D3KOGOX6.js +14 -0
- package/dist/doctor-D3KOGOX6.js.map +1 -0
- package/dist/holdout-BWVTTLEY.js +44 -0
- package/dist/holdout-BWVTTLEY.js.map +1 -0
- package/dist/index.js +94 -2240
- package/dist/index.js.map +1 -1
- package/dist/init-UBUDSJBD.js +15 -0
- package/dist/init-UBUDSJBD.js.map +1 -0
- package/dist/lifecycle-ZLZUTASA.js +12 -0
- package/dist/lifecycle-ZLZUTASA.js.map +1 -0
- package/dist/login-TIIX5V4O.js +18 -0
- package/dist/login-TIIX5V4O.js.map +1 -0
- package/dist/menu-S3WDS4JY.js +163 -0
- package/dist/menu-S3WDS4JY.js.map +1 -0
- package/dist/plan-G2A55FR7.js +135 -0
- package/dist/plan-G2A55FR7.js.map +1 -0
- package/dist/push-OJX4F2BH.js +16 -0
- package/dist/push-OJX4F2BH.js.map +1 -0
- package/dist/refs-RY7H3J7L.js +44 -0
- package/dist/refs-RY7H3J7L.js.map +1 -0
- package/dist/remove-EQQCDJRU.js +54 -0
- package/dist/remove-EQQCDJRU.js.map +1 -0
- package/dist/report-6WNO576N.js +69 -0
- package/dist/report-6WNO576N.js.map +1 -0
- package/dist/results-ZDOCWN6U.js +21 -0
- package/dist/results-ZDOCWN6U.js.map +1 -0
- package/dist/review-7E7JVMMV.js +34 -0
- package/dist/review-7E7JVMMV.js.map +1 -0
- package/dist/ship-MRBHWFZR.js +450 -0
- package/dist/ship-MRBHWFZR.js.map +1 -0
- package/dist/skills-JJOKGNO5.js +13 -0
- package/dist/skills-JJOKGNO5.js.map +1 -0
- package/dist/status-SATP23WK.js +12 -0
- package/dist/status-SATP23WK.js.map +1 -0
- package/dist/update-67QDJ6GF.js +37 -0
- package/dist/update-67QDJ6GF.js.map +1 -0
- package/package.json +22 -5
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/skills-self-hosted.ts","../src/skills-ship.ts","../src/skills.ts","../src/templates.ts"],"sourcesContent":["// Skill: SplitHub with the user's own database (self mode) and reporting from the cloud.\nexport const SKILL_SELF_HOSTED = `---\nname: splithub-self-hosted\ndescription: Keep SplitHub events in your own database (self mode): which tables, how to write a Store for Postgres, MySQL, Prisma, Drizzle, Supabase or anything else, where the ingest route lives, how the CLI reads results from it, and how to get cloud data into your own database for reports. Use when the user wants data in their infra or asks about tables, schema or reporting.\n---\n\n# SplitHub with your own database\n\nTwo ways to own the data:\n\n1. **Self mode** — the app hosts the ingest route and the store; nothing leaves your infra.\n \\`splithub.config.ts\\`: \\`storage: { mode: 'self', endpoint: '/api/splithub' }\\`.\n2. **Cloud + export** — events go to SplitHub Cloud; you pull summaries into your warehouse for reports.\n\n## Self mode: the pieces\n\n- Route: \\`app/api/splithub/[[...path]]/route.ts\\` (Next.js) or any fetch-style handler from\n \\`createSplitHubHandler({ config, store, adminToken })\\` in \\`@splithub/node\\`. It accepts\n \\`POST /events\\` from the SDK and answers \\`GET /status\\`, \\`GET /results/<id>\\` for the CLI\n (\\`npx splithub status\\` talks to the local dev server, or \\`--url https://your.app/api/splithub\\`).\n- Store: an object with three methods (below). Ready-made: \\`@splithub/adapter-sqlite\\`\n (Node 22.13+, \\`node:sqlite\\`, zero setup) and \\`@splithub/adapter-d1\\` (Cloudflare D1).\n \\`npx splithub init --mode self --db postgres|mysql|custom\\` writes \\`splithub/store.ts\\` with a\n working Postgres (pg) or MySQL (mysql2) store, or a documented stub for anything else.\n- Admin token: set \\`SPLITHUB_ADMIN_TOKEN\\` in the app and \\`--token\\` / \\`SPLITHUB_ADMIN_TOKEN\\` for\n the CLI, so results are not public.\n\n## The events table\n\nOne table, one row per event. Column names are yours; the templates use these:\n\n| column | type | meaning |\n|---|---|---|\n| id | text, primary key | client-generated, unique; dedupe on it (retries resend batches) |\n| type | text | \\`exposure\\` or \\`goal\\` |\n| experiment, variant | text | for exposures |\n| goal | text | for goals |\n| visitor_id, session_id | text | \\`sh_vid\\` cookie / session |\n| ts | bigint | client time, unix ms |\n| url | text | page |\n| props | json | goal payload, e.g. \\`{ \"revenue\": 49 }\\` |\n| received_at | bigint | server time, unix ms |\n| host, country, device, browser | text | request metadata (fill from headers if you want breakdowns) |\n\nIndexes: \\`(type, experiment, visitor_id, ts)\\` and \\`(type, goal, visitor_id, ts)\\`.\n\n## The Store interface (\\`@splithub/node\\`)\n\n\\`\\`\\`ts\ninterface Store {\n insertEvents(events: StoredEvent[]): Promise<number>; // skip existing ids, return inserted count\n variantCounts(experiment: string, goal: string): Promise<VariantCounts[]>; // per variant: exposures, conversions, revenue\n summaries(): Promise<ExperimentSummary[]>; // per experiment: exposures, visitors, first/last ts\n close?(): Promise<void> | void;\n}\n\\`\\`\\`\n\nAttribution rule — keep it exactly, so self-hosted results equal the cloud's:\n**a visitor converts for a variant when they sent the goal at or after their first exposure to\nthat variant; exposures and conversions count unique visitors.** The SQL in the generated\n\\`splithub/store.ts\\` implements it (CTE \\`first_exposure\\`, then goals joined with \\`ts >= first_ts\\`).\n\nPrisma / Drizzle: map the table in your schema and write the three methods with the ORM, or run the\none raw query for \\`variantCounts\\`. Supabase: the same SQL as a Postgres function called through\n\\`rpc\\`. Mongo: \\`$group\\` by variant and visitor with \\`$min ts\\`, then \\`$lookup\\` goals.\n\n## Reports from the cloud into your database\n\n- CSV: dashboard → experiment → Export (summary, daily timeline, raw events up to 50k rows).\n- Webhook channel for alerts and the digest: POST JSON to your endpoint when something happens.\n- Scheduled pull: a cron in your app calls \\`GET /v1/status\\` and \\`GET /v1/results/<id>\\` with the\n API key (\\`sk_…\\`) and stores the verdicts. Raw events live 30–90 days in the cloud (plan dependent),\n aggregates a year: pull summaries, not raw rows, for long-term reporting.\n- Need every raw event forever in your infra? Use self mode; one store per project.\n\n## Do not\n\n- Put database credentials in \\`splithub.config.ts\\` (it ships to the browser). They belong in the route's env.\n- Compute winners from raw rates in SQL; use \\`npx splithub results\\` (or \\`@splithub/stats\\`) for the verdict.\n`;\n","// Skills: ship the winner (`splithub ship`) and the full agent loop. Installed next\n// to the other skills by `splithub init` / `splithub skills`.\n\nexport const SKILL_SHIP = `---\nname: splithub-ship\ndescription: Ship the winner of an A/B test and remove the test from the code: keep only the winning value, component or page, delete the losing variants, finish the experiment and clean splithub.config.ts. Use when a test has a verdict and the user says ship it, roll out the winner, remove the test, or clean up; and whenever a test shows status finished and the user agreed on which variant to keep.\n---\n\n# Shipping the winner\n\nGoal: the app behaves exactly as the kept variant did, with no experiment left in the code or in the config.\n\n## 1. Decide\n\n\\`npx splithub results <id> --json\\` (MCP: \\`splithub_results\\`). Act on \\`verdict.status\\`:\n- \\`winner_found\\`: keep \\`verdict.winner\\`.\n- \\`no_significant_difference\\`: keep the control, or the simpler variant if the user agrees.\n- \\`insufficient_sample\\`: do not ship. Say when to look again: \\`daysToVerdict\\` days at the current traffic.\n- \\`srm_detected\\`: do not ship. The split is broken; fix the setup first.\n- \\`guardrail_breached\\`: do not ship the variant in \\`verdict.guardrailBreached\\`; it made a guardrail worse. Say what broke.\nAsk the user which variant to keep before going on. Never pick from raw rates.\n\n## 2. Preview, then ship\n\n\\`\\`\\`bash\nnpx splithub ship <id> --keep <variant> --dry-run --json # diff + leftovers, nothing written\nnpx splithub ship <id> --keep <variant> --json # apply\n\\`\\`\\`\nMCP: \\`splithub_ship\\` with \\`dryRun: true\\`, then without. It archives the test in the cloud (numbers stay readable), removes it from\n\\`splithub.config.ts\\` (checked by reloading the file) and rewrites the code:\n- \\`<Experiment id>\\` → the kept \\`<Variant>\\`'s children; the other variants are dropped.\n- \\`useValue('<id>')\\` / \\`sh.value('<id>')\\` → the kept value as a literal.\n- \\`useVariant('<id>')\\` comparisons, ternaries, \\`&&\\`, \\`if\\` and \\`switch\\` → the kept branch.\n- Unused \\`Experiment\\`, \\`Variant\\`, \\`useVariant\\`, \\`useValue\\` imports go.\n\n## 3. Finish by hand\n\n- \\`leftovers[]\\`: each \\`{ file, line, text, reason }\\` still mentions the experiment; rewrite those yourself.\n- component kind: delete components and files only the losing variants used (e.g. \\`splithub/<id>/\\`).\n- redirect kind: make the kept URL the real page (move the route or add a permanent redirect), delete the other page.\n- Goal: keep \\`track('<goal>')\\` when another test measures it (\\`next\\` says so); otherwise ask before removing it.\n- Cloud mode: \\`npx splithub push\\`. Then \\`npx splithub doctor\\` and the project's typecheck and tests.\n- \\`npx splithub refs <id> --json\\` must return \\`count: 0\\` for the experiment.\n\nReport what changed (files, the kept content) and the verdict in one line with its numbers. Never reuse the id.\n`;\n\nexport const SKILL_LOOP = `---\nname: splithub-loop\ndescription: The whole A/B test loop for an agent with SplitHub, step by step with the MCP tools and CLI --json fields: plan, create, implement, check, push, read results, finish, ship. Use when the user asks to run an A/B test end to end, or to pick up a test where it stands.\n---\n\n# The full loop\n\nEach step: MCP tool, or the CLI with \\`--json\\` (one line of JSON; errors are \\`{ error, code, hint }\\`, exit 1).\n\n| step | MCP tool | CLI | read |\n|---|---|---|---|\n| 0. size it | \\`splithub_plan\\` | \\`plan --baseline 0.05 --mde 0.1 [<id>]\\` | \\`perVariant\\`, \\`total\\`, \\`traffic.daysLeft\\` |\n| 1. create | \\`splithub_create_experiment\\` | \\`create --kind … --id … --goal … --yes\\` | \\`snippet\\`, \\`files\\`, \\`nextStep\\`, \\`handoff\\` |\n| 2. implement | (edit code) | | wire \\`snippet\\`; \\`track('<goal>')\\` where it converts |\n| 3. check | \\`splithub_doctor\\`, \\`splithub_refs\\` | \\`doctor\\`, \\`refs <id>\\` | \\`ok\\`, \\`checks[]\\`; \\`refs[]\\` shows the wiring |\n| 4. push (cloud) | \\`splithub_push\\` | \\`push\\` (\\`--dry-run\\` to preview) | \\`adopted\\`, \\`diff\\` |\n| 5. results | \\`splithub_status\\`, \\`splithub_results\\` | \\`status\\`, \\`results <id>\\` | \\`verdict.status\\`, \\`verdict.message\\`, \\`verdict.notes[]\\`, \\`daysToVerdict\\` |\n| 6. finish | \\`splithub_set_status\\` | \\`finish <id>\\` | \\`live\\`, \\`next\\` |\n| 7. ship | \\`splithub_ship\\` | \\`ship <id> --keep <v> [--dry-run]\\` | \\`diff\\`, \\`leftovers[]\\`, \\`next\\` |\n\nRules that keep the answer honest:\n- Plan before creating: if \\`traffic.daysLeft\\` is months, test a bigger change or a busier page.\n- \\`create\\` writes a running test; it goes live when the code is deployed (and pushed in cloud mode). Tell the user.\n- Verdicts are sequential (always-valid): checking any day is safe. Quote \\`verdict.message\\` and the always-valid\n \\`comparisons[].sequential.pValue\\`, not the fixed-horizon \\`pValue\\` (p = 0.01 there can still be not significant).\n Relay \\`verdict.notes[]\\`. Verdicts depend on sample size, not days. Say \\`verdict.sample.quality\\` (small | ok | good;\n good = every variant has its share of \\`goodSample\\`, default 5000 visitors, \\`create --good-sample\\`) next to the verdict.\n- \\`daysToVerdict\\` is the re-check date: schedule (or tell the user) a look in that many days instead of polling.\n- \\`srm_detected\\`: the split is broken; fix the setup, do not read the numbers.\n- \\`guardrail_breached\\`: don't ship. A variant made a guardrail worse (\\`verdict.guardrailBreached[]\\`), e.g. \"bold wins on\n signups but raises refunds\". \\`verdict.metrics[]\\` has the secondary metrics and guardrails, read like the main table.\n- Guard what matters when creating: \\`--secondary pricing_view --guardrail refunds:higher\\` (MCP: \\`secondary\\`, \\`guardrails\\`).\n Tests that could clash on one page go in one layer: \\`--layer checkout:0-50\\` and \\`checkout:50-100\\` never share a visitor.\n- \\`splithub holdout set 5\\` keeps 5% of visitors out of every test; the \\`__holdout\\` row (\"Holdout: long-term impact\")\n compares them with everyone else.\n- Ask the user which variant to keep before \\`ship\\`; preview with \\`dryRun\\` and show them the diff.\n- Error \\`code\\` → action: \\`NOT_LOGGED_IN\\` → \\`splithub_login\\`, show the link, wait, retry · \\`INSUFFICIENT_SCOPE\\` → the key\n can only read; ask for a key with write scope · \\`NO_CONFIG\\` → \\`splithub_init\\` ·\n \\`EXPERIMENT_NOT_FOUND\\` → \\`splithub_status\\` for ids · \\`CONFIG_UNSAFE\\` → edit the config by hand where the experiment is\n defined · \\`CONFIG_WRITE\\` → the file was restored; edit by hand, then \\`doctor\\`.\n`;\n","// Agent skills installed by `splithub init` / `splithub skills`. One folder per\n// topic under .claude/skills/, so an agent loads only what the task needs:\n// splithub — the library, the config file, the CLI, where things live\n// ab-testing — what a good test is, metrics, sample size, when to stop\n// splithub-api — the HTTP API (keys, endpoints, payloads) for scripts and servers\n// splithub-mcp — the MCP server: tools, how to register it in Claude Code / Cursor\n// splithub-ship — keep a variant and take the test out (`splithub ship`)\n// splithub-loop — the whole loop for an agent, tool by tool\n// Same text goes to .cursor/rules and the AGENTS.md section points here.\n\nexport interface Skill {\n dir: string;\n body: string;\n}\n\nexport const SKILL_SPLITHUB = `---\nname: splithub\ndescription: Add, change, run or read A/B experiments in an app that uses SplitHub (@splithub/react, splithub.config.ts, the splithub CLI, SplitHub Cloud). Use when the user asks to A/B test a component, headline, image, price, CTA or page, asks which variant is winning, or wants to start, pause or finish a test.\n---\n\n# SplitHub for coding agents\n\nThree pieces: \\`splithub.config.ts\\` (the experiments; source of truth when it exists),\n\\`@splithub/react\\` (renders variants), the \\`splithub\\` CLI (creates, pushes, reads results).\nEvents go to the app's own database (self mode) or to SplitHub Cloud (cloud mode).\n\n## Where things live\n\n- Config: \\`splithub.config.ts\\` in the project root. Experiments and goals. Ships to the browser: never put secrets in it.\n- Credentials: \\`npx splithub login\\` (or \\`init --mode cloud\\`) prints a link; the human approves it in the browser; the CLI\n stores a project API key for this folder. Never ask the user to paste keys. CI uses \\`SPLITHUB_API_KEY\\`.\n- The public key (\\`pk_\\`) ends up in \\`storage.key\\` of the config from any of: \\`login\\` (saved credentials), the first\n \\`push\\`, \\`init --key\\`, or \\`SPLITHUB_API_KEY\\` in the environment (init asks the cloud). Missing key = events go nowhere.\n- Cloud: \\`npx splithub push\\` uploads the config; \\`npx splithub pull\\` writes dashboard-created experiments into the file.\n- Results: \\`npx splithub status --json\\`, \\`npx splithub results <id> --json\\`. Sizing: \\`npx splithub plan --baseline 0.05 --mde 0.1\\`.\n- Code that uses a test: \\`npx splithub refs <id> --json\\`. Keep a variant and remove the test: \\`npx splithub ship <id> --keep <v>\\`.\n- \\`--json\\` prints one line; errors are \\`{ error, code, hint }\\` with exit code 1. The \\`splithub-loop\\` skill has the whole flow.\n- Everything interactive: \\`npx splithub\\` (menu). Everything scriptable: subcommands + \\`--json\\`.\n\n## Pick the kind\n\n| kind | use it when | create | read in code |\n|---|---|---|---|\n| \\`value\\` | only data changes: text, number, image URL, price, JSON. The default choice: no new components. | \\`--variants control,bold --value control=\"Old text\" --value bold=\"New text\"\\` | \\`const text = useValue<string>('<id>')\\`; server components: \\`getValue\\` from \\`@splithub/next\\`; no SDK: \\`sh.value('<id>')\\` |\n| \\`component\\` | structure changes: a layout, a section, a flow, a different component per variant | \\`--variants control,bold\\` (the CLI scaffolds \\`splithub/<id>/<Variant>.tsx\\` stubs) | \\`<Experiment id=\"<id>\"><Variant name=\"control\">…</Variant><Variant name=\"bold\">…</Variant></Experiment>\\`, or \\`useVariant('<id>')\\` in a switch |\n| \\`redirect\\` | a whole page changes: another URL per variant | \\`--match / --to control=/ --to v2=/home-v2\\` | nothing: \\`@splithub/next\\` middleware or the drop-in script \\`<script src=\"https://go.splithub.io/sh.js\" data-key=\"pk_…\">\\` sends visitors there |\n| \\`mvt\\` | several things change at once (headline × button × image) and the user wants to know which of them matters; every combination is tested and each factor is read on its own | \\`--factor headline=old:\"Old text\",new:\"New text\" --factor button=blue:#2563eb,green:#16a34a\\` (no \\`--variants\\`; first level = control; at most 4 factors × 4 levels, 16 combinations) | \\`const { headline, button } = useValue<{ headline: string; button: string }>('<id>')\\` |\n\nmvt needs more visitors than an A/B test: every combination must reach the minimum on its own (the results show the\ncombinations and, below them, each factor summed over its combinations). With little traffic, test one thing with \\`value\\`.\n\n## Create an experiment\n\n1. \\`npx splithub create --kind <value|component|redirect|mvt> --id <key> … --goal <goal> --yes --json\\` (see the table).\n2. Goal: pick an existing key or add one to \\`goals\\`:\n \\`{ type: 'event' }\\` + \\`track('<goal>')\\` where the conversion happens; \\`{ type: 'pageview', path: '/thanks' }\\`;\n \\`{ type: 'click', selector: '#buy' }\\` (tracked automatically by the SDK and the script).\n3. Wire it in code. \\`create\\` writes \\`status: 'running'\\`: the test goes live for every visitor, first page view\n included, as soon as this code is deployed. Use \\`--status draft\\` only when the user asks to hold it.\n4. Cloud mode: \\`npx splithub push\\` after any config change.\n5. Hand-off, always: end your reply by saying the test is **running in the config** and goes live when the user deploys\n this code; nothing is recorded before that, and local development never sends events to the cloud. Tell them how to\n check it locally (\\`?sh_force=<id>:<variant>\\`) and that Pause in the dashboard or \\`npx splithub pause <id>\\` holds it.\n In cloud mode a status set in the dashboard wins over the file for every visitor, first page included.\n For a draft (only if asked): say visitors do not see it and nothing is recorded until \\`npx splithub start <id>\\`.\n\nA draft is invisible to visitors. To check one before starting: \\`runDraftsInDev: true\\` at the top level of the config\nserves drafts like running tests in development only (\\`NODE_ENV=development\\`: \\`next dev\\`, \\`vite\\`);\n\\`?sh_force=<id>:<variant>\\` shows one variant on any page, draft or not, without recording an exposure;\n\\`?sh_debug=1\\` (or \\`npx splithub dev\\`) logs every decision and opens a panel to switch variants.\nIn development the SDK explains drafts, paused tests and held-back events in one line each. Vite apps: add \\`splithub()\\`\nfrom \\`@splithub/core/vite\\` to the plugins in vite.config so these lines and a summary of every experiment appear in the\nterminal that runs \\`vite\\`; Next.js server code prints them in its terminal anyway.\n\nKeys: lowercase a-z 0-9 - _, max 64 chars. First variant = control. Weights are relative (50/50, 1/1, 90/10).\nTargeting (country, device, UTM, cookie, time) is a \\`flow\\` graph on the experiment, edited in the dashboard; \\`pull\\` brings it into the file.\n\n## Read results\n\n\\`verdict.status\\` ∈ no_data | insufficient_sample | srm_detected | winner_found | no_significant_difference.\nReport \\`verdict.message\\` verbatim and the numbers behind it (visitors per variant, lift, the always-valid\n\\`comparisons[].sequential.pValue\\` and \\`sequential.ci\\`; the plain \\`pValue\\` is fixed-horizon, for display only) and relay\n\\`verdict.notes[]\\`. Verdicts are sequential and depend on sample size, not days: safe to check any time. \\`verdict.sample.quality\\` is\nsmall (below 100 visitors per variant or 50 conversions), ok, or good (every variant has its share of \\`goodSample\\`, default 5000\nvisitors; \\`create --good-sample\\`); say it next to the verdict. \\`verdict.sample.stars\\` (0..5) scores data quality from\n\\`verdict.sample.checks[]\\`: report the stars and the unmet labels; they never override the verdict. \\`daysToVerdict\\` says when to look again.\nNever declare a winner from raw rates. \\`srm_detected\\` means the split is broken: fix the setup before anything else.\n\n## Finish, and what to do when a test is finished\n\n\\`npx splithub finish <id>\\` stops the test (live within a minute in cloud mode). A finished test keeps its numbers for as long\nas it stays in the config; the code still contains every variant until someone removes it.\n\nWhenever you see a test with \\`status: finished\\` (in \\`status\\`, \\`results\\`, the MCP tools, or right after you finished it):\n1. Read \\`npx splithub results <id> --json\\` and tell the user the verdict in one line with its numbers.\n2. Ask two questions, do not assume: **which variant to keep** (recommend \\`verdict.winner\\`; with no significant\n difference, the control or the simpler one) and **whether to remove the test from the code now**.\n3. On yes, follow the \\`splithub-ship\\` skill: \\`npx splithub ship <id> --keep <variant> --dry-run\\`, show the diff, then run it\n without \\`--dry-run\\`; fix the \\`leftovers\\`, \\`push\\` in cloud mode, run \\`doctor\\` and the project's checks.\n4. On no, leave everything: a finished test costs nothing and the user may come back to it.\n\nA \\`paused\\` test is not over: say so and ask whether to resume (\\`start\\`) or finish it.\n\n## Do not\n\n- Nest an \\`<Experiment>\\` inside a \\`<Variant>\\` of the same id; call hooks conditionally.\n- Change \\`seed\\`, weights or variants of a running experiment (reshuffles visitors). Create a new experiment instead.\n- Edit in the dashboard what the config file owns: file experiments are read-only there (pause/finish still work).\n- Run more than 2-3 experiments on one page of a low-traffic site.\n- Put secrets, API keys or database credentials in the config or in the browser. \\`pk_\\` is public, \\`sk_\\` is not.\n`;\n\nexport const SKILL_AB_TESTING = `---\nname: ab-testing\ndescription: How to design, run and read an A/B test correctly: hypothesis, control, primary metric, sample size, significance, SRM, peeking, when to stop. Use before creating an experiment or when interpreting results, with any tool.\n---\n\n# A/B testing, the parts that matter\n\nAn A/B test splits visitors at random into variants and compares one metric. It answers one question:\n\"does this change move this metric?\" Everything below exists to keep that answer honest.\n\n## Before creating a test\n\n1. **One hypothesis, one primary metric.** \"A shorter signup form raises signup completion.\" The metric is a goal\n (event, page visit or click) that happens after the exposure. Pick it before looking at data.\n2. **Control first.** The first variant is what exists today. Change one thing per variant; several unrelated changes\n in one variant cannot be attributed.\n3. **Traffic reality.** Rough rule: to detect a relative lift L on a baseline rate p you need about\n \\`16 · p(1-p) / (p·L)²\\` visitors per variant (95% confidence, 80% power).\n Examples: baseline 5%, lift 20% → ~7,600 per variant; baseline 2%, lift 10% → ~78,000 per variant.\n If the site cannot reach that in 2-4 weeks, test a bigger change or a higher-traffic page.\n4. **Few tests per page.** Interactions between simultaneous tests on the same page muddy both.\n\n## While it runs\n\n- **Do not peek and stop early.** Checking daily and stopping at the first \"significant\" moment inflates false\n positives several times. Decide the sample or the duration up front; run whole weeks (weekday/weekend cycles).\n- **SRM (sample ratio mismatch).** If the split is 50/50 and you see 60/40, the assignment or the tracking is broken\n (bots, caching, redirect loops, a variant that errors). Fix first; results from an SRM run are meaningless.\n- **Keep variants frozen.** Changing copy, weights or targeting mid-run reshuffles who saw what. Start a new test.\n- **Watch the guardrails.** A variant that wins the primary metric but breaks revenue, errors or latency is not a win.\n\n## Reading results\n\n- Significance (p < 0.05 after correction for several variants) says the difference is unlikely to be noise;\n the confidence interval says how big it plausibly is. A \"winner\" with a CI that touches 0 is a weak win.\n- \"No significant difference\" with a sample smaller than needed means **keep running**, not \"no effect\".\n With enough sample it means the change does not matter: ship whichever is cheaper or cleaner.\n- Relative lift is on the baseline rate: 5% → 6% is +20% relative, +1 point absolute. Report both.\n- Attribution: conversions count only after the visitor's first exposure, per unique visitor.\n- Segments (country, device) are for hypotheses, not verdicts: slicing after the fact finds noise.\n\n## When to stop\n\n- Winner found with the planned sample → ship it, finish the test, remove the losing code.\n- Planned sample reached, no difference → ship the simpler variant, finish.\n- SRM or broken tracking → stop, fix, restart as a new test.\n- The business moved on → finish; do not leave tests running forever (they cost events and split traffic).\n\n## Vocabulary\n\nexposure = a visitor saw a variant · conversion = the goal happened after exposure · lift = relative change in rate ·\np-value = chance of seeing this difference by luck · CI = range the true difference likely lies in ·\nSRM = the split differs from the configured weights · guardrail = a metric that must not get worse.\n`;\n\nexport const SKILL_SPLITHUB_API = `---\nname: splithub-api\ndescription: Use the SplitHub Cloud HTTP API directly: public key event ingest, API-key endpoints for config, status, results and overrides, hosted experiments. Use when writing a server, a script, a non-JS client, or when the CLI is not available.\n---\n\n# SplitHub Cloud API\n\nBase URL: \\`https://go.splithub.io\\`. JSON in, JSON out (no envelope under \\`/v1/*\\`). Two credentials:\n\n- **Public key \\`pk_…\\`**: identifies the project to browsers and SDKs. Safe to ship. Can only send events and read\n the public overrides JSON.\n- **API key \\`sk_…\\`**: project-scoped secret for servers, CI and the CLI (\\`npx splithub login\\` mints one through a\n browser approval; or Settings → API keys in the dashboard). Never in client code.\n\n## Send events (public key)\n\n\\`\\`\\`http\nPOST /v1/events\nAuthorization: Bearer pk_live_…\nContent-Type: application/json\n\n{ \"project\": \"my-app\", \"sdk\": \"custom/1.0\", \"events\": [\n { \"id\": \"e_01h…\", \"type\": \"exposure\", \"exp\": \"hero-cta\", \"var\": \"bold\", \"vid\": \"v_abc\", \"sid\": \"s_1\", \"ts\": 1760000000000, \"url\": \"/\" },\n { \"id\": \"g_01h…\", \"type\": \"goal\", \"goal\": \"signup\", \"vid\": \"v_abc\", \"sid\": \"s_1\", \"ts\": 1760000010000, \"props\": { \"revenue\": 49 } }\n] }\n\\`\\`\\`\n\n- \\`id\\`: unique per event (8-64 chars, [a-z0-9_-]); duplicates are ignored. \\`vid\\`: stable visitor id (cookie \\`sh_vid\\`).\n- Reply \\`202 { ok, received, inserted, dropped }\\`. A project with nothing running or no config yet answers\n \\`202 { ok, inserted: 0, idle: 'no_running' | 'no_config' }\\`: nothing stored, not an error. \\`429\\` with \\`code\\` ∈\n quota_exceeded | deleted: back off for \\`Retry-After\\` seconds; nothing was stored. \\`403 origin_not_allowed\\` when the project restricts domains.\n- Exposures for experiments that are not running are dropped and not billed. Goals are never billed.\n\n## Live config for clients (public key)\n\n\\`GET /v1/overrides/<pk>.json\\` → \\`{ version, experiments: { <id>: { status?, traffic?, weights?, values? } }, added: [ …hosted experiments… ], goals: [...], ctx: { country, region, city } }\\`.\nPoll it at most once a minute; the SDK caches it in localStorage. \\`ctx\\` is the visitor's geo for targeting.\n\n## Project endpoints (API key)\n\n\\`\\`\\`http\nGET /v1/status every experiment: status, visitors, days, verdict headline\nGET /v1/results/<experiment> full verdict: sample, comparisons (lift, p-value, CI, P(beat)), srm, winner\nGET /v1/config the config stored in the cloud (file + dashboard experiments)\nPUT /v1/config { config } push a normalized config (what defineConfig returns); answers cloudOnly experiments to adopt\nGET /v1/overrides live overrides\nPUT /v1/overrides/<experiment> { status | traffic | weights | values } pause / resume / reweight without a deploy\nDELETE /v1/overrides/<experiment>\n\\`\\`\\`\n\n\\`Authorization: Bearer sk_live_…\\` on each. Status values: draft | running | paused | finished.\n\n## Dashboard-scope endpoints (session, same shapes the dashboard uses)\n\nUnder \\`/v1/projects/<projectId>/…\\` with a user session: \\`experiments\\` (hosted experiments CRUD incl. \\`flow\\` targeting),\n\\`goals\\`, \\`timeline\\`, \\`breakdown\\`, \\`realtime\\`, \\`events\\`, \\`export\\`, \\`notes\\`, \\`domains\\`, \\`links\\`, \\`alerts\\`, \\`billing\\`, \\`ai\\`.\nScripts should prefer the API-key endpoints above; the session ones exist for the UI.\n\n## Drop-in script (no SDK)\n\n\\`<script src=\"https://go.splithub.io/sh.js\" data-key=\"pk_live_…\"></script>\\` in \\`<head>\\`: redirect experiments, page-visit and\nclick goals from the cloud config, \\`window.splithub.track('goal')\\` for custom events,\n\\`<meta name=\"splithub-experiment\" content=\"<id>\">\\` on a tested page for an instant decision.\n\n## Errors\n\n\\`400\\` validation (message in \\`error\\`), \\`401\\` bad key, \\`403\\` forbidden, \\`404\\` unknown experiment/project, \\`409\\` state\n(e.g. \\`config_not_pushed\\`, \\`file_owned\\`), \\`429\\` gate or rate limit. Bodies: \\`{ error, code? }\\`.\n`;\n\nexport const SKILL_SPLITHUB_MCP = `---\nname: splithub-mcp\ndescription: Use or set up the SplitHub MCP server (@splithub/mcp) so an agent can create experiments, read verdicts, start/pause/finish tests and push config through tools instead of shelling out. Use when the user asks to connect SplitHub to Claude Code, Cursor, Codex or another MCP client.\n---\n\n# SplitHub MCP server\n\n\\`@splithub/mcp\\` is a stdio MCP server that wraps the \\`splithub\\` CLI for the current project folder.\nSame credentials and config file as the CLI; nothing extra to configure.\n\n## Register it\n\nClaude Code (project scope, committed in \\`.mcp.json\\`):\n\n\\`\\`\\`json\n{ \"mcpServers\": { \"splithub\": { \"command\": \"npx\", \"args\": [\"-y\", \"@splithub/mcp\"] } } }\n\\`\\`\\`\n\nor \\`claude mcp add splithub -- npx -y @splithub/mcp\\`. Cursor: the same object in \\`.cursor/mcp.json\\`.\nCodex / other clients: command \\`npx\\`, args \\`-y @splithub/mcp\\`. \\`npx splithub skills --mcp\\` writes both files.\nEnv: \\`SPLITHUB_PROJECT_DIR\\` (defaults to the MCP client's working directory), \\`SPLITHUB_API\\` (staging or self-hosted cloud), \\`SPLITHUB_API_KEY\\` (CI).\n\n## Tools\n\n| tool | does | notes |\n|---|---|---|\n| \\`splithub_status\\` | all experiments with verdict headline | start here |\n| \\`splithub_results\\` | full verdict for one experiment | report \\`verdict.message\\` verbatim; \\`daysToVerdict\\` = when to check again |\n| \\`splithub_review\\` | stored AI reviews (Pro): at start, on verdict changes, on finish, on demand | quote them as the analyst's opinion, the verdict stays the source of truth |\n| \\`splithub_plan\\` | visitors per variant for a baseline and a lift; days at an experiment's traffic | before creating |\n| \\`splithub_create_experiment\\` | adds an experiment to splithub.config.ts | kind value / component / redirect / mvt, id, variants (or factors for mvt), goal, values/to, weights, traffic, maxVisitors, start/end; running by default |\n| \\`splithub_refs\\` | files and lines that use an experiment and its goal | check the wiring, find leftovers |\n| \\`splithub_ship\\` | finish, remove from the config, rewrite the code to the kept variant | \\`dryRun: true\\` first; returns \\`diff\\` and \\`leftovers\\` |\n| \\`splithub_remove\\` | removes an experiment from the config (finished in the cloud) | when the code is already clean |\n| \\`splithub_projects\\` | every folder on this machine connected to a cloud project | |\n| \\`splithub_set_status\\` | start / pause / finish | applies live in cloud mode; after \\`finished\\`, ask which variant to keep and whether to clean the code (splithub-ship skill) |\n| \\`splithub_push\\` / \\`splithub_pull\\` | sync the config with the cloud | pull adopts dashboard-created experiments |\n| \\`splithub_doctor\\` | checks the installation | run after wiring code |\n| \\`splithub_login\\` | returns a sign-in link; the human approves it in the browser | show the link, then call \\`splithub_whoami\\` |\n| \\`splithub_whoami\\` | connected project and keys | |\n\n## Working rules\n\n- Create (running by default) → wire in code → \\`doctor\\` → \\`push\\` (cloud mode) → tell the user it goes live when they\n deploy this code and that \\`set_status paused\\` or Pause in the dashboard holds it.\n- Results: never call a winner from raw rates; quote the verdict. \\`srm_detected\\` = fix the setup first.\n- Running experiments are frozen: no weight / variant / seed edits; create a new one.\n- Errors carry a \\`code\\`. \\`NOT_LOGGED_IN\\`: call \\`splithub_login\\`, show the link to the user, wait for them, then retry.\n- The whole loop (plan → create → implement → doctor → push → results → finish → ship) is in the \\`splithub-loop\\` skill.\n- A \\`finished\\` experiment in \\`splithub_status\\` or \\`splithub_results\\` means the test is over but its code is still there: report\n the verdict, ask which variant to keep and whether to remove the test from the code, then follow the \\`splithub-ship\\` skill.\n`;\n\nexport { SKILL_SELF_HOSTED } from './skills-self-hosted';\nimport { SKILL_SELF_HOSTED } from './skills-self-hosted';\nexport { SKILL_LOOP, SKILL_SHIP } from './skills-ship';\nimport { SKILL_LOOP, SKILL_SHIP } from './skills-ship';\n\nexport const SKILLS: Skill[] = [\n { dir: 'splithub', body: SKILL_SPLITHUB },\n { dir: 'ab-testing', body: SKILL_AB_TESTING },\n { dir: 'splithub-api', body: SKILL_SPLITHUB_API },\n { dir: 'splithub-mcp', body: SKILL_SPLITHUB_MCP },\n { dir: 'splithub-self-hosted', body: SKILL_SELF_HOSTED },\n { dir: 'splithub-ship', body: SKILL_SHIP },\n { dir: 'splithub-loop', body: SKILL_LOOP },\n];\n\nexport const MCP_SERVER_ENTRY = { command: 'npx', args: ['-y', '@splithub/mcp'] };\n","export function configTemplate(project: string, mode: 'self' | 'cloud', key?: string): string {\n const storage = mode === 'self' ? `{ mode: 'self', endpoint: '/api/splithub' }` : `{ mode: 'cloud', key: ${key ? `'${key}'` : \"process.env.NEXT_PUBLIC_SPLITHUB_KEY ?? ''\"} }`;\n return `import { defineConfig } from '@splithub/core';\n\n// SplitHub experiments. This file is the source of truth: the SDK, the CLI\n// and the dashboard all read it. Keys: lowercase a-z 0-9 - _. First variant = control.\nexport default defineConfig({\n project: '${project}',\n storage: ${storage},\n // Drafts behave like running tests while you develop (next dev, vite); production ignores them.\n runDraftsInDev: true,\n\n goals: {\n // call track('signup') where the conversion happens\n signup: { type: 'event' },\n },\n\n experiments: {\n // run \\`npx splithub create\\` or add entries here:\n //\n // 'hero-title': {\n // kind: 'value',\n // status: 'running',\n // variants: {\n // control: { weight: 50, value: 'Ship faster' },\n // direct: { weight: 50, value: 'Know which version wins' },\n // },\n // goal: 'signup',\n // },\n },\n});\n`;\n}\n\nexport function nextMiddlewareTemplate(configImport: string): string {\n return `import { splithubMiddleware } from '@splithub/next';\nimport splithub from '${configImport}';\n\n// Sets the visitor cookie, forwards it to Server Components and serves\n// redirect experiments with an invisible rewrite.\nexport default splithubMiddleware(splithub);\n\nexport const config = {\n matcher: ['/((?!_next|api|.*\\\\\\\\..*).*)'],\n};\n`;\n}\n\nexport function nextRouteTemplate(configImport: string, store: { imports: string; factory: string } = { imports: \"import { sqliteStore } from '@splithub/adapter-sqlite';\", factory: \"sqliteStore({ file: process.env.SPLITHUB_DB ?? './splithub.db' })\" }): string {\n return `import { createSplitHubHandler } from '@splithub/node';\n${store.imports}\nimport config from '${configImport}';\n\n// Self-hosted ingest + results: events stay in your database. The CLI reads\n// status / results from this route; set SPLITHUB_ADMIN_TOKEN to protect it.\nconst handler = createSplitHubHandler({\n config,\n store: ${store.factory},\n adminToken: process.env.SPLITHUB_ADMIN_TOKEN,\n});\n\nexport const POST = handler;\nexport const GET = handler;\nexport const runtime = 'nodejs';\n`;\n}\n\nexport function componentVariantTemplate(name: string, experiment: string, variant: string): string {\n return `// Variant \"${variant}\" of experiment \"${experiment}\". Edit freely.\nexport function ${name}() {\n return (\n <section>\n <h2>${experiment} · ${variant}</h2>\n </section>\n );\n}\n`;\n}\n\nexport const AGENTS_SECTION = `\n## A/B testing (SplitHub)\n\nThis project uses SplitHub (\\`@splithub/react\\`). Experiments are declared in\n\\`splithub.config.ts\\` and rendered with \\`<Experiment id>\\` / \\`useVariant(id)\\` /\n\\`useValue(id)\\`. Conversions: \\`useTrack()\\` then \\`track('<goal>')\\`.\n\n- Not connected yet? \\`npx splithub init --mode cloud\\` (or \\`npx splithub login\\`) prints a link: show it to the user, they sign in and approve, the CLI gets the keys. Never ask the user to paste keys.\n- New experiment: \\`npx splithub create --kind value|component|redirect|mvt --id <key> --variants control,<other> --goal <goal> --yes\\`\n then wire it in code (value → \\`useValue\\`, component → \\`<Experiment>\\` with one \\`<Variant>\\` per key, redirect → nothing,\n mvt → \\`--factor name=level:value,…\\` and \\`const { a, b } = useValue(id)\\`).\n- Results: \\`npx splithub results <id> --json\\`; report \\`verdict.message\\` verbatim, never call a winner from raw rates.\n- Lifecycle: \\`npx splithub start|pause|finish <id>\\`. Check setup: \\`npx splithub doctor\\`.\n- A \\`finished\\` test still has all its variants in the code: report the verdict, ask which variant to keep and whether to\n remove the test from the code, then follow .claude/skills/splithub-ship/SKILL.md. Never delete test code unasked.\n- \\`create\\` writes \\`status: 'running'\\`: the test goes live when the code is deployed. Always tell the user that, and that\n \\`npx splithub pause <id>\\` or Pause in the dashboard holds it. Never change \\`seed\\` of a running experiment.\n- Config ships to the browser: no secrets in it. Full rules: .claude/skills/splithub/SKILL.md\n`;\n\nexport { SKILL_SPLITHUB as SKILL_MD } from './skills';\nimport { SKILL_SPLITHUB } from './skills';\n\nexport const CURSOR_RULE = `---\ndescription: SplitHub A/B testing conventions\nglobs: [\"splithub.config.ts\", \"splithub/**\", \"**/*.tsx\"]\nalwaysApply: false\n---\n${SKILL_SPLITHUB.replace(/^---[\\s\\S]*?---\\n/, '')}`;\n"],"mappings":";;;AACO,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACE1B,IAAM,aAAa;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA4CnB,IAAM,aAAa;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;AChCnB,IAAM,iBAAiB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiGvB,IAAM,mBAAmB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAuDzB,IAAM,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAsE3B,IAAM,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA0D3B,IAAM,SAAkB;AAAA,EAC7B,EAAE,KAAK,YAAY,MAAM,eAAe;AAAA,EACxC,EAAE,KAAK,cAAc,MAAM,iBAAiB;AAAA,EAC5C,EAAE,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,EAChD,EAAE,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,EAChD,EAAE,KAAK,wBAAwB,MAAM,kBAAkB;AAAA,EACvD,EAAE,KAAK,iBAAiB,MAAM,WAAW;AAAA,EACzC,EAAE,KAAK,iBAAiB,MAAM,WAAW;AAC3C;AAEO,IAAM,mBAAmB,EAAE,SAAS,OAAO,MAAM,CAAC,MAAM,eAAe,EAAE;;;ACjTzE,SAAS,eAAe,SAAiB,MAAwB,KAAsB;AAC5F,QAAM,UAAU,SAAS,SAAS,gDAAgD,yBAAyB,MAAM,IAAI,GAAG,MAAM,4CAA4C;AAC1K,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA,cAKK,OAAO;AAAA,aACR,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAwBpB;AAEO,SAAS,uBAAuB,cAA8B;AACnE,SAAO;AAAA,wBACe,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAUpC;AAEO,SAAS,kBAAkB,cAAsB,QAA8C,EAAE,SAAS,2DAA2D,SAAS,oEAAoE,GAAW;AAClQ,SAAO;AAAA,EACP,MAAM,OAAO;AAAA,sBACO,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAMvB,MAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQxB;AAEO,SAAS,yBAAyB,MAAc,YAAoB,SAAyB;AAClG,SAAO,eAAe,OAAO,oBAAoB,UAAU;AAAA,kBAC3C,IAAI;AAAA;AAAA;AAAA,YAGV,UAAU,SAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAKnC;AAEO,IAAM,iBAAiB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAuBvB,IAAM,cAAc;AAAA;AAAA;AAAA;AAAA;AAAA,EAKzB,eAAe,QAAQ,qBAAqB,EAAE,CAAC;","names":[]}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
fetchResults,
|
|
4
|
+
resolveTarget
|
|
5
|
+
} from "./chunk-BJQJGLOB.js";
|
|
6
|
+
import {
|
|
7
|
+
detectProject
|
|
8
|
+
} from "./chunk-PLXDG4B5.js";
|
|
9
|
+
import {
|
|
10
|
+
bar,
|
|
11
|
+
c,
|
|
12
|
+
fail,
|
|
13
|
+
pct,
|
|
14
|
+
printJson,
|
|
15
|
+
statusColor,
|
|
16
|
+
table,
|
|
17
|
+
verdictColor,
|
|
18
|
+
verdictIcon
|
|
19
|
+
} from "./chunk-V3DLUVOE.js";
|
|
20
|
+
|
|
21
|
+
// src/commands/results.ts
|
|
22
|
+
async function results(key, flags) {
|
|
23
|
+
if (!key) fail("USAGE", "Which experiment? npx splithub results <id>");
|
|
24
|
+
const target = resolveTarget(flags, detectProject());
|
|
25
|
+
const r = await fetchResults(target, key);
|
|
26
|
+
if (flags.json) {
|
|
27
|
+
printJson({ ...r, daysToVerdict: r.daysToVerdict ?? null, ...r.status === "finished" ? { next: FINISHED_NEXT } : {} });
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
const v = r.verdict;
|
|
31
|
+
const title = r.experiment === HOLDOUT ? "Holdout: long-term impact" : r.experiment;
|
|
32
|
+
console.log(`${c.bold(title)} ${c.dim(`\xB7 ${r.kind} \xB7 `)}${statusColor(r.status)} ${c.dim(`\xB7 goal: ${r.goal ?? "none"}`)}`);
|
|
33
|
+
if (r.description) console.log(c.dim(r.description));
|
|
34
|
+
console.log();
|
|
35
|
+
console.log(verdictColor(v.status, `${label(v.status)} ${v.message}`));
|
|
36
|
+
const eta = daysLeft(r.daysToVerdict);
|
|
37
|
+
if (eta && r.status === "running") console.log(c.dim(eta));
|
|
38
|
+
for (const line of dataLines(v)) console.log(c.dim(line));
|
|
39
|
+
for (const note of v.notes ?? []) console.log(c.dim(`\xB7 ${note}`));
|
|
40
|
+
console.log();
|
|
41
|
+
const rates = /* @__PURE__ */ new Map();
|
|
42
|
+
for (const cnt of v.sample.variants) rates.set(cnt.variant, cnt.exposures > 0 ? cnt.conversions / cnt.exposures : 0);
|
|
43
|
+
console.log(
|
|
44
|
+
table(
|
|
45
|
+
["VARIANT", "VISITORS", "CONV", "RATE", "LIFT", "P(BEAT)", "P-VALUE", "95% CI"],
|
|
46
|
+
// Always-valid (sequential) p-value and interval: the ones the verdict uses, safe to read any day.
|
|
47
|
+
v.sample.variants.map((cnt, i) => {
|
|
48
|
+
const cmp = v.comparisons.find((x) => x.variant === cnt.variant);
|
|
49
|
+
const name = i === 0 ? `${cnt.variant} ${c.dim("(control)")}` : cnt.variant;
|
|
50
|
+
const marked = v.winner === cnt.variant ? c.green(`\u2605 ${name}`) : name;
|
|
51
|
+
return [
|
|
52
|
+
marked,
|
|
53
|
+
String(cnt.exposures),
|
|
54
|
+
String(cnt.conversions),
|
|
55
|
+
pct(rates.get(cnt.variant)),
|
|
56
|
+
cmp ? pct(cmp.relativeLift, true) : "\u2013",
|
|
57
|
+
cmp ? pct(cmp.probabilityToBeat) : "\u2013",
|
|
58
|
+
cmp ? (cmp.sequential?.pValue ?? cmp.pValue).toFixed(4) : "\u2013",
|
|
59
|
+
cmp ? `${pct((cmp.sequential?.ci ?? cmp.ci).low, true)} \u2026 ${pct((cmp.sequential?.ci ?? cmp.ci).high, true)}` : "\u2013"
|
|
60
|
+
];
|
|
61
|
+
})
|
|
62
|
+
)
|
|
63
|
+
);
|
|
64
|
+
const maxRate = Math.max(1e-4, ...v.sample.variants.map((cnt) => rates.get(cnt.variant) ?? 0));
|
|
65
|
+
const nameWidth = Math.max(...v.sample.variants.map((cnt) => cnt.variant.length));
|
|
66
|
+
console.log();
|
|
67
|
+
console.log(c.dim("RATE"));
|
|
68
|
+
for (const cnt of v.sample.variants) {
|
|
69
|
+
const cmp = v.comparisons.find((x) => x.variant === cnt.variant);
|
|
70
|
+
const paint = v.winner === cnt.variant ? c.green : cmp && (cmp.relativeLift ?? 0) < 0 ? c.red : c.cyan;
|
|
71
|
+
const lift = cmp ? (cmp.relativeLift ?? 0) >= 0 ? c.green(`\u25B2 ${pct(cmp.relativeLift, true)}`) : c.red(`\u25BC ${pct(cmp.relativeLift, true)}`) : c.dim("baseline");
|
|
72
|
+
console.log(`${cnt.variant.padEnd(nameWidth)} ${bar(rates.get(cnt.variant) ?? 0, maxRate, 24, paint)} ${pct(rates.get(cnt.variant)).padStart(6)} ${lift}`);
|
|
73
|
+
}
|
|
74
|
+
if (v.comparisons.length) {
|
|
75
|
+
console.log();
|
|
76
|
+
console.log(c.dim("CHANCE TO BEAT THE CONTROL"));
|
|
77
|
+
for (const cmp of v.comparisons) {
|
|
78
|
+
const paint = cmp.probabilityToBeat >= 0.95 ? c.green : cmp.probabilityToBeat <= 0.05 ? c.red : c.yellow;
|
|
79
|
+
console.log(`${cmp.variant.padEnd(nameWidth)} ${bar(cmp.probabilityToBeat, 1, 24, paint)} ${pct(cmp.probabilityToBeat).padStart(6)}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (v.srm) {
|
|
83
|
+
console.log();
|
|
84
|
+
console.log(c.dim(`SRM check: chi\xB2=${v.srm.chiSquare.toFixed(2)} p=${v.srm.pValue.toExponential(2)} ${v.srm.detected ? c.red("DETECTED") : "ok"}`));
|
|
85
|
+
}
|
|
86
|
+
if (v.status === "insufficient_sample" && v.sample.neededPerVariant) {
|
|
87
|
+
console.log(c.dim(`Estimated ${v.sample.neededPerVariant} visitors per variant needed to confirm the current lift.`));
|
|
88
|
+
}
|
|
89
|
+
if (r.factors?.length) {
|
|
90
|
+
console.log();
|
|
91
|
+
console.log(c.bold("Each factor on its own") + c.dim(" (a level sums every combination that uses it, compared with the first level)"));
|
|
92
|
+
for (const f of r.factors) {
|
|
93
|
+
console.log(
|
|
94
|
+
table(
|
|
95
|
+
[f.key.toUpperCase(), "VISITORS", "CONV.", "RATE", "LIFT", "P-VALUE"],
|
|
96
|
+
f.levels.map((l) => [l.level + (f.best === l.level ? c.green(" \u2714") : ""), String(l.exposures), String(l.conversions), pct(l.rate), l.relativeLift === null ? "\u2013" : pct(l.relativeLift, true), l.pValue === null ? "\u2013" : l.pValue.toFixed(3)])
|
|
97
|
+
)
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (v.metrics?.length) {
|
|
102
|
+
console.log();
|
|
103
|
+
console.log(metricsText(v));
|
|
104
|
+
}
|
|
105
|
+
if (r.status === "finished" && r.experiment !== HOLDOUT) {
|
|
106
|
+
console.log();
|
|
107
|
+
console.log(c.dim(FINISHED_NEXT));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
var FINISHED_NEXT = "This test is finished but its variants are still in the code. Decide which variant to keep, then run npx splithub ship <id> --keep <variant> (agents: the splithub-ship skill).";
|
|
111
|
+
function daysLeft(days) {
|
|
112
|
+
if (days === null || days === void 0 || !Number.isFinite(days) || days <= 0) return null;
|
|
113
|
+
const d = Math.ceil(days);
|
|
114
|
+
return d === 1 ? "About 1 more day to a verdict at this traffic." : `About ${d} more days to a verdict at this traffic.`;
|
|
115
|
+
}
|
|
116
|
+
function starText(stars) {
|
|
117
|
+
const n = Math.max(0, Math.min(5, Math.round(stars)));
|
|
118
|
+
return "\u2605".repeat(n) + "\u2606".repeat(5 - n);
|
|
119
|
+
}
|
|
120
|
+
function dataLines(v) {
|
|
121
|
+
const { stars, checks } = v.sample;
|
|
122
|
+
if (stars === void 0 || !checks?.length) return [];
|
|
123
|
+
return [`Data ${starText(stars)} ${stars}/5`, ...checks.filter((x) => !x.met).map((x) => ` \u2717 ${x.label}`)];
|
|
124
|
+
}
|
|
125
|
+
function label(status) {
|
|
126
|
+
const icon = verdictIcon(status);
|
|
127
|
+
switch (status) {
|
|
128
|
+
case "winner_found":
|
|
129
|
+
return `${icon} WINNER`;
|
|
130
|
+
case "srm_detected":
|
|
131
|
+
return `${icon} SRM`;
|
|
132
|
+
case "insufficient_sample":
|
|
133
|
+
return `${icon} TOO EARLY`;
|
|
134
|
+
case "no_significant_difference":
|
|
135
|
+
return `${icon} NO DIFFERENCE`;
|
|
136
|
+
case "guardrail_breached":
|
|
137
|
+
return `${icon} DON'T SHIP`;
|
|
138
|
+
default:
|
|
139
|
+
return `${icon} NO DATA`;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
var HOLDOUT = "__holdout";
|
|
143
|
+
function metricsText(v) {
|
|
144
|
+
const out = [];
|
|
145
|
+
for (const m of v.metrics ?? []) {
|
|
146
|
+
const head = m.role === "guardrail" ? `GUARDRAIL ${m.metric}${m.worse === "higher" ? " (a rise is bad)" : ""}` : `ALSO ${m.metric}`;
|
|
147
|
+
out.push(c.dim(head));
|
|
148
|
+
out.push(
|
|
149
|
+
table(
|
|
150
|
+
["VARIANT", "CONV", "RATE", "LIFT", "P-VALUE", ""],
|
|
151
|
+
m.variants.map((cnt) => {
|
|
152
|
+
const cmp = m.comparisons.find((x) => x.variant === cnt.variant);
|
|
153
|
+
const breach = v.guardrailBreached?.find((b) => b.metric === m.metric && b.variant === cnt.variant);
|
|
154
|
+
return [cnt.variant, String(cnt.conversions), pct(cnt.exposures ? cnt.conversions / cnt.exposures : 0), cmp ? pct(cmp.relativeLift, true) : "\u2013", cmp ? (cmp.sequential?.pValue ?? cmp.pValue).toFixed(4) : "\u2013", breach ? c.red("\u2716 worse") : ""];
|
|
155
|
+
})
|
|
156
|
+
)
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
return out.join("\n");
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export {
|
|
163
|
+
results,
|
|
164
|
+
daysLeft,
|
|
165
|
+
starText,
|
|
166
|
+
dataLines,
|
|
167
|
+
HOLDOUT,
|
|
168
|
+
metricsText
|
|
169
|
+
};
|
|
170
|
+
//# sourceMappingURL=chunk-DUIJ5DTV.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/commands/results.ts"],"sourcesContent":["import type { ExperimentResults } from '@splithub/node';\nimport { fetchResults, resolveTarget } from '../api';\nimport { fail, printJson } from '../errors';\nimport { detectProject } from '../project';\nimport { bar, c, pct, statusColor, table, verdictColor, verdictIcon } from '../ui';\n\nexport async function results(key: string | undefined, flags: Record<string, unknown>): Promise<void> {\n if (!key) fail('USAGE', 'Which experiment? npx splithub results <id>');\n const target = resolveTarget(flags, detectProject());\n const r = await fetchResults(target, key);\n if (flags.json) {\n printJson({ ...r, daysToVerdict: r.daysToVerdict ?? null, ...(r.status === 'finished' ? { next: FINISHED_NEXT } : {}) });\n return;\n }\n const v = r.verdict;\n const title = r.experiment === HOLDOUT ? 'Holdout: long-term impact' : r.experiment;\n console.log(`${c.bold(title)} ${c.dim(`· ${r.kind} · `)}${statusColor(r.status)} ${c.dim(`· goal: ${r.goal ?? 'none'}`)}`);\n if (r.description) console.log(c.dim(r.description));\n console.log();\n console.log(verdictColor(v.status, `${label(v.status)} ${v.message}`));\n const eta = daysLeft(r.daysToVerdict);\n if (eta && r.status === 'running') console.log(c.dim(eta));\n for (const line of dataLines(v)) console.log(c.dim(line));\n for (const note of v.notes ?? []) console.log(c.dim(`· ${note}`));\n console.log();\n\n const rates = new Map<string, number>();\n for (const cnt of v.sample.variants) rates.set(cnt.variant, cnt.exposures > 0 ? cnt.conversions / cnt.exposures : 0);\n\n console.log(\n table(\n ['VARIANT', 'VISITORS', 'CONV', 'RATE', 'LIFT', 'P(BEAT)', 'P-VALUE', '95% CI'],\n // Always-valid (sequential) p-value and interval: the ones the verdict uses, safe to read any day.\n v.sample.variants.map((cnt, i) => {\n const cmp = v.comparisons.find((x) => x.variant === cnt.variant);\n const name = i === 0 ? `${cnt.variant} ${c.dim('(control)')}` : cnt.variant;\n const marked = v.winner === cnt.variant ? c.green(`★ ${name}`) : name;\n return [\n marked,\n String(cnt.exposures),\n String(cnt.conversions),\n pct(rates.get(cnt.variant)),\n cmp ? pct(cmp.relativeLift, true) : '–',\n cmp ? pct(cmp.probabilityToBeat) : '–',\n cmp ? (cmp.sequential?.pValue ?? cmp.pValue).toFixed(4) : '–',\n cmp ? `${pct((cmp.sequential?.ci ?? cmp.ci).low, true)} … ${pct((cmp.sequential?.ci ?? cmp.ci).high, true)}` : '–',\n ];\n })\n )\n );\n // Rates as bars, lift next to them; P(beat) as a second bar. Reads at a glance in a terminal.\n const maxRate = Math.max(0.0001, ...v.sample.variants.map((cnt) => rates.get(cnt.variant) ?? 0));\n const nameWidth = Math.max(...v.sample.variants.map((cnt) => cnt.variant.length));\n console.log();\n console.log(c.dim('RATE'));\n for (const cnt of v.sample.variants) {\n const cmp = v.comparisons.find((x) => x.variant === cnt.variant);\n const paint = v.winner === cnt.variant ? c.green : cmp && (cmp.relativeLift ?? 0) < 0 ? c.red : c.cyan;\n const lift = cmp ? (cmp.relativeLift ?? 0) >= 0 ? c.green(`▲ ${pct(cmp.relativeLift, true)}`) : c.red(`▼ ${pct(cmp.relativeLift, true)}`) : c.dim('baseline');\n console.log(`${cnt.variant.padEnd(nameWidth)} ${bar(rates.get(cnt.variant) ?? 0, maxRate, 24, paint)} ${pct(rates.get(cnt.variant)).padStart(6)} ${lift}`);\n }\n if (v.comparisons.length) {\n console.log();\n console.log(c.dim('CHANCE TO BEAT THE CONTROL'));\n for (const cmp of v.comparisons) {\n const paint = cmp.probabilityToBeat >= 0.95 ? c.green : cmp.probabilityToBeat <= 0.05 ? c.red : c.yellow;\n console.log(`${cmp.variant.padEnd(nameWidth)} ${bar(cmp.probabilityToBeat, 1, 24, paint)} ${pct(cmp.probabilityToBeat).padStart(6)}`);\n }\n }\n if (v.srm) {\n console.log();\n console.log(c.dim(`SRM check: chi²=${v.srm.chiSquare.toFixed(2)} p=${v.srm.pValue.toExponential(2)} ${v.srm.detected ? c.red('DETECTED') : 'ok'}`));\n }\n if (v.status === 'insufficient_sample' && v.sample.neededPerVariant) {\n console.log(c.dim(`Estimated ${v.sample.neededPerVariant} visitors per variant needed to confirm the current lift.`));\n }\n if (r.factors?.length) {\n console.log();\n console.log(c.bold('Each factor on its own') + c.dim(' (a level sums every combination that uses it, compared with the first level)'));\n for (const f of r.factors) {\n console.log(\n table(\n [f.key.toUpperCase(), 'VISITORS', 'CONV.', 'RATE', 'LIFT', 'P-VALUE'],\n f.levels.map((l) => [l.level + (f.best === l.level ? c.green(' ✔') : ''), String(l.exposures), String(l.conversions), pct(l.rate), l.relativeLift === null ? '–' : pct(l.relativeLift, true), l.pValue === null ? '–' : l.pValue.toFixed(3)])\n )\n );\n }\n }\n if (v.metrics?.length) {\n console.log();\n console.log(metricsText(v));\n }\n if (r.status === 'finished' && r.experiment !== HOLDOUT) {\n console.log();\n console.log(c.dim(FINISHED_NEXT));\n }\n}\n\n/** Printed for finished tests, so a human or an agent knows the code still carries every variant. */\nconst FINISHED_NEXT = 'This test is finished but its variants are still in the code. Decide which variant to keep, then run npx splithub ship <id> --keep <variant> (agents: the splithub-ship skill).';\n\n/** \"About 6 more days\" for a verdict ETA; null when unknown or already there. */\nexport function daysLeft(days: number | null | undefined): string | null {\n if (days === null || days === undefined || !Number.isFinite(days) || days <= 0) return null;\n const d = Math.ceil(days);\n return d === 1 ? 'About 1 more day to a verdict at this traffic.' : `About ${d} more days to a verdict at this traffic.`;\n}\n\nconst num = (n: number) => n.toLocaleString('en-US');\n\n/** \"★★★☆☆\" for 0..5 met checks. */\nexport function starText(stars: number): string {\n const n = Math.max(0, Math.min(5, Math.round(stars)));\n return '★'.repeat(n) + '☆'.repeat(5 - n);\n}\n\n/** \"Data ★★★☆☆ 3/5\" and one line per unmet check. Empty for servers without the score. */\nexport function dataLines(v: { sample: { stars?: number; checks?: Array<{ met: boolean; label: string }> } }): string[] {\n const { stars, checks } = v.sample;\n if (stars === undefined || !checks?.length) return [];\n return [`Data ${starText(stars)} ${stars}/5`, ...checks.filter((x) => !x.met).map((x) => ` ✗ ${x.label}`)];\n}\n\nfunction label(status: string): string {\n const icon = verdictIcon(status);\n switch (status) {\n case 'winner_found':\n return `${icon} WINNER`;\n case 'srm_detected':\n return `${icon} SRM`;\n case 'insufficient_sample':\n return `${icon} TOO EARLY`;\n case 'no_significant_difference':\n return `${icon} NO DIFFERENCE`;\n case 'guardrail_breached':\n return `${icon} DON'T SHIP`;\n default:\n return `${icon} NO DATA`;\n }\n}\n\nexport const HOLDOUT = '__holdout';\n\ntype V = ExperimentResults['verdict'];\n\n/** Secondary metrics and guardrails, one small table each. A breached guardrail is marked. */\nexport function metricsText(v: V): string {\n const out: string[] = [];\n for (const m of v.metrics ?? []) {\n const head = m.role === 'guardrail' ? `GUARDRAIL ${m.metric}${m.worse === 'higher' ? ' (a rise is bad)' : ''}` : `ALSO ${m.metric}`;\n out.push(c.dim(head));\n out.push(\n table(\n ['VARIANT', 'CONV', 'RATE', 'LIFT', 'P-VALUE', ''],\n m.variants.map((cnt) => {\n const cmp = m.comparisons.find((x) => x.variant === cnt.variant);\n const breach = v.guardrailBreached?.find((b) => b.metric === m.metric && b.variant === cnt.variant);\n return [cnt.variant, String(cnt.conversions), pct(cnt.exposures ? cnt.conversions / cnt.exposures : 0), cmp ? pct(cmp.relativeLift, true) : '–', cmp ? (cmp.sequential?.pValue ?? cmp.pValue).toFixed(4) : '–', breach ? c.red('✖ worse') : ''];\n })\n )\n );\n }\n return out.join('\\n');\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAMA,eAAsB,QAAQ,KAAyB,OAA+C;AACpG,MAAI,CAAC,IAAK,MAAK,SAAS,6CAA6C;AACrE,QAAM,SAAS,cAAc,OAAO,cAAc,CAAC;AACnD,QAAM,IAAI,MAAM,aAAa,QAAQ,GAAG;AACxC,MAAI,MAAM,MAAM;AACd,cAAU,EAAE,GAAG,GAAG,eAAe,EAAE,iBAAiB,MAAM,GAAI,EAAE,WAAW,aAAa,EAAE,MAAM,cAAc,IAAI,CAAC,EAAG,CAAC;AACvH;AAAA,EACF;AACA,QAAM,IAAI,EAAE;AACZ,QAAM,QAAQ,EAAE,eAAe,UAAU,8BAA8B,EAAE;AACzE,UAAQ,IAAI,GAAG,EAAE,KAAK,KAAK,CAAC,IAAI,EAAE,IAAI,QAAK,EAAE,IAAI,QAAK,CAAC,GAAG,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,cAAW,EAAE,QAAQ,MAAM,EAAE,CAAC,EAAE;AACzH,MAAI,EAAE,YAAa,SAAQ,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC;AACnD,UAAQ,IAAI;AACZ,UAAQ,IAAI,aAAa,EAAE,QAAQ,GAAG,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;AACtE,QAAM,MAAM,SAAS,EAAE,aAAa;AACpC,MAAI,OAAO,EAAE,WAAW,UAAW,SAAQ,IAAI,EAAE,IAAI,GAAG,CAAC;AACzD,aAAW,QAAQ,UAAU,CAAC,EAAG,SAAQ,IAAI,EAAE,IAAI,IAAI,CAAC;AACxD,aAAW,QAAQ,EAAE,SAAS,CAAC,EAAG,SAAQ,IAAI,EAAE,IAAI,QAAK,IAAI,EAAE,CAAC;AAChE,UAAQ,IAAI;AAEZ,QAAM,QAAQ,oBAAI,IAAoB;AACtC,aAAW,OAAO,EAAE,OAAO,SAAU,OAAM,IAAI,IAAI,SAAS,IAAI,YAAY,IAAI,IAAI,cAAc,IAAI,YAAY,CAAC;AAEnH,UAAQ;AAAA,IACN;AAAA,MACE,CAAC,WAAW,YAAY,QAAQ,QAAQ,QAAQ,WAAW,WAAW,QAAQ;AAAA;AAAA,MAE9E,EAAE,OAAO,SAAS,IAAI,CAAC,KAAK,MAAM;AAChC,cAAM,MAAM,EAAE,YAAY,KAAK,CAAC,MAAM,EAAE,YAAY,IAAI,OAAO;AAC/D,cAAM,OAAO,MAAM,IAAI,GAAG,IAAI,OAAO,IAAI,EAAE,IAAI,WAAW,CAAC,KAAK,IAAI;AACpE,cAAM,SAAS,EAAE,WAAW,IAAI,UAAU,EAAE,MAAM,UAAK,IAAI,EAAE,IAAI;AACjE,eAAO;AAAA,UACL;AAAA,UACA,OAAO,IAAI,SAAS;AAAA,UACpB,OAAO,IAAI,WAAW;AAAA,UACtB,IAAI,MAAM,IAAI,IAAI,OAAO,CAAC;AAAA,UAC1B,MAAM,IAAI,IAAI,cAAc,IAAI,IAAI;AAAA,UACpC,MAAM,IAAI,IAAI,iBAAiB,IAAI;AAAA,UACnC,OAAO,IAAI,YAAY,UAAU,IAAI,QAAQ,QAAQ,CAAC,IAAI;AAAA,UAC1D,MAAM,GAAG,KAAK,IAAI,YAAY,MAAM,IAAI,IAAI,KAAK,IAAI,CAAC,WAAM,KAAK,IAAI,YAAY,MAAM,IAAI,IAAI,MAAM,IAAI,CAAC,KAAK;AAAA,QACjH;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,UAAU,KAAK,IAAI,MAAQ,GAAG,EAAE,OAAO,SAAS,IAAI,CAAC,QAAQ,MAAM,IAAI,IAAI,OAAO,KAAK,CAAC,CAAC;AAC/F,QAAM,YAAY,KAAK,IAAI,GAAG,EAAE,OAAO,SAAS,IAAI,CAAC,QAAQ,IAAI,QAAQ,MAAM,CAAC;AAChF,UAAQ,IAAI;AACZ,UAAQ,IAAI,EAAE,IAAI,MAAM,CAAC;AACzB,aAAW,OAAO,EAAE,OAAO,UAAU;AACnC,UAAM,MAAM,EAAE,YAAY,KAAK,CAAC,MAAM,EAAE,YAAY,IAAI,OAAO;AAC/D,UAAM,QAAQ,EAAE,WAAW,IAAI,UAAU,EAAE,QAAQ,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,MAAM,EAAE;AAClG,UAAM,OAAO,OAAO,IAAI,gBAAgB,MAAM,IAAI,EAAE,MAAM,UAAK,IAAI,IAAI,cAAc,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,UAAK,IAAI,IAAI,cAAc,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,UAAU;AAC5J,YAAQ,IAAI,GAAG,IAAI,QAAQ,OAAO,SAAS,CAAC,KAAK,IAAI,MAAM,IAAI,IAAI,OAAO,KAAK,GAAG,SAAS,IAAI,KAAK,CAAC,KAAK,IAAI,MAAM,IAAI,IAAI,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,IAAI,EAAE;AAAA,EAC9J;AACA,MAAI,EAAE,YAAY,QAAQ;AACxB,YAAQ,IAAI;AACZ,YAAQ,IAAI,EAAE,IAAI,4BAA4B,CAAC;AAC/C,eAAW,OAAO,EAAE,aAAa;AAC/B,YAAM,QAAQ,IAAI,qBAAqB,OAAO,EAAE,QAAQ,IAAI,qBAAqB,OAAO,EAAE,MAAM,EAAE;AAClG,cAAQ,IAAI,GAAG,IAAI,QAAQ,OAAO,SAAS,CAAC,KAAK,IAAI,IAAI,mBAAmB,GAAG,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,iBAAiB,EAAE,SAAS,CAAC,CAAC,EAAE;AAAA,IACxI;AAAA,EACF;AACA,MAAI,EAAE,KAAK;AACT,YAAQ,IAAI;AACZ,YAAQ,IAAI,EAAE,IAAI,sBAAmB,EAAE,IAAI,UAAU,QAAQ,CAAC,CAAC,MAAM,EAAE,IAAI,OAAO,cAAc,CAAC,CAAC,IAAI,EAAE,IAAI,WAAW,EAAE,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;AAAA,EACpJ;AACA,MAAI,EAAE,WAAW,yBAAyB,EAAE,OAAO,kBAAkB;AACnE,YAAQ,IAAI,EAAE,IAAI,aAAa,EAAE,OAAO,gBAAgB,2DAA2D,CAAC;AAAA,EACtH;AACA,MAAI,EAAE,SAAS,QAAQ;AACrB,YAAQ,IAAI;AACZ,YAAQ,IAAI,EAAE,KAAK,wBAAwB,IAAI,EAAE,IAAI,+EAA+E,CAAC;AACrI,eAAW,KAAK,EAAE,SAAS;AACzB,cAAQ;AAAA,QACN;AAAA,UACE,CAAC,EAAE,IAAI,YAAY,GAAG,YAAY,SAAS,QAAQ,QAAQ,SAAS;AAAA,UACpE,EAAE,OAAO,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAI,IAAI,KAAK,OAAO,EAAE,SAAS,GAAG,OAAO,EAAE,WAAW,GAAG,IAAI,EAAE,IAAI,GAAG,EAAE,iBAAiB,OAAO,WAAM,IAAI,EAAE,cAAc,IAAI,GAAG,EAAE,WAAW,OAAO,WAAM,EAAE,OAAO,QAAQ,CAAC,CAAC,CAAC;AAAA,QAC9O;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,EAAE,SAAS,QAAQ;AACrB,YAAQ,IAAI;AACZ,YAAQ,IAAI,YAAY,CAAC,CAAC;AAAA,EAC5B;AACA,MAAI,EAAE,WAAW,cAAc,EAAE,eAAe,SAAS;AACvD,YAAQ,IAAI;AACZ,YAAQ,IAAI,EAAE,IAAI,aAAa,CAAC;AAAA,EAClC;AACF;AAGA,IAAM,gBAAgB;AAGf,SAAS,SAAS,MAAgD;AACvE,MAAI,SAAS,QAAQ,SAAS,UAAa,CAAC,OAAO,SAAS,IAAI,KAAK,QAAQ,EAAG,QAAO;AACvF,QAAM,IAAI,KAAK,KAAK,IAAI;AACxB,SAAO,MAAM,IAAI,mDAAmD,SAAS,CAAC;AAChF;AAKO,SAAS,SAAS,OAAuB;AAC9C,QAAM,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,CAAC,CAAC;AACpD,SAAO,SAAI,OAAO,CAAC,IAAI,SAAI,OAAO,IAAI,CAAC;AACzC;AAGO,SAAS,UAAU,GAA8F;AACtH,QAAM,EAAE,OAAO,OAAO,IAAI,EAAE;AAC5B,MAAI,UAAU,UAAa,CAAC,QAAQ,OAAQ,QAAO,CAAC;AACpD,SAAO,CAAC,QAAQ,SAAS,KAAK,CAAC,IAAI,KAAK,MAAM,GAAG,OAAO,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,YAAO,EAAE,KAAK,EAAE,CAAC;AAC5G;AAEA,SAAS,MAAM,QAAwB;AACrC,QAAM,OAAO,YAAY,MAAM;AAC/B,UAAQ,QAAQ;AAAA,IACd,KAAK;AACH,aAAO,GAAG,IAAI;AAAA,IAChB,KAAK;AACH,aAAO,GAAG,IAAI;AAAA,IAChB,KAAK;AACH,aAAO,GAAG,IAAI;AAAA,IAChB,KAAK;AACH,aAAO,GAAG,IAAI;AAAA,IAChB,KAAK;AACH,aAAO,GAAG,IAAI;AAAA,IAChB;AACE,aAAO,GAAG,IAAI;AAAA,EAClB;AACF;AAEO,IAAM,UAAU;AAKhB,SAAS,YAAY,GAAc;AACxC,QAAM,MAAgB,CAAC;AACvB,aAAW,KAAK,EAAE,WAAW,CAAC,GAAG;AAC/B,UAAM,OAAO,EAAE,SAAS,cAAc,aAAa,EAAE,MAAM,GAAG,EAAE,UAAU,WAAW,qBAAqB,EAAE,KAAK,QAAQ,EAAE,MAAM;AACjI,QAAI,KAAK,EAAE,IAAI,IAAI,CAAC;AACpB,QAAI;AAAA,MACF;AAAA,QACE,CAAC,WAAW,QAAQ,QAAQ,QAAQ,WAAW,EAAE;AAAA,QACjD,EAAE,SAAS,IAAI,CAAC,QAAQ;AACtB,gBAAM,MAAM,EAAE,YAAY,KAAK,CAAC,MAAM,EAAE,YAAY,IAAI,OAAO;AAC/D,gBAAM,SAAS,EAAE,mBAAmB,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,IAAI,OAAO;AAClG,iBAAO,CAAC,IAAI,SAAS,OAAO,IAAI,WAAW,GAAG,IAAI,IAAI,YAAY,IAAI,cAAc,IAAI,YAAY,CAAC,GAAG,MAAM,IAAI,IAAI,cAAc,IAAI,IAAI,UAAK,OAAO,IAAI,YAAY,UAAU,IAAI,QAAQ,QAAQ,CAAC,IAAI,UAAK,SAAS,EAAE,IAAI,cAAS,IAAI,EAAE;AAAA,QAChP,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO,IAAI,KAAK,IAAI;AACtB;","names":[]}
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
ensureLayer,
|
|
4
|
+
hasExperiment,
|
|
5
|
+
insertExperiment,
|
|
6
|
+
loadConfig,
|
|
7
|
+
readSource,
|
|
8
|
+
writeConfigChecked
|
|
9
|
+
} from "./chunk-B6FITBWN.js";
|
|
10
|
+
import {
|
|
11
|
+
componentVariantTemplate
|
|
12
|
+
} from "./chunk-DDWYTOX4.js";
|
|
13
|
+
import {
|
|
14
|
+
requireConfig
|
|
15
|
+
} from "./chunk-PLXDG4B5.js";
|
|
16
|
+
import {
|
|
17
|
+
c,
|
|
18
|
+
fail,
|
|
19
|
+
printJson
|
|
20
|
+
} from "./chunk-V3DLUVOE.js";
|
|
21
|
+
|
|
22
|
+
// src/commands/create.ts
|
|
23
|
+
import * as p from "@clack/prompts";
|
|
24
|
+
import { mvtVariants } from "@splithub/core";
|
|
25
|
+
import { existsSync, mkdirSync, writeFileSync } from "fs";
|
|
26
|
+
import { join, relative } from "path";
|
|
27
|
+
var KEY = /^[a-z0-9][a-z0-9_-]{0,63}$/;
|
|
28
|
+
function str(v) {
|
|
29
|
+
return typeof v === "string" && v.length > 0 ? v : void 0;
|
|
30
|
+
}
|
|
31
|
+
function list(v) {
|
|
32
|
+
if (Array.isArray(v)) return v.map(String);
|
|
33
|
+
if (typeof v === "string") return [v];
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
function parsePairs(items) {
|
|
37
|
+
const out = {};
|
|
38
|
+
for (const item of items) {
|
|
39
|
+
const i = item.indexOf("=");
|
|
40
|
+
if (i > 0) out[item.slice(0, i).trim()] = item.slice(i + 1);
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
function pascal(s) {
|
|
45
|
+
return s.replace(/(^|[-_])([a-z0-9])/g, (_, __, ch) => ch.toUpperCase());
|
|
46
|
+
}
|
|
47
|
+
async function ask(promise) {
|
|
48
|
+
const v = await promise;
|
|
49
|
+
if (p.isCancel(v)) {
|
|
50
|
+
p.cancel("Cancelled.");
|
|
51
|
+
process.exit(0);
|
|
52
|
+
}
|
|
53
|
+
return v;
|
|
54
|
+
}
|
|
55
|
+
async function create(flags) {
|
|
56
|
+
const json = Boolean(flags.json);
|
|
57
|
+
const yes = Boolean(flags.yes);
|
|
58
|
+
const project = requireConfig();
|
|
59
|
+
const config = await loadConfig(project.configFile);
|
|
60
|
+
const source = readSource(project.configFile);
|
|
61
|
+
const interactive = !yes && !json;
|
|
62
|
+
if (interactive) p.intro(c.bold("splithub create"));
|
|
63
|
+
let kind = str(flags.kind);
|
|
64
|
+
if (!kind) {
|
|
65
|
+
if (!interactive) fail("USAGE", "Which kind? Pass --kind value|component|redirect|mvt.");
|
|
66
|
+
kind = await ask(
|
|
67
|
+
p.select({
|
|
68
|
+
message: "What changes between variants?",
|
|
69
|
+
options: [
|
|
70
|
+
{ value: "value", label: "Text, image URL, number", hint: "no new components" },
|
|
71
|
+
{ value: "component", label: "Layout / structure", hint: "one component per variant" },
|
|
72
|
+
{ value: "redirect", label: "A whole page", hint: "needs @splithub/next middleware" },
|
|
73
|
+
{ value: "mvt", label: "Several things at once", hint: "multivariate: every combination, each factor read on its own" }
|
|
74
|
+
]
|
|
75
|
+
})
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
let id = str(flags.id);
|
|
79
|
+
if (!id) {
|
|
80
|
+
if (!interactive) fail("USAGE", "Which id? Pass --id <key>, e.g. --id hero-title.");
|
|
81
|
+
id = await ask(
|
|
82
|
+
p.text({
|
|
83
|
+
message: "Experiment id",
|
|
84
|
+
placeholder: "hero-title",
|
|
85
|
+
validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : hasExperiment(source, v) ? "already exists" : void 0
|
|
86
|
+
})
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
if (!["value", "component", "redirect", "mvt"].includes(kind)) fail("INVALID_FLAG", `Unknown kind "${kind}".`, "Use --kind value, component, redirect or mvt.");
|
|
90
|
+
if (!KEY.test(id)) fail("INVALID_FLAG", `"${id}" isn't a valid id.`, "Use lowercase a-z, 0-9, - and _, at most 64 characters.");
|
|
91
|
+
if (hasExperiment(source, id) || config.experiments.some((e) => e.key === id)) fail("EXPERIMENT_EXISTS", `There's already an experiment "${id}".`, "Pick another id: an old id keeps its history.");
|
|
92
|
+
const limits = parseLimits(flags);
|
|
93
|
+
const factors = {};
|
|
94
|
+
if (kind === "mvt") {
|
|
95
|
+
for (const raw of list(flags.factor)) {
|
|
96
|
+
const eq = raw.indexOf("=");
|
|
97
|
+
if (eq <= 0) fail("INVALID_FLAG", `--factor expects name=level:value,level:value, got "${raw}".`);
|
|
98
|
+
const name = raw.slice(0, eq).trim();
|
|
99
|
+
const levels = {};
|
|
100
|
+
for (const part of raw.slice(eq + 1).split(",")) {
|
|
101
|
+
const colon = part.indexOf(":");
|
|
102
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
103
|
+
if (!level) continue;
|
|
104
|
+
levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
105
|
+
}
|
|
106
|
+
factors[name] = levels;
|
|
107
|
+
}
|
|
108
|
+
while (interactive && Object.keys(factors).length === 0) {
|
|
109
|
+
const name = await ask(p.text({ message: "Factor name (what changes), e.g. headline", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
|
|
110
|
+
const levelsRaw = await ask(p.text({ message: `Levels of "${name}" as key:value, comma separated (first = control)`, placeholder: "old:Old headline, new:New headline" }));
|
|
111
|
+
const levels = {};
|
|
112
|
+
for (const part of levelsRaw.split(",")) {
|
|
113
|
+
const colon = part.indexOf(":");
|
|
114
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
115
|
+
if (level) levels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
116
|
+
}
|
|
117
|
+
factors[name] = levels;
|
|
118
|
+
while (await ask(p.confirm({ message: "Add another factor?", initialValue: false }))) {
|
|
119
|
+
const more = await ask(p.text({ message: "Factor name", validate: (v) => !KEY.test(v) ? "lowercase a-z 0-9 - _ only" : void 0 }));
|
|
120
|
+
const moreRaw = await ask(p.text({ message: `Levels of "${more}" as key:value, comma separated`, placeholder: "blue:#2563eb, green:#16a34a" }));
|
|
121
|
+
const moreLevels = {};
|
|
122
|
+
for (const part of moreRaw.split(",")) {
|
|
123
|
+
const colon = part.indexOf(":");
|
|
124
|
+
const level = (colon >= 0 ? part.slice(0, colon) : part).trim();
|
|
125
|
+
if (level) moreLevels[level] = colon >= 0 ? part.slice(colon + 1).trim() : level;
|
|
126
|
+
}
|
|
127
|
+
factors[more] = moreLevels;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (Object.keys(factors).length === 0) fail("USAGE", "An mvt experiment needs factors.", "Pass --factor name=level:value,level:value once per factor.");
|
|
131
|
+
if (flags.variants || flags.weights) fail("INVALID_FLAG", "An mvt experiment takes --factor, not --variants or --weights.", "Its variants are every combination of the factor levels, split equally.");
|
|
132
|
+
}
|
|
133
|
+
let mvt = null;
|
|
134
|
+
try {
|
|
135
|
+
mvt = kind === "mvt" ? mvtVariants(id, factors) : null;
|
|
136
|
+
} catch (error) {
|
|
137
|
+
fail("INVALID_FLAG", error instanceof Error ? error.message : String(error), "At most 4 factors \xD7 4 levels, 16 combinations.");
|
|
138
|
+
}
|
|
139
|
+
let variantKeys = mvt ? mvt.variants.map((v) => v.key) : str(flags.variants)?.split(",").map((s) => s.trim()).filter(Boolean);
|
|
140
|
+
if (!variantKeys || variantKeys.length < 2) {
|
|
141
|
+
if (!interactive) variantKeys = ["control", "b"];
|
|
142
|
+
else {
|
|
143
|
+
const v = await ask(p.text({ message: "Variant keys, comma separated (first = control)", initialValue: "control, b" }));
|
|
144
|
+
variantKeys = v.split(",").map((s) => s.trim()).filter(Boolean);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const k of variantKeys) if (!KEY.test(k)) fail("INVALID_FLAG", `"${k}" isn't a valid variant key.`, "Use lowercase a-z, 0-9, - and _.");
|
|
148
|
+
if (variantKeys.length < 2) fail("INVALID_FLAG", "An experiment needs at least 2 variants.", "Pass --variants control,b.");
|
|
149
|
+
if (new Set(variantKeys).size !== variantKeys.length) fail("INVALID_FLAG", "Two variants have the same key.");
|
|
150
|
+
const weights = mvt ? variantKeys.map(() => 1) : parseWeights(str(flags.weights), variantKeys.length);
|
|
151
|
+
const values = parsePairs(list(flags.value));
|
|
152
|
+
const tos = parsePairs(list(flags.to));
|
|
153
|
+
if (kind === "value" && interactive) {
|
|
154
|
+
for (const k of variantKeys) {
|
|
155
|
+
if (values[k] === void 0) values[k] = await ask(p.text({ message: `Value for "${k}"`, placeholder: k === variantKeys[0] ? "current text" : "new text" }));
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
let match = str(flags.match);
|
|
159
|
+
if (kind === "redirect") {
|
|
160
|
+
if (!match) {
|
|
161
|
+
if (!interactive) fail("USAGE", "A redirect experiment needs the path it runs on.", "Pass --match /path.");
|
|
162
|
+
match = await ask(p.text({ message: "Path to run the redirect test on", initialValue: "/" }));
|
|
163
|
+
}
|
|
164
|
+
for (const k of variantKeys) {
|
|
165
|
+
if (tos[k] === void 0) {
|
|
166
|
+
if (!interactive) tos[k] = k === variantKeys[0] ? match : `${match.replace(/\/$/, "")}/${k}`;
|
|
167
|
+
else tos[k] = await ask(p.text({ message: `URL for "${k}"`, initialValue: k === variantKeys[0] ? match : "" }));
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
let goal = str(flags.goal);
|
|
172
|
+
const goals = config.goals.map((g) => g.key);
|
|
173
|
+
if (!goal) {
|
|
174
|
+
if (!interactive) goal = goals[0];
|
|
175
|
+
else if (goals.length > 0) {
|
|
176
|
+
goal = await ask(
|
|
177
|
+
p.select({
|
|
178
|
+
message: "Goal (conversion) to optimise",
|
|
179
|
+
options: [...goals.map((g) => ({ value: g, label: g })), { value: "", label: "none yet (add later)" }]
|
|
180
|
+
})
|
|
181
|
+
);
|
|
182
|
+
if (!goal) goal = void 0;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
if (goal && !goals.includes(goal)) fail("GOAL_NOT_FOUND", `There's no goal "${goal}" in the config.`, `Declared goals: ${goals.join(", ") || "none"}. Add it to goals first.`);
|
|
186
|
+
const extras = parseExtras(flags, goals, goal);
|
|
187
|
+
const status = str(flags.status) ?? "running";
|
|
188
|
+
if (!["draft", "running", "paused"].includes(status)) fail("INVALID_FLAG", `A new experiment can't be "${status}".`, "Use --status running or --status draft.");
|
|
189
|
+
const description = str(flags.description);
|
|
190
|
+
const experiment = {
|
|
191
|
+
key: id,
|
|
192
|
+
kind,
|
|
193
|
+
status,
|
|
194
|
+
goal,
|
|
195
|
+
match,
|
|
196
|
+
description,
|
|
197
|
+
variants: mvt ? mvt.variants.map((v) => ({ key: v.key, weight: v.weight })) : variantKeys.map((k, i) => ({ key: k, weight: weights[i], value: values[k], to: tos[k] })),
|
|
198
|
+
...mvt ? { factors } : {},
|
|
199
|
+
...limits,
|
|
200
|
+
...extras
|
|
201
|
+
};
|
|
202
|
+
await writeConfigChecked(project.configFile, source, insertExperiment(extras.layer ? ensureLayer(source, extras.layer.key) : source, experiment), (loaded) => {
|
|
203
|
+
const e = loaded.experiments.find((x) => x.key === id);
|
|
204
|
+
if (!e) return `"${id}" is not in the file after the write`;
|
|
205
|
+
if (e.status !== status) return `"${id}" loads as ${e.status}, not ${status}`;
|
|
206
|
+
if (limits.minDays !== void 0 && e.minDays !== limits.minDays) return `"${id}" loads with minDays ${e.minDays ?? "unset"}, not ${limits.minDays}`;
|
|
207
|
+
if (limits.goodSample !== void 0 && e.goodSample !== limits.goodSample) return `"${id}" loads with goodSample ${e.goodSample ?? "unset"}, not ${limits.goodSample}`;
|
|
208
|
+
return null;
|
|
209
|
+
});
|
|
210
|
+
const files = [];
|
|
211
|
+
const componentNames = {};
|
|
212
|
+
if (kind === "component") {
|
|
213
|
+
const dir = join(project.root, project.hasSrc ? "src/splithub" : "splithub", id);
|
|
214
|
+
mkdirSync(dir, { recursive: true });
|
|
215
|
+
for (const k of variantKeys) {
|
|
216
|
+
const name = `${pascal(id)}${pascal(k)}`;
|
|
217
|
+
componentNames[k] = name;
|
|
218
|
+
const file = join(dir, `${pascal(k)}.tsx`);
|
|
219
|
+
if (!existsSync(file)) {
|
|
220
|
+
writeFileSync(file, componentVariantTemplate(name, id, k), "utf8");
|
|
221
|
+
files.push(relative(project.root, file));
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
const snippet = kind === "mvt" ? `const { ${Object.keys(factors).join(", ")} } = useValue<{ ${Object.keys(factors).map((f) => `${f}: string`).join("; ")} }>('${id}');` : kind === "value" ? `const value = useValue<string>('${id}');` : kind === "component" ? `<Experiment id="${id}">
|
|
226
|
+
${variantKeys.map((k) => ` <Variant name="${k}"><${componentNames[k]} /></Variant>`).join("\n")}
|
|
227
|
+
</Experiment>` : `// handled by @splithub/next middleware on ${match}; no code needed`;
|
|
228
|
+
const result = { experiment: id, kind, status, variants: experiment.variants, goal: goal ?? null, traffic: limits.traffic ?? 1, maxVisitors: limits.maxExposures ?? null, minDays: limits.minDays ?? null, goodSample: limits.goodSample ?? null, schedule: limits.schedule ?? null, configFile: relative(project.root, project.configFile), files, snippet };
|
|
229
|
+
if (json) {
|
|
230
|
+
const cloudMode = /mode\s*:\s*['"]cloud['"]/.test(readSource(project.configFile));
|
|
231
|
+
const handoff = status === "draft" ? `${id} is a draft: visitors do not see it and nothing is recorded until it is started. Tell the user and ask whether to start it now (npx splithub start ${id}).` : `${id} is running in the config: it goes live for visitors when this code is deployed${cloudMode ? " and pushed; local development never sends events to the cloud" : ""}. Tell the user, and that npx splithub pause ${id} or Pause in the dashboard holds it.`;
|
|
232
|
+
const nextStep = cloudMode ? `npx splithub push # the cloud does not know about ${id} until you push` : status === "draft" ? `npx splithub start ${id}` : "deploy the app";
|
|
233
|
+
printJson({ ...result, nextStep, handoff });
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
p.note(snippet, kind === "redirect" ? "Nothing to wire" : "Wire it in your code");
|
|
237
|
+
if (files.length) p.note(files.join("\n"), "Scaffolded");
|
|
238
|
+
const cloud = /mode\s*:\s*['"]cloud['"]/.test(readSource(project.configFile));
|
|
239
|
+
if (cloud) p.log.warn(`Cloud mode: the cloud does not know about it until you run ${c.cyan("npx splithub push")}.`);
|
|
240
|
+
if (status === "draft") p.log.warn(`${c.bold(id)} is a draft: visitors do not see it and nothing is recorded until you start it.`);
|
|
241
|
+
if (status === "running") p.log.info(`${c.bold(id)} goes live when you deploy this code. Check it first with ${c.cyan(`?sh_force=${id}:<variant>`)}; ${c.cyan(`npx splithub pause ${id}`)} holds it.`);
|
|
242
|
+
if (status === "draft") p.log.info(`See it before you start it: ${c.cyan("runDraftsInDev: true")} in the config serves drafts in development, ${c.cyan(`?sh_force=${id}:<variant>`)} shows one variant anywhere.`);
|
|
243
|
+
p.outro(`Added ${c.bold(id)} (${status}) to ${relative(project.root, project.configFile)}. ${status === "running" ? "" : `Start it with ${c.cyan(`npx splithub start ${id}`)}. `}${cloud ? `Then ${c.cyan("npx splithub push")}.` : ""}`);
|
|
244
|
+
}
|
|
245
|
+
function parseWeights(raw, count) {
|
|
246
|
+
if (!raw) return Array.from({ length: count }, () => 1);
|
|
247
|
+
const weights = raw.split(",").map((w) => Number(w.trim()));
|
|
248
|
+
if (weights.length !== count) fail("INVALID_FLAG", `--weights has ${weights.length} numbers for ${count} variants.`, "Give one weight per variant, in the same order: --weights 50,50.");
|
|
249
|
+
if (weights.some((w) => !Number.isFinite(w) || w < 0)) fail("INVALID_FLAG", `--weights "${raw}" has a value that isn't a number of 0 or more.`);
|
|
250
|
+
if (weights.every((w) => w === 0)) fail("INVALID_FLAG", "All weights are 0, so nobody would see the test.");
|
|
251
|
+
return weights;
|
|
252
|
+
}
|
|
253
|
+
function parseLimits(flags) {
|
|
254
|
+
const out = {};
|
|
255
|
+
const traffic = str(flags.traffic);
|
|
256
|
+
if (traffic !== void 0) {
|
|
257
|
+
const t = Number(traffic.replace(/%$/, ""));
|
|
258
|
+
if (!Number.isFinite(t) || t <= 0 || t > 100) fail("INVALID_FLAG", `--traffic "${traffic}" must be a percent between 0 and 100.`, "For example --traffic 50 puts half of the visitors in the test.");
|
|
259
|
+
if (t !== 100) out.traffic = Math.round(t * 100) / 1e4;
|
|
260
|
+
}
|
|
261
|
+
const max = str(flags["max-visitors"]);
|
|
262
|
+
if (max !== void 0) {
|
|
263
|
+
const m = Number(max);
|
|
264
|
+
if (!Number.isInteger(m) || m <= 0) fail("INVALID_FLAG", `--max-visitors "${max}" must be a whole number above 0.`);
|
|
265
|
+
out.maxExposures = m;
|
|
266
|
+
}
|
|
267
|
+
const minDays = str(flags["min-days"]);
|
|
268
|
+
if (minDays !== void 0) {
|
|
269
|
+
const d = Number(minDays);
|
|
270
|
+
if (!Number.isInteger(d) || d < 0 || d > 30) fail("INVALID_FLAG", `--min-days "${minDays}" must be a whole number from 0 to 30.`, "It is the fewest days before a verdict; 0 means none, the default.");
|
|
271
|
+
out.minDays = d;
|
|
272
|
+
}
|
|
273
|
+
const good = str(flags["good-sample"]);
|
|
274
|
+
if (good !== void 0) {
|
|
275
|
+
const g = Number(good);
|
|
276
|
+
if (!Number.isInteger(g) || g < 100 || g > 1e7) fail("INVALID_FLAG", `--good-sample "${good}" must be a whole number from 100 to 10,000,000.`, "It is the total visitors at which the sample counts as good; the default is 5000.");
|
|
277
|
+
out.goodSample = g;
|
|
278
|
+
}
|
|
279
|
+
const date = (name) => {
|
|
280
|
+
const v = str(flags[name]);
|
|
281
|
+
if (v === void 0) return null;
|
|
282
|
+
const ms = Date.parse(v);
|
|
283
|
+
if (Number.isNaN(ms)) fail("INVALID_FLAG", `--${name} "${v}" isn't a date.`, "Use an ISO date like 2026-11-01 or 2026-11-01T09:00:00Z.");
|
|
284
|
+
return new Date(ms).toISOString();
|
|
285
|
+
};
|
|
286
|
+
const startAt = date("start");
|
|
287
|
+
const endAt = date("end");
|
|
288
|
+
if (startAt && endAt && Date.parse(endAt) <= Date.parse(startAt)) fail("INVALID_FLAG", "--end must be after --start.");
|
|
289
|
+
if (startAt || endAt) out.schedule = { startAt, endAt };
|
|
290
|
+
return out;
|
|
291
|
+
}
|
|
292
|
+
function parseExtras(flags, goals, primary) {
|
|
293
|
+
const out = {};
|
|
294
|
+
const split = (v) => list(v).flatMap((x) => x.split(",")).map((x) => x.trim()).filter(Boolean);
|
|
295
|
+
const secondary = split(flags.secondary);
|
|
296
|
+
const guardrails = split(flags.guardrail).map((g) => {
|
|
297
|
+
const [goal = "", worse] = g.split(":");
|
|
298
|
+
if (worse !== void 0 && worse !== "lower" && worse !== "higher") fail("INVALID_FLAG", `--guardrail "${g}": after the colon comes lower or higher.`, "refunds:higher means more refunds is worse; signups (or signups:lower) means fewer is worse.");
|
|
299
|
+
return worse === "higher" ? { goal, worse: "higher" } : goal;
|
|
300
|
+
});
|
|
301
|
+
const named = [...secondary, ...guardrails.map((g) => typeof g === "string" ? g : g.goal)];
|
|
302
|
+
if (named.length) {
|
|
303
|
+
if (!primary) fail("USAGE", "Secondary metrics and guardrails need a primary goal.", "Pass --goal too.");
|
|
304
|
+
for (const g of named) if (!goals.includes(g)) fail("GOAL_NOT_FOUND", `There's no goal "${g}" in the config.`, `Declared goals: ${goals.join(", ") || "none"}. Add it to goals first.`);
|
|
305
|
+
const all = [primary, ...named];
|
|
306
|
+
if (new Set(all).size !== all.length) fail("INVALID_FLAG", "A goal is listed twice across --goal, --secondary and --guardrail.");
|
|
307
|
+
out.metrics = { ...secondary.length ? { secondary } : {}, ...guardrails.length ? { guardrails } : {} };
|
|
308
|
+
}
|
|
309
|
+
const layer = str(flags.layer);
|
|
310
|
+
if (layer) {
|
|
311
|
+
const m = /^([a-z0-9][a-z0-9_-]{0,63}):(\d{1,3})-(\d{1,3})$/.exec(layer);
|
|
312
|
+
const start = Number(m?.[2]);
|
|
313
|
+
const end = Number(m?.[3]);
|
|
314
|
+
if (!m || !(start < end) || end > 100) fail("INVALID_FLAG", `--layer "${layer}" must look like key:start-end with 0 <= start < end <= 100.`, "For example --layer checkout:0-50; another test in checkout:50-100 never shares a visitor.");
|
|
315
|
+
out.layer = { key: m[1], range: [start, end] };
|
|
316
|
+
}
|
|
317
|
+
return out;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
export {
|
|
321
|
+
create,
|
|
322
|
+
parseWeights,
|
|
323
|
+
parseLimits,
|
|
324
|
+
parseExtras
|
|
325
|
+
};
|
|
326
|
+
//# sourceMappingURL=chunk-E5VH4BQX.js.map
|