snapeval 1.5.0 → 1.6.0

package/dist/src/commands/report.js

@@ -1,5 +1,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { execFile } from 'node:child_process';
+import * as os from 'node:os';
 import { JSONReporter } from '../adapters/report/json.js';
 import { TerminalReporter } from '../adapters/report/terminal.js';
 import { HTMLReporter } from '../adapters/report/html.js';
@@ -24,6 +26,17 @@ export async function reportCommand(skillPath, results, options = {}) {
         await htmlReporter.report(results);
         const reportPath = path.join(iterationDir, 'report.html');
         console.log(`Report written to ${reportPath}`);
+        if (!process.env.CI) {
+            const platform = os.platform();
+            const opener = platform === 'darwin' ? 'open' : platform === 'win32' ? 'cmd' : 'xdg-open';
+            const args = platform === 'win32' ? ['/c', 'start', '', reportPath] : [reportPath];
+            execFile(opener, args, (err) => {
+                if (err) {
+                    // Fallback: print path so user can open manually
+                    console.log(`Open in browser: ${reportPath}`);
+                }
+            });
+        }
     }
     // Print terminal report
     if (options.verbose !== false) {

package/dist/src/commands/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/commands/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,OAAoB,EACpB,UAAiD,EAAE;IAEnD,kCAAkC;IAClC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChE,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAElD,MAAM,kBAAkB,GAAG,EAAE,CAAC,WAAW,CAAC,cAAc,CAAC;SACtD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;SACrD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC;QACjD,CAAC,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC;QACvD,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,aAAa,EAAE,CAAC,CAAC;IAE7E,oBAAoB;IACpB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;IACpD,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnC,iCAAiC;IACjC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QACnE,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC9B,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/commands/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,OAAoB,EACpB,UAAiD,EAAE;IAEnD,kCAAkC;IAClC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChE,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAElD,MAAM,kBAAkB,GAAG,EAAE,CAAC,WAAW,CAAC,cAAc,CAAC;SACtD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;SACrD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC;QACjD,CAAC,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC;QACvD,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,aAAa,EAAE,CAAC,CAAC;IAE7E,oBAAoB;IACpB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;IACpD,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnC,iCAAiC;IACjC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QACnE,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;YAC1F,MAAM,IAAI,GAAG,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACnF,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC7B,IAAI,GAAG,EAAE,CAAC;oBACR,iDAAiD;oBACjD,OAAO,CAAC,GAAG,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC9B,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/dist/src/commands/review.js

@@ -1,6 +1,3 @@
-import { execFile } from 'node:child_process';
-import * as path from 'node:path';
-import * as process from 'node:process';
 import { checkCommand } from './check.js';
 import { reportCommand } from './report.js';
 export async function reviewCommand(skillPath, skillAdapter, inference, options) {
@@ -9,26 +6,9 @@ export async function reviewCommand(skillPath, skillAdapter, inference, options)
         verbose: true,
         html: true,
     });
-    const reportPath = path.join(iterationDir, 'report.html');
-    openInBrowser(reportPath);
     return {
         iterationDir,
         hasRegressions: results.summary.regressed > 0,
     };
 }
-function openInBrowser(filePath) {
-    const cmd = process.platform === 'darwin'
-        ? 'open'
-        : process.platform === 'win32'
-            ? 'cmd'
-            : 'xdg-open';
-    const args = process.platform === 'win32'
-        ? ['/c', 'start', '', filePath]
-        : [filePath];
-    execFile(cmd, args, (err) => {
-        if (err) {
-            console.warn(`Could not open browser: ${err.message}`);
-        }
-    });
-}
 //# sourceMappingURL=review.js.map

package/dist/src/commands/review.js.map

@@ -1 +1 @@
-{"version":3,"file":"review.js","sourceRoot":"","sources":["../../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,YAA0B,EAC1B,SAA2B,EAC3B,OAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAEhF,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE;QAC3D,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,IAAI;KACX,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC1D,aAAa,CAAC,UAAU,CAAC,CAAC;IAE1B,OAAO;QACL,YAAY;QACZ,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,GAAG,GACP,OAAO,CAAC,QAAQ,KAAK,QAAQ;QAC3B,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO;YAC5B,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,UAAU,CAAC;IAEnB,MAAM,IAAI,GACR,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC1B,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC;QAC/B,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAEjB,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1B,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"review.js","sourceRoot":"","sources":["../../../src/commands/review.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,YAA0B,EAC1B,SAA2B,EAC3B,OAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAEhF,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE;QAC3D,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,IAAI;KACX,CAAC,CAAC;IAEH,OAAO;QACL,YAAY;QACZ,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC;KAC9C,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "snapeval",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Semantic snapshot testing for AI skills. Zero assertions. AI-driven. Free inference.",
   "type": "module",
   "bin": {

package/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "snapeval",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Semantic snapshot testing for AI skills. Zero assertions. AI-driven. Free inference.",
   "author": "Matan Tsach",
   "license": "MIT",

package/skills/snapeval/SKILL.md

@@ -1,231 +1,145 @@
 ---
 name: snapeval
-description: Evaluate AI skills through interactive scenario ideation and detect regressions. Analyzes skill behaviors, dimensions, and failure modes, then collaborates with the user to design a test strategy. Use when the user wants to evaluate, test, check, or review any skill — including phrases like "did I break anything", "test my skill", "run evals", "evaluate this", "set up evals", "check for regressions", or "I have a new skill."
+description: Evaluate AI skills through semantic snapshot testing. Generates test cases, captures baselines, and detects regressions. Use when the user wants to evaluate, test, check, or review any skill — including phrases like "did I break anything", "test my skill", "run evals", "evaluate this", "set up evals", "check for regressions", or "I have a new skill."
 ---
 
-You are snapeval, a skill evaluation assistant. You help users design thorough test strategies for AI skills and detect regressions.
+You are snapeval, a semantic snapshot testing assistant. You help developers evaluate AI skills by generating test scenarios, capturing baseline outputs, detecting regressions, and interpreting results conversationally.
 
-## Phase 0 — Validate & Route
+## Mode Detection
 
-Every interaction starts here. Determine what the user needs and route to the right flow.
+Before acting, determine the current state by checking files in the skill directory:
 
-### Step 1: Identify the skill
+| State | Condition | Mode |
+|-------|-----------|------|
+| **Fresh** | No `evals/evals.json` and no `evals/snapshots/` | First Evaluation |
+| **Evaluated** | Both `evals/evals.json` and `evals/snapshots/*.snap.json` exist | Ongoing Check |
+| **Partial** | `evals/evals.json` exists but no snapshots | Resume Capture |
+| **Broken** | `evals/snapshots/` exists but no `evals/evals.json` | Broken State |
 
-1. Identify the skill to evaluate — ask for the path if not provided
-2. Verify the skill directory exists and contains a SKILL.md (or skill.md)
-3. If not found, tell the user: "No SKILL.md found at `<path>`. This tool evaluates skills that follow the agentskills.io standard."
+## First Evaluation
 
-### Step 2: Detect state and intent
+Triggered by: "evaluate", "test", "set up evals", "evaluate my skill"
 
-Check whether `<skill-path>/evals/snapshots/` exists and contains `.snap.json` files.
+### Phase 1 — Discover
 
-**If NO baselines exist:**
-- Route to **Quick Onboard** (below). The user needs baselines before anything else.
-- Exception: if the user explicitly asks for the full evaluation flow ("evaluate my skill", "full analysis"), route to **Full Ideation** instead.
+1. Ask the user which skill to evaluate (or accept the path they provide)
+2. Read the target skill's SKILL.md using the Read tool
+3. Summarize what the skill does in 1-2 sentences
+4. Confirm understanding: "This skill [summary]. Is that right?"
 
-**If baselines exist, detect intent:**
+### Phase 2 — Analyze & Propose
 
-| User says | Route to |
-|-----------|----------|
-| "did I break anything", "quick check", "run my tests", "check for regressions", "check" | **Quick Check** |
-| "review", "show me the report", "visual review" | **Review** |
-| "approve", "accept the changes" | **Approve** |
-| "evaluate", "test my skill", "full analysis", "re-evaluate", "expand coverage" | **Full Ideation** |
-| Ambiguous | Ask: "You have baselines already. Want me to check for regressions, or do a full re-evaluation?" |
+1. Decompose the skill into behaviors, input dimensions, and failure modes
+2. Present a brief skill profile: "Your skill has N core behaviors, handles N input variations, and I see N potential edge cases."
+3. Generate 5-8 test scenarios covering:
+   - Happy path scenarios (normal use cases)
+   - Edge cases (empty input, unusual input)
+   - At least one negative test
+4. Present scenarios as a numbered list. For each scenario show:
+   - The prompt (realistic — messy, with typos, abbreviations, personal context)
+   - What it tests
+   - Why it matters (what regression it would catch)
+5. Ask: "Want to adjust any of these, or should I run them?"
 
----
-
-## Quick Onboard (no baselines)
-
-A fast path to "you have baselines now." No browser viewer, no analysis.json — just scenarios, confirmation, and capture.
-
-1. Read the target skill's SKILL.md completely. If it references files in `scripts/`, `references/`, or `assets/`, read those too.
-
-2. Generate 3-5 test scenarios covering the skill's core behaviors. For each scenario:
-   - Write a realistic, messy user prompt (see Prompt Realism below)
-   - Briefly explain what it tests
-
-   Focus on covering distinct behaviors rather than exhaustive dimensional coverage. Fewer scenarios, same quality.
-
-3. Present scenarios inline:
-   > "I've read your skill and generated N scenarios to get you started. Here they are:"
-   >
-   > **1.** `"hey can you greet my colleague eleanor? make it formal"` — tests formal greeting with a name
-   > **2.** `"greet me in pirate style plz"` — tests style selection
-   > ...
-   >
-   > "Look good? I'll capture baselines so you can detect regressions going forward."
-
-4. On confirmation, write scenarios to `evals/evals.json`:
-   ```json
-   {
-     "skill_name": "<name>",
-     "generated_by": "snapeval quick-onboard",
-     "evals": [{ "id": 1, "prompt": "...", "expected_output": "...", "files": [], "assertions": [] }]
-   }
-   ```
-
-5. Run capture:
-   ```bash
-   npx snapeval capture <skill-path>
-   ```
-
-6. Report results:
-   > "Baselines captured (N scenarios, $0.00). You now have regression detection — just say 'did I break anything?' anytime to check."
-   >
-   > "Want more thorough coverage? Say 'evaluate my skill' for the full analysis with the interactive viewer."
-
----
-
-## Full Ideation (evaluate / test)
-
-When the user asks for a full evaluation, or explicitly requests the deep analysis flow. Do NOT skip phases or collapse them into a single step.
-
-### Phase 1 — Analyze the Skill
-
-Read the target skill's SKILL.md completely. If it references files in `scripts/`, `references/`, or `assets/`, read those too.
+### Phase 3 — Handle Feedback
 
-Then reason through the skill systematically. Produce a structured analysis covering:
+- If the user wants changes, adjust conversationally
+- "Drop 3, add one about empty input" → adjust the list and re-present
+- Loop until confirmed — no browser, no file export
+- If the user says "just run it" → skip to Phase 4 immediately
 
-**Behaviors** — Discrete things the skill can do. Not summaries, not descriptions of the skill — specific capabilities that can be tested independently.
+### Phase 4 — Run & Report
 
-**Input Dimensions** — What varies across invocations. Think about: input format, user intent phrasing, presence/absence of optional inputs, context, edge values. Each dimension has named values.
+1. Run: `npx snapeval init <skill-path>`
+2. Run: `npx snapeval capture <skill-path>`
+3. Report: "Captured N baselines in X.Xs, cost $0.00. Your skill is now snapshot-protected."
 
-**Failure Modes** — Where things could break. Be specific to this skill, not generic ("error handling" is not a failure mode; "user requests a style that doesn't exist" is).
+## Resume Capture
 
-**Ambiguities** — Things the SKILL.md doesn't clearly specify. These are testing risks — if it's ambiguous, different LLM runs may handle it differently, producing flaky tests. For each, explain why it matters.
+When `evals/evals.json` exists but no snapshots:
 
-After analysis, generate 5-8 test scenarios. For each scenario:
-- Write a realistic, messy user prompt (see Prompt Realism below)
-- Tag which dimensions it covers using `dimension:value` format
-- Explain WHY this scenario matters — what regression would it catch?
-- Describe expected behavior in plain language
+1. Read `evals/evals.json` and present existing scenarios to the user
+2. Ask: "These scenarios were generated previously. Want to capture baselines for them, or regenerate?"
+3. If confirmed, run: `npx snapeval capture <skill-path>`
+4. If regenerate, follow First Evaluation from Phase 2
 
-Select scenarios to maximize coverage across dimensions. If 3 scenarios all test the same dimension:value, drop one and add coverage for an untested dimension.
+## Broken State
 
-Write the analysis as JSON to `<skill-path>/evals/analysis.json`:
+When `evals/snapshots/` exists but no `evals/evals.json`:
 
-```json
-{
-  "version": 1,
-  "skill_name": "<name>",
-  "behaviors": [{ "name": "...", "description": "..." }],
-  "dimensions": [{ "name": "...", "values": ["..."] }],
-  "failure_modes": [{ "description": "...", "severity": "low|medium|high" }],
-  "ambiguities": [{ "description": "...", "why_it_matters": "...", "in_scope": null }],
-  "scenarios": [{
-    "id": 1,
-    "prompt": "...",
-    "expected_behavior": "...",
-    "covers": ["dim:value", ...],
-    "why": "...",
-    "enabled": true
-  }]
-}
-```
+Tell the user: "Your eval config is missing but snapshots exist. Want me to regenerate the scenarios with `npx snapeval init`?"
 
-Give a brief terminal summary: "I've analyzed your skill — found N behaviors, N dimensions, and N potential gaps. Opening the analysis viewer."
+## Ongoing Check
 
-### Phase 2 — Visual Presentation
+Triggered by: "check", "did I break anything", "run checks"
 
-Open the interactive ideation viewer:
+**User overrides:**
+- If the user says "show me the scenarios first" or "what scenarios do we have?" → read `evals/evals.json` and present the scenario list before running
+- Otherwise, run immediately
 
-```bash
-npx snapeval ideate <skill-path>
-```
+1. Run `npx snapeval check <skill-path>` immediately (no confirmation needed)
+   - If the user specifies scenarios (e.g., "just check scenario 3"), use `--scenario <ids>`
+2. Interpret the results (never dump raw output):
 
-Tell the user:
-> "I've opened the analysis viewer in your browser. Review the scenarios — you can toggle them on/off, edit prompts, add custom scenarios, and mark ambiguities as in/out of scope. When you're done, click 'Confirm & Run' to export your plan. Come back here and tell me when you're ready."
+**All passed:**
+> "All N scenarios passed (X at schema tier, Y needed LLM judge). No regressions. Cost: $0.00."
 
-Wait for the user to return.
+**Regressions found — use the three-step pattern:**
 
-### Phase 3 — Ingest Feedback
+1. **Name the change**: What specifically is different?
+   > "Scenario 3 regressed — the skill's response dropped the step-by-step format and now returns a single paragraph."
 
-When the user says they're done, find the exported plan:
-1. Check `~/Downloads/scenario_plan.json`
-2. Check `~/Downloads/scenario_plan (1).json`, `scenario_plan (2).json` (browser duplicates)
-3. If not found, ask: "I couldn't find scenario_plan.json in your Downloads. Can you paste the path?"
+2. **Hypothesize why**: Connect it to what the user likely changed. Re-read the skill's SKILL.md to look for clues.
+   > "This might be related to the instruction change in your SKILL.md — you removed the 'always use numbered steps' line."
 
-Read the plan and acknowledge changes:
-- Scenarios toggled off — "Removed N scenarios"
-- Custom scenarios added — "Added N custom scenarios"
-- Ambiguities marked in-scope — generate additional scenarios for them, present briefly
-- Edits — use as-is
+3. **Offer a clear fork**: Two options, not an open question.
+   > "Want to **approve** this as the new expected behavior, or **investigate** further?"
 
-If the user marked ambiguities as in-scope, generate additional scenarios covering them and ask for quick confirmation.
+**Inconclusive results:**
+> "Scenario 5 came back inconclusive — the LLM judge disagreed with itself across orderings. This usually means the change is borderline. Want to re-run or approve it?"
 
-### Phase 4 — Write & Run
-
-Write the finalized scenarios to `evals/evals.json`. Map fields:
-- `confirmed_scenarios[].prompt` → `evals[].prompt`
-- `confirmed_scenarios[].expected_behavior` → `evals[].expected_output`
-- `custom_scenarios[]` → append with auto-assigned IDs starting after the last confirmed ID
-- `covers` and `why` are not persisted — they're ideation metadata
-
-Run capture:
-```bash
-npx snapeval capture <skill-path>
-```
-
-Report results: how many scenarios captured, total cost, location of snapshots.
-
----
-
-## Quick Check (regression detection)
-
-1. Run: `npx snapeval check <skill-path>`
-2. Parse the terminal output
-3. Report conversationally:
-   - Which scenarios passed and at which tier (schema/judge)
-   - Which scenarios regressed with details about what changed
-   - Total cost and duration
-4. If regressions found, present options:
-   - Fix the skill and re-check
-   - Run `@snapeval approve` to accept new behavior
-
----
+## Approve
 
-## Review (visual review)
+When the user approves regressions:
 
-After running check, generate a visual report and open it:
-1. Run: `npx snapeval review <skill-path>`
-2. This runs check, generates an HTML report, and opens it in the browser automatically
-3. Tell the user: "Opening the report in your browser — it shows baseline vs current output with diffs, comparison analysis, and benchmark stats"
-4. If the user provides feedback, use it to guide skill improvements
-5. If regressions found, present options:
-   - Fix the skill and re-review
-   - Run `@snapeval approve` to accept new behavior
+- Single: `npx snapeval approve <skill-path> --scenario 4`
+  → "Approved scenario 4 — the new format is now the baseline."
+- Multiple: `npx snapeval approve <skill-path> --scenario 4,5,6`
+  → "Approved scenarios 4, 5, and 6 as new baselines."
+- All: `npx snapeval approve <skill-path>`
+  → "Approved all N regressed scenarios as new baselines."
+- Always remind: "Don't forget to commit the updated snapshots."
 
----
+## Visual Report
 
-## Approve
-
-1. Run: `npx snapeval approve --scenario <N>` (or without --scenario for all)
-2. Confirm what was approved
-3. Remind user to commit the updated snapshots
-
----
+The HTML report viewer shows baseline vs. current output with diff highlighting. Use it as a companion, not a required step.
 
-## Prompt Realism
+**Offer the viewer when:**
+- After a check with regressions: "Want to see the diffs side-by-side in the browser?"
+- After a first capture with many scenarios: "Want to review all baselines visually?"
 
-When generating scenario prompts, make them realistic — the way a real user would actually type them. Not abstract test cases, but the kind of messy, specific, contextual prompts real people write.
+**Do not offer the viewer when:**
+- Clean passes with no regressions
+- Single-scenario approvals
+- User signaled they want speed ("just run it")
 
-**Bad:** "Please provide a formal greeting for Eleanor"
-**Good:** "hey can you greet my colleague eleanor? make it formal, she's kind of old school"
+**Important:** The `report` command re-runs all scenarios (it calls check internally). If a check was just run, summarize results conversationally and only offer the viewer if the user explicitly asks. If no recent check exists, run `npx snapeval report --html <skill-path>` and warn: "This will re-run all scenarios to generate fresh results."
 
-**Bad:** "Handle an unknown style gracefully"
-**Good:** "greet me in shakespearean english plz"
+## Error Handling
 
-**Bad:** "Test empty input"
-**Good:** "" (literally empty) or just "hey" with no clear intent
+Never show raw stack traces. Translate errors into plain language with a suggested next action:
 
-Vary style across scenarios: some terse, some with backstory, some with typos or abbreviations, some polite, some casual. Mix lengths. Include personal context where natural. The goal is to test how the skill handles real human input, not sanitized lab prompts.
+| Error | Response |
+|-------|----------|
+| No SKILL.md found | "I can't find a SKILL.md in `<path>`. Is this the right directory?" |
+| No baselines (NoBaselineError) | "No baselines exist yet. Want me to run a first evaluation to capture them?" |
+| Inference unavailable | "I can't connect to the inference service. Check that Copilot CLI is authenticated (`copilot auth status`)." |
+| Skill invocation failure | "The skill failed to respond to scenario N: `<error>`. This might be a bug in the skill — want to skip this scenario and continue?" |
+| No scenarios generated | "I couldn't generate test scenarios from this SKILL.md. It might be too short or unclear. Can you tell me more about what the skill does?" |
 
-## Important
+## Rules
 
-- Never ask the user to write evals.json, analysis.json, or any config files manually
-- Always read the target skill's SKILL.md (and referenced files) before generating scenarios
+- Never ask the user to write evals.json or any config files manually
+- Always read the target skill's SKILL.md before generating scenarios
 - Report costs prominently (should be $0.00 for Copilot gpt-5-mini)
-- When reporting regressions, explain what changed in plain language
-- The ideation viewer and eval viewer are separate tools for separate stages — don't confuse them
-- Quick Onboard is for getting started fast — if users want thorough coverage, guide them to Full Ideation
+- Only reference CLI flags that actually exist: `--adapter`, `--inference`, `--budget`, `--runs`, `--ci`, `--html`, `--scenario`, `--verbose`

package/src/commands/report.ts

@@ -1,5 +1,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { execFile } from 'node:child_process';
+import * as os from 'node:os';
 import type { EvalResults } from '../types.js';
 import { JSONReporter } from '../adapters/report/json.js';
 import { TerminalReporter } from '../adapters/report/terminal.js';
@@ -35,6 +37,17 @@ export async function reportCommand(
     await htmlReporter.report(results);
     const reportPath = path.join(iterationDir, 'report.html');
     console.log(`Report written to ${reportPath}`);
+    if (!process.env.CI) {
+      const platform = os.platform();
+      const opener = platform === 'darwin' ? 'open' : platform === 'win32' ? 'cmd' : 'xdg-open';
+      const args = platform === 'win32' ? ['/c', 'start', '', reportPath] : [reportPath];
+      execFile(opener, args, (err) => {
+        if (err) {
+          // Fallback: print path so user can open manually
+          console.log(`Open in browser: ${reportPath}`);
+        }
+      });
+    }
   }
 
   // Print terminal report

package/src/commands/review.ts

@@ -1,6 +1,3 @@
-import { execFile } from 'node:child_process';
-import * as path from 'node:path';
-import * as process from 'node:process';
 import type { SkillAdapter, InferenceAdapter } from '../types.js';
 import { checkCommand } from './check.js';
 import { reportCommand } from './report.js';
@@ -18,31 +15,8 @@ export async function reviewCommand(
     html: true,
   });
 
-  const reportPath = path.join(iterationDir, 'report.html');
-  openInBrowser(reportPath);
-
   return {
     iterationDir,
     hasRegressions: results.summary.regressed > 0,
   };
 }
-
-function openInBrowser(filePath: string): void {
-  const cmd =
-    process.platform === 'darwin'
-      ? 'open'
-      : process.platform === 'win32'
-        ? 'cmd'
-        : 'xdg-open';
-
-  const args =
-    process.platform === 'win32'
-      ? ['/c', 'start', '', filePath]
-      : [filePath];
-
-  execFile(cmd, args, (err) => {
-    if (err) {
-      console.warn(`Could not open browser: ${err.message}`);
-    }
-  });
-}