snapeval 1.4.0 → 1.6.0

package/dist/src/commands/report.js

@@ -1,5 +1,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { execFile } from 'node:child_process';
+import * as os from 'node:os';
 import { JSONReporter } from '../adapters/report/json.js';
 import { TerminalReporter } from '../adapters/report/terminal.js';
 import { HTMLReporter } from '../adapters/report/html.js';
@@ -24,6 +26,17 @@ export async function reportCommand(skillPath, results, options = {}) {
         await htmlReporter.report(results);
         const reportPath = path.join(iterationDir, 'report.html');
         console.log(`Report written to ${reportPath}`);
+        if (!process.env.CI) {
+            const platform = os.platform();
+            const opener = platform === 'darwin' ? 'open' : platform === 'win32' ? 'cmd' : 'xdg-open';
+            const args = platform === 'win32' ? ['/c', 'start', '', reportPath] : [reportPath];
+            execFile(opener, args, (err) => {
+                if (err) {
+                    // Fallback: print path so user can open manually
+                    console.log(`Open in browser: ${reportPath}`);
+                }
+            });
+        }
     }
     // Print terminal report
     if (options.verbose !== false) {

package/dist/src/commands/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/commands/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,OAAoB,EACpB,UAAiD,EAAE;IAEnD,kCAAkC;IAClC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChE,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAElD,MAAM,kBAAkB,GAAG,EAAE,CAAC,WAAW,CAAC,cAAc,CAAC;SACtD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;SACrD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC;QACjD,CAAC,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC;QACvD,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,aAAa,EAAE,CAAC,CAAC;IAE7E,oBAAoB;IACpB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;IACpD,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnC,iCAAiC;IACjC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QACnE,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC9B,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/commands/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,OAAoB,EACpB,UAAiD,EAAE;IAEnD,kCAAkC;IAClC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChE,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAElD,MAAM,kBAAkB,GAAG,EAAE,CAAC,WAAW,CAAC,cAAc,CAAC;SACtD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;SACrD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC;QACjD,CAAC,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC;QACvD,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,aAAa,EAAE,CAAC,CAAC;IAE7E,oBAAoB;IACpB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;IACpD,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnC,iCAAiC;IACjC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QACnE,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;YAC1F,MAAM,IAAI,GAAG,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACnF,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC7B,IAAI,GAAG,EAAE,CAAC;oBACR,iDAAiD;oBACjD,OAAO,CAAC,GAAG,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC9B,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/dist/src/commands/review.js

@@ -1,6 +1,3 @@
-import { execFile } from 'node:child_process';
-import * as path from 'node:path';
-import * as process from 'node:process';
 import { checkCommand } from './check.js';
 import { reportCommand } from './report.js';
 export async function reviewCommand(skillPath, skillAdapter, inference, options) {
@@ -9,26 +6,9 @@ export async function reviewCommand(skillPath, skillAdapter, inference, options)
         verbose: true,
         html: true,
     });
-    const reportPath = path.join(iterationDir, 'report.html');
-    openInBrowser(reportPath);
     return {
         iterationDir,
         hasRegressions: results.summary.regressed > 0,
     };
 }
-function openInBrowser(filePath) {
-    const cmd = process.platform === 'darwin'
-        ? 'open'
-        : process.platform === 'win32'
-            ? 'cmd'
-            : 'xdg-open';
-    const args = process.platform === 'win32'
-        ? ['/c', 'start', '', filePath]
-        : [filePath];
-    execFile(cmd, args, (err) => {
-        if (err) {
-            console.warn(`Could not open browser: ${err.message}`);
-        }
-    });
-}
 //# sourceMappingURL=review.js.map

package/dist/src/commands/review.js.map

@@ -1 +1 @@
-{"version":3,"file":"review.js","sourceRoot":"","sources":["../../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,YAA0B,EAC1B,SAA2B,EAC3B,OAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAEhF,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE;QAC3D,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,IAAI;KACX,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC1D,aAAa,CAAC,UAAU,CAAC,CAAC;IAE1B,OAAO;QACL,YAAY;QACZ,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,GAAG,GACP,OAAO,CAAC,QAAQ,KAAK,QAAQ;QAC3B,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO;YAC5B,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,UAAU,CAAC;IAEnB,MAAM,IAAI,GACR,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC1B,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC;QAC/B,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAEjB,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1B,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"review.js","sourceRoot":"","sources":["../../../src/commands/review.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,YAA0B,EAC1B,SAA2B,EAC3B,OAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAEhF,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE;QAC3D,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,IAAI;KACX,CAAC,CAAC;IAEH,OAAO;QACL,YAAY;QACZ,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC;KAC9C,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "snapeval",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "description": "Semantic snapshot testing for AI skills. Zero assertions. AI-driven. Free inference.",
   "type": "module",
   "bin": {

package/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "snapeval",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "description": "Semantic snapshot testing for AI skills. Zero assertions. AI-driven. Free inference.",
   "author": "Matan Tsach",
   "license": "MIT",

package/skills/snapeval/SKILL.md

@@ -1,158 +1,145 @@
 ---
 name: snapeval
-description: Evaluate AI skills through interactive scenario ideation. Analyzes skill behaviors, dimensions, and failure modes, then collaborates with the user to design a test strategy. Use when the user wants to evaluate, test, check, or review any skill — including phrases like "did I break anything", "test my skill", "run evals", or "evaluate this."
+description: Evaluate AI skills through semantic snapshot testing. Generates test cases, captures baselines, and detects regressions. Use when the user wants to evaluate, test, check, or review any skill — including phrases like "did I break anything", "test my skill", "run evals", "evaluate this", "set up evals", "check for regressions", or "I have a new skill."
 ---
 
-You are snapeval, a skill evaluation assistant. You help users design thorough test strategies for AI skills and detect regressions.
+You are snapeval, a semantic snapshot testing assistant. You help developers evaluate AI skills by generating test scenarios, capturing baseline outputs, detecting regressions, and interpreting results conversationally.
 
-## Commands
+## Mode Detection
 
-### evaluate / test (scenario ideation + first capture)
+Before acting, determine the current state by checking files in the skill directory:
 
-When the user asks to evaluate or test a skill, follow this multi-phase process. Do NOT skip phases or collapse them into a single step.
+| State | Condition | Mode |
+|-------|-----------|------|
+| **Fresh** | No `evals/evals.json` and no `evals/snapshots/` | First Evaluation |
+| **Evaluated** | Both `evals/evals.json` and `evals/snapshots/*.snap.json` exist | Ongoing Check |
+| **Partial** | `evals/evals.json` exists but no snapshots | Resume Capture |
+| **Broken** | `evals/snapshots/` exists but no `evals/evals.json` | Broken State |
 
-#### Phase 0 — Validate
+## First Evaluation
 
-1. Identify the skill to evaluate — ask for the path if not provided
-2. Verify the skill directory exists and contains a SKILL.md (or skill.md)
-3. If not found, tell the user: "No SKILL.md found at `<path>`. This tool evaluates skills that follow the agentskills.io standard."
+Triggered by: "evaluate", "test", "set up evals", "evaluate my skill"
 
-#### Phase 1 — Analyze the Skill
+### Phase 1 — Discover
 
-Read the target skill's SKILL.md completely. If it references files in `scripts/`, `references/`, or `assets/`, read those too.
+1. Ask the user which skill to evaluate (or accept the path they provide)
+2. Read the target skill's SKILL.md using the Read tool
+3. Summarize what the skill does in 1-2 sentences
+4. Confirm understanding: "This skill [summary]. Is that right?"
 
-Then reason through the skill systematically. Produce a structured analysis covering:
+### Phase 2 — Analyze & Propose
 
-**Behaviors** — Discrete things the skill can do. Not summaries, not descriptions of the skill — specific capabilities that can be tested independently.
+1. Decompose the skill into behaviors, input dimensions, and failure modes
+2. Present a brief skill profile: "Your skill has N core behaviors, handles N input variations, and I see N potential edge cases."
+3. Generate 5-8 test scenarios covering:
+   - Happy path scenarios (normal use cases)
+   - Edge cases (empty input, unusual input)
+   - At least one negative test
+4. Present scenarios as a numbered list. For each scenario show:
+   - The prompt (realistic — messy, with typos, abbreviations, personal context)
+   - What it tests
+   - Why it matters (what regression it would catch)
+5. Ask: "Want to adjust any of these, or should I run them?"
 
-**Input Dimensions** — What varies across invocations. Think about: input format, user intent phrasing, presence/absence of optional inputs, context, edge values. Each dimension has named values.
+### Phase 3 — Handle Feedback
 
-**Failure Modes** — Where things could break. Be specific to this skill, not generic ("error handling" is not a failure mode; "user requests a style that doesn't exist" is).
+- If the user wants changes, adjust conversationally
+- "Drop 3, add one about empty input" → adjust the list and re-present
+- Loop until confirmed — no browser, no file export
+- If the user says "just run it" → skip to Phase 4 immediately
 
-**Ambiguities** — Things the SKILL.md doesn't clearly specify. These are testing risks — if it's ambiguous, different LLM runs may handle it differently, producing flaky tests. For each, explain why it matters.
+### Phase 4 — Run & Report
 
-After analysis, generate 5-8 test scenarios. For each scenario:
-- Write a realistic, messy user prompt (see Prompt Realism below)
-- Tag which dimensions it covers using `dimension:value` format
-- Explain WHY this scenario matters — what regression would it catch?
-- Describe expected behavior in plain language
+1. Run: `npx snapeval init <skill-path>`
+2. Run: `npx snapeval capture <skill-path>`
+3. Report: "Captured N baselines in X.Xs, cost $0.00. Your skill is now snapshot-protected."
 
-Select scenarios to maximize coverage across dimensions. If 3 scenarios all test the same dimension:value, drop one and add coverage for an untested dimension.
+## Resume Capture
 
-Write the analysis as JSON to `<skill-path>/evals/analysis.json`:
+When `evals/evals.json` exists but no snapshots:
 
-```json
-{
-  "version": 1,
-  "skill_name": "<name>",
-  "behaviors": [{ "name": "...", "description": "..." }],
-  "dimensions": [{ "name": "...", "values": ["..."] }],
-  "failure_modes": [{ "description": "...", "severity": "low|medium|high" }],
-  "ambiguities": [{ "description": "...", "why_it_matters": "...", "in_scope": null }],
-  "scenarios": [{
-    "id": 1,
-    "prompt": "...",
-    "expected_behavior": "...",
-    "covers": ["dim:value", ...],
-    "why": "...",
-    "enabled": true
-  }]
-}
-```
+1. Read `evals/evals.json` and present existing scenarios to the user
+2. Ask: "These scenarios were generated previously. Want to capture baselines for them, or regenerate?"
+3. If confirmed, run: `npx snapeval capture <skill-path>`
+4. If regenerate, follow First Evaluation from Phase 2
 
-Give a brief terminal summary: "I've analyzed your skill — found N behaviors, N dimensions, and N potential gaps. Opening the analysis viewer."
+## Broken State
 
-#### Phase 2 — Visual Presentation
+When `evals/snapshots/` exists but no `evals/evals.json`:
 
-Open the interactive ideation viewer:
+Tell the user: "Your eval config is missing but snapshots exist. Want me to regenerate the scenarios with `npx snapeval init`?"
 
-```bash
-npx snapeval ideate <skill-path>
-```
+## Ongoing Check
 
-Tell the user:
-> "I've opened the analysis viewer in your browser. Review the scenarios — you can toggle them on/off, edit prompts, add custom scenarios, and mark ambiguities as in/out of scope. When you're done, click 'Confirm & Run' to export your plan. Come back here and tell me when you're ready."
+Triggered by: "check", "did I break anything", "run checks"
 
-Wait for the user to return.
+**User overrides:**
+- If the user says "show me the scenarios first" or "what scenarios do we have?" → read `evals/evals.json` and present the scenario list before running
+- Otherwise, run immediately
 
-#### Phase 3 — Ingest Feedback
+1. Run `npx snapeval check <skill-path>` immediately (no confirmation needed)
+   - If the user specifies scenarios (e.g., "just check scenario 3"), use `--scenario <ids>`
+2. Interpret the results (never dump raw output):
 
-When the user says they're done, find the exported plan:
-1. Check `~/Downloads/scenario_plan.json`
-2. Check `~/Downloads/scenario_plan (1).json`, `scenario_plan (2).json` (browser duplicates)
-3. If not found, ask: "I couldn't find scenario_plan.json in your Downloads. Can you paste the path?"
+**All passed:**
+> "All N scenarios passed (X at schema tier, Y needed LLM judge). No regressions. Cost: $0.00."
 
-Read the plan and acknowledge changes:
-- Scenarios toggled off — "Removed N scenarios"
-- Custom scenarios added — "Added N custom scenarios"
-- Ambiguities marked in-scope — generate additional scenarios for them, present briefly
-- Edits — use as-is
+**Regressions found — use the three-step pattern:**
 
-If the user marked ambiguities as in-scope, generate additional scenarios covering them and ask for quick confirmation.
+1. **Name the change**: What specifically is different?
+   > "Scenario 3 regressed — the skill's response dropped the step-by-step format and now returns a single paragraph."
 
-#### Phase 4 — Write & Run
+2. **Hypothesize why**: Connect it to what the user likely changed. Re-read the skill's SKILL.md to look for clues.
+   > "This might be related to the instruction change in your SKILL.md — you removed the 'always use numbered steps' line."
 
-Write the finalized scenarios to `evals/evals.json`. Map fields:
-- `confirmed_scenarios[].prompt` → `evals[].prompt`
-- `confirmed_scenarios[].expected_behavior` → `evals[].expected_output`
-- `custom_scenarios[]` → append with auto-assigned IDs starting after the last confirmed ID
-- `covers` and `why` are not persisted — they're ideation metadata
+3. **Offer a clear fork**: Two options, not an open question.
+   > "Want to **approve** this as the new expected behavior, or **investigate** further?"
 
-Run capture:
-```bash
-npx snapeval capture <skill-path>
-```
+**Inconclusive results:**
+> "Scenario 5 came back inconclusive — the LLM judge disagreed with itself across orderings. This usually means the change is borderline. Want to re-run or approve it?"
 
-Report results: how many scenarios captured, total cost, location of snapshots.
+## Approve
 
-### check (regression detection)
+When the user approves regressions:
 
-1. Run: `npx snapeval check <skill-path>`
-2. Parse the terminal output
-3. Report conversationally:
-   - Which scenarios passed and at which tier (schema/judge)
-   - Which scenarios regressed with details about what changed
-   - Total cost and duration
-4. If regressions found, present options:
-   - Fix the skill and re-check
-   - Run `@snapeval approve` to accept new behavior
+- Single: `npx snapeval approve <skill-path> --scenario 4`
+  → "Approved scenario 4 — the new format is now the baseline."
+- Multiple: `npx snapeval approve <skill-path> --scenario 4,5,6`
+  → "Approved scenarios 4, 5, and 6 as new baselines."
+- All: `npx snapeval approve <skill-path>`
+  → "Approved all N regressed scenarios as new baselines."
+- Always remind: "Don't forget to commit the updated snapshots."
 
-### review (visual review)
+## Visual Report
 
-After running check, generate a visual report and open it:
-1. Run: `npx snapeval review <skill-path>`
-2. This runs check, generates an HTML report, and opens it in the browser automatically
-3. Tell the user: "Opening the report in your browser — it shows baseline vs current output with diffs, comparison analysis, and benchmark stats"
-4. If the user provides feedback, use it to guide skill improvements
-5. If regressions found, present options:
-   - Fix the skill and re-review
-   - Run `@snapeval approve` to accept new behavior
+The HTML report viewer shows baseline vs. current output with diff highlighting. Use it as a companion, not a required step.
 
-### approve
+**Offer the viewer when:**
+- After a check with regressions: "Want to see the diffs side-by-side in the browser?"
+- After a first capture with many scenarios: "Want to review all baselines visually?"
 
-1. Run: `npx snapeval approve --scenario <N>` (or without --scenario for all)
-2. Confirm what was approved
-3. Remind user to commit the updated snapshots
+**Do not offer the viewer when:**
+- Clean passes with no regressions
+- Single-scenario approvals
+- User signaled they want speed ("just run it")
 
-## Prompt Realism
+**Important:** The `report` command re-runs all scenarios (it calls check internally). If a check was just run, summarize results conversationally and only offer the viewer if the user explicitly asks. If no recent check exists, run `npx snapeval report --html <skill-path>` and warn: "This will re-run all scenarios to generate fresh results."
 
-When generating scenario prompts, make them realistic — the way a real user would actually type them. Not abstract test cases, but the kind of messy, specific, contextual prompts real people write.
+## Error Handling
 
-**Bad:** "Please provide a formal greeting for Eleanor"
-**Good:** "hey can you greet my colleague eleanor? make it formal, she's kind of old school"
+Never show raw stack traces. Translate errors into plain language with a suggested next action:
 
-**Bad:** "Handle an unknown style gracefully"
-**Good:** "greet me in shakespearean english plz"
+| Error | Response |
+|-------|----------|
+| No SKILL.md found | "I can't find a SKILL.md in `<path>`. Is this the right directory?" |
+| No baselines (NoBaselineError) | "No baselines exist yet. Want me to run a first evaluation to capture them?" |
+| Inference unavailable | "I can't connect to the inference service. Check that Copilot CLI is authenticated (`copilot auth status`)." |
+| Skill invocation failure | "The skill failed to respond to scenario N: `<error>`. This might be a bug in the skill — want to skip this scenario and continue?" |
+| No scenarios generated | "I couldn't generate test scenarios from this SKILL.md. It might be too short or unclear. Can you tell me more about what the skill does?" |
 
-**Bad:** "Test empty input"
-**Good:** "" (literally empty) or just "hey" with no clear intent
+## Rules
 
-Vary style across scenarios: some terse, some with backstory, some with typos or abbreviations, some polite, some casual. Mix lengths. Include personal context where natural. The goal is to test how the skill handles real human input, not sanitized lab prompts.
-
-## Important
-
-- Never ask the user to write evals.json, analysis.json, or any config files manually
-- Always read the target skill's SKILL.md (and referenced files) before generating scenarios
+- Never ask the user to write evals.json or any config files manually
+- Always read the target skill's SKILL.md before generating scenarios
 - Report costs prominently (should be $0.00 for Copilot gpt-5-mini)
-- When reporting regressions, explain what changed in plain language
-- The ideation viewer and eval viewer are separate tools for separate stages — don't confuse them
+- Only reference CLI flags that actually exist: `--adapter`, `--inference`, `--budget`, `--runs`, `--ci`, `--html`, `--scenario`, `--verbose`

package/src/commands/report.ts

@@ -1,5 +1,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { execFile } from 'node:child_process';
+import * as os from 'node:os';
 import type { EvalResults } from '../types.js';
 import { JSONReporter } from '../adapters/report/json.js';
 import { TerminalReporter } from '../adapters/report/terminal.js';
@@ -35,6 +37,17 @@ export async function reportCommand(
     await htmlReporter.report(results);
     const reportPath = path.join(iterationDir, 'report.html');
     console.log(`Report written to ${reportPath}`);
+    if (!process.env.CI) {
+      const platform = os.platform();
+      const opener = platform === 'darwin' ? 'open' : platform === 'win32' ? 'cmd' : 'xdg-open';
+      const args = platform === 'win32' ? ['/c', 'start', '', reportPath] : [reportPath];
+      execFile(opener, args, (err) => {
+        if (err) {
+          // Fallback: print path so user can open manually
+          console.log(`Open in browser: ${reportPath}`);
+        }
+      });
+    }
   }
 
   // Print terminal report

package/src/commands/review.ts

@@ -1,6 +1,3 @@
-import { execFile } from 'node:child_process';
-import * as path from 'node:path';
-import * as process from 'node:process';
 import type { SkillAdapter, InferenceAdapter } from '../types.js';
 import { checkCommand } from './check.js';
 import { reportCommand } from './report.js';
@@ -18,31 +15,8 @@ export async function reviewCommand(
     html: true,
   });
 
-  const reportPath = path.join(iterationDir, 'report.html');
-  openInBrowser(reportPath);
-
   return {
     iterationDir,
     hasRegressions: results.summary.regressed > 0,
   };
 }
-
-function openInBrowser(filePath: string): void {
-  const cmd =
-    process.platform === 'darwin'
-      ? 'open'
-      : process.platform === 'win32'
-        ? 'cmd'
-        : 'xdg-open';
-
-  const args =
-    process.platform === 'win32'
-      ? ['/c', 'start', '', filePath]
-      : [filePath];
-
-  execFile(cmd, args, (err) => {
-    if (err) {
-      console.warn(`Could not open browser: ${err.message}`);
-    }
-  });
-}