swe-workflow-skills 0.1.0 → 0.3.0

package/roles.json

@@ -14,11 +14,13 @@
   "core": {
     "universal": [
       "skill-router",
+      "brainstorming",
       "feature-planning"
     ],
     "technical": [
       "skill-router",
       "feature-planning",
+      "plan-execution",
       "git-workflow",
       "code-reviewing",
       "verification-before-completion",
@@ -28,7 +30,8 @@
   },
 
   "meta_only": [
-    "writing-skills"
+    "writing-skills",
+    "subagent-orchestration"
   ],
 
   "roles": {
@@ -62,6 +65,7 @@
         "api-design",
         "tdd-workflow",
         "test-suite-design",
+        "browser-verification",
         "performance-optimization",
         "refactoring"
       ]
@@ -73,6 +77,7 @@
       "skills": [
         "containerization",
         "cicd-pipeline",
+        "release-management",
         "infrastructure-as-code",
         "deployment-repo",
         "gitops-delivery",
@@ -80,29 +85,70 @@
         "rollback-strategy",
         "incident-response",
         "observability-design",
-        "configuration-strategy",
-        "dependency-impact-analysis"
+        "resilience-engineering",
+        "configuration-strategy"
       ]
     },
     "ml": {
       "label": "ML Engineer / MLOps",
-      "description": "Skills for ML pipelines, experiment tracking, model deployment, and the data/observability around them.",
+      "description": "Skills for ML pipelines, experiment tracking, evaluation, model deployment, and the data/observability around them.",
       "core": "technical",
       "skills": [
         "ml-pipeline-design",
         "ml-experiment-tracking",
         "ml-model-deployment",
+        "ai-evaluation",
+        "data-modeling",
+        "test-data-strategy",
+        "observability-design"
+      ]
+    },
+    "ai": {
+      "label": "AI Engineer",
+      "description": "Skills for LLM-powered applications: prompt/RAG/agent design, evaluation, experiment tracking, model deployment, and the API around them.",
+      "core": "technical",
+      "skills": [
+        "llm-app-engineering",
+        "ai-evaluation",
+        "ml-experiment-tracking",
+        "ml-model-deployment",
+        "api-design"
+      ]
+    },
+    "data": {
+      "label": "Data Engineer",
+      "description": "Skills for data pipelines, analytics engineering, data quality/contracts, schema design, test data, and pipeline observability.",
+      "core": "technical",
+      "skills": [
+        "data-pipeline-design",
+        "data-quality",
         "data-modeling",
         "test-data-strategy",
         "observability-design"
       ]
     },
+    "data-scientist": {
+      "label": "Data Scientist",
+      "description": "Skills for exploratory data analysis, statistical rigor, notebook productionization, and the experiment-tracking/data-quality/modeling work around them.",
+      "core": "technical",
+      "skills": [
+        "exploratory-data-analysis",
+        "statistical-analysis",
+        "notebook-to-production",
+        "ml-experiment-tracking",
+        "data-quality",
+        "data-modeling",
+        "ai-evaluation"
+      ]
+    },
     "security": {
       "label": "Security Engineer",
-      "description": "Skills for security audits, dependency/CVE management, blast-radius analysis, and incident handling.",
+      "description": "Skills for design-time threat modeling, security audits, dependency/CVE management, blast-radius analysis, and incident handling.",
       "core": "technical",
       "skills": [
         "security-audit",
+        "threat-modeling",
+        "compliance-privacy",
         "dependency-management",
         "dependency-impact-analysis",
         "incident-response",
@@ -119,6 +165,7 @@
         "architecture-documentation",
         "api-design",
         "data-modeling",
+        "code-archaeology",
         "dependency-impact-analysis",
         "technical-debt-review",
         "project-review",
@@ -140,7 +187,9 @@
         "project-review",
         "strategic-review",
         "code-reviewing",
-        "incident-response"
+        "incident-response",
+        "release-management",
+        "dx-audit"
       ]
     },
     "pm": {
@@ -158,12 +207,14 @@
     },
     "strategy": {
       "label": "Strategy / Founder Review",
-      "description": "Skills for pre-public and strategic reviews: vision and positioning, market analysis, execution-vs-roadmap, and go/no-go synthesis.",
+      "description": "Skills for pre-public and strategic reviews: vision and positioning, market analysis, build-vs-buy decisions, execution-vs-roadmap, and go/no-go synthesis.",
       "core": "universal",
       "skills": [
         "strategic-review",
         "project-review",
         "project-proposal",
+        "build-vs-buy",
+        "finops-cost-optimization",
         "metrics-and-okrs"
       ]
     },
@@ -175,10 +226,24 @@
         "tdd-workflow",
         "test-suite-design",
         "test-data-strategy",
+        "browser-verification",
         "code-reviewing",
         "bug-investigating"
       ]
     },
+    "mobile": {
+      "label": "Mobile Engineer",
+      "description": "Skills for native and cross-platform mobile apps: architecture, offline/sync, UI/UX, accessibility, performance, API contracts, and app-store releases.",
+      "core": "technical",
+      "skills": [
+        "mobile-architecture",
+        "mobile-release",
+        "ui-ux-design",
+        "accessibility-design",
+        "performance-optimization",
+        "api-design"
+      ]
+    },
     "designer": {
       "label": "Designer / UX",
       "description": "Skills for user flows, accessibility, product requirements, and the frontend architecture that realizes them.",

package/scripts/resolve.mjs

@@ -104,10 +104,15 @@ function sortKeysDeep(value) {
 
 export function loadSettings(settingsPath) {
   if (existsSync(settingsPath) && statSync(settingsPath).isFile()) {
+    const text = readFileSync(settingsPath, "utf-8");
     try {
-      return JSON.parse(readFileSync(settingsPath, "utf-8")) || {};
-    } catch {
-      return {};
+      return JSON.parse(text) || {};
+    } catch (e) {
+      // A corrupt (e.g. hand-edited) file must NOT read as empty: downstream
+      // writers would rebuild it from {} and silently destroy the user's
+      // settings. Throw so callers skip the write (the SessionStart hook
+      // catches this and leaves the file alone; the CLI surfaces it).
+      throw new Error(`unparseable JSON in ${settingsPath} — fix or remove it (${e.message})`);
     }
   }
   return {};

package/skills/ai-evaluation/SKILL.md

@@ -0,0 +1,112 @@
+---
+name: ai-evaluation
+description: "Define how to evaluate ML models and GenAI/LLM apps — golden datasets, offline metrics, RAG evaluation (faithfulness, relevance, context precision/recall) with ragas/deepeval/promptfoo, LLM-as-judge design and calibration, eval regression gates in CI, online A/B and human feedback. Logging/comparing runs → ml-experiment-tracking; building the app → llm-app-engineering; general experiment statistics → statistical-analysis."
+when_to_use: "Triggers: evaluate the model, eval my chatbot, RAG evals, ragas, LLM as judge, golden dataset, eval suite, is the new prompt better, hallucination rate, benchmark our AI app."
+model: sonnet
+allowed-tools: Read, Grep, Glob, Write, Edit, Bash
+---
+
+# AI Evaluation
+
+Define **what to measure and how** for ML models and GenAI apps. Without an
+evaluation harness, every prompt tweak, retrieval change, or model swap ships on
+vibes — and quality silently regresses. The boundary with its neighbors:
+`ml-experiment-tracking` *records and compares* runs; this skill *defines the
+metrics, datasets, and judges* those runs report. `llm-app-engineering` designs
+the app; this skill proves whether a change to it helped.
+
+## Workflow
+
+### Step 1: Pin Down What "Good" Means
+
+Before choosing tools, write down 3–5 quality criteria in the product's terms
+(answers grounded in our docs, no fabricated order numbers, resolves the ticket).
+Vague goals ("high quality answers") make every later step unmeasurable. Decide
+the primary metric — the one that decides ship/no-ship — before seeing results,
+to avoid cherry-picking.
+
+### Step 2: Build a Golden Dataset
+
+Evaluate against a fixed, versioned dataset — not ad-hoc spot checks:
+
+- **Source**: real user questions (logs, support tickets) first; synthesize from
+  the corpus only to fill gaps, and have a human review synthetic cases.
+- **Size**: 30–50 cases is enough to start; grow with every production failure
+  (each becomes a regression case — same principle as a bug's regression test).
+- **Coverage**: include the unhappy paths — questions the corpus can't answer
+  (the correct behavior is refusal), ambiguous phrasing, adversarial inputs.
+- **Version it** alongside the code; a metric is only comparable across runs on
+  the same dataset version.
+
+With zero labeled data, this is still not optional — bootstrap: label a few dozen
+cases by hand (an afternoon), use reference-free metrics (groundedness against
+the source, rubric-based judging) while the labeled set grows.
+
+### Step 3: Choose Metrics by System Type
+
+- **Classical ML models**: standard offline metrics vs held-out data (accuracy,
+  F1, RMSE, AUC), sliced by segment — aggregate numbers hide subgroup failures.
+- **RAG systems** — score the stages separately, or you can't tell whether a bad
+  answer is a retrieval or a generation failure:
+  - *Retrieval*: context precision / context recall (did the right chunks come back?)
+  - *Generation*: faithfulness/groundedness (is every claim supported by the
+    retrieved context?) and answer relevance (does it address the question?)
+- **Agents**: end-to-end task success rate plus trajectory checks (right tool,
+  no loops, steps within budget).
+- **Reference-free** (no ground truth): groundedness against the source,
+  rubric-based judge scores, and pairwise A-vs-B comparison between variants.
+
+Pick **one** eval tool and wire it in — ragas (RAG metrics), deepeval
+(pytest-style, broad metric set), or promptfoo (config-driven prompt/model
+comparison). Selection detail: [references/eval-tooling.md](references/eval-tooling.md).
+
+### Step 4: Design the Judge Deliberately
+
+LLM-as-judge is the workhorse for GenAI evals, and an uncalibrated judge is a
+random-number generator with authority. Treat the judge as a component you
+design and validate:
+
+- **Rubric, not vibes**: decompose "good" into the Step-1 criteria, each scored
+  separately with a binary or 3-point scale — never a bare 1–10 overall score.
+- **Known biases, known counters**: position bias (swap A/B order and re-judge),
+  self-preference (judge with a different model family than the one being
+  judged), verbosity bias (instruct the judge to ignore length; cap output length).
+- **Reduce noise**: majority-of-k voting for important comparisons.
+- **Calibrate**: score 20–30 cases with both the judge and a human; if agreement
+  is poor, fix the rubric before trusting any judge-derived number.
+
+### Step 5: Gate Changes in CI
+
+An eval suite that runs once before launch rots immediately. Run it on every
+change to prompts, retrieval, or model version — like a test suite:
+
+- Gate on **regression vs the last accepted baseline** (metric dropped), not an
+  absolute threshold — absolute judge scores drift; movement is the signal.
+- Keep the CI subset small and fast (the golden set, k=1); run the full suite
+  nightly or pre-release with voting.
+- Log eval runs in the experiment tracker (`ml-experiment-tracking`) so results
+  are comparable over time.
+
+### Step 6: Close the Loop Online
+
+Offline evals predict; production confirms. Once users exist: lightweight
+feedback (thumbs up/down + optional reason), A/B test significant changes on the
+primary metric, and mine failures — every confirmed production failure becomes a
+new golden-set case (Step 2).
+
+## Principles Applied
+
+- **YAGNI**: 30 golden cases + one tool + one judge rubric beats a
+  six-metric dashboard nobody trusts. Add metrics when a decision needs them.
+- **KISS**: prefer binary per-criterion judgments over fine-grained scales —
+  humans and judges both agree more on yes/no.
+- **Measure movement, not absolutes**: judge scores are comparable only within
+  the same dataset + rubric + judge version; gate on deltas.
+
+## Cross-Skill References
+
+- `llm-app-engineering` — designing the system this skill measures
+- `ml-experiment-tracking` — recording and comparing the eval runs over time
+- `ml-model-deployment` — online monitoring and drift detection in production
+- `cicd-pipeline` — wiring the eval gate into CI
+- `metrics-and-okrs` — tying eval metrics to product-level success metrics

package/skills/ai-evaluation/evals/evals.json

@@ -0,0 +1,40 @@
+{
+  "skill_name": "ai-evaluation",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "We've built a RAG chatbot over our internal docs and we're planning to launch in two weeks. How should we evaluate it before launch? Someone on the team mentioned ragas.",
+      "expected_output": "Designs an evaluation plan: a golden dataset grounded in real user questions, the RAG triad (faithfulness, answer relevance, context precision/recall) separating retrieval from generation metrics, a concrete tool choice (ragas/deepeval/promptfoo) with rationale, LLM-as-judge reliability measures, and the eval suite wired in as a regression gate rather than a one-off check",
+      "assertions": [
+        "Proposes building a golden/evaluation dataset grounded in real user questions or the docs, rather than evaluating ad hoc",
+        "Covers the RAG triad or equivalent — faithfulness/groundedness, answer relevance, and retrieval/context quality — and distinguishes retrieval metrics from generation metrics",
+        "Recommends a concrete evaluation tool (ragas, deepeval, or promptfoo) and explains its role",
+        "Addresses LLM-as-judge reliability with at least one concrete measure: rubric design, position/self-preference bias, majority voting, or calibrating the judge against human spot-checks",
+        "Proposes running the evals as a repeatable regression gate (in CI or before each prompt/pipeline change), not only as a one-off pre-launch check"
+      ]
+    },
+    {
+      "id": 2,
+      "prompt": "Our LLM feature summarizes long sales calls. We have no ground-truth summaries and no users yet. How do we evaluate quality with zero labeled data?",
+      "expected_output": "Does not treat missing labels as a blocker: proposes reference-free evaluation (rubric-based LLM-as-judge, groundedness against the source call, pairwise comparison between variants), bootstraps a small human-labeled golden set, calibrates the judge against human spot-checks, and plans online signals (feedback, A/B) once users exist",
+      "assertions": [
+        "Does not claim evaluation is impossible without labels — proposes reference-free approaches such as rubric-based LLM-as-judge, groundedness against the source transcript, or pairwise comparison between variants",
+        "Recommends bootstrapping a small human-labeled golden set (even tens of examples) and explains what it buys",
+        "Designs the judge with an explicit rubric or criteria decomposition rather than a bare overall score",
+        "Mentions checking the judge against human spot-checks (calibration/agreement) or using voting to reduce judge noise",
+        "Plans for online signals after launch (user feedback, A/B testing) as a complement to offline evals"
+      ]
+    },
+    {
+      "id": 3,
+      "prompt": "Set up MLflow for our fine-tuning experiments so we can log runs and compare hyperparameters across the team.",
+      "expected_output": "Recognizes this as experiment-tracking infrastructure, not evaluation design — hands off to ml-experiment-tracking for the MLflow setup and run comparison, noting the boundary: tracking records and compares runs, while ai-evaluation defines what and how to measure",
+      "assertions": [
+        "Recognizes the request is about experiment-tracking/run-comparison infrastructure, not about defining evaluation metrics or datasets",
+        "Refers to the ml-experiment-tracking skill for the MLflow setup and run comparison",
+        "Does not produce an evaluation-design workflow (golden datasets, judges, RAG metrics) as the answer to a tracking-infrastructure request",
+        "States the boundary: tracking records and compares runs, while evaluation design defines what and how to measure"
+      ]
+    }
+  ]
+}

package/skills/ai-evaluation/references/eval-tooling.md

@@ -0,0 +1,63 @@
+# Evaluation Tooling — Selection and Judge-Design Detail
+
+Tool landscapes move fast; verify against each project's current docs before
+committing. What follows is the decision logic, which is stable.
+
+## Choosing the eval tool
+
+Pick **one** primary tool. They overlap heavily; running two means maintaining
+two golden-set formats and two CI integrations for no added signal.
+
+| Tool | Shape | Choose when |
+|---|---|---|
+| **ragas** | Python library of RAG-specific metrics (faithfulness, answer relevancy, context precision/recall, noise sensitivity) | The system is RAG and you want the standard names for the RAG triad with minimal ceremony. Integrates with LangChain/LlamaIndex datasets. |
+| **deepeval** | pytest-style framework — metrics as assertions, `assert_test(test_case, [metric])` | The team already lives in pytest and wants evals in the same runner and CI report as unit tests. Broad metric set (RAG triad, hallucination, bias, custom G-Eval rubrics). |
+| **promptfoo** | Config-driven (YAML) prompt/model comparison matrix + web viewer, CLI-first | The main activity is comparing prompt variants or models side-by-side; also does red-teaming. Good for non-Python stacks (Node CLI). |
+| **Roll your own** | A loop + a judge prompt + a JSON of cases | The metrics are product-specific rubrics anyway and a framework would only wrap your judge calls. Fine — the golden set and calibration discipline matter more than the harness. |
+
+Platform suites (LangSmith, Langfuse, Braintrust, W&B Weave) bundle eval +
+tracing + datasets; prefer them when the team already uses that platform for
+observability — the dataset/annotation UI is the real value.
+
+## RAG metric cheat sheet
+
+| Failure you saw | Stage | Metric to add |
+|---|---|---|
+| Answer cites facts not in the docs | Generation | Faithfulness / groundedness |
+| Answer is on-topic-ish but doesn't answer | Generation | Answer relevance |
+| Right doc exists but wasn't retrieved | Retrieval | Context recall |
+| Retrieved chunks are mostly padding | Retrieval | Context precision |
+| Correct refusal expected, model answered anyway | Both | Unanswerable-set accuracy |
+
+Debug order: retrieval metrics first. Generation metrics computed over bad
+context are noise — fix recall before tuning prompts.
+
+## LLM-as-judge bias catalog
+
+| Bias | Symptom | Counter |
+|---|---|---|
+| Position | Prefers whichever answer is shown first | Randomize/swap order, judge twice, keep only consistent verdicts |
+| Self-preference | Rates its own model family's style higher | Judge with a different model family than the generator |
+| Verbosity | Longer answer wins regardless of substance | Instruct explicitly; normalize or cap lengths; per-criterion rubric |
+| Sycophancy toward the prompt | Scores what the question implies is right | Hide metadata (which variant is "new"); blind the judge |
+| Scale drift | 7/10 today ≠ 7/10 next month | Binary/3-point criteria; compare deltas on the same rubric version |
+
+## Judge calibration procedure
+
+1. Sample 20–30 golden cases; have a human score them with the same rubric.
+2. Run the judge on the same cases (k=3 voting).
+3. Compute simple agreement (percent match per criterion). Below ~80%: the
+   rubric is ambiguous — tighten criteria wording, add anchored examples
+   (a passing and a failing answer per criterion), re-run.
+4. Re-calibrate whenever the rubric, judge model, or task distribution changes.
+
+## Online evaluation notes
+
+- **A/B tests**: randomize per user (not per request) for conversational
+  products; pick the primary metric before launch; expect judge-metric ↔
+  user-metric divergence and investigate it — it usually means the rubric is
+  measuring the wrong thing.
+- **Human feedback**: thumbs up/down rates are biased (angry users click more);
+  use as a trend signal and a failure-mining source, not an absolute score.
+- **Failure mining**: a weekly pass over flagged conversations feeding the
+  golden set is the highest-ROI eval activity that exists.

package/skills/architecture-design/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: architecture-design
-description: "Guide architectural and structural decisions using Architecture Decision Records (ADRs) — new services, pattern choices, database selection, component boundaries, state management, costly-to-reverse decisions. Triggers: how should I architect, which pattern, design decision, should I use X or Y, system design, ADR, component design, architectural trade-off."
+description: "Guide architectural and structural decisions using Architecture Decision Records (ADRs) — new services, pattern choices, database selection, component boundaries, state management, costly-to-reverse decisions. Triggers: how should I architect, which pattern, design decision, should I use X or Y, system design, ADR, component design, architectural trade-off. Owns internal-structure decisions — build vs buy/adopt a vendor or OSS capability → build-vs-buy."
 model: opus
 allowed-tools: Read, Grep, Glob, Write, Edit
 ---
@@ -78,6 +78,8 @@ ADRs are immutable records. If a decision is superseded, create a new ADR that r
 
 For visual architecture documentation (system diagrams, runtime flows, infrastructure topology), see the `architecture-documentation` skill.
 
+When the decision is whether a capability should be built at all versus bought or adopted (vendor/SaaS/OSS, TCO, lock-in, exit costs), use `build-vs-buy` — it produces the same ADR-style record, but for sourcing rather than structure.
+
 ## When to Split Decisions
 
 If the analysis reveals multiple independent decisions bundled together, split them into separate ADRs. For example, "How should we handle caching?" might split into:

package/skills/brainstorming/SKILL.md

@@ -0,0 +1,92 @@
+---
+name: brainstorming
+description: "Facilitate divergent ideation before committing to a spec — Socratic questioning, widening the option space, challenging assumptions and inherited constraints, then converging on candidates. Triggers: brainstorm, let's explore ideas, what are our options, help me think through, generate ideas, what could we build, not sure what I want, explore approaches, ideate. Once ideas converge, hand off to prd-writing (WHAT/WHY) or feature-planning (breakdown) — this skill opens options; those close them."
+model: sonnet
+allowed-tools: Read, Grep, Glob, Write, Edit
+---
+
+# Brainstorming
+
+Divergent exploration *before* convergent specification. The most expensive
+failure in product work is efficiently building the wrong thing — this skill
+exists to widen the option space and test the framing before anything gets
+specced. It sits upstream of `prd-writing` and `feature-planning`: they converge
+on one answer; this skill makes sure the right answers were on the table first.
+
+The mode is **Socratic**: the user's tacit knowledge of their domain beats any
+generated list. Ask questions that surface it; generate to provoke, not to
+replace their thinking.
+
+## Workflow
+
+### Step 1: Frame the Real Problem
+
+Before generating anything, ask: what problem, for whom, why now, and what would
+"solved" look like? Then challenge the framing — the stated problem is often a
+solution in disguise ("we need a dashboard" usually means "someone can't answer
+a question"). Ask "what does that get you?" until you hit the underlying need.
+If the user arrives with a solution already chosen, brainstorm the *problem
+behind it* before brainstorming within it.
+
+### Step 2: Separate Real Constraints from Inherited Ones
+
+List the constraints, then interrogate each: is it physics/budget/law (real), or
+habit/assumption/"we've always done it this way" (inherited)? Explicitly mark
+which are negotiable. Most breakthrough options live behind a constraint that
+turned out to be assumed.
+
+### Step 3: Diverge — Quantity First, Judgment Deferred
+
+Generate broadly, and do not evaluate mid-generation — criticism during
+divergence kills the unusual options that make the exercise worthwhile. Mix
+techniques rather than exhausting one:
+
+- **Vary the axes**: who else could this serve, what adjacent problem does it
+  solve, how else could it be delivered, when/where else does it apply?
+- **Invert**: how would we guarantee this fails? Each failure mode implies an option.
+- **Analogize**: how do other domains solve the structural equivalent?
+- **Recombine**: force-pair two existing ideas or systems.
+
+Alternate generating with asking — offer a batch, then ask a question that
+mines the user's context for the next batch.
+
+### Step 4: Widen Once More (the Wildcards)
+
+Before converging, deliberately add the options nobody proposes on their own:
+the **do-nothing / process-change** option (no build at all), the **1/10-scope**
+version (what's the smallest thing that dents the problem?), the **10x**
+version (what if this were the company's main bet?), and the
+**buy-instead-of-build** option (delegate that comparison to `build-vs-buy` if
+it becomes a contender).
+
+### Step 5: Converge with Explicit Criteria
+
+Cluster related ideas, then name the selection criteria *before* scoring
+(typical set: impact, effort, reversibility, confidence) — criteria chosen after
+seeing favorites just rationalize them. Score roughly, pick 1–3 candidates, and
+keep the rest in a **parking lot** with a one-line reason each — parked, not
+deleted; today's reject is next quarter's answer.
+
+### Step 6: Hand Off
+
+Write a one-paragraph brief per surviving candidate (problem, idea, why it won,
+open questions) and route it onward: `prd-writing` for requirements,
+`feature-planning` for task breakdown, `project-proposal` for a business case,
+`architecture-design` if the open questions are structural.
+
+## Principles Applied
+
+- **Defer judgment during divergence; be ruthless during convergence** — mixing
+  the modes does both badly.
+- **Ask > tell**: a question that unlocks the user's knowledge outperforms ten
+  generated ideas.
+- **YAGNI applies at convergence, not divergence** — generating a wild option
+  costs a line; building it is what needs justification.
+
+## Cross-Skill References
+
+- `prd-writing` — turn the winning idea into WHAT/WHY requirements
+- `feature-planning` — break the chosen idea into scoped tasks
+- `project-proposal` — wrap a candidate in a business case for a go/no-go
+- `build-vs-buy` — when a candidate could be bought or adopted instead of built
+- `architecture-design` — when the surviving questions are structural

package/skills/brainstorming/evals/evals.json

@@ -0,0 +1,40 @@
+{
+  "skill_name": "brainstorming",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "We want to improve developer onboarding at our company — new hires take about two months to ship their first meaningful change. I'm not sure where to start. Can we brainstorm?",
+      "expected_output": "Runs a Socratic divergent session: probes the problem first (where the two months actually go, for whom, what 'onboarded' means), separates real from inherited constraints, generates a broad and varied option set without evaluating mid-stream, deliberately includes wildcards (do-nothing/process-change, tiny-scope, buy-instead-of-build), then converges with named criteria and hands the 1–3 survivors to prd-writing or feature-planning with a parking lot for the rest",
+      "assertions": [
+        "Asks clarifying/Socratic questions about the underlying problem (where the time goes, for whom, what success looks like) before or while generating ideas, rather than immediately listing solutions",
+        "Generates a genuinely diverse option set spanning different kinds of interventions (e.g. docs, tooling, process, people/mentoring), not variations of a single idea",
+        "Does not evaluate, score, or eliminate ideas during the divergent generation phase",
+        "Includes at least one wildcard option such as a no-build/process change, a radically smaller-scope version, or buying/adopting instead of building",
+        "Converges using explicitly named selection criteria (e.g. impact, effort, reversibility) and identifies 1–3 candidates plus a parking lot, with a handoff toward prd-writing, feature-planning, or project-proposal"
+      ]
+    },
+    {
+      "id": 2,
+      "prompt": "I've decided we need to build a custom internal chat app for our 40-person company. Let's brainstorm features for it.",
+      "expected_output": "Widens before narrowing: probes the problem behind the already-chosen solution (what's failing with current communication, why existing tools don't fit), surfaces that 'build a custom chat app' is a solution not a problem, raises alternatives including buying/adopting existing tools (routing a serious buy-vs-build comparison to build-vs-buy), distinguishes real from assumed constraints — and still delivers the requested feature ideation rather than refusing or lecturing",
+      "assertions": [
+        "Probes the problem behind the chosen solution (what communication need is unmet, why existing tools fall short) instead of only listing features",
+        "Surfaces at least one alternative to building a custom chat app, such as adopting or configuring an existing tool, and mentions build-vs-buy for that comparison",
+        "Questions or tests the constraints/assumptions behind the decision (e.g. why custom, why now) and distinguishes real constraints from inherited ones",
+        "Still provides the requested divergent feature ideation for the chat app rather than refusing to engage with the user's framing",
+        "Does not converge on a final feature list without naming selection criteria first"
+      ]
+    },
+    {
+      "id": 3,
+      "prompt": "We've settled on building a CLI tool that syncs our design tokens to code. Break it into tasks with acceptance criteria and dependencies so the team can start next sprint.",
+      "expected_output": "Recognizes that ideation is complete and the user is asking for convergent task breakdown — refers to feature-planning for the scoped tasks, acceptance criteria, and dependency mapping rather than reopening divergent exploration",
+      "assertions": [
+        "Recognizes the request as convergent planning of an already-chosen idea, not ideation",
+        "Refers to the feature-planning skill for the task breakdown, acceptance criteria, and dependencies",
+        "Does not restart divergent idea generation or question the settled decision as the main response",
+        "States or implies the boundary: brainstorming opens the option space before a decision; feature-planning structures the work after it"
+      ]
+    }
+  ]
+}

package/skills/browser-verification/SKILL.md

@@ -0,0 +1,95 @@
+---
+name: browser-verification
+description: "Verify a web app change by driving the real UI in a browser — reproduce the user flow end to end, watch console errors and network failures, exercise loading/error/empty states, capture screenshots as evidence. Use before claiming a UI change works. Triggers: check it in the browser, verify the UI, click through the flow, console errors, test it like a user, smoke test the frontend. Designing persistent test suites → test-suite-design; the generic done-gate → verification-before-completion (this is its browser-specific practice)."
+model: sonnet
+allowed-tools: Read, Grep, Glob, Write, Edit, Bash
+---
+
+# Browser Verification
+
+Prove a web change works by *being the user*, not by reading the diff. Passing
+unit tests and a clean build say the code is consistent with itself — they say
+nothing about the page actually rendering, the handler actually firing, or the
+API call actually succeeding with real wiring. This skill is the
+browser-specific practice of `verification-before-completion`: the claim "the
+UI works" requires having driven the UI, in this session, with the evidence
+captured. Boundary: `test-suite-design` builds *persistent* automated suites;
+this skill is the verification pass you run on the change in front of you
+(what it finds often becomes a suite case afterward).
+
+## Workflow
+
+### Step 1: Define the Flow That Proves the Change
+
+Name the user-visible behavior the change promises ("submitting the form shows
+the success toast and the new record appears in the list") and the shortest
+real path to it. That flow — not "the page loads" — is what verification means.
+Include what should *stop* happening for bug fixes (the old broken behavior,
+reproduced first if possible).
+
+### Step 2: Run the Real App in a Realistic State
+
+Launch the actual stack the change ships in (dev server + backend + seeded
+data), not a mental model of it. State matters: an empty database hides list
+rendering bugs; a logged-out session hides authz redirects. Seed what the flow
+needs — including a user in the right role.
+
+### Step 3: Drive the Flow Like a User
+
+Automate the drive with a browser tool (Playwright or the environment's
+browser automation) or click it manually — but *interact*: navigate, type into
+the fields, submit, wait for the response, observe the outcome. Prefer
+selectors a user would recognize (roles, labels, text) over brittle internals.
+Loading a URL and seeing HTTP 200 is not driving the flow — most UI bugs live
+after first paint.
+
+### Step 4: Watch All Three Evidence Channels
+
+The page can *look* right while failing. During the drive, check:
+
+- **Console** — errors and warnings that fired during the flow (a caught-and-
+  swallowed error today is the un-reproducible bug report next week).
+- **Network** — the requests the flow triggered: right endpoints, right
+  payloads, right status codes; no 4xx/5xx, no CORS failures, no retry loops.
+- **UI state** — the visible outcome: content correct, no layout collapse, no
+  infinite spinner masquerading as "loading".
+
+### Step 5: Exercise Beyond the Happy Path
+
+The states users hit that developers don't: **error** (kill the backend or
+force a 500 — does the UI degrade or white-screen?), **empty** (zero items),
+**loading** (throttle and look — flicker? layout shift?), **repeat actions**
+(double-submit, back button, refresh mid-flow), and at least one non-desktop
+viewport if the change touches layout. Scope to what the change plausibly
+affects — a copy tweak doesn't need the full matrix; a checkout-flow change
+does. (`ui-ux-design` owns designing these states; this skill verifies them.)
+
+### Step 6: Capture Evidence and Report Precisely
+
+Screenshot the key outcomes (success state, plus anything broken), keep the
+console/network findings, and report in evidence terms: "drove signup →
+dashboard with a fresh user; form validates, POST /users returns 201, toast
+renders, zero console errors — screenshot attached" beats "the UI works".
+Anything found but not fixed is reported, not silently absorbed. If the flow
+proved valuable to verify, feed it to `test-suite-design` as a candidate e2e
+case so next time it verifies itself.
+
+## Principles Applied
+
+- **The rendered page is the truth**: code review and unit tests approximate
+  it; only the browser confirms it.
+- **Interaction over inspection**: most UI bugs are reachable only by doing
+  what a user does, in order.
+- **Evidence, not impressions**: a screenshot with zero console errors is a
+  verification; "looked fine" is a mood.
+
+## Cross-Skill References
+
+- `verification-before-completion` — the generic evidence gate this skill
+  instantiates for web UIs
+- `test-suite-design` — turning one-off verification flows into persistent e2e coverage
+- `ui-ux-design` — designing the loading/error/empty states verified here
+- `accessibility-design` — the deeper a11y pass when the change touches
+  interaction patterns
+- `bug-investigating` — when verification finds something broken and the cause
+  isn't obvious