skilldrop-cli 0.16.29 → 0.16.31

package/packs/dev-team/skills/rollout-plan/scripts/canary_math.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+"""How long each exposure step of a canary or flag rollout must run before its
+numbers can detect the regression you care about.
+
+Uses the normal approximation to the two-proportion z-test. The flag-on cohort
+(treatment) is compared with a concurrent flag-off cohort (control). By default
+the control is everyone not exposed (100% - exposure); pass --control-pct to fix
+it at a holdback size instead.
+
+For each step it prints the units (requests or users) each arm needs and the
+hours that takes at the given traffic, flags steps that cannot reach that in
+--max-hours, says the smallest regression the step *can* detect in that window,
+and suggests the smallest exposure that reaches sufficiency in the window.
+
+Rates and regressions accept a fraction (0.008), a percent ("0.8%") or, for an
+absolute regression, percentage points ("0.3pp").
+
+Usage:
+    python3 canary_math.py --baseline 0.8% --detect-abs 0.3pp --traffic 9000
+    python3 canary_math.py --baseline 3.1% --detect-rel 10% --direction down \\
+        --traffic 9000 --steps 1,5,25,50 --unit users
+    python3 canary_math.py ... --json
+
+Exit codes: 0 printed, 2 bad input.
+"""
+
+import argparse
+import json
+import math
+import sys
+from statistics import NormalDist
+
+Z = NormalDist()
+
+
+def fail(msg):
+    sys.stderr.write("error: " + msg + "\n")
+    sys.exit(2)
+
+
+def parse_rate(text, name, allow_pp=False):
+    s = str(text).strip().lower().replace(" ", "")
+    try:
+        if s.endswith("pp"):
+            if not allow_pp:
+                fail(name + ": 'pp' only applies to --detect-abs")
+            return float(s[:-2]) / 100.0
+        if s.endswith("%"):
+            return float(s[:-1]) / 100.0
+        v = float(s)
+    except ValueError:
+        fail(name + ": not a number: " + str(text))
+    return v
+
+
+def fmt_pct(x, digits=2):
+    return ("{:." + str(digits) + "f}").format(x * 100).rstrip("0").rstrip(".") + "%"
+
+
+def fmt_pp(x):
+    return "{:.3f}".format(x * 100).rstrip("0").rstrip(".") + " pp"
+
+
+def fmt_hours(h):
+    if h is None:
+        return "n/a"
+    if h < 1:
+        return "{:.0f} min".format(max(1, math.ceil(h * 60)))
+    if h < 48:
+        return "{:.1f} h".format(h)
+    return "{:.1f} h ({:.1f} d)".format(h, h / 24)
+
+
+def z_values(alpha, power, one_sided):
+    za = Z.inv_cdf(1 - alpha) if one_sided else Z.inv_cdf(1 - alpha / 2)
+    zb = Z.inv_cdf(power)
+    return za, zb
+
+
+def equal_arm_n(p1, p2, za, zb):
+    """Units per arm for two equal arms."""
+    pbar = (p1 + p2) / 2
+    num = za * math.sqrt(2 * pbar * (1 - pbar)) + zb * math.sqrt(p1 * (1 - p1) + p2 * (1 - p2))
+    return math.ceil((num / (p2 - p1)) ** 2)
+
+
+def hours_needed(p1, p2, e, c, traffic, za, zb):
+    """Hours until a treatment arm of share e and control of share c is powered.
+
+    n1 = traffic*e*t (treatment, rate p2), n2 = traffic*c*t (control, rate p1).
+    Both arms scale with t, so the power equation solves in closed form.
+    """
+    if e <= 0 or c <= 0:
+        return None
+    pbar = (e * p2 + c * p1) / (e + c)
+    a = za * math.sqrt(pbar * (1 - pbar) * (1 / e + 1 / c))
+    b = zb * math.sqrt(p2 * (1 - p2) / e + p1 * (1 - p1) / c)
+    return ((a + b) / abs(p2 - p1)) ** 2 / traffic
+
+
+def detectable_in(p1, direction, e, c, traffic, za, zb, hours):
+    """Smallest absolute regression detectable in `hours` (bisection)."""
+    if e <= 0 or c <= 0:
+        return None
+    hi = (1 - p1) if direction == "up" else p1
+    hi *= 0.999
+    lo = 1e-9
+    if hours_needed(p1, p1 + hi if direction == "up" else p1 - hi, e, c, traffic, za, zb) > hours:
+        return None
+    for _ in range(80):
+        mid = (lo + hi) / 2
+        p2 = p1 + mid if direction == "up" else p1 - mid
+        if hours_needed(p1, p2, e, c, traffic, za, zb) <= hours:
+            hi = mid
+        else:
+            lo = mid
+    return hi
+
+
+def smallest_sufficient_exposure(p1, p2, traffic, za, zb, max_hours, control_pct):
+    """Smallest exposure (0.1% steps, up to 50%) that is powered within max_hours."""
+    for tenth in range(1, 501):
+        e = tenth / 1000.0
+        c = control_pct if control_pct is not None else 1 - e
+        if e + c > 1 + 1e-9:
+            break
+        h = hours_needed(p1, p2, e, c, traffic, za, zb)
+        if h is not None and h <= max_hours:
+            return e, h
+    return None, None
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Hours each canary/flag exposure step must run to detect a regression.")
+    ap.add_argument("--baseline", required=True, help="baseline rate of the metric in control, e.g. 0.8%% or 0.008")
+    g = ap.add_mutually_exclusive_group(required=True)
+    g.add_argument("--detect-abs", help="smallest regression worth catching, absolute: 0.3pp or 0.003")
+    g.add_argument("--detect-rel", help="smallest regression worth catching, relative to baseline: 25%% or 0.25")
+    ap.add_argument("--traffic", required=True, type=float, help="units per hour across all users (requests or users)")
+    ap.add_argument("--unit", default="requests", help="what --traffic counts; use the unit the flag randomises on (default: requests)")
+    ap.add_argument("--steps", default="1,5,25,50,100", help="exposure percentages, comma-separated (default: 1,5,25,50,100)")
+    ap.add_argument("--control-pct", help="fixed control (holdback) size in percent; default: everyone not exposed")
+    ap.add_argument("--direction", choices=["up", "down"], default="up",
+                    help="up = a rise is the regression (error rate, default); down = a fall is (conversion)")
+    ap.add_argument("--alpha", type=float, default=0.05, help="false-alarm rate (default 0.05)")
+    ap.add_argument("--power", type=float, default=0.8, help="chance of catching a real regression (default 0.8)")
+    ap.add_argument("--one-sided", action="store_true", help="one-sided test (only the regression direction); default two-sided")
+    ap.add_argument("--max-hours", type=float, default=72.0, help="longest sensible step (default 72)")
+    ap.add_argument("--json", action="store_true", help="print JSON instead of a markdown table")
+    args = ap.parse_args()
+
+    p1 = parse_rate(args.baseline, "--baseline")
+    if not 0 < p1 < 1:
+        fail("--baseline must be between 0 and 100%")
+    if args.detect_abs is not None:
+        d = parse_rate(args.detect_abs, "--detect-abs", allow_pp=True)
+    else:
+        d = parse_rate(args.detect_rel, "--detect-rel") * p1
+    if d <= 0:
+        fail("the regression to detect must be greater than zero")
+    p2 = p1 + d if args.direction == "up" else p1 - d
+    if not 0 < p2 < 1:
+        fail("baseline plus the regression falls outside 0-100%")
+    if args.traffic <= 0:
+        fail("--traffic must be greater than zero")
+    if not 0 < args.alpha < 1 or not 0 < args.power < 1:
+        fail("--alpha and --power must be between 0 and 1")
+    if args.max_hours <= 0:
+        fail("--max-hours must be greater than zero")
+    control = None
+    if args.control_pct is not None:
+        control = parse_rate(args.control_pct.rstrip("%") + "%", "--control-pct")
+        if not 0 < control < 1:
+            fail("--control-pct must be between 0 and 100")
+    try:
+        steps = [float(s.strip().rstrip("%")) / 100.0 for s in args.steps.split(",") if s.strip()]
+    except ValueError:
+        fail("--steps must be comma-separated percentages, e.g. 1,5,25,50,100")
+    if not steps or any(not 0 < s <= 1 for s in steps):
+        fail("--steps must be percentages between 0 and 100")
+
+    za, zb = z_values(args.alpha, args.power, args.one_sided)
+    n_equal = equal_arm_n(p1, p2, za, zb)
+
+    rows = []
+    for e in steps:
+        c = control if control is not None else 1 - e
+        row = {"exposure_pct": round(e * 100, 4), "control_pct": round(c * 100, 4)}
+        if e + c > 1 + 1e-9:
+            row.update(status="invalid", note="exposure plus control exceeds 100%")
+        elif c <= 1e-9:
+            row.update(status="no-control", hours=None,
+                       note="no concurrent control at 100%: promote on the previous step's comparison, "
+                            "or keep a holdback (--control-pct) to keep comparing")
+        else:
+            h = hours_needed(p1, p2, e, c, args.traffic, za, zb)
+            row["hours"] = round(h, 3)
+            row["treatment_units"] = math.ceil(h * args.traffic * e)
+            row["control_units"] = math.ceil(h * args.traffic * c)
+            if h <= args.max_hours:
+                row["status"] = "sufficient"
+            else:
+                row["status"] = "insufficient"
+                mde = detectable_in(p1, args.direction, e, c, args.traffic, za, zb, args.max_hours)
+                row["detectable_in_max_hours_abs"] = None if mde is None else round(mde, 6)
+        rows.append(row)
+
+    best_e, best_h = smallest_sufficient_exposure(p1, p2, args.traffic, za, zb, args.max_hours, control)
+
+    result = {
+        "baseline": p1, "regression_abs": round(d, 6), "regressed_rate": round(p2, 6),
+        "direction": args.direction, "alpha": args.alpha, "power": args.power,
+        "test": "one-sided" if args.one_sided else "two-sided",
+        "traffic_per_hour": args.traffic, "unit": args.unit, "max_hours": args.max_hours,
+        "equal_arm_units_per_arm": n_equal, "steps": rows,
+        "smallest_sufficient_exposure_pct": None if best_e is None else round(best_e * 100, 1),
+        "smallest_sufficient_exposure_hours": None if best_h is None else round(best_h, 2),
+    }
+    if args.json:
+        print(json.dumps(result, indent=2))
+        return
+
+    u = args.unit
+    print("# Canary sufficiency")
+    print()
+    print("- Metric: baseline {} -> regression to catch {} ({} rate {}), {} test, alpha {}, power {}".format(
+        fmt_pct(p1, 3), fmt_pp(d), "rising to" if args.direction == "up" else "falling to",
+        fmt_pct(p2, 3), result["test"], args.alpha, args.power))
+    print("- Traffic: {:,.0f} {} per hour; control = {}".format(
+        args.traffic, u, "fixed {} holdback".format(fmt_pct(control)) if control is not None else "everyone not exposed"))
+    print("- Equal arms would need {:,} {} per arm".format(n_equal, u))
+    print()
+    print("| Exposure | Control | Treatment {u} needed | Control {u} needed | Minimum time | Status |".format(u=u))
+    print("|---|---|---|---|---|---|")
+    for r in rows:
+        exp = fmt_pct(r["exposure_pct"] / 100)
+        ctl = fmt_pct(r["control_pct"] / 100)
+        if r["status"] in ("no-control", "invalid"):
+            print("| {} | {} | - | - | n/a | {} |".format(exp, ctl, r["note"]))
+            continue
+        status = "ok" if r["status"] == "sufficient" else "INSUFFICIENT: over {:g} h".format(args.max_hours)
+        if r["status"] == "insufficient":
+            m = r.get("detectable_in_max_hours_abs")
+            status += "; in {:g} h it can only catch {}".format(
+                args.max_hours, "nothing measurable" if m is None else ">= " + fmt_pp(m))
+        print("| {} | {} | {:,} | {:,} | {} | {} |".format(
+            exp, ctl, r["treatment_units"], r["control_units"], fmt_hours(r["hours"]), status))
+    print()
+    if best_e is None:
+        print("No exposure up to 50% reaches sufficiency within {:g} h. Catch a larger regression, "
+              "run longer, or pick a metric with a higher baseline rate.".format(args.max_hours))
+    else:
+        print("Smallest exposure that reaches sufficiency within {:g} h: {} (about {}).".format(
+            args.max_hours, fmt_pct(best_e, 1), fmt_hours(best_h)))
+    print()
+    print("Minimum time is statistics only. A step also runs through at least one daily peak "
+          "and any delayed effect (nightly jobs, next-day emails) before it is promoted.")
+
+
+if __name__ == "__main__":
+    main()

package/packs/dev-team/skills/rollout-plan/templates/rollout-plan.md

@@ -0,0 +1,50 @@
+# Rollout plan — {change name}
+
+**{Two lines: the answer to the user's question, the input fact that settles it, and the shape of the plan.}**
+✅ *"No: at 2,000 users an hour a 1% step needs about 20 days to see a 0.3 pp error rise, so Tuesday-to-Friday proves nothing. Run internal for a day, 10% for 3 days, 50% for a day, and reach 100% the Wednesday after."*
+
+**Binding constraint:** {the fact that paces the plan: a partner cap, a billing or batch cycle, a deadline, the segments the flag can target, a shadow run, or volume. "None: traffic is high, the default ladder applies."}
+**Flag:** `{key}`, evaluated {server / client / edge}, targets by {user / account / tenant / %} · **Assumptions:** {one line, each tagged `[assumption]`}
+
+## Ladder
+
+| Step | Dates | Who is in it, and why | Judged on | Promote when |
+|---|---|---|---|---|
+| 0 | {Mon 7 Sep – Tue 8 Sep} | {internal / test accounts: crashes at no customer cost} | crashes only | {no abort for 24 h} |
+| 1 | {…} | {named cohort or `hash(salt + id) mod 100 < 10`; why these; who is held back} | {units or cycle this step sees, in one phrase} | {every threshold below holds at the step's end} |
+| 2 | {…} | | | |
+| {n} | {date} | everyone | {holdback or the last comparison} | — |
+
+{One sentence of arithmetic, with its inputs: what the smallest useful step is and why.}
+{Deadline line when there is one: ✅ *"100% on 14 Sep, 9 days before the deadline; one failed cycle misses it."*}
+
+## Thresholds
+
+All compare the flag-on cohort with the flag-off cohort over the same window. Counts, not
+percentages, where a step sees fewer than about 100 of the unit.
+
+| Metric | Baseline | Promote if | Abort (any time, {automatic / manual}) |
+|---|---|---|---|
+| {decision metric} | {from input} | {≤ control + 0.3 pp / zero unexplained} | {> 3× control for 5 min → flag off} |
+| {guardrail} | | | |
+
+- **Hold:** {sample short or metric between threshold and abort → extend once by {time or cycle}, then roll back}.
+- **Watcher:** {role} · **Sign-off before each promotion:** {role, if one is needed} · **Override:** {one role, reason written down}.
+
+## What turning the flag off does not undo
+
+Flag off: {who}, takes effect in {seconds / cache TTL / next app launch}. Only rows this change touches:
+
+| What | Undone? | Handling |
+|---|---|---|
+| {messages sent / money moved / records written in a new format / third-party records} | no | {correction or reissue path, reversal procedure, what keeps reading the new format} |
+
+**Point of no return:** {the step and why / none}.
+
+## Flag removal
+
+**Owner:** {name from the input, else role} · **Date:** {date} · **Gate:** {100% for 14 days or one full cycle, no rollback} · **Removes:** `{key}`, the old path, its config and dashboards · **Keeps:** {any reader the old records need}.
+
+## To confirm
+
+- {at most 4 facts, each one that would change a step}

package/packs/product-manager/loops/discover/LOOP.md

@@ -13,7 +13,7 @@ script: no check can tell you that you solved the wrong problem well.
 
 | # | Stage | Type | Skills | Gate |
 |---|---|---|---|---|
-| 1 | `gather` | generate | [`requirements-interview`](../../skills/requirements-interview/SKILL.md), [`user-journey-map`](../../skills/user-journey-map/SKILL.md), [`strategy-analysis`](../../skills/strategy-analysis/SKILL.md) | — |
+| 1 | `gather` | generate | [`requirements-interview`](../../skills/requirements-interview/SKILL.md), [`user-journey-map`](../../skills/user-journey-map/SKILL.md), [`strategy-analysis`](../../skills/strategy-analysis/SKILL.md), [`feedback-synthesis`](../../skills/feedback-synthesis/SKILL.md) | — |
 | 2 | `structure` | generate | [`brief-intake`](../../../core/skills/brief-intake/SKILL.md) | — |
 | 3 | `specify` | generate | [`prd-draft`](../../skills/prd-draft/SKILL.md) | — |
 | 4 | `ratify` | gate | [`doc-critique`](../../../core/skills/doc-critique/SKILL.md) | **G0** human |
@@ -22,8 +22,9 @@ script: no check can tell you that you solved the wrong problem well.
 
 1. **Gather from whichever direction the signal arrives.** A stakeholder conversation →
    `requirements-interview`. An existing experience with known friction → `user-journey-map`.
-   An open strategic question → `strategy-analysis`. Run one; running all three on the same
-   input produces three views of the same thing and no more information.
+   An open strategic question → `strategy-analysis`. A pile of tickets, survey verbatims or
+   interview notes already collected → `feedback-synthesis`. Run one; running several on the
+   same input produces several views of the same thing and no more information.
 2. **Structure once.** `brief-intake` tags every field `[explicit]` / `[implied]` /
    `[inferred]` / `[missing]` with verbatim quotes. The tags are the deliverable — an
    untagged brief hides which parts are somebody's assumption.
@@ -75,7 +76,7 @@ which end in `BLOCKED`.
 
 ## Anti-patterns to avoid
 
-- ❌ **Running all three `gather` skills to be thorough.** Three lenses on one input is not three inputs; pick the one that matches where the signal came from.
+- ❌ **Running every `gather` skill to be thorough.** Three lenses on one input is not three inputs; pick the one that matches where the signal came from.
 - ❌ **Letting `prd-draft` fill a `[missing]` field.** That converts an open question into a fact nobody agreed to.
 - ❌ **Treating `doc-critique`'s `SHIP IT` as ratification.** The critique judges the document; G0 judges the problem.
 - ❌ **Returning `REVISE` to `specify`.** Re-writing the PRD from the same thin brief reproduces the same gaps in better prose.

package/packs/product-manager/loops/discover/loop.json

@@ -1,7 +1,7 @@
 {
   "$comment": "RFC-0028. The first loop of the lifecycle: nothing is designed or built until a human has ratified what problem is being solved. Cheapest loop to redo and the most expensive to skip.",
   "name": "discover",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Take raw signal — interviews, journeys, a strategy question — to a requirement a human has ratified, so design starts from an agreed problem rather than an assumed one. Use when the user is deciding what to build, scoping a feature, or turning stakeholder conversations into requirements.",
   "entrypoint": "LOOP.md",
   "kind": "loop",
@@ -11,7 +11,7 @@
       "id": "gather",
       "type": "generate",
       "intent": "Collect the raw signal from whichever direction it arrives: a stakeholder conversation, an existing user journey, or a strategic question.",
-      "skills": ["requirements-interview", "user-journey-map", "strategy-analysis"]
+      "skills": ["requirements-interview", "user-journey-map", "strategy-analysis", "feedback-synthesis"]
     },
     {
       "id": "structure",

package/packs/product-manager/skills/feedback-synthesis/SKILL.md

@@ -0,0 +1,182 @@
+---
+name: feedback-synthesis
+description: Turn raw customer feedback — interview notes, support tickets, NPS or CSAT verbatims, app-store reviews, sales-call notes — into ranked themes a product decision can rest on, each a problem statement rather than a feature request, counted by distinct people or accounts not mentions, with segments, severity kept separate from frequency, verbatim quotes with source IDs, what people value, where sources disagree, what the sample cannot tell, and the questions for the next interviews. A stdlib script computes the counts. Use when the user says "what are customers telling us", "synthesise this feedback", "find the themes in these tickets / reviews / NPS comments", "affinity map these interview notes", or "what should we fix first based on feedback".
+---
+
+# feedback-synthesis
+
+Feedback arrives as a pile of tickets, survey comments, reviews and notes, and the loudest
+thing in the pile is rarely the most important. This skill turns the pile into a short list of
+problems worth acting on, with numbers that are counted rather than guessed and quotes that can
+be traced to the item they came from. It feeds discovery: the themes go to `brief-intake` to
+become a brief, and the open questions go into the next round of interviews.
+
+## How to respond
+
+**Five rules come before the steps and outrank them:**
+
+- **Answer what was asked, first.** Open with what to act on and why, in plain words. If the
+  data contradicts the user's premise ("the CSV export is our biggest problem"), say so in the
+  opening lines, with the count that shows it.
+- **Count people, not mentions.** One customer filing six tickets is one customer. Every number
+  in the reply is a count of distinct people or accounts, computed, and says which.
+- **Use only what the input contains.** No quote, count, segment, customer name, revenue figure
+  or reason that isn't in the feedback. What is missing goes in **What this can't tell us** and
+  the interview questions, never into a theme.
+- **Deliver from what there is.** Feedback without IDs, segments or a stated decision still gets
+  a synthesis, with the limit named once. Ask first only when there is no feedback at all.
+- **Write for someone who has never heard of this skill.** No skill names, file paths, script
+  output, theme codes or "S2" shorthand without its meaning. A next step is one plain sentence
+  at the end.
+
+1. **Read every item before coding any.** List the sources and, for each, what an item is
+   (a ticket, a response, an interview, a review), what identifies the person (account ID,
+   respondent ID, reviewer name, nothing), and what segment fields exist (plan, role, region,
+   tenure, company size, score). Decide the counting unit: **accounts** for B2B feedback where
+   account IDs exist (two staff from one customer are one account), **people** otherwise.
+   Match people across sources only on a shared ID: an interviewee whose account ID appears in
+   the tickets is the same account. Anonymous rows are never merged by guesswork. A rep's or
+   interviewer's guess about what someone needs ("they'll probably need SSO") is not that
+   person's feedback: leave it uncoded and say so if it matters to the answer.
+
+2. **Write the codebook as problem statements.** 4–12 themes, each one sentence saying who hits
+   what and what it costs them, plus a short code. A theme must be something the next interview
+   could prove wrong.
+   - ✅ *"Clients in the EU can't complete card payments; the error says authentication failed
+     and payment arrives weeks later by transfer."*
+   - ❌ *"Payment issues"* / *"Users want a better experience"* / *"Improve usability"* — banned:
+     nothing could disprove them.
+   - **A feature request is evidence of a need, not the need.** Rewrite it into the job behind
+     it and keep the request as a quote. ✅ *"Changing a rate means editing every recurring
+     invoice by hand — 30 to 58 invoices per price change"*, with "Please add bulk edit"
+     quoted as evidence. ❌ Theme title *"Bulk edit"*. Never let the loudest proposed solution
+     become the title.
+   - **A request with no stated reason is not a theme.** Three "add dark mode" comments with no
+     context go under **Requests with no stated need**, and the reason becomes an interview
+     question. Don't invent the need.
+   - Code what people value as themes too, prefixed `+` (`+reminders`). These become **What
+     not to break**.
+
+3. **Rate severity per item, separately from frequency.** From what happened to the person, not
+   how angry they sound; capitals and exclamation marks don't raise it.
+
+   | Level | Meaning | Example |
+   |---|---|---|
+   | **S1 — blocker** | A core job can't be done, or money, data, security or compliance is at stake | Clients can't pay; data lost; can't log in |
+   | **S2 — major** | Done only through a costly recurring workaround, or the person says they'll leave over it | Reformatting an export by hand every month |
+   | **S3 — minor** | Slower or annoying; the workaround is quick | An extra click to find a report |
+   | **S4 — cosmetic** | Looks, wording, preference, no stated cost | Blurry logo on a PDF |
+
+   More rules and edge cases in [`reference.md`](reference.md).
+
+4. **Fill the coding sheet and run the counts.** One row per item in the shape of
+   [`templates/coding-sheet.csv`](templates/coding-sheet.csv): `item_id`, `person_id` (blank
+   when the source has none), `source`, the segment columns, `themes` separated by `;` (a code
+   may carry its own severity as `code:S1`), and `severity`. When a raw segment is too fine to
+   show a pattern, add a derived column beside it (country → `market` EU / UK / US; employee
+   count → `size` band). Then run:
+
+   ```bash
+   # Claude Code
+   python3 "${CLAUDE_SKILL_DIR}/scripts/theme_counts.py" coded.csv --person-label account -o counts.md
+   # Other IDEs (from the skill folder)
+   python3 scripts/theme_counts.py coded.csv --person-label account -o counts.md
+   ```
+
+   It prints per theme the distinct people with an ID, the items with no ID (beside the count,
+   never added to it), mentions, repeat filers, sources, the segment breakdown with a flag when
+   a theme sits in one segment only, the ranking severity, and the items with no theme. If no script can run, count by hand from the sheet and show the
+   tally per theme. Above about 300 items, code the most recent full period or a stated sample,
+   and say the counts cover that sample.
+
+5. **Read the counts, then fix the codebook.** Re-read the unthemed items: three or more about
+   the same thing is a missing theme. Where one person drives most of a theme's mentions, the
+   theme is ranked on its people count. Where a theme lives in one segment, that is a finding
+   (✅ *"Only EU accounts: DE, FR, NL"*), not a footnote. Rerun after any recode.
+
+6. **Rank: severity first, then distinct people.** A theme's severity is the worst level at
+   least two people reached; a theme raised by one person takes that person's level and is
+   labelled *one report — confirm before acting*. The script's ranking severity column applies
+   this rule. A rare issue that stops payment outranks a common cosmetic gripe: say so in those
+   words when the ranking differs from the volume order.
+
+7. **Write the reply in the shape of [`templates/synthesis.md`](templates/synthesis.md)**,
+   sized to the ask (a few bullets when asked for bullets):
+   - **Headline** — what to act on first and why, correcting the premise if needed.
+   - **Themes, ranked** — problem statement; distinct people or accounts, plus mentions when
+     they differ; segments; severity in words; sources.
+   - **Evidence per theme** — 1–3 verbatim quotes, each followed by its source ID:
+     *"Took most of a Friday." (I-2)*. The quote is an exact substring of the item; cut with
+     "…", never reword, never stitch two items into one quote. An interviewer's or rep's summary
+     is reported without quotation marks and attributed to them. The requests behind the theme
+     are quoted here too.
+   - **What not to break** — what people value, with counts and quotes.
+   - **Where the sources disagree** — ✅ *"Interviewees call reminders the reason they stay;
+     four accounts report reminders going to a former contact."* Both sides, counted.
+   - **What this can't tell us** — each source's sampling bias in one line, and the
+     conclusions it rules out. No percentage of the user base from a self-selected sample:
+     ✅ *"7 of the 29 accounts that raised a ticket"* — ❌ *"24% of customers"*. Anonymous rows
+     that may overlap with identified ones are reported beside the count, never added to it:
+     ✅ *"3 accounts, plus one anonymous NPS comment that may be one of them"*.
+   - **Questions for the next interviews** — 3–6, each tied to a theme, open, non-leading, and
+     at least one that could disconfirm the top theme. ✅ *"Walk me through the last time a
+     client couldn't pay you. What happened next?"* — ❌ *"Would a fix for card payments help?"*
+
+8. **Keep the machinery out of the reply.** Theme codes, the coding sheet and the script's report
+   are working files; save them only when the user asked. The reply carries the answer.
+
+**Non-interactive runs** (subagent, CI, headless): with no feedback items at all, emit
+`BLOCKED: need the feedback itself (tickets, survey verbatims, interview notes or reviews)`,
+because a synthesis of what customers "usually say" is fabrication. A missing decision or
+audience becomes an `[assumption]` stated at the top (default: *"feeds next quarter's
+prioritisation"*). Missing person IDs or segment fields are not blockers: proceed, and state
+that the counts are mentions or unsegmented.
+
+## Useful references in this skill
+
+- [`reference.md`](reference.md) — the severity scale with edge cases, sampling bias by source, rewriting requests into problems, codebook rules, counting across sources
+- [`templates/coding-sheet.csv`](templates/coding-sheet.csv) — the CSV the script reads
+- [`templates/synthesis.md`](templates/synthesis.md) — the reply skeleton
+- [`scripts/theme_counts.py`](scripts/theme_counts.py) — distinct-person counts per theme, segments, severity, unthemed items (stdlib, Python 3.9+)
+- [`examples/booking-app-feedback.md`](examples/booking-app-feedback.md) — worked example: 16 items from three sources, the coding sheet, the real script output, and the finished reply
+
+## Quality bar
+
+- **Every count is distinct people or accounts,** computed from the coded items, and says which unit; mentions appear only beside it.
+- **The ranking puts severity before frequency** and says so where the two disagree.
+- **Every theme title is a falsifiable problem statement** naming who and what it costs; no theme is titled with a proposed feature.
+- **Every quote is verbatim with a source ID;** paraphrase never sits inside quotation marks.
+- **Segments are reported per theme,** and a one-segment theme is called out.
+- **Each source's sampling bias is named,** and no percentage of the whole user base is drawn from a self-selected sample.
+- **What people value and where the sources disagree** are both present.
+- **Interview questions are tied to themes,** open, and include one that could disconfirm the top theme.
+- **Nothing is invented:** no quote, count, segment, reason or customer that isn't in the input.
+
+## When to use this skill
+
+- ✅ A pile of support tickets, NPS or CSAT comments, reviews or interview notes, and the question "what are customers telling us?"
+- ✅ Checking a stakeholder's claim ("everyone asks for SSO") against the feedback before it goes into a brief
+- ✅ Affinity-mapping interview notes after a discovery round
+- ✅ Choosing what to fix first from a quarter's feedback
+
+## When NOT to use this skill
+
+- ❌ Planning or running the interviews themselves, or writing the question guide — `requirements-interview`
+- ❌ Mapping the stages of a journey a customer goes through — `user-journey-map`
+- ❌ Answering a research question from papers, reports or articles — `source-synthesis`
+- ❌ Ranking rival explanations for a metric change ("why did churn rise in March?") — `hypothesis-comparison`
+- ❌ Turning agreed findings into a brief for a PRD or design doc — `brief-intake`
+- ❌ Cleaning up a tracker backlog: duplicates, stale tickets, ordering — `backlog-triage`
+
+## Anti-patterns to avoid
+
+- ❌ **Counting tickets as customers.** "CSV export: 7 tickets, our biggest issue" when six came from one account.
+- ❌ **Ranking by volume alone.** Seven blurry-logo complaints above three accounts whose clients can't pay.
+- ❌ **The feature as the theme.** "Users want bulk edit" — the theme is the hours lost re-pricing recurring invoices.
+- ❌ **Inventing the need behind a bare request.** "Users want dark mode for eye strain" when nobody said why.
+- ❌ **Quotation marks around a summary.** A rep's note or the agent's paraphrase presented as the customer's words.
+- ❌ **Percentages of the base from a self-selected sample.** "40% of users want an Apple Watch app" from fifteen reviews.
+- ❌ **Merging anonymous rows into identified counts** by guessing who wrote them.
+- ❌ **Only problems.** Leaving out what people value invites the fix that breaks it.
+- ❌ **Vague themes.** "Pricing concerns", "usability", "better experience": name who, what happened, and what it cost.
+- ❌ **Leading interview questions.** "Would you use bulk edit?" confirms nothing.

package/packs/product-manager/skills/feedback-synthesis/evals/eval_queries.json

@@ -0,0 +1,46 @@
+[
+  {
+    "query": "Here are 200 support tickets and last quarter's NPS comments. What are customers actually telling us?",
+    "should_trigger": true
+  },
+  {
+    "query": "Affinity map these eight interview notes into themes with quotes",
+    "should_trigger": true
+  },
+  {
+    "query": "Find the themes in our app store reviews and tell me what to fix first",
+    "should_trigger": true
+  },
+  {
+    "query": "Sales says every prospect wants SSO. Check that against the call notes and churn survey before it goes in the brief",
+    "should_trigger": true
+  },
+  {
+    "query": "Synthesise this CSAT export into the top customer pain points by segment",
+    "should_trigger": true
+  },
+  {
+    "query": "Write me an interview guide for talking to churned customers next week",
+    "should_trigger": false
+  },
+  {
+    "query": "Map our customer's onboarding journey from sign-up to first invoice, with pains at each stage",
+    "should_trigger": false
+  },
+  {
+    "query": "Summarise what these five industry reports say about four-day weeks",
+    "should_trigger": false
+  },
+  {
+    "query": "Churn jumped 3 points in August. Which of these explanations fits the evidence best?",
+    "should_trigger": false
+  },
+  {
+    "query": "Clean up our Jira backlog: duplicates, stale tickets and what to pull next",
+    "should_trigger": false
+  },
+  {
+    "query": "Turn this Slack thread about the export bug into a brief for the PRD",
+    "should_trigger": false
+  }
+]