skilldrop-cli 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/bin/skilldrop.js +15 -10
- package/catalogue.json +5 -1
- package/contracts/catalogue.schema.json +2 -1
- package/model-routing.json +8 -8
- package/package.json +1 -1
- package/packs/ai-engineering/skills/judge-audit/SKILL.md +28 -11
- package/packs/ai-engineering/skills/judge-audit/evals/evals.json +30 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/12/judge_runs.csv +241 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/12/rosa_labels.csv +33 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/13/moderation_eval.csv +601 -0
- package/packs/ai-engineering/skills/judge-audit/manifest.json +2 -2
- package/packs/ai-engineering/skills/mcp-pairing/manifest.json +1 -1
- package/packs/ai-engineering/skills/mcp-pairing/reference.md +4 -3
- package/packs/ai-engineering/skills/subagent-design/SKILL.md +1 -0
- package/packs/ai-engineering/skills/subagent-design/examples/help-center-localization.md +146 -0
- package/packs/ai-engineering/skills/subagent-design/manifest.json +2 -2
- package/packs/api-builder/skills/prompt-caching-advisor/SKILL.md +4 -0
- package/packs/api-builder/skills/prompt-caching-advisor/examples/support-bot-block-order.md +120 -0
- package/packs/api-builder/skills/prompt-caching-advisor/manifest.json +2 -2
- package/packs/dev-team/loops/release/LOOP.md +4 -4
- package/packs/dev-team/loops/release/loop.json +1 -1
- package/packs/dev-team/skills/bug-triage/SKILL.md +1 -0
- package/packs/dev-team/skills/bug-triage/examples/export-row-limit-escalation.md +126 -0
- package/packs/dev-team/skills/bug-triage/manifest.json +2 -2
- package/packs/product-manager/skills/prd-draft/SKILL.md +1 -0
- package/packs/product-manager/skills/prd-draft/examples/approver-delay-not-slack.md +142 -0
- package/packs/product-manager/skills/prd-draft/manifest.json +2 -2
- package/profiles.json +1 -1
- package/packs/ai-engineering/loops/adopt/LOOP.md +0 -120
- package/packs/ai-engineering/loops/adopt/evals/eval_queries.json +0 -10
- package/packs/ai-engineering/loops/adopt/evals/evals.json +0 -69
- package/packs/ai-engineering/loops/adopt/evals/files/adopt-first-wave-to-sign-off/ai-ideas.md +0 -15
- package/packs/ai-engineering/loops/adopt/evals/files/adopt-first-wave-to-sign-off/org-context.md +0 -27
- package/packs/ai-engineering/loops/adopt/evals/files/adopt-first-wave-to-sign-off/staff-survey.csv +0 -11
- package/packs/ai-engineering/loops/adopt/evals/files/adopt-sponsor-slice-blocked-by-client-data/ai-ideas.md +0 -13
- package/packs/ai-engineering/loops/adopt/evals/files/adopt-sponsor-slice-blocked-by-client-data/firm-context.md +0 -23
- package/packs/ai-engineering/loops/adopt/loop.json +0 -53
- package/packs/career/loops/apply/LOOP.md +0 -97
- package/packs/career/loops/apply/evals/eval_queries.json +0 -9
- package/packs/career/loops/apply/evals/evals.json +0 -55
- package/packs/career/loops/apply/evals/files/apply-candidate-decided-full-run/application-form.md +0 -10
- package/packs/career/loops/apply/evals/files/apply-candidate-decided-full-run/job-posting.md +0 -33
- package/packs/career/loops/apply/evals/files/apply-candidate-decided-full-run/resume.md +0 -34
- package/packs/career/loops/apply/evals/files/apply-fit-gate-hard-requirement/cv.md +0 -27
- package/packs/career/loops/apply/evals/files/apply-fit-gate-hard-requirement/warehouse-supervisor.md +0 -30
- package/packs/career/loops/apply/loop.json +0 -52
- package/packs/product-manager/loops/discover/LOOP.md +0 -82
- package/packs/product-manager/loops/discover/evals/eval_queries.json +0 -30
- package/packs/product-manager/loops/discover/evals/evals.json +0 -54
- package/packs/product-manager/loops/discover/evals/files/discover-no-show-signal/interview-notes.md +0 -63
- package/packs/product-manager/loops/discover/evals/files/discover-no-show-signal/support-tickets.csv +0 -13
- package/packs/product-manager/loops/discover/evals/files/discover-vp-loyalty-no-evidence/vp-message.txt +0 -13
- package/packs/product-manager/loops/discover/loop.json +0 -41
- package/packs/sre-oncall/loops/operate/LOOP.md +0 -84
- package/packs/sre-oncall/loops/operate/evals/eval_queries.json +0 -30
- package/packs/sre-oncall/loops/operate/evals/evals.json +0 -53
- package/packs/sre-oncall/loops/operate/evals/files/operate-export-service-first-outage/alerts.log +0 -22
- package/packs/sre-oncall/loops/operate/evals/files/operate-export-service-first-outage/incident-chat.txt +0 -27
- package/packs/sre-oncall/loops/operate/evals/files/operate-export-service-first-outage/service-notes.md +0 -46
- package/packs/sre-oncall/loops/operate/evals/files/operate-incident-still-open/oncall-log.txt +0 -17
- package/packs/sre-oncall/loops/operate/evals/files/operate-incident-still-open/runbook-webhooks.md +0 -38
- package/packs/sre-oncall/loops/operate/loop.json +0 -41
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# Worked example: a seven-agent localization plan that grades its own work
|
|
2
|
+
|
|
3
|
+
Input → output oracle for `subagent-design`. Shows a fan-out that is justified but drawn
|
|
4
|
+
wrong: a translator that checks itself, a middle layer with no job, a summary agent standing
|
|
5
|
+
in for a merge, and auto-publishing. It also shows the one barrier the work actually needs.
|
|
6
|
+
|
|
7
|
+
## Input
|
|
8
|
+
|
|
9
|
+
> We're localizing our help center with AI agents. My team lead drew up the plan below. Is
|
|
10
|
+
> this the right setup? Our docs lead has to sign off, so keep it short.
|
|
11
|
+
>
|
|
12
|
+
> **Plan:**
|
|
13
|
+
> 1. A Coordinator agent spawns three Language Leads (de, fr, ja).
|
|
14
|
+
> 2. Each Language Lead spawns one Translator agent per article. The Translator translates
|
|
15
|
+
> the article, then reviews its own translation against the glossary and fixes anything it
|
|
16
|
+
> finds.
|
|
17
|
+
> 3. When all 960 translations are done, a Summary agent reads them all and writes a report of
|
|
18
|
+
> quality issues for the docs lead.
|
|
19
|
+
> 4. The Coordinator publishes everything to the help center.
|
|
20
|
+
> 5. We want 7+ agents because more agents run faster.
|
|
21
|
+
>
|
|
22
|
+
> **Facts:**
|
|
23
|
+
> - 320 English articles, about 1,500 words each, into three languages.
|
|
24
|
+
> - A glossary of 410 product terms with an approved translation per language. Style rule: UI
|
|
25
|
+
> labels stay in English.
|
|
26
|
+
> - Pilot of 40 articles into Japanese: the Translator's self-review marked all 40 "pass". Our
|
|
27
|
+
> Japanese linguist found 11 with UI labels translated, and 4 where the same term was
|
|
28
|
+
> translated two different ways in different articles.
|
|
29
|
+
> - Today a linguist per language approves each article before it goes live.
|
|
30
|
+
|
|
31
|
+
## Output
|
|
32
|
+
|
|
33
|
+
**Not as drawn. Splitting the work per article is right, but four parts of the plan need to
|
|
34
|
+
change.** Keep one translation job per article and language. A separate agent should check
|
|
35
|
+
each translation instead of the translator checking itself. Code should build the report, not
|
|
36
|
+
a summary agent. Drop the Language Lead layer. And nothing goes live without the linguist,
|
|
37
|
+
who stays the final approval. That leaves two agent roles, each run once per article and
|
|
38
|
+
language, plus code around them. The pilot is the reason: the translator passed its own work
|
|
39
|
+
40 times out of 40, and the linguist found problems in at least 11 of those articles.
|
|
40
|
+
|
|
41
|
+
Assumed: each article can be translated on its own, given the glossary.
|
|
42
|
+
|
|
43
|
+
# Orchestration plan: help-center localization (320 articles × de, fr, ja)
|
|
44
|
+
|
|
45
|
+
> Fan-out justification: **context separation**, because about 480,000 words of source won't
|
|
46
|
+
> fit one context well and each article stands alone. **Role conflict** for checking, because
|
|
47
|
+
> the agent that wrote a translation can't honestly grade it, as the 40/40 pilot showed.
|
|
48
|
+
> Speed comes with the per-article split. It is not a reason on its own.
|
|
49
|
+
|
|
50
|
+
## Role cards
|
|
51
|
+
|
|
52
|
+
### orchestrator (code, not an agent)
|
|
53
|
+
- **Mission:** Move each article-language pair through translation, checking and the linguist queue.
|
|
54
|
+
- **Inputs:** article list, glossary, UI-label rule
|
|
55
|
+
- **Denied (isolation):** it doesn't judge translation quality. It acts only on the fields each step returns
|
|
56
|
+
- **Output contract:** `{ article_id: string, lang: "de"|"fr"|"ja", status: "translated"|"checked"|"flagged"|"invalid"|"queued_for_linguist" }[]`
|
|
57
|
+
- **Tools/permissions:** read source articles, write to the linguist's review queue. **No publish permission**
|
|
58
|
+
- **On failure:** marks the item `invalid` with the reason and carries on with the rest
|
|
59
|
+
- **Model:** none needed: routing and merging are code
|
|
60
|
+
|
|
61
|
+
### translator ×960
|
|
62
|
+
- **Mission:** Translate one article into one language using the glossary.
|
|
63
|
+
- **Inputs:** one source article, that language's glossary, the UI-label rule
|
|
64
|
+
- **Denied (isolation):** other articles' translations, so each job stays small. Consistency across articles comes from the glossary and the term check below
|
|
65
|
+
- **Output contract:** `{ article_id: string, lang: string, body: string, terms: [{ source: string, target: string, in_glossary: bool }], status: "ok"|"blocked", blocked_reason: string|null }`
|
|
66
|
+
- **Tools/permissions:** none beyond its inputs. No write access anywhere
|
|
67
|
+
- **On failure:** `status: "blocked"` with a reason. It never sends a partial translation as `ok`
|
|
68
|
+
- **Model:** mid-size: translating with a fixed glossary doesn't need the largest model
|
|
69
|
+
|
|
70
|
+
### checker ×960
|
|
71
|
+
- **Mission:** Find every place the translation departs from the source, the glossary or the UI-label rule.
|
|
72
|
+
- **Inputs:** source article, translated `body`, glossary, UI-label rule
|
|
73
|
+
- **Denied (isolation):** the translator's `terms` list and status, so it checks the text instead of confirming the translator's claims about it
|
|
74
|
+
- **Output contract:** `{ article_id: string, lang: string, verdict: "pass"|"fail"|"blocked", findings: [{ type: "glossary"|"ui_label"|"meaning"|"omission", source_excerpt: string, target_excerpt: string, expected: string }] }`
|
|
75
|
+
- **Tools/permissions:** read-only
|
|
76
|
+
- **On failure:** `verdict: "blocked"`, handled exactly like `fail`. It is never counted as a pass
|
|
77
|
+
- **Model:** mid-size: a different agent from the translator, which is what makes the check independent
|
|
78
|
+
|
|
79
|
+
## Topology
|
|
80
|
+
|
|
81
|
+
**Shape:** a pipeline per article (translate, then check), **because** no article's
|
|
82
|
+
translation depends on another's. **One barrier per language**, **because** the pilot's 4
|
|
83
|
+
inconsistent terms only show up across articles. Code groups every translator's `terms` by
|
|
84
|
+
source term and flags any term with more than one translation. That check needs all of a
|
|
85
|
+
language's articles, and nothing else in the plan does.
|
|
86
|
+
**Depth:** one level. The Language Leads would only pass articles along, and the orchestrator
|
|
87
|
+
already does that.
|
|
88
|
+
|
|
89
|
+
```mermaid
|
|
90
|
+
flowchart LR
|
|
91
|
+
O[orchestrator] --> T["translator ×960"]
|
|
92
|
+
T --> C["checker ×960"]
|
|
93
|
+
T -. terms .-> X{"term check per language"}
|
|
94
|
+
C --> X
|
|
95
|
+
X --> L["linguist approval"]
|
|
96
|
+
L -->|approved| P[help center]
|
|
97
|
+
T -->|invalid output| F[failure list]
|
|
98
|
+
C -->|invalid output| F
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Aggregation & verification
|
|
102
|
+
|
|
103
|
+
- **Aggregation:** code. The term check groups by `(lang, source term)`. The report for the
|
|
104
|
+
docs lead is counts of `findings` by type and language, plus the list of `fail` and
|
|
105
|
+
`invalid` articles. Output that doesn't match its schema goes to the failure list and never
|
|
106
|
+
into the report.
|
|
107
|
+
- **Verification:** the checker's job is to find faults, and it never sees the translator's
|
|
108
|
+
own assessment. A failed article goes to the linguist with its findings attached.
|
|
109
|
+
- **Vote rule:** a single checker. An article is flagged if it has any finding. The linguist's
|
|
110
|
+
approval gates publishing, as it does today.
|
|
111
|
+
|
|
112
|
+
## Budget line
|
|
113
|
+
|
|
114
|
+
No spending cap was given, so none is set here. Model sizes are on each card.
|
|
115
|
+
|
|
116
|
+
**Left out of the plan:** the Coordinator publishing everything. Every article still goes
|
|
117
|
+
through linguist approval before it goes live. The pilot shows why: the self-check missed all
|
|
118
|
+
11 UI-label errors.
|
|
119
|
+
|
|
120
|
+
**To confirm**
|
|
121
|
+
- The spending cap for a full run.
|
|
122
|
+
- Whether source articles mark UI labels in a way code can detect. If they do, that check
|
|
123
|
+
can move from the checker to code.
|
|
124
|
+
- Who reworks a failed article: the linguist, or a second translation pass with the findings.
|
|
125
|
+
|
|
126
|
+
Next step: design the run around this, meaning retries, the stop conditions and what happens
|
|
127
|
+
when the cap is hit, before the first full batch.
|
|
128
|
+
|
|
129
|
+
## Why this is the oracle
|
|
130
|
+
|
|
131
|
+
- **The verifier is separate and adversarial, never the generator.** The plan's step 2 has
|
|
132
|
+
the translator check itself, and the pilot's 40/40 self-pass is the evidence that this
|
|
133
|
+
fails. The checker is denied the translator's own claims, so it can't just agree with them.
|
|
134
|
+
- **Aggregation is code, and a barrier needs a named dependency across items.** The summary
|
|
135
|
+
agent becomes a group-by over typed `findings`. The plan's "when all 960 are done"
|
|
136
|
+
synchronisation shrinks to one barrier per language, earned by cross-article term
|
|
137
|
+
consistency, the 4 inconsistent terms no per-article check can see.
|
|
138
|
+
- **Fan out only for a named reason, keep depth to one level, and take budget numbers only
|
|
139
|
+
from the user.** "More agents run faster" is rejected as a reason. The Language Lead layer
|
|
140
|
+
is cut. No cap is invented, and auto-publishing is removed and named as removed.
|
|
141
|
+
|
|
142
|
+
A cheaper pass approves the seven-agent plan and tunes the prompts. It keeps self-review,
|
|
143
|
+
lets a summary agent paraphrase 960 translations, and leaves the Coordinator publishing. That
|
|
144
|
+
ships the pilot's UI-label errors across three languages straight to the live help center. The
|
|
145
|
+
opposite cheap answer, "use one agent", fails too: 480,000 words don't fit one context, and it
|
|
146
|
+
leaves the role conflict in place.
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "subagent-design",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "Decompose a task into an orchestrator and subagents — one-mission role cards with typed output contracts, a topology (pipeline / parallel / judge panel) chosen with a reason, context-isolation rationale, and a verification stage that is never a generator. Use when the user wants to fan work out across multiple agents, design a multi-agent workflow or orchestration, or asks \"should this be one agent or several\".",
|
|
5
5
|
"entrypoint": "SKILL.md",
|
|
6
6
|
"deps": { "npm": [], "pip": [] },
|
|
7
7
|
"env": { "required": [], "optional": [] },
|
|
8
8
|
"related": ["agent-budget", "agent-loop-design", "council-review", "devils-advocate", "feature-implement-loop"],
|
|
9
9
|
"tags": ["subagents", "orchestration", "multi-agent", "fan-out", "agentic", "role-cards"],
|
|
10
|
-
"model": { "tier": "
|
|
10
|
+
"model": { "tier": "heavy", "rationale": "Measured (RFC-0073): on its held-out evals the heavy tier met 100% and 100% of checks in two tier trials, against standard's 86% and light's 85% and 86%, more than 5 points ahead both times; moved once its examples/ oracle shipped (RFC-0074)." }
|
|
11
11
|
}
|
|
@@ -75,6 +75,10 @@ Produce:
|
|
|
75
75
|
1. **Annotated structure** — show the messages array with `cache_control` placements marked and commented
|
|
76
76
|
2. **Cache strategy summary** — one paragraph: what is cached, why, expected hit rate, and the dominant cost driver before and after
|
|
77
77
|
|
|
78
|
+
## Useful references in this skill
|
|
79
|
+
|
|
80
|
+
- [`examples/support-bot-block-order.md`](examples/support-bot-block-order.md) — worked example: a breakpoint under the minimum size and a per-customer block splitting a shared cache, fixed by reordering
|
|
81
|
+
|
|
78
82
|
## Anti-patterns
|
|
79
83
|
|
|
80
84
|
- **Caching dynamic content.** If a user query, timestamp, or per-request database result is inside a cached block, every request is a cache miss — the structure defeats itself. Identify and flag before recommending.
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Worked example: a cache breakpoint that "didn't move the bill"
|
|
2
|
+
|
|
3
|
+
Input → output oracle for `prompt-caching-advisor`. Shows a breakpoint that never caches
|
|
4
|
+
because its prefix is under the minimum size, a per-customer block that splits a shared cache
|
|
5
|
+
in two, and the fix as a reordering, not an extra marker.
|
|
6
|
+
|
|
7
|
+
## Input
|
|
8
|
+
|
|
9
|
+
> Our support bot runs on Haiku. We added `cache_control` to the instructions block because
|
|
10
|
+
> that's the static part, and the input bill didn't move. Should we add another one on the
|
|
11
|
+
> catalogue? Numbers: about 150,000 requests a day, conversations average 6 turns with a turn
|
|
12
|
+
> every 1–2 minutes, busy all day and quiet overnight. The catalogue is rebuilt once a night
|
|
13
|
+
> at 02:00.
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
def build_request(customer, session, question):
|
|
17
|
+
system = [
|
|
18
|
+
{"type": "text", "text": INSTRUCTIONS, # ~3,200 characters
|
|
19
|
+
"cache_control": {"type": "ephemeral"}},
|
|
20
|
+
{"type": "text", "text": render_profile(customer)}, # ~1,400 characters, per customer
|
|
21
|
+
{"type": "text", "text": load_catalogue()}, # ~62,000 characters
|
|
22
|
+
]
|
|
23
|
+
messages = session.history + [{"role": "user", "content": question}]
|
|
24
|
+
return {"model": MODEL, "max_tokens": 800, "system": system, "messages": messages}
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Output
|
|
28
|
+
|
|
29
|
+
**Don't add a second marker yet. Fix the order first.** Two things are wrong, and a marker on
|
|
30
|
+
the catalogue in its current position would only fix half of it.
|
|
31
|
+
|
|
32
|
+
1. **Your current breakpoint caches nothing.** A breakpoint caches the whole prefix up to and
|
|
33
|
+
including its block. Here that is the instructions alone: 3,200 characters, or about 800
|
|
34
|
+
tokens. Haiku's minimum is about 1,024 tokens, so the API processes it as normal input,
|
|
35
|
+
with no error and no saving. That is why the bill didn't move.
|
|
36
|
+
2. **The customer profile sits in front of the catalogue.** The profile differs for every
|
|
37
|
+
customer, so any breakpoint after it makes a separate cache entry per customer. A marker
|
|
38
|
+
on the catalogue where it sits now would cache instructions + profile + catalogue together.
|
|
39
|
+
It would hit only on later turns of the same conversation, and every customer's first turn
|
|
40
|
+
would pay to write 16,600 tokens to the cache.
|
|
41
|
+
|
|
42
|
+
Move the catalogue above the profile, put one breakpoint at the end of the catalogue, and put
|
|
43
|
+
a second, moving breakpoint on the latest user turn.
|
|
44
|
+
|
|
45
|
+
### Annotated structure
|
|
46
|
+
|
|
47
|
+
| Block | Size (~4 chars/token) | Class | Cached by |
|
|
48
|
+
|---|---|---|---|
|
|
49
|
+
| Instructions | 3,200 ch ≈ 800 tok | Static | BP1 (as part of its prefix) |
|
|
50
|
+
| Catalogue | 62,000 ch ≈ 15,500 tok | Static until the 02:00 rebuild | BP1 |
|
|
51
|
+
| Customer profile | 1,400 ch ≈ 350 tok | Quasi-static (per conversation) | BP2 |
|
|
52
|
+
| Earlier turns | grows each turn | Fixed once sent | BP2 |
|
|
53
|
+
| Latest question | varies | Dynamic | written by BP2 for the *next* turn to read |
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
def build_request(customer, session, question):
|
|
57
|
+
system = [
|
|
58
|
+
# Static. ~800 tokens: under Haiku's ~1,024 minimum, so no marker of its own.
|
|
59
|
+
# It is still cached, because BP1's prefix includes it.
|
|
60
|
+
{"type": "text", "text": INSTRUCTIONS},
|
|
61
|
+
# Static until the 02:00 rebuild. ~15,500 tokens.
|
|
62
|
+
{"type": "text", "text": load_catalogue(),
|
|
63
|
+
"cache_control": {"type": "ephemeral"}}, # BP1: instructions + catalogue, ~16,300 tokens,
|
|
64
|
+
# identical for every customer
|
|
65
|
+
# Quasi-static: the same for a whole conversation, different per customer.
|
|
66
|
+
# It sits after BP1 so it can't split the shared entry.
|
|
67
|
+
{"type": "text", "text": render_profile(customer)},
|
|
68
|
+
]
|
|
69
|
+
latest = {
|
|
70
|
+
"role": "user",
|
|
71
|
+
"content": [{"type": "text", "text": question,
|
|
72
|
+
"cache_control": {"type": "ephemeral"}}], # BP2: profile + all earlier turns + this one
|
|
73
|
+
}
|
|
74
|
+
# session.history stores turns WITHOUT cache_control. If the marker is saved into history,
|
|
75
|
+
# markers pile up turn after turn, and from turn 4 the request carries more than the API's
|
|
76
|
+
# limit of four breakpoints.
|
|
77
|
+
messages = session.history + [latest]
|
|
78
|
+
return {"model": MODEL, "max_tokens": 800, "system": system, "messages": messages}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
BP2 is not caching dynamic content. It writes the conversation so far, so that the next turn,
|
|
82
|
+
where this question is now fixed history, can read it. Nothing that changes between requests
|
|
83
|
+
sits in front of BP1, and that is the prefix every customer shares.
|
|
84
|
+
|
|
85
|
+
Two breakpoints, one per logical boundary: content shared by everyone, then content shared
|
|
86
|
+
within a conversation. A third marker on the instructions would add nothing, because they are
|
|
87
|
+
under the minimum and already inside BP1's prefix.
|
|
88
|
+
|
|
89
|
+
### Cache strategy summary
|
|
90
|
+
|
|
91
|
+
BP1 caches the instructions and catalogue, about 16,300 tokens, as one entry shared by every
|
|
92
|
+
customer. At about 150,000 requests a day it stays warm all through the busy hours, so its hit
|
|
93
|
+
rate is near 100%. It misses on the first request after the 02:00 rebuild and after any
|
|
94
|
+
overnight gap longer than the cache's five-minute lifetime. BP2 caches each conversation's
|
|
95
|
+
profile and earlier turns. With turns 1–2 minutes apart, turns 2–6 read it, about 5 of 6
|
|
96
|
+
requests. Before the change, about 16,600 tokens of instructions, profile and catalogue were
|
|
97
|
+
processed at full price on every request, roughly 2.5 billion input tokens a day. After it,
|
|
98
|
+
almost all of that is a cache read, billed at about a tenth of the normal input price. The
|
|
99
|
+
largest costs left are output tokens and each turn's new question. This answer gives no dollar
|
|
100
|
+
figure: the request didn't include prices, and the hit rates above are estimates until the
|
|
101
|
+
response `usage` fields (`cache_read_input_tokens`, `cache_creation_input_tokens`) confirm
|
|
102
|
+
them.
|
|
103
|
+
|
|
104
|
+
## Why this is the oracle
|
|
105
|
+
|
|
106
|
+
- **The last block caches everything before it.** The user's mental model, "mark the static
|
|
107
|
+
block", is per block. A breakpoint marks a prefix. That one rule explains why the
|
|
108
|
+
instructions marker did nothing (its prefix is 800 tokens) and why a catalogue marker in
|
|
109
|
+
the current order would be keyed per customer (its prefix includes the profile).
|
|
110
|
+
- **Check the size before recommending a cache.** The fix removes the user's existing marker,
|
|
111
|
+
because it is under Haiku's minimum. The instructions still get cached, inside BP1.
|
|
112
|
+
- **Never let dynamic or per-user content split a shared prefix.** The profile is not dynamic
|
|
113
|
+
per request, but it is per customer. Moving it below BP1 turns a per-conversation cache into
|
|
114
|
+
a shared one. The comment on `session.history` heads off the four-breakpoint error that a
|
|
115
|
+
moving breakpoint causes when it is stored.
|
|
116
|
+
|
|
117
|
+
A cheaper pass answers the question as asked: "Yes, add `cache_control` to the catalogue
|
|
118
|
+
block." That gives each conversation a cache entry of its own, a premium cache write on every
|
|
119
|
+
first turn, and a bill that drops far less than expected. The team blames caching, and the
|
|
120
|
+
dead instructions marker stays in the code.
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "prompt-caching-advisor",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"description": "Analyzes a prompt or conversation structure and identifies the highest-value cache breakpoints — where to insert cache_control checkpoints to minimize repeated token processing. Use when designing or optimizing a Claude API integration that uses prompt caching, when input costs are dominated by a static system prompt or few-shot block, or when a multi-turn conversation reprocesses the same context on every turn.",
|
|
5
5
|
"entrypoint": "SKILL.md",
|
|
6
6
|
"deps": { "npm": [], "pip": [] },
|
|
7
7
|
"env": { "required": [], "optional": [] },
|
|
8
8
|
"related": ["token-budget-estimator"],
|
|
9
9
|
"tags": ["claude-api", "caching", "cost", "optimization", "prompt-engineering"],
|
|
10
|
-
"model": { "tier": "
|
|
10
|
+
"model": { "tier": "heavy", "rationale": "Measured (RFC-0073): on its held-out evals the heavy tier met 90% and 90% of checks in two tier trials, against standard's 63% and light's 83% and 73%, more than 5 points ahead both times; moved once its examples/ oracle shipped (RFC-0074)." }
|
|
11
11
|
}
|
|
@@ -5,7 +5,7 @@ description: Take merged code to live users through a planned, reversible rollou
|
|
|
5
5
|
|
|
6
6
|
# release
|
|
7
7
|
|
|
8
|
-
This loop covers the step between `build`, which ends when a human merges, and
|
|
8
|
+
This loop covers the step between `build`, which ends when a human merges, and running the service,
|
|
9
9
|
which assumes the service is already live. A merge can be undone with a revert. A launch that
|
|
10
10
|
has touched users or data often cannot be undone, so this loop gets its own gates instead of
|
|
11
11
|
being folded into either neighbour.
|
|
@@ -39,8 +39,8 @@ being folded into either neighbour.
|
|
|
39
39
|
`council-review`. `RECONSIDER` means it is ready but this is the wrong time, and the work
|
|
40
40
|
waits instead of going back to `plan`.
|
|
41
41
|
|
|
42
|
-
After `PROCEED`, the change is live and the work moves to
|
|
43
|
-
`observability-plan` or `runbook-generator` are the first things
|
|
42
|
+
After `PROCEED`, the change is live and the work moves to running it. Gaps that G3 sent to
|
|
43
|
+
`observability-plan` or `runbook-generator` are the first things to pick up there.
|
|
44
44
|
|
|
45
45
|
## Verdicts
|
|
46
46
|
|
|
@@ -83,7 +83,7 @@ which end in `BLOCKED`.
|
|
|
83
83
|
- **Rollback exists before readiness is judged.** A plan with no way back is the one thing that cannot be fixed after launch.
|
|
84
84
|
- **Readiness rests on evidence.** Every check links to something; "we're good" is `not met`.
|
|
85
85
|
- **Can and should are separate gates.** G3 says whether it can ship; G3.1 says whether it should ship now.
|
|
86
|
-
- **Detection gaps leave this loop as hand-offs.**
|
|
86
|
+
- **Detection gaps leave this loop as hand-offs.** Running the service starts with them, rather than discovering them during the first incident.
|
|
87
87
|
|
|
88
88
|
## Anti-patterns to avoid
|
|
89
89
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$comment": "RFC-0031. Covers the step between build (ends at merge, G2.1) and operate (starts at instrument). Its gates are G3 and G3.1: the whole number places the loop in the lifecycle (discover G0 through operate G4), and a decimal is a second gate inside the same loop (RFC-0044).",
|
|
3
3
|
"name": "release",
|
|
4
|
-
"version": "0.4.
|
|
4
|
+
"version": "0.4.1",
|
|
5
5
|
"description": "Take merged code to live users through a planned, reversible rollout — plan the phases and the rollback, judge readiness on evidence, draft the release notes, then a human makes the go/no-go call. Use when the user has merged a change and asks how to roll it out, whether they are ready to launch, or for a go/no-go decision.",
|
|
6
6
|
"entrypoint": "LOOP.md",
|
|
7
7
|
"kind": "loop",
|
|
@@ -33,6 +33,7 @@ Converts the bug report someone *sent* into the ticket an engineer can *start*
|
|
|
33
33
|
## Useful references in this skill
|
|
34
34
|
|
|
35
35
|
- [`templates/bug-ticket.md`](templates/bug-ticket.md) — the ticket skeleton with the severity/priority matrix and the no-repro diagnostics list
|
|
36
|
+
- [`examples/export-row-limit-escalation.md`](examples/export-row-limit-escalation.md) — worked example: a VP's S1/P1 for "exports losing data" that the log shows is a 10,000-row CSV cap, filed S3/P2
|
|
36
37
|
|
|
37
38
|
## Quality bar
|
|
38
39
|
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Worked example: an escalated "exports are losing data" report
|
|
2
|
+
|
|
3
|
+
Input → output oracle for `bug-triage`. Shows severity judged from the evidence rather than
|
|
4
|
+
the escalation, a paraphrased error replaced by the verbatim string, and a three-symptom
|
|
5
|
+
thread split into one bug, one more bug and one feature request.
|
|
6
|
+
|
|
7
|
+
## Input
|
|
8
|
+
|
|
9
|
+
> Can you turn this into a ticket for the Reports squad? Our VP of Customer Success wants it
|
|
10
|
+
> filed as S1/P1 today. Tallow & Finch renew next month and they're furious.
|
|
11
|
+
>
|
|
12
|
+
> **#cs-escalations (pasted):**
|
|
13
|
+
>
|
|
14
|
+
> **Dana (CSM), 09:12:** Tallow & Finch say exports are losing data. Their ops lead exported
|
|
15
|
+
> the Shipment Activity report for October and "half the shipments are gone". Got some random
|
|
16
|
+
> error when it finished. Started after last week's release I think.
|
|
17
|
+
>
|
|
18
|
+
> **Dana, 09:14:** Also their dates come out as MM/DD in the CSV even though their account is
|
|
19
|
+
> set to DD/MM. And they asked again for scheduled exports to SFTP.
|
|
20
|
+
>
|
|
21
|
+
> **Dana, 09:20:** Screenshot text from their ops lead: "Export ready. Some rows may not be
|
|
22
|
+
> included. (EXPORT_ROW_LIMIT_REACHED)"
|
|
23
|
+
>
|
|
24
|
+
> **Marco (support engineer), 09:41:** Pulled the job log:
|
|
25
|
+
>
|
|
26
|
+
> ```
|
|
27
|
+
> 2026-11-04T08:57:31Z export_job id=exp_88213 tenant=tallow-finch.example
|
|
28
|
+
> report=shipment_activity range=2026-10-01..2026-10-31 format=csv status=completed
|
|
29
|
+
> rows_matched=14212 rows_written=10000 warning=EXPORT_ROW_LIMIT_REACHED app=5.18.2
|
|
30
|
+
> ```
|
|
31
|
+
>
|
|
32
|
+
> **Marco, 09:43:** The shipments are all there in the app, btw. The report shows 14,212 rows
|
|
33
|
+
> on screen.
|
|
34
|
+
|
|
35
|
+
## Output
|
|
36
|
+
|
|
37
|
+
Filed as **S3/P2, not S1/P1.** Nothing stored was lost: the app still shows all 14,212 rows.
|
|
38
|
+
The export file stops at 10,000. That is a real bug with a likely workaround. The VP can raise
|
|
39
|
+
the priority to P1 for the renewal, but that changes priority only. The severity stays S3.
|
|
40
|
+
The thread also held two other items, split out at the bottom.
|
|
41
|
+
|
|
42
|
+
```markdown
|
|
43
|
+
# CSV export of Shipment Activity stops at 10,000 rows (EXPORT_ROW_LIMIT_REACHED); remaining rows missing from file
|
|
44
|
+
|
|
45
|
+
**Severity:** S3 — the export file is incomplete, stored data is intact, and splitting the date range likely works around it
|
|
46
|
+
**Priority:** P2 — one tenant confirmed, renewal next month; a workaround CS can hand over today removes the need to drop current work
|
|
47
|
+
**Blast radius:** any export over 10,000 rows, on any tenant `[assumption]`; one tenant confirmed `[verified]`
|
|
48
|
+
**Component (best guess):** report export job `[assumption]`
|
|
49
|
+
|
|
50
|
+
## Summary
|
|
51
|
+
|
|
52
|
+
A customer ops lead exported October's Shipment Activity report as CSV. The file holds 10,000
|
|
53
|
+
of the 14,212 matching rows `[verified: job log exp_88213]`. The dialog warned that rows "may
|
|
54
|
+
not be included" `[reported: screenshot text]`. All 14,212 rows still show in the app
|
|
55
|
+
`[verified: Marco, on screen]`.
|
|
56
|
+
|
|
57
|
+
## Environment
|
|
58
|
+
|
|
59
|
+
| Field | Value | Tag |
|
|
60
|
+
|---|---|---|
|
|
61
|
+
| App version | 5.18.2 | `[verified]` |
|
|
62
|
+
| Tenant | tallow-finch.example | `[verified]` |
|
|
63
|
+
| Report / range / format | shipment_activity, 2026-10-01..2026-10-31, CSV | `[verified]` |
|
|
64
|
+
| Time | 2026-11-04 08:57:31 UTC | `[verified]` |
|
|
65
|
+
| First seen / last worked | "after last week's release" | `[reported]` |
|
|
66
|
+
|
|
67
|
+
## Reproduction
|
|
68
|
+
|
|
69
|
+
**No repro yet.** One occurrence is confirmed in the log and nobody has reproduced it. Collect:
|
|
70
|
+
- An export of any report with more than 10,000 matching rows on a test tenant, run on 5.18.2.
|
|
71
|
+
Record rows_matched and rows_written from the job log.
|
|
72
|
+
- The downloaded file from exp_88213, to count its rows and check whether it ends cleanly at
|
|
73
|
+
row 10,000 or mid-row
|
|
74
|
+
- Export job logs from the 30 days before last week's release, filtered on
|
|
75
|
+
`rows_written=10000`
|
|
76
|
+
- A screenshot of the full export dialog, to check whether it states the limit or only "may
|
|
77
|
+
not be included"
|
|
78
|
+
|
|
79
|
+
## Expected vs actual
|
|
80
|
+
|
|
81
|
+
**Expected:** the CSV contains all 14,212 rows the report shows, or the export refuses and names the limit
|
|
82
|
+
**Actual:** the CSV contains 10,000 rows, the job status is `completed`, and the dialog shows a vague warning
|
|
83
|
+
**Error (verbatim):** `Export ready. Some rows may not be included. (EXPORT_ROW_LIMIT_REACHED)`
|
|
84
|
+
|
|
85
|
+
## Hypotheses (≤3 — each with a 5-minute check; none of these is a conclusion)
|
|
86
|
+
|
|
87
|
+
| # | Hypothesis | Quick check |
|
|
88
|
+
|---|---|---|
|
|
89
|
+
| H1 | 10,000 is a deliberate cap that was never documented | Search the export service config and docs for `10000` or `EXPORT_ROW_LIMIT` |
|
|
90
|
+
| H2 | The cap or its warning arrived in a recent release | Filter export logs before last week's release for `rows_written=10000` |
|
|
91
|
+
| H3 | Pagination stops after a fixed number of pages rather than a fixed row count | Export 10,001 rows and then 25,000 rows on a test tenant and compare rows_written |
|
|
92
|
+
|
|
93
|
+
## Questions for reporter (≤3, ranked by diagnostic value)
|
|
94
|
+
|
|
95
|
+
1. Has the ops lead had a complete export of more than 10,000 rows before? When was the last one?
|
|
96
|
+
2. Do two half-month exports (Oct 1–15 and Oct 16–31) each come out complete?
|
|
97
|
+
3. What does the file feed: a person reading it, or another system that will treat it as complete?
|
|
98
|
+
|
|
99
|
+
## Duplicate-search hints
|
|
100
|
+
|
|
101
|
+
`EXPORT_ROW_LIMIT_REACHED` · `export missing rows` · `csv export 10000`
|
|
102
|
+
|
|
103
|
+
## Split out (if the report contained more than one issue)
|
|
104
|
+
|
|
105
|
+
- CSV export writes dates as MM/DD when the account date format is DD/MM → to be filed as its own bug
|
|
106
|
+
- Scheduled exports to SFTP → feature request, not a bug; route to the product backlog
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Why this is the oracle
|
|
110
|
+
|
|
111
|
+
- **Severity is judged on worst plausible impact, and priority is judged separately.** The
|
|
112
|
+
reporter's words ("losing data", "half gone") and the VP's pressure both point at S1. The
|
|
113
|
+
evidence shows an incomplete file over intact data with a likely workaround, so the ticket
|
|
114
|
+
says S3. The VP's urgency is a legitimate input to priority, so the reply leaves that lever
|
|
115
|
+
with the VP and does not let it move severity.
|
|
116
|
+
- **Tags separate reported from verified.** "Half the shipments are gone" is `[reported]`. The
|
|
117
|
+
log shows 10,000 of 14,212, about 70%, written. "After last week's release" stays a
|
|
118
|
+
hypothesis with a check (H2), not a stated cause.
|
|
119
|
+
- **The verbatim error replaces the paraphrase, and one bug gets one ticket.** The "random
|
|
120
|
+
error" is quoted as the exact string, which is also the best duplicate-search term. The date
|
|
121
|
+
bug and the SFTP request get split out.
|
|
122
|
+
|
|
123
|
+
A cheaper pass titles it "Exports losing data", files S1/P1 as asked, writes "Cause: 10k row
|
|
124
|
+
limit introduced in 5.18" as fact, invents repro steps nobody ran, and puts the date bug in
|
|
125
|
+
the same ticket. That ticket gets fixed in a rush on a guessed cause, and the date bug goes
|
|
126
|
+
nowhere.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bug-triage",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Turn a vague bug report (Slack message, support ticket, \"it's broken on mobile sometimes\") into an actionable ticket — searchable symptom title, numbered repro steps from a clean state, expected vs actual, severity separated from priority, evidence tagged reported-vs-verified, and labeled hypotheses instead of root-cause guesses. Use when the user wants to triage a bug, write up a bug report, turn a complaint into a ticket, or make a flaky issue actionable.",
|
|
5
5
|
"entrypoint": "SKILL.md",
|
|
6
6
|
"deps": { "npm": [], "pip": [] },
|
|
@@ -10,5 +10,5 @@
|
|
|
10
10
|
{ "to": "feature-implement-loop", "when": "the defect is reproduced and scoped", "purpose": "Fix the triaged defect through the same capped loop a story gets.", "fallback": "If feature-implement-loop is absent, fix inline with a regression test that fails before the fix." }
|
|
11
11
|
],
|
|
12
12
|
"tags": ["bug-triage", "bug-report", "ticket", "qa", "repro", "severity"],
|
|
13
|
-
"model": { "tier": "
|
|
13
|
+
"model": { "tier": "heavy", "rationale": "Measured (RFC-0073): on its held-out evals the heavy tier met 92% and 92% of checks in two tier trials, against standard's 76% and light's 70% and 86%, more than 5 points ahead both times; moved once its examples/ oracle shipped (RFC-0074)." }
|
|
14
14
|
}
|
|
@@ -28,6 +28,7 @@ Turns "the business wants X" into the document that aligns everyone on **what pr
|
|
|
28
28
|
## Useful references in this skill
|
|
29
29
|
|
|
30
30
|
- [`templates/prd.md`](templates/prd.md) — the PRD skeleton with the solution-language test and MoSCoW table
|
|
31
|
+
- [`examples/approver-delay-not-slack.md`](examples/approver-delay-not-slack.md) — worked example: a brief that blames a missing Slack integration when the data says approvers sit on requests
|
|
31
32
|
|
|
32
33
|
## Quality bar
|
|
33
34
|
|