skilldrop-cli 0.13.7 → 0.13.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/bin/skilldrop.js +455 -41
  2. package/contracts/pack.schema.json +26 -0
  3. package/package.json +1 -1
  4. package/packs/ai-engineering/pack.json +36 -1
  5. package/packs/ai-engineering/skills/agent-threat-model/SKILL.md +5 -2
  6. package/packs/ai-engineering/skills/agent-threat-model/manifest.json +1 -1
  7. package/packs/ai-engineering/skills/agent-threat-model/reference.md +19 -0
  8. package/packs/ai-engineering/skills/agent-threat-model/templates/agent-threat-model.md +3 -3
  9. package/packs/ai-engineering/skills/ai-usage-report/evals/eval_queries.json +30 -0
  10. package/packs/ai-engineering/skills/ai-usage-report/evals/evals.json +19 -0
  11. package/packs/ai-engineering/skills/llm-eval-harness/evals/eval_queries.json +34 -0
  12. package/packs/ai-engineering/skills/llm-eval-harness/evals/evals.json +19 -0
  13. package/packs/claude-api/pack.json +27 -0
  14. package/packs/claude-api/skills/eval-harness-generator/evals/eval_queries.json +30 -0
  15. package/packs/claude-api/skills/eval-harness-generator/evals/evals.json +19 -0
  16. package/packs/claude-api/skills/prompt-caching-advisor/evals/eval_queries.json +30 -0
  17. package/packs/claude-api/skills/prompt-caching-advisor/evals/evals.json +19 -0
  18. package/packs/claude-api/skills/token-budget-estimator/evals/eval_queries.json +30 -0
  19. package/packs/claude-api/skills/token-budget-estimator/evals/evals.json +18 -0
  20. package/packs/claude-api/skills/tool-use-schema-writer/evals/eval_queries.json +30 -0
  21. package/packs/claude-api/skills/tool-use-schema-writer/evals/evals.json +18 -0
  22. package/packs/core/pack.json +29 -0
  23. package/packs/design/pack.json +39 -1
  24. package/packs/dev-team/pack.json +35 -1
  25. package/packs/dev-team/skills/accessibility-audit/evals/eval_queries.json +30 -0
  26. package/packs/dev-team/skills/accessibility-audit/evals/evals.json +19 -0
  27. package/packs/dev-team/skills/contribution-wizard/evals/eval_queries.json +30 -0
  28. package/packs/dev-team/skills/contribution-wizard/evals/evals.json +19 -0
  29. package/packs/dev-team/skills/feature-implement-loop/evals/eval_queries.json +30 -0
  30. package/packs/dev-team/skills/feature-implement-loop/evals/evals.json +19 -0
  31. package/packs/dev-team/skills/release-notes/evals/eval_queries.json +30 -0
  32. package/packs/dev-team/skills/release-notes/evals/evals.json +19 -0
  33. package/packs/product-manager/pack.json +40 -1
  34. package/packs/product-manager/skills/business-case/evals/eval_queries.json +30 -0
  35. package/packs/product-manager/skills/business-case/evals/evals.json +19 -0
  36. package/packs/solution-architect/pack.json +44 -1
  37. package/packs/solution-architect/skills/tech-comparison-matrix/evals/eval_queries.json +30 -0
  38. package/packs/solution-architect/skills/tech-comparison-matrix/evals/evals.json +18 -0
  39. package/packs/sre-oncall/pack.json +33 -1
  40. package/packs/sre-oncall/skills/capacity-cost-model/evals/eval_queries.json +30 -0
  41. package/packs/sre-oncall/skills/capacity-cost-model/evals/evals.json +19 -0
  42. package/packs/stakeholder-comms/pack.json +39 -1
  43. package/packs/stakeholder-comms/skills/guide-builder/evals/eval_queries.json +34 -0
  44. package/packs/stakeholder-comms/skills/guide-builder/evals/evals.json +18 -0
@@ -40,6 +40,32 @@
40
40
  "recovery": { "type": "string" }
41
41
  }
42
42
  },
43
+ "journey": {
44
+ "type": "object",
45
+ "additionalProperties": false,
46
+ "description": "A typical session with the pack, rendered as its walkthrough page: when you'd reach for it, what you bring, each step's skill with what you do, what you get and where you decide, and what you end with. Every step's skill must sit in this pack or in a pack it requires.",
47
+ "required": ["when", "you_bring", "steps", "you_end_with"],
48
+ "properties": {
49
+ "when": { "type": "string" },
50
+ "you_bring": { "type": "array", "items": { "type": "string" } },
51
+ "time": { "type": "string", "description": "A rough range for one session; left out when it depends too much on the work." },
52
+ "steps": {
53
+ "type": "array",
54
+ "items": {
55
+ "type": "object",
56
+ "additionalProperties": false,
57
+ "required": ["skill", "you_do", "you_get"],
58
+ "properties": {
59
+ "skill": { "type": "string" },
60
+ "you_do": { "type": "string" },
61
+ "you_get": { "type": "string" },
62
+ "decide": { "type": "string", "description": "The human decision at this step, if any: a loop gate or a judgment call." }
63
+ }
64
+ }
65
+ },
66
+ "you_end_with": { "type": "string" }
67
+ }
68
+ },
43
69
  "requires": {
44
70
  "type": "array",
45
71
  "items": { "type": "string" },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "skilldrop-cli",
3
- "version": "0.13.7",
3
+ "version": "0.13.8",
4
4
  "description": "Portable AI-agent skills for the deliverables knowledge workers ship \u2014 diagrams, ADRs, PRDs, runbooks, decks, reviews \u2014 installable into Claude Code, Cursor, Kiro, and any IDE that takes custom instructions. One command per skill or per role-based pack.",
5
5
  "author": {
6
6
  "name": "Sanjay Ananthanarayan",
@@ -18,5 +18,40 @@
18
18
  "verification": "ai-readiness-assessment scores six dimensions (data, tooling, skills, governance, process, culture) with evidence for each, and ranks the gaps that block adoption first.",
19
19
  "recovery": "If ai-readiness-assessment does not activate, ask for it by name and check that its folder exists in your skills folder (`~/.claude/skills/` by default)."
20
20
  },
21
- "requires": ["core"]
21
+ "requires": ["core"],
22
+ "journey": {
23
+ "when": "A team wants to adopt AI tools and needs to know whether it is ready, where to start, what the rules are, and how to roll out.",
24
+ "you_bring": [
25
+ "the team or organisation in scope and what the answer will feed (a go/no-go, a budget, a sequence)",
26
+ "evidence about current tools, data, policies and practices",
27
+ "the candidate AI use cases",
28
+ "any regulatory or contractual limits on data"
29
+ ],
30
+ "steps": [
31
+ {
32
+ "skill": "ai-readiness-assessment",
33
+ "you_do": "Describe the team, its tools and data, and the concerns already raised.",
34
+ "you_get": "Scores from 0 to 4 on data, tooling, skills, governance, process and culture, each with an evidence line, the gaps ranked by what blocks first, and a one-sentence posture.",
35
+ "decide": "Decide whether to go ahead, and at what scope, from the stated posture."
36
+ },
37
+ {
38
+ "skill": "ai-use-case-triage",
39
+ "you_do": "List the candidate use cases.",
40
+ "you_get": "A ranked portfolio scored on value, feasibility and risk with the weights shown, one first slice with the observable that proves or kills it, and a not-yet list with promotion conditions.",
41
+ "decide": "Confirm the first slice."
42
+ },
43
+ {
44
+ "skill": "ai-usage-policy",
45
+ "you_do": "Give the population, the tools and any data rules already in force.",
46
+ "you_get": "An acceptable-use policy with three data tiers, permitted and prohibited uses as behaviours, a human-review matrix, approved tools with an exception route, and an owner per section."
47
+ },
48
+ {
49
+ "skill": "ai-adoption-rollout",
50
+ "you_do": "Name the chosen tool or workflow and who it is for.",
51
+ "you_get": "A rollout in waves with enablement per cohort, named champions, a comms plan that says what will not change, and an adoption gate on every wave.",
52
+ "decide": "Each wave's gate must be met before the next cohort opens."
53
+ }
54
+ ],
55
+ "you_end_with": "A readiness baseline, a chosen first use case, a usage policy, and a gated rollout plan."
56
+ }
22
57
  }
@@ -38,13 +38,15 @@ Complements `threat-model`, which runs STRIDE on the system the agent lives in.
38
38
 
39
39
  6. **Write residual risk with owners.** Anything not fixed is accepted, in writing, by a **named role** with the trigger that would force revisiting it. An unowned residual risk is an unrecorded decision.
40
40
 
41
- 7. **Emit in one message**: capability inventory, the matrix, fixes per 🟥, the three sweeps, residual risk with owners, and a **pre-launch checklist** of what to verify before the agent faces real data. End with the revisit trigger — the model is stale the moment a tool, a content source, or a credential scope is added.
41
+ 7. **Tag every 🟥 and 🟧 path with its OWASP IDs** from the table in [`reference.md`](reference.md): LLM Top 10 2025 (`LLM01:2025` prompt injection, `LLM02:2025` disclosure, `LLM06:2025` excessive agency, …) and, where the agent loads skills or plugins, the Agentic Skills Top 10 (`AST01`–`AST10`). A security team files findings against a framework; the tag lets them, without changing how the path is ranked.
42
+
43
+ 8. **Emit in one message**: capability inventory, the matrix, fixes per 🟥, the three sweeps, residual risk with owners, and a **pre-launch checklist** of what to verify before the agent faces real data. End with the revisit trigger — the model is stale the moment a tool, a content source, or a credential scope is added.
42
44
 
43
45
  **Non-interactive runs:** unstated capabilities are tagged `[assumption]` and modeled at their worst plausible scope. If no tool list, data reach, or content source can be established at all, emit `BLOCKED: need the agent's tool/permission inventory` — with nothing to inventory there is no model, and a fabricated one is worse than none.
44
46
 
45
47
  ## Useful references in this skill
46
48
 
47
- - [`reference.md`](reference.md) — the egress catalog (the sweep for step 1), the trust test for content sources, the ranked mitigation menu, and the injection-vector bank per source type
49
+ - [`reference.md`](reference.md) — the egress catalog (the sweep for step 1), the trust test for content sources, the ranked mitigation menu, the injection-vector bank per source type, and the OWASP ID mapping for step 7
48
50
  - [`templates/agent-threat-model.md`](templates/agent-threat-model.md) — output skeleton
49
51
  - [`examples/support-triage-agent.md`](examples/support-triage-agent.md) — worked example: a Zendesk triage agent, 4 paths, 2 🟥, one split that fixes both
50
52
 
@@ -56,6 +58,7 @@ Complements `threat-model`, which runs STRIDE on the system the agent lives in.
56
58
  - **Egress is enumerated past the network tool.** A model that lists only `http_request` failed the sweep — rendered markdown, error text, logs, and file writes are all channels.
57
59
  - **Every 🟥 is broken or signed off** by a named role with a revisit trigger. No silent acceptance.
58
60
  - **Every fix has a verification step** a reviewer can actually run — a config assertion, a test, an observed denial.
61
+ - **Every 🟥 and 🟧 carries its OWASP IDs,** taken from the mapping table, not guessed from a risk's title.
59
62
  - **"Read-only" is never a safety claim.** Reading is how data reaches the attacker; the question is only where it goes next.
60
63
 
61
64
  ## When to use this skill
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-threat-model",
3
- "version": "0.1.0",
3
+ "version": "0.2.0",
4
4
  "description": "Threat-model an AI agent deployment against the lethal trifecta — private data, untrusted content, and an exfiltration vector — producing a per-capability matrix, a named architectural fix for every unsafe path, and a pre-launch checklist. Use before shipping an agent, when reviewing MCP server or tool permissions, when the user asks about prompt injection or data exfiltration risk, or when deciding whether an agent's capability surface is safe to expose.",
5
5
  "entrypoint": "SKILL.md",
6
6
  "deps": { "npm": [], "pip": [] },
@@ -80,3 +80,22 @@ Prompts to run each source against when enumerating paths:
80
80
  | 🟧 | All three legs, held only by a human gate or an allowlist — the control is load-bearing and must be named and verified |
81
81
  | 🟨 | Two legs; the third arrives with a plausible change (a new tool, a widened credential) |
82
82
  | ⚪ | One leg, or explicitly accepted by a named owner with a revisit trigger |
83
+
84
+ ## OWASP mapping — step 7
85
+
86
+ Tag by what the path *does*, not by its keywords. A path usually carries two or three IDs.
87
+ Lists verified against [genai.owasp.org/llm-top-10](https://genai.owasp.org/llm-top-10/) (2025)
88
+ and the [OWASP Agentic Skills Top 10](https://owasp.org/www-project-agentic-skills-top-10/) (v1.0-2026).
89
+
90
+ | What the finding is | LLM Top 10 2025 | Agentic Skills Top 10 |
91
+ |---|---|---|
92
+ | Untrusted content can steer a tool call (any 🟥 or 🟧 path) | `LLM01:2025` Prompt Injection | `AST05` Untrusted External Instructions, when the content arrives through a skill |
93
+ | Private data can leave through an egress leg | `LLM02:2025` Sensitive Information Disclosure | — |
94
+ | A tool, credential or subagent reaches further than the task needs | `LLM06:2025` Excessive Agency | `AST03` Over-Privileged Skills |
95
+ | Agent output is rendered or executed downstream (markdown images, HTML, SQL, shell) | `LLM05:2025` Improper Output Handling | — |
96
+ | A secret or a boundary lives only in the system prompt | `LLM07:2025` System Prompt Leakage | — |
97
+ | Retrieved documents or an index can be written by untrusted parties | `LLM08:2025` Vector and Embedding Weaknesses, `LLM04:2025` Data and Model Poisoning | — |
98
+ | No cap on loops, tokens or spend | `LLM10:2025` Unbounded Consumption | — |
99
+ | A third-party skill, plugin or MCP server is in the capability inventory | `LLM03:2025` Supply Chain | `AST01` Malicious Skills, `AST02` Supply Chain Compromise |
100
+ | A skill runs with the host agent's full permissions | `LLM06:2025` | `AST06` Weak Isolation |
101
+ | Installed skills update without review | `LLM03:2025` | `AST07` Update Drift |
@@ -25,9 +25,9 @@
25
25
 
26
26
  One row per **path**: content source → tools it can influence → egress.
27
27
 
28
- | # | Path | Private data | Untrusted content | Egress | Rank |
29
- |---|---|---|---|---|---|
30
- | 1 | {source} → {tool} → {egress} | ✅ {what} | ✅ {who writes} | ✅ {channel} | 🟥 |
28
+ | # | Path | Private data | Untrusted content | Egress | Rank | OWASP |
29
+ |---|---|---|---|---|---|---|
30
+ | 1 | {source} → {tool} → {egress} | ✅ {what} | ✅ {who writes} | ✅ {channel} | 🟥 | {LLM01:2025, LLM02:2025} |
31
31
  | 2 | … | ✅ | ❌ | ✅ | ⚪ |
32
32
 
33
33
  ## Fixes
@@ -0,0 +1,30 @@
1
+ [
2
+ {
3
+ "query": "Build me a monthly AI adoption report from this usage-event CSV",
4
+ "should_trigger": true
5
+ },
6
+ {
7
+ "query": "Which teams are generating AI output but never shipping it? Here's the telemetry export",
8
+ "should_trigger": true
9
+ },
10
+ {
11
+ "query": "Turn this JSONL of MCP usage events into a team rollup I can show execs",
12
+ "should_trigger": true
13
+ },
14
+ {
15
+ "query": "I suspect some people are invoking Copilot just to hit a usage target — can the logs show that?",
16
+ "should_trigger": true
17
+ },
18
+ {
19
+ "query": "Design a golden set and regression gate for our support-bot prompt",
20
+ "should_trigger": false
21
+ },
22
+ {
23
+ "query": "Define the product success metrics for our new AI summarization feature",
24
+ "should_trigger": false
25
+ },
26
+ {
27
+ "query": "Write the policy on which AI tools staff may use with customer data",
28
+ "should_trigger": false
29
+ }
30
+ ]
@@ -0,0 +1,19 @@
1
+ {
2
+ "skill_name": "ai-usage-report",
3
+ "evals": [
4
+ {
5
+ "id": 1,
6
+ "prompt": "Here's our AI usage export for September 1-30: northwind-ai-events.csv, 4,212 rows, 14 engineers on the platform team. Columns are timestamp, user_email, tool_name, session_id, output_consumed. There's no discard field. I want the team-rollup view for our VP's monthly staff meeting — she wants to know whether people are actually using what the AI gives them or just clicking the button.",
7
+ "assertions": [
8
+ "The report header states the exact period covered (2026-09-01 to 2026-09-30)",
9
+ "The team-rollup names no individual anywhere — no emails, names, or per-person rows, even in passing",
10
+ "The handoff is three sentences above the report: volume, what the data does and doesn't show, and the most actionable pattern",
11
+ "The discard-rate signal is omitted with a note that the input has no discard field, rather than estimated or shown with '~' or 'est.' placeholders",
12
+ "The output-consumed rate and other numbers are computed from the input rows and reconcile with the 4,212-row count; no metric is invented",
13
+ "The 'AI theater' framing appears only in section headers and pattern-level observations, never as a judgment about a person",
14
+ "Recommendations target team patterns (e.g. a workshop), not specific people",
15
+ "No prompt text is embedded in the report; it works from metadata only"
16
+ ]
17
+ }
18
+ ]
19
+ }
@@ -0,0 +1,34 @@
1
+ [
2
+ {
3
+ "query": "How do we test this prompt change without shipping a quality regression?",
4
+ "should_trigger": true
5
+ },
6
+ {
7
+ "query": "Build an eval set and pass bar for our ticket-classification LLM",
8
+ "should_trigger": true
9
+ },
10
+ {
11
+ "query": "We want an LLM judge to grade helpfulness on our chatbot — how should we set that up?",
12
+ "should_trigger": true
13
+ },
14
+ {
15
+ "query": "Set up a regression gate before we keep iterating on our RAG prompts",
16
+ "should_trigger": true
17
+ },
18
+ {
19
+ "query": "Generate eval cases for this skilldrop SKILL.md so I can drop them in its evals folder",
20
+ "should_trigger": false
21
+ },
22
+ {
23
+ "query": "Summarize how much each team used AI tools last month from this telemetry",
24
+ "should_trigger": false
25
+ },
26
+ {
27
+ "query": "Compare Claude, GPT, and Gemini for our document-extraction use case and recommend one",
28
+ "should_trigger": false
29
+ },
30
+ {
31
+ "query": "What adoption and handle-time metrics should we track for the new AI feature?",
32
+ "should_trigger": false
33
+ }
34
+ ]
@@ -0,0 +1,19 @@
1
+ {
2
+ "skill_name": "llm-eval-harness",
3
+ "evals": [
4
+ {
5
+ "id": 1,
6
+ "prompt": "We have a RAG assistant that answers HR policy questions for Acme's 3,000 employees from a 400-page handbook. We're about to swap the model and rewrite the system prompt. We have about 1,200 real logged questions from the last quarter, and HR has hand-labeled 80 of them with correct answers. The answer that hurts most is confidently wrong parental-leave or termination guidance. Design the eval harness so we don't ship a regression.",
7
+ "assertions": [
8
+ "The golden set is split into representative, adversarial/edge, and regression buckets, with a case count and sourcing stated for each",
9
+ "Representative cases are drawn from the 1,200 logged questions and 80 HR labels the user supplied; any synthetic or model-generated cases are flagged as weak evidence",
10
+ "Each bucket or case names its grading method, and programmatic or structured-assertion checks are preferred over LLM-as-judge where the output is checkable",
11
+ "If an LLM judge is used (e.g. for faithfulness), it has its own rubric, is validated against the HR human labels, and controls for position, verbosity, and self-preference bias",
12
+ "The metric, the no-regression gate against baseline, and a critical subset (parental-leave and termination answers) held at 100% are all set before any run",
13
+ "A held-out split is defined and the rule against tuning on the test set is stated explicitly",
14
+ "Cost per run and p95 latency are tracked alongside the quality metric",
15
+ "A failure taxonomy (e.g. hallucination, refusal, missed-edge-case) is defined so iteration targets the biggest bucket"
16
+ ]
17
+ }
18
+ ]
19
+ }
@@ -17,5 +17,32 @@
17
17
  "starter-prompt": "Here is our system prompt and a typical request: <paste them>. Where should the cache_control breakpoints go, and what hit rate should we expect?",
18
18
  "verification": "prompt-caching-advisor marks where each cache_control checkpoint goes and estimates the cache hit rate.",
19
19
  "recovery": "If prompt-caching-advisor does not activate, ask for it by name and check that its folder exists in your skills folder (`~/.claude/skills/` by default)."
20
+ },
21
+ "journey": {
22
+ "when": "You are building or tuning a Claude API integration and want its tools defined properly and its token costs under control.",
23
+ "you_bring": [
24
+ "the functions your agent should call: name, purpose and parameters",
25
+ "your system prompt and a typical request",
26
+ "a description of the workflow: turns, tool results, output length"
27
+ ],
28
+ "time": "30–60 minutes",
29
+ "steps": [
30
+ {
31
+ "skill": "tool-use-schema-writer",
32
+ "you_do": "Describe each function the model should call.",
33
+ "you_get": "A tool_use JSON schema per function with types, required fields and parameter descriptions, plus a Python client.messages.create snippet."
34
+ },
35
+ {
36
+ "skill": "token-budget-estimator",
37
+ "you_do": "Describe the workflow or share a sample request and response.",
38
+ "you_get": "A per-component token table, the dominant cost driver, and max_tokens and other budget controls per component."
39
+ },
40
+ {
41
+ "skill": "prompt-caching-advisor",
42
+ "you_do": "Paste the system prompt and a typical request.",
43
+ "you_get": "Each block classed as static, quasi-static or dynamic, where each cache_control breakpoint goes with code, and an expected cache hit rate."
44
+ }
45
+ ],
46
+ "you_end_with": "Tool schemas ready to wire in, a token budget with its main cost driver named, and a caching plan."
20
47
  }
21
48
  }
@@ -0,0 +1,30 @@
1
+ [
2
+ {
3
+ "query": "Write eval cases for this SKILL.md so I can put them in its evals folder",
4
+ "should_trigger": true
5
+ },
6
+ {
7
+ "query": "Generate trigger queries and pass/fail criteria for the release-notes skill",
8
+ "should_trigger": true
9
+ },
10
+ {
11
+ "query": "I'm about to edit a skill — build me a regression set from its SKILL.md first",
12
+ "should_trigger": true
13
+ },
14
+ {
15
+ "query": "Check this skill behaves as documented: draft eval cases covering its anti-patterns",
16
+ "should_trigger": true
17
+ },
18
+ {
19
+ "query": "Design an eval harness for our customer-support chatbot's RAG pipeline",
20
+ "should_trigger": false
21
+ },
22
+ {
23
+ "query": "Help me author a brand-new skilldrop skill from scratch, manifest and all",
24
+ "should_trigger": false
25
+ },
26
+ {
27
+ "query": "We need a golden set and LLM-judge rubric for our summarization feature in production",
28
+ "should_trigger": false
29
+ }
30
+ ]
@@ -0,0 +1,19 @@
1
+ {
2
+ "skill_name": "eval-harness-generator",
3
+ "evals": [
4
+ {
5
+ "id": 1,
6
+ "prompt": "Here's the full SKILL.md for our new skilldrop skill `vendor-risk-screen` (a gate skill that reads a vendor security questionnaire and emits PROCEED, REVISE, or BLOCKED; its quality bar says it must never pass a vendor that has no SOC 2 report or equivalent, and must never invent certifications not in the questionnaire). [SKILL.md pasted below] Generate the eval cases for its evals/ folder.",
7
+ "assertions": [
8
+ "Produces 8 to 12 cases, each with id, trigger, expected_shape, must_not_include, verdict_class, and notes",
9
+ "Every trigger is a realistic input a user would paste (e.g. a specific questionnaire excerpt), not 'run this skill'",
10
+ "Every expected_shape item is checkable — it names a specific section, field, or behavior, not 'output is good'",
11
+ "At least 2 cases test the quality bar's failure modes, including a vendor with no SOC 2 report that must not get PROCEED",
12
+ "Because the skill is a gate, verdict_class values come from contracts/terminals.json and at least one case expects revise or blocked",
13
+ "At least one must_not_include entry forbids certifications or facts not present in the input questionnaire",
14
+ "A 3 to 5 sentence eval strategy note follows the JSON, naming the most important case and what a regression on it would mean",
15
+ "Ends with the placement reminder for packs/<pack>/skills/vendor-risk-screen/evals/ and running python3 validate.py"
16
+ ]
17
+ }
18
+ ]
19
+ }
@@ -0,0 +1,30 @@
1
+ [
2
+ {
3
+ "query": "Where should I put cache_control in this messages array?",
4
+ "should_trigger": true
5
+ },
6
+ {
7
+ "query": "Our input costs are dominated by a static 10k-token system prompt — how do we cache it?",
8
+ "should_trigger": true
9
+ },
10
+ {
11
+ "query": "Every turn of our chat agent reprocesses the same docs; help me place cache breakpoints",
12
+ "should_trigger": true
13
+ },
14
+ {
15
+ "query": "Why is our prompt cache hit rate near zero? Here's the request structure",
16
+ "should_trigger": true
17
+ },
18
+ {
19
+ "query": "Estimate how many tokens per session our research agent will use and what drives it",
20
+ "should_trigger": false
21
+ },
22
+ {
23
+ "query": "Set a per-agent spending cap and kill switch for our autonomous agents",
24
+ "should_trigger": false
25
+ },
26
+ {
27
+ "query": "Size max_tokens for each component of this agentic workflow",
28
+ "should_trigger": false
29
+ }
30
+ ]
@@ -0,0 +1,19 @@
1
+ {
2
+ "skill_name": "prompt-caching-advisor",
3
+ "evals": [
4
+ {
5
+ "id": 1,
6
+ "prompt": "Our Claude contract-review service on Sonnet sends this on every request: a system prompt of about 9,000 characters of review instructions, then a 'reference' block with our 40,000-character clause library, then a line 'Today's date: {now}' appended to the end of that reference block, then 6 fixed few-shot examples (~12,000 characters), then the user's uploaded contract (varies per request). It's single-turn, roughly 2,000 requests a day. Where should the cache_control breakpoints go?",
7
+ "assertions": [
8
+ "Flags that the per-request timestamp appended to the reference block makes that block dynamic and would cause every request to miss, and recommends moving it out of the cached prefix before caching",
9
+ "No recommended cached block contains dynamic content — the user's contract and the date stay outside every breakpoint",
10
+ "Shows an annotated messages/system structure with each cache_control placement marked and commented",
11
+ "Explains that a breakpoint caches the entire prefix up to and including that block, not just the block itself",
12
+ "Uses at most three breakpoints, one per logical boundary",
13
+ "Checks block sizes against the Sonnet minimum (~2,048 tokens) using ~4 characters per token before recommending a cache",
14
+ "Gives a cache strategy summary paragraph with what is cached, why, expected hit rate, and the dominant cost driver before and after",
15
+ "Does not invent prompt sections, request volumes, or models beyond those in the prompt"
16
+ ]
17
+ }
18
+ ]
19
+ }
@@ -0,0 +1,30 @@
1
+ [
2
+ {
3
+ "query": "How many tokens will this agentic workflow use per session?",
4
+ "should_trigger": true
5
+ },
6
+ {
7
+ "query": "Size a cost budget for our new Claude integration before we build it",
8
+ "should_trigger": true
9
+ },
10
+ {
11
+ "query": "Our agent's token usage is way higher than expected — break down where it's going",
12
+ "should_trigger": true
13
+ },
14
+ {
15
+ "query": "Suggest max_tokens limits per component for this multi-turn tool-using agent",
16
+ "should_trigger": true
17
+ },
18
+ {
19
+ "query": "Exactly where should the cache_control checkpoints go in this prompt?",
20
+ "should_trigger": false
21
+ },
22
+ {
23
+ "query": "Put spending caps and budget kill-switches on our fleet of autonomous agents",
24
+ "should_trigger": false
25
+ },
26
+ {
27
+ "query": "Design the agent loop: when to stop, retry, and hand back to the user",
28
+ "should_trigger": false
29
+ }
30
+ ]
@@ -0,0 +1,18 @@
1
+ {
2
+ "skill_name": "token-budget-estimator",
3
+ "evals": [
4
+ {
5
+ "id": 1,
6
+ "prompt": "We're planning a Claude agent that triages inbound support tickets for Northwind. System prompt is about 6,000 characters of instructions plus 4 few-shot examples of about 1,500 characters each. Each session averages 8 tool calls: 3 calls to search_kb return about 3 KB of JSON each, and 5 calls to get_ticket_history return about 10 KB of JSON each. We keep the full conversation history. Final output is a 200-word triage note. We'll run around 5,000 sessions a day. Estimate the token budget.",
7
+ "assertions": [
8
+ "Produces a component table (system prompt, few-shot examples, tool definitions, tool results, context accumulation, output, total per session), not a single total",
9
+ "Estimates from the description are ranges, not falsely precise single numbers like '3,142 tokens'",
10
+ "Uses ~4 chars/token for prose and ~3 chars/token for the JSON tool results",
11
+ "Names the single dominant cost driver (likely accumulated tool results / context re-sent across turns) and explains why it dominates",
12
+ "Gives exactly three concrete controls ranked by expected savings, with specific API parameters (max_tokens, cache_control, truncating or paginating tool results)",
13
+ "The rough cost estimate states its pricing-tier assumption rather than guessing silently",
14
+ "Does not invent workflow components, session counts, or payload sizes not in the prompt"
15
+ ]
16
+ }
17
+ ]
18
+ }
@@ -0,0 +1,30 @@
1
+ [
2
+ {
3
+ "query": "Write the tool_use JSON schema for this function so Claude can call it",
4
+ "should_trigger": true
5
+ },
6
+ {
7
+ "query": "Turn this TypeScript API client method into an Anthropic tool definition",
8
+ "should_trigger": true
9
+ },
10
+ {
11
+ "query": "I have three functions for my agent — write their tool schemas so the model picks the right one",
12
+ "should_trigger": true
13
+ },
14
+ {
15
+ "query": "Document this existing endpoint as a tool the agent can consume, with required fields",
16
+ "should_trigger": true
17
+ },
18
+ {
19
+ "query": "Design the control loop for my tool-using agent — stop conditions and retries",
20
+ "should_trigger": false
21
+ },
22
+ {
23
+ "query": "Split this workflow across an orchestrator and specialist subagents",
24
+ "should_trigger": false
25
+ },
26
+ {
27
+ "query": "Draft the OpenAPI contract for our new public REST API",
28
+ "should_trigger": false
29
+ }
30
+ ]
@@ -0,0 +1,18 @@
1
+ {
2
+ "skill_name": "tool-use-schema-writer",
3
+ "evals": [
4
+ {
5
+ "id": 1,
6
+ "prompt": "Write the Claude tool definition for this Python function in our Acme order service:\n\ndef search_orders(customer_id: str, status: str | None = None, placed_after: str | None = None, sort: str = 'desc', limit: int = 20):\n \"\"\"Find a customer's orders. status is one of pending, shipped, delivered, cancelled. placed_after is an ISO date. sort is by order date, asc or desc. limit 1-100.\"\"\"\n\nThe agent also has a separate get_order(order_id) tool, so the descriptions need to keep them apart.",
7
+ "assertions": [
8
+ "Produces a valid Anthropic tool definition with name, description, and input_schema of type object",
9
+ "The top-level description names a decision condition (when to call search_orders versus get_order), not just a restatement of the function name",
10
+ "A required array is present and contains customer_id; every name in required exists in properties",
11
+ "Every parameter has a non-empty description, and limit states its valid range (1-100)",
12
+ "The status and sort enums list their values with a one-phrase explanation of each value",
13
+ "Includes a Python snippet passing the tool to client.messages.create(tools=[...]) and handling stop_reason == 'tool_use'",
14
+ "Does not invent parameters, enum values, or behaviors not in the supplied signature and docstring"
15
+ ]
16
+ }
17
+ ]
18
+ }
@@ -17,5 +17,34 @@
17
17
  "starter-prompt": "Here are my notes from today's planning call: <paste them>. Turn them into a brief, then critique the draft you write from it.",
18
18
  "verification": "brief-intake returns a structured brief with gaps tagged, and doc-critique ends with a verdict: SHIP IT, SHIP WITH CHANGES, MAJOR REWRITE or WRONG ARTIFACT.",
19
19
  "recovery": "If neither skill activates, check that `brief-intake/SKILL.md` and `doc-critique/SKILL.md` exist in your skills folder (`~/.claude/skills/` by default). In Claude Code, start a new session after installing."
20
+ },
21
+ "journey": {
22
+ "when": "You have rough notes, a transcript or a ticket and want a finished document, not a first draft.",
23
+ "you_bring": [
24
+ "the raw material: notes, a Slack thread, a transcript or a ticket",
25
+ "the kind of document you want at the end, such as an ADR, a runbook or an exec summary",
26
+ "answers to the gaps the brief flags as missing"
27
+ ],
28
+ "time": "20–45 minutes",
29
+ "steps": [
30
+ {
31
+ "skill": "brief-intake",
32
+ "you_do": "Paste the raw material and name the document you want it to become.",
33
+ "you_get": "A structured brief with every field tagged explicit, implied, inferred or missing, plus a list of what the next step will refuse to start without.",
34
+ "decide": "Answer or explicitly accept each missing field before drafting; the brief never fills them in for you."
35
+ },
36
+ {
37
+ "skill": "doc-critique",
38
+ "you_do": "Hand over the draft you wrote from the brief, by hand or with any generator skill.",
39
+ "you_get": "A critique that opens with a verdict and lists findings as blocker, major, minor or nit, each pointing to the section and suggesting a fix.",
40
+ "decide": "Gate G4: SHIP IT or SHIP WITH CHANGES moves on; MAJOR REWRITE sends you back to the brief; WRONG ARTIFACT means you need a different kind of document."
41
+ },
42
+ {
43
+ "skill": "output-hygiene",
44
+ "you_do": "Run it on the settled draft and say where it is going (markdown, commit message, plain text).",
45
+ "you_get": "The draft with invisible Unicode, provenance trailers and chat closers stripped, prose tells quoted with rewrites, and a 'Not touched' list."
46
+ }
47
+ ],
48
+ "you_end_with": "A reviewed, cleaned document with its verdict, the changes it carried, and a note of what the scrub did not touch."
20
49
  }
21
50
  }
@@ -18,5 +18,43 @@
18
18
  "verification": "brand-kit writes brand.json and a one-page BRAND.md with contrast checked, and marketing-flyer writes flyer.html in your colours with your logo, one call to action, and only the facts you gave it.",
19
19
  "recovery": "If the flyer comes out in default colours, check that marketing-flyer was given the brand.json path. If brand-kit does not activate, ask for it by name (\"use brand-kit\")."
20
20
  },
21
- "requires": ["core"]
21
+ "requires": ["core"],
22
+ "journey": {
23
+ "when": "You need a deck or a flyer in your own branding and do not want to answer the brand questions every time.",
24
+ "you_bring": [
25
+ "your logo as a PNG, and a white version if you have one",
26
+ "brand colours, fonts or a PowerPoint template, if you know them",
27
+ "the content: the doc or points for the deck, and the event facts for a flyer",
28
+ "the audience and how long you have to present"
29
+ ],
30
+ "steps": [
31
+ {
32
+ "skill": "brand-kit",
33
+ "you_do": "Hand over your logo, colours and any brand guidelines or template.",
34
+ "you_get": "A brand.json and a one-page BRAND.md, with text contrast checked and each value traced to where it came from."
35
+ },
36
+ {
37
+ "skill": "slide-outliner",
38
+ "you_do": "Give it the content, the audience and the time slot.",
39
+ "you_get": "A slide-by-slide outline with takeaway titles, key points, speaker notes, backup slides for likely questions, and a slide-count check against the time."
40
+ },
41
+ {
42
+ "skill": "deck-builder",
43
+ "you_do": "Hand it the outline, the audience and your brand.json.",
44
+ "you_get": "An editable .pptx in your brand, with charts, tables and images where the content calls for them."
45
+ },
46
+ {
47
+ "skill": "doc-critique",
48
+ "you_do": "Run it on the deck against the deck rubric, as deck-builder recommends.",
49
+ "you_get": "A verdict and severity-tagged findings with a fix for each.",
50
+ "decide": "Ship the deck, apply the changes, or rework the outline."
51
+ },
52
+ {
53
+ "skill": "marketing-flyer",
54
+ "you_do": "If you also need a flyer, give it the event facts and the same brand.json.",
55
+ "you_get": "A print-ready flyer.html with one call to action and only the facts you gave it, which you can save to PDF."
56
+ }
57
+ ],
58
+ "you_end_with": "A reusable brand kit, a reviewed on-brand .pptx, and optionally a flyer built from the same brand."
59
+ }
22
60
  }