pi-skill-shiori 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/Docs/recommendation-surface.md +62 -0
- package/README.md +54 -1
- package/package.json +2 -1
- package/src/index.ts +51 -9
- package/src/prompt.ts +6 -6
- package/src/query.ts +13 -4
- package/src/recommendation-feedback.ts +177 -0
- package/src/recommendation-reason.ts +43 -0
- package/src/retrieval.ts +19 -10
- package/src/types.ts +3 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
This project follows semantic versioning.
|
|
6
6
|
|
|
7
|
+
## [0.6.0] - 2026-06-29
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Bounded recommendation reason badges (`[trigger]`, `[description]`, `[low match]`) on recommended skills in auto-inject, `/shiori:recommend`, and `shiori_recommend` surfaces.
|
|
12
|
+
- `Docs/recommendation-surface.md` describing badge vocabulary, a multi-skill example, and how badges relate to Always Visible Skill policy and recommendation metrics.
|
|
13
|
+
- Tests for reason present, absent, low-confidence, and badge formatting.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- Candidate summaries use compact badges instead of freeform `Why:` lines in the default Shiori surfaces.
|
|
18
|
+
|
|
7
19
|
## [0.5.0] - 2026-06-28
|
|
8
20
|
|
|
9
21
|
### Added
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Recommendation surface
|
|
2
|
+
|
|
3
|
+
This document describes how Pi Skill Shiori surfaces recommended skills and the bounded reason badges shown with each match.
|
|
4
|
+
|
|
5
|
+
## Reason badges
|
|
6
|
+
|
|
7
|
+
Shiori attaches a compact reason badge to each recommended skill. The vocabulary is intentionally small:
|
|
8
|
+
|
|
9
|
+
| Badge | Meaning |
|
|
10
|
+
|---|---|
|
|
11
|
+
| `[trigger]` | A policy `triggers.include` phrase matched the user task. |
|
|
12
|
+
| `[description]` | Skill name/description tokens matched strongly enough to pass `candidateInjection.minScore`. |
|
|
13
|
+
| `[low match]` | The skill passed `minScore` but sits near the threshold; treat as a weaker suggestion. |
|
|
14
|
+
|
|
15
|
+
When Shiori cannot classify a match, it omits the badge instead of inventing a freeform explanation.
|
|
16
|
+
|
|
17
|
+
## Where badges appear
|
|
18
|
+
|
|
19
|
+
- **Auto-inject candidates** in the system prompt (`## Pi Skill Shiori Candidates`)
|
|
20
|
+
- **`/shiori:recommend` review mode** notifications and follow-up picker labels
|
|
21
|
+
- **`shiori_recommend` tool** collapsed result line (badges only, no long reports)
|
|
22
|
+
- **Pre-load summaries** when skills are queued for the agent
|
|
23
|
+
|
|
24
|
+
Badges are labels, not confidence scores. Numeric scores remain available in expanded detail views when needed.
|
|
25
|
+
|
|
26
|
+
## Example: multiple surfaced skills
|
|
27
|
+
|
|
28
|
+
Task: `browser scraping screenshot`
|
|
29
|
+
|
|
30
|
+
Possible matches:
|
|
31
|
+
|
|
32
|
+
```text
|
|
33
|
+
Matched 2 skill(s): playwright-cli [trigger], gstack-browse [description]
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Auto-inject candidate block:
|
|
37
|
+
|
|
38
|
+
```text
|
|
39
|
+
## Pi Skill Shiori Candidates
|
|
40
|
+
|
|
41
|
+
Relevant skills:
|
|
42
|
+
- playwright-cli [trigger]: Browser automation, web scraping, screenshots. Load: shiori_load_skill({ skill: "playwright-cli" })
|
|
43
|
+
- gstack-browse [description]: Headless browser QA for scripted flows. Load: shiori_load_skill({ skill: "gstack-browse" })
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
A near-threshold match degrades gracefully:
|
|
47
|
+
|
|
48
|
+
```text
|
|
49
|
+
- notes-helper [low match]: Search markdown notes in the vault. Load: shiori_load_skill({ skill: "notes-helper" })
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Relation to Always Visible Skill
|
|
53
|
+
|
|
54
|
+
`alwaysVisible` controls which skills stay in the Skill Catalog during Zero-Catalog Mode. Reason badges apply only to **recommended** (`triggerable`) skills surfaced by retrieval — not to always-visible catalog entries.
|
|
55
|
+
|
|
56
|
+
Always-visible skills may still appear as recommendations when they match a task; their badge explains the retrieval reason, not catalog visibility.
|
|
57
|
+
|
|
58
|
+
## Relation to recommendation metrics
|
|
59
|
+
|
|
60
|
+
`/shiori:stats` recommendation feedback records offers, loads, and follow-through by source (`auto-inject`, `command`, `tool`). Metrics do **not** store reason badges or per-reason tallies.
|
|
61
|
+
|
|
62
|
+
Use badges to understand individual suggestions during a session. Use metrics to see whether recommendations are being loaded after they are offered.
|
package/README.md
CHANGED
|
@@ -221,7 +221,7 @@ This writes a generated review file next to `.pi/skill-shiori.yml`. Review it be
|
|
|
221
221
|
| `/shiori:bootstrap` | Generate a review draft policy from discovered skill descriptions. |
|
|
222
222
|
| `/shiori:reload` | Ask what to reload, then rebuild Shiori’s skill inventory and optionally ask Pi to reload runtime resources. Code changes may still need full restart. |
|
|
223
223
|
| `/shiori:recommend` | Ask what you need help with, then either pre-load matches and queue the task for the agent or review recommendations only. Planning phrases (e.g. `計画を立てたい`) return the dev-plan intake template without dumping SKILL bodies. |
|
|
224
|
-
| `/shiori:stats` | Show operational counters. |
|
|
224
|
+
| `/shiori:stats` | Show operational counters and a compact recommendation quality summary. |
|
|
225
225
|
|
|
226
226
|
## Tool
|
|
227
227
|
|
|
@@ -267,6 +267,59 @@ retrievalBackend: sqlite-fts
|
|
|
267
267
|
code: prompt-boundary-v3
|
|
268
268
|
```
|
|
269
269
|
|
|
270
|
+
## Recommendation reason badges
|
|
271
|
+
|
|
272
|
+
Recommended skills show compact reason badges so you can see why Shiori surfaced them:
|
|
273
|
+
|
|
274
|
+
- `[trigger]` — a policy trigger phrase matched
|
|
275
|
+
- `[description]` — skill name/description tokens matched strongly
|
|
276
|
+
- `[low match]` — passed `minScore` but near the threshold
|
|
277
|
+
|
|
278
|
+
Badges appear in auto-inject candidates, `/shiori:recommend` review mode, pre-load summaries, and the `shiori_recommend` tool line. Unclassified matches omit the badge instead of showing a long explanation.
|
|
279
|
+
|
|
280
|
+
Example with multiple surfaced skills:
|
|
281
|
+
|
|
282
|
+
```text
|
|
283
|
+
Matched 2 skill(s): playwright-cli [trigger], gstack-browse [description]
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
See [`Docs/recommendation-surface.md`](Docs/recommendation-surface.md) for how badges relate to Always Visible Skill policy and recommendation feedback metrics.
|
|
287
|
+
|
|
288
|
+
## Recommendation feedback metrics
|
|
289
|
+
|
|
290
|
+
`/shiori:stats` includes session-local recommendation feedback so operators can tune trigger rules from real outcomes instead of guesswork.
|
|
291
|
+
|
|
292
|
+
Recorded (bounded, in-memory for the current Pi session):
|
|
293
|
+
|
|
294
|
+
- recommendation offers and candidate slot counts per source (`auto-inject`, `command`, `tool`)
|
|
295
|
+
- load follow-through when recommended skills are pre-loaded or later loaded via `shiori_load_skill`
|
|
296
|
+
- abandoned recommendation flows (candidates shown but not loaded, e.g. review-only mode)
|
|
297
|
+
- zero-candidate queries (obvious misses)
|
|
298
|
+
- per-skill offered/loaded tallies (up to 50 skill names, oldest entries dropped first)
|
|
299
|
+
|
|
300
|
+
Not recorded:
|
|
301
|
+
|
|
302
|
+
- user prompts, task descriptions, or transcript bodies
|
|
303
|
+
- skill `SKILL.md` contents
|
|
304
|
+
- filesystem paths beyond skill names already visible in policy/inventory
|
|
305
|
+
|
|
306
|
+
Retention boundary:
|
|
307
|
+
|
|
308
|
+
- counters reset when the Pi session ends
|
|
309
|
+
- no disk persistence or cross-session analytics in this slice
|
|
310
|
+
- inspect the compact summary at the top of `/shiori:stats`, with full JSON below
|
|
311
|
+
|
|
312
|
+
Example summary line:
|
|
313
|
+
|
|
314
|
+
```text
|
|
315
|
+
Recommendation feedback (session-local, no prompts stored)
|
|
316
|
+
offers: 4 | candidate slots: 7 | loaded: 3
|
|
317
|
+
abandoned: 1 | zero-candidate queries: 2 | follow-through: 42.9%
|
|
318
|
+
auto-inject: offers=2, loaded=1, zeroHits=1
|
|
319
|
+
command: offers=1, loaded=2, zeroHits=1
|
|
320
|
+
top skills: auth-helper(1/2), deploy-kit(1/1)
|
|
321
|
+
```
|
|
322
|
+
|
|
270
323
|
## Development
|
|
271
324
|
|
|
272
325
|
```bash
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-skill-shiori",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Pi extension that reduces Agent Skill catalog tokens with policy-based skill retrieval and on-demand skill loading.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
"LICENSE",
|
|
28
28
|
"SECURITY.md",
|
|
29
29
|
"CHANGELOG.md",
|
|
30
|
+
"Docs",
|
|
30
31
|
"src"
|
|
31
32
|
],
|
|
32
33
|
"scripts": {
|
package/src/index.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type { SkillIndex } from "./indexer.js";
|
|
|
8
8
|
import { loadRecommendedSkills } from "./load-skills.js";
|
|
9
9
|
import { formatPlanningKickoffMessage, isPlanningIntent } from "./planning-kickoff.js";
|
|
10
10
|
import { createStats } from "./metrics.js";
|
|
11
|
+
import { createRecommendationFeedback } from "./recommendation-feedback.js";
|
|
11
12
|
import { getPolicyPath, loadPolicy } from "./policy.js";
|
|
12
13
|
import {
|
|
13
14
|
formatCandidateDetail,
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
formatRecommendKickoffMessage,
|
|
17
18
|
suppressSkillCatalog,
|
|
18
19
|
} from "./prompt.js";
|
|
20
|
+
import { formatReasonBadgeSuffix } from "./recommendation-reason.js";
|
|
19
21
|
import { retrieveCandidatesExpanded } from "./retrieval-expanded.js";
|
|
20
22
|
import type { SkillRecord } from "./types.js";
|
|
21
23
|
|
|
@@ -40,6 +42,7 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
40
42
|
let inventoryRoots: string[] = [];
|
|
41
43
|
let warnedAlwaysVisibleMissing = new Set<string>();
|
|
42
44
|
const stats = createStats();
|
|
45
|
+
const recommendationFeedback = createRecommendationFeedback();
|
|
43
46
|
|
|
44
47
|
async function reload(cwd: string): Promise<SkillIndex> {
|
|
45
48
|
const policy = await loadPolicy(cwd);
|
|
@@ -126,8 +129,16 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
126
129
|
}
|
|
127
130
|
|
|
128
131
|
const candidates = retrieveCandidatesExpanded(event.prompt, current.skills, current.policy, current);
|
|
129
|
-
if (candidates.length === 0)
|
|
130
|
-
|
|
132
|
+
if (candidates.length === 0) {
|
|
133
|
+
stats.zeroCandidateCount += 1;
|
|
134
|
+
recommendationFeedback.recordZeroCandidates("auto-inject");
|
|
135
|
+
} else {
|
|
136
|
+
stats.candidateHitCount += 1;
|
|
137
|
+
recommendationFeedback.recordRecommendation(
|
|
138
|
+
"auto-inject",
|
|
139
|
+
candidates.map((candidate) => candidate.skill.name),
|
|
140
|
+
);
|
|
141
|
+
}
|
|
131
142
|
|
|
132
143
|
const injection = formatCandidateInjection(candidates);
|
|
133
144
|
const candidateSection = injection ? `\n\n## Pi Skill Shiori Candidates\n\n${injection}\n` : "";
|
|
@@ -153,7 +164,13 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
153
164
|
const current = await ensureIndex(ctx.cwd);
|
|
154
165
|
const policy = withRecommendLimits(current);
|
|
155
166
|
const candidates = retrieveCandidatesExpanded(params.task, current.skills, policy, current);
|
|
167
|
+
const candidateNames = candidates.map((candidate) => candidate.skill.name);
|
|
156
168
|
const loaded = await loadRecommendedSkills(candidates);
|
|
169
|
+
recommendationFeedback.recordRecommendation(
|
|
170
|
+
"tool",
|
|
171
|
+
candidateNames,
|
|
172
|
+
loaded.map((entry) => entry.candidate.skill.name),
|
|
173
|
+
);
|
|
157
174
|
const summary = formatLoadedSkillsSummary(loaded);
|
|
158
175
|
const blocks = loaded.map(({ candidate, content }) => ({
|
|
159
176
|
type: "text" as const,
|
|
@@ -163,11 +180,16 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
163
180
|
content: summary
|
|
164
181
|
? [{ type: "text", text: summary }, ...blocks]
|
|
165
182
|
: [{ type: "text", text: "No matching skills for that task." }],
|
|
166
|
-
details: {
|
|
183
|
+
details: {
|
|
184
|
+
query: params.task,
|
|
185
|
+
matches: candidates.map((c) => `${c.skill.name}${formatReasonBadgeSuffix(c.reason)}`),
|
|
186
|
+
},
|
|
167
187
|
};
|
|
168
188
|
},
|
|
169
189
|
renderResult(result, { expanded }, theme) {
|
|
170
|
-
const details = result.details as
|
|
190
|
+
const details = result.details as
|
|
191
|
+
| { query?: string; matches?: string[] }
|
|
192
|
+
| undefined;
|
|
171
193
|
const names = details?.matches?.join(", ") ?? "none";
|
|
172
194
|
const collapsed = `${theme.fg("success", "✓ Shiori")} ${theme.bold(names)}`;
|
|
173
195
|
if (!expanded) return new Text(collapsed, 0, 0);
|
|
@@ -193,6 +215,7 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
193
215
|
if (!record) throw new Error(`Skill not found: ${params.skill}`);
|
|
194
216
|
const content = await readFile(record.path, "utf8");
|
|
195
217
|
stats.loadedSkillCount += 1;
|
|
218
|
+
recommendationFeedback.recordSkillLoaded(record.name);
|
|
196
219
|
const details: ShioriLoadDetails = {
|
|
197
220
|
skill: record.name,
|
|
198
221
|
path: record.path,
|
|
@@ -305,12 +328,18 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
305
328
|
|
|
306
329
|
const policy = withRecommendLimits(current);
|
|
307
330
|
const candidates = retrieveCandidatesExpanded(query, current.skills, policy, current);
|
|
331
|
+
const candidateNames = candidates.map((candidate) => candidate.skill.name);
|
|
308
332
|
const planning = isPlanningIntent(query);
|
|
309
333
|
const reviewOnly = action === RECOMMEND_REVIEW_ONLY;
|
|
310
334
|
const loaded = planning || reviewOnly ? [] : await loadRecommendedSkills(candidates);
|
|
335
|
+
recommendationFeedback.recordRecommendation(
|
|
336
|
+
"command",
|
|
337
|
+
candidateNames,
|
|
338
|
+
loaded.map((entry) => entry.candidate.skill.name),
|
|
339
|
+
);
|
|
311
340
|
const summary = planning || reviewOnly
|
|
312
341
|
? candidates.length > 0
|
|
313
|
-
? `Matched ${candidates.length} skill(s): ${candidates.map((c) => c.skill.name).join(", ")}`
|
|
342
|
+
? `Matched ${candidates.length} skill(s): ${candidates.map((c) => `${c.skill.name}${formatReasonBadgeSuffix(c.reason)}`).join(", ")}`
|
|
314
343
|
: ""
|
|
315
344
|
: formatLoadedSkillsSummary(loaded);
|
|
316
345
|
|
|
@@ -341,14 +370,16 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
341
370
|
}
|
|
342
371
|
|
|
343
372
|
const choice = await ctx.ui.select("Follow up on a recommendation:", [
|
|
344
|
-
...candidates.map((candidate) => candidate.skill.name),
|
|
373
|
+
...candidates.map((candidate) => `${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}`),
|
|
345
374
|
"Done",
|
|
346
375
|
]);
|
|
347
376
|
if (!choice || choice === "Done") {
|
|
348
377
|
return;
|
|
349
378
|
}
|
|
350
379
|
|
|
351
|
-
const selected = candidates.find(
|
|
380
|
+
const selected = candidates.find(
|
|
381
|
+
(candidate) => choice === `${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}`,
|
|
382
|
+
);
|
|
352
383
|
if (!selected) {
|
|
353
384
|
return;
|
|
354
385
|
}
|
|
@@ -369,9 +400,20 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
369
400
|
});
|
|
370
401
|
|
|
371
402
|
pi.registerCommand("shiori:stats", {
|
|
372
|
-
description: "Show Pi Skill Shiori operational counters",
|
|
403
|
+
description: "Show Pi Skill Shiori operational counters and recommendation feedback",
|
|
373
404
|
handler: async (_args, ctx) => {
|
|
374
|
-
|
|
405
|
+
const payload = {
|
|
406
|
+
...stats,
|
|
407
|
+
recommendationFeedback: recommendationFeedback.summarize(),
|
|
408
|
+
};
|
|
409
|
+
ctx.ui.notify(
|
|
410
|
+
[
|
|
411
|
+
recommendationFeedback.formatSummary(),
|
|
412
|
+
"",
|
|
413
|
+
JSON.stringify(payload, null, 2),
|
|
414
|
+
].join("\n"),
|
|
415
|
+
"info",
|
|
416
|
+
);
|
|
375
417
|
},
|
|
376
418
|
});
|
|
377
419
|
|
package/src/prompt.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { normalizeAlwaysVisible } from "./always-visible.js";
|
|
2
|
+
import { formatReasonBadgeSuffix } from "./recommendation-reason.js";
|
|
2
3
|
import type { SkillCandidate, SkillRecord, SuppressionStatus } from "./types.js";
|
|
3
4
|
|
|
4
5
|
export interface CatalogSuppressionResult {
|
|
@@ -39,7 +40,7 @@ export function formatCandidateInjection(candidates: SkillCandidate[]): string {
|
|
|
39
40
|
for (const candidate of candidates) {
|
|
40
41
|
const desc = compactDescription(candidate.skill.description, COMPACT_DESCRIPTION_LIMIT);
|
|
41
42
|
lines.push(
|
|
42
|
-
`- ${candidate.skill.name}
|
|
43
|
+
`- ${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}: ${desc} Load: shiori_load_skill({ skill: "${candidate.skill.name}" })`,
|
|
43
44
|
);
|
|
44
45
|
}
|
|
45
46
|
return lines.join("\n");
|
|
@@ -70,8 +71,8 @@ export function formatRecommendKickoffMessage(
|
|
|
70
71
|
|
|
71
72
|
for (const { candidate, content } of loaded) {
|
|
72
73
|
lines.push(
|
|
73
|
-
`### Skill: ${candidate.skill.name}`,
|
|
74
|
-
`Score: ${candidate.score.toFixed(2)}
|
|
74
|
+
`### Skill: ${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}`,
|
|
75
|
+
`Score: ${candidate.score.toFixed(2)}`,
|
|
75
76
|
`Path: ${candidate.skill.path}`,
|
|
76
77
|
"",
|
|
77
78
|
content.trim(),
|
|
@@ -87,17 +88,16 @@ export function formatLoadedSkillsSummary(loaded: Array<{ candidate: SkillCandid
|
|
|
87
88
|
const lines = [`Pre-loaded ${loaded.length} skill(s):`];
|
|
88
89
|
for (const { candidate } of loaded) {
|
|
89
90
|
const desc = compactDescription(candidate.skill.description, COMPACT_DESCRIPTION_LIMIT);
|
|
90
|
-
lines.push(`- ${candidate.skill.name} (${candidate.score.toFixed(2)}): ${desc}`);
|
|
91
|
+
lines.push(`- ${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)} (${candidate.score.toFixed(2)}): ${desc}`);
|
|
91
92
|
}
|
|
92
93
|
return lines.join("\n");
|
|
93
94
|
}
|
|
94
95
|
|
|
95
96
|
export function formatCandidateDetail(candidate: SkillCandidate): string {
|
|
96
97
|
return [
|
|
97
|
-
candidate.skill.name
|
|
98
|
+
`${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}`,
|
|
98
99
|
`Description: ${candidate.skill.description}`,
|
|
99
100
|
`Score: ${candidate.score.toFixed(2)}`,
|
|
100
|
-
`Why: ${candidate.why}`,
|
|
101
101
|
`Path: ${candidate.skill.path}`,
|
|
102
102
|
`Load: shiori_load_skill({ skill: "${candidate.skill.name}" })`,
|
|
103
103
|
].join("\n");
|
package/src/query.ts
CHANGED
|
@@ -28,10 +28,19 @@ const TERM_ALIASES: Record<string, string[]> = {
|
|
|
28
28
|
};
|
|
29
29
|
|
|
30
30
|
/** Whole-prompt intents → English queries that match planning skills in descriptions. */
|
|
31
|
-
const INTENT_QUERY_PACKS: Array<{ test: RegExp; queries: string[] }> = [
|
|
32
|
-
{
|
|
33
|
-
test:
|
|
34
|
-
|
|
31
|
+
const INTENT_QUERY_PACKS: Array<{ test: RegExp; queries: string[] }> = [
|
|
32
|
+
{
|
|
33
|
+
test:
|
|
34
|
+
/\bx\s+(?:api|data|export|follower|monitor|post|search|social|timeline|tweet)\b|\b(?:twitter|tweet|timeline|follower)\b|social(?:\s+media)?/i,
|
|
35
|
+
queries: [
|
|
36
|
+
"tweet search social media data",
|
|
37
|
+
"twitter timeline follower export",
|
|
38
|
+
"x api tweet monitor",
|
|
39
|
+
],
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
test: /計画|プラン|立てたい|planning/i,
|
|
43
|
+
queries: [
|
|
35
44
|
"to-prd PRD plan",
|
|
36
45
|
"to-issues breakdown plan",
|
|
37
46
|
"grill me plan design",
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
export type RecommendationSource = "auto-inject" | "command" | "tool";
|
|
2
|
+
|
|
3
|
+
export interface SourceFeedbackCounts {
|
|
4
|
+
offers: number;
|
|
5
|
+
candidateSlots: number;
|
|
6
|
+
loaded: number;
|
|
7
|
+
zeroHits: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface SkillFeedbackCounts {
|
|
11
|
+
offered: number;
|
|
12
|
+
loaded: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface RecommendationFeedbackSummary {
|
|
16
|
+
totalOffers: number;
|
|
17
|
+
totalCandidateSlots: number;
|
|
18
|
+
totalLoaded: number;
|
|
19
|
+
totalAbandoned: number;
|
|
20
|
+
zeroCandidateQueries: number;
|
|
21
|
+
followThroughRate: number | null;
|
|
22
|
+
bySource: Record<RecommendationSource, SourceFeedbackCounts>;
|
|
23
|
+
topSkills: Array<{ name: string; offered: number; loaded: number }>;
|
|
24
|
+
retention: {
|
|
25
|
+
maxSkillEntries: number;
|
|
26
|
+
skillEntries: number;
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const SOURCES: RecommendationSource[] = ["auto-inject", "command", "tool"];
|
|
31
|
+
const MAX_SKILL_ENTRIES = 50;
|
|
32
|
+
|
|
33
|
+
function emptySourceCounts(): SourceFeedbackCounts {
|
|
34
|
+
return { offers: 0, candidateSlots: 0, loaded: 0, zeroHits: 0 };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function emptyBySource(): Record<RecommendationSource, SourceFeedbackCounts> {
|
|
38
|
+
return {
|
|
39
|
+
"auto-inject": emptySourceCounts(),
|
|
40
|
+
command: emptySourceCounts(),
|
|
41
|
+
tool: emptySourceCounts(),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface RecommendationFeedback {
|
|
46
|
+
recordRecommendation(source: RecommendationSource, candidates: string[], loaded?: string[]): void;
|
|
47
|
+
recordZeroCandidates(source: RecommendationSource): void;
|
|
48
|
+
recordSkillLoaded(skillName: string): void;
|
|
49
|
+
summarize(): RecommendationFeedbackSummary;
|
|
50
|
+
formatSummary(): string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function createRecommendationFeedback(): RecommendationFeedback {
|
|
54
|
+
const bySource = emptyBySource();
|
|
55
|
+
const bySkill = new Map<string, SkillFeedbackCounts>();
|
|
56
|
+
const recentAutoInjectCandidates = new Set<string>();
|
|
57
|
+
|
|
58
|
+
let totalOffers = 0;
|
|
59
|
+
let totalCandidateSlots = 0;
|
|
60
|
+
let totalLoaded = 0;
|
|
61
|
+
let totalAbandoned = 0;
|
|
62
|
+
let zeroCandidateQueries = 0;
|
|
63
|
+
|
|
64
|
+
function bumpSkill(name: string, field: "offered" | "loaded"): void {
|
|
65
|
+
const current = bySkill.get(name) ?? { offered: 0, loaded: 0 };
|
|
66
|
+
current[field] += 1;
|
|
67
|
+
bySkill.set(name, current);
|
|
68
|
+
while (bySkill.size > MAX_SKILL_ENTRIES) {
|
|
69
|
+
const oldest = bySkill.keys().next().value;
|
|
70
|
+
if (oldest === undefined) break;
|
|
71
|
+
bySkill.delete(oldest);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function recordLoadedSkills(source: RecommendationSource, loaded: string[]): void {
|
|
76
|
+
for (const name of loaded) {
|
|
77
|
+
bumpSkill(name, "loaded");
|
|
78
|
+
bySource[source].loaded += 1;
|
|
79
|
+
totalLoaded += 1;
|
|
80
|
+
recentAutoInjectCandidates.delete(name);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
recordRecommendation(source, candidates, loaded = []) {
|
|
86
|
+
if (candidates.length === 0) {
|
|
87
|
+
this.recordZeroCandidates(source);
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
totalOffers += 1;
|
|
92
|
+
totalCandidateSlots += candidates.length;
|
|
93
|
+
const sourceCounts = bySource[source];
|
|
94
|
+
sourceCounts.offers += 1;
|
|
95
|
+
sourceCounts.candidateSlots += candidates.length;
|
|
96
|
+
|
|
97
|
+
for (const name of candidates) {
|
|
98
|
+
bumpSkill(name, "offered");
|
|
99
|
+
if (source === "auto-inject") {
|
|
100
|
+
recentAutoInjectCandidates.add(name);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (loaded.length > 0) {
|
|
105
|
+
recordLoadedSkills(source, loaded);
|
|
106
|
+
} else if (source !== "auto-inject") {
|
|
107
|
+
totalAbandoned += 1;
|
|
108
|
+
}
|
|
109
|
+
},
|
|
110
|
+
|
|
111
|
+
recordZeroCandidates(source) {
|
|
112
|
+
zeroCandidateQueries += 1;
|
|
113
|
+
bySource[source].zeroHits += 1;
|
|
114
|
+
},
|
|
115
|
+
|
|
116
|
+
recordSkillLoaded(skillName) {
|
|
117
|
+
if (!recentAutoInjectCandidates.has(skillName)) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
recordLoadedSkills("auto-inject", [skillName]);
|
|
121
|
+
},
|
|
122
|
+
|
|
123
|
+
summarize() {
|
|
124
|
+
const followThroughRate =
|
|
125
|
+
totalCandidateSlots > 0 ? Number((totalLoaded / totalCandidateSlots).toFixed(3)) : null;
|
|
126
|
+
|
|
127
|
+
const topSkills = [...bySkill.entries()]
|
|
128
|
+
.map(([name, counts]) => ({ name, ...counts }))
|
|
129
|
+
.sort((a, b) => b.offered - a.offered || a.name.localeCompare(b.name))
|
|
130
|
+
.slice(0, 10);
|
|
131
|
+
|
|
132
|
+
return {
|
|
133
|
+
totalOffers,
|
|
134
|
+
totalCandidateSlots,
|
|
135
|
+
totalLoaded,
|
|
136
|
+
totalAbandoned,
|
|
137
|
+
zeroCandidateQueries,
|
|
138
|
+
followThroughRate,
|
|
139
|
+
bySource,
|
|
140
|
+
topSkills,
|
|
141
|
+
retention: {
|
|
142
|
+
maxSkillEntries: MAX_SKILL_ENTRIES,
|
|
143
|
+
skillEntries: bySkill.size,
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
},
|
|
147
|
+
|
|
148
|
+
formatSummary() {
|
|
149
|
+
const summary = this.summarize();
|
|
150
|
+
const rate =
|
|
151
|
+
summary.followThroughRate === null ? "n/a" : `${(summary.followThroughRate * 100).toFixed(1)}%`;
|
|
152
|
+
const lines = [
|
|
153
|
+
"Recommendation feedback (session-local, no prompts stored)",
|
|
154
|
+
`offers: ${summary.totalOffers} | candidate slots: ${summary.totalCandidateSlots} | loaded: ${summary.totalLoaded}`,
|
|
155
|
+
`abandoned: ${summary.totalAbandoned} | zero-candidate queries: ${summary.zeroCandidateQueries} | follow-through: ${rate}`,
|
|
156
|
+
];
|
|
157
|
+
|
|
158
|
+
for (const source of SOURCES) {
|
|
159
|
+
const counts = summary.bySource[source];
|
|
160
|
+
if (counts.offers === 0 && counts.zeroHits === 0) continue;
|
|
161
|
+
lines.push(
|
|
162
|
+
`${source}: offers=${counts.offers}, loaded=${counts.loaded}, zeroHits=${counts.zeroHits}`,
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (summary.topSkills.length > 0) {
|
|
167
|
+
const skillLine = summary.topSkills
|
|
168
|
+
.slice(0, 5)
|
|
169
|
+
.map((skill) => `${skill.name}(${skill.loaded}/${skill.offered})`)
|
|
170
|
+
.join(", ");
|
|
171
|
+
lines.push(`top skills: ${skillLine}`);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return lines.join("\n");
|
|
175
|
+
},
|
|
176
|
+
};
|
|
177
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { RecommendationReasonKind, SkillCandidate } from "./types.js";
|
|
2
|
+
|
|
3
|
+
/** Score within this margin above minScore is treated as low-confidence. */
|
|
4
|
+
export const LOW_CONFIDENCE_MARGIN = 0.08;
|
|
5
|
+
|
|
6
|
+
export const RECOMMENDATION_REASON_LABELS: Record<RecommendationReasonKind, string> = {
|
|
7
|
+
trigger: "trigger",
|
|
8
|
+
description: "description",
|
|
9
|
+
"low-confidence": "low match",
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export type RecommendationMatchSource = "trigger" | "description";
|
|
13
|
+
|
|
14
|
+
export function classifyRecommendationReason(
|
|
15
|
+
source: RecommendationMatchSource,
|
|
16
|
+
score: number,
|
|
17
|
+
minScore: number,
|
|
18
|
+
): RecommendationReasonKind {
|
|
19
|
+
if (source === "trigger") return "trigger";
|
|
20
|
+
return score < minScore + LOW_CONFIDENCE_MARGIN ? "low-confidence" : "description";
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function formatReasonBadge(reason: RecommendationReasonKind | null | undefined): string {
|
|
24
|
+
if (!reason) return "";
|
|
25
|
+
const label = RECOMMENDATION_REASON_LABELS[reason];
|
|
26
|
+
return label ? `[${label}]` : "";
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function formatReasonBadgeSuffix(reason: RecommendationReasonKind | null | undefined): string {
|
|
30
|
+
const badge = formatReasonBadge(reason);
|
|
31
|
+
return badge ? ` ${badge}` : "";
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function attachRecommendationReason(
|
|
35
|
+
candidate: Omit<SkillCandidate, "reason">,
|
|
36
|
+
source: RecommendationMatchSource,
|
|
37
|
+
minScore: number,
|
|
38
|
+
): SkillCandidate {
|
|
39
|
+
return {
|
|
40
|
+
...candidate,
|
|
41
|
+
reason: classifyRecommendationReason(source, candidate.score, minScore),
|
|
42
|
+
};
|
|
43
|
+
}
|
package/src/retrieval.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { attachRecommendationReason } from "./recommendation-reason.js";
|
|
1
2
|
import type { ShioriPolicy, SkillCandidate, SkillRecord } from "./types.js";
|
|
2
3
|
|
|
3
4
|
interface RetrievalIndex {
|
|
@@ -51,22 +52,30 @@ function evaluateSkill(
|
|
|
51
52
|
const includes = skillPolicy?.triggers?.include ?? [];
|
|
52
53
|
const matchedInclude = includes.find((trigger) => includesNormalized(normalizedQuery, trigger));
|
|
53
54
|
if (matchedInclude) {
|
|
54
|
-
return
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
55
|
+
return attachRecommendationReason(
|
|
56
|
+
{
|
|
57
|
+
skill,
|
|
58
|
+
score: 0.95,
|
|
59
|
+
why: `matched trigger "${matchedInclude}"`,
|
|
60
|
+
},
|
|
61
|
+
"trigger",
|
|
62
|
+
policy.candidateInjection.minScore,
|
|
63
|
+
);
|
|
59
64
|
}
|
|
60
65
|
if (exactTriggerOnly) return undefined;
|
|
61
66
|
|
|
62
67
|
const text = normalize(`${skill.name} ${skill.description} ${includes.join(" ")}`);
|
|
63
68
|
const tokenScore = scoreTokens(normalizedQuery, text);
|
|
64
69
|
if (tokenScore < policy.candidateInjection.minScore) return undefined;
|
|
65
|
-
return
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
+
return attachRecommendationReason(
|
|
71
|
+
{
|
|
72
|
+
skill,
|
|
73
|
+
score: tokenScore,
|
|
74
|
+
why: "matched skill description",
|
|
75
|
+
},
|
|
76
|
+
"description",
|
|
77
|
+
policy.candidateInjection.minScore,
|
|
78
|
+
);
|
|
70
79
|
}
|
|
71
80
|
|
|
72
81
|
function normalize(input: string): string {
|
package/src/types.ts
CHANGED
|
@@ -39,10 +39,13 @@ export interface SkillRecord {
|
|
|
39
39
|
source: string;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
+
export type RecommendationReasonKind = "trigger" | "description" | "low-confidence";
|
|
43
|
+
|
|
42
44
|
export interface SkillCandidate {
|
|
43
45
|
skill: SkillRecord;
|
|
44
46
|
score: number;
|
|
45
47
|
why: string;
|
|
48
|
+
reason: RecommendationReasonKind | null;
|
|
46
49
|
}
|
|
47
50
|
|
|
48
51
|
export interface ShioriStats {
|