pi-skill-shiori 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/Docs/recommendation-surface.md +62 -0
- package/README.md +95 -10
- package/package.json +2 -1
- package/src/discovery.ts +36 -3
- package/src/index.ts +83 -17
- package/src/inventory.ts +58 -0
- package/src/metrics.ts +2 -0
- package/src/policy.ts +11 -0
- package/src/prompt.ts +6 -6
- package/src/query.ts +13 -4
- package/src/recommendation-feedback.ts +177 -0
- package/src/recommendation-reason.ts +43 -0
- package/src/retrieval.ts +19 -10
- package/src/types.ts +11 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,32 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
This project follows semantic versioning.
|
|
6
6
|
|
|
7
|
+
## [0.6.0] - 2026-06-29
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Bounded recommendation reason badges (`[trigger]`, `[description]`, `[low match]`) on recommended skills in auto-inject, `/shiori:recommend`, and `shiori_recommend` surfaces.
|
|
12
|
+
- `Docs/recommendation-surface.md` describing badge vocabulary, a multi-skill example, and how badges relate to Always Visible Skill policy and recommendation metrics.
|
|
13
|
+
- Tests for reason present, absent, low-confidence, and badge formatting.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- Candidate summaries use compact badges instead of freeform `Why:` lines in the default Shiori surfaces.
|
|
18
|
+
|
|
19
|
+
## [0.5.0] - 2026-06-28
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- Vault-wide skill inventory: Shiori indexes skills from project `.pi/skills`, `.agents/skills`, the user-global `~/.pi/agent/skills` root, and optional policy `inventory.roots`.
|
|
24
|
+
- Automatic inventory refresh when skill files change under any indexed root (enabled by default; disable with `inventory.autoRefreshOnChange: false`).
|
|
25
|
+
- `/shiori:doctor` now lists indexed roots with per-root skill counts and auto-refresh status.
|
|
26
|
+
- `/shiori:stats` counters for manual and automatic inventory refreshes.
|
|
27
|
+
- Tests for multi-root discovery, refresh after local changes, and policy-aware retrieval.
|
|
28
|
+
|
|
29
|
+
### Changed
|
|
30
|
+
|
|
31
|
+
- `/shiori:reload` rebuilds the vault-wide inventory and SQLite FTS index instead of relying on the session-start snapshot only.
|
|
32
|
+
|
|
7
33
|
## [0.4.0] - 2026-06-28
|
|
8
34
|
|
|
9
35
|
### Added
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Recommendation surface
|
|
2
|
+
|
|
3
|
+
This document describes how Pi Skill Shiori surfaces recommended skills and the bounded reason badges shown with each match.
|
|
4
|
+
|
|
5
|
+
## Reason badges
|
|
6
|
+
|
|
7
|
+
Shiori attaches a compact reason badge to each recommended skill. The vocabulary is intentionally small:
|
|
8
|
+
|
|
9
|
+
| Badge | Meaning |
|
|
10
|
+
|---|---|
|
|
11
|
+
| `[trigger]` | A policy `triggers.include` phrase matched the user task. |
|
|
12
|
+
| `[description]` | Skill name/description tokens matched strongly enough to pass `candidateInjection.minScore`. |
|
|
13
|
+
| `[low match]` | The skill passed `minScore` but sits near the threshold; treat as a weaker suggestion. |
|
|
14
|
+
|
|
15
|
+
When Shiori cannot classify a match, it omits the badge instead of inventing a freeform explanation.
|
|
16
|
+
|
|
17
|
+
## Where badges appear
|
|
18
|
+
|
|
19
|
+
- **Auto-inject candidates** in the system prompt (`## Pi Skill Shiori Candidates`)
|
|
20
|
+
- **`/shiori:recommend` review mode** notifications and follow-up picker labels
|
|
21
|
+
- **`shiori_recommend` tool** collapsed result line (badges only, no long reports)
|
|
22
|
+
- **Pre-load summaries** when skills are queued for the agent
|
|
23
|
+
|
|
24
|
+
Badges are labels, not confidence scores. Numeric scores remain available in expanded detail views when needed.
|
|
25
|
+
|
|
26
|
+
## Example: multiple surfaced skills
|
|
27
|
+
|
|
28
|
+
Task: `browser scraping screenshot`
|
|
29
|
+
|
|
30
|
+
Possible matches:
|
|
31
|
+
|
|
32
|
+
```text
|
|
33
|
+
Matched 2 skill(s): playwright-cli [trigger], gstack-browse [description]
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Auto-inject candidate block:
|
|
37
|
+
|
|
38
|
+
```text
|
|
39
|
+
## Pi Skill Shiori Candidates
|
|
40
|
+
|
|
41
|
+
Relevant skills:
|
|
42
|
+
- playwright-cli [trigger]: Browser automation, web scraping, screenshots. Load: shiori_load_skill({ skill: "playwright-cli" })
|
|
43
|
+
- gstack-browse [description]: Headless browser QA for scripted flows. Load: shiori_load_skill({ skill: "gstack-browse" })
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
A near-threshold match degrades gracefully:
|
|
47
|
+
|
|
48
|
+
```text
|
|
49
|
+
- notes-helper [low match]: Search markdown notes in the vault. Load: shiori_load_skill({ skill: "notes-helper" })
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Relation to Always Visible Skill
|
|
53
|
+
|
|
54
|
+
`alwaysVisible` controls which skills stay in the Skill Catalog during Zero-Catalog Mode. Reason badges apply only to **recommended** (`triggerable`) skills surfaced by retrieval — not to always-visible catalog entries.
|
|
55
|
+
|
|
56
|
+
Always-visible skills may still appear as recommendations when they match a task; their badge explains the retrieval reason, not catalog visibility.
|
|
57
|
+
|
|
58
|
+
## Relation to recommendation metrics
|
|
59
|
+
|
|
60
|
+
`/shiori:stats` recommendation feedback records offers, loads, and follow-through by source (`auto-inject`, `command`, `tool`). Metrics do **not** store reason badges or per-reason tallies.
|
|
61
|
+
|
|
62
|
+
Use badges to understand individual suggestions during a session. Use metrics to see whether recommendations are being loaded after they are offered.
|
package/README.md
CHANGED
|
@@ -12,12 +12,13 @@ For maintenance priorities and the phased plan, see [`ROADMAP.md`](ROADMAP.md).
|
|
|
12
12
|
- **Policy-based retrieval**: treats skills as explicit by default and only candidates skills marked as triggerable.
|
|
13
13
|
- **Compact candidate injection**: injects short skill suggestions instead of full `SKILL.md` files.
|
|
14
14
|
- **On-demand loading**: exposes `shiori_load_skill` so the model can load one selected skill body when needed.
|
|
15
|
+
- **Vault-wide skill inventory**: indexes skills from project and user-global roots, with optional extra roots in policy.
|
|
15
16
|
- **SQLite FTS retrieval**: indexes skill names, descriptions, and policy triggers with `node:sqlite` + FTS5, with token-match fallback.
|
|
16
17
|
- **Compact UI output**: keeps skill-load logs short while still passing full skill text to the model.
|
|
17
18
|
|
|
18
19
|
## Status
|
|
19
20
|
|
|
20
|
-
`0.
|
|
21
|
+
`0.5.0` is an early working release. It is useful for local Pi workflows, but the prompt-boundary suppression logic is intentionally conservative: if Shiori cannot safely recognize a catalog boundary, it leaves the prompt untouched and warns instead of deleting too much.
|
|
21
22
|
|
|
22
23
|
## Requirements
|
|
23
24
|
|
|
@@ -39,7 +40,7 @@ Pi core packages are peer dependencies and should be supplied by the Pi runtime:
|
|
|
39
40
|
After the package is published to npm:
|
|
40
41
|
|
|
41
42
|
```bash
|
|
42
|
-
pi install npm:pi-skill-shiori@0.
|
|
43
|
+
pi install npm:pi-skill-shiori@0.5.0
|
|
43
44
|
```
|
|
44
45
|
|
|
45
46
|
Without a version pin:
|
|
@@ -53,7 +54,7 @@ pi install npm:pi-skill-shiori
|
|
|
53
54
|
Use `-l` to write the package to the current project’s `.pi/settings.json`:
|
|
54
55
|
|
|
55
56
|
```bash
|
|
56
|
-
pi install -l npm:pi-skill-shiori@0.
|
|
57
|
+
pi install -l npm:pi-skill-shiori@0.5.0
|
|
57
58
|
```
|
|
58
59
|
|
|
59
60
|
Without a version pin:
|
|
@@ -67,7 +68,7 @@ Equivalent pinned manual `.pi/settings.json` entry:
|
|
|
67
68
|
```json
|
|
68
69
|
{
|
|
69
70
|
"packages": [
|
|
70
|
-
"npm:pi-skill-shiori@0.
|
|
71
|
+
"npm:pi-skill-shiori@0.5.0"
|
|
71
72
|
]
|
|
72
73
|
}
|
|
73
74
|
```
|
|
@@ -85,7 +86,7 @@ Equivalent unpinned manual `.pi/settings.json` entry:
|
|
|
85
86
|
### Global install from GitHub
|
|
86
87
|
|
|
87
88
|
```bash
|
|
88
|
-
pi install git:github.com/eiei114/pi-skill-shiori@v0.
|
|
89
|
+
pi install git:github.com/eiei114/pi-skill-shiori@v0.5.0
|
|
89
90
|
```
|
|
90
91
|
|
|
91
92
|
Without a tag pin:
|
|
@@ -97,7 +98,7 @@ pi install git:github.com/eiei114/pi-skill-shiori
|
|
|
97
98
|
### Project-local install from GitHub
|
|
98
99
|
|
|
99
100
|
```bash
|
|
100
|
-
pi install -l git:github.com/eiei114/pi-skill-shiori@v0.
|
|
101
|
+
pi install -l git:github.com/eiei114/pi-skill-shiori@v0.5.0
|
|
101
102
|
```
|
|
102
103
|
|
|
103
104
|
Without a tag pin:
|
|
@@ -131,7 +132,38 @@ Or add a relative local path to `.pi/settings.json`:
|
|
|
131
132
|
}
|
|
132
133
|
```
|
|
133
134
|
|
|
134
|
-
> Note: policy/index changes can be refreshed with `/shiori:reload`,
|
|
135
|
+
> Note: policy/index changes can be refreshed with `/shiori:reload`, and Shiori auto-refreshes the vault-wide inventory when skill files change under indexed roots. Extension code changes still require restarting the Pi process because Node keeps imported extension modules cached.
|
|
136
|
+
|
|
137
|
+
## Vault-wide skill inventory
|
|
138
|
+
|
|
139
|
+
Shiori builds a searchable inventory from these roots (in discovery order; first match wins on duplicate skill names):
|
|
140
|
+
|
|
141
|
+
| Root | Scope |
|
|
142
|
+
|---|---|
|
|
143
|
+
| `<cwd>/.pi/skills` | Project-local Pi skills |
|
|
144
|
+
| `<cwd>/.agents/skills` | Project-local Agent skills |
|
|
145
|
+
| `~/.pi/agent/skills` | User-global Pi agent skills |
|
|
146
|
+
| `inventory.roots` entries | Optional extra roots (relative to `<cwd>` or absolute) |
|
|
147
|
+
|
|
148
|
+
Refresh behavior:
|
|
149
|
+
|
|
150
|
+
- **Session start** builds the initial inventory and SQLite FTS index.
|
|
151
|
+
- **Auto-refresh** (default) rescans roots before retrieval when any `SKILL.md` path changes. This is a directory walk only; it rebuilds the FTS index when the fingerprint changes.
|
|
152
|
+
- **Manual refresh** via `/shiori:reload` always rebuilds the inventory and FTS index immediately.
|
|
153
|
+
- Disable auto-refresh when you prefer a fixed snapshot:
|
|
154
|
+
|
|
155
|
+
```yaml
|
|
156
|
+
inventory:
|
|
157
|
+
autoRefreshOnChange: false
|
|
158
|
+
roots:
|
|
159
|
+
- custom-skills
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Cost/latency trade-offs:
|
|
163
|
+
|
|
164
|
+
- Auto-refresh adds a shallow filesystem walk on each agent turn while enabled. It avoids stale search results after local skill edits without restarting Pi.
|
|
165
|
+
- A full refresh rebuilds the SQLite FTS database and scales with total indexed skills and description length. Large vaults with many roots may prefer `autoRefreshOnChange: false` plus explicit `/shiori:reload` after bulk imports.
|
|
166
|
+
- Search stays description-first and policy-aware: only `triggerable` skills become candidates, with FTS ranking over names, descriptions, and policy triggers.
|
|
135
167
|
|
|
136
168
|
## Configure
|
|
137
169
|
|
|
@@ -185,11 +217,11 @@ This writes a generated review file next to `.pi/skill-shiori.yml`. Review it be
|
|
|
185
217
|
|
|
186
218
|
| Command | Purpose |
|
|
187
219
|
|---|---|
|
|
188
|
-
| `/shiori:doctor` | Show policy path, inventory count, retrieval backend, suppression status, and code marker. |
|
|
220
|
+
| `/shiori:doctor` | Show policy path, indexed roots, inventory count, retrieval backend, suppression status, and code marker. |
|
|
189
221
|
| `/shiori:bootstrap` | Generate a review draft policy from discovered skill descriptions. |
|
|
190
222
|
| `/shiori:reload` | Ask what to reload, then rebuild Shiori’s skill inventory and optionally ask Pi to reload runtime resources. Code changes may still need full restart. |
|
|
191
223
|
| `/shiori:recommend` | Ask what you need help with, then either pre-load matches and queue the task for the agent or review recommendations only. Planning phrases (e.g. `計画を立てたい`) return the dev-plan intake template without dumping SKILL bodies. |
|
|
192
|
-
| `/shiori:stats` | Show operational counters. |
|
|
224
|
+
| `/shiori:stats` | Show operational counters and a compact recommendation quality summary. |
|
|
193
225
|
|
|
194
226
|
## Tool
|
|
195
227
|
|
|
@@ -235,6 +267,59 @@ retrievalBackend: sqlite-fts
|
|
|
235
267
|
code: prompt-boundary-v3
|
|
236
268
|
```
|
|
237
269
|
|
|
270
|
+
## Recommendation reason badges
|
|
271
|
+
|
|
272
|
+
Recommended skills show compact reason badges so you can see why Shiori surfaced them:
|
|
273
|
+
|
|
274
|
+
- `[trigger]` — a policy trigger phrase matched
|
|
275
|
+
- `[description]` — skill name/description tokens matched strongly
|
|
276
|
+
- `[low match]` — passed `minScore` but near the threshold
|
|
277
|
+
|
|
278
|
+
Badges appear in auto-inject candidates, `/shiori:recommend` review mode, pre-load summaries, and the `shiori_recommend` tool line. Unclassified matches omit the badge instead of showing a long explanation.
|
|
279
|
+
|
|
280
|
+
Example with multiple surfaced skills:
|
|
281
|
+
|
|
282
|
+
```text
|
|
283
|
+
Matched 2 skill(s): playwright-cli [trigger], gstack-browse [description]
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
See [`Docs/recommendation-surface.md`](Docs/recommendation-surface.md) for how badges relate to Always Visible Skill policy and recommendation feedback metrics.
|
|
287
|
+
|
|
288
|
+
## Recommendation feedback metrics
|
|
289
|
+
|
|
290
|
+
`/shiori:stats` includes session-local recommendation feedback so operators can tune trigger rules from real outcomes instead of guesswork.
|
|
291
|
+
|
|
292
|
+
Recorded (bounded, in-memory for the current Pi session):
|
|
293
|
+
|
|
294
|
+
- recommendation offers and candidate slot counts per source (`auto-inject`, `command`, `tool`)
|
|
295
|
+
- load follow-through when recommended skills are pre-loaded or later loaded via `shiori_load_skill`
|
|
296
|
+
- abandoned recommendation flows (candidates shown but not loaded, e.g. review-only mode)
|
|
297
|
+
- zero-candidate queries (obvious misses)
|
|
298
|
+
- per-skill offered/loaded tallies (up to 50 skill names, oldest entries dropped first)
|
|
299
|
+
|
|
300
|
+
Not recorded:
|
|
301
|
+
|
|
302
|
+
- user prompts, task descriptions, or transcript bodies
|
|
303
|
+
- skill `SKILL.md` contents
|
|
304
|
+
- filesystem paths beyond skill names already visible in policy/inventory
|
|
305
|
+
|
|
306
|
+
Retention boundary:
|
|
307
|
+
|
|
308
|
+
- counters reset when the Pi session ends
|
|
309
|
+
- no disk persistence or cross-session analytics in this slice
|
|
310
|
+
- inspect the compact summary at the top of `/shiori:stats`, with full JSON below
|
|
311
|
+
|
|
312
|
+
Example summary line:
|
|
313
|
+
|
|
314
|
+
```text
|
|
315
|
+
Recommendation feedback (session-local, no prompts stored)
|
|
316
|
+
offers: 4 | candidate slots: 7 | loaded: 3
|
|
317
|
+
abandoned: 1 | zero-candidate queries: 2 | follow-through: 42.9%
|
|
318
|
+
auto-inject: offers=2, loaded=1, zeroHits=1
|
|
319
|
+
command: offers=1, loaded=2, zeroHits=1
|
|
320
|
+
top skills: auth-helper(1/2), deploy-kit(1/1)
|
|
321
|
+
```
|
|
322
|
+
|
|
238
323
|
## Development
|
|
239
324
|
|
|
240
325
|
```bash
|
|
@@ -281,7 +366,7 @@ npm login
|
|
|
281
366
|
After publishing, verify the Pi install path:
|
|
282
367
|
|
|
283
368
|
```bash
|
|
284
|
-
pi install npm:pi-skill-shiori@0.
|
|
369
|
+
pi install npm:pi-skill-shiori@0.5.0
|
|
285
370
|
pi install npm:pi-skill-shiori
|
|
286
371
|
pi list
|
|
287
372
|
/shiori:doctor
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-skill-shiori",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Pi extension that reduces Agent Skill catalog tokens with policy-based skill retrieval and on-demand skill loading.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
"LICENSE",
|
|
28
28
|
"SECURITY.md",
|
|
29
29
|
"CHANGELOG.md",
|
|
30
|
+
"Docs",
|
|
30
31
|
"src"
|
|
31
32
|
],
|
|
32
33
|
"scripts": {
|
package/src/discovery.ts
CHANGED
|
@@ -1,18 +1,42 @@
|
|
|
1
1
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
|
-
import { basename, dirname, join, resolve } from "node:path";
|
|
3
|
+
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
|
4
4
|
import { parse as parseYaml } from "yaml";
|
|
5
|
-
import type { SkillRecord } from "./types.js";
|
|
5
|
+
import type { ShioriPolicy, SkillRecord } from "./types.js";
|
|
6
6
|
|
|
7
7
|
const SKILL_FILE = "SKILL.md";
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
/** Project-local skill roots scanned on every vault-wide inventory build. */
|
|
10
|
+
export const VAULT_LOCAL_SKILL_ROOT_SEGMENTS = [".pi/skills", ".agents/skills"] as const;
|
|
11
|
+
|
|
12
|
+
/** User-global Pi agent skills root (lowest discovery precedence). */
|
|
13
|
+
export const GLOBAL_AGENT_SKILL_ROOT_SEGMENTS = ".pi/agent/skills";
|
|
14
|
+
|
|
15
|
+
export function resolveVaultSkillRoots(cwd: string, policy?: Pick<ShioriPolicy, "inventory">): string[] {
|
|
10
16
|
const roots = [
|
|
11
17
|
join(cwd, ".pi", "skills"),
|
|
12
18
|
join(cwd, ".agents", "skills"),
|
|
13
19
|
join(homedir(), ".pi", "agent", "skills"),
|
|
14
20
|
];
|
|
15
21
|
|
|
22
|
+
for (const entry of policy?.inventory?.roots ?? []) {
|
|
23
|
+
const trimmed = entry.trim();
|
|
24
|
+
if (!trimmed) continue;
|
|
25
|
+
const resolved = isAbsolute(trimmed) ? resolve(trimmed) : resolve(cwd, trimmed);
|
|
26
|
+
if (!roots.includes(resolved)) {
|
|
27
|
+
roots.push(resolved);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
return roots;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export async function discoverSkills(cwd: string, policy?: ShioriPolicy): Promise<SkillRecord[]> {
|
|
35
|
+
const roots = resolveVaultSkillRoots(cwd, policy);
|
|
36
|
+
return discoverSkillsFromRoots(roots);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function discoverSkillsFromRoots(roots: string[]): Promise<SkillRecord[]> {
|
|
16
40
|
const records: SkillRecord[] = [];
|
|
17
41
|
for (const root of roots) {
|
|
18
42
|
const files = await findSkillFiles(root);
|
|
@@ -23,6 +47,15 @@ export async function discoverSkills(cwd: string): Promise<SkillRecord[]> {
|
|
|
23
47
|
return dedupeByDiscoveryOrder(records);
|
|
24
48
|
}
|
|
25
49
|
|
|
50
|
+
export async function computeInventoryFingerprint(roots: string[]): Promise<string> {
|
|
51
|
+
const parts: string[] = [];
|
|
52
|
+
for (const root of roots) {
|
|
53
|
+
const files = await findSkillFiles(root);
|
|
54
|
+
parts.push(`${root}:${files.sort().join(",")}`);
|
|
55
|
+
}
|
|
56
|
+
return parts.join("|");
|
|
57
|
+
}
|
|
58
|
+
|
|
26
59
|
export function findDuplicates(records: SkillRecord[]): Map<string, SkillRecord[]> {
|
|
27
60
|
const groups = new Map<string, SkillRecord[]>();
|
|
28
61
|
for (const record of records) {
|
package/src/index.ts
CHANGED
|
@@ -3,11 +3,12 @@ import { Text } from "@earendil-works/pi-tui";
|
|
|
3
3
|
import { Type } from "typebox";
|
|
4
4
|
import { readFile, writeFile } from "node:fs/promises";
|
|
5
5
|
import { evaluateAlwaysVisible, formatAlwaysVisibleDiagnostics } from "./always-visible.js";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { findDuplicates, formatInventoryRoots, isAutoRefreshEnabled, isInventoryStale, refreshSkillInventory } from "./inventory.js";
|
|
7
|
+
import type { SkillIndex } from "./indexer.js";
|
|
8
8
|
import { loadRecommendedSkills } from "./load-skills.js";
|
|
9
9
|
import { formatPlanningKickoffMessage, isPlanningIntent } from "./planning-kickoff.js";
|
|
10
10
|
import { createStats } from "./metrics.js";
|
|
11
|
+
import { createRecommendationFeedback } from "./recommendation-feedback.js";
|
|
11
12
|
import { getPolicyPath, loadPolicy } from "./policy.js";
|
|
12
13
|
import {
|
|
13
14
|
formatCandidateDetail,
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
formatRecommendKickoffMessage,
|
|
17
18
|
suppressSkillCatalog,
|
|
18
19
|
} from "./prompt.js";
|
|
20
|
+
import { formatReasonBadgeSuffix } from "./recommendation-reason.js";
|
|
19
21
|
import { retrieveCandidatesExpanded } from "./retrieval-expanded.js";
|
|
20
22
|
import type { SkillRecord } from "./types.js";
|
|
21
23
|
|
|
@@ -36,23 +38,43 @@ const CANCEL_CHOICE = "Cancel";
|
|
|
36
38
|
|
|
37
39
|
export default function piSkillShiori(pi: ExtensionAPI) {
|
|
38
40
|
let index: SkillIndex | undefined;
|
|
41
|
+
let inventoryFingerprint: string | undefined;
|
|
42
|
+
let inventoryRoots: string[] = [];
|
|
39
43
|
let warnedAlwaysVisibleMissing = new Set<string>();
|
|
40
44
|
const stats = createStats();
|
|
45
|
+
const recommendationFeedback = createRecommendationFeedback();
|
|
41
46
|
|
|
42
47
|
async function reload(cwd: string): Promise<SkillIndex> {
|
|
43
|
-
const
|
|
44
|
-
index
|
|
45
|
-
index =
|
|
46
|
-
|
|
47
|
-
|
|
48
|
+
const policy = await loadPolicy(cwd);
|
|
49
|
+
const inventory = await refreshSkillInventory(cwd, policy, index);
|
|
50
|
+
index = inventory.index;
|
|
51
|
+
inventoryFingerprint = inventory.fingerprint;
|
|
52
|
+
inventoryRoots = inventory.roots;
|
|
53
|
+
stats.inventoryCount = index.skills.length;
|
|
54
|
+
stats.duplicateCount = findDuplicates(index.skills).size;
|
|
48
55
|
stats.lastReloadAt = index.builtAt;
|
|
49
56
|
stats.retrievalBackend = index.retrievalBackend;
|
|
50
57
|
stats.suppressionStatus = policy.zeroCatalog.enabled ? "not-needed" : "disabled";
|
|
58
|
+
stats.inventoryRefreshCount += 1;
|
|
51
59
|
return index;
|
|
52
60
|
}
|
|
53
61
|
|
|
54
62
|
async function ensureIndex(cwd: string): Promise<SkillIndex> {
|
|
55
|
-
|
|
63
|
+
if (!index || !inventoryFingerprint) {
|
|
64
|
+
return reload(cwd);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const policy = await loadPolicy(cwd);
|
|
68
|
+
if (!isAutoRefreshEnabled(policy)) {
|
|
69
|
+
return index;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
if (await isInventoryStale(cwd, policy, inventoryFingerprint)) {
|
|
73
|
+
stats.inventoryAutoRefreshCount += 1;
|
|
74
|
+
return reload(cwd);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return index;
|
|
56
78
|
}
|
|
57
79
|
|
|
58
80
|
function withRecommendLimits(current: SkillIndex) {
|
|
@@ -72,6 +94,8 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
72
94
|
pi.on("session_shutdown", () => {
|
|
73
95
|
index?.close?.();
|
|
74
96
|
index = undefined;
|
|
97
|
+
inventoryFingerprint = undefined;
|
|
98
|
+
inventoryRoots = [];
|
|
75
99
|
});
|
|
76
100
|
|
|
77
101
|
pi.on("input", async (event) => {
|
|
@@ -105,8 +129,16 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
105
129
|
}
|
|
106
130
|
|
|
107
131
|
const candidates = retrieveCandidatesExpanded(event.prompt, current.skills, current.policy, current);
|
|
108
|
-
if (candidates.length === 0)
|
|
109
|
-
|
|
132
|
+
if (candidates.length === 0) {
|
|
133
|
+
stats.zeroCandidateCount += 1;
|
|
134
|
+
recommendationFeedback.recordZeroCandidates("auto-inject");
|
|
135
|
+
} else {
|
|
136
|
+
stats.candidateHitCount += 1;
|
|
137
|
+
recommendationFeedback.recordRecommendation(
|
|
138
|
+
"auto-inject",
|
|
139
|
+
candidates.map((candidate) => candidate.skill.name),
|
|
140
|
+
);
|
|
141
|
+
}
|
|
110
142
|
|
|
111
143
|
const injection = formatCandidateInjection(candidates);
|
|
112
144
|
const candidateSection = injection ? `\n\n## Pi Skill Shiori Candidates\n\n${injection}\n` : "";
|
|
@@ -132,7 +164,13 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
132
164
|
const current = await ensureIndex(ctx.cwd);
|
|
133
165
|
const policy = withRecommendLimits(current);
|
|
134
166
|
const candidates = retrieveCandidatesExpanded(params.task, current.skills, policy, current);
|
|
167
|
+
const candidateNames = candidates.map((candidate) => candidate.skill.name);
|
|
135
168
|
const loaded = await loadRecommendedSkills(candidates);
|
|
169
|
+
recommendationFeedback.recordRecommendation(
|
|
170
|
+
"tool",
|
|
171
|
+
candidateNames,
|
|
172
|
+
loaded.map((entry) => entry.candidate.skill.name),
|
|
173
|
+
);
|
|
136
174
|
const summary = formatLoadedSkillsSummary(loaded);
|
|
137
175
|
const blocks = loaded.map(({ candidate, content }) => ({
|
|
138
176
|
type: "text" as const,
|
|
@@ -142,11 +180,16 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
142
180
|
content: summary
|
|
143
181
|
? [{ type: "text", text: summary }, ...blocks]
|
|
144
182
|
: [{ type: "text", text: "No matching skills for that task." }],
|
|
145
|
-
details: {
|
|
183
|
+
details: {
|
|
184
|
+
query: params.task,
|
|
185
|
+
matches: candidates.map((c) => `${c.skill.name}${formatReasonBadgeSuffix(c.reason)}`),
|
|
186
|
+
},
|
|
146
187
|
};
|
|
147
188
|
},
|
|
148
189
|
renderResult(result, { expanded }, theme) {
|
|
149
|
-
const details = result.details as
|
|
190
|
+
const details = result.details as
|
|
191
|
+
| { query?: string; matches?: string[] }
|
|
192
|
+
| undefined;
|
|
150
193
|
const names = details?.matches?.join(", ") ?? "none";
|
|
151
194
|
const collapsed = `${theme.fg("success", "✓ Shiori")} ${theme.bold(names)}`;
|
|
152
195
|
if (!expanded) return new Text(collapsed, 0, 0);
|
|
@@ -172,6 +215,7 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
172
215
|
if (!record) throw new Error(`Skill not found: ${params.skill}`);
|
|
173
216
|
const content = await readFile(record.path, "utf8");
|
|
174
217
|
stats.loadedSkillCount += 1;
|
|
218
|
+
recommendationFeedback.recordSkillLoaded(record.name);
|
|
175
219
|
const details: ShioriLoadDetails = {
|
|
176
220
|
skill: record.name,
|
|
177
221
|
path: record.path,
|
|
@@ -210,6 +254,9 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
210
254
|
`duplicates: ${duplicates.size}`,
|
|
211
255
|
`alwaysVisible: ${current.policy.alwaysVisible.join(", ") || "none"}`,
|
|
212
256
|
...formatAlwaysVisibleDiagnostics(alwaysVisible),
|
|
257
|
+
"inventoryRoots:",
|
|
258
|
+
...formatInventoryRoots(inventoryRoots, current.skills).map((line) => ` - ${line}`),
|
|
259
|
+
`autoRefreshOnChange: ${isAutoRefreshEnabled(current.policy) ? "enabled" : "disabled"}`,
|
|
213
260
|
`retrievalBackend: ${current.retrievalBackend}`,
|
|
214
261
|
`suppression: ${stats.suppressionStatus}`,
|
|
215
262
|
`code: ${SHIORI_CODE_MARKER}`,
|
|
@@ -281,12 +328,18 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
281
328
|
|
|
282
329
|
const policy = withRecommendLimits(current);
|
|
283
330
|
const candidates = retrieveCandidatesExpanded(query, current.skills, policy, current);
|
|
331
|
+
const candidateNames = candidates.map((candidate) => candidate.skill.name);
|
|
284
332
|
const planning = isPlanningIntent(query);
|
|
285
333
|
const reviewOnly = action === RECOMMEND_REVIEW_ONLY;
|
|
286
334
|
const loaded = planning || reviewOnly ? [] : await loadRecommendedSkills(candidates);
|
|
335
|
+
recommendationFeedback.recordRecommendation(
|
|
336
|
+
"command",
|
|
337
|
+
candidateNames,
|
|
338
|
+
loaded.map((entry) => entry.candidate.skill.name),
|
|
339
|
+
);
|
|
287
340
|
const summary = planning || reviewOnly
|
|
288
341
|
? candidates.length > 0
|
|
289
|
-
? `Matched ${candidates.length} skill(s): ${candidates.map((c) => c.skill.name).join(", ")}`
|
|
342
|
+
? `Matched ${candidates.length} skill(s): ${candidates.map((c) => `${c.skill.name}${formatReasonBadgeSuffix(c.reason)}`).join(", ")}`
|
|
290
343
|
: ""
|
|
291
344
|
: formatLoadedSkillsSummary(loaded);
|
|
292
345
|
|
|
@@ -317,14 +370,16 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
317
370
|
}
|
|
318
371
|
|
|
319
372
|
const choice = await ctx.ui.select("Follow up on a recommendation:", [
|
|
320
|
-
...candidates.map((candidate) => candidate.skill.name),
|
|
373
|
+
...candidates.map((candidate) => `${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}`),
|
|
321
374
|
"Done",
|
|
322
375
|
]);
|
|
323
376
|
if (!choice || choice === "Done") {
|
|
324
377
|
return;
|
|
325
378
|
}
|
|
326
379
|
|
|
327
|
-
const selected = candidates.find(
|
|
380
|
+
const selected = candidates.find(
|
|
381
|
+
(candidate) => choice === `${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}`,
|
|
382
|
+
);
|
|
328
383
|
if (!selected) {
|
|
329
384
|
return;
|
|
330
385
|
}
|
|
@@ -345,9 +400,20 @@ export default function piSkillShiori(pi: ExtensionAPI) {
|
|
|
345
400
|
});
|
|
346
401
|
|
|
347
402
|
pi.registerCommand("shiori:stats", {
|
|
348
|
-
description: "Show Pi Skill Shiori operational counters",
|
|
403
|
+
description: "Show Pi Skill Shiori operational counters and recommendation feedback",
|
|
349
404
|
handler: async (_args, ctx) => {
|
|
350
|
-
|
|
405
|
+
const payload = {
|
|
406
|
+
...stats,
|
|
407
|
+
recommendationFeedback: recommendationFeedback.summarize(),
|
|
408
|
+
};
|
|
409
|
+
ctx.ui.notify(
|
|
410
|
+
[
|
|
411
|
+
recommendationFeedback.formatSummary(),
|
|
412
|
+
"",
|
|
413
|
+
JSON.stringify(payload, null, 2),
|
|
414
|
+
].join("\n"),
|
|
415
|
+
"info",
|
|
416
|
+
);
|
|
351
417
|
},
|
|
352
418
|
});
|
|
353
419
|
|
package/src/inventory.ts
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import {
|
|
2
|
+
computeInventoryFingerprint,
|
|
3
|
+
discoverSkillsFromRoots,
|
|
4
|
+
findDuplicates,
|
|
5
|
+
resolveVaultSkillRoots,
|
|
6
|
+
} from "./discovery.js";
|
|
7
|
+
import { buildSkillIndex, type SkillIndex } from "./indexer.js";
|
|
8
|
+
import type { ShioriPolicy } from "./types.js";
|
|
9
|
+
|
|
10
|
+
export interface SkillInventory {
|
|
11
|
+
index: SkillIndex;
|
|
12
|
+
roots: string[];
|
|
13
|
+
fingerprint: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function isAutoRefreshEnabled(policy: ShioriPolicy): boolean {
|
|
17
|
+
return policy.inventory?.autoRefreshOnChange !== false;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export async function refreshSkillInventory(
|
|
21
|
+
cwd: string,
|
|
22
|
+
policy: ShioriPolicy,
|
|
23
|
+
previous?: SkillIndex,
|
|
24
|
+
): Promise<SkillInventory> {
|
|
25
|
+
previous?.close?.();
|
|
26
|
+
const roots = resolveVaultSkillRoots(cwd, policy);
|
|
27
|
+
const [skills, fingerprint] = await Promise.all([
|
|
28
|
+
discoverSkillsFromRoots(roots),
|
|
29
|
+
computeInventoryFingerprint(roots),
|
|
30
|
+
]);
|
|
31
|
+
const index = await buildSkillIndex(cwd, skills, policy);
|
|
32
|
+
return { index, roots, fingerprint };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export async function isInventoryStale(
|
|
36
|
+
cwd: string,
|
|
37
|
+
policy: ShioriPolicy,
|
|
38
|
+
fingerprint: string,
|
|
39
|
+
): Promise<boolean> {
|
|
40
|
+
const roots = resolveVaultSkillRoots(cwd, policy);
|
|
41
|
+
const current = await computeInventoryFingerprint(roots);
|
|
42
|
+
return current !== fingerprint;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function countSkillsByRoot(skills: SkillIndex["skills"], roots: string[]): Map<string, number> {
|
|
46
|
+
const counts = new Map(roots.map((root) => [root, 0]));
|
|
47
|
+
for (const skill of skills) {
|
|
48
|
+
counts.set(skill.source, (counts.get(skill.source) ?? 0) + 1);
|
|
49
|
+
}
|
|
50
|
+
return counts;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function formatInventoryRoots(roots: string[], skills: SkillIndex["skills"]): string[] {
|
|
54
|
+
const counts = countSkillsByRoot(skills, roots);
|
|
55
|
+
return roots.map((root) => `${root} (${counts.get(root) ?? 0} skills)`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export { findDuplicates };
|
package/src/metrics.ts
CHANGED
package/src/policy.ts
CHANGED
|
@@ -30,6 +30,10 @@ export async function loadPolicy(cwd: string): Promise<ShioriPolicy> {
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
function normalizePolicy(policy: Partial<ShioriPolicy> | undefined): ShioriPolicy {
|
|
33
|
+
const extraRoots = (policy?.inventory?.roots ?? [])
|
|
34
|
+
.map((root) => (typeof root === "string" ? root.trim() : ""))
|
|
35
|
+
.filter(Boolean);
|
|
36
|
+
|
|
33
37
|
return {
|
|
34
38
|
zeroCatalog: {
|
|
35
39
|
enabled: policy?.zeroCatalog?.enabled ?? DEFAULT_POLICY.zeroCatalog.enabled,
|
|
@@ -42,6 +46,13 @@ function normalizePolicy(policy: Partial<ShioriPolicy> | undefined): ShioriPolic
|
|
|
42
46
|
minScore: policy?.candidateInjection?.minScore ?? DEFAULT_POLICY.candidateInjection.minScore,
|
|
43
47
|
},
|
|
44
48
|
alwaysVisible: normalizeAlwaysVisible(policy?.alwaysVisible ?? DEFAULT_POLICY.alwaysVisible).allowlist,
|
|
49
|
+
inventory:
|
|
50
|
+
extraRoots.length > 0 || policy?.inventory?.autoRefreshOnChange === false
|
|
51
|
+
? {
|
|
52
|
+
roots: extraRoots.length > 0 ? extraRoots : undefined,
|
|
53
|
+
autoRefreshOnChange: policy?.inventory?.autoRefreshOnChange,
|
|
54
|
+
}
|
|
55
|
+
: undefined,
|
|
45
56
|
skills: policy?.skills ?? DEFAULT_POLICY.skills,
|
|
46
57
|
};
|
|
47
58
|
}
|
package/src/prompt.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { normalizeAlwaysVisible } from "./always-visible.js";
|
|
2
|
+
import { formatReasonBadgeSuffix } from "./recommendation-reason.js";
|
|
2
3
|
import type { SkillCandidate, SkillRecord, SuppressionStatus } from "./types.js";
|
|
3
4
|
|
|
4
5
|
export interface CatalogSuppressionResult {
|
|
@@ -39,7 +40,7 @@ export function formatCandidateInjection(candidates: SkillCandidate[]): string {
|
|
|
39
40
|
for (const candidate of candidates) {
|
|
40
41
|
const desc = compactDescription(candidate.skill.description, COMPACT_DESCRIPTION_LIMIT);
|
|
41
42
|
lines.push(
|
|
42
|
-
`- ${candidate.skill.name}
|
|
43
|
+
`- ${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}: ${desc} Load: shiori_load_skill({ skill: "${candidate.skill.name}" })`,
|
|
43
44
|
);
|
|
44
45
|
}
|
|
45
46
|
return lines.join("\n");
|
|
@@ -70,8 +71,8 @@ export function formatRecommendKickoffMessage(
|
|
|
70
71
|
|
|
71
72
|
for (const { candidate, content } of loaded) {
|
|
72
73
|
lines.push(
|
|
73
|
-
`### Skill: ${candidate.skill.name}`,
|
|
74
|
-
`Score: ${candidate.score.toFixed(2)}
|
|
74
|
+
`### Skill: ${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}`,
|
|
75
|
+
`Score: ${candidate.score.toFixed(2)}`,
|
|
75
76
|
`Path: ${candidate.skill.path}`,
|
|
76
77
|
"",
|
|
77
78
|
content.trim(),
|
|
@@ -87,17 +88,16 @@ export function formatLoadedSkillsSummary(loaded: Array<{ candidate: SkillCandid
|
|
|
87
88
|
const lines = [`Pre-loaded ${loaded.length} skill(s):`];
|
|
88
89
|
for (const { candidate } of loaded) {
|
|
89
90
|
const desc = compactDescription(candidate.skill.description, COMPACT_DESCRIPTION_LIMIT);
|
|
90
|
-
lines.push(`- ${candidate.skill.name} (${candidate.score.toFixed(2)}): ${desc}`);
|
|
91
|
+
lines.push(`- ${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)} (${candidate.score.toFixed(2)}): ${desc}`);
|
|
91
92
|
}
|
|
92
93
|
return lines.join("\n");
|
|
93
94
|
}
|
|
94
95
|
|
|
95
96
|
export function formatCandidateDetail(candidate: SkillCandidate): string {
|
|
96
97
|
return [
|
|
97
|
-
candidate.skill.name
|
|
98
|
+
`${candidate.skill.name}${formatReasonBadgeSuffix(candidate.reason)}`,
|
|
98
99
|
`Description: ${candidate.skill.description}`,
|
|
99
100
|
`Score: ${candidate.score.toFixed(2)}`,
|
|
100
|
-
`Why: ${candidate.why}`,
|
|
101
101
|
`Path: ${candidate.skill.path}`,
|
|
102
102
|
`Load: shiori_load_skill({ skill: "${candidate.skill.name}" })`,
|
|
103
103
|
].join("\n");
|
package/src/query.ts
CHANGED
|
@@ -28,10 +28,19 @@ const TERM_ALIASES: Record<string, string[]> = {
|
|
|
28
28
|
};
|
|
29
29
|
|
|
30
30
|
/** Whole-prompt intents → English queries that match planning skills in descriptions. */
|
|
31
|
-
const INTENT_QUERY_PACKS: Array<{ test: RegExp; queries: string[] }> = [
|
|
32
|
-
{
|
|
33
|
-
test:
|
|
34
|
-
|
|
31
|
+
const INTENT_QUERY_PACKS: Array<{ test: RegExp; queries: string[] }> = [
|
|
32
|
+
{
|
|
33
|
+
test:
|
|
34
|
+
/\bx\s+(?:api|data|export|follower|monitor|post|search|social|timeline|tweet)\b|\b(?:twitter|tweet|timeline|follower)\b|social(?:\s+media)?/i,
|
|
35
|
+
queries: [
|
|
36
|
+
"tweet search social media data",
|
|
37
|
+
"twitter timeline follower export",
|
|
38
|
+
"x api tweet monitor",
|
|
39
|
+
],
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
test: /計画|プラン|立てたい|planning/i,
|
|
43
|
+
queries: [
|
|
35
44
|
"to-prd PRD plan",
|
|
36
45
|
"to-issues breakdown plan",
|
|
37
46
|
"grill me plan design",
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
export type RecommendationSource = "auto-inject" | "command" | "tool";
|
|
2
|
+
|
|
3
|
+
export interface SourceFeedbackCounts {
|
|
4
|
+
offers: number;
|
|
5
|
+
candidateSlots: number;
|
|
6
|
+
loaded: number;
|
|
7
|
+
zeroHits: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface SkillFeedbackCounts {
|
|
11
|
+
offered: number;
|
|
12
|
+
loaded: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface RecommendationFeedbackSummary {
|
|
16
|
+
totalOffers: number;
|
|
17
|
+
totalCandidateSlots: number;
|
|
18
|
+
totalLoaded: number;
|
|
19
|
+
totalAbandoned: number;
|
|
20
|
+
zeroCandidateQueries: number;
|
|
21
|
+
followThroughRate: number | null;
|
|
22
|
+
bySource: Record<RecommendationSource, SourceFeedbackCounts>;
|
|
23
|
+
topSkills: Array<{ name: string; offered: number; loaded: number }>;
|
|
24
|
+
retention: {
|
|
25
|
+
maxSkillEntries: number;
|
|
26
|
+
skillEntries: number;
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const SOURCES: RecommendationSource[] = ["auto-inject", "command", "tool"];
|
|
31
|
+
const MAX_SKILL_ENTRIES = 50;
|
|
32
|
+
|
|
33
|
+
function emptySourceCounts(): SourceFeedbackCounts {
|
|
34
|
+
return { offers: 0, candidateSlots: 0, loaded: 0, zeroHits: 0 };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function emptyBySource(): Record<RecommendationSource, SourceFeedbackCounts> {
|
|
38
|
+
return {
|
|
39
|
+
"auto-inject": emptySourceCounts(),
|
|
40
|
+
command: emptySourceCounts(),
|
|
41
|
+
tool: emptySourceCounts(),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface RecommendationFeedback {
|
|
46
|
+
recordRecommendation(source: RecommendationSource, candidates: string[], loaded?: string[]): void;
|
|
47
|
+
recordZeroCandidates(source: RecommendationSource): void;
|
|
48
|
+
recordSkillLoaded(skillName: string): void;
|
|
49
|
+
summarize(): RecommendationFeedbackSummary;
|
|
50
|
+
formatSummary(): string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function createRecommendationFeedback(): RecommendationFeedback {
|
|
54
|
+
const bySource = emptyBySource();
|
|
55
|
+
const bySkill = new Map<string, SkillFeedbackCounts>();
|
|
56
|
+
const recentAutoInjectCandidates = new Set<string>();
|
|
57
|
+
|
|
58
|
+
let totalOffers = 0;
|
|
59
|
+
let totalCandidateSlots = 0;
|
|
60
|
+
let totalLoaded = 0;
|
|
61
|
+
let totalAbandoned = 0;
|
|
62
|
+
let zeroCandidateQueries = 0;
|
|
63
|
+
|
|
64
|
+
function bumpSkill(name: string, field: "offered" | "loaded"): void {
|
|
65
|
+
const current = bySkill.get(name) ?? { offered: 0, loaded: 0 };
|
|
66
|
+
current[field] += 1;
|
|
67
|
+
bySkill.set(name, current);
|
|
68
|
+
while (bySkill.size > MAX_SKILL_ENTRIES) {
|
|
69
|
+
const oldest = bySkill.keys().next().value;
|
|
70
|
+
if (oldest === undefined) break;
|
|
71
|
+
bySkill.delete(oldest);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function recordLoadedSkills(source: RecommendationSource, loaded: string[]): void {
|
|
76
|
+
for (const name of loaded) {
|
|
77
|
+
bumpSkill(name, "loaded");
|
|
78
|
+
bySource[source].loaded += 1;
|
|
79
|
+
totalLoaded += 1;
|
|
80
|
+
recentAutoInjectCandidates.delete(name);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
recordRecommendation(source, candidates, loaded = []) {
|
|
86
|
+
if (candidates.length === 0) {
|
|
87
|
+
this.recordZeroCandidates(source);
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
totalOffers += 1;
|
|
92
|
+
totalCandidateSlots += candidates.length;
|
|
93
|
+
const sourceCounts = bySource[source];
|
|
94
|
+
sourceCounts.offers += 1;
|
|
95
|
+
sourceCounts.candidateSlots += candidates.length;
|
|
96
|
+
|
|
97
|
+
for (const name of candidates) {
|
|
98
|
+
bumpSkill(name, "offered");
|
|
99
|
+
if (source === "auto-inject") {
|
|
100
|
+
recentAutoInjectCandidates.add(name);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (loaded.length > 0) {
|
|
105
|
+
recordLoadedSkills(source, loaded);
|
|
106
|
+
} else if (source !== "auto-inject") {
|
|
107
|
+
totalAbandoned += 1;
|
|
108
|
+
}
|
|
109
|
+
},
|
|
110
|
+
|
|
111
|
+
recordZeroCandidates(source) {
|
|
112
|
+
zeroCandidateQueries += 1;
|
|
113
|
+
bySource[source].zeroHits += 1;
|
|
114
|
+
},
|
|
115
|
+
|
|
116
|
+
recordSkillLoaded(skillName) {
|
|
117
|
+
if (!recentAutoInjectCandidates.has(skillName)) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
recordLoadedSkills("auto-inject", [skillName]);
|
|
121
|
+
},
|
|
122
|
+
|
|
123
|
+
summarize() {
|
|
124
|
+
const followThroughRate =
|
|
125
|
+
totalCandidateSlots > 0 ? Number((totalLoaded / totalCandidateSlots).toFixed(3)) : null;
|
|
126
|
+
|
|
127
|
+
const topSkills = [...bySkill.entries()]
|
|
128
|
+
.map(([name, counts]) => ({ name, ...counts }))
|
|
129
|
+
.sort((a, b) => b.offered - a.offered || a.name.localeCompare(b.name))
|
|
130
|
+
.slice(0, 10);
|
|
131
|
+
|
|
132
|
+
return {
|
|
133
|
+
totalOffers,
|
|
134
|
+
totalCandidateSlots,
|
|
135
|
+
totalLoaded,
|
|
136
|
+
totalAbandoned,
|
|
137
|
+
zeroCandidateQueries,
|
|
138
|
+
followThroughRate,
|
|
139
|
+
bySource,
|
|
140
|
+
topSkills,
|
|
141
|
+
retention: {
|
|
142
|
+
maxSkillEntries: MAX_SKILL_ENTRIES,
|
|
143
|
+
skillEntries: bySkill.size,
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
},
|
|
147
|
+
|
|
148
|
+
formatSummary() {
|
|
149
|
+
const summary = this.summarize();
|
|
150
|
+
const rate =
|
|
151
|
+
summary.followThroughRate === null ? "n/a" : `${(summary.followThroughRate * 100).toFixed(1)}%`;
|
|
152
|
+
const lines = [
|
|
153
|
+
"Recommendation feedback (session-local, no prompts stored)",
|
|
154
|
+
`offers: ${summary.totalOffers} | candidate slots: ${summary.totalCandidateSlots} | loaded: ${summary.totalLoaded}`,
|
|
155
|
+
`abandoned: ${summary.totalAbandoned} | zero-candidate queries: ${summary.zeroCandidateQueries} | follow-through: ${rate}`,
|
|
156
|
+
];
|
|
157
|
+
|
|
158
|
+
for (const source of SOURCES) {
|
|
159
|
+
const counts = summary.bySource[source];
|
|
160
|
+
if (counts.offers === 0 && counts.zeroHits === 0) continue;
|
|
161
|
+
lines.push(
|
|
162
|
+
`${source}: offers=${counts.offers}, loaded=${counts.loaded}, zeroHits=${counts.zeroHits}`,
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (summary.topSkills.length > 0) {
|
|
167
|
+
const skillLine = summary.topSkills
|
|
168
|
+
.slice(0, 5)
|
|
169
|
+
.map((skill) => `${skill.name}(${skill.loaded}/${skill.offered})`)
|
|
170
|
+
.join(", ");
|
|
171
|
+
lines.push(`top skills: ${skillLine}`);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return lines.join("\n");
|
|
175
|
+
},
|
|
176
|
+
};
|
|
177
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { RecommendationReasonKind, SkillCandidate } from "./types.js";
|
|
2
|
+
|
|
3
|
+
/** Score within this margin above minScore is treated as low-confidence. */
|
|
4
|
+
export const LOW_CONFIDENCE_MARGIN = 0.08;
|
|
5
|
+
|
|
6
|
+
export const RECOMMENDATION_REASON_LABELS: Record<RecommendationReasonKind, string> = {
|
|
7
|
+
trigger: "trigger",
|
|
8
|
+
description: "description",
|
|
9
|
+
"low-confidence": "low match",
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export type RecommendationMatchSource = "trigger" | "description";
|
|
13
|
+
|
|
14
|
+
export function classifyRecommendationReason(
|
|
15
|
+
source: RecommendationMatchSource,
|
|
16
|
+
score: number,
|
|
17
|
+
minScore: number,
|
|
18
|
+
): RecommendationReasonKind {
|
|
19
|
+
if (source === "trigger") return "trigger";
|
|
20
|
+
return score < minScore + LOW_CONFIDENCE_MARGIN ? "low-confidence" : "description";
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function formatReasonBadge(reason: RecommendationReasonKind | null | undefined): string {
|
|
24
|
+
if (!reason) return "";
|
|
25
|
+
const label = RECOMMENDATION_REASON_LABELS[reason];
|
|
26
|
+
return label ? `[${label}]` : "";
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function formatReasonBadgeSuffix(reason: RecommendationReasonKind | null | undefined): string {
|
|
30
|
+
const badge = formatReasonBadge(reason);
|
|
31
|
+
return badge ? ` ${badge}` : "";
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function attachRecommendationReason(
|
|
35
|
+
candidate: Omit<SkillCandidate, "reason">,
|
|
36
|
+
source: RecommendationMatchSource,
|
|
37
|
+
minScore: number,
|
|
38
|
+
): SkillCandidate {
|
|
39
|
+
return {
|
|
40
|
+
...candidate,
|
|
41
|
+
reason: classifyRecommendationReason(source, candidate.score, minScore),
|
|
42
|
+
};
|
|
43
|
+
}
|
package/src/retrieval.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { attachRecommendationReason } from "./recommendation-reason.js";
|
|
1
2
|
import type { ShioriPolicy, SkillCandidate, SkillRecord } from "./types.js";
|
|
2
3
|
|
|
3
4
|
interface RetrievalIndex {
|
|
@@ -51,22 +52,30 @@ function evaluateSkill(
|
|
|
51
52
|
const includes = skillPolicy?.triggers?.include ?? [];
|
|
52
53
|
const matchedInclude = includes.find((trigger) => includesNormalized(normalizedQuery, trigger));
|
|
53
54
|
if (matchedInclude) {
|
|
54
|
-
return
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
55
|
+
return attachRecommendationReason(
|
|
56
|
+
{
|
|
57
|
+
skill,
|
|
58
|
+
score: 0.95,
|
|
59
|
+
why: `matched trigger "${matchedInclude}"`,
|
|
60
|
+
},
|
|
61
|
+
"trigger",
|
|
62
|
+
policy.candidateInjection.minScore,
|
|
63
|
+
);
|
|
59
64
|
}
|
|
60
65
|
if (exactTriggerOnly) return undefined;
|
|
61
66
|
|
|
62
67
|
const text = normalize(`${skill.name} ${skill.description} ${includes.join(" ")}`);
|
|
63
68
|
const tokenScore = scoreTokens(normalizedQuery, text);
|
|
64
69
|
if (tokenScore < policy.candidateInjection.minScore) return undefined;
|
|
65
|
-
return
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
+
return attachRecommendationReason(
|
|
71
|
+
{
|
|
72
|
+
skill,
|
|
73
|
+
score: tokenScore,
|
|
74
|
+
why: "matched skill description",
|
|
75
|
+
},
|
|
76
|
+
"description",
|
|
77
|
+
policy.candidateInjection.minScore,
|
|
78
|
+
);
|
|
70
79
|
}
|
|
71
80
|
|
|
72
81
|
function normalize(input: string): string {
|
package/src/types.ts
CHANGED
|
@@ -14,6 +14,12 @@ export interface ShioriPolicy {
|
|
|
14
14
|
minScore: number;
|
|
15
15
|
};
|
|
16
16
|
alwaysVisible: string[];
|
|
17
|
+
inventory?: {
|
|
18
|
+
/** Extra skill roots relative to cwd or absolute paths. */
|
|
19
|
+
roots?: string[];
|
|
20
|
+
/** Rebuild the index when skill files change under any root (default: true). */
|
|
21
|
+
autoRefreshOnChange?: boolean;
|
|
22
|
+
};
|
|
17
23
|
skills: Record<string, SkillPolicy>;
|
|
18
24
|
}
|
|
19
25
|
|
|
@@ -33,10 +39,13 @@ export interface SkillRecord {
|
|
|
33
39
|
source: string;
|
|
34
40
|
}
|
|
35
41
|
|
|
42
|
+
export type RecommendationReasonKind = "trigger" | "description" | "low-confidence";
|
|
43
|
+
|
|
36
44
|
export interface SkillCandidate {
|
|
37
45
|
skill: SkillRecord;
|
|
38
46
|
score: number;
|
|
39
47
|
why: string;
|
|
48
|
+
reason: RecommendationReasonKind | null;
|
|
40
49
|
}
|
|
41
50
|
|
|
42
51
|
export interface ShioriStats {
|
|
@@ -49,4 +58,6 @@ export interface ShioriStats {
|
|
|
49
58
|
retrievalBackend: RetrievalBackend;
|
|
50
59
|
suppressionStatus: SuppressionStatus;
|
|
51
60
|
lastReloadAt?: string;
|
|
61
|
+
inventoryRefreshCount: number;
|
|
62
|
+
inventoryAutoRefreshCount: number;
|
|
52
63
|
}
|