recallnest 2.2.1 → 2.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/CHANGELOG.md +150 -0
  2. package/README.md +46 -35
  3. package/README_CN.md +41 -26
  4. package/assets/ui/app.js +1031 -0
  5. package/assets/ui/index.html +224 -0
  6. package/assets/ui/styles.css +781 -0
  7. package/config.json.example +4 -1
  8. package/connectors/examples/email-adapter.ts +73 -0
  9. package/connectors/examples/logs-adapter.ts +103 -0
  10. package/connectors/examples/rss-adapter.ts +95 -0
  11. package/docs/api-reference.md +1291 -0
  12. package/docs/architecture.md +168 -0
  13. package/docs/connector-spec.md +126 -0
  14. package/eval/cases-canary.json +58 -0
  15. package/eval/cases-scope-robustness.json +278 -0
  16. package/eval/cases.json +117 -20
  17. package/package.json +22 -1
  18. package/src/__tests__/access-tracker-novelty.test.ts +238 -0
  19. package/src/__tests__/activity-counter.test.ts +125 -0
  20. package/src/__tests__/adaptive-fidelity.test.ts +102 -0
  21. package/src/__tests__/admission-control.test.ts +206 -0
  22. package/src/__tests__/anchor-generator.test.ts +67 -0
  23. package/src/__tests__/auto-consolidation.test.ts +168 -0
  24. package/src/__tests__/auto-recall.test.ts +131 -0
  25. package/src/__tests__/babel-integration.test.ts +144 -0
  26. package/src/__tests__/batch-dedup.test.ts +65 -0
  27. package/src/__tests__/canary-eval.test.ts +114 -0
  28. package/src/__tests__/capture-engine.test.ts +1238 -0
  29. package/src/__tests__/capture-heuristic.test.ts +171 -0
  30. package/src/__tests__/cascade-forget.test.ts +210 -0
  31. package/src/__tests__/category-min-scores.test.ts +33 -0
  32. package/src/__tests__/cc8-cc1-essential-context.test.ts +298 -0
  33. package/src/__tests__/circuit-breaker.test.ts +99 -0
  34. package/src/__tests__/cluster-consolidation.test.ts +782 -0
  35. package/src/__tests__/confidence-metadata.test.ts +125 -0
  36. package/src/__tests__/confidence-tracker.test.ts +261 -0
  37. package/src/__tests__/conflict-advisor.test.ts +206 -0
  38. package/src/__tests__/conflict-escalation.test.ts +149 -0
  39. package/src/__tests__/conflict-lifecycle.test.ts +83 -0
  40. package/src/__tests__/conflict-output.test.ts +85 -0
  41. package/src/__tests__/conflict-store.test.ts +374 -0
  42. package/src/__tests__/connector-v1.test.ts +181 -0
  43. package/src/__tests__/consolidation-engine.test.ts +333 -0
  44. package/src/__tests__/constructive-retrieval.test.ts +509 -0
  45. package/src/__tests__/context-collapse-renderer.test.ts +214 -0
  46. package/src/__tests__/context-composer-pins.test.ts +64 -0
  47. package/src/__tests__/context-composer-task-selection.test.ts +24 -0
  48. package/src/__tests__/context-reconstructor.test.ts +65 -0
  49. package/src/__tests__/context-renderer.test.ts +91 -0
  50. package/src/__tests__/continuity-eval.test.ts +275 -0
  51. package/src/__tests__/continuity-schema.test.ts +97 -0
  52. package/src/__tests__/contracts.test.ts +216 -0
  53. package/src/__tests__/conversation-importer.test.ts +692 -0
  54. package/src/__tests__/core-recall-eval.test.ts +235 -0
  55. package/src/__tests__/core-summary.test.ts +109 -0
  56. package/src/__tests__/data-checkup.test.ts +276 -0
  57. package/src/__tests__/decay-exemption.test.ts +84 -0
  58. package/src/__tests__/dedup-actions.test.ts +58 -0
  59. package/src/__tests__/dedup-guard.test.ts +39 -0
  60. package/src/__tests__/deterministic-id.test.ts +39 -0
  61. package/src/__tests__/distill-lock.test.ts +305 -0
  62. package/src/__tests__/distill-session-argshape.test.ts +79 -0
  63. package/src/__tests__/doctor.test.ts +319 -0
  64. package/src/__tests__/dream-pipeline.test.ts +321 -0
  65. package/src/__tests__/drill-down.test.ts +108 -0
  66. package/src/__tests__/embedder.test.ts +122 -0
  67. package/src/__tests__/emotion-decay.test.ts +46 -0
  68. package/src/__tests__/emotion-detector.test.ts +56 -0
  69. package/src/__tests__/emotion-phase1-integration.test.ts +153 -0
  70. package/src/__tests__/emotion-retrieval.test.ts +56 -0
  71. package/src/__tests__/entity-resolver.test.ts +58 -0
  72. package/src/__tests__/env-config.test.ts +151 -0
  73. package/src/__tests__/error-signature-boost.test.ts +87 -0
  74. package/src/__tests__/error-signature.test.ts +95 -0
  75. package/src/__tests__/error-taxonomy.test.ts +129 -0
  76. package/src/__tests__/ethics-layer.test.ts +476 -0
  77. package/src/__tests__/event-segmenter.test.ts +153 -0
  78. package/src/__tests__/feature-flags-integration.test.ts +54 -0
  79. package/src/__tests__/frequency-tracker.test.ts +113 -0
  80. package/src/__tests__/graph-export.test.ts +489 -0
  81. package/src/__tests__/hotness-blend.test.ts +76 -0
  82. package/src/__tests__/ingest-chunking.test.ts +44 -0
  83. package/src/__tests__/ingest.test.ts +360 -0
  84. package/src/__tests__/interference-detector.test.ts +203 -0
  85. package/src/__tests__/kg-extractor.test.ts +192 -0
  86. package/src/__tests__/kg-store.test.ts +300 -0
  87. package/src/__tests__/llm-consolidation.test.ts +226 -0
  88. package/src/__tests__/maintenance-locks.test.ts +174 -0
  89. package/src/__tests__/mcp-efficiency.test.ts +288 -0
  90. package/src/__tests__/mcp-related-scopes.test.ts +151 -0
  91. package/src/__tests__/memory-boundaries.test.ts +250 -0
  92. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +345 -0
  93. package/src/__tests__/memory-evolution.test.ts +271 -0
  94. package/src/__tests__/memory-health-rebalance.test.ts +144 -0
  95. package/src/__tests__/memory-lint.test.ts +421 -0
  96. package/src/__tests__/memory-output.test.ts +215 -0
  97. package/src/__tests__/memory-promotion.test.ts +315 -0
  98. package/src/__tests__/migrate-dedup-ids.test.ts +65 -0
  99. package/src/__tests__/multi-vector.test.ts +242 -0
  100. package/src/__tests__/narrative-architecture.test.ts +456 -0
  101. package/src/__tests__/noise-filter.test.ts +205 -0
  102. package/src/__tests__/obsidian-connector.test.ts +363 -0
  103. package/src/__tests__/omitted-reporting.test.ts +98 -0
  104. package/src/__tests__/pending-review.test.ts +77 -0
  105. package/src/__tests__/pii-redact.test.ts +314 -0
  106. package/src/__tests__/post-retrieval-filter.test.ts +178 -0
  107. package/src/__tests__/ppr-traversal.test.ts +256 -0
  108. package/src/__tests__/prediction-engine.test.ts +349 -0
  109. package/src/__tests__/predictive-prospective-memory.test.ts +416 -0
  110. package/src/__tests__/preference-guard-smoke.test.ts +211 -0
  111. package/src/__tests__/preference-matcher.test.ts +230 -0
  112. package/src/__tests__/preference-slots.test.ts +205 -0
  113. package/src/__tests__/prospective-memory.test.ts +283 -0
  114. package/src/__tests__/query-entity-detector.test.ts +171 -0
  115. package/src/__tests__/query-expander-alias.test.ts +146 -0
  116. package/src/__tests__/query-expander.test.ts +34 -0
  117. package/src/__tests__/recall-governor.test.ts +227 -0
  118. package/src/__tests__/recall-mode.test.ts +64 -0
  119. package/src/__tests__/result-synthesizer.test.ts +198 -0
  120. package/src/__tests__/resume-context.test.ts +4898 -0
  121. package/src/__tests__/retrieval-trace.test.ts +213 -0
  122. package/src/__tests__/retriever-boundary-weight.test.ts +136 -0
  123. package/src/__tests__/retroactive-boost.test.ts +223 -0
  124. package/src/__tests__/salience-scoring.test.ts +106 -0
  125. package/src/__tests__/scope-inventory.test.ts +200 -0
  126. package/src/__tests__/scope-policy.test.ts +69 -0
  127. package/src/__tests__/scope-suggester.test.ts +80 -0
  128. package/src/__tests__/session-checkpoint.test.ts +291 -0
  129. package/src/__tests__/session-distiller.test.ts +640 -0
  130. package/src/__tests__/session-gc.test.ts +215 -0
  131. package/src/__tests__/skill-engine.test.ts +552 -0
  132. package/src/__tests__/skill-promotion.test.ts +601 -0
  133. package/src/__tests__/skill-verifier.test.ts +122 -0
  134. package/src/__tests__/source-heartbeat.test.ts +250 -0
  135. package/src/__tests__/store-patch-metadata.test.ts +205 -0
  136. package/src/__tests__/store-scope-audit.test.ts +81 -0
  137. package/src/__tests__/store-upsert.test.ts +184 -0
  138. package/src/__tests__/supersede-chain.test.ts +154 -0
  139. package/src/__tests__/temporal-parser.test.ts +241 -0
  140. package/src/__tests__/temporal-validity.test.ts +160 -0
  141. package/src/__tests__/term-registry.test.ts +121 -0
  142. package/src/__tests__/tool-error-surfacing.test.ts +98 -0
  143. package/src/__tests__/tool-output-compressor.test.ts +160 -0
  144. package/src/__tests__/topic-tag.test.ts +113 -0
  145. package/src/__tests__/trusted-memory.test.ts +839 -0
  146. package/src/__tests__/usage-tracker.test.ts +287 -0
  147. package/src/__tests__/version-manager.test.ts +213 -0
  148. package/src/__tests__/workflow-observation-skill-outcome.test.ts +352 -0
  149. package/src/__tests__/workflow-observation.test.ts +433 -0
  150. package/src/__tests__/write-verifier.test.ts +122 -0
  151. package/src/access-tracker.ts +38 -32
  152. package/src/activity-counter.ts +46 -18
  153. package/src/aliases.ts +133 -0
  154. package/src/api-server.ts +27 -7
  155. package/src/audit-log.ts +4 -2
  156. package/src/auto-consolidation.ts +8 -3
  157. package/src/auto-gc.ts +85 -21
  158. package/src/capture-engine.ts +54 -21
  159. package/src/cli.ts +199 -37
  160. package/src/confidence-tracker.ts +36 -18
  161. package/src/connector-types.ts +57 -0
  162. package/src/consolidation-engine.ts +109 -11
  163. package/src/context-composer.ts +8 -1
  164. package/src/conversation-importer.ts +25 -1
  165. package/src/data-checkup.ts +126 -7
  166. package/src/distill-lock.ts +194 -15
  167. package/src/doctor.ts +38 -2
  168. package/src/dream-pipeline.ts +118 -34
  169. package/src/env-config.ts +77 -0
  170. package/src/error-signature.ts +153 -0
  171. package/src/error-taxonomy.ts +115 -0
  172. package/src/eval.ts +251 -6
  173. package/src/idempotency.ts +8 -0
  174. package/src/ingest.ts +234 -10
  175. package/src/kg-extractor.ts +6 -2
  176. package/src/kg-store.ts +236 -44
  177. package/src/llm-consolidation.ts +2 -1
  178. package/src/mcp-server.ts +85 -1799
  179. package/src/mcp-tool-deps.ts +48 -0
  180. package/src/mcp-tools-advanced.ts +941 -0
  181. package/src/mcp-tools-core.ts +633 -0
  182. package/src/mcp-tools-governance.ts +475 -0
  183. package/src/memory-boundaries.ts +3 -1
  184. package/src/memory-evolution.ts +73 -1
  185. package/src/memory-lint.ts +183 -10
  186. package/src/memory-output.ts +114 -3
  187. package/src/memory-promotion.ts +246 -0
  188. package/src/memory-schema.ts +25 -2
  189. package/src/memory-store-port.ts +43 -0
  190. package/src/multi-vector.ts +2 -1
  191. package/src/narrative-schema.ts +2 -1
  192. package/src/noise-filter.ts +57 -6
  193. package/src/obsidian-connector.ts +282 -0
  194. package/src/pii-detector.ts +157 -5
  195. package/src/query-expander.ts +90 -9
  196. package/src/result-synthesizer.ts +2 -1
  197. package/src/retention-policy.ts +2 -1
  198. package/src/retriever.ts +259 -139
  199. package/src/runtime-config.ts +8 -1
  200. package/src/scope-suggester.ts +113 -0
  201. package/src/session-distiller.ts +23 -2
  202. package/src/session-engine.ts +2 -1
  203. package/src/session-schema.ts +3 -2
  204. package/src/session-store.ts +5 -0
  205. package/src/skill-engine.ts +78 -0
  206. package/src/skill-promotion.ts +247 -47
  207. package/src/skill-schema.ts +16 -3
  208. package/src/skill-verifier.ts +152 -0
  209. package/src/source-heartbeat.ts +108 -0
  210. package/src/store.ts +267 -38
  211. package/src/tool-output-compressor.ts +3 -1
  212. package/src/ui-server.ts +2 -1
  213. package/src/usage-tracker.ts +242 -0
  214. package/src/version-manager.ts +3 -2
  215. package/src/workflow-observation-engine.ts +2 -1
  216. package/src/workflow-observation-schema.ts +2 -1
  217. package/src/workflow-observation-store.ts +6 -2
  218. package/.env +0 -4
  219. package/CLAUDE.md +0 -46
  220. package/config.json +0 -55
  221. package/daily-recall.sh +0 -110
  222. package/logs/daily-recall.error.log +0 -0
  223. package/logs/daily-recall.log +0 -30
  224. package/logs/ingest-2026-04-11.log +0 -110
  225. package/logs/ingest-2026-04-12.log +0 -90
  226. package/logs/launchagent-stderr.log +0 -0
  227. package/logs/launchagent-stdout.log +0 -0
@@ -12,6 +12,7 @@
12
12
  import type { MemoryEntry, MemoryStore } from "./store.js";
13
13
  import { cosineSimilarity } from "./multi-vector.js";
14
14
  import { isActiveMemory } from "./memory-evolution.js";
15
+ import { verifyDraft, type VerifyResult } from "./skill-verifier.js";
15
16
 
16
17
  // ---------------------------------------------------------------------------
17
18
  // Types
@@ -25,12 +26,18 @@ export interface PromotionCandidate {
25
26
  /** For pattern_to_skill: extracted implementation steps */
26
27
  suggestedImplementation?: string;
27
28
  confidence: number; // 0-1
29
+ /** A2: zero-LLM verifier verdict (pattern_to_skill only). 失败候选保留披露、排后、不静默丢。 */
30
+ verification?: VerifyResult;
28
31
  }
29
32
 
30
33
  export interface PromotionScanResult {
31
34
  candidates: PromotionCandidate[];
32
35
  scannedCases: number;
33
36
  scannedPatterns: number;
37
+ /** No-silent-caps 披露:被桶上限截断、未参与聚类的 case 数。 */
38
+ truncatedCases: number;
39
+ /** 向量回填失败(库中无向量)而被跳过的条目数。 */
40
+ vectorlessSkipped: number;
34
41
  }
35
42
 
36
43
  export interface PromotionConfig {
@@ -40,24 +47,138 @@ export interface PromotionConfig {
40
47
  caseSimilarityThreshold: number;
41
48
  /** Max candidates to return (default: 5) */
42
49
  maxCandidates: number;
50
+ /** 全量扫描上限(防 OOM,default: 20000)。cases/patterns 各自独立适用。 */
51
+ maxScanEntries: number;
52
+ /** 单桶聚类上限(default: 2000)。桶内按 recency 保留,截断量计入 truncatedCases。 */
53
+ maxBucketSize: number;
54
+ /** listPage 翻页大小 (default: 1000)。 */
55
+ pageSize: number;
56
+ /** pattern_to_skill 通道最多探测的 structured patterns 数(每个一次 vectorSearch,
57
+ * default: 1000,按 recency 优先)。 */
58
+ maxPatternProbes: number;
43
59
  }
44
60
 
45
61
  const DEFAULT_CONFIG: PromotionConfig = {
46
62
  minCaseOccurrences: 3,
47
63
  caseSimilarityThreshold: 0.75,
48
64
  maxCandidates: 5,
65
+ maxScanEntries: 20_000,
66
+ maxBucketSize: 2_000,
67
+ pageSize: 1_000,
68
+ maxPatternProbes: 1_000,
49
69
  };
50
70
 
51
71
  // ---------------------------------------------------------------------------
52
72
  // Helpers
53
73
  // ---------------------------------------------------------------------------
54
74
 
55
- type PromotionStore = Pick<MemoryStore, "list" | "vectorSearch">;
75
+ type PromotionStore = Pick<MemoryStore, "listPage" | "getVectors" | "vectorSearch">;
56
76
 
57
77
  function isActive(entry: MemoryEntry): boolean {
58
78
  return isActiveMemory(entry.metadata);
59
79
  }
60
80
 
81
+ /**
82
+ * 翻页拉取一个 category 的全部条目(轻列,不含向量)。
83
+ * listPage 在 DB 层下推 where/limit/offset,不会像 list() 那样每页全表拉取。
84
+ */
85
+ async function listAllByCategory(
86
+ store: PromotionStore,
87
+ scope: string,
88
+ category: string,
89
+ maxEntries: number,
90
+ pageSize: number,
91
+ truncateTextTo?: number,
92
+ ): Promise<MemoryEntry[]> {
93
+ const out: MemoryEntry[] = [];
94
+ for (let offset = 0; out.length < maxEntries; offset += pageSize) {
95
+ const requested = Math.min(pageSize, maxEntries - out.length);
96
+ const page = await store.listPage({
97
+ scopeFilter: [scope],
98
+ category,
99
+ limit: requested,
100
+ offset,
101
+ });
102
+ for (const e of page) {
103
+ // 扫描只用 text 的头部(extractName 首行 / summarize 前 80 字 / 聚类靠向量),
104
+ // bridge 对话 case 全文很长,2 万条常驻能到 GB 级——截断防内存膨胀。
105
+ out.push(
106
+ truncateTextTo !== undefined && e.text.length > truncateTextTo
107
+ ? { ...e, text: e.text.slice(0, truncateTextTo) }
108
+ : e,
109
+ );
110
+ }
111
+ if (page.length < requested) break;
112
+ }
113
+ return out;
114
+ }
115
+
116
+ /** 从 metadata JSON 读 topicTag(缺失/解析失败 → undefined)。 */
117
+ function readTopicTag(entry: MemoryEntry): string | undefined {
118
+ try {
119
+ const parsed: unknown = JSON.parse(entry.metadata || "{}");
120
+ if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
121
+ const tag = (parsed as Record<string, unknown>).topicTag;
122
+ return typeof tag === "string" && tag.length > 0 ? tag : undefined;
123
+ }
124
+ } catch { /* fallthrough */ }
125
+ return undefined;
126
+ }
127
+
128
+ /**
129
+ * A2: 抽 entry 的工具名供 verifier tool-coverage 用。
130
+ * metadata.{caseMemory|workflowPattern}.tools 优先;fallback 解析正文 `Tools: a, b, c` 行。
131
+ */
132
+ function extractToolsFromEntry(entry: MemoryEntry): string[] {
133
+ try {
134
+ const parsed: unknown = JSON.parse(entry.metadata || "{}");
135
+ if (parsed !== null && typeof parsed === "object") {
136
+ const obj = parsed as Record<string, unknown>;
137
+ for (const key of ["caseMemory", "workflowPattern"]) {
138
+ const sub = obj[key];
139
+ if (sub !== null && typeof sub === "object") {
140
+ const tools = (sub as Record<string, unknown>).tools;
141
+ if (Array.isArray(tools)) {
142
+ return tools.filter((t): t is string => typeof t === "string" && t.length > 0);
143
+ }
144
+ }
145
+ }
146
+ }
147
+ } catch { /* fallthrough to text parse */ }
148
+ const m = entry.text.match(/(?:^|\n)Tools:\s*(.+)/);
149
+ if (m) return m[1].split(",").map(t => t.trim()).filter(Boolean);
150
+ return [];
151
+ }
152
+
153
+ /**
154
+ * 按 topicTag 分桶(无 tag → "untagged" 桶),桶内按 recency 保留 maxBucketSize 条。
155
+ * 分桶把 greedyCluster 的 O(n²) 最坏复杂度限制在桶内,同时避免全库向量常驻。
156
+ */
157
+ function bucketByTopic(
158
+ entries: MemoryEntry[],
159
+ maxBucketSize: number,
160
+ ): { buckets: Map<string, MemoryEntry[]>; truncated: number } {
161
+ const buckets = new Map<string, MemoryEntry[]>();
162
+ for (const e of entries) {
163
+ const key = readTopicTag(e) ?? "untagged";
164
+ let bucket = buckets.get(key);
165
+ if (!bucket) {
166
+ bucket = [];
167
+ buckets.set(key, bucket);
168
+ }
169
+ bucket.push(e);
170
+ }
171
+ let truncated = 0;
172
+ for (const [key, bucket] of buckets) {
173
+ if (bucket.length > maxBucketSize) {
174
+ bucket.sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0));
175
+ truncated += bucket.length - maxBucketSize;
176
+ buckets.set(key, bucket.slice(0, maxBucketSize));
177
+ }
178
+ }
179
+ return { buckets, truncated };
180
+ }
181
+
61
182
  /** Extract a short name from the first case's text (first line or first ~60 chars). */
62
183
  function extractName(text: string): string {
63
184
  const firstLine = text.split("\n")[0].trim();
@@ -95,12 +216,12 @@ function extractSteps(text: string): string {
95
216
  // Greedy Clustering (same approach as consolidation-engine's C-2)
96
217
  // ---------------------------------------------------------------------------
97
218
 
98
- interface Cluster {
219
+ export interface Cluster {
99
220
  seed: MemoryEntry;
100
221
  members: MemoryEntry[];
101
222
  }
102
223
 
103
- function greedyCluster(
224
+ export function greedyCluster(
104
225
  entries: MemoryEntry[],
105
226
  threshold: number,
106
227
  ): Cluster[] {
@@ -165,78 +286,140 @@ export async function scanForPromotions(
165
286
  ): Promise<PromotionScanResult> {
166
287
  const cfg: PromotionConfig = { ...DEFAULT_CONFIG, ...config };
167
288
 
168
- // 1. Load all entries in scope (high limit to get everything meaningful)
169
- const entries = await store.list([scope], undefined, 500, 0);
170
- const active = entries.filter(isActive);
289
+ // 1. Paged full scan, category pushed down to the DB layer.
290
+ // 旧实现 store.list([scope], undefined, 500, 0) 双重失效:① limit=500 在
291
+ // 34K+ 库上覆盖率 ~1.4%;② list() 性能优化恒返回 vector:[],聚类全部跳过,
292
+ // 管线产出恒为零(单测 mock 带向量所以从未暴露)。
293
+ // cases/patterns 各自独立预算——共享预算时大库的 cases 会把 patterns 饿死
294
+ // (首次生产 dry-run 实测:20000 cases 拉满后 patterns scanned=0,
295
+ // pattern_to_skill 通道整个没跑)。
296
+ const rawCases = await listAllByCategory(store, scope, "cases", cfg.maxScanEntries, cfg.pageSize, 2_000);
297
+ const rawPatterns = await listAllByCategory(store, scope, "patterns", cfg.maxScanEntries, cfg.pageSize);
171
298
 
172
- const cases = active.filter(e => e.category === "cases");
173
- const patterns = active.filter(e => e.category === "patterns");
299
+ const cases = rawCases.filter(isActive);
300
+ const patterns = rawPatterns.filter(isActive);
174
301
 
175
302
  const result: PromotionScanResult = {
176
303
  candidates: [],
177
304
  scannedCases: cases.length,
178
305
  scannedPatterns: patterns.length,
306
+ truncatedCases: 0,
307
+ vectorlessSkipped: 0,
179
308
  };
180
309
 
181
- // 2. Cluster cases by vector similarity
310
+ // 2. Cluster cases bucket-by-bucket (topicTag, untagged fallback)。
311
+ // 向量按桶回填、桶毕即释,全库向量绝不常驻;分桶同时把 greedyCluster 的
312
+ // 最坏 O(n²) 限制在 maxBucketSize 内。
313
+ // 配额 per-channel:case 通道最多产 maxCandidates 个,pattern 通道同;
314
+ // 最后合并按 confidence 排序截断——否则 case 候选多的库会让
315
+ // pattern_to_skill 通道在全局 break 处永远轮空(首次生产 dry-run 实测)。
316
+ let caseCandidates = 0;
182
317
  if (cases.length >= cfg.minCaseOccurrences) {
183
- const clusters = greedyCluster(cases, cfg.caseSimilarityThreshold);
184
-
185
- for (const cluster of clusters) {
186
- if (cluster.members.length < cfg.minCaseOccurrences) continue;
187
- if (result.candidates.length >= cfg.maxCandidates) break;
188
-
189
- // Compute average intra-cluster similarity for scoring
190
- const avgSim = computeAverageIntraClusterSimilarity(cluster.members);
191
-
192
- result.candidates.push({
193
- type: "case_to_pattern",
194
- sourceEntries: cluster.members.map(m => ({
195
- id: m.id,
196
- text: m.text,
197
- score: avgSim,
198
- })),
199
- suggestedName: extractName(cluster.seed.text),
200
- suggestedDescription: summarizeCluster(cluster.members),
201
- confidence: cluster.members.length / (cluster.members.length + 2), // Bayesian smoothing
202
- });
318
+ const { buckets, truncated } = bucketByTopic(cases, cfg.maxBucketSize);
319
+ result.truncatedCases = truncated;
320
+
321
+ for (const bucket of buckets.values()) {
322
+ if (bucket.length < cfg.minCaseOccurrences) continue;
323
+ if (caseCandidates >= cfg.maxCandidates) break;
324
+
325
+ const vectorMap = await store.getVectors(bucket.map(e => e.id));
326
+ const withVectors = bucket
327
+ .map(e => ({ ...e, vector: vectorMap.get(e.id) ?? [] }))
328
+ .filter(e => e.vector.length > 0);
329
+ result.vectorlessSkipped += bucket.length - withVectors.length;
330
+ if (withVectors.length < cfg.minCaseOccurrences) continue;
331
+
332
+ const clusters = greedyCluster(withVectors, cfg.caseSimilarityThreshold);
333
+
334
+ for (const cluster of clusters) {
335
+ if (cluster.members.length < cfg.minCaseOccurrences) continue;
336
+ if (caseCandidates >= cfg.maxCandidates) break;
337
+
338
+ // Compute average intra-cluster similarity for scoring
339
+ const avgSim = computeAverageIntraClusterSimilarity(cluster.members);
340
+
341
+ caseCandidates++;
342
+ result.candidates.push({
343
+ type: "case_to_pattern",
344
+ sourceEntries: cluster.members.map(m => ({
345
+ id: m.id,
346
+ text: m.text,
347
+ score: avgSim,
348
+ })),
349
+ suggestedName: extractName(cluster.seed.text),
350
+ suggestedDescription: summarizeCluster(cluster.members),
351
+ confidence: cluster.members.length / (cluster.members.length + 2), // Bayesian smoothing
352
+ });
353
+ }
203
354
  }
204
355
  }
205
356
 
206
- // 3. Detect pattern_to_skill candidates
207
- for (const pattern of patterns) {
208
- if (result.candidates.length >= cfg.maxCandidates) break;
209
- if (!hasStructuredSteps(pattern.text)) continue;
210
- if (!pattern.vector?.length) continue;
211
-
212
- // Find cases similar to this pattern
213
- const similarCases = cases.filter(c => {
214
- if (!c.vector?.length) return false;
215
- const sim = cosineSimilarity(pattern.vector, c.vector);
216
- return sim >= cfg.caseSimilarityThreshold;
217
- });
357
+ // 3. Detect pattern_to_skill candidates — similarity search pushed down to the
358
+ // vector index (one vectorSearch per structured pattern) instead of pairwise
359
+ // cosine against the full in-memory case corpus.
360
+ // vectorSearch score = 1/(1+cosineDistance) = 1/(2-cosineSim) is monotonic in
361
+ // cosineSim, so minScore = 1/(2-threshold) filters EXACTLY the same set as
362
+ // `cosineSim >= threshold`; per-hit cosine 反算 = 2 - 1/score.
363
+ const structuredPatterns = patterns
364
+ .filter(p => hasStructuredSteps(p.text))
365
+ .sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0))
366
+ .slice(0, cfg.maxPatternProbes);
367
+ const patternVectors = structuredPatterns.length > 0
368
+ ? await store.getVectors(structuredPatterns.map(p => p.id))
369
+ : new Map<string, number[]>();
370
+ const minSearchScore = 1 / (2 - cfg.caseSimilarityThreshold);
371
+
372
+ let patternCandidates = 0;
373
+ for (const pattern of structuredPatterns) {
374
+ if (patternCandidates >= cfg.maxCandidates) break;
375
+ const vec = patternVectors.get(pattern.id);
376
+ if (!vec || vec.length === 0) {
377
+ result.vectorlessSkipped++;
378
+ continue;
379
+ }
380
+
381
+ const hits = await store.vectorSearch(vec, 20, minSearchScore, [scope]);
382
+ const similarCases = hits.filter(
383
+ h => h.entry.category === "cases" && h.entry.id !== pattern.id && isActive(h.entry),
384
+ );
218
385
 
219
386
  if (similarCases.length < 2) continue;
220
387
 
388
+ // A2: zero-LLM verifier — pattern 当 draft,supporting cases 当 evidence。
389
+ // steps 传 [] 不是漏传:pattern.text 已含完整 Steps 块,resonance 用整段文本即可;
390
+ // extractSteps(pattern.text) 仅用于下面的 suggestedImplementation 字段。
391
+ const verification = verifyDraft(
392
+ { tools: extractToolsFromEntry(pattern), summary: pattern.text, steps: [] },
393
+ similarCases.map(h => ({ text: h.entry.text, tools: extractToolsFromEntry(h.entry) })),
394
+ );
395
+
396
+ patternCandidates++;
221
397
  result.candidates.push({
222
398
  type: "pattern_to_skill",
223
399
  sourceEntries: [
224
400
  { id: pattern.id, text: pattern.text, score: 1.0 },
225
- ...similarCases.map(c => ({
226
- id: c.id,
227
- text: c.text,
228
- score: cosineSimilarity(pattern.vector, c.vector),
401
+ ...similarCases.map(h => ({
402
+ id: h.entry.id,
403
+ text: h.entry.text,
404
+ score: 2 - 1 / h.score, // back-convert to cosine similarity
229
405
  })),
230
406
  ],
231
407
  suggestedName: extractName(pattern.text),
232
408
  suggestedDescription: `Skill derived from pattern with ${similarCases.length} supporting cases`,
233
409
  suggestedImplementation: extractSteps(pattern.text),
234
410
  confidence: similarCases.length / (similarCases.length + 2),
411
+ verification,
235
412
  });
236
413
  }
237
414
 
238
- // Sort by confidence descending, then truncate
239
- result.candidates.sort((a, b) => b.confidence - a.confidence);
415
+ // Sort: A2 verifier-passed first, then confidence desc, then truncate.
416
+ // 失败候选保留披露但排后;截断优先砍失败候选,通过的不被挤掉(Codex 验证补充)。
417
+ result.candidates.sort((a, b) => {
418
+ const aOk = a.verification?.ok ?? true;
419
+ const bOk = b.verification?.ok ?? true;
420
+ if (aOk !== bOk) return aOk ? -1 : 1;
421
+ return b.confidence - a.confidence;
422
+ });
240
423
  result.candidates = result.candidates.slice(0, cfg.maxCandidates);
241
424
 
242
425
  return result;
@@ -252,6 +435,14 @@ export function formatPromotionResult(result: PromotionScanResult): string {
252
435
  `Promotion scan: ${result.scannedCases} cases, ${result.scannedPatterns} patterns scanned.`,
253
436
  ];
254
437
 
438
+ // No-silent-caps:截断与向量缺失必须可见,否则"0 候选"会被误读成"扫全了没东西"。
439
+ if (result.truncatedCases > 0) {
440
+ lines.push(`⚠️ ${result.truncatedCases} cases truncated by bucket cap (kept most recent per topic bucket).`);
441
+ }
442
+ if (result.vectorlessSkipped > 0) {
443
+ lines.push(`⚠️ ${result.vectorlessSkipped} entries skipped (no vector in store).`);
444
+ }
445
+
255
446
  if (result.candidates.length === 0) {
256
447
  lines.push("No promotion candidates found.");
257
448
  return lines.join("\n");
@@ -264,6 +455,15 @@ export function formatPromotionResult(result: PromotionScanResult): string {
264
455
  lines.push(`Confidence: ${(c.confidence * 100).toFixed(1)}%`);
265
456
  lines.push(`Description: ${c.suggestedDescription}`);
266
457
  lines.push(`Sources: ${c.sourceEntries.length} entries`);
458
+ if (c.verification) {
459
+ const v = c.verification;
460
+ if (v.ok) {
461
+ lines.push(`Verifier: ✓ pass (coverage ${(v.coverage * 100).toFixed(0)}%, resonance ${(v.resonance * 100).toFixed(0)}%)`);
462
+ } else {
463
+ const unmapped = v.unmappedTools.length > 0 ? `; unmapped tools: ${v.unmappedTools.join(", ")}` : "";
464
+ lines.push(`Verifier: ⚠️ FAILED — ${v.reason ?? "?"} (coverage ${(v.coverage * 100).toFixed(0)}%, resonance ${(v.resonance * 100).toFixed(0)}%${unmapped})`);
465
+ }
466
+ }
267
467
  if (c.suggestedImplementation) {
268
468
  lines.push(`Implementation:\n\`\`\`\n${c.suggestedImplementation}\n\`\`\``);
269
469
  }
@@ -1,14 +1,27 @@
1
1
  /**
2
- * D-1: Skill Schema — defines the shape of executable skills stored in memory.
2
+ * D-1: Skill Schema — defines the shape of agent-readable skill runbooks stored in memory.
3
3
  *
4
4
  * Skills are "what an agent can do" (vs. memories which are "what an agent learned").
5
5
  * Alita/Alita-G showed MCP itself is a natural skill representation format.
6
+ *
7
+ * v2.5 (2026-05-27) — schema 收缩:
8
+ * 原 SkillImplementationTypeSchema 含 "bash" / "python" / "mcp_tool_chain" / "instruction_sequence"
9
+ * 四种,**implementation 字段从未真执行**(无 evaluator,仅作 context 给 agent 读)——是 schema
10
+ * 撒谎暗示可执行。源自 brgsk《agent memory: an anatomy》借鉴审计 + Codex trio 二审建议
11
+ * "P1 选收缩/删承诺,不接 evaluator"。
12
+ *
13
+ * 现在 enum 只保留 "instruction_sequence" 唯一类型,明确 skill 是 **agent-readable runbook**
14
+ * 而非可执行物。未来如真接 evaluator(产品决策另开),再展开 enum 加 "bash" / "python" 等。
15
+ *
16
+ * 兼容性:parseSkillFromEntry 读取老 metadata 用 type cast 不走 schema 校验,所以历史 bash/python
17
+ * skill records 仍可 retrieve;只是新写入受新 schema 约束。production 库 2026-05-27 实测无真实
18
+ * skill 数据,破坏面接近 zero。
6
19
  */
7
20
 
8
21
  import { z } from "zod";
9
22
 
10
23
  export const SkillImplementationTypeSchema = z.enum([
11
- "bash", "python", "mcp_tool_chain", "instruction_sequence",
24
+ "instruction_sequence",
12
25
  ]);
13
26
 
14
27
  export type SkillImplementationType = z.infer<typeof SkillImplementationTypeSchema>;
@@ -18,7 +31,7 @@ export const SkillInputSchema = z.object({
18
31
  description: z.string().min(1).max(500).describe("Natural language description (used for retrieval)"),
19
32
  triggerPattern: z.string().min(1).max(300).describe("When to suggest this skill"),
20
33
  implementationType: SkillImplementationTypeSchema,
21
- implementation: z.string().min(1).max(5000).describe("Executable content"),
34
+ implementation: z.string().min(1).max(5000).describe("Agent-readable runbook content: markdown steps, natural language workflow, or structured procedure. RecallNest does NOT execute this — it stores runbooks for agents to read and follow as context."),
22
35
  inputSchema: z.record(z.string(), z.unknown()).optional().describe("Parameter definition (JSON Schema)"),
23
36
  verification: z.string().max(500).optional().describe("How to verify execution success"),
24
37
  scope: z.string().min(1).max(160).describe("Project scope"),
@@ -0,0 +1,152 @@
1
+ /**
2
+ * A2 · Skill promotion verifier — zero-LLM consistency gate.
3
+ *
4
+ * 借鉴 MemOS local-plugin `core/skill/verifier.ts`(repo-insight 借鉴审计 2026-06-13)。
5
+ * 用在 `scan_skill_promotions` 的 pattern_to_skill 候选生成处:pattern 当 draft,
6
+ * supporting cases 当 evidence。两个确定性检查,**无 LLM 调用**:
7
+ *
8
+ * 1. **tool coverage**:draft 声称的 tools 必须 ⊆ evidence 里真实出现过的 tools
9
+ * (覆盖率 ≥ COVERAGE_THRESHOLD)。专治 LLM 蒸馏 skill 时编造证据里没有的命令/工具名。
10
+ * 2. **evidence resonance**:≥ minResonance 比例的 evidence 与 draft 的 summary/steps
11
+ * 共享 ≥2 个 token。专治"叙述跟例子对不上"。
12
+ *
13
+ * `tokensOf` 对中文做 2-gram bigram(RecallNest case/skill 大量中文,不能只认 ASCII token)。
14
+ * verdict 只描述、不决策——调用方(skill-promotion)据此附 `verification` 字段披露,
15
+ * 失败候选保留排后、不静默丢弃(no-silent-caps)。
16
+ */
17
+
18
+ export interface VerifyDraft {
19
+ /** draft 声称用到的工具/命令名 */
20
+ tools: string[];
21
+ /** draft 摘要文本(如 pattern 的 title + trigger / 正文) */
22
+ summary: string;
23
+ /** draft 步骤文本 */
24
+ steps: string[];
25
+ }
26
+
27
+ export interface VerifyEvidence {
28
+ /** 证据条目正文(如 supporting case 的 text) */
29
+ text: string;
30
+ /** 证据条目里真实出现过的工具/命令名 */
31
+ tools: string[];
32
+ }
33
+
34
+ export interface VerifyResult {
35
+ ok: boolean;
36
+ /** draft.tools ∩ evidenceTools / draft.tools(无声称工具时为 1) */
37
+ coverage: number;
38
+ /** 与 draft 共享 ≥2 token 的 evidence 比例 */
39
+ resonance: number;
40
+ /** draft 声称但任何 evidence 都没出现过的工具(疑似幻觉) */
41
+ unmappedTools: string[];
42
+ /** 失败原因;ok=true 时省略 */
43
+ reason?: string;
44
+ }
45
+
46
+ export const DEFAULT_MIN_RESONANCE = 0.5;
47
+ export const COVERAGE_THRESHOLD = 0.5;
48
+
49
+ /**
50
+ * 对一份 draft + 其证据做零 LLM 一致性校验。确定性、可单测、读写无副作用。
51
+ */
52
+ export function verifyDraft(
53
+ draft: VerifyDraft,
54
+ evidence: VerifyEvidence[],
55
+ options: { minResonance?: number } = {},
56
+ ): VerifyResult {
57
+ const minResonance = options.minResonance ?? DEFAULT_MIN_RESONANCE;
58
+
59
+ if (evidence.length === 0) {
60
+ return { ok: false, coverage: 0, resonance: 0, unmappedTools: [], reason: "no-evidence" };
61
+ }
62
+
63
+ // --- tool coverage(集合包含检查)---
64
+ const evidenceTools = new Set<string>();
65
+ for (const ev of evidence) {
66
+ for (const t of ev.tools) {
67
+ const norm = t.trim().toLowerCase();
68
+ if (norm) evidenceTools.add(norm);
69
+ }
70
+ }
71
+ const draftTools = draft.tools.map((t) => t.trim().toLowerCase()).filter(Boolean);
72
+ const matched: string[] = [];
73
+ const unmapped: string[] = [];
74
+ for (const tok of draftTools) {
75
+ if (evidenceTools.has(tok)) matched.push(tok);
76
+ else unmapped.push(tok);
77
+ }
78
+ // 无声称工具 → 无可证伪,coverage 视为满分(只看 resonance)。
79
+ const coverage = draftTools.length === 0 ? 1 : matched.length / draftTools.length;
80
+
81
+ // --- evidence resonance ---
82
+ const resonance = computeResonance(draft, evidence);
83
+
84
+ if (coverage < COVERAGE_THRESHOLD && draftTools.length > 0) {
85
+ return {
86
+ ok: false,
87
+ coverage,
88
+ resonance,
89
+ unmappedTools: unmapped,
90
+ reason: `coverage=${coverage.toFixed(2)}<${COVERAGE_THRESHOLD}`,
91
+ };
92
+ }
93
+ if (resonance < minResonance) {
94
+ return {
95
+ ok: false,
96
+ coverage,
97
+ resonance,
98
+ unmappedTools: unmapped,
99
+ reason: `resonance=${resonance.toFixed(2)}<${minResonance}`,
100
+ };
101
+ }
102
+ return { ok: true, coverage, resonance, unmappedTools: unmapped };
103
+ }
104
+
105
+ function computeResonance(draft: VerifyDraft, evidence: VerifyEvidence[]): number {
106
+ const needle = [draft.summary, ...draft.steps].join(" ").toLowerCase();
107
+ const draftTokens = tokensOf(needle);
108
+ if (draftTokens.size === 0) return 0;
109
+ let hit = 0;
110
+ for (const ev of evidence) {
111
+ const toks = tokensOf(ev.text.toLowerCase());
112
+ let overlap = 0;
113
+ for (const tok of draftTokens) {
114
+ if (toks.has(tok)) {
115
+ overlap += 1;
116
+ if (overlap >= 2) break; // 只需判定 ≥2,命中即可停
117
+ }
118
+ }
119
+ if (overlap >= 2) hit += 1;
120
+ }
121
+ return hit / evidence.length;
122
+ }
123
+
124
+ /**
125
+ * 分词:ASCII 标识符(≥4 字符、去停用词)+ CJK 2-gram bigram。
126
+ * CJK 部分不走停用词(2-gram 本身已足够稀疏)。
127
+ */
128
+ export function tokensOf(s: string): Set<string> {
129
+ const out = new Set<string>();
130
+ const asciiMatches = s.match(/[a-z0-9_][a-z0-9_./-]{3,}/g) ?? [];
131
+ for (const m of asciiMatches) {
132
+ const tok = m.toLowerCase();
133
+ if (RESONANCE_STOPWORDS.has(tok)) continue;
134
+ out.add(tok);
135
+ }
136
+ const cjkRuns = s.match(/[一-鿿぀-ヿ㐀-䶿]{2,}/g) ?? [];
137
+ for (const run of cjkRuns) {
138
+ for (let i = 0; i + 1 < run.length; i++) {
139
+ out.add(run.slice(i, i + 2));
140
+ }
141
+ }
142
+ return out;
143
+ }
144
+
145
+ const RESONANCE_STOPWORDS = new Set<string>([
146
+ "the", "and", "for", "with", "that", "this", "from", "will", "then",
147
+ "into", "when", "what", "where", "your", "user", "agent", "null", "true",
148
+ "false", "none", "let", "new", "old", "use", "used", "have", "has", "its",
149
+ "not", "any", "can", "does", "only", "just", "like", "please", "step",
150
+ "steps", "body", "title", "summary", "task", "tasks", "run", "see", "end",
151
+ "our", "their", "them", "being", "make", "made", "thing", "things",
152
+ ]);