recallnest 1.4.1 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +7 -3
  3. package/README.md +202 -130
  4. package/README_CN.md +263 -190
  5. package/bin/recallnest.mjs +62 -0
  6. package/config.json +55 -0
  7. package/daily-recall.sh +23 -0
  8. package/integrations/setup-all.sh +31 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +30 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/launchagent-stderr.log +0 -0
  14. package/logs/launchagent-stdout.log +0 -0
  15. package/package.json +10 -3
  16. package/src/access-tracker.ts +3 -2
  17. package/src/activity-counter.ts +109 -0
  18. package/src/adaptive-retrieval.ts +6 -5
  19. package/src/admission-control.ts +163 -0
  20. package/src/anchor-generator.ts +80 -0
  21. package/src/api-server.ts +10 -0
  22. package/src/audit-log.ts +122 -0
  23. package/src/auto-gc.ts +98 -29
  24. package/src/auto-recall.ts +33 -4
  25. package/src/capture-engine.ts +372 -23
  26. package/src/cascade-forget.ts +3 -2
  27. package/src/cli.ts +107 -5
  28. package/src/compat.ts +22 -0
  29. package/src/confidence-tracker.ts +210 -47
  30. package/src/conflict-store.ts +12 -3
  31. package/src/consolidation-engine.ts +296 -8
  32. package/src/context-collapse-renderer.ts +222 -0
  33. package/src/context-composer-stable-selection.ts +5 -1
  34. package/src/context-composer.ts +239 -4
  35. package/src/context-reconstructor.ts +404 -0
  36. package/src/conversation-importer.ts +441 -0
  37. package/src/data-checkup.ts +232 -0
  38. package/src/decay-engine.ts +84 -0
  39. package/src/distill-lock.ts +122 -0
  40. package/src/doctor.ts +8 -10
  41. package/src/dream-pipeline.ts +227 -0
  42. package/src/emotion-detector.ts +90 -0
  43. package/src/eval.ts +3 -2
  44. package/src/forget-engine.ts +269 -0
  45. package/src/frequency-tracker.ts +175 -0
  46. package/src/graph-export.ts +644 -0
  47. package/src/ingest.ts +36 -6
  48. package/src/interference-detector.ts +187 -0
  49. package/src/kg-extractor.ts +15 -24
  50. package/src/kg-store.ts +2 -1
  51. package/src/language-hook.ts +67 -0
  52. package/src/llm-client.ts +289 -6
  53. package/src/mcp-server.ts +707 -80
  54. package/src/memory-assets.ts +22 -5
  55. package/src/memory-evolution.ts +365 -0
  56. package/src/memory-lint.ts +429 -0
  57. package/src/memory-output.ts +69 -0
  58. package/src/memory-schema.ts +74 -2
  59. package/src/narrative-schema.ts +79 -0
  60. package/src/narrative-tagger.ts +164 -0
  61. package/src/pii-detector.ts +98 -0
  62. package/src/post-retrieval-filter.ts +76 -0
  63. package/src/prediction-engine.ts +253 -0
  64. package/src/preference-slots.ts +65 -2
  65. package/src/prospective-memory.ts +215 -0
  66. package/src/query-expander.ts +67 -0
  67. package/src/recall-governor.ts +183 -0
  68. package/src/retention-policy.ts +90 -0
  69. package/src/retriever.ts +828 -55
  70. package/src/rif.ts +34 -8
  71. package/src/runtime-config.ts +23 -8
  72. package/src/scope-inventory.ts +4 -3
  73. package/src/scope-policy.ts +4 -0
  74. package/src/session-distiller.ts +307 -0
  75. package/src/session-output.ts +40 -0
  76. package/src/session-schema.ts +37 -0
  77. package/src/session-store.ts +7 -2
  78. package/src/skill-engine.ts +208 -0
  79. package/src/skill-promotion.ts +293 -0
  80. package/src/skill-schema.ts +39 -0
  81. package/src/store.ts +96 -10
  82. package/src/topic-tag.ts +66 -0
  83. package/src/tracker.ts +3 -1
  84. package/src/ui-server.ts +80 -8
  85. package/src/workflow-observation-scope-review.ts +2 -1
  86. package/src/workflow-observation-store.ts +2 -1
  87. package/src/write-verifier.ts +126 -0
  88. package/.claude-plugin/CLAUDE.md +0 -29
  89. package/.claude-plugin/marketplace.json +0 -18
  90. package/.github/workflows/ci.yml +0 -28
  91. package/.mcp.json +0 -9
  92. package/CHANGELOG.md +0 -51
  93. package/DEMO.md +0 -94
  94. package/EVAL.md +0 -89
  95. package/FAILURES.md +0 -75
  96. package/OPERATIONS.md +0 -59
  97. package/ROADMAP.md +0 -198
  98. package/assets/recallnest-workbench.png +0 -0
  99. package/assets/ui/app.js +0 -807
  100. package/assets/ui/index.html +0 -154
  101. package/assets/ui/styles.css +0 -605
  102. package/assets/wechat-qr.jpg +0 -0
  103. package/daily-reflections/2026-03-05.md +0 -15
  104. package/daily-reflections/2026-03-06.md +0 -15
  105. package/daily-reflections/2026-03-07.md +0 -7
  106. package/daily-reflections/2026-03-08.md +0 -34
  107. package/daily-reflections/2026-03-09.md +0 -35
  108. package/daily-reflections/2026-03-10.md +0 -36
  109. package/daily-reflections/2026-03-11.md +0 -35
  110. package/daily-reflections/2026-03-12.md +0 -36
  111. package/daily-reflections/2026-03-13.md +0 -35
  112. package/daily-reflections/2026-03-14.md +0 -35
  113. package/daily-reflections/2026-03-15.md +0 -35
  114. package/daily-reflections/2026-03-16.md +0 -41
  115. package/daily-reflections/2026-03-17.md +0 -10
  116. package/daily-reflections/2026-03-18.md +0 -10
  117. package/daily-reflections/2026-03-19.md +0 -10
  118. package/daily-reflections/2026-03-20.md +0 -10
  119. package/daily-reflections/2026-03-21.md +0 -10
  120. package/daily-reflections/2026-03-22.md +0 -10
  121. package/daily-reflections/2026-03-23.md +0 -10
  122. package/daily-reflections/2026-03-24.md +0 -10
  123. package/daily-reflections/2026-03-25.md +0 -32
  124. package/daily-reflections/2026-03-26.md +0 -37
  125. package/daily-reflections/2026-03-27.md +0 -35
  126. package/daily-reflections/2026-03-28.md +0 -35
  127. package/daily-reflections/2026-03-29.md +0 -35
  128. package/daily-reflections/2026-03-30.md +0 -36
  129. package/daily-reflections/2026-03-31.md +0 -35
  130. package/daily-reflections/2026-04-01.md +0 -35
  131. package/docs/api-reference.md +0 -1291
  132. package/docs/architecture.md +0 -168
  133. package/docs/memory-boundary-contract.md +0 -226
  134. package/docs/memory-categories.md +0 -119
  135. package/docs/pipeline-audit-ultramemory.md +0 -177
  136. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  137. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  138. package/docs/plans/mcp-tiering-plan.md +0 -126
  139. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  140. package/docs/self-evolution.md +0 -198
  141. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  142. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  143. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  144. package/eval/continuity/results-2026-03-16.md +0 -117
  145. package/eval/continuity/results-2026-03-19.md +0 -453
  146. package/eval/continuity/runbook.md +0 -110
  147. package/eval/continuity/seed-cases.md +0 -54
  148. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  149. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  150. package/eval/reports/2026-03-06-baseline.md +0 -82
  151. package/eval/reports/query-expansion.md +0 -138
  152. package/lm +0 -19
  153. package/plan-v1.2-distributable.md +0 -105
  154. package/scripts/archive-distilled-junk.ts +0 -116
  155. package/scripts/backfill-l0l1l2.ts +0 -98
  156. package/scripts/blind-scan.ts +0 -588
  157. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  158. package/scripts/cleanup-step2-template-dups.ts +0 -113
  159. package/scripts/cleanup-step3-stale-content.ts +0 -149
  160. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  161. package/scripts/delete-distilled-junk.ts +0 -88
  162. package/scripts/distill-facts.ts +0 -305
  163. package/scripts/health-check.ts +0 -343
  164. package/scripts/incremental-ingest.sh +0 -71
  165. package/scripts/mlp-health-check.ts +0 -382
  166. package/scripts/pull-claude-desktop.ts +0 -515
  167. package/scripts/re-ingest-cc.sh +0 -66
  168. package/scripts/rebalance-memory-health.ts +0 -181
  169. package/scripts/run-desktop-ingest.sh +0 -11
  170. package/scripts/start-server.sh +0 -12
  171. package/scripts/viz-dashboard.html +0 -423
  172. package/scripts/viz-export.ts +0 -703
  173. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  174. package/src/__tests__/auto-consolidation.test.ts +0 -168
  175. package/src/__tests__/auto-recall.test.ts +0 -131
  176. package/src/__tests__/batch-dedup.test.ts +0 -65
  177. package/src/__tests__/capture-engine.test.ts +0 -1142
  178. package/src/__tests__/capture-heuristic.test.ts +0 -171
  179. package/src/__tests__/cascade-forget.test.ts +0 -210
  180. package/src/__tests__/category-min-scores.test.ts +0 -33
  181. package/src/__tests__/confidence-tracker.test.ts +0 -196
  182. package/src/__tests__/conflict-advisor.test.ts +0 -206
  183. package/src/__tests__/conflict-escalation.test.ts +0 -149
  184. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  185. package/src/__tests__/conflict-output.test.ts +0 -85
  186. package/src/__tests__/conflict-store.test.ts +0 -374
  187. package/src/__tests__/consolidation-engine.test.ts +0 -181
  188. package/src/__tests__/context-composer-pins.test.ts +0 -64
  189. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  190. package/src/__tests__/context-renderer.test.ts +0 -91
  191. package/src/__tests__/continuity-eval.test.ts +0 -275
  192. package/src/__tests__/continuity-schema.test.ts +0 -97
  193. package/src/__tests__/core-recall-eval.test.ts +0 -235
  194. package/src/__tests__/core-summary.test.ts +0 -109
  195. package/src/__tests__/dedup-actions.test.ts +0 -58
  196. package/src/__tests__/deterministic-id.test.ts +0 -39
  197. package/src/__tests__/doctor.test.ts +0 -303
  198. package/src/__tests__/drill-down.test.ts +0 -108
  199. package/src/__tests__/embedder.test.ts +0 -122
  200. package/src/__tests__/entity-resolver.test.ts +0 -58
  201. package/src/__tests__/event-segmenter.test.ts +0 -153
  202. package/src/__tests__/hotness-blend.test.ts +0 -76
  203. package/src/__tests__/ingest.test.ts +0 -360
  204. package/src/__tests__/kg-extractor.test.ts +0 -184
  205. package/src/__tests__/kg-store.test.ts +0 -198
  206. package/src/__tests__/llm-consolidation.test.ts +0 -226
  207. package/src/__tests__/memory-boundaries.test.ts +0 -237
  208. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/ppr-traversal.test.ts +0 -256
  212. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  213. package/src/__tests__/preference-matcher.test.ts +0 -230
  214. package/src/__tests__/preference-slots.test.ts +0 -106
  215. package/src/__tests__/prospective-memory.test.ts +0 -283
  216. package/src/__tests__/query-entity-detector.test.ts +0 -171
  217. package/src/__tests__/query-expander.test.ts +0 -34
  218. package/src/__tests__/recall-mode.test.ts +0 -64
  219. package/src/__tests__/result-synthesizer.test.ts +0 -198
  220. package/src/__tests__/resume-context.test.ts +0 -4743
  221. package/src/__tests__/retrieval-trace.test.ts +0 -213
  222. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  223. package/src/__tests__/retroactive-boost.test.ts +0 -223
  224. package/src/__tests__/scope-inventory.test.ts +0 -200
  225. package/src/__tests__/scope-policy.test.ts +0 -69
  226. package/src/__tests__/session-checkpoint.test.ts +0 -259
  227. package/src/__tests__/session-gc.test.ts +0 -215
  228. package/src/__tests__/store-scope-audit.test.ts +0 -81
  229. package/src/__tests__/temporal-parser.test.ts +0 -241
  230. package/src/__tests__/term-registry.test.ts +0 -121
  231. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  232. package/src/__tests__/version-manager.test.ts +0 -213
  233. package/src/__tests__/workflow-observation.test.ts +0 -400
@@ -0,0 +1,79 @@
1
+ /**
2
+ * Autobiographical Narrative Architecture — orthogonal metadata layer.
3
+ *
4
+ * Adds a three-level narrative hierarchy (life-period / general-event / specific-event)
5
+ * on top of the existing 6 durable memory categories. This enables temporal grouping
6
+ * and narrative retrieval without disrupting category logic.
7
+ *
8
+ * Philosophy: Conway's theory of autobiographical memory — memories organize
9
+ * hierarchically from life themes → general events → specific episodes.
10
+ *
11
+ * Feature flag: RECALLNEST_NARRATIVE_MODE=true
12
+ */
13
+
14
+ import { z } from "zod";
15
+
16
+ // ---------------------------------------------------------------------------
17
+ // Feature Flag
18
+ // ---------------------------------------------------------------------------
19
+
20
+ export function isNarrativeModeEnabled(): boolean {
21
+ return process.env.RECALLNEST_NARRATIVE_MODE === "true";
22
+ }
23
+
24
+ // ---------------------------------------------------------------------------
25
+ // Types
26
+ // ---------------------------------------------------------------------------
27
+
28
+ export interface NarrativeMetadata {
29
+ /** Life period identifier (auto-generated from scope prefix + quarterly time window) */
30
+ lifePeriodId: string;
31
+ /** Human-readable label for the life period */
32
+ lifePeriodLabel: string;
33
+ /** General event identifier (groups related memories within a day/session) */
34
+ generalEventId: string;
35
+ /** Human-readable label for the general event */
36
+ generalEventLabel: string;
37
+ /** Specific event identifier (unique episode) */
38
+ specificEventId: string;
39
+ /** Human-readable label for the specific event */
40
+ specificEventLabel: string;
41
+ /** Event start timestamp (epoch ms) */
42
+ startAt: number;
43
+ /** Event end timestamp (epoch ms, null if ongoing) */
44
+ endAt: number | null;
45
+ /** Sequence number within the general event for ordering */
46
+ sequence: number;
47
+ }
48
+
49
+ // ---------------------------------------------------------------------------
50
+ // Zod Schema
51
+ // ---------------------------------------------------------------------------
52
+
53
+ export const NarrativeMetadataSchema = z.object({
54
+ lifePeriodId: z.string().min(1).max(120),
55
+ lifePeriodLabel: z.string().min(1).max(120),
56
+ generalEventId: z.string().min(1).max(120),
57
+ generalEventLabel: z.string().min(1).max(200),
58
+ specificEventId: z.string().min(1).max(120),
59
+ specificEventLabel: z.string().min(1).max(200),
60
+ startAt: z.number(),
61
+ endAt: z.number().nullable(),
62
+ sequence: z.number().int().min(0),
63
+ });
64
+
65
+ // ---------------------------------------------------------------------------
66
+ // Parser
67
+ // ---------------------------------------------------------------------------
68
+
69
+ /** Parse narrative metadata from metadata JSON string, returns null if absent */
70
+ export function parseNarrative(metadata: string | undefined): NarrativeMetadata | null {
71
+ if (!metadata) return null;
72
+ try {
73
+ const parsed = JSON.parse(metadata);
74
+ if (parsed.narrative && typeof parsed.narrative.lifePeriodId === "string") {
75
+ return NarrativeMetadataSchema.parse(parsed.narrative);
76
+ }
77
+ } catch { /* malformed metadata — safe to ignore */ }
78
+ return null;
79
+ }
@@ -0,0 +1,164 @@
1
+ /**
2
+ * Rule-based Narrative Tagger — assigns autobiographical narrative metadata.
3
+ *
4
+ * Heuristics:
5
+ * 1. Life period = scope prefix + quarterly time window (Conway's lifetime periods)
6
+ * 2. General event = scope + daily cluster OR session ID (extended events)
7
+ * 3. Specific event = unique episode (scope + timestamp + text hash)
8
+ * 4. Keyword signals enrich labels (project-setup, debugging, learning, etc.)
9
+ *
10
+ * Zero LLM cost — pure rule-based, matching the emotion-detector pattern.
11
+ */
12
+
13
+ import type { NarrativeMetadata } from "./narrative-schema.js";
14
+ import { isNarrativeModeEnabled } from "./narrative-schema.js";
15
+
16
+ // ---------------------------------------------------------------------------
17
+ // Keyword Signals — enrich general event labels
18
+ // ---------------------------------------------------------------------------
19
+
20
+ const EVENT_KEYWORDS: ReadonlyArray<readonly [string, readonly string[]]> = [
21
+ ["project-setup", ["init", "setup", "bootstrap", "create", "scaffold", "new project", "初始化", "搭建"]],
22
+ ["development", ["implement", "build", "code", "feature", "refactor", "开发", "实现", "重构"]],
23
+ ["debugging", ["debug", "bug", "fix", "error", "crash", "investigate", "调试", "排查", "修复"]],
24
+ ["deployment", ["deploy", "ship", "release", "publish", "launch", "上线", "部署", "发布"]],
25
+ ["learning", ["learn", "study", "research", "explore", "understand", "学习", "研究", "探索"]],
26
+ ["migration", ["migrate", "upgrade", "move", "transfer", "port", "迁移", "升级"]],
27
+ ["review", ["review", "audit", "inspect", "check", "评审", "审查"]],
28
+ ["writing", ["write", "draft", "article", "blog", "post", "写作", "文章", "草稿"]],
29
+ ] as const;
30
+
31
+ // ---------------------------------------------------------------------------
32
+ // Helpers
33
+ // ---------------------------------------------------------------------------
34
+
35
+ function getQuarterLabel(date: Date): string {
36
+ const q = Math.floor(date.getMonth() / 3) + 1;
37
+ return `${date.getFullYear()}-Q${q}`;
38
+ }
39
+
40
+ function extractScopePrefix(scope: string): string {
41
+ return scope.split(":")[0] || "global";
42
+ }
43
+
44
+ /** Deterministic short hash for stable IDs */
45
+ function quickHash(str: string): string {
46
+ let hash = 0;
47
+ for (let i = 0; i < str.length; i++) {
48
+ const char = str.charCodeAt(i);
49
+ hash = ((hash << 5) - hash + char) | 0;
50
+ }
51
+ return Math.abs(hash).toString(36);
52
+ }
53
+
54
+ /** Detect the dominant event type from text via keyword matching */
55
+ function detectEventType(text: string): string | null {
56
+ const lower = text.toLowerCase();
57
+ for (const [label, keywords] of EVENT_KEYWORDS) {
58
+ if (keywords.some(kw => lower.includes(kw))) {
59
+ return label;
60
+ }
61
+ }
62
+ return null;
63
+ }
64
+
65
+ // ---------------------------------------------------------------------------
66
+ // ID Derivation
67
+ // ---------------------------------------------------------------------------
68
+
69
+ function deriveLifePeriodId(scope: string, timestamp: number): string {
70
+ const date = new Date(timestamp);
71
+ const quarter = getQuarterLabel(date);
72
+ const prefix = extractScopePrefix(scope);
73
+ return `lp:${prefix}:${quarter}`;
74
+ }
75
+
76
+ function deriveLifePeriodLabel(scope: string, timestamp: number): string {
77
+ const date = new Date(timestamp);
78
+ const quarter = getQuarterLabel(date);
79
+ const prefix = extractScopePrefix(scope);
80
+ return `${prefix} (${quarter})`;
81
+ }
82
+
83
+ function deriveGeneralEventId(scope: string, timestamp: number, sessionId?: string): string {
84
+ if (sessionId) {
85
+ return `ge:${sessionId}`;
86
+ }
87
+ const date = new Date(timestamp);
88
+ const day = date.toISOString().split("T")[0];
89
+ const scopeKey = scope.replace(/[^a-zA-Z0-9:_-]/g, "_").slice(0, 40);
90
+ return `ge:${scopeKey}:${day}`;
91
+ }
92
+
93
+ function deriveGeneralEventLabel(
94
+ scope: string,
95
+ text: string,
96
+ timestamp: number,
97
+ ): string {
98
+ const date = new Date(timestamp);
99
+ const day = date.toISOString().split("T")[0];
100
+ const eventType = detectEventType(text);
101
+ const scopeShort = scope.length > 30 ? scope.slice(0, 30) + "\u2026" : scope;
102
+
103
+ if (eventType) {
104
+ return `${eventType} @ ${scopeShort} (${day})`;
105
+ }
106
+ return `activity @ ${scopeShort} (${day})`;
107
+ }
108
+
109
+ function deriveSpecificEventId(scope: string, timestamp: number, textHash: string): string {
110
+ const prefix = extractScopePrefix(scope);
111
+ return `se:${prefix}:${timestamp}:${textHash}`;
112
+ }
113
+
114
+ function deriveSpecificEventLabel(text: string): string {
115
+ const clean = text.replace(/\s+/g, " ").trim();
116
+ return clean.length > 60 ? clean.slice(0, 57) + "\u2026" : clean;
117
+ }
118
+
119
+ // ---------------------------------------------------------------------------
120
+ // Public API
121
+ // ---------------------------------------------------------------------------
122
+
123
+ /**
124
+ * Tag a memory with narrative metadata using rule-based heuristics.
125
+ * Returns null when RECALLNEST_NARRATIVE_MODE is not enabled.
126
+ */
127
+ export function tagNarrative(params: {
128
+ scope: string;
129
+ text: string;
130
+ timestamp: number;
131
+ sessionId?: string;
132
+ sequence?: number;
133
+ }): NarrativeMetadata | null {
134
+ if (!isNarrativeModeEnabled()) return null;
135
+
136
+ const { scope, text, timestamp, sessionId, sequence } = params;
137
+ const textHash = quickHash(text.slice(0, 200));
138
+
139
+ return {
140
+ lifePeriodId: deriveLifePeriodId(scope, timestamp),
141
+ lifePeriodLabel: deriveLifePeriodLabel(scope, timestamp),
142
+ generalEventId: deriveGeneralEventId(scope, timestamp, sessionId),
143
+ generalEventLabel: deriveGeneralEventLabel(scope, text, timestamp),
144
+ specificEventId: deriveSpecificEventId(scope, timestamp, textHash),
145
+ specificEventLabel: deriveSpecificEventLabel(text),
146
+ startAt: timestamp,
147
+ endAt: null,
148
+ sequence: sequence ?? 0,
149
+ };
150
+ }
151
+
152
+ /**
153
+ * Conditionally tag narrative. Returns null when feature flag is off.
154
+ * Convenience wrapper matching the detectEmotionIfEnabled pattern.
155
+ */
156
+ export function tagNarrativeIfEnabled(params: {
157
+ scope: string;
158
+ text: string;
159
+ timestamp: number;
160
+ sessionId?: string;
161
+ sequence?: number;
162
+ }): NarrativeMetadata | null {
163
+ return tagNarrative(params);
164
+ }
@@ -0,0 +1,98 @@
1
+ /**
2
+ * F-3: Lightweight PII detection.
3
+ * Scans text for common sensitive patterns (passwords, tokens, IDs).
4
+ * Returns warnings, does not block writes.
5
+ */
6
+
7
+ export type PIISeverity = "high" | "medium" | "low";
8
+
9
+ export interface PIIDetection {
10
+ type: string; // "api_key" | "password" | "id_number" | "email" | "phone" | "credit_card"
11
+ severity: PIISeverity;
12
+ match: string; // the matched text (partially masked)
13
+ position: number; // char offset
14
+ }
15
+
16
+ export interface PIIScanResult {
17
+ hasPII: boolean;
18
+ detections: PIIDetection[];
19
+ summary: string; // human-readable summary
20
+ }
21
+
22
+ interface PIIRule {
23
+ type: string;
24
+ severity: PIISeverity;
25
+ pattern: RegExp;
26
+ }
27
+
28
+ const PII_RULES: PIIRule[] = [
29
+ {
30
+ type: "api_key",
31
+ severity: "high",
32
+ pattern: /(?:sk-[A-Za-z0-9_\-]{20,}|(?:api[_-]?key|token|secret)[=:\s]["']?[A-Za-z0-9_\-]{20,})/gi,
33
+ },
34
+ {
35
+ type: "password",
36
+ severity: "high",
37
+ pattern: /(?:password|passwd|pwd)[=:\s]["']?[^\s"']{8,}/gi,
38
+ },
39
+ {
40
+ type: "id_number",
41
+ severity: "high",
42
+ pattern: /\d{17}[\dXx]/g,
43
+ },
44
+ {
45
+ type: "credit_card",
46
+ severity: "high",
47
+ pattern: /\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}/g,
48
+ },
49
+ {
50
+ type: "phone",
51
+ severity: "medium",
52
+ pattern: /1[3-9]\d{9}/g,
53
+ },
54
+ {
55
+ type: "email",
56
+ severity: "low",
57
+ pattern: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
58
+ },
59
+ ];
60
+
61
+ function maskSensitive(value: string): string {
62
+ if (value.length <= 8) return value.slice(0, 2) + "***" + value.slice(-2);
63
+ return value.slice(0, 4) + "***" + value.slice(-4);
64
+ }
65
+
66
+ /** Scan text for potential PII. Pure regex, no LLM call. */
67
+ export function scanForPII(text: string): PIIScanResult {
68
+ const detections: PIIDetection[] = [];
69
+
70
+ for (const rule of PII_RULES) {
71
+ // Reset lastIndex for global regexes
72
+ rule.pattern.lastIndex = 0;
73
+ let m: RegExpExecArray | null;
74
+ while ((m = rule.pattern.exec(text)) !== null) {
75
+ detections.push({
76
+ type: rule.type,
77
+ severity: rule.severity,
78
+ match: maskSensitive(m[0]),
79
+ position: m.index,
80
+ });
81
+ }
82
+ }
83
+
84
+ const high = detections.filter((d) => d.severity === "high").length;
85
+ const medium = detections.filter((d) => d.severity === "medium").length;
86
+ const low = detections.filter((d) => d.severity === "low").length;
87
+
88
+ const summary =
89
+ detections.length === 0
90
+ ? "No PII detected"
91
+ : `Found ${detections.length} potential PII item${detections.length > 1 ? "s" : ""} (${high} high, ${medium} medium, ${low} low)`;
92
+
93
+ return {
94
+ hasPII: detections.length > 0,
95
+ detections,
96
+ summary,
97
+ };
98
+ }
@@ -0,0 +1,76 @@
1
+ /**
2
+ * E-2: Post-retrieval relevance filter.
3
+ * After retrieval returns top-K candidates, use LLM to judge relevance.
4
+ * Only fires for resume_context (batch recall), not search_memory (too slow for interactive).
5
+ */
6
+
7
+ import type { LLMClient } from "./llm-client.js";
8
+ import type { RetrievalResult } from "./retriever.js";
9
+
10
+ export interface FilterConfig {
11
+ /** Max items to filter per batch (default: 10) */
12
+ maxItems: number;
13
+ /** Minimum score to even consider filtering — below this, just drop (default: 0.40) */
14
+ minScoreForFilter: number;
15
+ }
16
+
17
+ const DEFAULT_FILTER_CONFIG: FilterConfig = {
18
+ maxItems: 10,
19
+ minScoreForFilter: 0.40,
20
+ };
21
+
22
+ const RELEVANCE_SYSTEM_PROMPT =
23
+ "你是记忆相关性过滤器。判断以下记忆是否与当前查询相关。\n" +
24
+ "只输出 JSON 数组:[true, false, true, ...](与输入顺序对应)";
25
+
26
+ /**
27
+ * Filter retrieval results by LLM relevance judgment.
28
+ * Returns only results judged as relevant.
29
+ * Falls back to returning all results on LLM failure.
30
+ */
31
+ export async function filterByRelevance(
32
+ results: RetrievalResult[],
33
+ query: string,
34
+ llm: LLMClient,
35
+ config?: Partial<FilterConfig>,
36
+ ): Promise<RetrievalResult[]> {
37
+ if (results.length === 0 || !query) return results;
38
+
39
+ const cfg: FilterConfig = { ...DEFAULT_FILTER_CONFIG, ...config };
40
+
41
+ // Split: items above threshold go to LLM filter, below threshold get dropped
42
+ const candidates = results.slice(0, cfg.maxItems).filter((r) => r.score >= cfg.minScoreForFilter);
43
+ if (candidates.length === 0) return [];
44
+
45
+ // Build user prompt: query + numbered memory excerpts (first 200 chars)
46
+ const excerpts = candidates
47
+ .map((r, i) => `[${i + 1}] ${extractExcerpt(r)}`)
48
+ .join("\n");
49
+ const userPrompt = `查询:${query}\n\n记忆列表:\n${excerpts}`;
50
+
51
+ try {
52
+ const verdicts = await llm.chatJson<boolean[]>(RELEVANCE_SYSTEM_PROMPT, userPrompt);
53
+ if (!Array.isArray(verdicts)) return candidates;
54
+
55
+ return candidates.filter((_, i) => i < verdicts.length && verdicts[i] === true);
56
+ } catch {
57
+ // Graceful fallback: return all candidates rather than losing data
58
+ return candidates;
59
+ }
60
+ }
61
+
62
+ /** Extract a short excerpt from a result for the LLM prompt. */
63
+ function extractExcerpt(result: RetrievalResult): string {
64
+ // Prefer l0_abstract from metadata if available, otherwise use raw text
65
+ if (result.entry.metadata) {
66
+ try {
67
+ const meta = JSON.parse(result.entry.metadata) as Record<string, unknown>;
68
+ if (typeof meta.l0_abstract === "string" && meta.l0_abstract.length > 0) {
69
+ return meta.l0_abstract.slice(0, 200);
70
+ }
71
+ } catch {
72
+ // Fall through to raw text
73
+ }
74
+ }
75
+ return result.entry.text.slice(0, 200);
76
+ }
@@ -0,0 +1,253 @@
1
+ /**
2
+ * Prediction Engine — HP-predictive (Phase 5)
3
+ *
4
+ * Pure heuristic prediction of prospective reminders.
5
+ * Zero LLM cost — all signals are derived from existing data structures:
6
+ * - Checkpoint openLoops (stale > 24h = high weight)
7
+ * - High-frequency but stale memories (not accessed recently)
8
+ * - Workflow observations with corrected/missed outcomes
9
+ * - Query topic tags that lack recent memory coverage
10
+ *
11
+ * Gated by RECALLNEST_PREDICTIVE_MEMORY=true feature flag.
12
+ */
13
+
14
+ import type { SessionCheckpointRecord } from "./session-schema.js";
15
+ import type { WorkflowObservationRecord } from "./workflow-observation-schema.js";
16
+
17
+ // ============================================================================
18
+ // Types
19
+ // ============================================================================
20
+
21
+ export interface PredictionSignal {
22
+ type: "stale_open_loop" | "stale_high_frequency" | "workflow_issue" | "uncovered_topic";
23
+ trigger: string;
24
+ action: string;
25
+ weight: number; // 0-1 raw signal strength
26
+ evidence: string[];
27
+ }
28
+
29
+ export interface ScoredPrediction {
30
+ trigger: string;
31
+ action: string;
32
+ confidence: number; // 0-1 final score
33
+ evidence: string[];
34
+ }
35
+
36
+ export interface PredictionContext {
37
+ /** Recent checkpoints (newest first) */
38
+ checkpoints: SessionCheckpointRecord[];
39
+ /** Recent workflow observations (newest first) */
40
+ workflowObservations: WorkflowObservationRecord[];
41
+ /** High-frequency memory texts + their last access timestamps */
42
+ frequentMemories: Array<{ text: string; topicTag?: string; lastAccessedAt: string; accessCount: number }>;
43
+ /** Topic tags from recent queries that had zero or few results */
44
+ uncoveredTopics: string[];
45
+ /** Current time for staleness calculation */
46
+ now?: Date;
47
+ }
48
+
49
+ // ============================================================================
50
+ // Constants
51
+ // ============================================================================
52
+
53
+ const CONFIDENCE_THRESHOLD = 0.6;
54
+ const STALE_OPEN_LOOP_HOURS = 24;
55
+ const STALE_MEMORY_DAYS = 14;
56
+ const MAX_PREDICTIONS = 5;
57
+
58
+ // Signal type weights for final scoring
59
+ const SIGNAL_WEIGHTS: Record<PredictionSignal["type"], number> = {
60
+ stale_open_loop: 0.9,
61
+ workflow_issue: 0.85,
62
+ stale_high_frequency: 0.7,
63
+ uncovered_topic: 0.65,
64
+ };
65
+
66
+ // ============================================================================
67
+ // Signal Collection
68
+ // ============================================================================
69
+
70
+ /**
71
+ * Collect raw prediction signals from available data sources.
72
+ * Each signal represents a potential predicted reminder.
73
+ */
74
+ export function collectSignals(context: PredictionContext): PredictionSignal[] {
75
+ const now = context.now ?? new Date();
76
+ const signals: PredictionSignal[] = [];
77
+
78
+ // --- Signal 1: Stale open loops from checkpoints ---
79
+ collectStaleOpenLoopSignals(context.checkpoints, now, signals);
80
+
81
+ // --- Signal 2: High-frequency but stale memories ---
82
+ collectStaleHighFrequencySignals(context.frequentMemories, now, signals);
83
+
84
+ // --- Signal 3: Workflow observations with corrected/missed outcomes ---
85
+ collectWorkflowIssueSignals(context.workflowObservations, signals);
86
+
87
+ // --- Signal 4: Uncovered topic tags ---
88
+ collectUncoveredTopicSignals(context.uncoveredTopics, signals);
89
+
90
+ return signals;
91
+ }
92
+
93
+ function collectStaleOpenLoopSignals(
94
+ checkpoints: SessionCheckpointRecord[],
95
+ now: Date,
96
+ signals: PredictionSignal[],
97
+ ): void {
98
+ const seenLoops = new Set<string>();
99
+
100
+ for (const cp of checkpoints) {
101
+ if (!cp.openLoops || cp.openLoops.length === 0) continue;
102
+
103
+ const cpTime = new Date(cp.updatedAt);
104
+ const hoursAgo = (now.getTime() - cpTime.getTime()) / (1000 * 60 * 60);
105
+
106
+ for (const loop of cp.openLoops) {
107
+ const normalized = loop.toLowerCase().trim();
108
+ if (seenLoops.has(normalized)) continue;
109
+ seenLoops.add(normalized);
110
+
111
+ if (hoursAgo >= STALE_OPEN_LOOP_HOURS) {
112
+ // Weight increases with staleness, capped at 1.0
113
+ const stalenessBoost = Math.min(hoursAgo / (STALE_OPEN_LOOP_HOURS * 3), 1.0);
114
+ signals.push({
115
+ type: "stale_open_loop",
116
+ trigger: loop,
117
+ action: `Unresolved from ${Math.floor(hoursAgo)}h ago: "${loop}"`,
118
+ weight: 0.6 + stalenessBoost * 0.4,
119
+ evidence: [
120
+ `Open loop from checkpoint ${cp.sessionId} (${Math.floor(hoursAgo)}h ago)`,
121
+ `Scope: ${cp.resolvedScope ?? cp.scope ?? "unknown"}`,
122
+ ],
123
+ });
124
+ }
125
+ }
126
+ }
127
+ }
128
+
129
+ function collectStaleHighFrequencySignals(
130
+ memories: PredictionContext["frequentMemories"],
131
+ now: Date,
132
+ signals: PredictionSignal[],
133
+ ): void {
134
+ for (const mem of memories) {
135
+ const lastAccess = new Date(mem.lastAccessedAt);
136
+ const daysSinceAccess = (now.getTime() - lastAccess.getTime()) / (1000 * 60 * 60 * 24);
137
+
138
+ if (daysSinceAccess >= STALE_MEMORY_DAYS && mem.accessCount >= 3) {
139
+ const stalenessRatio = Math.min(daysSinceAccess / (STALE_MEMORY_DAYS * 3), 1.0);
140
+ const frequencyBoost = Math.min(mem.accessCount / 10, 1.0);
141
+ const weight = 0.5 + (stalenessRatio * 0.3) + (frequencyBoost * 0.2);
142
+
143
+ const topicLabel = mem.topicTag ? ` [${mem.topicTag}]` : "";
144
+ const shortText = mem.text.length > 80 ? mem.text.slice(0, 77) + "..." : mem.text;
145
+
146
+ signals.push({
147
+ type: "stale_high_frequency",
148
+ trigger: mem.topicTag ?? shortText,
149
+ action: `Revisit frequently-used memory${topicLabel}: "${shortText}"`,
150
+ weight: Math.min(weight, 1.0),
151
+ evidence: [
152
+ `Accessed ${mem.accessCount} times but not in ${Math.floor(daysSinceAccess)} days`,
153
+ ...(mem.topicTag ? [`Topic: ${mem.topicTag}`] : []),
154
+ ],
155
+ });
156
+ }
157
+ }
158
+ }
159
+
160
+ function collectWorkflowIssueSignals(
161
+ observations: WorkflowObservationRecord[],
162
+ signals: PredictionSignal[],
163
+ ): void {
164
+ // Group by workflowId, count corrected/missed
165
+ const issueMap = new Map<string, { count: number; summaries: string[]; latestAt: string }>();
166
+
167
+ for (const obs of observations) {
168
+ if (obs.outcome !== "corrected" && obs.outcome !== "missed") continue;
169
+
170
+ const existing = issueMap.get(obs.workflowId);
171
+ if (existing) {
172
+ existing.count++;
173
+ if (existing.summaries.length < 3) existing.summaries.push(obs.summary);
174
+ } else {
175
+ issueMap.set(obs.workflowId, {
176
+ count: 1,
177
+ summaries: [obs.summary],
178
+ latestAt: obs.recordedAt,
179
+ });
180
+ }
181
+ }
182
+
183
+ for (const [workflowId, data] of issueMap) {
184
+ if (data.count < 1) continue; // Even a single correction is worth noting
185
+
186
+ const weight = Math.min(0.6 + data.count * 0.1, 1.0);
187
+ signals.push({
188
+ type: "workflow_issue",
189
+ trigger: workflowId,
190
+ action: `Workflow "${workflowId}" had ${data.count} issue(s): ${data.summaries[0]}`,
191
+ weight,
192
+ evidence: [
193
+ `${data.count} corrected/missed observations for workflow "${workflowId}"`,
194
+ `Latest: ${data.latestAt}`,
195
+ ...data.summaries.slice(0, 2),
196
+ ],
197
+ });
198
+ }
199
+ }
200
+
201
+ function collectUncoveredTopicSignals(
202
+ uncoveredTopics: string[],
203
+ signals: PredictionSignal[],
204
+ ): void {
205
+ for (const topic of uncoveredTopics) {
206
+ signals.push({
207
+ type: "uncovered_topic",
208
+ trigger: topic,
209
+ action: `No recent memories for queried topic "${topic}" — consider storing relevant context`,
210
+ weight: 0.65,
211
+ evidence: [`Topic "${topic}" appeared in queries but has no recent memory coverage`],
212
+ });
213
+ }
214
+ }
215
+
216
+ // ============================================================================
217
+ // Scoring
218
+ // ============================================================================
219
+
220
+ /**
221
+ * Score and rank prediction signals.
222
+ * Returns top predictions above the confidence threshold.
223
+ */
224
+ export function scorePredictions(signals: PredictionSignal[]): ScoredPrediction[] {
225
+ const scored: ScoredPrediction[] = signals.map(signal => {
226
+ const typeWeight = SIGNAL_WEIGHTS[signal.type];
227
+ const confidence = Math.round(signal.weight * typeWeight * 100) / 100;
228
+
229
+ return {
230
+ trigger: signal.trigger,
231
+ action: signal.action,
232
+ confidence,
233
+ evidence: signal.evidence,
234
+ };
235
+ });
236
+
237
+ return scored
238
+ .filter(p => p.confidence >= CONFIDENCE_THRESHOLD)
239
+ .sort((a, b) => b.confidence - a.confidence)
240
+ .slice(0, MAX_PREDICTIONS);
241
+ }
242
+
243
+ // ============================================================================
244
+ // Exports for testing
245
+ // ============================================================================
246
+
247
+ export const _testConstants = {
248
+ CONFIDENCE_THRESHOLD,
249
+ STALE_OPEN_LOOP_HOURS,
250
+ STALE_MEMORY_DAYS,
251
+ MAX_PREDICTIONS,
252
+ SIGNAL_WEIGHTS,
253
+ } as const;