recallnest 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +1 -1
  3. package/README.md +30 -5
  4. package/README_CN.md +30 -5
  5. package/config.json +55 -0
  6. package/connectors/examples/email-adapter.ts +73 -0
  7. package/connectors/examples/logs-adapter.ts +103 -0
  8. package/connectors/examples/rss-adapter.ts +95 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +60 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/ingest-2026-04-13.log +87 -0
  14. package/logs/launchagent-stderr.log +0 -0
  15. package/logs/launchagent-stdout.log +0 -0
  16. package/package.json +1 -1
  17. package/src/cli.ts +30 -0
  18. package/src/connector-types.ts +57 -0
  19. package/src/conversation-importer.ts +25 -1
  20. package/src/data-checkup.ts +31 -1
  21. package/src/doctor.ts +29 -0
  22. package/src/ingest.ts +203 -0
  23. package/src/mcp-server.ts +2 -2
  24. package/src/obsidian-connector.ts +282 -0
  25. package/src/session-schema.ts +2 -2
  26. package/src/source-heartbeat.ts +108 -0
  27. package/.claude-plugin/CLAUDE.md +0 -29
  28. package/.claude-plugin/marketplace.json +0 -18
  29. package/.dockerignore +0 -11
  30. package/.github/workflows/ci.yml +0 -28
  31. package/.mcp.json +0 -9
  32. package/CHANGELOG.md +0 -51
  33. package/DEMO.md +0 -94
  34. package/Dockerfile +0 -20
  35. package/EVAL.md +0 -89
  36. package/FAILURES.md +0 -75
  37. package/OPERATIONS.md +0 -59
  38. package/ROADMAP.md +0 -245
  39. package/SKILL.md +0 -250
  40. package/assets/dashboard.png +0 -0
  41. package/assets/knowledge-graph.png +0 -0
  42. package/assets/recallnest-workbench.png +0 -0
  43. package/assets/screenshots/ui-full.png +0 -0
  44. package/assets/screenshots/ui-overview.png +0 -0
  45. package/assets/screenshots/ui-search.png +0 -0
  46. package/assets/ui/app.js +0 -1031
  47. package/assets/ui/index.html +0 -224
  48. package/assets/ui/styles.css +0 -781
  49. package/assets/wechat-qr.jpg +0 -0
  50. package/bun.lock +0 -337
  51. package/daily-reflections/2026-03-05.md +0 -15
  52. package/daily-reflections/2026-03-06.md +0 -15
  53. package/daily-reflections/2026-03-07.md +0 -7
  54. package/daily-reflections/2026-03-08.md +0 -34
  55. package/daily-reflections/2026-03-09.md +0 -35
  56. package/daily-reflections/2026-03-10.md +0 -36
  57. package/daily-reflections/2026-03-11.md +0 -35
  58. package/daily-reflections/2026-03-12.md +0 -36
  59. package/daily-reflections/2026-03-13.md +0 -35
  60. package/daily-reflections/2026-03-14.md +0 -35
  61. package/daily-reflections/2026-03-15.md +0 -35
  62. package/daily-reflections/2026-03-16.md +0 -41
  63. package/daily-reflections/2026-03-17.md +0 -10
  64. package/daily-reflections/2026-03-18.md +0 -10
  65. package/daily-reflections/2026-03-19.md +0 -10
  66. package/daily-reflections/2026-03-20.md +0 -10
  67. package/daily-reflections/2026-03-21.md +0 -10
  68. package/daily-reflections/2026-03-22.md +0 -10
  69. package/daily-reflections/2026-03-23.md +0 -10
  70. package/daily-reflections/2026-03-24.md +0 -10
  71. package/daily-reflections/2026-03-25.md +0 -32
  72. package/daily-reflections/2026-03-26.md +0 -37
  73. package/daily-reflections/2026-03-27.md +0 -35
  74. package/daily-reflections/2026-03-28.md +0 -35
  75. package/daily-reflections/2026-03-29.md +0 -35
  76. package/daily-reflections/2026-03-30.md +0 -36
  77. package/daily-reflections/2026-03-31.md +0 -35
  78. package/daily-reflections/2026-04-01.md +0 -35
  79. package/daily-reflections/2026-04-02.md +0 -35
  80. package/daily-reflections/2026-04-03.md +0 -37
  81. package/daily-reflections/2026-04-04.md +0 -37
  82. package/daily-reflections/2026-04-05.md +0 -35
  83. package/daily-reflections/2026-04-06.md +0 -36
  84. package/daily-reflections/2026-04-07.md +0 -35
  85. package/daily-reflections/2026-04-08.md +0 -35
  86. package/daily-reflections/2026-04-12.md +0 -35
  87. package/data/alias-map.example.json +0 -22
  88. package/docs/api-reference.md +0 -1291
  89. package/docs/architecture.md +0 -168
  90. package/docs/memory-boundary-contract.md +0 -226
  91. package/docs/memory-categories.md +0 -119
  92. package/docs/pipeline-audit-ultramemory.md +0 -177
  93. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  94. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  95. package/docs/plans/2026-04-01-recall-boost.md +0 -100
  96. package/docs/plans/mcp-tiering-plan.md +0 -126
  97. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  98. package/docs/self-evolution.md +0 -198
  99. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  100. package/docs/superpowers/plans/2026-04-11-constructive-retrieval-emotion.md +0 -1195
  101. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  102. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  103. package/docs/superpowers/specs/2026-04-11-constructive-retrieval-emotion-design.md +0 -398
  104. package/eval/continuity/results-2026-03-16.md +0 -117
  105. package/eval/continuity/results-2026-03-19.md +0 -453
  106. package/eval/continuity/runbook.md +0 -110
  107. package/eval/continuity/seed-cases.md +0 -54
  108. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  109. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  110. package/eval/reports/2026-03-06-baseline.md +0 -82
  111. package/eval/reports/query-expansion.md +0 -138
  112. package/glama.json +0 -6
  113. package/lm +0 -19
  114. package/plan-philosophy-upgrades.md +0 -363
  115. package/plan-product-upgrades.md +0 -155
  116. package/plan-v1.2-distributable.md +0 -105
  117. package/plan.md +0 -140
  118. package/research-memory-evolution.md +0 -440
  119. package/research-rl-agent-memory.md +0 -330
  120. package/scripts/archive-distilled-junk.ts +0 -116
  121. package/scripts/backfill-anchors.ts +0 -130
  122. package/scripts/backfill-l0l1l2.ts +0 -98
  123. package/scripts/blind-scan.ts +0 -588
  124. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  125. package/scripts/cleanup-step2-template-dups.ts +0 -113
  126. package/scripts/cleanup-step3-stale-content.ts +0 -149
  127. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  128. package/scripts/delete-distilled-junk.ts +0 -88
  129. package/scripts/diagnose-vector-search.ts +0 -132
  130. package/scripts/distill-facts.ts +0 -305
  131. package/scripts/health-check.ts +0 -343
  132. package/scripts/incremental-ingest.sh +0 -71
  133. package/scripts/migrate-language.ts +0 -75
  134. package/scripts/mlp-health-check.ts +0 -382
  135. package/scripts/pull-claude-desktop.ts +0 -515
  136. package/scripts/re-ingest-cc.sh +0 -66
  137. package/scripts/rebalance-memory-health.ts +0 -181
  138. package/scripts/run-desktop-ingest.sh +0 -11
  139. package/scripts/start-server.sh +0 -12
  140. package/scripts/verify-emotion-decay.ts +0 -148
  141. package/scripts/viz-dashboard.html +0 -423
  142. package/scripts/viz-export.ts +0 -703
  143. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  144. package/src/__tests__/activity-counter.test.ts +0 -96
  145. package/src/__tests__/admission-control.test.ts +0 -206
  146. package/src/__tests__/anchor-generator.test.ts +0 -67
  147. package/src/__tests__/auto-consolidation.test.ts +0 -168
  148. package/src/__tests__/auto-recall.test.ts +0 -131
  149. package/src/__tests__/babel-integration.test.ts +0 -139
  150. package/src/__tests__/batch-dedup.test.ts +0 -65
  151. package/src/__tests__/capture-engine.test.ts +0 -1144
  152. package/src/__tests__/capture-heuristic.test.ts +0 -171
  153. package/src/__tests__/cascade-forget.test.ts +0 -210
  154. package/src/__tests__/category-min-scores.test.ts +0 -33
  155. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  156. package/src/__tests__/circuit-breaker.test.ts +0 -99
  157. package/src/__tests__/cluster-consolidation.test.ts +0 -712
  158. package/src/__tests__/confidence-metadata.test.ts +0 -125
  159. package/src/__tests__/confidence-tracker.test.ts +0 -249
  160. package/src/__tests__/conflict-advisor.test.ts +0 -206
  161. package/src/__tests__/conflict-escalation.test.ts +0 -149
  162. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  163. package/src/__tests__/conflict-output.test.ts +0 -85
  164. package/src/__tests__/conflict-store.test.ts +0 -374
  165. package/src/__tests__/consolidation-engine.test.ts +0 -181
  166. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  167. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  168. package/src/__tests__/context-composer-pins.test.ts +0 -64
  169. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  170. package/src/__tests__/context-reconstructor.test.ts +0 -65
  171. package/src/__tests__/context-renderer.test.ts +0 -91
  172. package/src/__tests__/continuity-eval.test.ts +0 -275
  173. package/src/__tests__/continuity-schema.test.ts +0 -97
  174. package/src/__tests__/conversation-importer.test.ts +0 -692
  175. package/src/__tests__/core-recall-eval.test.ts +0 -235
  176. package/src/__tests__/core-summary.test.ts +0 -109
  177. package/src/__tests__/data-checkup.test.ts +0 -225
  178. package/src/__tests__/decay-exemption.test.ts +0 -84
  179. package/src/__tests__/dedup-actions.test.ts +0 -58
  180. package/src/__tests__/dedup-guard.test.ts +0 -39
  181. package/src/__tests__/deterministic-id.test.ts +0 -39
  182. package/src/__tests__/distill-lock.test.ts +0 -176
  183. package/src/__tests__/doctor.test.ts +0 -303
  184. package/src/__tests__/dream-pipeline.test.ts +0 -232
  185. package/src/__tests__/drill-down.test.ts +0 -108
  186. package/src/__tests__/embedder.test.ts +0 -122
  187. package/src/__tests__/emotion-decay.test.ts +0 -46
  188. package/src/__tests__/emotion-detector.test.ts +0 -56
  189. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  190. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  191. package/src/__tests__/entity-resolver.test.ts +0 -58
  192. package/src/__tests__/ethics-layer.test.ts +0 -476
  193. package/src/__tests__/event-segmenter.test.ts +0 -153
  194. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  195. package/src/__tests__/frequency-tracker.test.ts +0 -113
  196. package/src/__tests__/graph-export.test.ts +0 -489
  197. package/src/__tests__/hotness-blend.test.ts +0 -76
  198. package/src/__tests__/ingest.test.ts +0 -360
  199. package/src/__tests__/interference-detector.test.ts +0 -203
  200. package/src/__tests__/kg-extractor.test.ts +0 -184
  201. package/src/__tests__/kg-store.test.ts +0 -198
  202. package/src/__tests__/llm-consolidation.test.ts +0 -226
  203. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  204. package/src/__tests__/memory-boundaries.test.ts +0 -237
  205. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -243
  206. package/src/__tests__/memory-evolution.test.ts +0 -271
  207. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  208. package/src/__tests__/memory-lint.test.ts +0 -271
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/narrative-architecture.test.ts +0 -456
  212. package/src/__tests__/pending-review.test.ts +0 -77
  213. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  214. package/src/__tests__/ppr-traversal.test.ts +0 -256
  215. package/src/__tests__/prediction-engine.test.ts +0 -349
  216. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  217. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  218. package/src/__tests__/preference-matcher.test.ts +0 -230
  219. package/src/__tests__/preference-slots.test.ts +0 -205
  220. package/src/__tests__/prospective-memory.test.ts +0 -283
  221. package/src/__tests__/query-entity-detector.test.ts +0 -171
  222. package/src/__tests__/query-expander-alias.test.ts +0 -89
  223. package/src/__tests__/query-expander.test.ts +0 -34
  224. package/src/__tests__/recall-governor.test.ts +0 -227
  225. package/src/__tests__/recall-mode.test.ts +0 -64
  226. package/src/__tests__/result-synthesizer.test.ts +0 -198
  227. package/src/__tests__/resume-context.test.ts +0 -4898
  228. package/src/__tests__/retrieval-trace.test.ts +0 -213
  229. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  230. package/src/__tests__/retroactive-boost.test.ts +0 -223
  231. package/src/__tests__/salience-scoring.test.ts +0 -106
  232. package/src/__tests__/scope-inventory.test.ts +0 -200
  233. package/src/__tests__/scope-policy.test.ts +0 -69
  234. package/src/__tests__/session-checkpoint.test.ts +0 -259
  235. package/src/__tests__/session-distiller.test.ts +0 -611
  236. package/src/__tests__/session-gc.test.ts +0 -215
  237. package/src/__tests__/skill-engine.test.ts +0 -546
  238. package/src/__tests__/skill-promotion.test.ts +0 -349
  239. package/src/__tests__/store-scope-audit.test.ts +0 -81
  240. package/src/__tests__/supersede-chain.test.ts +0 -154
  241. package/src/__tests__/temporal-parser.test.ts +0 -241
  242. package/src/__tests__/temporal-validity.test.ts +0 -160
  243. package/src/__tests__/term-registry.test.ts +0 -121
  244. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  245. package/src/__tests__/topic-tag.test.ts +0 -113
  246. package/src/__tests__/trusted-memory.test.ts +0 -828
  247. package/src/__tests__/version-manager.test.ts +0 -213
  248. package/src/__tests__/workflow-observation.test.ts +0 -400
  249. package/src/__tests__/write-verifier.test.ts +0 -122
@@ -1,703 +0,0 @@
1
- /**
2
- * 可视化数据导出 + HTML 仪表盘生成
3
- * 模块 6:把体检数据做成交互式可视化
4
- *
5
- * 用法:bun run scripts/viz-export.ts [lancedb-dir]
6
- * 输出:scripts/viz-dashboard.html(本地打开即可)
7
- */
8
-
9
- import lancedb from "@lancedb/lancedb";
10
- import { writeFileSync } from "fs";
11
- import { join, dirname } from "path";
12
-
13
- const DB_PATH = process.argv[2] || "./data/lancedb";
14
- const TABLE_NAME = "memories";
15
- const OUTPUT_HTML = join(dirname(new URL(import.meta.url).pathname), "viz-dashboard.html");
16
-
17
- interface MemoryRow {
18
- id: string;
19
- text: string;
20
- vector: number[];
21
- category: string;
22
- scope: string;
23
- importance: number;
24
- timestamp: number;
25
- metadata?: string;
26
- }
27
-
28
- function cosineSimilarity(a: number[], b: number[]): number {
29
- let dot = 0, normA = 0, normB = 0;
30
- for (let i = 0; i < a.length; i++) {
31
- dot += a[i] * b[i];
32
- normA += a[i] * a[i];
33
- normB += b[i] * b[i];
34
- }
35
- return dot / (Math.sqrt(normA) * Math.sqrt(normB));
36
- }
37
-
38
- async function main() {
39
- console.log("⏳ 读取数据...");
40
- const db = await lancedb.connect(DB_PATH);
41
- const table = await db.openTable(TABLE_NAME);
42
-
43
- const allRows: MemoryRow[] = await table
44
- .query()
45
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
46
- .toArray() as any;
47
-
48
- const total = allRows.length;
49
- console.log(`✅ ${total.toLocaleString()} 条记忆`);
50
-
51
- const parsedRows = allRows.map(row => ({
52
- ...row,
53
- meta: row.metadata ? JSON.parse(row.metadata) : {},
54
- }));
55
-
56
- // ── Prepare visualization data ──
57
-
58
- // 1. Category distribution
59
- const catMap = new Map<string, number>();
60
- for (const r of parsedRows) {
61
- catMap.set(r.category, (catMap.get(r.category) || 0) + 1);
62
- }
63
- const categoryData = Object.fromEntries([...catMap.entries()].sort((a, b) => b[1] - a[1]));
64
-
65
- // 2. Tier distribution
66
- const tierMap = new Map<string, number>();
67
- for (const r of parsedRows) {
68
- const tier = r.meta.tier ?? "unknown";
69
- tierMap.set(tier, (tierMap.get(tier) || 0) + 1);
70
- }
71
- const tierData = Object.fromEntries([...tierMap.entries()].sort((a, b) => b[1] - a[1]));
72
-
73
- // 3. Monthly timeline
74
- const monthMap = new Map<string, {total: number, cats: Record<string, number>}>();
75
- for (const r of parsedRows) {
76
- const month = new Date(r.timestamp).toISOString().slice(0, 7);
77
- if (!monthMap.has(month)) monthMap.set(month, { total: 0, cats: {} });
78
- const m = monthMap.get(month)!;
79
- m.total++;
80
- m.cats[r.category] = (m.cats[r.category] || 0) + 1;
81
- }
82
- const timelineData = Object.fromEntries([...monthMap.entries()].sort());
83
-
84
- // 4. Daily timeline (last 90 days)
85
- const now = Date.now();
86
- const dayMap = new Map<string, number>();
87
- for (const r of parsedRows) {
88
- if (now - r.timestamp < 90 * 24 * 60 * 60 * 1000) {
89
- const day = new Date(r.timestamp).toISOString().slice(0, 10);
90
- dayMap.set(day, (dayMap.get(day) || 0) + 1);
91
- }
92
- }
93
- const dailyData = Object.fromEntries([...dayMap.entries()].sort());
94
-
95
- // 5. Hourly pattern
96
- const hourData: number[] = new Array(24).fill(0);
97
- for (const r of parsedRows) {
98
- hourData[new Date(r.timestamp).getHours()]++;
99
- }
100
-
101
- // 6. Day of week
102
- const dowData: number[] = new Array(7).fill(0);
103
- for (const r of parsedRows) {
104
- dowData[new Date(r.timestamp).getDay()]++;
105
- }
106
-
107
- // 7. Importance distribution (histogram)
108
- const impBins = 20;
109
- const impHist: number[] = new Array(impBins).fill(0);
110
- for (const r of parsedRows) {
111
- const bin = Math.min(impBins - 1, Math.floor(r.importance * impBins));
112
- impHist[bin]++;
113
- }
114
-
115
- // 8. Access count distribution
116
- const accessBuckets: Record<string, number> = {
117
- "0 (死记忆)": 0, "1-2": 0, "3-5": 0, "6-10": 0, "11-50": 0, "50+": 0,
118
- };
119
- for (const r of parsedRows) {
120
- const ac = r.meta.accessCount ?? r.meta.access_count ?? 0;
121
- if (ac === 0) accessBuckets["0 (死记忆)"]++;
122
- else if (ac <= 2) accessBuckets["1-2"]++;
123
- else if (ac <= 5) accessBuckets["3-5"]++;
124
- else if (ac <= 10) accessBuckets["6-10"]++;
125
- else if (ac <= 50) accessBuckets["11-50"]++;
126
- else accessBuckets["50+"]++;
127
- }
128
-
129
- // 9. Top scopes
130
- const scopeMap = new Map<string, number>();
131
- for (const r of parsedRows) {
132
- const s = r.scope || "(empty)";
133
- scopeMap.set(s, (scopeMap.get(s) || 0) + 1);
134
- }
135
- const topScopes = Object.fromEntries(
136
- [...scopeMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, 15)
137
- );
138
-
139
- // 10. Text length distribution
140
- const lenBins = 30;
141
- const maxLen = 3000; // cap for histogram
142
- const lenHist: number[] = new Array(lenBins).fill(0);
143
- for (const r of parsedRows) {
144
- const bin = Math.min(lenBins - 1, Math.floor(r.text.length / maxLen * lenBins));
145
- lenHist[bin]++;
146
- }
147
-
148
- // 11. Embedding 2D projection (PCA on sample)
149
- console.log("⏳ 读取向量样本做 PCA 投影...");
150
- const pca_sample_size = 3000;
151
- const sampleRows: MemoryRow[] = await table
152
- .query()
153
- .select(["id", "text", "vector", "category", "scope", "timestamp"])
154
- .limit(pca_sample_size)
155
- .toArray() as any;
156
-
157
- // Simple PCA: project onto top 2 principal components
158
- const dim = sampleRows[0]?.vector?.length || 0;
159
- let pcaPoints: Array<{x: number, y: number, category: string, scope: string, text: string}> = [];
160
-
161
- if (dim > 0) {
162
- // Compute mean
163
- const mean = new Array(dim).fill(0);
164
- for (const r of sampleRows) {
165
- for (let d = 0; d < dim; d++) mean[d] += r.vector[d];
166
- }
167
- for (let d = 0; d < dim; d++) mean[d] /= sampleRows.length;
168
-
169
- // Center data (vector may be Float32Array, convert to plain array)
170
- const centered = sampleRows.map(r => {
171
- const vec = Array.from(r.vector);
172
- return vec.map((v, d) => v - mean[d]);
173
- });
174
-
175
- // Power iteration for top 2 eigenvectors
176
- function powerIteration(data: number[][], deflateVec?: number[]): number[] {
177
- const n = data.length;
178
- const d = data[0].length;
179
- let v = new Array(d).fill(0).map(() => Math.random() - 0.5);
180
- // Normalize
181
- let norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0));
182
- v = v.map(x => x / norm);
183
-
184
- // If deflating, project out previous component
185
- let dataToUse = data;
186
- if (deflateVec) {
187
- dataToUse = data.map(row => {
188
- const proj = row.reduce((s, x, i) => s + x * deflateVec[i], 0);
189
- return row.map((x, i) => x - proj * deflateVec[i]);
190
- });
191
- }
192
-
193
- for (let iter = 0; iter < 50; iter++) {
194
- // v_new = X^T * X * v
195
- const Xv = dataToUse.map(row => row.reduce((s, x, i) => s + x * v[i], 0));
196
- const newV = new Array(d).fill(0);
197
- for (let i = 0; i < n; i++) {
198
- for (let j = 0; j < d; j++) {
199
- newV[j] += dataToUse[i][j] * Xv[i];
200
- }
201
- }
202
- norm = Math.sqrt(newV.reduce((s, x) => s + x * x, 0));
203
- v = newV.map(x => x / norm);
204
- }
205
- return v;
206
- }
207
-
208
- const pc1 = powerIteration(centered);
209
- const pc2 = powerIteration(centered, pc1);
210
-
211
- // Project
212
- pcaPoints = sampleRows.map((r, i) => ({
213
- x: centered[i].reduce((s, v, d) => s + v * pc1[d], 0),
214
- y: centered[i].reduce((s, v, d) => s + v * pc2[d], 0),
215
- category: r.category,
216
- scope: (r.scope || "").slice(0, 40),
217
- text: r.text.slice(0, 80),
218
- }));
219
- }
220
-
221
- // 12. Scope × Category heatmap (top 10 scopes × all categories)
222
- const scopeCatMatrix: Record<string, Record<string, number>> = {};
223
- const heatmapScopes = [...scopeMap.entries()]
224
- .sort((a, b) => b[1] - a[1])
225
- .slice(0, 10)
226
- .map(([s]) => s);
227
- const allCats = [...catMap.keys()].sort();
228
-
229
- for (const s of heatmapScopes) {
230
- scopeCatMatrix[s] = {};
231
- for (const c of allCats) scopeCatMatrix[s][c] = 0;
232
- }
233
- for (const r of parsedRows) {
234
- const s = r.scope || "(empty)";
235
- if (scopeCatMatrix[s]) {
236
- scopeCatMatrix[s][r.category] = (scopeCatMatrix[s][r.category] || 0) + 1;
237
- }
238
- }
239
-
240
- // ── Summary stats ──
241
- const summaryStats = {
242
- total,
243
- categories: catMap.size,
244
- scopes: scopeMap.size,
245
- deadMemories: accessBuckets["0 (死记忆)"],
246
- deadPct: (accessBuckets["0 (死记忆)"] / total * 100).toFixed(1),
247
- oldest: new Date(Math.min(...parsedRows.map(r => r.timestamp))).toISOString().slice(0, 10),
248
- newest: new Date(Math.max(...parsedRows.map(r => r.timestamp))).toISOString().slice(0, 10),
249
- avgImportance: (parsedRows.reduce((s, r) => s + r.importance, 0) / total).toFixed(3),
250
- };
251
-
252
- // ── Generate HTML ──
253
- const vizData = {
254
- summaryStats,
255
- categoryData,
256
- tierData,
257
- timelineData,
258
- dailyData,
259
- hourData,
260
- dowData,
261
- impHist,
262
- accessBuckets,
263
- topScopes,
264
- lenHist,
265
- pcaPoints,
266
- scopeCatMatrix,
267
- heatmapScopes,
268
- allCats,
269
- };
270
-
271
- const html = generateHTML(vizData);
272
- writeFileSync(OUTPUT_HTML, html, "utf-8");
273
- console.log(`\n✅ 仪表盘已生成: ${OUTPUT_HTML}`);
274
- console.log(" 用浏览器打开即可查看交互图表");
275
- }
276
-
277
- function generateHTML(data: any): string {
278
- return `<!DOCTYPE html>
279
- <html lang="zh-CN">
280
- <head>
281
- <meta charset="UTF-8">
282
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
283
- <title>RecallNest 记忆体检仪表盘</title>
284
- <script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
285
- <style>
286
- * { margin: 0; padding: 0; box-sizing: border-box; }
287
- body {
288
- font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
289
- background: #0f0f23;
290
- color: #e0e0e0;
291
- padding: 20px;
292
- }
293
- h1 {
294
- text-align: center;
295
- font-size: 28px;
296
- color: #00d4ff;
297
- margin-bottom: 8px;
298
- }
299
- .subtitle {
300
- text-align: center;
301
- color: #888;
302
- margin-bottom: 30px;
303
- font-size: 14px;
304
- }
305
- .stats-bar {
306
- display: flex;
307
- justify-content: center;
308
- gap: 30px;
309
- margin-bottom: 30px;
310
- flex-wrap: wrap;
311
- }
312
- .stat-card {
313
- background: #1a1a3e;
314
- border: 1px solid #333;
315
- border-radius: 12px;
316
- padding: 16px 24px;
317
- text-align: center;
318
- min-width: 140px;
319
- }
320
- .stat-card .value {
321
- font-size: 28px;
322
- font-weight: bold;
323
- color: #00d4ff;
324
- }
325
- .stat-card .label {
326
- font-size: 12px;
327
- color: #888;
328
- margin-top: 4px;
329
- }
330
- .stat-card.alert .value { color: #ff6b6b; }
331
- .grid {
332
- display: grid;
333
- grid-template-columns: 1fr 1fr;
334
- gap: 20px;
335
- max-width: 1400px;
336
- margin: 0 auto;
337
- }
338
- .chart-box {
339
- background: #1a1a3e;
340
- border: 1px solid #333;
341
- border-radius: 12px;
342
- padding: 16px;
343
- }
344
- .chart-box.full-width {
345
- grid-column: 1 / -1;
346
- }
347
- .chart-box h3 {
348
- color: #00d4ff;
349
- font-size: 16px;
350
- margin-bottom: 12px;
351
- }
352
- .chart-container {
353
- width: 100%;
354
- min-height: 350px;
355
- }
356
- .chart-container.tall {
357
- min-height: 500px;
358
- }
359
- footer {
360
- text-align: center;
361
- color: #555;
362
- margin-top: 40px;
363
- font-size: 12px;
364
- }
365
- @media (max-width: 900px) {
366
- .grid { grid-template-columns: 1fr; }
367
- }
368
- </style>
369
- </head>
370
- <body>
371
-
372
- <h1>🧠 RecallNest 记忆体检仪表盘</h1>
373
- <p class="subtitle">生成时间: ${new Date().toISOString().slice(0, 19)} · 数据来源: LanceDB memories table</p>
374
-
375
- <div class="stats-bar">
376
- <div class="stat-card">
377
- <div class="value">${data.summaryStats.total.toLocaleString()}</div>
378
- <div class="label">总记忆数</div>
379
- </div>
380
- <div class="stat-card alert">
381
- <div class="value">${data.summaryStats.deadPct}%</div>
382
- <div class="label">死记忆率</div>
383
- </div>
384
- <div class="stat-card">
385
- <div class="value">${data.summaryStats.categories}</div>
386
- <div class="label">类别数</div>
387
- </div>
388
- <div class="stat-card">
389
- <div class="value">${data.summaryStats.scopes}</div>
390
- <div class="label">Scope 数</div>
391
- </div>
392
- <div class="stat-card">
393
- <div class="value">${data.summaryStats.avgImportance}</div>
394
- <div class="label">平均 Importance</div>
395
- </div>
396
- <div class="stat-card">
397
- <div class="value">${data.summaryStats.oldest}</div>
398
- <div class="label">最早记忆</div>
399
- </div>
400
- </div>
401
-
402
- <div class="grid">
403
-
404
- <!-- 1. Category Distribution -->
405
- <div class="chart-box">
406
- <h3>📊 类别分布</h3>
407
- <div class="chart-container" id="chart-category"></div>
408
- </div>
409
-
410
- <!-- 2. Tier Distribution -->
411
- <div class="chart-box">
412
- <h3>📊 Tier 分布</h3>
413
- <div class="chart-container" id="chart-tier"></div>
414
- </div>
415
-
416
- <!-- 3. Monthly Timeline -->
417
- <div class="chart-box full-width">
418
- <h3>📈 月度记忆增长</h3>
419
- <div class="chart-container" id="chart-timeline"></div>
420
- </div>
421
-
422
- <!-- 4. Daily Timeline (last 90 days) -->
423
- <div class="chart-box full-width">
424
- <h3>📈 近 90 天每日记忆量</h3>
425
- <div class="chart-container" id="chart-daily"></div>
426
- </div>
427
-
428
- <!-- 5. Hourly Pattern -->
429
- <div class="chart-box">
430
- <h3>⏰ 小时分布</h3>
431
- <div class="chart-container" id="chart-hourly"></div>
432
- </div>
433
-
434
- <!-- 6. Day of Week -->
435
- <div class="chart-box">
436
- <h3>📅 星期分布</h3>
437
- <div class="chart-container" id="chart-dow"></div>
438
- </div>
439
-
440
- <!-- 7. Access Count -->
441
- <div class="chart-box">
442
- <h3>🔍 访问次数分布</h3>
443
- <div class="chart-container" id="chart-access"></div>
444
- </div>
445
-
446
- <!-- 8. Importance Distribution -->
447
- <div class="chart-box">
448
- <h3>⚖️ Importance 分布</h3>
449
- <div class="chart-container" id="chart-importance"></div>
450
- </div>
451
-
452
- <!-- 9. Top Scopes -->
453
- <div class="chart-box full-width">
454
- <h3>🏷️ Top 15 Scopes</h3>
455
- <div class="chart-container" id="chart-scopes"></div>
456
- </div>
457
-
458
- <!-- 10. Scope × Category Heatmap -->
459
- <div class="chart-box full-width">
460
- <h3>🗺️ Scope × Category 热力图</h3>
461
- <div class="chart-container" id="chart-heatmap"></div>
462
- </div>
463
-
464
- <!-- 11. Text Length Distribution -->
465
- <div class="chart-box">
466
- <h3>📏 文本长度分布</h3>
467
- <div class="chart-container" id="chart-textlen"></div>
468
- </div>
469
-
470
- <!-- 12. Embedding PCA Scatter -->
471
- <div class="chart-box full-width">
472
- <h3>🧬 Embedding PCA 散点图 (采样 ${data.pcaPoints.length} 条)</h3>
473
- <div class="chart-container tall" id="chart-pca"></div>
474
- </div>
475
-
476
- </div>
477
-
478
- <footer>RecallNest Data Checkup · Module 6 Visualization · ${new Date().toISOString().slice(0, 10)}</footer>
479
-
480
- <script>
481
- const DATA = ${JSON.stringify(data)};
482
- const DARK_LAYOUT = {
483
- paper_bgcolor: 'rgba(0,0,0,0)',
484
- plot_bgcolor: 'rgba(0,0,0,0)',
485
- font: { color: '#e0e0e0', size: 12 },
486
- margin: { t: 30, r: 20, b: 50, l: 60 },
487
- };
488
- const COLORS = ['#00d4ff', '#ff6b6b', '#ffd93d', '#6bcb77', '#9b59b6', '#e67e22', '#1abc9c', '#e74c3c', '#3498db'];
489
-
490
- // 1. Category Donut
491
- Plotly.newPlot('chart-category', [{
492
- type: 'pie',
493
- labels: Object.keys(DATA.categoryData),
494
- values: Object.values(DATA.categoryData),
495
- hole: 0.45,
496
- marker: { colors: COLORS },
497
- textinfo: 'label+percent',
498
- textfont: { size: 11 },
499
- }], { ...DARK_LAYOUT, showlegend: false });
500
-
501
- // 2. Tier Donut
502
- Plotly.newPlot('chart-tier', [{
503
- type: 'pie',
504
- labels: Object.keys(DATA.tierData),
505
- values: Object.values(DATA.tierData),
506
- hole: 0.45,
507
- marker: { colors: ['#ff6b6b', '#ffd93d', '#6bcb77'] },
508
- textinfo: 'label+percent',
509
- textfont: { size: 12 },
510
- }], { ...DARK_LAYOUT, showlegend: false });
511
-
512
- // 3. Monthly Timeline (stacked area)
513
- {
514
- const months = Object.keys(DATA.timelineData);
515
- const allCats = [...new Set(months.flatMap(m => Object.keys(DATA.timelineData[m].cats)))];
516
- const traces = allCats.map((cat, i) => ({
517
- x: months,
518
- y: months.map(m => DATA.timelineData[m].cats[cat] || 0),
519
- name: cat,
520
- type: 'bar',
521
- marker: { color: COLORS[i % COLORS.length] },
522
- }));
523
- Plotly.newPlot('chart-timeline', traces, {
524
- ...DARK_LAYOUT,
525
- barmode: 'stack',
526
- xaxis: { title: '月份' },
527
- yaxis: { title: '记忆数' },
528
- legend: { orientation: 'h', y: -0.2 },
529
- });
530
- }
531
-
532
- // 4. Daily Timeline
533
- {
534
- const days = Object.keys(DATA.dailyData);
535
- Plotly.newPlot('chart-daily', [{
536
- x: days,
537
- y: days.map(d => DATA.dailyData[d]),
538
- type: 'bar',
539
- marker: { color: '#00d4ff', opacity: 0.7 },
540
- }], {
541
- ...DARK_LAYOUT,
542
- xaxis: { title: '日期' },
543
- yaxis: { title: '记忆数' },
544
- });
545
- }
546
-
547
- // 5. Hourly Pattern
548
- Plotly.newPlot('chart-hourly', [{
549
- x: DATA.hourData.map((_, i) => i + ':00'),
550
- y: DATA.hourData,
551
- type: 'bar',
552
- marker: {
553
- color: DATA.hourData.map((v, i) =>
554
- (i >= 22 || i <= 6) ? '#9b59b6' : (i >= 9 && i <= 18) ? '#00d4ff' : '#ffd93d'
555
- ),
556
- },
557
- }], {
558
- ...DARK_LAYOUT,
559
- xaxis: { title: '小时' },
560
- yaxis: { title: '记忆数' },
561
- });
562
-
563
- // 6. Day of Week
564
- {
565
- const dowNames = ['周日', '周一', '周二', '周三', '周四', '周五', '周六'];
566
- Plotly.newPlot('chart-dow', [{
567
- x: dowNames,
568
- y: DATA.dowData,
569
- type: 'bar',
570
- marker: { color: COLORS.slice(0, 7) },
571
- }], {
572
- ...DARK_LAYOUT,
573
- xaxis: { title: '星期' },
574
- yaxis: { title: '记忆数' },
575
- });
576
- }
577
-
578
- // 7. Access Count
579
- Plotly.newPlot('chart-access', [{
580
- x: Object.keys(DATA.accessBuckets),
581
- y: Object.values(DATA.accessBuckets),
582
- type: 'bar',
583
- marker: {
584
- color: Object.keys(DATA.accessBuckets).map((k, i) =>
585
- k.includes('死记忆') ? '#ff6b6b' : COLORS[i]
586
- ),
587
- },
588
- }], {
589
- ...DARK_LAYOUT,
590
- xaxis: { title: '访问次数' },
591
- yaxis: { title: '记忆数', type: 'log' },
592
- });
593
-
594
- // 8. Importance Histogram
595
- Plotly.newPlot('chart-importance', [{
596
- x: DATA.impHist.map((_, i) => (i / DATA.impHist.length).toFixed(2)),
597
- y: DATA.impHist,
598
- type: 'bar',
599
- marker: {
600
- color: DATA.impHist.map((_, i) => {
601
- const v = i / DATA.impHist.length;
602
- return v < 0.3 ? '#ff6b6b' : v < 0.6 ? '#ffd93d' : '#6bcb77';
603
- }),
604
- },
605
- }], {
606
- ...DARK_LAYOUT,
607
- xaxis: { title: 'Importance' },
608
- yaxis: { title: '记忆数' },
609
- });
610
-
611
- // 9. Top Scopes
612
- Plotly.newPlot('chart-scopes', [{
613
- y: Object.keys(DATA.topScopes),
614
- x: Object.values(DATA.topScopes),
615
- type: 'bar',
616
- orientation: 'h',
617
- marker: { color: '#00d4ff' },
618
- }], {
619
- ...DARK_LAYOUT,
620
- margin: { ...DARK_LAYOUT.margin, l: 250 },
621
- xaxis: { title: '记忆数' },
622
- });
623
-
624
- // 10. Heatmap
625
- Plotly.newPlot('chart-heatmap', [{
626
- z: DATA.heatmapScopes.map(s => DATA.allCats.map(c => DATA.scopeCatMatrix[s]?.[c] || 0)),
627
- x: DATA.allCats,
628
- y: DATA.heatmapScopes.map(s => s.length > 35 ? s.slice(0, 32) + '...' : s),
629
- type: 'heatmap',
630
- colorscale: [[0, '#0f0f23'], [0.5, '#00d4ff'], [1, '#ff6b6b']],
631
- showscale: true,
632
- }], {
633
- ...DARK_LAYOUT,
634
- margin: { ...DARK_LAYOUT.margin, l: 250 },
635
- });
636
-
637
- // 11. Text Length
638
- {
639
- const maxLen = 3000;
640
- Plotly.newPlot('chart-textlen', [{
641
- x: DATA.lenHist.map((_, i) => Math.round(i / DATA.lenHist.length * maxLen)),
642
- y: DATA.lenHist,
643
- type: 'bar',
644
- marker: { color: '#1abc9c' },
645
- }], {
646
- ...DARK_LAYOUT,
647
- xaxis: { title: '字符数' },
648
- yaxis: { title: '记忆数' },
649
- });
650
- }
651
-
652
- // 12. PCA Scatter
653
- if (DATA.pcaPoints.length > 0) {
654
- const cats = [...new Set(DATA.pcaPoints.map(p => p.category))];
655
- const traces = cats.map((cat, i) => {
656
- const points = DATA.pcaPoints.filter(p => p.category === cat);
657
- return {
658
- x: points.map(p => p.x),
659
- y: points.map(p => p.y),
660
- text: points.map(p => p.text),
661
- name: cat,
662
- mode: 'markers',
663
- type: 'scattergl',
664
- marker: {
665
- size: 3,
666
- color: COLORS[i % COLORS.length],
667
- opacity: 0.6,
668
- },
669
- hovertemplate: '<b>%{text}</b><br>scope: ' + points.map(p => p.scope).join('<br>') + '<extra>%{fullData.name}</extra>',
670
- };
671
- });
672
- // Fix hover: use customdata
673
- const pcaTraces = cats.map((cat, i) => {
674
- const points = DATA.pcaPoints.filter(p => p.category === cat);
675
- return {
676
- x: points.map(p => p.x),
677
- y: points.map(p => p.y),
678
- customdata: points.map(p => [p.text, p.scope]),
679
- name: cat,
680
- mode: 'markers',
681
- type: 'scattergl',
682
- marker: {
683
- size: 3,
684
- color: COLORS[i % COLORS.length],
685
- opacity: 0.6,
686
- },
687
- hovertemplate: '%{customdata[0]}<br><i>scope: %{customdata[1]}</i><extra>%{fullData.name}</extra>',
688
- };
689
- });
690
- Plotly.newPlot('chart-pca', pcaTraces, {
691
- ...DARK_LAYOUT,
692
- xaxis: { title: 'PC1', zeroline: false },
693
- yaxis: { title: 'PC2', zeroline: false },
694
- legend: { orientation: 'h', y: -0.15 },
695
- margin: { ...DARK_LAYOUT.margin, b: 80 },
696
- });
697
- }
698
- </script>
699
- </body>
700
- </html>`;
701
- }
702
-
703
- main().catch(console.error);