recallnest 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +1 -1
  3. package/README.md +30 -5
  4. package/README_CN.md +30 -5
  5. package/config.json +55 -0
  6. package/connectors/examples/email-adapter.ts +73 -0
  7. package/connectors/examples/logs-adapter.ts +103 -0
  8. package/connectors/examples/rss-adapter.ts +95 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +60 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/ingest-2026-04-13.log +87 -0
  14. package/logs/launchagent-stderr.log +0 -0
  15. package/logs/launchagent-stdout.log +0 -0
  16. package/package.json +1 -1
  17. package/src/cli.ts +30 -0
  18. package/src/connector-types.ts +57 -0
  19. package/src/conversation-importer.ts +25 -1
  20. package/src/data-checkup.ts +31 -1
  21. package/src/doctor.ts +29 -0
  22. package/src/ingest.ts +203 -0
  23. package/src/mcp-server.ts +2 -2
  24. package/src/obsidian-connector.ts +282 -0
  25. package/src/session-schema.ts +2 -2
  26. package/src/source-heartbeat.ts +108 -0
  27. package/.claude-plugin/CLAUDE.md +0 -29
  28. package/.claude-plugin/marketplace.json +0 -18
  29. package/.dockerignore +0 -11
  30. package/.github/workflows/ci.yml +0 -28
  31. package/.mcp.json +0 -9
  32. package/CHANGELOG.md +0 -51
  33. package/DEMO.md +0 -94
  34. package/Dockerfile +0 -20
  35. package/EVAL.md +0 -89
  36. package/FAILURES.md +0 -75
  37. package/OPERATIONS.md +0 -59
  38. package/ROADMAP.md +0 -245
  39. package/SKILL.md +0 -250
  40. package/assets/dashboard.png +0 -0
  41. package/assets/knowledge-graph.png +0 -0
  42. package/assets/recallnest-workbench.png +0 -0
  43. package/assets/screenshots/ui-full.png +0 -0
  44. package/assets/screenshots/ui-overview.png +0 -0
  45. package/assets/screenshots/ui-search.png +0 -0
  46. package/assets/ui/app.js +0 -1031
  47. package/assets/ui/index.html +0 -224
  48. package/assets/ui/styles.css +0 -781
  49. package/assets/wechat-qr.jpg +0 -0
  50. package/bun.lock +0 -337
  51. package/daily-reflections/2026-03-05.md +0 -15
  52. package/daily-reflections/2026-03-06.md +0 -15
  53. package/daily-reflections/2026-03-07.md +0 -7
  54. package/daily-reflections/2026-03-08.md +0 -34
  55. package/daily-reflections/2026-03-09.md +0 -35
  56. package/daily-reflections/2026-03-10.md +0 -36
  57. package/daily-reflections/2026-03-11.md +0 -35
  58. package/daily-reflections/2026-03-12.md +0 -36
  59. package/daily-reflections/2026-03-13.md +0 -35
  60. package/daily-reflections/2026-03-14.md +0 -35
  61. package/daily-reflections/2026-03-15.md +0 -35
  62. package/daily-reflections/2026-03-16.md +0 -41
  63. package/daily-reflections/2026-03-17.md +0 -10
  64. package/daily-reflections/2026-03-18.md +0 -10
  65. package/daily-reflections/2026-03-19.md +0 -10
  66. package/daily-reflections/2026-03-20.md +0 -10
  67. package/daily-reflections/2026-03-21.md +0 -10
  68. package/daily-reflections/2026-03-22.md +0 -10
  69. package/daily-reflections/2026-03-23.md +0 -10
  70. package/daily-reflections/2026-03-24.md +0 -10
  71. package/daily-reflections/2026-03-25.md +0 -32
  72. package/daily-reflections/2026-03-26.md +0 -37
  73. package/daily-reflections/2026-03-27.md +0 -35
  74. package/daily-reflections/2026-03-28.md +0 -35
  75. package/daily-reflections/2026-03-29.md +0 -35
  76. package/daily-reflections/2026-03-30.md +0 -36
  77. package/daily-reflections/2026-03-31.md +0 -35
  78. package/daily-reflections/2026-04-01.md +0 -35
  79. package/daily-reflections/2026-04-02.md +0 -35
  80. package/daily-reflections/2026-04-03.md +0 -37
  81. package/daily-reflections/2026-04-04.md +0 -37
  82. package/daily-reflections/2026-04-05.md +0 -35
  83. package/daily-reflections/2026-04-06.md +0 -36
  84. package/daily-reflections/2026-04-07.md +0 -35
  85. package/daily-reflections/2026-04-08.md +0 -35
  86. package/daily-reflections/2026-04-12.md +0 -35
  87. package/data/alias-map.example.json +0 -22
  88. package/docs/api-reference.md +0 -1291
  89. package/docs/architecture.md +0 -168
  90. package/docs/memory-boundary-contract.md +0 -226
  91. package/docs/memory-categories.md +0 -119
  92. package/docs/pipeline-audit-ultramemory.md +0 -177
  93. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  94. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  95. package/docs/plans/2026-04-01-recall-boost.md +0 -100
  96. package/docs/plans/mcp-tiering-plan.md +0 -126
  97. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  98. package/docs/self-evolution.md +0 -198
  99. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  100. package/docs/superpowers/plans/2026-04-11-constructive-retrieval-emotion.md +0 -1195
  101. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  102. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  103. package/docs/superpowers/specs/2026-04-11-constructive-retrieval-emotion-design.md +0 -398
  104. package/eval/continuity/results-2026-03-16.md +0 -117
  105. package/eval/continuity/results-2026-03-19.md +0 -453
  106. package/eval/continuity/runbook.md +0 -110
  107. package/eval/continuity/seed-cases.md +0 -54
  108. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  109. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  110. package/eval/reports/2026-03-06-baseline.md +0 -82
  111. package/eval/reports/query-expansion.md +0 -138
  112. package/glama.json +0 -6
  113. package/lm +0 -19
  114. package/plan-philosophy-upgrades.md +0 -363
  115. package/plan-product-upgrades.md +0 -155
  116. package/plan-v1.2-distributable.md +0 -105
  117. package/plan.md +0 -140
  118. package/research-memory-evolution.md +0 -440
  119. package/research-rl-agent-memory.md +0 -330
  120. package/scripts/archive-distilled-junk.ts +0 -116
  121. package/scripts/backfill-anchors.ts +0 -130
  122. package/scripts/backfill-l0l1l2.ts +0 -98
  123. package/scripts/blind-scan.ts +0 -588
  124. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  125. package/scripts/cleanup-step2-template-dups.ts +0 -113
  126. package/scripts/cleanup-step3-stale-content.ts +0 -149
  127. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  128. package/scripts/delete-distilled-junk.ts +0 -88
  129. package/scripts/diagnose-vector-search.ts +0 -132
  130. package/scripts/distill-facts.ts +0 -305
  131. package/scripts/health-check.ts +0 -343
  132. package/scripts/incremental-ingest.sh +0 -71
  133. package/scripts/migrate-language.ts +0 -75
  134. package/scripts/mlp-health-check.ts +0 -382
  135. package/scripts/pull-claude-desktop.ts +0 -515
  136. package/scripts/re-ingest-cc.sh +0 -66
  137. package/scripts/rebalance-memory-health.ts +0 -181
  138. package/scripts/run-desktop-ingest.sh +0 -11
  139. package/scripts/start-server.sh +0 -12
  140. package/scripts/verify-emotion-decay.ts +0 -148
  141. package/scripts/viz-dashboard.html +0 -423
  142. package/scripts/viz-export.ts +0 -703
  143. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  144. package/src/__tests__/activity-counter.test.ts +0 -96
  145. package/src/__tests__/admission-control.test.ts +0 -206
  146. package/src/__tests__/anchor-generator.test.ts +0 -67
  147. package/src/__tests__/auto-consolidation.test.ts +0 -168
  148. package/src/__tests__/auto-recall.test.ts +0 -131
  149. package/src/__tests__/babel-integration.test.ts +0 -139
  150. package/src/__tests__/batch-dedup.test.ts +0 -65
  151. package/src/__tests__/capture-engine.test.ts +0 -1144
  152. package/src/__tests__/capture-heuristic.test.ts +0 -171
  153. package/src/__tests__/cascade-forget.test.ts +0 -210
  154. package/src/__tests__/category-min-scores.test.ts +0 -33
  155. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  156. package/src/__tests__/circuit-breaker.test.ts +0 -99
  157. package/src/__tests__/cluster-consolidation.test.ts +0 -712
  158. package/src/__tests__/confidence-metadata.test.ts +0 -125
  159. package/src/__tests__/confidence-tracker.test.ts +0 -249
  160. package/src/__tests__/conflict-advisor.test.ts +0 -206
  161. package/src/__tests__/conflict-escalation.test.ts +0 -149
  162. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  163. package/src/__tests__/conflict-output.test.ts +0 -85
  164. package/src/__tests__/conflict-store.test.ts +0 -374
  165. package/src/__tests__/consolidation-engine.test.ts +0 -181
  166. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  167. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  168. package/src/__tests__/context-composer-pins.test.ts +0 -64
  169. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  170. package/src/__tests__/context-reconstructor.test.ts +0 -65
  171. package/src/__tests__/context-renderer.test.ts +0 -91
  172. package/src/__tests__/continuity-eval.test.ts +0 -275
  173. package/src/__tests__/continuity-schema.test.ts +0 -97
  174. package/src/__tests__/conversation-importer.test.ts +0 -692
  175. package/src/__tests__/core-recall-eval.test.ts +0 -235
  176. package/src/__tests__/core-summary.test.ts +0 -109
  177. package/src/__tests__/data-checkup.test.ts +0 -225
  178. package/src/__tests__/decay-exemption.test.ts +0 -84
  179. package/src/__tests__/dedup-actions.test.ts +0 -58
  180. package/src/__tests__/dedup-guard.test.ts +0 -39
  181. package/src/__tests__/deterministic-id.test.ts +0 -39
  182. package/src/__tests__/distill-lock.test.ts +0 -176
  183. package/src/__tests__/doctor.test.ts +0 -303
  184. package/src/__tests__/dream-pipeline.test.ts +0 -232
  185. package/src/__tests__/drill-down.test.ts +0 -108
  186. package/src/__tests__/embedder.test.ts +0 -122
  187. package/src/__tests__/emotion-decay.test.ts +0 -46
  188. package/src/__tests__/emotion-detector.test.ts +0 -56
  189. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  190. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  191. package/src/__tests__/entity-resolver.test.ts +0 -58
  192. package/src/__tests__/ethics-layer.test.ts +0 -476
  193. package/src/__tests__/event-segmenter.test.ts +0 -153
  194. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  195. package/src/__tests__/frequency-tracker.test.ts +0 -113
  196. package/src/__tests__/graph-export.test.ts +0 -489
  197. package/src/__tests__/hotness-blend.test.ts +0 -76
  198. package/src/__tests__/ingest.test.ts +0 -360
  199. package/src/__tests__/interference-detector.test.ts +0 -203
  200. package/src/__tests__/kg-extractor.test.ts +0 -184
  201. package/src/__tests__/kg-store.test.ts +0 -198
  202. package/src/__tests__/llm-consolidation.test.ts +0 -226
  203. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  204. package/src/__tests__/memory-boundaries.test.ts +0 -237
  205. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -243
  206. package/src/__tests__/memory-evolution.test.ts +0 -271
  207. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  208. package/src/__tests__/memory-lint.test.ts +0 -271
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/narrative-architecture.test.ts +0 -456
  212. package/src/__tests__/pending-review.test.ts +0 -77
  213. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  214. package/src/__tests__/ppr-traversal.test.ts +0 -256
  215. package/src/__tests__/prediction-engine.test.ts +0 -349
  216. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  217. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  218. package/src/__tests__/preference-matcher.test.ts +0 -230
  219. package/src/__tests__/preference-slots.test.ts +0 -205
  220. package/src/__tests__/prospective-memory.test.ts +0 -283
  221. package/src/__tests__/query-entity-detector.test.ts +0 -171
  222. package/src/__tests__/query-expander-alias.test.ts +0 -89
  223. package/src/__tests__/query-expander.test.ts +0 -34
  224. package/src/__tests__/recall-governor.test.ts +0 -227
  225. package/src/__tests__/recall-mode.test.ts +0 -64
  226. package/src/__tests__/result-synthesizer.test.ts +0 -198
  227. package/src/__tests__/resume-context.test.ts +0 -4898
  228. package/src/__tests__/retrieval-trace.test.ts +0 -213
  229. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  230. package/src/__tests__/retroactive-boost.test.ts +0 -223
  231. package/src/__tests__/salience-scoring.test.ts +0 -106
  232. package/src/__tests__/scope-inventory.test.ts +0 -200
  233. package/src/__tests__/scope-policy.test.ts +0 -69
  234. package/src/__tests__/session-checkpoint.test.ts +0 -259
  235. package/src/__tests__/session-distiller.test.ts +0 -611
  236. package/src/__tests__/session-gc.test.ts +0 -215
  237. package/src/__tests__/skill-engine.test.ts +0 -546
  238. package/src/__tests__/skill-promotion.test.ts +0 -349
  239. package/src/__tests__/store-scope-audit.test.ts +0 -81
  240. package/src/__tests__/supersede-chain.test.ts +0 -154
  241. package/src/__tests__/temporal-parser.test.ts +0 -241
  242. package/src/__tests__/temporal-validity.test.ts +0 -160
  243. package/src/__tests__/term-registry.test.ts +0 -121
  244. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  245. package/src/__tests__/topic-tag.test.ts +0 -113
  246. package/src/__tests__/trusted-memory.test.ts +0 -828
  247. package/src/__tests__/version-manager.test.ts +0 -213
  248. package/src/__tests__/workflow-observation.test.ts +0 -400
  249. package/src/__tests__/write-verifier.test.ts +0 -122
@@ -1,117 +0,0 @@
1
- /**
2
- * 清洗步骤 1:删除精确重复记忆
3
- * 每组重复保留 accessCount 最高的那条,其余删除
4
- *
5
- * 用法:
6
- * bun run scripts/cleanup-step1-exact-dups.ts [--execute]
7
- * 不加 --execute 只统计,加了才真删
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- interface MemoryRow {
17
- id: string;
18
- text: string;
19
- category: string;
20
- scope: string;
21
- importance: number;
22
- timestamp: number;
23
- metadata?: string;
24
- }
25
-
26
- async function main() {
27
- console.log(`\n🧹 清洗步骤 1:精确重复删除 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
28
- console.log("=".repeat(60));
29
-
30
- const db = await lancedb.connect(DB_PATH);
31
- const table = await db.openTable(TABLE_NAME);
32
-
33
- console.log("⏳ 读取全量数据...");
34
- const allRows: MemoryRow[] = await table
35
- .query()
36
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
37
- .toArray() as any;
38
-
39
- const total = allRows.length;
40
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
41
-
42
- // Parse metadata
43
- const parsedRows = allRows.map(row => ({
44
- ...row,
45
- meta: row.metadata ? JSON.parse(row.metadata) : {},
46
- }));
47
-
48
- // Group by normalized text
49
- const textGroups = new Map<string, typeof parsedRows>();
50
- for (const row of parsedRows) {
51
- const key = row.text.trim().toLowerCase();
52
- if (!textGroups.has(key)) textGroups.set(key, []);
53
- textGroups.get(key)!.push(row);
54
- }
55
-
56
- // Find duplicates
57
- const dupGroups = [...textGroups.entries()].filter(([, rows]) => rows.length > 1);
58
- const idsToDelete: string[] = [];
59
-
60
- for (const [, rows] of dupGroups) {
61
- // Sort: highest accessCount first, then oldest timestamp as tiebreaker (keep oldest)
62
- rows.sort((a, b) => {
63
- const acA = a.meta.accessCount ?? a.meta.access_count ?? 0;
64
- const acB = b.meta.accessCount ?? b.meta.access_count ?? 0;
65
- if (acB !== acA) return acB - acA; // higher access first
66
- return a.timestamp - b.timestamp; // older first
67
- });
68
- // Keep first, delete rest
69
- for (let i = 1; i < rows.length; i++) {
70
- idsToDelete.push(rows[i].id);
71
- }
72
- }
73
-
74
- console.log(`📊 统计结果:`);
75
- console.log(` 重复组数: ${dupGroups.length.toLocaleString()}`);
76
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
77
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
78
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
79
-
80
- // Show top 10 largest duplicate groups
81
- const topGroups = dupGroups.sort((a, b) => b[1].length - a[1].length).slice(0, 10);
82
- console.log(`\n Top 10 重复组:`);
83
- for (const [text, rows] of topGroups) {
84
- console.log(` [×${rows.length}] ${text.slice(0, 70)}...`);
85
- }
86
-
87
- if (!EXECUTE) {
88
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
89
- return;
90
- }
91
-
92
- // Execute deletion in batches
93
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
94
- const batchSize = 100;
95
- let deleted = 0;
96
-
97
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
98
- const batch = idsToDelete.slice(i, i + batchSize);
99
- // Build WHERE clause: id IN ('a', 'b', 'c')
100
- const escaped = batch.map(id => id.replace(/'/g, "''"));
101
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
102
- await table.delete(whereClause);
103
- deleted += batch.length;
104
- if (deleted % 1000 === 0 || deleted === idsToDelete.length) {
105
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
106
- }
107
- }
108
-
109
- // Verify
110
- const remaining = await table.countRows();
111
- console.log(`\n✅ 删除完成!`);
112
- console.log(` 删除前: ${total.toLocaleString()} 条`);
113
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
114
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
115
- }
116
-
117
- main().catch(console.error);
@@ -1,113 +0,0 @@
1
- /**
2
- * 清洗步骤 2:删除模板化重复内容
3
- * 前 100 字符相同的记忆,每组保留 accessCount 最高的 1 条
4
- * (步骤 1 已清精确重复,这里抓"近似重复"——同一段内容被切成不同长度存了多次)
5
- *
6
- * 用法:
7
- * bun run scripts/cleanup-step2-template-dups.ts [--execute]
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- interface MemoryRow {
17
- id: string;
18
- text: string;
19
- category: string;
20
- scope: string;
21
- importance: number;
22
- timestamp: number;
23
- metadata?: string;
24
- }
25
-
26
- async function main() {
27
- console.log(`\n🧹 清洗步骤 2:模板化重复删除 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
28
- console.log("=".repeat(60));
29
-
30
- const db = await lancedb.connect(DB_PATH);
31
- const table = await db.openTable(TABLE_NAME);
32
-
33
- console.log("⏳ 读取全量数据...");
34
- const allRows: MemoryRow[] = await table
35
- .query()
36
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
37
- .toArray() as any;
38
-
39
- const total = allRows.length;
40
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
41
-
42
- const parsedRows = allRows.map(row => ({
43
- ...row,
44
- meta: row.metadata ? JSON.parse(row.metadata) : {},
45
- }));
46
-
47
- // Group by first 100 chars (normalized)
48
- const fpGroups = new Map<string, typeof parsedRows>();
49
- for (const row of parsedRows) {
50
- const fp = row.text.trim().slice(0, 100).toLowerCase();
51
- if (!fpGroups.has(fp)) fpGroups.set(fp, []);
52
- fpGroups.get(fp)!.push(row);
53
- }
54
-
55
- // Find groups with duplicates (after step 1, exact dups are gone, so these are near-dups)
56
- const dupGroups = [...fpGroups.entries()].filter(([, rows]) => rows.length > 1);
57
- const idsToDelete: string[] = [];
58
-
59
- for (const [, rows] of dupGroups) {
60
- // Keep the one with highest accessCount, then longest text, then oldest
61
- rows.sort((a, b) => {
62
- const acA = a.meta.accessCount ?? a.meta.access_count ?? 0;
63
- const acB = b.meta.accessCount ?? b.meta.access_count ?? 0;
64
- if (acB !== acA) return acB - acA;
65
- if (b.text.length !== a.text.length) return b.text.length - a.text.length; // longer is more complete
66
- return a.timestamp - b.timestamp;
67
- });
68
- for (let i = 1; i < rows.length; i++) {
69
- idsToDelete.push(rows[i].id);
70
- }
71
- }
72
-
73
- console.log(`📊 统计结果:`);
74
- console.log(` 近似重复组数: ${dupGroups.length.toLocaleString()}`);
75
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
76
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
77
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
78
-
79
- // Show top 15 largest groups
80
- const topGroups = dupGroups.sort((a, b) => b[1].length - a[1].length).slice(0, 15);
81
- console.log(`\n Top 15 近似重复组:`);
82
- for (const [fp, rows] of topGroups) {
83
- console.log(` [×${rows.length.toString().padStart(3)}] ${fp.slice(0, 65)}...`);
84
- }
85
-
86
- if (!EXECUTE) {
87
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
88
- return;
89
- }
90
-
91
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
92
- const batchSize = 100;
93
- let deleted = 0;
94
-
95
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
96
- const batch = idsToDelete.slice(i, i + batchSize);
97
- const escaped = batch.map(id => id.replace(/'/g, "''"));
98
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
99
- await table.delete(whereClause);
100
- deleted += batch.length;
101
- if (deleted % 500 === 0 || deleted === idsToDelete.length) {
102
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
103
- }
104
- }
105
-
106
- const remaining = await table.countRows();
107
- console.log(`\n✅ 删除完成!`);
108
- console.log(` 删除前: ${total.toLocaleString()} 条`);
109
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
110
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
111
- }
112
-
113
- main().catch(console.error);
@@ -1,149 +0,0 @@
1
- /**
2
- * 清洗步骤 3:标记/清理过时内容
3
- * 扫描含 temporary/临时/HACK/TODO/FIXME 的记忆
4
- * 只删 accessCount=0 且含这些关键词的(从未被用过的过时内容)
5
- *
6
- * 用法:
7
- * bun run scripts/cleanup-step3-stale-content.ts [--execute]
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- const STALE_KEYWORDS = [
17
- "temporary", "临时", "HACK", "TODO", "FIXME",
18
- "workaround", "deprecated", "废弃", "待修复",
19
- ];
20
-
21
- interface MemoryRow {
22
- id: string;
23
- text: string;
24
- category: string;
25
- scope: string;
26
- importance: number;
27
- timestamp: number;
28
- metadata?: string;
29
- }
30
-
31
- async function main() {
32
- console.log(`\n🧹 清洗步骤 3:过时内容清理 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
33
- console.log("=".repeat(60));
34
-
35
- const db = await lancedb.connect(DB_PATH);
36
- const table = await db.openTable(TABLE_NAME);
37
-
38
- console.log("⏳ 读取全量数据...");
39
- const allRows: MemoryRow[] = await table
40
- .query()
41
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
42
- .toArray() as any;
43
-
44
- const total = allRows.length;
45
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
46
-
47
- const parsedRows = allRows.map(row => ({
48
- ...row,
49
- meta: row.metadata ? JSON.parse(row.metadata) : {},
50
- }));
51
-
52
- // Scan for stale keywords
53
- const keywordHits = new Map<string, typeof parsedRows>();
54
- for (const kw of STALE_KEYWORDS) {
55
- keywordHits.set(kw, []);
56
- }
57
-
58
- for (const row of parsedRows) {
59
- const textLower = row.text.toLowerCase();
60
- for (const kw of STALE_KEYWORDS) {
61
- if (textLower.includes(kw.toLowerCase())) {
62
- keywordHits.get(kw)!.push(row);
63
- }
64
- }
65
- }
66
-
67
- console.log("📊 关键词命中统计:");
68
- let totalHits = 0;
69
- for (const kw of STALE_KEYWORDS) {
70
- const hits = keywordHits.get(kw)!;
71
- if (hits.length > 0) {
72
- const deadHits = hits.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) === 0);
73
- console.log(` "${kw}": ${hits.length} 条 (其中 ${deadHits.length} 条从未访问)`);
74
- }
75
- }
76
-
77
- // Collect: only delete if accessCount=0 (dead memory with stale keyword)
78
- const idsToDeleteSet = new Set<string>();
79
- const deleteCandidates: typeof parsedRows = [];
80
-
81
- for (const row of parsedRows) {
82
- const ac = row.meta.accessCount ?? row.meta.access_count ?? 0;
83
- if (ac > 0) continue; // keep anything that was ever accessed
84
-
85
- const textLower = row.text.toLowerCase();
86
- for (const kw of STALE_KEYWORDS) {
87
- if (textLower.includes(kw.toLowerCase())) {
88
- if (!idsToDeleteSet.has(row.id)) {
89
- idsToDeleteSet.add(row.id);
90
- deleteCandidates.push(row);
91
- }
92
- break;
93
- }
94
- }
95
- }
96
-
97
- const idsToDelete = [...idsToDeleteSet];
98
-
99
- console.log(`\n📊 清理范围(仅 accessCount=0 + 含过时关键词):`);
100
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
101
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
102
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
103
-
104
- // Category breakdown of what we're deleting
105
- const catBreakdown = new Map<string, number>();
106
- for (const row of deleteCandidates) {
107
- catBreakdown.set(row.category, (catBreakdown.get(row.category) || 0) + 1);
108
- }
109
- console.log(`\n 待删除按类别:`);
110
- for (const [cat, count] of [...catBreakdown.entries()].sort((a, b) => b[1] - a[1])) {
111
- console.log(` ${cat.padEnd(14)} ${count}`);
112
- }
113
-
114
- // Show samples
115
- console.log(`\n 待删除示例(随机 10 条):`);
116
- const shuffled = deleteCandidates.sort(() => Math.random() - 0.5).slice(0, 10);
117
- for (const row of shuffled) {
118
- const matched = STALE_KEYWORDS.find(kw => row.text.toLowerCase().includes(kw.toLowerCase()));
119
- console.log(` [${row.category}] [kw="${matched}"] ${row.text.slice(0, 80)}...`);
120
- }
121
-
122
- if (!EXECUTE) {
123
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
124
- return;
125
- }
126
-
127
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
128
- const batchSize = 100;
129
- let deleted = 0;
130
-
131
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
132
- const batch = idsToDelete.slice(i, i + batchSize);
133
- const escaped = batch.map(id => id.replace(/'/g, "''"));
134
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
135
- await table.delete(whereClause);
136
- deleted += batch.length;
137
- if (deleted % 500 === 0 || deleted === idsToDelete.length) {
138
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
139
- }
140
- }
141
-
142
- const remaining = await table.countRows();
143
- console.log(`\n✅ 删除完成!`);
144
- console.log(` 删除前: ${total.toLocaleString()} 条`);
145
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
146
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
147
- }
148
-
149
- main().catch(console.error);
@@ -1,131 +0,0 @@
1
- /**
2
- * 清洗步骤 4:分析孤儿 scope(只有 1 条记忆的 scope)
3
- * 策略:不删记忆,而是把孤儿 scope 的记忆重新归类到更有意义的 scope
4
- * - accessCount=0 的孤儿 → 直接删(死记忆 + 孤儿 scope 双重废)
5
- * - accessCount>0 的孤儿 → 保留,只报告(不乱动有价值的)
6
- *
7
- * 用法:
8
- * bun run scripts/cleanup-step4-orphan-scopes.ts [--execute]
9
- */
10
-
11
- import lancedb from "@lancedb/lancedb";
12
-
13
- const DB_PATH = "./data/lancedb";
14
- const TABLE_NAME = "memories";
15
- const EXECUTE = process.argv.includes("--execute");
16
-
17
- interface MemoryRow {
18
- id: string;
19
- text: string;
20
- category: string;
21
- scope: string;
22
- importance: number;
23
- timestamp: number;
24
- metadata?: string;
25
- }
26
-
27
- async function main() {
28
- console.log(`\n🧹 清洗步骤 4:孤儿 scope 清理 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
29
- console.log("=".repeat(60));
30
-
31
- const db = await lancedb.connect(DB_PATH);
32
- const table = await db.openTable(TABLE_NAME);
33
-
34
- console.log("⏳ 读取全量数据...");
35
- const allRows: MemoryRow[] = await table
36
- .query()
37
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
38
- .toArray() as any;
39
-
40
- const total = allRows.length;
41
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
42
-
43
- const parsedRows = allRows.map(row => ({
44
- ...row,
45
- meta: row.metadata ? JSON.parse(row.metadata) : {},
46
- }));
47
-
48
- // Find orphan scopes (only 1 memory)
49
- const scopeCounts = new Map<string, number>();
50
- for (const row of parsedRows) {
51
- const s = row.scope || "(empty)";
52
- scopeCounts.set(s, (scopeCounts.get(s) || 0) + 1);
53
- }
54
-
55
- const orphanScopes = new Set(
56
- [...scopeCounts.entries()].filter(([, c]) => c === 1).map(([s]) => s)
57
- );
58
-
59
- const orphanRows = parsedRows.filter(r => orphanScopes.has(r.scope || "(empty)"));
60
-
61
- // Split: dead orphans vs alive orphans
62
- const deadOrphans = orphanRows.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) === 0);
63
- const aliveOrphans = orphanRows.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) > 0);
64
-
65
- console.log(`📊 孤儿 scope 统计:`);
66
- console.log(` 孤儿 scope 总数: ${orphanScopes.size}`);
67
- console.log(` 死记忆孤儿: ${deadOrphans.length} 条 → 可安全删除`);
68
- console.log(` 活记忆孤儿: ${aliveOrphans.length} 条 → 保留不动`);
69
-
70
- // Category breakdown
71
- const deadCatMap = new Map<string, number>();
72
- for (const r of deadOrphans) {
73
- deadCatMap.set(r.category, (deadCatMap.get(r.category) || 0) + 1);
74
- }
75
- console.log(`\n 死记忆孤儿按类别:`);
76
- for (const [cat, count] of [...deadCatMap.entries()].sort((a, b) => b[1] - a[1])) {
77
- console.log(` ${cat.padEnd(14)} ${count}`);
78
- }
79
-
80
- // Show alive orphan samples (these we keep)
81
- if (aliveOrphans.length > 0) {
82
- console.log(`\n 活记忆孤儿示例(保留):`);
83
- for (const r of aliveOrphans.slice(0, 5)) {
84
- const ac = r.meta.accessCount ?? r.meta.access_count ?? 0;
85
- console.log(` [${r.category}] [ac=${ac}] [scope=${r.scope.slice(0, 25)}] ${r.text.slice(0, 60)}...`);
86
- }
87
- }
88
-
89
- // Show dead orphan samples
90
- console.log(`\n 死记忆孤儿示例(待删):`);
91
- const samples = deadOrphans.sort(() => Math.random() - 0.5).slice(0, 10);
92
- for (const r of samples) {
93
- console.log(` [${r.category}] [scope=${r.scope.slice(0, 25)}] ${r.text.slice(0, 60)}...`);
94
- }
95
-
96
- const idsToDelete = deadOrphans.map(r => r.id);
97
-
98
- console.log(`\n📊 清理计划:`);
99
- console.log(` 待删除: ${idsToDelete.length} 条(死记忆 + 孤儿 scope)`);
100
- console.log(` 保留: ${(total - idsToDelete.length).toLocaleString()} 条`);
101
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
102
-
103
- if (!EXECUTE) {
104
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
105
- return;
106
- }
107
-
108
- console.log(`\n🔥 开始删除 ${idsToDelete.length} 条...`);
109
- const batchSize = 100;
110
- let deleted = 0;
111
-
112
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
113
- const batch = idsToDelete.slice(i, i + batchSize);
114
- const escaped = batch.map(id => id.replace(/'/g, "''"));
115
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
116
- await table.delete(whereClause);
117
- deleted += batch.length;
118
- if (deleted % 100 === 0 || deleted === idsToDelete.length) {
119
- console.log(` ✅ ${deleted} / ${idsToDelete.length}`);
120
- }
121
- }
122
-
123
- const remaining = await table.countRows();
124
- console.log(`\n✅ 删除完成!`);
125
- console.log(` 删除前: ${total.toLocaleString()} 条`);
126
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
127
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
128
- console.log(` 消灭孤儿 scope: ${deadOrphans.length} 个`);
129
- }
130
-
131
- main().catch(console.error);
@@ -1,88 +0,0 @@
1
- #!/usr/bin/env bun
2
- /**
3
- * Delete raw fact fragments that have already been distilled.
4
- * Targets: entries in distill-completed scopes with importance < 0.4, no L0, category=fact.
5
- * Safe: distilled replacements already exist; transcript originals are in ~/.claude/projects.
6
- *
7
- * Usage:
8
- * bun scripts/delete-distilled-junk.ts --dry-run # preview
9
- * bun scripts/delete-distilled-junk.ts # execute
10
- */
11
-
12
- import fs from "fs";
13
- import path from "path";
14
- import { loadDotEnv, loadConfig, expandHome } from "../src/runtime-config.js";
15
- import { loadLanceDB } from "../src/store.js";
16
-
17
- loadDotEnv();
18
- const config = loadConfig();
19
- const dryRun = process.argv.includes("--dry-run");
20
-
21
- const PROGRESS_FILE = path.join(import.meta.dir, "..", "data", "distill-progress.json");
22
-
23
- async function main() {
24
- const progress = JSON.parse(fs.readFileSync(PROGRESS_FILE, "utf-8"));
25
- const completedScopes = new Set<string>(progress.completedScopes);
26
- console.log(`=== Delete distilled junk${dryRun ? " (dry run)" : ""} ===\n`);
27
- console.log(` Distill-completed scopes: ${completedScopes.size}`);
28
-
29
- const lancedb = await loadLanceDB();
30
- const dbPath = expandHome(config.database?.path || "data/lancedb");
31
- const db = await lancedb.connect(dbPath);
32
- const table = await db.openTable("memories");
33
-
34
- // Read all fact entries (only need id, scope, importance, metadata — skip vector for speed)
35
- const allFacts = await table.query()
36
- .where("category = 'fact'")
37
- .select(["id", "scope", "importance", "metadata"])
38
- .toArray();
39
-
40
- console.log(` Total fact entries: ${allFacts.length}`);
41
-
42
- const toDelete: string[] = [];
43
-
44
- for (const row of allFacts) {
45
- const scope = (row.scope as string) || "";
46
- if (!completedScopes.has(scope)) continue;
47
-
48
- const imp = Number(row.importance);
49
- if (imp >= 0.4) continue;
50
-
51
- const metaStr = (row.metadata as string) || "{}";
52
- let meta: Record<string, unknown>;
53
- try { meta = JSON.parse(metaStr); } catch { continue; }
54
-
55
- if (typeof meta.l0_abstract === "string" && meta.l0_abstract.length > 0) continue;
56
-
57
- toDelete.push(row.id as string);
58
- }
59
-
60
- console.log(` Junk to delete: ${toDelete.length}`);
61
- console.log(` Would keep: ${allFacts.length - toDelete.length} fact entries\n`);
62
-
63
- if (dryRun) {
64
- console.log(` (dry run — no changes made)`);
65
- return;
66
- }
67
-
68
- if (toDelete.length === 0) {
69
- console.log(` Nothing to delete.`);
70
- return;
71
- }
72
-
73
- // Batch delete — no need to re-add
74
- const batchSize = 2000;
75
- for (let i = 0; i < toDelete.length; i += batchSize) {
76
- const batch = toDelete.slice(i, i + batchSize);
77
- const ids = batch.map(id => `'${id.replace(/'/g, "''")}'`).join(",");
78
- await table.delete(`id IN (${ids})`);
79
- console.log(` Batch ${Math.floor(i / batchSize) + 1}: deleted ${batch.length} entries`);
80
- }
81
-
82
- console.log(`\nDone. Deleted ${toDelete.length} junk fact entries. Freed ~${Math.round(toDelete.length * 6 / 1024)} MB (est).`);
83
- }
84
-
85
- main().catch(err => {
86
- console.error("Delete failed:", err);
87
- process.exit(1);
88
- });