recallnest 1.4.1 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +7 -3
  3. package/README.md +202 -130
  4. package/README_CN.md +263 -190
  5. package/bin/recallnest.mjs +62 -0
  6. package/config.json +55 -0
  7. package/daily-recall.sh +23 -0
  8. package/integrations/setup-all.sh +31 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +30 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/launchagent-stderr.log +0 -0
  14. package/logs/launchagent-stdout.log +0 -0
  15. package/package.json +10 -3
  16. package/src/access-tracker.ts +3 -2
  17. package/src/activity-counter.ts +109 -0
  18. package/src/adaptive-retrieval.ts +6 -5
  19. package/src/admission-control.ts +163 -0
  20. package/src/anchor-generator.ts +80 -0
  21. package/src/api-server.ts +10 -0
  22. package/src/audit-log.ts +122 -0
  23. package/src/auto-gc.ts +98 -29
  24. package/src/auto-recall.ts +33 -4
  25. package/src/capture-engine.ts +372 -23
  26. package/src/cascade-forget.ts +3 -2
  27. package/src/cli.ts +107 -5
  28. package/src/compat.ts +22 -0
  29. package/src/confidence-tracker.ts +210 -47
  30. package/src/conflict-store.ts +12 -3
  31. package/src/consolidation-engine.ts +296 -8
  32. package/src/context-collapse-renderer.ts +222 -0
  33. package/src/context-composer-stable-selection.ts +5 -1
  34. package/src/context-composer.ts +239 -4
  35. package/src/context-reconstructor.ts +404 -0
  36. package/src/conversation-importer.ts +441 -0
  37. package/src/data-checkup.ts +232 -0
  38. package/src/decay-engine.ts +84 -0
  39. package/src/distill-lock.ts +122 -0
  40. package/src/doctor.ts +8 -10
  41. package/src/dream-pipeline.ts +227 -0
  42. package/src/emotion-detector.ts +90 -0
  43. package/src/eval.ts +3 -2
  44. package/src/forget-engine.ts +269 -0
  45. package/src/frequency-tracker.ts +175 -0
  46. package/src/graph-export.ts +644 -0
  47. package/src/ingest.ts +36 -6
  48. package/src/interference-detector.ts +187 -0
  49. package/src/kg-extractor.ts +15 -24
  50. package/src/kg-store.ts +2 -1
  51. package/src/language-hook.ts +67 -0
  52. package/src/llm-client.ts +289 -6
  53. package/src/mcp-server.ts +707 -80
  54. package/src/memory-assets.ts +22 -5
  55. package/src/memory-evolution.ts +365 -0
  56. package/src/memory-lint.ts +429 -0
  57. package/src/memory-output.ts +69 -0
  58. package/src/memory-schema.ts +74 -2
  59. package/src/narrative-schema.ts +79 -0
  60. package/src/narrative-tagger.ts +164 -0
  61. package/src/pii-detector.ts +98 -0
  62. package/src/post-retrieval-filter.ts +76 -0
  63. package/src/prediction-engine.ts +253 -0
  64. package/src/preference-slots.ts +65 -2
  65. package/src/prospective-memory.ts +215 -0
  66. package/src/query-expander.ts +67 -0
  67. package/src/recall-governor.ts +183 -0
  68. package/src/retention-policy.ts +90 -0
  69. package/src/retriever.ts +828 -55
  70. package/src/rif.ts +34 -8
  71. package/src/runtime-config.ts +23 -8
  72. package/src/scope-inventory.ts +4 -3
  73. package/src/scope-policy.ts +4 -0
  74. package/src/session-distiller.ts +307 -0
  75. package/src/session-output.ts +40 -0
  76. package/src/session-schema.ts +37 -0
  77. package/src/session-store.ts +7 -2
  78. package/src/skill-engine.ts +208 -0
  79. package/src/skill-promotion.ts +293 -0
  80. package/src/skill-schema.ts +39 -0
  81. package/src/store.ts +96 -10
  82. package/src/topic-tag.ts +66 -0
  83. package/src/tracker.ts +3 -1
  84. package/src/ui-server.ts +80 -8
  85. package/src/workflow-observation-scope-review.ts +2 -1
  86. package/src/workflow-observation-store.ts +2 -1
  87. package/src/write-verifier.ts +126 -0
  88. package/.claude-plugin/CLAUDE.md +0 -29
  89. package/.claude-plugin/marketplace.json +0 -18
  90. package/.github/workflows/ci.yml +0 -28
  91. package/.mcp.json +0 -9
  92. package/CHANGELOG.md +0 -51
  93. package/DEMO.md +0 -94
  94. package/EVAL.md +0 -89
  95. package/FAILURES.md +0 -75
  96. package/OPERATIONS.md +0 -59
  97. package/ROADMAP.md +0 -198
  98. package/assets/recallnest-workbench.png +0 -0
  99. package/assets/ui/app.js +0 -807
  100. package/assets/ui/index.html +0 -154
  101. package/assets/ui/styles.css +0 -605
  102. package/assets/wechat-qr.jpg +0 -0
  103. package/daily-reflections/2026-03-05.md +0 -15
  104. package/daily-reflections/2026-03-06.md +0 -15
  105. package/daily-reflections/2026-03-07.md +0 -7
  106. package/daily-reflections/2026-03-08.md +0 -34
  107. package/daily-reflections/2026-03-09.md +0 -35
  108. package/daily-reflections/2026-03-10.md +0 -36
  109. package/daily-reflections/2026-03-11.md +0 -35
  110. package/daily-reflections/2026-03-12.md +0 -36
  111. package/daily-reflections/2026-03-13.md +0 -35
  112. package/daily-reflections/2026-03-14.md +0 -35
  113. package/daily-reflections/2026-03-15.md +0 -35
  114. package/daily-reflections/2026-03-16.md +0 -41
  115. package/daily-reflections/2026-03-17.md +0 -10
  116. package/daily-reflections/2026-03-18.md +0 -10
  117. package/daily-reflections/2026-03-19.md +0 -10
  118. package/daily-reflections/2026-03-20.md +0 -10
  119. package/daily-reflections/2026-03-21.md +0 -10
  120. package/daily-reflections/2026-03-22.md +0 -10
  121. package/daily-reflections/2026-03-23.md +0 -10
  122. package/daily-reflections/2026-03-24.md +0 -10
  123. package/daily-reflections/2026-03-25.md +0 -32
  124. package/daily-reflections/2026-03-26.md +0 -37
  125. package/daily-reflections/2026-03-27.md +0 -35
  126. package/daily-reflections/2026-03-28.md +0 -35
  127. package/daily-reflections/2026-03-29.md +0 -35
  128. package/daily-reflections/2026-03-30.md +0 -36
  129. package/daily-reflections/2026-03-31.md +0 -35
  130. package/daily-reflections/2026-04-01.md +0 -35
  131. package/docs/api-reference.md +0 -1291
  132. package/docs/architecture.md +0 -168
  133. package/docs/memory-boundary-contract.md +0 -226
  134. package/docs/memory-categories.md +0 -119
  135. package/docs/pipeline-audit-ultramemory.md +0 -177
  136. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  137. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  138. package/docs/plans/mcp-tiering-plan.md +0 -126
  139. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  140. package/docs/self-evolution.md +0 -198
  141. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  142. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  143. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  144. package/eval/continuity/results-2026-03-16.md +0 -117
  145. package/eval/continuity/results-2026-03-19.md +0 -453
  146. package/eval/continuity/runbook.md +0 -110
  147. package/eval/continuity/seed-cases.md +0 -54
  148. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  149. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  150. package/eval/reports/2026-03-06-baseline.md +0 -82
  151. package/eval/reports/query-expansion.md +0 -138
  152. package/lm +0 -19
  153. package/plan-v1.2-distributable.md +0 -105
  154. package/scripts/archive-distilled-junk.ts +0 -116
  155. package/scripts/backfill-l0l1l2.ts +0 -98
  156. package/scripts/blind-scan.ts +0 -588
  157. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  158. package/scripts/cleanup-step2-template-dups.ts +0 -113
  159. package/scripts/cleanup-step3-stale-content.ts +0 -149
  160. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  161. package/scripts/delete-distilled-junk.ts +0 -88
  162. package/scripts/distill-facts.ts +0 -305
  163. package/scripts/health-check.ts +0 -343
  164. package/scripts/incremental-ingest.sh +0 -71
  165. package/scripts/mlp-health-check.ts +0 -382
  166. package/scripts/pull-claude-desktop.ts +0 -515
  167. package/scripts/re-ingest-cc.sh +0 -66
  168. package/scripts/rebalance-memory-health.ts +0 -181
  169. package/scripts/run-desktop-ingest.sh +0 -11
  170. package/scripts/start-server.sh +0 -12
  171. package/scripts/viz-dashboard.html +0 -423
  172. package/scripts/viz-export.ts +0 -703
  173. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  174. package/src/__tests__/auto-consolidation.test.ts +0 -168
  175. package/src/__tests__/auto-recall.test.ts +0 -131
  176. package/src/__tests__/batch-dedup.test.ts +0 -65
  177. package/src/__tests__/capture-engine.test.ts +0 -1142
  178. package/src/__tests__/capture-heuristic.test.ts +0 -171
  179. package/src/__tests__/cascade-forget.test.ts +0 -210
  180. package/src/__tests__/category-min-scores.test.ts +0 -33
  181. package/src/__tests__/confidence-tracker.test.ts +0 -196
  182. package/src/__tests__/conflict-advisor.test.ts +0 -206
  183. package/src/__tests__/conflict-escalation.test.ts +0 -149
  184. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  185. package/src/__tests__/conflict-output.test.ts +0 -85
  186. package/src/__tests__/conflict-store.test.ts +0 -374
  187. package/src/__tests__/consolidation-engine.test.ts +0 -181
  188. package/src/__tests__/context-composer-pins.test.ts +0 -64
  189. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  190. package/src/__tests__/context-renderer.test.ts +0 -91
  191. package/src/__tests__/continuity-eval.test.ts +0 -275
  192. package/src/__tests__/continuity-schema.test.ts +0 -97
  193. package/src/__tests__/core-recall-eval.test.ts +0 -235
  194. package/src/__tests__/core-summary.test.ts +0 -109
  195. package/src/__tests__/dedup-actions.test.ts +0 -58
  196. package/src/__tests__/deterministic-id.test.ts +0 -39
  197. package/src/__tests__/doctor.test.ts +0 -303
  198. package/src/__tests__/drill-down.test.ts +0 -108
  199. package/src/__tests__/embedder.test.ts +0 -122
  200. package/src/__tests__/entity-resolver.test.ts +0 -58
  201. package/src/__tests__/event-segmenter.test.ts +0 -153
  202. package/src/__tests__/hotness-blend.test.ts +0 -76
  203. package/src/__tests__/ingest.test.ts +0 -360
  204. package/src/__tests__/kg-extractor.test.ts +0 -184
  205. package/src/__tests__/kg-store.test.ts +0 -198
  206. package/src/__tests__/llm-consolidation.test.ts +0 -226
  207. package/src/__tests__/memory-boundaries.test.ts +0 -237
  208. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/ppr-traversal.test.ts +0 -256
  212. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  213. package/src/__tests__/preference-matcher.test.ts +0 -230
  214. package/src/__tests__/preference-slots.test.ts +0 -106
  215. package/src/__tests__/prospective-memory.test.ts +0 -283
  216. package/src/__tests__/query-entity-detector.test.ts +0 -171
  217. package/src/__tests__/query-expander.test.ts +0 -34
  218. package/src/__tests__/recall-mode.test.ts +0 -64
  219. package/src/__tests__/result-synthesizer.test.ts +0 -198
  220. package/src/__tests__/resume-context.test.ts +0 -4743
  221. package/src/__tests__/retrieval-trace.test.ts +0 -213
  222. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  223. package/src/__tests__/retroactive-boost.test.ts +0 -223
  224. package/src/__tests__/scope-inventory.test.ts +0 -200
  225. package/src/__tests__/scope-policy.test.ts +0 -69
  226. package/src/__tests__/session-checkpoint.test.ts +0 -259
  227. package/src/__tests__/session-gc.test.ts +0 -215
  228. package/src/__tests__/store-scope-audit.test.ts +0 -81
  229. package/src/__tests__/temporal-parser.test.ts +0 -241
  230. package/src/__tests__/term-registry.test.ts +0 -121
  231. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  232. package/src/__tests__/version-manager.test.ts +0 -213
  233. package/src/__tests__/workflow-observation.test.ts +0 -400
@@ -1,117 +0,0 @@
1
- /**
2
- * 清洗步骤 1:删除精确重复记忆
3
- * 每组重复保留 accessCount 最高的那条,其余删除
4
- *
5
- * 用法:
6
- * bun run scripts/cleanup-step1-exact-dups.ts [--execute]
7
- * 不加 --execute 只统计,加了才真删
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- interface MemoryRow {
17
- id: string;
18
- text: string;
19
- category: string;
20
- scope: string;
21
- importance: number;
22
- timestamp: number;
23
- metadata?: string;
24
- }
25
-
26
- async function main() {
27
- console.log(`\n🧹 清洗步骤 1:精确重复删除 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
28
- console.log("=".repeat(60));
29
-
30
- const db = await lancedb.connect(DB_PATH);
31
- const table = await db.openTable(TABLE_NAME);
32
-
33
- console.log("⏳ 读取全量数据...");
34
- const allRows: MemoryRow[] = await table
35
- .query()
36
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
37
- .toArray() as any;
38
-
39
- const total = allRows.length;
40
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
41
-
42
- // Parse metadata
43
- const parsedRows = allRows.map(row => ({
44
- ...row,
45
- meta: row.metadata ? JSON.parse(row.metadata) : {},
46
- }));
47
-
48
- // Group by normalized text
49
- const textGroups = new Map<string, typeof parsedRows>();
50
- for (const row of parsedRows) {
51
- const key = row.text.trim().toLowerCase();
52
- if (!textGroups.has(key)) textGroups.set(key, []);
53
- textGroups.get(key)!.push(row);
54
- }
55
-
56
- // Find duplicates
57
- const dupGroups = [...textGroups.entries()].filter(([, rows]) => rows.length > 1);
58
- const idsToDelete: string[] = [];
59
-
60
- for (const [, rows] of dupGroups) {
61
- // Sort: highest accessCount first, then oldest timestamp as tiebreaker (keep oldest)
62
- rows.sort((a, b) => {
63
- const acA = a.meta.accessCount ?? a.meta.access_count ?? 0;
64
- const acB = b.meta.accessCount ?? b.meta.access_count ?? 0;
65
- if (acB !== acA) return acB - acA; // higher access first
66
- return a.timestamp - b.timestamp; // older first
67
- });
68
- // Keep first, delete rest
69
- for (let i = 1; i < rows.length; i++) {
70
- idsToDelete.push(rows[i].id);
71
- }
72
- }
73
-
74
- console.log(`📊 统计结果:`);
75
- console.log(` 重复组数: ${dupGroups.length.toLocaleString()}`);
76
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
77
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
78
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
79
-
80
- // Show top 10 largest duplicate groups
81
- const topGroups = dupGroups.sort((a, b) => b[1].length - a[1].length).slice(0, 10);
82
- console.log(`\n Top 10 重复组:`);
83
- for (const [text, rows] of topGroups) {
84
- console.log(` [×${rows.length}] ${text.slice(0, 70)}...`);
85
- }
86
-
87
- if (!EXECUTE) {
88
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
89
- return;
90
- }
91
-
92
- // Execute deletion in batches
93
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
94
- const batchSize = 100;
95
- let deleted = 0;
96
-
97
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
98
- const batch = idsToDelete.slice(i, i + batchSize);
99
- // Build WHERE clause: id IN ('a', 'b', 'c')
100
- const escaped = batch.map(id => id.replace(/'/g, "''"));
101
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
102
- await table.delete(whereClause);
103
- deleted += batch.length;
104
- if (deleted % 1000 === 0 || deleted === idsToDelete.length) {
105
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
106
- }
107
- }
108
-
109
- // Verify
110
- const remaining = await table.countRows();
111
- console.log(`\n✅ 删除完成!`);
112
- console.log(` 删除前: ${total.toLocaleString()} 条`);
113
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
114
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
115
- }
116
-
117
- main().catch(console.error);
@@ -1,113 +0,0 @@
1
- /**
2
- * 清洗步骤 2:删除模板化重复内容
3
- * 前 100 字符相同的记忆,每组保留 accessCount 最高的 1 条
4
- * (步骤 1 已清精确重复,这里抓"近似重复"——同一段内容被切成不同长度存了多次)
5
- *
6
- * 用法:
7
- * bun run scripts/cleanup-step2-template-dups.ts [--execute]
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- interface MemoryRow {
17
- id: string;
18
- text: string;
19
- category: string;
20
- scope: string;
21
- importance: number;
22
- timestamp: number;
23
- metadata?: string;
24
- }
25
-
26
- async function main() {
27
- console.log(`\n🧹 清洗步骤 2:模板化重复删除 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
28
- console.log("=".repeat(60));
29
-
30
- const db = await lancedb.connect(DB_PATH);
31
- const table = await db.openTable(TABLE_NAME);
32
-
33
- console.log("⏳ 读取全量数据...");
34
- const allRows: MemoryRow[] = await table
35
- .query()
36
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
37
- .toArray() as any;
38
-
39
- const total = allRows.length;
40
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
41
-
42
- const parsedRows = allRows.map(row => ({
43
- ...row,
44
- meta: row.metadata ? JSON.parse(row.metadata) : {},
45
- }));
46
-
47
- // Group by first 100 chars (normalized)
48
- const fpGroups = new Map<string, typeof parsedRows>();
49
- for (const row of parsedRows) {
50
- const fp = row.text.trim().slice(0, 100).toLowerCase();
51
- if (!fpGroups.has(fp)) fpGroups.set(fp, []);
52
- fpGroups.get(fp)!.push(row);
53
- }
54
-
55
- // Find groups with duplicates (after step 1, exact dups are gone, so these are near-dups)
56
- const dupGroups = [...fpGroups.entries()].filter(([, rows]) => rows.length > 1);
57
- const idsToDelete: string[] = [];
58
-
59
- for (const [, rows] of dupGroups) {
60
- // Keep the one with highest accessCount, then longest text, then oldest
61
- rows.sort((a, b) => {
62
- const acA = a.meta.accessCount ?? a.meta.access_count ?? 0;
63
- const acB = b.meta.accessCount ?? b.meta.access_count ?? 0;
64
- if (acB !== acA) return acB - acA;
65
- if (b.text.length !== a.text.length) return b.text.length - a.text.length; // longer is more complete
66
- return a.timestamp - b.timestamp;
67
- });
68
- for (let i = 1; i < rows.length; i++) {
69
- idsToDelete.push(rows[i].id);
70
- }
71
- }
72
-
73
- console.log(`📊 统计结果:`);
74
- console.log(` 近似重复组数: ${dupGroups.length.toLocaleString()}`);
75
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
76
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
77
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
78
-
79
- // Show top 15 largest groups
80
- const topGroups = dupGroups.sort((a, b) => b[1].length - a[1].length).slice(0, 15);
81
- console.log(`\n Top 15 近似重复组:`);
82
- for (const [fp, rows] of topGroups) {
83
- console.log(` [×${rows.length.toString().padStart(3)}] ${fp.slice(0, 65)}...`);
84
- }
85
-
86
- if (!EXECUTE) {
87
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
88
- return;
89
- }
90
-
91
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
92
- const batchSize = 100;
93
- let deleted = 0;
94
-
95
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
96
- const batch = idsToDelete.slice(i, i + batchSize);
97
- const escaped = batch.map(id => id.replace(/'/g, "''"));
98
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
99
- await table.delete(whereClause);
100
- deleted += batch.length;
101
- if (deleted % 500 === 0 || deleted === idsToDelete.length) {
102
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
103
- }
104
- }
105
-
106
- const remaining = await table.countRows();
107
- console.log(`\n✅ 删除完成!`);
108
- console.log(` 删除前: ${total.toLocaleString()} 条`);
109
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
110
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
111
- }
112
-
113
- main().catch(console.error);
@@ -1,149 +0,0 @@
1
- /**
2
- * 清洗步骤 3:标记/清理过时内容
3
- * 扫描含 temporary/临时/HACK/TODO/FIXME 的记忆
4
- * 只删 accessCount=0 且含这些关键词的(从未被用过的过时内容)
5
- *
6
- * 用法:
7
- * bun run scripts/cleanup-step3-stale-content.ts [--execute]
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- const STALE_KEYWORDS = [
17
- "temporary", "临时", "HACK", "TODO", "FIXME",
18
- "workaround", "deprecated", "废弃", "待修复",
19
- ];
20
-
21
- interface MemoryRow {
22
- id: string;
23
- text: string;
24
- category: string;
25
- scope: string;
26
- importance: number;
27
- timestamp: number;
28
- metadata?: string;
29
- }
30
-
31
- async function main() {
32
- console.log(`\n🧹 清洗步骤 3:过时内容清理 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
33
- console.log("=".repeat(60));
34
-
35
- const db = await lancedb.connect(DB_PATH);
36
- const table = await db.openTable(TABLE_NAME);
37
-
38
- console.log("⏳ 读取全量数据...");
39
- const allRows: MemoryRow[] = await table
40
- .query()
41
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
42
- .toArray() as any;
43
-
44
- const total = allRows.length;
45
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
46
-
47
- const parsedRows = allRows.map(row => ({
48
- ...row,
49
- meta: row.metadata ? JSON.parse(row.metadata) : {},
50
- }));
51
-
52
- // Scan for stale keywords
53
- const keywordHits = new Map<string, typeof parsedRows>();
54
- for (const kw of STALE_KEYWORDS) {
55
- keywordHits.set(kw, []);
56
- }
57
-
58
- for (const row of parsedRows) {
59
- const textLower = row.text.toLowerCase();
60
- for (const kw of STALE_KEYWORDS) {
61
- if (textLower.includes(kw.toLowerCase())) {
62
- keywordHits.get(kw)!.push(row);
63
- }
64
- }
65
- }
66
-
67
- console.log("📊 关键词命中统计:");
68
- let totalHits = 0;
69
- for (const kw of STALE_KEYWORDS) {
70
- const hits = keywordHits.get(kw)!;
71
- if (hits.length > 0) {
72
- const deadHits = hits.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) === 0);
73
- console.log(` "${kw}": ${hits.length} 条 (其中 ${deadHits.length} 条从未访问)`);
74
- }
75
- }
76
-
77
- // Collect: only delete if accessCount=0 (dead memory with stale keyword)
78
- const idsToDeleteSet = new Set<string>();
79
- const deleteCandidates: typeof parsedRows = [];
80
-
81
- for (const row of parsedRows) {
82
- const ac = row.meta.accessCount ?? row.meta.access_count ?? 0;
83
- if (ac > 0) continue; // keep anything that was ever accessed
84
-
85
- const textLower = row.text.toLowerCase();
86
- for (const kw of STALE_KEYWORDS) {
87
- if (textLower.includes(kw.toLowerCase())) {
88
- if (!idsToDeleteSet.has(row.id)) {
89
- idsToDeleteSet.add(row.id);
90
- deleteCandidates.push(row);
91
- }
92
- break;
93
- }
94
- }
95
- }
96
-
97
- const idsToDelete = [...idsToDeleteSet];
98
-
99
- console.log(`\n📊 清理范围(仅 accessCount=0 + 含过时关键词):`);
100
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
101
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
102
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
103
-
104
- // Category breakdown of what we're deleting
105
- const catBreakdown = new Map<string, number>();
106
- for (const row of deleteCandidates) {
107
- catBreakdown.set(row.category, (catBreakdown.get(row.category) || 0) + 1);
108
- }
109
- console.log(`\n 待删除按类别:`);
110
- for (const [cat, count] of [...catBreakdown.entries()].sort((a, b) => b[1] - a[1])) {
111
- console.log(` ${cat.padEnd(14)} ${count}`);
112
- }
113
-
114
- // Show samples
115
- console.log(`\n 待删除示例(随机 10 条):`);
116
- const shuffled = deleteCandidates.sort(() => Math.random() - 0.5).slice(0, 10);
117
- for (const row of shuffled) {
118
- const matched = STALE_KEYWORDS.find(kw => row.text.toLowerCase().includes(kw.toLowerCase()));
119
- console.log(` [${row.category}] [kw="${matched}"] ${row.text.slice(0, 80)}...`);
120
- }
121
-
122
- if (!EXECUTE) {
123
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
124
- return;
125
- }
126
-
127
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
128
- const batchSize = 100;
129
- let deleted = 0;
130
-
131
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
132
- const batch = idsToDelete.slice(i, i + batchSize);
133
- const escaped = batch.map(id => id.replace(/'/g, "''"));
134
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
135
- await table.delete(whereClause);
136
- deleted += batch.length;
137
- if (deleted % 500 === 0 || deleted === idsToDelete.length) {
138
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
139
- }
140
- }
141
-
142
- const remaining = await table.countRows();
143
- console.log(`\n✅ 删除完成!`);
144
- console.log(` 删除前: ${total.toLocaleString()} 条`);
145
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
146
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
147
- }
148
-
149
- main().catch(console.error);
@@ -1,131 +0,0 @@
1
- /**
2
- * 清洗步骤 4:分析孤儿 scope(只有 1 条记忆的 scope)
3
- * 策略:不删记忆,而是把孤儿 scope 的记忆重新归类到更有意义的 scope
4
- * - accessCount=0 的孤儿 → 直接删(死记忆 + 孤儿 scope 双重废)
5
- * - accessCount>0 的孤儿 → 保留,只报告(不乱动有价值的)
6
- *
7
- * 用法:
8
- * bun run scripts/cleanup-step4-orphan-scopes.ts [--execute]
9
- */
10
-
11
- import lancedb from "@lancedb/lancedb";
12
-
13
- const DB_PATH = "./data/lancedb";
14
- const TABLE_NAME = "memories";
15
- const EXECUTE = process.argv.includes("--execute");
16
-
17
- interface MemoryRow {
18
- id: string;
19
- text: string;
20
- category: string;
21
- scope: string;
22
- importance: number;
23
- timestamp: number;
24
- metadata?: string;
25
- }
26
-
27
- async function main() {
28
- console.log(`\n🧹 清洗步骤 4:孤儿 scope 清理 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
29
- console.log("=".repeat(60));
30
-
31
- const db = await lancedb.connect(DB_PATH);
32
- const table = await db.openTable(TABLE_NAME);
33
-
34
- console.log("⏳ 读取全量数据...");
35
- const allRows: MemoryRow[] = await table
36
- .query()
37
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
38
- .toArray() as any;
39
-
40
- const total = allRows.length;
41
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
42
-
43
- const parsedRows = allRows.map(row => ({
44
- ...row,
45
- meta: row.metadata ? JSON.parse(row.metadata) : {},
46
- }));
47
-
48
- // Find orphan scopes (only 1 memory)
49
- const scopeCounts = new Map<string, number>();
50
- for (const row of parsedRows) {
51
- const s = row.scope || "(empty)";
52
- scopeCounts.set(s, (scopeCounts.get(s) || 0) + 1);
53
- }
54
-
55
- const orphanScopes = new Set(
56
- [...scopeCounts.entries()].filter(([, c]) => c === 1).map(([s]) => s)
57
- );
58
-
59
- const orphanRows = parsedRows.filter(r => orphanScopes.has(r.scope || "(empty)"));
60
-
61
- // Split: dead orphans vs alive orphans
62
- const deadOrphans = orphanRows.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) === 0);
63
- const aliveOrphans = orphanRows.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) > 0);
64
-
65
- console.log(`📊 孤儿 scope 统计:`);
66
- console.log(` 孤儿 scope 总数: ${orphanScopes.size}`);
67
- console.log(` 死记忆孤儿: ${deadOrphans.length} 条 → 可安全删除`);
68
- console.log(` 活记忆孤儿: ${aliveOrphans.length} 条 → 保留不动`);
69
-
70
- // Category breakdown
71
- const deadCatMap = new Map<string, number>();
72
- for (const r of deadOrphans) {
73
- deadCatMap.set(r.category, (deadCatMap.get(r.category) || 0) + 1);
74
- }
75
- console.log(`\n 死记忆孤儿按类别:`);
76
- for (const [cat, count] of [...deadCatMap.entries()].sort((a, b) => b[1] - a[1])) {
77
- console.log(` ${cat.padEnd(14)} ${count}`);
78
- }
79
-
80
- // Show alive orphan samples (these we keep)
81
- if (aliveOrphans.length > 0) {
82
- console.log(`\n 活记忆孤儿示例(保留):`);
83
- for (const r of aliveOrphans.slice(0, 5)) {
84
- const ac = r.meta.accessCount ?? r.meta.access_count ?? 0;
85
- console.log(` [${r.category}] [ac=${ac}] [scope=${r.scope.slice(0, 25)}] ${r.text.slice(0, 60)}...`);
86
- }
87
- }
88
-
89
- // Show dead orphan samples
90
- console.log(`\n 死记忆孤儿示例(待删):`);
91
- const samples = deadOrphans.sort(() => Math.random() - 0.5).slice(0, 10);
92
- for (const r of samples) {
93
- console.log(` [${r.category}] [scope=${r.scope.slice(0, 25)}] ${r.text.slice(0, 60)}...`);
94
- }
95
-
96
- const idsToDelete = deadOrphans.map(r => r.id);
97
-
98
- console.log(`\n📊 清理计划:`);
99
- console.log(` 待删除: ${idsToDelete.length} 条(死记忆 + 孤儿 scope)`);
100
- console.log(` 保留: ${(total - idsToDelete.length).toLocaleString()} 条`);
101
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
102
-
103
- if (!EXECUTE) {
104
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
105
- return;
106
- }
107
-
108
- console.log(`\n🔥 开始删除 ${idsToDelete.length} 条...`);
109
- const batchSize = 100;
110
- let deleted = 0;
111
-
112
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
113
- const batch = idsToDelete.slice(i, i + batchSize);
114
- const escaped = batch.map(id => id.replace(/'/g, "''"));
115
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
116
- await table.delete(whereClause);
117
- deleted += batch.length;
118
- if (deleted % 100 === 0 || deleted === idsToDelete.length) {
119
- console.log(` ✅ ${deleted} / ${idsToDelete.length}`);
120
- }
121
- }
122
-
123
- const remaining = await table.countRows();
124
- console.log(`\n✅ 删除完成!`);
125
- console.log(` 删除前: ${total.toLocaleString()} 条`);
126
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
127
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
128
- console.log(` 消灭孤儿 scope: ${deadOrphans.length} 个`);
129
- }
130
-
131
- main().catch(console.error);
@@ -1,88 +0,0 @@
1
- #!/usr/bin/env bun
2
- /**
3
- * Delete raw fact fragments that have already been distilled.
4
- * Targets: entries in distill-completed scopes with importance < 0.4, no L0, category=fact.
5
- * Safe: distilled replacements already exist; transcript originals are in ~/.claude/projects.
6
- *
7
- * Usage:
8
- * bun scripts/delete-distilled-junk.ts --dry-run # preview
9
- * bun scripts/delete-distilled-junk.ts # execute
10
- */
11
-
12
- import fs from "fs";
13
- import path from "path";
14
- import { loadDotEnv, loadConfig, expandHome } from "../src/runtime-config.js";
15
- import { loadLanceDB } from "../src/store.js";
16
-
17
- loadDotEnv();
18
- const config = loadConfig();
19
- const dryRun = process.argv.includes("--dry-run");
20
-
21
- const PROGRESS_FILE = path.join(import.meta.dir, "..", "data", "distill-progress.json");
22
-
23
- async function main() {
24
- const progress = JSON.parse(fs.readFileSync(PROGRESS_FILE, "utf-8"));
25
- const completedScopes = new Set<string>(progress.completedScopes);
26
- console.log(`=== Delete distilled junk${dryRun ? " (dry run)" : ""} ===\n`);
27
- console.log(` Distill-completed scopes: ${completedScopes.size}`);
28
-
29
- const lancedb = await loadLanceDB();
30
- const dbPath = expandHome(config.database?.path || "data/lancedb");
31
- const db = await lancedb.connect(dbPath);
32
- const table = await db.openTable("memories");
33
-
34
- // Read all fact entries (only need id, scope, importance, metadata — skip vector for speed)
35
- const allFacts = await table.query()
36
- .where("category = 'fact'")
37
- .select(["id", "scope", "importance", "metadata"])
38
- .toArray();
39
-
40
- console.log(` Total fact entries: ${allFacts.length}`);
41
-
42
- const toDelete: string[] = [];
43
-
44
- for (const row of allFacts) {
45
- const scope = (row.scope as string) || "";
46
- if (!completedScopes.has(scope)) continue;
47
-
48
- const imp = Number(row.importance);
49
- if (imp >= 0.4) continue;
50
-
51
- const metaStr = (row.metadata as string) || "{}";
52
- let meta: Record<string, unknown>;
53
- try { meta = JSON.parse(metaStr); } catch { continue; }
54
-
55
- if (typeof meta.l0_abstract === "string" && meta.l0_abstract.length > 0) continue;
56
-
57
- toDelete.push(row.id as string);
58
- }
59
-
60
- console.log(` Junk to delete: ${toDelete.length}`);
61
- console.log(` Would keep: ${allFacts.length - toDelete.length} fact entries\n`);
62
-
63
- if (dryRun) {
64
- console.log(` (dry run — no changes made)`);
65
- return;
66
- }
67
-
68
- if (toDelete.length === 0) {
69
- console.log(` Nothing to delete.`);
70
- return;
71
- }
72
-
73
- // Batch delete — no need to re-add
74
- const batchSize = 2000;
75
- for (let i = 0; i < toDelete.length; i += batchSize) {
76
- const batch = toDelete.slice(i, i + batchSize);
77
- const ids = batch.map(id => `'${id.replace(/'/g, "''")}'`).join(",");
78
- await table.delete(`id IN (${ids})`);
79
- console.log(` Batch ${Math.floor(i / batchSize) + 1}: deleted ${batch.length} entries`);
80
- }
81
-
82
- console.log(`\nDone. Deleted ${toDelete.length} junk fact entries. Freed ~${Math.round(toDelete.length * 6 / 1024)} MB (est).`);
83
- }
84
-
85
- main().catch(err => {
86
- console.error("Delete failed:", err);
87
- process.exit(1);
88
- });