recallnest 2.2.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/.env +4 -0
  2. package/config.json +55 -0
  3. package/logs/daily-recall.error.log +0 -0
  4. package/logs/daily-recall.log +30 -0
  5. package/logs/ingest-2026-04-11.log +110 -0
  6. package/logs/ingest-2026-04-12.log +90 -0
  7. package/logs/launchagent-stderr.log +0 -0
  8. package/logs/launchagent-stdout.log +0 -0
  9. package/package.json +1 -1
  10. package/.claude-plugin/CLAUDE.md +0 -29
  11. package/.claude-plugin/marketplace.json +0 -18
  12. package/.dockerignore +0 -11
  13. package/.github/workflows/ci.yml +0 -28
  14. package/.mcp.json +0 -9
  15. package/CHANGELOG.md +0 -51
  16. package/DEMO.md +0 -94
  17. package/Dockerfile +0 -20
  18. package/EVAL.md +0 -89
  19. package/FAILURES.md +0 -75
  20. package/OPERATIONS.md +0 -59
  21. package/ROADMAP.md +0 -245
  22. package/SKILL.md +0 -250
  23. package/assets/dashboard.png +0 -0
  24. package/assets/knowledge-graph.png +0 -0
  25. package/assets/recallnest-workbench.png +0 -0
  26. package/assets/screenshots/ui-full.png +0 -0
  27. package/assets/screenshots/ui-overview.png +0 -0
  28. package/assets/screenshots/ui-search.png +0 -0
  29. package/assets/ui/app.js +0 -1031
  30. package/assets/ui/index.html +0 -224
  31. package/assets/ui/styles.css +0 -781
  32. package/assets/wechat-qr.jpg +0 -0
  33. package/bun.lock +0 -337
  34. package/daily-reflections/2026-03-05.md +0 -15
  35. package/daily-reflections/2026-03-06.md +0 -15
  36. package/daily-reflections/2026-03-07.md +0 -7
  37. package/daily-reflections/2026-03-08.md +0 -34
  38. package/daily-reflections/2026-03-09.md +0 -35
  39. package/daily-reflections/2026-03-10.md +0 -36
  40. package/daily-reflections/2026-03-11.md +0 -35
  41. package/daily-reflections/2026-03-12.md +0 -36
  42. package/daily-reflections/2026-03-13.md +0 -35
  43. package/daily-reflections/2026-03-14.md +0 -35
  44. package/daily-reflections/2026-03-15.md +0 -35
  45. package/daily-reflections/2026-03-16.md +0 -41
  46. package/daily-reflections/2026-03-17.md +0 -10
  47. package/daily-reflections/2026-03-18.md +0 -10
  48. package/daily-reflections/2026-03-19.md +0 -10
  49. package/daily-reflections/2026-03-20.md +0 -10
  50. package/daily-reflections/2026-03-21.md +0 -10
  51. package/daily-reflections/2026-03-22.md +0 -10
  52. package/daily-reflections/2026-03-23.md +0 -10
  53. package/daily-reflections/2026-03-24.md +0 -10
  54. package/daily-reflections/2026-03-25.md +0 -32
  55. package/daily-reflections/2026-03-26.md +0 -37
  56. package/daily-reflections/2026-03-27.md +0 -35
  57. package/daily-reflections/2026-03-28.md +0 -35
  58. package/daily-reflections/2026-03-29.md +0 -35
  59. package/daily-reflections/2026-03-30.md +0 -36
  60. package/daily-reflections/2026-03-31.md +0 -35
  61. package/daily-reflections/2026-04-01.md +0 -35
  62. package/daily-reflections/2026-04-02.md +0 -35
  63. package/daily-reflections/2026-04-03.md +0 -37
  64. package/daily-reflections/2026-04-04.md +0 -37
  65. package/daily-reflections/2026-04-05.md +0 -35
  66. package/daily-reflections/2026-04-06.md +0 -36
  67. package/daily-reflections/2026-04-07.md +0 -35
  68. package/daily-reflections/2026-04-08.md +0 -35
  69. package/daily-reflections/2026-04-12.md +0 -35
  70. package/data/alias-map.example.json +0 -22
  71. package/docs/api-reference.md +0 -1291
  72. package/docs/architecture.md +0 -168
  73. package/docs/memory-boundary-contract.md +0 -226
  74. package/docs/memory-categories.md +0 -119
  75. package/docs/pipeline-audit-ultramemory.md +0 -177
  76. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  77. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  78. package/docs/plans/2026-04-01-recall-boost.md +0 -100
  79. package/docs/plans/mcp-tiering-plan.md +0 -126
  80. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  81. package/docs/self-evolution.md +0 -198
  82. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  83. package/docs/superpowers/plans/2026-04-11-constructive-retrieval-emotion.md +0 -1195
  84. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  85. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  86. package/docs/superpowers/specs/2026-04-11-constructive-retrieval-emotion-design.md +0 -398
  87. package/eval/continuity/results-2026-03-16.md +0 -117
  88. package/eval/continuity/results-2026-03-19.md +0 -453
  89. package/eval/continuity/runbook.md +0 -110
  90. package/eval/continuity/seed-cases.md +0 -54
  91. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  92. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  93. package/eval/reports/2026-03-06-baseline.md +0 -82
  94. package/eval/reports/query-expansion.md +0 -138
  95. package/glama.json +0 -6
  96. package/lm +0 -19
  97. package/plan-philosophy-upgrades.md +0 -363
  98. package/plan-product-upgrades.md +0 -155
  99. package/plan-v1.2-distributable.md +0 -105
  100. package/plan.md +0 -140
  101. package/research-memory-evolution.md +0 -440
  102. package/research-rl-agent-memory.md +0 -330
  103. package/scripts/archive-distilled-junk.ts +0 -116
  104. package/scripts/backfill-anchors.ts +0 -130
  105. package/scripts/backfill-l0l1l2.ts +0 -98
  106. package/scripts/blind-scan.ts +0 -588
  107. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  108. package/scripts/cleanup-step2-template-dups.ts +0 -113
  109. package/scripts/cleanup-step3-stale-content.ts +0 -149
  110. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  111. package/scripts/delete-distilled-junk.ts +0 -88
  112. package/scripts/diagnose-vector-search.ts +0 -132
  113. package/scripts/distill-facts.ts +0 -305
  114. package/scripts/health-check.ts +0 -343
  115. package/scripts/incremental-ingest.sh +0 -71
  116. package/scripts/migrate-language.ts +0 -75
  117. package/scripts/mlp-health-check.ts +0 -382
  118. package/scripts/pull-claude-desktop.ts +0 -515
  119. package/scripts/re-ingest-cc.sh +0 -66
  120. package/scripts/rebalance-memory-health.ts +0 -181
  121. package/scripts/run-desktop-ingest.sh +0 -11
  122. package/scripts/start-server.sh +0 -12
  123. package/scripts/verify-emotion-decay.ts +0 -148
  124. package/scripts/viz-dashboard.html +0 -423
  125. package/scripts/viz-export.ts +0 -703
  126. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  127. package/src/__tests__/activity-counter.test.ts +0 -96
  128. package/src/__tests__/admission-control.test.ts +0 -206
  129. package/src/__tests__/anchor-generator.test.ts +0 -67
  130. package/src/__tests__/auto-consolidation.test.ts +0 -168
  131. package/src/__tests__/auto-recall.test.ts +0 -131
  132. package/src/__tests__/babel-integration.test.ts +0 -139
  133. package/src/__tests__/batch-dedup.test.ts +0 -65
  134. package/src/__tests__/capture-engine.test.ts +0 -1144
  135. package/src/__tests__/capture-heuristic.test.ts +0 -171
  136. package/src/__tests__/cascade-forget.test.ts +0 -210
  137. package/src/__tests__/category-min-scores.test.ts +0 -33
  138. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  139. package/src/__tests__/circuit-breaker.test.ts +0 -99
  140. package/src/__tests__/cluster-consolidation.test.ts +0 -712
  141. package/src/__tests__/confidence-metadata.test.ts +0 -125
  142. package/src/__tests__/confidence-tracker.test.ts +0 -249
  143. package/src/__tests__/conflict-advisor.test.ts +0 -206
  144. package/src/__tests__/conflict-escalation.test.ts +0 -149
  145. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  146. package/src/__tests__/conflict-output.test.ts +0 -85
  147. package/src/__tests__/conflict-store.test.ts +0 -374
  148. package/src/__tests__/consolidation-engine.test.ts +0 -181
  149. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  150. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  151. package/src/__tests__/context-composer-pins.test.ts +0 -64
  152. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  153. package/src/__tests__/context-reconstructor.test.ts +0 -65
  154. package/src/__tests__/context-renderer.test.ts +0 -91
  155. package/src/__tests__/continuity-eval.test.ts +0 -275
  156. package/src/__tests__/continuity-schema.test.ts +0 -97
  157. package/src/__tests__/conversation-importer.test.ts +0 -692
  158. package/src/__tests__/core-recall-eval.test.ts +0 -235
  159. package/src/__tests__/core-summary.test.ts +0 -109
  160. package/src/__tests__/data-checkup.test.ts +0 -225
  161. package/src/__tests__/decay-exemption.test.ts +0 -84
  162. package/src/__tests__/dedup-actions.test.ts +0 -58
  163. package/src/__tests__/dedup-guard.test.ts +0 -39
  164. package/src/__tests__/deterministic-id.test.ts +0 -39
  165. package/src/__tests__/distill-lock.test.ts +0 -176
  166. package/src/__tests__/doctor.test.ts +0 -303
  167. package/src/__tests__/dream-pipeline.test.ts +0 -232
  168. package/src/__tests__/drill-down.test.ts +0 -108
  169. package/src/__tests__/embedder.test.ts +0 -122
  170. package/src/__tests__/emotion-decay.test.ts +0 -46
  171. package/src/__tests__/emotion-detector.test.ts +0 -56
  172. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  173. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  174. package/src/__tests__/entity-resolver.test.ts +0 -58
  175. package/src/__tests__/ethics-layer.test.ts +0 -476
  176. package/src/__tests__/event-segmenter.test.ts +0 -153
  177. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  178. package/src/__tests__/frequency-tracker.test.ts +0 -113
  179. package/src/__tests__/graph-export.test.ts +0 -489
  180. package/src/__tests__/hotness-blend.test.ts +0 -76
  181. package/src/__tests__/ingest.test.ts +0 -360
  182. package/src/__tests__/interference-detector.test.ts +0 -203
  183. package/src/__tests__/kg-extractor.test.ts +0 -184
  184. package/src/__tests__/kg-store.test.ts +0 -198
  185. package/src/__tests__/llm-consolidation.test.ts +0 -226
  186. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  187. package/src/__tests__/memory-boundaries.test.ts +0 -237
  188. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -243
  189. package/src/__tests__/memory-evolution.test.ts +0 -271
  190. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  191. package/src/__tests__/memory-lint.test.ts +0 -271
  192. package/src/__tests__/memory-output.test.ts +0 -215
  193. package/src/__tests__/multi-vector.test.ts +0 -242
  194. package/src/__tests__/narrative-architecture.test.ts +0 -456
  195. package/src/__tests__/pending-review.test.ts +0 -77
  196. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  197. package/src/__tests__/ppr-traversal.test.ts +0 -256
  198. package/src/__tests__/prediction-engine.test.ts +0 -349
  199. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  200. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  201. package/src/__tests__/preference-matcher.test.ts +0 -230
  202. package/src/__tests__/preference-slots.test.ts +0 -205
  203. package/src/__tests__/prospective-memory.test.ts +0 -283
  204. package/src/__tests__/query-entity-detector.test.ts +0 -171
  205. package/src/__tests__/query-expander-alias.test.ts +0 -89
  206. package/src/__tests__/query-expander.test.ts +0 -34
  207. package/src/__tests__/recall-governor.test.ts +0 -227
  208. package/src/__tests__/recall-mode.test.ts +0 -64
  209. package/src/__tests__/result-synthesizer.test.ts +0 -198
  210. package/src/__tests__/resume-context.test.ts +0 -4898
  211. package/src/__tests__/retrieval-trace.test.ts +0 -213
  212. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  213. package/src/__tests__/retroactive-boost.test.ts +0 -223
  214. package/src/__tests__/salience-scoring.test.ts +0 -106
  215. package/src/__tests__/scope-inventory.test.ts +0 -200
  216. package/src/__tests__/scope-policy.test.ts +0 -69
  217. package/src/__tests__/session-checkpoint.test.ts +0 -259
  218. package/src/__tests__/session-distiller.test.ts +0 -611
  219. package/src/__tests__/session-gc.test.ts +0 -215
  220. package/src/__tests__/skill-engine.test.ts +0 -546
  221. package/src/__tests__/skill-promotion.test.ts +0 -349
  222. package/src/__tests__/store-scope-audit.test.ts +0 -81
  223. package/src/__tests__/supersede-chain.test.ts +0 -154
  224. package/src/__tests__/temporal-parser.test.ts +0 -241
  225. package/src/__tests__/temporal-validity.test.ts +0 -160
  226. package/src/__tests__/term-registry.test.ts +0 -121
  227. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  228. package/src/__tests__/topic-tag.test.ts +0 -113
  229. package/src/__tests__/trusted-memory.test.ts +0 -828
  230. package/src/__tests__/version-manager.test.ts +0 -213
  231. package/src/__tests__/workflow-observation.test.ts +0 -400
  232. package/src/__tests__/write-verifier.test.ts +0 -122
@@ -1,117 +0,0 @@
1
- /**
2
- * 清洗步骤 1:删除精确重复记忆
3
- * 每组重复保留 accessCount 最高的那条,其余删除
4
- *
5
- * 用法:
6
- * bun run scripts/cleanup-step1-exact-dups.ts [--execute]
7
- * 不加 --execute 只统计,加了才真删
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- interface MemoryRow {
17
- id: string;
18
- text: string;
19
- category: string;
20
- scope: string;
21
- importance: number;
22
- timestamp: number;
23
- metadata?: string;
24
- }
25
-
26
- async function main() {
27
- console.log(`\n🧹 清洗步骤 1:精确重复删除 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
28
- console.log("=".repeat(60));
29
-
30
- const db = await lancedb.connect(DB_PATH);
31
- const table = await db.openTable(TABLE_NAME);
32
-
33
- console.log("⏳ 读取全量数据...");
34
- const allRows: MemoryRow[] = await table
35
- .query()
36
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
37
- .toArray() as any;
38
-
39
- const total = allRows.length;
40
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
41
-
42
- // Parse metadata
43
- const parsedRows = allRows.map(row => ({
44
- ...row,
45
- meta: row.metadata ? JSON.parse(row.metadata) : {},
46
- }));
47
-
48
- // Group by normalized text
49
- const textGroups = new Map<string, typeof parsedRows>();
50
- for (const row of parsedRows) {
51
- const key = row.text.trim().toLowerCase();
52
- if (!textGroups.has(key)) textGroups.set(key, []);
53
- textGroups.get(key)!.push(row);
54
- }
55
-
56
- // Find duplicates
57
- const dupGroups = [...textGroups.entries()].filter(([, rows]) => rows.length > 1);
58
- const idsToDelete: string[] = [];
59
-
60
- for (const [, rows] of dupGroups) {
61
- // Sort: highest accessCount first, then oldest timestamp as tiebreaker (keep oldest)
62
- rows.sort((a, b) => {
63
- const acA = a.meta.accessCount ?? a.meta.access_count ?? 0;
64
- const acB = b.meta.accessCount ?? b.meta.access_count ?? 0;
65
- if (acB !== acA) return acB - acA; // higher access first
66
- return a.timestamp - b.timestamp; // older first
67
- });
68
- // Keep first, delete rest
69
- for (let i = 1; i < rows.length; i++) {
70
- idsToDelete.push(rows[i].id);
71
- }
72
- }
73
-
74
- console.log(`📊 统计结果:`);
75
- console.log(` 重复组数: ${dupGroups.length.toLocaleString()}`);
76
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
77
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
78
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
79
-
80
- // Show top 10 largest duplicate groups
81
- const topGroups = dupGroups.sort((a, b) => b[1].length - a[1].length).slice(0, 10);
82
- console.log(`\n Top 10 重复组:`);
83
- for (const [text, rows] of topGroups) {
84
- console.log(` [×${rows.length}] ${text.slice(0, 70)}...`);
85
- }
86
-
87
- if (!EXECUTE) {
88
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
89
- return;
90
- }
91
-
92
- // Execute deletion in batches
93
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
94
- const batchSize = 100;
95
- let deleted = 0;
96
-
97
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
98
- const batch = idsToDelete.slice(i, i + batchSize);
99
- // Build WHERE clause: id IN ('a', 'b', 'c')
100
- const escaped = batch.map(id => id.replace(/'/g, "''"));
101
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
102
- await table.delete(whereClause);
103
- deleted += batch.length;
104
- if (deleted % 1000 === 0 || deleted === idsToDelete.length) {
105
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
106
- }
107
- }
108
-
109
- // Verify
110
- const remaining = await table.countRows();
111
- console.log(`\n✅ 删除完成!`);
112
- console.log(` 删除前: ${total.toLocaleString()} 条`);
113
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
114
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
115
- }
116
-
117
- main().catch(console.error);
@@ -1,113 +0,0 @@
1
- /**
2
- * 清洗步骤 2:删除模板化重复内容
3
- * 前 100 字符相同的记忆,每组保留 accessCount 最高的 1 条
4
- * (步骤 1 已清精确重复,这里抓"近似重复"——同一段内容被切成不同长度存了多次)
5
- *
6
- * 用法:
7
- * bun run scripts/cleanup-step2-template-dups.ts [--execute]
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- interface MemoryRow {
17
- id: string;
18
- text: string;
19
- category: string;
20
- scope: string;
21
- importance: number;
22
- timestamp: number;
23
- metadata?: string;
24
- }
25
-
26
- async function main() {
27
- console.log(`\n🧹 清洗步骤 2:模板化重复删除 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
28
- console.log("=".repeat(60));
29
-
30
- const db = await lancedb.connect(DB_PATH);
31
- const table = await db.openTable(TABLE_NAME);
32
-
33
- console.log("⏳ 读取全量数据...");
34
- const allRows: MemoryRow[] = await table
35
- .query()
36
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
37
- .toArray() as any;
38
-
39
- const total = allRows.length;
40
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
41
-
42
- const parsedRows = allRows.map(row => ({
43
- ...row,
44
- meta: row.metadata ? JSON.parse(row.metadata) : {},
45
- }));
46
-
47
- // Group by first 100 chars (normalized)
48
- const fpGroups = new Map<string, typeof parsedRows>();
49
- for (const row of parsedRows) {
50
- const fp = row.text.trim().slice(0, 100).toLowerCase();
51
- if (!fpGroups.has(fp)) fpGroups.set(fp, []);
52
- fpGroups.get(fp)!.push(row);
53
- }
54
-
55
- // Find groups with duplicates (after step 1, exact dups are gone, so these are near-dups)
56
- const dupGroups = [...fpGroups.entries()].filter(([, rows]) => rows.length > 1);
57
- const idsToDelete: string[] = [];
58
-
59
- for (const [, rows] of dupGroups) {
60
- // Keep the one with highest accessCount, then longest text, then oldest
61
- rows.sort((a, b) => {
62
- const acA = a.meta.accessCount ?? a.meta.access_count ?? 0;
63
- const acB = b.meta.accessCount ?? b.meta.access_count ?? 0;
64
- if (acB !== acA) return acB - acA;
65
- if (b.text.length !== a.text.length) return b.text.length - a.text.length; // longer is more complete
66
- return a.timestamp - b.timestamp;
67
- });
68
- for (let i = 1; i < rows.length; i++) {
69
- idsToDelete.push(rows[i].id);
70
- }
71
- }
72
-
73
- console.log(`📊 统计结果:`);
74
- console.log(` 近似重复组数: ${dupGroups.length.toLocaleString()}`);
75
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
76
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
77
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
78
-
79
- // Show top 15 largest groups
80
- const topGroups = dupGroups.sort((a, b) => b[1].length - a[1].length).slice(0, 15);
81
- console.log(`\n Top 15 近似重复组:`);
82
- for (const [fp, rows] of topGroups) {
83
- console.log(` [×${rows.length.toString().padStart(3)}] ${fp.slice(0, 65)}...`);
84
- }
85
-
86
- if (!EXECUTE) {
87
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
88
- return;
89
- }
90
-
91
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
92
- const batchSize = 100;
93
- let deleted = 0;
94
-
95
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
96
- const batch = idsToDelete.slice(i, i + batchSize);
97
- const escaped = batch.map(id => id.replace(/'/g, "''"));
98
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
99
- await table.delete(whereClause);
100
- deleted += batch.length;
101
- if (deleted % 500 === 0 || deleted === idsToDelete.length) {
102
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
103
- }
104
- }
105
-
106
- const remaining = await table.countRows();
107
- console.log(`\n✅ 删除完成!`);
108
- console.log(` 删除前: ${total.toLocaleString()} 条`);
109
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
110
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
111
- }
112
-
113
- main().catch(console.error);
@@ -1,149 +0,0 @@
1
- /**
2
- * 清洗步骤 3:标记/清理过时内容
3
- * 扫描含 temporary/临时/HACK/TODO/FIXME 的记忆
4
- * 只删 accessCount=0 且含这些关键词的(从未被用过的过时内容)
5
- *
6
- * 用法:
7
- * bun run scripts/cleanup-step3-stale-content.ts [--execute]
8
- */
9
-
10
- import lancedb from "@lancedb/lancedb";
11
-
12
- const DB_PATH = "./data/lancedb";
13
- const TABLE_NAME = "memories";
14
- const EXECUTE = process.argv.includes("--execute");
15
-
16
- const STALE_KEYWORDS = [
17
- "temporary", "临时", "HACK", "TODO", "FIXME",
18
- "workaround", "deprecated", "废弃", "待修复",
19
- ];
20
-
21
- interface MemoryRow {
22
- id: string;
23
- text: string;
24
- category: string;
25
- scope: string;
26
- importance: number;
27
- timestamp: number;
28
- metadata?: string;
29
- }
30
-
31
- async function main() {
32
- console.log(`\n🧹 清洗步骤 3:过时内容清理 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
33
- console.log("=".repeat(60));
34
-
35
- const db = await lancedb.connect(DB_PATH);
36
- const table = await db.openTable(TABLE_NAME);
37
-
38
- console.log("⏳ 读取全量数据...");
39
- const allRows: MemoryRow[] = await table
40
- .query()
41
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
42
- .toArray() as any;
43
-
44
- const total = allRows.length;
45
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
46
-
47
- const parsedRows = allRows.map(row => ({
48
- ...row,
49
- meta: row.metadata ? JSON.parse(row.metadata) : {},
50
- }));
51
-
52
- // Scan for stale keywords
53
- const keywordHits = new Map<string, typeof parsedRows>();
54
- for (const kw of STALE_KEYWORDS) {
55
- keywordHits.set(kw, []);
56
- }
57
-
58
- for (const row of parsedRows) {
59
- const textLower = row.text.toLowerCase();
60
- for (const kw of STALE_KEYWORDS) {
61
- if (textLower.includes(kw.toLowerCase())) {
62
- keywordHits.get(kw)!.push(row);
63
- }
64
- }
65
- }
66
-
67
- console.log("📊 关键词命中统计:");
68
- let totalHits = 0;
69
- for (const kw of STALE_KEYWORDS) {
70
- const hits = keywordHits.get(kw)!;
71
- if (hits.length > 0) {
72
- const deadHits = hits.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) === 0);
73
- console.log(` "${kw}": ${hits.length} 条 (其中 ${deadHits.length} 条从未访问)`);
74
- }
75
- }
76
-
77
- // Collect: only delete if accessCount=0 (dead memory with stale keyword)
78
- const idsToDeleteSet = new Set<string>();
79
- const deleteCandidates: typeof parsedRows = [];
80
-
81
- for (const row of parsedRows) {
82
- const ac = row.meta.accessCount ?? row.meta.access_count ?? 0;
83
- if (ac > 0) continue; // keep anything that was ever accessed
84
-
85
- const textLower = row.text.toLowerCase();
86
- for (const kw of STALE_KEYWORDS) {
87
- if (textLower.includes(kw.toLowerCase())) {
88
- if (!idsToDeleteSet.has(row.id)) {
89
- idsToDeleteSet.add(row.id);
90
- deleteCandidates.push(row);
91
- }
92
- break;
93
- }
94
- }
95
- }
96
-
97
- const idsToDelete = [...idsToDeleteSet];
98
-
99
- console.log(`\n📊 清理范围(仅 accessCount=0 + 含过时关键词):`);
100
- console.log(` 待删除条数: ${idsToDelete.length.toLocaleString()}`);
101
- console.log(` 保留条数: ${(total - idsToDelete.length).toLocaleString()}`);
102
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
103
-
104
- // Category breakdown of what we're deleting
105
- const catBreakdown = new Map<string, number>();
106
- for (const row of deleteCandidates) {
107
- catBreakdown.set(row.category, (catBreakdown.get(row.category) || 0) + 1);
108
- }
109
- console.log(`\n 待删除按类别:`);
110
- for (const [cat, count] of [...catBreakdown.entries()].sort((a, b) => b[1] - a[1])) {
111
- console.log(` ${cat.padEnd(14)} ${count}`);
112
- }
113
-
114
- // Show samples
115
- console.log(`\n 待删除示例(随机 10 条):`);
116
- const shuffled = deleteCandidates.sort(() => Math.random() - 0.5).slice(0, 10);
117
- for (const row of shuffled) {
118
- const matched = STALE_KEYWORDS.find(kw => row.text.toLowerCase().includes(kw.toLowerCase()));
119
- console.log(` [${row.category}] [kw="${matched}"] ${row.text.slice(0, 80)}...`);
120
- }
121
-
122
- if (!EXECUTE) {
123
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
124
- return;
125
- }
126
-
127
- console.log(`\n🔥 开始删除 ${idsToDelete.length.toLocaleString()} 条...`);
128
- const batchSize = 100;
129
- let deleted = 0;
130
-
131
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
132
- const batch = idsToDelete.slice(i, i + batchSize);
133
- const escaped = batch.map(id => id.replace(/'/g, "''"));
134
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
135
- await table.delete(whereClause);
136
- deleted += batch.length;
137
- if (deleted % 500 === 0 || deleted === idsToDelete.length) {
138
- console.log(` ✅ ${deleted.toLocaleString()} / ${idsToDelete.length.toLocaleString()}`);
139
- }
140
- }
141
-
142
- const remaining = await table.countRows();
143
- console.log(`\n✅ 删除完成!`);
144
- console.log(` 删除前: ${total.toLocaleString()} 条`);
145
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
146
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
147
- }
148
-
149
- main().catch(console.error);
@@ -1,131 +0,0 @@
1
- /**
2
- * 清洗步骤 4:分析孤儿 scope(只有 1 条记忆的 scope)
3
- * 策略:不删记忆,而是把孤儿 scope 的记忆重新归类到更有意义的 scope
4
- * - accessCount=0 的孤儿 → 直接删(死记忆 + 孤儿 scope 双重废)
5
- * - accessCount>0 的孤儿 → 保留,只报告(不乱动有价值的)
6
- *
7
- * 用法:
8
- * bun run scripts/cleanup-step4-orphan-scopes.ts [--execute]
9
- */
10
-
11
- import lancedb from "@lancedb/lancedb";
12
-
13
- const DB_PATH = "./data/lancedb";
14
- const TABLE_NAME = "memories";
15
- const EXECUTE = process.argv.includes("--execute");
16
-
17
- interface MemoryRow {
18
- id: string;
19
- text: string;
20
- category: string;
21
- scope: string;
22
- importance: number;
23
- timestamp: number;
24
- metadata?: string;
25
- }
26
-
27
- async function main() {
28
- console.log(`\n🧹 清洗步骤 4:孤儿 scope 清理 ${EXECUTE ? "[执行模式]" : "[预览模式]"}`);
29
- console.log("=".repeat(60));
30
-
31
- const db = await lancedb.connect(DB_PATH);
32
- const table = await db.openTable(TABLE_NAME);
33
-
34
- console.log("⏳ 读取全量数据...");
35
- const allRows: MemoryRow[] = await table
36
- .query()
37
- .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"])
38
- .toArray() as any;
39
-
40
- const total = allRows.length;
41
- console.log(`✅ ${total.toLocaleString()} 条记忆\n`);
42
-
43
- const parsedRows = allRows.map(row => ({
44
- ...row,
45
- meta: row.metadata ? JSON.parse(row.metadata) : {},
46
- }));
47
-
48
- // Find orphan scopes (only 1 memory)
49
- const scopeCounts = new Map<string, number>();
50
- for (const row of parsedRows) {
51
- const s = row.scope || "(empty)";
52
- scopeCounts.set(s, (scopeCounts.get(s) || 0) + 1);
53
- }
54
-
55
- const orphanScopes = new Set(
56
- [...scopeCounts.entries()].filter(([, c]) => c === 1).map(([s]) => s)
57
- );
58
-
59
- const orphanRows = parsedRows.filter(r => orphanScopes.has(r.scope || "(empty)"));
60
-
61
- // Split: dead orphans vs alive orphans
62
- const deadOrphans = orphanRows.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) === 0);
63
- const aliveOrphans = orphanRows.filter(r => (r.meta.accessCount ?? r.meta.access_count ?? 0) > 0);
64
-
65
- console.log(`📊 孤儿 scope 统计:`);
66
- console.log(` 孤儿 scope 总数: ${orphanScopes.size}`);
67
- console.log(` 死记忆孤儿: ${deadOrphans.length} 条 → 可安全删除`);
68
- console.log(` 活记忆孤儿: ${aliveOrphans.length} 条 → 保留不动`);
69
-
70
- // Category breakdown
71
- const deadCatMap = new Map<string, number>();
72
- for (const r of deadOrphans) {
73
- deadCatMap.set(r.category, (deadCatMap.get(r.category) || 0) + 1);
74
- }
75
- console.log(`\n 死记忆孤儿按类别:`);
76
- for (const [cat, count] of [...deadCatMap.entries()].sort((a, b) => b[1] - a[1])) {
77
- console.log(` ${cat.padEnd(14)} ${count}`);
78
- }
79
-
80
- // Show alive orphan samples (these we keep)
81
- if (aliveOrphans.length > 0) {
82
- console.log(`\n 活记忆孤儿示例(保留):`);
83
- for (const r of aliveOrphans.slice(0, 5)) {
84
- const ac = r.meta.accessCount ?? r.meta.access_count ?? 0;
85
- console.log(` [${r.category}] [ac=${ac}] [scope=${r.scope.slice(0, 25)}] ${r.text.slice(0, 60)}...`);
86
- }
87
- }
88
-
89
- // Show dead orphan samples
90
- console.log(`\n 死记忆孤儿示例(待删):`);
91
- const samples = deadOrphans.sort(() => Math.random() - 0.5).slice(0, 10);
92
- for (const r of samples) {
93
- console.log(` [${r.category}] [scope=${r.scope.slice(0, 25)}] ${r.text.slice(0, 60)}...`);
94
- }
95
-
96
- const idsToDelete = deadOrphans.map(r => r.id);
97
-
98
- console.log(`\n📊 清理计划:`);
99
- console.log(` 待删除: ${idsToDelete.length} 条(死记忆 + 孤儿 scope)`);
100
- console.log(` 保留: ${(total - idsToDelete.length).toLocaleString()} 条`);
101
- console.log(` 瘦身比例: ${(idsToDelete.length / total * 100).toFixed(1)}%`);
102
-
103
- if (!EXECUTE) {
104
- console.log(`\n⚠️ 预览模式,未执行删除。加 --execute 参数执行。`);
105
- return;
106
- }
107
-
108
- console.log(`\n🔥 开始删除 ${idsToDelete.length} 条...`);
109
- const batchSize = 100;
110
- let deleted = 0;
111
-
112
- for (let i = 0; i < idsToDelete.length; i += batchSize) {
113
- const batch = idsToDelete.slice(i, i + batchSize);
114
- const escaped = batch.map(id => id.replace(/'/g, "''"));
115
- const whereClause = `id IN (${escaped.map(id => `'${id}'`).join(",")})`;
116
- await table.delete(whereClause);
117
- deleted += batch.length;
118
- if (deleted % 100 === 0 || deleted === idsToDelete.length) {
119
- console.log(` ✅ ${deleted} / ${idsToDelete.length}`);
120
- }
121
- }
122
-
123
- const remaining = await table.countRows();
124
- console.log(`\n✅ 删除完成!`);
125
- console.log(` 删除前: ${total.toLocaleString()} 条`);
126
- console.log(` 删除后: ${remaining.toLocaleString()} 条`);
127
- console.log(` 实际删除: ${(total - remaining).toLocaleString()} 条`);
128
- console.log(` 消灭孤儿 scope: ${deadOrphans.length} 个`);
129
- }
130
-
131
- main().catch(console.error);
@@ -1,88 +0,0 @@
1
- #!/usr/bin/env bun
2
- /**
3
- * Delete raw fact fragments that have already been distilled.
4
- * Targets: entries in distill-completed scopes with importance < 0.4, no L0, category=fact.
5
- * Safe: distilled replacements already exist; transcript originals are in ~/.claude/projects.
6
- *
7
- * Usage:
8
- * bun scripts/delete-distilled-junk.ts --dry-run # preview
9
- * bun scripts/delete-distilled-junk.ts # execute
10
- */
11
-
12
- import fs from "fs";
13
- import path from "path";
14
- import { loadDotEnv, loadConfig, expandHome } from "../src/runtime-config.js";
15
- import { loadLanceDB } from "../src/store.js";
16
-
17
- loadDotEnv();
18
- const config = loadConfig();
19
- const dryRun = process.argv.includes("--dry-run");
20
-
21
- const PROGRESS_FILE = path.join(import.meta.dir, "..", "data", "distill-progress.json");
22
-
23
- async function main() {
24
- const progress = JSON.parse(fs.readFileSync(PROGRESS_FILE, "utf-8"));
25
- const completedScopes = new Set<string>(progress.completedScopes);
26
- console.log(`=== Delete distilled junk${dryRun ? " (dry run)" : ""} ===\n`);
27
- console.log(` Distill-completed scopes: ${completedScopes.size}`);
28
-
29
- const lancedb = await loadLanceDB();
30
- const dbPath = expandHome(config.database?.path || "data/lancedb");
31
- const db = await lancedb.connect(dbPath);
32
- const table = await db.openTable("memories");
33
-
34
- // Read all fact entries (only need id, scope, importance, metadata — skip vector for speed)
35
- const allFacts = await table.query()
36
- .where("category = 'fact'")
37
- .select(["id", "scope", "importance", "metadata"])
38
- .toArray();
39
-
40
- console.log(` Total fact entries: ${allFacts.length}`);
41
-
42
- const toDelete: string[] = [];
43
-
44
- for (const row of allFacts) {
45
- const scope = (row.scope as string) || "";
46
- if (!completedScopes.has(scope)) continue;
47
-
48
- const imp = Number(row.importance);
49
- if (imp >= 0.4) continue;
50
-
51
- const metaStr = (row.metadata as string) || "{}";
52
- let meta: Record<string, unknown>;
53
- try { meta = JSON.parse(metaStr); } catch { continue; }
54
-
55
- if (typeof meta.l0_abstract === "string" && meta.l0_abstract.length > 0) continue;
56
-
57
- toDelete.push(row.id as string);
58
- }
59
-
60
- console.log(` Junk to delete: ${toDelete.length}`);
61
- console.log(` Would keep: ${allFacts.length - toDelete.length} fact entries\n`);
62
-
63
- if (dryRun) {
64
- console.log(` (dry run — no changes made)`);
65
- return;
66
- }
67
-
68
- if (toDelete.length === 0) {
69
- console.log(` Nothing to delete.`);
70
- return;
71
- }
72
-
73
- // Batch delete — no need to re-add
74
- const batchSize = 2000;
75
- for (let i = 0; i < toDelete.length; i += batchSize) {
76
- const batch = toDelete.slice(i, i + batchSize);
77
- const ids = batch.map(id => `'${id.replace(/'/g, "''")}'`).join(",");
78
- await table.delete(`id IN (${ids})`);
79
- console.log(` Batch ${Math.floor(i / batchSize) + 1}: deleted ${batch.length} entries`);
80
- }
81
-
82
- console.log(`\nDone. Deleted ${toDelete.length} junk fact entries. Freed ~${Math.round(toDelete.length * 6 / 1024)} MB (est).`);
83
- }
84
-
85
- main().catch(err => {
86
- console.error("Delete failed:", err);
87
- process.exit(1);
88
- });