sensorium-mcp 2.17.27 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (224) hide show
  1. package/Install-Sensorium.ps1 +327 -0
  2. package/README.md +14 -0
  3. package/dist/config.d.ts +16 -1
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/config.js +39 -2
  6. package/dist/config.js.map +1 -1
  7. package/dist/daily-session.d.ts +2 -1
  8. package/dist/daily-session.d.ts.map +1 -1
  9. package/dist/daily-session.js +23 -26
  10. package/dist/daily-session.js.map +1 -1
  11. package/dist/dashboard/routes/settings.d.ts +4 -0
  12. package/dist/dashboard/routes/settings.d.ts.map +1 -1
  13. package/dist/dashboard/routes/settings.js +57 -1
  14. package/dist/dashboard/routes/settings.js.map +1 -1
  15. package/dist/dashboard/routes/threads.d.ts +1 -0
  16. package/dist/dashboard/routes/threads.d.ts.map +1 -1
  17. package/dist/dashboard/routes/threads.js +23 -27
  18. package/dist/dashboard/routes/threads.js.map +1 -1
  19. package/dist/dashboard/routes.d.ts.map +1 -1
  20. package/dist/dashboard/routes.js +7 -2
  21. package/dist/dashboard/routes.js.map +1 -1
  22. package/dist/dashboard/spa.html +11 -11
  23. package/dist/data/interfaces.d.ts +36 -0
  24. package/dist/data/interfaces.d.ts.map +1 -0
  25. package/dist/data/interfaces.js +2 -0
  26. package/dist/data/interfaces.js.map +1 -0
  27. package/dist/data/memory/bootstrap.d.ts +36 -16
  28. package/dist/data/memory/bootstrap.d.ts.map +1 -1
  29. package/dist/data/memory/bootstrap.js +71 -217
  30. package/dist/data/memory/bootstrap.js.map +1 -1
  31. package/dist/data/memory/consolidation.d.ts +35 -34
  32. package/dist/data/memory/consolidation.d.ts.map +1 -1
  33. package/dist/data/memory/consolidation.js +43 -555
  34. package/dist/data/memory/consolidation.js.map +1 -1
  35. package/dist/data/memory/index.d.ts +0 -1
  36. package/dist/data/memory/index.d.ts.map +1 -1
  37. package/dist/data/memory/index.js +0 -1
  38. package/dist/data/memory/index.js.map +1 -1
  39. package/dist/data/memory/migration-runner.d.ts +5 -0
  40. package/dist/data/memory/migration-runner.d.ts.map +1 -0
  41. package/dist/data/memory/migration-runner.js +403 -0
  42. package/dist/data/memory/migration-runner.js.map +1 -0
  43. package/dist/data/memory/reflection.js +1 -1
  44. package/dist/data/memory/schema-ddl.d.ts +4 -0
  45. package/dist/data/memory/schema-ddl.d.ts.map +1 -0
  46. package/dist/data/memory/schema-ddl.js +194 -0
  47. package/dist/data/memory/schema-ddl.js.map +1 -0
  48. package/dist/data/memory/schema-guard.d.ts +3 -0
  49. package/dist/data/memory/schema-guard.d.ts.map +1 -0
  50. package/dist/data/memory/schema-guard.js +184 -0
  51. package/dist/data/memory/schema-guard.js.map +1 -0
  52. package/dist/data/memory/schema.d.ts +2 -5
  53. package/dist/data/memory/schema.d.ts.map +1 -1
  54. package/dist/data/memory/schema.js +6 -834
  55. package/dist/data/memory/schema.js.map +1 -1
  56. package/dist/data/memory/semantic.d.ts +0 -1
  57. package/dist/data/memory/semantic.d.ts.map +1 -1
  58. package/dist/data/memory/semantic.js +2 -8
  59. package/dist/data/memory/semantic.js.map +1 -1
  60. package/dist/data/memory/synthesis.js +2 -2
  61. package/dist/data/memory/synthesis.js.map +1 -1
  62. package/dist/data/memory/thread-registry.d.ts +18 -4
  63. package/dist/data/memory/thread-registry.d.ts.map +1 -1
  64. package/dist/data/memory/thread-registry.js +25 -0
  65. package/dist/data/memory/thread-registry.js.map +1 -1
  66. package/dist/data/sent-message.repository.d.ts +12 -0
  67. package/dist/data/sent-message.repository.d.ts.map +1 -0
  68. package/dist/data/sent-message.repository.js +31 -0
  69. package/dist/data/sent-message.repository.js.map +1 -0
  70. package/dist/http-server.d.ts.map +1 -1
  71. package/dist/http-server.js +23 -2
  72. package/dist/http-server.js.map +1 -1
  73. package/dist/index.js +27 -48
  74. package/dist/index.js.map +1 -1
  75. package/dist/logger.d.ts +7 -2
  76. package/dist/logger.d.ts.map +1 -1
  77. package/dist/logger.js +89 -12
  78. package/dist/logger.js.map +1 -1
  79. package/dist/scheduler.d.ts +8 -0
  80. package/dist/scheduler.d.ts.map +1 -1
  81. package/dist/scheduler.js +15 -0
  82. package/dist/scheduler.js.map +1 -1
  83. package/dist/server/factory.d.ts +2 -1
  84. package/dist/server/factory.d.ts.map +1 -1
  85. package/dist/server/factory.js +11 -4
  86. package/dist/server/factory.js.map +1 -1
  87. package/dist/services/agent-spawn.service.d.ts +39 -0
  88. package/dist/services/agent-spawn.service.d.ts.map +1 -0
  89. package/dist/services/agent-spawn.service.js +348 -0
  90. package/dist/services/agent-spawn.service.js.map +1 -0
  91. package/dist/services/background-runner.d.ts +26 -0
  92. package/dist/services/background-runner.d.ts.map +1 -0
  93. package/dist/services/background-runner.js +71 -0
  94. package/dist/services/background-runner.js.map +1 -0
  95. package/dist/services/consolidation.service.d.ts +16 -0
  96. package/dist/services/consolidation.service.d.ts.map +1 -0
  97. package/dist/services/consolidation.service.js +508 -0
  98. package/dist/services/consolidation.service.js.map +1 -0
  99. package/dist/services/dispatcher/broker.d.ts +2 -0
  100. package/dist/services/dispatcher/broker.d.ts.map +1 -1
  101. package/dist/services/dispatcher/broker.js +5 -10
  102. package/dist/services/dispatcher/broker.js.map +1 -1
  103. package/dist/services/dispatcher/index.d.ts +1 -1
  104. package/dist/services/dispatcher/index.d.ts.map +1 -1
  105. package/dist/services/dispatcher/index.js +1 -1
  106. package/dist/services/dispatcher/index.js.map +1 -1
  107. package/dist/services/dispatcher/lock.d.ts.map +1 -1
  108. package/dist/services/dispatcher/lock.js +7 -11
  109. package/dist/services/dispatcher/lock.js.map +1 -1
  110. package/dist/services/maintenance-signal.d.ts +18 -0
  111. package/dist/services/maintenance-signal.d.ts.map +1 -0
  112. package/dist/services/maintenance-signal.js +48 -0
  113. package/dist/services/maintenance-signal.js.map +1 -0
  114. package/dist/services/memory-briefing.service.d.ts +4 -0
  115. package/dist/services/memory-briefing.service.d.ts.map +1 -0
  116. package/dist/services/memory-briefing.service.js +143 -0
  117. package/dist/services/memory-briefing.service.js.map +1 -0
  118. package/dist/services/process.service.d.ts +31 -0
  119. package/dist/services/process.service.d.ts.map +1 -0
  120. package/dist/services/process.service.js +100 -0
  121. package/dist/services/process.service.js.map +1 -0
  122. package/dist/services/thread-health.service.d.ts +18 -0
  123. package/dist/services/thread-health.service.d.ts.map +1 -0
  124. package/dist/services/thread-health.service.js +118 -0
  125. package/dist/services/thread-health.service.js.map +1 -0
  126. package/dist/services/thread-lifecycle.service.d.ts +52 -0
  127. package/dist/services/thread-lifecycle.service.d.ts.map +1 -0
  128. package/dist/services/thread-lifecycle.service.js +174 -0
  129. package/dist/services/thread-lifecycle.service.js.map +1 -0
  130. package/dist/services/topic.service.d.ts +25 -0
  131. package/dist/services/topic.service.d.ts.map +1 -0
  132. package/dist/services/topic.service.js +65 -0
  133. package/dist/services/topic.service.js.map +1 -0
  134. package/dist/services/worker-cleanup.service.d.ts +8 -0
  135. package/dist/services/worker-cleanup.service.d.ts.map +1 -0
  136. package/dist/services/worker-cleanup.service.js +82 -0
  137. package/dist/services/worker-cleanup.service.js.map +1 -0
  138. package/dist/sessions.d.ts +14 -0
  139. package/dist/sessions.d.ts.map +1 -1
  140. package/dist/sessions.js +55 -0
  141. package/dist/sessions.js.map +1 -1
  142. package/dist/telegram.d.ts +13 -6
  143. package/dist/telegram.d.ts.map +1 -1
  144. package/dist/telegram.js +43 -14
  145. package/dist/telegram.js.map +1 -1
  146. package/dist/tools/defs/memory-defs.d.ts.map +1 -1
  147. package/dist/tools/defs/memory-defs.js +0 -19
  148. package/dist/tools/defs/memory-defs.js.map +1 -1
  149. package/dist/tools/delegate-tool.d.ts +4 -0
  150. package/dist/tools/delegate-tool.d.ts.map +1 -1
  151. package/dist/tools/delegate-tool.js +48 -109
  152. package/dist/tools/delegate-tool.js.map +1 -1
  153. package/dist/tools/memory-tools.d.ts.map +1 -1
  154. package/dist/tools/memory-tools.js +1 -16
  155. package/dist/tools/memory-tools.js.map +1 -1
  156. package/dist/tools/shared-agent-utils.d.ts +9 -1
  157. package/dist/tools/shared-agent-utils.d.ts.map +1 -1
  158. package/dist/tools/shared-agent-utils.js +24 -42
  159. package/dist/tools/shared-agent-utils.js.map +1 -1
  160. package/dist/tools/start-session-tool.d.ts +2 -0
  161. package/dist/tools/start-session-tool.d.ts.map +1 -1
  162. package/dist/tools/start-session-tool.js +66 -106
  163. package/dist/tools/start-session-tool.js.map +1 -1
  164. package/dist/tools/thread-lifecycle.d.ts +5 -127
  165. package/dist/tools/thread-lifecycle.d.ts.map +1 -1
  166. package/dist/tools/thread-lifecycle.js +5 -1163
  167. package/dist/tools/thread-lifecycle.js.map +1 -1
  168. package/dist/tools/utility-tools.js +5 -2
  169. package/dist/tools/utility-tools.js.map +1 -1
  170. package/dist/tools/wait/drive-handler.d.ts +0 -1
  171. package/dist/tools/wait/drive-handler.d.ts.map +1 -1
  172. package/dist/tools/wait/drive-handler.js +5 -22
  173. package/dist/tools/wait/drive-handler.js.map +1 -1
  174. package/dist/tools/wait/message-delivery.js +1 -1
  175. package/dist/tools/wait/message-delivery.js.map +1 -1
  176. package/dist/tools/wait/message-processing.d.ts.map +1 -1
  177. package/dist/tools/wait/message-processing.js +9 -8
  178. package/dist/tools/wait/message-processing.js.map +1 -1
  179. package/dist/tools/wait/poll-loop.d.ts +2 -0
  180. package/dist/tools/wait/poll-loop.d.ts.map +1 -1
  181. package/dist/tools/wait/poll-loop.js +27 -29
  182. package/dist/tools/wait/poll-loop.js.map +1 -1
  183. package/dist/tools/wait/task-handler.d.ts +0 -3
  184. package/dist/tools/wait/task-handler.d.ts.map +1 -1
  185. package/dist/tools/wait/task-handler.js +3 -2
  186. package/dist/tools/wait/task-handler.js.map +1 -1
  187. package/dist/types.d.ts +0 -1
  188. package/dist/types.d.ts.map +1 -1
  189. package/package.json +4 -8
  190. package/supervisor/config.go +182 -69
  191. package/supervisor/config_test.go +78 -0
  192. package/supervisor/go.mod +12 -0
  193. package/supervisor/go.sum +20 -0
  194. package/supervisor/health.go +60 -11
  195. package/supervisor/health_test.go +29 -0
  196. package/supervisor/keeper.go +15 -10
  197. package/supervisor/log.go +109 -28
  198. package/supervisor/log_test.go +86 -6
  199. package/supervisor/main.go +150 -19
  200. package/supervisor/main_test.go +130 -0
  201. package/supervisor/process.go +47 -4
  202. package/supervisor/process_test.go +14 -0
  203. package/supervisor/secrets.go +95 -0
  204. package/supervisor/secrets_securevault_test.go +98 -0
  205. package/supervisor/secrets_test.go +119 -0
  206. package/supervisor/self_update.go +282 -0
  207. package/supervisor/self_update_test.go +177 -0
  208. package/supervisor/service_restart_stub.go +9 -0
  209. package/supervisor/service_restart_windows.go +63 -0
  210. package/supervisor/service_stub.go +15 -0
  211. package/supervisor/service_windows.go +216 -0
  212. package/supervisor/update_state.go +264 -0
  213. package/supervisor/update_state_test.go +306 -0
  214. package/supervisor/updater.go +311 -10
  215. package/supervisor/updater_test.go +64 -0
  216. package/dist/data/memory/quality-scoring.d.ts +0 -32
  217. package/dist/data/memory/quality-scoring.d.ts.map +0 -1
  218. package/dist/data/memory/quality-scoring.js +0 -182
  219. package/dist/data/memory/quality-scoring.js.map +0 -1
  220. package/scripts/install-supervisor.ps1 +0 -67
  221. package/scripts/install-supervisor.sh +0 -43
  222. package/scripts/start-supervisor.ps1 +0 -46
  223. package/scripts/start-supervisor.sh +0 -20
  224. package/templates/coding-task.default.md +0 -12
@@ -1,58 +1,62 @@
1
1
  import { cleanupOldSentMessages } from "./schema.js";
2
- import { getUnconsolidatedEpisodes, markConsolidated } from "./episodes.js";
3
- import { saveSemanticNote, searchSemanticNotesRanked, supersedeNote, archiveNotesForThread, getThreadIdsWithActiveNotes, searchByEmbedding, saveNoteEmbedding, } from "./semantic.js";
2
+ import { archiveNotesForThread, getThreadIdsWithActiveNotes } from "./semantic.js";
4
3
  import { log } from "../../logger.js";
5
- import { resolveKnowledgeThreadId } from "../../config.js";
6
- import { nowISO, repairAndParseJSON } from "./utils.js";
7
- import { chatCompletion, generateEmbedding } from "../../integrations/openai/chat.js";
8
- import { errorMessage } from "../../utils.js";
4
+ import { nowISO } from "./utils.js";
9
5
  import { getAllThreads } from "./thread-registry.js";
10
- // ─── Helpers ─────────────────────────────────────────────────────────────────
11
- function logConsolidation(db, entry) {
6
+ const TERMINAL_THREAD_STATUSES = new Set(["archived", "expired", "exited"]);
7
+ export function getUnconsolidatedThreadIds(db) {
8
+ const rows = db
9
+ .prepare(`SELECT DISTINCT thread_id FROM episodes WHERE consolidated = 0`)
10
+ .all();
11
+ return rows.map((row) => row.thread_id);
12
+ }
13
+ export function logConsolidation(db, entry) {
12
14
  db.prepare(`INSERT INTO meta_consolidation_log
13
15
  (run_at, episodes_processed, notes_created, duration_ms)
14
16
  VALUES (?, ?, ?, ?)`).run(nowISO(), entry.episodesProcessed, entry.notesCreated, entry.durationMs);
15
17
  }
16
- // ─── Write-time deduplication ────────────────────────────────────────────────
17
- const CONSOLIDATION_DEDUP_THRESHOLD = 0.85;
18
- async function checkConsolidationDuplicate(db, content, apiKey, threadId) {
19
- try {
20
- const embedding = await generateEmbedding(content, apiKey);
21
- const matches = searchByEmbedding(db, embedding, {
22
- maxResults: 1,
23
- minSimilarity: CONSOLIDATION_DEDUP_THRESHOLD,
24
- skipAccessTracking: true,
25
- threadId,
26
- });
27
- const match = matches.find((m) => m.similarity >= CONSOLIDATION_DEDUP_THRESHOLD);
28
- if (match) {
29
- return { isDuplicate: true, matchId: match.noteId, similarity: match.similarity, embedding };
30
- }
31
- return { isDuplicate: false, embedding };
32
- }
33
- catch (err) {
34
- log.warn(`[consolidation] Dedup embedding check failed, proceeding without dedup: ${err instanceof Error ? err.message : err}`);
35
- return { isDuplicate: false, embedding: null };
18
+ export function cleanupConsolidationHousekeeping(db) {
19
+ cleanupOldSentMessages(db);
20
+ }
21
+ export function getCandidateNotesForPruning(db, maxNotes) {
22
+ return db.prepare(`
23
+ SELECT * FROM semantic_notes
24
+ WHERE valid_to IS NULL
25
+ AND superseded_by IS NULL
26
+ AND is_guardrail = 0
27
+ AND pinned = 0
28
+ ORDER BY access_count ASC, created_at ASC
29
+ LIMIT ?
30
+ `).all(maxNotes);
31
+ }
32
+ export function hasActiveNote(db, noteId) {
33
+ const row = db.prepare(`SELECT note_id FROM semantic_notes WHERE note_id = ? AND valid_to IS NULL AND superseded_by IS NULL`).get(noteId);
34
+ return Boolean(row);
35
+ }
36
+ export function getActiveNoteContent(db, noteId) {
37
+ const row = db.prepare(`SELECT note_id, content FROM semantic_notes WHERE note_id = ? AND valid_to IS NULL AND superseded_by IS NULL`).get(noteId);
38
+ if (!row)
39
+ return null;
40
+ return { noteId: row.note_id, content: row.content };
41
+ }
42
+ export function expireNote(db, noteId, now) {
43
+ db.prepare(`UPDATE semantic_notes SET valid_to = ?, updated_at = ? WHERE note_id = ?`).run(now, now, noteId);
44
+ }
45
+ export function mergeDuplicateNote(db, keepId, expireId, now, mergedContent) {
46
+ if (mergedContent) {
47
+ db.prepare(`UPDATE semantic_notes SET content = ?, updated_at = ? WHERE note_id = ?`).run(mergedContent, now, keepId);
36
48
  }
49
+ db.prepare(`UPDATE semantic_notes SET valid_to = ?, superseded_by = ?, updated_at = ? WHERE note_id = ?`).run(now, keepId, now, expireId);
37
50
  }
38
- // ─── Orphaned Notes Sweep ────────────────────────────────────────────────────
39
- /** Terminal thread statuses that indicate notes should be expired. */
40
- const TERMINAL_THREAD_STATUSES = new Set(['archived', 'expired', 'exited']);
41
- /**
42
- * Find threads with active semantic notes whose registry status is terminal
43
- * (archived/expired/exited) and expire those notes.
44
- * Returns total number of notes archived.
45
- */
46
- function sweepOrphanedNotes(db) {
51
+ export function sweepOrphanedNotes(db) {
47
52
  const threadIdsWithNotes = getThreadIdsWithActiveNotes(db);
48
53
  if (threadIdsWithNotes.length === 0)
49
54
  return 0;
50
55
  const allThreads = getAllThreads(db);
51
- const threadStatusMap = new Map(allThreads.map(t => [t.threadId, t.status]));
56
+ const threadStatusMap = new Map(allThreads.map((thread) => [thread.threadId, thread.status]));
52
57
  let totalArchived = 0;
53
58
  for (const threadId of threadIdsWithNotes) {
54
59
  const status = threadStatusMap.get(threadId);
55
- // Archive notes for terminal threads AND threads missing from registry
56
60
  if (!status || TERMINAL_THREAD_STATUSES.has(status)) {
57
61
  totalArchived += archiveNotesForThread(db, threadId);
58
62
  }
@@ -62,520 +66,4 @@ function sweepOrphanedNotes(db) {
62
66
  }
63
67
  return totalArchived;
64
68
  }
65
- // ─── Intelligent Consolidation ───────────────────────────────────────────────
66
- // PRIVACY NOTE: This function sends conversation episode excerpts to OpenAI's
67
- // API for knowledge extraction and consolidation. Operators can disable this
68
- // by setting the environment variable CONSOLIDATION_ENABLED=false (or "0").
69
- let consolidationInProgress = false;
70
- /**
71
- * Consolidate ALL threads that have unconsolidated episodes.
72
- * Iterates distinct thread_ids and runs per-thread consolidation for each.
73
- * Returns an aggregated report.
74
- *
75
- * Owns the global `consolidationInProgress` lock so that per-thread calls
76
- * within the loop don't block each other and concurrent invocations are
77
- * properly serialized.
78
- */
79
- export async function runConsolidationAllThreads(db, options) {
80
- if (consolidationInProgress) {
81
- log.info("Consolidation already in progress — skipping (all-threads)");
82
- return {
83
- episodesProcessed: 0,
84
- notesCreated: 0,
85
- durationMs: 0,
86
- details: ["Skipped — consolidation already in progress."],
87
- };
88
- }
89
- consolidationInProgress = true;
90
- try {
91
- const startMs = Date.now();
92
- const threadRows = db
93
- .prepare(`SELECT DISTINCT thread_id FROM episodes WHERE consolidated = 0`)
94
- .all();
95
- if (threadRows.length === 0) {
96
- return {
97
- episodesProcessed: 0,
98
- notesCreated: 0,
99
- durationMs: Date.now() - startMs,
100
- details: ["Nothing to consolidate across any thread."],
101
- };
102
- }
103
- let totalProcessed = 0;
104
- let totalNotes = 0;
105
- const allDetails = [];
106
- for (const { thread_id } of threadRows) {
107
- const report = await runIntelligentConsolidation(db, thread_id, {
108
- maxEpisodes: options?.maxEpisodesPerThread ?? 30,
109
- dryRun: options?.dryRun,
110
- _skipLock: true, // lock is held by this function
111
- });
112
- totalProcessed += report.episodesProcessed;
113
- totalNotes += report.notesCreated;
114
- allDetails.push(`Thread ${thread_id}: ${report.episodesProcessed} eps → ${report.notesCreated} notes`);
115
- }
116
- // Phase 2: Memory pruning — scan for outdated, duplicate, or low-quality notes
117
- try {
118
- const pruneReport = await runMemoryPruning(db);
119
- if (pruneReport.notesExpired + pruneReport.notesMerged > 0) {
120
- allDetails.push(`Pruning: scanned ${pruneReport.notesScanned}, expired ${pruneReport.notesExpired}, merged ${pruneReport.notesMerged}`);
121
- allDetails.push(...pruneReport.details);
122
- }
123
- }
124
- catch (err) {
125
- log.warn(`[memory] Pruning phase failed: ${errorMessage(err)}`);
126
- }
127
- // Phase 3: Orphaned notes sweep — expire notes for dead/archived threads
128
- try {
129
- const orphanedCount = sweepOrphanedNotes(db);
130
- if (orphanedCount > 0) {
131
- allDetails.push(`Orphan sweep: archived ${orphanedCount} notes from dead threads`);
132
- }
133
- }
134
- catch (err) {
135
- log.warn(`[memory] Orphan notes sweep failed: ${errorMessage(err)}`);
136
- }
137
- return {
138
- episodesProcessed: totalProcessed,
139
- notesCreated: totalNotes,
140
- durationMs: Date.now() - startMs,
141
- details: allDetails,
142
- };
143
- }
144
- finally {
145
- consolidationInProgress = false;
146
- }
147
- }
148
- export async function runIntelligentConsolidation(db, threadId, options) {
149
- // Opt-out: allow operators to disable consolidation for privacy reasons
150
- const consolidationEnabled = process.env.CONSOLIDATION_ENABLED;
151
- if (consolidationEnabled === "false" || consolidationEnabled === "0") {
152
- return {
153
- episodesProcessed: 0,
154
- notesCreated: 0,
155
- durationMs: 0,
156
- details: ["Consolidation disabled via CONSOLIDATION_ENABLED env var."],
157
- };
158
- }
159
- const skipLock = options?._skipLock ?? false;
160
- if (!skipLock && consolidationInProgress) {
161
- log.info("Consolidation already in progress — skipping");
162
- return {
163
- episodesProcessed: 0,
164
- notesCreated: 0,
165
- durationMs: 0,
166
- details: ["Skipped — consolidation already in progress."],
167
- };
168
- }
169
- if (!skipLock)
170
- consolidationInProgress = true;
171
- try {
172
- const startMs = Date.now();
173
- const maxEpisodes = options?.maxEpisodes ?? 30;
174
- const dryRun = options?.dryRun ?? false;
175
- const episodes = getUnconsolidatedEpisodes(db, threadId, maxEpisodes);
176
- if (episodes.length === 0) {
177
- return {
178
- episodesProcessed: 0,
179
- notesCreated: 0,
180
- durationMs: Date.now() - startMs,
181
- details: ["Nothing to consolidate."],
182
- };
183
- }
184
- // Format episodes for the prompt
185
- const episodesText = episodes
186
- .map((ep, i) => {
187
- const c = ep.content;
188
- let content;
189
- if (c && typeof c === "object") {
190
- content = c.text ?? c.caption ?? JSON.stringify(c);
191
- }
192
- else {
193
- content = JSON.stringify(ep.content);
194
- }
195
- return `[${i + 1}] (${ep.type}/${ep.modality}, ${ep.timestamp}) ${content}`;
196
- })
197
- .join("\n");
198
- // ── Contradiction detection: find existing notes related to these episodes ──
199
- // Extract keywords from episodes to search for potentially conflicting notes
200
- const episodeWords = episodesText.toLowerCase()
201
- .replace(/[^a-z0-9\s]/g, " ")
202
- .split(/\s+/)
203
- .filter(w => w.length > 3);
204
- const wordFreq = new Map();
205
- const stopWords = new Set(["this", "that", "with", "from", "have", "been", "will", "would", "could", "should", "about", "there", "their", "which", "when", "what", "were", "they", "than", "then", "also", "just", "more", "some", "into", "over", "after", "before", "other", "very", "your", "here"]);
206
- for (const w of episodeWords) {
207
- if (!stopWords.has(w))
208
- wordFreq.set(w, (wordFreq.get(w) ?? 0) + 1);
209
- }
210
- const topKeywords = [...wordFreq.entries()]
211
- .sort((a, b) => b[1] - a[1])
212
- .slice(0, 12)
213
- .map(([w]) => w);
214
- let existingNotesSection = "";
215
- if (topKeywords.length > 0) {
216
- try {
217
- const related = searchSemanticNotesRanked(db, topKeywords.join(" "), {
218
- maxResults: 15,
219
- skipAccessTracking: true,
220
- minMatchRatio: 0.2, // broader recall for contradiction scan
221
- });
222
- if (related.length > 0) {
223
- existingNotesSection = `\n\nExisting memory notes (potentially related):
224
- ${related.map(n => `[${n.noteId}] (${n.type}, conf: ${n.confidence}) ${n.content}`).join("\n")}`;
225
- }
226
- }
227
- catch (err) {
228
- log.warn(`[consolidation] searchSemanticNotesRanked failed during contradiction scan: ${errorMessage(err)}`);
229
- }
230
- }
231
- const systemPrompt = `You are a memory consolidation agent. Analyze these conversation episodes and extract knowledge that should be remembered across sessions.
232
-
233
- Episodes:
234
- ${episodesText}${existingNotesSection}
235
-
236
- Output a JSON object with:
237
- {
238
- "notes": [
239
- {
240
- "type": "fact" | "preference" | "pattern" | "entity" | "relationship",
241
- "content": "One clear sentence describing the knowledge",
242
- "keywords": ["keyword1", "keyword2", "keyword3"],
243
- "confidence": 0.0-1.0,
244
- "priority": 0 | 1 | 2
245
- }
246
- ],
247
- "supersede": [
248
- {
249
- "oldNoteId": "sn_xxx",
250
- "reason": "Why the old note is outdated/contradicted",
251
- "newContent": "Updated version of the knowledge",
252
- "type": "fact",
253
- "keywords": ["keyword1", "keyword2"],
254
- "confidence": 0.8,
255
- "priority": 0 | 1 | 2
256
- }
257
- ]
258
- }
259
-
260
- Rules:
261
- - Only extract information that would be useful in future sessions
262
- - Preferences are stronger signals than facts (confidence: 0.9)
263
- - Do not extract trivial/transient information
264
- - If the operator corrected the agent, extract the correction as a preference
265
- - Focus on: operator name, preferences, communication style, technical choices, project context
266
- - CRITICAL: Check existing notes for CONTRADICTIONS. If a new episode contradicts or updates an existing note, add a "supersede" entry. The new episodes represent MORE RECENT information.
267
- - Common contradictions: decisions changed, projects completed/abandoned, preferences updated, tools/tech switched
268
- - PRIORITY DETECTION: Infer priority from the operator's language and emotional investment:
269
- - priority 2 (high importance): operator says "important", "crucial", "I really need", "don't forget", shows strong emotional investment, repeated emphasis
270
- - priority 1 (notable): operator says "would be nice", "I'd like", "should", mentions something multiple times across conversations
271
- - priority 0 (normal): default for routine facts, observations, patterns
272
- - Return {"notes": [], "supersede": []} if nothing notable`;
273
- let notesCreated = 0;
274
- const details = [];
275
- try {
276
- const apiKey = process.env.OPENAI_API_KEY;
277
- if (!apiKey) {
278
- throw new Error("OPENAI_API_KEY not set");
279
- }
280
- const messages = [
281
- { role: "system", content: systemPrompt },
282
- { role: "user", content: "Extract knowledge from the episodes above." },
283
- ];
284
- const raw = await chatCompletion(messages, apiKey, {
285
- model: process.env.CONSOLIDATION_MODEL ?? "gpt-4o-mini",
286
- maxTokens: 4096,
287
- responseFormat: { type: "json_object" },
288
- timeoutMs: 60_000,
289
- });
290
- const parsed = repairAndParseJSON(raw);
291
- const extractedNotes = parsed.notes ?? [];
292
- const supersedeActions = parsed.supersede ?? [];
293
- const episodeIds = episodes.map((ep) => ep.episodeId);
294
- if (!dryRun) {
295
- const knowledgeThreadId = resolveKnowledgeThreadId(threadId);
296
- for (const note of extractedNotes) {
297
- if (typeof note.content !== 'string' || note.content.length === 0 || note.content.length >= 2000) {
298
- log.warn(`[consolidation] Skipping note with invalid content (type=${typeof note.content}, len=${typeof note.content === 'string' ? note.content.length : 'N/A'})`);
299
- continue;
300
- }
301
- const validTypes = ["fact", "preference", "pattern", "entity", "relationship"];
302
- const noteType = validTypes.includes(note.type)
303
- ? note.type
304
- : "fact";
305
- // Write-time dedup: skip notes that are too similar to existing ones
306
- const dedup = await checkConsolidationDuplicate(db, note.content, apiKey, knowledgeThreadId);
307
- if (dedup.isDuplicate) {
308
- log.debug(`[consolidation] Dedup: skipping note similar to ${dedup.matchId} at ${dedup.similarity?.toFixed(3)}`);
309
- continue;
310
- }
311
- const noteId = saveSemanticNote(db, {
312
- type: noteType,
313
- content: note.content,
314
- keywords: Array.isArray(note.keywords) ? note.keywords : [],
315
- confidence: Math.max(0, Math.min(1, note.confidence ?? 0.5)),
316
- priority: Math.max(0, Math.min(2, note.priority ?? 0)),
317
- threadId: knowledgeThreadId,
318
- sourceEpisodes: episodeIds,
319
- });
320
- // Persist the embedding computed during dedup check
321
- if (dedup.embedding) {
322
- saveNoteEmbedding(db, noteId, dedup.embedding);
323
- }
324
- notesCreated++;
325
- details.push(`[${noteType}] ${note.content}`);
326
- }
327
- // Execute supersede actions — resolve contradictions with existing notes
328
- let supersededCount = 0;
329
- for (const action of supersedeActions) {
330
- if (!action.oldNoteId || !action.newContent)
331
- continue;
332
- // Verify old note exists and is still active
333
- const oldNote = db.prepare(`SELECT note_id FROM semantic_notes WHERE note_id = ? AND valid_to IS NULL AND superseded_by IS NULL`).get(action.oldNoteId);
334
- if (!oldNote) {
335
- details.push(`[skip-supersede] ${action.oldNoteId} not found or already superseded`);
336
- continue;
337
- }
338
- try {
339
- const validTypes = ["fact", "preference", "pattern", "entity", "relationship"];
340
- const noteType = validTypes.includes(action.type) ? action.type : "fact";
341
- const newId = supersedeNote(db, action.oldNoteId, {
342
- type: noteType,
343
- content: action.newContent,
344
- keywords: Array.isArray(action.keywords) ? action.keywords : [],
345
- confidence: Math.max(0, Math.min(1, action.confidence ?? 0.8)),
346
- priority: Math.max(0, Math.min(2, action.priority ?? 0)),
347
- sourceEpisodes: episodeIds,
348
- });
349
- supersededCount++;
350
- details.push(`[supersede] ${action.oldNoteId} → ${newId}: ${action.reason}`);
351
- }
352
- catch (err) {
353
- details.push(`[supersede-error] ${action.oldNoteId}: ${errorMessage(err)}`);
354
- }
355
- }
356
- if (supersededCount > 0) {
357
- log.info(`[memory] Contradiction resolution: superseded ${supersededCount} outdated note(s)`);
358
- }
359
- // Mark episodes as consolidated
360
- markConsolidated(db, episodeIds);
361
- // Log the consolidation
362
- logConsolidation(db, {
363
- episodesProcessed: episodes.length,
364
- notesCreated: notesCreated + supersededCount,
365
- durationMs: Date.now() - startMs,
366
- });
367
- }
368
- else {
369
- for (const note of extractedNotes) {
370
- details.push(`[dry-run] [${note.type}] ${note.content}`);
371
- notesCreated++;
372
- }
373
- for (const action of supersedeActions) {
374
- details.push(`[dry-run] [supersede] ${action.oldNoteId} → ${action.reason}`);
375
- }
376
- }
377
- }
378
- catch (err) {
379
- // Do NOT mark episodes as consolidated on failure — they should be
380
- // retried on the next consolidation run. Previously this was a silent
381
- // data-loss bug: a transient OpenAI outage would permanently lose the
382
- // episodes' knowledge without extracting anything.
383
- const msg = errorMessage(err);
384
- log.error(`[memory] Intelligent consolidation failed (episodes NOT marked): ${msg}`);
385
- details.push(`Consolidation failed (will retry): ${msg}`);
386
- }
387
- // Housekeeping: clean up old sent_messages entries (>7 days)
388
- cleanupOldSentMessages(db);
389
- return {
390
- episodesProcessed: episodes.length,
391
- notesCreated,
392
- durationMs: Date.now() - startMs,
393
- details,
394
- };
395
- }
396
- finally {
397
- if (!skipLock)
398
- consolidationInProgress = false;
399
- }
400
- }
401
- /**
402
- * Scan a batch of existing semantic notes for quality issues:
403
- * - Outdated facts that no longer reflect reality
404
- * - Duplicate notes conveying the same information
405
- * - Low-quality notes that are vague or non-actionable
406
- *
407
- * Designed to run as a post-consolidation phase, gradually cleaning
408
- * the memory DB over successive runs.
409
- */
410
- export async function runMemoryPruning(db, options) {
411
- const pruningEnabled = process.env.PRUNING_ENABLED;
412
- if (pruningEnabled === "false" || pruningEnabled === "0") {
413
- return {
414
- notesScanned: 0, notesExpired: 0, notesMerged: 0, durationMs: 0,
415
- details: ["Pruning disabled via PRUNING_ENABLED env var."],
416
- };
417
- }
418
- const apiKey = process.env.OPENAI_API_KEY;
419
- if (!apiKey) {
420
- return {
421
- notesScanned: 0, notesExpired: 0, notesMerged: 0, durationMs: 0,
422
- details: ["Pruning skipped — OPENAI_API_KEY not set."],
423
- };
424
- }
425
- const startMs = Date.now();
426
- const envSampleSize = parseInt(process.env.PRUNING_SAMPLE_SIZE ?? "", 10);
427
- const maxNotes = options?.maxNotes ?? (Number.isFinite(envSampleSize) && envSampleSize > 0 ? envSampleSize : 30);
428
- const dryRun = options?.dryRun ?? false;
429
- const details = [];
430
- // Sample candidate notes: lowest access count first, then oldest.
431
- // Guardrails and pinned notes are excluded — they're intentionally permanent.
432
- const candidates = db.prepare(`
433
- SELECT * FROM semantic_notes
434
- WHERE valid_to IS NULL
435
- AND superseded_by IS NULL
436
- AND is_guardrail = 0
437
- AND pinned = 0
438
- ORDER BY access_count ASC, created_at ASC
439
- LIMIT ?
440
- `).all(maxNotes);
441
- if (candidates.length < 5) {
442
- return {
443
- notesScanned: 0, notesExpired: 0, notesMerged: 0,
444
- durationMs: Date.now() - startMs,
445
- details: ["Too few candidate notes for pruning (< 5)."],
446
- };
447
- }
448
- const now = new Date();
449
- const notesText = candidates.map((row) => {
450
- const ageMs = now.getTime() - new Date(row.created_at).getTime();
451
- const ageDays = Math.round(ageMs / (1000 * 60 * 60 * 24));
452
- return `[${row.note_id}] (${row.type}, conf: ${row.confidence}, accesses: ${row.access_count}, age: ${ageDays}d) ${row.content}`;
453
- }).join("\n");
454
- const systemPrompt = `You are a memory quality manager. Review these semantic notes and identify problems.
455
-
456
- Notes to review:
457
- ${notesText}
458
-
459
- Output a JSON object:
460
- {
461
- "expire": [
462
- { "noteId": "sn_xxx", "reason": "Brief explanation" }
463
- ],
464
- "merge": [
465
- { "keepId": "sn_xxx", "expireId": "sn_yyy", "mergedContent": "Best combined version", "reason": "Brief explanation" }
466
- ]
467
- }
468
-
469
- Decision criteria:
470
- - EXPIRE if: the note is vague/non-actionable ("the system has issues"), trivially obvious, or clearly outdated (references old versions, completed tasks still marked pending)
471
- - MERGE if: two or more notes convey the same information — keep the more specific/accurate one. Provide mergedContent combining the best of both when useful.
472
- - KEEP (omit from output) if: the note contains unique, specific, actionable information
473
-
474
- Be conservative — when in doubt, keep the note. Only expire notes you are confident are low-value.
475
- Do NOT expire preferences or guardrails unless they clearly contradict each other.
476
- Return {"expire": [], "merge": []} if nothing needs pruning.`;
477
- try {
478
- const messages = [
479
- { role: "system", content: systemPrompt },
480
- { role: "user", content: "Review the notes above and identify any that should be expired or merged." },
481
- ];
482
- const raw = await chatCompletion(messages, apiKey, {
483
- model: process.env.CONSOLIDATION_MODEL ?? "gpt-4o-mini",
484
- maxTokens: 4096,
485
- responseFormat: { type: "json_object" },
486
- timeoutMs: 60_000,
487
- });
488
- const parsed = repairAndParseJSON(raw);
489
- if (parsed == null || typeof parsed !== "object" || Array.isArray(parsed)) {
490
- throw new Error(`LLM returned non-object response: ${typeof parsed}`);
491
- }
492
- const result = parsed;
493
- const expireActions = (Array.isArray(result.expire) ? result.expire : []);
494
- const mergeActions = (Array.isArray(result.merge) ? result.merge : []);
495
- // Only allow actions targeting notes in the candidate set — prevents
496
- // LLM hallucinations from expiring pinned/guardrail notes outside the batch.
497
- const candidateIds = new Set(candidates.map((c) => c.note_id));
498
- let expired = 0;
499
- let merged = 0;
500
- const nowStr = nowISO();
501
- if (!dryRun) {
502
- // Wrap all mutations in a transaction so partial failures roll back.
503
- db.transaction(() => {
504
- for (const action of expireActions) {
505
- if (!action.noteId)
506
- continue;
507
- if (!candidateIds.has(action.noteId)) {
508
- details.push(`[skip-expire] ${action.noteId} not in candidate set`);
509
- continue;
510
- }
511
- const exists = db.prepare(`SELECT note_id FROM semantic_notes WHERE note_id = ? AND valid_to IS NULL`).get(action.noteId);
512
- if (!exists) {
513
- details.push(`[skip-expire] ${action.noteId} not found or already expired`);
514
- continue;
515
- }
516
- db.prepare(`UPDATE semantic_notes SET valid_to = ?, updated_at = ? WHERE note_id = ?`).run(nowStr, nowStr, action.noteId);
517
- expired++;
518
- details.push(`[pruned] ${action.noteId}: ${action.reason}`);
519
- }
520
- for (const action of mergeActions) {
521
- if (!action.keepId || !action.expireId)
522
- continue;
523
- if (action.keepId === action.expireId) {
524
- details.push(`[skip-merge] ${action.keepId}: keepId === expireId`);
525
- continue;
526
- }
527
- if (!candidateIds.has(action.keepId) || !candidateIds.has(action.expireId)) {
528
- details.push(`[skip-merge] ${action.keepId}/${action.expireId} not in candidate set`);
529
- continue;
530
- }
531
- const keepNote = db.prepare(`SELECT note_id, content FROM semantic_notes WHERE note_id = ? AND valid_to IS NULL`).get(action.keepId);
532
- const expireNote = db.prepare(`SELECT note_id FROM semantic_notes WHERE note_id = ? AND valid_to IS NULL`).get(action.expireId);
533
- if (!keepNote || !expireNote) {
534
- details.push(`[skip-merge] ${action.keepId}/${action.expireId} not found`);
535
- continue;
536
- }
537
- // Update kept note with merged content when provided
538
- if (action.mergedContent && action.mergedContent !== keepNote.content) {
539
- db.prepare(`UPDATE semantic_notes SET content = ?, updated_at = ? WHERE note_id = ?`).run(action.mergedContent, nowStr, keepNote.note_id);
540
- }
541
- // Expire the duplicate, linking to the kept note
542
- db.prepare(`UPDATE semantic_notes SET valid_to = ?, superseded_by = ?, updated_at = ? WHERE note_id = ?`).run(nowStr, keepNote.note_id, nowStr, action.expireId);
543
- merged++;
544
- details.push(`[merged] ${action.expireId} → ${action.keepId}: ${action.reason}`);
545
- }
546
- })();
547
- }
548
- else {
549
- for (const action of expireActions) {
550
- details.push(`[dry-run] [prune] ${action.noteId}: ${action.reason}`);
551
- expired++;
552
- }
553
- for (const action of mergeActions) {
554
- details.push(`[dry-run] [merge] ${action.expireId} → ${action.keepId}: ${action.reason}`);
555
- merged++;
556
- }
557
- }
558
- if (expired + merged > 0) {
559
- log.info(`[memory] Pruning: expired ${expired}, merged ${merged} note(s)`);
560
- }
561
- return {
562
- notesScanned: candidates.length,
563
- notesExpired: expired,
564
- notesMerged: merged,
565
- durationMs: Date.now() - startMs,
566
- details,
567
- };
568
- }
569
- catch (err) {
570
- const msg = errorMessage(err);
571
- log.error(`[memory] Pruning failed: ${msg}`);
572
- return {
573
- notesScanned: candidates.length,
574
- notesExpired: 0,
575
- notesMerged: 0,
576
- durationMs: Date.now() - startMs,
577
- details: [`Pruning failed: ${msg}`],
578
- };
579
- }
580
- }
581
69
  //# sourceMappingURL=consolidation.js.map