total-recall-brain 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +173 -0
  3. package/bin/antigravity.mjs +212 -0
  4. package/bin/init.mjs +36 -0
  5. package/bin/oci-sniper.sh +69 -0
  6. package/bin/total-recall.mjs +229 -0
  7. package/docs/ARCHITECTURE.md +251 -0
  8. package/docs/IDE_INTEGRATION.md +112 -0
  9. package/package.json +71 -0
  10. package/scaffold/.agent/skills/total-recall/SKILL.md +195 -0
  11. package/scaffold/.agent/skills/total-recall/evals/evals.json +20 -0
  12. package/scaffold/.agent/skills/total-recall/memory-vault/concepts/cli-help-reference.md +145 -0
  13. package/scaffold/.agent/skills/total-recall/memory-vault/invariants/operating-instructions.md +75 -0
  14. package/scaffold/.agent/skills/total-recall/memory-vault/preferences/always-websearch-gap.md +16 -0
  15. package/scaffold/.agent/skills/total-recall/memory-vault/preferences/topic-research-sop.md +17 -0
  16. package/scaffold/.agent/skills/total-recall/references/api-reference.md +187 -0
  17. package/scaffold/.agent/skills/total-recall/references/architecture-reference.md +81 -0
  18. package/scaffold/.agent/skills/total-recall/references/cli-reference.md +103 -0
  19. package/scaffold/.agent/skills/total-recall/references/ssss-reference.md +88 -0
  20. package/scaffold/.agent/skills/total-recall/references/troubleshooting.md +68 -0
  21. package/scaffold/.agent/skills/total-recall/scripts/sync-repo.mjs +83 -0
  22. package/scaffold/.agent/skills/total-recall/skills/cli-agents/SKILL.md +122 -0
  23. package/scaffold/.agent/skills/total-recall/skills/cli-agents/agents.yml +27 -0
  24. package/scaffold/.agent/skills/total-recall/skills/cli-agents/references/antigravity.txt +20 -0
  25. package/scaffold/.agent/skills/total-recall/skills/cli-agents/references/claude.txt +58 -0
  26. package/scaffold/.agent/skills/total-recall/skills/cli-agents/references/codex.txt +81 -0
  27. package/scaffold/.agent/skills/total-recall/skills/cli-agents/references/gemini.txt +44 -0
  28. package/scaffold/.agent/skills/total-recall/skills/research/SKILL.md +337 -0
  29. package/scaffold/.agent/skills/total-recall/skills/skill/SKILL.md +281 -0
  30. package/scaffold/.agent/skills/total-recall/skills/skill/assets/skill-template/SKILL.md +20 -0
  31. package/scaffold/.agent/skills/total-recall/skills/skill/evals/evals.json +20 -0
  32. package/scaffold/.agent/skills/total-recall/skills/skill/hooks/on-skill-add.sh +14 -0
  33. package/scaffold/.agent/skills/total-recall/skills/skill/hooks/on-skill-delete.sh +14 -0
  34. package/scaffold/.agent/skills/total-recall/skills/skill/learnings/.gitkeep +0 -0
  35. package/scaffold/.agent/skills/total-recall/skills/skill/references/.gitkeep +0 -0
  36. package/scaffold/.agent/skills/total-recall/skills/skill/scripts/.gitkeep +0 -0
  37. package/scaffold/.agent/skills/total-recall/skills/skill/scripts/create-skill.sh +65 -0
  38. package/scaffold/.agent/skills/total-recall/skills/skill/scripts/enforce-skill-optimization.mjs +135 -0
  39. package/scaffold/.agent/skills/total-recall/skills/skill/scripts/find-skills.mjs +124 -0
  40. package/scaffold/.agent/skills/total-recall/skills/skill/scripts/scan-skill.mjs +207 -0
  41. package/scaffold/.agent/skills/total-recall/skills/skill/scripts/skill-watcher.mjs +64 -0
  42. package/scaffold/.agent/skills/total-recall/skills/skill/subagents/audit-all-skills.md +62 -0
  43. package/scaffold/.agent/skills/total-recall/skills/ssss/SKILL.md +27 -0
  44. package/scaffold/.agent/skills/total-recall/skills/ssss/evals/evals.json +17 -0
  45. package/scaffold/.agent/skills/total-recall/skills/ssss/references/ssss-spec.md +117 -0
  46. package/scaffold/.agent/skills/total-recall/skills/ssss/scripts/validate-schema.mjs +54 -0
  47. package/scaffold/.agent/skills/total-recall/skills/ssss/subagents/audit-ssss-node.md +9 -0
  48. package/scaffold/.agent/skills/total-recall/subagents/total-recall-diagnostician.md +13 -0
  49. package/src/cli/agent-dir.mjs +105 -0
  50. package/src/cli/backup.mjs +313 -0
  51. package/src/cli/brain.mjs +224 -0
  52. package/src/cli/chat.mjs +74 -0
  53. package/src/cli/config.mjs +247 -0
  54. package/src/cli/connect.mjs +872 -0
  55. package/src/cli/daemon.mjs +228 -0
  56. package/src/cli/deploy-ui.mjs +1793 -0
  57. package/src/cli/deploy.mjs +1726 -0
  58. package/src/cli/doctor.mjs +254 -0
  59. package/src/cli/dream.mjs +80 -0
  60. package/src/cli/forget.mjs +92 -0
  61. package/src/cli/friction.mjs +21 -0
  62. package/src/cli/generate-pat.mjs +65 -0
  63. package/src/cli/hash-password.mjs +22 -0
  64. package/src/cli/help.mjs +153 -0
  65. package/src/cli/import-rules.mjs +102 -0
  66. package/src/cli/ingest/index.mjs +318 -0
  67. package/src/cli/ingest/parsers/chrome-bookmarks.mjs +238 -0
  68. package/src/cli/ingest/parsers/google-keep.mjs +147 -0
  69. package/src/cli/ingest/parsers/search-history.mjs +142 -0
  70. package/src/cli/ingest/parsers/youtube-history.mjs +181 -0
  71. package/src/cli/ingest/utils/dedup.mjs +69 -0
  72. package/src/cli/ingest/utils/takeout-walker.mjs +101 -0
  73. package/src/cli/ingest.mjs +102 -0
  74. package/src/cli/init.mjs +736 -0
  75. package/src/cli/integration-dispatcher.mjs +128 -0
  76. package/src/cli/lint.mjs +166 -0
  77. package/src/cli/map.mjs +194 -0
  78. package/src/cli/migrate.mjs +87 -0
  79. package/src/cli/rebuild.mjs +88 -0
  80. package/src/cli/recall.mjs +219 -0
  81. package/src/cli/relay.mjs +518 -0
  82. package/src/cli/remember.mjs +306 -0
  83. package/src/cli/research.mjs +381 -0
  84. package/src/cli/reset-password.mjs +75 -0
  85. package/src/cli/restore.mjs +135 -0
  86. package/src/cli/setup.mjs +616 -0
  87. package/src/cli/share.mjs +180 -0
  88. package/src/cli/skill.mjs +313 -0
  89. package/src/cli/snapshot.mjs +108 -0
  90. package/src/cli/start.mjs +63 -0
  91. package/src/cli/status.mjs +296 -0
  92. package/src/cli/sync.mjs +383 -0
  93. package/src/cli/uninstall.mjs +613 -0
  94. package/src/cli/upgrade.mjs +141 -0
  95. package/src/cli/wizard.html +3682 -0
  96. package/src/core/blackboard.mjs +54 -0
  97. package/src/core/clarity-rewriter.mjs +770 -0
  98. package/src/core/conclusion-writer.mjs +259 -0
  99. package/src/core/config.mjs +277 -0
  100. package/src/core/conflict-detector.mjs +536 -0
  101. package/src/core/crypto.mjs +107 -0
  102. package/src/core/daemon-control.mjs +158 -0
  103. package/src/core/daemon-loop.mjs +364 -0
  104. package/src/core/dream.mjs +347 -0
  105. package/src/core/drift-detector.mjs +143 -0
  106. package/src/core/embeddings.mjs +632 -0
  107. package/src/core/emergency-alerts.mjs +137 -0
  108. package/src/core/evolution.mjs +107 -0
  109. package/src/core/fact-seeker.mjs +1602 -0
  110. package/src/core/friction.mjs +95 -0
  111. package/src/core/import-rules.mjs +308 -0
  112. package/src/core/inference-engine.mjs +367 -0
  113. package/src/core/logger.mjs +75 -0
  114. package/src/core/memory-layers.mjs +96 -0
  115. package/src/core/migrate.mjs +116 -0
  116. package/src/core/notifications.mjs +77 -0
  117. package/src/core/operation-validator.mjs +272 -0
  118. package/src/core/optimizer.mjs +330 -0
  119. package/src/core/pattern_detector.mjs +113 -0
  120. package/src/core/post-mortem.mjs +413 -0
  121. package/src/core/protect-instructions.mjs +148 -0
  122. package/src/core/quick-capture.mjs +145 -0
  123. package/src/core/research-queue.mjs +431 -0
  124. package/src/core/research.mjs +461 -0
  125. package/src/core/runtime.mjs +515 -0
  126. package/src/core/sandbox.mjs +163 -0
  127. package/src/core/scheduler.mjs +569 -0
  128. package/src/core/schema.mjs +333 -0
  129. package/src/core/search.mjs +219 -0
  130. package/src/core/semantic-index.mjs +276 -0
  131. package/src/core/session-watcher.mjs +850 -0
  132. package/src/core/snapshot.mjs +133 -0
  133. package/src/core/source-adapters.mjs +1038 -0
  134. package/src/core/steering.mjs +137 -0
  135. package/src/core/surface.mjs +444 -0
  136. package/src/core/task_runner.mjs +125 -0
  137. package/src/core/tts.mjs +142 -0
  138. package/src/core/usage-tracker.mjs +349 -0
  139. package/src/core/vault-cache.mjs +76 -0
  140. package/src/core/vault.mjs +273 -0
  141. package/src/core/watchdog.mjs +236 -0
  142. package/src/server/api.mjs +944 -0
  143. package/src/server/auth.mjs +410 -0
  144. package/src/server/index.mjs +868 -0
  145. package/src/server/keys.mjs +181 -0
  146. package/src/server/rest.mjs +1652 -0
  147. package/src/server/route-manifest.json +77 -0
  148. package/src/server/routes/_shared.mjs +93 -0
  149. package/src/server/routes/keys.mjs +55 -0
  150. package/src/server/routes/memory.mjs +249 -0
  151. package/src/server/routes/sessions.mjs +336 -0
  152. package/src/server/routes/share.mjs +138 -0
  153. package/src/server/tools.mjs +618 -0
  154. package/templates/Caddyfile +37 -0
  155. package/templates/com.totalrecall.backup.plist +42 -0
  156. package/templates/com.totalrecall.daemon.plist +40 -0
  157. package/templates/com.totalrecall.duckdns.plist +23 -0
  158. package/templates/com.totalrecall.server.plist +46 -0
  159. package/templates/default-config/budget.yml +6 -0
  160. package/templates/default-config/runtime.yml +11 -0
  161. package/templates/default-config/security.yml +53 -0
  162. package/templates/default-config/voice.yml +34 -0
  163. package/templates/obsidian-queries/conflicts.md +26 -0
  164. package/templates/obsidian-queries/daily.md +17 -0
  165. package/templates/obsidian-queries/index.md +38 -0
  166. package/templates/obsidian-queries/skills.md +14 -0
  167. package/templates/onboarding-interview.md +72 -0
  168. package/templates/skill_engineering.md +40 -0
  169. package/templates/total-recall-daemon.service +21 -0
  170. package/templates/total-recall-server.service +20 -0
@@ -0,0 +1,77 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { promisify } from 'node:util';
3
+ import { logger } from './logger.mjs';
4
+
5
+ const execFileAsync = promisify(execFile);
6
+
7
+ /**
8
+ * Sends a macOS desktop notification using terminal-notifier if available,
9
+ * falling back to osascript.
10
+ *
11
+ * @param {string} title - The notification title
12
+ * @param {string} message - The notification message content
13
+ * @param {object} [options] - Additional parameters
14
+ * @param {string} [options.open] - Absolute path or URL to open on clicking "Open" or "Show"
15
+ * @param {string} [options.sound] - macOS sound name (e.g., 'Glass', 'Hero', 'Sosumi', 'default')
16
+ * @param {string} [options.subtitle] - Subtitle text to show
17
+ * @param {string} [options.group] - Group ID to allow notification replacement rather than stacking
18
+ */
19
+ export async function sendSystemNotification(title, message, options = {}) {
20
+ // Bypassed in tests
21
+ const isTest = process.env.VITEST || process.env._TR_TEST_AGENT_DIR || process.env.NODE_ENV === 'test';
22
+ if (isTest) {
23
+ logger.debug('notifications: bypassing system notification in test environment');
24
+ return;
25
+ }
26
+
27
+ const {
28
+ open,
29
+ sound = 'default',
30
+ subtitle = 'Total Recall',
31
+ group = 'total-recall-system'
32
+ } = options;
33
+
34
+ // Sanitize inputs to prevent unexpected arguments/escaping issues
35
+ const cleanMessage = String(message).replace(/[^ -~]/g, '').replace(/["'`$\\]/g, '').slice(0, 200);
36
+ const cleanTitle = String(title).replace(/[^ -~]/g, '').replace(/["'`$\\]/g, '').slice(0, 100);
37
+ const cleanSubtitle = subtitle ? String(subtitle).replace(/[^ -~]/g, '').replace(/["'`$\\]/g, '').slice(0, 100) : '';
38
+
39
+ try {
40
+ // 1. Try terminal-notifier first (supports click actions, custom sounds, subtitles, grouping)
41
+ const args = [
42
+ '-title', cleanTitle,
43
+ '-message', cleanMessage,
44
+ '-group', group,
45
+ ];
46
+ if (cleanSubtitle) {
47
+ args.push('-subtitle', cleanSubtitle);
48
+ }
49
+ if (open) {
50
+ // Ensure file url is correct format or HTTP(S) URL
51
+ const openTarget = (open.startsWith('/') || open.startsWith('~')) ? `file://${open}` : open;
52
+ args.push('-open', openTarget);
53
+ args.push('-actions', 'Open');
54
+ }
55
+ if (sound && sound !== 'default') {
56
+ args.push('-sound', sound);
57
+ }
58
+
59
+ await execFileAsync('terminal-notifier', args, { timeout: 3000 });
60
+ logger.debug('notifications: successfully sent notification via terminal-notifier', { title, message });
61
+ } catch (err) {
62
+ // 2. Fall back to plain AppleScript display notification if terminal-notifier fails
63
+ try {
64
+ const appleScriptParts = [
65
+ `display notification "${cleanMessage.replace(/"/g, '\\"')}"`,
66
+ `with title "${cleanTitle.replace(/"/g, '\\"')}"`
67
+ ];
68
+ if (sound && sound !== 'default') {
69
+ appleScriptParts.push(`sound name "${sound}"`);
70
+ }
71
+ await execFileAsync('osascript', ['-e', appleScriptParts.join(' ')], { timeout: 3000 });
72
+ logger.debug('notifications: successfully sent fallback osascript notification', { title, message });
73
+ } catch (fallbackErr) {
74
+ logger.debug('notifications: both terminal-notifier and osascript failed', { err: fallbackErr.message });
75
+ }
76
+ }
77
+ }
@@ -0,0 +1,272 @@
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import matter from 'gray-matter';
4
+ import crypto from 'node:crypto';
5
+ import {
6
+ SSSS_SCHEMAS,
7
+ OperationEnvelopeSchema,
8
+ PatchEnvelopeSchema,
9
+ EventEnvelopeSchema,
10
+ } from './schema.mjs';
11
+ import { atomicWrite, safeStringify } from './vault.mjs';
12
+ import { logger } from './logger.mjs';
13
+
14
+ const APPEND_TYPES = new Set(['conversation', 'run']);
15
+ const PROTOCOL_PATHS = [
16
+ 'references/ssss-spec.md',
17
+ 'references/admin-protocol-evolution-policy.md',
18
+ 'references/user-local-optimizer-boundary-policy.md',
19
+ ];
20
+ const PROTOCOL_PATH_PREFIXES = ['fixtures/'];
21
+ const idempotencyCache = new Map();
22
+ const IDEMPOTENCY_TTL_MS = 10 * 60 * 1000;
23
+ const warmedVaults = new Set();
24
+
25
+ function cacheKey(wid, key) { return `${wid}:${key}`; }
26
+
27
+ function pruneExpired() {
28
+ const now = Date.now();
29
+ for (const [k, v] of idempotencyCache) {
30
+ if (now - v.ts > IDEMPOTENCY_TTL_MS) idempotencyCache.delete(k);
31
+ }
32
+ }
33
+
34
+ export function warmIdempotencyCache(vaultRoot, eventLogDir) {
35
+ const vaultKey = path.resolve(vaultRoot);
36
+ if (warmedVaults.has(vaultKey)) return;
37
+ warmedVaults.add(vaultKey);
38
+
39
+ const auditDir = eventLogDir || path.join(vaultRoot, '.events');
40
+ const auditFile = path.join(auditDir, 'audit.jsonl');
41
+ if (!fs.existsSync(auditFile)) return;
42
+
43
+ try {
44
+ const lines = fs.readFileSync(auditFile, 'utf8').split('\n');
45
+ const now = Date.now();
46
+ for (const line of lines) {
47
+ if (!line.trim()) continue;
48
+ try {
49
+ const record = JSON.parse(line);
50
+ if (record.event_type !== 'audit' || !record.payload) continue;
51
+ const ts = Date.parse(record.ts);
52
+ if (isNaN(ts) || now - ts > IDEMPOTENCY_TTL_MS) continue;
53
+
54
+ const ck = cacheKey(record.payload.workspace_id, record.payload.idempotency_key);
55
+ const response = {
56
+ success: true,
57
+ type: record.payload.envelope_type,
58
+ operation_id: record.correlation_id,
59
+ path: record.subject,
60
+ committed_at: record.ts,
61
+ validation: {
62
+ valid: true,
63
+ type: record.payload.resolved_type,
64
+ errors: [],
65
+ warnings: []
66
+ }
67
+ };
68
+ idempotencyCache.set(ck, { response, ts });
69
+ } catch { /* skip corrupt lines */ }
70
+ }
71
+ } catch (err) {
72
+ logger.error('operation-validator', `Failed to warm idempotency cache: ${err.message}`);
73
+ }
74
+ }
75
+
76
+ function isProtocolPath(fp) {
77
+ for (const p of PROTOCOL_PATHS) if (fp === p || fp.endsWith(`/${p}`)) return true;
78
+ for (const prefix of PROTOCOL_PATH_PREFIXES) if (fp.startsWith(prefix) || fp.includes(`/${prefix}`)) return true;
79
+ if (fp.endsWith('schema.mjs') || fp.endsWith('schema.spec.mjs')) return true;
80
+ return false;
81
+ }
82
+
83
+ function buildRepair(zodError) {
84
+ return { field_errors: (zodError.issues || []).map(i => ({ field: i.path.join('.') || '(root)', issue: i.message })) };
85
+ }
86
+
87
+ function resolveVfsPath(vaultRoot, vfsPath) {
88
+ const cleaned = vfsPath.replace(/^\/+/, '');
89
+ const resolved = path.resolve(vaultRoot, cleaned);
90
+ if (!resolved.startsWith(path.resolve(vaultRoot))) throw new Error(`Path traversal: ${vfsPath}`);
91
+ return resolved;
92
+ }
93
+
94
+ function makeErrorResponse(type, opId, p, errors) {
95
+ return { success: false, type, operation_id: opId, path: p, committed_at: null, validation: { valid: false, errors, warnings: [] } };
96
+ }
97
+
98
+ /**
99
+ * Process an SSSS operation through the full §6.3 pipeline.
100
+ * @param {object} envelope - The operation envelope.
101
+ * @param {string} vaultRoot - Absolute path to the vault directory.
102
+ * @param {object} [options] - { agentRole, leaseStore, eventLogDir }
103
+ * @returns {object} OperationResponse (§6.4).
104
+ */
105
+ export function processOperation(envelope, vaultRoot, options = {}) {
106
+ const { agentRole = 'admin', leaseStore, eventLogDir } = options;
107
+ const operationId = crypto.randomUUID();
108
+ const warnings = [];
109
+
110
+ // Stage 1: Envelope Validation
111
+ const schemaMap = { operation: OperationEnvelopeSchema, patch: PatchEnvelopeSchema, event: EventEnvelopeSchema };
112
+ const envSchema = schemaMap[envelope.type];
113
+ if (!envSchema) return makeErrorResponse(envelope.type || 'unknown', operationId, envelope.path || '', [`Unknown envelope type: ${envelope.type}`]);
114
+ const envResult = envSchema.safeParse(envelope);
115
+ if (!envResult.success) return { ...makeErrorResponse(envelope.type, operationId, envelope.path || '', envResult.error.issues.map(i => `${i.path.join('.')}: ${i.message}`)), repair: buildRepair(envResult.error) };
116
+
117
+ // Warm idempotency cache from audit logs if not already done
118
+ warmIdempotencyCache(vaultRoot, eventLogDir);
119
+
120
+ // Stage 2: Idempotency
121
+ pruneExpired();
122
+ const ck = cacheKey(envelope.workspace_id, envelope.idempotency_key);
123
+ const cached = idempotencyCache.get(ck);
124
+ if (cached) return { ...cached.response, replay: cached.response };
125
+
126
+ // Stage 3: Authorization
127
+ if (isProtocolPath(envelope.path) && agentRole !== 'admin') return makeErrorResponse(envelope.type, operationId, envelope.path, [`Protocol path '${envelope.path}' requires admin role.`]);
128
+
129
+ // Stage 4: Lease Check
130
+ if (leaseStore) {
131
+ const lc = checkLease(envelope, leaseStore);
132
+ if (!lc.ok) return makeErrorResponse(envelope.type, operationId, envelope.path, [lc.error]);
133
+ }
134
+
135
+ // Stage 5: Content Validation
136
+ let resolvedType = null;
137
+ const errors = [];
138
+
139
+ if (envelope.type === 'operation' || envelope.type === 'patch') {
140
+ try {
141
+ let fmData;
142
+ if (envelope.type === 'operation') {
143
+ fmData = matter(envelope.content).data;
144
+ } else {
145
+ const absPath = resolveVfsPath(vaultRoot, envelope.path);
146
+ if (!fs.existsSync(absPath)) { errors.push(`Target file does not exist for patch: ${envelope.path}`); }
147
+ else {
148
+ const { data, content: body } = matter(fs.readFileSync(absPath, 'utf8'));
149
+ fmData = { ...data, ...envelope.patches };
150
+ if (envelope.patches.__body__ !== undefined && APPEND_TYPES.has(data.type)) {
151
+ const eb = body.trim(), nb = String(envelope.patches.__body__).trim();
152
+ if (eb && !nb.startsWith(eb)) errors.push(`Append-type '${data.type}' does not allow rewriting existing records.`);
153
+ }
154
+ }
155
+ }
156
+ if (fmData && errors.length === 0) {
157
+ resolvedType = fmData.type;
158
+ if (!resolvedType) errors.push('Missing required frontmatter field: type');
159
+ else if (!SSSS_SCHEMAS[resolvedType]) errors.push(`Unknown SSSS type: '${resolvedType}'.`);
160
+ else {
161
+ const r = SSSS_SCHEMAS[resolvedType].safeParse(fmData);
162
+ if (!r.success) r.error.issues.forEach(i => errors.push(`${i.path.join('.')}: ${i.message}`));
163
+ if (envelope.type === 'operation' && APPEND_TYPES.has(resolvedType)) {
164
+ const ap = resolveVfsPath(vaultRoot, envelope.path);
165
+ if (fs.existsSync(ap)) {
166
+ const eb = matter(fs.readFileSync(ap, 'utf8')).content.trim();
167
+ const nb = matter(envelope.content).content.trim();
168
+ if (eb && !nb.startsWith(eb)) errors.push(`Append-type '${resolvedType}' does not allow rewriting existing records.`);
169
+ }
170
+ }
171
+ if (agentRole === 'optimizer' && resolvedType === 'memory') {
172
+ if (fmData.priority === 'absolute') warnings.push('Optimizer writing Tier 1 (priority: absolute) node.');
173
+ if (fmData.immutable === true) warnings.push('Optimizer writing immutable node.');
174
+ }
175
+ if (resolvedType === 'schema-proposal' && fmData.status === 'accepted') {
176
+ if (agentRole !== 'admin') errors.push('Only admin may accept schema-proposals.');
177
+ if (!fmData.reviewed_by) errors.push('Accepted schema-proposals must have reviewed_by.');
178
+ }
179
+ }
180
+ }
181
+ } catch (err) { errors.push(`Content parsing error: ${err.message}`); }
182
+ }
183
+ if (envelope.type === 'event') { try { JSON.parse(envelope.content); } catch { errors.push('Event content must be valid JSON.'); } }
184
+
185
+ const isValid = errors.length === 0;
186
+ if (envelope.dry_run || !isValid) {
187
+ const resp = { success: isValid, type: envelope.type, operation_id: operationId, path: envelope.path, committed_at: null, dry_run: !!envelope.dry_run, validation: { valid: isValid, type: resolvedType, errors, warnings } };
188
+ if (!isValid) resp.repair = { field_errors: errors.map(e => ({ field: e.split(': ')[0] || '(unknown)', issue: e })) };
189
+ return resp;
190
+ }
191
+
192
+ // Stage 6: Commit
193
+ const committedAt = new Date().toISOString();
194
+ try {
195
+ const absPath = resolveVfsPath(vaultRoot, envelope.path);
196
+ const dir = path.dirname(absPath);
197
+ if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
198
+ if (envelope.type === 'operation') {
199
+ let contentToWrite = envelope.content;
200
+ try {
201
+ const parsed = matter(contentToWrite);
202
+ if (parsed.data && parsed.data.type === 'memory') {
203
+ parsed.data.updated = committedAt;
204
+ parsed.data.last_accessed = committedAt;
205
+ contentToWrite = safeStringify(parsed.content, parsed.data);
206
+ }
207
+ } catch {
208
+ // Fallback to raw content if parsing fails
209
+ }
210
+ atomicWrite(absPath, contentToWrite);
211
+ }
212
+ else if (envelope.type === 'patch') {
213
+ const { data, content: body } = matter(fs.readFileSync(absPath, 'utf8'));
214
+ const merged = { ...data };
215
+ for (const [k, v] of Object.entries(envelope.patches)) { if (k !== '__body__') merged[k] = v; }
216
+ if (merged.type === 'memory') {
217
+ merged.updated = committedAt;
218
+ merged.last_accessed = committedAt;
219
+ }
220
+ const newBody = envelope.patches.__body__ !== undefined ? (APPEND_TYPES.has(data.type) ? body + '\n' + envelope.patches.__body__ : envelope.patches.__body__) : body;
221
+ atomicWrite(absPath, safeStringify(newBody, merged));
222
+ } else if (envelope.type === 'event') {
223
+ const logDir = eventLogDir || path.join(vaultRoot, '.events');
224
+ if (!fs.existsSync(logDir)) fs.mkdirSync(logDir, { recursive: true });
225
+ fs.appendFileSync(path.join(logDir, `${envelope.workspace_id}.jsonl`), JSON.stringify({ event_id: operationId, event_type: 'operation-audit', ts: committedAt, path: envelope.path, content: envelope.content, idempotency_key: envelope.idempotency_key }) + '\n');
226
+ }
227
+ } catch (err) {
228
+ logger.error('operation-validator', `Commit failed: ${err.message}`);
229
+ return makeErrorResponse(envelope.type, operationId, envelope.path, [`Commit failed: ${err.message}`]);
230
+ }
231
+
232
+ // Stage 7: Audit
233
+ try {
234
+ const auditDir = eventLogDir || path.join(vaultRoot, '.events');
235
+ if (!fs.existsSync(auditDir)) fs.mkdirSync(auditDir, { recursive: true });
236
+ fs.appendFileSync(path.join(auditDir, 'audit.jsonl'), JSON.stringify({ event_id: crypto.randomUUID(), event_type: 'audit', correlation_id: operationId, ts: committedAt, subject: envelope.path, payload: { envelope_type: envelope.type, idempotency_key: envelope.idempotency_key, workspace_id: envelope.workspace_id, agent_role: agentRole, resolved_type: resolvedType } }) + '\n');
237
+ } catch (err) { logger.error('operation-validator', `Audit failed: ${err.message}`); }
238
+
239
+ const response = { success: true, type: envelope.type, operation_id: operationId, path: envelope.path, committed_at: committedAt, validation: { valid: true, type: resolvedType, errors: [], warnings } };
240
+ idempotencyCache.set(ck, { response, ts: Date.now() });
241
+ return response;
242
+ }
243
+
244
+ function checkLease(envelope, leaseStore) {
245
+ const lp = path.join(leaseStore, envelope.workspace_id, `${envelope.path.replace(/\//g, '__')}.lease.json`);
246
+ if (!fs.existsSync(lp)) return { ok: true };
247
+ try {
248
+ const lease = JSON.parse(fs.readFileSync(lp, 'utf8'));
249
+ if (new Date(lease.expires_at) < new Date()) { fs.unlinkSync(lp); return { ok: true }; }
250
+ if (!envelope.lease_id) return { ok: false, error: `Path '${envelope.path}' is leased (${lease.lease_id}). Supply lease_id.` };
251
+ if (envelope.lease_id !== lease.lease_id) return { ok: false, error: `Lease mismatch: got '${envelope.lease_id}', expected '${lease.lease_id}'.` };
252
+ return { ok: true };
253
+ } catch { try { fs.unlinkSync(lp); } catch { } return { ok: true }; }
254
+ }
255
+
256
+ export function acquireLease(workspaceId, vfsPath, leaseStore, ttlMs = 30000) {
257
+ const dir = path.join(leaseStore, workspaceId);
258
+ if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
259
+ const lp = path.join(dir, `${vfsPath.replace(/\//g, '__')}.lease.json`);
260
+ if (fs.existsSync(lp)) { try { const e = JSON.parse(fs.readFileSync(lp, 'utf8')); if (new Date(e.expires_at) >= new Date()) return { error: `Path '${vfsPath}' already leased.` }; } catch { } }
261
+ const leaseId = crypto.randomUUID();
262
+ const expiresAt = new Date(Date.now() + ttlMs).toISOString();
263
+ fs.writeFileSync(lp, JSON.stringify({ lease_id: leaseId, path: vfsPath, workspace_id: workspaceId, expires_at: expiresAt }, null, 2));
264
+ return { lease_id: leaseId, expires_at: expiresAt };
265
+ }
266
+
267
+ export function releaseLease(workspaceId, vfsPath, leaseId, leaseStore) {
268
+ const lp = path.join(leaseStore, workspaceId, `${vfsPath.replace(/\//g, '__')}.lease.json`);
269
+ if (!fs.existsSync(lp)) return { released: true };
270
+ try { const e = JSON.parse(fs.readFileSync(lp, 'utf8')); if (e.lease_id !== leaseId) return { error: 'Lease ID mismatch.' }; fs.unlinkSync(lp); return { released: true }; }
271
+ catch (err) { return { error: err.message }; }
272
+ }
@@ -0,0 +1,330 @@
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import crypto from 'crypto';
4
+ import matter from 'gray-matter';
5
+ import { loadNodes, writeNode } from './vault.mjs';
6
+ import { logger } from './logger.mjs';
7
+ import { brainDir } from './config.mjs';
8
+
9
+ /**
10
+ * Total Recall Optimizer
11
+ * Instead of mutating memory directly, the optimizer generates Proposals.
12
+ * Proposals must pass a local evaluation gate before being accepted.
13
+ */
14
+
15
+ export function createProposal(category, summary, targetPath = null, rationale = '') {
16
+ const proposalId = `prop_${crypto.randomBytes(6).toString('hex')}`;
17
+ return {
18
+ type: 'proposal',
19
+ proposal_id: proposalId,
20
+ category,
21
+ status: 'draft',
22
+ target_path: targetPath,
23
+ summary,
24
+ rationale,
25
+ proposed_by: 'kernel_optimizer_v1',
26
+ proposed_at: new Date().toISOString(),
27
+ };
28
+ }
29
+
30
+ /**
31
+ * Scans the vault for duplicate or contradictory memories and generates a cleanup proposal.
32
+ */
33
+ export async function generateMemoryCleanupProposals(vaultDir) {
34
+ const nodes = loadNodes(vaultDir);
35
+ const proposals = [];
36
+
37
+ // Simple heuristic: find nodes with the same predicate and object but different subjects, or duplicate subjects.
38
+ // In a real system, we would use local LLM / embeddings to find semantic duplicates.
39
+ const conceptMap = new Map();
40
+
41
+ for (const node of nodes) {
42
+ if (node.type !== 'memory') continue;
43
+ const key = `${node.predicate}:${node.object}`;
44
+ if (!conceptMap.has(key)) {
45
+ conceptMap.set(key, []);
46
+ }
47
+ conceptMap.get(key).push(node);
48
+ }
49
+
50
+ for (const [key, similarNodes] of conceptMap.entries()) {
51
+ if (similarNodes.length > 1) {
52
+ const paths = similarNodes.map(n => n.slug).join(', ');
53
+ const proposal = createProposal(
54
+ 'memory-cleanup',
55
+ `Merge similar memory nodes for concept [${key}]`,
56
+ paths,
57
+ `Detected ${similarNodes.length} nodes with identical predicate:object signatures. Merging them will reduce fragmentation.`
58
+ );
59
+ proposals.push(proposal);
60
+ }
61
+ }
62
+
63
+ return proposals;
64
+ }
65
+
66
+ /**
67
+ * Scans skill execution logs (if available) and proposes improvements.
68
+ */
69
+ /**
70
+ * Scan skill files for quality issues and propose improvements.
71
+ * Looks at skill files that haven't been updated in >30 days and
72
+ * checks for common issues: missing examples, vague instructions, outdated commands.
73
+ */
74
+ export async function generateSkillImprovementProposals(vaultDir) {
75
+ const agentDir = path.join(path.dirname(vaultDir));
76
+ const skillsDir = path.join(agentDir, 'skills');
77
+ const proposals = [];
78
+
79
+ if (!fs.existsSync(skillsDir)) return proposals;
80
+
81
+ const now = Date.now();
82
+ const staleCutoff = 30 * 24 * 60 * 60 * 1000; // 30 days
83
+
84
+ try {
85
+ const skillDirs = fs.readdirSync(skillsDir, { withFileTypes: true })
86
+ .filter(d => d.isDirectory())
87
+ .map(d => path.join(skillsDir, d.name, 'SKILL.md'))
88
+ .filter(p => fs.existsSync(p));
89
+
90
+ for (const skillFile of skillDirs) {
91
+ const stat = fs.statSync(skillFile);
92
+ const ageMs = now - stat.mtimeMs;
93
+ if (ageMs < staleCutoff) continue;
94
+
95
+ const raw = fs.readFileSync(skillFile, 'utf8');
96
+ const hasExamples = /##\s*example/i.test(raw);
97
+ const hasCaution = /caution|warning|never|must not/i.test(raw);
98
+ const lineCount = raw.split('\n').length;
99
+
100
+ const issues = [];
101
+ if (!hasExamples) issues.push('No examples section');
102
+ if (!hasCaution) issues.push('No caution/constraint section');
103
+ if (lineCount < 20) issues.push('Very short — may lack detail');
104
+
105
+ if (issues.length > 0) {
106
+ const skillName = path.basename(path.dirname(skillFile));
107
+ proposals.push(createProposal(
108
+ 'skill-improvement',
109
+ `Improve skill: ${skillName} (${issues.join(', ')})`,
110
+ skillFile,
111
+ `Skill file is ${Math.floor(ageMs / 86400000)} days old. Issues: ${issues.join('; ')}`,
112
+ ));
113
+ }
114
+ }
115
+ } catch (err) {
116
+ logger.debug('optimizer: generateSkillImprovementProposals failed to read skillsDir', { err: err.message });
117
+ }
118
+
119
+ return proposals;
120
+ }
121
+
122
+ /**
123
+ * Scan the task queue for stuck or stalled workflows.
124
+ * Tasks that have been 'in-progress' for >1 hour are considered stalled.
125
+ */
126
+ export async function generateWorkflowRepairProposals(vaultDir) {
127
+ const queueDir = path.join(brainDir, 'scheduler', 'queue');
128
+ const proposals = [];
129
+
130
+ if (!fs.existsSync(queueDir)) return proposals;
131
+
132
+ const now = Date.now();
133
+ const stalledCutoff = 60 * 60 * 1000; // 1 hour
134
+
135
+ try {
136
+ const files = fs.readdirSync(queueDir).filter(f => f.endsWith('.md'));
137
+ for (const file of files) {
138
+ const raw = fs.readFileSync(path.join(queueDir, file), 'utf8');
139
+ const { data } = matter(raw);
140
+ if (data.status !== 'in-progress') continue;
141
+
142
+ const updatedAt = new Date(data.updated_at || data.created || 0).getTime();
143
+ if (now - updatedAt < stalledCutoff) continue;
144
+
145
+ proposals.push(createProposal(
146
+ 'workflow-repair',
147
+ `Stalled task: ${data.slug || file}`,
148
+ path.join(queueDir, file),
149
+ `Task has been in-progress for ${Math.floor((now - updatedAt) / 60000)} minutes. May need retry or cancellation.`,
150
+ ));
151
+ }
152
+ } catch (err) {
153
+ logger.debug('optimizer: generateWorkflowRepairProposals failed to read queueDir', { err: err.message });
154
+ }
155
+
156
+ return proposals;
157
+ }
158
+
159
+ /**
160
+ * Analyze the vault's model usage patterns and propose routing improvements.
161
+ * Flags cases where high-priority tasks were sent to the local LLM
162
+ * that should have used a frontier model.
163
+ */
164
+ export async function generateModelRoutingProposals(vaultDir) {
165
+ const proposals = [];
166
+ const nodes = loadNodes(vaultDir);
167
+
168
+ // Find high-importance facts with low confidence — likely needed frontier model
169
+ const underconfident = nodes.filter(n =>
170
+ n.status === 'active' &&
171
+ n.category === 'facts' &&
172
+ n.importance >= 4 &&
173
+ n.confidence < 0.6 &&
174
+ n.source?.agent === 'fact-seeker',
175
+ );
176
+
177
+ for (const node of underconfident.slice(0, 5)) {
178
+ proposals.push(createProposal(
179
+ 'model-routing',
180
+ `Re-research with frontier model: "${node.title}"`,
181
+ node.slug,
182
+ `High-importance fact (importance: ${node.importance}) has low confidence (${node.confidence}). Originally researched by local model. Frontier model may produce better synthesis.`,
183
+ ));
184
+ }
185
+
186
+ return proposals;
187
+ }
188
+
189
+ export async function generateStaleKnowledgeRefreshProposals(vaultDir) {
190
+ const nodes = loadNodes(vaultDir);
191
+ const proposals = [];
192
+
193
+ const now = Date.now();
194
+ const ONE_MONTH_MS = 30 * 24 * 60 * 60 * 1000;
195
+
196
+ for (const node of nodes) {
197
+ if (node.type === 'memory' && node.last_accessed) {
198
+ const lastAccessed = new Date(node.last_accessed).getTime();
199
+ if (now - lastAccessed > ONE_MONTH_MS && node.importance > 3) {
200
+ proposals.push(createProposal(
201
+ 'stale-knowledge-refresh',
202
+ `Refresh stale high-importance memory: ${node.title}`,
203
+ node.slug,
204
+ `Node has not been accessed in over 30 days but maintains high importance. Requires verification against current state.`
205
+ ));
206
+ }
207
+ }
208
+ }
209
+
210
+ return proposals;
211
+ }
212
+
213
+ /**
214
+ * Local Eval Gate for Proposal Promotion
215
+ * Uses the local runtime to evaluate the validity of a proposal.
216
+ */
217
+ export async function evaluateProposalGate(proposal, runtimeConfig) {
218
+ // In a production scenario, this calls the local LLM runtime to grade the proposal.
219
+ // For now, we simulate the evaluation gate based on deterministic rules or simple heuristics.
220
+ if (proposal.category === 'memory-cleanup' && proposal.rationale.includes('identical predicate:object')) {
221
+ proposal.status = 'accepted';
222
+ proposal.reviewed_at = new Date().toISOString();
223
+ proposal.reviewed_by = 'local_eval_gate';
224
+ return true;
225
+ }
226
+
227
+ if (proposal.category === 'stale-knowledge-refresh') {
228
+ proposal.status = 'accepted';
229
+ proposal.reviewed_at = new Date().toISOString();
230
+ proposal.reviewed_by = 'local_eval_gate';
231
+ return true;
232
+ }
233
+
234
+ proposal.status = 'rejected';
235
+ proposal.rejection_reason = 'Failed to meet minimum evaluation threshold for automatic promotion.';
236
+ proposal.reviewed_at = new Date().toISOString();
237
+ proposal.reviewed_by = 'local_eval_gate';
238
+ return false;
239
+ }
240
+
241
+ // ─── Smart Decay ────────────────────────────────────────────────────────────────
242
+
243
+ const DECAY_SYSTEM = `You are a Memory Retention Analyst for a sovereign AI memory system.
244
+
245
+ Given a memory node that has not been accessed recently, evaluate whether it should be:
246
+ 1. RETAINED — still relevant and valuable
247
+ 2. ARCHIVED — no longer active but worth keeping as historical record
248
+ 3. DECAYED — safe to archive with low confidence flag
249
+
250
+ Output valid JSON:
251
+ {
252
+ "verdict": "RETAINED" | "ARCHIVED" | "DECAYED",
253
+ "confidence_adjustment": -0.2 to 0.0,
254
+ "reasoning": "string"
255
+ }
256
+
257
+ Criteria:
258
+ - RETAINED: Foundational principles, architectural decisions, active rules
259
+ - ARCHIVED: Historical context, superseded patterns, solved problems
260
+ - DECAYED: Temporary workarounds, version-specific hacks, stale configs
261
+
262
+ Output ONLY valid JSON.`;
263
+
264
+ /**
265
+ * LLM-powered Smart Decay — evaluates each node individually rather than
266
+ * applying blind half-life. Produces decay proposals for the optimizer gate.
267
+ *
268
+ * @param {string} vaultDir
269
+ * @param {object} runtimeConfig
270
+ * @param {number} [ageThresholdDays=60] Minimum age before considering decay
271
+ * @returns {object[]} decay proposals
272
+ */
273
+ export async function runSmartDecay(vaultDir, runtimeConfig, ageThresholdDays = 60) {
274
+ const { callLocalRuntime } = await import('./runtime.mjs');
275
+ const nodes = loadNodes(vaultDir);
276
+ const now = Date.now();
277
+ const ageThresholdMs = ageThresholdDays * 24 * 60 * 60 * 1000;
278
+ const proposals = [];
279
+
280
+ // Candidates: active, not absolute priority, not accessed recently
281
+ const candidates = nodes.filter(n => {
282
+ if (n.status !== 'active') return false;
283
+ if (n.priority === 'absolute') return false;
284
+ if (n.importance >= 5) return false;
285
+ const lastAccessed = new Date(n.last_accessed || n.updated || 0).getTime();
286
+ return (now - lastAccessed) > ageThresholdMs;
287
+ });
288
+
289
+ // Process up to 5 candidates per run to avoid excessive LLM calls
290
+ const batch = candidates.slice(0, 5);
291
+
292
+ for (const node of batch) {
293
+ const ageDays = Math.floor((now - new Date(node.last_accessed || node.updated || 0).getTime()) / (1000 * 60 * 60 * 24));
294
+
295
+ const prompt = [
296
+ `## Memory Node: ${node.slug}`,
297
+ `Title: "${node.title}"`,
298
+ `Category: ${node.category} | Importance: ${node.importance} | Age: ${ageDays} days`,
299
+ `Tags: ${(node.tags || []).join(', ')}`,
300
+ '',
301
+ '## Body',
302
+ (node.body || '').slice(0, 800),
303
+ '',
304
+ `This node has not been accessed in ${ageDays} days. Should it be retained, archived, or decayed?`,
305
+ ].join('\n');
306
+
307
+ try {
308
+ const rawResponse = await callLocalRuntime(prompt, DECAY_SYSTEM, runtimeConfig);
309
+ const jsonMatch = rawResponse.match(/\{[\s\S]*\}/);
310
+ if (!jsonMatch) continue;
311
+
312
+ const result = JSON.parse(jsonMatch[0]);
313
+ const verdict = result.verdict || 'RETAINED';
314
+
315
+ if (verdict !== 'RETAINED') {
316
+ proposals.push(createProposal(
317
+ 'smart-decay',
318
+ `Smart Decay [${verdict}]: "${node.title}"`,
319
+ node.slug,
320
+ `Age: ${ageDays} days. Reasoning: ${result.reasoning || 'No reasoning provided.'} Confidence adjustment: ${result.confidence_adjustment || 0}`,
321
+ ));
322
+ }
323
+ } catch {
324
+ // Skip failed evaluations silently
325
+ }
326
+ }
327
+
328
+ return proposals;
329
+ }
330
+