persyst-mcp 2.1.3 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/init.js +7 -0
- package/index.js +41 -0
- package/package.json +2 -2
- package/src/attestation.js +7 -1
- package/src/database.js +973 -877
- package/src/extractor-heuristic.js +324 -250
- package/src/git.js +7 -1
- package/src/search.js +597 -456
- package/src/server.js +72 -67
- package/src/tools.js +157 -20
- package/src/watcher.js +306 -0
package/src/server.js
CHANGED
|
@@ -1,67 +1,72 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* server.js — MCP Server Setup
|
|
3
|
-
*
|
|
4
|
-
* Creates the MCP server, registers all tools, and connects
|
|
5
|
-
* via stdio transport (the standard MCP communication method).
|
|
6
|
-
* Sets up hourly temporal decay and daily consolidation background tasks.
|
|
7
|
-
*
|
|
8
|
-
* IMPORTANT: Never write to stdout — it's reserved for MCP protocol.
|
|
9
|
-
* All logging goes to stderr via console.error().
|
|
10
|
-
*/
|
|
11
|
-
|
|
12
|
-
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
13
|
-
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
14
|
-
import { registerTools, cleanupWatchers } from './tools.js';
|
|
15
|
-
import { applyTemporalDecay, closeDatabase } from './database.js';
|
|
16
|
-
import { consolidateMemories } from './search.js';
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
*
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
//
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
// --- Start
|
|
38
|
-
// Runs every
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
1
|
+
/**
|
|
2
|
+
* server.js — MCP Server Setup
|
|
3
|
+
*
|
|
4
|
+
* Creates the MCP server, registers all tools, and connects
|
|
5
|
+
* via stdio transport (the standard MCP communication method).
|
|
6
|
+
* Sets up hourly temporal decay and daily consolidation background tasks.
|
|
7
|
+
*
|
|
8
|
+
* IMPORTANT: Never write to stdout — it's reserved for MCP protocol.
|
|
9
|
+
* All logging goes to stderr via console.error().
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
13
|
+
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
14
|
+
import { registerTools, cleanupWatchers } from './tools.js';
|
|
15
|
+
import { applyTemporalDecay, closeDatabase } from './database.js';
|
|
16
|
+
import { consolidateMemories } from './search.js';
|
|
17
|
+
import { startWatcher, stopWatcher } from './watcher.js';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Start the Persyst MCP server.
|
|
21
|
+
* This is called from index.js (the entry point).
|
|
22
|
+
*/
|
|
23
|
+
export async function startServer() {
|
|
24
|
+
// --- Create MCP server ---
|
|
25
|
+
const server = new McpServer({
|
|
26
|
+
name: 'persyst',
|
|
27
|
+
version: '2.1.3'
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
// --- Register all tools ---
|
|
31
|
+
const registeredCount = registerTools(server);
|
|
32
|
+
console.error(`[persyst] ${registeredCount} tools registered ✓`);
|
|
33
|
+
|
|
34
|
+
// --- Start background log watcher daemon ---
|
|
35
|
+
startWatcher();
|
|
36
|
+
|
|
37
|
+
// --- Start temporal decay timer ---
|
|
38
|
+
// Runs every hour: reduces importance of memories not accessed in 7+ days
|
|
39
|
+
const decayTimer = setInterval(applyTemporalDecay, 3600000);
|
|
40
|
+
|
|
41
|
+
// --- Start daily consolidation sweep ---
|
|
42
|
+
// Runs every 24 hours: merges similar memories (similarity > 0.85)
|
|
43
|
+
const consolidationTimer = setInterval(async () => {
|
|
44
|
+
console.error('[persyst] Running scheduled daily memory consolidation sweep...');
|
|
45
|
+
try {
|
|
46
|
+
const report = await consolidateMemories();
|
|
47
|
+
console.error(`[persyst] Consolidation sweep completed: consolidated ${report.consolidated_groups} duplicate groups.`);
|
|
48
|
+
} catch (err) {
|
|
49
|
+
console.error('[persyst] Daily consolidation sweep failed:', err.message);
|
|
50
|
+
}
|
|
51
|
+
}, 86400000);
|
|
52
|
+
|
|
53
|
+
// --- Graceful shutdown (Bug 3 fix: also cleans up git watchers) ---
|
|
54
|
+
const shutdown = () => {
|
|
55
|
+
console.error('[persyst] Shutting down...');
|
|
56
|
+
clearInterval(decayTimer);
|
|
57
|
+
clearInterval(consolidationTimer);
|
|
58
|
+
stopWatcher(); // Stop background log watcher
|
|
59
|
+
cleanupWatchers(); // Bug 3 fix: stop all git repo watchers
|
|
60
|
+
closeDatabase();
|
|
61
|
+
process.exit(0);
|
|
62
|
+
};
|
|
63
|
+
process.on('SIGINT', shutdown);
|
|
64
|
+
process.on('SIGTERM', shutdown);
|
|
65
|
+
|
|
66
|
+
// --- Connect via stdio ---
|
|
67
|
+
const transport = new StdioServerTransport();
|
|
68
|
+
await server.connect(transport);
|
|
69
|
+
|
|
70
|
+
console.error('[persyst] MCP server running on stdio ✓');
|
|
71
|
+
console.error('[persyst] Ready to receive tool calls');
|
|
72
|
+
}
|
package/src/tools.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
import { z } from 'zod';
|
|
15
15
|
import { generateEmbedding } from './embeddings.js';
|
|
16
|
-
import {
|
|
16
|
+
import db, {
|
|
17
17
|
insertMemory,
|
|
18
18
|
insertVector,
|
|
19
19
|
getMemory,
|
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
getMemoryByContent,
|
|
33
33
|
boostMemory,
|
|
34
34
|
logContradiction,
|
|
35
|
+
getProvenance,
|
|
35
36
|
getAllAgentStats,
|
|
36
37
|
getAttestationsByDateRange,
|
|
37
38
|
getMemoryHistoryChain,
|
|
@@ -51,8 +52,8 @@ import { searchCache } from './cache.js';
|
|
|
51
52
|
// CONSTANTS
|
|
52
53
|
// ============================================================
|
|
53
54
|
|
|
54
|
-
/** Maximum allowed memory content length (
|
|
55
|
-
const MAX_MEMORY_CONTENT_LENGTH =
|
|
55
|
+
/** Maximum allowed memory content length (10,000 characters) */
|
|
56
|
+
const MAX_MEMORY_CONTENT_LENGTH = 10000;
|
|
56
57
|
|
|
57
58
|
/** Minimum content length (must have actual content) */
|
|
58
59
|
const MIN_MEMORY_CONTENT_LENGTH = 1;
|
|
@@ -174,22 +175,54 @@ export function registerTools(server) {
|
|
|
174
175
|
const existingMemory = getMemoryById(hitId, namespace);
|
|
175
176
|
if (!existingMemory) continue;
|
|
176
177
|
|
|
177
|
-
// Check if content is substantially different (Jaccard distance > 0.5)
|
|
178
178
|
const jaccard = jaccardDistance(content, existingMemory.content);
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
}
|
|
179
|
+
// Contradiction: similar topic (high similarity), but differing key terms
|
|
180
|
+
if (jaccard > 0 && jaccard < 0.5) {
|
|
181
|
+
// Fetch provenances for trust calculation
|
|
182
|
+
const oldProv = getProvenance(hitId);
|
|
183
|
+
let oldReputation = 1.0;
|
|
184
|
+
if (oldProv && oldProv.source_type === 'agent' && oldProv.source_id) {
|
|
185
|
+
const agentRow = db.prepare('SELECT reputation_score FROM agent_stats WHERE agent_id = ?').get(oldProv.source_id);
|
|
186
|
+
if (agentRow) oldReputation = agentRow.reputation_score;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
let newReputation = 1.0;
|
|
190
|
+
if (agent_id) {
|
|
191
|
+
const agentRow = db.prepare('SELECT reputation_score FROM agent_stats WHERE agent_id = ?').get(agent_id);
|
|
192
|
+
if (agentRow) newReputation = agentRow.reputation_score;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const trustOld = (oldProv ? oldProv.confidence : 1.0) * oldReputation;
|
|
196
|
+
const trustNew = 1.0 * newReputation; // New confidence is 1.0
|
|
197
|
+
|
|
198
|
+
const isSelfUpdate = oldProv && oldProv.source_type === 'agent' && oldProv.source_id === agent_id;
|
|
199
|
+
|
|
200
|
+
if (isSelfUpdate || trustNew > trustOld) {
|
|
201
|
+
// New is preferred
|
|
202
|
+
logContradiction(hitId, id, `Auto-detected contradiction: new memory is more trustworthy (similarity: ${sim.toFixed(3)}, content_diff: ${jaccard.toFixed(3)})`);
|
|
203
|
+
contradictions.push({
|
|
204
|
+
old_memory_id: hitId,
|
|
205
|
+
old_content_preview: existingMemory.content.slice(0, 100),
|
|
206
|
+
similarity: sim.toFixed(4),
|
|
207
|
+
content_difference: jaccard.toFixed(4),
|
|
208
|
+
resolution: 'replaced_old'
|
|
209
|
+
});
|
|
210
|
+
} else {
|
|
211
|
+
// Old is preferred
|
|
212
|
+
logContradiction(id, hitId, `Auto-detected contradiction: existing memory is more trustworthy (similarity: ${sim.toFixed(3)}, content_diff: ${jaccard.toFixed(3)})`);
|
|
213
|
+
contradictions.push({
|
|
214
|
+
old_memory_id: hitId,
|
|
215
|
+
old_content_preview: existingMemory.content.slice(0, 100),
|
|
216
|
+
similarity: sim.toFixed(4),
|
|
217
|
+
content_difference: jaccard.toFixed(4),
|
|
218
|
+
resolution: 'kept_old'
|
|
219
|
+
});
|
|
220
|
+
break; // New memory was archived, stop contradiction check
|
|
221
|
+
}
|
|
188
222
|
}
|
|
189
223
|
}
|
|
190
224
|
}
|
|
191
225
|
} catch (e) {
|
|
192
|
-
// Contradiction detection is best-effort, don't fail the memory insertion
|
|
193
226
|
console.error(`[persyst] Contradiction detection error: ${e.message}`);
|
|
194
227
|
}
|
|
195
228
|
|
|
@@ -271,12 +304,22 @@ export function registerTools(server) {
|
|
|
271
304
|
const oldMemory = getMemory(id);
|
|
272
305
|
if (!oldMemory) return text({ error: `Memory #${id} not found` });
|
|
273
306
|
|
|
307
|
+
// Retrieve old agent_id from provenance
|
|
308
|
+
const oldProv = getProvenance(id);
|
|
309
|
+
const resolvedAgentId = agent_id || (oldProv && oldProv.source_type === 'agent' ? oldProv.source_id : null);
|
|
310
|
+
|
|
274
311
|
// Insert new version
|
|
275
|
-
const newId = insertMemory(
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
312
|
+
const newId = insertMemory(
|
|
313
|
+
content,
|
|
314
|
+
oldMemory.importance_score,
|
|
315
|
+
{
|
|
316
|
+
source_type: resolvedAgentId ? 'agent' : 'manual',
|
|
317
|
+
source_id: resolvedAgentId,
|
|
318
|
+
confidence: 1.0
|
|
319
|
+
},
|
|
320
|
+
oldMemory.namespace || 'shared',
|
|
321
|
+
id
|
|
322
|
+
);
|
|
280
323
|
|
|
281
324
|
const embedding = await generateEmbedding(content);
|
|
282
325
|
insertVector(newId, embedding);
|
|
@@ -499,14 +542,50 @@ export function registerTools(server) {
|
|
|
499
542
|
},
|
|
500
543
|
async ({ query }) => {
|
|
501
544
|
try {
|
|
502
|
-
|
|
545
|
+
let hits = [];
|
|
546
|
+
const queryAsId = Number(query);
|
|
547
|
+
if (!isNaN(queryAsId) && Number.isInteger(queryAsId)) {
|
|
548
|
+
const mem = getAnyMemoryById(queryAsId);
|
|
549
|
+
if (mem) {
|
|
550
|
+
hits.push({ id: mem.id });
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
if (hits.length === 0) {
|
|
555
|
+
hits = searchAllMemoriesFts(query, 5);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// Fallback to LIKE query on memories content if FTS is empty or fails
|
|
559
|
+
if (hits.length === 0) {
|
|
560
|
+
try {
|
|
561
|
+
const likeRows = db.prepare("SELECT id FROM memories WHERE content LIKE ? LIMIT 5").all(`%${query}%`);
|
|
562
|
+
hits = likeRows;
|
|
563
|
+
} catch (_) {}
|
|
564
|
+
}
|
|
565
|
+
|
|
503
566
|
if (hits.length === 0) {
|
|
504
567
|
return text({ message: 'No memories matching query found.' });
|
|
505
568
|
}
|
|
506
569
|
|
|
507
570
|
const histories = {};
|
|
571
|
+
const seenChainKeys = new Set();
|
|
508
572
|
for (const hit of hits) {
|
|
509
573
|
const chain = getMemoryHistoryChain(hit.id);
|
|
574
|
+
if (chain.length === 0) continue;
|
|
575
|
+
|
|
576
|
+
// Deduplicate chains to prevent duplicate entries in history response
|
|
577
|
+
const chainKey = chain.map(c => c.id).sort((a, b) => a - b).join(',');
|
|
578
|
+
if (seenChainKeys.has(chainKey)) continue;
|
|
579
|
+
seenChainKeys.add(chainKey);
|
|
580
|
+
|
|
581
|
+
// Decorate chain versions with semantic diffs from the previous version
|
|
582
|
+
for (let idx = 0; idx < chain.length; idx++) {
|
|
583
|
+
if (idx > 0) {
|
|
584
|
+
chain[idx].diff_from_previous = diffWords(chain[idx - 1].content, chain[idx].content);
|
|
585
|
+
} else {
|
|
586
|
+
chain[idx].diff_from_previous = null;
|
|
587
|
+
}
|
|
588
|
+
}
|
|
510
589
|
histories[hit.id] = chain;
|
|
511
590
|
}
|
|
512
591
|
|
|
@@ -716,3 +795,61 @@ function jaccardDistance(a, b) {
|
|
|
716
795
|
if (union === 0) return 0;
|
|
717
796
|
return 1 - (intersection / union);
|
|
718
797
|
}
|
|
798
|
+
|
|
799
|
+
/**
|
|
800
|
+
* Compute word-level diff between two text strings using dynamic programming.
|
|
801
|
+
* Highlights additions as [+added+] and deletions as [-deleted-].
|
|
802
|
+
* @param {string} oldStr - Original text
|
|
803
|
+
* @param {string} newStr - New version of text
|
|
804
|
+
* @returns {string} Diff string
|
|
805
|
+
*/
|
|
806
|
+
function diffWords(oldStr, newStr) {
|
|
807
|
+
const oldWords = oldStr.split(/(\s+)/);
|
|
808
|
+
const newWords = newStr.split(/(\s+)/);
|
|
809
|
+
|
|
810
|
+
const dp = Array(oldWords.length + 1).fill(0).map(() => Array(newWords.length + 1).fill(0));
|
|
811
|
+
|
|
812
|
+
for (let i = 1; i <= oldWords.length; i++) {
|
|
813
|
+
for (let j = 1; j <= newWords.length; j++) {
|
|
814
|
+
if (oldWords[i-1] === newWords[j-1]) {
|
|
815
|
+
dp[i][j] = dp[i-1][j-1] + 1;
|
|
816
|
+
} else {
|
|
817
|
+
dp[i][j] = Math.max(dp[i-1][j], dp[i][j-1]);
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
let i = oldWords.length;
|
|
823
|
+
let j = newWords.length;
|
|
824
|
+
const result = [];
|
|
825
|
+
|
|
826
|
+
while (i > 0 || j > 0) {
|
|
827
|
+
if (i > 0 && j > 0 && oldWords[i-1] === newWords[j-1]) {
|
|
828
|
+
result.unshift({ type: 'common', value: oldWords[i-1] });
|
|
829
|
+
i--;
|
|
830
|
+
j--;
|
|
831
|
+
} else if (j > 0 && (i === 0 || dp[i][j-1] >= dp[i-1][j])) {
|
|
832
|
+
result.unshift({ type: 'added', value: newWords[j-1] });
|
|
833
|
+
j--;
|
|
834
|
+
} else {
|
|
835
|
+
result.unshift({ type: 'removed', value: oldWords[i-1] });
|
|
836
|
+
i--;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
// Combine consecutive items of the same type
|
|
841
|
+
const combined = [];
|
|
842
|
+
for (const part of result) {
|
|
843
|
+
if (combined.length > 0 && combined[combined.length - 1].type === part.type) {
|
|
844
|
+
combined[combined.length - 1].value += part.value;
|
|
845
|
+
} else {
|
|
846
|
+
combined.push({ ...part });
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
return combined.map(part => {
|
|
851
|
+
if (part.type === 'added') return `[+${part.value}+]`;
|
|
852
|
+
if (part.type === 'removed') return `[-${part.value}-]`;
|
|
853
|
+
return part.value;
|
|
854
|
+
}).join('');
|
|
855
|
+
}
|
package/src/watcher.js
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* watcher.js — Persyst Automatic Log Watcher Daemon
|
|
3
|
+
*
|
|
4
|
+
* Periodically scans configured log directories for coding agents (Antigravity, Roo-Code, etc.),
|
|
5
|
+
* reads new appends/messages from transcripts, runs heuristics to extract high-value memories,
|
|
6
|
+
* and stores them in the local database.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { join, resolve } from 'path';
|
|
10
|
+
import { homedir } from 'os';
|
|
11
|
+
import { existsSync, readFileSync, writeFileSync, readdirSync, statSync } from 'fs';
|
|
12
|
+
import {
|
|
13
|
+
getWatchPosition,
|
|
14
|
+
upsertWatchPosition,
|
|
15
|
+
insertMemory,
|
|
16
|
+
insertVector,
|
|
17
|
+
memoryExists
|
|
18
|
+
} from './database.js';
|
|
19
|
+
import { generateEmbedding } from './embeddings.js';
|
|
20
|
+
import { extractHeuristic } from './extractor-heuristic.js';
|
|
21
|
+
import { searchHybrid } from './search.js';
|
|
22
|
+
import { searchCache } from './cache.js';
|
|
23
|
+
|
|
24
|
+
// Config path: ~/.persyst/config.json
|
|
25
|
+
const CONFIG_FILE = join(homedir(), '.persyst', 'config.json');
|
|
26
|
+
|
|
27
|
+
let intervalId = null;
|
|
28
|
+
const DEDUP_THRESHOLD = 0.80;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Load configured directories to watch. Generates a default config if missing.
|
|
32
|
+
* @returns {Array<string>} List of directory paths to scan
|
|
33
|
+
*/
|
|
34
|
+
export function loadWatchedDirs() {
|
|
35
|
+
const defaultDirs = [
|
|
36
|
+
join(homedir(), '.gemini', 'antigravity-ide', 'brain').replace(/\\/g, '/')
|
|
37
|
+
];
|
|
38
|
+
|
|
39
|
+
// Also probe standard paths for Cline / Roo Code
|
|
40
|
+
const platform = process.platform;
|
|
41
|
+
if (platform === 'win32') {
|
|
42
|
+
const appData = process.env.APPDATA || join(homedir(), 'AppData', 'Roaming');
|
|
43
|
+
const rooPath = join(appData, 'Roo-Code', 'tasks').replace(/\\/g, '/');
|
|
44
|
+
if (existsSync(rooPath)) defaultDirs.push(rooPath);
|
|
45
|
+
const clinePath = join(appData, 'Code', 'User', 'globalStorage', 'saoudrizwan.claude-dev', 'tasks').replace(/\\/g, '/');
|
|
46
|
+
if (existsSync(clinePath)) defaultDirs.push(clinePath);
|
|
47
|
+
} else if (platform === 'darwin') {
|
|
48
|
+
const rooPath = join(homedir(), 'Library', 'Application Support', 'Roo-Code', 'tasks');
|
|
49
|
+
if (existsSync(rooPath)) defaultDirs.push(rooPath);
|
|
50
|
+
const clinePath = join(homedir(), 'Library', 'Application Support', 'Code', 'User', 'globalStorage', 'saoudrizwan.claude-dev', 'tasks');
|
|
51
|
+
if (existsSync(clinePath)) defaultDirs.push(clinePath);
|
|
52
|
+
} else {
|
|
53
|
+
const rooPath = join(homedir(), '.config', 'Roo-Code', 'tasks');
|
|
54
|
+
if (existsSync(rooPath)) defaultDirs.push(rooPath);
|
|
55
|
+
const clinePath = join(homedir(), '.config', 'Code', 'User', 'globalStorage', 'saoudrizwan.claude-dev', 'tasks');
|
|
56
|
+
if (existsSync(clinePath)) defaultDirs.push(clinePath);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
try {
|
|
60
|
+
if (existsSync(CONFIG_FILE)) {
|
|
61
|
+
const config = JSON.parse(readFileSync(CONFIG_FILE, 'utf8'));
|
|
62
|
+
if (Array.isArray(config.watch_dirs)) {
|
|
63
|
+
return config.watch_dirs;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
} catch (_) {
|
|
67
|
+
// Fail-safe: rewrite or fallback
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Create default config file if it does not exist
|
|
71
|
+
try {
|
|
72
|
+
writeFileSync(CONFIG_FILE, JSON.stringify({ watch_dirs: defaultDirs }, null, 2));
|
|
73
|
+
} catch (_) {}
|
|
74
|
+
|
|
75
|
+
return defaultDirs;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Scan a single transcript file in JSONL format (Antigravity).
|
|
80
|
+
* @param {string} filePath
|
|
81
|
+
*/
|
|
82
|
+
async function processJsonlFile(filePath) {
|
|
83
|
+
try {
|
|
84
|
+
const stat = statSync(filePath);
|
|
85
|
+
const lastPos = getWatchPosition(filePath);
|
|
86
|
+
|
|
87
|
+
if (stat.size <= lastPos) return;
|
|
88
|
+
|
|
89
|
+
// Read only new content appended to the file
|
|
90
|
+
const fileBuffer = readFileSync(filePath);
|
|
91
|
+
const newContentBuffer = fileBuffer.subarray(lastPos, stat.size);
|
|
92
|
+
const text = newContentBuffer.toString('utf8');
|
|
93
|
+
|
|
94
|
+
const lines = text.split('\n');
|
|
95
|
+
let addedCount = 0;
|
|
96
|
+
|
|
97
|
+
for (const line of lines) {
|
|
98
|
+
if (!line.trim()) continue;
|
|
99
|
+
|
|
100
|
+
let record;
|
|
101
|
+
try {
|
|
102
|
+
record = JSON.parse(line);
|
|
103
|
+
} catch (_) {
|
|
104
|
+
// Line might be incomplete/partially written — skip and parse next time
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Check if it's user prompt or assistant response
|
|
109
|
+
if (
|
|
110
|
+
record.content &&
|
|
111
|
+
(record.type === 'USER_INPUT' || record.type === 'PLANNER_RESPONSE' || record.source === 'MODEL')
|
|
112
|
+
) {
|
|
113
|
+
// Strip XML/markdown wrapper tags (like <USER_REQUEST> or <ADDITIONAL_METADATA>)
|
|
114
|
+
const cleanText = record.content.replace(/<[^>]+>[\s\S]*?<\/[^>]+>/g, '').trim();
|
|
115
|
+
if (cleanText.length < 15) continue;
|
|
116
|
+
|
|
117
|
+
const facts = extractHeuristic(cleanText);
|
|
118
|
+
for (const fact of facts) {
|
|
119
|
+
// Verify against exact duplicate
|
|
120
|
+
if (memoryExists(fact.content)) continue;
|
|
121
|
+
|
|
122
|
+
// Verify against semantic similarity
|
|
123
|
+
const similar = await searchHybrid(fact.content, 1);
|
|
124
|
+
if (similar.length > 0 && parseFloat(similar[0].similarity) >= DEDUP_THRESHOLD) {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Insert memory with provenance
|
|
129
|
+
const id = insertMemory(fact.content, fact.confidence, {
|
|
130
|
+
source_type: 'agent',
|
|
131
|
+
source_id: record.source === 'MODEL' ? 'antigravity-worker' : 'user-dialogue',
|
|
132
|
+
confidence: fact.confidence
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const embedding = await generateEmbedding(fact.content);
|
|
136
|
+
insertVector(id, embedding);
|
|
137
|
+
addedCount++;
|
|
138
|
+
console.error(`[persyst-watcher] Auto-extracted fact: "${fact.content}" (Memory #${id})`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (addedCount > 0) {
|
|
144
|
+
searchCache.invalidate();
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Persist new byte offset position
|
|
148
|
+
upsertWatchPosition(filePath, stat.size);
|
|
149
|
+
} catch (err) {
|
|
150
|
+
console.error(`[persyst-watcher] Failed to process JSONL file ${filePath}: ${err.message}`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Scan a single task file in JSON format (Roo Code / Cline).
|
|
156
|
+
* @param {string} filePath
|
|
157
|
+
*/
|
|
158
|
+
async function processJsonFile(filePath) {
|
|
159
|
+
try {
|
|
160
|
+
const lastMsgCount = getWatchPosition(filePath);
|
|
161
|
+
|
|
162
|
+
// Read full JSON (JSON objects are written entirely, not appended)
|
|
163
|
+
const contentText = readFileSync(filePath, 'utf8');
|
|
164
|
+
let task;
|
|
165
|
+
try {
|
|
166
|
+
task = JSON.parse(contentText);
|
|
167
|
+
} catch (_) {
|
|
168
|
+
return; // incomplete JSON, try again later
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const history = task.history;
|
|
172
|
+
if (!Array.isArray(history) || history.length <= lastMsgCount) return;
|
|
173
|
+
|
|
174
|
+
let addedCount = 0;
|
|
175
|
+
// Process only newly added messages
|
|
176
|
+
for (let i = lastMsgCount; i < history.length; i++) {
|
|
177
|
+
const msg = history[i];
|
|
178
|
+
if (!msg.content || typeof msg.content !== 'string') continue;
|
|
179
|
+
|
|
180
|
+
// Filter out system message structures
|
|
181
|
+
if (msg.role === 'user' || msg.role === 'assistant') {
|
|
182
|
+
const facts = extractHeuristic(msg.content);
|
|
183
|
+
for (const fact of facts) {
|
|
184
|
+
if (memoryExists(fact.content)) continue;
|
|
185
|
+
|
|
186
|
+
const similar = await searchHybrid(fact.content, 1);
|
|
187
|
+
if (similar.length > 0 && parseFloat(similar[0].similarity) >= DEDUP_THRESHOLD) {
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const id = insertMemory(fact.content, fact.confidence, {
|
|
192
|
+
source_type: 'agent',
|
|
193
|
+
source_id: msg.role === 'assistant' ? 'roo-worker' : 'user-dialogue',
|
|
194
|
+
confidence: fact.confidence
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
const embedding = await generateEmbedding(fact.content);
|
|
198
|
+
insertVector(id, embedding);
|
|
199
|
+
addedCount++;
|
|
200
|
+
console.error(`[persyst-watcher] Auto-extracted fact: "${fact.content}" (Memory #${id})`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (addedCount > 0) {
|
|
206
|
+
searchCache.invalidate();
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Persist message count index
|
|
210
|
+
upsertWatchPosition(filePath, history.length);
|
|
211
|
+
} catch (err) {
|
|
212
|
+
console.error(`[persyst-watcher] Failed to process JSON file ${filePath}: ${err.message}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Find files ending with a given extension recursively in a folder up to a certain depth.
|
|
218
|
+
* @param {string} dir
|
|
219
|
+
* @param {string} ext
|
|
220
|
+
* @param {number} depth
|
|
221
|
+
* @returns {Array<string>}
|
|
222
|
+
*/
|
|
223
|
+
function findFiles(dir, ext, depth = 3) {
|
|
224
|
+
const results = [];
|
|
225
|
+
if (depth < 0) return results;
|
|
226
|
+
|
|
227
|
+
try {
|
|
228
|
+
if (!existsSync(dir)) return results;
|
|
229
|
+
const items = readdirSync(dir);
|
|
230
|
+
for (const item of items) {
|
|
231
|
+
const path = join(dir, item);
|
|
232
|
+
let stat;
|
|
233
|
+
try { stat = statSync(path); } catch (_) { continue; }
|
|
234
|
+
|
|
235
|
+
if (stat.isDirectory()) {
|
|
236
|
+
results.push(...findFiles(path, ext, depth - 1));
|
|
237
|
+
} else if (item.endsWith(ext)) {
|
|
238
|
+
results.push(path);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
} catch (_) {}
|
|
242
|
+
|
|
243
|
+
return results;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Perform a single scan of watched directories.
|
|
248
|
+
*/
|
|
249
|
+
export async function scanDirectories() {
|
|
250
|
+
const watchDirs = loadWatchedDirs();
|
|
251
|
+
|
|
252
|
+
for (const dir of watchDirs) {
|
|
253
|
+
if (!existsSync(dir)) continue;
|
|
254
|
+
|
|
255
|
+
// Scan for JSONL (Antigravity transcripts)
|
|
256
|
+
const jsonlFiles = findFiles(dir, 'transcript.jsonl', 3);
|
|
257
|
+
for (const file of jsonlFiles) {
|
|
258
|
+
await processJsonlFile(file);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// Scan for JSON (Roo Code / Cline task files)
|
|
262
|
+
const jsonFiles = findFiles(dir, '.json', 2);
|
|
263
|
+
for (const file of jsonFiles) {
|
|
264
|
+
// Avoid processing general configurations/settings files
|
|
265
|
+
if (file.includes('tasks')) {
|
|
266
|
+
await processJsonFile(file);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Start the background log watcher daemon.
|
|
274
|
+
*/
|
|
275
|
+
export function startWatcher() {
|
|
276
|
+
if (intervalId) return;
|
|
277
|
+
|
|
278
|
+
console.error('[persyst-watcher] Starting background log watcher daemon...');
|
|
279
|
+
// Warm up config/paths
|
|
280
|
+
loadWatchedDirs();
|
|
281
|
+
|
|
282
|
+
// Run initial scan
|
|
283
|
+
scanDirectories().catch(err => {
|
|
284
|
+
console.error(`[persyst-watcher] Initial scan failed: ${err.message}`);
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
// Polling directory scan every 5 seconds
|
|
288
|
+
intervalId = setInterval(async () => {
|
|
289
|
+
try {
|
|
290
|
+
await scanDirectories();
|
|
291
|
+
} catch (err) {
|
|
292
|
+
console.error(`[persyst-watcher] Folder scan failed: ${err.message}`);
|
|
293
|
+
}
|
|
294
|
+
}, 5000);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* Stop the background log watcher daemon.
|
|
299
|
+
*/
|
|
300
|
+
export function stopWatcher() {
|
|
301
|
+
if (intervalId) {
|
|
302
|
+
clearInterval(intervalId);
|
|
303
|
+
intervalId = null;
|
|
304
|
+
console.error('[persyst-watcher] Background log watcher daemon stopped.');
|
|
305
|
+
}
|
|
306
|
+
}
|