twinclaw 1.3.2 → 1.4.0

package/dist/services/db.js

@@ -1,12 +1,10 @@
 import Database from 'better-sqlite3';
-import * as sqliteVec from 'sqlite-vec';
 import path from 'path';
 import fs from 'fs';
 import { getConfigValue } from '../config/json-config.js';
 import { getDatabasePath, ensureWorkspaceSubdirs } from '../config/workspace.js';
 ensureWorkspaceSubdirs();
 const DB_PATH = getDatabasePath();
-const MEMORY_EMBEDDING_DIM = Number(getConfigValue('MEMORY_EMBEDDING_DIM') ?? '1536') || 1536;
 const DEFAULT_MEMORY_TOP_K = 5;
 const MAX_MEMORY_TOP_K = 50;
 const ALLOW_CROSS_SESSION_MEMORY_FALLBACK = getConfigValue('MEMORY_ALLOW_CROSS_SESSION_FALLBACK')?.trim().toLowerCase() === 'true';
@@ -14,8 +12,6 @@ if (!fs.existsSync(path.dirname(DB_PATH))) {
     fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
 }
 export const db = new Database(DB_PATH);
-// Load sqlite-vec C extension
-sqliteVec.load(db);
 db.pragma('foreign_keys = ON');
 db.exec(`
   CREATE TABLE IF NOT EXISTS sessions (
@@ -58,11 +54,11 @@ db.exec(`
     FOREIGN KEY(session_id) REFERENCES sessions(session_id)
   );
 
-  -- Virtual table for vector search
-  CREATE VIRTUAL TABLE IF NOT EXISTS vec_memory USING vec0(
-    embedding float[${MEMORY_EMBEDDING_DIM}],
-    session_id TEXT,
-    fact_text TEXT
+  -- Virtual table for full-text search (FTS5) memory
+  CREATE VIRTUAL TABLE IF NOT EXISTS semantic_memory_fts USING fts5(
+    session_id UNINDEXED,
+    fact_text,
+    tokenize="porter unicode61"
   );
 
   CREATE TABLE IF NOT EXISTS reasoning_nodes (
@@ -298,13 +294,6 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_runtime_budget_events_created
     ON runtime_budget_events(created_at DESC);
 `);
-function serializeEmbedding(embedding) {
-    const sqliteVecWithSerializer = sqliteVec;
-    if (typeof sqliteVecWithSerializer.serializeFloat32 === 'function') {
-        return sqliteVecWithSerializer.serializeFloat32(embedding);
-    }
-    return Buffer.from(new Float32Array(embedding).buffer);
-}
 function normalizeTopK(topK) {
     if (!Number.isFinite(topK)) {
         return DEFAULT_MEMORY_TOP_K;
@@ -327,55 +316,26 @@ export function getSessionMessages(sessionId) {
     const stmt = db.prepare('SELECT * FROM messages WHERE session_id = ? ORDER BY rowid ASC');
     return stmt.all(sessionId);
 }
-export function saveMemoryEmbedding(sessionId, factText, embedding) {
-    const stmt = db.prepare('INSERT INTO vec_memory (embedding, session_id, fact_text) VALUES (?, ?, ?)');
-    const result = stmt.run(serializeEmbedding(embedding), sessionId, factText);
+export function saveMemoryFact(sessionId, factText) {
+    const stmt = db.prepare('INSERT INTO semantic_memory_fts (session_id, fact_text) VALUES (?, ?)');
+    const result = stmt.run(sessionId, factText);
     return Number(result.lastInsertRowid);
 }
-export function getNearestMemories(queryEmbedding, topK = 5, currentSessionId) {
-    const boundedTopK = normalizeTopK(topK);
-    const matcher = serializeEmbedding(queryEmbedding);
-    const stmt = db.prepare('SELECT rowid AS memory_rowid, session_id, fact_text, distance FROM vec_memory WHERE embedding MATCH ? ORDER BY distance ASC LIMIT ?');
-    const rows = stmt.all(matcher, boundedTopK * 3);
-    if (!currentSessionId) {
-        return rows.slice(0, boundedTopK);
-    }
-    const scoped = rows.filter((row) => row.session_id === currentSessionId);
-    if (!ALLOW_CROSS_SESSION_MEMORY_FALLBACK) {
-        return scoped.slice(0, boundedTopK);
-    }
-    const global = rows.filter((row) => row.session_id !== currentSessionId);
-    return [...scoped, ...global].slice(0, boundedTopK);
-}
-/**
- * Keyword-based fallback search when vector embeddings are unavailable.
- * Uses simple SQL LIKE matching.
- */
-export function keywordSearchMemories(query, topK = 5, currentSessionId) {
+export function searchMemoriesFTS(query, topK = 5, currentSessionId) {
     const boundedTopK = normalizeTopK(topK);
-    // Simple tokenization: remove common punctuation and split by whitespace
-    const tokens = query
+    const words = query
         .toLowerCase()
         .replace(/[^\w\s]/g, ' ')
         .split(/\s+/)
-        .filter((t) => t.length > 2);
-    if (tokens.length === 0) {
+        .filter((w) => w.length > 2);
+    if (words.length === 0) {
         return [];
     }
-    // Build a query with LIKE for each token (OR logic)
-    // Distance is mocked as 1.0 for keyword matches
-    const placeholders = tokens.map(() => 'fact_text LIKE ?').join(' OR ');
-    const sql = `
-    SELECT rowid AS memory_rowid, session_id, fact_text, 1.0 AS distance 
-    FROM vec_memory 
-    WHERE ${placeholders}
-    ORDER BY rowid DESC
-    LIMIT ?
-  `;
+    // Use natural language query format or standard FTS query
+    const ftsQuery = words.map((w) => `"${w}"*`).join(' OR ');
+    const stmt = db.prepare('SELECT rowid AS memory_rowid, session_id, fact_text, rank AS distance FROM semantic_memory_fts WHERE semantic_memory_fts MATCH ? ORDER BY rank ASC LIMIT ?');
     try {
-        const stmt = db.prepare(sql);
-        const values = tokens.map((t) => `%${t}%`);
-        const rows = stmt.all(...values, boundedTopK * 3);
+        const rows = stmt.all(ftsQuery, boundedTopK * 3);
         if (!currentSessionId) {
             return rows.slice(0, boundedTopK);
         }
@@ -386,10 +346,10 @@ export function keywordSearchMemories(query, topK = 5, currentSessionId) {
         const global = rows.filter((row) => row.session_id !== currentSessionId);
         return [...scoped, ...global].slice(0, boundedTopK);
     }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        console.error(`[DB] keywordSearchMemories failed (tokens=${tokens.length}, topK=${boundedTopK}, scoped=${currentSessionId ? 'yes' : 'no'}): ${message}`);
-        throw error;
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.warn(`[DB] FTS search failed for query '${ftsQuery}': ${message}`);
+        return [];
     }
 }
 // Re-export domain-specific DB helpers for backwards compatibility.

package/dist/services/device-pairing.js

@@ -2,6 +2,7 @@ import { randomBytes } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 import { logThought } from '../utils/logger.js';
+import { getWorkspaceDir } from '../config/workspace.js';
 const CODE_ALPHABET = 'ABCDEFGHJKLMNPQRSTUVWXYZ23456789';
 const CODE_LENGTH = 8;
 const DEFAULT_CODE_TTL_MS = 5 * 60 * 1000; // 5 minutes for device pairing
@@ -33,7 +34,7 @@ export class DevicePairingService {
     constructor(options = {}) {
         this.#credentialsDir = options.credentialsDir
             ? path.resolve(options.credentialsDir)
-            : path.resolve('memory', 'devices');
+            : path.join(getWorkspaceDir(), 'memory', 'devices');
         this.#codeTtlMs = options.codeTtlMs ?? DEFAULT_CODE_TTL_MS;
         this.#maxPending = options.maxPendingPerChannel ?? DEFAULT_MAX_PENDING;
         // Load existing devices

package/dist/services/dm-pairing.js

@@ -1,6 +1,7 @@
 import { randomBytes } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
+import { getWorkspaceDir } from '../config/workspace.js';
 const CODE_ALPHABET = 'ABCDEFGHJKLMNPQRSTUVWXYZ23456789';
 const CODE_LENGTH = 8;
 const DEFAULT_CODE_TTL_MS = 60 * 60 * 1000;
@@ -66,7 +67,7 @@ export class DmPairingService {
     constructor(options = {}) {
         this.#credentialsDir = options.credentialsDir
             ? path.resolve(options.credentialsDir)
-            : path.resolve('memory', 'credentials');
+            : path.join(getWorkspaceDir(), 'memory', 'credentials');
         this.#codeTtlMs =
             Number.isFinite(options.codeTtlMs) && (options.codeTtlMs ?? 0) > 0
                 ? Number(options.codeTtlMs)

package/dist/services/hooks.js

@@ -93,6 +93,18 @@ class HooksService {
         }
         return results;
     }
+    async executeHook(event, data = {}) {
+        const context = {
+            event,
+            timestamp: new Date().toISOString(),
+            data,
+        };
+        // Use the sessionId if provided in the data payload
+        if (typeof data.sessionId === 'string') {
+            context.sessionId = data.sessionId;
+        }
+        return this.runHook(event, context);
+    }
     async #runScript(scriptPath, context) {
         const fullPath = path.isAbsolute(scriptPath)
             ? scriptPath

package/dist/services/job-scheduler.js

@@ -2,6 +2,7 @@ import cron from 'node-cron';
 import { logThought } from '../utils/logger.js';
 import fs from 'node:fs';
 import path from 'node:path';
+import { getWorkspaceDir } from '../config/workspace.js';
 /**
  * Centralized job scheduler for TwinClaw's proactive execution layer.
  *
@@ -30,7 +31,8 @@ export class JobScheduler {
      * @param persistenceDir - Directory to store job persistence files
      */
     constructor(persistenceDir) {
-        this.#persistencePath = path.resolve(persistenceDir || './memory', 'scheduler-jobs.json');
+        const workspaceMemoryDir = path.join(getWorkspaceDir(), 'memory');
+        this.#persistencePath = path.resolve(persistenceDir || workspaceMemoryDir, 'scheduler-jobs.json');
         this.#loadPersistedJobs();
     }
     /** Register a new repeating job. Throws if a job with the same ID already exists. */

package/dist/services/model-router.js

@@ -290,6 +290,7 @@ export class ModelRouter {
             if (lastTriedModelId && lastTriedModelId !== config.id) {
                 this.metrics.failoverCount += 1;
                 this.recordEvent('failover', config, `Automatic fallback ${lastTriedModelId} -> ${config.id}.`);
+                console.warn(`[ModelRouter] ⚠️ Fallback triggered: Switching from ${lastTriedModelId} to ${config.id}`);
             }
             const payload = {
                 model: config.model,
@@ -317,6 +318,7 @@ export class ModelRouter {
                 const retryWaitMs = Math.min(firstAttempt.rateLimitCooldownMs, this.intelligentPacingMaxWaitMs);
                 if (retryWaitMs > 0) {
                     this.recordEvent('cooldown_wait', config, `Intelligent pacing wait ${retryWaitMs}ms before retrying ${config.id}.`);
+                    console.warn(`[ModelRouter] ⏳ Rate limited. Intelligent pacing: waiting ${retryWaitMs}ms before retrying ${config.id}...`);
                     await this.sleepFn(retryWaitMs);
                 }
                 const retryCooldown = this.getModelCooldownState(config.id);
@@ -400,6 +402,7 @@ export class ModelRouter {
             if (lastTriedModelId && lastTriedModelId !== config.id) {
                 this.metrics.failoverCount += 1;
                 this.recordEvent('failover', config, `Automatic fallback ${lastTriedModelId} -> ${config.id}.`);
+                console.warn(`[ModelRouter] ⚠️ Streaming Fallback triggered: Switching from ${lastTriedModelId} to ${config.id}`);
             }
             const payload = {
                 model: config.model,

package/dist/services/semantic-memory.js

@@ -1,8 +1,8 @@
 import { createHash } from 'node:crypto';
-import { chunkText, EmbeddingService } from './embedding-service.js';
-import { getNearestMemories, keywordSearchMemories, saveMemoryEmbedding, } from './db.js';
+import { chunkText } from './embedding-service.js';
+import { searchMemoriesFTS, saveMemoryFact, } from './db.js';
 import { getMemoryProvenanceRows, getReasoningEvidenceExpansion, getReasoningNodesByClaimKey, linkMemoryProvenance, upsertReasoningEdge, upsertReasoningNode, } from './db-reasoning.js';
-const embeddingService = new EmbeddingService();
+import { getHooksService } from './hooks.js';
 const NEGATION_PATTERN = /\b(no|not|never|without|cannot|can't|won't|dont|don't|didnt|didn't|isnt|isn't|arent|aren't|wasnt|wasn't|weren't)\b/i;
 const CLAIM_KEY_STOPWORDS = /\b(a|an|the|and|or|to|of|for|in|on|at|with|is|are|was|were|be|been|being|do|does|did|this|that|it|as|by|from|no|not|never|without|cannot|cant|wont)\b/g;
 const TASK_HINT_PATTERN = /\b(todo|task|implement|fix|build|create|refactor|ship|deploy)\b/i;
@@ -20,11 +20,7 @@ export async function indexConversationTurn(sessionId, role, content) {
         const limitedChunks = chunks.slice(0, 2);
         for (const chunk of limitedChunks) {
             const taggedChunk = `${role.toUpperCase()}: ${chunk}`;
-            const embedding = await embeddingService.embedText(taggedChunk);
-            if (!embedding) {
-                continue;
-            }
-            const memoryRowId = saveMemoryEmbedding(sessionId, taggedChunk, embedding);
+            const memoryRowId = saveMemoryFact(sessionId, taggedChunk);
             const node = buildReasoningNode(sessionId, role, taggedChunk);
             upsertReasoningNode(node);
             linkMemoryProvenance(memoryRowId, node.nodeId, sessionId);
@@ -41,6 +37,11 @@ export async function indexConversationTurn(sessionId, role, content) {
             previousNodeId = node.nodeId;
             reconcileClaimRelations(node);
         }
+        // Trigger proactive context alignment hook
+        const hooks = getHooksService();
+        hooks.executeHook('afterIndexTurn', { sessionId, role, content }).catch((err) => {
+            console.warn(`[SemanticMemory] Hook execution failed for afterIndexTurn:`, err);
+        });
     }
     catch (err) {
         const message = err instanceof Error ? err.message : String(err);
@@ -48,24 +49,13 @@ export async function indexConversationTurn(sessionId, role, content) {
     }
 }
 export async function retrieveEvidenceAwareMemoryContext(sessionId, prompt, topK = 5) {
-    const embedding = await embeddingService.embedText(prompt);
     const candidateLimit = Math.max(topK * 3, topK);
-    let nearest;
-    let fallbackUsed = false;
-    if (embedding) {
-        nearest = getNearestMemories(embedding, candidateLimit, sessionId);
-    }
-    else {
-        nearest = keywordSearchMemories(prompt, candidateLimit, sessionId);
-        fallbackUsed = true;
-    }
+    const nearest = searchMemoriesFTS(prompt, candidateLimit, sessionId);
     if (nearest.length === 0) {
         return {
             context: '',
             diagnostics: [
-                fallbackUsed
-                    ? 'Keyword-fallback memory retrieval found no candidates for the active session scope.'
-                    : 'Memory retrieval found no nearest vector candidates for the active session scope.'
+                'Memory retrieval found no FTS candidates for the active session scope.'
             ],
             conflictCount: 0,
             hitCount: 0,
@@ -93,9 +83,7 @@ export async function retrieveEvidenceAwareMemoryContext(sessionId, prompt, topK
     return {
         context: `Retrieved evidence-backed memories:\n${lines.join('\n')}${conflictNote}`,
         diagnostics: [
-            fallbackUsed
-                ? `Keyword-fallback memory retrieval ranked ${nearest.length} candidates down to ${ranked.length} items.`
-                : `Hybrid memory retrieval ranked ${nearest.length} vector candidates down to ${ranked.length} evidence-backed items.`,
+            `FTS memory retrieval ranked ${nearest.length} candidates down to ${ranked.length} evidence-backed items.`,
             `Graph traversal depth=${MAX_GRAPH_TRAVERSAL_DEPTH} collected ${expandedEdges.length} relation edge(s).`,
             conflictCount > 0
                 ? `Detected contradiction signals in ${conflictCount} candidate(s).`
@@ -207,7 +195,9 @@ function buildGraphRelationCounts(edges) {
 }
 function scoreCandidate(sessionId, factText, memoryRowId, distance, options) {
     const provenance = options.provenance;
-    const vectorScore = 1 / (1 + Math.max(0, distance));
+    // SQLite FTS5 bm25 rank is negative. More negative means better match.
+    const ftsRelevance = Math.max(0, -distance);
+    const ftsScore = Math.min(1, ftsRelevance * 0.15); // Scale BM25 to 0.0 - 1.0 range
     const graphCounts = provenance
         ? options.graphRelationCounts.get(provenance.nodeId) ?? {
             supports: 0,
@@ -223,7 +213,7 @@ function scoreCandidate(sessionId, factText, memoryRowId, distance, options) {
     const relationScore = Math.min(1, supports * 0.2 + depends * 0.11 + derived * 0.08);
     const recencyScore = estimateRecencyScore(provenance?.updatedAt);
     const contradictionPenalty = Math.min(0.35, contradicts * 0.08);
-    const score = vectorScore * 0.62 + relationScore * 0.26 + recencyScore * 0.12 - contradictionPenalty;
+    const score = ftsScore * 0.62 + relationScore * 0.26 + recencyScore * 0.12 - contradictionPenalty;
     return {
         sessionId,
         factText,

package/dist/skills/builtin.js

@@ -5,6 +5,7 @@ import { randomUUID } from 'node:crypto';
 import { executeProgram, executeShell } from './shell.js';
 import { logToolCall } from '../utils/logger.js';
 import { validatePathInWorkspace } from '../config/workspace.js';
+import { getPersonaEditorTool } from '../tools/persona-editor.js';
 function buildReadFileSkill() {
     return {
         name: 'fs.read',
@@ -1038,6 +1039,39 @@ function buildVenvSkill() {
         },
     };
 }
+function buildPersonaEditorSkill() {
+    return {
+        name: 'persona.edit',
+        group: 'group:core',
+        aliases: ['edit_persona', 'update_user_profile', 'update_soul'],
+        description: 'Programmatically edit the agent\'s identity (soul.md) or user profile (user.md).',
+        parameters: {
+            type: 'object',
+            properties: {
+                target: { type: 'string', enum: ['user', 'soul'] },
+                operation: { type: 'string', enum: ['append', 'replace', 'clear'] },
+                content: { type: 'string' },
+            },
+            required: ['target', 'operation'],
+        },
+        async execute(input) {
+            if (input.operation !== 'clear' && typeof input.content !== 'string') {
+                return { ok: false, output: 'Content must be provided for append/replace operations' };
+            }
+            const request = input;
+            const editor = getPersonaEditorTool();
+            const result = await editor.editPersona({
+                target: request.target,
+                operation: request.operation,
+                content: request.content || '',
+            });
+            return {
+                ok: result.success,
+                output: result.message,
+            };
+        },
+    };
+}
 export function createBuiltinSkills() {
     return [
         buildReadFileSkill(),
@@ -1073,5 +1107,6 @@ export function createBuiltinSkills() {
         buildDefenderSkill(),
         buildPowerShellISESkill(),
         buildVenvSkill(),
+        buildPersonaEditorSkill(),
     ];
 }

package/dist/tools/persona-editor.js

@@ -0,0 +1,56 @@
+import { ensureIdentityFiles } from '../config/identity-bootstrap.js';
+import { getWorkspaceDir } from '../config/workspace.js';
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import { logThought } from '../utils/logger.js';
+export class PersonaEditorTool {
+    async editPersona(request) {
+        try {
+            const workspaceDir = getWorkspaceDir();
+            const identityDir = path.join(workspaceDir, 'identity');
+            // Ensure the directory and foundation files exist
+            ensureIdentityFiles();
+            const targetFile = request.target === 'user' ? 'user.md' : 'soul.md';
+            const filePath = path.join(identityDir, targetFile);
+            let currentContent = '';
+            try {
+                currentContent = await fs.readFile(filePath, 'utf-8');
+            }
+            catch (err) {
+                // If it doesn't exist, we treat it as empty
+                currentContent = '';
+            }
+            let newContent = currentContent;
+            switch (request.operation) {
+                case 'append':
+                    // Append with an explicit timestamp so the agent knows when this was learned
+                    const timestamp = new Date().toISOString();
+                    newContent = `${currentContent.trim()}\n\n[Learned at ${timestamp}]\n${request.content.trim()}`;
+                    break;
+                case 'replace':
+                    newContent = request.content.trim();
+                    break;
+                case 'clear':
+                    newContent = '';
+                    break;
+                default:
+                    return { success: false, message: `Unknown operation: ${request.operation}` };
+            }
+            await fs.writeFile(filePath, newContent, 'utf-8');
+            await logThought(`[PersonaEditor] Updated ${targetFile} via operation '${request.operation}'. Let the system know.`);
+            return { success: true, message: `Successfully updated ${targetFile}` };
+        }
+        catch (err) {
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            await logThought(`[PersonaEditor] Error updating persona: ${errorMsg}`);
+            return { success: false, message: `Failed to edit persona: ${errorMsg}` };
+        }
+    }
+}
+let personaEditorTool = null;
+export function getPersonaEditorTool() {
+    if (!personaEditorTool) {
+        personaEditorTool = new PersonaEditorTool();
+    }
+    return personaEditorTool;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "twinclaw",
-  "version": "1.3.2",
+  "version": "1.4.0",
   "description": "Eagle-eyed agentic AI gateway with multi-modal hooks and proactive memory.",
   "main": "dist/index.js",
   "bin": {
@@ -53,7 +53,6 @@
     "playwright": "^1.58.2",
     "playwright-core": "^1.58.2",
     "qrcode-terminal": "^0.12.0",
-    "sqlite-vec": "^0.1.7-alpha.2",
     "ts-node": "^10.9.2",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",
@@ -83,4 +82,4 @@
     "request": "npm:@cypress/request@^3.0.10",
     "request-promise": "npm:@cypress/request-promise@^5.0.0"
   }
-}
+}