wendkeep 0.73.0 → 0.75.0

package/src/observer-sql-store.mjs

@@ -4,9 +4,12 @@ import { createRequire } from 'node:module';
 import { join, basename, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { decodeTranscript, encodeTranscript } from './observer-transcript-store.mjs';
+import {
+  chunkMarkdownDocument, recallEvidence, recallTerms,
+} from '../packages/vault/src/evidence-recall.mjs';
 
 export const OBSERVER_SQL_FILE = 'observer.sqlite';
-export const OBSERVER_SQL_SCHEMA_VERSION = 3;
+export const OBSERVER_SQL_SCHEMA_VERSION = 5;
 export const OBSERVER_EVENT_SCHEMA_VERSION = 1;
 
 const SCHEMA_DIR = fileURLToPath(new URL('../schema/observer/', import.meta.url));
@@ -19,6 +22,7 @@ const EVENT_KINDS = new Set([
 const OBSERVER_SQL_MINIMUM_NODE = '22.13.0';
 const require = createRequire(import.meta.url);
 let DatabaseSync;
+const DATABASE_PATHS = new WeakMap();
 
 export function observerSqlRuntimeSupport(version = process.versions.node) {
   const current = String(version || '0.0.0');
@@ -71,6 +75,10 @@ function hash(value) {
   return createHash('sha256').update(typeof value === 'string' ? value : json(value)).digest('hex');
 }
 
+function scopedIdentity(projectId, externalId) {
+  return `${projectId}\u001f${externalId}`;
+}
+
 function validProjectId(value) { return typeof value === 'string' && PROJECT_ID_RE.test(value); }
 
 function requireProject(db, projectId) {
@@ -94,7 +102,9 @@ export function openObserverDatabase(dataDir) {
   if (!dataDir) throw new Error('dataDir é obrigatório.');
   mkdirSync(dataDir, { recursive: true });
   const SqliteDatabase = observerDatabaseSync();
-  const db = new SqliteDatabase(join(dataDir, OBSERVER_SQL_FILE));
+  const databasePath = join(dataDir, OBSERVER_SQL_FILE);
+  const db = new SqliteDatabase(databasePath);
+  DATABASE_PATHS.set(db, databasePath);
   db.exec('PRAGMA foreign_keys = ON; PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;');
   return db;
 }
@@ -102,27 +112,146 @@ export function openObserverDatabase(dataDir) {
 export function migrateObserverDatabase(db) {
   if (!db) throw new Error('db é obrigatório.');
   db.exec('CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)');
-  const applied = new Set(db.prepare('SELECT version FROM schema_migrations').all().map((row) => Number(row.version)));
+  const migrationColumns = db.prepare('PRAGMA table_info(schema_migrations)').all().map((row) => row.name);
+  if (!migrationColumns.includes('checksum')) db.exec("ALTER TABLE schema_migrations ADD COLUMN checksum TEXT NOT NULL DEFAULT ''");
+  const appliedRows = db.prepare('SELECT version, name, checksum FROM schema_migrations ORDER BY version').all();
+  const applied = new Map(appliedRows.map((row) => [Number(row.version), row]));
+  const backups = [];
   for (const file of migrationFiles()) {
     const version = Number(file.split('-')[0]);
-    if (applied.has(version)) continue;
     const sql = readFileSync(join(SCHEMA_DIR, file), 'utf8');
+    const checksum = hash(sql);
+    const previous = applied.get(version);
+    if (previous) {
+      if (previous.name !== file || (previous.checksum && previous.checksum !== checksum)) {
+        const error = new Error(`Checksum da migração ${version} não corresponde ao arquivo ${file}.`);
+        error.code = 'WENDKEEP_OBSERVER_MIGRATION_CHECKSUM_MISMATCH';
+        throw error;
+      }
+      if (!previous.checksum) {
+        db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(checksum, version);
+      }
+      continue;
+    }
+    if (/^\s*--\s*wendkeep:structural\b/m.test(sql) && applied.size > 0) {
+      const databasePath = DATABASE_PATHS.get(db);
+      if (!databasePath) throw new Error('Caminho do Observer desconhecido para backup estrutural.');
+      const backupPath = `${databasePath}.pre-${String(version).padStart(3, '0')}-${Date.now()}.bak`;
+      const escaped = backupPath.replaceAll("'", "''");
+      db.exec(`VACUUM INTO '${escaped}'`);
+      backups.push(backupPath);
+    }
     db.exec('BEGIN IMMEDIATE');
     try {
       db.exec(sql);
-      db.prepare('INSERT INTO schema_migrations(version, name, applied_at) VALUES (?, ?, ?)').run(version, file, now());
+      db.prepare('INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)').run(version, file, now(), checksum);
       db.exec('COMMIT');
     } catch (error) {
       db.exec('ROLLBACK');
       throw error;
     }
   }
+  rebuildSqlEvidenceIndex(db, { missingOnly: true });
   return {
     version: Number(db.prepare('SELECT COALESCE(MAX(version), 0) AS version FROM schema_migrations').get().version || 0),
-    applied: db.prepare('SELECT version, name, applied_at FROM schema_migrations ORDER BY version').all(),
+    applied: db.prepare('SELECT version, name, applied_at, checksum FROM schema_migrations ORDER BY version').all(),
+    backups,
   };
 }
 
+export function observerFts5Support(db) {
+  try {
+    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS evidence_chunks_fts USING fts5(
+      chunk_id UNINDEXED,
+      project_id UNINDEXED,
+      logical_path,
+      title,
+      heading,
+      content,
+      tokenize = 'unicode61 remove_diacritics 2'
+    )`);
+    return { supported: true, engine: 'fts5' };
+  } catch (error) {
+    return { supported: false, engine: 'lexical-fallback', reason: text(error?.message || error) };
+  }
+}
+
+function synchronizeSqlFts(db) {
+  const support = observerFts5Support(db);
+  if (!support.supported) return support;
+  const chunks = Number(db.prepare('SELECT COUNT(*) AS count FROM document_chunks').get().count || 0);
+  const indexed = Number(db.prepare('SELECT COUNT(*) AS count FROM evidence_chunks_fts').get().count || 0);
+  if (chunks === indexed) return support;
+  db.exec(`DELETE FROM evidence_chunks_fts;
+    INSERT INTO evidence_chunks_fts(chunk_id, project_id, logical_path, title, heading, content)
+    SELECT chunk_id, project_id, logical_path, title, heading, content
+    FROM document_chunks;`);
+  return { ...support, rebuilt: true, chunks };
+}
+
+function writeSqlDocumentChunks(db, {
+  projectId, logicalPath, content, metadata = {}, entityType = 'document', capturedAt = '',
+} = {}) {
+  const chunks = chunkMarkdownDocument({
+    projectId,
+    logicalPath,
+    content,
+    metadata: { ...metadata, observed_at: metadata.observed_at || capturedAt },
+    entityType,
+  });
+  db.prepare('DELETE FROM document_chunks WHERE project_id = ? AND logical_path = ?').run(projectId, logicalPath);
+  const fts = observerFts5Support(db);
+  if (fts.supported) {
+    db.prepare('DELETE FROM evidence_chunks_fts WHERE project_id = ? AND logical_path = ?').run(projectId, logicalPath);
+  }
+  const insert = db.prepare(`INSERT INTO document_chunks(
+    chunk_id, project_id, logical_path, title, heading, entity_type, change_slug,
+    session_id, work_session_id, authority, observed_at, validity, content_hash, ordinal, content
+  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+  const insertFts = fts.supported
+    ? db.prepare('INSERT INTO evidence_chunks_fts(chunk_id, project_id, logical_path, title, heading, content) VALUES (?, ?, ?, ?, ?, ?)')
+    : null;
+  for (const chunk of chunks) {
+    insert.run(
+      chunk.chunk_id, chunk.project_id, chunk.logical_path, chunk.title, chunk.heading,
+      chunk.entity_type, chunk.change_slug, chunk.session_id, chunk.work_session_id,
+      chunk.authority, chunk.observed_at, chunk.validity, chunk.content_hash,
+      chunk.ordinal, chunk.content,
+    );
+    insertFts?.run(chunk.chunk_id, chunk.project_id, chunk.logical_path, chunk.title, chunk.heading, chunk.content);
+  }
+  return { chunks: chunks.length, fts };
+}
+
+export function rebuildSqlEvidenceIndex(db, { missingOnly = false } = {}) {
+  const table = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'document_chunks'").get();
+  if (!table) return { documents: 0, chunks: 0, fts: { supported: false, engine: 'unavailable' } };
+  const documents = db.prepare(`SELECT d.project_id, d.logical_path, d.content, d.metadata_json, d.entity_type, d.captured_at
+    FROM documents d
+    WHERE d.deleted_at IS NULL
+      AND (? = 0 OR NOT EXISTS (
+        SELECT 1 FROM document_chunks c
+        WHERE c.project_id = d.project_id AND c.logical_path = d.logical_path
+      ))
+    ORDER BY d.project_id, d.logical_path`).all(missingOnly ? 1 : 0);
+  let chunks = 0;
+  let fts = observerFts5Support(db);
+  for (const document of documents) {
+    const result = writeSqlDocumentChunks(db, {
+      projectId: document.project_id,
+      logicalPath: document.logical_path,
+      content: document.content,
+      metadata: parseJson(document.metadata_json),
+      entityType: document.entity_type,
+      capturedAt: document.captured_at,
+    });
+    chunks += result.chunks;
+    fts = result.fts;
+  }
+  fts = synchronizeSqlFts(db);
+  return { documents: documents.length, chunks, fts };
+}
+
 export function ensureObserverDatabase(dataDir) {
   const db = openObserverDatabase(dataDir);
   try {
@@ -171,9 +300,9 @@ function ensureSession(db, projectId, payload) {
   const sessionId = text(payload.session_id || payload.sessionId);
   if (!sessionId) throw new Error('session_id ausente.');
   db.prepare(`
-    INSERT INTO sessions(session_id, project_id, provider, status, summary, change_slug, started_at, ended_at, updated_at, metadata_json)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(session_id) DO UPDATE SET
+    INSERT INTO sessions(session_pk, session_id, project_id, provider, status, summary, change_slug, started_at, ended_at, updated_at, metadata_json)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(project_id, session_id) DO UPDATE SET
       provider = CASE WHEN excluded.provider <> '' THEN excluded.provider ELSE sessions.provider END,
       status = CASE WHEN excluded.status <> '' THEN excluded.status ELSE sessions.status END,
       summary = CASE WHEN excluded.summary <> '' THEN excluded.summary ELSE sessions.summary END,
@@ -183,7 +312,7 @@ function ensureSession(db, projectId, payload) {
       updated_at = excluded.updated_at,
       metadata_json = CASE WHEN excluded.metadata_json <> '{}' THEN excluded.metadata_json ELSE sessions.metadata_json END
   `).run(
-    sessionId, projectId, text(payload.provider), text(payload.status, 'unknown'), text(payload.summary), text(payload.change_slug || payload.changeSlug),
+    scopedIdentity(projectId, sessionId), sessionId, projectId, text(payload.provider), text(payload.status, 'unknown'), text(payload.summary), text(payload.change_slug || payload.changeSlug),
     payload.started_at || payload.startedAt || null, payload.ended_at || payload.endedAt || null, now(), json(payload.metadata),
   );
   return sessionId;
@@ -194,9 +323,9 @@ function ensureAgent(db, projectId, payload) {
   if (!agentId) throw new Error('agent_id ausente.');
   const sessionId = ensureSession(db, projectId, payload);
   db.prepare(`
-    INSERT INTO agent_runs(agent_id, project_id, session_id, parent_agent_id, role, agent_name, agent_type, workflow, status, model, effort, started_at, ended_at, metadata_json)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(agent_id) DO UPDATE SET
+    INSERT INTO agent_runs(agent_pk, agent_id, project_id, session_id, parent_agent_id, role, agent_name, agent_type, workflow, status, model, effort, started_at, ended_at, metadata_json)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(project_id, agent_id) DO UPDATE SET
       parent_agent_id = COALESCE(excluded.parent_agent_id, agent_runs.parent_agent_id),
       role = excluded.role,
       agent_name = CASE WHEN excluded.agent_name <> '' THEN excluded.agent_name ELSE agent_runs.agent_name END,
@@ -209,7 +338,7 @@ function ensureAgent(db, projectId, payload) {
       ended_at = COALESCE(excluded.ended_at, agent_runs.ended_at),
       metadata_json = CASE WHEN excluded.metadata_json <> '{}' THEN excluded.metadata_json ELSE agent_runs.metadata_json END
   `).run(
-    agentId, projectId, sessionId, payload.parent_agent_id || payload.parentAgentId || null, text(payload.role, 'main'),
+    scopedIdentity(projectId, agentId), agentId, projectId, sessionId, payload.parent_agent_id || payload.parentAgentId || null, text(payload.role, 'main'),
     text(payload.agent_name || payload.agentName), text(payload.agent_type || payload.agentType), text(payload.workflow),
     text(payload.status, 'unknown'), text(payload.model), text(payload.effort), payload.started_at || null, payload.ended_at || null, json(payload.metadata),
   );
@@ -266,6 +395,14 @@ function applyDocument(db, event) {
     VALUES (?, ?, ?, ?, 'upsert', ?, ?, ?, ?, ?, ?)
   `).run(event.event_id, event.project_id, text(p.entity_type || p.entityType, 'memory'), logicalPath, revision, contentHash,
     text(p.source_session_id || p.sourceSessionId), text(p.source_turn_id || p.sourceTurnId), event.occurred_at, json(p));
+  writeSqlDocumentChunks(db, {
+    projectId: event.project_id,
+    logicalPath,
+    content,
+    metadata: p.metadata,
+    entityType: text(p.entity_type || p.entityType, 'memory'),
+    capturedAt: text(p.captured_at || event.occurred_at),
+  });
   return { stale: false };
 }
 
@@ -279,6 +416,10 @@ function applyDocumentDelete(db, event) {
     VALUES (?, ?, ?, ?, 'delete', ?, '', ?, ?, ?, ?)
   `).run(event.event_id, event.project_id, text(event.payload.entity_type || event.payload.entityType, 'memory'), logicalPath,
     integer(event.payload.revision, 1), text(event.payload.source_session_id || event.payload.sourceSessionId), text(event.payload.source_turn_id || event.payload.sourceTurnId), event.occurred_at, json(event.payload));
+  db.prepare('DELETE FROM document_chunks WHERE project_id = ? AND logical_path = ?').run(event.project_id, logicalPath);
+  if (observerFts5Support(db).supported) {
+    db.prepare('DELETE FROM evidence_chunks_fts WHERE project_id = ? AND logical_path = ?').run(event.project_id, logicalPath);
+  }
   return { stale: false };
 }
 
@@ -289,19 +430,19 @@ function applyUsageRollup(db, event) {
   const [input, cacheWrite, cacheRead, output, reasoning, total] = tokenFields(p.tokens);
   const rollupKey = text(p.rollup_key || p.rollupKey) || [event.project_id, sessionId, agentId, p.model_provider || p.modelProvider || '', p.model || '', p.effort || ''].join(':');
   const revision = integer(p.revision, 1);
-  const current = db.prepare('SELECT revision FROM usage_rollups WHERE rollup_key = ?').get(rollupKey);
+  const current = db.prepare('SELECT revision FROM usage_rollups WHERE project_id = ? AND rollup_key = ?').get(event.project_id, rollupKey);
   if (current && revision < Number(current.revision)) return { stale: true };
   db.prepare(`
-    INSERT INTO usage_rollups(rollup_key, project_id, session_id, agent_id, role, provider, model_provider, model, effort, calls, tokens_input, tokens_cache_write, tokens_cache_read, tokens_output, tokens_reasoning, tokens_total, cost_usd, cost_status, pricing_source, pricing_version, wasted_usd, revision, occurred_at, source_event_id, metadata_json)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(rollup_key) DO UPDATE SET
+    INSERT INTO usage_rollups(rollup_pk, rollup_key, project_id, session_id, agent_id, role, provider, model_provider, model, effort, calls, tokens_input, tokens_cache_write, tokens_cache_read, tokens_output, tokens_reasoning, tokens_total, cost_usd, cost_status, pricing_source, pricing_version, wasted_usd, revision, occurred_at, source_event_id, metadata_json)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(project_id, rollup_key) DO UPDATE SET
       calls = excluded.calls, tokens_input = excluded.tokens_input, tokens_cache_write = excluded.tokens_cache_write,
       tokens_cache_read = excluded.tokens_cache_read, tokens_output = excluded.tokens_output, tokens_reasoning = excluded.tokens_reasoning,
       tokens_total = excluded.tokens_total, cost_usd = excluded.cost_usd, cost_status = excluded.cost_status,
       pricing_source = excluded.pricing_source, pricing_version = excluded.pricing_version, wasted_usd = excluded.wasted_usd,
       revision = excluded.revision, occurred_at = excluded.occurred_at, source_event_id = excluded.source_event_id, metadata_json = excluded.metadata_json
   `).run(
-    rollupKey, event.project_id, sessionId, agentId, text(p.role, 'main'), text(p.provider), text(p.model_provider || p.modelProvider), text(p.model), text(p.effort),
+    scopedIdentity(event.project_id, rollupKey), rollupKey, event.project_id, sessionId, agentId, text(p.role, 'main'), text(p.provider), text(p.model_provider || p.modelProvider), text(p.model), text(p.effort),
     integer(p.calls), input, cacheWrite, cacheRead, output, reasoning, total, number(p.cost_usd ?? p.costUsd), text(p.cost_status || p.costStatus, 'unknown'),
     text(p.pricing_source || p.pricingSource), text(p.pricing_version || p.pricingVersion), number(p.wasted_usd ?? p.wastedUsd), revision, event.occurred_at, event.event_id, json(p.metadata),
   );
@@ -314,17 +455,17 @@ function applyCall(db, event) {
   const agentId = ensureAgent(db, event.project_id, p);
   const callId = text(p.call_id || p.callId);
   if (!callId) throw new Error('call_id ausente.');
-  if (db.prepare('SELECT call_id FROM llm_calls WHERE call_id = ?').get(callId)) {
+  if (db.prepare('SELECT call_id FROM llm_calls WHERE project_id = ? AND call_id = ?').get(event.project_id, callId)) {
     const error = new Error(`call_id já existe: ${callId}`);
     error.code = 'call_conflict';
     throw error;
   }
   const [input, cacheWrite, cacheRead, output, reasoning, total] = tokenFields(p.tokens);
   db.prepare(`
-    INSERT INTO llm_calls(call_id, project_id, session_id, agent_id, role, provider, model_provider, model, effort, sequence, occurred_at, tokens_input, tokens_cache_write, tokens_cache_read, tokens_output, tokens_reasoning, tokens_total, cost_usd, cost_status, transcript_id, prompt_text, response_text, status, metadata_json)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    INSERT INTO llm_calls(call_pk, call_id, project_id, session_id, agent_id, role, provider, model_provider, model, effort, sequence, occurred_at, tokens_input, tokens_cache_write, tokens_cache_read, tokens_output, tokens_reasoning, tokens_total, cost_usd, cost_status, transcript_id, prompt_text, response_text, status, metadata_json)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `).run(
-    callId, event.project_id, sessionId, agentId, text(p.role, 'main'), text(p.provider), text(p.model_provider || p.modelProvider), text(p.model), text(p.effort), integer(p.sequence),
+    scopedIdentity(event.project_id, callId), callId, event.project_id, sessionId, agentId, text(p.role, 'main'), text(p.provider), text(p.model_provider || p.modelProvider), text(p.model), text(p.effort), integer(p.sequence),
     text(p.occurred_at || event.occurred_at), input, cacheWrite, cacheRead, output, reasoning, total, number(p.cost_usd ?? p.costUsd), text(p.cost_status || p.costStatus, 'unknown'),
     p.transcript_id || p.transcriptId || null, text(p.prompt_text || p.promptText || p.prompt), text(p.response_text || p.responseText || p.response), text(p.status, 'complete'), json(p.metadata),
   );
@@ -339,14 +480,14 @@ function applyTranscript(db, event) {
   if (!transcriptId) throw new Error('transcript_id ausente.');
   const encoded = encodeTranscript(p.content);
   db.prepare(`
-    INSERT INTO transcripts(transcript_id, project_id, session_id, agent_id, coverage, codec, content_gzip, content_sha256, original_bytes, compressed_bytes, source, occurred_at, metadata_json)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(transcript_id) DO UPDATE SET
+    INSERT INTO transcripts(transcript_pk, transcript_id, project_id, session_id, agent_id, coverage, codec, content_gzip, content_sha256, original_bytes, compressed_bytes, source, occurred_at, metadata_json)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(project_id, transcript_id) DO UPDATE SET
       coverage = excluded.coverage, codec = excluded.codec, content_gzip = excluded.content_gzip,
       content_sha256 = excluded.content_sha256, original_bytes = excluded.original_bytes, compressed_bytes = excluded.compressed_bytes,
       source = excluded.source, occurred_at = excluded.occurred_at, metadata_json = excluded.metadata_json
   `).run(
-    transcriptId, event.project_id, sessionId, agentId, text(p.coverage, 'complete'), encoded.codec, encoded.content_gzip, encoded.content_sha256,
+    scopedIdentity(event.project_id, transcriptId), transcriptId, event.project_id, sessionId, agentId, text(p.coverage, 'complete'), encoded.codec, encoded.content_gzip, encoded.content_sha256,
     encoded.original_bytes, encoded.compressed_bytes, text(p.source), event.occurred_at, json(p.metadata),
   );
   return { stale: false };
@@ -392,6 +533,8 @@ export function ingestObserverEvents(db, { projectId, events = [] } = {}) {
       }
       continue;
     }
+    db.exec('SAVEPOINT ingest_one');
+    let savepointOpen = true;
     try {
       db.prepare('INSERT INTO ingest_events(event_id, project_id, kind, payload_hash, payload_json, occurred_at, ingested_at, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
         .run(event.event_id, projectId, event.kind, payloadHash, json(event.payload), event.occurred_at, now(), 'accepted');
@@ -404,8 +547,12 @@ export function ingestObserverEvents(db, { projectId, events = [] } = {}) {
         result.accepted += 1;
         result.results.push({ accepted: true, event_id: event.event_id });
       }
+      db.exec('RELEASE ingest_one');
+      savepointOpen = false;
     } catch (error) {
-      db.prepare('DELETE FROM ingest_events WHERE event_id = ?').run(event.event_id);
+      if (savepointOpen) {
+        try { db.exec('ROLLBACK TO ingest_one'); } finally { db.exec('RELEASE ingest_one'); }
+      }
       if (error?.code === 'call_conflict' || error?.code === 'document_conflict') {
         result.conflicts += 1;
         result.results.push({ accepted: false, conflict: true, event_id: event.event_id, errors: [error.message] });
@@ -542,11 +689,38 @@ export function readSqlTree(db, projectId, prefix = '') {
   return { schema_version: OBSERVER_SQL_SCHEMA_VERSION, project_id: projectId, prefix, documents: rows };
 }
 
-export function searchSqlDocuments(db, projectId, query = '') {
+export function searchSqlDocuments(db, projectId, query = '', { forceLexical = false } = {}) {
   requireProject(db, projectId);
-  const q = `%${text(query).replace(/[%_]/g, '\\$&')}%`;
-  const rows = db.prepare(`SELECT logical_path, entity_type, title, content_hash, revision, captured_at, source_session_id, content FROM documents WHERE project_id = ? AND deleted_at IS NULL AND (logical_path LIKE ? ESCAPE '\\' OR title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\') ORDER BY captured_at DESC, logical_path LIMIT 100`).all(projectId, q, q, q);
-  return rows.map((row) => ({ ...row, excerpt: row.content.slice(0, 240) }));
+  const terms = recallTerms(query);
+  if (!terms.length) return [];
+  const fts = forceLexical
+    ? { supported: false, engine: 'lexical-fallback' }
+    : observerFts5Support(db);
+  const columns = `c.chunk_id, c.project_id, c.logical_path, c.title, c.heading,
+    c.entity_type, c.change_slug, c.session_id, c.work_session_id, c.authority,
+    c.observed_at, c.validity, c.content_hash AS chunk_content_hash, c.ordinal, c.content,
+    d.content_hash, d.revision, d.captured_at, d.source_session_id`;
+  let rows;
+  if (fts.supported) {
+    const expression = [...new Set(terms)]
+      .map((term) => `"${term.replaceAll('"', '""')}"`)
+      .join(' OR ');
+    rows = db.prepare(`SELECT ${columns} FROM evidence_chunks_fts f
+      JOIN document_chunks c ON c.chunk_id = f.chunk_id
+      JOIN documents d ON d.project_id = c.project_id AND d.logical_path = c.logical_path
+      WHERE evidence_chunks_fts MATCH ? AND c.project_id = ? AND d.deleted_at IS NULL
+      ORDER BY bm25(evidence_chunks_fts, 0, 0, 1.5, 3.0, 2.5, 1.0), c.observed_at DESC
+      LIMIT 200`).all(expression, projectId);
+  } else {
+    // FTS5 may be unavailable in a valid Observer runtime. Rank the complete project corpus
+    // instead of truncating by recency before matching, which would make old exact evidence
+    // permanently unreachable.
+    rows = db.prepare(`SELECT ${columns} FROM document_chunks c
+      JOIN documents d ON d.project_id = c.project_id AND d.logical_path = c.logical_path
+      WHERE c.project_id = ? AND d.deleted_at IS NULL
+      ORDER BY c.observed_at DESC, c.logical_path, c.ordinal`).all(projectId);
+  }
+  return recallEvidence(rows, query, { topK: 5 });
 }
 
 export function readSqlSync(db, projectId) {
@@ -568,6 +742,48 @@ export function readSqlProject(db, projectId) {
   return db.prepare('SELECT * FROM projects WHERE project_id = ?').get(projectId);
 }
 
+export function upsertSqlProjectSnapshot(db, snapshot) {
+  const projectId = text(snapshot?.project_id || snapshot?.projectId);
+  requireProject(db, projectId);
+  const eventId = text(snapshot?.event_id);
+  const capturedAt = text(snapshot?.captured_at);
+  if (!eventId || !capturedAt || Number.isNaN(Date.parse(capturedAt))) throw new Error('snapshot SQL inválido.');
+  const current = db.prepare('SELECT event_id, captured_at FROM project_snapshots WHERE project_id = ?').get(projectId);
+  if (current?.event_id === eventId) return { accepted: false, duplicate: true, event_id: eventId };
+  if (current && (current.captured_at > capturedAt
+    || (current.captured_at === capturedAt && current.event_id >= eventId))) {
+    return { accepted: false, stale: true, event_id: eventId };
+  }
+  db.prepare(`INSERT INTO project_snapshots(project_id, event_id, captured_at, snapshot_json)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(project_id) DO UPDATE SET
+      event_id = excluded.event_id,
+      captured_at = excluded.captured_at,
+      snapshot_json = excluded.snapshot_json`).run(projectId, eventId, capturedAt, json(snapshot));
+  return { accepted: true, duplicate: false, event_id: eventId };
+}
+
+export function readSqlProjectSnapshot(db, projectId) {
+  requireProject(db, projectId);
+  const row = db.prepare('SELECT snapshot_json FROM project_snapshots WHERE project_id = ?').get(projectId);
+  return row ? parseJson(row.snapshot_json, null) : null;
+}
+
+export function readSqlProjectOverview(db, projectId) {
+  const project = readSqlProject(db, projectId);
+  const snapshot = readSqlProjectSnapshot(db, projectId);
+  const events = db.prepare('SELECT COUNT(*) AS count FROM ingest_events WHERE project_id = ?').get(projectId);
+  return {
+    projectId: project.project_id,
+    projectName: project.project_name,
+    wendkeepVersion: project.wendkeep_version,
+    registeredAt: project.registered_at,
+    updatedAt: project.updated_at,
+    eventCount: integer(events.count),
+    snapshot,
+  };
+}
+
 export function listSqlProjects(db) {
   return db.prepare('SELECT * FROM projects ORDER BY project_name, project_id').all();
 }

package/src/observer-store.mjs

@@ -86,8 +86,7 @@ function newer(left, right) {
   return leftTime > rightTime || (leftTime === rightTime && String(left?.event_id).localeCompare(String(right?.event_id)) > 0);
 }
 
-export function rebuildObserverIndex(dataDir) {
-  ensureDataDir(dataDir);
+function deriveObserverIndex(dataDir) {
   const byProject = new Map();
   for (const event of readEvents(dataDir)) {
     const current = byProject.get(event.project_id);
@@ -106,15 +105,28 @@ export function rebuildObserverIndex(dataDir) {
     const item = byProject.get(event.project_id);
     if (item) item.eventCount += 1;
   }
-  const index = {
+  return {
     schema_version: OBSERVER_DATA_SCHEMA_VERSION,
     generated_at: new Date().toISOString(),
     projects: [...byProject.values()].sort((a, b) => a.projectId.localeCompare(b.projectId)),
   };
+}
+
+export function rebuildObserverIndex(dataDir) {
+  ensureDataDir(dataDir);
+  const index = deriveObserverIndex(dataDir);
   atomicJson(join(dataDir, OBSERVER_INDEX_FILE), index);
   return index;
 }
 
+/** Read a legacy source without creating or refreshing legacy authority files. */
+export function readObserverIndexSource(dataDir) {
+  ensureDataDir(dataDir);
+  const index = readJson(join(dataDir, OBSERVER_INDEX_FILE), null);
+  if (index?.schema_version === OBSERVER_DATA_SCHEMA_VERSION && Array.isArray(index.projects)) return index;
+  return deriveObserverIndex(dataDir);
+}
+
 export function readObserverIndex(dataDir) {
   ensureDataDir(dataDir);
   const path = join(dataDir, OBSERVER_INDEX_FILE);

package/src/observer.mjs

@@ -1,11 +1,12 @@
 import { homedir } from 'node:os';
 import { isAbsolute, resolve } from 'node:path';
-import { appendObserverEvent, readObserverIndex, registerObserverProject } from './observer-store.mjs';
+import { readObserverIndexSource } from './observer-store.mjs';
 import { buildProjectSnapshot } from './observer-snapshot.mjs';
 import { compareMemoryParity } from './observer-memory-publish.mjs';
 import { publishObserverSql } from './observer-sql-publish.mjs';
+import { migrateObserverData } from './observer-sql-migrate.mjs';
 import { startObserverServer } from './observer-server.mjs';
-import { ensureObserverDatabase, migrateObserverDatabase, listSqlProjects, OBSERVER_SQL_FILE, OBSERVER_SQL_SCHEMA_VERSION } from './observer-sql-store.mjs';
+import { ensureObserverDatabase, migrateObserverDatabase, listSqlProjects, registerSqlProject, upsertSqlProjectSnapshot, OBSERVER_SQL_FILE, OBSERVER_SQL_SCHEMA_VERSION } from './observer-sql-store.mjs';
 import { resolveProjectVault } from '../packages/vault/src/project-vault.mjs';
 import { observerAuthHeaders, resolveObserverToken } from './observer-auth.mjs';
 
@@ -16,12 +17,15 @@ Uso:
                           [--allow-non-loopback] [--token TOKEN]
   wendkeep observer register --project P [--vault V] [--data-dir D] [--json]
   wendkeep observer publish --project P [--vault V] [--data-dir D] [--json]
+  wendkeep observer reconcile --project P [--vault V] [--data-dir D] [--url U]
+                              [--capture-level metadata|messages|full-transcript] [--json]
   wendkeep observer memory import --project P [--vault V] [--url U] [--token TOKEN]
                                     [--capture-level metadata|messages|full-transcript] [--json]
   wendkeep observer status [--data-dir D] [--json]
 
 O Observer local pode manter snapshots operacionais e uma cópia completa da memória em volume
 Docker. O comando memory import faz a primeira migração de um vault para o container.
+Hooks apenas enfileiram/drenam alterações incrementais; reconcile é a varredura integral explícita.
 `;
 
 function optionValue(argv, name) {
@@ -73,6 +77,19 @@ function databaseSummary(dir) {
   } finally { db.close(); }
 }
 
+function sqlProjectsSummary(dir) {
+  const db = ensureObserverDatabase(dir);
+  try {
+    return listSqlProjects(db).map((project) => ({
+      projectId: project.project_id,
+      projectName: project.project_name,
+      wendkeepVersion: project.wendkeep_version,
+      updatedAt: project.updated_at,
+      authority: 'sqlite',
+    }));
+  } finally { db.close(); }
+}
+
 export async function runObserver(argv = [], { write = (chunk) => process.stdout.write(chunk) } = {}) {
   const [sub] = argv;
   const asJson = argv.includes('--json');
@@ -84,7 +101,8 @@ export async function runObserver(argv = [], { write = (chunk) => process.stdout
   const token = resolveObserverToken(optionValue(argv, '--token'));
 
   if (sub === 'status') {
-    print({ ...summary(readObserverIndex(dir)), database: databaseSummary(dir) }, asJson, write);
+    const legacy = summary(readObserverIndexSource(dir));
+    print({ ...legacy, projects: sqlProjectsSummary(dir), legacy_projects: legacy.projects, database: databaseSummary(dir) }, asJson, write);
     return 0;
   }
 
@@ -131,6 +149,12 @@ export async function runObserver(argv = [], { write = (chunk) => process.stdout
       token,
       captureLevel: optionValue(argv, '--capture-level') || process.env.WENDKEEP_OBSERVER_CAPTURE_LEVEL || 'metadata',
     });
+    const snapshotResponse = await fetch(`${String(url).replace(/\/$/, '')}/v1/projects/${encodeURIComponent(snapshot.project_id)}/snapshot`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(snapshot),
+    });
+    if (!snapshotResponse.ok) throw new Error(`Observer não importou o snapshot: HTTP ${snapshotResponse.status}.`);
     const parity = await compareMemoryParity({
       vaultBase: vault,
       projectId: snapshot.project_id,
@@ -148,24 +172,90 @@ export async function runObserver(argv = [], { write = (chunk) => process.stdout
     return result.ok ? 0 : 1;
   }
 
+  if (sub === 'reconcile') {
+    const root = projectRoot(argv);
+    const vault = vaultBase(argv, root);
+    const snapshot = buildProjectSnapshot({ vaultBase: vault, projectRoot: root });
+    const url = optionValue(argv, '--url') || process.env.WENDKEEP_OBSERVER_URL || '';
+    if (!url) {
+      const db = ensureObserverDatabase(dir);
+      let migration;
+      try {
+        migration = migrateObserverData({
+          dataDir: dir,
+          vaultBase: vault,
+          projectId: snapshot.project_id,
+          projectName: snapshot.project_name,
+          database: db,
+        });
+        upsertSqlProjectSnapshot(db, snapshot);
+      } finally { db.close(); }
+      const result = { ok: migration.rejected === 0 && migration.conflicts === 0, project_id: snapshot.project_id, mode: 'local-sqlite', migration };
+      print(result, asJson, write);
+      return result.ok ? 0 : 1;
+    }
+    const registration = await fetch(`${String(url).replace(/\/$/, '')}/v1/projects/${encodeURIComponent(snapshot.project_id)}`, {
+      method: 'PUT',
+      headers: observerAuthHeaders(token, { 'content-type': 'application/json', accept: 'application/json' }),
+      body: JSON.stringify({
+        project_id: snapshot.project_id,
+        project_name: snapshot.project_name,
+        wendkeep_version: snapshot.wendkeep_version,
+      }),
+    });
+    if (!registration.ok) throw new Error(`Observer não registrou o projeto: HTTP ${registration.status}.`);
+    const sql = await publishObserverSql({
+      vaultBase: vault,
+      projectId: snapshot.project_id,
+      url,
+      token,
+      captureLevel: optionValue(argv, '--capture-level') || process.env.WENDKEEP_OBSERVER_CAPTURE_LEVEL || 'metadata',
+    });
+    const snapshotResponse = await fetch(`${String(url).replace(/\/$/, '')}/v1/projects/${encodeURIComponent(snapshot.project_id)}/snapshot`, {
+      method: 'POST',
+      headers: observerAuthHeaders(token, { 'content-type': 'application/json', accept: 'application/json' }),
+      body: JSON.stringify(snapshot),
+    });
+    if (!snapshotResponse.ok) throw new Error(`Observer não reconciliou o snapshot: HTTP ${snapshotResponse.status}.`);
+    const parity = await compareMemoryParity({ vaultBase: vault, projectId: snapshot.project_id, url, token });
+    const result = { ok: sql.ok && parity.missing === 0 && parity.mismatched === 0, project_id: snapshot.project_id, mode: 'remote-sqlite', sql, parity };
+    print(result, asJson, write);
+    return result.ok ? 0 : 1;
+  }
+
   if (!['register', 'publish'].includes(sub)) throw new Error('observer: subcomando desconhecido: ' + sub);
   const root = projectRoot(argv);
   const vault = vaultBase(argv, root);
   const snapshot = buildProjectSnapshot({ vaultBase: vault, projectRoot: root });
 
   if (sub === 'register') {
-    const result = registerObserverProject(dir, {
-      projectId: snapshot.project_id,
-      projectName: snapshot.project_name,
-      wendkeepVersion: snapshot.wendkeep_version,
-    });
-    if (!result.registered) throw new Error(result.errors.join(' '));
-    print(result, asJson, write);
+    const db = ensureObserverDatabase(dir);
+    let sql;
+    try {
+      sql = registerSqlProject(db, {
+        projectId: snapshot.project_id,
+        projectName: snapshot.project_name,
+        wendkeepVersion: snapshot.wendkeep_version,
+      });
+    } finally { db.close(); }
+    if (!sql.registered) throw new Error(sql.errors.join(' '));
+    print({ ...sql, authority: 'sqlite' }, asJson, write);
     return 0;
   }
 
-  const result = appendObserverEvent(dir, snapshot);
-  if (!result.accepted && !result.duplicate) throw new Error(result.errors.join(' '));
-  print({ ok: true, ...result }, asJson, write);
-  return 0;
+  const db = ensureObserverDatabase(dir);
+  let migration;
+  try {
+    migration = migrateObserverData({
+      dataDir: dir,
+      vaultBase: vault,
+      projectId: snapshot.project_id,
+      projectName: snapshot.project_name,
+      database: db,
+    });
+    upsertSqlProjectSnapshot(db, snapshot);
+  } finally { db.close(); }
+  const result = { ok: migration.rejected === 0 && migration.conflicts === 0, authority: 'sqlite', migration };
+  print(result, asJson, write);
+  return result.ok ? 0 : 1;
 }