wendkeep 0.74.0 → 0.75.0

package/src/observer-sql-publish.mjs

@@ -5,6 +5,8 @@ import {
   readdirSync,
   readFileSync,
   renameSync,
+  rmSync,
+  statSync,
   unlinkSync,
   writeFileSync,
 } from 'node:fs';
@@ -18,6 +20,7 @@ import { sanitizeObserverContent, sanitizeObserverMetadata } from './observer-pr
 
 export const SQL_OUTBOX_REL = '.brain/observer-sql-outbox';
 export const SQL_STATE_REL = '.brain/observer-sql-state.json';
+export const SQL_PUBLISHER_LEASE_REL = '.brain/observer-sql-publisher.lock';
 const SQL_SCHEMA_VERSION = 1;
 export const SQL_EVENT_BATCH_SIZE = 64;
 export const SQL_EVENT_BATCH_BYTES = 8 * 1024 * 1024;
@@ -158,7 +161,11 @@ function transcriptCalls({ projectId, sessionId, agentId, role, provider, modelF
     const tokens = tokenPayload(turn.usage);
     if (!prompt && !response && !tokens.total) return [];
     const model = text(turn.model || parsed.model || modelFallback, '?');
-    const callId = eventId('call', projectId, `${transcriptId}:${agentId}:${turn.turnId || index + 1}:${hash(content).slice(0, 16)}`);
+    const stableTurn = turn.turnId || index + 1;
+    const turnRevision = turn.status === 'complete'
+      ? 'complete'
+      : hash({ prompt, response, tokens, status: turn.status }).slice(0, 16);
+    const callId = eventId('call', projectId, `${transcriptId}:${agentId}:${stableTurn}:${turnRevision}`);
     return [{
       schema_version: 1,
       event_id: eventId('call-event', projectId, callId),
@@ -321,6 +328,136 @@ export function buildObserverSqlEventBatch({ vaultBase, projectId, input = {}, n
   return { events: dedupeEvents(events), nextState, scanned: Object.keys(nextState.files).length, changed };
 }
 
+function hookEventName(input = {}) {
+  return text(input.hook_event_name || input.hookEventName || input.event_name || input.event).toLowerCase();
+}
+
+function incrementalSessionContext(vaultBase, input = {}) {
+  const registry = readRegistry(vaultBase);
+  const requestedId = text(
+    input.session_id || input.sessionId || input.thread_id || input.threadId
+      || input.conversation_id || input.conversationId,
+  );
+  const pair = Object.entries(registry.sessions || {}).find(([sessionId, entry]) => (
+    (requestedId && sessionId === requestedId)
+    || (input.session_file && normalizePath(entry?.session_file) === normalizePath(input.session_file))
+  ));
+  if (!pair?.[1]?.session_file) return null;
+  const logicalPath = normalizePath(pair[1].session_file);
+  if (!logicalPath || logicalPath.startsWith('/') || logicalPath.includes('..')) return null;
+  const absolute = join(vaultBase, logicalPath);
+  if (!existsSync(absolute)) return null;
+  const content = readFileSync(absolute, 'utf8');
+  return {
+    sessionId: pair[0],
+    entry: pair[1],
+    file: { logicalPath, absolute },
+    content,
+    fm: parseFrontmatter(content),
+  };
+}
+
+/** Build only the session/subagent evidence named by the current hook payload. */
+export function buildObserverSqlIncrementalBatch({
+  vaultBase,
+  projectId,
+  input = {},
+  now = new Date(),
+  state = readState(vaultBase),
+  captureLevel = process.env.WENDKEEP_OBSERVER_CAPTURE_LEVEL || 'metadata',
+} = {}) {
+  if (!vaultBase || !projectId) throw new Error('vaultBase e projectId são obrigatórios.');
+  const eventName = hookEventName(input);
+  const nextState = {
+    schema_version: SQL_SCHEMA_VERSION,
+    files: { ...(state.files || {}) },
+    transcripts: { ...(state.transcripts || {}) },
+  };
+  if (eventName.includes('sessionstart')) {
+    return { events: [], nextState, scanned: 0, changed: 0, scope: 'drain-only' };
+  }
+  const context = incrementalSessionContext(vaultBase, input);
+  if (!context) return { events: [], nextState, scanned: 0, changed: 0, scope: 'drain-only' };
+
+  const occurredAt = isoNow(now);
+  const resolvedCaptureLevel = normalizeObserverCaptureLevel(captureLevel);
+  const events = [];
+  let changed = 0;
+  const safeContent = sanitizeObserverContent(context.content);
+  const contentHash = hash(safeContent);
+  const previous = state.files?.[context.file.logicalPath];
+  const revision = Math.max(1, Number(previous?.revision || 0) + (previous?.content_hash === contentHash ? 0 : 1));
+  nextState.files[context.file.logicalPath] = { content_hash: contentHash, revision };
+  if (previous?.content_hash !== contentHash) {
+    changed += 1;
+    events.push(documentEvent({
+      projectId,
+      logicalPath: context.file.logicalPath,
+      content: context.content,
+      metadata: context.fm,
+      revision,
+      occurredAt,
+    }));
+    const cost = parseSessionCost(context.content)
+      || { model: '?', mainCost: 0, subCost: 0, tokens: 0, subTokens: 0, ledger: [] };
+    const sessionProjection = sessionEvents({
+      projectId,
+      logicalPath: context.file.logicalPath,
+      content: context.content,
+      cost,
+      revision,
+      sessionId: context.sessionId,
+    }).events;
+    events.push(...sessionProjection.filter((event) => !(
+      eventName.includes('subagentstop') && event.kind === 'transcript.upsert'
+    )));
+  }
+
+  const mainAgentId = `${projectId}:${context.sessionId}:main`;
+  const provider = text(context.fm.provider);
+  const model = text(context.fm.modelo || context.fm.custo_modelo_label, '?');
+  let sources = sourceCandidates({
+    vaultBase,
+    logicalPath: context.file.logicalPath,
+    fm: context.fm,
+    input,
+  });
+  if (input.agent_transcript_path || input.agentTranscriptPath || eventName.includes('subagentstop')) {
+    sources = sources.filter((source) => source.role === 'subagent').slice(0, 1);
+  } else {
+    sources = sources.filter((source) => source.role === 'main').slice(0, 1);
+  }
+  for (const source of sources) {
+    const transcriptContent = readFileSync(source.path, 'utf8');
+    const fingerprint = hash(transcriptContent);
+    const previousTranscript = state.transcripts?.[source.transcriptId];
+    if (previousTranscript?.content_hash === fingerprint && previousTranscript?.coverage === resolvedCaptureLevel) continue;
+    const complete = completeTranscriptEvents({
+      projectId,
+      sessionId: context.sessionId,
+      mainAgentId,
+      provider,
+      model,
+      source,
+      now: occurredAt,
+      captureLevel: resolvedCaptureLevel,
+    });
+    nextState.transcripts[source.transcriptId] = {
+      content_hash: complete.fingerprint,
+      coverage: resolvedCaptureLevel,
+    };
+    events.push(...complete.events);
+    changed += 1;
+  }
+  return {
+    events: dedupeEvents(events),
+    nextState,
+    scanned: 1,
+    changed,
+    scope: `session:${context.sessionId}`,
+  };
+}
+
 function queueOutbox(vaultBase, batch) {
   mkdirSync(outboxDir(vaultBase), { recursive: true });
   const path = outboxPath(vaultBase, batch);
@@ -328,14 +465,184 @@ function queueOutbox(vaultBase, batch) {
   return path;
 }
 
+function eventCoalesceKey(event) {
+  const payload = event.payload || {};
+  return [
+    event.project_id,
+    event.kind,
+    payload.logical_path || payload.session_id || payload.agent_id || payload.rollup_key
+      || payload.call_id || payload.transcript_id || event.event_id,
+  ].join('\u001f');
+}
+
+const WAIT_ARRAY = new Int32Array(new SharedArrayBuffer(4));
+function waitSync(milliseconds) { Atomics.wait(WAIT_ARRAY, 0, 0, milliseconds); }
+
+function acquireBatchFileLease(path, waitMs = 0) {
+  const lock = `${path}.lock`;
+  const deadline = Date.now() + Math.max(0, waitMs);
+  do {
+    try {
+      mkdirSync(lock);
+      return lock;
+    } catch (error) {
+      if (error?.code !== 'EEXIST') throw error;
+      try {
+        if (Date.now() - statSync(lock).mtimeMs > 60_000) {
+          const stale = `${lock}.stale-${process.pid}-${Date.now()}`;
+          renameSync(lock, stale);
+          rmSync(stale, { recursive: true, force: true });
+          continue;
+        }
+      } catch { /* another process recovered or released it */ }
+      if (Date.now() >= deadline) return null;
+      waitSync(Math.min(5, Math.max(1, deadline - Date.now())));
+    }
+  } while (Date.now() <= deadline);
+  return null;
+}
+
+function releaseBatchFileLease(lock) {
+  if (lock) rmSync(lock, { recursive: true, force: true });
+}
+
+/** Queue a precise writer batch and replace older pending state for the same logical scope. */
+export function enqueueObserverSqlBatch(vaultBase, batch, { scope = 'incremental', now = new Date() } = {}) {
+  if (!batch?.project_id || !Array.isArray(batch.events)) throw new Error('Batch incremental do Observer inválido.');
+  if (!batch.events.length) return { queued: false, events: 0, path: '' };
+  mkdirSync(outboxDir(vaultBase), { recursive: true });
+  const path = join(outboxDir(vaultBase), `sql-live-${hash(`${batch.project_id}:${scope}`).slice(0, 24)}.json`);
+  const lease = acquireBatchFileLease(path, 200);
+  if (!lease) {
+    const fallback = queueOutbox(vaultBase, { ...batch, enqueued_at: isoNow(now), scope });
+    return { queued: true, events: batch.events.length, path: fallback, coalesced: false };
+  }
+  try {
+    const existing = readJson(path, { events: [] });
+    const merged = new Map();
+    for (const event of [...(existing.events || []), ...batch.events]) merged.set(eventCoalesceKey(event), event);
+    const timestamp = isoNow(now);
+    atomicJson(path, {
+      schema_version: SQL_SCHEMA_VERSION,
+      project_id: batch.project_id,
+      scope,
+      enqueued_at: existing.enqueued_at || timestamp,
+      updated_at: timestamp,
+      events: [...merged.values()],
+    });
+    return { queued: true, events: merged.size, path, coalesced: true };
+  } finally {
+    releaseBatchFileLease(lease);
+  }
+}
+
+/** Writer seam for note/archive commands: enqueue exactly one known logical document. */
+export function enqueueObserverDocumentChange({
+  vaultBase,
+  projectId,
+  logicalPath,
+  deleted = false,
+  now = new Date(),
+} = {}) {
+  const normalized = normalizePath(logicalPath);
+  if (!vaultBase || !projectId || !normalized || normalized.startsWith('/') || normalized.includes('..')) {
+    throw new Error('Documento incremental do Observer inválido.');
+  }
+  const state = readState(vaultBase);
+  const nextState = {
+    schema_version: SQL_SCHEMA_VERSION,
+    files: { ...(state.files || {}) },
+    transcripts: { ...(state.transcripts || {}) },
+  };
+  const previous = state.files?.[normalized];
+  const revision = Math.max(1, Number(previous?.revision || 0) + 1);
+  const occurredAt = isoNow(now);
+  let event;
+  if (deleted) {
+    if (!previous) return { queued: false, unchanged: true, events: 0 };
+    event = {
+      schema_version: 1,
+      event_id: eventId('document-delete', projectId, `${normalized}:${revision}`),
+      kind: 'document.delete',
+      project_id: projectId,
+      occurred_at: occurredAt,
+      payload: { logical_path: normalized, revision },
+    };
+    nextState.files[normalized] = { ...previous, revision, deleted: true };
+  } else {
+    const absolute = join(vaultBase, normalized);
+    if (!existsSync(absolute)) return { queued: false, missing: true, events: 0 };
+    const content = readFileSync(absolute, 'utf8');
+    const metadata = parseFrontmatter(content);
+    const contentHash = hash(sanitizeObserverContent(content));
+    if (previous?.content_hash === contentHash && previous?.deleted !== true) {
+      return { queued: false, unchanged: true, events: 0 };
+    }
+    event = documentEvent({ projectId, logicalPath: normalized, content, metadata, revision, occurredAt });
+    nextState.files[normalized] = { content_hash: contentHash, revision };
+  }
+  const queued = enqueueObserverSqlBatch(vaultBase, {
+    schema_version: SQL_SCHEMA_VERSION,
+    project_id: projectId,
+    events: [event],
+  }, { scope: `document:${normalized}`, now });
+  atomicJson(statePath(vaultBase), nextState);
+  return { ...queued, event_id: event.event_id, deleted };
+}
+
 export function listSqlOutbox(vaultBase) {
   const dir = outboxDir(vaultBase);
   if (!existsSync(dir)) return [];
-  return readdirSync(dir).filter((name) => /^sql-[a-f0-9]{24}\.json$/.test(name)).sort().flatMap((name) => {
+  return readdirSync(dir).filter((name) => /^sql-(?:live-)?[a-f0-9]{24}\.json$/.test(name)).sort().flatMap((name) => {
     try { return [{ path: join(dir, name), ...JSON.parse(readFileSync(join(dir, name), 'utf8')) }]; } catch { return []; }
   });
 }
 
+export function inspectObserverSqlOutbox(vaultBase, currentTime = Date.now()) {
+  const batches = listSqlOutbox(vaultBase);
+  let bytes = 0;
+  let oldestAt = '';
+  let events = 0;
+  for (const batch of batches) {
+    try { bytes += statSync(batch.path).size; } catch { /* raced with a drain */ }
+    events += Array.isArray(batch.events) ? batch.events.length : 0;
+    const candidate = text(batch.enqueued_at || batch.updated_at);
+    if (candidate && (!oldestAt || candidate < oldestAt)) oldestAt = candidate;
+  }
+  const oldestAgeMs = oldestAt ? Math.max(0, Number(currentTime) - Date.parse(oldestAt)) : 0;
+  return { batches: batches.length, events, bytes, oldest_at: oldestAt, oldest_age_ms: oldestAgeMs };
+}
+
+function leasePath(vaultBase) { return join(vaultBase, SQL_PUBLISHER_LEASE_REL); }
+
+function acquirePublisherLease(vaultBase, currentTime = Date.now()) {
+  const path = leasePath(vaultBase);
+  mkdirSync(join(path, '..'), { recursive: true });
+  const token = `${process.pid}-${currentTime}-${Math.random().toString(16).slice(2)}`;
+  try {
+    mkdirSync(path);
+  } catch (error) {
+    if (error?.code !== 'EEXIST') throw error;
+    let age = 0;
+    try { age = currentTime - statSync(path).mtimeMs; } catch { return null; }
+    if (age <= 60_000) return null;
+    const stale = `${path}.stale-${token}`;
+    try {
+      renameSync(path, stale);
+      rmSync(stale, { recursive: true, force: true });
+      mkdirSync(path);
+    } catch { return null; }
+  }
+  atomicJson(join(path, 'owner.json'), { token, pid: process.pid, acquired_at: new Date(currentTime).toISOString() });
+  return { path, token };
+}
+
+function releasePublisherLease(lease) {
+  if (!lease) return;
+  const owner = readJson(join(lease.path, 'owner.json'), {});
+  if (owner.token === lease.token) rmSync(lease.path, { recursive: true, force: true });
+}
+
 async function postSqlChunk({ url, projectId, events, fetchImpl = globalThis.fetch, token = '' }) {
   const controller = new AbortController();
   const rawBody = Buffer.from(JSON.stringify({ events }), 'utf8');
@@ -402,17 +709,65 @@ async function postSqlBatch({ url, projectId, events, fetchImpl = globalThis.fet
 export async function retryObserverSqlOutbox({ vaultBase, projectId, url, fetchImpl = globalThis.fetch, token = process.env.WENDKEEP_OBSERVER_TOKEN || '' } = {}) {
   const pending = listSqlOutbox(vaultBase);
   if (!url) return { attempted: 0, confirmed: 0, pending: pending.length };
+  const lease = acquirePublisherLease(vaultBase);
+  if (!lease) return { attempted: 0, confirmed: 0, pending: pending.length, busy: true };
   let attempted = 0;
   let confirmed = 0;
-  for (const batch of pending) {
-    attempted += 1;
-    try {
-      await postSqlBatch({ url, projectId, events: batch.events, fetchImpl, token });
-      unlinkSync(batch.path);
-      confirmed += 1;
-    } catch { break; }
+  try {
+    for (const batch of pending) {
+      if (batch.project_id && batch.project_id !== projectId) continue;
+      const batchLease = acquireBatchFileLease(batch.path);
+      if (!batchLease) continue;
+      attempted += 1;
+      try {
+        const current = readJson(batch.path, null);
+        if (!current?.events?.length) continue;
+        await postSqlBatch({ url, projectId, events: current.events, fetchImpl, token });
+        if (existsSync(batch.path)) unlinkSync(batch.path);
+        confirmed += 1;
+      } catch { break; }
+      finally { releaseBatchFileLease(batchLease); }
+    }
+    return { attempted, confirmed, pending: listSqlOutbox(vaultBase).length, busy: false };
+  } finally {
+    releasePublisherLease(lease);
   }
-  return { attempted, confirmed, pending: listSqlOutbox(vaultBase).length };
+}
+
+/** Hook path: enqueue at most one touched session/subagent, then drain under one lease. */
+export async function publishObserverSqlIncremental({
+  vaultBase,
+  projectId,
+  url = process.env.WENDKEEP_OBSERVER_URL || '',
+  input = {},
+  now = new Date(),
+  fetchImpl = globalThis.fetch,
+  token = process.env.WENDKEEP_OBSERVER_TOKEN || '',
+  captureLevel = process.env.WENDKEEP_OBSERVER_CAPTURE_LEVEL || 'metadata',
+} = {}) {
+  if (!vaultBase || !projectId) throw new Error('vaultBase e projectId são obrigatórios.');
+  const state = readState(vaultBase);
+  const batch = buildObserverSqlIncrementalBatch({
+    vaultBase, projectId, input, now, state, captureLevel,
+  });
+  const queued = enqueueObserverSqlBatch(vaultBase, {
+    schema_version: SQL_SCHEMA_VERSION,
+    project_id: projectId,
+    events: batch.events,
+  }, { scope: batch.scope, now });
+  if (batch.events.length) atomicJson(statePath(vaultBase), batch.nextState);
+  const replay = await retryObserverSqlOutbox({ vaultBase, projectId, url, fetchImpl, token });
+  const pending = listSqlOutbox(vaultBase).length;
+  return {
+    ok: pending === 0,
+    queued: pending > 0,
+    scanned: batch.scanned,
+    changed: batch.changed,
+    enqueued_events: queued.events || 0,
+    pending,
+    replay,
+    hookExitCode: 0,
+  };
 }
 
 export async function publishObserverSql({ vaultBase, projectId, url = process.env.WENDKEEP_OBSERVER_URL || '', input = {}, now = new Date(), fetchImpl = globalThis.fetch, token = process.env.WENDKEEP_OBSERVER_TOKEN || '', captureLevel = process.env.WENDKEEP_OBSERVER_CAPTURE_LEVEL || 'metadata' } = {}) {
@@ -423,17 +778,22 @@ export async function publishObserverSql({ vaultBase, projectId, url = process.e
     ? await readRemoteDocuments({ url, projectId, fetchImpl, token })
     : {};
   const batch = buildObserverSqlEventBatch({ vaultBase, projectId, input, now, state, remoteDocuments, captureLevel });
-  atomicJson(statePath(vaultBase), batch.nextState);
-  if (!batch.events.length) return { ok: true, queued: false, scanned: batch.scanned, changed: batch.changed, pending: listSqlOutbox(vaultBase).length, replay };
+  if (!batch.events.length) {
+    atomicJson(statePath(vaultBase), batch.nextState);
+    return { ok: true, queued: false, scanned: batch.scanned, changed: batch.changed, pending: listSqlOutbox(vaultBase).length, replay };
+  }
   if (!url) {
     queueOutbox(vaultBase, { schema_version: SQL_SCHEMA_VERSION, project_id: projectId, events: batch.events });
+    atomicJson(statePath(vaultBase), batch.nextState);
     return { ok: false, queued: true, scanned: batch.scanned, changed: batch.changed, pending: listSqlOutbox(vaultBase).length, replay, hookExitCode: 0 };
   }
   try {
     const response = await postSqlBatch({ url, projectId, events: batch.events, fetchImpl, token });
+    atomicJson(statePath(vaultBase), batch.nextState);
     return { ok: true, queued: false, scanned: batch.scanned, changed: batch.changed, pending: listSqlOutbox(vaultBase).length, replay, response };
   } catch (error) {
     queueOutbox(vaultBase, { schema_version: SQL_SCHEMA_VERSION, project_id: projectId, events: batch.events });
+    atomicJson(statePath(vaultBase), batch.nextState);
     return { ok: false, queued: true, scanned: batch.scanned, changed: batch.changed, pending: listSqlOutbox(vaultBase).length, replay, hookExitCode: 0, error: error.message };
   }
 }