wendkeep 0.70.0 → 0.72.0

package/schema/observer/001-authority.sql

@@ -0,0 +1,107 @@
+PRAGMA foreign_keys = ON;
+
+CREATE TABLE IF NOT EXISTS schema_migrations (
+  version INTEGER PRIMARY KEY,
+  name TEXT NOT NULL,
+  applied_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS projects (
+  project_id TEXT PRIMARY KEY,
+  project_name TEXT NOT NULL,
+  wendkeep_version TEXT NOT NULL DEFAULT '',
+  registered_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS ingest_events (
+  event_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE,
+  kind TEXT NOT NULL,
+  payload_hash TEXT NOT NULL,
+  payload_json TEXT NOT NULL,
+  occurred_at TEXT NOT NULL,
+  ingested_at TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'accepted'
+);
+
+CREATE INDEX IF NOT EXISTS idx_ingest_events_project_time
+  ON ingest_events(project_id, occurred_at);
+
+CREATE TABLE IF NOT EXISTS memory_events (
+  event_id TEXT PRIMARY KEY REFERENCES ingest_events(event_id) ON DELETE CASCADE,
+  project_id TEXT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE,
+  entity_type TEXT NOT NULL,
+  logical_path TEXT NOT NULL,
+  operation TEXT NOT NULL,
+  revision INTEGER NOT NULL DEFAULT 1,
+  content_hash TEXT NOT NULL DEFAULT '',
+  source_session_id TEXT NOT NULL DEFAULT '',
+  source_turn_id TEXT NOT NULL DEFAULT '',
+  occurred_at TEXT NOT NULL,
+  payload_json TEXT NOT NULL,
+  UNIQUE(project_id, logical_path, revision)
+);
+
+CREATE INDEX IF NOT EXISTS idx_memory_events_project_path
+  ON memory_events(project_id, logical_path, revision);
+
+CREATE TABLE IF NOT EXISTS documents (
+  document_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE,
+  logical_path TEXT NOT NULL,
+  entity_type TEXT NOT NULL,
+  title TEXT NOT NULL DEFAULT '',
+  content TEXT NOT NULL DEFAULT '',
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  content_hash TEXT NOT NULL DEFAULT '',
+  revision INTEGER NOT NULL DEFAULT 1,
+  source_session_id TEXT NOT NULL DEFAULT '',
+  source_turn_id TEXT NOT NULL DEFAULT '',
+  captured_at TEXT NOT NULL,
+  deleted_at TEXT,
+  UNIQUE(project_id, logical_path)
+);
+
+CREATE INDEX IF NOT EXISTS idx_documents_project_type
+  ON documents(project_id, entity_type, deleted_at);
+
+CREATE INDEX IF NOT EXISTS idx_documents_project_revision
+  ON documents(project_id, revision);
+
+CREATE TABLE IF NOT EXISTS sessions (
+  session_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE,
+  provider TEXT NOT NULL DEFAULT '',
+  status TEXT NOT NULL DEFAULT 'unknown',
+  summary TEXT NOT NULL DEFAULT '',
+  change_slug TEXT NOT NULL DEFAULT '',
+  started_at TEXT,
+  ended_at TEXT,
+  updated_at TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_sessions_project_time
+  ON sessions(project_id, started_at, ended_at);
+
+CREATE TABLE IF NOT EXISTS agent_runs (
+  agent_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE,
+  session_id TEXT NOT NULL REFERENCES sessions(session_id) ON DELETE CASCADE,
+  parent_agent_id TEXT,
+  role TEXT NOT NULL DEFAULT 'main',
+  agent_name TEXT NOT NULL DEFAULT '',
+  agent_type TEXT NOT NULL DEFAULT '',
+  workflow TEXT NOT NULL DEFAULT '',
+  status TEXT NOT NULL DEFAULT 'unknown',
+  model TEXT NOT NULL DEFAULT '',
+  effort TEXT NOT NULL DEFAULT '',
+  started_at TEXT,
+  ended_at TEXT,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  FOREIGN KEY(parent_agent_id) REFERENCES agent_runs(agent_id) ON DELETE SET NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_agent_runs_project_session
+  ON agent_runs(project_id, session_id, role);

package/schema/observer/002-usage.sql

@@ -0,0 +1,72 @@
+CREATE TABLE IF NOT EXISTS usage_rollups (
+  rollup_key TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE,
+  session_id TEXT NOT NULL REFERENCES sessions(session_id) ON DELETE CASCADE,
+  agent_id TEXT NOT NULL REFERENCES agent_runs(agent_id) ON DELETE CASCADE,
+  role TEXT NOT NULL DEFAULT 'main',
+  provider TEXT NOT NULL DEFAULT '',
+  model_provider TEXT NOT NULL DEFAULT '',
+  model TEXT NOT NULL DEFAULT '',
+  effort TEXT NOT NULL DEFAULT '',
+  calls INTEGER NOT NULL DEFAULT 0,
+  tokens_input INTEGER NOT NULL DEFAULT 0,
+  tokens_cache_write INTEGER NOT NULL DEFAULT 0,
+  tokens_cache_read INTEGER NOT NULL DEFAULT 0,
+  tokens_output INTEGER NOT NULL DEFAULT 0,
+  tokens_reasoning INTEGER NOT NULL DEFAULT 0,
+  tokens_total INTEGER NOT NULL DEFAULT 0,
+  cost_usd REAL NOT NULL DEFAULT 0,
+  cost_status TEXT NOT NULL DEFAULT 'unknown',
+  pricing_source TEXT NOT NULL DEFAULT '',
+  pricing_version TEXT NOT NULL DEFAULT '',
+  wasted_usd REAL NOT NULL DEFAULT 0,
+  revision INTEGER NOT NULL DEFAULT 1,
+  occurred_at TEXT NOT NULL,
+  source_event_id TEXT NOT NULL REFERENCES ingest_events(event_id) ON DELETE CASCADE,
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_usage_rollups_project_time
+  ON usage_rollups(project_id, occurred_at);
+
+CREATE INDEX IF NOT EXISTS idx_usage_rollups_project_agent
+  ON usage_rollups(project_id, agent_id, role);
+
+CREATE INDEX IF NOT EXISTS idx_usage_rollups_project_model
+  ON usage_rollups(project_id, model_provider, model);
+
+CREATE TABLE IF NOT EXISTS llm_calls (
+  call_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE,
+  session_id TEXT NOT NULL REFERENCES sessions(session_id) ON DELETE CASCADE,
+  agent_id TEXT NOT NULL REFERENCES agent_runs(agent_id) ON DELETE CASCADE,
+  role TEXT NOT NULL DEFAULT 'main',
+  provider TEXT NOT NULL DEFAULT '',
+  model_provider TEXT NOT NULL DEFAULT '',
+  model TEXT NOT NULL DEFAULT '',
+  effort TEXT NOT NULL DEFAULT '',
+  sequence INTEGER NOT NULL DEFAULT 0,
+  occurred_at TEXT NOT NULL,
+  tokens_input INTEGER NOT NULL DEFAULT 0,
+  tokens_cache_write INTEGER NOT NULL DEFAULT 0,
+  tokens_cache_read INTEGER NOT NULL DEFAULT 0,
+  tokens_output INTEGER NOT NULL DEFAULT 0,
+  tokens_reasoning INTEGER NOT NULL DEFAULT 0,
+  tokens_total INTEGER NOT NULL DEFAULT 0,
+  cost_usd REAL NOT NULL DEFAULT 0,
+  cost_status TEXT NOT NULL DEFAULT 'unknown',
+  transcript_id TEXT,
+  prompt_text TEXT NOT NULL DEFAULT '',
+  response_text TEXT NOT NULL DEFAULT '',
+  status TEXT NOT NULL DEFAULT 'complete',
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_llm_calls_project_time
+  ON llm_calls(project_id, occurred_at);
+
+CREATE INDEX IF NOT EXISTS idx_llm_calls_project_agent
+  ON llm_calls(project_id, agent_id, sequence);
+
+CREATE INDEX IF NOT EXISTS idx_llm_calls_project_model
+  ON llm_calls(project_id, model_provider, model);

package/schema/observer/003-transcripts.sql

@@ -0,0 +1,21 @@
+CREATE TABLE IF NOT EXISTS transcripts (
+  transcript_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL REFERENCES projects(project_id) ON DELETE CASCADE,
+  session_id TEXT NOT NULL REFERENCES sessions(session_id) ON DELETE CASCADE,
+  agent_id TEXT NOT NULL REFERENCES agent_runs(agent_id) ON DELETE CASCADE,
+  coverage TEXT NOT NULL DEFAULT 'summary_only',
+  codec TEXT NOT NULL DEFAULT 'gzip',
+  content_gzip BLOB NOT NULL,
+  content_sha256 TEXT NOT NULL,
+  original_bytes INTEGER NOT NULL,
+  compressed_bytes INTEGER NOT NULL,
+  source TEXT NOT NULL DEFAULT '',
+  occurred_at TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_transcripts_project_session
+  ON transcripts(project_id, session_id, occurred_at);
+
+CREATE INDEX IF NOT EXISTS idx_transcripts_project_coverage
+  ON transcripts(project_id, coverage);

package/src/observer-memory-publish.mjs

@@ -0,0 +1,335 @@
+import { createHash } from 'node:crypto';
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { MAX_MEMORY_CONTENT_BYTES } from './observer-memory.mjs';
+
+export const MEMORY_OUTBOX_REL = '.brain/observer-memory-outbox';
+export const MEMORY_STATE_FILE = '.brain/observer-memory-state.json';
+const MEMORY_SCHEMA_VERSION = 1;
+const ROOT_FILES = new Set(['CORE.md', 'DIGEST.md', 'SHARED_MEMORY.md']);
+const ROOTS = [
+  '02-Sessões',
+  '04-Decisões',
+  '05-Bugs',
+  '06-Aprendizados',
+  '07-Specs',
+  '08-Mudanças',
+  '.brain',
+];
+const TRANSIENT_NAMES = new Set(['observer-memory-outbox', 'observer-outbox']);
+
+function isoNow(value) {
+  const date = value instanceof Date ? value : new Date(value || Date.now());
+  if (Number.isNaN(date.getTime())) throw new Error('captured_at inválido.');
+  return date.toISOString();
+}
+
+function hash(value) {
+  return createHash('sha256').update(value).digest('hex');
+}
+
+function readJson(path, fallback) {
+  if (!existsSync(path)) return fallback;
+  try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return fallback; }
+}
+
+function atomicJson(path, value) {
+  mkdirSync(join(path, '..'), { recursive: true });
+  const temp = path + '.' + process.pid + '.' + Date.now() + '.tmp';
+  writeFileSync(temp, JSON.stringify(value, null, 2) + '\n', 'utf8');
+  renameSync(temp, path);
+}
+
+function readState(vaultBase) {
+  return readJson(join(vaultBase, MEMORY_STATE_FILE), {
+    schema_version: MEMORY_SCHEMA_VERSION,
+    files: {},
+  });
+}
+
+function entityType(logicalPath) {
+  if (logicalPath.startsWith('02-Sessões/')) return 'session';
+  if (logicalPath.startsWith('04-Decisões/')) return 'decision';
+  if (logicalPath.startsWith('05-Bugs/')) return 'bug';
+  if (logicalPath.startsWith('06-Aprendizados/')) return 'learning';
+  if (logicalPath.startsWith('07-Specs/')) return 'spec';
+  if (logicalPath.startsWith('08-Mudanças/')) return 'change';
+  return 'memory';
+}
+
+function shouldSkip(name, relativePath) {
+  if (TRANSIENT_NAMES.has(name) || name === 'observer-sql-state.json' || name === 'observer-sql-outbox'
+    || name.endsWith('.tmp') || name.endsWith('.lock')) return true;
+  if (relativePath === MEMORY_STATE_FILE) return true;
+  return false;
+}
+
+function walkDirectory(root, relativeRoot, output) {
+  if (!existsSync(root)) return;
+  for (const entry of readdirSync(root, { withFileTypes: true })) {
+    const logicalPath = relativeRoot ? relativeRoot + '/' + entry.name : entry.name;
+    if (shouldSkip(entry.name, logicalPath)) continue;
+    const absolute = join(root, entry.name);
+    if (entry.isDirectory()) {
+      walkDirectory(absolute, logicalPath, output);
+    } else if (entry.isFile()) {
+      output.push({ absolute, logicalPath });
+    }
+  }
+}
+
+function memoryFiles(vaultBase) {
+  const files = [];
+  for (const root of ROOTS) {
+    const absolute = join(vaultBase, root);
+    if (root === '.brain' || root.includes('-')) walkDirectory(absolute, root, files);
+  }
+  for (const rootFile of ROOT_FILES) {
+    const absolute = join(vaultBase, rootFile);
+    if (existsSync(absolute) && statSync(absolute).isFile()) files.push({ absolute, logicalPath: rootFile });
+  }
+  return files.sort((a, b) => a.logicalPath.localeCompare(b.logicalPath));
+}
+
+export function localMemoryManifest(vaultBase) {
+  return Object.fromEntries(memoryFiles(vaultBase).map((file) => {
+    const content = readFileSync(file.absolute, 'utf8');
+    return [file.logicalPath, {
+      logical_path: file.logicalPath,
+      content_hash: hash(content),
+      bytes: Buffer.byteLength(content, 'utf8'),
+    }];
+  }));
+}
+
+function projectIdFromVault(vaultBase) {
+  const project = readJson(join(vaultBase, '.brain', 'PROJECT.json'), null);
+  return project?.projectId || project?.project_id || '';
+}
+
+function eventId(projectId, logicalPath, revision, contentHash) {
+  return 'mem-' + hash([projectId, logicalPath, revision, contentHash].join(':')).slice(0, 24);
+}
+
+function makeEvent({ projectId, logicalPath, content, revision, sourceSessionId, sourceTurnId, capturedAt, operation = 'upsert' }) {
+  const body = content || '';
+  const contentHash = hash(body);
+  return {
+    schema_version: MEMORY_SCHEMA_VERSION,
+    event_id: eventId(projectId, logicalPath, revision, contentHash),
+    project_id: projectId,
+    entity_type: entityType(logicalPath),
+    logical_path: logicalPath,
+    operation,
+    ...(operation === 'upsert' ? { content: body } : {}),
+    content_hash: contentHash,
+    revision,
+    source_session_id: String(sourceSessionId || ''),
+    source_turn_id: String(sourceTurnId || ''),
+    captured_at: capturedAt,
+  };
+}
+
+export function buildMemoryEventBatch({
+  vaultBase,
+  projectId = projectIdFromVault(vaultBase),
+  sourceSessionId = '',
+  sourceTurnId = '',
+  now = new Date(),
+  state = readState(vaultBase),
+} = {}) {
+  if (!vaultBase || !projectId) throw new Error('vaultBase e projectId são obrigatórios.');
+  const capturedAt = isoNow(now);
+  const currentFiles = memoryFiles(vaultBase);
+  const currentPaths = new Set(currentFiles.map((file) => file.logicalPath));
+  const nextState = {
+    schema_version: MEMORY_SCHEMA_VERSION,
+    files: { ...(state.files || {}) },
+  };
+  const events = [];
+
+  for (const file of currentFiles) {
+    const content = readFileSync(file.absolute, 'utf8');
+    if (Buffer.byteLength(content, 'utf8') > MAX_MEMORY_CONTENT_BYTES) {
+      throw new Error('arquivo excede o limite de memória: ' + file.logicalPath);
+    }
+    const contentHash = hash(content);
+    const previous = state.files?.[file.logicalPath];
+    if (previous?.content_hash === contentHash) {
+      nextState.files[file.logicalPath] = previous;
+      continue;
+    }
+    const revision = Number(previous?.revision || 0) + 1;
+    const next = { content_hash: contentHash, revision, event_id: eventId(projectId, file.logicalPath, revision, contentHash) };
+    nextState.files[file.logicalPath] = next;
+    events.push(makeEvent({
+      projectId,
+      logicalPath: file.logicalPath,
+      content,
+      revision,
+      sourceSessionId,
+      sourceTurnId,
+      capturedAt,
+    }));
+  }
+
+  for (const [logicalPath, previous] of Object.entries(state.files || {})) {
+    if (currentPaths.has(logicalPath)) continue;
+    const revision = Number(previous.revision || 0) + 1;
+    events.push(makeEvent({
+      projectId,
+      logicalPath,
+      content: '',
+      revision,
+      sourceSessionId,
+      sourceTurnId,
+      capturedAt,
+      operation: 'delete',
+    }));
+    delete nextState.files[logicalPath];
+  }
+
+  return { events, nextState, scanned: currentFiles.length, changed: events.length };
+}
+
+export function commitMemoryPublishState(vaultBase, state) {
+  atomicJson(join(vaultBase, MEMORY_STATE_FILE), state);
+}
+
+function outboxDir(vaultBase) {
+  return join(vaultBase, MEMORY_OUTBOX_REL);
+}
+
+function outboxPath(vaultBase, events) {
+  const id = hash(JSON.stringify(events)).slice(0, 24);
+  return join(outboxDir(vaultBase), id + '.json');
+}
+
+function queueMemoryBatch(vaultBase, events) {
+  mkdirSync(outboxDir(vaultBase), { recursive: true });
+  const path = outboxPath(vaultBase, events);
+  if (!existsSync(path)) atomicJson(path, { schema_version: MEMORY_SCHEMA_VERSION, events });
+  return path;
+}
+
+export function listMemoryOutbox(vaultBase) {
+  const dir = outboxDir(vaultBase);
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((name) => name.endsWith('.json'))
+    .sort()
+    .map((name) => join(dir, name));
+}
+
+async function postBatch(url, projectId, events, fetchImpl = globalThis.fetch) {
+  const response = await fetchImpl(
+    String(url).replace(/\/$/, '') + '/v1/projects/' + encodeURIComponent(projectId) + '/memory/events',
+    {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', accept: 'application/json' },
+      body: JSON.stringify({ events }),
+    },
+  );
+  if (!response.ok) throw new Error('Observer respondeu HTTP ' + response.status + '.');
+  return response.json();
+}
+
+export async function retryObserverMemoryOutbox({
+  vaultBase,
+  projectId = projectIdFromVault(vaultBase),
+  url,
+  fetchImpl = globalThis.fetch,
+} = {}) {
+  const files = listMemoryOutbox(vaultBase);
+  if (!url) return { attempted: 0, confirmed: 0, pending: files.length };
+  let attempted = 0;
+  let confirmed = 0;
+  for (const path of files) {
+    attempted += 1;
+    try {
+      const batch = readJson(path, null);
+      if (!batch?.events?.length) {
+        unlinkSync(path);
+        continue;
+      }
+      await postBatch(url, projectId, batch.events, fetchImpl);
+      unlinkSync(path);
+      confirmed += 1;
+    } catch {
+      break;
+    }
+  }
+  return { attempted, confirmed, pending: listMemoryOutbox(vaultBase).length };
+}
+
+export async function compareMemoryParity({
+  vaultBase,
+  projectId = projectIdFromVault(vaultBase),
+  url,
+  fetchImpl = globalThis.fetch,
+} = {}) {
+  const response = await fetchImpl(
+    String(url).replace(/\/$/, '') + '/v1/projects/' + encodeURIComponent(projectId) + '/memory/tree',
+    { headers: { accept: 'application/json' } },
+  );
+  if (!response.ok) throw new Error('Observer respondeu HTTP ' + response.status + '.');
+  const body = await response.json();
+  const local = localMemoryManifest(vaultBase);
+  const remote = Object.fromEntries((body.documents || []).map((item) => [item.logical_path, item]));
+  const missing = Object.keys(local).filter((path) => !remote[path]);
+  const mismatched = Object.keys(local).filter((path) => remote[path] && remote[path].content_hash !== local[path].content_hash);
+  const extra = Object.keys(remote).filter((path) => !local[path]);
+  return {
+    files: Object.keys(local).length,
+    remote_files: Object.keys(remote).length,
+    missing: missing.length,
+    mismatched: mismatched.length,
+    extra: extra.length,
+    missing_paths: missing,
+    mismatched_paths: mismatched,
+    extra_paths: extra,
+  };
+}
+
+export async function publishObserverMemory({
+  vaultBase,
+  projectId = projectIdFromVault(vaultBase),
+  url,
+  sourceSessionId = '',
+  sourceTurnId = '',
+  now = new Date(),
+  fetchImpl = globalThis.fetch,
+} = {}) {
+  if (!vaultBase || !projectId) throw new Error('vaultBase e projectId são obrigatórios.');
+  await retryObserverMemoryOutbox({ vaultBase, projectId, url, fetchImpl });
+  const state = readState(vaultBase);
+  const batch = buildMemoryEventBatch({ vaultBase, projectId, sourceSessionId, sourceTurnId, now, state });
+  if (batch.events.length === 0) {
+    commitMemoryPublishState(vaultBase, batch.nextState);
+    return { ok: true, queued: false, scanned: batch.scanned, changed: 0, pending: listMemoryOutbox(vaultBase).length };
+  }
+  if (!url) {
+    queueMemoryBatch(vaultBase, batch.events);
+    commitMemoryPublishState(vaultBase, batch.nextState);
+    return { ok: false, queued: true, scanned: batch.scanned, changed: batch.changed, pending: listMemoryOutbox(vaultBase).length, hookExitCode: 0 };
+  }
+  try {
+    await postBatch(url, projectId, batch.events, fetchImpl);
+    commitMemoryPublishState(vaultBase, batch.nextState);
+    return { ok: true, queued: false, scanned: batch.scanned, changed: batch.changed, pending: listMemoryOutbox(vaultBase).length };
+  } catch (error) {
+    queueMemoryBatch(vaultBase, batch.events);
+    commitMemoryPublishState(vaultBase, batch.nextState);
+    return { ok: false, queued: true, scanned: batch.scanned, changed: batch.changed, pending: listMemoryOutbox(vaultBase).length, hookExitCode: 0, error: error.message };
+  }
+}