wendkeep 0.87.0 → 0.88.0

package/packages/observer/src/retention.mjs

@@ -0,0 +1,39 @@
+import { purgeObserverData } from './purge.mjs';
+
+const CLASSES = ['documents', 'calls', 'transcripts'];
+
+export function observerRetentionCutoffs(policy = {}, now = new Date()) {
+  const timestamp = new Date(now);
+  if (Number.isNaN(timestamp.getTime())) throw Object.assign(new Error('retention clock inválido.'), { code: 'observer_retention_invalid' });
+  return Object.fromEntries(CLASSES.flatMap((dataClass) => {
+    const days = Number(policy[dataClass] ?? 0);
+    if (!Number.isInteger(days) || days < 0) throw Object.assign(new Error(`retention inválida para ${dataClass}.`), { code: 'observer_retention_invalid' });
+    return days === 0 ? [] : [[dataClass, new Date(timestamp.getTime() - days * 86_400_000).toISOString()]];
+  }));
+}
+
+export function runObserverRetention(db, {
+  projectId, policy = {}, clock = () => new Date(), dryRun = false, operationId = '', receiptSink,
+} = {}) {
+  const observedAt = clock();
+  const cutoffs = observerRetentionCutoffs(policy, observedAt);
+  const receipts = [];
+  for (const [dataClass, before] of Object.entries(cutoffs)) {
+    receipts.push(purgeObserverData(db, {
+      projectId,
+      before,
+      classes: [dataClass],
+      dryRun,
+      operationId: operationId ? `${operationId}:${dataClass}` : '',
+      now: observedAt,
+      receiptSink,
+    }));
+  }
+  return {
+    schema_version: 1,
+    project_id: projectId,
+    observed_at: new Date(observedAt).toISOString(),
+    cutoffs,
+    receipts,
+  };
+}

package/packages/observer/src/token-registry.mjs

@@ -0,0 +1,122 @@
+import { createHash, randomBytes } from 'node:crypto';
+
+const ROLES = new Set(['viewer', 'auditor', 'publisher', 'admin']);
+
+export function hashObserverToken(token) {
+  const value = String(token || '');
+  if (!value) throw Object.assign(new Error('token vazio.'), { code: 'observer_token_invalid' });
+  return `sha256:${createHash('sha256').update(value, 'utf8').digest('hex')}`;
+}
+
+function json(value) { return JSON.stringify(value); }
+function parse(value) { try { return JSON.parse(value || '[]'); } catch { return []; } }
+function validTime(value, field) {
+  const parsed = new Date(value);
+  if (Number.isNaN(parsed.getTime())) throw Object.assign(new Error(`${field} inválido.`), { code: 'observer_token_invalid' });
+  return parsed.toISOString();
+}
+
+export function registerObserverToken(db, {
+  tokenId = randomBytes(12).toString('hex'), token, role, projectIds = [], scopes = [],
+  createdAt = new Date().toISOString(), expiresAt, rotatedFrom = null,
+} = {}) {
+  if (!ROLES.has(role)) throw Object.assign(new Error('role inválida.'), { code: 'observer_role_invalid' });
+  if (!Array.isArray(projectIds) || projectIds.length === 0) throw Object.assign(new Error('projectIds é obrigatório.'), { code: 'observer_projects_invalid' });
+  if (!Array.isArray(scopes) || scopes.length === 0) throw Object.assign(new Error('scopes é obrigatório.'), { code: 'observer_scopes_invalid' });
+  const row = {
+    token_id: String(tokenId), token_hash: hashObserverToken(token), role,
+    project_ids_json: json([...new Set(projectIds.map(String))].sort()),
+    scopes_json: json([...new Set(scopes.map(String))].sort()),
+    created_at: validTime(createdAt, 'createdAt'), expires_at: validTime(expiresAt, 'expiresAt'),
+    revoked_at: null, rotated_from: rotatedFrom ? String(rotatedFrom) : null,
+  };
+  db.prepare(`INSERT INTO observer_tokens(token_id, token_hash, role, project_ids_json, scopes_json, created_at, expires_at, revoked_at, rotated_from)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(...Object.values(row));
+  return { token_id: row.token_id, role, project_ids: parse(row.project_ids_json), scopes: parse(row.scopes_json), expires_at: row.expires_at, rotated_from: row.rotated_from };
+}
+
+export function resolveObserverPrincipal(db, token, { now = new Date().toISOString() } = {}) {
+  let tokenHash;
+  try { tokenHash = hashObserverToken(token); } catch { return { ok: false, code: 'observer_token_missing' }; }
+  const row = db.prepare('SELECT * FROM observer_tokens WHERE token_hash = ?').get(tokenHash);
+  if (!row) return { ok: false, code: 'observer_token_invalid' };
+  const timestamp = new Date(now).getTime();
+  if (row.revoked_at && new Date(row.revoked_at).getTime() <= timestamp) return { ok: false, code: 'observer_token_revoked', token_id: row.token_id };
+  if (new Date(row.expires_at).getTime() <= timestamp) return { ok: false, code: 'observer_token_expired', token_id: row.token_id };
+  return {
+    ok: true, token_id: row.token_id, role: row.role,
+    project_ids: parse(row.project_ids_json), scopes: parse(row.scopes_json), expires_at: row.expires_at,
+  };
+}
+
+export function ensureObserverBootstrapToken(db, {
+  token, tokenId = '', role, projectIds = [], scopes = [], expiresAt, createdAt = new Date().toISOString(), now = createdAt,
+} = {}) {
+  const tokenHash = hashObserverToken(token);
+  const existing = db.prepare('SELECT * FROM observer_tokens WHERE token_hash = ?').get(tokenHash);
+  if (existing) {
+    const expired = new Date(existing.expires_at).getTime() <= new Date(validTime(now, 'now')).getTime();
+    return {
+      created: false,
+      token_id: existing.token_id,
+      role: existing.role,
+      project_ids: parse(existing.project_ids_json),
+      scopes: parse(existing.scopes_json),
+      expires_at: existing.expires_at,
+      revoked: Boolean(existing.revoked_at),
+      expired,
+    };
+  }
+  if (!Array.isArray(projectIds) || projectIds.length === 0 || projectIds.includes('*')) {
+    throw Object.assign(new Error('bootstrap exige projectIds explícitos e não aceita wildcard.'), { code: 'observer_bootstrap_projects_invalid' });
+  }
+  const normalizedExpiry = validTime(expiresAt, 'expiresAt');
+  if (new Date(normalizedExpiry).getTime() <= new Date(validTime(now, 'now')).getTime()) {
+    throw Object.assign(new Error('bootstrap exige expiração futura.'), { code: 'observer_bootstrap_expired' });
+  }
+  const id = String(tokenId || `bootstrap-${tokenHash.slice(-24)}`);
+  if (db.prepare('SELECT token_id FROM observer_tokens WHERE token_id = ?').get(id)) {
+    throw Object.assign(new Error('tokenId de bootstrap já pertence a outra credencial.'), { code: 'observer_bootstrap_token_id_conflict' });
+  }
+  return {
+    created: true,
+    ...registerObserverToken(db, {
+      tokenId: id,
+      token,
+      role,
+      projectIds,
+      scopes,
+      createdAt,
+      expiresAt: normalizedExpiry,
+    }),
+    revoked: false,
+    expired: false,
+  };
+}
+
+export function revokeObserverToken(db, { tokenId, revokedAt = new Date().toISOString() } = {}) {
+  const result = db.prepare('UPDATE observer_tokens SET revoked_at = ? WHERE token_id = ? AND revoked_at IS NULL')
+    .run(validTime(revokedAt, 'revokedAt'), String(tokenId || ''));
+  return { token_id: String(tokenId || ''), revoked: Number(result.changes) > 0 };
+}
+
+export function rotateObserverToken(db, {
+  tokenId, newTokenId, newToken, rotatedAt = new Date().toISOString(), expiresAt,
+} = {}) {
+  const current = db.prepare('SELECT * FROM observer_tokens WHERE token_id = ?').get(String(tokenId || ''));
+  if (!current || current.revoked_at) throw Object.assign(new Error('token não pode ser rotacionado.'), { code: 'observer_token_rotation_invalid' });
+  db.exec('BEGIN IMMEDIATE');
+  try {
+    revokeObserverToken(db, { tokenId, revokedAt: rotatedAt });
+    const created = registerObserverToken(db, {
+      tokenId: newTokenId, token: newToken, role: current.role,
+      projectIds: parse(current.project_ids_json), scopes: parse(current.scopes_json),
+      createdAt: rotatedAt, expiresAt, rotatedFrom: tokenId,
+    });
+    db.exec('COMMIT');
+    return created;
+  } catch (error) {
+    db.exec('ROLLBACK');
+    throw error;
+  }
+}

package/schema/observer/006-observer-security.sql

@@ -0,0 +1,64 @@
+-- wendkeep:structural
+-- Security lifecycle: hash-only credentials, access audit, retention, purge receipts and encrypted columns.
+
+ALTER TABLE llm_calls ADD COLUMN prompt_envelope TEXT NOT NULL DEFAULT '';
+ALTER TABLE llm_calls ADD COLUMN response_envelope TEXT NOT NULL DEFAULT '';
+ALTER TABLE llm_calls ADD COLUMN metadata_envelope TEXT NOT NULL DEFAULT '';
+ALTER TABLE documents ADD COLUMN content_envelope TEXT NOT NULL DEFAULT '';
+ALTER TABLE documents ADD COLUMN metadata_envelope TEXT NOT NULL DEFAULT '';
+ALTER TABLE transcripts ADD COLUMN metadata_envelope TEXT NOT NULL DEFAULT '';
+ALTER TABLE project_snapshots ADD COLUMN snapshot_envelope TEXT NOT NULL DEFAULT '';
+
+CREATE TABLE IF NOT EXISTS observer_tokens (
+  token_id TEXT PRIMARY KEY,
+  token_hash TEXT NOT NULL UNIQUE,
+  role TEXT NOT NULL CHECK(role IN ('viewer', 'auditor', 'publisher', 'admin')),
+  project_ids_json TEXT NOT NULL,
+  scopes_json TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  expires_at TEXT NOT NULL,
+  revoked_at TEXT,
+  rotated_from TEXT REFERENCES observer_tokens(token_id) ON DELETE SET NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_observer_tokens_hash_active ON observer_tokens(token_hash, revoked_at, expires_at);
+
+CREATE TABLE IF NOT EXISTS observer_access_audit (
+  audit_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL,
+  token_id TEXT REFERENCES observer_tokens(token_id) ON DELETE SET NULL,
+  capability TEXT NOT NULL,
+  outcome TEXT NOT NULL,
+  occurred_at TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_observer_audit_project_time ON observer_access_audit(project_id, occurred_at);
+
+CREATE TABLE IF NOT EXISTS observer_retention_policies (
+  project_id TEXT PRIMARY KEY REFERENCES projects(project_id) ON DELETE CASCADE,
+  policy_json TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS observer_purge_receipts (
+  receipt_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL,
+  request_hash TEXT NOT NULL,
+  cutoff_at TEXT NOT NULL,
+  classes_json TEXT NOT NULL,
+  counts_json TEXT NOT NULL,
+  purged_at TEXT NOT NULL,
+  receipt_hash TEXT NOT NULL UNIQUE,
+  receipt_json TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_observer_purge_project_time ON observer_purge_receipts(project_id, purged_at);
+CREATE INDEX IF NOT EXISTS idx_observer_purge_request ON observer_purge_receipts(project_id, request_hash, purged_at);
+
+CREATE TABLE IF NOT EXISTS observer_security_backfill (
+  project_id TEXT PRIMARY KEY REFERENCES projects(project_id) ON DELETE CASCADE,
+  status TEXT NOT NULL,
+  protected_rows INTEGER NOT NULL DEFAULT 0,
+  updated_at TEXT NOT NULL
+);

package/schema/observer-policy-v1.schema.json

@@ -0,0 +1,63 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://wendkeep.dev/schema/observer-policy-v1.schema.json",
+  "title": "WendKeep Observer policy v1",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "document_capture": { "enum": ["none", "metadata", "selected", "full"] },
+    "transcript_capture": { "enum": ["none", "metadata", "messages", "full"] },
+    "prompt_capture": { "enum": ["none", "redacted", "full"] },
+    "response_capture": { "enum": ["none", "redacted", "full"] },
+    "usage_capture": { "enum": ["none", "aggregate", "calls"] },
+    "require_loopback_auth": { "type": "boolean" },
+    "encryption_required": { "type": "boolean" },
+    "rules": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["data_class", "capture"],
+        "properties": {
+          "project_id": { "type": "string", "minLength": 1, "maxLength": 121 },
+          "data_class": { "enum": ["document", "transcript", "prompt", "response", "usage"] },
+          "path": { "type": "string", "maxLength": 2048 },
+          "entity_type": { "type": "string", "maxLength": 160 },
+          "capture": { "enum": ["none", "metadata", "selected", "full", "messages", "redacted", "aggregate", "calls"] }
+        }
+      }
+    },
+    "redaction": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "rules": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["pattern"],
+            "properties": {
+              "pattern": { "type": "string", "minLength": 1, "maxLength": 512 },
+              "replacement": { "type": "string", "maxLength": 160 }
+            }
+          }
+        }
+      }
+    },
+    "retention": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "document": { "type": "integer", "minimum": 0 },
+        "transcript": { "type": "integer", "minimum": 0 },
+        "prompt": { "type": "integer", "minimum": 0 },
+        "response": { "type": "integer", "minimum": 0 },
+        "usage": { "type": "integer", "minimum": 0 },
+        "documents": { "type": "integer", "minimum": 0 },
+        "calls": { "type": "integer", "minimum": 0 },
+        "transcripts": { "type": "integer", "minimum": 0 }
+      }
+    }
+  }
+}

package/schema/sync-event-v1.schema.json

@@ -20,6 +20,16 @@
     "observed_at": { "type": "string", "format": "date-time" },
     "operation": { "enum": ["put", "tombstone", "resolve"] },
     "privacy": { "enum": ["shared", "private"] },
+    "policy_ref": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["policy_id", "version", "hash"],
+      "properties": {
+        "policy_id": { "type": "string", "minLength": 1, "maxLength": 160 },
+        "version": { "type": "integer", "minimum": 1 },
+        "hash": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }
+      }
+    },
     "payload": {}
   }
 }

package/src/observer-auth.mjs

@@ -8,3 +8,11 @@ export function observerAuthHeaders(token, headers = {}) {
     ? { ...headers, authorization: `Bearer ${resolved}` }
     : { ...headers };
 }
+
+export {
+  registerObserverToken,
+  resolveObserverPrincipal,
+  revokeObserverToken,
+  rotateObserverToken,
+} from '../packages/observer/src/token-registry.mjs';
+export { authorizeObserverPrincipal } from '../packages/observer/src/authz.mjs';

package/src/observer-privacy.mjs

@@ -1,4 +1,5 @@
 import { basename } from 'node:path';
+import { redactObserverText, redactObserverValue } from '../packages/observer/src/redaction.mjs';
 
 const TRANSCRIPT_PATH_KEY = /^(?:transcript_path|agent_transcript_path|transcriptPath|agentTranscriptPath)$/i;
 const TRANSCRIPT_PATH_LINE = /^(\s*["']?(?:transcript_path|agent_transcript_path|transcriptPath|agentTranscriptPath)["']?\s*:\s*)(["']?)(.*?)(\2)(\s*,?\s*)$/gmi;
@@ -8,16 +9,19 @@ function sourceLabel(value) {
 }
 
 export function sanitizeObserverContent(content) {
-  return String(content || '').replace(TRANSCRIPT_PATH_LINE, (_line, prefix, quote, value, _closing, suffix) => (
+  const safePaths = String(content || '').replace(TRANSCRIPT_PATH_LINE, (_line, prefix, quote, value, _closing, suffix) => (
     `${prefix}${quote}${sourceLabel(value)}${quote}${suffix}`
   ));
+  return redactObserverText(safePaths);
 }
 
 export function sanitizeObserverMetadata(value) {
   if (Array.isArray(value)) return value.map(sanitizeObserverMetadata);
   if (!value || typeof value !== 'object') return value;
-  return Object.fromEntries(Object.entries(value).map(([key, item]) => [
+  return redactObserverValue(Object.fromEntries(Object.entries(value).map(([key, item]) => [
     key,
     TRANSCRIPT_PATH_KEY.test(key) ? sourceLabel(item) : sanitizeObserverMetadata(item),
-  ]));
+  ])));
 }
+
+export { redactObserverText, redactObserverValue };

package/src/observer-publish.mjs

@@ -3,10 +3,37 @@ import { join } from 'node:path';
 import { publishObserverSqlIncremental } from './observer-sql-publish.mjs';
 import { observerAuthHeaders, resolveObserverToken } from './observer-auth.mjs';
 import { readProjectForValidation } from '../packages/vault/src/validate-memory.mjs';
+import { createObserverPolicy } from '../packages/observer/src/policy.mjs';
+import { createObserverEncryption } from '../packages/observer/src/encryption.mjs';
 
 const OUTBOX_REL = join('.brain', 'observer-outbox');
 const REQUEST_TIMEOUT_MS = 500;
 
+function decodeObserverKey(value) {
+  const raw = String(value || '').trim();
+  const key = /^[a-f0-9]{64}$/i.test(raw) ? Buffer.from(raw, 'hex') : Buffer.from(raw, 'base64');
+  if (key.byteLength !== 32) throw Object.assign(new Error('Observer outbox key deve ter 32 bytes em hex/base64.'), { code: 'observer_encryption_key_invalid' });
+  return key;
+}
+
+export function resolveObserverPublisherSecurity({ env = process.env } = {}) {
+  const policyFile = String(env.WENDKEEP_OBSERVER_POLICY_FILE || '').trim();
+  const policy = policyFile
+    ? createObserverPolicy(JSON.parse(readFileSync(policyFile, 'utf8')))
+    : createObserverPolicy();
+  const keyEnvName = String(env.WENDKEEP_OBSERVER_OUTBOX_KEY_ENV || '').trim();
+  if (!keyEnvName) return { policy, outboxEncryption: null };
+  const key = decodeObserverKey(env[keyEnvName]);
+  return {
+    policy,
+    outboxEncryption: createObserverEncryption({
+      required: true,
+      keyId: String(env.WENDKEEP_OBSERVER_OUTBOX_KEY_ID || keyEnvName),
+      keyProvider: () => key,
+    }),
+  };
+}
+
 function outboxDir(vaultBase) {
   return join(vaultBase, OUTBOX_REL);
 }
@@ -92,6 +119,8 @@ export async function publishObserverSnapshot({
   now = new Date(),
   input = {},
   token = process.env.WENDKEEP_OBSERVER_TOKEN || '',
+  policy = null,
+  outboxEncryption = null,
 } = {}) {
   try {
     const project = readProjectForValidation(vaultBase);
@@ -103,6 +132,8 @@ export async function publishObserverSnapshot({
       input,
       now,
       token,
+      policy,
+      outboxEncryption,
     });
     // SQLite is the only live authority. The legacy snapshot store remains
     // readable solely as a migration source for pre-SQL installations.