total-recall-brain 3.13.0 → 3.13.2

package/src/core/secrets-env-export.mjs

@@ -0,0 +1,323 @@
+/**
+ * Materialize .env files FROM the Total Recall secrets store.
+ *
+ * Model: TR secrets.enc is the SSOT for all user API keys.
+ * Repo / deploy .env files are projections — never the other way around.
+ *
+ * Never write secret values into vault markdown, openwiki, or surfaces.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { loadSecrets, listSecretsMeta, resolveSecretsPath } from './secrets-store.mjs';
+
+const META_KEY = '__tr_secrets_meta';
+
+const BEGIN_MARK = '# BEGIN TOTAL-RECALL-SECRETS';
+const END_MARK = '# END TOTAL-RECALL-SECRETS';
+
+function envHeader() {
+  return `${BEGIN_MARK}
+# Generated by Total Recall — do not hand-edit secret values here.
+# SSOT: secrets store (npx total-recall secret …)
+# Re-export: npx total-recall secret export-env --path <repo>
+# ${new Date().toISOString()}
+`;
+}
+
+/**
+ * Merge a managed TR secrets block into an existing .env without wiping
+ * non-secret / hand-managed keys outside the BEGIN/END markers.
+ *
+ * @param {string} existing - current file contents (or '')
+ * @param {string} managedBody - full block including BEGIN/END marks
+ * @returns {string}
+ */
+export function mergeEnvManagedBlock(existing, managedBody) {
+  const managed = String(managedBody || '').trimEnd() + '\n';
+  const text = existing == null ? '' : String(existing);
+
+  if (!text.trim()) return managed;
+
+  const beginIdx = text.indexOf(BEGIN_MARK);
+  const endIdx = text.indexOf(END_MARK);
+
+  if (beginIdx !== -1 && endIdx !== -1 && endIdx > beginIdx) {
+    const before = text.slice(0, beginIdx).replace(/\s+$/, '');
+    const after = text.slice(endIdx + END_MARK.length).replace(/^\r?\n?/, '');
+    const parts = [];
+    if (before) parts.push(before.endsWith('\n') ? before : `${before}\n`);
+    parts.push(managed.endsWith('\n') ? managed : `${managed}\n`);
+    if (after.trim()) {
+      parts.push(after.startsWith('\n') ? after : `\n${after}`);
+    }
+    return parts.join('').replace(/\n{3,}/g, '\n\n');
+  }
+
+  // No prior TR block — append managed section after existing content
+  const base = text.endsWith('\n') ? text : `${text}\n`;
+  return `${base}\n${managed}`;
+}
+
+/**
+ * Resolve a project slug from a filesystem path (last segment).
+ * @param {string} projectPath
+ */
+export function projectSlugFromPath(projectPath) {
+  if (!projectPath) return null;
+  const abs = path.resolve(projectPath);
+  return path.basename(abs);
+}
+
+/**
+ * Load project-registry entries (if any).
+ */
+export function loadProjectRegistry() {
+  const registryPath = path.join(
+    os.homedir(),
+    '.agent',
+    'skills',
+    'total-recall',
+    'config',
+    'project-registry.json',
+  );
+  if (!fs.existsSync(registryPath)) return [];
+  try {
+    const data = JSON.parse(fs.readFileSync(registryPath, 'utf8'));
+    return Array.isArray(data) ? data : [];
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Decide whether a secret meta row applies to a target project.
+ *
+ * @param {object} metaRow - from listSecretsMeta
+ * @param {{ projectPath?: string, projectSlug?: string, includeGlobal?: boolean, keys?: string[] }} target
+ */
+export function secretMatchesTarget(metaRow, target = {}) {
+  if (target.keys?.length && !target.keys.includes(metaRow.key)) return false;
+
+  const slug = target.projectSlug || projectSlugFromPath(target.projectPath);
+  const repos = Array.isArray(metaRow.repos) ? metaRow.repos : [];
+  const projectPath = metaRow.project_path || '';
+
+  // Explicit key list mode
+  if (target.keys?.length) return true;
+
+  // Bound to this repo by slug or path (policy: at most one product repo per key)
+  // Multi-repo keys are invalid — do not project them until fixed
+  if (repos.length > 1) return false;
+  if (slug && repos.some((r) => r === slug || r === '*' || r.endsWith(`/${slug}`))) {
+    return true;
+  }
+  if (target.projectPath && projectPath) {
+    const a = path.resolve(target.projectPath);
+    const b = path.resolve(projectPath);
+    if (a === b || a.startsWith(b + path.sep) || b.startsWith(a + path.sep)) return true;
+  }
+
+  // Unbound secrets: include when includeGlobal (default true for full SSOT export)
+  if (target.includeGlobal !== false && repos.length === 0 && !projectPath) {
+    return true;
+  }
+
+  // Also include when tagged global scope with empty repos
+  if (target.includeGlobal !== false && metaRow.scope === 'global' && repos.length === 0) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Build dotenv body (KEY=value lines) for matching secrets.
+ * @returns {{ body: string, keys: string[], exampleBody: string, count: number }}
+ */
+export async function buildEnvProjection(brainDir, target = {}) {
+  const secrets = await loadSecrets(brainDir);
+  const metaRows = await listSecretsMeta(brainDir);
+  const byKey = new Map(metaRows.map((m) => [m.key, m]));
+
+  const selected = [];
+  for (const [key, value] of Object.entries(secrets)) {
+    if (key === META_KEY) continue;
+    if (value == null || value === '') continue;
+    const meta = byKey.get(key) || { key, repos: [], scope: 'global' };
+    if (!secretMatchesTarget(meta, target)) continue;
+    selected.push({ key, value: String(value), meta });
+  }
+
+  selected.sort((a, b) => a.key.localeCompare(b.key));
+
+  const lines = selected.map(({ key, value }) => {
+    // Quote if needed
+    const needsQuote = /[\s#"'$\\]/.test(value);
+    const v = needsQuote ? JSON.stringify(value) : value;
+    return `${key}=${v}`;
+  });
+
+  const exampleLines = selected.map(({ key }) => `${key}=`);
+
+  const body = `${envHeader()}${lines.join('\n')}${lines.length ? '\n' : ''}${END_MARK}\n`;
+  const exampleBody = `# Example env — names only. Values live in Total Recall secrets store.
+# Generate real .env: npx total-recall secret export-env --path .
+${exampleLines.join('\n')}${exampleLines.length ? '\n' : ''}`;
+
+  return {
+    body,
+    exampleBody,
+    keys: selected.map((s) => s.key),
+    count: selected.length,
+    store: resolveSecretsPath(brainDir),
+  };
+}
+
+/**
+ * Write .env (and optional .env.example) into a project directory.
+ *
+ * @param {string} brainDir
+ * @param {string} projectPath - repo root
+ * @param {{
+ *   filename?: string,
+ *   example?: boolean,
+ *   dryRun?: boolean,
+ *   includeGlobal?: boolean,
+ *   keys?: string[],
+ *   mode?: number,
+ * }} opts
+ */
+export async function exportEnvToProject(brainDir, projectPath, opts = {}) {
+  const abs = path.resolve(projectPath);
+  if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) {
+    throw new Error(`Project path is not a directory: ${abs}`);
+  }
+
+  const filename = opts.filename || '.env';
+  const target = {
+    projectPath: abs,
+    projectSlug: projectSlugFromPath(abs),
+    includeGlobal: opts.includeGlobal !== false,
+    keys: opts.keys,
+  };
+
+  const projection = await buildEnvProjection(brainDir, target);
+  const envPath = path.join(abs, filename);
+  const examplePath = path.join(abs, `${filename}.example`);
+
+  if (opts.dryRun) {
+    return {
+      dryRun: true,
+      envPath,
+      examplePath: opts.example !== false ? examplePath : null,
+      keys: projection.keys,
+      count: projection.count,
+      store: projection.store,
+    };
+  }
+
+  const mode = opts.mode ?? 0o600;
+  // Merge by default so non-secret APP_NAME=… etc. outside the TR block survive
+  const replaceAll = opts.replaceAll === true;
+  let finalBody = projection.body;
+  let merged = false;
+  if (!replaceAll && fs.existsSync(envPath)) {
+    try {
+      const existing = fs.readFileSync(envPath, 'utf8');
+      finalBody = mergeEnvManagedBlock(existing, projection.body);
+      merged = true;
+    } catch {
+      finalBody = projection.body;
+    }
+  }
+  fs.writeFileSync(envPath, finalBody, { encoding: 'utf8', mode });
+  try {
+    fs.chmodSync(envPath, mode);
+  } catch {
+    /* windows / fs without chmod */
+  }
+
+  let wroteExample = false;
+  if (opts.example !== false) {
+    fs.writeFileSync(examplePath, projection.exampleBody, { encoding: 'utf8', mode: 0o644 });
+    wroteExample = true;
+  }
+
+  // Ensure .env is gitignored
+  ensureGitignoreHasEnv(abs);
+
+  return {
+    dryRun: false,
+    envPath,
+    examplePath: wroteExample ? examplePath : null,
+    keys: projection.keys,
+    count: projection.count,
+    store: projection.store,
+    projectSlug: target.projectSlug,
+    merged,
+  };
+}
+
+/**
+ * Export to multiple registered projects.
+ */
+export async function exportEnvToRegistry(brainDir, opts = {}) {
+  const registry = loadProjectRegistry();
+  const results = [];
+  for (const entry of registry) {
+    const projectPath = entry.path || entry.project_root;
+    if (!projectPath || !fs.existsSync(projectPath)) {
+      results.push({
+        name: entry.name,
+        path: projectPath,
+        ok: false,
+        error: 'path missing',
+      });
+      continue;
+    }
+    try {
+      const r = await exportEnvToProject(brainDir, projectPath, opts);
+      results.push({ name: entry.name, ...r, ok: true });
+    } catch (err) {
+      results.push({ name: entry.name, path: projectPath, ok: false, error: err.message });
+    }
+  }
+  return results;
+}
+
+/**
+ * Build a deploy payload: dotenv text + optional JSON map (no disk write).
+ */
+export async function buildDeploySecretsPayload(brainDir, target = {}) {
+  const projection = await buildEnvProjection(brainDir, {
+    includeGlobal: true,
+    ...target,
+  });
+  const secrets = await loadSecrets(brainDir);
+  const map = {};
+  for (const key of projection.keys) {
+    if (secrets[key] != null) map[key] = String(secrets[key]);
+  }
+  return {
+    dotenv: projection.body,
+    map,
+    keys: projection.keys,
+    count: projection.count,
+    store: projection.store,
+  };
+}
+
+function ensureGitignoreHasEnv(projectPath) {
+  const gi = path.join(projectPath, '.gitignore');
+  const linesNeeded = ['.env', '.env.local', '.env.*.local'];
+  let content = '';
+  if (fs.existsSync(gi)) {
+    content = fs.readFileSync(gi, 'utf8');
+  }
+  const missing = linesNeeded.filter((l) => !content.split(/\r?\n/).some((line) => line.trim() === l));
+  if (!missing.length) return;
+  const block = `\n# Total Recall secrets projections\n${missing.join('\n')}\n`;
+  fs.appendFileSync(gi, content.endsWith('\n') || !content ? block : `\n${block}`);
+}

package/src/core/secrets-rotate.mjs

@@ -0,0 +1,245 @@
+/**
+ * Secrets rotation orchestration.
+ *
+ * - Detect overdue keys (schedule metadata)
+ * - Enqueue assisted rotation tasks (optional browser-use)
+ * - After rotate: re-export .env to bound repos
+ * - TR PATs can be auto-rotated when explicitly requested
+ *
+ * Never log secret values. Never write values into vault/openwiki.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  listRotationDue,
+  listSecretsMeta,
+  rotateSecret,
+  getSecret,
+  updateSecretMeta,
+} from './secrets-store.mjs';
+import { exportEnvToProject, exportEnvToRegistry, loadProjectRegistry, projectSlugFromPath } from './secrets-env-export.mjs';
+import { getProvider, providerForKeyName } from './provider-catalog.mjs';
+import { atomicWrite, safeStringify } from './vault.mjs';
+
+/**
+ * Build a supervised browser-use prompt for rotating one key.
+ * Does not perform automation itself.
+ */
+export function buildBrowserRotatePrompt(secretMeta) {
+  const catalog = getProvider(secretMeta.provider) || providerForKeyName(secretMeta.key);
+  const consoleUrl = secretMeta.console_url || catalog?.console_url || '(provider console)';
+  const docsUrl = secretMeta.api_docs_url || catalog?.docs_url || '';
+
+  return [
+    `Rotate API credential for secret key: ${secretMeta.key}`,
+    `Provider: ${secretMeta.provider || catalog?.id || 'unknown'}`,
+    '',
+    'SUPERVISED browser workflow (do NOT auto-revoke until TR confirms store+export):',
+    `1. Open: ${consoleUrl}`,
+    '2. Complete login / 2FA with the human if needed.',
+    '3. Create a NEW API key (do not delete the old one yet).',
+    '4. Copy the new key value ONCE — never paste into chat logs or vault markdown.',
+    '5. Tell the human to run (or run via tool if value is only in a secure channel):',
+    `   npx total-recall secret rotate ${secretMeta.key} "<NEW_VALUE>" --export-env`,
+    '6. After TR confirms export succeeded, revoke/disable the OLD key in the console.',
+    '',
+    docsUrl ? `API docs: ${docsUrl}` : '',
+    secretMeta.repos?.length ? `Bound repos: ${secretMeta.repos.join(', ')}` : 'Repos: (unbound / global projection)',
+    secretMeta.next_rotate_due ? `Was due: ${secretMeta.next_rotate_due}` : '',
+    '',
+    'Safety: never screenshot the full key; never write the value into remember/openwiki.',
+  ]
+    .filter(Boolean)
+    .join('\n');
+}
+
+/**
+ * Enqueue one queue task per overdue secret (or refresh existing pending).
+ * @param {string} brainDir
+ * @param {{ autoOnly?: boolean, queueDir?: string }} opts
+ */
+export async function enqueueRotationDueTasks(brainDir, opts = {}) {
+  const due = await listRotationDue(brainDir, { autoOnly: !!opts.autoOnly });
+  const queueDir =
+    opts.queueDir || path.join(brainDir, 'scheduler', 'queue');
+  if (!fs.existsSync(queueDir)) fs.mkdirSync(queueDir, { recursive: true });
+
+  const created = [];
+  const now = new Date().toISOString();
+
+  for (const secret of due) {
+    const slug = `secret-rotate-${secret.key}`
+      .toLowerCase()
+      .replace(/[^a-z0-9_-]+/g, '-')
+      .slice(0, 80);
+    const filePath = path.join(queueDir, `${slug}.md`);
+    if (fs.existsSync(filePath)) {
+      // leave existing pending task
+      created.push({ key: secret.key, slug, status: 'exists' });
+      continue;
+    }
+
+    const prompt = buildBrowserRotatePrompt(secret);
+    const front = {
+      type: 'task',
+      slug,
+      category: 'secrets-rotation',
+      title: `Rotate secret: ${secret.key}`,
+      status: 'pending',
+      priority: secret.auto_rotate ? 1 : 2,
+      executor: 'secrets-rotation-assist',
+      capabilities: ['secrets:rotate', 'notify'],
+      intent: `Rotate overdue secret ${secret.key} (supervised browser or provider API)`,
+      payload: {
+        secret_key: secret.key,
+        provider: secret.provider,
+        auto_rotate: !!secret.auto_rotate,
+        console_url: secret.console_url,
+        browser_prompt: true,
+      },
+      created: now,
+      updated: now,
+    };
+
+    const body = [
+      `## Objective`,
+      ``,
+      `Rotate \`${secret.key}\` — schedule overdue.`,
+      ``,
+      `## Browser-use instructions`,
+      ``,
+      '```',
+      prompt,
+      '```',
+      ``,
+      `## After new value is obtained`,
+      ``,
+      '```bash',
+      `npx total-recall secret rotate ${secret.key} "<NEW_VALUE>" --export-env`,
+      '```',
+    ].join('\n');
+
+    atomicWrite(filePath, safeStringify(body, front));
+    created.push({ key: secret.key, slug, status: 'created', path: filePath });
+  }
+
+  return { due: due.length, tasks: created };
+}
+
+/**
+ * Rotate value then re-export .env to bound repos (and optionally all registry).
+ */
+export async function rotateSecretAndExport(brainDir, key, newValue, opts = {}) {
+  const result = await rotateSecret(brainDir, key, newValue, {
+    actor: opts.actor || 'cli',
+    provider: opts.provider,
+  });
+
+  const metaRows = await listSecretsMeta(brainDir);
+  const row = metaRows.find((r) => r.key === key);
+  const exports = [];
+
+  if (opts.exportEnv !== false) {
+    const registry = loadProjectRegistry();
+    const repos = row?.repos || [];
+    const boundPaths = new Set();
+
+    for (const r of repos) {
+      // match registry by name/slug
+      for (const entry of registry) {
+        if (entry.name === r || projectSlugFromPath(entry.path) === r) {
+          if (entry.path && fs.existsSync(entry.path)) boundPaths.add(path.resolve(entry.path));
+        }
+      }
+      // also allow absolute path in repos
+      if (r.startsWith('/') && fs.existsSync(r)) boundPaths.add(path.resolve(r));
+    }
+
+    if (row?.project_path && fs.existsSync(row.project_path)) {
+      boundPaths.add(path.resolve(row.project_path));
+    }
+
+    if (opts.exportAllProjects) {
+      const all = await exportEnvToRegistry(brainDir, {
+        includeGlobal: true,
+        example: true,
+      });
+      for (const r of all) exports.push(r);
+    } else if (boundPaths.size) {
+      for (const p of boundPaths) {
+        try {
+          const r = await exportEnvToProject(brainDir, p, {
+            includeGlobal: opts.includeGlobal !== false,
+            example: true,
+          });
+          exports.push({ ok: true, ...r });
+        } catch (err) {
+          exports.push({ ok: false, path: p, error: err.message });
+        }
+      }
+    } else if (opts.exportCwd) {
+      const r = await exportEnvToProject(brainDir, process.cwd(), {
+        includeGlobal: true,
+        example: true,
+      });
+      exports.push({ ok: true, ...r });
+    }
+  }
+
+  return {
+    ...result,
+    secret: row,
+    exports,
+  };
+}
+
+/**
+ * Daemon executor: scan due secrets and enqueue assist tasks.
+ */
+export async function runSecretsRotationCheck(ctx) {
+  const result = await enqueueRotationDueTasks(ctx.brainDir, {
+    autoOnly: false,
+    queueDir: ctx.queueDir,
+  });
+  return {
+    success: true,
+    output: `Rotation check: ${result.due} due, ${result.tasks.filter((t) => t.status === 'created').length} tasks created`,
+    executor: 'secrets-rotation-check',
+    ...result,
+  };
+}
+
+/**
+ * Daemon executor: re-export all project envs from SSOT.
+ */
+export async function runSecretsExportAll(ctx) {
+  const results = await exportEnvToRegistry(ctx.brainDir, {
+    includeGlobal: true,
+    example: true,
+  });
+  const ok = results.filter((r) => r.ok).length;
+  return {
+    success: true,
+    output: `export-env: ${ok}/${results.length} projects updated`,
+    executor: 'secrets-export-env',
+    results,
+  };
+}
+
+/**
+ * Print browser prompt for a key (CLI helper).
+ */
+export async function getBrowserRotateAssist(brainDir, key) {
+  const rows = await listSecretsMeta(brainDir);
+  const row = rows.find((r) => r.key === key);
+  if (!row) throw new Error(`Secret not found: ${key}`);
+  return {
+    key,
+    prompt: buildBrowserRotatePrompt(row),
+    console_url: row.console_url,
+    docs_url: row.api_docs_url,
+    overdue: row.rotation_overdue,
+    next_rotate_due: row.next_rotate_due,
+  };
+}