total-recall-brain 3.12.2 → 3.13.1

package/src/server/routes/secrets.mjs

@@ -0,0 +1,470 @@
+/**
+ * /api/secrets/* — provider secrets catalog (not dashboard PATs; see /api/keys)
+ *
+ * SSOT: secrets store. Repo/deploy .env files are projections via export-env.
+ *
+ * - GET/POST catalog, meta, rotate, usage, import-env (migrate-in only)
+ * - POST /api/secrets/export-env   write .env from store → path or all projects
+ * - GET  /api/secrets/export-env   dry-run / preview keys for a path
+ */
+
+import express from 'express';
+import path from 'node:path';
+import { requireAuth, requireScope, verifyStepUpToken } from '../auth.mjs';
+import { badRequest, serverError, notFound, BRAIN_DIR, resolveVaultFromQuery } from './_shared.mjs';
+import {
+  listSecretsMeta,
+  setSecret,
+  getSecret,
+  getSecretsCatalog,
+  updateSecretMeta,
+  rotateSecret,
+  listRotationDue,
+  recordUsage,
+  summarizeUsage,
+  summarizeUsageByKey,
+  listUsageEvents,
+  deleteSecret,
+} from '../../core/secrets-store.mjs';
+import { logger } from '../../core/logger.mjs';
+import { listProviders, getProvider } from '../../core/provider-catalog.mjs';
+import {
+  scanEnvSources,
+  publicScanResult,
+  importEnvSecrets,
+  candidatesFromPaste,
+  inferProvider,
+  isCandidateKey,
+} from '../../core/env-import.mjs';
+import {
+  exportEnvToProject,
+  exportEnvToRegistry,
+  buildEnvProjection,
+  buildDeploySecretsPayload,
+} from '../../core/secrets-env-export.mjs';
+
+const router = express.Router();
+
+/**
+ * Provider secrets are a single host SSOT — NOT per project-vault brain.
+ * Selecting project:ultrachat for memory/graph must not empty the Secrets page.
+ * Repo binding lives in secret metadata (repos[]), not separate secrets.enc files.
+ *
+ * Override only via TR_SECRETS_BRAIN / TR_BRAIN env (deploy/ops).
+ */
+function secretsBrainDir(_req) {
+  if (process.env.TR_SECRETS_BRAIN) return process.env.TR_SECRETS_BRAIN;
+  return BRAIN_DIR;
+}
+
+/** @deprecated alias — all secrets routes use global SSOT */
+function brainDirFromReq(req) {
+  return secretsBrainDir(req);
+}
+
+router.get('/api/secrets/providers', requireAuth, requireScope('keys:read', 'config:read'), (_req, res) => {
+  res.json({ providers: listProviders() });
+});
+
+router.get('/api/secrets/usage', requireAuth, requireScope('keys:read', 'config:read'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const days = Math.min(90, Math.max(1, parseInt(req.query.days, 10) || 30));
+    const summary = summarizeUsage(brainDir, { days });
+    const by = summarizeUsageByKey(brainDir, { days });
+    const events = listUsageEvents(brainDir, {
+      limit: Math.min(200, parseInt(req.query.limit, 10) || 50),
+      key_ref: req.query.key_ref || null,
+      days,
+    });
+    res.json({ summary, ...by, events });
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+router.post('/api/secrets/usage', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const body = req.body || {};
+    const row = recordUsage(brainDir, {
+      provider: body.provider || 'unknown',
+      model: body.model,
+      input_tokens: body.input_tokens || 0,
+      output_tokens: body.output_tokens || 0,
+      cost_usd: body.cost_usd ?? body.cost ?? null,
+      key_ref: body.key_ref || body.key || null,
+      source: body.source || 'api',
+    });
+    res.status(201).json(row);
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+router.get('/api/secrets/rotation-due', requireAuth, requireScope('keys:read', 'config:read'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const due = await listRotationDue(brainDir, { autoOnly: req.query.auto === '1' });
+    res.json({ due, count: due.length });
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+router.post('/api/secrets/rotation-due/enqueue', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const { enqueueRotationDueTasks } = await import('../../core/secrets-rotate.mjs');
+    const result = await enqueueRotationDueTasks(brainDir, {
+      autoOnly: !!req.body?.autoOnly,
+    });
+    res.json(result);
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+router.get('/api/secrets', requireAuth, requireScope('keys:read', 'config:read'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    if (req.query.catalog === '0') {
+      const keys = await listSecretsMeta(brainDir);
+      return res.json({ keys, store: path.join(brainDir, 'config', 'secrets.enc') });
+    }
+    const catalog = await getSecretsCatalog(brainDir);
+    res.json(catalog);
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+/**
+ * Preview or write .env projections from secrets store.
+ * Body/query: { path?, all_projects?, dry_run?, filename?, include_global?, keys? }
+ */
+router.post('/api/secrets/export-env', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const body = req.body || {};
+    const dryRun = !!body.dry_run || !!body.dryRun;
+    const exportOpts = {
+      filename: body.filename || '.env',
+      example: body.example !== false,
+      dryRun,
+      includeGlobal: body.include_global !== false && body.includeGlobal !== false,
+      keys: Array.isArray(body.keys) ? body.keys : undefined,
+    };
+
+    if (body.all_projects || body.allProjects) {
+      const results = await exportEnvToRegistry(brainDir, exportOpts);
+      return res.json({ results, store: path.join(brainDir, 'config', 'secrets.enc') });
+    }
+
+    const targetPath = body.path || body.project_path || process.cwd();
+    if (body.preview_only || body.previewOnly) {
+      const projection = await buildEnvProjection(brainDir, {
+        projectPath: targetPath,
+        includeGlobal: exportOpts.includeGlobal,
+        keys: exportOpts.keys,
+      });
+      return res.json({
+        path: targetPath,
+        keys: projection.keys,
+        count: projection.count,
+        store: projection.store,
+        // never return dotenv body with values over API unless explicit
+        has_values: false,
+      });
+    }
+
+    const result = await exportEnvToProject(brainDir, targetPath, exportOpts);
+    res.json({
+      ...result,
+      // strip any accidental value leakage — only paths and key names
+      keys: result.keys,
+      count: result.count,
+    });
+  } catch (err) {
+    if (err.message?.includes('not a directory')) return badRequest(res, err.message);
+    serverError(res, err);
+  }
+});
+
+router.get('/api/secrets/export-env', requireAuth, requireScope('keys:read', 'config:read'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const targetPath = req.query.path || process.cwd();
+    const projection = await buildEnvProjection(brainDir, {
+      projectPath: targetPath,
+      includeGlobal: req.query.no_global !== '1',
+    });
+    res.json({
+      path: targetPath,
+      keys: projection.keys,
+      count: projection.count,
+      store: projection.store,
+    });
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+/** Deploy helper: returns dotenv text for host agent (auth required). */
+router.post('/api/secrets/deploy-payload', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const body = req.body || {};
+    const payload = await buildDeploySecretsPayload(brainDir, {
+      projectPath: body.path,
+      projectSlug: body.project_slug || body.projectSlug,
+      includeGlobal: body.include_global !== false,
+      keys: body.keys,
+    });
+    // For deploy automation — values included once under auth
+    res.json(payload);
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+router.get('/api/secrets/scan-env', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const existing = await listSecretsMeta(brainDir);
+    const existingSet = new Set(existing.map((e) => e.key));
+    const scan = scanEnvSources({
+      brainDir,
+      includeProcessEnv: true,
+      cwd: process.cwd(),
+    });
+    res.json(publicScanResult(scan, existingSet));
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+router.post('/api/secrets/parse-env', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const text = req.body?.text;
+    if (!text || typeof text !== 'string') {
+      return badRequest(res, 'text is required (paste .env contents)');
+    }
+    if (text.length > 200_000) {
+      return badRequest(res, 'text too large (max 200KB)');
+    }
+    const brainDir = brainDirFromReq(req);
+    const existing = await listSecretsMeta(brainDir);
+    const existingSet = new Set(existing.map((e) => e.key));
+    const { candidates } = candidatesFromPaste(text);
+    res.json({
+      candidates: candidates.map((c) => ({
+        ...c,
+        already_set: existingSet.has(c.key),
+      })),
+      count: candidates.length,
+    });
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+router.post('/api/secrets/import-env', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const body = req.body || {};
+    const overwrite = !!body.overwrite;
+
+    if (body.pairs && typeof body.pairs === 'object') {
+      let pairs = body.pairs;
+      if (Array.isArray(body.keys) && body.keys.length) {
+        const filtered = {};
+        for (const k of body.keys) {
+          if (pairs[k] != null) filtered[k] = pairs[k];
+        }
+        pairs = filtered;
+      }
+      const clean = {};
+      for (const [k, v] of Object.entries(pairs)) {
+        // Pattern-based only — any secret-shaped name, no product API whitelist
+        if (isCandidateKey(k) && v) clean[k] = String(v);
+      }
+      const result = await importEnvSecrets(brainDir, {
+        pairs: clean,
+        overwrite,
+        actor: 'dashboard-import',
+      });
+      return res.json(result);
+    }
+
+    const result = await importEnvSecrets(brainDir, {
+      keys: Array.isArray(body.keys) ? body.keys : undefined,
+      all: !!body.all,
+      overwrite,
+      actor: 'dashboard-import',
+    });
+    res.json(result);
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+router.post('/api/secrets', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const body = req.body || {};
+    const { key, value } = body;
+    if (!key || !value) return badRequest(res, 'key and value are required');
+    const brainDir = brainDirFromReq(req);
+    const provider = body.provider || inferProvider(key);
+    const catalog = getProvider(provider);
+    await setSecret(brainDir, key, value, {
+      provider,
+      scope: body.scope || 'global',
+      repos: body.repos,
+      subscription_tier: body.subscription_tier,
+      monthly_cost_usd: body.monthly_cost_usd,
+      monthly_cap_usd: body.monthly_cap_usd ?? catalog?.default_monthly_cap_usd,
+      api_docs_url: body.api_docs_url || catalog?.docs_url,
+      rotate_every_days: body.rotate_every_days,
+      auto_rotate: body.auto_rotate,
+      notes: body.notes,
+      project_path: body.project_path,
+      label: body.label,
+      actor: 'dashboard',
+    });
+    const keys = await listSecretsMeta(brainDir);
+    res.status(201).json(keys.find((k) => k.key === key));
+  } catch (err) {
+    if (err.message?.includes('Invalid')) return badRequest(res, err.message);
+    serverError(res, err);
+  }
+});
+
+router.patch('/api/secrets/:key', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const row = await updateSecretMeta(brainDir, req.params.key, req.body || {}, {
+      actor: 'dashboard',
+    });
+    if (!row) return notFound(res, `Secret not found: ${req.params.key}`);
+    res.json(row);
+  } catch (err) {
+    if (err.message?.includes('not found')) return notFound(res, err.message);
+    serverError(res, err);
+  }
+});
+
+/**
+ * Reveal full secret value — requires short-lived step_up_token from passkey
+ * (or password re-entry). Never logged. Audit records key name only.
+ */
+router.post(
+  '/api/secrets/:key/reveal',
+  requireAuth,
+  requireScope('keys:read', 'config:read'),
+  async (req, res) => {
+    try {
+      const token =
+        req.body?.step_up_token ||
+        req.headers['x-tr-step-up'] ||
+        req.headers['x-total-recall-step-up'];
+      const check = verifyStepUpToken(token, 'secrets:reveal');
+      if (!check.ok) {
+        return res.status(403).json({
+          error: check.error,
+          code: 'step_up_required',
+          hint: 'Complete passkey prompt or password step-up, then retry reveal.',
+        });
+      }
+
+      const brainDir = brainDirFromReq(req);
+      const key = req.params.key;
+      const got = await getSecret(brainDir, key, {
+        action: 'reveal',
+        actor: `step-up:${check.payload?.actor || 'dashboard'}`,
+      });
+      if (!got.found) return notFound(res, `Secret not found: ${key}`);
+
+      logger.info('secrets', 'Secret revealed (step-up)', {
+        key,
+        actor: check.payload?.actor || 'dashboard',
+      });
+
+      res.json({
+        key,
+        value: got.value,
+        // client should display briefly; server does not store this response
+        revealed_at: new Date().toISOString(),
+        step_up_actor: check.payload?.actor || null,
+      });
+    } catch (err) {
+      serverError(res, err);
+    }
+  },
+);
+
+router.post('/api/secrets/:key/rotate', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const body = req.body || {};
+    const { value } = body;
+    if (!value) return badRequest(res, 'value is required for rotate');
+    const brainDir = brainDirFromReq(req);
+
+    // Default: after rotate, re-project .env to bound repos / all projects when asked
+    const wantExport =
+      body.export_env === true ||
+      body.exportEnv === true ||
+      body.export_all === true ||
+      body.exportAll === true;
+
+    if (wantExport) {
+      const { rotateSecretAndExport } = await import('../../core/secrets-rotate.mjs');
+      const result = await rotateSecretAndExport(brainDir, req.params.key, value, {
+        actor: 'dashboard',
+        provider: body.provider,
+        exportEnv: true,
+        exportAllProjects: body.export_all === true || body.exportAll === true,
+        exportCwd: false,
+        includeGlobal: true,
+      });
+      // Never echo secret value
+      const keys = await listSecretsMeta(brainDir);
+      return res.json({
+        key: result.key,
+        rotated: true,
+        next_rotate_due: result.next_rotate_due,
+        secret: keys.find((k) => k.key === req.params.key),
+        exports: (result.exports || []).map((e) => ({
+          ok: e.ok !== false,
+          envPath: e.envPath,
+          count: e.count,
+          name: e.name,
+          error: e.error,
+        })),
+      });
+    }
+
+    const result = await rotateSecret(brainDir, req.params.key, value, {
+      actor: 'dashboard',
+      provider: body.provider,
+    });
+    const keys = await listSecretsMeta(brainDir);
+    res.json({ ...result, secret: keys.find((k) => k.key === req.params.key) });
+  } catch (err) {
+    if (err.message?.includes('not found')) return notFound(res, err.message);
+    serverError(res, err);
+  }
+});
+
+router.delete('/api/secrets/:key', requireAuth, requireScope('keys:write', 'config:write'), async (req, res) => {
+  try {
+    const brainDir = brainDirFromReq(req);
+    const r = await deleteSecret(brainDir, req.params.key, { actor: 'dashboard' });
+    if (!r.found) return notFound(res, `Secret not found: ${req.params.key}`);
+    res.json(r);
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
+export { router as secretsRouter };

package/src/server/routes/share.mjs

@@ -9,7 +9,7 @@
 import express from 'express';
 import crypto from 'node:crypto';
 import { createMemoryNode } from '../../core/vault.mjs';
-import { writeNodeValidated } from '../../core/validated-write.mjs';
+import { writeNodeValidatedAsync } from '../../core/validated-write.mjs';
 import { invalidate } from '../../core/vault-cache.mjs';
 import { addToQueue } from '../../core/research-queue.mjs';
 import { requireAuth, requireScope } from '../auth.mjs';
@@ -122,7 +122,7 @@ router.post('/api/share', requireAuth, requireScope('memory:write'), async (req,
       }];
     }
 
-    const vaultResult = writeNodeValidated(node, vaultDir);
+    const vaultResult = await writeNodeValidatedAsync(node, vaultDir);
     if (!vaultResult.success) {
       return res.status(422).json({
         error: 'Validation failed',