total-recall-brain 3.21.2 → 3.22.0

package/src/core/ollama-embeddings.mjs

@@ -0,0 +1,336 @@
+/**
+ * src/core/ollama-embeddings.mjs
+ *
+ * Local / mesh Ollama embedding provider.
+ *
+ * Two properties here are load-bearing:
+ *
+ *   1. No hardcoded host. The endpoint comes from explicit config, then the
+ *      loopback default, then live mesh discovery. A machine name baked into
+ *      product code would leak one operator's topology into every install.
+ *
+ *   2. No hardcoded model. The model is selected from what the server actually
+ *      reports — filtered by its declared capability and matched against the
+ *      vault's vector width. Pinning a name would break the day it is renamed,
+ *      unpulled, or superseded by a better one.
+ *
+ * Calls go through plain fetch rather than throttledFetch: this is local
+ * infrastructure on the operator's own mesh, not a rate-limited third-party
+ * API, and it must keep working when the outbound network policy is strict.
+ */
+
+import { logger } from './logger.mjs';
+
+const OLLAMA_DEFAULT_PORT = 11434;
+const PROBE_TIMEOUT_MS = 2500;
+const SHOW_TIMEOUT_MS = 5000;
+const EMBED_TIMEOUT_MS = 30000;
+const DISCOVERY_TTL_MS = 5 * 60_000;
+
+/** fetch with a hard deadline. Manual controller — AbortSignal.timeout has bitten this file before. */
+async function fetchWithDeadline(url, options = {}, timeoutMs = PROBE_TIMEOUT_MS) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    return await fetch(url, { ...options, signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** Accepts `host`, `host:port`, or a full URL — OLLAMA_HOST is used all three ways in the wild. */
+export function normalizeBaseUrl(value) {
+  const raw = String(value || '').trim();
+  if (!raw) return null;
+  const withScheme = /^https?:\/\//i.test(raw) ? raw : `http://${raw}`;
+  try {
+    const u = new URL(withScheme);
+    if (!u.port && !/:\d+$/.test(u.host)) u.port = String(OLLAMA_DEFAULT_PORT);
+    return u.origin;
+  } catch {
+    return null;
+  }
+}
+
+/** Explicitly configured endpoints, highest precedence first. */
+function configuredBaseUrls() {
+  return [process.env.TR_OLLAMA_URL, process.env.OLLAMA_HOST]
+    .map(normalizeBaseUrl)
+    .filter(Boolean);
+}
+
+/**
+ * Online mesh peers, as endpoint candidates. Discovery, not configuration —
+ * this is how a laptop finds the box that actually holds the models without
+ * anyone writing an address into the repo.
+ */
+async function meshCandidateUrls() {
+  try {
+    const { readTailscaleStatus } = await import('./mesh-enroll.mjs');
+    const status = readTailscaleStatus();
+    if (!status) return [];
+    const peers = Object.values(status.Peer || {});
+    return peers
+      .filter((p) => p?.Online)
+      .flatMap((p) => (p.TailscaleIPs || []).filter((ip) => !ip.includes(':')))
+      .map((ip) => normalizeBaseUrl(`${ip}:${OLLAMA_DEFAULT_PORT}`))
+      .filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+/** @returns {Promise<Array<object>|null>} the server's model list, or null when it is not an Ollama endpoint. */
+export async function listOllamaModels(baseUrl) {
+  try {
+    const res = await fetchWithDeadline(`${baseUrl}/api/tags`, {}, PROBE_TIMEOUT_MS);
+    if (!res.ok) return null;
+    const data = await res.json();
+    return Array.isArray(data?.models) ? data.models : null;
+  } catch {
+    return null;
+  }
+}
+
+/** @type {{url: string, at: number}|null} */
+let endpointCache = null;
+
+/**
+ * Resolve a reachable Ollama endpoint: configured → loopback → mesh peers.
+ * @param {{force?: boolean}} [opts]
+ * @returns {Promise<string|null>}
+ */
+export async function resolveOllamaEndpoint({ force = false } = {}) {
+  if (!force && endpointCache && Date.now() - endpointCache.at < DISCOVERY_TTL_MS) {
+    return endpointCache.url;
+  }
+
+  const configured = configuredBaseUrls();
+  const candidates = [
+    ...configured,
+    normalizeBaseUrl(`127.0.0.1:${OLLAMA_DEFAULT_PORT}`),
+    // Mesh discovery is last and only consulted when nothing local answers, so
+    // the common case costs one loopback probe rather than a network sweep.
+    ...(configured.length ? [] : await meshCandidateUrls()),
+  ];
+
+  const seen = new Set();
+  for (const url of candidates) {
+    if (!url || seen.has(url)) continue;
+    seen.add(url);
+    const models = await listOllamaModels(url);
+    if (models) {
+      endpointCache = { url, at: Date.now() };
+      logger.debug('embeddings: resolved Ollama endpoint', { url, models: models.length });
+      return url;
+    }
+  }
+
+  endpointCache = null;
+  return null;
+}
+
+/** Full metadata for one model, including declared capabilities. */
+async function describeModel(baseUrl, name) {
+  try {
+    const res = await fetchWithDeadline(
+      `${baseUrl}/api/show`,
+      {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ model: name }),
+      },
+      SHOW_TIMEOUT_MS,
+    );
+    if (!res.ok) return null;
+    return await res.json();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The embedding width a model actually produces.
+ *
+ * Read the family-scoped key exactly. Generative models publish several
+ * *.embedding_length entries (a vision tower's width, an audio tower's width)
+ * and a loose regex will happily match one of those — picking a chat model
+ * that cannot embed at all.
+ */
+export function embeddingWidth(info) {
+  const family = info?.details?.family;
+  const modelInfo = info?.model_info || {};
+  if (family && Number.isFinite(modelInfo[`${family}.embedding_length`])) {
+    return modelInfo[`${family}.embedding_length`];
+  }
+  return null;
+}
+
+/** Ollama declares this explicitly; trust it over any name-based heuristic. */
+export function canEmbed(info) {
+  return Array.isArray(info?.capabilities) && info.capabilities.includes('embedding');
+}
+
+/** @type {Map<string, string|null>} */
+const modelCache = new Map();
+
+/**
+ * Choose an embedding model from what the server reports.
+ *
+ * @param {string} baseUrl
+ * @param {{dims: number, preferred?: string|null, force?: boolean}} opts
+ *   `dims` is the vault's vector width — a model of any other width cannot be
+ *   stored in the existing index, so width is a hard filter, not a preference.
+ * @returns {Promise<string|null>}
+ */
+export async function selectEmbeddingModel(baseUrl, { dims, preferred = null, force = false } = {}) {
+  const cacheKey = `${baseUrl}|${dims}|${preferred || ''}`;
+  if (!force && modelCache.has(cacheKey)) return modelCache.get(cacheKey);
+
+  const models = await listOllamaModels(baseUrl);
+  if (!models?.length) {
+    modelCache.set(cacheKey, null);
+    return null;
+  }
+
+  const names = models.map((m) => m.name || m.model).filter(Boolean);
+  // An operator-pinned model still has to prove it can embed at the right
+  // width; a stale pin should fall through to discovery, not hard-fail.
+  const ordered = preferred
+    ? [...names.filter((n) => n === preferred || n.split(':')[0] === preferred), ...names]
+    : names;
+
+  const bySize = new Map(models.map((m) => [m.name || m.model, m.size || Number.MAX_SAFE_INTEGER]));
+  const viable = [];
+
+  for (const name of [...new Set(ordered)]) {
+    const info = await describeModel(baseUrl, name);
+    if (!info || !canEmbed(info)) continue;
+    const width = embeddingWidth(info);
+    if (width !== dims) {
+      logger.debug('embeddings: skipping Ollama model on width mismatch', { name, width, want: dims });
+      continue;
+    }
+    if (preferred && (name === preferred || name.split(':')[0] === preferred)) {
+      modelCache.set(cacheKey, name);
+      return name;
+    }
+    viable.push(name);
+  }
+
+  // Smallest viable model wins: embedding models at equal width are near-equal
+  // in quality, and the small one keeps recall latency down.
+  viable.sort((a, b) => (bySize.get(a) || 0) - (bySize.get(b) || 0));
+  const chosen = viable[0] || null;
+  if (chosen) logger.debug('embeddings: selected Ollama model', { model: chosen, dims });
+  modelCache.set(cacheKey, chosen);
+  return chosen;
+}
+
+/**
+ * Embed one string. Returns the vector, or throws with a reason the caller can log.
+ *
+ * @param {string} text
+ * @param {{baseUrl: string, model: string, dims: number}} opts
+ * @returns {Promise<number[]>}
+ */
+export async function getOllamaEmbedding(text, { baseUrl, model, dims }) {
+  const res = await fetchWithDeadline(
+    `${baseUrl}/api/embeddings`,
+    {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ model, prompt: text }),
+    },
+    EMBED_TIMEOUT_MS,
+  );
+
+  if (!res.ok) {
+    const body = await res.text().catch(() => '');
+    throw new Error(`Ollama embedding error ${res.status}: ${body || 'no response body'}`);
+  }
+
+  const data = await res.json();
+  const embedding = data?.embedding;
+  if (!Array.isArray(embedding) || !embedding.length) {
+    throw new Error('Ollama returned no embedding vector');
+  }
+  // Storing a wrong-width vector corrupts the index silently; fail loudly instead.
+  if (Number.isFinite(dims) && embedding.length !== dims) {
+    throw new Error(`Ollama model ${model} returned ${embedding.length} dims, vault requires ${dims}`);
+  }
+  return embedding;
+}
+
+/**
+ * Every model the server holds, annotated for an operator choosing one:
+ * whether it can embed at all, how wide its vectors are, and whether that
+ * width fits the vault index. Models that cannot be used are still returned,
+ * with the reason visible, so the UI can explain the absence rather than
+ * silently showing an empty list.
+ *
+ * @param {string} baseUrl
+ * @param {{dims?: number}} [opts]
+ */
+export async function describeEmbeddingCandidates(baseUrl, { dims } = {}) {
+  const models = await listOllamaModels(baseUrl);
+  if (!models?.length) return [];
+
+  const out = [];
+  for (const m of models) {
+    const name = m.name || m.model;
+    if (!name) continue;
+    const info = await describeModel(baseUrl, name);
+    const embeds = info ? canEmbed(info) : false;
+    const width = info ? embeddingWidth(info) : null;
+    out.push({
+      name,
+      size: m.size ?? null,
+      family: info?.details?.family ?? null,
+      capabilities: info?.capabilities ?? [],
+      embedding: embeds,
+      dims: width,
+      compatible: Boolean(embeds && Number.isFinite(dims) && width === dims),
+    });
+  }
+  return out;
+}
+
+/**
+ * Current state of the local/mesh embedding provider, for display.
+ * Never throws — an unreachable endpoint is a reportable state, not an error.
+ *
+ * @param {{dims: number, preferred?: string|null}} opts
+ */
+export async function getOllamaProviderStatus({ dims, preferred = null } = {}) {
+  const endpoint = await resolveOllamaEndpoint();
+  if (!endpoint) {
+    return {
+      available: false,
+      endpoint: null,
+      selected: null,
+      preferred,
+      dims,
+      candidates: [],
+      reason: 'No Ollama endpoint answered on this host or any online mesh peer.',
+    };
+  }
+
+  const candidates = await describeEmbeddingCandidates(endpoint, { dims });
+  const selected = await selectEmbeddingModel(endpoint, { dims, preferred });
+  return {
+    available: Boolean(selected),
+    endpoint,
+    selected,
+    preferred,
+    dims,
+    candidates,
+    reason: selected ? null : `No model on ${endpoint} embeds at ${dims} dims.`,
+  };
+}
+
+/** Reset memoised discovery — used by tests and after a topology change. */
+export function resetOllamaDiscovery() {
+  endpointCache = null;
+  modelCache.clear();
+}

package/src/core/provider-catalog.mjs

@@ -11,6 +11,11 @@
  *   console_url?: string,
  *   key_patterns: string[],
  *   schema: { auth: string, header?: string, env_keys: string[], notes?: string },
+ *   rotation?: {
+ *     class?: 'provider_api' | 'provider_browser' | 'self_generated' | 'manual',
+ *     high_risk?: boolean,
+ *     manual_keys?: string[],
+ *   },
  *   default_monthly_cap_usd?: number | null,
  *   tiers?: { id: string, label: string, monthly_usd?: number | null }[],
  * }} ProviderDef */
@@ -295,6 +300,124 @@ export const PROVIDER_CATALOG = [
     default_monthly_cap_usd: 0,
     tiers: [{ id: 'self-hosted', label: 'Self-hosted', monthly_usd: 0 }],
   },
+  {
+    id: 'stripe',
+    name: 'Stripe',
+    docs_url: 'https://docs.stripe.com/api',
+    pricing_url: 'https://stripe.com/pricing',
+    console_url: 'https://dashboard.stripe.com/apikeys',
+    key_patterns: ['STRIPE_SECRET_KEY', 'STRIPE_API_KEY', 'STRIPE_PUBLISHABLE_KEY', 'STRIPE_WEBHOOK_SECRET'],
+    schema: {
+      auth: 'bearer',
+      header: 'Authorization: Bearer <key>',
+      env_keys: ['STRIPE_SECRET_KEY'],
+      notes: 'Live keys are payment-critical. Roll with a grace window; never revoke before export confirms.',
+    },
+    // Stripe exposes no API to mint or roll secret keys — dashboard only, by design.
+    rotation: { class: 'provider_browser', high_risk: true },
+    default_monthly_cap_usd: null,
+    tiers: [{ id: 'payg', label: 'Pay as you go', monthly_usd: null }],
+  },
+  {
+    id: 'vercel',
+    name: 'Vercel',
+    docs_url: 'https://vercel.com/docs/rest-api',
+    console_url: 'https://vercel.com/account/settings/tokens',
+    key_patterns: ['VERCEL_API_TOKEN', 'VERCEL_TOKEN'],
+    schema: { auth: 'bearer', header: 'Authorization: Bearer <token>', env_keys: ['VERCEL_API_TOKEN'] },
+    rotation: { class: 'provider_browser' },
+    default_monthly_cap_usd: 20,
+    tiers: [{ id: 'hobby', label: 'Hobby', monthly_usd: 0 }, { id: 'pro', label: 'Pro', monthly_usd: 20 }],
+  },
+  {
+    id: 'telnyx',
+    name: 'Telnyx',
+    docs_url: 'https://developers.telnyx.com/api',
+    console_url: 'https://portal.telnyx.com/#/app/api-keys',
+    key_patterns: ['TELNYX_API_KEY', 'TELNYX_SIP_PASSWORD'],
+    schema: { auth: 'bearer', header: 'Authorization: Bearer <key>', env_keys: ['TELNYX_API_KEY'] },
+    rotation: { class: 'provider_browser' },
+    default_monthly_cap_usd: null,
+    tiers: [{ id: 'payg', label: 'Pay as you go', monthly_usd: null }],
+  },
+  {
+    id: 'fal',
+    name: 'fal.ai',
+    docs_url: 'https://docs.fal.ai',
+    console_url: 'https://fal.ai/dashboard/keys',
+    key_patterns: ['FAL_KEY', 'FAL_AI_API_KEY'],
+    schema: { auth: 'header', header: 'Authorization: Key <key>', env_keys: ['FAL_KEY'] },
+    rotation: { class: 'provider_browser' },
+    default_monthly_cap_usd: 20,
+    tiers: [{ id: 'payg', label: 'Pay as you go', monthly_usd: null }],
+  },
+  {
+    id: 'tavily',
+    name: 'Tavily',
+    docs_url: 'https://docs.tavily.com',
+    console_url: 'https://app.tavily.com/home',
+    key_patterns: ['TAVILY_API_KEY'],
+    schema: { auth: 'bearer', header: 'Authorization: Bearer <key>', env_keys: ['TAVILY_API_KEY'] },
+    rotation: { class: 'provider_browser' },
+    default_monthly_cap_usd: 10,
+    tiers: [{ id: 'free', label: 'Free', monthly_usd: 0 }],
+  },
+  {
+    id: 'exa',
+    name: 'Exa',
+    docs_url: 'https://docs.exa.ai',
+    console_url: 'https://dashboard.exa.ai/api-keys',
+    key_patterns: ['EXA_API_KEY'],
+    schema: { auth: 'header', header: 'x-api-key: <key>', env_keys: ['EXA_API_KEY'] },
+    rotation: { class: 'provider_browser' },
+    default_monthly_cap_usd: 10,
+    tiers: [{ id: 'payg', label: 'Pay as you go', monthly_usd: null }],
+  },
+  {
+    id: 'npm',
+    name: 'npm',
+    docs_url: 'https://docs.npmjs.com/about-access-tokens',
+    console_url: 'https://www.npmjs.com/settings/~/tokens',
+    key_patterns: ['NPM_TOKEN', 'NPM_RECOVERY_CODE'],
+    schema: { auth: 'bearer', header: 'Authorization: Bearer <token>', env_keys: ['NPM_TOKEN'] },
+    // Recovery codes are single-issue artifacts, never machine-rotatable.
+    rotation: { class: 'provider_browser', manual_keys: ['NPM_RECOVERY_CODE'] },
+    default_monthly_cap_usd: 0,
+    tiers: [{ id: 'free', label: 'Free', monthly_usd: 0 }],
+  },
+  {
+    id: 'pexels',
+    name: 'Pexels',
+    docs_url: 'https://www.pexels.com/api/documentation/',
+    console_url: 'https://www.pexels.com/api/new/',
+    key_patterns: ['PEXELS_API_KEY'],
+    schema: { auth: 'header', header: 'Authorization: <key>', env_keys: ['PEXELS_API_KEY'] },
+    rotation: { class: 'provider_browser' },
+    default_monthly_cap_usd: 0,
+    tiers: [{ id: 'free', label: 'Free', monthly_usd: 0 }],
+  },
+  {
+    id: 'artificial-analysis',
+    name: 'Artificial Analysis',
+    docs_url: 'https://artificialanalysis.ai/documentation',
+    console_url: 'https://artificialanalysis.ai/account/api-keys',
+    key_patterns: ['ARTIFICIAL_ANALYSIS_API_KEY'],
+    schema: { auth: 'header', header: 'x-api-key: <key>', env_keys: ['ARTIFICIAL_ANALYSIS_API_KEY'] },
+    rotation: { class: 'provider_browser' },
+    default_monthly_cap_usd: 0,
+    tiers: [{ id: 'free', label: 'Free', monthly_usd: 0 }],
+  },
+  {
+    id: 'browser-use',
+    name: 'Browser Use',
+    docs_url: 'https://docs.browser-use.com',
+    console_url: 'https://cloud.browser-use.com/settings/api-keys',
+    key_patterns: ['BROWSER_USE_API_KEY'],
+    schema: { auth: 'bearer', header: 'Authorization: Bearer <key>', env_keys: ['BROWSER_USE_API_KEY'] },
+    rotation: { class: 'provider_browser' },
+    default_monthly_cap_usd: 10,
+    tiers: [{ id: 'payg', label: 'Pay as you go', monthly_usd: null }],
+  },
 ];
 
 const byId = new Map(PROVIDER_CATALOG.map((p) => [p.id, p]));
@@ -310,24 +433,70 @@ export function getProvider(providerId) {
 /**
  * @param {string} keyName
  */
+/**
+ * Minimum length for a derived stem before it may be used for a fuzzy match.
+ * Guards against short stems swallowing unrelated keys — e.g. the pattern
+ * `TR_TOKEN` reduces to the stem `TR`, which as a bare substring matches
+ * S-TR-IPE_SECRET_KEY and mis-attributes Stripe credentials to Total Recall.
+ */
+const MIN_FUZZY_STEM = 4;
+
+/** Does `k` contain `stem` on an underscore token boundary (not mid-word)? */
+function matchesOnTokenBoundary(k, stem) {
+  return (
+    k === stem ||
+    k.startsWith(stem + '_') ||
+    k.endsWith('_' + stem) ||
+    k.includes('_' + stem + '_')
+  );
+}
+
+/**
+ * Resolve the provider that owns a secret key name.
+ *
+ * Matching is tiered strongest→weakest so an exact pattern always wins over a
+ * fuzzy stem. Fuzzy tiers require a token boundary and a minimum stem length;
+ * bare substring matching is never used.
+ *
+ * @param {string} keyName
+ * @returns {ProviderDef|null}
+ */
 export function providerForKeyName(keyName) {
   // Strip packaging prefixes so DEVELOPER_BRAVE_SEARCH_API_KEY → BRAVE_SEARCH_API_KEY
   let k = String(keyName || '').toUpperCase();
   k = k.replace(/^(DEVELOPER_|VITE_|NEXT_PUBLIC_|PUBLIC_|NUXT_PUBLIC_|PORTFOLIO_|ULTRACHAT_)+/g, '');
+
+  // Tier 1 — exact pattern match.
   for (const p of PROVIDER_CATALOG) {
-    if (p.key_patterns.some((pat) => {
-      const P = pat.toUpperCase();
-      return P === k || k === P || k.endsWith('_' + P) || k.includes(P.replace(/_API_KEY$|_TOKEN$|_KEY$/i, ''));
-    })) {
+    if (p.key_patterns.some((pat) => pat.toUpperCase() === k)) return p;
+  }
+
+  // Tier 2 — key is a suffixed variant of a full pattern (FOO_OPENAI_API_KEY).
+  for (const p of PROVIDER_CATALOG) {
+    if (p.key_patterns.some((pat) => k.endsWith('_' + pat.toUpperCase()))) return p;
+  }
+
+  // Tier 3 — pattern stem on a token boundary (GITHUB_TOKEN → GITHUB_APP_CLIENT_SECRET).
+  for (const p of PROVIDER_CATALOG) {
+    if (
+      p.key_patterns.some((pat) => {
+        const stem = pat.toUpperCase().replace(/_API_KEY$|_TOKEN$|_KEY$|_SECRET$/, '');
+        if (stem.length < MIN_FUZZY_STEM) return false;
+        return matchesOnTokenBoundary(k, stem);
+      })
+    ) {
       return p;
     }
   }
-  // fuzzy product id in the remaining name
+
+  // Tier 4 — provider id on a token boundary.
   const compact = k.replace(/-/g, '_');
   for (const p of PROVIDER_CATALOG) {
     const id = p.id.toUpperCase().replace(/-/g, '_');
-    if (compact.includes(id) || compact.startsWith(id + '_')) return p;
+    if (id.length < MIN_FUZZY_STEM) continue;
+    if (matchesOnTokenBoundary(compact, id)) return p;
   }
+
   return null;
 }