thumbgate 1.26.8 → 1.27.3

package/scripts/dashboard-chat.js

@@ -2,53 +2,138 @@
 
 // scripts/dashboard-chat.js
 // -----------------------------------------------------------------------------
-// "Chat with your data" — the dashboard chat backend. Answers a natural-language
-// question about THIS install's ThumbGate data (captured lessons + prevention
-// rules) by retrieving the most relevant lessons and asking Gemini to answer
-// grounded ONLY in that retrieved context (RAG). No data leaves the box except
-// the retrieved snippets + the question, sent to the configured Gemini endpoint.
+// "Chat with your data" — the dashboard chat backend. Local-first RAG over
+// this install's ThumbGate data (lessons, raw feedback memories via LanceDB
+// vectors, receipts, gate stats). Retrieval is local (lesson search + optional
+// vector-store.searchSimilar). Generation uses your configured LLM: a local
+// OpenAI-compatible endpoint first, then Gemini or Perplexity when explicitly
+// configured.
 //
-// Enterprise framing: this is the in-product "chat with your governed data"
-// experience. (The Dialogflow CX messenger widget is the separate path where a
-// customer connects their own DFCX agent + the ThumbGate webhook gate.)
+// Dialogflow/Google is not the dashboard chatbot brain. It remains an optional
+// guard-adapter path for buyers who already run their own Google agent tenancy.
 // -----------------------------------------------------------------------------
 
 const path = require('path');
 
 const GEMINI_ENDPOINT = 'https://generativelanguage.googleapis.com/v1beta/models';
+const PERPLEXITY_ENDPOINT = 'https://api.perplexity.ai/chat/completions';
 const DEFAULT_MODEL = 'gemini-2.5-flash';
 const MAX_QUESTION_CHARS = 2000;
 const MAX_CONTEXT_LESSONS = 8;
 
+// Allowlist the model so a user-supplied `model` cannot route the call to an
+// arbitrary / unexpected (or more expensive) endpoint. Anything not on the list
+// falls back to the default.
+const ALLOWED_MODELS = new Set([
+  'gemini-2.5-flash', 'gemini-2.5-flash-lite', 'gemini-2.5-pro',
+  'gemini-2.0-flash', 'gemini-2.0-flash-lite',
+  'gemini-flash-latest', 'gemini-flash-lite-latest', 'gemini-pro-latest',
+]);
+
+function resolveModel(requested) {
+  const r = String(requested || '').trim();
+  if (r && ALLOWED_MODELS.has(r)) return r;
+  const envModel = String(process.env.THUMBGATE_GEMINI_MODEL || '').trim();
+  if (envModel && ALLOWED_MODELS.has(envModel)) return envModel;
+  return DEFAULT_MODEL;
+}
+
 function resolveApiKey(opts = {}) {
-  return opts.apiKey || process.env.GEMINI_API_KEY || process.env.THUMBGATE_GEMINI_API_KEY || '';
+  let key = '';
+  if (Object.hasOwn(opts, 'apiKey')) {
+    key = opts.apiKey || '';
+  } else {
+    key = opts.apiKey || process.env.GEMINI_API_KEY || process.env.THUMBGATE_GEMINI_API_KEY || process.env.GOOGLE_API_KEY || process.env.PERPLEXITY_API_KEY || process.env.THUMBGATE_PERPLEXITY_API_KEY || '';
+  }
+  if (!key) return '';
+  return key.trim().replace(/^["']|["']$/g, '');
+}
+
+function debugChatFallback(label, err) {
+  if (process.env.THUMBGATE_DEBUG_CHAT !== '1') return;
+  const detail = err?.message ? err.message : String(err);
+  console.warn(`[dashboard-chat] ${label}: ${detail}`);
 }
 
-// Retrieve the most relevant stored lessons for the question.
-function retrieveContext(question, opts = {}) {
-  let searchLessons;
+function loadLessonSearcher() {
   try {
-    ({ searchLessons } = require(path.join(__dirname, 'lesson-search')));
-  } catch (_) {
-    return [];
+    return require(path.join(__dirname, 'lesson-search')).searchLessons;
+  } catch (err) {
+    debugChatFallback('lesson search unavailable', err);
+    return null;
   }
-  let res;
+}
+
+function lessonToContextItem(lesson) {
+  return {
+    id: lesson.id,
+    signal: lesson.signal || lesson.feedback || '',
+    title: (lesson.title || '').replace(/^(?:MISTAKE|SUCCESS):\s*/i, '').slice(0, 160),
+    content: String(lesson.content || lesson.context || '').replace(/\s+/g, ' ').trim().slice(0, 600),
+    tags: lesson.tags || [],
+    source: 'lessons',
+  };
+}
+
+function vectorMatchToContextItem(match, index) {
+  return {
+    id: match.id || `vec-${index}`,
+    signal: match.signal || '',
+    title: String(match.context || match.text || '').slice(0, 100),
+    content: match.text || match.context || '',
+    tags: match.tags ? String(match.tags).split(',').filter(Boolean) : [],
+    source: 'lancedb-vector',
+  };
+}
+
+function dedupeContextItems(items, limit = MAX_CONTEXT_LESSONS + 3) {
+  const seen = new Set();
+  return items.filter((item) => {
+    if (!(item.content || item.title)) return false;
+    const key = item.id || item.content.slice(0, 80);
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  }).slice(0, limit);
+}
+
+function retrieveLessonContext(question, opts = {}) {
+  const searchLessons = loadLessonSearcher();
+  if (!searchLessons) return [];
   try {
-    res = searchLessons(String(question || ''), {
+    const res = searchLessons(String(question || ''), {
       limit: MAX_CONTEXT_LESSONS,
       feedbackDir: opts.feedbackDir,
     });
-  } catch (_) {
+    const rows = res?.results || res?.lessons || [];
+    return rows.slice(0, MAX_CONTEXT_LESSONS).map(lessonToContextItem);
+  } catch (err) {
+    debugChatFallback('lesson retrieval failed', err);
+    return [];
+  }
+}
+
+async function retrieveVectorContext(question, opts = {}) {
+  if (opts.useVectorSearch === false) return [];
+  try {
+    const vectorStore = require(path.join(__dirname, 'vector-store'));
+    const vecResults = vectorStore.searchSimilar
+      ? await vectorStore.searchSimilar(String(question || ''), opts.vectorLimit || 4)
+      : [];
+    return vecResults
+      .filter((match) => match?.text)
+      .map(vectorMatchToContextItem);
+  } catch (err) {
+    debugChatFallback('vector retrieval failed', err);
     return [];
   }
-  const rows = (res && (res.results || res.lessons)) || [];
-  return rows.slice(0, MAX_CONTEXT_LESSONS).map((l) => ({
-    id: l.id,
-    signal: l.signal || l.feedback || '',
-    title: (l.title || '').replace(/^(?:MISTAKE|SUCCESS):\s*/i, '').slice(0, 160),
-    content: String(l.content || l.context || '').replace(/\s+/g, ' ').trim().slice(0, 600),
-    tags: l.tags || [],
-  })).filter((l) => l.content || l.title);
+}
+
+// Retrieve relevant stored lessons and optional raw feedback vector matches.
+async function retrieveContext(question, opts = {}) {
+  const lessons = retrieveLessonContext(question, opts);
+  const vectors = await retrieveVectorContext(question, opts);
+  return dedupeContextItems([...lessons, ...vectors]);
 }
 
 // Build a grounded RAG prompt. Pure function (testable).
@@ -72,15 +157,87 @@ function buildChatPrompt(question, lessons) {
 
 // Parse the Gemini generateContent response into plain text. Pure (testable).
 function parseGeminiAnswer(body) {
-  const parts = body
-    && body.candidates
-    && body.candidates[0]
-    && body.candidates[0].content
-    && body.candidates[0].content.parts;
+  const parts = body?.candidates?.[0]?.content?.parts;
   if (!Array.isArray(parts)) return '';
   return parts.map((p) => (p && typeof p.text === 'string' ? p.text : '')).join('').trim();
 }
 
+function buildOpenAiChatPayload(prompt, model) {
+  return JSON.stringify({
+    model,
+    messages: [{ role: 'user', content: prompt }],
+    temperature: 0.2,
+    max_tokens: 1024,
+  });
+}
+
+function parseOpenAiChatAnswer(json) {
+  return json?.choices?.[0]?.message?.content || '';
+}
+
+function parseModelError(json, status) {
+  return json?.error?.message ? String(json.error.message).split('\n')[0] : `HTTP ${status}`;
+}
+
+function trimTrailingSlashes(value) {
+  let text = String(value || '');
+  while (text.endsWith('/')) {
+    text = text.slice(0, -1);
+  }
+  return text;
+}
+
+async function callLocalOpenAiEndpoint({ endpoint, apiKey, model, prompt, fetchImpl, sources }) {
+  const url = endpoint.includes('/chat/completions')
+    ? endpoint
+    : `${trimTrailingSlashes(endpoint)}/chat/completions`;
+  const res = await fetchImpl(url, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      'Authorization': `Bearer ${apiKey || 'local'}`
+    },
+    body: buildOpenAiChatPayload(prompt, model),
+  });
+  const json = await res.json().catch(() => ({}));
+  if (!res.ok) {
+    return { ok: false, error: 'local_llm_error', status: res.status, message: parseModelError(json, res.status), sources };
+  }
+  const answer = parseOpenAiChatAnswer(json);
+  return { ok: true, answer: answer.trim() || '(no answer returned)', sources, model: json.model || model };
+}
+
+async function callPerplexityEndpoint({ apiKey, prompt, fetchImpl, sources }) {
+  const res = await fetchImpl(PERPLEXITY_ENDPOINT, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', 'Authorization': `Bearer ${apiKey}` },
+    body: buildOpenAiChatPayload(prompt, 'sonar'),
+  });
+  const json = await res.json().catch(() => ({}));
+  if (!res.ok) {
+    return { ok: false, error: 'perplexity_error', status: res.status, message: parseModelError(json, res.status), sources };
+  }
+  const answer = parseOpenAiChatAnswer(json);
+  return { ok: true, answer: answer.trim() || '(no answer returned)', sources, model: json.model || 'perplexity-hybrid' };
+}
+
+async function callGeminiEndpoint({ apiKey, model, prompt, fetchImpl, sources }) {
+  const res = await fetchImpl(`${GEMINI_ENDPOINT}/${encodeURIComponent(model)}:generateContent`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', 'x-goog-api-key': apiKey },
+    body: JSON.stringify({
+      contents: [{ role: 'user', parts: [{ text: prompt }] }],
+      generationConfig: { temperature: 0.2, maxOutputTokens: 1024 },
+    }),
+  });
+  const json = await res.json().catch(() => ({}));
+  if (!res.ok) {
+    return { ok: false, error: 'gemini_error', status: res.status, message: parseModelError(json, res.status), sources };
+  }
+  const answer = parseGeminiAnswer(json);
+  return { ok: true, answer: answer || '(no answer returned)', sources, model: json.modelVersion || model };
+}
+
 // Answer a question grounded in this install's lessons. Returns
 // { ok, answer, sources, model } or { ok:false, error, ... }.
 async function answerDataQuestion(question, opts = {}) {
@@ -90,41 +247,32 @@ async function answerDataQuestion(question, opts = {}) {
     return { ok: false, error: 'question_too_long', message: `Question exceeds ${MAX_QUESTION_CHARS} characters.` };
   }
 
+  const localEndpoint = opts.localEndpoint || process.env.THUMBGATE_LOCAL_LLM_ENDPOINT || '';
+  const localModel = opts.localModel || process.env.THUMBGATE_LOCAL_LLM_MODEL || 'llama3';
   const apiKey = resolveApiKey(opts);
-  const lessons = retrieveContext(q, opts);
+  const lessons = await retrieveContext(q, opts);
   const sources = lessons.map((l) => ({ id: l.id, title: l.title, signal: l.signal }));
 
-  if (!apiKey) {
+  if (!apiKey && !localEndpoint) {
     return {
       ok: false,
       error: 'no_api_key',
-      message: 'Chat is not configured. Set GEMINI_API_KEY (e.g. `npx thumbgate setup-vertex --write`) to enable "chat with your data".',
+      message: 'Chat is not configured. Set a valid GEMINI_API_KEY, PERPLEXITY_API_KEY, or THUMBGATE_LOCAL_LLM_ENDPOINT in the project .env.',
       sources,
     };
   }
 
-  const model = opts.model || process.env.THUMBGATE_GEMINI_MODEL || DEFAULT_MODEL;
+  const model = resolveModel(opts.model);
   const prompt = buildChatPrompt(q, lessons);
   const fetchImpl = opts.fetch || globalThis.fetch;
+  const isPerplexity = apiKey && (apiKey.startsWith('pplx-') || apiKey.includes('perplexity'));
 
   try {
-    const res = await fetchImpl(`${GEMINI_ENDPOINT}/${encodeURIComponent(model)}:generateContent`, {
-      method: 'POST',
-      headers: { 'content-type': 'application/json', 'x-goog-api-key': apiKey },
-      body: JSON.stringify({
-        contents: [{ role: 'user', parts: [{ text: prompt }] }],
-        generationConfig: { temperature: 0.2, maxOutputTokens: 1024 },
-      }),
-    });
-    const json = await res.json().catch(() => ({}));
-    if (!res.ok) {
-      const msg = (json && json.error && json.error.message) ? String(json.error.message).split('\n')[0] : `HTTP ${res.status}`;
-      return { ok: false, error: 'gemini_error', status: res.status, message: msg, sources };
-    }
-    const answer = parseGeminiAnswer(json);
-    return { ok: true, answer: answer || '(no answer returned)', sources, model: json.modelVersion || model };
+    if (localEndpoint) return await callLocalOpenAiEndpoint({ endpoint: localEndpoint, apiKey, model: localModel, prompt, fetchImpl, sources });
+    if (isPerplexity) return await callPerplexityEndpoint({ apiKey, prompt, fetchImpl, sources });
+    return await callGeminiEndpoint({ apiKey, model, prompt, fetchImpl, sources });
   } catch (err) {
-    return { ok: false, error: 'network', message: err && err.message ? err.message : String(err), sources };
+    return { ok: false, error: 'network', message: err?.message || String(err), sources };
   }
 }
 

package/scripts/feedback-sanitizer.js

@@ -0,0 +1,105 @@
+'use strict';
+
+const crypto = require('crypto');
+
+const TRANSPORT_KEYS = new Set([
+  'hookeventname',
+  'hook_event_name',
+  'sessionid',
+  'session_id',
+  'transcriptpath',
+  'transcript_path',
+  'timestamp',
+  'createdat',
+  'created_at',
+  'updatedat',
+  'updated_at',
+  'cwd',
+  'pid',
+  'processid',
+  'process_id',
+  'promptid',
+  'prompt_id',
+  'traceid',
+  'trace_id',
+  'requestid',
+  'request_id',
+  'installid',
+  'install_id',
+  'visitorsessionid',
+  'visitor_session_id',
+  'toolinput',
+  'tool_input',
+]);
+
+const TRANSPORT_WORDS = new Set([
+  ...TRANSPORT_KEYS,
+  'hook',
+  'event',
+  'userpromptsubmit',
+  'user_prompt_submit',
+  'pretooluse',
+  'pre_tool_use',
+  'posttooluse',
+  'post_tool_use',
+  'claude',
+  'codex',
+  'projects',
+  'redacted',
+  'tmp',
+  'private',
+  'folders',
+  'json',
+]);
+
+function stripEphemeralText(text) {
+  if (!text || typeof text !== 'string') return '';
+  return String(text)
+    .replace(/["']?(?:hook_?event_?name|session_?id|transcript_?path|timestamp|created_?at|updated_?at|cwd|pid|process_?id|prompt_?id|trace_?id|request_?id|install_?id|visitor_?session_?id)["']?\s*[:=]\s*["']?[^"',}\]\s]+["']?/gi, ' ')
+    .replace(/\/(?:private\/)?tmp\/[^\s"',}\]]+/gi, ' ')
+    .replace(/\/var\/folders\/[^\s"',}\]]+/gi, ' ')
+    .replace(/\/Users\/[^/\s]+\/\.(?:claude|codex|thumbgate)\/[^\s"',}\]]+/gi, ' ')
+    .replace(/\/Users\/[^/\s]+\/\.config\/thumbgate\/[^\s"',}\]]+/gi, ' ')
+    .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, ' ')
+    .replace(/\b[0-9a-f]{24,}\b/gi, ' ')
+    .replace(/\b\d{4}-\d{2}-\d{2}t\d{2}:\d{2}:\d{2}(?:\.\d+)?z?\b/gi, ' ')
+    .replace(/\b\d{10,13}\b/g, ' ')
+    .replace(/:\d{4,5}\b/g, ':PORT');
+}
+
+function transportWordsOnly(text) {
+  const tokens = String(text || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9_ -]/g, ' ')
+    .split(/\s+/)
+    .filter((token) => token.length >= 3);
+  if (tokens.length === 0) return true;
+  return tokens.every((token) => TRANSPORT_WORDS.has(token));
+}
+
+function sanitizeFeedbackText(text) {
+  const stripped = stripEphemeralText(text)
+    .replace(/\/Users\/[^\s/]+/g, '/Users/redacted')
+    .replace(/\s+/g, ' ')
+    .trim();
+  if (transportWordsOnly(stripped)) return '';
+  return stripped;
+}
+
+function actionFingerprint(parts) {
+  const raw = Array.isArray(parts) ? parts.join(' ') : String(parts || '');
+  const stable = sanitizeFeedbackText(raw)
+    .toLowerCase()
+    .replace(/[^a-z0-9._:/ -]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+  if (!stable || transportWordsOnly(stable)) return null;
+  return crypto.createHash('sha256').update(stable).digest('hex').slice(0, 16);
+}
+
+module.exports = {
+  TRANSPORT_WORDS,
+  sanitizeFeedbackText,
+  actionFingerprint,
+  transportWordsOnly,
+};