thumbgate 1.15.0 → 1.16.1

package/src/api/server.js

@@ -3,7 +3,12 @@ const http = require('http');
 const https = require('https');
 const fs = require('fs');
 const path = require('path');
+const { EventEmitter } = require('node:events');
 const pkg = require('../../package.json');
+const {
+  createUnavailableAsyncOperation,
+  loadOptionalModule,
+} = require('../../scripts/private-core-boundary');
 
 const POSTHOG_API_PATHS = new Set(['/capture', '/batch', '/decide', '/e', '/engage']);
 const POSTHOG_INGEST_HOST = 'us.i.posthog.com';
@@ -50,7 +55,9 @@ const {
 } = require('../../scripts/feedback-paths');
 const {
   readRecentConversationWindow,
-} = require('../../scripts/feedback-history-distiller');
+} = loadOptionalModule(path.join(__dirname, '../../scripts/feedback-history-distiller'), () => ({
+  readRecentConversationWindow: () => [],
+}));
 const {
   readJSONL,
   exportDpoFromMemories,
@@ -69,14 +76,6 @@ const {
 const {
   buildRubricEvaluation,
 } = require('../../scripts/rubric-engine');
-const {
-  listIntents,
-  planIntent,
-} = require('../../scripts/intent-router');
-const {
-  startHandoff,
-  completeHandoff,
-} = require('../../scripts/delegation-runtime');
 const {
   bootstrapInternalAgent,
 } = require('../../scripts/internal-agent-bootstrap');
@@ -118,14 +117,6 @@ const {
   buildHostedSuccessUrl,
   buildHostedCancelUrl,
 } = require('../../scripts/hosted-config');
-const {
-  PRO_MONTHLY_PRICE_DOLLARS,
-  PRO_ANNUAL_PRICE_DOLLARS,
-  TEAM_MONTHLY_PRICE_DOLLARS,
-  normalizePlanId,
-  normalizeBillingCycle,
-  normalizeSeatCount,
-} = require('../../scripts/commercial-offer');
 const {
   generateSkills,
 } = require('../../scripts/skill-generator');
@@ -146,6 +137,9 @@ const {
 const {
   evaluateWorkflowSentinel,
 } = require('../../scripts/workflow-sentinel');
+const {
+  normalizeProviderAction,
+} = require('../../scripts/provider-action-normalizer');
 const {
   recordDecisionEvaluation,
   recordDecisionOutcome,
@@ -163,14 +157,6 @@ const {
 const {
   getSettingsStatus,
 } = require('../../scripts/settings-hierarchy');
-const {
-  searchLessons,
-} = require('../../scripts/lesson-search');
-const {
-  updateRecordInJsonl,
-  deleteRecordFromJsonl,
-  readJSONLLocal,
-} = require('../../scripts/lesson-synthesis');
 const {
   searchThumbgate,
 } = require('../../scripts/thumbgate-search');
@@ -187,17 +173,6 @@ const {
 const {
   resolveAnalyticsWindow,
 } = require('../../scripts/analytics-window');
-const {
-  launchDpoExportJob,
-  launchHarnessJob,
-  pauseQueuedJob,
-  cancelQueuedJob,
-  resumeHostedJob,
-} = require('../../scripts/hosted-job-launcher');
-const {
-  appendWorkflowSprintLead,
-  advanceWorkflowSprintLead,
-} = require('../../scripts/workflow-sprint-intake');
 const {
   importDocument,
   listImportedDocuments,
@@ -209,6 +184,7 @@ const {
 } = require('../../scripts/rate-limiter');
 const { sendProblem, PROBLEM_TYPES } = require('../../scripts/problem-detail');
 const { TOOLS: MCP_TOOLS } = require('../../scripts/tool-registry');
+const resendMailer = require('../../scripts/mailer/resend-mailer');
 const {
   buildContextFootprintReport,
 } = require('../../scripts/context-footprint');
@@ -253,6 +229,80 @@ const STATIC_MIME_BY_EXT = Object.freeze({
   '.txt': 'text/plain; charset=utf-8',
   '.json': 'application/json; charset=utf-8',
 });
+const PRIVATE_API_MODULES = Object.freeze({
+  intentRouter: path.resolve(__dirname, '../../scripts/intent-router.js'),
+  delegationRuntime: path.resolve(__dirname, '../../scripts/delegation-runtime.js'),
+  hostedJobLauncher: path.resolve(__dirname, '../../scripts/hosted-job-launcher.js'),
+  workflowSprintIntake: path.resolve(__dirname, '../../scripts/workflow-sprint-intake.js'),
+  lessonSearch: path.resolve(__dirname, '../../scripts/lesson-search.js'),
+  lessonSynthesis: path.resolve(__dirname, '../../scripts/lesson-synthesis.js'),
+  semanticLayer: path.resolve(__dirname, '../../scripts/semantic-layer.js'),
+  commercialOffer: path.resolve(__dirname, '../../scripts/commercial-offer.js'),
+});
+
+function createPrivateCoreUnavailableError(feature) {
+  const error = new Error(`${feature} is only available in the ThumbGate private core or hosted runtime.`);
+  error.statusCode = 503;
+  error.code = 'PRIVATE_CORE_REQUIRED';
+  return error;
+}
+
+function loadPrivateApiModule(key) {
+  const modulePath = PRIVATE_API_MODULES[key];
+  if (!modulePath) {
+    throw new Error(`Unknown private API module: ${key}`);
+  }
+  try {
+    return require(modulePath);
+  } catch (error) {
+    const message = String(error && error.message || '');
+    if ((error && (error.code === 'MODULE_NOT_FOUND' || error.code === 'ERR_MODULE_NOT_FOUND'))
+      && (message.includes(modulePath) || message.includes(path.basename(modulePath)))) {
+      return null;
+    }
+    throw error;
+  }
+}
+
+function requirePrivateApiModule(key, feature) {
+  const module = loadPrivateApiModule(key);
+  if (!module) {
+    throw createPrivateCoreUnavailableError(feature);
+  }
+  return module;
+}
+
+function getCommercialOfferModule() {
+  return requirePrivateApiModule('commercialOffer', 'Commercial offer planning');
+}
+
+function normalizePlanId(value) {
+  return getCommercialOfferModule().normalizePlanId(value);
+}
+
+function normalizeBillingCycle(value) {
+  return getCommercialOfferModule().normalizeBillingCycle(value);
+}
+
+function normalizeSeatCount(value, fallback) {
+  return getCommercialOfferModule().normalizeSeatCount(value, fallback);
+}
+
+function getLessonSynthesisModule() {
+  return requirePrivateApiModule('lessonSynthesis', 'Lesson synthesis');
+}
+
+function readLessonJsonl(filePath, options) {
+  return getLessonSynthesisModule().readJSONLLocal(filePath, options);
+}
+
+function updateLessonJsonlRecord(filePath, recordId, record) {
+  return getLessonSynthesisModule().updateRecordInJsonl(filePath, recordId, record);
+}
+
+function deleteLessonJsonlRecord(filePath, recordId) {
+  return getLessonSynthesisModule().deleteRecordFromJsonl(filePath, recordId);
+}
 
 function serveStaticFile(res, filePath, { headOnly = false, cacheSeconds = 86400 } = {}) {
   const ext = path.extname(filePath).toLowerCase();
@@ -506,11 +556,11 @@ function findRecordById(id, feedbackDir) {
   const feedbackLogPath = path.join(feedbackDir, 'feedback-log.jsonl');
   let memoryRecord = null;
   let feedbackEvent = null;
-  const memoryRecords = readJSONLLocal(memoryLogPath, { maxLines: 0 });
+  const memoryRecords = readLessonJsonl(memoryLogPath, { maxLines: 0 });
   for (const rec of memoryRecords) {
     if (rec.id === id) { memoryRecord = rec; break; }
   }
-  const feedbackRecords = readJSONLLocal(feedbackLogPath, { maxLines: 0 });
+  const feedbackRecords = readLessonJsonl(feedbackLogPath, { maxLines: 0 });
   for (const rec of feedbackRecords) {
     if (rec.id === id) { feedbackEvent = rec; break; }
   }
@@ -535,8 +585,8 @@ function updateLessonRecord(feedbackDir, lessonId, updater) {
   if (!updated) return null;
   const memoryLogPath = path.join(feedbackDir, 'memory-log.jsonl');
   const feedbackLogPath = path.join(feedbackDir, 'feedback-log.jsonl');
-  const updatedMemory = updateRecordInJsonl(memoryLogPath, lessonId, updated);
-  const updatedFeedback = updateRecordInJsonl(feedbackLogPath, lessonId, updated);
+  const updatedMemory = updateLessonJsonlRecord(memoryLogPath, lessonId, updated);
+  const updatedFeedback = updateLessonJsonlRecord(feedbackLogPath, lessonId, updated);
   if (!updatedMemory && !updatedFeedback) return null;
   return updated;
 }
@@ -1115,6 +1165,61 @@ function resolveBillingSummaryOptions(parsed) {
   });
 }
 
+function sendInvalidAnalyticsWindowProblem(res, title, err) {
+  sendProblem(res, {
+    type: PROBLEM_TYPES.INVALID_REQUEST,
+    title,
+    status: 400,
+    detail: err && err.message ? err.message : 'Invalid analytics window request.',
+  });
+}
+
+function resolveBillingSummaryOptionsOrRespondProblem(res, parsed, invalidTitle) {
+  try {
+    return resolveBillingSummaryOptions(parsed);
+  } catch (err) {
+    sendInvalidAnalyticsWindowProblem(res, invalidTitle, err);
+    return null;
+  }
+}
+
+async function buildLiveDashboardData(parsed, feedbackDir) {
+  const summaryOptions = resolveBillingSummaryOptions(parsed);
+  const billingSummary = await getBillingSummaryLive(summaryOptions);
+  const data = generateDashboard(feedbackDir, {
+    analyticsWindow: summaryOptions,
+    billingSummary,
+    billingSource: 'live',
+    authContext: { tier: 'pro' },
+  });
+  return { summaryOptions, data };
+}
+
+async function loadLiveDashboardDataOrRespondProblem(res, parsed, feedbackDir, invalidTitle) {
+  try {
+    return await buildLiveDashboardData(parsed, feedbackDir);
+  } catch (err) {
+    sendInvalidAnalyticsWindowProblem(res, invalidTitle, err);
+    return null;
+  }
+}
+
+function buildLossAnalyticsResponse(data, summaryOptions) {
+  return {
+    window: data.analytics.window || summaryOptions,
+    lossAnalysis: data.analytics.lossAnalysis || null,
+    buyerLoss: data.analytics.buyerLoss || null,
+    funnel: data.analytics.funnel || null,
+    revenue: data.analytics.revenue || null,
+    telemetry: {
+      conversionFunnel: data.analytics.telemetry && data.analytics.telemetry.conversionFunnel,
+      behavior: data.analytics.telemetry && data.analytics.telemetry.behavior,
+      ctas: data.analytics.telemetry && data.analytics.telemetry.ctas,
+      visitors: data.analytics.telemetry && data.analytics.telemetry.visitors,
+    },
+  };
+}
+
 function createJourneyId(prefix) {
   return createTraceId(prefix).replace(/^trace_/, `${prefix}_`);
 }
@@ -1338,6 +1443,7 @@ function serveTrackedLinkRedirect({ req, res, parsed, hostedConfig, isHeadReques
 }
 
 function resolveCheckoutOfferSummary(metadata = {}) {
+  const commercialOffer = getCommercialOfferModule();
   const planId = normalizePlanId(metadata.planId);
   const billingCycle = normalizeBillingCycle(metadata.billingCycle);
 
@@ -1348,8 +1454,8 @@ function resolveCheckoutOfferSummary(metadata = {}) {
       billingCycle: 'monthly',
       seatCount,
       type: 'subscription',
-      price: TEAM_MONTHLY_PRICE_DOLLARS * seatCount,
-      priceLabel: `$${TEAM_MONTHLY_PRICE_DOLLARS}/seat/mo`,
+      price: commercialOffer.TEAM_MONTHLY_PRICE_DOLLARS * seatCount,
+      priceLabel: `$${commercialOffer.TEAM_MONTHLY_PRICE_DOLLARS}/seat/mo`,
     };
   }
 
@@ -1359,7 +1465,7 @@ function resolveCheckoutOfferSummary(metadata = {}) {
       billingCycle: 'annual',
       seatCount: 1,
       type: 'subscription',
-      price: PRO_ANNUAL_PRICE_DOLLARS,
+      price: commercialOffer.PRO_ANNUAL_PRICE_DOLLARS,
       priceLabel: '$149/yr',
     };
   }
@@ -1369,7 +1475,7 @@ function resolveCheckoutOfferSummary(metadata = {}) {
     billingCycle: 'monthly',
     seatCount: 1,
     type: 'subscription',
-    price: PRO_MONTHLY_PRICE_DOLLARS,
+    price: commercialOffer.PRO_MONTHLY_PRICE_DOLLARS,
     priceLabel: '$19/mo',
   };
 }
@@ -3133,6 +3239,18 @@ function createApiServer() {
   const expectedApiKey = getExpectedApiKey();
   const expectedOperatorKey = getExpectedOperatorKey();
 
+  // Live-event bus. Feedback captures, prevention-rule regenerations, and
+  // gate decisions push to this emitter; the /v1/events SSE endpoint streams
+  // those events to connected dashboard clients so they render in real time
+  // instead of waiting for the next manual refresh.
+  //
+  // See .changeset/dashboard-sse-live.md for the ROI rationale — this is a
+  // direct application of the "persistent channel beats per-turn HTTP" pattern
+  // to ThumbGate's dashboard surface (the primary UI for watching team
+  // feedback flow).
+  const eventBus = new EventEmitter();
+  eventBus.setMaxListeners(200);
+
   return http.createServer(async (req, res) => {
     const parsed = new URL(req.url, 'http://localhost');
     const pathname = parsed.pathname;
@@ -3366,6 +3484,19 @@ function createApiServer() {
               landingPath,
               attribution,
             }) + '\n');
+            // Fire-and-forget welcome email. Never blocks the 200 response, never
+            // throws — the mailer returns a structured result even on failure, so
+            // the signup still succeeds if Resend is down or RESEND_API_KEY is unset.
+            Promise.resolve()
+              .then(() => resendMailer.sendNewsletterWelcomeEmail({ to: email }))
+              .then((result) => {
+                if (!result || result.sent !== true) {
+                  console.warn('[newsletter] welcome email not sent:', email, result && result.reason);
+                }
+              })
+              .catch((err) => {
+                console.warn('[newsletter] welcome email threw:', email, err && err.message);
+              });
           }
           const journeyState = resolveJourneyState(req, parsed);
           appendBestEffortTelemetry(getFeedbackPaths().FEEDBACK_DIR, {
@@ -3688,8 +3819,8 @@ async function addContext(){
       const feedbackDir = requestSafeDataDir;
       const memoryLogPath = path.join(feedbackDir, 'memory-log.jsonl');
       const feedbackLogPath = path.join(feedbackDir, 'feedback-log.jsonl');
-      const deletedMemory = deleteRecordFromJsonl(memoryLogPath, lessonId);
-      const deletedFeedback = deleteRecordFromJsonl(feedbackLogPath, lessonId);
+      const deletedMemory = deleteLessonJsonlRecord(memoryLogPath, lessonId);
+      const deletedFeedback = deleteLessonJsonlRecord(feedbackLogPath, lessonId);
       if (!deletedMemory && !deletedFeedback) {
         sendJson(res, 404, { error: 'Record not found' });
         return;
@@ -4426,7 +4557,8 @@ async function addContext(){
         const body = isFormSubmission
           ? await parseFormBody(req, 24 * 1024)
           : await parseJsonBody(req, 24 * 1024);
-        const lead = appendWorkflowSprintLead({
+        const workflowSprintIntake = requirePrivateApiModule('workflowSprintIntake', 'Workflow sprint intake');
+        const lead = workflowSprintIntake.appendWorkflowSprintLead({
           ...body,
           traceId: body.traceId || traceId,
           acquisitionId: body.acquisitionId || journeyState.acquisitionId,
@@ -4837,15 +4969,71 @@ async function addContext(){
         return;
       }
 
+      // Server-Sent Events stream of live feedback / rule-regen / gate events.
+      // Dashboard clients subscribe once (with the same Bearer auth already
+      // required for /v1/feedback/stats) and receive pushed events as they
+      // happen — no polling, no per-event HTTP round trip. Replaces the
+      // implicit "refresh the page" loop that used to be the only way to see
+      // new feedback land.
+      if (req.method === 'GET' && pathname === '/v1/events') {
+        res.writeHead(200, {
+          'Content-Type': 'text/event-stream',
+          'Cache-Control': 'no-cache, no-transform',
+          'Connection': 'keep-alive',
+          'X-Accel-Buffering': 'no', // disable nginx buffering if any proxy is in front
+        });
+        // Initial handshake so the client knows the stream is live. Carries
+        // the server version so clients can detect mid-session upgrades.
+        res.write(`event: connected\ndata: ${JSON.stringify({ version: pkg.version, ts: Date.now() })}\n\n`);
+
+        // Both writes below can fail once the client has disconnected (the
+        // socket is destroyed but our subscriber hasn't been removed yet).
+        // We intentionally swallow the error and rely on the 'close'/'aborted'
+        // handlers below to unsubscribe and clear the heartbeat — there is no
+        // useful recovery action to take inline for a closed stream.
+        const safeWrite = (chunk) => {
+          try {
+            res.write(chunk);
+          } catch (writeErr) {
+            // Connection closed between the emit and the flush; cleanup runs
+            // via the 'close' listener so we don't need to act here. Keep the
+            // exception binding so Sonar's "handle or don't catch" rule is
+            // satisfied without adding log noise on every disconnect.
+            void writeErr;
+          }
+        };
+
+        const onEvent = (payload) => {
+          safeWrite(`event: ${payload.type}\ndata: ${JSON.stringify(payload)}\n\n`);
+        };
+        eventBus.on('broadcast', onEvent);
+
+        // Heartbeat every 25s keeps proxies (Railway, CDNs) from idle-closing
+        // the connection. Clients ignore comment frames per the SSE spec.
+        const heartbeat = setInterval(() => {
+          safeWrite(':ping\n\n');
+        }, 25_000);
+
+        const cleanup = () => {
+          clearInterval(heartbeat);
+          eventBus.removeListener('broadcast', onEvent);
+        };
+        req.on('close', cleanup);
+        req.on('aborted', cleanup);
+        res.on('close', cleanup);
+        return;
+      }
+
       if (req.method === 'GET' && pathname === '/v1/intents/catalog') {
         const mcpProfile = parsed.searchParams.get('mcpProfile') || undefined;
         const bundleId = parsed.searchParams.get('bundleId') || undefined;
         const partnerProfile = parsed.searchParams.get('partnerProfile') || undefined;
         try {
-          const catalog = listIntents({ mcpProfile, bundleId, partnerProfile });
+          const intentRouter = requirePrivateApiModule('intentRouter', 'Intent catalog');
+          const catalog = intentRouter.listIntents({ mcpProfile, bundleId, partnerProfile });
           sendJson(res, 200, catalog);
         } catch (err) {
-          throw createHttpError(400, err.message || 'Invalid intent catalog request');
+          throw createHttpError(err.statusCode || 400, err.message || 'Invalid intent catalog request');
         }
         return;
       }
@@ -4853,7 +5041,8 @@ async function addContext(){
       if (req.method === 'POST' && pathname === '/v1/intents/plan') {
         const body = await parseJsonBody(req);
         try {
-          const plan = planIntent({
+          const intentRouter = requirePrivateApiModule('intentRouter', 'Intent planning');
+          const plan = intentRouter.planIntent({
             intentId: body.intentId,
             context: body.context || '',
             mcpProfile: body.mcpProfile,
@@ -4865,7 +5054,7 @@ async function addContext(){
           });
           sendJson(res, 200, plan);
         } catch (err) {
-          throw createHttpError(400, err.message || 'Invalid intent plan request');
+          throw createHttpError(err.statusCode || 400, err.message || 'Invalid intent plan request');
         }
         return;
       }
@@ -4873,7 +5062,9 @@ async function addContext(){
       if (req.method === 'POST' && pathname === '/v1/handoffs/start') {
         const body = await parseJsonBody(req);
         try {
-          const plan = planIntent({
+          const intentRouter = requirePrivateApiModule('intentRouter', 'Handoff planning');
+          const delegationRuntime = requirePrivateApiModule('delegationRuntime', 'Sequential handoffs');
+          const plan = intentRouter.planIntent({
             intentId: body.intentId,
             context: body.context || '',
             mcpProfile: body.mcpProfile,
@@ -4883,7 +5074,7 @@ async function addContext(){
             approved: body.approved === true,
             repoPath: body.repoPath,
           });
-          const result = startHandoff({
+          const result = delegationRuntime.startHandoff({
             plan,
             context: body.context || '',
             mcpProfile: body.mcpProfile || plan.mcpProfile,
@@ -4902,7 +5093,8 @@ async function addContext(){
       if (req.method === 'POST' && pathname === '/v1/handoffs/complete') {
         const body = await parseJsonBody(req);
         try {
-          const result = completeHandoff({
+          const delegationRuntime = requirePrivateApiModule('delegationRuntime', 'Sequential handoffs');
+          const result = delegationRuntime.completeHandoff({
             handoffId: body.handoffId,
             outcome: body.outcome,
             resultContext: body.resultContext || '',
@@ -4993,7 +5185,8 @@ async function addContext(){
         }
         const inputs = parseOptionalObject(body.inputs, 'inputs') || {};
         try {
-          const launched = launchHarnessJob(identifier, inputs, {
+          const hostedJobLauncher = requirePrivateApiModule('hostedJobLauncher', 'Hosted harness jobs');
+          const launched = hostedJobLauncher.launchHarnessJob(identifier, inputs, {
             jobId: normalizeNullableText(body.jobId) || undefined,
             skill: normalizeNullableText(body.skill) || undefined,
             partnerProfile: normalizeNullableText(body.partnerProfile) || undefined,
@@ -5052,8 +5245,9 @@ async function addContext(){
             throw createHttpError(409, `Job ${jobId} is already ${state.status}`);
           }
 
+          const hostedJobLauncher = requirePrivateApiModule('hostedJobLauncher', 'Hosted job control');
           if (action === 'resume') {
-            const launched = resumeHostedJob(jobId);
+            const launched = hostedJobLauncher.resumeHostedJob(jobId);
             sendJson(res, 202, {
               accepted: true,
               action,
@@ -5067,8 +5261,8 @@ async function addContext(){
 
           if (IDLE_JOB_STATUSES.has(state.status)) {
             const job = action === 'pause'
-              ? pauseQueuedJob(jobId, parseOptionalObject(body.metadata, 'metadata') || {})
-              : cancelQueuedJob(jobId, parseOptionalObject(body.metadata, 'metadata') || {});
+              ? hostedJobLauncher.pauseQueuedJob(jobId, parseOptionalObject(body.metadata, 'metadata') || {})
+              : hostedJobLauncher.cancelQueuedJob(jobId, parseOptionalObject(body.metadata, 'metadata') || {});
             sendJson(res, 202, {
               accepted: true,
               action,
@@ -5197,7 +5391,8 @@ async function addContext(){
           .split(',')
           .map((tag) => tag.trim())
           .filter(Boolean);
-        const results = searchLessons(query, {
+        const lessonSearch = requirePrivateApiModule('lessonSearch', 'Lesson search');
+        const results = lessonSearch.searchLessons(query, {
           limit: Number.isFinite(limit) ? limit : 10,
           category,
           tags,
@@ -5311,6 +5506,19 @@ async function addContext(){
           tags: extractTags(body.tags),
           skill: body.skill,
         });
+        if (result?.accepted) {
+          // Fan out to any connected dashboard clients so they re-render
+          // without polling. Non-sensitive summary only (no chat history,
+          // no evidence blobs).
+          eventBus.emit('broadcast', {
+            type: 'feedback',
+            signal: body.signal,
+            tags: Array.isArray(body.tags) ? body.tags.slice(0, 8) : [],
+            feedbackId: result.feedbackId,
+            promoted: Boolean(result.promoted),
+            ts: Date.now(),
+          });
+        }
         const code = result.accepted ? 200 : 422;
         sendJson(res, code, result);
         return;
@@ -5323,6 +5531,13 @@ async function addContext(){
           ? resolveSafePath(body.outputPath, { safeDataDir: requestSafeDataDir })
           : undefined;
         const result = writePreventionRules(outputPath, Number.isFinite(minOccurrences) ? minOccurrences : 2);
+        // Tell live dashboard clients the rules file just changed so they can
+        // re-fetch the summary without waiting on a poll tick.
+        eventBus.emit('broadcast', {
+          type: 'rules-updated',
+          path: result.path,
+          ts: Date.now(),
+        });
         sendJson(res, 200, {
           path: result.path,
           markdown: result.markdown,
@@ -5380,7 +5595,8 @@ async function addContext(){
 
         if (wantsAsync) {
           try {
-            const launched = launchDpoExportJob(paths, {
+            const hostedJobLauncher = requirePrivateApiModule('hostedJobLauncher', 'Hosted DPO export jobs');
+            const launched = hostedJobLauncher.launchDpoExportJob(paths, {
               jobId: normalizeNullableText(body.jobId) || undefined,
             });
             sendJson(res, 202, {
@@ -5423,8 +5639,8 @@ async function addContext(){
         const feedbackDir = requestSafeDataDir;
         const memoryLogPath = path.join(feedbackDir, 'memory-log.jsonl');
         const feedbackLogPath = path.join(feedbackDir, 'feedback-log.jsonl');
-        const memories = readJSONLLocal(memoryLogPath, { maxLines: 0 });
-        const feedbacks = readJSONLLocal(feedbackLogPath, { maxLines: 0 });
+        const memories = readLessonJsonl(memoryLogPath, { maxLines: 0 });
+        const feedbacks = readLessonJsonl(feedbackLogPath, { maxLines: 0 });
 
         // Merge into unified lesson records
         const lessonMap = new Map();
@@ -5514,7 +5730,7 @@ async function addContext(){
         const feedbackLogPath = path.join(feedbackDir, 'feedback-log.jsonl');
 
         // Load existing IDs for dedup
-        const existing = readJSONLLocal(feedbackLogPath, { maxLines: 0 });
+        const existing = readLessonJsonl(feedbackLogPath, { maxLines: 0 });
         const existingIds = new Set(existing.map((r) => r.id).filter(Boolean));
         // Also dedup by title+signal content hash
         const existingHashes = new Set(existing.map((r) => {
@@ -5699,12 +5915,12 @@ async function addContext(){
 
       // GET /v1/semantic/describe — get canonical definition of a business entity
       if (req.method === 'GET' && pathname === '/v1/semantic/describe') {
-        const { describeSemanticSchema } = require('../../scripts/semantic-layer');
+        const semanticLayer = requirePrivateApiModule('semanticLayer', 'Semantic schema');
         const type = parsed.query.type;
         if (!type) {
           throw createHttpError(400, 'type query parameter is required');
         }
-        const schema = describeSemanticSchema();
+        const schema = semanticLayer.describeSemanticSchema();
         const entity = schema.entities[type] || schema.metrics[type];
         if (!entity) {
           sendProblem(res, {
@@ -5780,16 +5996,12 @@ async function addContext(){
           return;
         }
 
-        let summaryOptions;
-        try {
-          summaryOptions = resolveBillingSummaryOptions(parsed);
-        } catch (err) {
-          sendProblem(res, {
-            type: PROBLEM_TYPES.INVALID_REQUEST,
-            title: 'Invalid billing summary query',
-            status: 400,
-            detail: err && err.message ? err.message : 'Invalid analytics window request.',
-          });
+        const summaryOptions = resolveBillingSummaryOptionsOrRespondProblem(
+          res,
+          parsed,
+          'Invalid billing summary query',
+        );
+        if (!summaryOptions) {
           return;
         }
 
@@ -5813,7 +6025,8 @@ async function addContext(){
         const { FEEDBACK_DIR } = getFeedbackPaths();
         try {
           const body = await parseJsonBody(req, 24 * 1024);
-          const result = advanceWorkflowSprintLead(body, { feedbackDir: FEEDBACK_DIR });
+          const workflowSprintIntake = requirePrivateApiModule('workflowSprintIntake', 'Workflow sprint intake');
+          const result = workflowSprintIntake.advanceWorkflowSprintLead(body, { feedbackDir: FEEDBACK_DIR });
 
           appendBestEffortTelemetry(FEEDBACK_DIR, {
             eventType: 'workflow_sprint_lead_advanced',
@@ -5910,28 +6123,36 @@ async function addContext(){
         return;
       }
 
+      // GET /v1/analytics/losses -- explain where buyer dollars are falling out of the funnel
+      if (req.method === 'GET' && pathname === '/v1/analytics/losses') {
+        const dashboardResult = await loadLiveDashboardDataOrRespondProblem(
+          res,
+          parsed,
+          requestFeedbackDir,
+          'Invalid loss analytics query',
+        );
+        if (!dashboardResult) {
+          return;
+        }
+        const { summaryOptions, data } = dashboardResult;
+
+        sendJson(res, 200, buildLossAnalyticsResponse(data, summaryOptions));
+        return;
+      }
+
       // GET /v1/dashboard -- Full ThumbGate dashboard JSON
       if (req.method === 'GET' && pathname === '/v1/dashboard') {
-        let summaryOptions;
-        try {
-          summaryOptions = resolveBillingSummaryOptions(parsed);
-        } catch (err) {
-          sendProblem(res, {
-            type: PROBLEM_TYPES.INVALID_REQUEST,
-            title: 'Invalid dashboard query',
-            status: 400,
-            detail: err && err.message ? err.message : 'Invalid analytics window request.',
-          });
+        const dashboardResult = await loadLiveDashboardDataOrRespondProblem(
+          res,
+          parsed,
+          requestFeedbackDir,
+          'Invalid dashboard query',
+        );
+        if (!dashboardResult) {
           return;
         }
+        const { data } = dashboardResult;
 
-        const billingSummary = await getBillingSummaryLive(summaryOptions);
-        const data = generateDashboard(requestFeedbackDir, {
-          analyticsWindow: summaryOptions,
-          billingSummary,
-          billingSource: 'live',
-          authContext: { tier: 'pro' },
-        });
         sendJson(res, 200, data);
         return;
       }
@@ -5968,27 +6189,18 @@ async function addContext(){
 
       // GET /v1/dashboard/render-spec -- Constrained hosted dashboard JSON spec
       if (req.method === 'GET' && pathname === '/v1/dashboard/render-spec') {
-        let summaryOptions;
-        try {
-          summaryOptions = resolveBillingSummaryOptions(parsed);
-        } catch (err) {
-          sendProblem(res, {
-            type: PROBLEM_TYPES.INVALID_REQUEST,
-            title: 'Invalid render-spec query',
-            status: 400,
-            detail: err && err.message ? err.message : 'Invalid analytics window request.',
-          });
+        const dashboardResult = await loadLiveDashboardDataOrRespondProblem(
+          res,
+          parsed,
+          requestFeedbackDir,
+          'Invalid render-spec query',
+        );
+        if (!dashboardResult) {
           return;
         }
+        const { summaryOptions, data } = dashboardResult;
 
         try {
-          const billingSummary = await getBillingSummaryLive(summaryOptions);
-          const data = generateDashboard(requestFeedbackDir, {
-            analyticsWindow: summaryOptions,
-            billingSummary,
-            billingSource: 'live',
-            authContext: { tier: 'pro' },
-          });
           const renderSpec = buildDashboardRenderSpec(data, {
             view: parsed.searchParams.get('view') || undefined,
             now: summaryOptions.now,
@@ -6007,42 +6219,87 @@ async function addContext(){
 
       if (req.method === 'POST' && pathname === '/v1/decisions/evaluate') {
         const body = await parseJsonBody(req);
-        if (!body.toolName) {
+        const normalizedRequestAction = normalizeProviderAction(body);
+        const hasProviderNativeAction = Boolean(
+          body.providerToolCall
+            || body.toolCall
+            || body.toolUse
+            || body.content
+            || body.mcp
+            || body.mcpToolCall
+            || body.method === 'tools/call'
+        );
+        if (!body.toolName && !hasProviderNativeAction) {
           sendProblem(res, {
             type: PROBLEM_TYPES.BAD_REQUEST,
             title: 'Bad Request',
             status: 400,
-            detail: 'toolName is required.',
+            detail: 'toolName or provider tool call is required.',
           });
           return;
         }
 
-        const report = evaluateWorkflowSentinel(body.toolName, {
+        const changedFiles = Array.isArray(body.changedFiles)
+          ? body.changedFiles
+          : normalizedRequestAction.affectedFiles;
+        const scopeState = getScopeState();
+        const toolInput = {
           command: body.command,
           path: body.filePath,
-          changed_files: Array.isArray(body.changedFiles) ? body.changedFiles : [],
+          changed_files: changedFiles,
           repoPath: body.repoPath,
           baseBranch: body.baseBranch,
-        }, {
+          providerToolCall: body.providerToolCall,
+          toolCall: body.toolCall,
+          toolUse: body.toolUse,
+          content: body.content,
+          input: body.input,
+          arguments: body.arguments,
+          method: body.method,
+          params: body.params,
+          mcp: body.mcp,
+          mcpToolCall: body.mcpToolCall,
+          budget: body.budget,
+          usage: body.usage,
+        };
+
+        const report = evaluateWorkflowSentinel(normalizedRequestAction.toolName || body.toolName, toolInput, {
+          provider: body.provider,
+          model: body.model,
+          normalizedAction: normalizedRequestAction,
+          usage: body.usage,
+          tokenEstimate: body.tokenEstimate,
+          costUsd: body.costUsd,
+          budget: body.budget,
           repoPath: body.repoPath,
           baseBranch: body.baseBranch,
-          affectedFiles: Array.isArray(body.changedFiles) ? body.changedFiles : undefined,
+          affectedFiles: changedFiles.length > 0 ? changedFiles : undefined,
           requirePrForReleaseSensitive: body.requirePrForReleaseSensitive === true,
           requireVersionNotBehindBase: body.requireVersionNotBehindBase === true,
-          governanceState: getScopeState(),
+          governanceState: {
+            ...scopeState,
+            branchGovernance: body.workflowDispatch && typeof body.workflowDispatch === 'object'
+              ? {
+                ...(scopeState.branchGovernance || {}),
+                workflowDispatch: body.workflowDispatch,
+              }
+              : scopeState.branchGovernance,
+          },
           feedbackDir: requestFeedbackDir,
         });
         const evaluation = recordDecisionEvaluation(report, {
           source: 'api',
-          toolName: body.toolName,
+          toolName: report.toolName,
           toolInput: {
             command: body.command,
             filePath: body.filePath,
-            changedFiles: Array.isArray(body.changedFiles) ? body.changedFiles : [],
+            changedFiles,
             repoPath: body.repoPath,
             baseBranch: body.baseBranch,
+            normalizedAction: report.normalizedAction,
+            costControl: report.costControl,
           },
-          changedFiles: Array.isArray(body.changedFiles) ? body.changedFiles : [],
+          changedFiles,
         }, {
           feedbackDir: requestFeedbackDir,
         });
@@ -6252,9 +6509,13 @@ module.exports = {
   startServer,
   __test__: {
     buildCheckoutFallbackUrl,
+    createPrivateCoreUnavailableError,
     buildPosthogProxyRequestOptions,
     getPosthogProxyPath,
     isAllowedPosthogProxyPath,
+    PRIVATE_API_MODULES,
+    loadPrivateApiModule,
+    requirePrivateApiModule,
     renderSitemapXml,
     renderPackagedDashboardHtml,
     renderPackagedLessonsHtml,