thumbgate 1.30.0 → 1.31.0

package/src/api/server.js

@@ -210,7 +210,7 @@ const {
   getSettingsStatus,
 } = require('../../scripts/settings-hierarchy');
 const {
-  searchThumbgate,
+  searchThumbgateAsync,
 } = require('../../scripts/thumbgate-search');
 const {
   appendTelemetryPing,
@@ -480,6 +480,25 @@ const TRACKED_LINK_TARGETS = Object.freeze({
     },
     allowCustomerEmail: true,
   },
+  // Human skip from Pro interstitial: same-origin hop so crawlers never see buy.stripe.com
+  // in the interstitial HTML (bot-guard contract). Bots are deflected to /checkout/pro.
+  'pro-direct': {
+    fallbackHref: PRO_CHECKOUT_URL,
+    external: true,
+    botDeflectPath: '/checkout/pro',
+    ctaId: 'pro_checkout_direct_stripe',
+    ctaPlacement: 'checkout_interstitial',
+    eventType: 'pro_checkout_direct_stripe',
+    defaults: {
+      utm_source: 'website',
+      utm_medium: 'checkout_interstitial',
+      utm_campaign: 'pro_upgrade',
+      plan_id: 'pro',
+      billing_cycle: 'monthly',
+    },
+    allowCustomerEmail: true,
+    prefillStripeEmail: true,
+  },
   // 2026-06-02: Teams/Aiventyx deprecated. Redirect legacy links to Pro.
   teams: {
     path: '/go/pro?utm_source=legacy_teams&utm_medium=redirect',
@@ -837,6 +856,67 @@ function getRequestFeedbackPaths(req, parsed) {
   });
 }
 
+const DATA_IDENTITY_CACHE = new Map();
+const DATA_IDENTITY_CACHE_LIMIT = 2048;
+
+function hashDataIdentity(prefix, value) {
+  const normalized = String(value || '');
+  const cacheKey = `${prefix}\0${normalized}`;
+  const cached = DATA_IDENTITY_CACHE.get(cacheKey);
+  if (cached) return cached;
+
+  // Billing identifiers can be email-like and API keys must never be made
+  // guessable through a fast digest. A memory-hard KDF gives us a stable,
+  // path-safe pseudonym without persisting the source identifier.
+  const derived = `${prefix}_${crypto.scryptSync(
+    normalized,
+    `thumbgate-${prefix}-data-identity-v1`,
+    16,
+    { N: 16384, r: 8, p: 1 }
+  ).toString('hex').slice(0, 24)}`;
+
+  if (DATA_IDENTITY_CACHE.size >= DATA_IDENTITY_CACHE_LIMIT) {
+    DATA_IDENTITY_CACHE.clear();
+  }
+  DATA_IDENTITY_CACHE.set(cacheKey, derived);
+  return derived;
+}
+
+function buildHostedTenantIdentity(validation = {}) {
+  if (!validation || validation.valid !== true || !validation.customerId) return null;
+  const tenantSeed = validation.customerId;
+  const principalSeed = validation.installId || tenantSeed;
+  const tenantId = hashDataIdentity('tenant', tenantSeed);
+  return {
+    partition: tenantId,
+    accessContext: {
+      tenantId,
+      principalId: hashDataIdentity('principal', principalSeed),
+      isAdmin: false,
+    },
+  };
+}
+
+function resolveRequestDataIdentity(req, expectedApiKey) {
+  const token = extractApiKey(req);
+  if (!token || (expectedApiKey && safeKeyEqual(token, expectedApiKey))) {
+    return { partition: null, accessContext: null };
+  }
+  const validation = validateApiKey(token);
+  const identity = buildHostedTenantIdentity(validation);
+  if (validation.valid === true && !identity) {
+    return { partition: null, accessContext: null, invalidHostedIdentity: true };
+  }
+  return identity || { partition: null, accessContext: null };
+}
+
+function partitionFeedbackPaths(paths, dataIdentity) {
+  if (!dataIdentity?.partition) return paths;
+  return getFeedbackPaths({
+    feedbackDir: path.join(paths.FEEDBACK_DIR, 'tenants', dataIdentity.partition),
+  });
+}
+
 function getSafeDataDir(req, parsed) {
   const { FEEDBACK_LOG_PATH } = getRequestFeedbackPaths(req, parsed);
   return path.resolve(path.dirname(FEEDBACK_LOG_PATH));
@@ -2501,8 +2581,9 @@ ${hiddenInputs}
 <p class="email-note">Required so the checkout has attributable buyer intent and your receipt can be recovered.</p>
 <button type="submit" class="primary">Pay $19/mo with Stripe →</button>
 </form>
+<a class="secondary" data-i="pro_checkout_direct_stripe" rel="nofollow" href="/go/pro-direct?utm_source=website&amp;utm_medium=checkout_interstitial&amp;utm_campaign=pro_upgrade&amp;utm_content=skip_form&amp;cta_id=pro_checkout_direct_stripe&amp;cta_placement=checkout_interstitial">Skip form — open Stripe checkout directly →</a>
 <a class="secondary" data-i="workflow_sprint_intake" href="/#workflow-sprint-intake">Not sure yet? Send the workflow first</a>
-<p class="choice-note">Cancel anytime. 7-day refund, no questions. Diagnostics and sprints have their own pages.</p>
+<p class="choice-note">Cancel anytime. 7-day refund, no questions. Diagnostics and sprints have their own pages. Email path keeps receipts attributable; skip uses a same-origin hop that bot-deflects crawlers.</p>
 <div class="objection-box" aria-label="Checkout feedback">
 <strong>Not buying today?</strong>
 <p class="objection-note">Tap one reason so we know what to fix. This does not sign you up.</p>
@@ -2825,6 +2906,37 @@ function serveTrackedLinkRedirect({ req, res, parsed, hostedConfig, isHeadReques
 
   const { FEEDBACK_DIR } = getFeedbackPaths();
   const journeyState = resolveJourneyState(req, parsed);
+  const botClassification = classifyRequester(req.headers);
+  if (target.botDeflectPath && botClassification.isBot) {
+    const deflectUrl = new URL(target.botDeflectPath, hostedConfig.appOrigin);
+    appendTrackedLinkQueryParams(deflectUrl, parsed, target);
+    if (!isHeadRequest) {
+      appendBestEffortTelemetry(
+        FEEDBACK_DIR,
+        {
+          eventType: 'checkout_bot_deflected',
+          clientType: 'web',
+          page: `/go/${target.slug}`,
+          reason: botClassification.reason || 'bot',
+          destinationSlug: target.slug,
+          acquisitionId: journeyState.acquisitionId,
+          visitorId: journeyState.visitorId,
+          sessionId: journeyState.sessionId,
+        },
+        req.headers,
+        `tracked_link_bot_deflect:${target.slug}`
+      );
+    }
+    res.writeHead(302, {
+      ...(journeyState.setCookieHeaders.length ? { 'Set-Cookie': journeyState.setCookieHeaders } : {}),
+      'Cache-Control': 'no-store',
+      'X-Robots-Tag': 'noindex,nofollow',
+      'X-ThumbGate-Link-Slug': target.slug,
+      Location: deflectUrl.toString(),
+    });
+    res.end();
+    return;
+  }
   const destinationUrl = buildTrackedLinkDestination(target, hostedConfig, parsed);
   const attribution = buildTrackedLinkAttribution(target, parsed, req, journeyState, destinationUrl);
   if (target.external) {
@@ -5224,9 +5336,10 @@ function createApiServer() {
       sendJson(res, 403, { error: 'project selection is only available on localhost requests' });
       return;
     }
-    const requestFeedbackPaths = getRequestFeedbackPaths(req, parsed);
-    const requestFeedbackDir = requestFeedbackPaths.FEEDBACK_DIR;
-    const requestSafeDataDir = getSafeDataDir(req, parsed);
+    let requestFeedbackPaths = getRequestFeedbackPaths(req, parsed);
+    let requestFeedbackDir = requestFeedbackPaths.FEEDBACK_DIR;
+    let requestSafeDataDir = getSafeDataDir(req, parsed);
+    let requestDocumentAccessContext = null;
 
     // PostHog reverse proxy -- bypasses ad blockers.
     // Only allow known PostHog API paths to prevent SSRF (CodeQL js/request-forgery).
@@ -8645,6 +8758,25 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
       return;
     }
 
+    // Provisioned hosted keys are authenticated identities, not merely valid
+    // booleans. Bind every private data path to a hash of the billing customer
+    // before any endpoint reads or writes feedback, documents, traces, or jobs.
+    // Static admin/local operation retains the existing single-tenant path.
+    const requestDataIdentity = resolveRequestDataIdentity(req, expectedApiKey);
+    if (requestDataIdentity.invalidHostedIdentity) {
+      sendProblem(res, {
+        type: PROBLEM_TYPES.UNAUTHORIZED,
+        title: 'Unauthorized',
+        status: 403,
+        detail: 'The hosted API key is not bound to a customer identity.',
+      });
+      return;
+    }
+    requestFeedbackPaths = partitionFeedbackPaths(requestFeedbackPaths, requestDataIdentity);
+    requestFeedbackDir = requestFeedbackPaths.FEEDBACK_DIR;
+    requestSafeDataDir = path.resolve(requestFeedbackDir);
+    requestDocumentAccessContext = requestDataIdentity.accessContext;
+
     // Usage metering — record request for billing keys (not static THUMBGATE_API_KEY)
     const _token = extractBearerToken(req);
     if (_token && _token !== expectedApiKey) {
@@ -8665,6 +8797,33 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
           stats.tier = 'Pro';
         }
 
+        // Active gates for the dashboard hero cards. Connect only fetches
+        // /v1/feedback/stats first; without these fields Active Gates stays
+        // stuck on the HTML placeholder "—" until a later /v1/dashboard call
+        // succeeds (and never updates if that call fails).
+        try {
+          const { computeGateStats } = require('../../scripts/dashboard');
+          const gateStats = computeGateStats();
+          stats.activeGateCount = Number(gateStats.totalGates) || 0;
+          stats.gateStats = {
+            totalGates: Number(gateStats.totalGates) || 0,
+            manualCount: Number(gateStats.manualCount) || 0,
+            autoCount: Number(gateStats.autoCount) || 0,
+            blocked: Number(gateStats.blocked) || 0,
+            warned: Number(gateStats.warned) || 0,
+          };
+        } catch (error) {
+          debugApiFallback('gate stats unavailable for feedback/stats', error);
+          stats.activeGateCount = 0;
+          stats.gateStats = {
+            totalGates: 0,
+            manualCount: 0,
+            autoCount: 0,
+            blocked: 0,
+            warned: 0,
+          };
+        }
+
         const projectChatSettings = readProjectChatSettings(req, parsed);
 
         stats.geminiConfigured = Boolean(
@@ -9364,13 +9523,19 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
         const signal = parsed.searchParams.get('signal') || null;
         let results;
         try {
-          const requestFeedbackPaths = getRequestFeedbackPaths(req, parsed);
-          results = searchThumbgate({
+          results = await searchThumbgateAsync({
             query,
             limit: Number.isFinite(limit) ? limit : 10,
             source,
             signal,
-            feedbackDir: requestFeedbackPaths.FEEDBACK_DIR,
+            feedbackDir: requestFeedbackDir,
+            metadataFilters: {
+              tags: (parsed.searchParams.get('tags') || '').split(',').map((tag) => tag.trim()).filter(Boolean),
+              sourceFormat: parsed.searchParams.get('sourceFormat') || undefined,
+              sourceType: parsed.searchParams.get('sourceType') || undefined,
+            },
+            queryRewrite: parsed.searchParams.get('queryRewrite') !== 'false',
+            accessContext: requestDocumentAccessContext,
           });
         } catch (err) {
           throw createHttpError(400, err.message || 'Invalid ThumbGate search request');
@@ -9388,13 +9553,15 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
         const body = await parseJsonBody(req);
         let results;
         try {
-          const requestFeedbackPaths = getRequestFeedbackPaths(req, parsed);
-          results = searchThumbgate({
+          results = await searchThumbgateAsync({
             query: body.query || body.q || '',
             limit: body.limit,
             source: body.source,
             signal: body.signal,
-            feedbackDir: requestFeedbackPaths.FEEDBACK_DIR,
+            feedbackDir: requestFeedbackDir,
+            metadataFilters: body.filters,
+            queryRewrite: body.queryRewrite !== false,
+            accessContext: requestDocumentAccessContext,
           });
         } catch (err) {
           throw createHttpError(400, err.message || 'Invalid ThumbGate search request');
@@ -9412,6 +9579,7 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
           limit: Number.isFinite(limit) ? limit : 20,
           query,
           tag,
+          accessContext: requestDocumentAccessContext,
         });
         sendJson(res, 200, results);
         return;
@@ -9425,6 +9593,7 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
           }
           const document = readImportedDocument(documentId, {
             feedbackDir: requestFeedbackDir,
+            accessContext: requestDocumentAccessContext,
           });
           if (!document) {
             throw createHttpError(404, `Imported document not found: ${escapeHtml(documentId)}`);
@@ -9557,6 +9726,11 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
           tags: extractTags(body.tags),
           proposeGates: body.proposeGates !== false,
           feedbackDir: requestFeedbackDir,
+          accessContext: requestDocumentAccessContext,
+          visibility: body.visibility === 'private' ? 'private' : 'tenant',
+          allowedPrincipals: Array.isArray(body.allowedPrincipals)
+            ? body.allowedPrincipals.map((value) => String(value || '').trim()).filter(Boolean)
+            : [],
         });
         sendJson(res, 201, {
           ok: true,
@@ -9607,14 +9781,19 @@ a{color:#8b9}</style></head><body><form class="card" method="post" action="/oaut
           fs.writeFileSync(paths.outputPath, result.jsonl);
         }
 
-        sendJson(res, 200, {
+        const response = {
           pairs: result.pairs.length,
+          pairCount: result.pairs.length,
           errors: result.errors.length,
           learnings: result.learnings.length,
           unpairedErrors: result.unpairedErrors.length,
           unpairedLearnings: result.unpairedLearnings.length,
           outputPath: paths.outputPath,
-        });
+        };
+        if (body.includePairs === true) {
+          response.records = result.pairs;
+        }
+        sendJson(res, 200, response);
         return;
       }
 
@@ -10640,6 +10819,8 @@ module.exports = {
     buildLossAnalyticsResponse,
     buildIntakeAlertRateLimitKey,
     sanitizeHtmlUnsafeJsonValue,
+    buildHostedTenantIdentity,
+    partitionFeedbackPaths,
   },
 };