thumbgate 0.9.13 → 1.0.0

package/src/api/server.js

@@ -13,6 +13,10 @@ const {
   getFeedbackPaths,
   appendDiagnosticRecord,
 } = require('../../scripts/feedback-loop');
+const {
+  readActiveProjectState,
+  resolveProjectDir,
+} = require('../../scripts/feedback-paths');
 const {
   readRecentConversationWindow,
 } = require('../../scripts/feedback-history-distiller');
@@ -48,6 +52,11 @@ const {
 const {
   buildCloudflareSandboxPlan,
 } = require('../../scripts/cloudflare-dynamic-sandbox');
+const {
+  listJobStates,
+  readJobState,
+  requestJobControl,
+} = require('../../scripts/async-job-runner');
 const {
   loadModel,
   getReliability,
@@ -132,6 +141,13 @@ const {
 const {
   resolveAnalyticsWindow,
 } = require('../../scripts/analytics-window');
+const {
+  launchDpoExportJob,
+  launchHarnessJob,
+  pauseQueuedJob,
+  cancelQueuedJob,
+  resumeHostedJob,
+} = require('../../scripts/hosted-job-launcher');
 const {
   appendWorkflowSprintLead,
   advanceWorkflowSprintLead,
@@ -161,6 +177,9 @@ const SESSION_COOKIE_NAME = 'thumbgate_session_id';
 const ACQUISITION_COOKIE_NAME = 'thumbgate_acquisition_id';
 const VISITOR_COOKIE_MAX_AGE_SECONDS = 60 * 60 * 24 * 90;
 const BUILD_METADATA = resolveBuildMetadata();
+const TERMINAL_JOB_STATUSES = new Set(['completed', 'failed', 'cancelled']);
+const IDLE_JOB_STATUSES = new Set(['queued', 'paused', 'resume_requested']);
+const JOB_CONTROL_ACTIONS = new Set(['pause', 'cancel', 'resume']);
 
 // ---------------------------------------------------------------------------
 // Stripe event tracking helpers
@@ -203,8 +222,55 @@ function computeConversionStats(events) {
 }
 // ---------------------------------------------------------------------------
 
-function getSafeDataDir() {
-  const { FEEDBACK_LOG_PATH } = getFeedbackPaths();
+function getRequestedProjectSelection(req, parsed) {
+  const projectFromQuery = parsed && parsed.searchParams
+    ? parsed.searchParams.get('project')
+    : null;
+  const projectFromHeaders = req && req.headers
+    ? req.headers['x-thumbgate-project-dir'] || req.headers['x-thumbgate-project']
+    : null;
+  return projectFromQuery || projectFromHeaders || null;
+}
+
+function getEffectiveRequestedProjectSelection(req, parsed) {
+  return isProjectSelectionAllowed(req, parsed)
+    ? getRequestedProjectSelection(req, parsed)
+    : null;
+}
+
+function isProjectSelectionAllowed(req, parsed) {
+  const explicitProject = getRequestedProjectSelection(req, parsed);
+  if (!explicitProject) return true;
+  return isLoopbackHost(getRequestHostHeader(req));
+}
+
+function resolveRequestProjectDir(req, parsed) {
+  const explicitProject = getEffectiveRequestedProjectSelection(req, parsed);
+  return resolveProjectDir({
+    projectDir: explicitProject,
+    env: process.env,
+  });
+}
+
+function shouldPreferProjectScopedFeedback(req, parsed) {
+  const explicitProject = getEffectiveRequestedProjectSelection(req, parsed);
+  if (explicitProject) return true;
+  if (process.env.THUMBGATE_PROJECT_DIR || process.env.CLAUDE_PROJECT_DIR) return true;
+  if (process.env.THUMBGATE_FEEDBACK_DIR) return false;
+  return Boolean(readActiveProjectState({ env: process.env }));
+}
+
+function getRequestFeedbackPaths(req, parsed) {
+  const explicitProject = getEffectiveRequestedProjectSelection(req, parsed);
+  return getFeedbackPaths({
+    projectDir: resolveRequestProjectDir(req, parsed),
+    explicitProjectDir: explicitProject,
+    skipExplicitFeedbackDir: shouldPreferProjectScopedFeedback(req, parsed),
+  });
+}
+
+function getSafeDataDir(req, parsed) {
+  const { FEEDBACK_LOG_PATH } = getRequestFeedbackPaths(req, parsed);
   return path.resolve(path.dirname(FEEDBACK_LOG_PATH));
 }
 
@@ -857,6 +923,14 @@ function getPublicOrigin(req) {
   return `${proto}://${host}`;
 }
 
+function getRequestHostHeader(req) {
+  const forwardedHost = req.headers['x-forwarded-host'];
+  if (Array.isArray(forwardedHost)) {
+    return forwardedHost[0] || req.headers.host || '';
+  }
+  return forwardedHost || req.headers.host || '';
+}
+
 function isLoopbackHost(hostValue) {
   const rawHost = String(hostValue || '').split(',')[0].trim();
   if (!rawHost) {
@@ -1156,6 +1230,7 @@ nav .container { display: flex; justify-content: space-between; align-items: cen
   .actions-bar { flex-direction: column; }
 }
 </style>
+<script defer data-domain="thumbgate-production.up.railway.app" src="https://plausible.io/js/script.js"></script>
 </head>
 <body>
 <nav><div class="container">
@@ -1486,6 +1561,7 @@ function renderCheckoutSuccessPage(runtimeConfig) {
       font-size: 14px;
     }
   </style>
+<script defer data-domain="thumbgate-production.up.railway.app" src="https://plausible.io/js/script.js"></script>
 </head>
 <body>
   <main>
@@ -1765,6 +1841,7 @@ function renderCheckoutCancelledPage(runtimeConfig) {
       margin-top: 12px;
     }
   </style>
+<script defer data-domain="thumbgate-production.up.railway.app" src="https://plausible.io/js/script.js"></script>
 </head>
 <body>
   <main>
@@ -1937,6 +2014,7 @@ function renderWorkflowSprintIntakeResultPage(runtimeConfig, { title, detail, le
       border: 1px solid var(--line);
     }
   </style>
+<script defer data-domain="thumbgate-production.up.railway.app" src="https://plausible.io/js/script.js"></script>
 </head>
 <body>
   <main>
@@ -2094,10 +2172,10 @@ function extractTags(input) {
   return [];
 }
 
-function resolveSafePath(inputPath, { mustExist = false } = {}) {
+function resolveSafePath(inputPath, { mustExist = false, safeDataDir } = {}) {
   const allowExternal = process.env.THUMBGATE_ALLOW_EXTERNAL_PATHS === 'true';
   const resolved = path.resolve(String(inputPath || ''));
-  const SAFE_DATA_DIR = getSafeDataDir();
+  const SAFE_DATA_DIR = safeDataDir || getSafeDataDir();
   const inSafeRoot = resolved === SAFE_DATA_DIR || resolved.startsWith(`${SAFE_DATA_DIR}${path.sep}`);
 
   if (!allowExternal && !inSafeRoot) {
@@ -2111,6 +2189,56 @@ function resolveSafePath(inputPath, { mustExist = false } = {}) {
   return resolved;
 }
 
+function resolveDpoExportPaths(body = {}, options = {}) {
+  const { safeDataDir, fallbackMemoryLogPath = null } = options;
+  return {
+    inputPath: body.inputPath
+      ? resolveSafePath(body.inputPath, { mustExist: true, safeDataDir })
+      : null,
+    memoryLogPath: body.memoryLogPath
+      ? resolveSafePath(body.memoryLogPath, { mustExist: true, safeDataDir })
+      : null,
+    outputPath: body.outputPath
+      ? resolveSafePath(body.outputPath, { safeDataDir })
+      : null,
+    fallbackMemoryLogPath,
+  };
+}
+
+function loadDpoExportMemories({ inputPath, memoryLogPath, fallbackMemoryLogPath = null }) {
+  if (inputPath) {
+    const raw = fs.readFileSync(inputPath, 'utf-8');
+    const parsedMemories = JSON.parse(raw);
+    return Array.isArray(parsedMemories) ? parsedMemories : parsedMemories.memories || [];
+  }
+
+  return readJSONL(memoryLogPath || fallbackMemoryLogPath || DEFAULT_LOCAL_MEMORY_LOG);
+}
+
+function parseJobStatuses(value) {
+  if (!value) return [];
+  return String(value)
+    .split(',')
+    .map((entry) => entry.trim())
+    .filter(Boolean);
+}
+
+function readHostedJobOrThrow(jobId) {
+  const state = readJobState(jobId);
+  if (!state) {
+    throw createHttpError(404, `Job not found: ${jobId}`);
+  }
+  return state;
+}
+
+function normalizeJobIdFromPath(pathname, suffix = '') {
+  const pattern = suffix
+    ? new RegExp(`^/v1/jobs/([^/]+)${suffix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`)
+    : /^\/v1\/jobs\/([^/]+)$/;
+  const match = pathname.match(pattern);
+  return match ? decodeURIComponent(match[1]) : null;
+}
+
 function createApiServer() {
   const expectedApiKey = getExpectedApiKey();
 
@@ -2121,6 +2249,13 @@ function createApiServer() {
     const isGetLikeRequest = req.method === 'GET' || isHeadRequest;
     const publicOrigin = getPublicOrigin(req);
     const hostedConfig = resolveHostedBillingConfig({ requestOrigin: publicOrigin });
+    if (!isProjectSelectionAllowed(req, parsed)) {
+      sendJson(res, 403, { error: 'project selection is only available on localhost requests' });
+      return;
+    }
+    const requestFeedbackPaths = getRequestFeedbackPaths(req, parsed);
+    const requestFeedbackDir = requestFeedbackPaths.FEEDBACK_DIR;
+    const requestSafeDataDir = getSafeDataDir(req, parsed);
 
     // Public MCP endpoint — responds to Smithery registry scanning and MCP initialize
     // The initialize handshake is unauthenticated; subsequent tool calls require Bearer auth
@@ -2394,7 +2529,7 @@ function createApiServer() {
       const signal = parsed.searchParams.get('signal');
       if (signal === 'up' || signal === 'down') {
         const chatHistory = readRecentConversationWindow({
-          feedbackDir: getSafeDataDir(),
+          feedbackDir: requestSafeDataDir,
           limit: 10,
         });
         const result = captureFeedback({
@@ -2484,7 +2619,7 @@ async function addContext(){
         sendJson(res, 400, { error: 'relatedFeedbackId is required' });
         return;
       }
-      const feedbackDir = getSafeDataDir();
+      const feedbackDir = requestSafeDataDir;
       const detailField = signal === 'down' ? 'whatWentWrong' : 'whatWorked';
       const updated = updateLessonRecord(feedbackDir, relatedFeedbackId, (existing) => {
         const nextTags = Array.from(new Set([
@@ -2536,7 +2671,7 @@ async function addContext(){
     const lessonUpdateMatch = pathname.match(/^\/lessons\/([^/]+)\/update$/);
     if (req.method === 'POST' && lessonUpdateMatch) {
       const lessonId = decodeURIComponent(lessonUpdateMatch[1]);
-      const feedbackDir = getSafeDataDir();
+      const feedbackDir = requestSafeDataDir;
       const body = await parseJsonBody(req);
       const record = findRecordById(lessonId, feedbackDir);
       if (!record) {
@@ -2568,7 +2703,7 @@ async function addContext(){
     const lessonDeleteMatch = pathname.match(/^\/lessons\/([^/]+)\/delete$/);
     if (req.method === 'POST' && lessonDeleteMatch) {
       const lessonId = decodeURIComponent(lessonDeleteMatch[1]);
-      const feedbackDir = getSafeDataDir();
+      const feedbackDir = requestSafeDataDir;
       const memoryLogPath = path.join(feedbackDir, 'memory-log.jsonl');
       const feedbackLogPath = path.join(feedbackDir, 'feedback-log.jsonl');
       const deletedMemory = deleteRecordFromJsonl(memoryLogPath, lessonId);
@@ -2585,7 +2720,7 @@ async function addContext(){
     const lessonDetailMatch = pathname.match(/^\/lessons\/([^/]+)$/);
     if (isGetLikeRequest && lessonDetailMatch && lessonDetailMatch[1] !== '') {
       const lessonId = decodeURIComponent(lessonDetailMatch[1]);
-      const feedbackDir = getSafeDataDir();
+      const feedbackDir = requestSafeDataDir;
       const record = findRecordById(lessonId, feedbackDir);
       if (!record) {
         sendHtml(res, 404, renderLessonDetailHtml(null, lessonId));
@@ -2713,7 +2848,7 @@ async function addContext(){
           version: pkg.version,
           status: 'ok',
           docs: 'https://github.com/IgorGanapolsky/ThumbGate',
-          endpoints: ['/health', '/dashboard', '/guide', '/learn', '/pro', '/v1/feedback/capture', '/v1/feedback/stats', '/v1/feedback/summary', '/v1/lessons/search', '/v1/search', '/v1/dashboard', '/v1/dashboard/render-spec', '/v1/settings/status', '/v1/dpo/export', '/v1/analytics/databricks/export'],
+          endpoints: ['/health', '/dashboard', '/guide', '/learn', '/pro', '/v1/feedback/capture', '/v1/feedback/stats', '/v1/feedback/summary', '/v1/lessons/search', '/v1/search', '/v1/dashboard', '/v1/dashboard/render-spec', '/v1/settings/status', '/v1/dpo/export', '/v1/jobs', '/v1/jobs/harness', '/v1/analytics/databricks/export'],
         }, {}, {
           headOnly: isHeadRequest,
         });
@@ -2959,7 +3094,7 @@ async function addContext(){
     }
 
     if (isGetLikeRequest && pathname === '/healthz') {
-      const { FEEDBACK_LOG_PATH, MEMORY_LOG_PATH } = getFeedbackPaths();
+      const { FEEDBACK_LOG_PATH, MEMORY_LOG_PATH } = requestFeedbackPaths;
       sendJson(res, 200, {
         status: 'ok',
         feedbackLogPath: FEEDBACK_LOG_PATH,
@@ -3497,7 +3632,7 @@ async function addContext(){
 
     try {
       if (req.method === 'GET' && pathname === '/v1/feedback/stats') {
-        sendJson(res, 200, analyzeFeedback());
+        sendJson(res, 200, analyzeFeedback(requestFeedbackPaths.FEEDBACK_LOG_PATH));
         return;
       }
 
@@ -3649,6 +3784,112 @@ async function addContext(){
         return;
       }
 
+      if (req.method === 'POST' && pathname === '/v1/jobs/harness') {
+        const body = await parseJsonBody(req);
+        const identifier = body.harness || body.harnessId;
+        if (!identifier) {
+          throw createHttpError(400, 'harness is required');
+        }
+        const inputs = parseOptionalObject(body.inputs, 'inputs') || {};
+        try {
+          const launched = launchHarnessJob(identifier, inputs, {
+            jobId: normalizeNullableText(body.jobId) || undefined,
+            skill: normalizeNullableText(body.skill) || undefined,
+            partnerProfile: normalizeNullableText(body.partnerProfile) || undefined,
+            autoImprove: body.autoImprove !== false,
+          });
+          sendJson(res, 202, {
+            accepted: true,
+            jobId: launched.jobId,
+            status: launched.state.status,
+            launchMode: launched.launchMode,
+            pid: launched.pid,
+            statusUrl: `/v1/jobs/${encodeURIComponent(launched.jobId)}`,
+            job: launched.state,
+          });
+        } catch (err) {
+          throw createHttpError(err.statusCode || 400, err.message || 'Invalid hosted harness request');
+        }
+        return;
+      }
+
+      if (req.method === 'GET' && pathname === '/v1/jobs') {
+        const limit = Number(parsed.searchParams.get('limit') || 20);
+        const statuses = parseJobStatuses(parsed.searchParams.get('status'));
+        const jobs = listJobStates({
+          limit: Number.isFinite(limit) ? limit : 20,
+          statuses,
+        });
+        sendJson(res, 200, {
+          total: jobs.length,
+          jobs,
+        });
+        return;
+      }
+
+      {
+        const jobId = normalizeJobIdFromPath(pathname);
+        if (req.method === 'GET' && jobId) {
+          sendJson(res, 200, {
+            job: readHostedJobOrThrow(jobId),
+          });
+          return;
+        }
+      }
+
+      {
+        const jobId = normalizeJobIdFromPath(pathname, '/control');
+        if (req.method === 'POST' && jobId) {
+          const state = readHostedJobOrThrow(jobId);
+          const body = await parseJsonBody(req);
+          const action = normalizeNullableText(body.action);
+          if (!action || !JOB_CONTROL_ACTIONS.has(action)) {
+            throw createHttpError(400, 'action must be one of pause, cancel, or resume');
+          }
+
+          if (TERMINAL_JOB_STATUSES.has(state.status)) {
+            throw createHttpError(409, `Job ${jobId} is already ${state.status}`);
+          }
+
+          if (action === 'resume') {
+            const launched = resumeHostedJob(jobId);
+            sendJson(res, 202, {
+              accepted: true,
+              action,
+              jobId,
+              launchMode: launched.launchMode,
+              pid: launched.pid,
+              job: launched.state,
+            });
+            return;
+          }
+
+          if (IDLE_JOB_STATUSES.has(state.status)) {
+            const job = action === 'pause'
+              ? pauseQueuedJob(jobId, parseOptionalObject(body.metadata, 'metadata') || {})
+              : cancelQueuedJob(jobId, parseOptionalObject(body.metadata, 'metadata') || {});
+            sendJson(res, 202, {
+              accepted: true,
+              action,
+              jobId,
+              job,
+            });
+            return;
+          }
+
+          const metadata = parseOptionalObject(body.metadata, 'metadata') || {};
+          const control = requestJobControl(jobId, action, metadata);
+          sendJson(res, 202, {
+            accepted: true,
+            action,
+            jobId,
+            control,
+            job: readHostedJobOrThrow(jobId),
+          });
+          return;
+        }
+      }
+
       if (req.method === 'POST' && pathname === '/v1/gates/constraint') {
         const body = await parseJsonBody(req);
         if (!body.key || body.value === undefined) {
@@ -3740,7 +3981,9 @@ async function addContext(){
 
       if (req.method === 'GET' && pathname === '/v1/feedback/summary') {
         const recent = Number(parsed.searchParams.get('recent') || 20);
-        const summary = feedbackSummary(Number.isFinite(recent) ? recent : 20);
+        const summary = feedbackSummary(Number.isFinite(recent) ? recent : 20, {
+          feedbackDir: requestFeedbackDir,
+        });
         sendJson(res, 200, { summary });
         return;
       }
@@ -3757,6 +4000,7 @@ async function addContext(){
           limit: Number.isFinite(limit) ? limit : 10,
           category,
           tags,
+          feedbackDir: requestFeedbackDir,
         });
         sendJson(res, 200, results);
         return;
@@ -3846,7 +4090,9 @@ async function addContext(){
       if (req.method === 'POST' && pathname === '/v1/feedback/rules') {
         const body = await parseJsonBody(req);
         const minOccurrences = Number(body.minOccurrences || 2);
-        const outputPath = body.outputPath ? resolveSafePath(body.outputPath) : undefined;
+        const outputPath = body.outputPath
+          ? resolveSafePath(body.outputPath, { safeDataDir: requestSafeDataDir })
+          : undefined;
         const result = writePreventionRules(outputPath, Number.isFinite(minOccurrences) ? minOccurrences : 2);
         sendJson(res, 200, {
           path: result.path,
@@ -3872,25 +4118,38 @@ async function addContext(){
 
       if (req.method === 'POST' && pathname === '/v1/dpo/export') {
         const body = await parseJsonBody(req);
-        let memories = [];
+        const paths = resolveDpoExportPaths(body, {
+          safeDataDir: requestSafeDataDir,
+          fallbackMemoryLogPath: requestFeedbackPaths.MEMORY_LOG_PATH,
+        });
+        const wantsAsync = body.async === true || normalizeNullableText(body.mode) === 'async';
 
-        if (body.inputPath) {
-          const safeInputPath = resolveSafePath(body.inputPath, { mustExist: true });
-          const raw = fs.readFileSync(safeInputPath, 'utf-8');
-          const parsedMemories = JSON.parse(raw);
-          memories = Array.isArray(parsedMemories) ? parsedMemories : parsedMemories.memories || [];
-        } else {
-          const localPath = body.memoryLogPath
-            ? resolveSafePath(body.memoryLogPath, { mustExist: true })
-            : DEFAULT_LOCAL_MEMORY_LOG;
-          memories = readJSONL(localPath);
+        if (wantsAsync) {
+          try {
+            const launched = launchDpoExportJob(paths, {
+              jobId: normalizeNullableText(body.jobId) || undefined,
+            });
+            sendJson(res, 202, {
+              accepted: true,
+              async: true,
+              jobId: launched.jobId,
+              status: launched.state.status,
+              outputPath: paths.outputPath,
+              statusUrl: `/v1/jobs/${encodeURIComponent(launched.jobId)}`,
+              launchMode: launched.launchMode,
+              job: launched.state,
+            });
+          } catch (err) {
+            throw createHttpError(err.statusCode || 400, err.message || 'Invalid DPO export request');
+          }
+          return;
         }
 
+        const memories = loadDpoExportMemories(paths);
         const result = exportDpoFromMemories(memories);
-        if (body.outputPath) {
-          const safeOutputPath = resolveSafePath(body.outputPath);
-          fs.mkdirSync(path.dirname(safeOutputPath), { recursive: true });
-          fs.writeFileSync(safeOutputPath, result.jsonl);
+        if (paths.outputPath) {
+          fs.mkdirSync(path.dirname(paths.outputPath), { recursive: true });
+          fs.writeFileSync(paths.outputPath, result.jsonl);
         }
 
         sendJson(res, 200, {
@@ -3899,14 +4158,16 @@ async function addContext(){
           learnings: result.learnings.length,
           unpairedErrors: result.unpairedErrors.length,
           unpairedLearnings: result.unpairedLearnings.length,
-          outputPath: body.outputPath ? resolveSafePath(body.outputPath) : null,
+          outputPath: paths.outputPath,
         });
         return;
       }
 
       if (req.method === 'POST' && pathname === '/v1/analytics/databricks/export') {
         const body = await parseJsonBody(req);
-        const outputPath = body.outputPath ? resolveSafePath(body.outputPath) : undefined;
+        const outputPath = body.outputPath
+          ? resolveSafePath(body.outputPath, { safeDataDir: requestSafeDataDir })
+          : undefined;
         const result = exportDatabricksBundle(undefined, outputPath);
         sendJson(res, 200, result);
         return;
@@ -3971,7 +4232,7 @@ async function addContext(){
       // ----------------------------------------------------------------
 
       if (req.method === 'GET' && pathname === '/v1/quality/scores') {
-        const modelPath = path.join(getSafeDataDir(), 'feedback_model.json');
+        const modelPath = path.join(requestSafeDataDir, 'feedback_model.json');
         const model = loadModel(modelPath);
         const reliability = getReliability(model);
         const category = parsed.searchParams.get('category');
@@ -3991,7 +4252,7 @@ async function addContext(){
       }
 
       if (req.method === 'GET' && pathname === '/v1/quality/rules') {
-        const rulesPath = path.join(getSafeDataDir(), 'prevention-rules.md');
+        const rulesPath = path.join(requestSafeDataDir, 'prevention-rules.md');
         let markdown = '';
         if (fs.existsSync(rulesPath)) {
           markdown = fs.readFileSync(rulesPath, 'utf8').trim();
@@ -4008,7 +4269,7 @@ async function addContext(){
       }
 
       if (req.method === 'GET' && pathname === '/v1/quality/posteriors') {
-        const modelPath = path.join(getSafeDataDir(), 'feedback_model.json');
+        const modelPath = path.join(requestSafeDataDir, 'feedback_model.json');
         const model = loadModel(modelPath);
         const posteriors = samplePosteriors(model);
         sendJson(res, 200, { posteriors });
@@ -4247,9 +4508,8 @@ async function addContext(){
           return;
         }
 
-        const { FEEDBACK_DIR } = getFeedbackPaths();
         const billingSummary = await getBillingSummaryLive(summaryOptions);
-        const data = generateDashboard(FEEDBACK_DIR, {
+        const data = generateDashboard(requestFeedbackDir, {
           analyticsWindow: summaryOptions,
           billingSummary,
           billingSource: 'live',
@@ -4275,9 +4535,8 @@ async function addContext(){
         }
 
         try {
-          const { FEEDBACK_DIR } = getFeedbackPaths();
           const billingSummary = await getBillingSummaryLive(summaryOptions);
-          const data = generateDashboard(FEEDBACK_DIR, {
+          const data = generateDashboard(requestFeedbackDir, {
             analyticsWindow: summaryOptions,
             billingSummary,
             billingSource: 'live',