thumbgate 1.29.1 → 1.29.2

package/scripts/prompt-eval.js

@@ -143,6 +143,8 @@ function handleRejectExpectation(checks, result, expected) {
 
   const wasRejected = result.accepted === false
     || result.status === 'rejected'
+    || result.status === 'clarification_required'
+    || result.needsClarification === true
     || result.actionType === 'no-action';
   checks.push({
     criterion: 'shouldReject',
@@ -466,8 +468,9 @@ function runSuiteObject(suite, options = {}) {
   const skipped = results.filter((r) => r.status === 'skip').length;
   const totalScore = results.length > 0
     ? Math.round(results.reduce((s, r) => s + r.score, 0) / results.length)
-    : 100;
+    : 0;
   const minScore = options.minScore ?? 80;
+  const insufficientEvidence = results.length === 0;
 
   return {
     suite: suite.name,
@@ -478,8 +481,9 @@ function runSuiteObject(suite, options = {}) {
     skipped,
     score: totalScore,
     minScore,
-    pass: totalScore >= minScore,
-    noCases: results.length === 0,
+    pass: !insufficientEvidence && totalScore >= minScore,
+    noCases: insufficientEvidence,
+    evidenceStatus: insufficientEvidence ? 'insufficient_evidence' : 'measured',
     feedbackDerived: suite.source && suite.source.type === 'feedback-log',
     generatedAt: new Date().toISOString(),
     results,
@@ -685,6 +689,7 @@ function runSuite(suitePath = DEFAULT_SUITE, options = {}) {
 
 function compareReports(currentReport, baselineReport) {
   const baselineById = new Map((baselineReport?.results || []).map((result) => [result.id, result]));
+  const currentById = new Map((currentReport?.results || []).map((result) => [result.id, result]));
   const regressions = [];
   const improvements = [];
 
@@ -717,12 +722,83 @@ function compareReports(currentReport, baselineReport) {
     }
   }
 
+  for (const baseline of baselineReport?.results || []) {
+    if (currentById.has(baseline.id)) continue;
+    regressions.push({
+      id: baseline.id,
+      baselineScore: baseline.score,
+      currentScore: null,
+      delta: null,
+      baselineStatus: baseline.status,
+      currentStatus: 'missing',
+    });
+  }
+
   return {
     baselineSuite: baselineReport?.suite || null,
     baselineScore: Number.isFinite(Number(baselineReport?.score)) ? Number(baselineReport.score) : null,
     scoreDelta: Number.isFinite(Number(baselineReport?.score)) ? currentReport.score - Number(baselineReport.score) : null,
     regressions,
     improvements,
+    baselineCases: baselineById.size,
+    currentCases: currentById.size,
+    baselineCoverageRate: baselineById.size
+      ? Math.round((Array.from(baselineById.keys()).filter((id) => currentById.has(id)).length / baselineById.size) * 10000) / 10000
+      : null,
+  };
+}
+
+function logSafeResult(result, index) {
+  const allowedStatuses = new Set(['pass', 'fail', 'error', 'skip']);
+  return {
+    case: index + 1,
+    status: allowedStatuses.has(result.status) ? result.status : 'error',
+    score: Number(result.score || 0),
+    passCount: Number(result.passCount || 0),
+    totalChecks: Number(result.totalChecks || 0),
+  };
+}
+
+function logSafeReport(report, suite, fromFeedback) {
+  const source = suite?.source || {};
+  const comparison = report.comparison
+    ? {
+        baselineScore: Number(report.comparison.baselineScore || 0),
+        scoreDelta: Number(report.comparison.scoreDelta || 0),
+        regressionCount: Number(report.comparison.regressions?.length || 0),
+        improvementCount: Number(report.comparison.improvements?.length || 0),
+        baselineCases: Number(report.comparison.baselineCases || 0),
+        currentCases: Number(report.comparison.currentCases || 0),
+        baselineCoverageRate: report.comparison.baselineCoverageRate,
+      }
+    : undefined;
+  return {
+    suiteType: fromFeedback ? 'feedback-derived' : 'configured',
+    total: Number(report.total || 0),
+    passed: Number(report.passed || 0),
+    failed: Number(report.failed || 0),
+    errors: Number(report.errors || 0),
+    skipped: Number(report.skipped || 0),
+    score: Number(report.score || 0),
+    minScore: Number(report.minScore || 0),
+    pass: report.pass === true,
+    noCases: report.noCases === true,
+    evidenceStatus: report.evidenceStatus === 'measured' ? 'measured' : 'insufficient_evidence',
+    feedbackDerived: report.feedbackDerived === true,
+    syntheticCount: Number(report.syntheticCount || 0),
+    comparison,
+    results: (report.results || []).map(logSafeResult),
+    suiteDefinition: fromFeedback
+      ? {
+          version: Number(suite?.version || 0),
+          source: {
+            type: 'feedback-log',
+            totalEntries: Number(source.totalEntries || 0),
+            selectedCases: Number(source.selectedCases || 0),
+          },
+          evaluationCount: Number(suite?.evaluations?.length || 0),
+        }
+      : undefined,
   };
 }
 
@@ -847,7 +923,7 @@ if (isCliInvocation()) {
   }
 
   if (json) {
-    console.log(JSON.stringify({ ...report, suiteDefinition: fromFeedback ? suite : undefined }, null, 2));
+    console.log(JSON.stringify(logSafeReport(report, suite, fromFeedback), null, 2));
   } else {
     console.log(`\n${report.suite}`);
     console.log('='.repeat(50));
@@ -883,6 +959,7 @@ module.exports = {
   gradeOutput,
   loadSuite,
   loadReport,
+  logSafeReport,
   compareReports,
   readJsonl,
   runEvaluation,

package/scripts/schedule-manager.js

@@ -0,0 +1,249 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { execSync } = require('child_process');
+const { buildAgenticDataPipelineJobSpec } = require('./agentic-data-pipeline');
+const { ensureDir } = require('./fs-utils');
+
+const SCHEDULES_DIR = path.join(os.homedir(), '.thumbgate', 'schedules');
+const PLIST_PREFIX = 'com.thumbgate.schedule';
+
+
+function escapePlistString(value) {
+  return String(value || '')
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#39;');
+}
+
+/**
+ * Parse a simple cron-like spec into LaunchAgent calendar intervals
+ * Supports: "daily 9:00", "weekly monday 8:30", "hourly", "every 6h"
+ */
+function parseCronSpec(spec) {
+  const s = spec.toLowerCase().trim();
+
+  if (s === 'hourly') {
+    return { Minute: 0 };
+  }
+
+  const everyHMatch = s.match(/^every\s+(\d+)\s*h/);
+  if (everyHMatch) {
+    return { Minute: 0 }; // LaunchAgent doesn't support "every Nh" natively, use hourly
+  }
+
+  const dailyMatch = s.match(/^daily\s+(\d{1,2}):(\d{2})$/);
+  if (dailyMatch) {
+    return { Hour: parseInt(dailyMatch[1]), Minute: parseInt(dailyMatch[2]) };
+  }
+
+  const weeklyMatch = s.match(/^weekly\s+(monday|tuesday|wednesday|thursday|friday|saturday|sunday)\s+(\d{1,2}):(\d{2})$/);
+  if (weeklyMatch) {
+    const dayMap = { sunday: 0, monday: 1, tuesday: 2, wednesday: 3, thursday: 4, friday: 5, saturday: 6 };
+    return {
+      Weekday: dayMap[weeklyMatch[1]],
+      Hour: parseInt(weeklyMatch[2]),
+      Minute: parseInt(weeklyMatch[3]),
+    };
+  }
+
+  // Fallback: try to parse as "HH:MM" (daily)
+  const timeMatch = s.match(/^(\d{1,2}):(\d{2})$/);
+  if (timeMatch) {
+    return { Hour: parseInt(timeMatch[1]), Minute: parseInt(timeMatch[2]) };
+  }
+
+  return null;
+}
+
+function generatePlist(schedule) {
+  const label = escapePlistString(`${PLIST_PREFIX}.${schedule.id}`);
+  const interval = schedule.calendarInterval;
+
+  let intervalXml = '<dict>\n';
+  for (const [key, value] of Object.entries(interval)) {
+    intervalXml += `        <key>${key}</key>\n        <integer>${value}</integer>\n`;
+  }
+  intervalXml += '    </dict>';
+
+  const logDir = escapePlistString(path.join(os.homedir(), '.thumbgate', 'logs'));
+  const workingDirectory = escapePlistString(schedule.workingDirectory || os.homedir());
+  const command = escapePlistString(schedule.command);
+  const homeDir = escapePlistString(os.homedir());
+  const escapedScheduleId = escapePlistString(schedule.id);
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>
+    <string>${label}</string>
+    <key>ProgramArguments</key>
+    <array>
+        <string>${process.execPath}</string>
+        <string>-e</string>
+        <string>${command}</string>
+    </array>
+    <key>WorkingDirectory</key>
+    <string>${workingDirectory}</string>
+    <key>StartCalendarInterval</key>
+    ${intervalXml}
+    <key>StandardOutPath</key>
+    <string>${logDir}/schedule-${escapedScheduleId}.log</string>
+    <key>StandardErrorPath</key>
+    <string>${logDir}/schedule-${escapedScheduleId}-error.log</string>
+    <key>EnvironmentVariables</key>
+    <dict>
+        <key>PATH</key>
+        <string>/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin</string>
+        <key>HOME</key>
+        <string>${homeDir}</string>
+    </dict>
+</dict>
+</plist>`;
+}
+
+function buildManagedScheduleCommand(params = {}) {
+  if (!params.jobFile) {
+    throw new Error('buildManagedScheduleCommand requires jobFile');
+  }
+
+  const runnerPath = path.join(__dirname, 'async-job-runner.js');
+  const jobFile = path.resolve(params.jobFile);
+  const autoResume = params.autoResume !== false;
+
+  return [
+    `const runner = require(${JSON.stringify(runnerPath)});`,
+    `const result = runner.runJobFromFile(${JSON.stringify(jobFile)}, ${JSON.stringify({ autoResume })});`,
+    'process.stdout.write(JSON.stringify(result, null, 2) + "\\n");',
+    'if (["failed", "cancelled"].includes(result.status)) process.exit(1);',
+  ].join(' ');
+}
+
+function buildAgenticDataPipelineSchedule(params = {}) {
+  const id = params.id || params.name || 'agentic-data-pipeline';
+  const jobFile = path.resolve(
+    params.jobFile || path.join(SCHEDULES_DIR, `${id}.job.json`)
+  );
+  const jobSpec = buildAgenticDataPipelineJobSpec({
+    jobId: id,
+    feedbackDir: params.feedbackDir,
+    outDir: params.outDir,
+    window: params.window,
+    liveBilling: params.liveBilling,
+    recordWorkflowRun: params.recordWorkflowRun,
+  });
+
+  return {
+    id,
+    jobFile,
+    jobSpec,
+    command: buildManagedScheduleCommand({
+      jobFile,
+      autoResume: params.autoResume !== false,
+    }),
+  };
+}
+
+function createSchedule(params) {
+  ensureDir(SCHEDULES_DIR);
+
+  const id = params.id || params.name || `sched_${Date.now()}`;
+  const calendarInterval = parseCronSpec(params.schedule);
+  if (!calendarInterval) {
+    return { success: false, error: `Cannot parse schedule: "${params.schedule}". Use formats like "daily 9:00", "weekly monday 8:30", "hourly"` };
+  }
+
+  const jobFile = params.jobFile ? path.resolve(params.jobFile) : null;
+  const command = params.command || (jobFile ? buildManagedScheduleCommand({
+    jobFile,
+    autoResume: params.autoResume !== false,
+  }) : null);
+
+  if (!command) {
+    return { success: false, error: 'Schedule requires command or jobFile' };
+  }
+
+  const schedule = {
+    id,
+    name: params.name || id,
+    description: params.description || '',
+    schedule: params.schedule,
+    command,
+    jobFile,
+    resumePolicy: jobFile ? (params.autoResume !== false ? 'auto_resume' : 'fresh_only') : null,
+    workingDirectory: params.workingDirectory || (jobFile ? path.dirname(jobFile) : process.cwd()),
+    calendarInterval,
+    createdAt: new Date().toISOString(),
+  };
+
+  // Save schedule metadata
+  const metaPath = path.join(SCHEDULES_DIR, `${id}.json`);
+  fs.writeFileSync(metaPath, JSON.stringify(schedule, null, 2), 'utf8');
+
+  // Generate and install LaunchAgent
+  if (process.platform === 'darwin') {
+    const plistContent = generatePlist(schedule);
+    const plistPath = path.join(os.homedir(), 'Library', 'LaunchAgents', `${PLIST_PREFIX}.${id}.plist`);
+    const logDir = path.join(os.homedir(), '.thumbgate', 'logs');
+    if (!fs.existsSync(logDir)) fs.mkdirSync(logDir, { recursive: true });
+    fs.mkdirSync(path.dirname(plistPath), { recursive: true });
+
+    fs.writeFileSync(plistPath, plistContent, 'utf8');
+    try {
+      execSync(`launchctl unload "${plistPath}" 2>/dev/null`, { stdio: 'pipe' });
+    } catch { /* not loaded */ }
+    try {
+      execSync(`launchctl load "${plistPath}"`, { stdio: 'pipe' });
+    } catch (e) {
+      return { success: false, error: `Failed to load LaunchAgent: ${e.message}`, schedule };
+    }
+
+    return { success: true, schedule, plistPath, message: `Schedule "${id}" created and loaded` };
+  }
+
+  // Linux keeps the schedule metadata so operators can install it via user crontab tooling.
+  return { success: true, schedule, message: `Schedule "${id}" saved for Linux crontab installation` };
+}
+
+function listSchedules() {
+  ensureDir(SCHEDULES_DIR);
+  const files = fs.readdirSync(SCHEDULES_DIR).filter(f => f.endsWith('.json'));
+  return files.map(f => {
+    try {
+      return JSON.parse(fs.readFileSync(path.join(SCHEDULES_DIR, f), 'utf8'));
+    } catch {
+      return { id: f.replace('.json', ''), error: 'corrupt' };
+    }
+  });
+}
+
+function deleteSchedule(id) {
+  const metaPath = path.join(SCHEDULES_DIR, `${id}.json`);
+  const plistPath = path.join(os.homedir(), 'Library', 'LaunchAgents', `${PLIST_PREFIX}.${id}.plist`);
+
+  try {
+    execSync(`launchctl unload "${plistPath}" 2>/dev/null`, { stdio: 'pipe' });
+  } catch { /* not loaded */ }
+
+  if (fs.existsSync(plistPath)) fs.unlinkSync(plistPath);
+  if (fs.existsSync(metaPath)) fs.unlinkSync(metaPath);
+
+  return { success: true, message: `Schedule "${id}" deleted` };
+}
+
+module.exports = {
+  createSchedule,
+  listSchedules,
+  deleteSchedule,
+  escapePlistString,
+  generatePlist,
+  parseCronSpec,
+  buildManagedScheduleCommand,
+  buildAgenticDataPipelineSchedule,
+};