thumbgate 0.9.9

package/scripts/rubric-engine.js

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const DEFAULT_RUBRIC_PATH = path.join(PROJECT_ROOT, 'config', 'rubrics', 'default-v1.json');
+
+function getRubricPath() {
+  return process.env.THUMBGATE_RUBRIC_PATH || DEFAULT_RUBRIC_PATH;
+}
+
+function loadRubricConfig() {
+  const raw = fs.readFileSync(getRubricPath(), 'utf-8');
+  const rubric = JSON.parse(raw);
+
+  if (!rubric || typeof rubric !== 'object') {
+    throw new Error('Invalid rubric config: expected object');
+  }
+  if (!rubric.rubricId || typeof rubric.rubricId !== 'string') {
+    throw new Error('Invalid rubric config: rubricId is required');
+  }
+  if (!Array.isArray(rubric.criteria) || rubric.criteria.length === 0) {
+    throw new Error('Invalid rubric config: criteria must be a non-empty array');
+  }
+
+  const seen = new Set();
+  let totalWeight = 0;
+  rubric.criteria.forEach((criterion) => {
+    if (!criterion.id || typeof criterion.id !== 'string') {
+      throw new Error('Invalid rubric config: criterion id is required');
+    }
+    if (seen.has(criterion.id)) {
+      throw new Error(`Invalid rubric config: duplicate criterion '${criterion.id}'`);
+    }
+    seen.add(criterion.id);
+
+    const weight = Number(criterion.weight);
+    if (!Number.isFinite(weight) || weight <= 0) {
+      throw new Error(`Invalid rubric config: criterion '${criterion.id}' has invalid weight`);
+    }
+    totalWeight += weight;
+
+    const minPassingScore = Number(criterion.minPassingScore || 3);
+    if (!Number.isFinite(minPassingScore) || minPassingScore < 1 || minPassingScore > 5) {
+      throw new Error(`Invalid rubric config: criterion '${criterion.id}' has invalid minPassingScore`);
+    }
+  });
+
+  if (Math.abs(totalWeight - 1) > 0.001) {
+    throw new Error(`Invalid rubric config: weights must sum to 1.0 (got ${totalWeight.toFixed(3)})`);
+  }
+
+  return rubric;
+}
+
+function parseRubricScores(input) {
+  if (input == null) return [];
+  if (Array.isArray(input)) return input;
+  if (typeof input === 'string') {
+    const trimmed = input.trim();
+    if (!trimmed) return [];
+    const parsed = JSON.parse(trimmed);
+    if (!Array.isArray(parsed)) {
+      throw new Error('rubricScores must be an array');
+    }
+    return parsed;
+  }
+  throw new Error('rubricScores must be array or JSON string');
+}
+
+function normalizeRubricScores(rawScores, rubric = loadRubricConfig()) {
+  const scores = parseRubricScores(rawScores);
+  if (scores.length === 0) return [];
+
+  const criterionMap = new Map(rubric.criteria.map((c) => [c.id, c]));
+  return scores.map((item, index) => {
+    if (!item || typeof item !== 'object') {
+      throw new Error(`rubricScores[${index}] must be an object`);
+    }
+    const criterion = String(item.criterion || '').trim();
+    if (!criterionMap.has(criterion)) {
+      throw new Error(`rubricScores[${index}] unknown criterion '${criterion}'`);
+    }
+
+    const score = Number(item.score);
+    if (!Number.isFinite(score) || score < 1 || score > 5) {
+      throw new Error(`rubricScores[${index}] score must be between 1 and 5`);
+    }
+
+    return {
+      criterion,
+      score,
+      evidence: item.evidence ? String(item.evidence).trim() : '',
+      judge: item.judge ? String(item.judge).trim() : 'unknown',
+    };
+  });
+}
+
+function groupByCriterion(scores) {
+  const grouped = {};
+  scores.forEach((scoreItem) => {
+    if (!grouped[scoreItem.criterion]) grouped[scoreItem.criterion] = [];
+    grouped[scoreItem.criterion].push(scoreItem);
+  });
+  return grouped;
+}
+
+function evaluateGuardrails(guardrails, rubric = loadRubricConfig()) {
+  const input = guardrails && typeof guardrails === 'object' ? guardrails : {};
+  const expected = Array.isArray(rubric.guardrails) ? rubric.guardrails : [];
+
+  const status = {};
+  const failed = [];
+  expected.forEach((g) => {
+    const value = input[g.key];
+    const normalized = value === true ? true : value === false ? false : null;
+    status[g.key] = normalized;
+    if (normalized === false) failed.push(g.key);
+  });
+
+  return {
+    status,
+    failed,
+  };
+}
+
+function evaluateJudgeAgreement(scoresByCriterion) {
+  const disagreements = [];
+  for (const [criterion, entries] of Object.entries(scoresByCriterion)) {
+    if (entries.length < 2) continue;
+    const values = entries.map((e) => e.score);
+    const max = Math.max(...values);
+    const min = Math.min(...values);
+    if (max - min >= 2) {
+      disagreements.push({
+        criterion,
+        max,
+        min,
+        judges: entries.map((e) => e.judge),
+      });
+    }
+  }
+  return disagreements;
+}
+
+function buildRubricEvaluation({ rubricScores, guardrails } = {}) {
+  const rubric = loadRubricConfig();
+  const normalizedScores = normalizeRubricScores(rubricScores, rubric);
+  const scoresByCriterion = groupByCriterion(normalizedScores);
+  const guardrailResult = evaluateGuardrails(guardrails, rubric);
+
+  const criterionBreakdown = {};
+  const failingCriteria = [];
+  const missingEvidenceClaims = [];
+  let weightedScore = 0;
+
+  rubric.criteria.forEach((criterion) => {
+    const entries = scoresByCriterion[criterion.id] || [];
+    const avg = entries.length > 0
+      ? entries.reduce((sum, item) => sum + item.score, 0) / entries.length
+      : null;
+
+    criterionBreakdown[criterion.id] = {
+      averageScore: avg,
+      minPassingScore: Number(criterion.minPassingScore || 3),
+      judgeCount: entries.length,
+      label: criterion.label || criterion.id,
+    };
+
+    if (avg != null) {
+      weightedScore += (avg / 5) * Number(criterion.weight);
+      if (avg < Number(criterion.minPassingScore || 3)) {
+        failingCriteria.push(criterion.id);
+      }
+    }
+
+    if (criterion.requiresEvidence && entries.some((entry) => entry.score >= 4 && !entry.evidence)) {
+      missingEvidenceClaims.push(criterion.id);
+    }
+  });
+
+  const judgeDisagreements = evaluateJudgeAgreement(scoresByCriterion);
+  const blockReasons = [];
+  if (failingCriteria.length > 0) blockReasons.push(`failing_criteria:${failingCriteria.join(',')}`);
+  if (guardrailResult.failed.length > 0) blockReasons.push(`failed_guardrails:${guardrailResult.failed.join(',')}`);
+  if (judgeDisagreements.length > 0) blockReasons.push('judge_disagreement');
+  if (missingEvidenceClaims.length > 0) blockReasons.push(`missing_evidence:${missingEvidenceClaims.join(',')}`);
+
+  return {
+    rubricId: rubric.rubricId,
+    rubricVersion: rubric.version || 1,
+    weightedScore: Math.round(weightedScore * 1000) / 1000,
+    criterionBreakdown,
+    failingCriteria,
+    guardrails: guardrailResult.status,
+    failingGuardrails: guardrailResult.failed,
+    judgeDisagreements,
+    missingEvidenceClaims,
+    promotionEligible: blockReasons.length === 0,
+    blockReasons,
+    rubricScores: normalizedScores,
+  };
+}
+
+module.exports = {
+  DEFAULT_RUBRIC_PATH,
+  getRubricPath,
+  loadRubricConfig,
+  parseRubricScores,
+  normalizeRubricScores,
+  evaluateGuardrails,
+  evaluateJudgeAgreement,
+  buildRubricEvaluation,
+};
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const scoresArg = args.find((arg) => arg.startsWith('--scores='));
+  const guardrailsArg = args.find((arg) => arg.startsWith('--guardrails='));
+
+  if (!scoresArg) {
+    console.log(JSON.stringify(loadRubricConfig(), null, 2));
+    process.exit(0);
+  }
+
+  const scores = scoresArg.replace('--scores=', '');
+  const guardrails = guardrailsArg ? JSON.parse(guardrailsArg.replace('--guardrails=', '')) : {};
+  const result = buildRubricEvaluation({ rubricScores: scores, guardrails });
+  console.log(JSON.stringify(result, null, 2));
+}

package/scripts/schedule-manager.js

@@ -0,0 +1,251 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { execSync } = require('child_process');
+const { buildAgenticDataPipelineJobSpec } = require('./agentic-data-pipeline');
+
+const SCHEDULES_DIR = path.join(os.homedir(), '.rlhf', 'schedules');
+const PLIST_PREFIX = 'com.thumbgate.schedule';
+
+function ensureDir() {
+  if (!fs.existsSync(SCHEDULES_DIR)) fs.mkdirSync(SCHEDULES_DIR, { recursive: true });
+}
+
+function escapePlistString(value) {
+  return String(value || '')
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#39;');
+}
+
+/**
+ * Parse a simple cron-like spec into LaunchAgent calendar intervals
+ * Supports: "daily 9:00", "weekly monday 8:30", "hourly", "every 6h"
+ */
+function parseCronSpec(spec) {
+  const s = spec.toLowerCase().trim();
+
+  if (s === 'hourly') {
+    return { Minute: 0 };
+  }
+
+  const everyHMatch = s.match(/^every\s+(\d+)\s*h/);
+  if (everyHMatch) {
+    return { Minute: 0 }; // LaunchAgent doesn't support "every Nh" natively, use hourly
+  }
+
+  const dailyMatch = s.match(/^daily\s+(\d{1,2}):(\d{2})$/);
+  if (dailyMatch) {
+    return { Hour: parseInt(dailyMatch[1]), Minute: parseInt(dailyMatch[2]) };
+  }
+
+  const weeklyMatch = s.match(/^weekly\s+(monday|tuesday|wednesday|thursday|friday|saturday|sunday)\s+(\d{1,2}):(\d{2})$/);
+  if (weeklyMatch) {
+    const dayMap = { sunday: 0, monday: 1, tuesday: 2, wednesday: 3, thursday: 4, friday: 5, saturday: 6 };
+    return {
+      Weekday: dayMap[weeklyMatch[1]],
+      Hour: parseInt(weeklyMatch[2]),
+      Minute: parseInt(weeklyMatch[3]),
+    };
+  }
+
+  // Fallback: try to parse as "HH:MM" (daily)
+  const timeMatch = s.match(/^(\d{1,2}):(\d{2})$/);
+  if (timeMatch) {
+    return { Hour: parseInt(timeMatch[1]), Minute: parseInt(timeMatch[2]) };
+  }
+
+  return null;
+}
+
+function generatePlist(schedule) {
+  const label = escapePlistString(`${PLIST_PREFIX}.${schedule.id}`);
+  const interval = schedule.calendarInterval;
+
+  let intervalXml = '<dict>\n';
+  for (const [key, value] of Object.entries(interval)) {
+    intervalXml += `        <key>${key}</key>\n        <integer>${value}</integer>\n`;
+  }
+  intervalXml += '    </dict>';
+
+  const logDir = escapePlistString(path.join(os.homedir(), '.rlhf', 'logs'));
+  const workingDirectory = escapePlistString(schedule.workingDirectory || os.homedir());
+  const command = escapePlistString(schedule.command);
+  const homeDir = escapePlistString(os.homedir());
+  const escapedScheduleId = escapePlistString(schedule.id);
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>
+    <string>${label}</string>
+    <key>ProgramArguments</key>
+    <array>
+        <string>${process.execPath}</string>
+        <string>-e</string>
+        <string>${command}</string>
+    </array>
+    <key>WorkingDirectory</key>
+    <string>${workingDirectory}</string>
+    <key>StartCalendarInterval</key>
+    ${intervalXml}
+    <key>StandardOutPath</key>
+    <string>${logDir}/schedule-${escapedScheduleId}.log</string>
+    <key>StandardErrorPath</key>
+    <string>${logDir}/schedule-${escapedScheduleId}-error.log</string>
+    <key>EnvironmentVariables</key>
+    <dict>
+        <key>PATH</key>
+        <string>/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin</string>
+        <key>HOME</key>
+        <string>${homeDir}</string>
+    </dict>
+</dict>
+</plist>`;
+}
+
+function buildManagedScheduleCommand(params = {}) {
+  if (!params.jobFile) {
+    throw new Error('buildManagedScheduleCommand requires jobFile');
+  }
+
+  const runnerPath = path.join(__dirname, 'async-job-runner.js');
+  const jobFile = path.resolve(params.jobFile);
+  const autoResume = params.autoResume !== false;
+
+  return [
+    `const runner = require(${JSON.stringify(runnerPath)});`,
+    `const result = runner.runJobFromFile(${JSON.stringify(jobFile)}, ${JSON.stringify({ autoResume })});`,
+    'process.stdout.write(JSON.stringify(result, null, 2) + "\\n");',
+    'if (["failed", "cancelled"].includes(result.status)) process.exit(1);',
+  ].join(' ');
+}
+
+function buildAgenticDataPipelineSchedule(params = {}) {
+  const id = params.id || params.name || 'agentic-data-pipeline';
+  const jobFile = path.resolve(
+    params.jobFile || path.join(SCHEDULES_DIR, `${id}.job.json`)
+  );
+  const jobSpec = buildAgenticDataPipelineJobSpec({
+    jobId: id,
+    feedbackDir: params.feedbackDir,
+    outDir: params.outDir,
+    window: params.window,
+    liveBilling: params.liveBilling,
+    recordWorkflowRun: params.recordWorkflowRun,
+  });
+
+  return {
+    id,
+    jobFile,
+    jobSpec,
+    command: buildManagedScheduleCommand({
+      jobFile,
+      autoResume: params.autoResume !== false,
+    }),
+  };
+}
+
+function createSchedule(params) {
+  ensureDir();
+
+  const id = params.id || params.name || `sched_${Date.now()}`;
+  const calendarInterval = parseCronSpec(params.schedule);
+  if (!calendarInterval) {
+    return { success: false, error: `Cannot parse schedule: "${params.schedule}". Use formats like "daily 9:00", "weekly monday 8:30", "hourly"` };
+  }
+
+  const jobFile = params.jobFile ? path.resolve(params.jobFile) : null;
+  const command = params.command || (jobFile ? buildManagedScheduleCommand({
+    jobFile,
+    autoResume: params.autoResume !== false,
+  }) : null);
+
+  if (!command) {
+    return { success: false, error: 'Schedule requires command or jobFile' };
+  }
+
+  const schedule = {
+    id,
+    name: params.name || id,
+    description: params.description || '',
+    schedule: params.schedule,
+    command,
+    jobFile,
+    resumePolicy: jobFile ? (params.autoResume !== false ? 'auto_resume' : 'fresh_only') : null,
+    workingDirectory: params.workingDirectory || (jobFile ? path.dirname(jobFile) : process.cwd()),
+    calendarInterval,
+    createdAt: new Date().toISOString(),
+  };
+
+  // Save schedule metadata
+  const metaPath = path.join(SCHEDULES_DIR, `${id}.json`);
+  fs.writeFileSync(metaPath, JSON.stringify(schedule, null, 2), 'utf8');
+
+  // Generate and install LaunchAgent
+  if (process.platform === 'darwin') {
+    const plistContent = generatePlist(schedule);
+    const plistPath = path.join(os.homedir(), 'Library', 'LaunchAgents', `${PLIST_PREFIX}.${id}.plist`);
+    const logDir = path.join(os.homedir(), '.rlhf', 'logs');
+    if (!fs.existsSync(logDir)) fs.mkdirSync(logDir, { recursive: true });
+    fs.mkdirSync(path.dirname(plistPath), { recursive: true });
+
+    fs.writeFileSync(plistPath, plistContent, 'utf8');
+    try {
+      execSync(`launchctl unload "${plistPath}" 2>/dev/null`, { stdio: 'pipe' });
+    } catch { /* not loaded */ }
+    try {
+      execSync(`launchctl load "${plistPath}"`, { stdio: 'pipe' });
+    } catch (e) {
+      return { success: false, error: `Failed to load LaunchAgent: ${e.message}`, schedule };
+    }
+
+    return { success: true, schedule, plistPath, message: `Schedule "${id}" created and loaded` };
+  }
+
+  // Linux keeps the schedule metadata so operators can install it via user crontab tooling.
+  return { success: true, schedule, message: `Schedule "${id}" saved for Linux crontab installation` };
+}
+
+function listSchedules() {
+  ensureDir();
+  const files = fs.readdirSync(SCHEDULES_DIR).filter(f => f.endsWith('.json'));
+  return files.map(f => {
+    try {
+      return JSON.parse(fs.readFileSync(path.join(SCHEDULES_DIR, f), 'utf8'));
+    } catch {
+      return { id: f.replace('.json', ''), error: 'corrupt' };
+    }
+  });
+}
+
+function deleteSchedule(id) {
+  const metaPath = path.join(SCHEDULES_DIR, `${id}.json`);
+  const plistPath = path.join(os.homedir(), 'Library', 'LaunchAgents', `${PLIST_PREFIX}.${id}.plist`);
+
+  try {
+    execSync(`launchctl unload "${plistPath}" 2>/dev/null`, { stdio: 'pipe' });
+  } catch { /* not loaded */ }
+
+  if (fs.existsSync(plistPath)) fs.unlinkSync(plistPath);
+  if (fs.existsSync(metaPath)) fs.unlinkSync(metaPath);
+
+  return { success: true, message: `Schedule "${id}" deleted` };
+}
+
+module.exports = {
+  createSchedule,
+  listSchedules,
+  deleteSchedule,
+  escapePlistString,
+  generatePlist,
+  parseCronSpec,
+  buildManagedScheduleCommand,
+  buildAgenticDataPipelineSchedule,
+};