thumbgate 1.16.13 → 1.16.20

package/scripts/scaling-law-claims.js

@@ -1,60 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-function normalizeText(value) {
-  if (value === undefined || value === null) return '';
-  return String(value).trim();
-}
-
-function classifyScalingClaim(claim) {
-  const text = normalizeText(claim).toLowerCase();
-  if (/\b(pretrain|pretraining|parameters|training tokens|flops|cross entropy|test loss)\b/.test(text)) {
-    return 'pretraining_scaling';
-  }
-  if (/\b(rl|reinforcement|feedback|dpo|kto|reward|policy|thumbs[-\s]?(up|down)|gate|prevention rule)\b/.test(text)) {
-    return 'feedback_policy_scaling';
-  }
-  return 'general_scaling';
-}
-
-function evaluateScalingClaim(input = {}) {
-  const claim = normalizeText(input.claim);
-  const claimType = classifyScalingClaim(claim);
-  const evidence = Array.isArray(input.evidence) ? input.evidence.filter(Boolean) : [];
-  const heldout = evidence.some((entry) => /held[-\s]?out|validation|eval|ablation|backtest/i.test(String(entry)));
-  const production = evidence.some((entry) => /production|real user|workflow run|decision journal|blocked action/i.test(String(entry)));
-  const rlCompute = evidence.some((entry) => /sampling compute|rollout|trajectory|policy update|reward model|rl compute/i.test(String(entry)));
-  const sampling = evidence.some((entry) => /pass@|best-of-n|majority vote|sample budget|sampling/i.test(String(entry)));
-  const issues = [];
-
-  if (!claim) issues.push('missing_claim');
-  if (claimType === 'feedback_policy_scaling' && !heldout) {
-    issues.push('missing_heldout_feedback_eval');
-  }
-  if (claimType === 'feedback_policy_scaling' && /rl|reinforcement|sampling/i.test(claim) && !rlCompute) {
-    issues.push('missing_rl_compute_evidence');
-  }
-  if (claimType === 'feedback_policy_scaling' && /sampling|best-of|vote|pass@/i.test(claim) && !sampling) {
-    issues.push('missing_sampling_budget_evidence');
-  }
-  if (claimType === 'pretraining_scaling' && evidence.length === 0) {
-    issues.push('missing_model_scaling_evidence');
-  }
-  if (/guarantee|always|never|100%|proves?/i.test(claim) && !production) {
-    issues.push('absolute_claim_without_production_evidence');
-  }
-
-  return {
-    claimType,
-    decision: issues.length === 0 ? 'allow' : 'warn',
-    issues,
-    requiredEvidence: claimType === 'feedback_policy_scaling'
-      ? ['held-out eval', 'ablation or backtest', 'RL/sampling compute budget when claimed', 'decision-journal production sample']
-      : ['source data', 'validation metric', 'scope limits'],
-  };
-}
-
-module.exports = {
-  classifyScalingClaim,
-  evaluateScalingClaim,
-};

package/scripts/student-consistent-training.js

@@ -1,73 +0,0 @@
-'use strict';
-
-function buildStudentConsistentTrainingPlan(options = {}) {
-  const student = options.student || 'target-agent-policy';
-  const teacher = options.teacher || 'frontier-reviewer';
-  const dataset = options.dataset || 'thumbgate-feedback-lessons';
-  const holdout = options.holdout || 'feedback-gate-holdout';
-
-  return {
-    method: 'student_consistent_synthetic_sft',
-    dataset,
-    teacher,
-    student,
-    generationContract: {
-      teacherRole: 'adds capability tokens: corrected decision, missing evidence, safer action',
-      studentRole: 'preserves target agent style: terse format, tool discipline, gate vocabulary',
-      rejectIf: [
-        'teacher rewrites the answer into unsupported style',
-        'lesson cannot be traced to source feedback',
-        'sample contains secrets or private customer context',
-        'sample teaches a shortcut that bypasses evidence gates',
-      ],
-    },
-    requiredArtifacts: [
-      'source feedback id',
-      'student baseline response',
-      'teacher correction',
-      'student-consistent final sample',
-      'redaction report',
-      'holdout eval result',
-    ],
-    evals: {
-      holdout,
-      compareAgainst: ['raw_teacher_sft', 'self_distill_only', 'no_training_baseline'],
-      metrics: ['gate_precision', 'gate_recall', 'unsupported_claim_rate', 'style_drift_rate'],
-    },
-  };
-}
-
-function evaluateStudentConsistentTrainingSample(sample = {}) {
-  const issues = [];
-
-  if (!sample.sourceFeedbackId) issues.push('missing_source_feedback_id');
-  if (!sample.studentBaseline) issues.push('missing_student_baseline');
-  if (!sample.teacherCorrection) issues.push('missing_teacher_correction');
-  if (!sample.finalSample) issues.push('missing_final_sample');
-  if (!sample.redacted) issues.push('redaction_required');
-  if (!sample.holdoutEval) issues.push('holdout_eval_required');
-
-  const text = [
-    sample.studentBaseline,
-    sample.teacherCorrection,
-    sample.finalSample,
-  ].filter(Boolean).join('\n');
-  if (/(api[_-]?key|secret|token|password)\s*[:=]/i.test(text)) {
-    issues.push('secret_like_content');
-  }
-
-  if (sample.styleDriftRate !== undefined && Number(sample.styleDriftRate) > 0.15) {
-    issues.push('style_drift_too_high');
-  }
-
-  return {
-    decision: issues.length ? 'warn' : 'allow',
-    issues,
-    onPolicy: !issues.includes('style_drift_too_high') && Boolean(sample.studentBaseline),
-  };
-}
-
-module.exports = {
-  buildStudentConsistentTrainingPlan,
-  evaluateStudentConsistentTrainingSample,
-};