textopt 0.1.0 → 0.2.0

package/dist/index.cjs

@@ -1,8 +1,8 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_reporting = require("./reporting-CNHzbJC-.cjs");
+const require_warnings = require("./warnings-CWRJF-jA.cjs");
 const require_concurrency = require("./concurrency-C-cFzWW2.cjs");
 const require_math = require("./math-BhlziRPc.cjs");
-const require_demos = require("./demos-D2o0qSSo.cjs");
+const require_demos = require("./demos-9v5ts7F3.cjs");
 const require_text = require("./text-CK_HB3su.cjs");
 //#region src/compare.ts
 const EXACT_LIMIT = 20;
@@ -239,6 +239,10 @@ function createJudge(args) {
 	const { model, criteria, scale = DEFAULT_SCALE, renderInput = stringify, renderOutput = stringify, buildPrompt = buildJudgePrompt } = args;
 	if (criteria.length === 0) throw new Error("createJudge requires at least one criterion");
 	if (!Number.isFinite(scale) || scale <= 0) throw new Error(`scale must be a positive number, received ${scale}`);
+	assertCriteria({
+		criteria,
+		scale
+	});
 	return async ({ input, output, expected, signal }) => {
 		return readVerdict({
 			response: await model({
@@ -282,6 +286,7 @@ function buildJudgePrompt(args) {
 		`Grade each criterion from 0 to ${scale}, where ${scale} is a perfect answer.`,
 		"",
 		"Then write feedback. It is read by a program that rewrites the system's instructions, not by a person reviewing this output, so say what the instructions should tell the system to do differently. Feedback about this particular answer is of no use to it.",
+		...expected === void 0 ? [] : ["", "Do not restate the expected answer, or any fact drawn from it, in the feedback. The instruction it is rewritten into is reused on inputs whose answers you have not seen: a fact copied out of the expected answer becomes an answer key memorised in the prompt, which raises the score on this input and teaches the system nothing. Name the kind of thing the answer was missing, not the thing itself."],
 		"",
 		"Reply in exactly this format and nothing else:",
 		...criteria.map((criterion) => `<score name="${criterion.name}">…</score>`),
@@ -300,6 +305,9 @@ function readVerdict(args) {
 	const graded = /* @__PURE__ */ new Map();
 	for (const match of response.matchAll(SCORE)) graded.set(match[1], Number(match[2]));
 	const objectiveScores = {};
+	let gated = false;
+	let weighted = 0;
+	let totalWeight = 0;
 	for (const criterion of criteria) {
 		const grade = graded.get(criterion.name);
 		if (grade === void 0 || Number.isNaN(grade)) return {
@@ -307,15 +315,30 @@ function readVerdict(args) {
 			feedback: `Judge did not grade "${criterion.name}".`,
 			transient: true
 		};
-		objectiveScores[criterion.name] = clamp(grade / scale);
+		const normalized = clamp(grade / scale);
+		objectiveScores[criterion.name] = normalized;
+		if (criterion.gate !== void 0 && grade < criterion.gate) gated = true;
+		const weight = criterion.weight ?? 1;
+		weighted += normalized * weight;
+		totalWeight += weight;
 	}
-	const grades = Object.values(objectiveScores);
 	return {
-		score: grades.reduce((total, grade) => total + grade, 0) / grades.length,
+		score: gated ? 0 : weighted / totalWeight,
 		feedback,
 		objectiveScores
 	};
 }
+function assertCriteria(args) {
+	const { criteria, scale } = args;
+	let totalWeight = 0;
+	for (const { name, weight = 1, gate } of criteria) {
+		if (!Number.isFinite(weight) || weight < 0) throw new Error(`weight on criterion "${name}" must be a non-negative number, received ${weight}`);
+		totalWeight += weight;
+		if (gate !== void 0 && (!Number.isFinite(gate) || gate <= 0)) throw new Error(`gate on criterion "${name}" must be greater than 0, received ${gate}; no grade can fall below 0`);
+		if (gate !== void 0 && gate > scale) throw new Error(`gate on criterion "${name}" is ${gate}, above the scale of ${scale}; no grade can reach it`);
+	}
+	if (totalWeight <= 0) throw new Error("criteria weights must sum to more than 0; at least one criterion has to count towards the score");
+}
 function stringify(value) {
 	return typeof value === "string" ? value : JSON.stringify(value);
 }
@@ -341,28 +364,30 @@ function priceUsage(args) {
 	};
 }
 //#endregion
-exports.BudgetExhausted = require_reporting.BudgetExhausted;
-exports.assertResumable = require_reporting.assertResumable;
+exports.BudgetExhausted = require_warnings.BudgetExhausted;
+exports.assertResumable = require_warnings.assertResumable;
 exports.buildJudgePrompt = buildJudgePrompt;
-exports.candidateFingerprint = require_reporting.candidateFingerprint;
+exports.candidateFingerprint = require_warnings.candidateFingerprint;
 exports.compare = compare;
-exports.componentNames = require_reporting.componentNames;
-exports.costExhausted = require_reporting.costExhausted;
-exports.createBudget = require_reporting.createBudget;
-exports.createDeadline = require_reporting.createDeadline;
-exports.createEvaluator = require_reporting.createEvaluator;
+exports.componentNames = require_warnings.componentNames;
+exports.costExhausted = require_warnings.costExhausted;
+exports.createBudget = require_warnings.createBudget;
+exports.createDeadline = require_warnings.createDeadline;
+exports.createEvaluator = require_warnings.createEvaluator;
 exports.createJudge = createJudge;
-exports.createMemoryCache = require_reporting.createMemoryCache;
+exports.createMemoryCache = require_warnings.createMemoryCache;
 exports.formatDemos = require_demos.formatDemos;
 exports.harvestFewShotExamples = require_demos.harvestFewShotExamples;
 exports.harvestRollouts = require_demos.harvestRollouts;
-exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
-exports.isRunFinished = require_reporting.isRunFinished;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;
 exports.mapWithConcurrency = require_concurrency.mapWithConcurrency;
-exports.measuredMean = require_reporting.measuredMean;
+exports.measuredMean = require_warnings.measuredMean;
 exports.parseDemos = require_demos.parseDemos;
 exports.parseProposedText = require_text.parseProposedText;
 exports.priceUsage = priceUsage;
-exports.requireMeasuredMean = require_reporting.requireMeasuredMean;
-exports.runFingerprint = require_reporting.runFingerprint;
+exports.requireMeasuredMean = require_warnings.requireMeasuredMean;
+exports.resolveValidationSet = require_warnings.resolveValidationSet;
+exports.runFingerprint = require_warnings.runFingerprint;
+exports.seedScoreWarnings = require_warnings.seedScoreWarnings;
 exports.toTrainingJsonl = toTrainingJsonl;

package/dist/index.d.cts

@@ -1,8 +1,8 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, d as EvaluationBatch, f as EvaluationContext, g as ScoreResult, h as RolloutUsage, i as Reporter, l as Candidate, m as EvaluationSplit, n as OptimizerEvent, o as isCandidateAccepted, p as EvaluationPhase, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, u as EvaluateArgs, v as UsageTotals, y as componentNames } from "./reporting-DQbAohc9.cjs";
+import { C as componentNames, S as UsageTotals, _ as EvaluationPhase, a as RunFinished, b as ScoreResult, c as RunWarning, d as seedScoreWarnings, f as Adapter, g as EvaluationContext, h as EvaluationBatch, i as Reporter, l as RunWarningCode, m as EvaluateArgs, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, u as resolveValidationSet, v as EvaluationSplit, x as TextModel, y as RolloutUsage } from "./reporting-bq007_2z.cjs";
 import { n as EvaluationCache, r as createMemoryCache, t as CachedScore } from "./cache-CuSo0NJ8.cjs";
 import { t as Rng } from "./rng-BR5MOedA.cjs";
-import { a as harvestFewShotExamples, i as formatDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-CU9dy2oT.cjs";
-import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-Bh5DPRMH.cjs";
+import { a as harvestFewShotExamples, i as formatDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-ByaLZy-Z.cjs";
+import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-4Zv-Zt2t.cjs";
 import { t as BatchSampler } from "./sampling-axOwfZf5.cjs";
 //#region src/checkpoint.d.ts
 /**
@@ -252,7 +252,7 @@ interface TrainingExample {
  * candidate to leave in the input. Dropping it entirely moves the whole prompt
  * into weights and gives up the ability to steer the model with text
  * afterwards; keeping a short task statement distills away only the tokens the
- * search accreted. See `docs/distillation.md`.
+ * search accreted. See `docs/distillation.md` in this package.
  */
 type TrainingExampleRenderer<Datum, Output> = (args: {
   rollout: Rollout<Datum, Output>;
@@ -279,6 +279,38 @@ declare function toTrainingJsonl<Datum, Output>(args: {
 interface JudgeCriterion {
   name: string;
   description: string;
+  /**
+   * Share of the instance score this criterion carries, relative to the other
+   * criteria. Default 1, which is the unweighted mean.
+   *
+   * 0 removes a criterion from the aggregate, not from the search. It is still
+   * graded and still recorded in `objectiveScores`, which
+   * `paretoSelector({ frontier: "objective" })` and `"hybrid"` build their
+   * selection fronts from — so a candidate leading a zero-weight criterion
+   * still earns parent selection under those. Under the default instance
+   * frontier, 0 is enough. For a number that can never steer the search,
+   * compute it outside the judge.
+   */
+  weight?: number;
+  /**
+   * Grade, on the judge's own scale, that this criterion must reach for the
+   * instance to score at all. Below it the instance scores 0 whatever the
+   * other criteria said.
+   *
+   * A mean lets a search trade a hard requirement away: a candidate that tanks
+   * one non-negotiable criterion and aces three cosmetic ones outranks the
+   * incumbent that kept the rule. Anything a caller would not ship without is
+   * a gate rather than a term in the average.
+   *
+   * Enforce it once. A gate already makes the requirement non-negotiable, so a
+   * heavy `weight` on the same criterion redistributes score only among
+   * candidates that all cleared it — and pins that share of the aggregate near
+   * its ceiling, narrowing the range the search has left to move in. Gate it,
+   * then weight it low.
+   *
+   * @see docs/metric-preflight.md
+   */
+  gate?: number;
 }
 type JudgePromptBuilder = (args: {
   input: string;
@@ -356,4 +388,4 @@ declare function priceUsage(args: {
   pricing?: TokenPricing;
 }): RolloutUsage;
 //#endregion
-export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type CandidateAccepted, type ChatMessage, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type HarvestResult, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerEvent, type OptimizerResult, type OptimizerTask, type ReportableEvent, type Reporter, type RetryPolicy, type Rng, type Rollout, type RolloutUsage, type RunFinished, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type TrainingExample, type TrainingExampleRenderer, type UsageTotals, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint, toTrainingJsonl };
+export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type CandidateAccepted, type ChatMessage, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type HarvestResult, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerEvent, type OptimizerResult, type OptimizerTask, type ReportableEvent, type Reporter, type RetryPolicy, type Rng, type Rollout, type RolloutUsage, type RunFinished, type RunWarning, type RunWarningCode, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type TrainingExample, type TrainingExampleRenderer, type UsageTotals, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, resolveValidationSet, runFingerprint, seedScoreWarnings, toTrainingJsonl };

package/dist/index.d.mts

@@ -1,8 +1,8 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, d as EvaluationBatch, f as EvaluationContext, g as ScoreResult, h as RolloutUsage, i as Reporter, l as Candidate, m as EvaluationSplit, n as OptimizerEvent, o as isCandidateAccepted, p as EvaluationPhase, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, u as EvaluateArgs, v as UsageTotals, y as componentNames } from "./reporting-DQbAohc9.mjs";
+import { C as componentNames, S as UsageTotals, _ as EvaluationPhase, a as RunFinished, b as ScoreResult, c as RunWarning, d as seedScoreWarnings, f as Adapter, g as EvaluationContext, h as EvaluationBatch, i as Reporter, l as RunWarningCode, m as EvaluateArgs, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, u as resolveValidationSet, v as EvaluationSplit, x as TextModel, y as RolloutUsage } from "./reporting-bq007_2z.mjs";
 import { n as EvaluationCache, r as createMemoryCache, t as CachedScore } from "./cache-CuSo0NJ8.mjs";
 import { t as Rng } from "./rng-BR5MOedA.mjs";
-import { a as harvestFewShotExamples, i as formatDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-1b7JiUue.mjs";
-import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-Ck6-e_8o.mjs";
+import { a as harvestFewShotExamples, i as formatDemos, n as Demo, o as parseDemos, r as DemoRenderer, t as BootstrapResult } from "./demos-ASsSXYXA.mjs";
+import { a as EvaluateBatchArgs, c as Evaluator, d as costExhausted, f as createEvaluator, g as createBudget, h as Budget, i as BudgetExhausted, l as RetryPolicy, m as requireMeasuredMean, n as OptimizerResult, o as EvaluateTracedArgs, p as measuredMean, r as OptimizerTask, s as EvaluationEvent, t as Optimizer, u as ScoredBatch } from "./optimizer-Ds5mzYjz.mjs";
 import { t as BatchSampler } from "./sampling-DFo_7RNJ.mjs";
 //#region src/checkpoint.d.ts
 /**
@@ -252,7 +252,7 @@ interface TrainingExample {
  * candidate to leave in the input. Dropping it entirely moves the whole prompt
  * into weights and gives up the ability to steer the model with text
  * afterwards; keeping a short task statement distills away only the tokens the
- * search accreted. See `docs/distillation.md`.
+ * search accreted. See `docs/distillation.md` in this package.
  */
 type TrainingExampleRenderer<Datum, Output> = (args: {
   rollout: Rollout<Datum, Output>;
@@ -279,6 +279,38 @@ declare function toTrainingJsonl<Datum, Output>(args: {
 interface JudgeCriterion {
   name: string;
   description: string;
+  /**
+   * Share of the instance score this criterion carries, relative to the other
+   * criteria. Default 1, which is the unweighted mean.
+   *
+   * 0 removes a criterion from the aggregate, not from the search. It is still
+   * graded and still recorded in `objectiveScores`, which
+   * `paretoSelector({ frontier: "objective" })` and `"hybrid"` build their
+   * selection fronts from — so a candidate leading a zero-weight criterion
+   * still earns parent selection under those. Under the default instance
+   * frontier, 0 is enough. For a number that can never steer the search,
+   * compute it outside the judge.
+   */
+  weight?: number;
+  /**
+   * Grade, on the judge's own scale, that this criterion must reach for the
+   * instance to score at all. Below it the instance scores 0 whatever the
+   * other criteria said.
+   *
+   * A mean lets a search trade a hard requirement away: a candidate that tanks
+   * one non-negotiable criterion and aces three cosmetic ones outranks the
+   * incumbent that kept the rule. Anything a caller would not ship without is
+   * a gate rather than a term in the average.
+   *
+   * Enforce it once. A gate already makes the requirement non-negotiable, so a
+   * heavy `weight` on the same criterion redistributes score only among
+   * candidates that all cleared it — and pins that share of the aggregate near
+   * its ceiling, narrowing the range the search has left to move in. Gate it,
+   * then weight it low.
+   *
+   * @see docs/metric-preflight.md
+   */
+  gate?: number;
 }
 type JudgePromptBuilder = (args: {
   input: string;
@@ -356,4 +388,4 @@ declare function priceUsage(args: {
   pricing?: TokenPricing;
 }): RolloutUsage;
 //#endregion
-export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type CandidateAccepted, type ChatMessage, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type HarvestResult, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerEvent, type OptimizerResult, type OptimizerTask, type ReportableEvent, type Reporter, type RetryPolicy, type Rng, type Rollout, type RolloutUsage, type RunFinished, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type TrainingExample, type TrainingExampleRenderer, type UsageTotals, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint, toTrainingJsonl };
+export { type Adapter, type BatchSampler, type BootstrapResult, type Budget, BudgetExhausted, type CachedScore, type Candidate, type CandidateAccepted, type ChatMessage, type Comparison, type ComparisonRun, type ComparisonSummary, type Deadline, type Demo, type DemoRenderer, type EvaluateArgs, type EvaluateBatchArgs, type EvaluateTracedArgs, type EvaluationBatch, type EvaluationCache, type EvaluationContext, type EvaluationEvent, type EvaluationPhase, type EvaluationSplit, type Evaluator, type HarvestResult, type Judge, type JudgeCriterion, type JudgePromptBuilder, type Optimizer, type OptimizerEvent, type OptimizerResult, type OptimizerTask, type ReportableEvent, type Reporter, type RetryPolicy, type Rng, type Rollout, type RolloutUsage, type RunFinished, type RunWarning, type RunWarningCode, type ScoreResult, type ScoredBatch, type TextModel, type TokenPricing, type TrainingExample, type TrainingExampleRenderer, type UsageTotals, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, resolveValidationSet, runFingerprint, seedScoreWarnings, toTrainingJsonl };

package/dist/index.mjs

@@ -1,7 +1,7 @@
-import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, o as BudgetExhausted, p as candidateFingerprint, s as costExhausted, u as requireMeasuredMean } from "./reporting-DY-DC4HG.mjs";
+import { S as createBudget, _ as componentNames, c as BudgetExhausted, d as measuredMean, f as requireMeasuredMean, g as runFingerprint, h as candidateFingerprint, l as costExhausted, m as assertResumable, n as seedScoreWarnings, o as isCandidateAccepted, p as createDeadline, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, y as createMemoryCache } from "./warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "./concurrency-D58PWeSk.mjs";
 import { i as signFlipPValue, n as holmAdjust } from "./math-Dqme4rYz.mjs";
-import { a as harvestRollouts, n as harvestFewShotExamples, r as parseDemos, t as formatDemos } from "./demos-DE2oxNWX.mjs";
+import { a as harvestRollouts, n as harvestFewShotExamples, r as parseDemos, t as formatDemos } from "./demos-Brobjfuc.mjs";
 import { t as parseProposedText } from "./text--v4Ffbus.mjs";
 //#region src/compare.ts
 const EXACT_LIMIT = 20;
@@ -238,6 +238,10 @@ function createJudge(args) {
 	const { model, criteria, scale = DEFAULT_SCALE, renderInput = stringify, renderOutput = stringify, buildPrompt = buildJudgePrompt } = args;
 	if (criteria.length === 0) throw new Error("createJudge requires at least one criterion");
 	if (!Number.isFinite(scale) || scale <= 0) throw new Error(`scale must be a positive number, received ${scale}`);
+	assertCriteria({
+		criteria,
+		scale
+	});
 	return async ({ input, output, expected, signal }) => {
 		return readVerdict({
 			response: await model({
@@ -281,6 +285,7 @@ function buildJudgePrompt(args) {
 		`Grade each criterion from 0 to ${scale}, where ${scale} is a perfect answer.`,
 		"",
 		"Then write feedback. It is read by a program that rewrites the system's instructions, not by a person reviewing this output, so say what the instructions should tell the system to do differently. Feedback about this particular answer is of no use to it.",
+		...expected === void 0 ? [] : ["", "Do not restate the expected answer, or any fact drawn from it, in the feedback. The instruction it is rewritten into is reused on inputs whose answers you have not seen: a fact copied out of the expected answer becomes an answer key memorised in the prompt, which raises the score on this input and teaches the system nothing. Name the kind of thing the answer was missing, not the thing itself."],
 		"",
 		"Reply in exactly this format and nothing else:",
 		...criteria.map((criterion) => `<score name="${criterion.name}">…</score>`),
@@ -299,6 +304,9 @@ function readVerdict(args) {
 	const graded = /* @__PURE__ */ new Map();
 	for (const match of response.matchAll(SCORE)) graded.set(match[1], Number(match[2]));
 	const objectiveScores = {};
+	let gated = false;
+	let weighted = 0;
+	let totalWeight = 0;
 	for (const criterion of criteria) {
 		const grade = graded.get(criterion.name);
 		if (grade === void 0 || Number.isNaN(grade)) return {
@@ -306,15 +314,30 @@ function readVerdict(args) {
 			feedback: `Judge did not grade "${criterion.name}".`,
 			transient: true
 		};
-		objectiveScores[criterion.name] = clamp(grade / scale);
+		const normalized = clamp(grade / scale);
+		objectiveScores[criterion.name] = normalized;
+		if (criterion.gate !== void 0 && grade < criterion.gate) gated = true;
+		const weight = criterion.weight ?? 1;
+		weighted += normalized * weight;
+		totalWeight += weight;
 	}
-	const grades = Object.values(objectiveScores);
 	return {
-		score: grades.reduce((total, grade) => total + grade, 0) / grades.length,
+		score: gated ? 0 : weighted / totalWeight,
 		feedback,
 		objectiveScores
 	};
 }
+function assertCriteria(args) {
+	const { criteria, scale } = args;
+	let totalWeight = 0;
+	for (const { name, weight = 1, gate } of criteria) {
+		if (!Number.isFinite(weight) || weight < 0) throw new Error(`weight on criterion "${name}" must be a non-negative number, received ${weight}`);
+		totalWeight += weight;
+		if (gate !== void 0 && (!Number.isFinite(gate) || gate <= 0)) throw new Error(`gate on criterion "${name}" must be greater than 0, received ${gate}; no grade can fall below 0`);
+		if (gate !== void 0 && gate > scale) throw new Error(`gate on criterion "${name}" is ${gate}, above the scale of ${scale}; no grade can reach it`);
+	}
+	if (totalWeight <= 0) throw new Error("criteria weights must sum to more than 0; at least one criterion has to count towards the score");
+}
 function stringify(value) {
 	return typeof value === "string" ? value : JSON.stringify(value);
 }
@@ -340,4 +363,4 @@ function priceUsage(args) {
 	};
 }
 //#endregion
-export { BudgetExhausted, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, runFingerprint, toTrainingJsonl };
+export { BudgetExhausted, assertResumable, buildJudgePrompt, candidateFingerprint, compare, componentNames, costExhausted, createBudget, createDeadline, createEvaluator, createJudge, createMemoryCache, formatDemos, harvestFewShotExamples, harvestRollouts, isCandidateAccepted, isRunFinished, mapWithConcurrency, measuredMean, parseDemos, parseProposedText, priceUsage, requireMeasuredMean, resolveValidationSet, runFingerprint, seedScoreWarnings, toTrainingJsonl };

package/dist/mipro/index.cjs

@@ -1,8 +1,8 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_reporting = require("../reporting-CNHzbJC-.cjs");
+const require_warnings = require("../warnings-CWRJF-jA.cjs");
 const require_concurrency = require("../concurrency-C-cFzWW2.cjs");
 const require_math = require("../math-BhlziRPc.cjs");
-const require_demos = require("../demos-D2o0qSSo.cjs");
+const require_demos = require("../demos-9v5ts7F3.cjs");
 const require_text = require("../text-CK_HB3su.cjs");
 const require_rng = require("../rng-DbA_rPIo.cjs");
 const require_sampling = require("../sampling-Dars7ctR.cjs");
@@ -218,7 +218,7 @@ var MiproOptimizer = class {
 				task
 			});
 		} finally {
-			await require_reporting.flushReporters(task.reporters ?? []);
+			await require_warnings.flushReporters(task.reporters ?? []);
 		}
 	}
 };
@@ -278,10 +278,14 @@ function buildDatasetSummaryPrompt(examples) {
 async function runMipro(args) {
 	const { config, task } = args;
 	const { instructionsPerComponent = DEFAULT_INSTRUCTIONS, minibatchSize = DEFAULT_MINIBATCH_SIZE, maxTrials = DEFAULT_MAX_TRIALS, startupTrials, gamma, surrogateSamples, multivariate, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, demoSets = DEFAULT_DEMO_SETS, maxDemos = DEFAULT_MAX_DEMOS, demoMinScore, exemplars = DEFAULT_EXEMPLARS, datasetSummary = true, summaryExamples = DEFAULT_SUMMARY_EXAMPLES, concurrency = 1, seed = 0, buildPrompt = buildMiproPrompt, tips = DEFAULT_TIPS, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_reporting.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
-	const emit = require_reporting.createEmitter(reporters);
-	const deadline = require_reporting.createDeadline({ maxWallClockMs });
-	const components = require_reporting.componentNames(seedCandidate);
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = require_sampling.createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = require_warnings.defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = require_warnings.resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
+	const emit = require_warnings.createEmitter(reporters);
+	const deadline = require_warnings.createDeadline({ maxWallClockMs });
+	const components = require_warnings.componentNames(seedCandidate);
 	if (trainingSet.length === 0) throw new Error("optimize requires a non-empty trainingSet");
 	if (validationSet.length === 0) throw new Error("optimize requires a non-empty validationSet");
 	if (components.length === 0) throw new Error("optimize requires a seed candidate with at least one component");
@@ -299,24 +303,24 @@ async function runMipro(args) {
 		datum,
 		index
 	})) ?? [];
-	const fingerprint = require_reporting.runFingerprint({
+	const fingerprint = require_warnings.runFingerprint({
 		seedCandidate,
 		trainingIds,
 		validationIds,
 		seed,
 		...cacheNamespace === void 0 ? {} : { cacheNamespace }
 	});
-	require_reporting.assertResumable({
+	require_warnings.assertResumable({
 		fingerprint,
 		...resumeFrom === void 0 ? {} : { snapshot: resumeFrom }
 	});
 	const rng = require_rng.createSeededRng(seed, resumeFrom?.rngState);
-	const budget = require_reporting.createBudget({
+	const budget = require_warnings.createBudget({
 		maxMetricCalls,
 		spent: resumeFrom?.metricCalls ?? 0
 	});
-	const evaluationCache = cache === false ? void 0 : cache ?? require_reporting.createMemoryCache();
-	const evaluator = require_reporting.createEvaluator({
+	const evaluationCache = cache === false ? void 0 : cache ?? require_warnings.createMemoryCache();
+	const evaluator = require_warnings.createEvaluator({
 		adapter,
 		budget,
 		...retry === void 0 ? {} : { retry },
@@ -380,7 +384,7 @@ async function runMipro(args) {
 		for (let index = 0; index < demoSets; index += 1) {
 			const affordable = Math.min(trainingSet.length, budget.remaining() - validationSet.length);
 			if (affordable < 1) break;
-			if (require_reporting.costExhausted({
+			if (require_warnings.costExhausted({
 				usage: evaluator.usage(),
 				maxCostUsd
 			})) break;
@@ -474,7 +478,7 @@ async function runMipro(args) {
 		});
 	}
 	const seedEvaluation = resumeFrom === void 0 ? await fullSweep(seedCandidate, "seed") : void 0;
-	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_reporting.requireMeasuredMean({
+	const seedScore = seedEvaluation === void 0 ? resumeFrom.seedScore : require_warnings.requireMeasuredMean({
 		batch: seedEvaluation,
 		phase: "seed"
 	});
@@ -485,7 +489,7 @@ async function runMipro(args) {
 		candidateId: 0,
 		candidate: seedCandidate,
 		aggregateScore: seedScore,
-		instanceScores: require_reporting.instanceRow(seedEvaluation),
+		instanceScores: require_warnings.instanceRow(seedEvaluation),
 		...trackBestOutputs ? { outputs: seedEvaluation.outputs } : {}
 	});
 	if (resumeFrom === void 0) surrogateInput.push({
@@ -559,12 +563,12 @@ async function runMipro(args) {
 		try {
 			evaluation = await fullSweep(candidate, "validation");
 		} catch (err) {
-			if (err instanceof require_reporting.BudgetExhausted) return "budgetExhausted";
+			if (err instanceof require_warnings.BudgetExhausted) return "budgetExhausted";
 			if (signal?.aborted) return "aborted";
 			throw err;
 		}
 		fullEvaluations += 1;
-		const score = require_reporting.measuredMean(evaluation);
+		const score = require_warnings.measuredMean(evaluation);
 		if (score === void 0) return "swept";
 		surrogateInput.push({
 			choices,
@@ -585,7 +589,7 @@ async function runMipro(args) {
 				candidateId: acceptedCandidates,
 				candidate,
 				aggregateScore: score,
-				instanceScores: require_reporting.instanceRow(evaluation),
+				instanceScores: require_warnings.instanceRow(evaluation),
 				...trackBestOutputs ? { outputs: evaluation.outputs } : {}
 			});
 		}
@@ -596,7 +600,7 @@ async function runMipro(args) {
 			stopReason = "aborted";
 			break;
 		}
-		if (require_reporting.costExhausted({
+		if (require_warnings.costExhausted({
 			usage: evaluator.usage(),
 			maxCostUsd
 		})) {
@@ -641,9 +645,9 @@ async function runMipro(args) {
 				candidateId: null,
 				iteration: trial
 			});
-			minibatchScore = require_reporting.measuredMean(evaluation);
+			minibatchScore = require_warnings.measuredMean(evaluation);
 		} catch (err) {
-			if (err instanceof require_reporting.BudgetExhausted) {
+			if (err instanceof require_warnings.BudgetExhausted) {
 				stopReason = "budgetExhausted";
 				break;
 			}
@@ -700,15 +704,16 @@ async function runMipro(args) {
 		iteration: trial,
 		charge: false
 	});
-	const testScore = heldOut === void 0 ? void 0 : require_reporting.measuredMean(heldOut);
+	const testScore = heldOut === void 0 ? void 0 : require_warnings.measuredMean(heldOut);
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
 		...testScore === void 0 ? {} : { testScore },
-		...heldOut === void 0 ? {} : { testInstanceScores: require_reporting.instanceRow(heldOut) },
+		...heldOut === void 0 ? {} : { testInstanceScores: require_warnings.instanceRow(heldOut) },
 		...heldOut === void 0 || !trackBestOutputs ? {} : { testOutputs: heldOut.outputs }
 	});
 	return {
@@ -731,6 +736,7 @@ async function runMipro(args) {
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }
@@ -762,6 +768,6 @@ function assertConfig(config) {
 //#endregion
 exports.MiproOptimizer = MiproOptimizer;
 exports.buildMiproPrompt = buildMiproPrompt;
-exports.isCandidateAccepted = require_reporting.isCandidateAccepted;
-exports.isRunFinished = require_reporting.isRunFinished;
+exports.isCandidateAccepted = require_warnings.isCandidateAccepted;
+exports.isRunFinished = require_warnings.isRunFinished;
 exports.proposeConfiguration = proposeConfiguration;

package/dist/mipro/index.d.cts

@@ -1,8 +1,8 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.cjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.cjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.cjs";
 import { t as Rng } from "../rng-BR5MOedA.cjs";
-import { r as DemoRenderer } from "../demos-CU9dy2oT.cjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Bh5DPRMH.cjs";
+import { r as DemoRenderer } from "../demos-ByaLZy-Z.cjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-4Zv-Zt2t.cjs";
 import { t as BatchSampler } from "../sampling-axOwfZf5.cjs";
 //#region src/mipro/optimize.d.ts
 type MiproPromptBuilder = (args: {

package/dist/mipro/index.d.mts

@@ -1,8 +1,8 @@
-import { _ as TextModel, a as RunFinished, c as Adapter, i as Reporter, l as Candidate, n as OptimizerEvent, o as isCandidateAccepted, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, v as UsageTotals } from "../reporting-DQbAohc9.mjs";
+import { S as UsageTotals, a as RunFinished, f as Adapter, i as Reporter, n as OptimizerEvent, o as isCandidateAccepted, p as Candidate, r as ReportableEvent, s as isRunFinished, t as CandidateAccepted, x as TextModel } from "../reporting-bq007_2z.mjs";
 import { n as EvaluationCache, t as CachedScore } from "../cache-CuSo0NJ8.mjs";
 import { t as Rng } from "../rng-BR5MOedA.mjs";
-import { r as DemoRenderer } from "../demos-1b7JiUue.mjs";
-import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ck6-e_8o.mjs";
+import { r as DemoRenderer } from "../demos-ASsSXYXA.mjs";
+import { n as OptimizerResult, r as OptimizerTask, s as EvaluationEvent, t as Optimizer } from "../optimizer-Ds5mzYjz.mjs";
 import { t as BatchSampler } from "../sampling-DFo_7RNJ.mjs";
 //#region src/mipro/optimize.d.ts
 type MiproPromptBuilder = (args: {

package/dist/mipro/index.mjs

@@ -1,7 +1,7 @@
-import { _ as createMemoryCache, a as isRunFinished, b as createBudget, c as createEvaluator, d as createDeadline, f as assertResumable, h as componentNames, i as isCandidateAccepted, l as measuredMean, m as runFingerprint, n as flushReporters, o as BudgetExhausted, r as instanceRow, s as costExhausted, t as createEmitter, u as requireMeasuredMean, v as defaultInstanceId } from "../reporting-DY-DC4HG.mjs";
+import { S as createBudget, _ as componentNames, a as instanceRow, b as defaultInstanceId, c as BudgetExhausted, d as measuredMean, f as requireMeasuredMean, g as runFingerprint, i as flushReporters, l as costExhausted, m as assertResumable, o as isCandidateAccepted, p as createDeadline, r as createEmitter, s as isRunFinished, t as resolveValidationSet, u as createEvaluator, y as createMemoryCache } from "../warnings-OxvDi9kN.mjs";
 import { t as mapWithConcurrency } from "../concurrency-D58PWeSk.mjs";
 import { r as mean } from "../math-Dqme4rYz.mjs";
-import { n as harvestFewShotExamples, t as formatDemos } from "../demos-DE2oxNWX.mjs";
+import { n as harvestFewShotExamples, t as formatDemos } from "../demos-Brobjfuc.mjs";
 import { t as parseProposedText } from "../text--v4Ffbus.mjs";
 import { t as createSeededRng } from "../rng-Dtc5eZ_W.mjs";
 import { t as createEpochShuffledSampler } from "../sampling-CfHt7Gue.mjs";
@@ -277,7 +277,11 @@ function buildDatasetSummaryPrompt(examples) {
 async function runMipro(args) {
 	const { config, task } = args;
 	const { instructionsPerComponent = DEFAULT_INSTRUCTIONS, minibatchSize = DEFAULT_MINIBATCH_SIZE, maxTrials = DEFAULT_MAX_TRIALS, startupTrials, gamma, surrogateSamples, multivariate, fullEvalInterval = DEFAULT_FULL_EVAL_INTERVAL, demoSets = DEFAULT_DEMO_SETS, maxDemos = DEFAULT_MAX_DEMOS, demoMinScore, exemplars = DEFAULT_EXEMPLARS, datasetSummary = true, summaryExamples = DEFAULT_SUMMARY_EXAMPLES, concurrency = 1, seed = 0, buildPrompt = buildMiproPrompt, tips = DEFAULT_TIPS, trackBestOutputs = false, checkpointCache = true } = config;
-	const { seedCandidate, trainingSet, validationSet = trainingSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { seedCandidate, trainingSet, validationSet: requestedValidationSet, testSet, adapter, reflect, componentOptions, demoComponents, renderDemo, goldOutput, maxMetricCalls, renderDatum = renderDefault, batchSampler = createEpochShuffledSampler({ minibatchSize }), cache, cacheNamespace, retry, maxCostUsd, maxWallClockMs, instanceId = defaultInstanceId, reporters = [], onCheckpoint, resumeFrom, signal } = task;
+	const { validationSet, warnings } = resolveValidationSet({
+		validationSet: requestedValidationSet,
+		trainingSet
+	});
 	const emit = createEmitter(reporters);
 	const deadline = createDeadline({ maxWallClockMs });
 	const components = componentNames(seedCandidate);
@@ -703,6 +707,7 @@ async function runMipro(args) {
 	emit({
 		type: "finish",
 		reason: stopReason,
+		warnings,
 		bestCandidateId: acceptedCandidates,
 		bestScore,
 		metricCalls: budget.spent(),
@@ -730,6 +735,7 @@ async function runMipro(args) {
 		metricCalls: budget.spent(),
 		reflectionCalls,
 		cacheHits: evaluator.cacheHits(),
+		warnings,
 		stopReason
 	};
 }