reeval 0.0.1

package/dist/matchers-DyLvm7EG.js

@@ -0,0 +1,374 @@
+import { t as UsageError } from "./errors-CSxfJ4qu.js";
+import { t as Scheduler } from "./core-tLN-z1Be.js";
+import { AsyncLocalStorage } from "node:async_hooks";
+import { ASYMMETRIC_MATCHERS_OBJECT, GLOBAL_EXPECT, JestAsymmetricMatchers, JestChaiExpect, JestExtend, addCustomEqualityTesters, chai, customMatchers, getState, setState } from "@vitest/expect";
+//#region src/engine/expect.ts
+chai.use(JestExtend);
+chai.use(JestChaiExpect);
+chai.use(JestAsymmetricMatchers);
+/** An `expect` of its own for one attempt (or, without a task, the worker's). */
+function createExpect(task) {
+	const expect = ((value, message) => {
+		const { assertionCalls } = getState(expect);
+		setState({ assertionCalls: assertionCalls + 1 }, expect);
+		const assertion = chai.expect(value, message);
+		return task ? assertion.withTest(task) : assertion;
+	});
+	Object.assign(expect, chai.expect);
+	Object.assign(expect, globalThis[ASYMMETRIC_MATCHERS_OBJECT]);
+	expect.getState = () => getState(expect);
+	expect.setState = (state) => setState(state, expect);
+	setState({
+		assertionCalls: 0,
+		isExpectingAssertions: false,
+		isExpectingAssertionsError: null,
+		expectedAssertionsNumber: null,
+		expectedAssertionsNumberErrorGen: null,
+		currentTestName: task?.fullTestName ?? ""
+	}, expect);
+	expect.assert = chai.assert;
+	expect.extend = (matchers) => chai.expect.extend(expect, matchers);
+	expect.addEqualityTesters = (testers) => addCustomEqualityTesters(testers);
+	expect.soft = (value, message) => expect(value, message).withContext({ soft: true });
+	expect.poll = createPoll(expect);
+	expect.unreachable = (message) => {
+		chai.assert.fail(`expected${message ? ` "${message}" ` : " "}not to be reached`);
+	};
+	function assertions(expected) {
+		const errorGen = () => /* @__PURE__ */ new Error(`expected number of assertions to be ${expected}, but got ${expect.getState().assertionCalls}`);
+		setState({
+			expectedAssertionsNumber: expected,
+			expectedAssertionsNumberErrorGen: errorGen
+		}, expect);
+	}
+	function hasAssertions() {
+		setState({
+			isExpectingAssertions: true,
+			isExpectingAssertionsError: /* @__PURE__ */ new Error("expected any number of assertion, but got none")
+		}, expect);
+	}
+	chai.util.addMethod(expect, "assertions", assertions);
+	chai.util.addMethod(expect, "hasAssertions", hasAssertions);
+	expect.extend(customMatchers);
+	return expect;
+}
+/**
+* `expect.poll(fn).matcher()`: calls `fn` and the matcher until the matcher passes, every
+* `interval` ms, for at most `timeout` ms, then fails with the last error.
+*/
+function createPoll(expect) {
+	return (fn, options = {}) => {
+		const { interval = 50, timeout = 1e3, message } = options;
+		const assertion = expect(null, message);
+		chai.util.flag(assertion, "poll", true);
+		const proxy = new Proxy(assertion, { get(target, key, receiver) {
+			const matcher = Reflect.get(target, key, receiver);
+			if (typeof matcher !== "function") return matcher instanceof chai.Assertion ? proxy : matcher;
+			return (...args) => {
+				const location = /* @__PURE__ */ new Error("expect.poll");
+				return (async () => {
+					const deadline = Date.now() + timeout;
+					const signal = AbortSignal.timeout(timeout);
+					for (;;) try {
+						chai.util.flag(assertion, "object", await fn({ signal }));
+						return await Reflect.apply(matcher, assertion, args);
+					} catch (error) {
+						if (Date.now() + interval >= deadline) {
+							if (error instanceof Error && location.stack) error.stack = location.stack.replace(location.message, error.message);
+							throw error;
+						}
+						await new Promise((resolve) => setTimeout(resolve, interval));
+					}
+				})();
+			};
+		} });
+		return proxy;
+	};
+}
+/** The failure `expect.assertions`, `expect.hasAssertions` or `expect.soft` left on an attempt. */
+function attemptExpectError(expect, task) {
+	const [soft] = task.result?.errors ?? [];
+	if (soft) return Object.assign(/* @__PURE__ */ new Error(), soft);
+	const state = expect.getState();
+	if (state.expectedAssertionsNumber !== null && state.expectedAssertionsNumber !== state.assertionCalls) return state.expectedAssertionsNumberErrorGen();
+	if (state.isExpectingAssertions && state.assertionCalls === 0) return state.isExpectingAssertionsError;
+}
+const globalExpect = createExpect();
+Object.defineProperty(globalThis, GLOBAL_EXPECT, {
+	value: globalExpect,
+	writable: true,
+	configurable: true
+});
+const currentExpect = new AsyncLocalStorage();
+const currentPending = new AsyncLocalStorage();
+const isThenable = (value) => typeof value === "object" && value !== null && typeof value.then === "function";
+/**
+* Wraps an assertion chain so each promise one of its matchers returns is tracked: an eval that
+* doesn't await it would otherwise pass and tear its sandbox down while the check still runs.
+*/
+function tracked(assertion, pending, path) {
+	return new Proxy(assertion, { get(target, property) {
+		const value = Reflect.get(target, property);
+		const name = `${path}.${String(property)}`;
+		if (typeof value === "function") return (...args) => {
+			const result = Reflect.apply(value, target, args);
+			if (!isThenable(result)) return result;
+			const entry = {
+				assertion: `${name}()`,
+				settled: Promise.resolve(result).then(() => void 0, (error) => error ?? /* @__PURE__ */ new Error(`${name}() failed`)),
+				observed: false
+			};
+			pending.push(entry);
+			const observe = (fn) => {
+				entry.observed = true;
+				return fn(Promise.resolve(result));
+			};
+			return {
+				then: (onFulfilled, onRejected) => observe((p) => p.then(onFulfilled, onRejected)),
+				catch: (onRejected) => observe((p) => p.catch(onRejected)),
+				finally: (onFinally) => observe((p) => p.finally(onFinally)),
+				[Symbol.toStringTag]: "Promise"
+			};
+		};
+		return typeof value === "object" && value !== null && !isThenable(value) ? tracked(value, pending, name) : value;
+	} });
+}
+const track = (assertion, path) => {
+	const pending = currentPending.getStore();
+	return pending && typeof assertion === "object" && assertion !== null ? tracked(assertion, pending, path) : assertion;
+};
+/** Assertions that return a chain, so the chain's async matchers are tracked too. */
+const CHAINS = {
+	soft: "expect.soft(…)",
+	poll: "expect.poll(…)"
+};
+const expect = new Proxy(globalExpect, {
+	apply: (target, thisArg, args) => track(Reflect.apply(currentExpect.getStore() ?? target, thisArg, args), "expect(…)"),
+	get: (target, property) => {
+		const bound = currentExpect.getStore() ?? target;
+		const value = Reflect.get(bound, property);
+		if (typeof value !== "function" || property === "assert") return value;
+		const chain = CHAINS[property];
+		return (...args) => {
+			const result = Reflect.apply(value, bound, args);
+			return chain ? track(result, chain) : result;
+		};
+	}
+});
+/** Runs an attempt's body with its own `expect` and unawaited-assertion list. */
+function withAttemptExpect(own, pending, body) {
+	return currentExpect.run(own, () => currentPending.run(pending, body));
+}
+/**
+* Waits for the async assertions the eval started but didn't await, and returns the error that
+* fails the attempt: the first one's failure, or that it wasn't awaited.
+*/
+async function unawaitedError(pending) {
+	const missed = pending.filter((p) => !p.observed);
+	if (missed.length === 0) return void 0;
+	for (const p of missed) p.observed = true;
+	const outcomes = await Promise.all(missed.map((p) => p.settled));
+	const i = Math.max(0, outcomes.findIndex((o) => o !== void 0));
+	const hint = `${missed[i]?.assertion} wasn't awaited. Its check is async: write \`await ${missed[i]?.assertion}\`.`;
+	const failure = outcomes[i];
+	if (failure instanceof Error) {
+		failure.message = `${failure.message}\n\n${hint}`;
+		return failure;
+	}
+	const error = new Error(hint);
+	error.name = "AssertionError";
+	return error;
+}
+function extendExpect(matchers) {
+	globalExpect.extend(matchers);
+}
+//#endregion
+//#region src/engine/runtime.ts
+let engine;
+/** Installs the engine this worker runs eval files with. */
+function useEngine(next) {
+	engine = next;
+}
+function active() {
+	if (!engine) throw new UsageError("evaluate() and group() only work in an eval file that reeval run loads. Run it with reeval run.");
+	return engine;
+}
+const registerTest = (name, options, body) => active().registerTest(name, options, body);
+const registerGroup = (name, mode, body) => active().registerGroup(name, mode, body);
+const getRunContext = () => active().getRunContext();
+const collectingFile = () => active().collectingFile();
+//#endregion
+//#region src/scheduler/client.ts
+var LocalTransport = class {
+	#pending = /* @__PURE__ */ new Map();
+	#scheduler = new Scheduler((r) => {
+		this.#pending.get(r.id)?.resolve();
+		this.#pending.delete(r.id);
+	});
+	request(id, limits, rank, pending) {
+		this.#pending.set(id, pending);
+		this.#scheduler.request({
+			id,
+			owner: "local",
+			limits,
+			...rank ? { rank } : {}
+		});
+	}
+	release(id) {
+		this.#pending.delete(id);
+		this.#scheduler.release(id);
+	}
+};
+/** Over the worker's IPC channel to reeval's own engine: requests out, grants back. */
+var ChannelTransport = class {
+	#pending = /* @__PURE__ */ new Map();
+	#send;
+	constructor(send) {
+		this.#send = send;
+	}
+	grant(id) {
+		const pending = this.#pending.get(id);
+		this.#pending.delete(id);
+		pending?.resolve();
+	}
+	request(id, limits, rank, pending) {
+		this.#pending.set(id, pending);
+		this.#send({
+			t: "request",
+			id,
+			limits,
+			...rank ? { rank } : {}
+		});
+	}
+	release(id) {
+		this.#pending.delete(id);
+		this.#send({
+			t: "release",
+			id
+		});
+	}
+};
+let transport;
+let ids = 0;
+/** Points this process at the run's scheduler over `send`; returns what delivers its grants. */
+function useSchedulerChannel(send) {
+	const channel = new ChannelTransport(send);
+	transport = channel;
+	return (id) => channel.grant(id);
+}
+/**
+* Waits until every resource in `limits` has room, then holds a lease on each until released.
+* Rejects with the signal's reason if it aborts first.
+*/
+function acquire(limits, options = {}) {
+	const { rank, signal } = options;
+	if (signal?.aborted) return Promise.reject(signal.reason);
+	transport ??= new LocalTransport();
+	const via = transport;
+	const id = String(++ids);
+	let released = false;
+	const lease = { release() {
+		if (released) return;
+		released = true;
+		via.release(id);
+	} };
+	return new Promise((resolve, reject) => {
+		const onAbort = () => {
+			lease.release();
+			reject(signal?.reason);
+		};
+		signal?.addEventListener("abort", onAbort, { once: true });
+		via.request(id, limits, rank, {
+			resolve: () => {
+				signal?.removeEventListener("abort", onAbort);
+				resolve(lease);
+			},
+			reject: (error) => {
+				signal?.removeEventListener("abort", onAbort);
+				reject(error);
+			}
+		});
+	});
+}
+/** Runs `fn` while holding leases on `limits`. */
+async function withLease(limits, fn, signal) {
+	const lease = await acquire(limits, signal ? { signal } : {});
+	try {
+		return await fn();
+	} finally {
+		lease.release();
+	}
+}
+//#endregion
+//#region src/matchers.ts
+/** A check the turn's harness can't answer either way, e.g. a web fetch it never reports. */
+var HarnessCapabilityError = class extends Error {
+	name = "HarnessCapabilityError";
+};
+const isTurn = (value) => typeof value === "object" && value !== null && Array.isArray(value.transcript) && Array.isArray(value.unreported);
+const matches = (value, pattern) => pattern === void 0 || (typeof pattern === "string" ? value.includes(pattern) : pattern.test(value));
+/** Passes when the turn has a `type` event whose `field` matches; refuses when it can't know. */
+function webEvent(matcher, actual, type, field, pattern) {
+	if (!isTurn(actual)) throw new TypeError(`${matcher}() expects a turn from t.send()`);
+	const what = type === "web-search" ? "web searches" : "web fetches";
+	if (actual.unreported.includes(type)) throw new HarnessCapabilityError(`${matcher}() can't check this turn: its harness doesn't report ${what} ('${type}' is in turn.unreported), so the agent may have made them with no event. Check the reply or files instead, or use a harness that reports them.`);
+	const seen = actual.transcript.filter((e) => e.type === type).map(field);
+	const pass = seen.some((v) => matches(v, pattern));
+	const target = pattern === void 0 ? "" : ` matching ${pattern}`;
+	return {
+		pass,
+		message: () => pass ? `expected no ${what}${target}, got ${JSON.stringify(seen)}` : `expected ${what}${target}, got ${seen.length ? JSON.stringify(seen) : "none"}`
+	};
+}
+/** The last `lines` lines, and at most `chars` characters of them: one line can be megabytes. */
+const tail = (s, lines = 20, chars = 2e3) => {
+	const last = s.trimEnd().split("\n").slice(-lines).join("\n");
+	return last.length <= chars ? last : `[… ${last.length - chars} earlier characters]${last.slice(-chars)}`;
+};
+const hasFiles = (value) => typeof value === "object" && value !== null && typeof value.readFile === "function";
+const matchers = {
+	async toHaveFile(actual, path, pattern) {
+		if (!hasFiles(actual)) throw new TypeError("toHaveFile() expects a turn from t.send(), or t");
+		const content = await actual.readFile(path);
+		if (content === null) return {
+			pass: false,
+			message: () => `expected file ${path} to exist`
+		};
+		if (pattern === void 0) return {
+			pass: true,
+			message: () => `expected no file at ${path}`
+		};
+		const pass = typeof pattern === "string" ? content.includes(pattern) : pattern.test(content);
+		return {
+			pass,
+			message: () => `expected ${path} ${pass ? "not " : ""}to match ${pattern}\n--- ${path} ---\n${tail(content)}`
+		};
+	},
+	async toSucceed(actual) {
+		const r = await actual;
+		if (typeof r?.exitCode !== "number") throw new TypeError("toSucceed() expects run.exec(...)");
+		return {
+			pass: r.exitCode === 0,
+			message: () => r.exitCode === 0 ? "expected command to fail" : `expected command to succeed, exit code ${r.exitCode}\n--- stderr ---\n${tail(r.stderr)}\n--- stdout ---\n${tail(r.stdout)}`
+		};
+	},
+	toHaveSearched(actual, query) {
+		return webEvent("toHaveSearched", actual, "web-search", (e) => "query" in e ? e.query : "", query);
+	},
+	toHaveFetched(actual, url) {
+		return webEvent("toHaveFetched", actual, "web-fetch", (e) => "url" in e ? e.url : "", url);
+	},
+	toPass(actual) {
+		const v = actual;
+		if (typeof v?.score !== "number") throw new TypeError("toPass() expects a t.judge() verdict");
+		const score = v.score.toFixed(2);
+		return {
+			pass: v.verdict === "pass",
+			message: () => v.verdict === "pass" ? `expected judge to fail: score ${score} ≥ ${v.threshold} for "${v.question}"` : `judge score ${score} is below ${v.threshold} for "${v.question}" (${v.model})`
+		};
+	}
+};
+//#endregion
+export { withLease as a, registerGroup as c, attemptExpectError as d, createExpect as f, withAttemptExpect as g, unawaitedError as h, useSchedulerChannel as i, registerTest as l, extendExpect as m, matchers as n, collectingFile as o, expect as p, acquire as r, getRunContext as s, HarnessCapabilityError as t, useEngine as u };
+
+//# sourceMappingURL=matchers-DyLvm7EG.js.map

package/dist/matchers-DyLvm7EG.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers-DyLvm7EG.js","names":[],"sources":["../src/engine/expect.ts","../src/engine/runtime.ts","../src/scheduler/client.ts","../src/matchers.ts"],"mappings":";;;;;AAkDA,KAAK,IAAI,UAAwB;AACjC,KAAK,IAAI,cAA4B;AACrC,KAAK,IAAI,sBAAoC;;AAkB7C,SAAgB,aAAa,MAAiC;CAC5D,MAAM,WAAW,OAAgB,YAAqB;EACpD,MAAM,EAAE,mBAAmB,SAAS,MAAM;EAC1C,SAAS,EAAE,gBAAgB,iBAAiB,EAAE,GAAG,MAAM;EACvD,MAAM,YAAY,KAAK,OAAO,OAAO,OAAO;EAC5C,OAAO,OAAO,UAAU,SAAS,IAAI,IAAI;CAC3C;CACA,OAAO,OAAO,QAAQ,KAAK,MAAM;CACjC,OAAO,OAAO,QAAS,WAAsC,2BAA2B;CACxF,OAAO,iBAAiB,SAAuB,MAAM;CACrD,OAAO,YAAY,UAAU,SAAS,OAAO,MAAM;CACnD,SACE;EACE,gBAAgB;EAChB,uBAAuB;EACvB,4BAA4B;EAC5B,0BAA0B;EAC1B,kCAAkC;EAClC,iBAAiB,MAAM,gBAAgB;CACzC,GACA,MACF;CACA,OAAO,SAAS,KAAK;CACrB,OAAO,UAAU,aACd,KAAK,OAAwE,OAC5E,QACA,QACF;CACF,OAAO,sBAAsB,YAAY,yBAAyB,OAAO;CACzE,OAAO,QAAQ,OAAO,YACnB,OAAO,OAAO,OAAO,CAAC,CAAkD,YAAY,EACnF,MAAM,KACR,CAAC;CACH,OAAO,OAAO,WAAW,MAAM;CAC/B,OAAO,eAAe,YAAY;EAChC,KAAK,OAAO,KAAK,WAAW,UAAU,KAAK,QAAQ,MAAM,IAAI,kBAAkB;CACjF;CACA,SAAS,WAAW,UAAkB;EACpC,MAAM,iCACJ,IAAI,MACF,uCAAuC,SAAS,YAAY,OAAO,SAAS,CAAC,CAAC,gBAChF;EACF,SACE;GAAE,0BAA0B;GAAU,kCAAkC;EAAS,GACjF,MACF;CACF;CACA,SAAS,gBAAgB;EAEvB,SACE;GAAE,uBAAuB;GAAM,4CAA4B,IAF3C,MAAM,gDAEyC;EAAE,GACjE,MACF;CACF;CACA,KAAK,KAAK,UAAU,QAAQ,cAAc,UAAU;CACpD,KAAK,KAAK,UAAU,QAAQ,iBAAiB,aAAa;CAC1D,OAAO,OAAO,cAAc;CAC5B,OAAO;AACT;;;;;AAMA,SAAS,WAAW,QAA4C;CAC9D,QAAQ,IAAI,UAAU,CAAC,MAAM;EAC3B,MAAM,EAAE,WAAW,IAAI,UAAU,KAAM,YAAY;EACnD,MAAM,YAAY,OAAO,MAAM,OAAO;EACtC,KAAK,KAAK,KAAK,WAAW,QAAQ,IAAI;EACtC,MAAM,QAAgB,IAAI,MAAM,WAAW,EACzC,IAAI,QAAQ,KAAK,UAAU;GACzB,MAAM,UAAU,QAAQ,IAAI,QAAQ,KAAK,QAAQ;GACjD,IAAI,OAAO,YAAY,YACrB,OAAO,mBAAmB,KAAK,YAAY,QAAQ;GACrD,QAAQ,GAAG,SAAoB;IAC7B,MAAM,2BAAW,IAAI,MAAM,aAAa;IACxC,QAAQ,YAAY;KAClB,MAAM,WAAW,KAAK,IAAI,IAAI;KAC9B,MAAM,SAAS,YAAY,QAAQ,OAAO;KAC1C,SACE,IAAI;MACF,KAAK,KAAK,KAAK,WAAW,UAAU,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;MACxD,OAAO,MAAM,QAAQ,MAAM,SAAS,WAAW,IAAI;KACrD,SAAS,OAAO;MACd,IAAI,KAAK,IAAI,IAAI,YAAY,UAAU;OACrC,IAAI,iBAAiB,SAAS,SAAS,OACrC,MAAM,QAAQ,SAAS,MAAM,QAAQ,SAAS,SAAS,MAAM,OAAO;OACtE,MAAM;MACR;MACA,MAAM,IAAI,SAAS,YAAY,WAAW,SAAS,QAAQ,CAAC;KAC9D;IAEJ,EAAA,CAAG;GACL;EACF,EACF,CAAC;EACD,OAAO;CACT;AACF;;AAGA,SAAgB,mBAAmB,QAAsB,MAAqC;CAC5F,MAAM,CAAC,QAAQ,KAAK,QAAQ,UAAU,CAAC;CAEvC,IAAI,MAAM,OAAO,OAAO,uBAAO,IAAI,MAAM,GAAG,IAAI;CAChD,MAAM,QAAQ,OAAO,SAAS;CAC9B,IACE,MAAM,6BAA6B,QACnC,MAAM,6BAA6B,MAAM,gBAEzC,OAAQ,MAAM,iCAAiD;CACjE,IAAI,MAAM,yBAAyB,MAAM,mBAAmB,GAC1D,OAAO,MAAM;AAEjB;AAEA,MAAM,eAAe,aAAa;AAClC,OAAO,eAAe,YAAY,eAAe;CAC/C,OAAO;CACP,UAAU;CACV,cAAc;AAChB,CAAC;AAED,MAAM,gBAAgB,IAAI,kBAAgC;AAI1D,MAAM,iBAAiB,IAAI,kBAA6B;AAExD,MAAM,cAAc,UAClB,OAAO,UAAU,YACjB,UAAU,QACV,OAAQ,MAA2B,SAAS;;;;;AAM9C,SAAS,QAA0B,WAAc,SAAoB,MAAiB;CACpF,OAAO,IAAI,MAAM,WAAW,EAC1B,IAAI,QAAQ,UAAU;EACpB,MAAM,QAAQ,QAAQ,IAAI,QAAQ,QAAQ;EAC1C,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,QAAQ;EACvC,IAAI,OAAO,UAAU,YACnB,QAAQ,GAAG,SAAoB;GAC7B,MAAM,SAAS,QAAQ,MAAM,OAAO,QAAQ,IAAI;GAChD,IAAI,CAAC,WAAW,MAAM,GAAG,OAAO;GAChC,MAAM,QAAiB;IACrB,WAAW,GAAG,KAAK;IACnB,SAAS,QAAQ,QAAQ,MAAM,CAAC,CAAC,WACzB,KAAA,IACL,UAAmB,yBAAS,IAAI,MAAM,GAAG,KAAK,UAAU,CAC3D;IACA,UAAU;GACZ;GACA,QAAQ,KAAK,KAAK;GAClB,MAAM,WAAc,OAAmC;IACrD,MAAM,WAAW;IACjB,OAAO,GAAG,QAAQ,QAAQ,MAAM,CAAC;GACnC;GACA,OAAO;IAEL,OAAO,aAAqB,eAC1B,SAAS,MAAM,EAAE,KAAK,aAAa,UAAU,CAAC;IAChD,QAAQ,eAAuB,SAAS,MAAM,EAAE,MAAM,UAAU,CAAC;IACjE,UAAU,cAAsB,SAAS,MAAM,EAAE,QAAQ,SAAS,CAAC;KAClE,OAAO,cAAc;GACxB;EACF;EAEF,OAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,WAAW,KAAK,IACnE,QAAQ,OAAO,SAAS,IAAI,IAC5B;CACN,EACF,CAAC;AACH;AAEA,MAAM,SAAS,WAAoB,SAAiB;CAClD,MAAM,UAAU,eAAe,SAAS;CACxC,OAAO,WAAW,OAAO,cAAc,YAAY,cAAc,OAC7D,QAAQ,WAAW,SAAS,IAAI,IAChC;AACN;;AAGA,MAAM,SAAiC;CAAE,MAAM;CAAkB,MAAM;AAAiB;AAExF,MAAa,SAAuB,IAAI,MAAM,cAAc;CAC1D,QAAQ,QAAQ,SAAS,SACvB,MAAM,QAAQ,MAAM,cAAc,SAAS,KAAK,QAAQ,SAAS,IAAI,GAAG,WAAW;CACrF,MAAM,QAAQ,aAAa;EACzB,MAAM,QAAQ,cAAc,SAAS,KAAK;EAC1C,MAAM,QAAQ,QAAQ,IAAI,OAAO,QAAQ;EAEzC,IAAI,OAAO,UAAU,cAAc,aAAa,UAAU,OAAO;EACjE,MAAM,QAAQ,OAAO;EAErB,QAAQ,GAAG,SAAoB;GAC7B,MAAM,SAAS,QAAQ,MAAM,OAAO,OAAO,IAAI;GAC/C,OAAO,QAAQ,MAAM,QAAQ,KAAK,IAAI;EACxC;CACF;AACF,CAAC;;AAGD,SAAgB,kBACd,KACA,SACA,MACY;CACZ,OAAO,cAAc,IAAI,WAAW,eAAe,IAAI,SAAS,IAAI,CAAC;AACvE;;;;;AAMA,eAAsB,eAAe,SAAgD;CACnF,MAAM,SAAS,QAAQ,QAAQ,MAAM,CAAC,EAAE,QAAQ;CAChD,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;CAChC,KAAK,MAAM,KAAK,QAAQ,EAAE,WAAW;CACrC,MAAM,WAAW,MAAM,QAAQ,IAAI,OAAO,KAAK,MAAM,EAAE,OAAO,CAAC;CAC/D,MAAM,IAAI,KAAK,IACb,GACA,SAAS,WAAW,MAAM,MAAM,KAAA,CAAS,CAC3C;CACA,MAAM,OAAO,GAAG,OAAO,EAAE,EAAE,UAAU,qDAAqD,OAAO,EAAE,EAAE,UAAU;CAC/G,MAAM,UAAU,SAAS;CACzB,IAAI,mBAAmB,OAAO;EAC5B,QAAQ,UAAU,GAAG,QAAQ,QAAQ,MAAM;EAC3C,OAAO;CACT;CACA,MAAM,QAAQ,IAAI,MAAM,IAAI;CAC5B,MAAM,OAAO;CACb,OAAO;AACT;AAQA,SAAgB,aAAa,UAA0B;CACrD,aAAa,OAAO,QAAiD;AACvE;;;ACvQA,IAAI;;AAGJ,SAAgB,UAAU,MAAoB;CAC5C,SAAS;AACX;AAEA,SAAS,SAAiB;CACxB,IAAI,CAAC,QACH,MAAM,IAAI,WACR,iGACF;CACF,OAAO;AACT;AAEA,MAAa,gBAAwC,MAAM,SAAS,SAClE,OAAO,CAAC,CAAC,aAAa,MAAM,SAAS,IAAI;AAE3C,MAAa,iBAA0C,MAAM,MAAM,SACjE,OAAO,CAAC,CAAC,cAAc,MAAM,MAAM,IAAI;AAEzC,MAAa,sBAAkC,OAAO,CAAC,CAAC,cAAc;AAEtE,MAAa,uBAA2C,OAAO,CAAC,CAAC,eAAe;;;ACzDhF,IAAM,iBAAN,MAA0C;CACxC,2BAAoB,IAAI,IAAqB;CAC7C,aAAsB,IAAI,WAAW,MAAM;EACzC,KAAK,SAAS,IAAI,EAAE,EAAE,CAAC,EAAE,QAAQ;EACjC,KAAK,SAAS,OAAO,EAAE,EAAE;CAC3B,CAAC;CAED,QAAQ,IAAY,QAAgB,MAAwB,SAAkB;EAC5E,KAAK,SAAS,IAAI,IAAI,OAAO;EAC7B,KAAK,WAAW,QAAQ;GAAE;GAAI,OAAO;GAAS;GAAQ,GAAI,OAAO,EAAE,KAAK,IAAI,CAAC;EAAG,CAAC;CACnF;CAEA,QAAQ,IAAY;EAClB,KAAK,SAAS,OAAO,EAAE;EACvB,KAAK,WAAW,QAAQ,EAAE;CAC5B;AACF;;AAGA,IAAM,mBAAN,MAA4C;CAC1C,2BAAoB,IAAI,IAAqB;CAC7C;CAEA,YAAY,MAAwC;EAClD,KAAK,QAAQ;CACf;CAEA,MAAM,IAAY;EAChB,MAAM,UAAU,KAAK,SAAS,IAAI,EAAE;EACpC,KAAK,SAAS,OAAO,EAAE;EACvB,SAAS,QAAQ;CACnB;CAEA,QAAQ,IAAY,QAAgB,MAAwB,SAAkB;EAC5E,KAAK,SAAS,IAAI,IAAI,OAAO;EAC7B,KAAK,MAAM;GAAE,GAAG;GAAW;GAAI;GAAQ,GAAI,OAAO,EAAE,KAAK,IAAI,CAAC;EAAG,CAAC;CACpE;CAEA,QAAQ,IAAY;EAClB,KAAK,SAAS,OAAO,EAAE;EACvB,KAAK,MAAM;GAAE,GAAG;GAAW;EAAG,CAAC;CACjC;AACF;AAEA,IAAI;AACJ,IAAI,MAAM;;AAGV,SAAgB,oBAAoB,MAA8D;CAChG,MAAM,UAAU,IAAI,iBAAiB,IAAI;CACzC,YAAY;CACZ,QAAQ,OAAO,QAAQ,MAAM,EAAE;AACjC;;;;;AAMA,SAAgB,QACd,QACA,UAAiD,CAAC,GAClC;CAChB,MAAM,EAAE,MAAM,WAAW;CACzB,IAAI,QAAQ,SAAS,OAAO,QAAQ,OAAO,OAAO,MAAM;CACxD,cAAc,IAAI,eAAe;CACjC,MAAM,MAAM;CACZ,MAAM,KAAK,OAAO,EAAE,GAAG;CACvB,IAAI,WAAW;CACf,MAAM,QAAe,EACnB,UAAU;EACR,IAAI,UAAU;EACd,WAAW;EACX,IAAI,QAAQ,EAAE;CAChB,EACF;CACA,OAAO,IAAI,SAAgB,SAAS,WAAW;EAC7C,MAAM,gBAAgB;GACpB,MAAM,QAAQ;GACd,OAAO,QAAQ,MAAM;EACvB;EACA,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;EACzD,IAAI,QAAQ,IAAI,QAAQ,MAAM;GAC5B,eAAe;IACb,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,QAAQ,KAAK;GACf;GACA,SAAS,UAAU;IACjB,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,OAAO,KAAK;GACd;EACF,CAAC;CACH,CAAC;AACH;;AAGA,eAAsB,UACpB,QACA,IACA,QACY;CACZ,MAAM,QAAQ,MAAM,QAAQ,QAAQ,SAAS,EAAE,OAAO,IAAI,CAAC,CAAC;CAC5D,IAAI;EACF,OAAO,MAAM,GAAG;CAClB,UAAU;EACR,MAAM,QAAQ;CAChB;AACF;;;;ACvHA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAgB;AAClB;AAEA,MAAM,UAAU,UACd,OAAO,UAAU,YACjB,UAAU,QACV,MAAM,QAAS,MAAe,UAAU,KACxC,MAAM,QAAS,MAAe,UAAU;AAE1C,MAAM,WAAW,OAAe,YAC9B,YAAY,KAAA,MACX,OAAO,YAAY,WAAW,MAAM,SAAS,OAAO,IAAI,QAAQ,KAAK,KAAK;;AAG7E,SAAS,SACP,SACA,QACA,MACA,OACA,SACQ;CACR,IAAI,CAAC,OAAO,MAAM,GAAG,MAAM,IAAI,UAAU,GAAG,QAAQ,gCAAgC;CACpF,MAAM,OAAO,SAAS,eAAe,iBAAiB;CACtD,IAAI,OAAO,WAAW,SAAS,IAAI,GACjC,MAAM,IAAI,uBACR,GAAG,QAAQ,uDAAuD,KAAK,KAAK,KAAK,+IACnF;CAEF,MAAM,OAAO,OAAO,WAAW,QAAQ,MAAM,EAAE,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK;CACvE,MAAM,OAAO,KAAK,MAAM,MAAM,QAAQ,GAAG,OAAO,CAAC;CACjD,MAAM,SAAS,YAAY,KAAA,IAAY,KAAK,aAAa;CACzD,OAAO;EACL;EACA,eACE,OACI,eAAe,OAAO,OAAO,QAAQ,KAAK,UAAU,IAAI,MACxD,YAAY,OAAO,OAAO,QAAQ,KAAK,SAAS,KAAK,UAAU,IAAI,IAAI;CAC/E;AACF;;AAGA,MAAM,QAAQ,GAAW,QAAQ,IAAI,QAAQ,QAAS;CACpD,MAAM,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI;CAC5D,OAAO,KAAK,UAAU,QAClB,OACA,MAAM,KAAK,SAAS,MAAM,sBAAsB,KAAK,MAAM,CAAC,KAAK;AACvE;AAIA,MAAM,YAAY,UAChB,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAQ,MAAmB,aAAa;AAEzF,MAAa,WAAW;CACtB,MAAM,WAAW,QAAiB,MAAc,SAA4C;EAC1F,IAAI,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,UAAU,iDAAiD;EAC5F,MAAM,UAAU,MAAM,OAAO,SAAS,IAAI;EAC1C,IAAI,YAAY,MAAM,OAAO;GAAE,MAAM;GAAO,eAAe,iBAAiB,KAAK;EAAW;EAC5F,IAAI,YAAY,KAAA,GAAW,OAAO;GAAE,MAAM;GAAM,eAAe,uBAAuB;EAAO;EAC7F,MAAM,OAAO,OAAO,YAAY,WAAW,QAAQ,SAAS,OAAO,IAAI,QAAQ,KAAK,OAAO;EAC3F,OAAO;GACL;GACA,eACE,YAAY,KAAK,GAAG,OAAO,SAAS,GAAG,WAAW,QAAQ,QAAQ,KAAK,QAAQ,KAAK,OAAO;EAC/F;CACF;CAEA,MAAM,UAAU,QAAkC;EAChD,MAAM,IAAK,MAAM;EACjB,IAAI,OAAO,GAAG,aAAa,UAAU,MAAM,IAAI,UAAU,mCAAmC;EAC5F,OAAO;GACL,MAAM,EAAE,aAAa;GACrB,eACE,EAAE,aAAa,IACX,6BACA,0CAA0C,EAAE,SAAS,oBAClC,KAAK,EAAE,MAAM,EAAE,oBAAoB,KAAK,EAAE,MAAM;EAC3E;CACF;CAEA,eAAe,QAAiB,OAAiC;EAC/D,OAAO,SACL,kBACA,QACA,eACC,MAAO,WAAW,IAAI,EAAE,QAAQ,IACjC,KACF;CACF;CAEA,cAAc,QAAiB,KAA+B;EAC5D,OAAO,SAAS,iBAAiB,QAAQ,cAAc,MAAO,SAAS,IAAI,EAAE,MAAM,IAAK,GAAG;CAC7F;CAEA,OAAO,QAAyB;EAC9B,MAAM,IAAI;EACV,IAAI,OAAO,GAAG,UAAU,UAAU,MAAM,IAAI,UAAU,sCAAsC;EAC5F,MAAM,QAAQ,EAAE,MAAM,QAAQ,CAAC;EAC/B,OAAO;GACL,MAAM,EAAE,YAAY;GACpB,eACE,EAAE,YAAY,SACV,iCAAiC,MAAM,KAAK,EAAE,UAAU,QAAQ,EAAE,SAAS,KAC3E,eAAe,MAAM,YAAY,EAAE,UAAU,QAAQ,EAAE,SAAS,KAAK,EAAE,MAAM;EACrF;CACF;AACF"}

package/dist/model-BdO79gWz.js

@@ -0,0 +1,44 @@
+import { c as judgeRoute } from "./judge-D8Uf4dA9.js";
+import { i as debugTrace } from "./attempt-BdRQ9jKX.js";
+import { experimental_evaluate } from "ai";
+import { google } from "@ai-sdk/google";
+//#region src/judge/model.ts
+/** See `judgeRoute`: a known provider's own package when its key is set, else the AI Gateway. */
+async function resolveModel(model) {
+	if (typeof model !== "string") return model;
+	const route = judgeRoute(model);
+	if (route.kind === "gateway") return model;
+	const id = model.slice(route.provider.length + 1);
+	if (route.provider === "google") return google.evaluationModel(id);
+	const { typeSafeAi } = await import("@ai-sdk/typesafe-ai");
+	return typeSafeAi.evaluationModel(id);
+}
+async function scoreWithModel(model, question, state, providerOptions, signal) {
+	const result = await experimental_evaluate({
+		model: await resolveModel(model),
+		state,
+		questions: { answer: {
+			type: "boolean",
+			instructions: question
+		} },
+		providerOptions,
+		abortSignal: signal,
+		maxRetries: 0
+	});
+	const { answers, usage, warnings, response, providerMetadata } = result;
+	debugTrace("judge", "provider-response", {
+		answers,
+		usage,
+		warnings,
+		response,
+		providerMetadata
+	}, "judge provider response");
+	return {
+		score: result.answers.answer.probability,
+		resolvedModel: result.response?.modelId
+	};
+}
+//#endregion
+export { scoreWithModel };
+
+//# sourceMappingURL=model-BdO79gWz.js.map

package/dist/model-BdO79gWz.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-BdO79gWz.js","names":["evaluate"],"sources":["../src/judge/model.ts"],"mappings":";;;;;;AASA,eAAe,aACb,OAC0B;CAC1B,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,MAAM,QAAQ,WAAW,KAAK;CAC9B,IAAI,MAAM,SAAS,WAAW,OAAO;CACrC,MAAM,KAAK,MAAM,MAAM,MAAM,SAAS,SAAS,CAAC;CAChD,IAAI,MAAM,aAAa,UAAU,OAAO,OAAO,gBAAgB,EAAE;CACjE,MAAM,EAAE,eAAe,MAAM,OAAO;CACpC,OAAO,WAAW,gBAAgB,EAAE;AACtC;AAEA,eAAsB,eACpB,OACA,UACA,OACA,iBACA,QAC+D;CAC/D,MAAM,SAAS,MAAMA,sBAAS;EAC5B,OAAO,MAAM,aAAa,KAAK;EAC/B;EACA,WAAW,EAAE,QAAQ;GAAE,MAAM;GAAW,cAAc;EAAS,EAAE;EACjE;EACA,aAAa;EAEb,YAAY;CACd,CAAC;CACD,MAAM,EAAE,SAAS,OAAO,UAAU,UAAU,qBAAqB;CACjE,WACE,SACA,qBACA;EAAE;EAAS;EAAO;EAAU;EAAU;CAAiB,GACvD,yBACF;CACA,OAAO;EAAE,OAAO,OAAO,QAAQ,OAAO;EAAa,eAAe,OAAO,UAAU;CAAQ;AAC7F"}