stageflow 0.40.0 → 0.42.0

package/dist/config/installEvalSample.js

@@ -0,0 +1,182 @@
+import { existsSync } from "node:fs";
+import { access, readFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { isMap, parse as parseYaml, parseDocument } from "yaml";
+import { isAgentError } from "../agents/errors.js";
+import { EvalsWriteError, writeFileAtomic } from "./atomicWrite.js";
+import { ensureCatalogScanRoot } from "./ensureCatalogScanRoot.js";
+import { explicitRoots, findEvalFile, manifestForWrite, resolveWriteDir, } from "./evalWriteRoots.js";
+import { peekDefaultModel } from "../models/catalog.js";
+import { loadStageflowManifestOutcome } from "./loadStageflowManifest.js";
+import { resolveSeededExamplesPath } from "./seededCatalog.js";
+export const EVAL_SAMPLE_FILES = {
+    pipeline: "eval-sample.pipeline.yaml",
+    eval: "critique-sample.eval.yaml",
+    dataset: "critique-sample.dataset.jsonl",
+};
+export const EVAL_SAMPLE_ID = "critique-sample";
+export const AGENT_EVAL_SAMPLE_FILES = {
+    agent: "support-sample.agent.yaml",
+    eval: "support-sample.eval.yaml",
+    dataset: "support-sample.dataset.jsonl",
+};
+export const AGENT_EVAL_SAMPLE_ID = "support-sample";
+export const AGENT_EVAL_SAMPLE_AGENT_ID = "support_sample";
+async function exists(abs) {
+    try {
+        await access(abs);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function relativeRef(fromDir, toFile) {
+    const rel = path.relative(fromDir, toFile).split(path.sep).join("/");
+    return rel.startsWith(".") ? rel : `./${rel}`;
+}
+function rewriteEvalPaths(text, pipelineRef, datasetRef) {
+    const doc = parseDocument(text);
+    if (!isMap(doc.contents))
+        throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", "Sample eval file is not a mapping");
+    if (pipelineRef !== undefined)
+        doc.setIn(["target", "pipeline"], pipelineRef);
+    doc.set("dataset", datasetRef);
+    const out = doc.toString({ lineWidth: 0 });
+    return out.endsWith("\n") ? out : `${out}\n`;
+}
+export function resolveEvalSampleDir(fromUrl = import.meta.url, marker = EVAL_SAMPLE_FILES.eval) {
+    const examples = resolveSeededExamplesPath(fromUrl);
+    const here = path.dirname(fileURLToPath(fromUrl));
+    const candidates = [
+        ...(examples === undefined ? [] : [path.join(examples, "evals")]),
+        path.resolve(here, "../../examples/evals"),
+        path.resolve(here, "../../../examples/evals"),
+        path.resolve(process.cwd(), "examples/evals"),
+    ];
+    return candidates.find((dir) => existsSync(path.join(dir, marker)));
+}
+export async function installEvalSample(input) {
+    if (input.kind === "agent")
+        return installAgentEvalSample(input);
+    const { projectRoot } = input;
+    const sourceDir = input.sourceDir ?? resolveEvalSampleDir();
+    if (sourceDir === undefined) {
+        throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", "The packaged sample files were not found");
+    }
+    const sources = {
+        pipeline: path.join(sourceDir, EVAL_SAMPLE_FILES.pipeline),
+        eval: path.join(sourceDir, EVAL_SAMPLE_FILES.eval),
+        dataset: path.join(sourceDir, EVAL_SAMPLE_FILES.dataset),
+    };
+    const texts = { pipeline: "", eval: "", dataset: "" };
+    for (const key of Object.keys(sources)) {
+        try {
+            texts[key] = await readFile(sources[key], "utf8");
+        }
+        catch {
+            throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", `Sample file ${EVAL_SAMPLE_FILES[key]} not found in ${sourceDir}`);
+        }
+    }
+    const manifest = await manifestForWrite(projectRoot);
+    const dirs = {
+        pipeline: await resolveWriteDir(projectRoot, manifest, "pipelines"),
+        eval: await resolveWriteDir(projectRoot, manifest, "evals"),
+        dataset: await resolveWriteDir(projectRoot, manifest, "datasets"),
+    };
+    const targets = {
+        pipeline: path.join(dirs.pipeline.absDir, EVAL_SAMPLE_FILES.pipeline),
+        eval: path.join(dirs.eval.absDir, EVAL_SAMPLE_FILES.eval),
+        dataset: path.join(dirs.dataset.absDir, EVAL_SAMPLE_FILES.dataset),
+    };
+    const taken = (await findEvalFile(projectRoot, EVAL_SAMPLE_ID, manifest)) !== undefined ||
+        (await exists(targets.pipeline)) ||
+        (await exists(targets.eval)) ||
+        (await exists(targets.dataset));
+    if (taken)
+        throw new EvalsWriteError("SAMPLE_EXISTS", "The sample eval is already installed");
+    const evalText = rewriteEvalPaths(texts.eval, relativeRef(dirs.eval.absDir, targets.pipeline), relativeRef(dirs.eval.absDir, targets.dataset));
+    const projectManifest = await loadStageflowManifestOutcome(projectRoot);
+    const hasDefaultModel = projectManifest.ok && projectManifest.value.manifest.model !== undefined;
+    const pipelineText = hasDefaultModel
+        ? texts.pipeline
+        : texts.pipeline.replace(/^id: .*\n/m, (line) => `${line}model: ${peekDefaultModel()}\n`);
+    await writeFileAtomic(targets.pipeline, pipelineText);
+    await writeFileAtomic(targets.dataset, texts.dataset);
+    await writeFileAtomic(targets.eval, evalText);
+    const scanKeys = {
+        pipeline: { kind: "pipelines", key: "pipelines" },
+        eval: { kind: "evals", key: "evals" },
+        dataset: { kind: "datasets", key: "datasets" },
+    };
+    for (const name of Object.keys(dirs)) {
+        const { kind, key } = scanKeys[name];
+        if (explicitRoots(manifest, kind) !== undefined) {
+            await ensureCatalogScanRoot(projectRoot, dirs[name].relDir, [key]);
+        }
+    }
+    return { evalId: EVAL_SAMPLE_ID };
+}
+async function agentExists(agents, projectRoot) {
+    try {
+        await agents.get(projectRoot, AGENT_EVAL_SAMPLE_AGENT_ID);
+        return true;
+    }
+    catch (err) {
+        if (isAgentError(err) && err.code === "agent.not_found")
+            return false;
+        throw err;
+    }
+}
+async function installAgentEvalSample(input) {
+    const { projectRoot, agents } = input;
+    const sourceDir = input.sourceDir ?? resolveEvalSampleDir(undefined, AGENT_EVAL_SAMPLE_FILES.eval);
+    if (sourceDir === undefined) {
+        throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", "The packaged sample files were not found");
+    }
+    if (agents === undefined) {
+        throw new EvalsWriteError("SAMPLE_AGENTS_UNAVAILABLE", "Agents are not available on this Host");
+    }
+    const texts = { agent: "", eval: "", dataset: "" };
+    for (const key of Object.keys(AGENT_EVAL_SAMPLE_FILES)) {
+        try {
+            texts[key] = await readFile(path.join(sourceDir, AGENT_EVAL_SAMPLE_FILES[key]), "utf8");
+        }
+        catch {
+            throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", `Sample file ${AGENT_EVAL_SAMPLE_FILES[key]} not found in ${sourceDir}`);
+        }
+    }
+    const projectManifest = await loadStageflowManifestOutcome(projectRoot);
+    if (!projectManifest.ok || projectManifest.value.manifest.model === undefined) {
+        throw new EvalsWriteError("SAMPLE_NO_DEFAULT_MODEL", "The agent sample needs a default model. Set model in stageflow.yaml, then install the sample again.");
+    }
+    const manifest = await manifestForWrite(projectRoot);
+    const dirs = {
+        eval: await resolveWriteDir(projectRoot, manifest, "evals"),
+        dataset: await resolveWriteDir(projectRoot, manifest, "datasets"),
+    };
+    const targets = {
+        eval: path.join(dirs.eval.absDir, AGENT_EVAL_SAMPLE_FILES.eval),
+        dataset: path.join(dirs.dataset.absDir, AGENT_EVAL_SAMPLE_FILES.dataset),
+    };
+    const taken = (await findEvalFile(projectRoot, AGENT_EVAL_SAMPLE_ID, manifest)) !== undefined ||
+        (await exists(targets.eval)) ||
+        (await exists(targets.dataset)) ||
+        (await agentExists(agents, projectRoot));
+    if (taken)
+        throw new EvalsWriteError("SAMPLE_EXISTS", "The sample eval is already installed");
+    const agentDefinition = parseYaml(texts.agent);
+    const evalText = rewriteEvalPaths(texts.eval, undefined, relativeRef(dirs.eval.absDir, targets.dataset));
+    await agents.create(projectRoot, { ...agentDefinition, web_chat: { enabled: false } }, { source: "api" });
+    await agents.setEnabled(projectRoot, AGENT_EVAL_SAMPLE_AGENT_ID, true, { source: "api" });
+    await writeFileAtomic(targets.dataset, texts.dataset);
+    await writeFileAtomic(targets.eval, evalText);
+    const scanKeys = { eval: "evals", dataset: "datasets" };
+    for (const name of Object.keys(dirs)) {
+        if (explicitRoots(manifest, scanKeys[name]) !== undefined) {
+            await ensureCatalogScanRoot(projectRoot, dirs[name].relDir, [scanKeys[name]]);
+        }
+    }
+    return { evalId: AGENT_EVAL_SAMPLE_ID };
+}

package/dist/config/loadDataset.d.ts

@@ -0,0 +1,16 @@
+import type { DatasetFormat, DatasetLimits, DatasetLoadResult, ParseDatasetResult } from "./datasetTypes.js";
+export declare const DATASET_LIMITS: DatasetLimits;
+export declare const DATASET_MIN_CASES = 5;
+export type ParseDatasetOptions = {
+    limits?: Partial<DatasetLimits>;
+    path?: string;
+};
+export declare function datasetFormatFromPath(filePath: string): DatasetFormat | undefined;
+export declare function parseDatasetText(text: string, format: DatasetFormat, options?: ParseDatasetOptions & {
+    csvDelimiter?: string;
+}): ParseDatasetResult;
+export declare function parseDatasetCases(rawCases: readonly unknown[], options?: ParseDatasetOptions): ParseDatasetResult;
+export declare function loadDatasetFile(absPath: string, options?: ParseDatasetOptions & {
+    root?: string;
+    csvDelimiter?: string;
+}): Promise<DatasetLoadResult>;

package/dist/config/loadDataset.js

@@ -0,0 +1,318 @@
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { CsvParseError, parseCsv } from "./csv.js";
+import { datasetContentHash } from "./datasetHash.js";
+export const DATASET_LIMITS = {
+    maxCases: 5000,
+    maxCaseBytes: 1024 * 1024,
+    maxFileBytes: 50 * 1024 * 1024,
+};
+export const DATASET_MIN_CASES = 5;
+const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+const TAG_RE = /^[a-z0-9][a-z0-9-]{0,31}$/;
+const MAX_TAGS = 10;
+const MAX_TITLE = 120;
+const CASE_KEYS = new Set(["id", "input", "title", "expected", "tags", "goal", "metadata"]);
+function isPlainObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+export function datasetFormatFromPath(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext === ".jsonl")
+        return "jsonl";
+    if (ext === ".json")
+        return "json";
+    if (ext === ".csv")
+        return "csv";
+    return undefined;
+}
+function finding(severity, code, filePath, message, pos) {
+    const out = {
+        severity,
+        code,
+        path: filePath,
+        message: pos ? `${pos.label}: ${message}` : message,
+        category: "dataset",
+    };
+    if (pos && pos.line > 0)
+        out.line = pos.line;
+    return out;
+}
+function parseJsonl(text, filePath, findings) {
+    const entries = [];
+    const lines = text.charCodeAt(0) === 0xfeff ? text.slice(1).split(/\r?\n/) : text.split(/\r?\n/);
+    lines.forEach((raw, idx) => {
+        if (raw.trim() === "")
+            return;
+        const pos = { line: idx + 1, label: `line ${idx + 1}` };
+        try {
+            entries.push({ value: JSON.parse(raw), pos });
+        }
+        catch (err) {
+            const detail = err instanceof Error ? err.message : String(err);
+            findings.push(finding("error", "dataset.load_error", filePath, `invalid JSON (${detail})`, pos));
+        }
+    });
+    return entries;
+}
+function parseJson(text, filePath, findings) {
+    let doc;
+    try {
+        doc = JSON.parse(text.charCodeAt(0) === 0xfeff ? text.slice(1) : text);
+    }
+    catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        findings.push(finding("error", "dataset.load_error", filePath, `invalid JSON (${detail})`));
+        return [];
+    }
+    let list;
+    if (Array.isArray(doc))
+        list = doc;
+    else if (isPlainObject(doc) && Array.isArray(doc.cases))
+        list = doc.cases;
+    if (!Array.isArray(list)) {
+        findings.push(finding("error", "dataset.invalid_shape", filePath, 'JSON dataset must be an array of cases or an object { "cases": [...] }'));
+        return [];
+    }
+    return list.map((value, idx) => ({
+        value,
+        pos: { line: 0, label: `case ${idx + 1}` },
+    }));
+}
+const STRING_COLUMNS = new Set(["id", "title", "goal"]);
+const JSON_BASES = new Set(["id", "title", "goal", "tags", "input", "expected", "metadata"]);
+function classifyHeader(header) {
+    const json = header.endsWith(":json");
+    const name = json ? header.slice(0, -":json".length) : header;
+    if (json && JSON_BASES.has(name))
+        return { base: name, json };
+    if (!json && (STRING_COLUMNS.has(name) || name === "tags" || name === "expected")) {
+        return { base: name, json };
+    }
+    for (const prefix of ["input.", "expected."]) {
+        if (name.startsWith(prefix) && name.length > prefix.length) {
+            return { base: prefix.slice(0, -1), key: name.slice(prefix.length), json };
+        }
+    }
+    return undefined;
+}
+function parseCsvEntries(text, filePath, findings, delimiter) {
+    let records;
+    try {
+        records = parseCsv(text, { delimiter });
+    }
+    catch (err) {
+        if (err instanceof CsvParseError) {
+            findings.push(finding("error", "dataset.load_error", filePath, err.message, {
+                line: err.record,
+                label: `row ${err.record}`,
+            }));
+            return [];
+        }
+        throw err;
+    }
+    const headerIdx = records.findIndex((r) => !(r.length === 1 && r[0] === ""));
+    if (headerIdx === -1)
+        return [];
+    const headers = records[headerIdx].map((h) => h.trim());
+    const targets = headers.map((h, col) => {
+        const target = classifyHeader(h);
+        if (!target) {
+            findings.push(finding("error", "dataset.csv_unknown_column", filePath, `unknown CSV column "${h}" (column ${col + 1}); use id, title, goal, tags, input.<name>, expected.<name> or a :json suffix`, { line: headerIdx + 1, label: `row ${headerIdx + 1}` }));
+        }
+        return target;
+    });
+    const entries = [];
+    for (let r = headerIdx + 1; r < records.length; r += 1) {
+        const cells = records[r];
+        if (cells.length === 1 && cells[0] === "")
+            continue;
+        const pos = { line: r + 1, label: `row ${r + 1}` };
+        const obj = {};
+        let broken = false;
+        targets.forEach((target, col) => {
+            const cell = cells[col];
+            if (!target || cell === undefined || cell === "")
+                return;
+            let value = cell;
+            if (target.json) {
+                try {
+                    value = JSON.parse(cell);
+                }
+                catch {
+                    findings.push(finding("error", "dataset.load_error", filePath, `column "${headers[col]}" holds invalid JSON`, pos));
+                    broken = true;
+                    return;
+                }
+            }
+            else if (target.base === "tags") {
+                value = cell
+                    .split(";")
+                    .map((t) => t.trim())
+                    .filter((t) => t !== "");
+            }
+            if (target.key !== undefined) {
+                const parent = (obj[target.base] ??= {});
+                parent[target.key] = value;
+            }
+            else {
+                obj[target.base] = value;
+            }
+        });
+        if (!broken)
+            entries.push({ value: obj, pos });
+    }
+    return entries;
+}
+function validateEntries(entries, filePath, limits, findings) {
+    const cases = [];
+    const seen = new Set();
+    let list = entries;
+    if (entries.length > limits.maxCases) {
+        findings.push(finding("error", "dataset.too_many_cases", filePath, `dataset has ${entries.length} cases; the limit is ${limits.maxCases}`));
+        list = entries.slice(0, limits.maxCases);
+    }
+    for (const { value, pos } of list) {
+        const report = (code, message) => {
+            findings.push(finding("error", code, filePath, message, pos));
+        };
+        if (!isPlainObject(value)) {
+            report("dataset.not_object", "case must be a JSON object");
+            continue;
+        }
+        let ok = true;
+        const fail = (code, message) => {
+            report(code, message);
+            ok = false;
+        };
+        for (const key of Object.keys(value)) {
+            if (!CASE_KEYS.has(key))
+                fail("dataset.unknown_key", `unknown case key "${key}"`);
+        }
+        const id = value.id;
+        if (id === undefined) {
+            fail("dataset.case_missing_id", "case has no id (hand-authored datasets must carry ids)");
+        }
+        else if (typeof id !== "string" || !ID_RE.test(id)) {
+            fail("dataset.case_invalid_id", `id ${JSON.stringify(id)} must match ${ID_RE.source}`);
+        }
+        else if (seen.has(id)) {
+            fail("dataset.duplicate_id", `duplicate id "${id}"`);
+        }
+        else {
+            seen.add(id);
+        }
+        if (value.input === undefined) {
+            fail("dataset.case_missing_input", "case has no input");
+        }
+        else if (!isPlainObject(value.input)) {
+            fail("dataset.invalid_field", "input must be an object");
+        }
+        if (value.title !== undefined && (typeof value.title !== "string" || value.title.length > MAX_TITLE)) {
+            fail("dataset.invalid_field", `title must be a string of at most ${MAX_TITLE} characters`);
+        }
+        if (value.goal !== undefined && typeof value.goal !== "string") {
+            fail("dataset.invalid_field", "goal must be a string");
+        }
+        if (value.metadata !== undefined && !isPlainObject(value.metadata)) {
+            fail("dataset.invalid_field", "metadata must be an object");
+        }
+        if (value.tags !== undefined) {
+            const tags = value.tags;
+            if (!Array.isArray(tags) || tags.length > MAX_TAGS) {
+                fail("dataset.invalid_field", `tags must be an array of at most ${MAX_TAGS} strings`);
+            }
+            else {
+                for (const tag of tags) {
+                    if (typeof tag !== "string" || !TAG_RE.test(tag)) {
+                        fail("dataset.invalid_field", `tag ${JSON.stringify(tag)} must match ${TAG_RE.source}`);
+                    }
+                }
+            }
+        }
+        if (ok && Buffer.byteLength(JSON.stringify(value), "utf8") > limits.maxCaseBytes) {
+            fail("dataset.case_too_large", `case "${String(id)}" is larger than ${limits.maxCaseBytes} bytes`);
+        }
+        if (ok)
+            cases.push(value);
+    }
+    return cases;
+}
+export function parseDatasetText(text, format, options = {}) {
+    const limits = { ...DATASET_LIMITS, ...options.limits };
+    const filePath = options.path ?? "";
+    const findings = [];
+    if (Buffer.byteLength(text, "utf8") > limits.maxFileBytes) {
+        findings.push(finding("error", "dataset.file_too_large", filePath, `dataset file is larger than ${limits.maxFileBytes} bytes`));
+        return { cases: [], findings, contentHash: datasetContentHash([]) };
+    }
+    let entries;
+    if (format === "jsonl")
+        entries = parseJsonl(text, filePath, findings);
+    else if (format === "json")
+        entries = parseJson(text, filePath, findings);
+    else
+        entries = parseCsvEntries(text, filePath, findings, options.csvDelimiter);
+    return finishParse(entries, filePath, limits, findings);
+}
+function finishParse(entries, filePath, limits, findings) {
+    const cases = validateEntries(entries, filePath, limits, findings);
+    if (entries.length === 0 && !findings.some((f) => f.severity === "error")) {
+        findings.push(finding("error", "dataset.empty", filePath, "dataset has no cases"));
+    }
+    else if (entries.length > 0 &&
+        cases.length < DATASET_MIN_CASES &&
+        !findings.some((f) => f.severity === "error")) {
+        findings.push(finding("warning", "dataset.few_cases", filePath, `dataset has ${cases.length} cases; fewer than ${DATASET_MIN_CASES} gives wide error bars`));
+    }
+    return { cases, findings, contentHash: datasetContentHash(cases) };
+}
+export function parseDatasetCases(rawCases, options = {}) {
+    const limits = { ...DATASET_LIMITS, ...options.limits };
+    const entries = rawCases.map((value, idx) => ({
+        value,
+        pos: { line: 0, label: `case ${idx + 1}` },
+    }));
+    return finishParse(entries, options.path ?? "", limits, []);
+}
+export async function loadDatasetFile(absPath, options = {}) {
+    const limits = { ...DATASET_LIMITS, ...options.limits };
+    const relPath = (options.root ? path.relative(options.root, absPath) : path.basename(absPath))
+        .split(path.sep)
+        .join("/");
+    const base = {
+        path: absPath,
+        relPath,
+        format: datasetFormatFromPath(absPath),
+        cases: [],
+        contentHash: datasetContentHash([]),
+        fileBytes: 0,
+    };
+    const fail = (code, message, fileBytes = 0) => ({
+        ...base,
+        fileBytes,
+        findings: [finding("error", code, relPath, message)],
+    });
+    if (base.format === undefined) {
+        return fail("dataset.load_error", `unknown dataset format "${path.extname(absPath)}"; use .jsonl, .json or .csv`);
+    }
+    let size;
+    try {
+        size = (await stat(absPath)).size;
+    }
+    catch (err) {
+        return fail("dataset.load_error", `cannot read dataset: ${err.message}`);
+    }
+    if (size > limits.maxFileBytes) {
+        return fail("dataset.file_too_large", `dataset file is larger than ${limits.maxFileBytes} bytes`, size);
+    }
+    let text;
+    try {
+        text = await readFile(absPath, "utf8");
+    }
+    catch (err) {
+        return fail("dataset.load_error", `cannot read dataset: ${err.message}`, size);
+    }
+    const parsed = parseDatasetText(text, base.format, { ...options, path: relPath });
+    return { ...base, ...parsed, fileBytes: size };
+}

package/dist/config/loadEval.d.ts

@@ -0,0 +1,32 @@
+import type { EvalConversation, EvalFile, ResolvedConversation, ResolvedGrader } from "../evals/types.js";
+import { type LoadOutcome } from "./loadOutcome.js";
+type UnhashedGrader = ResolvedGrader extends infer G ? (G extends unknown ? Omit<G, "hash"> : never) : never;
+export type LoadedEval = {
+    path: string;
+    relPath: string;
+    id: string;
+    fileHash: string;
+    file: EvalFile;
+    graders: ResolvedGrader[];
+};
+export declare const EVAL_DEFAULT_REPEATS = 3;
+export declare const EVAL_DEFAULT_PASS_THRESHOLD = 0.8;
+export declare const EVAL_DEFAULT_MAX_COST_USD = 2;
+export declare const EVAL_DEFAULT_CONCURRENCY = 4;
+export declare const EVAL_AGENT_DEFAULT_CONCURRENCY = 2;
+export declare const EVAL_AGENT_DEFAULT_MAX_COST_USD = 1;
+export declare const EVAL_CONVERSATION_DEFAULTS: ResolvedConversation;
+export declare function resolveConversation(conversation: EvalConversation | undefined): ResolvedConversation;
+export declare function evalFileStem(filePath: string): string;
+export type EvalPaths = {
+    pipelinePath?: string;
+    stageFile?: string;
+    datasetPath: string;
+};
+/** Paths in an eval file resolve against the eval file's directory (P6). */
+export declare function resolveEvalPaths(file: EvalFile, evalAbsPath: string): EvalPaths;
+export declare function graderHash(grader: UnhashedGrader): string;
+export declare function graderHashShort(grader: Pick<ResolvedGrader, "hash">): string;
+export declare function parseEvalOutcome(yamlText: string, filePath: string, projectRoot: string): LoadOutcome<LoadedEval>;
+export declare function loadEvalOutcome(absPath: string, projectRoot: string): Promise<LoadOutcome<LoadedEval>>;
+export {};