stageflow 0.40.0 → 0.44.0

package/dist/config/evalWriteRoots.js

@@ -0,0 +1,109 @@
+import { realpathSync } from "node:fs";
+import { stat } from "node:fs/promises";
+import path from "node:path";
+import { CatalogPathError, isInsideProjectRoot } from "./catalogRelativePath.js";
+import { normalizeCatalogDir } from "./ensureCatalogScanRoot.js";
+import { listDatasetFiles, listEvalFiles } from "./evalCatalog.js";
+import { datasetFormatFromPath } from "./loadDataset.js";
+import { loadStageflowManifestOutcome, manifestPathForProject, parseStageflowManifestOutcome, } from "./loadStageflowManifest.js";
+const DEFAULT_DIR = {
+    evals: "evals",
+    datasets: "datasets",
+    pipelines: "pipelines",
+};
+export async function manifestForWrite(projectRoot) {
+    const outcome = await loadStageflowManifestOutcome(projectRoot);
+    if (outcome.ok)
+        return outcome.value;
+    const fallback = parseStageflowManifestOutcome("version: 1\ncatalog:\n  pipelines: [pipelines]\n  tasks: [tasks]\n", manifestPathForProject(projectRoot), projectRoot);
+    if (!fallback.ok)
+        throw new Error("default manifest failed to parse");
+    return fallback.value;
+}
+export function explicitRoots(manifest, kind) {
+    const entries = manifest.manifest.catalog[kind];
+    return entries === undefined ? undefined : entries.map(normalizeCatalogDir);
+}
+async function isFile(abs) {
+    try {
+        return (await stat(abs)).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+function containedInProject(projectRoot, absPath) {
+    let realRoot = path.resolve(projectRoot);
+    try {
+        realRoot = realpathSync(realRoot);
+    }
+    catch {
+        // project root missing: fall back to the resolved path
+    }
+    const rel = path.relative(path.resolve(projectRoot), absPath);
+    if (rel.startsWith("..") || path.isAbsolute(rel))
+        return false;
+    return isInsideProjectRoot(projectRoot, path.resolve(realRoot, rel));
+}
+export async function resolveWriteDir(projectRoot, manifest, kind, directory) {
+    let relDir;
+    if (directory !== undefined) {
+        if (path.isAbsolute(directory)) {
+            throw new CatalogPathError("Absolute directory is not allowed; use a path relative to the project root", "absolute_path_not_allowed", [projectRoot]);
+        }
+        relDir = normalizeCatalogDir(directory) || ".";
+    }
+    else {
+        relDir = DEFAULT_DIR[kind];
+        for (const entry of explicitRoots(manifest, kind) ?? []) {
+            if (!(await isFile(path.join(projectRoot, entry)))) {
+                relDir = entry;
+                break;
+            }
+        }
+    }
+    const absDir = path.resolve(projectRoot, relDir);
+    if (!containedInProject(projectRoot, absDir)) {
+        throw new CatalogPathError(`directory escapes project root ${projectRoot}`, "path_outside_project_root", [projectRoot]);
+    }
+    return { absDir, relDir, explicitDirectory: directory !== undefined };
+}
+export function assertInsideProject(projectRoot, absPath, what) {
+    if (!containedInProject(projectRoot, absPath)) {
+        throw new CatalogPathError(`${what} escapes project root ${projectRoot}`, "path_outside_project_root", [
+            projectRoot,
+        ]);
+    }
+}
+export function evalStem(fileName) {
+    return fileName.replace(/\.eval\.ya?ml$/, "");
+}
+export function datasetStem(fileName) {
+    return fileName.replace(/\.dataset\.(jsonl|json|csv)$/, "");
+}
+export async function findEvalFile(projectRoot, id, manifest) {
+    const m = manifest ?? (await manifestForWrite(projectRoot));
+    const refs = await listEvalFiles(m);
+    return refs.find((ref) => evalStem(path.basename(ref.absPath)) === id);
+}
+export function datasetIdsFor(refs) {
+    const byStem = new Map();
+    for (const ref of refs) {
+        const stem = datasetStem(path.basename(ref.absPath));
+        byStem.set(stem, [...(byStem.get(stem) ?? []), ref]);
+    }
+    const ids = new Map();
+    for (const [stem, group] of byStem) {
+        for (const ref of group) {
+            const format = datasetFormatFromPath(ref.absPath);
+            ids.set(ref.absPath, group.length > 1 && format !== "jsonl" ? `${stem}.${format ?? "unknown"}` : stem);
+        }
+    }
+    return ids;
+}
+export async function findDatasetFile(projectRoot, id, manifest) {
+    const m = manifest ?? (await manifestForWrite(projectRoot));
+    const refs = await listDatasetFiles(m);
+    const ids = datasetIdsFor(refs);
+    return refs.find((ref) => ids.get(ref.absPath) === id);
+}

package/dist/config/githubTriggerAction.d.ts

@@ -0,0 +1,8 @@
+export declare const GITHUB_TRIGGER_ACTIONS: readonly ["opened", "synchronize", "updated", "merged", "closed"];
+export type GithubTriggerAction = (typeof GITHUB_TRIGGER_ACTIONS)[number];
+export declare function isGithubTriggerAction(value: unknown): value is GithubTriggerAction;
+/**
+ * For `event.source` values starting with `github.`, `match.action` must be set
+ * to one of the five PR poll actions. Returns an error message or undefined.
+ */
+export declare function githubMatchActionError(source: string, match: Record<string, unknown> | undefined): string | undefined;

package/dist/config/githubTriggerAction.js

@@ -0,0 +1,29 @@
+export const GITHUB_TRIGGER_ACTIONS = [
+    "opened",
+    "synchronize",
+    "updated",
+    "merged",
+    "closed",
+];
+const ACTION_SET = new Set(GITHUB_TRIGGER_ACTIONS);
+export function isGithubTriggerAction(value) {
+    return typeof value === "string" && ACTION_SET.has(value);
+}
+/**
+ * For `event.source` values starting with `github.`, `match.action` must be set
+ * to one of the five PR poll actions. Returns an error message or undefined.
+ */
+export function githubMatchActionError(source, match) {
+    if (!source.startsWith("github."))
+        return undefined;
+    const action = match?.action;
+    if (action === undefined) {
+        return ("event.match.action is required for github.* sources " +
+            `(one of: ${GITHUB_TRIGGER_ACTIONS.join(", ")})`);
+    }
+    if (!isGithubTriggerAction(action)) {
+        return (`event.match.action must be one of: ${GITHUB_TRIGGER_ACTIONS.join(", ")} ` +
+            `(got ${JSON.stringify(action)})`);
+    }
+    return undefined;
+}

package/dist/config/hostConfig.js

@@ -27,6 +27,8 @@ export const KNOWN_STAGEFLOW_ENV_KEYS = new Set([
     "STAGEFLOW_READ_TOKEN",
     "STAGEFLOW_READ_TOKEN_FILE",
     "STAGEFLOW_ALLOW_UNKNOWN_CONFIG",
+    "STAGEFLOW_EVALS_KEEP_WORKDIR",
+    "STAGEFLOW_SHOW_INTERNAL_AGENTS",
     "STAGEFLOW_MAX_CONCURRENT_RUNS",
     "STAGEFLOW_MAX_CONCURRENT_RUNS_PER_PROJECT",
     "STAGEFLOW_MAX_QUEUED",
@@ -58,6 +60,7 @@ export const KNOWN_STAGEFLOW_ENV_KEYS = new Set([
     "STAGEFLOW_STAGE_EXECUTION",
     "STAGEFLOW_STAGE_CONTAINER_IMAGE",
     "STAGEFLOW_STAGE_CONTAINER_DOCKER_BIN",
+    "STAGEFLOW_STAGE_READONLY_DIR",
     "STAGEFLOW_STAGE_ENV_ALLOW",
     "STAGEFLOW_STAGE_ENV_PASSTHROUGH",
     "STAGEFLOW_GC_INTERVAL_MS",

package/dist/config/installEvalSample.d.ts

@@ -0,0 +1,34 @@
+import type { AgentDetail } from "../agents/service.js";
+export declare const EVAL_SAMPLE_FILES: {
+    readonly pipeline: "eval-sample.pipeline.yaml";
+    readonly eval: "critique-sample.eval.yaml";
+    readonly dataset: "critique-sample.dataset.jsonl";
+};
+export declare const EVAL_SAMPLE_ID = "critique-sample";
+export declare const AGENT_EVAL_SAMPLE_FILES: {
+    readonly agent: "support-sample.agent.yaml";
+    readonly eval: "support-sample.eval.yaml";
+    readonly dataset: "support-sample.dataset.jsonl";
+};
+export declare const AGENT_EVAL_SAMPLE_ID = "support-sample";
+export declare const AGENT_EVAL_SAMPLE_AGENT_ID = "support_sample";
+export type EvalSampleKind = "stage" | "agent";
+/** The slice of `AgentService` the agent sample install needs. */
+export type EvalSampleAgents = {
+    get(projectRoot: string, agentId: string): Promise<AgentDetail>;
+    create(projectRoot: string, rawInput: unknown, actor: {
+        source: "api";
+    }): Promise<AgentDetail>;
+    setEnabled(projectRoot: string, agentId: string, enabled: boolean, actor: {
+        source: "api";
+    }): Promise<AgentDetail>;
+};
+export declare function resolveEvalSampleDir(fromUrl?: string, marker?: string): string | undefined;
+export declare function installEvalSample(input: {
+    projectRoot: string;
+    sourceDir?: string;
+    kind?: EvalSampleKind;
+    agents?: EvalSampleAgents;
+}): Promise<{
+    evalId: string;
+}>;

package/dist/config/installEvalSample.js

@@ -0,0 +1,182 @@
+import { existsSync } from "node:fs";
+import { access, readFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { isMap, parse as parseYaml, parseDocument } from "yaml";
+import { isAgentError } from "../agents/errors.js";
+import { EvalsWriteError, writeFileAtomic } from "./atomicWrite.js";
+import { ensureCatalogScanRoot } from "./ensureCatalogScanRoot.js";
+import { explicitRoots, findEvalFile, manifestForWrite, resolveWriteDir, } from "./evalWriteRoots.js";
+import { peekDefaultModel } from "../models/catalog.js";
+import { loadStageflowManifestOutcome } from "./loadStageflowManifest.js";
+import { resolveSeededExamplesPath } from "./seededCatalog.js";
+export const EVAL_SAMPLE_FILES = {
+    pipeline: "eval-sample.pipeline.yaml",
+    eval: "critique-sample.eval.yaml",
+    dataset: "critique-sample.dataset.jsonl",
+};
+export const EVAL_SAMPLE_ID = "critique-sample";
+export const AGENT_EVAL_SAMPLE_FILES = {
+    agent: "support-sample.agent.yaml",
+    eval: "support-sample.eval.yaml",
+    dataset: "support-sample.dataset.jsonl",
+};
+export const AGENT_EVAL_SAMPLE_ID = "support-sample";
+export const AGENT_EVAL_SAMPLE_AGENT_ID = "support_sample";
+async function exists(abs) {
+    try {
+        await access(abs);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function relativeRef(fromDir, toFile) {
+    const rel = path.relative(fromDir, toFile).split(path.sep).join("/");
+    return rel.startsWith(".") ? rel : `./${rel}`;
+}
+function rewriteEvalPaths(text, pipelineRef, datasetRef) {
+    const doc = parseDocument(text);
+    if (!isMap(doc.contents))
+        throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", "Sample eval file is not a mapping");
+    if (pipelineRef !== undefined)
+        doc.setIn(["target", "pipeline"], pipelineRef);
+    doc.set("dataset", datasetRef);
+    const out = doc.toString({ lineWidth: 0 });
+    return out.endsWith("\n") ? out : `${out}\n`;
+}
+export function resolveEvalSampleDir(fromUrl = import.meta.url, marker = EVAL_SAMPLE_FILES.eval) {
+    const examples = resolveSeededExamplesPath(fromUrl);
+    const here = path.dirname(fileURLToPath(fromUrl));
+    const candidates = [
+        ...(examples === undefined ? [] : [path.join(examples, "evals")]),
+        path.resolve(here, "../../examples/evals"),
+        path.resolve(here, "../../../examples/evals"),
+        path.resolve(process.cwd(), "examples/evals"),
+    ];
+    return candidates.find((dir) => existsSync(path.join(dir, marker)));
+}
+export async function installEvalSample(input) {
+    if (input.kind === "agent")
+        return installAgentEvalSample(input);
+    const { projectRoot } = input;
+    const sourceDir = input.sourceDir ?? resolveEvalSampleDir();
+    if (sourceDir === undefined) {
+        throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", "The packaged sample files were not found");
+    }
+    const sources = {
+        pipeline: path.join(sourceDir, EVAL_SAMPLE_FILES.pipeline),
+        eval: path.join(sourceDir, EVAL_SAMPLE_FILES.eval),
+        dataset: path.join(sourceDir, EVAL_SAMPLE_FILES.dataset),
+    };
+    const texts = { pipeline: "", eval: "", dataset: "" };
+    for (const key of Object.keys(sources)) {
+        try {
+            texts[key] = await readFile(sources[key], "utf8");
+        }
+        catch {
+            throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", `Sample file ${EVAL_SAMPLE_FILES[key]} not found in ${sourceDir}`);
+        }
+    }
+    const manifest = await manifestForWrite(projectRoot);
+    const dirs = {
+        pipeline: await resolveWriteDir(projectRoot, manifest, "pipelines"),
+        eval: await resolveWriteDir(projectRoot, manifest, "evals"),
+        dataset: await resolveWriteDir(projectRoot, manifest, "datasets"),
+    };
+    const targets = {
+        pipeline: path.join(dirs.pipeline.absDir, EVAL_SAMPLE_FILES.pipeline),
+        eval: path.join(dirs.eval.absDir, EVAL_SAMPLE_FILES.eval),
+        dataset: path.join(dirs.dataset.absDir, EVAL_SAMPLE_FILES.dataset),
+    };
+    const taken = (await findEvalFile(projectRoot, EVAL_SAMPLE_ID, manifest)) !== undefined ||
+        (await exists(targets.pipeline)) ||
+        (await exists(targets.eval)) ||
+        (await exists(targets.dataset));
+    if (taken)
+        throw new EvalsWriteError("SAMPLE_EXISTS", "The sample eval is already installed");
+    const evalText = rewriteEvalPaths(texts.eval, relativeRef(dirs.eval.absDir, targets.pipeline), relativeRef(dirs.eval.absDir, targets.dataset));
+    const projectManifest = await loadStageflowManifestOutcome(projectRoot);
+    const hasDefaultModel = projectManifest.ok && projectManifest.value.manifest.model !== undefined;
+    const pipelineText = hasDefaultModel
+        ? texts.pipeline
+        : texts.pipeline.replace(/^id: .*\n/m, (line) => `${line}model: ${peekDefaultModel()}\n`);
+    await writeFileAtomic(targets.pipeline, pipelineText);
+    await writeFileAtomic(targets.dataset, texts.dataset);
+    await writeFileAtomic(targets.eval, evalText);
+    const scanKeys = {
+        pipeline: { kind: "pipelines", key: "pipelines" },
+        eval: { kind: "evals", key: "evals" },
+        dataset: { kind: "datasets", key: "datasets" },
+    };
+    for (const name of Object.keys(dirs)) {
+        const { kind, key } = scanKeys[name];
+        if (explicitRoots(manifest, kind) !== undefined) {
+            await ensureCatalogScanRoot(projectRoot, dirs[name].relDir, [key]);
+        }
+    }
+    return { evalId: EVAL_SAMPLE_ID };
+}
+async function agentExists(agents, projectRoot) {
+    try {
+        await agents.get(projectRoot, AGENT_EVAL_SAMPLE_AGENT_ID);
+        return true;
+    }
+    catch (err) {
+        if (isAgentError(err) && err.code === "agent.not_found")
+            return false;
+        throw err;
+    }
+}
+async function installAgentEvalSample(input) {
+    const { projectRoot, agents } = input;
+    const sourceDir = input.sourceDir ?? resolveEvalSampleDir(undefined, AGENT_EVAL_SAMPLE_FILES.eval);
+    if (sourceDir === undefined) {
+        throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", "The packaged sample files were not found");
+    }
+    if (agents === undefined) {
+        throw new EvalsWriteError("SAMPLE_AGENTS_UNAVAILABLE", "Agents are not available on this Host");
+    }
+    const texts = { agent: "", eval: "", dataset: "" };
+    for (const key of Object.keys(AGENT_EVAL_SAMPLE_FILES)) {
+        try {
+            texts[key] = await readFile(path.join(sourceDir, AGENT_EVAL_SAMPLE_FILES[key]), "utf8");
+        }
+        catch {
+            throw new EvalsWriteError("SAMPLE_SOURCE_MISSING", `Sample file ${AGENT_EVAL_SAMPLE_FILES[key]} not found in ${sourceDir}`);
+        }
+    }
+    const projectManifest = await loadStageflowManifestOutcome(projectRoot);
+    if (!projectManifest.ok || projectManifest.value.manifest.model === undefined) {
+        throw new EvalsWriteError("SAMPLE_NO_DEFAULT_MODEL", "The agent sample needs a default model. Set model in stageflow.yaml, then install the sample again.");
+    }
+    const manifest = await manifestForWrite(projectRoot);
+    const dirs = {
+        eval: await resolveWriteDir(projectRoot, manifest, "evals"),
+        dataset: await resolveWriteDir(projectRoot, manifest, "datasets"),
+    };
+    const targets = {
+        eval: path.join(dirs.eval.absDir, AGENT_EVAL_SAMPLE_FILES.eval),
+        dataset: path.join(dirs.dataset.absDir, AGENT_EVAL_SAMPLE_FILES.dataset),
+    };
+    const taken = (await findEvalFile(projectRoot, AGENT_EVAL_SAMPLE_ID, manifest)) !== undefined ||
+        (await exists(targets.eval)) ||
+        (await exists(targets.dataset)) ||
+        (await agentExists(agents, projectRoot));
+    if (taken)
+        throw new EvalsWriteError("SAMPLE_EXISTS", "The sample eval is already installed");
+    const agentDefinition = parseYaml(texts.agent);
+    const evalText = rewriteEvalPaths(texts.eval, undefined, relativeRef(dirs.eval.absDir, targets.dataset));
+    await agents.create(projectRoot, { ...agentDefinition, web_chat: { enabled: false } }, { source: "api" });
+    await agents.setEnabled(projectRoot, AGENT_EVAL_SAMPLE_AGENT_ID, true, { source: "api" });
+    await writeFileAtomic(targets.dataset, texts.dataset);
+    await writeFileAtomic(targets.eval, evalText);
+    const scanKeys = { eval: "evals", dataset: "datasets" };
+    for (const name of Object.keys(dirs)) {
+        if (explicitRoots(manifest, scanKeys[name]) !== undefined) {
+            await ensureCatalogScanRoot(projectRoot, dirs[name].relDir, [scanKeys[name]]);
+        }
+    }
+    return { evalId: AGENT_EVAL_SAMPLE_ID };
+}

package/dist/config/loadDataset.d.ts

@@ -0,0 +1,16 @@
+import type { DatasetFormat, DatasetLimits, DatasetLoadResult, ParseDatasetResult } from "./datasetTypes.js";
+export declare const DATASET_LIMITS: DatasetLimits;
+export declare const DATASET_MIN_CASES = 5;
+export type ParseDatasetOptions = {
+    limits?: Partial<DatasetLimits>;
+    path?: string;
+};
+export declare function datasetFormatFromPath(filePath: string): DatasetFormat | undefined;
+export declare function parseDatasetText(text: string, format: DatasetFormat, options?: ParseDatasetOptions & {
+    csvDelimiter?: string;
+}): ParseDatasetResult;
+export declare function parseDatasetCases(rawCases: readonly unknown[], options?: ParseDatasetOptions): ParseDatasetResult;
+export declare function loadDatasetFile(absPath: string, options?: ParseDatasetOptions & {
+    root?: string;
+    csvDelimiter?: string;
+}): Promise<DatasetLoadResult>;