pi-extensible-workflows 0.3.2 → 1.0.1

package/src/ambient-workflow-evals.ts

@@ -1,12 +1,13 @@
 import { spawn, type ChildProcess } from "node:child_process";
 import { execFileSync } from "node:child_process";
 import { randomUUID } from "node:crypto";
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join, relative } from "node:path";
 import { fileURLToPath } from "node:url";
 import { CAPTURE_IDENTITY } from "./eval-capture-extension.js";
-import { extractCapturedWorkflows, extractParentOracleFile } from "./workflow-evals.js";
+import { isObject } from "./index.js";
+import { extractCapturedWorkflows, extractParentOracleFile, findSessionFile } from "./workflow-evals.js";
 
 export const AMBIENT_OPT_IN = "PI_WORKFLOW_EVAL_AMBIENT";
 export const AMBIENT_CAPTURE_NOTE = "Ambient Tier D uses the explicit capture extension. Pi 0.80.6 orders CLI extensions before discovered extensions and the first tool registration wins, so ambient tools and skills remain available while workflow execution stays capture-only.";
@@ -327,26 +328,13 @@ function ambientAgentDir(environment: NodeJS.ProcessEnv): string {
   return environment.PI_CODING_AGENT_DIR ?? process.env.PI_CODING_AGENT_DIR ?? join(homedir(), ".pi", "agent");
 }
 
-function isObject(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
-
-function findSessionFile(directory: string, sessionId: string): string | undefined {
-  if (!existsSync(directory)) return undefined;
-  for (const entry of readdirSync(directory, { withFileTypes: true })) {
-    const path = join(directory, entry.name);
-    if (entry.isDirectory()) { const found = findSessionFile(path, sessionId); if (found) return found; }
-    else if (entry.name.endsWith(".jsonl")) {
-      try { const header = JSON.parse(readFileSync(path, "utf8").split("\n")[0] ?? "{}") as unknown; if (isObject(header) && header.id === sessionId) return path; } catch { /* Ignore incomplete sessions. */ }
-    }
-  }
-  return undefined;
-}
-
-function emptyResult(candidate: AmbientEvalCase, repository: AmbientFixtureRepository, worktree: AmbientCaseWorktree, environment: NodeJS.ProcessEnv, error: string): AmbientCaseResult {
+type AmbientCaseResultDraft = Omit<AmbientCaseResult, "manifest"> & { manifest: Omit<AmbientManifest, "gitStatusAfter"> };
+function emptyResult(candidate: AmbientEvalCase, repository: AmbientFixtureRepository, worktree: AmbientCaseWorktree, environment: NodeJS.ProcessEnv, error: string): AmbientCaseResultDraft {
   const accounting = emptyAccounting();
   return {
     id: candidate.id, status: "failed", workflows: [], accounting, accountingTrustworthy: false, diagnostics: [], errors: [error],
     manifest: {
-      invocationMode: AMBIENT_INVOCATION_MODE, fixtureRoot: repository.fixtureRoot, worktreePath: worktree.path, ambientAgentDir: ambientAgentDir(environment), fixtureFileList: worktree.fixtureFiles, gitStatusBefore: worktree.gitStatusBefore, gitStatusAfter: worktree.gitStatusBefore, parentToolSequence: [], skillReads: [], workflowCalls: [], workflowCallCount: 0, tokenCost: accounting,
+      invocationMode: AMBIENT_INVOCATION_MODE, fixtureRoot: repository.fixtureRoot, worktreePath: worktree.path, ambientAgentDir: ambientAgentDir(environment), fixtureFileList: worktree.fixtureFiles, gitStatusBefore: worktree.gitStatusBefore, parentToolSequence: [], skillReads: [], workflowCalls: [], workflowCallCount: 0, tokenCost: accounting,
       cleanup: { processExited: false, processGroupTerminated: false, worktreeRemoved: false, fixtureRepoRemoved: false, tempRootRemoved: false, captureIdentityVerified: false, realWorkflowAgentsLaunched: 0 },
     },
   };
@@ -356,12 +344,13 @@ async function runAmbientCase(repository: AmbientFixtureRepository, candidate: A
   const worktree = createAmbientCaseWorktree(repository, candidate.id);
   const sessionId = randomUUID();
   const sessionDir = join(repository.root, "sessions", candidate.id);
-  let result: AmbientCaseResult | undefined;
-  let gitStatusAfter: readonly string[] = worktree.gitStatusBefore;
+  let result: AmbientCaseResultDraft | undefined;
+  let failure: string | undefined;
+  let gitStatusAfter: readonly string[];
   try {
     const pi = await runAmbientPiProcess({ worktree: worktree.path, sessionDir, sessionId, prompt: candidate.prompt, provider, model, ...(thinking ? { thinking } : {}), ...(piCommand ? { piCommand } : {}), timeoutMs: candidate.timeoutMs, maxCost: candidate.maxCost, environment });
     const sessionFile = findSessionFile(sessionDir, sessionId);
-    if (!sessionFile) result = emptyResult(candidate, repository, worktree, environment, "Ambient parent session was not written.");
+    if (!sessionFile) failure = "Ambient parent session was not written.";
     else {
       const oracle = extractParentOracleFile(sessionFile);
       const workflows = extractCapturedWorkflows(oracle);
@@ -371,24 +360,25 @@ async function runAmbientCase(repository: AmbientFixtureRepository, candidate: A
       result = {
         id: candidate.id, status, workflows, accounting: oracle.usage, accountingTrustworthy: !pi.timedOut && !pi.budgetExceeded && pi.exitCode === 0, diagnostics: [pi.stderr].filter(Boolean), errors,
         manifest: {
-          invocationMode: AMBIENT_INVOCATION_MODE, fixtureRoot: repository.fixtureRoot, worktreePath: worktree.path, ambientAgentDir: ambientAgentDir(environment), fixtureFileList: worktree.fixtureFiles, gitStatusBefore: worktree.gitStatusBefore, gitStatusAfter, parentToolSequence: oracle.parentToolSequence, skillReads: oracle.skillReads, workflowCalls: workflows, workflowCallCount: oracle.workflowCallCount, tokenCost: oracle.usage,
+          invocationMode: AMBIENT_INVOCATION_MODE, fixtureRoot: repository.fixtureRoot, worktreePath: worktree.path, ambientAgentDir: ambientAgentDir(environment), fixtureFileList: worktree.fixtureFiles, gitStatusBefore: worktree.gitStatusBefore, parentToolSequence: oracle.parentToolSequence, skillReads: oracle.skillReads, workflowCalls: workflows, workflowCallCount: oracle.workflowCallCount, tokenCost: oracle.usage,
           cleanup: { processExited: pi.exitCode !== null, processGroupTerminated: pi.processGroupTerminated, worktreeRemoved: false, fixtureRepoRemoved: false, tempRootRemoved: false, captureIdentityVerified, realWorkflowAgentsLaunched: 0 },
         },
       };
     }
   } catch (error) {
-    result = emptyResult(candidate, repository, worktree, environment, error instanceof Error ? error.message : String(error));
+    failure = error instanceof Error ? error.message : String(error);
   } finally {
     try { gitStatusAfter = gitStatus(worktree.path); } catch { gitStatusAfter = ["<unavailable>"]; }
     const worktreeRemoved = removeAmbientCaseWorktree(repository, worktree);
-    if (!result) result = emptyResult(candidate, repository, worktree, environment, "Ambient case produced no result.");
+    if (!result) result = emptyResult(candidate, repository, worktree, environment, failure ?? "Ambient case produced no result.");
     const cleanup = { ...result.manifest.cleanup, worktreeRemoved };
-    result = { ...result, manifest: { ...result.manifest, gitStatusAfter, cleanup } };
-    writeFileSync(join(artifactsDir, `${candidate.id}.json`), `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 });
+    result = { ...result, manifest: { ...result.manifest, cleanup } };
   }
-  return result;
-}
 
+  const finalized: AmbientCaseResult = { ...result, manifest: { ...result.manifest, gitStatusAfter } };
+  writeFileSync(join(artifactsDir, `${candidate.id}.json`), `${JSON.stringify(finalized, null, 2)}\n`, { mode: 0o600 });
+  return finalized;
+}
 export function assertAmbientOptIn(environment: NodeJS.ProcessEnv = process.env): void {
   if (environment[AMBIENT_OPT_IN] !== "1") throw new Error(`Ambient Tier D evals are opt-in. Set ${AMBIENT_OPT_IN}=1 to run them.`);
 }

package/src/doctor.ts

@@ -13,13 +13,17 @@ import {
 import {
   DEFAULT_SETTINGS,
   loadSettings,
-  parseModelReference,
+  resolveAgentResourcePolicy,
+  resolveModelReference,
   parseRoleMarkdown,
   preflight,
   registeredWorkflowDefinitions,
   workflowRoleDirectories,
+  workflowProjectSettingsPath,
   workflowSettingsPath,
   WorkflowError,
+  type AgentResourceExclusions,
+  type AgentResourcePolicy,
   type WorkflowScriptDefinition,
   type WorkflowSettings,
 } from "./index.js";
@@ -36,6 +40,8 @@ export interface DoctorPiState {
   knownModels: readonly string[];
   availableModels: readonly string[];
   extensionErrors: readonly { path?: string; message: string }[];
+  extensions?: readonly string[];
+  skills?: readonly string[];
   workflows: Readonly<Record<string, WorkflowScriptDefinition>>;
 }
 export interface DoctorReport {
@@ -47,6 +53,7 @@ export interface DoctorReport {
   activeTools: readonly string[];
   roles: readonly DoctorRole[];
   workflows: readonly DoctorWorkflow[];
+  resourcePolicy: AgentResourcePolicy;
   diagnostics: readonly DoctorDiagnostic[];
 }
 export interface DoctorOptions {
@@ -107,7 +114,7 @@ async function discoverPi(cwd: string, agentDir: string): Promise<DoctorPiState>
       agentDir,
       settingsManager,
       modelRuntime,
-      resourceLoaderOptions: { noSkills: true, noPromptTemplates: true, noThemes: true, noContextFiles: true },
+      resourceLoaderOptions: { noPromptTemplates: true, noThemes: true, noContextFiles: true },
       resourceLoaderReloadOptions: { resolveProjectTrust: async () => trusted },
     });
     const allModels = services.modelRuntime.getModels();
@@ -117,11 +124,14 @@ async function discoverPi(cwd: string, agentDir: string): Promise<DoctorPiState>
     const { session } = await createAgentSessionFromServices({ services, sessionManager: SessionManager.inMemory(), model });
     const activeTools = session.agent.state.tools.map(({ name }) => name).filter((name) => name !== "workflow" && name !== "workflow_respond" && name !== "workflow_catalog");
     const extensions = services.resourceLoader.getExtensions();
+    const skills = services.resourceLoader.getSkills().skills;
     return {
       trust: { required, trusted, source },
       activeTools,
       knownModels: allModels.map(({ provider, id }) => `${provider}/${id}`),
       availableModels: availableModels.map(({ provider, id }) => `${provider}/${id}`),
+      extensions: extensions.extensions.map(({ resolvedPath }) => resolvedPath),
+      skills: skills.map(({ name }) => name),
       extensionErrors: [
         ...extensions.errors.map(({ path, error }) => ({ path, message: error })),
         ...services.diagnostics.filter(({ type }) => type === "error").map(({ message }) => ({ message })),
@@ -147,10 +157,13 @@ function roleFilesFrom(dirs: readonly string[]): string[] {
 function diagnostic(severity: DoctorSeverity, code: string, message: string, source?: string, hint?: string): DoctorDiagnostic {
   return { severity, code, message, ...(source ? { source } : {}), ...(hint ? { hint } : {}) };
 }
-
-function validateModel(value: string, known: ReadonlySet<string>, available: ReadonlySet<string>, source: string, diagnostics: DoctorDiagnostic[]): void {
+function emptyResourcePolicy(globalSettingsPath: string, cwd: string, projectTrusted: boolean): AgentResourcePolicy {
+  const empty: AgentResourceExclusions = { skills: [], extensions: [] };
+  return { globalSettingsPath, projectSettingsPath: workflowProjectSettingsPath(cwd), projectTrusted, global: empty, project: empty, effective: empty, unmatchedSkills: [], unmatchedExtensions: [] };
+}
+function validateModel(value: string, known: ReadonlySet<string>, available: ReadonlySet<string>, source: string, diagnostics: DoctorDiagnostic[], aliases: Readonly<Record<string, string>>, settingsPath: string): void {
   try {
-    const parsed = parseModelReference(value);
+    const parsed = resolveModelReference(value, aliases, known, settingsPath);
     const name = `${parsed.provider}/${parsed.model}`;
     if (!known.has(name) || !available.has(name)) diagnostics.push(diagnostic("warning", "MODEL_UNAVAILABLE", `Model is valid-shaped but unavailable: ${name}`, source));
   } catch (error) {
@@ -158,9 +171,9 @@ function validateModel(value: string, known: ReadonlySet<string>, available: Rea
   }
 }
 
-function inspectRole(path: string, activeTools: ReadonlySet<string>, knownModels: ReadonlySet<string>, availableModels: ReadonlySet<string>, diagnostics: DoctorDiagnostic[]): AgentDefinition | undefined {
+function inspectRole(path: string, activeTools: ReadonlySet<string>, knownModels: ReadonlySet<string>, availableModels: ReadonlySet<string>, diagnostics: DoctorDiagnostic[], aliases: Readonly<Record<string, string>>, settingsPath: string): AgentDefinition | undefined {
   let definition: AgentDefinition;
-  try { definition = parseRoleMarkdown(readFileSync(path, "utf8"), true); }
+  try { definition = parseRoleMarkdown(readFileSync(path, "utf8"), true, path); }
   catch (error) {
     diagnostics.push(diagnostic("error", "ROLE_FRONTMATTER", (error as Error).message, path, "Fix the role YAML frontmatter."));
     return undefined;
@@ -169,15 +182,19 @@ function inspectRole(path: string, activeTools: ReadonlySet<string>, knownModels
   if (body.trim() === "") diagnostics.push(diagnostic("warning", "ROLE_BODY_EMPTY", "Role body is empty", path));
   if (Buffer.byteLength(body) > 50 * 1024) diagnostics.push(diagnostic("warning", "ROLE_BODY_LARGE", "Role body exceeds 50KB", path));
   if (/{{\s*[^{}]+\s*}}/.test(body)) diagnostics.push(diagnostic("warning", "ROLE_PLACEHOLDER", "Role body contains an unsupported placeholder-looking token", path));
-  if (definition.model) validateModel(definition.model, knownModels, availableModels, path, diagnostics);
+  if (definition.model) validateModel(definition.model, knownModels, availableModels, path, diagnostics, aliases, settingsPath);
   for (const tool of definition.tools ?? []) if (!activeTools.has(tool)) diagnostics.push(diagnostic("error", "ROLE_TOOL_INACTIVE", `Tool is unknown or inactive: ${tool}`, path, "Use a tool listed under Active tools or enable its Pi extension."));
   return definition;
 }
 
-class DoctorModelSet extends Set<string> {
-  override has(value: string): boolean { parseModelReference(value); return true; }
+function matchResourcePolicy(policy: AgentResourcePolicy, pi: DoctorPiState): AgentResourcePolicy {
+  const extensions = new Set((pi.extensions ?? []).map(canonical));
+  const skills = new Set(pi.skills ?? []);
+  return { ...policy, unmatchedSkills: policy.effective.skills.filter((name) => !skills.has(name)), unmatchedExtensions: policy.effective.extensions.filter((path) => !extensions.has(canonical(path))) };
+}
+function resourcePolicySource(policy: AgentResourcePolicy, kind: keyof AgentResourceExclusions, selector: string): string {
+  return policy.project[kind].includes(selector) && !policy.global[kind].includes(selector) ? policy.projectSettingsPath : policy.globalSettingsPath;
 }
-
 export async function doctor(options: DoctorOptions = {}): Promise<DoctorReport> {
   const cwd = canonical(options.cwd ?? process.cwd());
   const agentDir = canonical(options.agentDir ?? getAgentDir());
@@ -196,10 +213,21 @@ export async function doctor(options: DoctorOptions = {}): Promise<DoctorReport>
   if (options.activeTools) pi = { ...pi, activeTools: options.activeTools.filter((tool) => tool !== "workflow" && tool !== "workflow_respond" && tool !== "workflow_catalog") };
   if (pi.trust.required && !pi.trust.trusted) diagnostics.push(diagnostic("warning", "PROJECT_UNTRUSTED", "Pi project resources are inactive because the project is not trusted", cwd, "Open this project in Pi, choose Trust, then rerun doctor."));
   for (const error of pi.extensionErrors) diagnostics.push(diagnostic("error", "EXTENSION_LOAD", error.message, error.path, "Fix or disable the failing Pi extension."));
+  let resourcePolicy: AgentResourcePolicy;
+  try {
+    resourcePolicy = matchResourcePolicy(resolveAgentResourcePolicy(cwd, pi.trust.trusted, settingsPath), pi);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (!diagnostics.some(({ code, source }) => code === "SETTINGS_INVALID" && source === settingsPath)) diagnostics.push(diagnostic("error", "SETTINGS_INVALID", message, message.includes(".json") ? message.match(/(?:settings: )?([^ )]+\.json)/)?.[1] : undefined, "Fix or remove the invalid workflow settings file."));
+    resourcePolicy = emptyResourcePolicy(settingsPath, cwd, pi.trust.trusted);
+  }
+  for (const skill of resourcePolicy.unmatchedSkills) diagnostics.push(diagnostic("warning", "AGENT_RESOURCE_UNMATCHED", `Disabled skill selector currently matches no discovered skill: ${skill}`, `${resourcePolicySource(resourcePolicy, "skills", skill)}.disabledAgentResources.skills`));
+  for (const extension of resourcePolicy.unmatchedExtensions) diagnostics.push(diagnostic("warning", "AGENT_RESOURCE_UNMATCHED", `Disabled extension selector currently matches no discovered extension source: ${extension}`, `${resourcePolicySource(resourcePolicy, "extensions", extension)}.disabledAgentResources.extensions`));
 
   const activeTools = new Set(pi.activeTools);
   const knownModels = new Set(pi.knownModels);
   const availableModels = new Set(pi.availableModels);
+  const aliases = settings.modelAliases ?? {};
   const roles: DoctorRole[] = [];
   const definitions = new Map<string, AgentDefinition>();
   const globalPaths = new Map<string, string>();
@@ -208,7 +236,7 @@ export async function doctor(options: DoctorOptions = {}): Promise<DoctorReport>
     const name = basename(path, ".md");
     roles.push({ name, path, scope: "global", active: true });
     globalPaths.set(name, path);
-    const definition = inspectRole(path, activeTools, knownModels, availableModels, diagnostics);
+    const definition = inspectRole(path, activeTools, knownModels, availableModels, diagnostics, aliases, settingsPath);
     if (definition) definitions.set(name, definition);
   }
   for (const path of roleFilesFrom([join(cwd, ".pi", "pi-extensible-workflows", "roles")])) {
@@ -221,7 +249,7 @@ export async function doctor(options: DoctorOptions = {}): Promise<DoctorReport>
       const global = roles.find((role) => role.path === globalPath);
       if (global) { global.active = false; global.overriddenBy = path; }
     }
-    const definition = inspectRole(path, activeTools, knownModels, availableModels, diagnostics);
+    const definition = inspectRole(path, activeTools, knownModels, availableModels, diagnostics, aliases, settingsPath);
     if (definition) definitions.set(name, definition); else definitions.delete(name);
   }
 
@@ -230,9 +258,13 @@ export async function doctor(options: DoctorOptions = {}): Promise<DoctorReport>
     let valid = true;
     try {
       const checked = preflight(workflow.script, {
-        models: new DoctorModelSet(pi.knownModels),
+        models: new Set(pi.knownModels),
         tools: activeTools,
         agentTypes: new Set(definitions.keys()),
+        modelAliases: aliases,
+        knownModels,
+        settingsPath,
+        skipModelAvailability: true,
       }, [], { name, description: workflow.description });
       for (const model of checked.referenced.models) if (!knownModels.has(model) || !availableModels.has(model)) diagnostics.push(diagnostic("warning", "MODEL_UNAVAILABLE", `Model is valid-shaped but unavailable: ${model}`, name));
     } catch (error) {
@@ -246,7 +278,7 @@ export async function doctor(options: DoctorOptions = {}): Promise<DoctorReport>
   const severityOrder: Record<DoctorSeverity, number> = { error: 0, warning: 1 };
   diagnostics.sort((left, right) => severityOrder[left.severity] - severityOrder[right.severity] || (left.source ?? "").localeCompare(right.source ?? "") || left.code.localeCompare(right.code) || left.message.localeCompare(right.message));
   roles.sort((left, right) => left.name.localeCompare(right.name) || left.scope.localeCompare(right.scope));
-  return { cwd, agentDir, settingsPath, settings, trust: pi.trust, activeTools: [...activeTools].sort(), roles, workflows, diagnostics };
+  return { cwd, agentDir, settingsPath, settings, trust: pi.trust, activeTools: [...activeTools].sort(), roles, workflows, resourcePolicy, diagnostics };
 }
 
 function count(report: DoctorReport, severity: DoctorSeverity): number { return report.diagnostics.filter((item) => item.severity === severity).length; }
@@ -261,11 +293,23 @@ export function formatDoctorReport(report: DoctorReport): string {
     `- CWD: \`${report.cwd}\``,
     `- Agent dir: \`${report.agentDir}\``,
     `- Workflow settings: \`${report.settingsPath}\``,
-    `- Limits: concurrency=${String(report.settings.concurrency)}, maxAgentLaunches=${String(report.settings.maxAgentLaunches)}`,
+    `- Limits: concurrency=${String(report.settings.concurrency)}`,
     "",
     "## Trust/resources",
     `- [${report.trust.trusted ? "ok" : "warning"}] ${report.trust.source}`,
     "",
+    "## Agent resource exclusions",
+    `- Global settings: \`${report.resourcePolicy.globalSettingsPath}\``,
+    `- Global skills: ${report.resourcePolicy.global.skills.join(", ") || "(none)"}`,
+    `- Global extensions: ${report.resourcePolicy.global.extensions.join(", ") || "(none)"}`,
+    `- Project settings: \`${report.resourcePolicy.projectSettingsPath}\` (${report.resourcePolicy.projectTrusted ? "trusted" : "ignored: project untrusted"})`,
+    `- Project skills: ${report.resourcePolicy.project.skills.join(", ") || "(none)"}`,
+    `- Project extensions: ${report.resourcePolicy.project.extensions.join(", ") || "(none)"}`,
+    `- Effective skills: ${report.resourcePolicy.effective.skills.join(", ") || "(none)"}`,
+    `- Effective extensions: ${report.resourcePolicy.effective.extensions.join(", ") || "(none)"}`,
+    `- Unmatched skills: ${report.resourcePolicy.unmatchedSkills.join(", ") || "(none)"}`,
+    `- Unmatched extensions: ${report.resourcePolicy.unmatchedExtensions.join(", ") || "(none)"}`,
+    "",
     "## Active tools",
     ...(report.activeTools.length ? report.activeTools.map((tool) => `- \`${tool}\``) : ["- None resolved"]),
     "",