pi-extensible-workflows 3.0.0 → 3.2.0

package/src/validation.ts

@@ -8,9 +8,10 @@ import { Script } from "node:vm";
 import { Value } from "typebox/value";
 import { getAgentDir, parseFrontmatter } from "@earendil-works/pi-coding-agent";
 import { WorkflowError } from "./types.js";
-import type { AgentDefinition, AgentResourceExclusions, AgentResourcePolicy, CheckpointInput, JsonSchema, JsonValue, LaunchSnapshot, PreflightCapabilities, PreflightResult, ShellOptions, StaticWorkflowCall, StaticWorkflowExecution, StaticWorkflowScope, ValidatedWorkflowLaunch, WorkflowCallKind, WorkflowErrorCode, WorkflowMetadata, WorkflowSettings, WorkflowValidationContext, WorkflowValidationParameters } from "./types.js";
+import type { AgentDefinition, AgentResourceExclusions, AgentResourcePolicy, CheckpointInput, JsonSchema, JsonValue, LaunchSnapshot, PreflightCapabilities, PreflightResult, ShellOptions, StaticWorkflowCall, StaticWorkflowExecution, StaticWorkflowScope, ValidatedWorkflowLaunch, WorkflowCallKind, WorkflowErrorCode, WorkflowExtensionMetadata, WorkflowMetadata, WorkflowRoleDirectoryRegistration, WorkflowSettings, WorkflowSettingsOverrides, WorkflowSettingsResolution, WorkflowSettingsSources, WorkflowValidationContext, WorkflowValidationParameters } from "./types.js";
 import type { WorkflowRegistryApi } from "./registry.js";
-import { deepFreeze, errorText, fail, jsonValue, mergeAgentResourceExclusions, modelAliasName, modelCapability, object, parseThinking, positiveInteger, resolveModelReference, unknownModel, validateModelAliases } from "./utils.js";
+import { registeredWorkflowRoleDirectoryRegistrations } from "./registry.js";
+import { annotateModelAliasError, deepFreeze, errorText, fail, jsonValue, modelAliasName, modelCapability, object, parseThinking, positiveInteger, resolveModelReference, unknownModel, validateModelAliases, validateResourcePattern } from "./utils.js";
 import { WORKFLOW_CALL_KINDS } from "./types.js";
 
 export const DEFAULT_SETTINGS: Readonly<WorkflowSettings> = Object.freeze({ concurrency: 8 });
@@ -25,10 +26,20 @@ export function validateCheckpoint(value: unknown): CheckpointInput {
 export function workflowSettingsPath(agentDir = getAgentDir()): string { return join(agentDir, ROLE_DIRECTORY, "settings.json"); }
 export function workflowProjectSettingsPath(cwd: string): string { return join(cwd, ".pi", ROLE_DIRECTORY, "settings.json"); }
 const EMPTY_AGENT_RESOURCE_EXCLUSIONS: AgentResourceExclusions = Object.freeze({ skills: [], extensions: [] });
+function resourcePatternHasMagic(value: string): boolean { return /[*?\x5b\x5d{}()]/.test(value); }
 function normalizedResourcePath(value: string, settingsPath: string): string {
   let expanded = value === "~" ? homedir() : value.startsWith("~/") || value.startsWith("~\\") ? join(homedir(), value.slice(2)) : value;
   if (expanded.startsWith("file://")) expanded = fileURLToPath(expanded);
   const resolved = resolve(dirname(settingsPath), expanded);
+  if (expanded === "**" || expanded.startsWith("**/") || expanded.startsWith("**\\")) return expanded;
+  if (resourcePatternHasMagic(expanded)) {
+    const magicIndex = resolved.search(/[*?\x5b\x5d{}()]/);
+    const separatorIndex = Math.max(resolved.lastIndexOf("/", magicIndex), resolved.lastIndexOf("\\", magicIndex));
+    const rootBoundary = separatorIndex === 0 || (separatorIndex === 2 && /^[A-Za-z]:[\\/]/.test(resolved));
+    const prefix = rootBoundary ? resolved.slice(0, separatorIndex + 1) : separatorIndex >= 0 ? resolved.slice(0, separatorIndex) : resolved;
+    const suffix = rootBoundary ? resolved.slice(separatorIndex + 1) : separatorIndex >= 0 ? resolved.slice(separatorIndex) : "";
+    try { return `${realpathSync(prefix)}${suffix}`; } catch { return resolved; }
+  }
   try { return realpathSync(resolved); } catch { return resolved; }
 }
 function validateAgentResourceExclusions(value: unknown, settingsPath: string, errorCode: "INVALID_SETTINGS" | "INVALID_METADATA" = "INVALID_SETTINGS"): AgentResourceExclusions | undefined {
@@ -41,41 +52,74 @@ function validateAgentResourceExclusions(value: unknown, settingsPath: string, e
     const entries = value[kind];
     if (entries === undefined) continue;
     if (!Array.isArray(entries)) fail(errorCode, `${base}.${kind} must be an array`);
-    const seen = new Set<string>();
     for (const [index, entry] of entries.entries()) {
       if (typeof entry !== "string" || !entry.trim()) fail(errorCode, `${base}.${kind}[${String(index)}] must be a non-empty string`);
       let selector = entry.trim();
       if (kind === "extensions") {
-        try { selector = normalizedResourcePath(selector, settingsPath); } catch (error) { fail(errorCode, `${base}.${kind}[${String(index)}] must be a valid path: ${errorText(error)}`); }
+        const negated = selector.startsWith("!");
+        const body = negated ? selector.slice(1) : selector;
+        if (!body) fail(errorCode, `${base}.${kind}[${String(index)}] must be a valid minimatch pattern: Empty minimatch pattern ${JSON.stringify(selector)}`);
+        try { selector = `${negated ? "!" : ""}${normalizedResourcePath(body, settingsPath)}`; } catch (error) { fail(errorCode, `${base}.${kind}[${String(index)}] must be a valid path: ${errorText(error)}`); }
       }
-      if (!seen.has(selector)) { seen.add(selector); normalized[kind].push(selector); }
+      try { validateResourcePattern(selector); } catch (error) { fail(errorCode, `${base}.${kind}[${String(index)}] must be a valid minimatch pattern: ${errorText(error)}`); }
+      normalized[kind].push(selector);
     }
   }
   return Object.freeze({ skills: Object.freeze(normalized.skills), extensions: Object.freeze(normalized.extensions) });
 }
-export function loadSettings(path = workflowSettingsPath()): Readonly<WorkflowSettings> {
+function parseSettings(path: string, partial: false): Readonly<WorkflowSettings>;
+function parseSettings(path: string, partial: true): Readonly<WorkflowSettingsOverrides>;
+function parseSettings(path: string, partial: boolean): Readonly<WorkflowSettings | WorkflowSettingsOverrides> {
   let parsed: unknown;
   try { parsed = JSON.parse(readFileSync(path, "utf8")); }
   catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return DEFAULT_SETTINGS;
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return partial ? Object.freeze({}) : DEFAULT_SETTINGS;
     fail("CONFIG_ERROR", `Invalid workflow settings JSON at ${path}: ${errorText(error)}`);
   }
   if (!object(parsed)) fail("INVALID_SETTINGS", `Workflow settings at ${path} must be an object`);
   const allowed = new Set(["concurrency", "modelAliases", "disabledAgentResources"]);
   const unknown = Object.keys(parsed).find((key) => !allowed.has(key));
   if (unknown) fail("INVALID_SETTINGS", `Unknown workflow setting at ${path}: ${unknown}`);
-  const concurrency = parsed.concurrency === undefined ? DEFAULT_SETTINGS.concurrency : parsed.concurrency;
-  if (!positiveInteger(concurrency) || concurrency > 16) fail("INVALID_SETTINGS", `${path}.concurrency must be an integer from 1 to 16`);
+  const concurrency = parsed.concurrency === undefined ? (partial ? undefined : DEFAULT_SETTINGS.concurrency) : parsed.concurrency;
+  if (concurrency !== undefined && (!positiveInteger(concurrency) || concurrency > 16)) fail("INVALID_SETTINGS", `${path}.concurrency must be an integer from 1 to 16`);
   const modelAliases = parsed.modelAliases === undefined ? undefined : validateModelAliases(parsed.modelAliases, path);
   const disabledAgentResources = validateAgentResourceExclusions(parsed.disabledAgentResources, path);
-  return Object.freeze({ concurrency, ...(modelAliases ? { modelAliases } : {}), ...(disabledAgentResources ? { disabledAgentResources } : {}) });
+  return Object.freeze({ ...(concurrency === undefined ? {} : { concurrency }), ...(modelAliases === undefined ? {} : { modelAliases }), ...(disabledAgentResources === undefined ? {} : { disabledAgentResources }) });
 }
-export function resolveAgentResourcePolicy(cwd: string, projectTrusted: boolean, globalSettingsPath = workflowSettingsPath()): AgentResourcePolicy {
+export function loadSettings(path = workflowSettingsPath()): Readonly<WorkflowSettings> { return parseSettings(path, false); }
+export function loadSettingsOverrides(path: string): Readonly<WorkflowSettingsOverrides> { return parseSettings(path, true); }
+export function resolveWorkflowSettings(cwd: string, projectTrusted: boolean, globalSettingsPath = workflowSettingsPath()): WorkflowSettingsResolution {
   const projectSettingsPath = workflowProjectSettingsPath(cwd);
-  const global = loadSettings(globalSettingsPath).disabledAgentResources ?? EMPTY_AGENT_RESOURCE_EXCLUSIONS;
-  const project = projectTrusted ? loadSettings(projectSettingsPath).disabledAgentResources ?? EMPTY_AGENT_RESOURCE_EXCLUSIONS : EMPTY_AGENT_RESOURCE_EXCLUSIONS;
-  const effective = mergeAgentResourceExclusions(global, project);
-  return { globalSettingsPath, projectSettingsPath, projectTrusted, global, project, effective, unmatchedSkills: [], unmatchedExtensions: [] };
+  const global = loadSettings(globalSettingsPath);
+  const project: Readonly<WorkflowSettingsOverrides> = projectTrusted ? loadSettingsOverrides(projectSettingsPath) : Object.freeze({});
+  const projectHas = (key: keyof WorkflowSettingsOverrides): boolean => Object.prototype.hasOwnProperty.call(project, key);
+  const sources: WorkflowSettingsSources = {
+    concurrency: projectHas("concurrency") ? projectSettingsPath : globalSettingsPath,
+    modelAliases: projectHas("modelAliases") ? projectSettingsPath : globalSettingsPath,
+    disabledAgentResources: projectHas("disabledAgentResources") ? projectSettingsPath : globalSettingsPath,
+  };
+  const effective = Object.freeze({
+    concurrency: project.concurrency ?? global.concurrency,
+    ...(projectHas("modelAliases") ? { modelAliases: project.modelAliases } : global.modelAliases === undefined ? {} : { modelAliases: global.modelAliases }),
+    ...(projectHas("disabledAgentResources") ? { disabledAgentResources: project.disabledAgentResources } : global.disabledAgentResources === undefined ? {} : { disabledAgentResources: global.disabledAgentResources }),
+  });
+  return { globalSettingsPath, projectSettingsPath, projectTrusted, global, project, effective, sources };
+}
+export function validateModelAliasAvailability(aliases: Readonly<Record<string, string>>, names: readonly string[], availableModels: ReadonlySet<string>, knownModels: ReadonlySet<string>, settingsPath?: string): void {
+  for (const name of names) {
+    try {
+      const target = modelCapability(name, aliases, knownModels, settingsPath);
+      if (!availableModels.has(target)) unknownModel(name, target, settingsPath);
+    } catch (error) { throw annotateModelAliasError(error, name); }
+  }
+}
+export function resolveAgentResourcePolicy(cwd: string, projectTrusted: boolean, globalSettingsPath = workflowSettingsPath()): AgentResourcePolicy {
+  const resolved = resolveWorkflowSettings(cwd, projectTrusted, globalSettingsPath);
+  const empty = EMPTY_AGENT_RESOURCE_EXCLUSIONS;
+  const global = resolved.global.disabledAgentResources ?? empty;
+  const project = resolved.project.disabledAgentResources ?? empty;
+  const effective = resolved.effective.disabledAgentResources ?? empty;
+  return { globalSettingsPath: resolved.globalSettingsPath, projectSettingsPath: resolved.projectSettingsPath, projectTrusted, global, project, effective, unmatchedSkills: [], unmatchedExtensions: [] };
 }
 export function saveModelAliases(path = workflowSettingsPath(), aliases: Readonly<Record<string, string>> = {}): void {
   const normalized = validateModelAliases(aliases, path);
@@ -135,23 +179,51 @@ function projectRoleDirectories(root: string): readonly string[] {
   return [join(root, ROLE_DIRECTORY, "roles")];
 }
 
-function readAgentDefinitions(dir: string): Record<string, AgentDefinition> {
-  try {
-    return Object.fromEntries(readdirSync(dir, { withFileTypes: true })
-      .filter((entry) => entry.isFile() && extname(entry.name) === ".md")
-      .map((entry) => { const path = join(dir, entry.name); return [basename(entry.name, ".md"), parseRoleMarkdown(readFileSync(path, "utf8"), true, path)]; }));
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") return {};
-    throw error;
-  }
+type RoleDirectorySource = { path: string; extension?: WorkflowExtensionMetadata };
+export type WorkflowRoleDirectoryInput = string | WorkflowRoleDirectoryRegistration;
+type RoleFile = { name: string; path: string; source: RoleDirectorySource };
+function roleDirectorySources(dirs: readonly WorkflowRoleDirectoryInput[]): RoleDirectorySource[] {
+  const seen = new Set<string>();
+  return dirs.flatMap((value) => {
+    const source = typeof value === "string" ? { path: value } : { path: value.path, extension: value.extension };
+    if (seen.has(source.path)) return [];
+    seen.add(source.path);
+    return [source];
+  });
 }
-
-function readRoleDefinitions(dirs: readonly string[]): Record<string, AgentDefinition> {
-  return Object.fromEntries(dirs.flatMap((dir) => Object.entries(readAgentDefinitions(dir))));
+function roleDirectoryLabel(source: RoleDirectorySource, extension = true): string {
+  return source.extension ? `Extension "${source.extension.headline}" (${source.extension.version})` : extension ? "Registered workflow extension" : "Standard workflow";
+}
+function scanRoleFiles(dirs: readonly WorkflowRoleDirectoryInput[], extension: boolean): RoleFile[] {
+  const files: RoleFile[] = [];
+  for (const source of roleDirectorySources(dirs)) {
+    let entries: import("node:fs").Dirent[];
+    try { entries = readdirSync(source.path, { withFileTypes: true }); }
+    catch (error) {
+      if (!extension && (error as NodeJS.ErrnoException).code === "ENOENT") continue;
+      fail("INVALID_METADATA", `${roleDirectoryLabel(source, extension)} role directory "${source.path}" could not be scanned: ${errorText(error)}`);
+    }
+    for (const entry of entries) if (entry.isFile() && extname(entry.name) === ".md") files.push({ name: basename(entry.name, ".md"), path: join(source.path, entry.name), source });
+  }
+  return files.sort((left, right) => left.name.localeCompare(right.name) || left.path.localeCompare(right.path));
+}
+function readRoleDefinitions(dirs: readonly WorkflowRoleDirectoryInput[], extension = false): Record<string, AgentDefinition> {
+  const files = scanRoleFiles(dirs, extension);
+  if (extension) {
+    const byName = new Map<string, RoleFile[]>();
+    for (const file of files) byName.set(file.name, [...(byName.get(file.name) ?? []), file]);
+    for (const [name, matches] of byName) if (matches.length > 1) fail("INVALID_METADATA", `Duplicate extension role "${name}": ${matches.map(({ path, source }) => `${roleDirectoryLabel(source)} role directory "${source.path}" (${path})`).join("; ")}`);
+  }
+  return Object.fromEntries(files.map(({ name, path, source }) => {
+    try { return [name, parseRoleMarkdown(readFileSync(path, "utf8"), true, path)]; }
+    catch (error) {
+      if (extension) fail("INVALID_METADATA", `${roleDirectoryLabel(source)} role directory "${source.path}" contains invalid role "${name}" at "${path}": ${errorText(error)}`);
+      throw error;
+    }
+  }));
 }
-
-export function loadAgentDefinitions(cwd: string, agentDir = getAgentDir(), projectTrusted = true): Readonly<Record<string, AgentDefinition>> {
-  return deepFreeze({ ...readRoleDefinitions(workflowRoleDirectories(agentDir)), ...(projectTrusted ? readRoleDefinitions(projectRoleDirectories(join(cwd, ".pi"))) : {}) });
+export function loadAgentDefinitions(cwd: string, agentDir = getAgentDir(), projectTrusted = true, extensionRoleDirectories: readonly WorkflowRoleDirectoryInput[] = registeredWorkflowRoleDirectoryRegistrations()): Readonly<Record<string, AgentDefinition>> {
+  return deepFreeze({ ...readRoleDefinitions(extensionRoleDirectories, true), ...readRoleDefinitions(workflowRoleDirectories(agentDir)), ...(projectTrusted ? readRoleDefinitions(projectRoleDirectories(join(cwd, ".pi"))) : {}) });
 }
 function validateRolePolicies(definitions: Readonly<Record<string, AgentDefinition>>, roles: readonly string[], availableModels: ReadonlySet<string>, rootTools: ReadonlySet<string>, aliases: Readonly<Record<string, string>> = {}, knownModels = availableModels, settingsPath?: string): void {
   for (const role of roles) {
@@ -252,6 +324,32 @@ function workflowCallsWithStructure(program: acorn.Program): Array<{ call: Workf
   visit(program, { execution: "sequential", structure: [] });
   return calls.sort((left, right) => left.call.start - right.call.start);
 }
+function memberCall(node: acorn.AnyNode | undefined, objectName: string, propertyName: string): boolean {
+  if (node?.type !== "CallExpression" || node.callee.type !== "MemberExpression" || node.callee.computed || node.callee.object.type !== "Identifier" || node.callee.object.name !== objectName || node.callee.property.type !== "Identifier") return false;
+  return node.callee.property.name === propertyName;
+}
+function mapCallback(node: acorn.AnyNode): acorn.AnyNode | undefined {
+  if (!memberCall(node, "Promise", "all") && !memberCall(node, "Promise", "allSettled")) return undefined;
+  if (node.type !== "CallExpression") return undefined;
+  const source = node.arguments[0];
+  if (source?.type !== "CallExpression" || source.callee.type !== "MemberExpression" || source.callee.computed || source.callee.property.type !== "Identifier" || !["map", "flatMap"].includes(source.callee.property.name)) return undefined;
+  const callback = source.arguments[0];
+  return callback?.type === "ArrowFunctionExpression" || callback?.type === "FunctionExpression" ? callback : undefined;
+}
+function hasUnscopedAgent(node: acorn.AnyNode, scoped = false): boolean {
+  const operation = workflowCallKind(node);
+  if (operation === "agent") return !scoped;
+  const nestedScope = scoped || operation === "parallel" || operation === "pipeline";
+  return astChildren(node).some((child) => hasUnscopedAgent(child, nestedScope));
+}
+function validateObviousConcurrentAgentCalls(program: acorn.Program): void {
+  const visit = (node: acorn.AnyNode): void => {
+    const callback = mapCallback(node);
+    if (callback && hasUnscopedAgent(callback)) fail("INVALID_METADATA", "Promise.all/map agent fan-out cannot prove stable call-site identity; use parallel(...) or pipeline(...)");
+    for (const child of astChildren(node)) visit(child);
+  };
+  visit(program);
+}
 function validateDirectPrimitiveReferences(program: acorn.AnyNode, name: string): void {
   const visit = (node: acorn.AnyNode, parent?: acorn.AnyNode): void => {
     if (node.type === "Identifier" && node.name === name) {
@@ -534,6 +632,7 @@ export function preflight(script: string, capabilities: PreflightCapabilities, s
   validateDirectPrimitiveReferences(program, "shell");
   for (const [index, schema] of schemas.entries()) validateSchema(schema, `schema[${String(index)}]`);
   const calls = workflowCalls(program);
+  validateObviousConcurrentAgentCalls(program);
   const phases = calls.filter((call) => call.callee.name === "phase").map((call) => literalString(call.arguments[0])).filter((phase): phase is string => phase !== undefined);
   for (const call of calls) {
     const operation = call.callee.name;
@@ -590,7 +689,7 @@ export function validateWorkflowLaunchWithRegistry(params: WorkflowValidationPar
   const script = functionName !== undefined && fn ? functionLaunchScript(functionName) : typeof params.script === "string" && params.script.trim() ? params.script : "";
   if (!script) fail("INVALID_SYNTAX", "Provide script or registered function");
   const metadata = validateWorkflowMetadata({ name: workflowName, ...(typeof params.description === "string" ? { description: params.description } : fn?.description ? { description: fn.description } : {}) });
-  const globalAgentDefinitions = loadAgentDefinitions(context.cwd, context.agentDir, false);
+  const globalAgentDefinitions = loadAgentDefinitions(context.cwd, context.agentDir, false, registry && typeof registry.roleDirectoryRegistrations === "function" ? registry.roleDirectoryRegistrations() : registry && typeof registry.roleDirectories === "function" ? registry.roleDirectories() : undefined);
   const projectAgentDefinitions = context.projectTrusted ? readRoleDefinitions(projectRoleDirectories(join(context.cwd, ".pi"))) : {};
   const agentDefinitions = deepFreeze({ ...globalAgentDefinitions, ...projectAgentDefinitions });
   const aliases = context.modelAliases ?? {};

package/src/workflow-artifacts.ts

@@ -0,0 +1,34 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { JsonValue } from "./types.js";
+
+export interface WorkflowArtifact { extension: ".js" | ".json" | ".md"; content: string }
+export type WorkflowTui = { stop(): void; start(): void; requestRender(force?: boolean): void };
+
+export function workflowScriptArtifact(script: string): WorkflowArtifact { return { extension: ".js", content: script }; }
+export function workflowResultArtifact(value: JsonValue): WorkflowArtifact { return typeof value === "string" ? { extension: ".md", content: value } : { extension: ".json", content: `${JSON.stringify(value, null, 2)}\n` }; }
+
+async function spawnWorkflowEditor(command: string, path: string): Promise<number | null> {
+  const [editor, ...editorArgs] = command.split(" ");
+  if (!editor) return null;
+  return new Promise((resolve) => {
+    try {
+      const child = spawn(editor, [...editorArgs, path], { stdio: "inherit", shell: process.platform === "win32" });
+      child.once("error", () => { resolve(null); });
+      child.once("close", (code) => { resolve(code); });
+    } catch { resolve(null); }
+  });
+}
+
+export async function openWorkflowArtifact(tui: WorkflowTui, command: string, artifact: WorkflowArtifact): Promise<number | null> {
+  const directory = await mkdtemp(join(tmpdir(), "pi-workflow-editor-"));
+  const path = join(directory, `artifact${artifact.extension}`);
+  try {
+    await writeFile(path, artifact.content, { encoding: "utf8", mode: 0o600 });
+    tui.stop();
+    try { return await spawnWorkflowEditor(command, path); }
+    finally { tui.start(); tui.requestRender(true); }
+  } finally { await rm(directory, { recursive: true, force: true }); }
+}