runspec-node 0.26.1 → 0.28.0

package/src/testing.ts

@@ -0,0 +1,334 @@
+/**
+ * runspec-node/testing — test helpers for runspec runnables.
+ *
+ * The Node port of Python's `runspec.testing`. Import it from the subpath
+ * export so it stays out of the main entry:
+ *
+ *     import { ParseHarness } from 'runspec-node/testing';
+ *
+ * `parse()` already takes `argv` and `configPath`, so testing a runnable never
+ * requires monkeypatching `process.argv` or mocking the parser. `ParseHarness`
+ * removes the remaining boilerplate — pointing `parse()` at a spec, asserting on
+ * clean exits without the process actually exiting, and walking the subcommand
+ * tree to auto-cover every command.
+ *
+ * Node specifics this handles for you: `parse()` calls `process.exit(0)` on
+ * `--help` and *throws* `RunSpecError` on validation/require-command errors. The
+ * harness intercepts `process.exit` (so `--help` doesn't kill your test runner)
+ * and maps a thrown `RunSpecError` to a non-zero "exit", unifying both under one
+ * exit-code abstraction that mirrors Python's `SystemExit`-based helpers.
+ */
+
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { parse } from './parser';
+import { loadRaw } from './loader';
+import { inferScript } from './inference';
+import { RunSpecError } from './errors';
+import type { ParsedArgs, ScriptSpec, ArgSpec } from './models';
+
+export interface ParseHarnessOptions {
+  scriptName: string;
+  /** An existing runspec.toml (a fixture, or your package's real spec). */
+  configPath?: string;
+  /** An inline TOML string written to a throwaway temp file. */
+  toml?: string;
+  /** Pin the per-run summary off during a parse (default true). */
+  suppressSummary?: boolean;
+}
+
+export interface ExpectExitOptions {
+  /** Expected exit code; `null` accepts any non-zero. Default 1; use 0 for --help. */
+  code?: number | null;
+  /** Substring(s) the captured output must include. */
+  contains?: string | string[];
+}
+
+export interface SmokeOptions {
+  /** Arg-name → value map used to build valid invocations for each leaf. */
+  values?: Record<string, unknown>;
+  /** Assert `--help` exits 0 for the runnable and every subcommand (default true). */
+  checkHelp?: boolean;
+}
+
+/** Thrown in place of `process.exit()` so a parse that exits can be caught. */
+class ExitSignal extends Error {
+  constructor(public exitCode: number) {
+    super(`process.exit(${exitCode})`);
+  }
+}
+
+export class ParseHarness {
+  readonly scriptName: string;
+  readonly configPath: string;
+  private readonly suppressSummary: boolean;
+  private tmpDir: string | null = null;
+  private readonly spec: ScriptSpec;
+
+  constructor(opts: ParseHarnessOptions) {
+    const { scriptName, configPath, toml, suppressSummary = true } = opts;
+    if ((configPath == null) === (toml == null)) {
+      throw new Error('pass exactly one of configPath or toml');
+    }
+
+    this.scriptName = scriptName;
+    this.suppressSummary = suppressSummary;
+
+    let cfgPath = configPath;
+    if (toml != null) {
+      this.tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'runspec-test-'));
+      cfgPath = path.join(this.tmpDir, 'runspec.toml');
+      fs.writeFileSync(cfgPath, toml, 'utf-8');
+    }
+    this.configPath = path.resolve(cfgPath as string);
+
+    try {
+      const raw = loadRaw(this.configPath);
+      if (!(scriptName in raw.runnables)) {
+        const have = Object.keys(raw.runnables).sort().join(', ') || '(none)';
+        throw new Error(`runnable '${scriptName}' not in ${this.configPath} (have: ${have})`);
+      }
+      // Infer so `required` is resolved from defaults/types (raw leaves it unset).
+      this.spec = inferScript(raw.runnables[scriptName], raw.config.autonomyDefault);
+    } catch (e) {
+      // Tear down the temp dir synchronously on a failed construction.
+      this.close();
+      throw e;
+    }
+  }
+
+  // ── lifecycle ─────────────────────────────────────────────────────────────
+
+  close(): void {
+    if (this.tmpDir != null) {
+      fs.rmSync(this.tmpDir, { recursive: true, force: true });
+      this.tmpDir = null;
+    }
+  }
+
+  // ── core parse entry points ───────────────────────────────────────────────
+
+  parse(argv: string[]): ParsedArgs {
+    const { code, output, parsed } = this.runParse(argv);
+    if (parsed === null) {
+      throw new Error(`expected \`${this.scriptName} ${argv.join(' ')}\` to parse, but it exited ${code}:\n${output}`);
+    }
+    return parsed;
+  }
+
+  expectOk(argv: string[]): ParsedArgs {
+    return this.parse(argv);
+  }
+
+  expectExit(argv: string[], opts: ExpectExitOptions = {}): string {
+    const { code = 1, contains } = opts;
+    const res = this.runParse(argv);
+    if (res.code === null) {
+      throw new Error(`expected \`${this.scriptName} ${argv.join(' ')}\` to exit, but it parsed cleanly`);
+    }
+    if (code !== null && res.code !== code) {
+      throw new Error(`\`${this.scriptName} ${argv.join(' ')}\`: expected exit ${code}, got ${res.code}:\n${res.output}`);
+    }
+    if (contains != null) {
+      const needles = Array.isArray(contains) ? contains : [contains];
+      for (const needle of needles) {
+        if (!res.output.includes(needle)) {
+          throw new Error(`\`${this.scriptName} ${argv.join(' ')}\`: ${JSON.stringify(needle)} not found in output:\n${res.output}`);
+        }
+      }
+    }
+    return res.output;
+  }
+
+  // ── spec introspection ────────────────────────────────────────────────────
+
+  commandPaths(): string[][] {
+    const paths: string[][] = [];
+    const walk = (node: ScriptSpec, prefix: string[]): void => {
+      for (const [name, sub] of Object.entries(node.commands ?? {})) {
+        const here = [...prefix, name];
+        paths.push(here);
+        walk(sub, here);
+      }
+    };
+    walk(this.spec, []);
+    return paths;
+  }
+
+  leafCommandPaths(): string[][] {
+    return this.commandPaths().filter((p) => Object.keys(this.node(p).commands ?? {}).length === 0);
+  }
+
+  argsFor(p: string[] = []): Record<string, ArgSpec> {
+    return this.node(p).args ?? {};
+  }
+
+  requiredArgs(p: string[] = []): string[] {
+    return Object.entries(this.argsFor(p))
+      .filter(([, a]) => a.required)
+      .map(([name]) => name);
+  }
+
+  requiresCommand(p: string[] = []): boolean {
+    return Boolean(this.node(p).requireCommand);
+  }
+
+  // ── auto coverage ─────────────────────────────────────────────────────────
+
+  smoke(opts: SmokeOptions = {}): string[] {
+    const { values = {}, checkHelp = true } = opts;
+    const done: string[] = [];
+
+    if (checkHelp) {
+      this.expectExit(['--help'], { code: 0 });
+      done.push('--help');
+    }
+    if (this.requiresCommand()) {
+      this.expectExit([]);
+      done.push('(no command) -> require-command');
+    }
+
+    for (const p of this.commandPaths()) {
+      const label = p.join(' ');
+      const node = this.node(p);
+
+      if (checkHelp) {
+        this.expectExit([...p, '--help'], { code: 0 });
+        done.push(`${label} --help`);
+      }
+
+      if (node.requireCommand) {
+        this.expectExit(p);
+        done.push(`${label} -> require-command`);
+        continue;
+      }
+      if (Object.keys(node.commands ?? {}).length > 0) {
+        continue; // intermediate group without require-command
+      }
+
+      if (this.requiredArgs(p).length > 0) {
+        this.expectExit(p);
+        done.push(`${label} -> missing required`);
+      }
+
+      const built = this.buildValidArgv(p, values);
+      if (built !== null) {
+        this.expectOk([...p, ...built]);
+        done.push(`${label} (valid)`);
+      }
+    }
+
+    return done;
+  }
+
+  // ── internals ─────────────────────────────────────────────────────────────
+
+  private runParse(argv: string[]): { code: number | null; output: string; parsed: ParsedArgs | null } {
+    // Capture the *original* (unbound) handles so restoration preserves identity.
+    const realExit = process.exit;
+    const realOut = process.stdout.write;
+    const realErr = process.stderr.write;
+    const realLog = console.log;
+    const realErrLog = console.error;
+    let captured = '';
+
+    (process as unknown as { exit: (c?: number) => never }).exit = (c?: number): never => {
+      throw new ExitSignal(c ?? 0);
+    };
+    process.stdout.write = ((chunk: unknown): boolean => {
+      captured += String(chunk);
+      return true;
+    }) as typeof process.stdout.write;
+    process.stderr.write = ((chunk: unknown): boolean => {
+      captured += String(chunk);
+      return true;
+    }) as typeof process.stderr.write;
+    console.log = (...a: unknown[]): void => {
+      captured += a.map(String).join(' ') + '\n';
+    };
+    console.error = (...a: unknown[]): void => {
+      captured += a.map(String).join(' ') + '\n';
+    };
+
+    return this.withSummaryPinned(() => {
+      try {
+        const parsed = parse({
+          scriptName: this.scriptName,
+          argv,
+          configPath: this.configPath,
+          _promptSecrets: false,
+        });
+        return { code: null, output: captured, parsed };
+      } catch (e) {
+        if (e instanceof ExitSignal) {
+          return { code: e.exitCode, output: captured, parsed: null };
+        }
+        if (e instanceof RunSpecError) {
+          // Validation / require-command throw rather than exit; fold the
+          // message into the captured output so `contains:` can match on it.
+          return { code: 1, output: `${captured}${e.message}\n`, parsed: null };
+        }
+        throw e;
+      } finally {
+        process.exit = realExit;
+        process.stdout.write = realOut;
+        process.stderr.write = realErr;
+        console.log = realLog;
+        console.error = realErrLog;
+      }
+    });
+  }
+
+  private node(p: string[]): ScriptSpec {
+    let node = this.spec;
+    for (const part of p) node = (node.commands ?? {})[part];
+    return node;
+  }
+
+  private mergedArgs(p: string[]): Record<string, ArgSpec> {
+    const merged: Record<string, ArgSpec> = { ...this.argsFor([]) };
+    const acc: string[] = [];
+    for (const part of p) {
+      acc.push(part);
+      Object.assign(merged, this.argsFor(acc));
+    }
+    return merged;
+  }
+
+  private buildValidArgv(p: string[], values: Record<string, unknown>): string[] | null {
+    const argv: string[] = [];
+    for (const [name, a] of Object.entries(this.mergedArgs(p))) {
+      const supplied = values[name];
+      if (supplied === undefined || supplied === null) {
+        if (a.required) return null;
+        continue;
+      }
+      const flag = `--${name}`;
+      if (a.type === 'flag') {
+        if (supplied) argv.push(flag);
+      } else {
+        argv.push(flag, String(supplied));
+      }
+    }
+    return argv;
+  }
+
+  private withSummaryPinned<T>(fn: () => T): T {
+    if (!this.suppressSummary) return fn();
+    const prefix = this.scriptName.toUpperCase().replace(/-/g, '_');
+    const key = `RUNSPEC_${prefix}_ARG_NO_SUMMARY`;
+    const prev = process.env[key];
+    process.env[key] = '1';
+    try {
+      return fn();
+    } finally {
+      if (prev === undefined) delete process.env[key];
+      else process.env[key] = prev;
+    }
+  }
+}
+
+export function harness(scriptName: string, opts: Omit<ParseHarnessOptions, 'scriptName'> = {}): ParseHarness {
+  return new ParseHarness({ scriptName, ...opts });
+}

package/src/types.ts

@@ -63,6 +63,13 @@ function coerceStr(value: unknown, spec: ArgSpec): string {
   return coerced;
 }
 
+// `password` coerces exactly like `str` (it is a string and honours
+// pattern/min-length/max-length); the type exists to mark the value as secret so
+// it is masked in UIs, refused on the command line, and kept out of logs.
+function coercePassword(value: unknown, spec: ArgSpec): string {
+  return coerceStr(value, spec);
+}
+
 function coerceInt(value: unknown, spec: ArgSpec): number {
   const n = Number(value);
   if (!Number.isFinite(n) || !Number.isInteger(n)) {
@@ -136,6 +143,7 @@ function checkPattern(value: string, spec: ArgSpec): void {
 }
 
 registerType('str', coerceStr);
+registerType('password', coercePassword);
 registerType('int', coerceInt);
 registerType('float', coerceFloat);
 registerType('bool', coerceBool);

package/tests/test_cli_test.test.ts

@@ -0,0 +1,134 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { cmdTest, testOne } from '../src/cli';
+
+const GOOD_TOML = `
+[config]
+autonomy-default = "autonomous"
+
+[greet]
+description = "Greet"
+[greet.args]
+name = {type = "str", required = false, default = "world"}
+
+[farewell]
+description = "Bye"
+[farewell.args]
+name = {type = "str", required = false, default = "world"}
+`;
+
+const cleanups: string[] = [];
+afterEach(() => {
+  while (cleanups.length) fs.rmSync(cleanups.pop()!, { recursive: true, force: true });
+  jest.restoreAllMocks();
+});
+
+/** Build a venv-shaped folder: runspec.toml + node_modules/.bin/<name> shims
+ * exiting with the given code (a shim of `null` means: don't create it). */
+function folder(toml: string, shims: Record<string, number | null>): string {
+  const dir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'runspec-test-'));
+  cleanups.push(dir);
+  fs.writeFileSync(path.join(dir, 'runspec.toml'), toml);
+  const binDir = path.join(dir, 'node_modules', '.bin');
+  fs.mkdirSync(binDir, { recursive: true });
+  for (const [name, code] of Object.entries(shims)) {
+    if (code === null) continue;
+    const p = path.join(binDir, name);
+    fs.writeFileSync(p, `#!/bin/sh\nexit ${code}\n`);
+    fs.chmodSync(p, 0o755);
+  }
+  return dir;
+}
+
+function item(dir: string, runnable: string) {
+  return { source: path.join(dir, 'runspec.toml'), runnable, spec: {} as any };
+}
+
+// ── testOne (the per-runnable unit) ─────────────────────────────────────────
+
+test('passes when spec smoke ok and entry point exits 0', () => {
+  const dir = folder(GOOD_TOML, { greet: 0 });
+  const r = testOne(item(dir, 'greet'));
+  expect(r.ok).toBe(true);
+  expect(r.checks.spec_smoke.ok).toBe(true);
+  expect(r.checks.entry_point.ok).toBe(true);
+});
+
+test('fails when the entry-point binary exits non-zero', () => {
+  const dir = folder(GOOD_TOML, { greet: 1 });
+  const r = testOne(item(dir, 'greet'));
+  expect(r.ok).toBe(false);
+  expect(r.checks.spec_smoke.ok).toBe(true); // spec is fine
+  expect(r.checks.entry_point.ok).toBe(false);
+  expect(r.checks.entry_point.error).toContain('exit 1');
+});
+
+test('fails when the entry point is missing', () => {
+  const dir = folder(GOOD_TOML, { greet: null });
+  const r = testOne(item(dir, 'greet'));
+  expect(r.ok).toBe(false);
+  expect(r.checks.entry_point.error).toContain('entry point not found');
+});
+
+test('fails spec smoke when the runnable is not in the TOML', () => {
+  const dir = folder(GOOD_TOML, { ghost: 0 });
+  const r = testOne(item(dir, 'ghost'));
+  expect(r.ok).toBe(false);
+  expect(r.checks.spec_smoke.ok).toBe(false);
+  expect(r.checks.spec_smoke.error).toContain('ghost');
+});
+
+// ── cmdTest (orchestration: exit code, filter, json) ────────────────────────
+
+function runCmd(dir: string, args: string[]): { exitCode: number | undefined; out: string } {
+  const cwd = process.cwd();
+  let exitCode: number | undefined;
+  const exitSpy = jest.spyOn(process, 'exit').mockImplementation(((c?: number) => {
+    exitCode = c;
+    return undefined as never;
+  }) as never);
+  let out = '';
+  const logSpy = jest.spyOn(console, 'log').mockImplementation((...a: unknown[]) => {
+    out += a.map(String).join(' ') + '\n';
+  });
+  try {
+    process.chdir(dir);
+    cmdTest(args);
+  } finally {
+    process.chdir(cwd);
+    logSpy.mockRestore();
+    exitSpy.mockRestore();
+  }
+  return { exitCode, out };
+}
+
+test('all pass → no exit-1, summary reports passed', () => {
+  const dir = folder(GOOD_TOML, { greet: 0, farewell: 0 });
+  const { exitCode, out } = runCmd(dir, []);
+  expect(exitCode).toBeUndefined();
+  expect(out).toContain('2 passed, 0 failed');
+});
+
+test('one broken runnable → exit 1', () => {
+  const dir = folder(GOOD_TOML, { greet: 1, farewell: 0 });
+  const { exitCode, out } = runCmd(dir, []);
+  expect(exitCode).toBe(1);
+  expect(out).toContain('1 passed, 1 failed');
+});
+
+test('--runnable filters to a single runnable', () => {
+  const dir = folder(GOOD_TOML, { greet: 0, farewell: 0 });
+  const { out } = runCmd(dir, ['--runnable', 'greet']);
+  expect(out).toContain('1 passed');
+  expect(out).not.toContain('farewell');
+});
+
+test('--format json emits results + summary', () => {
+  const dir = folder(GOOD_TOML, { greet: 0, farewell: 0 });
+  const { out } = runCmd(dir, ['--format', 'json']);
+  const payload = JSON.parse(out);
+  expect(payload.summary).toEqual({ total: 2, passed: 2, failed: 0 });
+  expect(payload.results).toHaveLength(2);
+  expect(Object.keys(payload.results[0].checks)).toEqual(['spec_smoke', 'entry_point']);
+});

package/tests/test_password.test.ts

@@ -0,0 +1,106 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { parse, loadSpec } from '../src/parser';
+import { loadRaw } from '../src/loader';
+import { inferScript } from '../src/inference';
+import { buildSchema } from '../src/cli';
+import { coerce, listTypes } from '../src/types';
+import type { ArgSpec } from '../src/models';
+
+function makeTmpConfig(toml: string): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'runspec-pw-test-'));
+  const file = path.join(dir, 'runspec.toml');
+  fs.writeFileSync(file, toml);
+  return file;
+}
+
+const TOML = `
+[deploy]
+description = "Deploy"
+[deploy.args.host]
+type = "str"
+hint = "web-01.example.com"
+required = true
+[deploy.args.secret]
+type = "password"
+description = "DB password"
+`;
+
+// ── coercion ──────────────────────────────────────────────────────────────────
+
+test('password coerces like str', () => {
+  const spec = { name: 'secret', type: 'password' } as ArgSpec;
+  expect(coerce('hunter2', spec)).toBe('hunter2');
+});
+
+test('password honours min-length', () => {
+  const spec = { name: 'secret', type: 'password', minLength: 8 } as ArgSpec;
+  expect(() => coerce('ab', spec)).toThrow();
+});
+
+test('password is a registered type', () => {
+  expect(listTypes()).toContain('password');
+});
+
+// ── refused on the command line ─────────────────────────────────────────────────
+
+test('password refused as --secret value', () => {
+  const configPath = makeTmpConfig(TOML);
+  expect(() =>
+    parse({ scriptName: 'deploy', argv: ['--host', 'h', '--secret', 'x'], configPath }),
+  ).toThrow(/cannot be passed on the command line/);
+});
+
+test('password refused as --secret=value', () => {
+  const configPath = makeTmpConfig(TOML);
+  expect(() =>
+    parse({ scriptName: 'deploy', argv: ['--host', 'h', '--secret=x'], configPath }),
+  ).toThrow(/cannot be passed on the command line/);
+});
+
+// ── resolved from the environment ────────────────────────────────────────────────
+
+test('password resolves from RUNSPEC_<RUNNABLE>_ARG_<NAME>', () => {
+  const configPath = makeTmpConfig(TOML);
+  process.env['RUNSPEC_DEPLOY_ARG_SECRET'] = 'fromenv';
+  try {
+    const args = parse({ scriptName: 'deploy', argv: ['--host', 'h'], configPath });
+    expect(args['secret']).toBe('fromenv');
+  } finally {
+    delete process.env['RUNSPEC_DEPLOY_ARG_SECRET'];
+  }
+});
+
+// ── hint plumbing ───────────────────────────────────────────────────────────────
+
+test('hint is carried onto the spec', () => {
+  const configPath = makeTmpConfig(TOML);
+  const raw = loadRaw(configPath);
+  expect(raw.runnables['deploy'].args['host'].hint).toBe('web-01.example.com');
+  expect(raw.runnables['deploy'].args['secret'].hint).toBeUndefined();
+});
+
+// ── omitted from emitted agent schemas ───────────────────────────────────────────
+
+test('password arg is excluded from emitted MCP schema', () => {
+  const configPath = makeTmpConfig(TOML);
+  const raw = loadRaw(configPath);
+  const script = inferScript(raw.runnables['deploy'], raw.config.autonomyDefault);
+  const schema = buildSchema('deploy', script, 'mcp');
+  const input = schema['inputSchema'] as Record<string, unknown>;
+  const props = input['properties'] as Record<string, unknown>;
+  expect('host' in props).toBe(true);
+  expect('secret' in props).toBe(false);
+  expect((input['required'] as string[] | undefined) ?? []).not.toContain('secret');
+});
+
+// ── loadSpec never prompts ───────────────────────────────────────────────────────
+
+test('loadSpec does not prompt for a required password', () => {
+  const configPath = makeTmpConfig(TOML);
+  // loadSpec disables prompting; required args make it throw at validation, but
+  // it must never block on a prompt. (No TTY in tests anyway, but the gate is
+  // what guarantees it.)
+  expect(() => loadSpec({ scriptName: 'deploy', configPath })).toThrow();
+});