runspec-node 0.27.0 → 0.28.0

package/src/testing.ts

@@ -0,0 +1,334 @@
+/**
+ * runspec-node/testing — test helpers for runspec runnables.
+ *
+ * The Node port of Python's `runspec.testing`. Import it from the subpath
+ * export so it stays out of the main entry:
+ *
+ *     import { ParseHarness } from 'runspec-node/testing';
+ *
+ * `parse()` already takes `argv` and `configPath`, so testing a runnable never
+ * requires monkeypatching `process.argv` or mocking the parser. `ParseHarness`
+ * removes the remaining boilerplate — pointing `parse()` at a spec, asserting on
+ * clean exits without the process actually exiting, and walking the subcommand
+ * tree to auto-cover every command.
+ *
+ * Node specifics this handles for you: `parse()` calls `process.exit(0)` on
+ * `--help` and *throws* `RunSpecError` on validation/require-command errors. The
+ * harness intercepts `process.exit` (so `--help` doesn't kill your test runner)
+ * and maps a thrown `RunSpecError` to a non-zero "exit", unifying both under one
+ * exit-code abstraction that mirrors Python's `SystemExit`-based helpers.
+ */
+
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { parse } from './parser';
+import { loadRaw } from './loader';
+import { inferScript } from './inference';
+import { RunSpecError } from './errors';
+import type { ParsedArgs, ScriptSpec, ArgSpec } from './models';
+
+export interface ParseHarnessOptions {
+  scriptName: string;
+  /** An existing runspec.toml (a fixture, or your package's real spec). */
+  configPath?: string;
+  /** An inline TOML string written to a throwaway temp file. */
+  toml?: string;
+  /** Pin the per-run summary off during a parse (default true). */
+  suppressSummary?: boolean;
+}
+
+export interface ExpectExitOptions {
+  /** Expected exit code; `null` accepts any non-zero. Default 1; use 0 for --help. */
+  code?: number | null;
+  /** Substring(s) the captured output must include. */
+  contains?: string | string[];
+}
+
+export interface SmokeOptions {
+  /** Arg-name → value map used to build valid invocations for each leaf. */
+  values?: Record<string, unknown>;
+  /** Assert `--help` exits 0 for the runnable and every subcommand (default true). */
+  checkHelp?: boolean;
+}
+
+/** Thrown in place of `process.exit()` so a parse that exits can be caught. */
+class ExitSignal extends Error {
+  constructor(public exitCode: number) {
+    super(`process.exit(${exitCode})`);
+  }
+}
+
+export class ParseHarness {
+  readonly scriptName: string;
+  readonly configPath: string;
+  private readonly suppressSummary: boolean;
+  private tmpDir: string | null = null;
+  private readonly spec: ScriptSpec;
+
+  constructor(opts: ParseHarnessOptions) {
+    const { scriptName, configPath, toml, suppressSummary = true } = opts;
+    if ((configPath == null) === (toml == null)) {
+      throw new Error('pass exactly one of configPath or toml');
+    }
+
+    this.scriptName = scriptName;
+    this.suppressSummary = suppressSummary;
+
+    let cfgPath = configPath;
+    if (toml != null) {
+      this.tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'runspec-test-'));
+      cfgPath = path.join(this.tmpDir, 'runspec.toml');
+      fs.writeFileSync(cfgPath, toml, 'utf-8');
+    }
+    this.configPath = path.resolve(cfgPath as string);
+
+    try {
+      const raw = loadRaw(this.configPath);
+      if (!(scriptName in raw.runnables)) {
+        const have = Object.keys(raw.runnables).sort().join(', ') || '(none)';
+        throw new Error(`runnable '${scriptName}' not in ${this.configPath} (have: ${have})`);
+      }
+      // Infer so `required` is resolved from defaults/types (raw leaves it unset).
+      this.spec = inferScript(raw.runnables[scriptName], raw.config.autonomyDefault);
+    } catch (e) {
+      // Tear down the temp dir synchronously on a failed construction.
+      this.close();
+      throw e;
+    }
+  }
+
+  // ── lifecycle ─────────────────────────────────────────────────────────────
+
+  close(): void {
+    if (this.tmpDir != null) {
+      fs.rmSync(this.tmpDir, { recursive: true, force: true });
+      this.tmpDir = null;
+    }
+  }
+
+  // ── core parse entry points ───────────────────────────────────────────────
+
+  parse(argv: string[]): ParsedArgs {
+    const { code, output, parsed } = this.runParse(argv);
+    if (parsed === null) {
+      throw new Error(`expected \`${this.scriptName} ${argv.join(' ')}\` to parse, but it exited ${code}:\n${output}`);
+    }
+    return parsed;
+  }
+
+  expectOk(argv: string[]): ParsedArgs {
+    return this.parse(argv);
+  }
+
+  expectExit(argv: string[], opts: ExpectExitOptions = {}): string {
+    const { code = 1, contains } = opts;
+    const res = this.runParse(argv);
+    if (res.code === null) {
+      throw new Error(`expected \`${this.scriptName} ${argv.join(' ')}\` to exit, but it parsed cleanly`);
+    }
+    if (code !== null && res.code !== code) {
+      throw new Error(`\`${this.scriptName} ${argv.join(' ')}\`: expected exit ${code}, got ${res.code}:\n${res.output}`);
+    }
+    if (contains != null) {
+      const needles = Array.isArray(contains) ? contains : [contains];
+      for (const needle of needles) {
+        if (!res.output.includes(needle)) {
+          throw new Error(`\`${this.scriptName} ${argv.join(' ')}\`: ${JSON.stringify(needle)} not found in output:\n${res.output}`);
+        }
+      }
+    }
+    return res.output;
+  }
+
+  // ── spec introspection ────────────────────────────────────────────────────
+
+  commandPaths(): string[][] {
+    const paths: string[][] = [];
+    const walk = (node: ScriptSpec, prefix: string[]): void => {
+      for (const [name, sub] of Object.entries(node.commands ?? {})) {
+        const here = [...prefix, name];
+        paths.push(here);
+        walk(sub, here);
+      }
+    };
+    walk(this.spec, []);
+    return paths;
+  }
+
+  leafCommandPaths(): string[][] {
+    return this.commandPaths().filter((p) => Object.keys(this.node(p).commands ?? {}).length === 0);
+  }
+
+  argsFor(p: string[] = []): Record<string, ArgSpec> {
+    return this.node(p).args ?? {};
+  }
+
+  requiredArgs(p: string[] = []): string[] {
+    return Object.entries(this.argsFor(p))
+      .filter(([, a]) => a.required)
+      .map(([name]) => name);
+  }
+
+  requiresCommand(p: string[] = []): boolean {
+    return Boolean(this.node(p).requireCommand);
+  }
+
+  // ── auto coverage ─────────────────────────────────────────────────────────
+
+  smoke(opts: SmokeOptions = {}): string[] {
+    const { values = {}, checkHelp = true } = opts;
+    const done: string[] = [];
+
+    if (checkHelp) {
+      this.expectExit(['--help'], { code: 0 });
+      done.push('--help');
+    }
+    if (this.requiresCommand()) {
+      this.expectExit([]);
+      done.push('(no command) -> require-command');
+    }
+
+    for (const p of this.commandPaths()) {
+      const label = p.join(' ');
+      const node = this.node(p);
+
+      if (checkHelp) {
+        this.expectExit([...p, '--help'], { code: 0 });
+        done.push(`${label} --help`);
+      }
+
+      if (node.requireCommand) {
+        this.expectExit(p);
+        done.push(`${label} -> require-command`);
+        continue;
+      }
+      if (Object.keys(node.commands ?? {}).length > 0) {
+        continue; // intermediate group without require-command
+      }
+
+      if (this.requiredArgs(p).length > 0) {
+        this.expectExit(p);
+        done.push(`${label} -> missing required`);
+      }
+
+      const built = this.buildValidArgv(p, values);
+      if (built !== null) {
+        this.expectOk([...p, ...built]);
+        done.push(`${label} (valid)`);
+      }
+    }
+
+    return done;
+  }
+
+  // ── internals ─────────────────────────────────────────────────────────────
+
+  private runParse(argv: string[]): { code: number | null; output: string; parsed: ParsedArgs | null } {
+    // Capture the *original* (unbound) handles so restoration preserves identity.
+    const realExit = process.exit;
+    const realOut = process.stdout.write;
+    const realErr = process.stderr.write;
+    const realLog = console.log;
+    const realErrLog = console.error;
+    let captured = '';
+
+    (process as unknown as { exit: (c?: number) => never }).exit = (c?: number): never => {
+      throw new ExitSignal(c ?? 0);
+    };
+    process.stdout.write = ((chunk: unknown): boolean => {
+      captured += String(chunk);
+      return true;
+    }) as typeof process.stdout.write;
+    process.stderr.write = ((chunk: unknown): boolean => {
+      captured += String(chunk);
+      return true;
+    }) as typeof process.stderr.write;
+    console.log = (...a: unknown[]): void => {
+      captured += a.map(String).join(' ') + '\n';
+    };
+    console.error = (...a: unknown[]): void => {
+      captured += a.map(String).join(' ') + '\n';
+    };
+
+    return this.withSummaryPinned(() => {
+      try {
+        const parsed = parse({
+          scriptName: this.scriptName,
+          argv,
+          configPath: this.configPath,
+          _promptSecrets: false,
+        });
+        return { code: null, output: captured, parsed };
+      } catch (e) {
+        if (e instanceof ExitSignal) {
+          return { code: e.exitCode, output: captured, parsed: null };
+        }
+        if (e instanceof RunSpecError) {
+          // Validation / require-command throw rather than exit; fold the
+          // message into the captured output so `contains:` can match on it.
+          return { code: 1, output: `${captured}${e.message}\n`, parsed: null };
+        }
+        throw e;
+      } finally {
+        process.exit = realExit;
+        process.stdout.write = realOut;
+        process.stderr.write = realErr;
+        console.log = realLog;
+        console.error = realErrLog;
+      }
+    });
+  }
+
+  private node(p: string[]): ScriptSpec {
+    let node = this.spec;
+    for (const part of p) node = (node.commands ?? {})[part];
+    return node;
+  }
+
+  private mergedArgs(p: string[]): Record<string, ArgSpec> {
+    const merged: Record<string, ArgSpec> = { ...this.argsFor([]) };
+    const acc: string[] = [];
+    for (const part of p) {
+      acc.push(part);
+      Object.assign(merged, this.argsFor(acc));
+    }
+    return merged;
+  }
+
+  private buildValidArgv(p: string[], values: Record<string, unknown>): string[] | null {
+    const argv: string[] = [];
+    for (const [name, a] of Object.entries(this.mergedArgs(p))) {
+      const supplied = values[name];
+      if (supplied === undefined || supplied === null) {
+        if (a.required) return null;
+        continue;
+      }
+      const flag = `--${name}`;
+      if (a.type === 'flag') {
+        if (supplied) argv.push(flag);
+      } else {
+        argv.push(flag, String(supplied));
+      }
+    }
+    return argv;
+  }
+
+  private withSummaryPinned<T>(fn: () => T): T {
+    if (!this.suppressSummary) return fn();
+    const prefix = this.scriptName.toUpperCase().replace(/-/g, '_');
+    const key = `RUNSPEC_${prefix}_ARG_NO_SUMMARY`;
+    const prev = process.env[key];
+    process.env[key] = '1';
+    try {
+      return fn();
+    } finally {
+      if (prev === undefined) delete process.env[key];
+      else process.env[key] = prev;
+    }
+  }
+}
+
+export function harness(scriptName: string, opts: Omit<ParseHarnessOptions, 'scriptName'> = {}): ParseHarness {
+  return new ParseHarness({ scriptName, ...opts });
+}

package/tests/test_cli_test.test.ts

@@ -0,0 +1,134 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { cmdTest, testOne } from '../src/cli';
+
+const GOOD_TOML = `
+[config]
+autonomy-default = "autonomous"
+
+[greet]
+description = "Greet"
+[greet.args]
+name = {type = "str", required = false, default = "world"}
+
+[farewell]
+description = "Bye"
+[farewell.args]
+name = {type = "str", required = false, default = "world"}
+`;
+
+const cleanups: string[] = [];
+afterEach(() => {
+  while (cleanups.length) fs.rmSync(cleanups.pop()!, { recursive: true, force: true });
+  jest.restoreAllMocks();
+});
+
+/** Build a venv-shaped folder: runspec.toml + node_modules/.bin/<name> shims
+ * exiting with the given code (a shim of `null` means: don't create it). */
+function folder(toml: string, shims: Record<string, number | null>): string {
+  const dir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'runspec-test-'));
+  cleanups.push(dir);
+  fs.writeFileSync(path.join(dir, 'runspec.toml'), toml);
+  const binDir = path.join(dir, 'node_modules', '.bin');
+  fs.mkdirSync(binDir, { recursive: true });
+  for (const [name, code] of Object.entries(shims)) {
+    if (code === null) continue;
+    const p = path.join(binDir, name);
+    fs.writeFileSync(p, `#!/bin/sh\nexit ${code}\n`);
+    fs.chmodSync(p, 0o755);
+  }
+  return dir;
+}
+
+function item(dir: string, runnable: string) {
+  return { source: path.join(dir, 'runspec.toml'), runnable, spec: {} as any };
+}
+
+// ── testOne (the per-runnable unit) ─────────────────────────────────────────
+
+test('passes when spec smoke ok and entry point exits 0', () => {
+  const dir = folder(GOOD_TOML, { greet: 0 });
+  const r = testOne(item(dir, 'greet'));
+  expect(r.ok).toBe(true);
+  expect(r.checks.spec_smoke.ok).toBe(true);
+  expect(r.checks.entry_point.ok).toBe(true);
+});
+
+test('fails when the entry-point binary exits non-zero', () => {
+  const dir = folder(GOOD_TOML, { greet: 1 });
+  const r = testOne(item(dir, 'greet'));
+  expect(r.ok).toBe(false);
+  expect(r.checks.spec_smoke.ok).toBe(true); // spec is fine
+  expect(r.checks.entry_point.ok).toBe(false);
+  expect(r.checks.entry_point.error).toContain('exit 1');
+});
+
+test('fails when the entry point is missing', () => {
+  const dir = folder(GOOD_TOML, { greet: null });
+  const r = testOne(item(dir, 'greet'));
+  expect(r.ok).toBe(false);
+  expect(r.checks.entry_point.error).toContain('entry point not found');
+});
+
+test('fails spec smoke when the runnable is not in the TOML', () => {
+  const dir = folder(GOOD_TOML, { ghost: 0 });
+  const r = testOne(item(dir, 'ghost'));
+  expect(r.ok).toBe(false);
+  expect(r.checks.spec_smoke.ok).toBe(false);
+  expect(r.checks.spec_smoke.error).toContain('ghost');
+});
+
+// ── cmdTest (orchestration: exit code, filter, json) ────────────────────────
+
+function runCmd(dir: string, args: string[]): { exitCode: number | undefined; out: string } {
+  const cwd = process.cwd();
+  let exitCode: number | undefined;
+  const exitSpy = jest.spyOn(process, 'exit').mockImplementation(((c?: number) => {
+    exitCode = c;
+    return undefined as never;
+  }) as never);
+  let out = '';
+  const logSpy = jest.spyOn(console, 'log').mockImplementation((...a: unknown[]) => {
+    out += a.map(String).join(' ') + '\n';
+  });
+  try {
+    process.chdir(dir);
+    cmdTest(args);
+  } finally {
+    process.chdir(cwd);
+    logSpy.mockRestore();
+    exitSpy.mockRestore();
+  }
+  return { exitCode, out };
+}
+
+test('all pass → no exit-1, summary reports passed', () => {
+  const dir = folder(GOOD_TOML, { greet: 0, farewell: 0 });
+  const { exitCode, out } = runCmd(dir, []);
+  expect(exitCode).toBeUndefined();
+  expect(out).toContain('2 passed, 0 failed');
+});
+
+test('one broken runnable → exit 1', () => {
+  const dir = folder(GOOD_TOML, { greet: 1, farewell: 0 });
+  const { exitCode, out } = runCmd(dir, []);
+  expect(exitCode).toBe(1);
+  expect(out).toContain('1 passed, 1 failed');
+});
+
+test('--runnable filters to a single runnable', () => {
+  const dir = folder(GOOD_TOML, { greet: 0, farewell: 0 });
+  const { out } = runCmd(dir, ['--runnable', 'greet']);
+  expect(out).toContain('1 passed');
+  expect(out).not.toContain('farewell');
+});
+
+test('--format json emits results + summary', () => {
+  const dir = folder(GOOD_TOML, { greet: 0, farewell: 0 });
+  const { out } = runCmd(dir, ['--format', 'json']);
+  const payload = JSON.parse(out);
+  expect(payload.summary).toEqual({ total: 2, passed: 2, failed: 0 });
+  expect(payload.results).toHaveLength(2);
+  expect(Object.keys(payload.results[0].checks)).toEqual(['spec_smoke', 'entry_point']);
+});

package/tests/testing.test.ts

@@ -0,0 +1,132 @@
+import { ParseHarness, harness } from '../src/testing';
+
+// A spec with a required subcommand, a leaf with a required arg + a flag, and a
+// leaf with only optional args — enough to exercise the whole smoke() walk.
+const SPEC = `
+[config]
+autonomy-default = "autonomous"
+
+[tool]
+description = "A tool with subcommands"
+require-command = true
+
+[tool.commands.create]
+description = "Create a thing"
+[tool.commands.create.args]
+name = {type = "str", required = true}
+force = {type = "flag", default = false}
+
+[tool.commands.show]
+description = "Show a thing"
+[tool.commands.show.args]
+id = {type = "str", required = false, default = "x"}
+`;
+
+function h(): ParseHarness {
+  return new ParseHarness({ scriptName: 'tool', toml: SPEC });
+}
+
+// ── core parse entry points ────────────────────────────────────────────────
+
+test('expectExit(--help) returns 0 WITHOUT killing the process', () => {
+  const harn = h();
+  // If process.exit weren't intercepted, this would terminate the jest worker.
+  const out = harn.expectExit(['create', '--help'], { code: 0 });
+  expect(out).toContain('Create a thing');
+  harn.close();
+});
+
+test('require-command: empty argv exits non-zero (RunSpecError mapped to code 1)', () => {
+  const harn = h();
+  harn.expectExit([]); // default code = 1
+  harn.close();
+});
+
+test('missing required arg exits 1 and names the flag', () => {
+  const harn = h();
+  const out = harn.expectExit(['create'], { code: 1, contains: 'name' });
+  expect(out).toContain('name');
+  harn.close();
+});
+
+test('expectOk returns the parsed args with coercion applied', () => {
+  const harn = h();
+  const parsed = harn.expectOk(['create', '--name', 'widget', '--force']);
+  expect(String(parsed.name)).toBe('widget');
+  expect(parsed.force).toBe(true);
+  harn.close();
+});
+
+// ── smoke() auto-coverage ───────────────────────────────────────────────────
+
+test('smoke walks the whole tree and reports the checks performed', () => {
+  const harn = h();
+  const done = harn.smoke({ values: { name: 'widget' } });
+  expect(done).toContain('--help');
+  expect(done).toContain('(no command) -> require-command');
+  expect(done).toContain('create --help');
+  expect(done).toContain('create -> missing required');
+  expect(done).toContain('create (valid)');
+  expect(done).toContain('show (valid)');
+  harn.close();
+});
+
+test('smoke without values still does the structural checks', () => {
+  const harn = h();
+  const done = harn.smoke();
+  expect(done).toContain('--help');
+  expect(done).toContain('create -> missing required');
+  // show has no required args, so it still gets a (valid) check with empty argv
+  expect(done).toContain('show (valid)');
+  harn.close();
+});
+
+// ── introspection ───────────────────────────────────────────────────────────
+
+test('command/leaf paths and arg introspection', () => {
+  const harn = h();
+  expect(harn.commandPaths()).toEqual([['create'], ['show']]);
+  expect(harn.leafCommandPaths()).toEqual([['create'], ['show']]);
+  expect(harn.requiredArgs(['create'])).toEqual(['name']);
+  expect(harn.requiredArgs(['show'])).toEqual([]);
+  expect(harn.requiresCommand()).toBe(true);
+  expect(Object.keys(harn.argsFor(['create']))).toEqual(['name', 'force']);
+  harn.close();
+});
+
+// ── lifecycle / API guards ──────────────────────────────────────────────────
+
+test('factory helper mirrors the constructor', () => {
+  const harn = harness('tool', { toml: SPEC });
+  expect(harn.scriptName).toBe('tool');
+  harn.close();
+});
+
+test('rejects passing both config and toml', () => {
+  expect(() => new ParseHarness({ scriptName: 'tool', toml: SPEC, configPath: '/x/runspec.toml' })).toThrow(/exactly one/);
+});
+
+test('rejects passing neither config nor toml', () => {
+  expect(() => new ParseHarness({ scriptName: 'tool' })).toThrow(/exactly one/);
+});
+
+test('unknown runnable throws with the available names', () => {
+  expect(() => new ParseHarness({ scriptName: 'ghost', toml: SPEC })).toThrow(/ghost/);
+});
+
+// ── interception is fully restored ──────────────────────────────────────────
+
+test('process.exit / stdout.write / console.log are restored after a run', () => {
+  const harn = h();
+  const exitBefore = process.exit;
+  const stdoutBefore = process.stdout.write;
+  const logBefore = console.log;
+
+  harn.expectExit(['--help'], { code: 0 });
+  harn.expectExit(['create']); // also exercise the throw path
+
+  expect(process.exit).toBe(exitBefore);
+  expect(process.stdout.write).toBe(stdoutBefore);
+  expect(console.log).toBe(logBefore);
+  harn.close();
+});