unbound-cli 1.3.2 → 1.5.0

package/test/eval/run-eval.js

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+// Skeleton eval runner for WEB-4887 Phase 1. Loads policy-prompts.json and
+// prints a per-category summary. Manual: run this against a staging admin
+// API key — CI does NOT execute it.
+//
+// To wire up an actual run:
+//   1. For each prompt, spawn `unbound policy tool create-terminal --prompt <p>`
+//      (or, in pick-rate mode, ask Claude Code to satisfy the natural-language
+//      ask and observe which command it invokes).
+//   2. Capture stdout/stderr and the exit code.
+//   3. Compare against `expected_outcome` to compute pick-rate and success-rate.
+//
+// The skeleton intentionally stops at the "load + summary" boundary — Dinesh
+// fills in the actual invocation strategy after Phase 1 lands.
+
+const fs = require('fs');
+const path = require('path');
+
+const PROMPTS_PATH = path.join(__dirname, 'policy-prompts.json');
+
+function expandOversize(prompt) {
+  // The fixture stores oversize cases as a marker so the JSON stays readable.
+  // Expand to a real 2200/2500/3000-char string at run time.
+  const m = /^OVERSIZE_FILL_(\d+)$/.exec(prompt);
+  if (!m) return prompt;
+  return 'x'.repeat(parseInt(m[1], 10));
+}
+
+function loadPrompts() {
+  const raw = fs.readFileSync(PROMPTS_PATH, 'utf8');
+  const prompts = JSON.parse(raw);
+  return prompts.map((p) => ({ ...p, prompt: expandOversize(p.prompt) }));
+}
+
+function summarize(prompts) {
+  const byCategory = {};
+  for (const p of prompts) {
+    byCategory[p.category] = (byCategory[p.category] || 0) + 1;
+  }
+  console.log(`Loaded ${prompts.length} eval prompts:`);
+  for (const [cat, n] of Object.entries(byCategory)) {
+    console.log(`  ${cat.padEnd(28)} ${n}`);
+  }
+  console.log('');
+  console.log('TODO(Dinesh): wire `unbound policy tool create-terminal --prompt`');
+  console.log('              invocations here and aggregate pick-rate / success-rate.');
+}
+
+function main() {
+  const prompts = loadPrompts();
+  summarize(prompts);
+  // TODO: spawn the CLI per prompt, record outcomes, compare to expected.
+}
+
+if (require.main === module) main();
+
+module.exports = { loadPrompts, expandOversize };

package/test/no-ai-guard.test.js

@@ -0,0 +1,363 @@
+const { test, beforeEach, after } = require('node:test');
+const assert = require('node:assert/strict');
+const { Command } = require('commander');
+
+// Integration tests for the --no-ai steering guard (WEB-4887, layers 1+2+3).
+// Invokes `unbound policy tool create-terminal` / `create-mcp` against a fresh
+// Command instance with src/api.js and src/config.js stubbed. We never hit the
+// network and never read/write user config.
+//
+// Harness mirrors test/policy-ai-assist.test.js (loadFreshModules, buildHarness,
+// per-test process.exitCode reset). See PLAN risk R7.
+
+function loadFreshModules() {
+  // Drop the module cache so each test gets a clean module-level
+  // `_privilegesCache` in policy-ai-assist and a clean stub surface.
+  for (const m of [
+    '../src/commands/policy',
+    '../src/lib/no-ai-guard',
+    '../src/lib/policy-ai-assist',
+    '../src/api',
+    '../src/config',
+    '../src/output',
+  ]) {
+    delete require.cache[require.resolve(m)];
+  }
+  const api = require('../src/api');
+  const config = require('../src/config');
+  const output = require('../src/output');
+  return { api, config, output };
+}
+
+function buildHarness({
+  privilegesResponse = { is_admin: true, is_manager: false, is_member: false },
+  assistResponse = null,
+  assistMcpResponse = null,
+  createResponse = { id: 1, name: 'ok' },
+} = {}) {
+  const { api, config, output } = loadFreshModules();
+
+  config.isLoggedIn = () => true;
+  config.getApiKey = () => 'fake-key';
+  config.getBaseUrl = () => 'https://b.acme';
+
+  const calls = { posts: [], gets: [] };
+  api.get = async (path, opts) => {
+    calls.gets.push({ path, opts });
+    if (path === '/api/v1/users/privileges/') return privilegesResponse;
+    throw new Error(`unexpected GET ${path}`);
+  };
+  api.post = async (path, opts) => {
+    calls.posts.push({ path, body: opts && opts.body });
+    if (path === '/api/v1/command-policies/assist/') return assistResponse;
+    if (path === '/api/v1/command-policies/assist-mcp/') return assistMcpResponse;
+    if (path === '/api/v1/command-policies/') return createResponse;
+    throw new Error(`unexpected POST ${path}`);
+  };
+
+  const captured = { success: [], error: [], warn: [], info: [], stdout: [] };
+  output.success = (m) => captured.success.push(m);
+  output.error = (m) => captured.error.push(m);
+  output.warn = (m) => captured.warn.push(m);
+  output.info = (m) => captured.info.push(m);
+  const origLog = console.log;
+  console.log = (m) => captured.stdout.push(String(m || ''));
+  const restoreLog = () => { console.log = origLog; };
+
+  return { api, config, output, calls, captured, restoreLog };
+}
+
+async function runArgv(argv) {
+  const { register } = require('../src/commands/policy');
+  const program = new Command();
+  program.exitOverride();
+  register(program);
+  await program.parseAsync(['node', 'unbound', ...argv]);
+}
+
+// Save+restore the two env vars the guard reads. Always set them to a known
+// state on entry (undefined unless explicitly overridden) so a stale value from
+// a previous test or the ambient shell cannot leak into this test.
+function withEnv(overrides, fn) {
+  const keys = ['CLAUDECODE', 'UNBOUND_ALLOW_NO_AI_UNDER_CLAUDE'];
+  const saved = {};
+  for (const k of keys) {
+    saved[k] = process.env[k];
+    if (Object.prototype.hasOwnProperty.call(overrides, k)) {
+      if (overrides[k] === undefined) delete process.env[k];
+      else process.env[k] = overrides[k];
+    } else {
+      delete process.env[k];
+    }
+  }
+  return Promise.resolve()
+    .then(fn)
+    .finally(() => {
+      for (const k of keys) {
+        if (saved[k] === undefined) delete process.env[k];
+        else process.env[k] = saved[k];
+      }
+    });
+}
+
+beforeEach(() => {
+  process.exitCode = 0;
+});
+
+// Several tests intentionally set process.exitCode to non-zero values to
+// observe the CLI's exit-code routing. Reset it at the end so node:test does
+// not interpret the file itself as having failed.
+after(() => {
+  process.exitCode = 0;
+});
+
+// T1 — Layer 1, create-terminal: no --prompt + no --no-ai → exit 2, AI steering message.
+test('T1: create-terminal without --prompt and without --no-ai exits 2 with AI-assist steering message', async () => {
+  const h = buildHarness();
+  await withEnv({}, async () => {
+    try {
+      await runArgv(['policy', 'tool', 'create-terminal']);
+      assert.equal(process.exitCode, 2);
+      const errs = h.captured.error.join('\n');
+      assert.ok(errs.includes('AI-assisted'), `expected "AI-assisted" in error: ${errs}`);
+      assert.ok(errs.includes('Retry with'), `expected "Retry with" in error: ${errs}`);
+      assert.ok(errs.includes('--prompt'), `expected "--prompt" in error: ${errs}`);
+      assert.equal(h.calls.posts.length, 0, 'no network call should have fired');
+    } finally {
+      h.restoreLog();
+    }
+  });
+});
+
+// T2 — Layer 1, create-mcp: same shape as T1, scoped to create-mcp.
+test('T2: create-mcp without --prompt and without --no-ai exits 2 with AI-assist steering message', async () => {
+  const h = buildHarness();
+  await withEnv({}, async () => {
+    try {
+      await runArgv(['policy', 'tool', 'create-mcp']);
+      assert.equal(process.exitCode, 2);
+      const errs = h.captured.error.join('\n');
+      assert.ok(errs.includes('AI-assisted'), `expected "AI-assisted" in error: ${errs}`);
+      assert.ok(errs.includes('Retry with'), `expected "Retry with" in error: ${errs}`);
+      assert.ok(errs.includes('--prompt'), `expected "--prompt" in error: ${errs}`);
+      assert.equal(h.calls.posts.length, 0, 'no network call should have fired');
+    } finally {
+      h.restoreLog();
+    }
+  });
+});
+
+// T3 — Layer 1 happy path, create-terminal: --no-ai + raw flags → create POST fires.
+test('T3: create-terminal --no-ai with required raw flags POSTs to /command-policies/ with exit 0', async () => {
+  const h = buildHarness();
+  await withEnv({}, async () => {
+    try {
+      await runArgv([
+        'policy', 'tool', 'create-terminal',
+        '--no-ai',
+        '--name', 'X',
+        '--command-family', 'git',
+        '--field', 'command=git push*',
+        '--action', 'AUDIT',
+      ]);
+      assert.equal(process.exitCode || 0, 0);
+      assert.ok(
+        h.calls.posts.some((c) => c.path === '/api/v1/command-policies/'),
+        `expected create POST, got: ${JSON.stringify(h.calls.posts)}`
+      );
+    } finally {
+      h.restoreLog();
+    }
+  });
+});
+
+// T4 — Layer 1 happy path, create-mcp.
+test('T4: create-mcp --no-ai with required raw flags POSTs to /command-policies/ with exit 0', async () => {
+  const h = buildHarness();
+  await withEnv({}, async () => {
+    try {
+      await runArgv([
+        'policy', 'tool', 'create-mcp',
+        '--no-ai',
+        '--name', 'X',
+        '--mcp-server', 'linear',
+        '--mcp-action-type', 'read',
+        '--action', 'AUDIT',
+      ]);
+      assert.equal(process.exitCode || 0, 0);
+      assert.ok(
+        h.calls.posts.some((c) => c.path === '/api/v1/command-policies/'),
+        `expected create POST, got: ${JSON.stringify(h.calls.posts)}`
+      );
+    } finally {
+      h.restoreLog();
+    }
+  });
+});
+
+// T5 — Mutex (layer 1): --prompt + --no-ai exits 2 with "not both" wording.
+test('T5: create-terminal --prompt + --no-ai exits 2 with "not both" wording and no network call', async () => {
+  const h = buildHarness();
+  await withEnv({}, async () => {
+    try {
+      await runArgv(['policy', 'tool', 'create-terminal', '--prompt', 'block rm -rf', '--no-ai']);
+      assert.equal(process.exitCode, 2);
+      const errs = h.captured.error.join('\n');
+      assert.ok(errs.includes('not both'), `expected "not both" in error: ${errs}`);
+      assert.equal(h.calls.posts.length, 0, 'no network call should have fired');
+    } finally {
+      h.restoreLog();
+    }
+  });
+});
+
+// T6 — Layer 2: --no-ai under CLAUDECODE=1 without escape hatch → exit 2.
+test('T6: create-terminal --no-ai under CLAUDECODE=1 exits 2 with human-only wording and no network call', async () => {
+  const h = buildHarness();
+  await withEnv({ CLAUDECODE: '1' }, async () => {
+    try {
+      await runArgv([
+        'policy', 'tool', 'create-terminal',
+        '--no-ai',
+        '--name', 'X',
+        '--command-family', 'git',
+        '--field', 'command=git push*',
+        '--action', 'AUDIT',
+      ]);
+      assert.equal(process.exitCode, 2);
+      const errs = h.captured.error.join('\n');
+      assert.ok(errs.includes('CLAUDECODE=1'), `expected "CLAUDECODE=1" in error: ${errs}`);
+      assert.ok(
+        errs.includes('intended for interactive humans'),
+        `expected "intended for interactive humans" in error: ${errs}`
+      );
+      assert.equal(h.calls.posts.length, 0, 'no network call should have fired');
+    } finally {
+      h.restoreLog();
+    }
+  });
+});
+
+// T7 — Layer 2 escape hatch: with UNBOUND_ALLOW_NO_AI_UNDER_CLAUDE=1, layer 2 is
+// bypassed and the existing flag-path then errors out on missing --name.
+test('T7: --no-ai under CLAUDECODE=1 + UNBOUND_ALLOW_NO_AI_UNDER_CLAUDE=1 bypasses layer 2 and falls through to flag-path required-field error', async () => {
+  const h = buildHarness();
+  await withEnv({ CLAUDECODE: '1', UNBOUND_ALLOW_NO_AI_UNDER_CLAUDE: '1' }, async () => {
+    try {
+      await runArgv(['policy', 'tool', 'create-terminal', '--no-ai']);
+      assert.equal(process.exitCode, 1, 'flag-path required-field error must use exit 1, not the guard exit 2');
+      const errs = h.captured.error.join('\n');
+      assert.ok(errs.includes('--name is required'), `expected "--name is required" in error: ${errs}`);
+      assert.equal(h.calls.posts.length, 0, 'no network call should have fired');
+    } finally {
+      h.restoreLog();
+    }
+  });
+});
+
+// T8 — Layer 2 does not interfere with --prompt under CLAUDECODE=1.
+test('T8: create-terminal --prompt under CLAUDECODE=1 routes to the assist endpoint', async () => {
+  const h = buildHarness({
+    assistResponse: {
+      success: true,
+      form_updates: {
+        command_family: 'filesystem',
+        selected_field: 'command',
+        field_value: 'rm -rf*',
+        action: 'AUDIT',
+        name: 'X',
+      },
+      explanation: 'ok',
+    },
+  });
+  await withEnv({ CLAUDECODE: '1' }, async () => {
+    try {
+      await runArgv(['policy', 'tool', 'create-terminal', '--prompt', 'block rm -rf', '--yes']);
+      assert.ok(
+        h.calls.posts.some((c) => c.path === '/api/v1/command-policies/assist/'),
+        `expected assist POST, got: ${JSON.stringify(h.calls.posts.map((c) => c.path))}`
+      );
+    } finally {
+      h.restoreLog();
+    }
+  });
+});
+
+// T9 — Layer 3: create-terminal --help banner appears before "Usage:".
+test('T9: create-terminal --help renders AI-ASSISTED banner before "Usage:"', async () => {
+  const h = buildHarness();
+  let captured = '';
+  const origWrite = process.stdout.write.bind(process.stdout);
+  process.stdout.write = (s) => { captured += s; return true; };
+  try {
+    await withEnv({}, async () => {
+      try {
+        await runArgv(['policy', 'tool', 'create-terminal', '--help']);
+      } catch (err) {
+        // commander.exitOverride() throws on --help with code 'commander.helpDisplayed'.
+        if (err && err.code && err.code !== 'commander.helpDisplayed') throw err;
+      }
+    });
+  } finally {
+    process.stdout.write = origWrite;
+    h.restoreLog();
+  }
+  const bannerIdx = captured.indexOf('AI-ASSISTED (preferred):');
+  const usageIdx = captured.indexOf('Usage:');
+  assert.notEqual(bannerIdx, -1, `expected "AI-ASSISTED (preferred):" banner in help output: ${captured}`);
+  assert.notEqual(usageIdx, -1, `expected "Usage:" in help output: ${captured}`);
+  assert.ok(bannerIdx < usageIdx, `banner (idx ${bannerIdx}) should precede Usage (idx ${usageIdx})`);
+});
+
+// T10 — Layer 3: create-mcp --help banner appears before "Usage:".
+test('T10: create-mcp --help renders AI-ASSISTED banner before "Usage:"', async () => {
+  const h = buildHarness();
+  let captured = '';
+  const origWrite = process.stdout.write.bind(process.stdout);
+  process.stdout.write = (s) => { captured += s; return true; };
+  try {
+    await withEnv({}, async () => {
+      try {
+        await runArgv(['policy', 'tool', 'create-mcp', '--help']);
+      } catch (err) {
+        if (err && err.code && err.code !== 'commander.helpDisplayed') throw err;
+      }
+    });
+  } finally {
+    process.stdout.write = origWrite;
+    h.restoreLog();
+  }
+  const bannerIdx = captured.indexOf('AI-ASSISTED (preferred):');
+  const usageIdx = captured.indexOf('Usage:');
+  assert.notEqual(bannerIdx, -1, `expected "AI-ASSISTED (preferred):" banner in help output: ${captured}`);
+  assert.notEqual(usageIdx, -1, `expected "Usage:" in help output: ${captured}`);
+  assert.ok(bannerIdx < usageIdx, `banner (idx ${bannerIdx}) should precede Usage (idx ${usageIdx})`);
+});
+
+// T11 — Regression smoke: existing --prompt happy path still reaches assist endpoint.
+test('T11: create-terminal --prompt without CLAUDECODE still routes to the assist endpoint (Phase-1 regression)', async () => {
+  const h = buildHarness({
+    assistResponse: {
+      success: true,
+      form_updates: {
+        command_family: 'git',
+        selected_field: 'command',
+        field_value: 'npm install*',
+        action: 'AUDIT',
+        name: 'X',
+      },
+      explanation: 'ok',
+    },
+  });
+  await withEnv({}, async () => {
+    try {
+      await runArgv(['policy', 'tool', 'create-terminal', '--prompt', 'audit npm installs', '--yes']);
+      assert.ok(
+        h.calls.posts.some((c) => c.path === '/api/v1/command-policies/assist/'),
+        `expected assist POST, got: ${JSON.stringify(h.calls.posts.map((c) => c.path))}`
+      );
+    } finally {
+      h.restoreLog();
+    }
+  });
+});