testronaut 1.1.2 → 1.2.1

package/llm/llmFactory.js

@@ -0,0 +1,50 @@
+/**
+ * llmFactory.js
+ * --------------
+ * Purpose:
+ *   Provide a single entry point to obtain an LLM provider adapter.
+ *   Each adapter exposes a unified `chat({ model, messages, tools })`
+ *   and returns `{ message, usage }` where `message` is OpenAI-like.
+ *
+ * Responsibilities:
+ *   - Map provider name → concrete adapter instance.
+ *   - Normalize constructor options (env API keys + optional overrides).
+ *
+ * Message contract (internal, OpenAI-like):
+ *   - messages: Array<{ role: 'system'|'user'|'assistant'|'tool', content?: string, ... }>
+ *   - Assistant tool calls:
+ *       message.tool_calls = [{ id, type:'function', function:{ name, arguments:string }}]
+ *   - Tool messages:
+ *       { role:'tool', tool_call_id, name, type:'function', content:string }
+ *
+ * Related tests:
+ *   Located in `tests/llmTests/llmFactory.test.js`
+ *
+ * Used by:
+ *   - core/turnLoop.js (to obtain the active provider implementation)
+ */
+
+import { OpenAIProvider } from './openAI/openaiProvider.js';
+import { GeminiProvider } from './gemini/geminiProvider.js';
+
+/**
+ * Return an adapter implementing:
+ *   chat({ model, messages, tools }) -> Promise<{ message, usage }>
+ *
+ * @param {string} providerName - 'openai' | 'gemini'
+ * @param {object} [opts]       - Optional overrides (e.g., { apiKey, baseURL, ... })
+ * @returns {{ chat: Function }} provider adapter
+ */
+export function getLLM(providerName, opts = {}) {
+  switch ((providerName || '').toLowerCase()) {
+    case 'openai':
+      // NOTE: opts last → caller can override env key if needed (useful for tests)
+      return new OpenAIProvider({ apiKey: process.env.OPENAI_API_KEY, ...opts });
+
+    case 'gemini':
+      return new GeminiProvider({ apiKey: process.env.GEMINI_API_KEY, ...opts });
+
+    default:
+      throw new Error(`Unsupported LLM provider: ${providerName}`);
+  }
+}

package/llm/modelResolver.js

@@ -0,0 +1,65 @@
+/**
+ * modelResolver.js
+ * -----------------
+ * Purpose:
+ *   Determines which LLM provider and model Testronaut should use.
+ *   Reads from env first, then `testronaut-config.json`, then defaults.
+ *
+ * Resolution priority:
+ *   1) Env: TESTRONAUT_PROVIDER / TESTRONAUT_MODEL
+ *   2) Config file (provider + model)
+ *   3) Legacy config (model === "openai", no provider)
+ *   4) Default: { provider: "openai", model: "gpt-4o" }
+ *
+ * Related tests: tests/llmTests/modelResolver.test.js
+ * Used by: core/turnLoop.js, llm/llmFactory.js
+ */
+
+import path from 'path';
+import fs from 'fs';
+
+/**
+ * Resolve provider & model, honoring env overrides and safe defaults.
+ * @param {{cwd?: string}} [opts] - optional current working directory override (useful in tests)
+ * @returns {{provider: string, model: string}}
+ */
+export function resolveProviderModel(opts = {}) {
+  const envProvider = process.env.TESTRONAUT_PROVIDER?.trim();
+  const envModel    = process.env.TESTRONAUT_MODEL?.trim();
+
+  // 1) Env override (both provided)
+  if (envProvider && envModel) {
+    return { provider: envProvider, model: envModel };
+  }
+
+  // 2) Config lookup (compute path at call time)
+  const root = opts.cwd || process.cwd();
+  const configPath = path.join(root, 'testronaut-config.json');
+
+  try {
+    if (fs.existsSync(configPath)) {
+      const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+
+      // Modern config: explicit provider + model
+      if (cfg?.provider && cfg?.model) {
+        return {
+          provider: envProvider || cfg.provider,
+          model: envModel || cfg.model,
+        };
+      }
+
+      // Legacy: model === "openai" and no provider
+      if (!cfg?.provider && cfg?.model === 'openai') {
+        return { provider: 'openai', model: envModel || 'gpt-4o' };
+      }
+    }
+  } catch (e) {
+    console.warn('⚠️ Could not read testronaut-config.json; using defaults. Error:', e.message);
+  }
+
+  // 3) Defaults
+  return {
+    provider: envProvider || 'openai',
+    model: envModel || 'gpt-4o',
+  };
+}

package/llm/openAI/openaiProvider.js

@@ -0,0 +1,54 @@
+/**
+ * openaiProvider.js
+ * ------------------
+ * Purpose:
+ *   Adapter that wraps OpenAI's SDK and exposes a normalized `chat()` API
+ *   consistent with Testronaut’s OpenAI-like message format.
+ *
+ * Responsibilities:
+ *   - Call OpenAI Chat Completions with { model, messages, tools }.
+ *   - Return { message, usage, headers } where:
+ *       • message is OpenAI-like (already native)
+ *       • usage.total_tokens is forwarded from OpenAI
+ *       • headers (if present) can be used by the token limit learner
+ *
+ * Related tests:
+ *   Located in `tests/llmTests/openAIProvider.test.js`
+ *
+ * Used by:
+ *   - llm/llmFactory.js
+ */
+
+import OpenAI from 'openai';
+
+export class OpenAIProvider {
+  constructor({ apiKey } = {}) {
+    if (!apiKey) throw new Error('OPENAI_API_KEY is required for OpenAI provider');
+    this.client = new OpenAI({ apiKey });
+  }
+
+  /**
+   * Execute a chat turn via OpenAI and normalize the response.
+   * @param {{model:string, messages:any[], tools?:any[]}} params
+   * @returns {Promise<{message:any, usage?:{total_tokens?:number, providerRaw?:any}, headers?:any}>}
+   */
+  async chat({ model, messages, tools }) {
+    const res = await this.client.chat.completions.create({
+      model,
+      messages,
+      tools,
+    });
+
+    // OpenAI already returns an OpenAI-like message and usage structure.
+    const message = res.choices?.[0]?.message ?? { role: 'assistant', content: '' };
+    const usage = {
+      total_tokens: res.usage?.total_tokens,
+      providerRaw: res.usage,
+    };
+
+    // Some SDK versions surface headers on `res.headers`, others on `res.response.headers`.
+    const headers = res.headers || res.response?.headers;
+
+    return { message, usage, headers };
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.1.2",
+  "version": "1.2.1",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "./bin/cli.js"
@@ -12,7 +12,8 @@
   },
   "type": "module",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "start": "node ./bin/cli.js"
   },
   "author": "",
@@ -20,6 +21,7 @@
   "description": "",
   "dependencies": {
     "@dqbd/tiktoken": "^1.0.21",
+    "@google/generative-ai": "^0.24.1",
     "cheerio": "^1.1.0",
     "dotenv": "^17.2.0",
     "inquirer": "^12.9.1",
@@ -34,5 +36,9 @@
     "automation",
     "node",
     "qa"
-  ]
+  ],
+  "devDependencies": {
+    "@types/node": "^24.9.1",
+    "vitest": "^4.0.1"
+  }
 }

package/tests/binTests/init.test.js

@@ -0,0 +1,189 @@
+// tests/cliTests/init.test.js
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+// ─────────────────────────────────────────────
+// Hoisted queue for the `prompts` mock
+// ─────────────────────────────────────────────
+const { shared } = vi.hoisted(() => ({
+  shared: {
+    answers: [], // each element should be the resolved object for one prompts() call
+    calls: 0,
+  },
+}));
+
+// Mock `prompts` to return queued answers
+vi.mock('prompts', () => ({
+  default: async () => {
+    shared.calls += 1;
+    if (!shared.answers.length) {
+      throw new Error('Test prompts queue empty. Push expected answers to shared.answers.');
+    }
+    return shared.answers.shift();
+  },
+}));
+
+// Important: import SUT after mocks
+import { initializeTestronautProject } from '../../bin/init';
+
+function makeTempProject() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-init-'));
+}
+function readJson(p) {
+  return JSON.parse(fs.readFileSync(p, 'utf8'));
+}
+function fileExists(p) {
+  try { return fs.statSync(p).isFile(); } catch { return false; }
+}
+function dirExists(p) {
+  try { return fs.statSync(p).isDirectory(); } catch { return false; }
+}
+
+const originalCwd = process.cwd();
+const ORIGINAL_ENV = { ...process.env };
+
+describe('initializeTestronautProject', () => {
+  beforeEach(() => {
+    process.env = { ...ORIGINAL_ENV };
+    shared.answers = [];
+    shared.calls = 0;
+  });
+
+  afterEach(() => {
+    process.env = { ...ORIGINAL_ENV };
+    process.chdir(originalCwd);
+  });
+
+  it('scaffolds a new OpenAI project: config, .env, folders, welcome mission', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // Queue answers for two prompts:
+    // 1) provider
+    // 2) openai model
+    shared.answers.push({ llmProvider: 'openai' });
+    shared.answers.push({ openaiModel: 'gpt-4o' });
+
+    await initializeTestronautProject();
+
+    // Config
+    const configPath = path.join(temp, 'testronaut-config.json');
+    expect(fileExists(configPath)).toBe(true);
+    const cfg = readJson(configPath);
+    expect(cfg.provider).toBe('openai');
+    expect(cfg.model).toBe('gpt-4o');
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.outputDir).toBe('missions/mission_reports');
+    expect(typeof cfg.maxTurns).toBe('number');
+
+    // .env
+    const envPath = path.join(temp, '.env');
+    expect(fileExists(envPath)).toBe(true);
+    const envTxt = fs.readFileSync(envPath, 'utf8');
+    expect(envTxt).toMatch(/OPENAI_API_KEY=sk-/);
+
+    // Folders
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    // Welcome mission (name depends on your implementation; we check presence in missions/)
+    const files = fs.readdirSync(path.join(temp, 'missions'));
+    expect(files.some(f => /welcome/i.test(f))).toBe(true);
+
+    // Exactly two prompts were asked
+    expect(shared.calls).toBe(2);
+  });
+
+  it('scaffolds a new Gemini project: config, .env, folders, welcome mission', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    shared.answers.push({ llmProvider: 'gemini' });
+    shared.answers.push({ geminiModel: 'gemini-2.5-flash' });
+
+    await initializeTestronautProject();
+
+    const cfg = readJson(path.join(temp, 'testronaut-config.json'));
+    expect(cfg.provider).toBe('gemini');
+    expect(cfg.model).toBe('gemini-2.5-flash');
+
+    const envTxt = fs.readFileSync(path.join(temp, '.env'), 'utf8');
+    expect(envTxt).toMatch(/GEMINI_API_KEY=AIza/);
+
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    const files = fs.readdirSync(path.join(temp, 'missions'));
+    expect(files.some(f => /welcome/i.test(f))).toBe(true);
+
+    expect(shared.calls).toBe(2);
+  });
+
+  it('is idempotent: when initialized, it skips prompts and does not overwrite .env', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // Pre-create a config marked as initialized + a non-empty .env
+    const configPath = path.join(temp, 'testronaut-config.json');
+    fs.writeFileSync(
+      configPath,
+      JSON.stringify({
+        initialized: true,
+        provider: 'openai',
+        model: 'gpt-4o',
+        outputDir: 'missions/mission_reports',
+        projectName: 'demo',
+        maxTurns: 20,
+      }, null, 2),
+      'utf8'
+    );
+
+    const envPath = path.join(temp, '.env');
+    fs.writeFileSync(envPath, 'DO_NOT_TOUCH=1\n', 'utf8');
+
+    // Also ensure folders are missing to verify folder creation still happens
+    // (initialize should still ensure dirs even when skipping prompts)
+    await initializeTestronautProject();
+
+    // Prompts should not have been called
+    expect(shared.calls).toBe(0);
+
+    // .env untouched
+    const envTxt = fs.readFileSync(envPath, 'utf8');
+    expect(envTxt).toBe('DO_NOT_TOUCH=1\n');
+
+    // Folders ensured
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    // Config left as-is (no clobber)
+    const cfg = readJson(configPath);
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.provider).toBe('openai');
+    expect(cfg.model).toBe('gpt-4o');
+  });
+
+  it('preserves pre-existing fields and merges defaults without clobbering', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // seed a partial config (not initialized yet)
+    fs.writeFileSync(
+      path.join(temp, 'testronaut-config.json'),
+      JSON.stringify({ projectName: 'my-project' }, null, 2),
+      'utf8'
+    );
+
+    shared.answers.push({ llmProvider: 'openai' });
+    shared.answers.push({ openaiModel: 'gpt-4.1-mini' });
+
+    await initializeTestronautProject();
+
+    const cfg = readJson(path.join(temp, 'testronaut-config.json'));
+    expect(cfg.projectName).toBe('my-project'); // preserved
+    expect(cfg.outputDir).toBe('missions/mission_reports'); // default filled in
+    expect(cfg.maxTurns).toBe(20); // default filled in
+  });
+});

package/tests/binTests/initHelpersTests/defaultConfig.test.js

@@ -0,0 +1,12 @@
+import { describe, it, expect } from 'vitest';
+import { defaultConfig } from '../../../bin/initHelpers';
+
+describe('defaultConfig', () => {
+  it('creates sane defaults', () => {
+    const cfg = defaultConfig('my-app');
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.outputDir).toBe('missions/mission_reports');
+    expect(cfg.projectName).toBe('my-app');
+    expect(cfg.maxTurns).toBe(20);
+  });
+});

package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js

@@ -0,0 +1,14 @@
+import { describe, it, expect } from 'vitest';
+import { makeEnvTemplate } from '../../../bin/initHelpers';
+
+describe('makeEnvTemplate', () => {
+  it('returns OpenAI template', () => {
+    const s = makeEnvTemplate('openai');
+    expect(s).toMatch(/OPENAI_API_KEY=sk-\.\.\./);
+  });
+
+  it('returns Gemini template', () => {
+    const s = makeEnvTemplate('gemini');
+    expect(s).toMatch(/GEMINI_API_KEY=AIza\.\.\./);
+  });
+});

package/tests/binTests/initHelpersTests/migrateLegacyConfig.test.js

@@ -0,0 +1,19 @@
+import { describe, it, expect } from 'vitest';
+import { migrateLegacyConfig } from '../../../bin/initHelpers';
+
+describe('migrateLegacyConfig', () => {
+  it('sets provider=openai when model=openai and provider is missing', () => {
+    const out = migrateLegacyConfig({ model: 'openai' });
+    expect(out.provider).toBe('openai');
+  });
+
+  it('does not override existing provider', () => {
+    const out = migrateLegacyConfig({ model: 'openai', provider: 'gemini' });
+    expect(out.provider).toBe('gemini');
+  });
+
+  it('keeps unrelated configs intact', () => {
+    const out = migrateLegacyConfig({ foo: 1, bar: 'x' });
+    expect(out).toEqual({ foo: 1, bar: 'x' });
+  });
+});

package/tests/binTests/initHelpersTests/models.test.js

@@ -0,0 +1,21 @@
+import { describe, it, expect } from 'vitest';
+import { openAIModels, geminiModels, isKnownModel, pickInitialIndex } from '../../../bin/initHelpers';
+
+describe('model helpers', () => {
+  it('lists known models', () => {
+    expect(openAIModels()).toContain('gpt-4o');
+    expect(geminiModels()).toContain('gemini-2.5-flash');
+  });
+
+  it('recognizes known models by provider', () => {
+    expect(isKnownModel('openai', 'gpt-4.1-mini')).toBe(true);
+    expect(isKnownModel('gemini', 'gpt-4.1-mini')).toBe(false);
+  });
+
+  it('picks initial index from list or fallback', () => {
+    const list = ['a','b','c'];
+    expect(pickInitialIndex(list, 'b', 'c')).toBe(1);
+    expect(pickInitialIndex(list, 'z', 'c')).toBe(2);
+    expect(pickInitialIndex(list, 'z', 'y')).toBe(0);
+  });
+});

package/tests/coreTests/redation.test.js

@@ -0,0 +1,118 @@
+/**
+ * redaction.test.js
+ * -----------------
+ * Purpose:
+ *   Vitest unit tests for redaction utilities:
+ *   - Centralized keyword usage
+ *   - Tool arg masking (selector & key-name based)
+ *   - Free-text masking for plans/instructions
+ *
+ * Test runner:
+ *   Vitest (https://vitest.dev/)
+ *
+ * Location:
+ *   tests/core/redaction.test.js
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  SENSITIVE_KEYWORDS,
+  maskPreview,
+  redactArgs,
+  redactPasswordInText,
+} from '../../core/redaction.js';
+
+describe('redaction utilities', () => {
+  describe('SENSITIVE_KEYWORDS (centralized list)', () => {
+    it('contains expected baseline keywords', () => {
+      const expected = ['password', 'token', 'access', 'api-key', 'auth'];
+      for (const k of expected) {
+        expect(SENSITIVE_KEYWORDS).toContain(k);
+      }
+    });
+  });
+
+  describe('maskPreview', () => {
+    it('masks and includes length by default', () => {
+      expect(maskPreview('hunter2')).toBe('•••••• (7)');
+    });
+
+    it('masks empty-ish values without length', () => {
+      expect(maskPreview('')).toBe('••••••');
+      expect(maskPreview(null)).toBe('••••••');
+      expect(maskPreview(undefined)).toBe('••••••');
+    });
+
+    it('respects showLength=false', () => {
+      expect(maskPreview('abcdef', false)).toBe('••••••');
+    });
+  });
+
+  describe('redactArgs', () => {
+    it('masks when selector contains #access', () => {
+      const args = { selector: '#access', text: 'Imp0st3r123!' };
+      const redacted = redactArgs('fill', args);
+      expect(redacted.text).toMatch(/^•••••• \(\d+\)$/);
+      expect(redacted.selector).toBe('#access'); // selector remains visible
+    });
+
+    it('masks when inputType is password', () => {
+      const args = { selector: '#login', inputType: 'password', value: 'superSecret' };
+      const redacted = redactArgs('type', args);
+      expect(redacted.value).toBe('•••••• (11)');
+    });
+
+    it('masks values by sensitive arg key names (e.g., apiKey, token)', () => {
+      const args = { apiKey: 'sk-live-abc', token: 'abcd1234', other: 'notSecret' };
+      const redacted = redactArgs('noop', args); // fnName not type/fill still masks by key name
+      expect(redacted.apiKey).toBe('•••••• (11)');
+      expect(redacted.token).toBe('•••••• (8)');
+      expect(redacted.other).toBe('notSecret');
+    });
+
+    it('does not mask non-sensitive calls', () => {
+      const args = { selector: '#search', text: 'hello world' };
+      const redacted = redactArgs('fill', args);
+      expect(redacted.text).toBe('hello world');
+    });
+  });
+
+  describe('redactPasswordInText', () => {
+    it('Case 1: keyword ... "VALUE"', () => {
+      const input = `My password is "hunter2".`;
+      const out = redactPasswordInText(input);
+      expect(out).toContain(`password is "•••••• (7)"`);
+    });
+
+    it('Case 2: keyword ... (with|as|to|=) VALUE', () => {
+      const input = `token=abcd1234 and auth with xyz`;
+      const out = redactPasswordInText(input);
+      expect(out).toContain(`token=•••••• (8)`);
+      expect(out).toContain(`auth with •••••• (3)`);
+    });
+
+    it('Case 3: "VALUE" into sensitive target (#access)', () => {
+      const input = `Type "Imp0st3r123!" into #access`;
+      const out = redactPasswordInText(input);
+      expect(out).toBe(`Type "•••••• (12)" into #access`);
+    });
+
+    it('non-sensitive text is unchanged', () => {
+      const input = `Click the button and type "hello" into #search`;
+      const out = redactPasswordInText(input);
+      expect(out).toBe(input);
+    });
+
+    it('handles api key forms with spaces/underscores/hyphens', () => {
+      const cases = [
+        `api-key "abcDEF123"`,
+        `api key "abcDEF123"`,
+        `api_key "abcDEF123"`,
+      ];
+      for (const text of cases) {
+        const out = redactPasswordInText(text);
+        expect(out).toContain(`"•••••• (9)"`);
+      }
+    });
+  });
+});