testronaut 1.1.2 → 1.2.0

package/llm/llmFactory.js

@@ -0,0 +1,50 @@
+/**
+ * llmFactory.js
+ * --------------
+ * Purpose:
+ *   Provide a single entry point to obtain an LLM provider adapter.
+ *   Each adapter exposes a unified `chat({ model, messages, tools })`
+ *   and returns `{ message, usage }` where `message` is OpenAI-like.
+ *
+ * Responsibilities:
+ *   - Map provider name → concrete adapter instance.
+ *   - Normalize constructor options (env API keys + optional overrides).
+ *
+ * Message contract (internal, OpenAI-like):
+ *   - messages: Array<{ role: 'system'|'user'|'assistant'|'tool', content?: string, ... }>
+ *   - Assistant tool calls:
+ *       message.tool_calls = [{ id, type:'function', function:{ name, arguments:string }}]
+ *   - Tool messages:
+ *       { role:'tool', tool_call_id, name, type:'function', content:string }
+ *
+ * Related tests:
+ *   Located in `tests/llmTests/llmFactory.test.js`
+ *
+ * Used by:
+ *   - core/turnLoop.js (to obtain the active provider implementation)
+ */
+
+import { OpenAIProvider } from './openAI/openaiProvider.js';
+import { GeminiProvider } from './gemini/geminiProvider.js';
+
+/**
+ * Return an adapter implementing:
+ *   chat({ model, messages, tools }) -> Promise<{ message, usage }>
+ *
+ * @param {string} providerName - 'openai' | 'gemini'
+ * @param {object} [opts]       - Optional overrides (e.g., { apiKey, baseURL, ... })
+ * @returns {{ chat: Function }} provider adapter
+ */
+export function getLLM(providerName, opts = {}) {
+  switch ((providerName || '').toLowerCase()) {
+    case 'openai':
+      // NOTE: opts last → caller can override env key if needed (useful for tests)
+      return new OpenAIProvider({ apiKey: process.env.OPENAI_API_KEY, ...opts });
+
+    case 'gemini':
+      return new GeminiProvider({ apiKey: process.env.GEMINI_API_KEY, ...opts });
+
+    default:
+      throw new Error(`Unsupported LLM provider: ${providerName}`);
+  }
+}

package/llm/modelResolver.js

@@ -0,0 +1,65 @@
+/**
+ * modelResolver.js
+ * -----------------
+ * Purpose:
+ *   Determines which LLM provider and model Testronaut should use.
+ *   Reads from env first, then `testronaut-config.json`, then defaults.
+ *
+ * Resolution priority:
+ *   1) Env: TESTRONAUT_PROVIDER / TESTRONAUT_MODEL
+ *   2) Config file (provider + model)
+ *   3) Legacy config (model === "openai", no provider)
+ *   4) Default: { provider: "openai", model: "gpt-4o" }
+ *
+ * Related tests: tests/llmTests/modelResolver.test.js
+ * Used by: core/turnLoop.js, llm/llmFactory.js
+ */
+
+import path from 'path';
+import fs from 'fs';
+
+/**
+ * Resolve provider & model, honoring env overrides and safe defaults.
+ * @param {{cwd?: string}} [opts] - optional current working directory override (useful in tests)
+ * @returns {{provider: string, model: string}}
+ */
+export function resolveProviderModel(opts = {}) {
+  const envProvider = process.env.TESTRONAUT_PROVIDER?.trim();
+  const envModel    = process.env.TESTRONAUT_MODEL?.trim();
+
+  // 1) Env override (both provided)
+  if (envProvider && envModel) {
+    return { provider: envProvider, model: envModel };
+  }
+
+  // 2) Config lookup (compute path at call time)
+  const root = opts.cwd || process.cwd();
+  const configPath = path.join(root, 'testronaut-config.json');
+
+  try {
+    if (fs.existsSync(configPath)) {
+      const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+
+      // Modern config: explicit provider + model
+      if (cfg?.provider && cfg?.model) {
+        return {
+          provider: envProvider || cfg.provider,
+          model: envModel || cfg.model,
+        };
+      }
+
+      // Legacy: model === "openai" and no provider
+      if (!cfg?.provider && cfg?.model === 'openai') {
+        return { provider: 'openai', model: envModel || 'gpt-4o' };
+      }
+    }
+  } catch (e) {
+    console.warn('⚠️ Could not read testronaut-config.json; using defaults. Error:', e.message);
+  }
+
+  // 3) Defaults
+  return {
+    provider: envProvider || 'openai',
+    model: envModel || 'gpt-4o',
+  };
+}

package/llm/openAI/openaiProvider.js

@@ -0,0 +1,54 @@
+/**
+ * openaiProvider.js
+ * ------------------
+ * Purpose:
+ *   Adapter that wraps OpenAI's SDK and exposes a normalized `chat()` API
+ *   consistent with Testronaut’s OpenAI-like message format.
+ *
+ * Responsibilities:
+ *   - Call OpenAI Chat Completions with { model, messages, tools }.
+ *   - Return { message, usage, headers } where:
+ *       • message is OpenAI-like (already native)
+ *       • usage.total_tokens is forwarded from OpenAI
+ *       • headers (if present) can be used by the token limit learner
+ *
+ * Related tests:
+ *   Located in `tests/llmTests/openAIProvider.test.js`
+ *
+ * Used by:
+ *   - llm/llmFactory.js
+ */
+
+import OpenAI from 'openai';
+
+export class OpenAIProvider {
+  constructor({ apiKey } = {}) {
+    if (!apiKey) throw new Error('OPENAI_API_KEY is required for OpenAI provider');
+    this.client = new OpenAI({ apiKey });
+  }
+
+  /**
+   * Execute a chat turn via OpenAI and normalize the response.
+   * @param {{model:string, messages:any[], tools?:any[]}} params
+   * @returns {Promise<{message:any, usage?:{total_tokens?:number, providerRaw?:any}, headers?:any}>}
+   */
+  async chat({ model, messages, tools }) {
+    const res = await this.client.chat.completions.create({
+      model,
+      messages,
+      tools,
+    });
+
+    // OpenAI already returns an OpenAI-like message and usage structure.
+    const message = res.choices?.[0]?.message ?? { role: 'assistant', content: '' };
+    const usage = {
+      total_tokens: res.usage?.total_tokens,
+      providerRaw: res.usage,
+    };
+
+    // Some SDK versions surface headers on `res.headers`, others on `res.response.headers`.
+    const headers = res.headers || res.response?.headers;
+
+    return { message, usage, headers };
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.1.2",
+  "version": "1.2.0",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "./bin/cli.js"
@@ -12,7 +12,8 @@
   },
   "type": "module",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "start": "node ./bin/cli.js"
   },
   "author": "",
@@ -20,6 +21,7 @@
   "description": "",
   "dependencies": {
     "@dqbd/tiktoken": "^1.0.21",
+    "@google/generative-ai": "^0.24.1",
     "cheerio": "^1.1.0",
     "dotenv": "^17.2.0",
     "inquirer": "^12.9.1",
@@ -34,5 +36,9 @@
     "automation",
     "node",
     "qa"
-  ]
+  ],
+  "devDependencies": {
+    "@types/node": "^24.9.1",
+    "vitest": "^4.0.1"
+  }
 }

package/tests/binTests/init.test.js

@@ -0,0 +1,189 @@
+// tests/cliTests/init.test.js
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+// ─────────────────────────────────────────────
+// Hoisted queue for the `prompts` mock
+// ─────────────────────────────────────────────
+const { shared } = vi.hoisted(() => ({
+  shared: {
+    answers: [], // each element should be the resolved object for one prompts() call
+    calls: 0,
+  },
+}));
+
+// Mock `prompts` to return queued answers
+vi.mock('prompts', () => ({
+  default: async () => {
+    shared.calls += 1;
+    if (!shared.answers.length) {
+      throw new Error('Test prompts queue empty. Push expected answers to shared.answers.');
+    }
+    return shared.answers.shift();
+  },
+}));
+
+// Important: import SUT after mocks
+import { initializeTestronautProject } from '../../bin/init';
+
+function makeTempProject() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-init-'));
+}
+function readJson(p) {
+  return JSON.parse(fs.readFileSync(p, 'utf8'));
+}
+function fileExists(p) {
+  try { return fs.statSync(p).isFile(); } catch { return false; }
+}
+function dirExists(p) {
+  try { return fs.statSync(p).isDirectory(); } catch { return false; }
+}
+
+const originalCwd = process.cwd();
+const ORIGINAL_ENV = { ...process.env };
+
+describe('initializeTestronautProject', () => {
+  beforeEach(() => {
+    process.env = { ...ORIGINAL_ENV };
+    shared.answers = [];
+    shared.calls = 0;
+  });
+
+  afterEach(() => {
+    process.env = { ...ORIGINAL_ENV };
+    process.chdir(originalCwd);
+  });
+
+  it('scaffolds a new OpenAI project: config, .env, folders, welcome mission', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // Queue answers for two prompts:
+    // 1) provider
+    // 2) openai model
+    shared.answers.push({ llmProvider: 'openai' });
+    shared.answers.push({ openaiModel: 'gpt-4o' });
+
+    await initializeTestronautProject();
+
+    // Config
+    const configPath = path.join(temp, 'testronaut-config.json');
+    expect(fileExists(configPath)).toBe(true);
+    const cfg = readJson(configPath);
+    expect(cfg.provider).toBe('openai');
+    expect(cfg.model).toBe('gpt-4o');
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.outputDir).toBe('missions/mission_reports');
+    expect(typeof cfg.maxTurns).toBe('number');
+
+    // .env
+    const envPath = path.join(temp, '.env');
+    expect(fileExists(envPath)).toBe(true);
+    const envTxt = fs.readFileSync(envPath, 'utf8');
+    expect(envTxt).toMatch(/OPENAI_API_KEY=sk-/);
+
+    // Folders
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    // Welcome mission (name depends on your implementation; we check presence in missions/)
+    const files = fs.readdirSync(path.join(temp, 'missions'));
+    expect(files.some(f => /welcome/i.test(f))).toBe(true);
+
+    // Exactly two prompts were asked
+    expect(shared.calls).toBe(2);
+  });
+
+  it('scaffolds a new Gemini project: config, .env, folders, welcome mission', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    shared.answers.push({ llmProvider: 'gemini' });
+    shared.answers.push({ geminiModel: 'gemini-2.5-flash' });
+
+    await initializeTestronautProject();
+
+    const cfg = readJson(path.join(temp, 'testronaut-config.json'));
+    expect(cfg.provider).toBe('gemini');
+    expect(cfg.model).toBe('gemini-2.5-flash');
+
+    const envTxt = fs.readFileSync(path.join(temp, '.env'), 'utf8');
+    expect(envTxt).toMatch(/GEMINI_API_KEY=AIza/);
+
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    const files = fs.readdirSync(path.join(temp, 'missions'));
+    expect(files.some(f => /welcome/i.test(f))).toBe(true);
+
+    expect(shared.calls).toBe(2);
+  });
+
+  it('is idempotent: when initialized, it skips prompts and does not overwrite .env', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // Pre-create a config marked as initialized + a non-empty .env
+    const configPath = path.join(temp, 'testronaut-config.json');
+    fs.writeFileSync(
+      configPath,
+      JSON.stringify({
+        initialized: true,
+        provider: 'openai',
+        model: 'gpt-4o',
+        outputDir: 'missions/mission_reports',
+        projectName: 'demo',
+        maxTurns: 20,
+      }, null, 2),
+      'utf8'
+    );
+
+    const envPath = path.join(temp, '.env');
+    fs.writeFileSync(envPath, 'DO_NOT_TOUCH=1\n', 'utf8');
+
+    // Also ensure folders are missing to verify folder creation still happens
+    // (initialize should still ensure dirs even when skipping prompts)
+    await initializeTestronautProject();
+
+    // Prompts should not have been called
+    expect(shared.calls).toBe(0);
+
+    // .env untouched
+    const envTxt = fs.readFileSync(envPath, 'utf8');
+    expect(envTxt).toBe('DO_NOT_TOUCH=1\n');
+
+    // Folders ensured
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    // Config left as-is (no clobber)
+    const cfg = readJson(configPath);
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.provider).toBe('openai');
+    expect(cfg.model).toBe('gpt-4o');
+  });
+
+  it('preserves pre-existing fields and merges defaults without clobbering', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // seed a partial config (not initialized yet)
+    fs.writeFileSync(
+      path.join(temp, 'testronaut-config.json'),
+      JSON.stringify({ projectName: 'my-project' }, null, 2),
+      'utf8'
+    );
+
+    shared.answers.push({ llmProvider: 'openai' });
+    shared.answers.push({ openaiModel: 'gpt-4.1-mini' });
+
+    await initializeTestronautProject();
+
+    const cfg = readJson(path.join(temp, 'testronaut-config.json'));
+    expect(cfg.projectName).toBe('my-project'); // preserved
+    expect(cfg.outputDir).toBe('missions/mission_reports'); // default filled in
+    expect(cfg.maxTurns).toBe(20); // default filled in
+  });
+});

package/tests/binTests/initHelpersTests/defaultConfig.test.js

@@ -0,0 +1,12 @@
+import { describe, it, expect } from 'vitest';
+import { defaultConfig } from '../../../bin/initHelpers';
+
+describe('defaultConfig', () => {
+  it('creates sane defaults', () => {
+    const cfg = defaultConfig('my-app');
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.outputDir).toBe('missions/mission_reports');
+    expect(cfg.projectName).toBe('my-app');
+    expect(cfg.maxTurns).toBe(20);
+  });
+});

package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js

@@ -0,0 +1,14 @@
+import { describe, it, expect } from 'vitest';
+import { makeEnvTemplate } from '../../../bin/initHelpers';
+
+describe('makeEnvTemplate', () => {
+  it('returns OpenAI template', () => {
+    const s = makeEnvTemplate('openai');
+    expect(s).toMatch(/OPENAI_API_KEY=sk-\.\.\./);
+  });
+
+  it('returns Gemini template', () => {
+    const s = makeEnvTemplate('gemini');
+    expect(s).toMatch(/GEMINI_API_KEY=AIza\.\.\./);
+  });
+});

package/tests/binTests/initHelpersTests/migrateLegacyConfig.test.js

@@ -0,0 +1,19 @@
+import { describe, it, expect } from 'vitest';
+import { migrateLegacyConfig } from '../../../bin/initHelpers';
+
+describe('migrateLegacyConfig', () => {
+  it('sets provider=openai when model=openai and provider is missing', () => {
+    const out = migrateLegacyConfig({ model: 'openai' });
+    expect(out.provider).toBe('openai');
+  });
+
+  it('does not override existing provider', () => {
+    const out = migrateLegacyConfig({ model: 'openai', provider: 'gemini' });
+    expect(out.provider).toBe('gemini');
+  });
+
+  it('keeps unrelated configs intact', () => {
+    const out = migrateLegacyConfig({ foo: 1, bar: 'x' });
+    expect(out).toEqual({ foo: 1, bar: 'x' });
+  });
+});

package/tests/binTests/initHelpersTests/models.test.js

@@ -0,0 +1,21 @@
+import { describe, it, expect } from 'vitest';
+import { openAIModels, geminiModels, isKnownModel, pickInitialIndex } from '../../../bin/initHelpers';
+
+describe('model helpers', () => {
+  it('lists known models', () => {
+    expect(openAIModels()).toContain('gpt-4o');
+    expect(geminiModels()).toContain('gemini-2.5-flash');
+  });
+
+  it('recognizes known models by provider', () => {
+    expect(isKnownModel('openai', 'gpt-4.1-mini')).toBe(true);
+    expect(isKnownModel('gemini', 'gpt-4.1-mini')).toBe(false);
+  });
+
+  it('picks initial index from list or fallback', () => {
+    const list = ['a','b','c'];
+    expect(pickInitialIndex(list, 'b', 'c')).toBe(1);
+    expect(pickInitialIndex(list, 'z', 'c')).toBe(2);
+    expect(pickInitialIndex(list, 'z', 'y')).toBe(0);
+  });
+});

package/tests/coreTests/turnLoop.test.js

@@ -0,0 +1,177 @@
+// tests/turnLoopTests/turnLoop.test.js
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+
+// ─────────────────────────────────────────────
+// Hoisted shared state (visible to mock factories)
+// ─────────────────────────────────────────────
+const { shared } = vi.hoisted(() => {
+  return {
+    shared: {
+      // LLM chat mock will be attached here by the llmFactory mock
+      chatMock: null,
+
+      // Browser tool spies available to the chromeBrowser mock
+      chromeToolSpies: {
+        click_text: vi.fn(async () => JSON.stringify({ ok: true, clicked: 'Hello' })),
+        get_dom: vi.fn(async () => '<html>dom</html>'),
+        screenshot: vi.fn(async () =>
+          JSON.stringify({
+            _testronaut_file_event: 'download',
+            fileName: 'shot.png',
+            bytes: 1234,
+            mode: 'save',
+          })
+        ),
+      },
+    },
+  };
+});
+
+// ─────────────────────────────────────────────
+// Mocks
+// ─────────────────────────────────────────────
+
+vi.mock('../../llm/modelResolver.js', () => ({
+  resolveProviderModel: () => ({ provider: 'stub', model: 'stub-model' }),
+}));
+
+// Use hoisted shared.chatMock
+vi.mock('../../llm/llmFactory.js', () => ({
+  getLLM: () => {
+    shared.chatMock = vi.fn();
+    return { chat: shared.chatMock };
+  },
+}));
+
+vi.mock('../../tools/tokenControl.js', () => ({
+  tokenEstimate: vi.fn(async () => 0),
+  tokenUseCoolOff: vi.fn(async (totalTokensUsed, turnTimestamps) => ({
+    shouldBackoff: false,
+    totalTokensUsed,
+    turnTimestamps,
+  })),
+  recordTokenUsage: vi.fn(() => {}),
+  pruneOldTokenUsage: vi.fn((turnTimestamps) => ({
+    turnTimestamps,
+    totalTokensUsed: turnTimestamps.reduce((a, [, t]) => a + t, 0),
+  })),
+  updateLimitsFromHeaders: vi.fn(() => {}),
+}));
+
+vi.mock('../../tools/toolSchema.js', () => ({
+  default: [
+    { type: 'function', function: { name: 'click_text', description: 'click by text', parameters: {} } },
+    { type: 'function', function: { name: 'get_dom', description: 'get DOM html', parameters: {} } },
+    { type: 'function', function: { name: 'screenshot', description: 'take a screenshot', parameters: {} } },
+  ],
+}));
+
+// Use hoisted shared.chromeToolSpies
+vi.mock('../../tools/chromeBrowser.js', () => ({
+  CHROME_TOOL_MAP: shared.chromeToolSpies,
+}));
+
+vi.mock('../../tools/turnLoopUtils.js', async () => {
+  const real = await vi.importActual('../../tools/turnLoopUtils.js');
+  return {
+    ...real,
+    finalResponseHandler: (msg) => {
+      const content = (msg?.content || '').toString();
+      if (content.includes('FINAL')) {
+        return { finalMessage: 'Mission complete', success: true };
+      }
+      return null;
+    },
+    validateAndInsertMissingToolResponses: () => true,
+    wait: vi.fn(async () => {}),
+  };
+});
+
+vi.mock('../../core/turnIntent.js', () => ({
+  summarizeTurnIntentFromMessage: (msg) => `plan:${(msg?.content || '').slice(0, 20)}`,
+}));
+
+vi.mock('../../core/redaction.js', () => ({
+  redactArgs: (name, args) => args,
+}));
+
+// Import SUT after mocks
+import { turnLoop } from '../../core/turnLoop.js';
+
+// Helper
+function baseMessages() {
+  return [
+    { role: 'system', content: 'You are Testronaut.' },
+    { role: 'user', content: 'Start mission' },
+  ];
+}
+
+describe('turnLoop', () => {
+  let browser;
+
+  beforeEach(() => {
+    browser = {};
+    // reset spies and chat mock
+    shared.chromeToolSpies.click_text.mockClear();
+    shared.chromeToolSpies.get_dom.mockClear();
+    shared.chromeToolSpies.screenshot.mockClear();
+    if (shared.chatMock) shared.chatMock.mockReset();
+  });
+
+  it('handles a final response in a single turn (no tools)', async () => {
+    shared.chatMock.mockResolvedValueOnce({
+      message: { role: 'assistant', content: 'FINAL: success' },
+      usage: { total_tokens: 42 },
+      headers: {},
+    });
+
+    const messages = baseMessages();
+    const result = await turnLoop(browser, messages, 1, 0, 0, {}, { steps: [], missionName: 'demo' });
+
+    expect(result).toBeTruthy();
+    expect(result.steps?.length).toBe(1);
+    const step = result.steps[0];
+    expect(step.tokensUsed).toBe(42);
+    expect(step.summary).toContain('plan:FINAL: success');
+    expect(step.result).toMatch(/✅|Success/);
+
+    expect(shared.chromeToolSpies.click_text).not.toHaveBeenCalled();
+    expect(shared.chromeToolSpies.get_dom).not.toHaveBeenCalled();
+  });
+
+  it('executes a tool call, injects DOM, then finishes on next turn', async () => {
+    shared.chatMock.mockResolvedValueOnce({
+      message: {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'tool_1',
+            type: 'function',
+            function: { name: 'click_text', arguments: JSON.stringify({ text: 'Hello' }) },
+          },
+        ],
+      },
+      usage: { total_tokens: 10 },
+    });
+
+    shared.chatMock.mockResolvedValueOnce({
+      message: { role: 'assistant', content: 'FINAL: done' },
+      usage: { total_tokens: 5 },
+    });
+
+    const messages = baseMessages();
+    const res = await turnLoop(browser, messages, 2, 0, 0, {}, { steps: [], missionName: 'demo' });
+
+    expect(res).toBeTruthy();
+    expect(res.steps.length).toBe(2);
+
+    const step1 = res.steps[0];
+    expect(step1.result).toBe('✅ Passed');
+    expect(shared.chromeToolSpies.click_text).toHaveBeenCalledTimes(1);
+    expect(shared.chromeToolSpies.get_dom).toHaveBeenCalled();
+
+    const step2 = res.steps[1];
+    expect(step2.result).toMatch(/Success/);
+  });
+});