testronaut 1.1.1 → 1.2.0

package/llm/gemini/geminiProvider.js

@@ -0,0 +1,188 @@
+/**
+ * geminiProvider.js
+ * ------------------
+ * Purpose:
+ *   Adapter that normalizes Testronaut's OpenAI-like chat format to
+ *   Google Gemini's SDK and back. Exposes a single `chat()` method.
+ *
+ * Responsibilities:
+ *   - Convert OpenAI-like messages → Gemini "contents" (roles/parts).
+ *   - Map tool/function calling both ways:
+ *       • assistant.tool_calls → Gemini functionCall parts
+ *       • tool messages → user parts (so the model can read results)
+ *   - Return an OpenAI-like assistant message and usage metadata.
+ *
+ * Message contract (OpenAI-like, internal):
+ *   - messages: Array<{ role: 'system'|'user'|'assistant'|'tool', content?: string|Array, ... }>
+ *   - Assistant tool calls:
+ *       message.tool_calls = [{ id, type:'function', function:{ name, arguments:string }}]
+ *   - Tool messages:
+ *       { role:'tool', tool_call_id, name, type:'function', content:string }
+ *
+ * Related tests:
+ *   Located in `tests/llmTests/geminiProvider.test.js`
+ *
+ * Used by:
+ *   - llm/llmFactory.js
+ */
+
+import { GoogleGenerativeAI } from '@google/generative-ai';
+
+/**
+ * Convert OpenAI-like messages → Gemini "contents" array.
+ * - System messages are coalesced and injected as a prefix into the next user turn.
+ * - Assistant tool calls are represented as model turns with functionCall parts.
+ * - Tool results are encoded as a user turn with a JSON payload part.
+ * - Text/images are mapped to Gemini `parts` (text / inlineData).
+ */
+function toGeminiContents(messages) {
+  const contents = [];
+  let systemPrefix = '';
+
+  for (const m of messages) {
+    if (m.role === 'system') {
+      // Collect multi-system messages; inject once on the next user message
+      const sysText = Array.isArray(m.content)
+        ? m.content.map(p => (typeof p === 'string' ? p : p.text || '')).join('\n')
+        : (m.content || '');
+      systemPrefix += (systemPrefix ? '\n' : '') + sysText;
+      continue;
+    }
+
+    // Assistant tool calls → Gemini functionCall parts on a model turn
+    if (m.role === 'assistant' && Array.isArray(m.tool_calls) && m.tool_calls.length) {
+      contents.push({
+        role: 'model',
+        parts: m.tool_calls.map(tc => ({
+          functionCall: {
+            name: tc.function?.name,
+            args: safeJsonParse(tc.function?.arguments) ?? {},
+          },
+        })),
+      });
+      continue;
+    }
+
+    // Tool results → encode as a user turn with a JSON part the model can parse
+    if (m.role === 'tool') {
+      const parts = [{
+        text: JSON.stringify({
+          _tool_result: true,
+          tool_call_id: m.tool_call_id,
+          name: m.name,
+          content: m.content,
+        }),
+      }];
+      contents.push({ role: 'user', parts });
+      continue;
+    }
+
+    // Normal user/assistant content (text and/or images)
+    const baseParts = [];
+    const asArray = Array.isArray(m.content) ? m.content : [{ type: 'text', text: m.content }];
+    for (const p of asArray) {
+      if (p?.type === 'image') {
+        baseParts.push({
+          inlineData: {
+            mimeType: p.mimeType,
+            data: Buffer.from(p.data).toString('base64'),
+          }
+        });
+      } else {
+        baseParts.push({ text: typeof p === 'string' ? p : (p.text ?? '') });
+      }
+    }
+
+    // Include system prefix exactly once on the first user turn after system
+    if (systemPrefix && m.role === 'user') {
+      baseParts.unshift({ text: `[system]\n${systemPrefix}` });
+      systemPrefix = '';
+    }
+
+    contents.push({ role: m.role === 'assistant' ? 'model' : 'user', parts: baseParts });
+  }
+
+  return contents;
+}
+
+function safeJsonParse(s) {
+  try { return JSON.parse(s ?? '{}'); } catch { return null; }
+}
+
+/**
+ * Convert a Gemini candidate → OpenAI-like assistant message.
+ * - Collects text parts into `content`.
+ * - Translates functionCall parts into `tool_calls`.
+ */
+function fromGeminiCandidate(cand) {
+  const parts = cand?.content?.parts ?? [];
+  const tool_calls = [];
+  const texts = [];
+
+  for (const p of parts) {
+    if (p.functionCall) {
+      tool_calls.push({
+        id: cryptoRandomId(),
+        type: 'function',
+        function: {
+          name: p.functionCall.name,
+          arguments: JSON.stringify(p.functionCall.args ?? {}),
+        }
+      });
+    } else if (typeof p.text === 'string') {
+      texts.push(p.text);
+    }
+  }
+
+  const content = texts.join('');
+  return { role: 'assistant', content, tool_calls: tool_calls.length ? tool_calls : undefined };
+}
+
+function cryptoRandomId() {
+  return 'tool_' + Math.random().toString(36).slice(2);
+}
+
+export class GeminiProvider {
+  constructor({ apiKey } = {}) {
+    if (!apiKey) throw new Error('GEMINI_API_KEY is required for Gemini provider');
+    this.client = new GoogleGenerativeAI(apiKey);
+  }
+
+  /**
+   * Execute a chat turn via Gemini and normalize the response.
+   * @param {{model:string, messages:any[], tools?:any[]}} params
+   * @returns {Promise<{message:any, usage?:{total_tokens?:number, providerRaw?:any}}>}
+   */
+  async chat({ model, messages, tools }) {
+    // Map OpenAI-like tool schema → Gemini functionDeclarations
+    const genTools = tools?.length
+      ? [{
+          functionDeclarations: tools.map(t => ({
+            name: t.function?.name ?? t.name,
+            description: t.description ?? '',
+            parameters: t.function?.parameters ?? t.parameters ?? {}, // JSON schema
+          })),
+        }]
+      : undefined;
+
+    const gmodel = this.client.getGenerativeModel({
+      model,
+      tools: genTools,
+      generationConfig: {}, // temperature/topP can be added upstream if needed
+    });
+
+    const contents = toGeminiContents(messages);
+    const res = await gmodel.generateContent({ contents });
+
+    const cand = res?.response?.candidates?.[0];
+    const message = fromGeminiCandidate(cand);
+
+    const usageMeta = res?.response?.usageMetadata;
+    const usage = {
+      total_tokens: usageMeta?.totalTokenCount,
+      providerRaw: usageMeta,
+    };
+
+    return { message, usage };
+  }
+}

package/llm/llmFactory.js

@@ -0,0 +1,50 @@
+/**
+ * llmFactory.js
+ * --------------
+ * Purpose:
+ *   Provide a single entry point to obtain an LLM provider adapter.
+ *   Each adapter exposes a unified `chat({ model, messages, tools })`
+ *   and returns `{ message, usage }` where `message` is OpenAI-like.
+ *
+ * Responsibilities:
+ *   - Map provider name → concrete adapter instance.
+ *   - Normalize constructor options (env API keys + optional overrides).
+ *
+ * Message contract (internal, OpenAI-like):
+ *   - messages: Array<{ role: 'system'|'user'|'assistant'|'tool', content?: string, ... }>
+ *   - Assistant tool calls:
+ *       message.tool_calls = [{ id, type:'function', function:{ name, arguments:string }}]
+ *   - Tool messages:
+ *       { role:'tool', tool_call_id, name, type:'function', content:string }
+ *
+ * Related tests:
+ *   Located in `tests/llmTests/llmFactory.test.js`
+ *
+ * Used by:
+ *   - core/turnLoop.js (to obtain the active provider implementation)
+ */
+
+import { OpenAIProvider } from './openAI/openaiProvider.js';
+import { GeminiProvider } from './gemini/geminiProvider.js';
+
+/**
+ * Return an adapter implementing:
+ *   chat({ model, messages, tools }) -> Promise<{ message, usage }>
+ *
+ * @param {string} providerName - 'openai' | 'gemini'
+ * @param {object} [opts]       - Optional overrides (e.g., { apiKey, baseURL, ... })
+ * @returns {{ chat: Function }} provider adapter
+ */
+export function getLLM(providerName, opts = {}) {
+  switch ((providerName || '').toLowerCase()) {
+    case 'openai':
+      // NOTE: opts last → caller can override env key if needed (useful for tests)
+      return new OpenAIProvider({ apiKey: process.env.OPENAI_API_KEY, ...opts });
+
+    case 'gemini':
+      return new GeminiProvider({ apiKey: process.env.GEMINI_API_KEY, ...opts });
+
+    default:
+      throw new Error(`Unsupported LLM provider: ${providerName}`);
+  }
+}

package/llm/modelResolver.js

@@ -0,0 +1,65 @@
+/**
+ * modelResolver.js
+ * -----------------
+ * Purpose:
+ *   Determines which LLM provider and model Testronaut should use.
+ *   Reads from env first, then `testronaut-config.json`, then defaults.
+ *
+ * Resolution priority:
+ *   1) Env: TESTRONAUT_PROVIDER / TESTRONAUT_MODEL
+ *   2) Config file (provider + model)
+ *   3) Legacy config (model === "openai", no provider)
+ *   4) Default: { provider: "openai", model: "gpt-4o" }
+ *
+ * Related tests: tests/llmTests/modelResolver.test.js
+ * Used by: core/turnLoop.js, llm/llmFactory.js
+ */
+
+import path from 'path';
+import fs from 'fs';
+
+/**
+ * Resolve provider & model, honoring env overrides and safe defaults.
+ * @param {{cwd?: string}} [opts] - optional current working directory override (useful in tests)
+ * @returns {{provider: string, model: string}}
+ */
+export function resolveProviderModel(opts = {}) {
+  const envProvider = process.env.TESTRONAUT_PROVIDER?.trim();
+  const envModel    = process.env.TESTRONAUT_MODEL?.trim();
+
+  // 1) Env override (both provided)
+  if (envProvider && envModel) {
+    return { provider: envProvider, model: envModel };
+  }
+
+  // 2) Config lookup (compute path at call time)
+  const root = opts.cwd || process.cwd();
+  const configPath = path.join(root, 'testronaut-config.json');
+
+  try {
+    if (fs.existsSync(configPath)) {
+      const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+
+      // Modern config: explicit provider + model
+      if (cfg?.provider && cfg?.model) {
+        return {
+          provider: envProvider || cfg.provider,
+          model: envModel || cfg.model,
+        };
+      }
+
+      // Legacy: model === "openai" and no provider
+      if (!cfg?.provider && cfg?.model === 'openai') {
+        return { provider: 'openai', model: envModel || 'gpt-4o' };
+      }
+    }
+  } catch (e) {
+    console.warn('⚠️ Could not read testronaut-config.json; using defaults. Error:', e.message);
+  }
+
+  // 3) Defaults
+  return {
+    provider: envProvider || 'openai',
+    model: envModel || 'gpt-4o',
+  };
+}

package/llm/openAI/openaiProvider.js

@@ -0,0 +1,54 @@
+/**
+ * openaiProvider.js
+ * ------------------
+ * Purpose:
+ *   Adapter that wraps OpenAI's SDK and exposes a normalized `chat()` API
+ *   consistent with Testronaut’s OpenAI-like message format.
+ *
+ * Responsibilities:
+ *   - Call OpenAI Chat Completions with { model, messages, tools }.
+ *   - Return { message, usage, headers } where:
+ *       • message is OpenAI-like (already native)
+ *       • usage.total_tokens is forwarded from OpenAI
+ *       • headers (if present) can be used by the token limit learner
+ *
+ * Related tests:
+ *   Located in `tests/llmTests/openAIProvider.test.js`
+ *
+ * Used by:
+ *   - llm/llmFactory.js
+ */
+
+import OpenAI from 'openai';
+
+export class OpenAIProvider {
+  constructor({ apiKey } = {}) {
+    if (!apiKey) throw new Error('OPENAI_API_KEY is required for OpenAI provider');
+    this.client = new OpenAI({ apiKey });
+  }
+
+  /**
+   * Execute a chat turn via OpenAI and normalize the response.
+   * @param {{model:string, messages:any[], tools?:any[]}} params
+   * @returns {Promise<{message:any, usage?:{total_tokens?:number, providerRaw?:any}, headers?:any}>}
+   */
+  async chat({ model, messages, tools }) {
+    const res = await this.client.chat.completions.create({
+      model,
+      messages,
+      tools,
+    });
+
+    // OpenAI already returns an OpenAI-like message and usage structure.
+    const message = res.choices?.[0]?.message ?? { role: 'assistant', content: '' };
+    const usage = {
+      total_tokens: res.usage?.total_tokens,
+      providerRaw: res.usage,
+    };
+
+    // Some SDK versions surface headers on `res.headers`, others on `res.response.headers`.
+    const headers = res.headers || res.response?.headers;
+
+    return { message, usage, headers };
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "./bin/cli.js"
@@ -12,7 +12,8 @@
   },
   "type": "module",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "start": "node ./bin/cli.js"
   },
   "author": "",
@@ -20,6 +21,7 @@
   "description": "",
   "dependencies": {
     "@dqbd/tiktoken": "^1.0.21",
+    "@google/generative-ai": "^0.24.1",
     "cheerio": "^1.1.0",
     "dotenv": "^17.2.0",
     "inquirer": "^12.9.1",
@@ -34,5 +36,9 @@
     "automation",
     "node",
     "qa"
-  ]
+  ],
+  "devDependencies": {
+    "@types/node": "^24.9.1",
+    "vitest": "^4.0.1"
+  }
 }

package/tests/binTests/init.test.js

@@ -0,0 +1,189 @@
+// tests/cliTests/init.test.js
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+// ─────────────────────────────────────────────
+// Hoisted queue for the `prompts` mock
+// ─────────────────────────────────────────────
+const { shared } = vi.hoisted(() => ({
+  shared: {
+    answers: [], // each element should be the resolved object for one prompts() call
+    calls: 0,
+  },
+}));
+
+// Mock `prompts` to return queued answers
+vi.mock('prompts', () => ({
+  default: async () => {
+    shared.calls += 1;
+    if (!shared.answers.length) {
+      throw new Error('Test prompts queue empty. Push expected answers to shared.answers.');
+    }
+    return shared.answers.shift();
+  },
+}));
+
+// Important: import SUT after mocks
+import { initializeTestronautProject } from '../../bin/init';
+
+function makeTempProject() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-init-'));
+}
+function readJson(p) {
+  return JSON.parse(fs.readFileSync(p, 'utf8'));
+}
+function fileExists(p) {
+  try { return fs.statSync(p).isFile(); } catch { return false; }
+}
+function dirExists(p) {
+  try { return fs.statSync(p).isDirectory(); } catch { return false; }
+}
+
+const originalCwd = process.cwd();
+const ORIGINAL_ENV = { ...process.env };
+
+describe('initializeTestronautProject', () => {
+  beforeEach(() => {
+    process.env = { ...ORIGINAL_ENV };
+    shared.answers = [];
+    shared.calls = 0;
+  });
+
+  afterEach(() => {
+    process.env = { ...ORIGINAL_ENV };
+    process.chdir(originalCwd);
+  });
+
+  it('scaffolds a new OpenAI project: config, .env, folders, welcome mission', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // Queue answers for two prompts:
+    // 1) provider
+    // 2) openai model
+    shared.answers.push({ llmProvider: 'openai' });
+    shared.answers.push({ openaiModel: 'gpt-4o' });
+
+    await initializeTestronautProject();
+
+    // Config
+    const configPath = path.join(temp, 'testronaut-config.json');
+    expect(fileExists(configPath)).toBe(true);
+    const cfg = readJson(configPath);
+    expect(cfg.provider).toBe('openai');
+    expect(cfg.model).toBe('gpt-4o');
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.outputDir).toBe('missions/mission_reports');
+    expect(typeof cfg.maxTurns).toBe('number');
+
+    // .env
+    const envPath = path.join(temp, '.env');
+    expect(fileExists(envPath)).toBe(true);
+    const envTxt = fs.readFileSync(envPath, 'utf8');
+    expect(envTxt).toMatch(/OPENAI_API_KEY=sk-/);
+
+    // Folders
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    // Welcome mission (name depends on your implementation; we check presence in missions/)
+    const files = fs.readdirSync(path.join(temp, 'missions'));
+    expect(files.some(f => /welcome/i.test(f))).toBe(true);
+
+    // Exactly two prompts were asked
+    expect(shared.calls).toBe(2);
+  });
+
+  it('scaffolds a new Gemini project: config, .env, folders, welcome mission', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    shared.answers.push({ llmProvider: 'gemini' });
+    shared.answers.push({ geminiModel: 'gemini-2.5-flash' });
+
+    await initializeTestronautProject();
+
+    const cfg = readJson(path.join(temp, 'testronaut-config.json'));
+    expect(cfg.provider).toBe('gemini');
+    expect(cfg.model).toBe('gemini-2.5-flash');
+
+    const envTxt = fs.readFileSync(path.join(temp, '.env'), 'utf8');
+    expect(envTxt).toMatch(/GEMINI_API_KEY=AIza/);
+
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    const files = fs.readdirSync(path.join(temp, 'missions'));
+    expect(files.some(f => /welcome/i.test(f))).toBe(true);
+
+    expect(shared.calls).toBe(2);
+  });
+
+  it('is idempotent: when initialized, it skips prompts and does not overwrite .env', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // Pre-create a config marked as initialized + a non-empty .env
+    const configPath = path.join(temp, 'testronaut-config.json');
+    fs.writeFileSync(
+      configPath,
+      JSON.stringify({
+        initialized: true,
+        provider: 'openai',
+        model: 'gpt-4o',
+        outputDir: 'missions/mission_reports',
+        projectName: 'demo',
+        maxTurns: 20,
+      }, null, 2),
+      'utf8'
+    );
+
+    const envPath = path.join(temp, '.env');
+    fs.writeFileSync(envPath, 'DO_NOT_TOUCH=1\n', 'utf8');
+
+    // Also ensure folders are missing to verify folder creation still happens
+    // (initialize should still ensure dirs even when skipping prompts)
+    await initializeTestronautProject();
+
+    // Prompts should not have been called
+    expect(shared.calls).toBe(0);
+
+    // .env untouched
+    const envTxt = fs.readFileSync(envPath, 'utf8');
+    expect(envTxt).toBe('DO_NOT_TOUCH=1\n');
+
+    // Folders ensured
+    expect(dirExists(path.join(temp, 'missions'))).toBe(true);
+    expect(dirExists(path.join(temp, 'missions', 'mission_reports'))).toBe(true);
+
+    // Config left as-is (no clobber)
+    const cfg = readJson(configPath);
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.provider).toBe('openai');
+    expect(cfg.model).toBe('gpt-4o');
+  });
+
+  it('preserves pre-existing fields and merges defaults without clobbering', async () => {
+    const temp = makeTempProject();
+    process.chdir(temp);
+
+    // seed a partial config (not initialized yet)
+    fs.writeFileSync(
+      path.join(temp, 'testronaut-config.json'),
+      JSON.stringify({ projectName: 'my-project' }, null, 2),
+      'utf8'
+    );
+
+    shared.answers.push({ llmProvider: 'openai' });
+    shared.answers.push({ openaiModel: 'gpt-4.1-mini' });
+
+    await initializeTestronautProject();
+
+    const cfg = readJson(path.join(temp, 'testronaut-config.json'));
+    expect(cfg.projectName).toBe('my-project'); // preserved
+    expect(cfg.outputDir).toBe('missions/mission_reports'); // default filled in
+    expect(cfg.maxTurns).toBe(20); // default filled in
+  });
+});

package/tests/binTests/initHelpersTests/defaultConfig.test.js

@@ -0,0 +1,12 @@
+import { describe, it, expect } from 'vitest';
+import { defaultConfig } from '../../../bin/initHelpers';
+
+describe('defaultConfig', () => {
+  it('creates sane defaults', () => {
+    const cfg = defaultConfig('my-app');
+    expect(cfg.initialized).toBe(true);
+    expect(cfg.outputDir).toBe('missions/mission_reports');
+    expect(cfg.projectName).toBe('my-app');
+    expect(cfg.maxTurns).toBe(20);
+  });
+});

package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js

@@ -0,0 +1,14 @@
+import { describe, it, expect } from 'vitest';
+import { makeEnvTemplate } from '../../../bin/initHelpers';
+
+describe('makeEnvTemplate', () => {
+  it('returns OpenAI template', () => {
+    const s = makeEnvTemplate('openai');
+    expect(s).toMatch(/OPENAI_API_KEY=sk-\.\.\./);
+  });
+
+  it('returns Gemini template', () => {
+    const s = makeEnvTemplate('gemini');
+    expect(s).toMatch(/GEMINI_API_KEY=AIza\.\.\./);
+  });
+});

package/tests/binTests/initHelpersTests/migrateLegacyConfig.test.js

@@ -0,0 +1,19 @@
+import { describe, it, expect } from 'vitest';
+import { migrateLegacyConfig } from '../../../bin/initHelpers';
+
+describe('migrateLegacyConfig', () => {
+  it('sets provider=openai when model=openai and provider is missing', () => {
+    const out = migrateLegacyConfig({ model: 'openai' });
+    expect(out.provider).toBe('openai');
+  });
+
+  it('does not override existing provider', () => {
+    const out = migrateLegacyConfig({ model: 'openai', provider: 'gemini' });
+    expect(out.provider).toBe('gemini');
+  });
+
+  it('keeps unrelated configs intact', () => {
+    const out = migrateLegacyConfig({ foo: 1, bar: 'x' });
+    expect(out).toEqual({ foo: 1, bar: 'x' });
+  });
+});

package/tests/binTests/initHelpersTests/models.test.js

@@ -0,0 +1,21 @@
+import { describe, it, expect } from 'vitest';
+import { openAIModels, geminiModels, isKnownModel, pickInitialIndex } from '../../../bin/initHelpers';
+
+describe('model helpers', () => {
+  it('lists known models', () => {
+    expect(openAIModels()).toContain('gpt-4o');
+    expect(geminiModels()).toContain('gemini-2.5-flash');
+  });
+
+  it('recognizes known models by provider', () => {
+    expect(isKnownModel('openai', 'gpt-4.1-mini')).toBe(true);
+    expect(isKnownModel('gemini', 'gpt-4.1-mini')).toBe(false);
+  });
+
+  it('picks initial index from list or fallback', () => {
+    const list = ['a','b','c'];
+    expect(pickInitialIndex(list, 'b', 'c')).toBe(1);
+    expect(pickInitialIndex(list, 'z', 'c')).toBe(2);
+    expect(pickInitialIndex(list, 'z', 'y')).toBe(0);
+  });
+});