testronaut 1.3.3 → 1.3.5

package/.devcontainer/devcontainer.json

@@ -0,0 +1,49 @@
+{
+  "name": "Testronaut Multi-Repo Codespace",
+
+  "image": "mcr.microsoft.com/devcontainers/javascript-node:22",
+
+  "customizations": {
+    "codespaces": {
+      "repositories": {
+        "mission-testronaut/testronaut-cli": {
+          "permissions": "write-all"
+        },
+        "mission-testronaut/testronaut-examples": {
+          "permissions": "write-all"
+        }
+      }
+    },
+    "vscode": {
+      "settings": {
+        "files.autoSave": "onFocusChange",
+        "terminal.integrated.defaultProfile.linux": "bash",
+        "git.confirmSync": false,
+        "explorer.confirmDelete": false,
+        "extensions.autoCheckUpdates": true,
+        "extensions.autoUpdate": true
+      },
+      "extensions": [
+        "github.vscode-github-actions",
+        "github.codespaces",
+        "eamodio.gitlens",
+        "esbenp.prettier-vscode",
+        "dbaeumer.vscode-eslint",
+        "openai.chatgpt",
+        "anthropic.claude-code"
+      ]
+    }
+  },
+
+  "workspaceFolder": "/workspaces",
+
+  "postCreateCommand": "bash -lc 'set -e; cd /workspaces; if [ ! -d \"testronaut-examples/.git\" ]; then git clone https://github.com/mission-testronaut/testronaut-examples.git; fi'",
+
+  "postStartCommand": "bash -lc 'git config --global pull.rebase false && git config --global init.defaultBranch main'",
+
+  "hostRequirements": {
+    "cpus": 4,
+    "memory": "8gb",
+    "storage": "32gb"
+  }
+}

package/.github/workflows/prepare-release.yml

@@ -0,0 +1,168 @@
+name: Prepare Package Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_type:
+        description: Semver increment to prepare
+        required: true
+        type: choice
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+permissions:
+  contents: read
+
+concurrency:
+  group: prepare-package-release
+  cancel-in-progress: false
+
+jobs:
+  authorize-release:
+    name: Authorize Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require main branch
+        run: |
+          if [ "${GITHUB_REF_NAME}" != "main" ]; then
+            echo "Manual release preparation is only allowed from main."
+            exit 1
+          fi
+
+      - name: Require repository admin permission
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const actor = context.actor;
+
+            if (owner.toLowerCase() === actor.toLowerCase()) {
+              core.info(`Actor ${actor} matches the repository owner and is treated as admin.`);
+              return;
+            }
+
+            const response = await github.rest.repos.getCollaboratorPermissionLevel({
+              owner,
+              repo,
+              username: actor,
+            });
+
+            const { permission, role_name: roleName } = response.data;
+            core.info(`Actor ${actor} permission=${permission}, role_name=${roleName}`);
+
+            if (permission !== "admin" && roleName !== "admin") {
+              core.setFailed(`Only repository admins can prepare package releases. ${actor} has ${roleName || permission}.`);
+            }
+
+  prepare-release:
+    name: Prepare Release Pull Request
+    needs: authorize-release
+    runs-on: ubuntu-latest
+    permissions:
+      # Required because this workflow pushes a release branch and opens a PR.
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run automated test suite
+        run: npm test
+
+      - name: Bump package version
+        run: npm version "${{ inputs.release_type }}" --no-git-tag-version
+
+      - name: Read new version
+        id: version
+        run: |
+          VERSION="$(node -e 'process.stdout.write(require("./package.json").version)')"
+          echo "value=${VERSION}" >> "$GITHUB_OUTPUT"
+          echo "branch=release/v${VERSION}" >> "$GITHUB_OUTPUT"
+
+      - name: Ensure release branch is not already open
+        env:
+          RELEASE_BRANCH: ${{ steps.version.outputs.branch }}
+        run: |
+          if git ls-remote --exit-code --heads origin "${RELEASE_BRANCH}" >/dev/null 2>&1; then
+            echo "Release branch ${RELEASE_BRANCH} already exists on origin."
+            exit 1
+          fi
+
+      - name: Create release branch
+        env:
+          RELEASE_BRANCH: ${{ steps.version.outputs.branch }}
+        run: git switch -c "${RELEASE_BRANCH}"
+
+      - name: Configure git author
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Commit release version
+        env:
+          VERSION: ${{ steps.version.outputs.value }}
+        run: |
+          git add package.json package-lock.json
+          git commit -m "chore(release): v${VERSION}"
+
+      - name: Push release branch
+        env:
+          RELEASE_BRANCH: ${{ steps.version.outputs.branch }}
+        run: git push --set-upstream origin "${RELEASE_BRANCH}"
+
+      - name: Open release pull request
+        id: pull-request
+        uses: actions/github-script@v7
+        env:
+          RELEASE_BRANCH: ${{ steps.version.outputs.branch }}
+          VERSION: ${{ steps.version.outputs.value }}
+        with:
+          script: |
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const head = process.env.RELEASE_BRANCH;
+            const version = process.env.VERSION;
+
+            const pr = await github.rest.pulls.create({
+              owner,
+              repo,
+              title: `chore(release): v${version}`,
+              head,
+              base: "main",
+              body: [
+                `Prepares \`v${version}\` for npm publication.`,
+                "",
+                "After this pull request merges into `main`, the publish workflow will release the package to npm.",
+              ].join("\n"),
+            });
+
+            core.setOutput("url", pr.data.html_url);
+
+      - name: Write job summary
+        env:
+          VERSION: ${{ steps.version.outputs.value }}
+          RELEASE_BRANCH: ${{ steps.version.outputs.branch }}
+          PR_URL: ${{ steps.pull-request.outputs.url }}
+        run: |
+          {
+            echo "## Release PR created"
+            echo
+            echo "- Version: \`${VERSION}\`"
+            echo "- Release branch: \`${RELEASE_BRANCH}\`"
+            echo "- Pull request: ${PR_URL}"
+          } >> "$GITHUB_STEP_SUMMARY"

package/.github/workflows/publish.yml

@@ -0,0 +1,133 @@
+name: Publish Package
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - package.json
+      - package-lock.json
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+
+concurrency:
+  group: publish-package
+  cancel-in-progress: false
+
+jobs:
+  publish-package:
+    name: Publish Package
+    runs-on: ubuntu-latest
+    environment:
+      # Use the same environment name in npm trusted publisher settings if you
+      # want npm to bind publishing to this protected environment.
+      name: package-publish
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Detect release version
+        id: release
+        env:
+          BEFORE_SHA: ${{ github.event.before }}
+          EVENT_NAME: ${{ github.event_name }}
+          REF_NAME: ${{ github.ref_name }}
+        run: |
+          VERSION="$(node -e 'process.stdout.write(require("./package.json").version)')"
+          PACKAGE_NAME="$(node -e 'process.stdout.write(require("./package.json").name)')"
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+          echo "package_name=${PACKAGE_NAME}" >> "$GITHUB_OUTPUT"
+
+          if [ "${EVENT_NAME}" = "workflow_dispatch" ]; then
+            if [ "${REF_NAME}" != "main" ]; then
+              echo "Manual publishing is only allowed from main."
+              exit 1
+            fi
+
+            echo "should_publish=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if [ -z "${BEFORE_SHA}" ] || [ "${BEFORE_SHA}" = "0000000000000000000000000000000000000000" ]; then
+            echo "should_publish=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          PREVIOUS_VERSION="$(git show "${BEFORE_SHA}:package.json" | node -e 'let data=""; process.stdin.on("data", chunk => data += chunk); process.stdin.on("end", () => process.stdout.write(JSON.parse(data).version));')"
+
+          if [ "${PREVIOUS_VERSION}" = "${VERSION}" ]; then
+            echo "Version is unchanged at ${VERSION}; skipping publish."
+            echo "should_publish=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "Version changed from ${PREVIOUS_VERSION} to ${VERSION}."
+            echo "should_publish=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Setup Node
+        if: steps.release.outputs.should_publish == 'true'
+        uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: npm
+          registry-url: https://registry.npmjs.org
+
+      - name: Install dependencies
+        if: steps.release.outputs.should_publish == 'true'
+        run: npm ci
+
+      - name: Run automated test suite
+        if: steps.release.outputs.should_publish == 'true'
+        run: npm test
+
+      - name: Check whether version already exists on npm
+        if: steps.release.outputs.should_publish == 'true'
+        id: npm-check
+        env:
+          PACKAGE_NAME: ${{ steps.release.outputs.package_name }}
+          VERSION: ${{ steps.release.outputs.version }}
+        run: |
+          if npm view "${PACKAGE_NAME}@${VERSION}" version >/dev/null 2>&1; then
+            echo "should_publish=false" >> "$GITHUB_OUTPUT"
+            echo "Version ${VERSION} is already published to npm."
+          else
+            echo "should_publish=true" >> "$GITHUB_OUTPUT"
+            echo "Version ${VERSION} is not yet published to npm."
+          fi
+
+      - name: Publish to npm with trusted publishing
+        if: steps.release.outputs.should_publish == 'true' && steps.npm-check.outputs.should_publish == 'true'
+        run: npm publish --provenance --access public
+
+      - name: Tag published version
+        if: steps.release.outputs.should_publish == 'true' && steps.npm-check.outputs.should_publish == 'true'
+        env:
+          VERSION: ${{ steps.release.outputs.version }}
+        run: |
+          if git rev-parse "v${VERSION}" >/dev/null 2>&1; then
+            echo "Tag v${VERSION} already exists locally."
+            exit 0
+          fi
+
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git tag "v${VERSION}"
+          git push origin "refs/tags/v${VERSION}"
+
+      - name: Write job summary
+        env:
+          VERSION: ${{ steps.release.outputs.version }}
+          SHOULD_PUBLISH: ${{ steps.release.outputs.should_publish }}
+          NPM_PUBLISH: ${{ steps.npm-check.outputs.should_publish }}
+        run: |
+          {
+            echo "## Publish summary"
+            echo
+            echo "- Version: \`${VERSION}\`"
+            echo "- Trigger requested publish: \`${SHOULD_PUBLISH:-false}\`"
+            echo "- npm publish executed: \`${NPM_PUBLISH:-false}\`"
+          } >> "$GITHUB_STEP_SUMMARY"

package/bin/cli.js

@@ -11,6 +11,7 @@
  *
  * Key flags:
  *   --model <id> / --model=<id>         → sets TESTRONAUT_MODEL env (wins over config file)
+ *   --provider <id> / --provider=<id>   → sets TESTRONAUT_PROVIDER env (wins over config file)
  *   --turns <n> / --turns=<n>           → sets TESTRONAUT_TURNS env (wins over config file)
  *   --init                               → scaffolds project + optional Playwright browsers
  *   --dev                                → use staging API base URL
@@ -43,6 +44,7 @@ import inquirer from 'inquirer';
 import fetch from 'node-fetch';
 import crypto from 'crypto';
 import http from 'http';
+import { resolveProviderModel } from '../llm/modelResolver.js';
 import { exec as execCmd } from 'child_process';
 import { promisify } from 'util';
 const exec = promisify(execCmd);
@@ -103,6 +105,40 @@ function parseVercelBypassArgs(argsList) {
   return { secret, args: nextArgs, invalid };
 }
 
+function parseProviderArgs(argsList) {
+  const nextArgs = [...argsList];
+  let provider;
+  let invalid = false;
+  const idx = nextArgs.findIndex(a => a === '--provider' || a.startsWith('--provider='));
+  if (idx >= 0) {
+    const rawArg = nextArgs[idx];
+    const hasInline = rawArg.includes('=');
+    const nextVal = !hasInline && nextArgs[idx + 1] && !nextArgs[idx + 1].startsWith('-') ? nextArgs[idx + 1] : undefined;
+    if (hasInline) {
+      provider = rawArg.split('=')[1];
+    } else if (nextVal) {
+      provider = nextVal;
+    }
+
+    if (provider) {
+      provider = provider.trim();
+    }
+
+    const supportedProviders = new Set(['openai', 'gemini']);
+    const isValid = (v) => !!v && supportedProviders.has(String(v).toLowerCase());
+    if (!isValid(provider)) {
+      invalid = true;
+      provider = undefined;
+    }
+
+    // Remove flag & value so they aren’t treated as filenames
+    const consume = hasInline ? 1 : (nextVal ? 2 : 1);
+    nextArgs.splice(idx, consume);
+  }
+  return { provider, args: nextArgs, invalid };
+}
+
+
 const vercelBypassResult = parseVercelBypassArgs(args);
 if (vercelBypassResult.invalid) {
   console.warn('⚠️ Invalid --vercel-bypass value. Provide a non-empty secret.');
@@ -197,6 +233,7 @@ export const __test__ = {
   readJsonlSteps,
   parseVercelBypassArgs,
   createVercelBypassHeader,
+  parseProviderArgs,
 };
 
 // Look for --model=<id> or --model <id>
@@ -218,6 +255,18 @@ if (modelFlagIndex >= 0) {
   args.splice(modelFlagIndex, modelOverride ? 2 : 1);
 }
 
+// Look for --provider=<id> or --provider <id>
+const providerResult = parseProviderArgs(args);
+if (providerResult.invalid) {
+  console.warn('⚠️ Invalid --provider value. Provide a non-empty provider id.');
+}
+args = providerResult.args;
+const providerOverride = providerResult.provider;
+if (providerOverride) {
+  process.env.TESTRONAUT_PROVIDER = providerOverride.trim();
+  console.log(`🧩 Provider override: ${process.env.TESTRONAUT_PROVIDER}`);
+}
+
 // Look for --debug / --debug=<bool> / --no-debug
 // Normalize CLI boolean strings into true/false/null for optional flags.
 const parseBool = (v) => {
@@ -334,6 +383,7 @@ Options:
   --init                    Scaffold project folders and a welcome mission
   --turns=<n>               Override max turns for this run (e.g., --turns=30)
   --debug[=<bool>]          Enable verbose debug logs (or set TESTRONAUT_DEBUG=1)
+  --provider=<id>           Override LLM provider for this run (e.g., --provider=openai)
   --dev                     Use the staging API base URL
   --vercel-bypass=<secret>  Send Vercel protection bypass header for protected deployments
   --help                    Show this help message
@@ -469,25 +519,8 @@ const flatMissions = allResults.flatMap(entry => {
   });
 });
 
-// Read provider/model from config (allow env override for model)
-const cfgPath = path.resolve(process.cwd(), 'testronaut-config.json');
-let llmProvider = 'openai';
-let llmModel = 'gpt-4o';
-
-if (fs.existsSync(cfgPath)) {
-  try {
-    const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf8'));
-    if (cfg?.provider) llmProvider = String(cfg.provider);
-    if (cfg?.model) llmModel = String(cfg.model);
-  } catch (e) {
-    console.warn('⚠️ Could not read testronaut-config.json:', e.message);
-  }
-}
-
-// Env override takes precedence for the model (matches runtime behavior)
-if (process.env.TESTRONAUT_MODEL?.trim()) {
-  llmModel = process.env.TESTRONAUT_MODEL.trim();
-}
+// Read provider/model from config (allow env override)
+const { provider: llmProvider, model: llmModel } = resolveProviderModel({ cwd: process.cwd() });
 
 
 const report = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.3.3",
+  "version": "1.3.5",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "bin/cli.js"
@@ -19,6 +19,10 @@
   "author": "",
   "license": "MIT",
   "description": "",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mission-testronaut/testronaut-cli"
+  },
   "dependencies": {
     "@dqbd/tiktoken": "^1.0.21",
     "@google/generative-ai": "^0.24.1",

package/tests/binTests/cli.helpers.test.js

@@ -67,6 +67,49 @@ describe('cli helpers', () => {
     expect(res.invalid).toBe(true);
   });
 
+  it('parses provider flag with inline value and removes it from args', () => {
+    const { parseProviderArgs } = __test__;
+    const res = parseProviderArgs(['--provider=openai', 'login']);
+    expect(res.provider).toBe('openai');
+    expect(res.args).toEqual(['login']);
+    expect(res.invalid).toBe(false);
+  });
+
+  it('parses provider flag value from next arg', () => {
+    const { parseProviderArgs } = __test__;
+    const res = parseProviderArgs(['--provider', 'gemini', 'login']);
+    expect(res.provider).toBe('gemini');
+    expect(res.args).toEqual(['login']);
+    expect(res.invalid).toBe(false);
+  });
+
+  it('accepts supported provider names', () => {
+    const { parseProviderArgs } = __test__;
+    const res1 = parseProviderArgs(['--provider', 'openai', 'login']);
+    expect(res1.provider).toBe('openai');
+    expect(res1.invalid).toBe(false);
+
+    const res2 = parseProviderArgs(['--provider=gemini', 'login']);
+    expect(res2.provider).toBe('gemini');
+    expect(res2.invalid).toBe(false);
+  });
+
+  it('flags invalid provider name', () => {
+    const { parseProviderArgs } = __test__;
+    const res = parseProviderArgs(['--provider', 'bad provider', 'login']);
+    expect(res.provider).toBeUndefined();
+    expect(res.args).toEqual(['login']);
+    expect(res.invalid).toBe(true);
+  });
+
+  it('handles missing provider value', () => {
+    const { parseProviderArgs } = __test__;
+    const res = parseProviderArgs(['--provider']);
+    expect(res.provider).toBeUndefined();
+    expect(res.args).toEqual([]);
+    expect(res.invalid).toBe(true);
+  });
+
   it('builds the vercel bypass header when provided', () => {
     const { createVercelBypassHeader } = __test__;
     expect(createVercelBypassHeader('abc')).toEqual({ 'x-vercel-protection-bypass': 'abc' });

package/tools/generateHtmlReport.js

@@ -35,8 +35,8 @@ export function generateHtmlReport(report, outputPath) {
   // Escaping helpers for text and HTML attributes
   const esc = (s) => String(s ?? '')
     .replace(/&/g, '&amp;').replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;');
-  const escAttr = (s) => esc(s).replace(/"/g, '&quot;'); // for HTML attributes
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
   const badge = (status) =>
     status === 'passed' ? '✅ Passed' :
     status === 'failed' ? '❌ Failed' : (status || '—');
@@ -70,7 +70,7 @@ export function generateHtmlReport(report, outputPath) {
         ? step.summary.trim()
         : '';
       const planSpan = plan
-        ? `<span class="plan" title="${escAttr(plan)}">${esc(plan)}</span>`
+        ? `<span class="plan" title="${esc(plan)}">${esc(plan)}</span>`
         : '';
 
       return `
@@ -78,7 +78,7 @@ export function generateHtmlReport(report, outputPath) {
           <summary>
             <span class="turn">Turn ${esc((step.turn ?? idx) + 1)}${esc(attemptLabel)}</span>
             ${planSpan}
-            <span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${escAttr(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
+            <span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
             <span class="tokens">tokens: ${esc(step.tokensUsed ?? '—')} / total: ${esc(step.totalTokensUsed ?? '—')}</span>
           </summary>
           <pre class="events">${esc(events.join('\n')) || '(no events)'}</pre>