testronaut 1.2.5 → 1.3.0

package/core/agent.js

@@ -25,6 +25,7 @@
 import 'dotenv/config';
 import { ChromeBrowser } from '../tools/chromeBrowser.js';
 import { turnLoop } from './turnLoop.js';
+import { createEmptyGroundControl, summarizeGroundControlForPrompt } from '../tools/contextControl.js';
 import fs from 'fs';
 import path from 'path';
 
@@ -46,6 +47,9 @@ export async function runAgent(goals, missionName, maxTurns = 20) {
     const tmpDir = path.resolve(process.cwd(), 'missions/tmp');
     fs.mkdirSync(tmpDir, { recursive: true });
 
+    // 🔭 Shared Ground Control state across all goals/missions
+    const groundControl = createEmptyGroundControl();
+
     for (const goal of goals) {
       const steps = [];
       // Unique-ish JSONL file for this mission’s steps
@@ -62,22 +66,79 @@ export async function runAgent(goals, missionName, maxTurns = 20) {
           ? goal.goal
           : goal?.goal?.toString?.() ?? String(goal.goal);
 
-      const messages = [
-        {
-          role: 'system',
-          content: `
-            You are an autonomous web agent. Use function calls to complete the user's goal.
-            If you are unsure of the selectors for inputs or buttons, call 'get_dom' to retrieve page HTML,
-            analyze it, then make your best guess based on labels, names, types, and placeholder values.
-            If you want to click a button labeled "Sign out", prefer:
+      // 🔭 Build a compact Ground Control snapshot for the prompt
+      const groundSummary = summarizeGroundControlForPrompt(groundControl);
+      let systemContent = `
+You are an autonomous web agent piloting a browser for end-to-end testing.
+
+Core behavior:
+- Use function calls to complete the user's goal.
+- If you are unsure of selectors for inputs or buttons, call 'get_dom' to retrieve page HTML,
+  analyze it, then choose selectors based on labels, names, types, and placeholder values.
+- If you want to click a button labeled "Sign out", prefer:
+
+  click_text({ text: "Sign out" })
+
+  Do NOT use CSS selectors like 'button:contains("Sign out")' — they are invalid.
+
+After completing the goal, respond with a final plain-text message starting with
+either "SUCCESS:" or "FAILURE:" followed by a short explanation.
+
+──── Ground Control (persistent mission state) ────
+You also have special tools to manage a compact, non-pruned state called "Ground Control".
+Ground Control tracks high-level truths about the app and session, such as:
 
-            click_text({ text: "Sign out" })
+- app:       { baseUrl, currentUrl, routeRole }          // where the app lives and where you are now
+- session:   { loggedIn, userLabel, tenant }             // login state and identity
+- navigation:{ currentLabel }                            // human label for the current view
+- constraints:{ stayWithinBaseUrl }                      // domain / navigation constraints
+- telemetry: [{ ts, kind, text, status, turn, extra }]   // optional breadcrumbs
 
-            Do NOT use CSS selectors like 'button:contains("Sign out")' — they are invalid.
+You have two Ground Control tools:
 
-            After completing the goal, respond with a final plain-text message starting with SUCCESS or FAILURE.
-          `.trim(),
-        },
+1) establish_ground_control
+   - Use this EARLY in a mission/phase once you understand key facts, e.g.:
+     - The base URL (e.g. "https://ult.ultimarii.app")
+     - The current URL and what the page represents (routeRole: "login", "chat", "dashboard", etc.)
+     - Whether you are logged in or not.
+   - Call it once per mission phase (premission / mission / postmission) when you first
+     have a clear picture of where you are and what the app state is.
+   - Provide as many known fields as you can, but do not guess wildly. Unknown fields can be omitted.
+
+2) update_ground_control
+   - Use this when important truths change or become clearer, for example:
+     - URL or routeRole changes after navigation or login.
+     - Login state changes (loggedIn false → true or vice versa).
+     - You discover the displayed user name or tenant.
+     - You identify a new page role (e.g. "chat", "settings", "feedback-form").
+   - Only update fields that changed or that you now know with more confidence.
+   - Optionally add short telemetry breadcrumbs about major milestones,
+     e.g. "Logged in and reached chat workspace".
+
+General rules:
+- Keep Ground Control high-level and stable; do NOT spam small, noisy updates.
+- Before leaving a page for a new flow, make sure Ground Control reflects the currentUrl
+  and routeRole so you can reason about where you are and where you’ve been.
+- Respect constraints (e.g. stayWithinBaseUrl = true) when deciding whether it is safe to navigate.
+
+Use Ground Control as your persistent mission memory about:
+- Where the app lives (baseUrl),
+- Where you are (currentUrl, routeRole),
+- Who you are logged in as (session),
+- Any constraints about staying on the correct site.
+      `.trim();
+
+      if (groundSummary) {
+        systemContent +=
+          '\n\n' +
+          `
+──── Ground Control Snapshot ────
+🛰️ ${JSON.stringify(groundSummary, null, 2)}
+          `.trim();
+      }
+
+      const messages = [
+        { role: 'system', content: systemContent },
         { role: 'user', content: userContent },
       ];
 
@@ -92,6 +153,7 @@ export async function runAgent(goals, missionName, maxTurns = 20) {
         {
           steps,
           missionName,
+          groundControl,
           onStep: (s) => {
             // Append each step as a JSON line
             try {

package/core/turnLoop.js

@@ -47,6 +47,13 @@ import { resolveProviderModel } from '../llm/modelResolver.js';
 import { getLLM } from '../llm/llmFactory.js';
 import { summarizeTurnIntentFromMessage } from './turnIntent.js';
 import { redactArgs } from './redaction.js';
+import { 
+  sanitizeHeavyToolHistory, 
+  pruneConversationContext,
+  createEmptyGroundControl,
+  applyGroundControlUpdate,
+  recordGroundTelemetry,
+ } from '../tools/contextControl.js';
 
 // ─────────────────────────────────────────────
 // STEP 0: Resolve provider + model and initialize adapter
@@ -60,7 +67,26 @@ const llm = getLLM(PROVIDER_ID);
 let totalTokensUsed = 0;
 let turnTimestamps = [];
 let shouldBackoff;
-// let steps = [];
+
+// Certain tools mutate the browser or produce side effects that are
+// *useful for humans* (reports/screenshots), but do not carry semantic
+// information that the LLM needs on subsequent turns.
+// For these, we respond to the tool_call with a tiny stub ("OK"/error)
+// instead of the full result payload to avoid bloating context.
+//
+// NOTE: We *still* log the full result in `step.events` and step metadata.
+const FIRE_AND_FORGET_TOOLS = new Set([
+  'screenshot',
+  'click',
+  'click_text',
+  // 'expand_menu',
+  'fill',
+  'upload_file',
+  'download_file',
+  'click_and_follow_popup',
+  'switch_to_page',
+  'close_current_page',
+]);
 
 /**
  * Utility: format byte counts into human-readable strings.
@@ -143,9 +169,10 @@ export const turnLoop = async (
   currentTurn = 0, 
   retryCount = 0, 
   currentStep = {},
-  ctx = {} // { steps, missionName }
+  ctx = {} // { steps, missionName, groundControl }
 ) => {
-  const { steps = [], missionName } = ctx;
+  const { steps = [], missionName, groundControl = createEmptyGroundControl() } = ctx;
+  ctx.groundControl = groundControl;
   let agentMemory = { lastMenuExpanded: false };
   
   // Centralized recorder: push to in-memory buffer AND fire optional callback for streaming
@@ -173,11 +200,9 @@ export const turnLoop = async (
       ({ totalTokensUsed, turnTimestamps, shouldBackoff } =
         await tokenUseCoolOff(totalTokensUsed, turnTimestamps, MODEL_ID));
       if (shouldBackoff) {
-        // step.result = '🟡 In Progress';
         recordStep(step);         // ✅ write the partial step before sleeping
         turn -= 1;                // retry same turn index
         continue;
-        // return await turnLoop(browser, messages, maxTurns, turn, currentStep);
       }
 
       // Verify message structure integrity before requesting next model step
@@ -209,6 +234,18 @@ export const turnLoop = async (
         return steps;
       }
 
+      // Before calling the model, trim/sanitize the conversation context.
+      // 1) Replace older heavy tool payloads with small stubs.
+      sanitizeHeavyToolHistory(messages, { keepRecentPerTool: 2 });
+
+      // 2) Hard-cap total history length (system messages + last N others).
+      {
+        const pruned = pruneConversationContext(messages, { maxNonSystemMessages: 40 });
+        messages.length = 0;
+        messages.push(...pruned);
+      }
+
+
       // ─────────────────────────────────────────────
       // STEP 2: Request next reasoning turn from model
       // ─────────────────────────────────────────────
@@ -228,7 +265,6 @@ export const turnLoop = async (
         const delay = Math.min(60000, 2 ** retryCount * 2000);
         console.warn(`⚠️ Rate limited. Retrying in ${delay / 1000}s... (retry ${retryCount + 1})`);
         step.events.push(`⏳ Rate limit: waiting ${Math.round(delay/1000)}s (retry ${retryCount + 1})`);
-        // step.result = '🟡 In Progress';
         recordStep(step);         // ✅ persist this step before sleeping
         await wait(delay);
 
@@ -239,8 +275,6 @@ export const turnLoop = async (
           recordStep(step);
           return steps;
         }
-        // return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, step);
-        // reattempt same turn after delay; keep same loop, bump retryCount
         retryCount += 1;
         turn -= 1;
         continue;
@@ -303,7 +337,7 @@ export const turnLoop = async (
           result = errorMessage;
         }
 
-        // Capture and log any file upload/download events
+        // Capture and log any file upload/download events (for reports)
         try {
           const maybeJson = JSON.parse(result);
           if (maybeJson && maybeJson._testronaut_file_event) {
@@ -315,14 +349,16 @@ export const turnLoop = async (
             step.events.push(msgLine);
             console.log(msgLine);
           }
-        } catch { /* non-JSON results ignored */ }
+        } catch {
+          // non-JSON results ignored
+        }
 
         console.log(`[tool ] ← ${fnName} result:`, errorMessage ? '❌ Failed' : '✅ Success');
         const truncated = result.length > 1000 ? result.slice(0, 1000) + '…' : result;
         step.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`);
         step.events.push(`[tool ] ← ${truncated}`);
 
-        // Screenshot detection
+        // Screenshot detection (for report metadata only)
         if (fnName === 'screenshot') {
           const match = result.match(/screenshot.*?saved at: (.+\.png)/i);
           if (match && match[1]) {
@@ -331,13 +367,33 @@ export const turnLoop = async (
           }
         }
 
-        // Push corresponding tool message back to LLM
+        if (fnName === 'set_ground_control_state') {
+          applyGroundControlUpdate(groundControl, args);
+          result = JSON.stringify({ ok: true, groundControl });
+        }
+
+        if (fnName === 'record_mission_telemetry') {
+          const recorded = recordGroundTelemetry(groundControl, args, { turn });
+          result = JSON.stringify({ ok: true, recorded });
+        }
+
+        // Decide what to send back to the LLM for this tool.
+        // - For "fire-and-forget" tools, send a tiny stub (OK/error) to avoid
+        //   bloating context with large payloads (file JSON, etc.).
+        // - For semantic tools (get_dom, check_text, etc.), send full result.
+        let contentForModel;
+        if (FIRE_AND_FORGET_TOOLS.has(fnName)) {
+          contentForModel = errorMessage || 'OK';
+        } else {
+          contentForModel = result;
+        }
+
         toolResponses.push({
           role: 'tool',
           tool_call_id: call.id,
           name: fnName,
           type: 'function',
-          content: result,
+          content: contentForModel,
         });
 
         // After interactive DOM actions, refresh model context

package/llm/gemini/geminiProvider.js

@@ -109,6 +109,19 @@ function safeJsonParse(s) {
   try { return JSON.parse(s ?? '{}'); } catch { return null; }
 }
 
+// Gemini's functionDeclarations reject JSON Schema's `additionalProperties`; prune it recursively.
+function stripAdditionalProperties(obj) {
+  if (!obj || typeof obj !== 'object') return obj;
+  if (Array.isArray(obj)) return obj.map(stripAdditionalProperties);
+
+  const out = {};
+  for (const [k, v] of Object.entries(obj)) {
+    if (k === 'additionalProperties') continue; // Gemini function declarations do not support this key
+    out[k] = stripAdditionalProperties(v);
+  }
+  return out;
+}
+
 /**
  * Convert a Gemini candidate → OpenAI-like assistant message.
  * - Collects text parts into `content`.
@@ -160,7 +173,7 @@ export class GeminiProvider {
           functionDeclarations: tools.map(t => ({
             name: t.function?.name ?? t.name,
             description: t.description ?? '',
-            parameters: t.function?.parameters ?? t.parameters ?? {}, // JSON schema
+            parameters: stripAdditionalProperties(t.function?.parameters ?? t.parameters ?? {}), // JSON schema sans additionalProperties (unsupported by Gemini)
           })),
         }]
       : undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.2.5",
+  "version": "1.3.0",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "./bin/cli.js"

package/tests/llmTests/geminiProvider.test.js

@@ -77,6 +77,35 @@ describe('GeminiProvider', () => {
     expect(shared.getModelOpts.tools[0].functionDeclarations.map(d => d.name)).toEqual(['foo', 'bar']);
   });
 
+  it('strips additionalProperties from tool schemas (unsupported by Gemini)', async () => {
+    const prov = new GeminiProvider({ apiKey: 'gk-123' });
+    const tools = [
+      {
+        type: 'function',
+        function: {
+          name: 'set_ground_control_state',
+          parameters: {
+            type: 'object',
+            properties: {
+              app: {
+                type: 'object',
+                additionalProperties: false,
+                properties: { baseUrl: { type: 'string' } },
+              },
+            },
+            additionalProperties: false,
+          },
+        },
+      },
+    ];
+
+    await prov.chat({ model: 'gemini-2.5-flash', messages: [{ role: 'user', content: 'go' }], tools });
+
+    const decl = shared.getModelOpts.tools[0].functionDeclarations[0];
+    expect(decl.parameters.additionalProperties).toBeUndefined();
+    expect(decl.parameters.properties.app.additionalProperties).toBeUndefined();
+  });
+
   it('converts OpenAI-like messages → Gemini contents (system prefix + tool calls + tool results)', async () => {
     const prov = new GeminiProvider({ apiKey: 'gk-xyz' });