playwright-mimic 0.1.0 → 0.1.2

package/dist/mimic/click.js

@@ -0,0 +1,348 @@
+import { generateText, Output } from 'ai';
+import { zClickActionResult } from './schema/action.js';
+import { countTokens } from '../utils/token-counter.js';
+import { addAnnotation } from './annotations.js';
+import { generateBestSelectorForElement } from './selector.js';
+import { selectorToPlaywrightCode, generateClickCode } from './playwrightCodeGenerator.js';
+import { captureScreenshot, generateAriaSnapshot } from './markers.js';
+/**
+ * Get click action by matching Gherkin step against captured target elements
+ *
+ * This function uses AI to analyze a Gherkin step and match it against
+ * all available target elements on the page. It returns the top 5 most
+ * likely candidates along with the appropriate click type.
+ *
+ * @param page - Playwright Page object (currently unused but kept for consistency)
+ * @param brain - LanguageModel instance for AI analysis
+ * @param gherkinStep - The Gherkin step to match (e.g., "I click on the Submit button")
+ * @param targetElements - Array of captured elements with marker IDs from the page
+ * @param testContext - Optional test context with previous steps and current state
+ * @returns Promise resolving to ClickActionResult with top candidates and click type
+ */
+export const getClickAction = async (page, brain, gherkinStep, testContext) => {
+    const startTime = Date.now();
+    // Capture screenshot with markers and positioning data
+    console.log('📸 [getClickAction] Starting screenshot capture with markers...');
+    const screenshotStart = Date.now();
+    const { image: screenshot, markers: markerData } = await captureScreenshot(page);
+    const screenshotTime = Date.now() - screenshotStart;
+    console.log(`📸 [getClickAction] Screenshot captured in ${screenshotTime}ms (${(screenshotTime / 1000).toFixed(2)}s)`);
+    const base64Start = Date.now();
+    const screenshotBase64 = screenshot.toString('base64');
+    const base64Time = Date.now() - base64Start;
+    console.log(`📸 [getClickAction] Screenshot converted to base64 in ${base64Time}ms (${(base64Time / 1000).toFixed(2)}s), size: ${(screenshotBase64.length / 1024).toFixed(2)}KB`);
+    // Generate accessibility snapshot to explain the screenshot structure
+    console.log('🔍 [getClickAction] Generating accessibility snapshot...');
+    const ariaSnapshotStart = Date.now();
+    const ariaSnapshot = await generateAriaSnapshot(page);
+    const ariaSnapshotTime = Date.now() - ariaSnapshotStart;
+    console.log(`🔍 [getClickAction] Accessibility snapshot generated in ${ariaSnapshotTime}ms (${(ariaSnapshotTime / 1000).toFixed(2)}s), length: ${ariaSnapshot.length} chars`);
+    // Convert marker data to format expected by prompt
+    const markerStart = Date.now();
+    const markerInfo = markerData?.map(m => {
+        return {
+            id: m.mimicId,
+            tag: m.tag,
+            text: m.text,
+            role: m.role,
+            ariaLabel: m.ariaLabel,
+        };
+    });
+    const markerTime = Date.now() - markerStart;
+    console.log(`🔍 [getClickAction] Processed ${markerInfo.length} markers in ${markerTime}ms (${(markerTime / 1000).toFixed(2)}s)`);
+    // Build marker summary for the prompt
+    const summaryStart = Date.now();
+    const markerSummary = markerInfo
+        .slice(0, 50) // Limit to first 50 markers to avoid prompt size issues
+        .map(m => `  Marker ${m.id}: ${m.tag}${m.role ? ` (role: ${m.role})` : ''}${m.text ? ` - "${m.text.substring(0, 50)}"` : ''}${m.ariaLabel ? ` [aria-label: "${m.ariaLabel}"]` : ''}`)
+        .join('\n');
+    const summaryTime = Date.now() - summaryStart;
+    console.log(`📝 [getClickAction] Built marker summary in ${summaryTime}ms`);
+    // Build context description for the prompt
+    const promptStart = Date.now();
+    // Build context description with defensive checks for optional testContext
+    // Ensure previousSteps exists and is an array before calling .map()
+    const contextDescription = testContext ? `
+**Test Context:**
+- Current URL: ${testContext.currentState.url}
+- Current Page Title: ${testContext.currentState.pageTitle}
+- Step ${testContext.currentStepIndex + 1} of ${testContext.totalSteps}
+${testContext.previousSteps && Array.isArray(testContext.previousSteps) && testContext.previousSteps.length > 0 ? `
+**Previous Steps Executed:**
+${testContext.previousSteps.map((prevStep, idx) => `${idx + 1}. Step ${prevStep.stepIndex + 1}: "${prevStep.stepText}" (${prevStep.actionKind}${prevStep.url ? ` → ${prevStep.url}` : ''})`).join('\n')}
+` : ''}
+` : '';
+    const prompt = `You are an expert Playwright test engineer specializing in mapping Gherkin steps to concrete DOM interactions using visual analysis.
+
+Your task is to analyze:
+1. A screenshot of the page with numbered marker badges on elements
+2. An accessibility snapshot (provided below) that describes the page structure with roles, names, data-testid, and data-mimic-* attributes
+3. A single Gherkin step that implies a click action
+
+**CRITICAL: Click Type Determination**
+You must determine the correct click type from the Gherkin step. Use these rules:
+
+**Click Type Equivalencies (these are the SAME):**
+- "click", "left click", and "primary click" → use "primary"
+- "right click" and "secondary click" → use "secondary"  
+- "middle click" and "tertiary click" → use "tertiary"
+- "double click" or "double-click" → use "double"
+
+**Default Behavior:**
+- If the step just says "click" (without specifying left/right/middle/double), ALWAYS use "primary"
+- "click" by itself means a standard left/primary click, NOT a hover
+
+**Hover Rules (STRICT):**
+- ONLY use "hover" when the Gherkin step EXPLICITLY mentions "hover", "hover over", or "move mouse over"
+- NEVER use "hover" for a step that says "click" - these are DIFFERENT actions
+- Hover is ONLY for revealing tooltips, dropdowns, or additional UI elements that appear on mouseover
+- If the step says "click", it means click, NOT hover
+
+**Modifier Keys:**
+- Detect if modifier keys are mentioned: shift, control (ctrl), alt, meta (command)
+- These are optional and should be included in the modifiers field if present
+
+**IMPORTANT**: Look at the screenshot to identify elements by their marker numbers. Each element has a numbered badge:
+- **RED badges** = Interactive elements (buttons, links, inputs, etc.)
+- **BLUE badges** = Display-only content elements
+- **GREEN badges** = Structure/test anchor elements
+
+You must return the **top 5 most likely elements** that the Gherkin step is referring to, identified by their **marker ID numbers** (the numbers shown on the badges in the screenshot).
+
+---
+
+### IMPORTANT RULES
+
+- **ALWAYS use the marker ID (mimicId)** from the screenshot - this is the number shown on the element's badge
+- Rank elements from **most likely (rank 1)** to **least likely (rank 5)**
+- Prefer **semantic matches** first:
+  - Visible text (what you can read in the screenshot)
+  - Element position and visual appearance
+  - Accessible name (label, aria-label, role)
+  - Button / link intent
+- Do NOT invent elements or marker IDs - only use marker IDs that are visible in the screenshot
+- Do NOT include more than 5 results
+- If fewer than 5 reasonable matches exist, return fewer
+- Do NOT assume navigation or side effects — this task is only about **what element is clicked**
+- For each candidate, provide:
+  - The **mimicId** (marker number from the screenshot badge)
+  - A **clear, human-readable description** that identifies the element (e.g., "Submit button", "Login link with text 'Sign in'")
+  - Element metadata (tag, text, role, etc.) based on what you can see in the screenshot and the marker information provided
+- Consider the test context - what steps came before may help identify the correct element
+
+${contextDescription}
+**Gherkin Step:**
+${gherkinStep}
+
+**Available Markers (${markerInfo.length} total):**
+${markerSummary}
+${markerInfo.length > 50 ? `\n... and ${markerInfo.length - 50} more markers` : ''}
+
+**Accessibility Snapshot (explains the screenshot structure):**
+The following accessibility snapshot describes the page structure with roles, accessible names, data-testid attributes, and data-mimic-* attributes. Use this to understand the page structure alongside the screenshot:
+\`\`\`
+${ariaSnapshot}
+\`\`\`
+
+## Analyze the screenshot and accessibility snapshot, then return up to the top 5 most likely elements that the Gherkin step is referring to.
+Use the marker ID numbers (mimicId) shown on the badges in the screenshot and referenced in the accessibility snapshot to identify elements.
+
+**Remember:**
+- "click" = primary click (left click) - the default for any click action
+- "hover" is ONLY for explicit hover instructions, NEVER for click steps
+- When in doubt about click type, default to "primary"
+`;
+    const promptTime = Date.now() - promptStart;
+    console.log(`📝 [getClickAction] Built prompt in ${promptTime}ms, prompt length: ${prompt.length} chars`);
+    // Build message content - try without image first, then retry with image if needed
+    const messageStart = Date.now();
+    // First attempt: text-only (no image) - faster and cheaper
+    const messageContentTextOnly = [
+        { type: 'text', text: prompt }
+    ];
+    // Second attempt: with image (if first attempt fails)
+    const messageContentWithImage = [
+        { type: 'text', text: prompt },
+        { type: 'image', image: screenshotBase64 }
+    ];
+    const messageTime = Date.now() - messageStart;
+    console.log(`📨 [getClickAction] Built message content in ${messageTime}ms`);
+    // Set explicit timeout for AI calls to prevent indefinite hangs
+    // 2 minutes should be sufficient for most AI responses, even with retries
+    const aiTimeout = 120_000; // 2 minutes
+    let res;
+    let aiTime;
+    let usedImage = false;
+    // First attempt: try without image (text-only with accessibility snapshot)
+    console.log('🤖 [getClickAction] Calling AI model (text-only, no image)...');
+    const aiStart = Date.now();
+    try {
+        res = await Promise.race([
+            generateText({
+                model: brain,
+                messages: [
+                    {
+                        role: 'user',
+                        content: messageContentTextOnly
+                    }
+                ],
+                maxRetries: 2, // Fewer retries for first attempt
+                output: Output.object({ schema: zClickActionResult, name: 'clickActionResult' }),
+            }),
+            new Promise((_, reject) => setTimeout(() => reject(new Error(`AI model call timed out after ${aiTimeout}ms`)), aiTimeout))
+        ]);
+        // Validate the response
+        if (!res.output || !res.output.candidates || !Array.isArray(res.output.candidates) || res.output.candidates.length === 0) {
+            throw new Error('First attempt returned invalid result, retrying with image...');
+        }
+        aiTime = Date.now() - aiStart;
+        console.log(`✅ [getClickAction] AI model responded successfully (text-only) in ${aiTime}ms (${(aiTime / 1000).toFixed(2)}s)`);
+    }
+    catch (error) {
+        // First attempt failed - retry with image
+        console.log(`⚠️  [getClickAction] First attempt failed, retrying with image: ${error instanceof Error ? error.message : String(error)}`);
+        const retryStart = Date.now();
+        usedImage = true;
+        try {
+            res = await Promise.race([
+                generateText({
+                    model: brain,
+                    messages: [
+                        {
+                            role: 'user',
+                            content: messageContentWithImage
+                        }
+                    ],
+                    maxRetries: 3,
+                    output: Output.object({ schema: zClickActionResult, name: 'clickActionResult' }),
+                }),
+                new Promise((_, reject) => setTimeout(() => reject(new Error(`AI model call timed out after ${aiTimeout}ms`)), aiTimeout))
+            ]);
+            aiTime = Date.now() - retryStart;
+            console.log(`✅ [getClickAction] AI model responded successfully (with image) in ${aiTime}ms (${(aiTime / 1000).toFixed(2)}s)`);
+        }
+        catch (retryError) {
+            const elapsed = Date.now() - aiStart;
+            throw new Error(`AI model call failed after ${elapsed}ms (tried both text-only and with image): ${retryError instanceof Error ? retryError.message : String(retryError)}`);
+        }
+    }
+    await countTokens(res);
+    const totalTime = Date.now() - startTime;
+    console.log(`⏱️  [getClickAction] Total time: ${totalTime}ms (${(totalTime / 1000).toFixed(2)}s)${usedImage ? ' (used image on retry)' : ' (text-only, no image needed)'}`);
+    console.log(`   Breakdown: screenshot=${screenshotTime}ms, base64=${base64Time}ms, markers=${markerTime}ms, summary=${summaryTime}ms, prompt=${promptTime}ms, message=${messageTime}ms, AI=${aiTime}ms`);
+    // Validate that the AI model returned a valid structured output
+    // The output should always be defined when using structured outputs, but add defensive check
+    if (!res.output) {
+        throw new Error('AI model failed to generate valid click action result. The output is undefined.');
+    }
+    // Validate that candidates array exists and is not empty
+    if (!res.output.candidates || !Array.isArray(res.output.candidates) || res.output.candidates.length === 0) {
+        throw new Error(`AI model returned invalid click action result: candidates array is missing, not an array, or empty. Received: ${JSON.stringify(res.output)}`);
+    }
+    return res.output;
+};
+/**
+ * Execute a click action on a page element with plain English annotation
+ *
+ * This function performs the actual click interaction and logs what action
+ * is being performed in human-readable terms for better test traceability.
+ * Uses the LLM-generated description from the selected candidate.
+ *
+ * @param element - Playwright Locator for the target element to interact with
+ * @param clickActionResult - Click action result containing click type and target information
+ * @param selectedCandidate - The selected candidate element with LLM-generated description
+ * @param testInfo - Playwright TestInfo for adding annotations
+ * @param gherkinStep - The original Gherkin step for annotation type
+ * @returns Promise that resolves to an object containing the action result and selector (for snapshot storage)
+ */
+export const executeClickAction = async (element, clickActionResult, selectedCandidate, testInfo, gherkinStep) => {
+    // Use the LLM-generated description from the candidate
+    // This description was created by the AI when matching the element
+    const elementDescription = selectedCandidate.description || 'element';
+    // Check if element is valid before attempting click
+    if (!element) {
+        throw new Error(`Cannot click: element not found or page may be closed`);
+    }
+    // Build annotation description based on click type
+    let annotationDescription = '';
+    switch (clickActionResult.clickType) {
+        case 'primary':
+            annotationDescription = `→ Clicking on ${elementDescription} with left mouse button`;
+            break;
+        case 'secondary':
+            annotationDescription = `→ Right-clicking on ${elementDescription} to open context menu`;
+            break;
+        case 'double':
+            annotationDescription = `→ Double-clicking on ${elementDescription} to activate`;
+            break;
+        case 'tertiary':
+            annotationDescription = `→ Clicking on ${elementDescription} with middle mouse button`;
+            break;
+        case 'hover':
+            annotationDescription = `→ Hovering over ${elementDescription} to reveal additional options`;
+            break;
+        default:
+            throw new Error(`Unknown click type: ${clickActionResult.clickType}`);
+    }
+    // Generate Playwright code equivalent BEFORE performing the action
+    // This ensures the element is still available (before navigation/closure)
+    let playwrightCode;
+    let selector = null;
+    try {
+        // Generate the best selector descriptor from the element
+        // This gives us a descriptive, stable selector for snapshot storage
+        // Use 30-second timeout for selector generation
+        const selectorDescriptor = await generateBestSelectorForElement(element, { timeout: 30000 });
+        const selectorCode = selectorToPlaywrightCode(selectorDescriptor);
+        playwrightCode = generateClickCode(selectorCode, clickActionResult.clickType);
+        // Store the selector descriptor for snapshot storage
+        selector = selectorDescriptor;
+    }
+    catch (error) {
+        // If generating from element fails, fall back to mimicId if available
+        // This can happen if the element is not available or page is closing
+        const errorMessage = error?.message || String(error);
+        // Check if page closed - this can happen with navigation after clicks
+        if (errorMessage.includes('closed') || errorMessage.includes('Target page')) {
+            console.warn('Page closed during selector generation, using mimicId fallback');
+        }
+        if (selectedCandidate.mimicId) {
+            const selectorCode = `page.locator('[data-mimic-id="${selectedCandidate.mimicId}"]')`;
+            playwrightCode = generateClickCode(selectorCode, clickActionResult.clickType);
+            // Create a CSS selector descriptor as fallback
+            selector = {
+                type: 'css',
+                selector: `[data-mimic-id="${selectedCandidate.mimicId}"]`
+            };
+        }
+        else {
+            // If we can't generate the code, that's okay - just skip it
+            console.warn('Could not generate Playwright code for click action:', errorMessage);
+        }
+    }
+    // Add annotation using centralized utility (includes Playwright code if available)
+    addAnnotation(testInfo, gherkinStep, annotationDescription, playwrightCode);
+    // Perform the click action
+    switch (clickActionResult.clickType) {
+        case 'primary':
+            await element.click();
+            break;
+        case 'secondary':
+            await element.click({ button: 'right' });
+            break;
+        case 'double':
+            await element.dblclick();
+            break;
+        case 'tertiary':
+            await element.click({ button: 'middle' });
+            break;
+        case 'hover':
+            await element.hover();
+            break;
+        default:
+            throw new Error(`Unknown click type: ${clickActionResult.clickType}`);
+    }
+    // Return action result and selector for snapshot storage
+    return { actionResult: clickActionResult, selector };
+};
+//# sourceMappingURL=click.js.map

package/dist/mimic/click.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../src/mimic/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,YAAY,EAAE,MAAM,EAAE,MAAM,IAAI,CAAA;AAG7D,OAAO,EACL,kBAAkB,EAEnB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,8BAA8B,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAC3F,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAGvE;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,IAAU,EACV,KAAoB,EACpB,WAAmB,EACnB,WAAyB,EACG,EAAE;IAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,uDAAuD;IACvD,OAAO,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC;IAC/E,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACnC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,8CAA8C,cAAc,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAEvH,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC/B,MAAM,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,yDAAyD,UAAU,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAElL,sEAAsE;IACtE,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,iBAAiB,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,2DAA2D,gBAAgB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,YAAY,CAAC,MAAM,QAAQ,CAAC,CAAC;IAE9K,mDAAmD;IACnD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC/B,MAAM,UAAU,GAMX,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE;QAEvB,OAAO;YACL,EAAE,EAAE,CAAC,CAAC,OAAO;YACb,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,SAAS,EAAE,CAAC,CAAC,SAAS;SACvB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,iCAAiC,UAAU,CAAC,MAAM,eAAe,UAAU,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAElI,sCAAsC;IACtC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,aAAa,GAAG,UAAU;SAC7B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,wDAAwD;SACrE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;SACpL,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,+CAA+C,WAAW,IAAI,CAAC,CAAC;IAE5E,2CAA2C;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC/B,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,kBAAkB,GAAG,WAAW,CAAC,CAAC,CAAC;;iBAE1B,WAAW,CAAC,YAAY,CAAC,GAAG;wBACrB,WAAW,CAAC,YAAY,CAAC,SAAS;SACjD,WAAW,CAAC,gBAAgB,GAAG,CAAC,OAAO,WAAW,CAAC,UAAU;EACpE,WAAW,CAAC,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;;EAEhH,WAAW,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,CAChD,GAAG,GAAG,GAAG,CAAC,UAAU,QAAQ,CAAC,SAAS,GAAG,CAAC,MAAM,QAAQ,CAAC,QAAQ,MAAM,QAAQ,CAAC,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CACzI,CAAC,IAAI,CAAC,IAAI,CAAC;CACX,CAAC,CAAC,CAAC,EAAE;CACL,CAAC,CAAC,CAAC,EAAE,CAAC;IAEL,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA0Df,kBAAkB;;EAElB,WAAW;;uBAEU,UAAU,CAAC,MAAM;EACtC,aAAa;EACb,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,aAAa,UAAU,CAAC,MAAM,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE;;;;;EAKhF,YAAY;;;;;;;;;;CAUb,CAAC;IACA,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,uCAAuC,UAAU,sBAAsB,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;IAE1G,mFAAmF;IACnF,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,2DAA2D;IAC3D,MAAM,sBAAsB,GAA0C;QACpE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;KAC/B,CAAC;IAEF,sDAAsD;IACtD,MAAM,uBAAuB,GAA6E;QACxG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;QAC9B,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE;KAC3C,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,gDAAgD,WAAW,IAAI,CAAC,CAAC;IAE7E,gEAAgE;IAChE,0EAA0E;IAC1E,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,YAAY;IACvC,IAAI,GAAG,CAAC;IACR,IAAI,MAAc,CAAC;IACnB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,2EAA2E;IAC3E,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACvB,YAAY,CAAC;gBACX,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,sBAAsB;qBAChC;iBACF;gBACD,UAAU,EAAE,CAAC,EAAE,kCAAkC;gBACjD,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC;aACjF,CAAC;YACF,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAC/F;SACF,CAAC,CAAC;QAEH,wBAAwB;QACxB,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzH,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,qEAAqE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAChI,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,0CAA0C;QAC1C,OAAO,CAAC,GAAG,CAAC,mEAAmE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzI,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,SAAS,GAAG,IAAI,CAAC;QAEjB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBACvB,YAAY,CAAC;oBACX,KAAK,EAAE,KAAK;oBACZ,QAAQ,EAAE;wBACR;4BACE,IAAI,EAAE,MAAM;4BACZ,OAAO,EAAE,uBAAuB;yBACjC;qBACF;oBACD,UAAU,EAAE,CAAC;oBACb,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC;iBACjF,CAAC;gBACF,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAC/F;aACF,CAAC,CAAC;YAEH,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,sEAAsE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjI,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,6CAA6C,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC7K,CAAC;IACH,CAAC;IAED,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IAEvB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,oCAAoC,SAAS,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,+BAA+B,EAAE,CAAC,CAAC;IAC5K,OAAO,CAAC,GAAG,CAAC,4BAA4B,cAAc,cAAc,UAAU,eAAe,UAAU,eAAe,WAAW,cAAc,UAAU,eAAe,WAAW,UAAU,MAAM,IAAI,CAAC,CAAC;IAEzM,gEAAgE;IAChE,6FAA6F;IAC7F,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;IACrG,CAAC;IAED,yDAAyD;IACzD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1G,MAAM,IAAI,KAAK,CAAC,iHAAiH,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACjK,CAAC;IAED,OAAO,GAAG,CAAC,MAAM,CAAC;AACpB,CAAC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,OAAuB,EACvB,iBAAoC,EACpC,iBAAqD,EACrD,QAA8B,EAC9B,WAAmB,EACgE,EAAE;IACrF,uDAAuD;IACvD,mEAAmE;IACnE,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,WAAW,IAAI,SAAS,CAAC;IAEtE,oDAAoD;IACpD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IAED,mDAAmD;IACnD,IAAI,qBAAqB,GAAG,EAAE,CAAC;IAC/B,QAAQ,iBAAiB,CAAC,SAAS,EAAE,CAAC;QACpC,KAAK,SAAS;YACZ,qBAAqB,GAAG,iBAAiB,kBAAkB,yBAAyB,CAAC;YACrF,MAAM;QACR,KAAK,WAAW;YACd,qBAAqB,GAAG,uBAAuB,kBAAkB,uBAAuB,CAAC;YACzF,MAAM;QACR,KAAK,QAAQ;YACX,qBAAqB,GAAG,wBAAwB,kBAAkB,cAAc,CAAC;YACjF,MAAM;QACR,KAAK,UAAU;YACb,qBAAqB,GAAG,iBAAiB,kBAAkB,2BAA2B,CAAC;YACvF,MAAM;QACR,KAAK,OAAO;YACV,qBAAqB,GAAG,mBAAmB,kBAAkB,+BAA+B,CAAC;YAC7F,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,uBAAuB,iBAAiB,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,mEAAmE;IACnE,0EAA0E;IAC1E,IAAI,cAAkC,CAAC;IACvC,IAAI,QAAQ,GAA8B,IAAI,CAAC;IAE/C,IAAI,CAAC;QACH,yDAAyD;QACzD,oEAAoE;QACpE,gDAAgD;QAChD,MAAM,kBAAkB,GAAG,MAAM,8BAA8B,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7F,MAAM,YAAY,GAAG,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;QAClE,cAAc,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAE9E,qDAAqD;QACrD,QAAQ,GAAG,kBAAkB,CAAC;IAChC,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,sEAAsE;QACtE,qEAAqE;QACrE,MAAM,YAAY,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;QAErD,sEAAsE;QACtE,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YAC5E,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,iBAAiB,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,iCAAiC,iBAAiB,CAAC,OAAO,MAAM,CAAC;YACtF,cAAc,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAC9E,+CAA+C;YAC/C,QAAQ,GAAG;gBACT,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,mBAAmB,iBAAiB,CAAC,OAAO,IAAI;aAC3D,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,4DAA4D;YAC5D,OAAO,CAAC,IAAI,CAAC,sDAAsD,EAAE,YAAY,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAED,mFAAmF;IACnF,aAAa,CAAC,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE,cAAc,CAAC,CAAC;IAE5E,2BAA2B;IAC3B,QAAQ,iBAAiB,CAAC,SAAS,EAAE,CAAC;QACpC,KAAK,SAAS;YACZ,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,MAAM;QACR,KAAK,WAAW;YACd,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YACzC,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM;QACR,KAAK,UAAU;YACb,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC1C,MAAM;QACR,KAAK,OAAO;YACV,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,uBAAuB,iBAAiB,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,yDAAyD;IACzD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,CAAC;AACvD,CAAC,CAAC"}

package/dist/mimic/forms.d.ts

@@ -0,0 +1,52 @@
+import { Locator, Page, TestInfo } from '@playwright/test';
+import { type LanguageModel } from 'ai';
+import z from 'zod';
+import type { TestContext } from '../mimic.js';
+import type { SelectorDescriptor } from './selectorTypes.js';
+declare const zFormActionResult: z.ZodObject<{
+    mimicId: z.ZodNumber;
+    type: z.ZodEnum<{
+        click: "click";
+        type: "type";
+        fill: "fill";
+        select: "select";
+        uncheck: "uncheck";
+        check: "check";
+        setInputFiles: "setInputFiles";
+        clear: "clear";
+        keypress: "keypress";
+    }>;
+    params: z.ZodObject<{
+        value: z.ZodString;
+        modifiers: z.ZodArray<z.ZodEnum<{
+            Alt: "Alt";
+            Control: "Control";
+            Meta: "Meta";
+            Shift: "Shift";
+            none: "none";
+        }>>;
+    }, z.core.$strip>;
+    elementDescription: z.ZodString;
+}, z.core.$strip>;
+export type FormActionResult = z.infer<typeof zFormActionResult>;
+export declare const getFormAction: (page: Page, brain: LanguageModel, gherkinStep: string, testContext?: TestContext) => Promise<FormActionResult>;
+/**
+ * Execute a form action on a page element with plain English annotation
+ *
+ * This function performs form interactions (typing, selecting, checking, etc.)
+ * and adds test annotations for better traceability and validation.
+ * Uses the LLM-generated element description from the form action result.
+ *
+ * @param page - Playwright Page object for keyboard actions
+ * @param formActionResult - Form action result containing action type, parameters, and element description
+ * @param targetElement - Playwright Locator for the target form element
+ * @param testInfo - Playwright TestInfo for adding annotations (optional)
+ * @param gherkinStep - The original Gherkin step for annotation type (optional)
+ * @returns Promise that resolves when the form action is complete
+ */
+export declare const executeFormAction: (page: Page, formActionResult: FormActionResult, targetElement: Locator | null, testInfo?: TestInfo, gherkinStep?: string) => Promise<{
+    actionResult: FormActionResult;
+    selector: SelectorDescriptor | null;
+}>;
+export {};
+//# sourceMappingURL=forms.d.ts.map

package/dist/mimic/forms.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"forms.d.ts","sourceRoot":"","sources":["../../src/mimic/forms.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,KAAK,aAAa,EAAwB,MAAM,IAAI,CAAA;AAC7D,OAAO,CAAC,MAAM,KAAK,CAAC;AAIpB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7D,QAAA,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;iBAgBrB,CAAA;AAGF,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAEjE,eAAO,MAAM,aAAa,GACxB,MAAM,IAAI,EACV,OAAO,aAAa,EACpB,aAAa,MAAM,EACnB,cAAc,WAAW,KACxB,OAAO,CAAC,gBAAgB,CAsP1B,CAAA;AAGD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iBAAiB,GAC5B,MAAM,IAAI,EACV,kBAAkB,gBAAgB,EAClC,eAAe,OAAO,GAAG,IAAI,EAC7B,WAAW,QAAQ,EACnB,cAAc,MAAM,KACnB,OAAO,CAAC;IAAE,YAAY,EAAE,gBAAgB,CAAC;IAAC,QAAQ,EAAE,kBAAkB,GAAG,IAAI,CAAA;CAAE,CAmQjF,CAAA"}