tamash-playwright 0.3.0 → 0.5.0

package/dist/healer/providers/gemini.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createGeminiProvider = createGeminiProvider;
 const prompt_1 = require("./prompt");
+const vision_models_1 = require("./vision-models");
 // Fallback only — normally overridden by SuggestSelectorInput.timeoutMs, which mirrors the
 // Playwright project's configured actionTimeout so the healer never outlasts the action budget.
 const REQUEST_TIMEOUT_MS = 15000;
@@ -14,8 +15,10 @@ function createGeminiProvider() {
     // Gemini's OpenAI-compatible surface — same request/response shape as OpenAI's Chat
     // Completions API, so this reuses the same prompt/parsing logic as providers/openai.ts.
     const baseUrl = (process.env.GEMINI_BASE_URL ?? 'https://generativelanguage.googleapis.com/v1beta/openai').replace(/\/+$/, '');
+    const supportsVision = (0, vision_models_1.isVisionCapableModel)('gemini', model);
     return {
         name: `gemini:${model}`,
+        supportsVision,
         async suggestSelector(input) {
             const controller = new AbortController();
             const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
@@ -61,6 +64,102 @@ function createGeminiProvider() {
                 clearTimeout(timeout);
             }
         },
+        async suggestSelectorFromImage(input) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
+            try {
+                const response = await fetch(`${baseUrl}/chat/completions`, {
+                    method: 'POST',
+                    signal: controller.signal,
+                    headers: {
+                        'content-type': 'application/json',
+                        authorization: `Bearer ${apiKey}`,
+                    },
+                    body: JSON.stringify({
+                        model,
+                        temperature: 0,
+                        response_format: { type: 'json_object' },
+                        messages: [
+                            { role: 'system', content: prompt_1.VISION_SYSTEM_PROMPT },
+                            {
+                                role: 'user',
+                                content: [
+                                    { type: 'text', text: (0, prompt_1.buildVisionUserPrompt)(input) },
+                                    { type: 'image_url', image_url: { url: `data:image/png;base64,${input.imageBase64}` } },
+                                ],
+                            },
+                        ],
+                    }),
+                });
+                if (!response.ok) {
+                    console.warn(`[self-healer] Gemini vision request failed: ${response.status} ${response.statusText}`);
+                    return null;
+                }
+                const payload = await response.json();
+                const content = payload.choices?.[0]?.message?.content;
+                if (!content) {
+                    return null;
+                }
+                const point = (0, prompt_1.parseVisionSuggestion)(content);
+                if (!point) {
+                    return null;
+                }
+                return { point, usage: (0, prompt_1.extractOpenAICompatibleUsage)(payload) };
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                console.warn(`[self-healer] Gemini vision provider error: ${reason}`);
+                return null;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        },
+        async suggestActionTactic(input) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
+            try {
+                const response = await fetch(`${baseUrl}/chat/completions`, {
+                    method: 'POST',
+                    signal: controller.signal,
+                    headers: {
+                        'content-type': 'application/json',
+                        authorization: `Bearer ${apiKey}`,
+                    },
+                    body: JSON.stringify({
+                        model,
+                        temperature: 0,
+                        response_format: { type: 'json_object' },
+                        messages: [
+                            { role: 'system', content: prompt_1.ACTION_RECOVERY_SYSTEM_PROMPT },
+                            { role: 'user', content: (0, prompt_1.buildActionRecoveryUserPrompt)(input) },
+                        ],
+                    }),
+                });
+                if (!response.ok) {
+                    console.warn(`[self-healer] Gemini action-recovery request failed: ${response.status} ${response.statusText}`);
+                    return null;
+                }
+                const payload = await response.json();
+                const content = payload.choices?.[0]?.message?.content;
+                if (!content) {
+                    return null;
+                }
+                const tactic = (0, prompt_1.parseActionTacticSuggestion)(content);
+                if (!tactic) {
+                    return null;
+                }
+                return { tactic, usage: (0, prompt_1.extractOpenAICompatibleUsage)(payload) };
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                console.warn(`[self-healer] Gemini action-recovery provider error: ${reason}`);
+                return null;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        },
     };
 }
 //# sourceMappingURL=gemini.js.map

package/dist/healer/providers/gemini.js.map

@@ -1 +1 @@
-{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../../src/healer/providers/gemini.ts"],"names":[],"mappings":";;;AACA,qCAAyG;AAEzG,2FAA2F;AAC3F,gGAAgG;AAChG,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC;IACE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAEvC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oFAAoF;IACpF,wFAAwF;IACxF,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yDAAyD,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAE/H,OAAO;QACL,IAAI,EAAE,UAAU,KAAK,EAAE;QACvB,KAAK,CAAC,eAAe,CAAC,KAA2B;YAC/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;oBAC1D,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,WAAW,EAAE,CAAC;wBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;wBACxC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sBAAa,EAAE;4BAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,wBAAe,EAAC,KAAK,CAAC,EAAE;yBAClD;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wCAAwC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/F,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;gBACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,UAAU,GAAG,IAAA,wBAAe,EAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,qCAA4B,EAAC,OAAO,CAAC,EAAE,CAAC;YACtE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../../src/healer/providers/gemini.ts"],"names":[],"mappings":";;;AACA,qCAAwQ;AACxQ,mDAAuD;AAEvD,2FAA2F;AAC3F,gGAAgG;AAChG,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC;IACE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAEvC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oFAAoF;IACpF,wFAAwF;IACxF,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yDAAyD,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/H,MAAM,cAAc,GAAG,IAAA,oCAAoB,EAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAE7D,OAAO;QACL,IAAI,EAAE,UAAU,KAAK,EAAE;QACvB,cAAc;QACd,KAAK,CAAC,eAAe,CAAC,KAA2B;YAC/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;oBAC1D,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,WAAW,EAAE,CAAC;wBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;wBACxC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sBAAa,EAAE;4BAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,wBAAe,EAAC,KAAK,CAAC,EAAE;yBAClD;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wCAAwC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/F,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;gBACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,UAAU,GAAG,IAAA,wBAAe,EAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,qCAA4B,EAAC,OAAO,CAAC,EAAE,CAAC;YACtE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,wBAAwB,CAAC,KAAmC;YAChE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;oBAC1D,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,WAAW,EAAE,CAAC;wBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;wBACxC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,6BAAoB,EAAE;4BACjD;gCACE,IAAI,EAAE,MAAM;gCACZ,OAAO,EAAE;oCACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAA,8BAAqB,EAAC,KAAK,CAAC,EAAE;oCACpD,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,yBAAyB,KAAK,CAAC,WAAW,EAAE,EAAE,EAAE;iCACxF;6BACF;yBACF;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,+CAA+C,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBACtG,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;gBACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,KAAK,GAAG,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;gBAC7C,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAA,qCAA4B,EAAC,OAAO,CAAC,EAAE,CAAC;YACjE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;gBACtE,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,mBAAmB,CAAC,KAA+B;YACvD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;oBAC1D,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,WAAW,EAAE,CAAC;wBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;wBACxC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sCAA6B,EAAE;4BAC1D,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,sCAA6B,EAAC,KAAK,CAAC,EAAE;yBAChE;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wDAAwD,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/G,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;gBACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,MAAM,GAAG,IAAA,oCAA2B,EAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAA,qCAA4B,EAAC,OAAO,CAAC,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wDAAwD,MAAM,EAAE,CAAC,CAAC;gBAC/E,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/healer/providers/ollama.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../../src/healer/providers/ollama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAwC,MAAM,SAAS,CAAC;AAOlF,wBAAgB,oBAAoB,IAAI,YAAY,GAAG,IAAI,CA6E1D"}
+{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../../src/healer/providers/ollama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAsB,YAAY,EAAsH,MAAM,SAAS,CAAC;AAQpL,wBAAgB,oBAAoB,IAAI,YAAY,GAAG,IAAI,CA2M1D"}

package/dist/healer/providers/ollama.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createOllamaProvider = createOllamaProvider;
 const prompt_1 = require("./prompt");
+const vision_models_1 = require("./vision-models");
 // Fallback only — normally overridden by SuggestSelectorInput.timeoutMs, which mirrors the
 // Playwright project's configured actionTimeout so the healer never outlasts the action budget.
 const REQUEST_TIMEOUT_MS = 15000;
@@ -12,8 +13,10 @@ function createOllamaProvider() {
         return null;
     }
     const baseUrl = (process.env.OLLAMA_BASE_URL ?? 'https://ollama.com').replace(/\/+$/, '');
+    const supportsVision = (0, vision_models_1.isVisionCapableModel)('ollama', model);
     return {
         name: `ollama:${model}`,
+        supportsVision,
         async suggestSelector(input) {
             const controller = new AbortController();
             const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
@@ -71,6 +74,116 @@ function createOllamaProvider() {
                 clearTimeout(timeout);
             }
         },
+        async suggestSelectorFromImage(input) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
+            try {
+                const response = await fetch(`${baseUrl}/api/chat`, {
+                    method: 'POST',
+                    signal: controller.signal,
+                    headers: {
+                        'content-type': 'application/json',
+                        authorization: `Bearer ${apiKey}`,
+                    },
+                    body: JSON.stringify({
+                        model,
+                        stream: false,
+                        format: 'json',
+                        options: { temperature: 0 },
+                        messages: [
+                            { role: 'system', content: prompt_1.VISION_SYSTEM_PROMPT },
+                            { role: 'user', content: (0, prompt_1.buildVisionUserPrompt)(input), images: [input.imageBase64] },
+                        ],
+                    }),
+                });
+                if (!response.ok) {
+                    console.warn(`[self-healer] Ollama vision request failed: ${response.status} ${response.statusText}`);
+                    return null;
+                }
+                const payload = await response.json();
+                const content = payload.message?.content;
+                if (!content) {
+                    return null;
+                }
+                const point = (0, prompt_1.parseVisionSuggestion)(content);
+                if (!point) {
+                    return null;
+                }
+                const inputTokens = payload.prompt_eval_count;
+                const outputTokens = payload.eval_count;
+                return {
+                    point,
+                    usage: {
+                        inputTokens,
+                        outputTokens,
+                        totalTokens: inputTokens !== undefined && outputTokens !== undefined ? inputTokens + outputTokens : undefined,
+                    },
+                };
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                console.warn(`[self-healer] Ollama vision provider error: ${reason}`);
+                return null;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        },
+        async suggestActionTactic(input) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
+            try {
+                const response = await fetch(`${baseUrl}/api/chat`, {
+                    method: 'POST',
+                    signal: controller.signal,
+                    headers: {
+                        'content-type': 'application/json',
+                        authorization: `Bearer ${apiKey}`,
+                    },
+                    body: JSON.stringify({
+                        model,
+                        stream: false,
+                        format: 'json',
+                        options: { temperature: 0 },
+                        messages: [
+                            { role: 'system', content: prompt_1.ACTION_RECOVERY_SYSTEM_PROMPT },
+                            { role: 'user', content: (0, prompt_1.buildActionRecoveryUserPrompt)(input) },
+                        ],
+                    }),
+                });
+                if (!response.ok) {
+                    console.warn(`[self-healer] Ollama action-recovery request failed: ${response.status} ${response.statusText}`);
+                    return null;
+                }
+                const payload = await response.json();
+                const content = payload.message?.content;
+                if (!content) {
+                    return null;
+                }
+                const tactic = (0, prompt_1.parseActionTacticSuggestion)(content);
+                if (!tactic) {
+                    return null;
+                }
+                const inputTokens = payload.prompt_eval_count;
+                const outputTokens = payload.eval_count;
+                return {
+                    tactic,
+                    usage: {
+                        inputTokens,
+                        outputTokens,
+                        totalTokens: inputTokens !== undefined && outputTokens !== undefined ? inputTokens + outputTokens : undefined,
+                    },
+                };
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                console.warn(`[self-healer] Ollama action-recovery provider error: ${reason}`);
+                return null;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        },
     };
 }
 //# sourceMappingURL=ollama.js.map

package/dist/healer/providers/ollama.js.map

@@ -1 +1 @@
-{"version":3,"file":"ollama.js","sourceRoot":"","sources":["../../../src/healer/providers/ollama.ts"],"names":[],"mappings":";;;AACA,qCAA2E;AAE3E,2FAA2F;AAC3F,gGAAgG;AAChG,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC;IACE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAEvC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,oBAAoB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAE1F,OAAO;QACL,IAAI,EAAE,UAAU,KAAK,EAAE;QACvB,KAAK,CAAC,eAAe,CAAC,KAA2B;YAC/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,iGAAiG;gBACjG,uCAAuC;gBACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,WAAW,EAAE;oBAClD,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE;wBAC3B,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sBAAa,EAAE;4BAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,wBAAe,EAAC,KAAK,CAAC,EAAE;yBAClD;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wCAAwC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/F,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAIlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC;gBACzC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,UAAU,GAAG,IAAA,wBAAe,EAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC;gBAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC;gBACxC,OAAO;oBACL,UAAU;oBACV,KAAK,EAAE;wBACL,WAAW;wBACX,YAAY;wBACZ,WAAW,EAAE,WAAW,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS;qBAC9G;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"ollama.js","sourceRoot":"","sources":["../../../src/healer/providers/ollama.ts"],"names":[],"mappings":";;;AACA,qCAA0O;AAC1O,mDAAuD;AAEvD,2FAA2F;AAC3F,gGAAgG;AAChG,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC;IACE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAEvC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,oBAAoB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1F,MAAM,cAAc,GAAG,IAAA,oCAAoB,EAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAE7D,OAAO;QACL,IAAI,EAAE,UAAU,KAAK,EAAE;QACvB,cAAc;QACd,KAAK,CAAC,eAAe,CAAC,KAA2B;YAC/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,iGAAiG;gBACjG,uCAAuC;gBACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,WAAW,EAAE;oBAClD,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE;wBAC3B,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sBAAa,EAAE;4BAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,wBAAe,EAAC,KAAK,CAAC,EAAE;yBAClD;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wCAAwC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/F,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAIlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC;gBACzC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,UAAU,GAAG,IAAA,wBAAe,EAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC;gBAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC;gBACxC,OAAO;oBACL,UAAU;oBACV,KAAK,EAAE;wBACL,WAAW;wBACX,YAAY;wBACZ,WAAW,EAAE,WAAW,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS;qBAC9G;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,wBAAwB,CAAC,KAAmC;YAChE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,WAAW,EAAE;oBAClD,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE;wBAC3B,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,6BAAoB,EAAE;4BACjD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,8BAAqB,EAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE;yBACrF;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,+CAA+C,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBACtG,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAIlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC;gBACzC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,KAAK,GAAG,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;gBAC7C,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC;gBAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC;gBACxC,OAAO;oBACL,KAAK;oBACL,KAAK,EAAE;wBACL,WAAW;wBACX,YAAY;wBACZ,WAAW,EAAE,WAAW,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS;qBAC9G;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;gBACtE,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,mBAAmB,CAAC,KAA+B;YACvD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,WAAW,EAAE;oBAClD,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE;wBAC3B,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sCAA6B,EAAE;4BAC1D,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,sCAA6B,EAAC,KAAK,CAAC,EAAE;yBAChE;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wDAAwD,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/G,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAIlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC;gBACzC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,MAAM,GAAG,IAAA,oCAA2B,EAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC;gBAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC;gBACxC,OAAO;oBACL,MAAM;oBACN,KAAK,EAAE;wBACL,WAAW;wBACX,YAAY;wBACZ,WAAW,EAAE,WAAW,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS;qBAC9G;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wDAAwD,MAAM,EAAE,CAAC,CAAC;gBAC/E,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/healer/providers/openai.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../src/healer/providers/openai.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAwC,MAAM,SAAS,CAAC;AAOlF,wBAAgB,oBAAoB,IAAI,YAAY,GAAG,IAAI,CAgE1D"}
+{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../src/healer/providers/openai.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAsB,YAAY,EAAsH,MAAM,SAAS,CAAC;AAQpL,wBAAgB,oBAAoB,IAAI,YAAY,GAAG,IAAI,CA8K1D"}

package/dist/healer/providers/openai.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createOpenAIProvider = createOpenAIProvider;
 const prompt_1 = require("./prompt");
+const vision_models_1 = require("./vision-models");
 // Fallback only — normally overridden by SuggestSelectorInput.timeoutMs, which mirrors the
 // Playwright project's configured actionTimeout so the healer never outlasts the action budget.
 const REQUEST_TIMEOUT_MS = 15000;
@@ -12,8 +13,10 @@ function createOpenAIProvider() {
         return null;
     }
     const baseUrl = (process.env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1').replace(/\/+$/, '');
+    const supportsVision = (0, vision_models_1.isVisionCapableModel)('openai', model);
     return {
         name: `openai:${model}`,
+        supportsVision,
         async suggestSelector(input) {
             const controller = new AbortController();
             const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
@@ -59,6 +62,102 @@ function createOpenAIProvider() {
                 clearTimeout(timeout);
             }
         },
+        async suggestSelectorFromImage(input) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
+            try {
+                const response = await fetch(`${baseUrl}/chat/completions`, {
+                    method: 'POST',
+                    signal: controller.signal,
+                    headers: {
+                        'content-type': 'application/json',
+                        authorization: `Bearer ${apiKey}`,
+                    },
+                    body: JSON.stringify({
+                        model,
+                        temperature: 0,
+                        response_format: { type: 'json_object' },
+                        messages: [
+                            { role: 'system', content: prompt_1.VISION_SYSTEM_PROMPT },
+                            {
+                                role: 'user',
+                                content: [
+                                    { type: 'text', text: (0, prompt_1.buildVisionUserPrompt)(input) },
+                                    { type: 'image_url', image_url: { url: `data:image/png;base64,${input.imageBase64}` } },
+                                ],
+                            },
+                        ],
+                    }),
+                });
+                if (!response.ok) {
+                    console.warn(`[self-healer] OpenAI vision request failed: ${response.status} ${response.statusText}`);
+                    return null;
+                }
+                const payload = await response.json();
+                const content = payload.choices?.[0]?.message?.content;
+                if (!content) {
+                    return null;
+                }
+                const point = (0, prompt_1.parseVisionSuggestion)(content);
+                if (!point) {
+                    return null;
+                }
+                return { point, usage: (0, prompt_1.extractOpenAICompatibleUsage)(payload) };
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                console.warn(`[self-healer] OpenAI vision provider error: ${reason}`);
+                return null;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        },
+        async suggestActionTactic(input) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), input.timeoutMs ?? REQUEST_TIMEOUT_MS);
+            try {
+                const response = await fetch(`${baseUrl}/chat/completions`, {
+                    method: 'POST',
+                    signal: controller.signal,
+                    headers: {
+                        'content-type': 'application/json',
+                        authorization: `Bearer ${apiKey}`,
+                    },
+                    body: JSON.stringify({
+                        model,
+                        temperature: 0,
+                        response_format: { type: 'json_object' },
+                        messages: [
+                            { role: 'system', content: prompt_1.ACTION_RECOVERY_SYSTEM_PROMPT },
+                            { role: 'user', content: (0, prompt_1.buildActionRecoveryUserPrompt)(input) },
+                        ],
+                    }),
+                });
+                if (!response.ok) {
+                    console.warn(`[self-healer] OpenAI action-recovery request failed: ${response.status} ${response.statusText}`);
+                    return null;
+                }
+                const payload = await response.json();
+                const content = payload.choices?.[0]?.message?.content;
+                if (!content) {
+                    return null;
+                }
+                const tactic = (0, prompt_1.parseActionTacticSuggestion)(content);
+                if (!tactic) {
+                    return null;
+                }
+                return { tactic, usage: (0, prompt_1.extractOpenAICompatibleUsage)(payload) };
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                console.warn(`[self-healer] OpenAI action-recovery provider error: ${reason}`);
+                return null;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        },
     };
 }
 //# sourceMappingURL=openai.js.map

package/dist/healer/providers/openai.js.map

@@ -1 +1 @@
-{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../../src/healer/providers/openai.ts"],"names":[],"mappings":";;;AACA,qCAAyG;AAEzG,2FAA2F;AAC3F,gGAAgG;AAChG,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC;IACE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAEvC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,2BAA2B,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO;QACL,IAAI,EAAE,UAAU,KAAK,EAAE;QACvB,KAAK,CAAC,eAAe,CAAC,KAA2B;YAC/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;oBAC1D,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,WAAW,EAAE,CAAC;wBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;wBACxC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sBAAa,EAAE;4BAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,wBAAe,EAAC,KAAK,CAAC,EAAE;yBAClD;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wCAAwC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/F,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;gBACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,UAAU,GAAG,IAAA,wBAAe,EAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,qCAA4B,EAAC,OAAO,CAAC,EAAE,CAAC;YACtE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../../src/healer/providers/openai.ts"],"names":[],"mappings":";;;AACA,qCAAwQ;AACxQ,mDAAuD;AAEvD,2FAA2F;AAC3F,gGAAgG;AAChG,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC;IACE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAEvC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,2BAA2B,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACjG,MAAM,cAAc,GAAG,IAAA,oCAAoB,EAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAE7D,OAAO;QACL,IAAI,EAAE,UAAU,KAAK,EAAE;QACvB,cAAc;QACd,KAAK,CAAC,eAAe,CAAC,KAA2B;YAC/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;oBAC1D,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,WAAW,EAAE,CAAC;wBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;wBACxC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sBAAa,EAAE;4BAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,wBAAe,EAAC,KAAK,CAAC,EAAE;yBAClD;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wCAAwC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/F,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;gBACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,UAAU,GAAG,IAAA,wBAAe,EAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,qCAA4B,EAAC,OAAO,CAAC,EAAE,CAAC;YACtE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wCAAwC,MAAM,EAAE,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,wBAAwB,CAAC,KAAmC;YAChE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;oBAC1D,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,WAAW,EAAE,CAAC;wBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;wBACxC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,6BAAoB,EAAE;4BACjD;gCACE,IAAI,EAAE,MAAM;gCACZ,OAAO,EAAE;oCACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAA,8BAAqB,EAAC,KAAK,CAAC,EAAE;oCACpD,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,yBAAyB,KAAK,CAAC,WAAW,EAAE,EAAE,EAAE;iCACxF;6BACF;yBACF;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,+CAA+C,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBACtG,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;gBACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,KAAK,GAAG,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;gBAC7C,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAA,qCAA4B,EAAC,OAAO,CAAC,EAAE,CAAC;YACjE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;gBACtE,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,mBAAmB,CAAC,KAA+B;YACvD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;YAE5F,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;oBAC1D,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;qBAClC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,WAAW,EAAE,CAAC;wBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;wBACxC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sCAA6B,EAAE;4BAC1D,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAA,sCAA6B,EAAC,KAAK,CAAC,EAAE;yBAChE;qBACF,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,CAAC,IAAI,CAAC,wDAAwD,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC/G,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAGlC,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;gBACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,MAAM,GAAG,IAAA,oCAA2B,EAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAA,qCAA4B,EAAC,OAAO,CAAC,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,wDAAwD,MAAM,EAAE,CAAC,CAAC;gBAC/E,OAAO,IAAI,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/healer/providers/prompt.d.ts

@@ -1,7 +1,13 @@
-import type { SelectorSuggestion, SuggestSelectorInput, TokenUsage } from './types';
+import type { ActionTactic, SelectorSuggestion, SuggestActionTacticInput, SuggestElementFromImageInput, SuggestSelectorInput, TokenUsage, VisionPoint } from './types';
 export declare const SYSTEM_PROMPT = "You are a Playwright self-healing assistant.\nYou will be given an ARIA accessibility snapshot of a web page, an action that just failed, and a human description of the element the test intended to interact with.\nPick the single best-matching element that is LITERALLY present in the snapshot and respond with strict JSON only (no markdown, no prose) matching one of:\n{\"strategy\":\"role\",\"role\":\"<aria role>\",\"name\":\"<accessible name>\"}\n{\"strategy\":\"text\",\"text\":\"<visible text>\"}\n{\"strategy\":\"testId\",\"testId\":\"<test id>\"}\n{\"strategy\":\"label\",\"label\":\"<label text>\"}\n{\"strategy\":\"placeholder\",\"placeholder\":\"<placeholder text>\"}\n{\"strategy\":\"css\",\"css\":\"<css selector>\"}\n{\"strategy\":\"none\"}\nIf nothing in the snapshot plausibly matches the description, respond with {\"strategy\":\"none\"}. Never invent an element that isn't in the snapshot.";
 export declare function buildUserPrompt(input: SuggestSelectorInput): string;
 export declare function parseSuggestion(content: string): SelectorSuggestion | null;
+export declare const VISION_SYSTEM_PROMPT = "You are a Playwright self-healing assistant.\nYou will be given a screenshot of a web page, an action that just failed, and a human description of the element the test intended to interact with.\nFind the single best-matching element that is LITERALLY visible in the screenshot and respond with strict JSON only (no markdown, no prose):\n{\"found\":true,\"x\":<0-1000>,\"y\":<0-1000>}\nWhere x and y are the CENTER of the matching element, expressed as its position within the image scaled to a 0-1000 range (0 = left/top edge, 1000 = right/bottom edge) \u2014 NOT raw pixel coordinates.\nIf nothing in the screenshot plausibly matches the description, respond with {\"found\":false}. Never invent an element that isn't visible in the image.";
+export declare function buildVisionUserPrompt(input: SuggestElementFromImageInput): string;
+export declare function parseVisionSuggestion(content: string): VisionPoint | null;
+export declare const ACTION_RECOVERY_SYSTEM_PROMPT = "You are a Playwright self-healing assistant.\nAn element was already correctly located, but the action on it failed for the reason described below \u2014 this is NOT a selector problem, so do not suggest one. Given the error, choose the single tactic most likely to help:\n{\"tactic\":\"scroll\"} \u2014 the element may be outside the current viewport; scroll it into view, then retry the same action.\n{\"tactic\":\"force\"} \u2014 the error looks like an overly strict actionability check rather than something genuinely blocking the click (e.g. flagged as \"not stable\" during a finished animation, or a borderline visibility/bounding-box check); retry the same action bypassing Playwright's actionability checks. Note: force still performs a real coordinate-based click, so it will NOT help if another element is genuinely covering this one \u2014 use \"dispatch\" for that instead.\n{\"tactic\":\"wait\"} \u2014 the error suggests something transient (an animation or transition still settling, content still loading); wait briefly, then retry the same action.\n{\"tactic\":\"dispatch\"} \u2014 the element is genuinely covered/intercepted by another element on top of it, or force did not help; dispatch the underlying DOM event directly on the element instead of a real coordinate-based interaction. Last resort, since it skips real hit-testing entirely.\n{\"tactic\":\"none\"} \u2014 none of the above would plausibly help.\nRespond with strict JSON only (no markdown, no prose), exactly one of the five objects above.";
+export declare function buildActionRecoveryUserPrompt(input: SuggestActionTacticInput): string;
+export declare function parseActionTacticSuggestion(content: string): ActionTactic | null;
 export declare function extractOpenAICompatibleUsage(payload: {
     usage?: {
         prompt_tokens?: number;

package/dist/healer/providers/prompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../src/healer/providers/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAIpF,eAAO,MAAM,aAAa,g5BAU0H,CAAC;AAErJ,wBAAgB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,MAAM,CAWnE;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CA4C1E;AAID,wBAAgB,4BAA4B,CAAC,OAAO,EAAE;IAAE,KAAK,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAAG,UAAU,GAAG,SAAS,CAWvK"}
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../src/healer/providers/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,oBAAoB,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAIvK,eAAO,MAAM,aAAa,g5BAU0H,CAAC;AAErJ,wBAAgB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,MAAM,CAWnE;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CA4C1E;AAKD,eAAO,MAAM,oBAAoB,0uBAKsH,CAAC;AAExJ,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,4BAA4B,GAAG,MAAM,CAMjF;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAiCzE;AASD,eAAO,MAAM,6BAA6B,4/CAOoD,CAAC;AAE/F,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAKrF;AAED,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAsBhF;AAID,wBAAgB,4BAA4B,CAAC,OAAO,EAAE;IAAE,KAAK,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAAG,UAAU,GAAG,SAAS,CAWvK"}

package/dist/healer/providers/prompt.js

@@ -1,8 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SYSTEM_PROMPT = void 0;
+exports.ACTION_RECOVERY_SYSTEM_PROMPT = exports.VISION_SYSTEM_PROMPT = exports.SYSTEM_PROMPT = void 0;
 exports.buildUserPrompt = buildUserPrompt;
 exports.parseSuggestion = parseSuggestion;
+exports.buildVisionUserPrompt = buildVisionUserPrompt;
+exports.parseVisionSuggestion = parseVisionSuggestion;
+exports.buildActionRecoveryUserPrompt = buildActionRecoveryUserPrompt;
+exports.parseActionTacticSuggestion = parseActionTacticSuggestion;
 exports.extractOpenAICompatibleUsage = extractOpenAICompatibleUsage;
 const MAX_SNAPSHOT_CHARS = 8000;
 exports.SYSTEM_PROMPT = `You are a Playwright self-healing assistant.
@@ -71,6 +75,96 @@ function parseSuggestion(content) {
     }
     return null;
 }
+// Used only as a fallback, after the ARIA/text-based prompt above has already failed to produce
+// a working heal — this is strictly the second attempt, never the first (the text path is
+// cheaper and already handles the common case).
+exports.VISION_SYSTEM_PROMPT = `You are a Playwright self-healing assistant.
+You will be given a screenshot of a web page, an action that just failed, and a human description of the element the test intended to interact with.
+Find the single best-matching element that is LITERALLY visible in the screenshot and respond with strict JSON only (no markdown, no prose):
+{"found":true,"x":<0-1000>,"y":<0-1000>}
+Where x and y are the CENTER of the matching element, expressed as its position within the image scaled to a 0-1000 range (0 = left/top edge, 1000 = right/bottom edge) — NOT raw pixel coordinates.
+If nothing in the screenshot plausibly matches the description, respond with {"found":false}. Never invent an element that isn't visible in the image.`;
+function buildVisionUserPrompt(input) {
+    return [
+        `Failed action: ${input.action}`,
+        `Element description: ${input.description ?? '(none provided)'}`,
+        'A screenshot of the page is attached.',
+    ].join('\n\n');
+}
+function parseVisionSuggestion(content) {
+    let parsed;
+    try {
+        parsed = JSON.parse(content);
+    }
+    catch {
+        const match = content.match(/\{[\s\S]*\}/);
+        if (!match) {
+            return null;
+        }
+        try {
+            parsed = JSON.parse(match[0]);
+        }
+        catch {
+            return null;
+        }
+    }
+    if (!parsed || typeof parsed !== 'object' || !('found' in parsed)) {
+        return null;
+    }
+    const point = parsed;
+    if (point.found === false) {
+        return { found: false };
+    }
+    if (point.found === true
+        && typeof point.x === 'number' && point.x >= 0 && point.x <= 1000
+        && typeof point.y === 'number' && point.y >= 0 && point.y <= 1000) {
+        return point;
+    }
+    return null;
+}
+const ACTION_TACTICS = ['scroll', 'force', 'wait', 'dispatch', 'none'];
+// Used only after a locator has already been successfully healed (by the text or vision prompt
+// above) but replaying the original action on it still failed for a reason that has nothing to do
+// with the selector — the element was found, the action itself just couldn't complete. The AI's
+// role here is deliberately narrow: pick ONE of a small fixed set of known-safe remediation
+// tactics, never invent a new way to interact with the page.
+exports.ACTION_RECOVERY_SYSTEM_PROMPT = `You are a Playwright self-healing assistant.
+An element was already correctly located, but the action on it failed for the reason described below — this is NOT a selector problem, so do not suggest one. Given the error, choose the single tactic most likely to help:
+{"tactic":"scroll"} — the element may be outside the current viewport; scroll it into view, then retry the same action.
+{"tactic":"force"} — the error looks like an overly strict actionability check rather than something genuinely blocking the click (e.g. flagged as "not stable" during a finished animation, or a borderline visibility/bounding-box check); retry the same action bypassing Playwright's actionability checks. Note: force still performs a real coordinate-based click, so it will NOT help if another element is genuinely covering this one — use "dispatch" for that instead.
+{"tactic":"wait"} — the error suggests something transient (an animation or transition still settling, content still loading); wait briefly, then retry the same action.
+{"tactic":"dispatch"} — the element is genuinely covered/intercepted by another element on top of it, or force did not help; dispatch the underlying DOM event directly on the element instead of a real coordinate-based interaction. Last resort, since it skips real hit-testing entirely.
+{"tactic":"none"} — none of the above would plausibly help.
+Respond with strict JSON only (no markdown, no prose), exactly one of the five objects above.`;
+function buildActionRecoveryUserPrompt(input) {
+    return [
+        `Failed action: ${input.action}`,
+        `Error: ${input.errorMessage}`,
+    ].join('\n\n');
+}
+function parseActionTacticSuggestion(content) {
+    let parsed;
+    try {
+        parsed = JSON.parse(content);
+    }
+    catch {
+        const match = content.match(/\{[\s\S]*\}/);
+        if (!match) {
+            return null;
+        }
+        try {
+            parsed = JSON.parse(match[0]);
+        }
+        catch {
+            return null;
+        }
+    }
+    if (!parsed || typeof parsed !== 'object' || !('tactic' in parsed)) {
+        return null;
+    }
+    const { tactic } = parsed;
+    return typeof tactic === 'string' && ACTION_TACTICS.includes(tactic) ? tactic : null;
+}
 // Shared by any provider whose HTTP response mimics OpenAI's Chat Completions shape
 // (OpenAI itself, and Gemini's OpenAI-compatible endpoint).
 function extractOpenAICompatibleUsage(payload) {

package/dist/healer/providers/prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../../../src/healer/providers/prompt.ts"],"names":[],"mappings":";;;;;;AAEA,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEnB,QAAA,aAAa,GAAG;;;;;;;;;;oJAUuH,CAAC;AAErJ,yBAAgC,KAA2B;IACzD,MAAM,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,kBAAkB;QAC7D,CAAC,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,mBAAmB;QACvE,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;IAEvB,OAAO;QACL,kBAAkB,KAAK,CAAC,MAAM,EAAE;QAChC,wBAAwB,KAAK,CAAC,WAAW,IAAI,iBAAiB,EAAE;QAChE,gBAAgB;QAChB,QAAQ;KACT,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjB,CAAC;AAED,yBAAgC,OAAe;IAC7C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,IAAI,MAAM,CAAC,EAAE,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,UAAU,GAAG,MAA4B,CAAC;IAChD,IAAI,UAAU,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QACnC,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,MAAM,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,MAAM,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,UAAU,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC9E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,OAAO,IAAI,OAAO,UAAU,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC5E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,aAAa,IAAI,OAAO,UAAU,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxF,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,KAAK,IAAI,OAAO,UAAU,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;QACxE,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,oFAAoF;AACpF,4DAA4D;AAC5D,sCAA6C,OAAkG;IAC7I,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,aAAa;QAChC,YAAY,EAAE,KAAK,CAAC,iBAAiB;QACrC,WAAW,EAAE,KAAK,CAAC,YAAY;KAChC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../../../src/healer/providers/prompt.ts"],"names":[],"mappings":";;;;;;;;;;AAEA,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEnB,QAAA,aAAa,GAAG;;;;;;;;;;oJAUuH,CAAC;AAErJ,yBAAgC,KAA2B;IACzD,MAAM,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,kBAAkB;QAC7D,CAAC,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,mBAAmB;QACvE,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;IAEvB,OAAO;QACL,kBAAkB,KAAK,CAAC,MAAM,EAAE;QAChC,wBAAwB,KAAK,CAAC,WAAW,IAAI,iBAAiB,EAAE;QAChE,gBAAgB;QAChB,QAAQ;KACT,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjB,CAAC;AAED,yBAAgC,OAAe;IAC7C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,IAAI,MAAM,CAAC,EAAE,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,UAAU,GAAG,MAA4B,CAAC;IAChD,IAAI,UAAU,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QACnC,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,MAAM,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,MAAM,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,UAAU,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC9E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,OAAO,IAAI,OAAO,UAAU,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC5E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,aAAa,IAAI,OAAO,UAAU,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxF,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,KAAK,KAAK,IAAI,OAAO,UAAU,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;QACxE,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,gGAAgG;AAChG,0FAA0F;AAC1F,gDAAgD;AACnC,QAAA,oBAAoB,GAAG;;;;;uJAKmH,CAAC;AAExJ,+BAAsC,KAAmC;IACvE,OAAO;QACL,kBAAkB,KAAK,CAAC,MAAM,EAAE;QAChC,wBAAwB,KAAK,CAAC,WAAW,IAAI,iBAAiB,EAAE;QAChE,uCAAuC;KACxC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjB,CAAC;AAED,+BAAsC,OAAe;IACnD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,EAAE,CAAC;QAClE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,MAAqB,CAAC;IACpC,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;QAC1B,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;IACD,IACE,KAAK,CAAC,KAAK,KAAK,IAAI;WACjB,OAAO,KAAK,CAAC,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,IAAI;WAC9D,OAAO,KAAK,CAAC,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,IAAI,EACjE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,cAAc,GAA4B,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AAEhG,+FAA+F;AAC/F,kGAAkG;AAClG,gGAAgG;AAChG,4FAA4F;AAC5F,6DAA6D;AAChD,QAAA,6BAA6B,GAAG;;;;;;;8FAOiD,CAAC;AAE/F,uCAA8C,KAA+B;IAC3E,OAAO;QACL,kBAAkB,KAAK,CAAC,MAAM,EAAE;QAChC,UAAU,KAAK,CAAC,YAAY,EAAE;KAC/B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjB,CAAC;AAED,qCAA4C,OAAe;IACzD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,MAAM,CAAC,EAAE,CAAC;QACnE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,MAA6B,CAAC;IACjD,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAK,cAAoC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAsB,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9H,CAAC;AAED,oFAAoF;AACpF,4DAA4D;AAC5D,sCAA6C,OAAkG;IAC7I,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,aAAa;QAChC,YAAY,EAAE,KAAK,CAAC,iBAAiB;QACrC,WAAW,EAAE,KAAK,CAAC,YAAY;KAChC,CAAC;AACJ,CAAC"}