zan-browser 3.0.66 → 3.0.69

package/dist/agent/extraction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction.d.ts","sourceRoot":"","sources":["../../src/agent/extraction.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAE1C,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb;AAWD,2EAA2E;AAC3E,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,SAAS,EAAE,EACxB,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,WAAW,EAChB,YAAY,EAAE,mBAAmB,GAChC,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAgClC"}

package/dist/agent/extraction.js

@@ -0,0 +1,96 @@
+"use strict";
+// ─── Seed page extraction ────────────────────────────────────────────────────
+// Multi-strategy extraction for pages whose URL contains a seed value.
+// Tries scrape → dismiss modal → scrape again → eval_js for SSR data.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.tryExtraction = tryExtraction;
+const DISMISS_KEYWORDS = /cookie|accept|agree|close|dismiss|got it|i understand|consent/i;
+const SSR_GLOBALS = [
+    "window.__NEXT_DATA__",
+    "window.__NUXT__",
+    "window.__APP_STATE__",
+    "window.__INITIAL_STATE__",
+];
+/** Try to extract data from the current page using multiple strategies. */
+async function tryExtraction(pageUrl, _seedValues, registry, ctx, conversation) {
+    // Strategy 1: scrape directly
+    const scrape1 = await tryScrape(registry, ctx);
+    if (scrape1 && scrape1.length > 200) {
+        return { data: scrape1, url: pageUrl };
+    }
+    // Strategy 2: dismiss modal/banner if present, then scrape again
+    const observeRes = await tryObserve(registry, ctx);
+    if (observeRes) {
+        const dismissId = findDismissButton(observeRes);
+        if (dismissId) {
+            try {
+                await registry.execute("click", { elementId: dismissId }, ctx);
+                // Brief wait for modal animation
+                await new Promise((r) => setTimeout(r, 500));
+            }
+            catch { /* non-fatal */ }
+            const scrape2 = await tryScrape(registry, ctx);
+            if (scrape2 && scrape2.length > 200) {
+                return { data: scrape2, url: pageUrl };
+            }
+        }
+    }
+    // Strategy 3: eval_js for SSR embedded data
+    const ssrData = await trySSRExtraction(registry, ctx);
+    if (ssrData && ssrData.length > 200) {
+        return { data: ssrData, url: pageUrl };
+    }
+    return null;
+}
+async function tryScrape(registry, ctx) {
+    try {
+        const res = await registry.execute("scrape", {}, ctx);
+        if (res.success) {
+            const data = res.data;
+            return data?.text ?? (typeof res.data === "string" ? res.data : null);
+        }
+    }
+    catch { /* non-fatal */ }
+    return null;
+}
+async function tryObserve(registry, ctx) {
+    try {
+        const res = await registry.execute("observe", {}, ctx);
+        if (res.success) {
+            return res.summary;
+        }
+    }
+    catch { /* non-fatal */ }
+    return null;
+}
+function findDismissButton(observeSummary) {
+    // Parse the ActionSpace output looking for buttons/links with dismiss keywords
+    const lines = observeSummary.split("\n");
+    for (const line of lines) {
+        if (DISMISS_KEYWORDS.test(line)) {
+            // Look for element IDs like B1, L2, etc.
+            const idMatch = line.match(/\b([BLO]\d+)\b/);
+            if (idMatch) {
+                return idMatch[1];
+            }
+        }
+    }
+    return null;
+}
+async function trySSRExtraction(registry, ctx) {
+    const expr = SSR_GLOBALS
+        .map((g) => `(typeof ${g} !== 'undefined' ? JSON.stringify(${g}).slice(0, 5000) : null)`)
+        .join(" || ");
+    try {
+        const res = await registry.execute("eval_js", { expression: expr }, ctx);
+        if (res.success && res.data) {
+            const text = typeof res.data === "string" ? res.data : JSON.stringify(res.data);
+            if (text && text !== "null" && text.length > 10) {
+                return text;
+            }
+        }
+    }
+    catch { /* non-fatal */ }
+    return null;
+}
+//# sourceMappingURL=extraction.js.map

package/dist/agent/extraction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction.js","sourceRoot":"","sources":["../../src/agent/extraction.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,uEAAuE;AACvE,sEAAsE;;AAsBtE,sCAsCC;AAhDD,MAAM,gBAAgB,GAAG,gEAAgE,CAAC;AAE1F,MAAM,WAAW,GAAG;IAClB,sBAAsB;IACtB,iBAAiB;IACjB,sBAAsB;IACtB,0BAA0B;CAC3B,CAAC;AAEF,2EAA2E;AACpE,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,WAAwB,EACxB,QAAsB,EACtB,GAAgB,EAChB,YAAiC;IAEjC,8BAA8B;IAC9B,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;IACzC,CAAC;IAED,iEAAiE;IACjE,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACnD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,SAAS,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAChD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,CAAC;gBAC/D,iCAAiC;gBACjC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC;YAE3B,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,4CAA4C;IAC5C,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACtD,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;IACzC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAsB,EAAE,GAAgB;IAC/D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACtD,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAqC,CAAC;YACvD,OAAO,IAAI,EAAE,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAsB,EAAE,GAAgB;IAChE,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACvD,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO,GAAG,CAAC,OAAO,CAAC;QACrB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,cAAsB;IAC/C,+EAA+E;IAC/E,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChC,yCAAyC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;YAC7C,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAsB,EAAE,GAAgB;IACtE,MAAM,IAAI,GAAG,WAAW;SACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,qCAAqC,CAAC,0BAA0B,CAAC;SACxF,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QACzE,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChF,IAAI,IAAI,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBAChD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/agent/goal-builder.d.ts

@@ -0,0 +1,11 @@
+import type { SeedValue } from "../types";
+import type { Rejection } from "./orchestrator";
+import type { DiscoveryState } from "./discovery-state";
+export interface GoalBuilderInput {
+    originalGoal: string;
+    seedValues: SeedValue[];
+    rejections: Rejection[];
+    state: DiscoveryState;
+}
+export declare function buildEnrichedGoal(input: GoalBuilderInput): string;
+//# sourceMappingURL=goal-builder.d.ts.map

package/dist/agent/goal-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"goal-builder.d.ts","sourceRoot":"","sources":["../../src/agent/goal-builder.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,KAAK,EAAE,cAAc,CAAC;CACvB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,gBAAgB,GAAG,MAAM,CA4DjE"}

package/dist/agent/goal-builder.js

@@ -0,0 +1,57 @@
+"use strict";
+// ─── Goal builder ────────────────────────────────────────────────────────────
+// Builds the enriched goal string injected into each agent attempt.
+// Contains scrape hints, success criteria, seed values, and accumulated context.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildEnrichedGoal = buildEnrichedGoal;
+function buildEnrichedGoal(input) {
+    const { originalGoal, seedValues, rejections, state } = input;
+    const lines = [];
+    // What counts as success — permissive if a prior rejection suggested scrape
+    const scrapeHinted = rejections.some((r) => r.reason.toLowerCase().includes("scrape"));
+    // If a prior rejection identified a seed page, inject immediate action directive
+    if (scrapeHinted) {
+        const seedPageMatch = rejections
+            .map((r) => r.reason.match(/The page (https?:\/\/\S+) has the seed value/))
+            .filter((m) => m !== null).pop();
+        if (seedPageMatch) {
+            const seedPageUrl = seedPageMatch[1];
+            lines.push(`IMMEDIATE ACTION: The previous attempt found that ${seedPageUrl} contains the data rendered in HTML. ` +
+                `You are already on ${seedPageUrl}. Observe the page immediately. If the DOM shows the data you need, ` +
+                "use scrape as your first action — do not navigate away, do not read network logs first.");
+            lines.push("");
+        }
+    }
+    // Core goal
+    lines.push(`GOAL: ${originalGoal}`);
+    lines.push("");
+    if (scrapeHinted) {
+        lines.push("SUCCESS CRITERIA: Find a reproducible URL that returns the requested data. " +
+            "A JSON API endpoint is preferred, but if the data is only available via HTML rendering, " +
+            "scraping a parameterizable page URL (one that contains the seed value in the path) is also " +
+            "valid — set data_found=true and declare done.");
+    }
+    else {
+        lines.push("SUCCESS CRITERIA: Find an HTTP endpoint (API, JSON feed, or structured data URL) " +
+            "that can be called directly with fetch/curl — no browser session required at runtime. " +
+            "DOM extractions, scraped HTML text, and framework data-transport routes (/_next/data/, " +
+            "__data.json) do NOT count. The result must be a reproducible URL with a real HTTP response.");
+    }
+    // Seed values context
+    if (seedValues.length > 0) {
+        lines.push("");
+        lines.push("SEED VALUES (use these verbatim when searching/filling):");
+        for (const s of seedValues) {
+            lines.push(`  • ${s.paramName} (${s.description}): "${s.exampleValue}"`);
+        }
+    }
+    // Accumulated discovery context from shared state
+    const stateSummary = state.summarize();
+    if (stateSummary) {
+        lines.push("");
+        lines.push("ACCUMULATED CONTEXT:");
+        lines.push(stateSummary);
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=goal-builder.js.map

package/dist/agent/goal-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"goal-builder.js","sourceRoot":"","sources":["../../src/agent/goal-builder.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,oEAAoE;AACpE,iFAAiF;;AAajF,8CA4DC;AA5DD,SAAgB,iBAAiB,CAAC,KAAuB;IACvD,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,4EAA4E;IAC5E,MAAM,YAAY,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEvF,iFAAiF;IACjF,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,aAAa,GAAG,UAAU;aAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;aAC1E,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;QACnC,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CACR,qDAAqD,WAAW,uCAAuC;gBACvG,sBAAsB,WAAW,sEAAsE;gBACvG,yFAAyF,CAC1F,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,YAAY;IACZ,KAAK,CAAC,IAAI,CAAC,SAAS,YAAY,EAAE,CAAC,CAAC;IACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CACR,6EAA6E;YAC7E,0FAA0F;YAC1F,6FAA6F;YAC7F,+CAA+C,CAChD,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CACR,mFAAmF;YACnF,wFAAwF;YACxF,yFAAyF;YACzF,6FAA6F,CAC9F,CAAC;IACJ,CAAC;IAED,sBAAsB;IACtB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;QACvE,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,WAAW,OAAO,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;IACvC,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/agent/orchestrator.d.ts

@@ -25,6 +25,7 @@ export interface EndpointResult {
     reproducible: boolean;
     whyReproducible: string;
     goodExamples?: string[];
+    notes?: string;
 }
 export interface OrchestratorConfig {
     goal: string;
@@ -55,13 +56,7 @@ export declare class AgentOrchestrator {
     run(config: OrchestratorConfig): Promise<OrchestratorResult>;
     /** Inject external feedback from the pipeline (validator, request-selector, etc.) */
     injectFeedback(rejection: Rejection): void;
-    private buildEnrichedGoal;
-    private evaluateResult;
-    private preValidateEndpoint;
-    private buildUrlTemplate;
-    private explainReproducibility;
     private diagnoseFailure;
-    private recordFailedEndpoints;
     private buildFailureSummary;
 }
 //# sourceMappingURL=orchestrator.d.ts.map

package/dist/agent/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/agent/orchestrator.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAI3D,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,WAAW,GAAG,kBAAkB,GAAG,UAAU,CAAC;IACtD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,eAAe,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,OAAO,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,CAAC;CACtD;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,YAAY,CAAC,EAAE,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAe;IAC/B,OAAO,CAAC,GAAG,CAAc;IACzB,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,YAAY,CAAM;IAC1B,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,SAAS,CAAK;gBAEV,QAAQ,EAAE,YAAY,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,CAAC,EAAE,MAAM;IAOrF,GAAG,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAsIlE,qFAAqF;IACrF,cAAc,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAM1C,OAAO,CAAC,iBAAiB;IA6DzB,OAAO,CAAC,cAAc;YA6JR,mBAAmB;IAsIjC,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,sBAAsB;IAY9B,OAAO,CAAC,eAAe;IAwBvB,OAAO,CAAC,qBAAqB;IAiB7B,OAAO,CAAC,mBAAmB;CAe5B"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/agent/orchestrator.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAQ3D,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,WAAW,GAAG,kBAAkB,GAAG,UAAU,CAAC;IACtD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,eAAe,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,OAAO,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,IAAI,CAAC;CACtD;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,YAAY,CAAC,EAAE,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAe;IAC/B,OAAO,CAAC,GAAG,CAAc;IACzB,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,YAAY,CAAM;IAC1B,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,SAAS,CAAK;gBAEV,QAAQ,EAAE,YAAY,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,CAAC,EAAE,MAAM;IAOrF,GAAG,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAwKlE,qFAAqF;IACrF,cAAc,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAM1C,OAAO,CAAC,eAAe;IAwBvB,OAAO,CAAC,mBAAmB;CAe5B"}

package/dist/agent/orchestrator.js

@@ -7,14 +7,14 @@
 // The core insight: the agent's real goal is not "find data" but "find a
 // reproducible source that works without a browser at runtime". The orchestrator
 // communicates that distinction and feeds back why previous attempts failed.
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AgentOrchestrator = void 0;
-const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
 const agent_1 = require("./agent");
 const conversation_1 = require("./conversation");
+const goal_builder_1 = require("./goal-builder");
+const candidate_evaluator_1 = require("./candidate-evaluator");
+const pre_validator_1 = require("./pre-validator");
+const replicability_checker_1 = require("./replicability-checker");
 // ─── Orchestrator ────────────────────────────────────────────────────────────
 class AgentOrchestrator {
     registry;
@@ -53,7 +53,12 @@ class AgentOrchestrator {
                 this.ctx.interceptor?.clear();
             }
             // 1. Build enriched goal with accumulated context
-            const enrichedGoal = this.buildEnrichedGoal();
+            const enrichedGoal = (0, goal_builder_1.buildEnrichedGoal)({
+                originalGoal: this.originalGoal,
+                seedValues: this.seedValues,
+                rejections: this.state.rejections,
+                state: this.state,
+            });
             // 2. Fresh conversation for each attempt
             const conversation = new conversation_1.ConversationHistory();
             // 2b. If a prior rejection identified a seed page, navigate there before the loop starts
@@ -79,7 +84,12 @@ class AgentOrchestrator {
                 priorContext: attempt > 0 ? this.state.summarize() : undefined,
             });
             // 4. Evaluate result — is it a reproducible endpoint?
-            const endpoint = this.evaluateResult(agentResult);
+            const endpoint = (0, candidate_evaluator_1.evaluateResult)({
+                agentResult,
+                state: this.state,
+                seedValues: this.seedValues,
+                ctx: this.ctx,
+            });
             const record = {
                 agentResult,
                 goalGiven: enrichedGoal,
@@ -100,9 +110,32 @@ class AgentOrchestrator {
                         totalTime: Date.now() - this.startTime,
                     };
                 }
+                // Replicability check — skip pre-validation for non-replicable endpoints
+                const replicability = (0, replicability_checker_1.checkReplicability)(endpoint.request, this.seedValues);
+                if (!replicability.replicable) {
+                    progress(`Endpoint not replicable: ${replicability.reason}`);
+                    record.rejectionReason = `Endpoint not replicable: ${replicability.reason}`;
+                    this.state.triedEndpoints.set(endpoint.request.url, replicability.reason);
+                    this.state.rejections.push({
+                        source: "validator",
+                        reason: `Endpoint ${endpoint.request.url} rejected: ${replicability.reason}`,
+                        capturedUrl: endpoint.request.url,
+                    });
+                    continue;
+                }
+                if (replicability.notes) {
+                    endpoint.notes = replicability.notes;
+                }
                 // Pre-validate: confirm endpoint returns data for different inputs
                 progress(`Pre-validating endpoint: ${endpoint.urlTemplate}`);
-                const validation = await this.preValidateEndpoint(endpoint, progress);
+                const validation = await (0, pre_validator_1.preValidateEndpoint)({
+                    endpoint,
+                    seedValues: this.seedValues,
+                    originalGoal: this.originalGoal,
+                    ctx: this.ctx,
+                    state: this.state,
+                    progress,
+                });
                 if (validation.valid) {
                     endpoint.goodExamples = validation.goodExamples;
                     progress(`Attempt ${attempt + 1}: endpoint validated → ${endpoint.urlTemplate}`);
@@ -148,333 +181,6 @@ class AgentOrchestrator {
         this.state.rejections.push(rejection);
     }
     // ─── Private ───────────────────────────────────────────────────────────────
-    buildEnrichedGoal() {
-        const lines = [];
-        // What counts as success — permissive if a prior rejection suggested scrape
-        const scrapeHinted = this.state.rejections.some((r) => r.reason.toLowerCase().includes("scrape"));
-        // If a prior rejection identified a seed page, inject immediate action directive
-        if (scrapeHinted) {
-            const seedPageMatch = this.state.rejections
-                .map((r) => r.reason.match(/The page (https?:\/\/\S+) has the seed value/))
-                .filter((m) => m !== null).pop();
-            if (seedPageMatch) {
-                const seedPageUrl = seedPageMatch[1];
-                lines.push(`IMMEDIATE ACTION: The previous attempt found that ${seedPageUrl} contains the data rendered in HTML. ` +
-                    `You are already on ${seedPageUrl}. Observe the page immediately. If the DOM shows the data you need, ` +
-                    "use scrape as your first action — do not navigate away, do not read network logs first.");
-                lines.push("");
-            }
-        }
-        // Core goal
-        lines.push(`GOAL: ${this.originalGoal}`);
-        lines.push("");
-        if (scrapeHinted) {
-            lines.push("SUCCESS CRITERIA: Find a reproducible URL that returns the requested data. " +
-                "A JSON API endpoint is preferred, but if the data is only available via HTML rendering, " +
-                "scraping a parameterizable page URL (one that contains the seed value in the path) is also " +
-                "valid — set data_found=true and declare done.");
-        }
-        else {
-            lines.push("SUCCESS CRITERIA: Find an HTTP endpoint (API, JSON feed, or structured data URL) " +
-                "that can be called directly with fetch/curl — no browser session required at runtime. " +
-                "DOM extractions, scraped HTML text, and framework data-transport routes (/_next/data/, " +
-                "__data.json) do NOT count. The result must be a reproducible URL with a real HTTP response.");
-        }
-        // Seed values context
-        if (this.seedValues.length > 0) {
-            lines.push("");
-            lines.push("SEED VALUES (use these verbatim when searching/filling):");
-            for (const s of this.seedValues) {
-                lines.push(`  • ${s.paramName} (${s.description}): "${s.exampleValue}"`);
-            }
-        }
-        // Accumulated discovery context from shared state
-        const stateSummary = this.state.summarize();
-        if (stateSummary) {
-            lines.push("");
-            lines.push("ACCUMULATED CONTEXT:");
-            lines.push(stateSummary);
-        }
-        return lines.join("\n");
-    }
-    evaluateResult(agentResult) {
-        // Build set of URLs the agent explicitly tested via fetch_url with success
-        const verifiedUrls = new Set();
-        for (const step of agentResult.steps) {
-            if (step.toolName === "fetch_url" && step.result.success && step.params.url) {
-                verifiedUrls.add(String(step.params.url));
-            }
-        }
-        const candidates = [];
-        const allCaptured = [
-            ...agentResult.capturedRequests,
-            ...(this.ctx.interceptor?.getAll() ?? []),
-        ];
-        // Deduplicate by URL
-        const seenUrls = new Set();
-        // Path 1: fetch_url verified endpoints with score >= 50
-        for (const url of verifiedUrls) {
-            const match = allCaptured.find((r) => r.url === url && !r.isDomExtraction && !r.isAsset && r.score >= 50);
-            if (match && !seenUrls.has(url)) {
-                candidates.push(match);
-                seenUrls.add(url);
-            }
-        }
-        // Path 2: high-score XHRs from browser navigation (THE CRITICAL FIX)
-        // Accepts XHRs captured by navigating and interacting — not just fetch_url verified
-        for (const r of allCaptured) {
-            if (r.score >= 60 &&
-                !r.isDomExtraction &&
-                !r.isAsset &&
-                r.isXHR &&
-                !this.state.triedEndpoints.has(r.url) &&
-                !seenUrls.has(r.url)) {
-                candidates.push(r);
-                seenUrls.add(r.url);
-            }
-        }
-        // Path 3: successful fetch_url in the last 5 steps of a successful run
-        // fetch_url bypasses the browser — its results never appear in the interceptor.
-        // If the agent succeeded and a recent fetch_url returned data, accept it directly.
-        if (agentResult.success) {
-            const recentSteps = agentResult.steps.slice(-5).reverse();
-            for (const step of recentSteps) {
-                if (step.toolName === "fetch_url" &&
-                    step.result.success &&
-                    step.params.url &&
-                    !seenUrls.has(String(step.params.url)) &&
-                    !this.state.triedEndpoints.has(String(step.params.url))) {
-                    const fetchUrl = String(step.params.url);
-                    const fetchData = step.result.data;
-                    const bodyLen = fetchData?.body?.length ?? 0;
-                    // Only accept if it returned substantial content
-                    if (bodyLen > 100) {
-                        const synthetic = {
-                            id: `fetch_url_${Date.now()}`,
-                            timestamp: Date.now(),
-                            method: String(step.params.method ?? "GET"),
-                            url: fetchUrl,
-                            requestHeaders: {},
-                            requestBody: step.params.body ? String(step.params.body) : undefined,
-                            status: fetchData?.status ?? 200,
-                            responseHeaders: {},
-                            responseBody: fetchData?.body,
-                            contentType: fetchData?.contentType ?? "",
-                            isXHR: false,
-                            isAsset: false,
-                            score: 75,
-                            isDomExtraction: false,
-                        };
-                        candidates.push(synthetic);
-                        seenUrls.add(fetchUrl);
-                    }
-                }
-            }
-        }
-        // Path 4: successful scrape on a page whose URL contains a seed value
-        const seeds = this.seedValues.map((s) => s.exampleValue.toLowerCase());
-        for (let i = 0; i < agentResult.steps.length; i++) {
-            const step = agentResult.steps[i];
-            if (step.toolName === "scrape" && step.result.success) {
-                // Find the most recent navigate before this scrape
-                const pageUrl = [...agentResult.steps]
-                    .slice(0, i)
-                    .reverse()
-                    .find((s) => s.toolName === "navigate" && s.result.success && s.params.url)
-                    ?.params.url;
-                if (pageUrl && seeds.some((sv) => pageUrl.toLowerCase().includes(sv))) {
-                    const scrapeData = step.result.data;
-                    if (scrapeData?.text && scrapeData.text.length > 100) {
-                        const syntheticUrl = String(pageUrl);
-                        if (!seenUrls.has(syntheticUrl)) {
-                            candidates.push({
-                                id: `scrape_${Date.now()}`,
-                                timestamp: Date.now(),
-                                method: "GET",
-                                url: syntheticUrl,
-                                requestHeaders: {},
-                                status: 200,
-                                responseHeaders: {},
-                                responseBody: scrapeData.text,
-                                contentType: "text/html",
-                                isXHR: false,
-                                isAsset: false,
-                                score: 70,
-                                isDomExtraction: true,
-                            });
-                            seenUrls.add(syntheticUrl);
-                        }
-                    }
-                }
-            }
-        }
-        // Exclude endpoints that already failed or were pre-validated in previous attempts
-        const viable = candidates.filter((r) => !this.state.triedEndpoints.has(r.url) &&
-            !this.state.preValidatedEndpoints.has(r.url));
-        if (viable.length === 0) {
-            this.recordFailedEndpoints(agentResult);
-            return null;
-        }
-        // Prefer endpoints where seed value appears in URL (parameterizable), then by score
-        viable.sort((a, b) => {
-            const aHasSeed = seeds.some((sv) => a.url.toLowerCase().includes(sv));
-            const bHasSeed = seeds.some((sv) => b.url.toLowerCase().includes(sv));
-            if (aHasSeed && !bHasSeed)
-                return -1;
-            if (!aHasSeed && bHasSeed)
-                return 1;
-            return b.score - a.score;
-        });
-        const best = viable[0];
-        // Build URL template by replacing seed values
-        const urlTemplate = this.buildUrlTemplate(best.url);
-        return {
-            request: best,
-            urlTemplate,
-            score: best.score,
-            reproducible: true,
-            whyReproducible: this.explainReproducibility(best),
-        };
-    }
-    async preValidateEndpoint(endpoint, progress) {
-        this.state.preValidatedEndpoints.add(endpoint.request.url);
-        const template = endpoint.urlTemplate;
-        const hasParams = template.includes("{");
-        // Build test URL sets from seed values
-        const paramSets = [];
-        if (hasParams && this.seedValues.length > 0) {
-            // Primary set: exampleValues
-            const primary = new Map();
-            for (const s of this.seedValues) {
-                primary.set(s.paramName, s.exampleValue);
-            }
-            paramSets.push({ label: this.seedValues.map((s) => s.exampleValue).join(", "), values: primary });
-            // Alternative sets from alternativeExamples
-            const maxAlternatives = 3;
-            // Find the max number of alternative examples across all seeds
-            const altCount = Math.min(maxAlternatives, Math.max(...this.seedValues.map((s) => s.alternativeExamples?.length ?? 0)));
-            for (let i = 0; i < altCount; i++) {
-                const alt = new Map();
-                for (const s of this.seedValues) {
-                    const altVal = s.alternativeExamples?.[i];
-                    alt.set(s.paramName, altVal ?? s.exampleValue);
-                }
-                paramSets.push({ label: [...alt.values()].join(", "), values: alt });
-            }
-        }
-        else {
-            // No params or no seeds — test the URL as-is
-            paramSets.push({ label: "original", values: new Map() });
-        }
-        // Limit to 4 sets total
-        const testSets = paramSets.slice(0, 4);
-        // Fetch each URL
-        const results = [];
-        for (let i = 0; i < testSets.length; i++) {
-            const set = testSets[i];
-            let testUrl = template;
-            for (const [param, value] of set.values) {
-                testUrl = testUrl.split(`{${param}}`).join(value);
-            }
-            progress(`Pre-validate [${i + 1}/${testSets.length}]: ${set.label}`);
-            try {
-                const resp = await fetch(testUrl, {
-                    signal: AbortSignal.timeout(10_000),
-                    headers: { "User-Agent": "Mozilla/5.0 (compatible; ZanBrowser/1.0)" },
-                });
-                const body = await resp.text();
-                results.push({
-                    label: set.label,
-                    url: testUrl,
-                    status: resp.status,
-                    bodyPreview: body.slice(0, 500),
-                });
-            }
-            catch (err) {
-                results.push({
-                    label: set.label,
-                    url: testUrl,
-                    status: 0,
-                    bodyPreview: "",
-                    error: err.message,
-                });
-            }
-            // Rate limit pause between calls (skip after last)
-            if (i < testSets.length - 1) {
-                await new Promise((r) => setTimeout(r, 5000));
-            }
-        }
-        // LLM reasoning call (Haiku — lightweight)
-        const client = new sdk_1.default({ apiKey: this.ctx.anthropicApiKey });
-        const llmPrompt = `You are validating whether an API endpoint is generic and returns real data for different inputs.
-
-GOAL: ${this.originalGoal}
-ENDPOINT TEMPLATE: ${template}
-
-TEST RESULTS:
-${results.map((r) => `- Input: ${r.label}\n  URL: ${r.url}\n  Status: ${r.status}${r.error ? ` (error: ${r.error})` : ""}\n  Body preview: ${r.bodyPreview.slice(0, 300)}`).join("\n\n")}
-
-Answer these questions:
-1. Do the results confirm this endpoint returns real data relevant to the goal?
-2. Are null/empty responses expected (e.g. a product that doesn't exist) or do they indicate the endpoint is wrong?
-3. Is there enough evidence that the endpoint is generic (works for different inputs)?
-
-Respond ONLY with valid JSON:
-{"valid": true/false, "reason": "one sentence explanation", "goodExamples": ["values that returned real data"]}`;
-        try {
-            const resp = await client.messages.create({
-                model: "claude-sonnet-4-6",
-                max_tokens: 400,
-                messages: [{ role: "user", content: llmPrompt }],
-            });
-            const text = resp.content[0].type === "text" ? resp.content[0].text : "{}";
-            // Extract JSON from response (handle possible markdown wrapping)
-            const jsonMatch = text.match(/\{[\s\S]*\}/);
-            if (jsonMatch) {
-                const parsed = JSON.parse(jsonMatch[0]);
-                return {
-                    valid: parsed.valid === true,
-                    reason: String(parsed.reason ?? "unknown"),
-                    goodExamples: Array.isArray(parsed.goodExamples) ? parsed.goodExamples : [],
-                };
-            }
-        }
-        catch (err) {
-            progress(`Pre-validation LLM error: ${err.message}`);
-        }
-        // If LLM fails, check if at least one fetch returned 200 with substantial body
-        const anySuccess = results.some((r) => r.status >= 200 && r.status < 400 && r.bodyPreview.length > 100);
-        return {
-            valid: anySuccess,
-            reason: anySuccess ? "LLM validation failed but at least one fetch returned data" : "Could not validate endpoint",
-            goodExamples: results.filter((r) => r.status >= 200 && r.status < 400 && r.bodyPreview.length > 100).map((r) => r.label),
-        };
-    }
-    buildUrlTemplate(url) {
-        let template = url;
-        for (const seed of this.seedValues) {
-            if (template.includes(seed.exampleValue)) {
-                template = template.split(seed.exampleValue).join(`{${seed.paramName}}`);
-            }
-        }
-        return template;
-    }
-    explainReproducibility(req) {
-        const parts = [];
-        if (req.isXHR)
-            parts.push("real XHR/fetch request");
-        if (req.contentType?.includes("json"))
-            parts.push("returns JSON");
-        if (req.score >= 80)
-            parts.push("high relevance score");
-        if (req.method === "GET")
-            parts.push("stateless GET — no session dependency");
-        if (req.method === "POST" && req.requestBody)
-            parts.push("POST with known payload structure");
-        return parts.length > 0
-            ? `Reproducible: ${parts.join(", ")}`
-            : "Captured as a real network request with structured response";
-    }
     diagnoseFailure(agentResult) {
         if (!agentResult.success) {
             return agentResult.reason;
@@ -494,19 +200,6 @@ Respond ONLY with valid JSON:
         }
         return "Captured requests did not meet reproducibility criteria";
     }
-    recordFailedEndpoints(agentResult) {
-        // Record fetch_url attempts that didn't produce a good result
-        for (const step of agentResult.steps) {
-            if (step.toolName === "fetch_url" && step.params.url) {
-                const url = String(step.params.url);
-                if (!this.state.triedEndpoints.has(url)) {
-                    this.state.triedEndpoints.set(url, step.result.success
-                        ? "returned data but not structured/useful enough"
-                        : String(step.result.error ?? "request failed"));
-                }
-            }
-        }
-    }
     buildFailureSummary() {
         const parts = [];
         parts.push(`Exhausted ${this.state.attempts.length} attempts (${this.state.totalSteps} total steps)`);