unbrowse 2.12.2 → 2.12.7

package/runtime-src/execution/index.ts

@@ -6,11 +6,10 @@ import { publishSkill, mergeEndpoints } from "../marketplace/index.js";
 import { updateEndpointScore } from "../marketplace/index.js";
 import { getCredential, storeCredential, deleteCredential } from "../vault/index.js";
 import { getStoredAuth, getAuthCookies, refreshAuthFromBrowser } from "../auth/index.js";
-import { resolvePreExecutionAuth } from "../auth/dependency-runtime.js";
-import { authRuntime } from "../auth/runtime.js";
 import { applyProjection, inferSchema } from "../transform/index.js";
 import { detectSchemaDrift } from "../transform/drift.js";
-import { recordExecution, recordTransaction, cachePublishedSkill, findExistingSkillForDomain, getLocalWalletContext, updateEndpointSchema } from "../client/index.js";
+import { generateExtractionHints } from "../transform/schema-hints.js";
+import { recordExecution, cachePublishedSkill, findExistingSkillForDomain, updateEndpointSchema } from "../client/index.js";
 import { validateManifest } from "../client/index.js";
 import { withRetry, isRetryableStatus } from "./retry.js";
 import type { EndpointDescriptor, ExecutionOptions, ExecutionTrace, ProjectionOptions, SkillManifest } from "../types/index.js";
@@ -23,21 +22,7 @@ import { log } from "../logger.js";
 import { TRACE_VERSION } from "../version.js";
 import { buildQueryBindingMap, extractTemplateQueryBindings, mergeContextTemplateParams } from "../template-params.js";
 import { assessIntentResult, projectIntentData } from "../intent-match.js";
-import { isStructuredSearchForm, detectSearchForms, type SearchFormSpec } from "./search-forms.js";
-import { attributeLifecycle, type LifecycleEvent, type LifecyclePhase } from "../runtime/lifecycle.js";
-import { queueBackgroundIndex } from "../indexer/index.js";
-import {
-  writeSkillSnapshot,
-  domainSkillCache,
-  persistDomainCache,
-  getDomainReuseKey,
-  scopedCacheKey,
-  buildResolveCacheKey,
-  snapshotPathForCacheKey,
-  generateLocalDescription,
-} from "../orchestrator/index.js";
-import { checkPaymentRequirement } from "../payments/index.js";
-import { isAllowedByRobots } from "./robots.js";
+
 /** Stamp every trace with the code version hash for telemetry tracking */
 function stampTrace(trace: ExecutionTrace): ExecutionTrace {
   trace.trace_version = TRACE_VERSION;
@@ -192,6 +177,10 @@ export interface ExecutionResult {
   trace: ExecutionTrace;
   result: unknown;
   learned_skill?: SkillManifest;
+  /** Inferred JSON schema of the endpoint's response, for agent-side extraction */
+  response_schema?: import("../types/index.js").ResponseSchema;
+  /** Ready-to-use extraction hints derived from response_schema */
+  extraction_hints?: import("../transform/schema-hints.js").ExtractionHint;
 }
 
 export function projectResultForIntent(data: unknown, intent?: string): unknown {
@@ -217,9 +206,7 @@ function sanitizeNavigationQueryParams(url: URL): URL {
 }
 
 function restoreTemplatePlaceholderEncoding(url: string): string {
-  // Only restore template placeholders like {variable_name}, not arbitrary JSON braces.
-  // Template placeholders: %7Bword_chars%7D (no spaces, no quotes, no colons inside)
-  return url.replace(/%7B(\w+)%7D/gi, "{$1}");
+  return url.replace(/%7B/gi, "{").replace(/%7D/gi, "}");
 }
 
 function compactSchemaSample(value: unknown, depth = 0): unknown {
@@ -575,21 +562,6 @@ export function buildStructuredReplayHeaders(
   return headers;
 }
 
-function normalizeReplayHeaders(
-  ...bags: Array<Record<string, string> | undefined>
-): Record<string, string> {
-  const normalized: Record<string, string> = {};
-  for (const bag of bags) {
-    for (const [key, value] of Object.entries(bag ?? {})) {
-      if (typeof value !== "string") continue;
-      const trimmed = value.trim();
-      if (!trimmed) continue;
-      normalized[key.toLowerCase()] = trimmed;
-    }
-  }
-  return normalized;
-}
-
 function shouldFallbackToBrowserReplay(
   data: unknown,
   endpoint: EndpointDescriptor,
@@ -620,7 +592,6 @@ export function buildPageArtifactCapture(
   endpoint?: EndpointDescriptor;
   result?: { data: unknown; _extraction: Record<string, unknown> };
   quality_note?: string;
-  search_form?: SearchFormSpec;
 } {
   const extracted = extractFromDOM(html, intent);
   if (!extracted.data || extracted.confidence <= 0.2) return {};
@@ -632,11 +603,6 @@ export function buildPageArtifactCapture(
   if (semanticAssessment.verdict === "fail") {
     return { quality_note: semanticAssessment.reason };
   }
-
-  // Detect structured search forms from the captured HTML
-  const searchForms = detectSearchForms(html);
-  const validSearchForm = searchForms.find((spec: SearchFormSpec) => isStructuredSearchForm(spec));
-
   const response_schema = inferSchema([extracted.data]);
   const endpoint: EndpointDescriptor = {
     endpoint_id: nanoid(),
@@ -645,15 +611,12 @@ export function buildPageArtifactCapture(
     idempotency: "safe" as const,
     verification_status: "verified" as const,
     reliability_score: extracted.confidence,
-    description: validSearchForm
-      ? `Captured search form artifact for ${intent}`
-      : `Captured page artifact for ${intent}`,
+    description: `Captured page artifact for ${intent}`,
     response_schema,
     dom_extraction: {
       extraction_method: extracted.extraction_method,
       confidence: extracted.confidence,
       ...(extracted.selector ? { selector: extracted.selector } : {}),
-      ...(validSearchForm ? { search_form: validSearchForm } : {}),
     },
     trigger_url: url,
   };
@@ -666,14 +629,6 @@ export function buildPageArtifactCapture(
     }),
     ...(authRequired ? { auth_required: true } : {}),
   };
-  if (validSearchForm && endpoint.semantic) {
-    endpoint.semantic.action_kind = "search";
-  }
-
-  if (validSearchForm) {
-    log("execution", `detected structured search form: ${validSearchForm.form_selector} with ${validSearchForm.fields.length} fields`);
-  }
-
   return {
     endpoint,
     result: {
@@ -682,10 +637,8 @@ export function buildPageArtifactCapture(
         method: extracted.extraction_method,
         confidence: extracted.confidence,
         source: "dom-fallback",
-        ...(validSearchForm ? { search_form_detected: true } : {}),
       },
     },
-    ...(validSearchForm ? { search_form: validSearchForm } : {}),
   };
 }
 
@@ -960,35 +913,16 @@ export async function executeSkill(
   options?: ExecutionOptions
 ): Promise<ExecutionResult> {
   if (skill.execution_type === "browser-capture") {
-    return executeBrowserCapture(skill, params, options);
+    return executeBrowserCapture(skill, params);
   }
 
-  // Allow targeting a specific endpoint by ID — never silently fall back
+  // Allow targeting a specific endpoint by ID
   if (params.endpoint_id) {
     const target = skill.endpoints.find((e) => e.endpoint_id === params.endpoint_id);
     if (target) {
       const { endpoint_id: _, ...cleanParams } = params;
       return executeEndpoint(skill, target, cleanParams, projection, options);
     }
-    // Agent explicitly chose this endpoint — don't silently swap to a different one
-    log("exec", `endpoint ${params.endpoint_id} not found in skill ${skill.skill_id} (${skill.endpoints.length} endpoints: ${skill.endpoints.map(e => e.endpoint_id).join(", ")})`);
-    const trace: ExecutionTrace = {
-      trace_id: nanoid(),
-      skill_id: skill.skill_id,
-      endpoint_id: String(params.endpoint_id),
-      started_at: new Date().toISOString(),
-      completed_at: new Date().toISOString(),
-      success: false,
-      error: `endpoint_not_found: ${params.endpoint_id} not in skill ${skill.skill_id}`,
-    };
-    return {
-      trace,
-      result: {
-        error: "endpoint_not_found",
-        message: `Endpoint ${params.endpoint_id} not found in skill ${skill.skill_id}. Available: ${skill.endpoints.map(e => `${e.endpoint_id} (${e.description?.slice(0, 50)})`).join(", ")}`,
-        available_endpoints: skill.endpoints.map(e => ({ endpoint_id: e.endpoint_id, description: e.description })),
-      },
-    };
   }
 
   // Use the caller's intent for ranking when available, fall back to skill's original intent
@@ -998,8 +932,7 @@ export async function executeSkill(
 
 async function executeBrowserCapture(
   skill: SkillManifest,
-  params: Record<string, unknown>,
-  options?: ExecutionOptions,
+  params: Record<string, unknown>
 ): Promise<ExecutionResult> {
   const fallbackUrl =
     (typeof params.context_url === "string" && params.context_url) ||
@@ -1019,9 +952,9 @@ async function executeBrowserCapture(
   let cookies = params.cookies as Array<{ name: string; value: string; domain: string }> | undefined;
   let usedStoredAuth = !!(cookies && cookies.length > 0) || !!(authHeaders && Object.keys(authHeaders).length > 0);
 
-  // Auto-resolve cookies from vault, falling back to browser extraction
+  // Bird-style: auto-resolve cookies from vault → browser fallback
   if (!cookies || cookies.length === 0) {
-    const resolved = await getAuthCookies(targetDomain, { autoExtract: true });
+    const resolved = await getAuthCookies(targetDomain, { autoExtract: false });
     if (resolved && resolved.length > 0) {
       cookies = resolved;
       usedStoredAuth = true;
@@ -1108,22 +1041,6 @@ async function executeBrowserCapture(
 
   const endpoints = extractEndpoints(captured.requests, captured.ws_messages, { pageUrl: url, finalUrl: captured.final_url, intent });
 
-  // Detect structured search forms from captured HTML and attach to search-like endpoints
-  if (captured.html) {
-    const detectedForms = detectSearchForms(captured.html);
-    if (detectedForms.length > 0) {
-      for (const ep of endpoints) {
-        if (!ep.search_form && ep.method === "GET") {
-          const matchingForm = detectedForms.find((f) => isStructuredSearchForm(f));
-          if (matchingForm) {
-            ep.search_form = matchingForm;
-            break; // attach the best form to the first search-like GET endpoint
-          }
-        }
-      }
-    }
-  }
-
   // JS bundle scanning: discover API routes not seen in network traffic
   if (captured.js_bundles && captured.js_bundles.size > 0) {
     const pageOrigin = new URL(url).origin;
@@ -1414,60 +1331,32 @@ async function executeBrowserCapture(
     description: `API skill for ${domain}`,
     owner_type: "agent" as const,
     endpoints: localEndpoints,
+    operation_graph: buildSkillOperationGraph(localEndpoints),
     intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
     ...(auth_profile_ref ? { auth_profile_ref } : {}),
   };
-  // Generate local descriptions immediately so BM25 ranking works on first cache hit
-  for (const ep of localDraft.endpoints) {
-    if (!ep.description) {
-      ep.description = generateLocalDescription(ep);
-    }
-  }
-
-  // PHASE 2: Write local cache IMMEDIATELY (~1ms) — populates cache before auto-exec
-  const bgCacheKey = buildResolveCacheKey(domain, intent, url);
-  const bgScopedKey = scopedCacheKey(options?.client_scope ?? "global", bgCacheKey);
-  writeSkillSnapshot(bgScopedKey, localDraft);
-  const bgDomainKey = getDomainReuseKey(url ?? domain);
-  if (bgDomainKey) {
-    domainSkillCache.set(bgDomainKey, {
-      skillId: localDraft.skill_id,
-      localSkillPath: snapshotPathForCacheKey(bgScopedKey),
-      ts: Date.now(),
-    });
-    persistDomainCache();
-  }
-
-  // PHASE 2: Queue heavy work for background (graph + validate + publish)
-  queueBackgroundIndex({
-    skill: { ...localDraft },
-    domain,
-    intent,
-    contextUrl: url,
-    clientScope: options?.client_scope,
-    cacheKey: bgCacheKey,
-  });
-
-  // Return the local draft as learned_skill — no blocking on marketplace publish
   let learned: SkillManifest = localDraft;
-  try { cachePublishedSkill(localDraft, options?.client_scope); } catch { /* best-effort */ }
-
-  // Attribute lifecycle phases for this capture-to-publish flow
-  const completedAt = new Date().toISOString();
-  const captureDurationMs = new Date(completedAt).getTime() - new Date(startedAt).getTime();
-  const lifecycleEvents: LifecycleEvent[] = [
-    { phase: "capture", skill_id: learned.skill_id, timestamp: startedAt, duration_ms: captureDurationMs, source: "live-capture" },
-    { phase: "publish", skill_id: learned.skill_id, timestamp: completedAt, duration_ms: 0, source: publishableEndpoints.length > 0 ? "marketplace" : "cache" },
-  ];
-  const lifecycleAttribution = attributeLifecycle(lifecycleEvents);
-  log("execution", `lifecycle attribution: capture=${lifecycleAttribution.get("capture") ?? 0}ms, publish=${lifecycleAttribution.get("publish") ?? 0}ms`);
+  if (publishableEndpoints.length > 0) {
+    const { operation_graph: _graph, ...publishBase } = localDraft;
+    const publishDraft: SkillManifest = { ...publishBase, endpoints: publishableEndpoints };
+    const validation = await validateManifest({ ...publishDraft, skill_id: "__validate__" });
+    if (!validation.valid) throw new Error(`Skill validation failed: ${validation.hardErrors.join("; ")}`);
+    const published = await publishSkill(publishDraft);
+    learned = {
+      ...published,
+      endpoints: localEndpoints,
+      operation_graph: localDraft.operation_graph,
+      ...(auth_profile_ref ? { auth_profile_ref } : {}),
+    };
+  }
+  try { cachePublishedSkill(learned, options?.client_scope); } catch { /* local cache best-effort */ }
 
   const trace: ExecutionTrace = stampTrace({
     trace_id: traceId,
     skill_id: learned.skill_id,
     endpoint_id: "browser-capture",
     started_at: startedAt,
-    completed_at: completedAt,
+    completed_at: new Date().toISOString(),
     success: true,
     result: { learned_skill_id: learned.skill_id, endpoints_discovered: cleanEndpoints.length },
   });
@@ -1527,16 +1416,6 @@ async function tryHttpFetch(
   }
 }
 
-/** When extraction returns "multiple" candidates, pick the best one's data to avoid duplicates */
-function flattenExtracted(data: unknown): unknown {
-  if (!Array.isArray(data)) return data;
-  const first = data[0];
-  if (first && typeof first === "object" && "type" in first && "data" in first && "relevance_score" in first) {
-    return data.reduce((best: any, cur: any) => (cur.relevance_score ?? 0) > (best.relevance_score ?? 0) ? cur : best).data;
-  }
-  return data;
-}
-
 async function executeDomExtractionEndpoint(
   endpoint: EndpointDescriptor,
   url: string,
@@ -1555,7 +1434,16 @@ async function executeDomExtractionEndpoint(
         if (ssrSemantic.verdict !== "fail") {
           console.log(`[ssr-fast] hit — extracted via HTTP fetch`);
           return {
-            data: flattenExtracted(ssrExtracted.data),
+            data: {
+              data: ssrExtracted.data,
+              _extraction: {
+                method: ssrExtracted.extraction_method,
+                confidence: ssrExtracted.confidence,
+                source: "ssr-fast",
+                final_url: ssrResult.final_url,
+                ...(ssrExtracted.selector ? { selector: ssrExtracted.selector } : {}),
+              },
+            },
             status: 200,
             trace_id: nanoid(),
           };
@@ -1567,33 +1455,8 @@ async function executeDomExtractionEndpoint(
     console.log(`[ssr-fast] miss, falling back to browser`);
   }
 
-  // Browser fallback — captures both intercepted API requests AND page HTML
+  // Browser fallback
   const captured = await captureSession(url, authHeaders, cookies, intent);
-
-  // Check intercepted requests first — if the site's JS made API calls,
-  // those have the actual filtered data (not the initial HTML page load)
-  if (captured.requests.length > 0) {
-    const { extractEndpoints: extractEps } = await import("../reverse-engineer/index.js");
-    const apiEndpoints = extractEps(captured.requests, undefined, { pageUrl: url, finalUrl: captured.final_url });
-    const jsonEndpoints = apiEndpoints.filter(ep => ep.response_schema && !ep.dom_extraction);
-    if (jsonEndpoints.length > 0) {
-      // Found real API responses — return the best one's data
-      const best = jsonEndpoints[0];
-      const matchingReq = captured.requests.find(r =>
-        r.url.includes(best.url_template.split("?")[0].split("{")[0]) &&
-        r.response_body && r.response_status >= 200 && r.response_status < 400
-      );
-      if (matchingReq?.response_body) {
-        try {
-          const data = JSON.parse(matchingReq.response_body);
-          console.log(`[dom-exec] found API response from browser capture: ${matchingReq.url.substring(0, 80)}`);
-          return { data, status: matchingReq.response_status, trace_id: nanoid() };
-        } catch { /* not JSON, fall through to DOM extraction */ }
-      }
-    }
-  }
-
-  // Fall back to DOM extraction from rendered HTML
   const html = captured.html ?? "";
   const extracted = extractFromDOMWithHint(html, intent, endpoint.dom_extraction);
   if (extracted.data) {
@@ -1620,7 +1483,16 @@ async function executeDomExtractionEndpoint(
       };
     }
     return {
-      data: flattenExtracted(extracted.data),
+      data: {
+        data: extracted.data,
+        _extraction: {
+          method: extracted.extraction_method,
+          confidence: extracted.confidence,
+          source: "rendered-dom",
+          final_url: captured.final_url,
+          ...(extracted.selector ? { selector: extracted.selector } : {}),
+        },
+      },
       status: 200,
       trace_id: nanoid(),
     };
@@ -1670,6 +1542,8 @@ export async function executeEndpoint(
       }
       return {
         trace, result: resultData,
+        ...(endpoint.response_schema ? { response_schema: endpoint.response_schema } : {}),
+        ...(endpoint.response_schema ? { extraction_hints: generateExtractionHints(endpoint.response_schema, skill.intent_signature) ?? undefined } : {}),
       };
     } catch (err) {
       const trace: ExecutionTrace = stampTrace({
@@ -1681,38 +1555,6 @@ export async function executeEndpoint(
     }
   }
 
-  // Payment gate — check if marketplace skill requires payment before executing
-  if (!skill.skill_id.startsWith("local:") && skill.execution_type === "http" && skill.owner_type !== "agent") {
-    const wallet = getLocalWalletContext();
-    const gate = await checkPaymentRequirement(skill.skill_id, endpoint.endpoint_id, {
-      wallet_configured: !!wallet.wallet_address,
-    });
-    if (gate.status === "payment_required" || gate.status === "wallet_not_configured" || gate.status === "insufficient_balance") {
-      const trace: ExecutionTrace = stampTrace({
-        trace_id: nanoid(),
-        skill_id: skill.skill_id,
-        endpoint_id: endpoint.endpoint_id,
-        started_at: new Date().toISOString(),
-        completed_at: new Date().toISOString(),
-        success: false,
-        status_code: 402,
-        error: "payment_required",
-      });
-      return {
-        trace,
-        result: {
-          error: "payment_required",
-          price_usd: gate.requirement?.amount,
-          payment_status: gate.status,
-          message: gate.message,
-          wallet_provider: wallet.wallet_provider ?? "lobster.cash",
-          wallet_address: wallet.wallet_address,
-          indexing_fallback_available: true,
-        },
-      };
-    }
-  }
-
   // Mutation safety gate
   if (endpoint.method !== "GET" && endpoint.idempotency === "unsafe") {
     if (options?.dry_run) {
@@ -1899,45 +1741,18 @@ export async function executeEndpoint(
     }
   }
 
-
-  // robots.txt compliance gate — block disallowed paths before any network call.
-  if (!options?.skip_robots_check) {
-    const allowed = await isAllowedByRobots(url);
-    if (!allowed) {
-      const traceId = nanoid();
-      log("exec", `robots.txt blocked ${url}`);
-      return {
-        trace: stampTrace({
-          trace_id: traceId,
-          skill_id: skill.skill_id,
-          endpoint_id: endpoint.endpoint_id,
-          started_at: startedAt,
-          completed_at: new Date().toISOString(),
-          success: false,
-          error: "robots_txt_disallowed",
-        }),
-        result: {
-          error: "robots_txt_disallowed",
-          message: `robots.txt disallows access to ${url} for the Unbrowse user-agent.`,
-        },
-      };
-    }
-  }
   const structuredReplayUrl = isSafe ? deriveStructuredDataReplayUrl(url) : url;
   const hasStructuredReplay = structuredReplayUrl !== url;
 
   const serverFetch = async (): Promise<{ data: unknown; status: number; trace_id: string }> => {
-    const endpointHeaders = normalizeReplayHeaders(endpoint.headers_template);
-    const sessionHeaders = normalizeReplayHeaders(authHeaders);
-
     // Default accept to JSON, but never overwrite the endpoint's own accept header
     // (e.g. LinkedIn uses "application/vnd.linkedin.normalized+json+2.1")
-    const defaultAccept: Record<string, string> = (!endpoint.dom_extraction && !endpointHeaders["accept"] && !sessionHeaders["accept"])
+    const defaultAccept: Record<string, string> = (!endpoint.dom_extraction && !endpoint.headers_template?.["accept"])
       ? { "accept": "application/json" } : {};
     const headers: Record<string, string> = {
       ...defaultAccept,
-      ...endpointHeaders,
-      ...sessionHeaders,
+      ...endpoint.headers_template,
+      ...authHeaders,
     };
     // Strip browser-only headers that cause issues server-side
     delete headers["sec-ch-ua"];
@@ -1956,15 +1771,15 @@ export async function executeEndpoint(
       headers["cookie"] = cookieStr;
 
       // CSRF token auto-detection (bird pattern): many sites require CSRF tokens
-      // as both a cookie AND a header. The cookie value is always fresher than
-      // any stored vault header, so it ALWAYS overrides.
-      const csrfCookie = cookies.find((c) =>
-        /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf)$/i.test(c.name)
-      );
-      if (csrfCookie) {
-        const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
-        headers["x-csrf-token"] = v;
-        headers["x-xsrf-token"] = v;
+      // as both a cookie AND a header. Detect common patterns and replay them.
+      if (!headers["x-csrf-token"] && !headers["x-xsrf-token"]) {
+        const csrfCookie = cookies.find((c) =>
+          /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf)$/i.test(c.name)
+        );
+        if (csrfCookie) {
+          const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
+          headers["x-csrf-token"] = v;
+        }
       }
     }
 
@@ -1975,7 +1790,7 @@ export async function executeEndpoint(
       if (csrfCookie) {
         const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
         if (endpoint.csrf_plan.source === "cookie" || endpoint.csrf_plan.source === "header") {
-          headers[endpoint.csrf_plan.param_name.toLowerCase()] = v;
+          headers[endpoint.csrf_plan.param_name.toLowerCase()] ??= v;
         } else if (endpoint.csrf_plan.source === "form" && body && typeof body === "object" && !Array.isArray(body)) {
           (body as Record<string, unknown>)[endpoint.csrf_plan.param_name] ??= v;
         }
@@ -1987,7 +1802,6 @@ export async function executeEndpoint(
 
     for (const replayUrl of replayUrls) {
       const replayHeaders = buildStructuredReplayHeaders(url, replayUrl, headers);
-      log("exec", `server-fetch: ${endpoint.method} ${replayUrl.substring(0, 80)} csrf=${replayHeaders["x-csrf-token"]?.substring(0, 10)}... cookies=${(replayHeaders["cookie"]?.length ?? 0)}chars`);
       const res = await fetch(replayUrl, {
         method: endpoint.method,
         headers: replayHeaders,
@@ -2056,20 +1870,7 @@ export async function executeEndpoint(
       if (result.status >= 200 && result.status < 400 && !shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
         strategy = "server";
       } else if (endpoint.trigger_url && isSafe) {
-        // Build trigger URL with agent's params applied — don't replay original captured search
-        let triggerUrl = endpoint.trigger_url;
-        if (Object.keys(mergedParams).length > 0) {
-          try {
-            const tu = new URL(endpoint.trigger_url);
-            for (const [k, v] of Object.entries(mergedParams)) {
-              if (v != null && !reservedMetaParams.has(k)) {
-                tu.searchParams.set(k, String(v));
-              }
-            }
-            triggerUrl = tu.toString();
-          } catch { /* keep original trigger_url */ }
-        }
-        result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
+        result = await triggerAndIntercept(endpoint.trigger_url, endpoint.url_template, cookies, authHeaders);
         strategy = "trigger-intercept";
       } else {
         result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
@@ -2086,18 +1887,8 @@ export async function executeEndpoint(
       }
     } else if (endpointStrategy === "trigger-intercept" && endpoint.trigger_url && isSafe) {
       // Proven: this endpoint needs trigger-intercept
-      let triggerUrl = endpoint.trigger_url;
-      if (Object.keys(mergedParams).length > 0) {
-        try {
-          const tu = new URL(endpoint.trigger_url);
-          for (const [k, v] of Object.entries(mergedParams)) {
-            if (v != null && !reservedMetaParams.has(k)) tu.searchParams.set(k, String(v));
-          }
-          triggerUrl = tu.toString();
-        } catch { /* keep original */ }
-      }
-      log("exec", `using learned strategy trigger-intercept via ${triggerUrl}`);
-      result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
+      log("exec", `using learned strategy trigger-intercept via ${endpoint.trigger_url}`);
+      result = await triggerAndIntercept(endpoint.trigger_url, endpoint.url_template, cookies, authHeaders);
       strategy = "trigger-intercept";
     } else if (endpointStrategy === "browser") {
       if (shouldIgnoreLearnedBrowserStrategy(endpoint, url)) {
@@ -2120,9 +1911,7 @@ export async function executeEndpoint(
       try {
         result = await serverFetch();
         if (result.status >= 200 && result.status < 400) {
-          // For API endpoints, trust the server response — no fallback to browser
-          const isApiEndpoint = /\/(api|graphql)\b/i.test(endpoint.url_template) || /\.(json)(\?|$)/.test(endpoint.url_template);
-          if (!isApiEndpoint && shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
+          if (shouldFallbackToBrowserReplay(result.data, endpoint, options?.intent ?? skill.intent_signature, options?.contextUrl)) {
             result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
             strategy = "browser";
           } else {
@@ -2131,17 +1920,7 @@ export async function executeEndpoint(
         } else {
           log("exec", `server fetch returned ${result.status}, falling back`);
           if (endpoint.trigger_url && isSafe) {
-            let triggerUrl = endpoint.trigger_url;
-            if (Object.keys(mergedParams).length > 0) {
-              try {
-                const tu = new URL(endpoint.trigger_url);
-                for (const [k, v] of Object.entries(mergedParams)) {
-                  if (v != null && !reservedMetaParams.has(k)) tu.searchParams.set(k, String(v));
-                }
-                triggerUrl = tu.toString();
-              } catch { /* keep original */ }
-            }
-            result = await triggerAndIntercept(triggerUrl, endpoint.url_template, cookies, authHeaders);
+            result = await triggerAndIntercept(endpoint.trigger_url, endpoint.url_template, cookies, authHeaders);
             strategy = "trigger-intercept";
           } else {
             result = await withRetry(browserCall, (r) => isRetryableStatus(r.status));
@@ -2199,41 +1978,15 @@ export async function executeEndpoint(
     trace.result = data;
   }
 
-  // Stale credential detection: on 401/403, attempt auth recovery before giving up.
-  // Chain: authRuntime.refreshSession (lightweight) → refreshAuthFromBrowser (re-extract)
-  //        → authRuntime.loginIfNeeded (full interactive login)
+  // Stale credential detection: on 401/403, try refreshing from browser (bird pattern)
+  // instead of just deleting. Next request will use fresh cookies.
   if (status === 401 || status === 403) {
-    let authRecovered = false;
     try {
-      // 1. Lightweight session refresh via authRuntime
-      const sessionRefreshed = await authRuntime.refreshSession(epDomain);
-      if (sessionRefreshed) {
-        log("auth", `session refreshed via authRuntime for ${epDomain} — retry should succeed`);
-        authRecovered = true;
-      }
-
-      // 2. Re-extract cookies from browser SQLite (bird pattern)
-      if (!authRecovered) {
-        const browserRefreshed = await refreshAuthFromBrowser(epDomain);
-        if (browserRefreshed) {
-          log("auth", `credentials refreshed from browser for ${epDomain}`);
-          authRecovered = true;
-        }
-      }
-
-      // 3. Full login flow via authRuntime as last resort
-      if (!authRecovered) {
-        const loginResult = await authRuntime.loginIfNeeded(epDomain);
-        if (loginResult) {
-          log("auth", `loginIfNeeded succeeded for ${epDomain}`);
-          authRecovered = true;
-        }
-      }
-
-      if (authRecovered) {
-        trace.error = `${trace.error} (credentials refreshed — retry should succeed)`;
+      const refreshed = await refreshAuthFromBrowser(epDomain);
+      if (refreshed) {
+        trace.error = `${trace.error} (credentials refreshed from browser — retry should succeed)`;
       } else {
-        // No recovery path worked — delete stale credentials
+        // No fresh cookies available — delete stale ones
         if (skill.auth_profile_ref) {
           await deleteCredential(skill.auth_profile_ref);
         }
@@ -2326,27 +2079,8 @@ export async function executeEndpoint(
   }
 
   // Record execution for reliability scoring (fire-and-forget — don't block response)
-  recordExecution(skill.skill_id, endpoint.endpoint_id, trace, skill).catch(() => {});
+  recordExecution(skill.skill_id, endpoint.endpoint_id, trace).catch(() => {});
 
-  // Record transaction if this was a paid execution (fire-and-forget)
-  if (trace.success && options?.payment_verified === true && skill.base_price_usd && skill.base_price_usd > 0) {
-    const consumerConfig = (() => {
-      try { return JSON.parse(require("fs").readFileSync(require("os").homedir() + "/.unbrowse/config.json", "utf-8")); }
-      catch { return {}; }
-    })();
-    if (consumerConfig.agent_id) {
-      const wallet = getLocalWalletContext();
-      recordTransaction({
-        transaction_id: trace.trace_id,
-        consumer_id: consumerConfig.agent_id,
-        creator_id: skill.indexer_id,
-        skill_id: skill.skill_id,
-        endpoint_id: endpoint.endpoint_id,
-        price_usd: skill.base_price_usd,
-        payment_proof: wallet.wallet_address ? `wallet:${wallet.wallet_address}` : undefined,
-      }).catch(() => {});
-    }
-  }
   // Apply field projection
   let resultData = data;
   if (projection?.raw) {
@@ -2357,8 +2091,14 @@ export async function executeEndpoint(
     resultData = projectResultForIntent(data, effectiveIntent);
   }
 
+  const rawResultShape = resultData === data;
+
   return {
     trace, result: resultData,
+    ...(endpoint.response_schema && rawResultShape ? { response_schema: endpoint.response_schema } : {}),
+    ...(endpoint.response_schema && rawResultShape
+      ? { extraction_hints: generateExtractionHints(endpoint.response_schema, effectiveIntent) ?? undefined }
+      : {}),
   };
 }
 
@@ -2367,7 +2107,7 @@ export async function executeEndpoint(
  * e.g. /search?q=books&page=1 → /search?q={q}&page={page}
  * Path stays untouched — only query string is templatized.
  */
-export function templatizeQueryParams(url: string): string {
+function templatizeQueryParams(url: string): string {
   try {
     const u = sanitizeNavigationQueryParams(new URL(url));
     if (u.search.length <= 1) return url; // no query params