unbrowse 3.1.0-experiments.995f8bb → 3.1.0-experiments.9cbcb13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "3.1.0-experiments.995f8bb",
+  "version": "3.1.0-experiments.9cbcb13",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {
@@ -9,7 +9,6 @@
   "files": [
     "bin",
     "dist",
-    "runtime-src",
     "vendor/kuri",
     "scripts/release-assets.mjs",
     "scripts/postinstall.mjs",

package/vendor/kuri/manifest.json

@@ -1,7 +1,7 @@
 {
   "repo_url": "https://github.com/justrach/kuri.git",
   "branch": "adding-extensions",
-  "source_sha": "08eecbe3740f046a46f656eed7ebfc66c1bad9bb",
+  "source_sha": "eadfaa5f921f7152e1762aed5ed64b3a4fbefbf3",
   "built_at": "2026-04-05T06:43:57.212Z",
   "binaries": {
     "darwin-arm64": {

package/runtime-src/agent-outcome.ts

@@ -1,166 +0,0 @@
-import type { OrchestrationTiming, SkillManifest } from "./types/index.js";
-
-export interface AgentImpact {
-  source: string;
-  cache_hit: boolean;
-  browser_avoided: boolean;
-  baseline_total_ms?: number;
-  actual_total_ms?: number;
-  time_saved_ms?: number;
-  time_saved_pct: number;
-  tokens_saved: number;
-  tokens_saved_pct: number;
-  baseline_cost_uc?: number;
-  actual_cost_uc?: number;
-  cost_saved_uc?: number;
-}
-
-export interface AgentNextAction {
-  endpoint_id: string;
-  operation_id: string;
-  title: string;
-  why: string;
-  command: string;
-}
-
-const BROWSER_SOURCES = new Set(["live-capture", "first-pass", "browser-action"]);
-
-function edgePriority(kind: string): number {
-  switch (kind) {
-    case "parent_child":
-      return 4;
-    case "pagination":
-      return 3;
-    case "dependency":
-      return 2;
-    case "hint":
-      return 1;
-    case "auth":
-      return 0;
-    default:
-      return -1;
-  }
-}
-
-function nextActionWhy(kind: string, bindingKey: string, title: string): string {
-  switch (kind) {
-    case "parent_child":
-      return `Likely next detail step after this result. Exposes ${title}.`;
-    case "pagination":
-      return `Likely next page or continuation step. Carries ${bindingKey || "cursor"} forward.`;
-    case "dependency":
-      return `Unlocks the next dependent call using ${bindingKey || "known bindings"}.`;
-    case "auth":
-      return "Useful once authentication is in place.";
-    case "hint":
-      return "Common follow-up action from the current result.";
-    default:
-      return "Likely follow-up action.";
-  }
-}
-
-function operationTitle(operation: NonNullable<SkillManifest["operation_graph"]>["operations"][number]): string {
-  const semantic = [operation.action_kind, operation.resource_kind]
-    .filter(Boolean)
-    .join(" ")
-    .replace(/_/g, " ")
-    .trim();
-  return operation.description_out || semantic || operation.endpoint_id;
-}
-
-export function buildAgentImpact(
-  timing?: Partial<OrchestrationTiming> | null,
-): AgentImpact | undefined {
-  if (!timing?.source) return undefined;
-  return {
-    source: timing.source,
-    cache_hit: timing.cache_hit === true,
-    browser_avoided: !BROWSER_SOURCES.has(timing.source),
-    baseline_total_ms: timing.baseline_total_ms,
-    actual_total_ms: timing.actual_total_ms,
-    time_saved_ms: timing.time_saved_ms,
-    time_saved_pct: timing.time_saved_pct ?? 0,
-    tokens_saved: timing.tokens_saved ?? 0,
-    tokens_saved_pct: timing.tokens_saved_pct ?? 0,
-    baseline_cost_uc: timing.baseline_cost_uc,
-    actual_cost_uc: timing.actual_cost_uc,
-    cost_saved_uc: timing.cost_saved_uc,
-  };
-}
-
-export function buildNextActions(
-  skill: SkillManifest | undefined,
-  endpointId: string | undefined,
-  maxActions = 3,
-): AgentNextAction[] {
-  if (!skill?.operation_graph || !endpointId) return [];
-  const graph = skill.operation_graph;
-  const current = graph.operations.find((operation) => operation.endpoint_id === endpointId);
-  if (!current) return [];
-
-  const byOperationId = new Map(graph.operations.map((operation) => [operation.operation_id, operation]));
-  const scored = new Map<string, {
-    operation_id: string;
-    endpoint_id: string;
-    title: string;
-    why: string;
-    score: number;
-  }>();
-
-  for (const edge of graph.edges) {
-    if (edge.from_operation_id !== current.operation_id) continue;
-    const target = byOperationId.get(edge.to_operation_id);
-    if (!target) continue;
-
-    const candidate = {
-      operation_id: target.operation_id,
-      endpoint_id: target.endpoint_id,
-      title: operationTitle(target),
-      why: nextActionWhy(edge.kind, edge.binding_key, operationTitle(target)),
-      score: (edgePriority(edge.kind) * 10) + Math.round(edge.confidence * 10),
-    };
-    const existing = scored.get(target.operation_id);
-    if (!existing || candidate.score > existing.score) {
-      scored.set(target.operation_id, candidate);
-    }
-  }
-
-  return [...scored.values()]
-    .sort((a, b) => b.score - a.score || a.title.localeCompare(b.title))
-    .slice(0, maxActions)
-    .map((candidate) => ({
-      endpoint_id: candidate.endpoint_id,
-      operation_id: candidate.operation_id,
-      title: candidate.title,
-      why: candidate.why,
-      command: `unbrowse execute --skill ${skill.skill_id} --endpoint ${candidate.endpoint_id}`,
-    }));
-}
-
-export function attachAgentOutcomeHints<T extends Record<string, unknown>>(
-  payload: T,
-  opts?: {
-    skill?: SkillManifest;
-    endpointId?: string;
-    timing?: Partial<OrchestrationTiming> | null;
-  },
-): T & {
-  impact?: AgentImpact;
-  next_actions?: AgentNextAction[];
-} {
-  const target = payload as Record<string, unknown>;
-  const impact = buildAgentImpact(opts?.timing);
-  if (impact) {
-    target.impact = impact;
-  }
-
-  const nextActions = buildNextActions(opts?.skill, opts?.endpointId);
-  if (nextActions.length > 0) {
-    target.next_actions = nextActions;
-  }
-
-  return target as T & {
-    impact?: AgentImpact;
-    next_actions?: AgentNextAction[];
-  };
-}

package/runtime-src/analytics-session.ts

@@ -1,55 +0,0 @@
-import type { ExecutionTrace, OrchestrationTiming } from "./types/index.js";
-
-export interface AnalyticsSessionPayload {
-  session_id: string;
-  started_at: string;
-  completed_at?: string;
-  trace_version?: string;
-  api_calls: number;
-  discovery_queries: number;
-  cached_skill_calls: number;
-  fresh_index_calls: number;
-  browser_mode: "default" | "replaced" | "manual" | "unknown";
-  success?: boolean;
-  source?: string;
-  time_saved_ms?: number;
-  time_saved_pct?: number;
-  tokens_saved?: number;
-  tokens_saved_pct?: number;
-  cost_saved_uc?: number;
-}
-
-export function buildAnalyticsSessionPayload(
-  sessionId: string,
-  startedAt: string,
-  source: OrchestrationTiming["source"] | "first-pass",
-  trace: Pick<ExecutionTrace, "completed_at" | "trace_version" | "success" | "tokens_saved" | "tokens_saved_pct" | "api_call_count"> & {
-    network_events?: unknown[];
-  },
-  timing?: Pick<OrchestrationTiming, "time_saved_ms" | "time_saved_pct" | "cost_saved_uc">,
-): AnalyticsSessionPayload {
-  const cacheLike = source === "marketplace" || source === "route-cache";
-  const browserMode = source === "live-capture" || source === "browser-action"
-    ? "default"
-    : source === "first-pass"
-      ? "default"
-      : "replaced";
-  return {
-    session_id: sessionId,
-    started_at: startedAt,
-    completed_at: trace.completed_at,
-    trace_version: trace.trace_version,
-    api_calls: trace.api_call_count ?? Math.max(1, trace.network_events?.length ?? 0),
-    discovery_queries: cacheLike ? 1 : 0,
-    cached_skill_calls: cacheLike ? 1 : 0,
-    fresh_index_calls: source === "live-capture" || source === "first-pass" || source === "browser-action" ? 1 : 0,
-    browser_mode: browserMode,
-    success: trace.success ?? true,
-    source,
-    time_saved_ms: timing?.time_saved_ms,
-    time_saved_pct: timing?.time_saved_pct,
-    tokens_saved: trace.tokens_saved,
-    tokens_saved_pct: trace.tokens_saved_pct,
-    cost_saved_uc: timing?.cost_saved_uc,
-  };
-}

package/runtime-src/api/browse-index.ts

@@ -1,343 +0,0 @@
-import { nanoid } from "nanoid";
-import { readFileSync } from "node:fs";
-import { log } from "../logger.js";
-import { extractEndpoints, extractAuthHeaders } from "../reverse-engineer/index.js";
-import { enrichEndpointsWithTokenSources } from "../reverse-engineer/token-sources.js";
-import { buildSkillOperationGraph, inferEndpointSemantic } from "../graph/index.js";
-import { validateExtractionQuality } from "../execution/index.js";
-import { assessIntentResult } from "../intent-match.js";
-import type { KuriHarEntry } from "../kuri/client.js";
-import type { EndpointDescriptor, SkillManifest } from "../types/index.js";
-import type { RawRequest } from "../capture/index.js";
-import { cachePublishedSkill, findExistingSkillForDomain } from "../client/index.js";
-import { mergeEndpoints } from "../marketplace/index.js";
-import { upsertDagEdgesFromOperationGraph } from "../orchestrator/dag-feedback.js";
-import { storeCredential } from "../vault/index.js";
-import { getRegistrableDomain } from "../domain.js";
-import {
-  buildResolveCacheKey,
-  domainSkillCache,
-  generateLocalDescription,
-  getDomainReuseKey,
-  invalidateRouteCacheForDomain,
-  persistDomainCache,
-  scopedCacheKey,
-  snapshotPathForCacheKey,
-  writeSkillSnapshot,
-} from "../orchestrator/index.js";
-
-function normalizeBrowseUrl(url: string, baseUrl?: string): string {
-  if (!url) return url;
-  try {
-    return new URL(url).toString();
-  } catch {
-    if (!baseUrl) return url;
-    try {
-      return new URL(url, baseUrl).toString();
-    } catch {
-      return url;
-    }
-  }
-}
-
-export function harEntriesToRawRequests(entries: KuriHarEntry[], baseUrl?: string): RawRequest[] {
-  return entries
-    .filter((entry) => entry.request && entry.response)
-    .map((entry) => ({
-      url: normalizeBrowseUrl(entry.request.url, baseUrl),
-      method: entry.request.method,
-      request_headers: Object.fromEntries((entry.request.headers ?? []).map((header) => [header.name.toLowerCase(), header.value])),
-      request_body: entry.request.postData?.text,
-      response_status: entry.response.status,
-      response_headers: Object.fromEntries((entry.response.headers ?? []).map((header) => [header.name.toLowerCase(), header.value])),
-      response_body: entry.response.content?.text,
-      timestamp: entry.startedDateTime ?? new Date().toISOString(),
-    }));
-}
-
-export function buildBrowseRequestKey(request: RawRequest): string {
-  return [
-    request.method,
-    request.url,
-    typeof request.request_body === "string" ? request.request_body : JSON.stringify(request.request_body ?? null),
-  ].join(":");
-}
-
-export function mergeBrowseRequests(intercepted: RawRequest[], harEntries: KuriHarEntry[], baseUrl?: string): RawRequest[] {
-  const normalizedIntercepted = intercepted.map((request) => ({
-    ...request,
-    url: normalizeBrowseUrl(request.url, baseUrl),
-  }));
-  const harRequests = harEntriesToRawRequests(harEntries, baseUrl);
-  const seen = new Set<string>();
-  const allRequests: RawRequest[] = [];
-
-  for (const request of normalizedIntercepted) {
-    const key = buildBrowseRequestKey(request);
-    if (!seen.has(key)) {
-      seen.add(key);
-      allRequests.push(request);
-    }
-  }
-
-  for (const request of harRequests) {
-    const key = buildBrowseRequestKey(request);
-    if (!seen.has(key)) {
-      seen.add(key);
-      allRequests.push(request);
-    }
-  }
-
-  return allRequests;
-}
-
-export interface BrowseIndexResult {
-  domain: string;
-  indexed: boolean;
-  mode: "http" | "dom" | "none";
-  skill: SkillManifest | null;
-}
-
-export function shouldIndexDomBrowseFallback(params: {
-  requestCount: number;
-  intent: string;
-  extractedData: unknown;
-  extractedConfidence: number;
-  hasStructuredForm: boolean;
-}): {
-  allow: boolean;
-  reason?: string;
-  intentLooksSearch: boolean;
-} {
-  const { requestCount, intent, extractedData, extractedConfidence, hasStructuredForm } = params;
-  const intentLooksSearch = /\b(search|find|lookup|filter)\b/i.test(intent);
-
-  if (!extractedData) {
-    if (hasStructuredForm && requestCount > 0 && intentLooksSearch) {
-      return { allow: true, intentLooksSearch };
-    }
-    return {
-      allow: false,
-      reason: hasStructuredForm ? "form_only_without_network_evidence" : "no_dom_data",
-      intentLooksSearch,
-    };
-  }
-
-  const quality = validateExtractionQuality(extractedData, extractedConfidence, intent);
-  if (!quality.valid) {
-    if (hasStructuredForm && requestCount > 0 && intentLooksSearch) {
-      return { allow: true, intentLooksSearch };
-    }
-    return {
-      allow: false,
-      reason: quality.quality_note ?? "low_quality_dom_extraction",
-      intentLooksSearch,
-    };
-  }
-
-  const semanticAssessment = assessIntentResult(extractedData, intent);
-  if (semanticAssessment.verdict === "fail") {
-    if (hasStructuredForm && requestCount > 0 && intentLooksSearch) {
-      return { allow: true, intentLooksSearch };
-    }
-    return {
-      allow: false,
-      reason: semanticAssessment.reason ?? "dom_extraction_did_not_match_intent",
-      intentLooksSearch,
-    };
-  }
-
-  return { allow: true, intentLooksSearch };
-}
-
-export async function cacheBrowseRequests(params: {
-  sessionUrl: string;
-  sessionDomain: string;
-  requests: RawRequest[];
-  getPageHtml?: () => Promise<string>;
-  jsBundles?: Map<string, string>;
-  intent?: string;
-}): Promise<BrowseIndexResult> {
-  const { sessionUrl, sessionDomain, requests, getPageHtml, jsBundles } = params;
-  let domain: string;
-  try { domain = new URL(sessionUrl).hostname; } catch { domain = sessionDomain; }
-  const intent = params.intent ?? `browse ${domain}`;
-
-  const rawEndpoints = extractEndpoints(requests, undefined, { pageUrl: sessionUrl, finalUrl: sessionUrl });
-
-  // Extract and persist auth headers (authorization, csrf, bearer tokens)
-  // so serverFetch can replay them. Use registrable domain for vault key
-  // so ads.x.com and ads-api.x.com share the same session.
-  const capturedAuthHeaders = extractAuthHeaders(requests);
-  if (Object.keys(capturedAuthHeaders).length > 0) {
-    const sessionKey = `${getRegistrableDomain(domain)}-session`;
-    await storeCredential(sessionKey, JSON.stringify({ headers: capturedAuthHeaders })).catch(() => {});
-  }
-
-  if (rawEndpoints.length > 0) {
-    const existingSkill = findExistingSkillForDomain(domain);
-    let allExisting = existingSkill?.endpoints ?? [];
-
-    const domainKey = getDomainReuseKey(sessionUrl ?? domain);
-    if (domainKey) {
-      const cached = domainSkillCache.get(domainKey);
-      if (cached?.localSkillPath) {
-        try {
-          const snapshot = JSON.parse(readFileSync(cached.localSkillPath, "utf-8"));
-          if (snapshot?.endpoints?.length > 0) {
-            allExisting = mergeEndpoints(allExisting, snapshot.endpoints);
-          }
-        } catch {
-          // ignore stale snapshot
-        }
-      }
-    }
-
-    const mergedEndpoints = allExisting.length > 0 ? mergeEndpoints(allExisting, rawEndpoints) : rawEndpoints;
-    if (!existingSkill || mergedEndpoints.length >= existingSkill.endpoints.length) {
-      for (const endpoint of mergedEndpoints) {
-        if (!endpoint.description) endpoint.description = generateLocalDescription(endpoint);
-        if (!endpoint.semantic) endpoint.semantic = inferEndpointSemantic(endpoint);
-      }
-      const quickSkill: SkillManifest = {
-        skill_id: existingSkill?.skill_id ?? nanoid(),
-        version: "1.0.0",
-        schema_version: "1",
-        lifecycle: "active",
-        execution_type: "http",
-        created_at: existingSkill?.created_at ?? new Date().toISOString(),
-        updated_at: new Date().toISOString(),
-        name: domain,
-        intent_signature: intent,
-        domain,
-        description: `API skill for ${domain}`,
-        owner_type: "agent",
-        endpoints: mergedEndpoints,
-        operation_graph: buildSkillOperationGraph(mergedEndpoints),
-        intents: Array.from(new Set([...(existingSkill?.intents ?? []), intent])),
-      };
-
-      // Token source discovery: scan live HTML for tokens used in captured
-      // request headers and attach AuthTokenBinding entries so serverFetch
-      // can rescrape fresh tokens on replay.
-      try {
-        const html = getPageHtml ? await getPageHtml() : undefined;
-        if (html && html.startsWith("<")) {
-          const preCheck = requests.filter(r => r.request_headers["authorization"] || r.request_headers["x-csrf-token"]).length;
-          const enriched = enrichEndpointsWithTokenSources(quickSkill.endpoints, requests, html, jsBundles);
-          log("browse-index", `token enrichment: ${enriched} bindings, ${preCheck} auth-reqs pre-call, ${quickSkill.endpoints.length} eps`);
-        }
-      } catch (e) { log("browse-index", `token enrichment failed: ${e}`); }
-
-      const cacheKey = buildResolveCacheKey(domain, intent, sessionUrl);
-      const scopedKey = scopedCacheKey("global", cacheKey);
-      writeSkillSnapshot(scopedKey, quickSkill);
-      if (domainKey) {
-        domainSkillCache.set(domainKey, {
-          skillId: quickSkill.skill_id,
-          localSkillPath: snapshotPathForCacheKey(scopedKey),
-          ts: Date.now(),
-        });
-        persistDomainCache();
-      }
-      try { cachePublishedSkill(quickSkill); } catch {}
-      upsertDagEdgesFromOperationGraph(quickSkill);
-      invalidateRouteCacheForDomain(domain);
-      return { domain, indexed: true, mode: "http", skill: quickSkill };
-    }
-
-    return { domain, indexed: false, mode: "http", skill: existingSkill ?? null };
-  }
-
-  if (!getPageHtml) return { domain, indexed: false, mode: "none", skill: null };
-
-  try {
-    const html = await getPageHtml();
-    if (!html || !html.startsWith("<")) return { domain, indexed: false, mode: "none", skill: null };
-
-    const { extractFromDOM } = await import("../extraction/index.js");
-    const { detectSearchForms, isStructuredSearchForm } = await import("../execution/search-forms.js");
-    const { inferSchema } = await import("../transform/index.js");
-    const { templatizeQueryParams } = await import("../execution/index.js");
-
-    const extracted = extractFromDOM(html, intent);
-    const searchForms = detectSearchForms(html);
-    const validForm = searchForms.find((form: { form_selector: string; fields: unknown[] }) => isStructuredSearchForm(form));
-    const domDecision = shouldIndexDomBrowseFallback({
-      requestCount: requests.length,
-      intent,
-      extractedData: extracted.data,
-      extractedConfidence: extracted.confidence,
-      hasStructuredForm: !!validForm,
-    });
-
-    if (!domDecision.allow || !extracted.data) return { domain, indexed: false, mode: "none", skill: null };
-
-    const urlTemplate = templatizeQueryParams(sessionUrl);
-    const endpoint: EndpointDescriptor = {
-      endpoint_id: nanoid(),
-      method: "GET",
-      url_template: urlTemplate,
-      idempotency: "safe",
-      verification_status: "verified",
-      reliability_score: extracted.confidence ?? 0.7,
-      description: validForm && domDecision.intentLooksSearch ? `Search form for ${domain}` : `Page content from ${domain}`,
-      response_schema: inferSchema([extracted.data]),
-      dom_extraction: {
-        extraction_method: extracted.extraction_method ?? "repeated-elements",
-        confidence: extracted.confidence ?? 0.7,
-        ...(extracted.selector ? { selector: extracted.selector } : {}),
-      },
-      trigger_url: sessionUrl,
-      ...(validForm && domDecision.intentLooksSearch ? { search_form: validForm } : {}),
-    };
-
-    endpoint.semantic = inferEndpointSemantic(endpoint, {
-      sampleResponse: extracted.data,
-      observedAt: new Date().toISOString(),
-      sampleRequestUrl: sessionUrl,
-    });
-
-    const existing = findExistingSkillForDomain(domain);
-    const allEndpoints = existing ? mergeEndpoints(existing.endpoints, [endpoint]) : [endpoint];
-    for (const candidate of allEndpoints) {
-      if (!candidate.description) candidate.description = generateLocalDescription(candidate);
-    }
-
-    const skill: SkillManifest = {
-      skill_id: existing?.skill_id ?? nanoid(),
-      version: "1.0.0",
-      schema_version: "1",
-      lifecycle: "active",
-      execution_type: "http",
-      created_at: existing?.created_at ?? new Date().toISOString(),
-      updated_at: new Date().toISOString(),
-      name: domain,
-      intent_signature: intent,
-      domain,
-      description: `DOM skill for ${domain}`,
-      owner_type: "agent",
-      endpoints: allEndpoints,
-      operation_graph: buildSkillOperationGraph(allEndpoints),
-      intents: [...new Set([...(existing?.intents ?? []), intent])],
-    };
-    const cacheKey = buildResolveCacheKey(domain, intent, sessionUrl);
-    const scopedKey = scopedCacheKey("global", cacheKey);
-    writeSkillSnapshot(scopedKey, skill);
-    const domainReuseKey = getDomainReuseKey(sessionUrl ?? domain);
-    if (domainReuseKey) {
-      domainSkillCache.set(domainReuseKey, {
-        skillId: skill.skill_id,
-        localSkillPath: snapshotPathForCacheKey(scopedKey),
-        ts: Date.now(),
-      });
-      persistDomainCache();
-    }
-    try { cachePublishedSkill(skill); } catch {}
-    upsertDagEdgesFromOperationGraph(skill);
-    invalidateRouteCacheForDomain(domain);
-    return { domain, indexed: true, mode: "dom", skill };
-  } catch {
-    return { domain, indexed: false, mode: "none", skill: null };
-  }
-}