unbrowse 3.1.0 → 3.2.0

package/runtime-src/router.ts

@@ -1,17 +0,0 @@
-import type { EndpointDescriptor } from "./types/index.js";
-
-export const UNSAFE_ACTION_BLOCK_THRESHOLD = 0.6;
-
-export function computeUnsafeActionScore(endpoint: EndpointDescriptor): number {
-  let score = 0;
-  if (endpoint.idempotency === "unsafe") score += 0.4;
-  if (endpoint.method === "POST" || endpoint.method === "PUT" || endpoint.method === "DELETE") score += 0.2;
-  const inferredFromBundle = /inferred from js bundle/i.test(endpoint.description ?? "");
-  if (inferredFromBundle) score += 0.2;
-  if (!endpoint.response_schema) score += 0.1;
-  if (endpoint.verification_status === "failed") score += 0.1;
-  if (endpoint.reliability_score < 0.3) score += 0.1;
-  if (endpoint.trigger_url) score -= 0.1;
-  if (endpoint.verification_status === "verified") score -= 0.15;
-  return Math.max(0, Math.min(1, score));
-}

package/runtime-src/routing-telemetry.ts

@@ -1,395 +0,0 @@
-import { createHash } from "node:crypto";
-import { nanoid } from "nanoid";
-import type {
-  ExecutionOptions,
-  ProjectionOptions,
-  RoutingCandidateEvent,
-  RoutingCandidateSnapshot,
-  RoutingContextBuckets,
-  RoutingSessionCompletedEvent,
-  RoutingSessionEvent,
-  RoutingSessionOutcome,
-  RoutingStepEvent,
-  RoutingTelemetryEvent,
-  RoutingTelemetrySource,
-  RoutingRunType,
-  SkillManifest,
-} from "./types/index.js";
-import { anonymizeUrl, classifyFailure, hashResponseBody, hashValue } from "./telemetry.js";
-
-const BLOCKED_KEY_PATTERN =
-  /(cookie|token|auth|secret|password|session|bearer|credential|apikey|response_body|request_body|transcript|email)/i;
-
-type StateBindings = Record<string, unknown>;
-
-function stableHash(value: unknown): string {
-  return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 24);
-}
-
-function sanitizeScalar(value: unknown): string | number | boolean | null {
-  if (value == null) return null;
-  if (typeof value === "string") return hashValue(value);
-  if (typeof value === "number" || typeof value === "boolean") return value;
-  return stableHash(value);
-}
-
-function sanitizeObject(input: Record<string, unknown>): Record<string, string | number | boolean | null> {
-  const out: Record<string, string | number | boolean | null> = {};
-  for (const [key, value] of Object.entries(input)) {
-    if (BLOCKED_KEY_PATTERN.test(key)) continue;
-    out[key] = sanitizeScalar(value);
-  }
-  return out;
-}
-
-export function hashRoutingState(bindings: StateBindings = {}): string {
-  const safeEntries = Object.entries(bindings)
-    .filter(([key]) => !BLOCKED_KEY_PATTERN.test(key))
-    .sort(([a], [b]) => a.localeCompare(b))
-    .map(([key, value]) => [key, sanitizeScalar(value)]);
-  return stableHash(safeEntries);
-}
-
-export function hashSchemaFingerprint(schema: unknown): string | undefined {
-  if (!schema || typeof schema !== "object") return undefined;
-  return stableHash(schema);
-}
-
-export function buildRoutingContextBuckets(
-  intent: string,
-  projection?: ProjectionOptions,
-  options?: ExecutionOptions,
-  hasPriorHistory = false,
-): RoutingContextBuckets {
-  const lower = intent.toLowerCase();
-  const role = /\b(code|repo|pull request|issue|commit|package)\b/.test(lower)
-    ? "developer"
-    : /\b(research|paper|literature|study)\b/.test(lower)
-      ? "researcher"
-      : /\b(price|stock|trade|portfolio|market)\b/.test(lower)
-        ? "analyst"
-        : "general";
-  const outputPreference = projection?.raw
-    ? "raw"
-    : projection?.fields?.length
-      ? "structured"
-      : "mixed";
-  return {
-    role,
-    cost_sensitivity: options?.force_capture ? "low" : "medium",
-    latency_sensitivity: options?.force_capture ? "low" : "high",
-    output_preference: outputPreference,
-    task_horizon: /\b(workflow|sequence|multi|then|after|before|across)\b/.test(lower) ? "long" : "short",
-    has_prior_history: hasPriorHistory,
-  };
-}
-
-function normalizeDomain(value?: string): string | null {
-  if (!value) return null;
-  try {
-    return new URL(value).hostname.toLowerCase();
-  } catch {
-    return value.toLowerCase();
-  }
-}
-
-function routingSourceFromOrchestratorSource(
-  source: "marketplace" | "live-capture" | "dom-fallback" | "route-cache" | "browser-action" | "first-pass",
-): RoutingTelemetrySource {
-  if (source === "dom-fallback") return "live-capture";
-  if (source === "first-pass") return "browser-action";
-  return source;
-}
-
-function routeFingerprint(endpoint: Pick<SkillManifest["endpoints"][number], "method" | "url_template">): string {
-  return stableHash(`${endpoint.method}:${anonymizeUrl(endpoint.url_template)}`);
-}
-
-function operationIdForEndpoint(skill: SkillManifest | undefined, endpointId?: string): string | undefined {
-  if (!skill?.operation_graph || !endpointId) return undefined;
-  return skill.operation_graph.operations.find((operation) => operation.endpoint_id === endpointId)?.operation_id;
-}
-
-type CollectorOptions = {
-  sessionId: string;
-  startedAt: string;
-  intent: string;
-  traceVersion?: string;
-  anonymizedAgentId?: string;
-  runType: RoutingRunType;
-  contextBuckets: RoutingContextBuckets;
-  normalizedDomains?: string[];
-};
-
-type CandidateRankArgs = {
-  stepIndex?: number;
-  source: RoutingTelemetrySource;
-  stateHashBefore: string;
-  candidateCount: number;
-  reachableOperationCount?: number;
-  availableBindingCount?: number;
-  missingBindingCount?: number;
-  selectedEndpointId?: string;
-  selectedOperationId?: string;
-  candidates: RoutingCandidateSnapshot[];
-};
-
-type StepArgs = {
-  stepIndex?: number;
-  source: RoutingTelemetrySource;
-  stateHashBefore: string;
-  stateHashAfter: string;
-  selectedSkillId?: string;
-  selectedEndpointId?: string;
-  selectedOperationId?: string;
-  reachableOperationCount?: number;
-  availableBindingCount?: number;
-  missingBindingCount?: number;
-  candidateCount: number;
-  executionLatencyMs?: number;
-  statusCode?: number;
-  success?: boolean;
-  failureReason?: string;
-  schemaFingerprint?: string;
-  responseHash?: string;
-  crossDomainTransition?: boolean;
-  retryCount?: number;
-  userOverride?: boolean;
-  didStepUnlockNextStep?: boolean;
-  requiredRecovery?: boolean;
-};
-
-export function buildRoutingCandidateSnapshots(
-  skill: SkillManifest,
-  ranked: Array<{ endpoint: SkillManifest["endpoints"][number]; score: number }>,
-  options?: {
-    reachableEndpointIds?: Set<string>;
-    selectedEndpointId?: string;
-    rejectionReasons?: Record<string, string | undefined>;
-  },
-): RoutingCandidateSnapshot[] {
-  return ranked.map((candidate, index) => ({
-    candidate_id: nanoid(),
-    rank: index + 1,
-    skill_id: skill.skill_id,
-    endpoint_id: candidate.endpoint.endpoint_id,
-    operation_id: operationIdForEndpoint(skill, candidate.endpoint.endpoint_id),
-    route_fingerprint: routeFingerprint(candidate.endpoint),
-    score: Math.round(candidate.score * 1000) / 1000,
-    chosen: candidate.endpoint.endpoint_id === options?.selectedEndpointId,
-    reachable: options?.reachableEndpointIds
-      ? options.reachableEndpointIds.has(candidate.endpoint.endpoint_id)
-      : true,
-    rejection_reason: options?.rejectionReasons?.[candidate.endpoint.endpoint_id],
-    feature_snapshot: {
-      method: candidate.endpoint.method,
-      has_response_schema: !!candidate.endpoint.response_schema,
-      dom_extraction: !!candidate.endpoint.dom_extraction,
-      verification_status: candidate.endpoint.verification_status,
-      reliability_score: candidate.endpoint.reliability_score,
-      unsafe_action_score: candidate.endpoint.idempotency === "unsafe" ? 1 : 0,
-    },
-  }));
-}
-
-export function createRoutingTelemetryCollector(options: CollectorOptions) {
-  const events: RoutingTelemetryEvent[] = [];
-  const normalizedDomains = new Set(options.normalizedDomains?.filter(Boolean) ?? []);
-  let stepIndex = 0;
-  let completed = false;
-
-  const sessionStarted: RoutingSessionEvent = {
-    event_id: nanoid(),
-    event_type: "routing_session_started",
-    session_id: options.sessionId,
-    created_at: options.startedAt,
-    trace_version: options.traceVersion,
-    anonymized_agent_id: options.anonymizedAgentId,
-    top_level_intent: options.intent,
-    normalized_domains: [...normalizedDomains],
-    run_type: options.runType,
-    context_buckets: options.contextBuckets,
-  };
-  events.push(sessionStarted);
-
-  function syncDomains() {
-    sessionStarted.normalized_domains = [...normalizedDomains];
-  }
-
-  function nextStep(explicit?: number): number {
-    if (explicit && explicit > stepIndex) stepIndex = explicit;
-    else if (!explicit) stepIndex += 1;
-    return explicit ?? stepIndex;
-  }
-
-  return {
-    sessionId: options.sessionId,
-    addDomain(value?: string) {
-      const domain = normalizeDomain(value);
-      if (!domain) return;
-      normalizedDomains.add(domain);
-      syncDomains();
-    },
-    recordCandidates(args: CandidateRankArgs): number {
-      const index = nextStep(args.stepIndex);
-      const event: RoutingCandidateEvent = {
-        event_id: nanoid(),
-        event_type: "routing_candidates_ranked",
-        session_id: options.sessionId,
-        created_at: new Date().toISOString(),
-        trace_version: options.traceVersion,
-        anonymized_agent_id: options.anonymizedAgentId,
-        top_level_intent: options.intent,
-        normalized_domains: [...normalizedDomains],
-        run_type: options.runType,
-        step_id: `${options.sessionId}:${index}`,
-        step_index: index,
-        source: args.source,
-        state_hash_before: args.stateHashBefore,
-        candidate_count: args.candidateCount,
-        reachable_operation_count: args.reachableOperationCount,
-        available_binding_count: args.availableBindingCount,
-        missing_binding_count: args.missingBindingCount,
-        selected_endpoint_id: args.selectedEndpointId,
-        selected_operation_id: args.selectedOperationId,
-        candidates: args.candidates,
-      };
-      events.push(event);
-      return index;
-    },
-    recordStep(args: StepArgs): number {
-      const index = nextStep(args.stepIndex);
-      const event: RoutingStepEvent = {
-        event_id: nanoid(),
-        event_type: "routing_step_executed",
-        session_id: options.sessionId,
-        created_at: new Date().toISOString(),
-        trace_version: options.traceVersion,
-        anonymized_agent_id: options.anonymizedAgentId,
-        top_level_intent: options.intent,
-        normalized_domains: [...normalizedDomains],
-        run_type: options.runType,
-        step_id: `${options.sessionId}:${index}`,
-        step_index: index,
-        source: args.source,
-        state_hash_before: args.stateHashBefore,
-        state_hash_after: args.stateHashAfter,
-        selected_skill_id: args.selectedSkillId,
-        selected_endpoint_id: args.selectedEndpointId,
-        selected_operation_id: args.selectedOperationId,
-        reachable_operation_count: args.reachableOperationCount,
-        available_binding_count: args.availableBindingCount,
-        missing_binding_count: args.missingBindingCount,
-        candidate_count: args.candidateCount,
-        execution_latency_ms: args.executionLatencyMs,
-        status_code: args.statusCode,
-        success: args.success,
-        failure_reason: args.failureReason,
-        schema_fingerprint: args.schemaFingerprint,
-        response_hash: args.responseHash,
-        cross_domain_transition: args.crossDomainTransition ?? false,
-        retry_count: args.retryCount ?? 0,
-        user_override: args.userOverride ?? false,
-        did_step_unlock_next_step: args.didStepUnlockNextStep ?? false,
-        required_recovery: args.requiredRecovery ?? false,
-      };
-      events.push(event);
-      return index;
-    },
-    complete(args: {
-      outcome: RoutingSessionOutcome;
-      completedAt: string;
-      totalApiCalls: number;
-      retryCount: number;
-      userOverride: boolean;
-      requiredRecovery: boolean;
-    }) {
-      if (completed) return;
-      completed = true;
-      const rankedEvents = events.filter((event) => event.event_type === "routing_candidates_ranked");
-      const stepEvents = events.filter((event) => event.event_type === "routing_step_executed");
-      const completedEvent: RoutingSessionCompletedEvent = {
-        event_id: nanoid(),
-        event_type: "routing_session_completed",
-        session_id: options.sessionId,
-        created_at: args.completedAt,
-        completed_at: args.completedAt,
-        trace_version: options.traceVersion,
-        anonymized_agent_id: options.anonymizedAgentId,
-        top_level_intent: options.intent,
-        normalized_domains: [...normalizedDomains],
-        run_type: options.runType,
-        final_outcome: args.outcome,
-        final_success: args.outcome === "success",
-        total_steps: stepEvents.length,
-        total_candidates_ranked: rankedEvents.reduce((sum, event) => sum + event.candidate_count, 0),
-        total_api_calls: args.totalApiCalls,
-        retry_count: args.retryCount,
-        user_override: args.userOverride,
-        required_recovery: args.requiredRecovery,
-      };
-      events.push(completedEvent);
-    },
-    toBatch(): RoutingTelemetryEvent[] {
-      return events.map((event) => ({
-        ...event,
-        normalized_domains: [...normalizedDomains],
-      }));
-    },
-  };
-}
-
-export function sanitizeRoutingEventBatch(events: RoutingTelemetryEvent[]): RoutingTelemetryEvent[] {
-  return events.map((event) => {
-    if (event.event_type === "routing_session_started") {
-      return {
-        ...event,
-        top_level_intent: event.top_level_intent.trim(),
-      };
-    }
-    if (event.event_type === "routing_candidates_ranked") {
-      return {
-        ...event,
-        candidates: event.candidates.map((candidate) => ({
-          ...candidate,
-          rejection_reason: candidate.rejection_reason?.slice(0, 120),
-          feature_snapshot: sanitizeObject(candidate.feature_snapshot) as RoutingCandidateSnapshot["feature_snapshot"],
-        })),
-      };
-    }
-    if (event.event_type === "routing_step_executed") {
-      return {
-        ...event,
-        failure_reason: event.failure_reason ? classifyFailure(event.failure_reason) : undefined,
-      };
-    }
-    return event;
-  });
-}
-
-export function deriveRoutingStepArtifacts(args: {
-  result: unknown;
-  skill?: SkillManifest;
-  selectedEndpointId?: string;
-  source: "marketplace" | "live-capture" | "dom-fallback" | "route-cache" | "browser-action" | "first-pass";
-  bindingsBefore?: StateBindings;
-  bindingsAfter?: StateBindings;
-}): {
-  source: RoutingTelemetrySource;
-  stateHashBefore: string;
-  stateHashAfter: string;
-  responseHash?: string;
-  schemaFingerprint?: string;
-  selectedOperationId?: string;
-} {
-  return {
-    source: routingSourceFromOrchestratorSource(args.source),
-    stateHashBefore: hashRoutingState(args.bindingsBefore),
-    stateHashAfter: hashRoutingState(args.bindingsAfter ?? args.bindingsBefore),
-    responseHash: args.result != null ? hashResponseBody(args.result) : undefined,
-    schemaFingerprint: hashSchemaFingerprint(
-      args.skill?.endpoints.find((endpoint) => endpoint.endpoint_id === args.selectedEndpointId)?.response_schema,
-    ),
-    selectedOperationId: operationIdForEndpoint(args.skill, args.selectedEndpointId),
-  };
-}

package/runtime-src/runtime/browser-access.ts

@@ -1,11 +0,0 @@
-export interface BrowserAccessConfig {
-  default_path: "unbrowse" | "direct" | "proxy";
-  fallback_path: "direct" | "proxy";
-  supported_frameworks: string[];
-}
-
-export const DEFAULT_BROWSER_ACCESS: BrowserAccessConfig = {
-  default_path: "unbrowse",
-  fallback_path: "direct",
-  supported_frameworks: ["openclaw", "mcp", "langchain", "hermes", "elizaos"],
-};

package/runtime-src/runtime/browser-auth.ts

@@ -1,12 +0,0 @@
-function isFalseyEnv(value: string | undefined): boolean {
-  if (!value) return false;
-  const normalized = value.trim().toLowerCase();
-  return normalized === "0" || normalized === "false" || normalized === "no" || normalized === "off";
-}
-
-/**
- * Default on for auth-gated browsing; explicit opt-out for clean-room E2E runs.
- */
-export function shouldImportBrowserCookies(): boolean {
-  return !isFalseyEnv(process.env.UNBROWSE_IMPORT_BROWSER_COOKIES);
-}

package/runtime-src/runtime/browser-host.ts

@@ -1,48 +0,0 @@
-export type HostEnvironment = "openclaw" | "openai" | "native" | "mcp" | "unknown";
-
-export interface BrowserPathConfig {
-  binary_path?: string;
-  cdp_port?: number;
-  headless: boolean;
-  user_data_dir?: string;
-}
-
-/** Detect the host environment from environment variables */
-export function detectHostEnvironment(): HostEnvironment {
-  if (process.env.OPENCLAW_RUNTIME) return "openclaw";
-  if (process.env.OPENAI_TOOL_RUNTIME) return "openai";
-  if (process.env.MCP_SERVER_MODE) return "mcp";
-  if (process.env.UNBROWSE_NATIVE) return "native";
-  return "unknown";
-}
-
-/** Get browser configuration appropriate for the detected host */
-export function getBrowserConfig(env?: HostEnvironment): BrowserPathConfig {
-  const detected = env ?? detectHostEnvironment();
-  switch (detected) {
-    case "openclaw":
-      return {
-        binary_path: process.env.OPENCLAW_BROWSER_PATH ?? "/usr/bin/chromium",
-        headless: true,
-        user_data_dir: process.env.OPENCLAW_USER_DATA ?? "/tmp/openclaw-chrome",
-      };
-    case "openai":
-      return {
-        binary_path: process.env.OPENAI_BROWSER_PATH,
-        headless: true,
-        cdp_port: parseInt(process.env.OPENAI_CDP_PORT ?? "9222"),
-      };
-    case "mcp":
-      return {
-        headless: true,
-        cdp_port: parseInt(process.env.CDP_PORT ?? "0"),
-      };
-    case "native":
-      return {
-        headless: false,
-        user_data_dir: process.env.UNBROWSE_USER_DATA,
-      };
-    default:
-      return { headless: false };
-  }
-}

package/runtime-src/runtime/lifecycle.ts

@@ -1,17 +0,0 @@
-export type LifecyclePhase = "discover" | "capture" | "resolve" | "execute" | "publish";
-
-export interface LifecycleEvent {
-  phase: LifecyclePhase;
-  skill_id: string;
-  timestamp: string;
-  duration_ms: number;
-  source: "cache" | "marketplace" | "live-capture";
-}
-
-export function attributeLifecycle(events: LifecycleEvent[]): Map<LifecyclePhase, number> {
-  const totals = new Map<LifecyclePhase, number>();
-  for (const e of events) {
-    totals.set(e.phase, (totals.get(e.phase) ?? 0) + e.duration_ms);
-  }
-  return totals;
-}