unbrowse 3.1.0 → 3.2.0

package/runtime-src/payments/lobster-pay.ts

@@ -1,182 +0,0 @@
-/**
- * Lobster.cash x402 payment bridge.
- *
- * Handles the pay-and-retry cycle:
- *   1. Receives x402 payment terms from a 402 response
- *   2. Calls `lobstercash x402 fetch <url>` to pay + retry
- *   3. Returns the paid response body
- *
- * Delegation boundary:
- * - Unbrowse owns: detecting 402, passing the URL, using the result
- * - Lobster owns: wallet signing, transaction broadcast, proof construction
- */
-
-import { execFile, execFileSync } from "node:child_process";
-import { existsSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-
-const LOBSTER_PAY_TIMEOUT_MS = 30_000;
-
-/**
- * Resolve the lobster CLI command. Prefers the direct binary if
- * installed globally, otherwise falls back to npx.
- */
-function getLobsterCommand(): { cmd: string; prefix: string[] } | null {
-  // 1. Direct binary in PATH
-  try {
-    execFileSync("lobstercash", ["--version"], { stdio: "ignore", timeout: 3_000 });
-    return { cmd: "lobstercash", prefix: [] };
-  } catch (_e) { /* not in PATH */ }
-
-  // 2. Check npm global bin (often not in agent PATH)
-  try {
-    const npmPrefix = execFileSync("npm", ["config", "get", "prefix"], { encoding: "utf8", timeout: 5_000 }).trim();
-    const lobsterPath = join(npmPrefix, "bin", "lobstercash");
-    if (existsSync(lobsterPath)) {
-      execFileSync(lobsterPath, ["--version"], { stdio: "ignore", timeout: 3_000 });
-      return { cmd: lobsterPath, prefix: [] };
-    }
-  } catch (_e) { /* npm not available */ }
-
-  return null;
-}
-
-let cachedCommand: { cmd: string; prefix: string[] } | null | undefined = undefined;
-function lobsterCmd(): { cmd: string; prefix: string[] } | null {
-  if (cachedCommand === undefined) cachedCommand = getLobsterCommand();
-  return cachedCommand;
-}
-
-export interface LobsterPayResult {
-  success: boolean;
-  body: string;
-  statusCode?: number;
-  error?: string;
-}
-
-/**
- * Check if the lobster CLI is available and wallet is configured.
- */
-export function isLobsterAvailable(): boolean {
-  const agentsPath = join(process.env.HOME || homedir(), ".lobster", "agents.json");
-  return existsSync(agentsPath);
-}
-
-/**
- * Pay for an x402-gated URL via lobster.cash and return the response.
- *
- * Shells out to `lobstercash x402 fetch <url>` which handles:
- * - Reading 402 + payment terms
- * - Signing + broadcasting the USDC transfer
- * - Retrying with PAYMENT-SIGNATURE header
- * - Returning the paid response
- */
-export function lobsterX402Fetch(
-  url: string,
-  options?: {
-    jsonBody?: string;
-    headers?: Record<string, string>;
-    timeoutMs?: number;
-  },
-): Promise<LobsterPayResult> {
-  return new Promise((resolve) => {
-    const resolved = lobsterCmd();
-    if (!resolved) {
-      resolve({ success: false, body: "", error: "lobstercash CLI not in PATH" });
-      return;
-    }
-    const { cmd, prefix } = resolved;
-    const args = [...prefix, "x402", "fetch", url, "--debug"];
-
-    if (options?.jsonBody) {
-      args.push("--json", options.jsonBody);
-    }
-
-    if (options?.headers) {
-      for (const [key, value] of Object.entries(options.headers)) {
-        args.push("--header", `${key}:${value}`);
-      }
-    }
-
-    const timeout = options?.timeoutMs ?? LOBSTER_PAY_TIMEOUT_MS;
-    args.push("--timeout", String(timeout));
-
-    execFile(cmd, args, { timeout: timeout + 5_000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
-      if (err) {
-        const msg = stderr?.trim() || err.message;
-        console.warn(`[lobster-pay] x402 fetch failed: ${msg}`);
-        resolve({ success: false, body: "", error: msg });
-        return;
-      }
-
-      if (stderr) {
-        for (const line of stderr.split("\n").filter(Boolean)) {
-          console.log(`[lobster-pay] ${line}`);
-        }
-      }
-
-      // Parse status from stdout header line: "Status: 200"
-      const statusMatch = stdout.match(/^Status:\s*(\d+)/m);
-      const statusCode = statusMatch ? parseInt(statusMatch[1], 10) : undefined;
-
-      if (statusCode && statusCode >= 400) {
-        resolve({ success: false, body: stdout, statusCode, error: `HTTP ${statusCode}` });
-        return;
-      }
-
-      resolve({ success: true, body: stdout, statusCode });
-    });
-  });
-}
-
-/**
- * Pay for an x402-gated API request and return the parsed JSON response.
- *
- * This is the high-level entry point for the pay-and-retry loop.
- * Returns null if payment fails or lobster is unavailable.
- */
-export async function payAndRetry<T = unknown>(
-  fullUrl: string,
-  options?: {
-    body?: unknown;
-    headers?: Record<string, string>;
-  },
-): Promise<{ data: T; paid: true } | null> {
-  if (!isLobsterAvailable()) {
-    console.log("[lobster-pay] lobster.cash not configured — skipping payment");
-    return null;
-  }
-
-  console.log(`[lobster-pay] attempting x402 payment for ${fullUrl}`);
-
-  const result = await lobsterX402Fetch(fullUrl, {
-    jsonBody: options?.body ? JSON.stringify(options.body) : undefined,
-    headers: options?.headers,
-  });
-
-  if (!result.success) {
-    console.warn(`[lobster-pay] payment failed: ${result.error}`);
-    return null;
-  }
-  try {
-    // lobstercash x402 fetch outputs header lines before the JSON body:
-    //   x402 FETCH <url>
-    //   Status: 200
-    //   Content-Type: application/json
-    //   <blank line>
-    //   {"actual":"json",...}
-    // Extract the JSON portion by finding the first '{' or '['
-    const raw = result.body;
-    const jsonStart = Math.min(
-      ...[raw.indexOf("{"), raw.indexOf("[")].filter((i) => i >= 0),
-    );
-    const jsonStr = jsonStart >= 0 ? raw.slice(jsonStart) : raw;
-    const data = JSON.parse(jsonStr) as T;
-    console.log("[lobster-pay] payment successful — got paid response");
-    return { data, paid: true };
-  } catch (_e) {
-    console.warn("[lobster-pay] paid response was not valid JSON");
-    return null;
-  }
-}

package/runtime-src/payments/wallet.ts

@@ -1,98 +0,0 @@
-import { existsSync, readFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-
-/**
- * Wallet precheck — lobster.cash compatible.
- *
- * This module only checks whether the agent has a wallet configured.
- * It does NOT generate wallets, manage keys, or call wallet APIs.
- * Wallet provisioning and transaction execution are owned by
- * the agent's wallet plugin (e.g. lobster.cash).
- */
-
-export type WalletCheckResult = {
-  configured: boolean;
-  provider?: string;
-};
-
-type WalletContext = {
-  wallet_address?: string;
-  wallet_provider?: string;
-};
-
-type LobsterAgentRecord = {
-  walletAddress?: unknown;
-  wallet_address?: unknown;
-  authorizedWallets?: {
-    solana?: unknown;
-    [key: string]: unknown;
-  };
-};
-
-type LobsterAgentsFile = {
-  activeAgentId?: unknown;
-  agents?: Record<string, LobsterAgentRecord> | LobsterAgentRecord[];
-};
-
-function asNonEmptyString(value: unknown): string | undefined {
-  return typeof value === "string" && value.trim() ? value.trim() : undefined;
-}
-
-function getLobsterWalletFromLocalConfig(): string | undefined {
-  const agentsPath = join(process.env.HOME || homedir(), ".lobster", "agents.json");
-  if (!existsSync(agentsPath)) return undefined;
-
-  try {
-    const raw = JSON.parse(readFileSync(agentsPath, "utf8")) as LobsterAgentsFile;
-    const activeAgentId = asNonEmptyString(raw.activeAgentId);
-    const activeAgent = Array.isArray(raw.agents)
-      ? raw.agents.find((agent) => asNonEmptyString((agent as { id?: unknown }).id) === activeAgentId)
-      : activeAgentId
-        ? raw.agents?.[activeAgentId]
-        : undefined;
-
-    return asNonEmptyString(activeAgent?.authorizedWallets?.solana)
-      ?? asNonEmptyString(activeAgent?.walletAddress)
-      ?? asNonEmptyString(activeAgent?.wallet_address);
-  } catch {
-    return undefined;
-  }
-}
-
-export function getWalletContext(): WalletContext {
-  const lobsterWallet = asNonEmptyString(process.env.LOBSTER_WALLET_ADDRESS);
-  if (lobsterWallet) {
-    return { wallet_address: lobsterWallet, wallet_provider: "lobster.cash" };
-  }
-
-  const genericWallet = asNonEmptyString(process.env.AGENT_WALLET_ADDRESS);
-  if (genericWallet) {
-    return {
-      wallet_address: genericWallet,
-      wallet_provider: asNonEmptyString(process.env.AGENT_WALLET_PROVIDER),
-    };
-  }
-
-  const localLobsterWallet = getLobsterWalletFromLocalConfig();
-  if (localLobsterWallet) {
-    return { wallet_address: localLobsterWallet, wallet_provider: "lobster.cash" };
-  }
-
-  return {};
-}
-
-/**
- * Check if the agent has a wallet configured.
- *
- * Looks for wallet context signals that a wallet plugin would set.
- * Does NOT create or modify wallet state.
- */
-export function checkWalletConfigured(): WalletCheckResult {
-  const wallet = getWalletContext();
-  if (!wallet.wallet_address) return { configured: false };
-  return {
-    configured: true,
-    provider: wallet.wallet_provider ?? "unknown",
-  };
-}

package/runtime-src/publish/review-context.ts

@@ -1,93 +0,0 @@
-import { buildSkillOperationGraph } from "../graph/index.js";
-import type { SkillManifest } from "../types/index.js";
-import { readWorkflowArtifact } from "../workflow/artifact.js";
-import { buildSafeParameterSpecView, flattenResponseSchemaFields } from "./schema-review.js";
-
-function operationTitle(actionKind: string, resourceKind: string): string {
-  return `${actionKind.replace(/_/g, " ")} ${resourceKind.replace(/_/g, " ")}`.trim();
-}
-
-export function buildEndpointReviewContext(skill: SkillManifest, endpointId: string): Record<string, unknown> | null {
-  const graph = skill.operation_graph ?? buildSkillOperationGraph(skill.endpoints);
-  const operation = graph.operations.find((candidate) => candidate.endpoint_id === endpointId);
-  const endpoint = skill.endpoints.find((candidate) => candidate.endpoint_id === endpointId);
-  if (!operation || !endpoint) return null;
-  const workflowArtifact = readWorkflowArtifact(skill.skill_id);
-  const recipe = workflowArtifact?.recipes.find((candidate) => candidate.endpoint_id === endpointId) ?? null;
-
-  const dependencies = graph.edges
-    .filter((edge) => edge.to_operation_id === operation.operation_id)
-    .map((edge) => {
-      const source = graph.operations.find((candidate) => candidate.operation_id === edge.from_operation_id);
-      if (!source) return null;
-      return {
-        endpoint_id: source.endpoint_id,
-        operation_id: source.operation_id,
-        title: operationTitle(source.action_kind, source.resource_kind),
-        binding_key: edge.binding_key,
-        kind: edge.kind,
-        confidence: edge.confidence,
-        description_out: source.description_out,
-      };
-    })
-    .filter((entry): entry is NonNullable<typeof entry> => !!entry);
-
-  const unlocks = graph.edges
-    .filter((edge) => edge.from_operation_id === operation.operation_id)
-    .map((edge) => {
-      const target = graph.operations.find((candidate) => candidate.operation_id === edge.to_operation_id);
-      if (!target) return null;
-      return {
-        endpoint_id: target.endpoint_id,
-        operation_id: target.operation_id,
-        title: operationTitle(target.action_kind, target.resource_kind),
-        binding_key: edge.binding_key,
-        kind: edge.kind,
-        confidence: edge.confidence,
-        description_out: target.description_out,
-      };
-    })
-    .filter((entry): entry is NonNullable<typeof entry> => !!entry);
-
-  const triggerSiblings = skill.endpoints
-    .filter((candidate) => candidate.endpoint_id !== endpointId && candidate.trigger_url && candidate.trigger_url === endpoint.trigger_url)
-    .slice(0, 5)
-    .map((candidate) => ({
-      endpoint_id: candidate.endpoint_id,
-      method: candidate.method,
-      description: candidate.description ?? candidate.semantic?.description_out ?? "",
-      dom_extraction: !!candidate.dom_extraction,
-      trigger_url: candidate.trigger_url,
-    }));
-
-  return {
-    operation_id: operation.operation_id,
-    title: operationTitle(operation.action_kind, operation.resource_kind),
-    action_kind: operation.action_kind,
-    resource_kind: operation.resource_kind,
-    description_in: operation.description_in,
-    response_summary: operation.response_summary,
-    requires: operation.requires ?? [],
-    provides: operation.provides ?? [],
-    negative_tags: operation.negative_tags ?? [],
-    provenance: endpoint.dom_extraction
-      ? "dom_extraction"
-      : endpoint.method === "WS"
-        ? "websocket"
-        : "http_replay",
-    trigger_url: endpoint.trigger_url ?? null,
-    verification_status: endpoint.verification_status,
-    reliability_score: endpoint.reliability_score,
-    request_schema: recipe
-      ? recipe.replay_contract.parameter_specs.map((spec) => buildSafeParameterSpecView(spec))
-      : [],
-    response_schema: flattenResponseSchemaFields(endpoint.response_schema),
-    prerequisites: recipe?.replay_contract.prerequisite_specs ?? [],
-    token_bindings: recipe?.token_bindings ?? [],
-    next_state: recipe?.replay_contract.next_state ?? [],
-    path_binding_candidates: endpoint._path_binding_candidates ?? [],
-    dependencies,
-    unlocks,
-    trigger_siblings: triggerSiblings,
-  };
-}

package/runtime-src/publish/sanitize.ts

@@ -1,197 +0,0 @@
-import type { EndpointDescriptor } from "../types/index.js";
-
-const SECRET_VALUE_PATTERNS = [
-  /^eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/,
-  /^Bearer\s+\S+/i,
-  /^Basic\s+[A-Za-z0-9+/=]+/i,
-  /^ghp_[A-Za-z0-9]{36}/,
-  /^sk-[A-Za-z0-9]{20,}/,
-  /^pk_(live|test)_[A-Za-z0-9]+/,
-  /^xox[bsrp]-[A-Za-z0-9-]+/,
-  /^AKIA[A-Z0-9]{16}/,
-  /^[A-Za-z0-9+/]{40,}={0,2}$/,
-  /^v2\.[A-Za-z0-9_-]{20,}/,
-];
-
-const SECRET_KEY_PATTERNS = /^(api[_-]?key|access[_-]?token|auth[_-]?token|secret[_-]?key|private[_-]?key|password|passwd|session[_-]?id|session[_-]?token|csrf[_-]?token|client[_-]?secret|bearer|refresh[_-]?token|id[_-]?token|jwt|nonce|otp|pin|ssn|credit[_-]?card)$/i;
-const EMAIL_PATTERN = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;
-const PHONE_PATTERN = /(?<!\w)(?:\+?\d[\d\s-]{7,}\d)(?!\w)/g;
-const UUID_PATTERN = /\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b/gi;
-const URL_PATTERN = /\bhttps?:\/\/[^\s"'<>]+/gi;
-const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9._-]+\.[A-Za-z0-9._-]+\b/g;
-const CREDIT_CARD_PATTERN = /\b(?:\d[ -]*?){13,19}\b/g;
-const LONG_ID_PATTERN = /\b[A-Za-z0-9_-]{20,}\b/g;
-
-export function looksLikeSecret(key: string, value: unknown): boolean {
-  if (typeof value !== "string" || value.length < 8) return false;
-  if (SECRET_KEY_PATTERNS.test(key)) return true;
-  return SECRET_VALUE_PATTERNS.some((pattern) => pattern.test(value));
-}
-
-export function sanitizeAgentVisibleText(text: string): string {
-  return text
-    .replace(JWT_PATTERN, "[secret-token]")
-    .replace(EMAIL_PATTERN, "user@example.com")
-    .replace(PHONE_PATTERN, "[phone]")
-    .replace(UUID_PATTERN, "00000000-0000-0000-0000-000000000000")
-    .replace(CREDIT_CARD_PATTERN, "[sensitive-number]")
-    .replace(URL_PATTERN, (value) => {
-      if (/^https?:\/\/example\.com/i.test(value)) return value;
-      return "https://example.com/resource";
-    })
-    .replace(LONG_ID_PATTERN, (value) => {
-      if (looksLikeSecret("", value)) return "[secret-token]";
-      return value;
-    });
-}
-
-export function sanitizeAgentVisibleValue(key: string, value: unknown): unknown {
-  if (value == null) return value;
-  if (typeof value === "string") {
-    return synthesizePlaceholder(key, sanitizeAgentVisibleText(value));
-  }
-  if (Array.isArray(value)) return value.slice(0, 3).map((entry) => sanitizeAgentVisibleValue(key, entry));
-  if (typeof value === "object") {
-    const out: Record<string, unknown> = {};
-    for (const [childKey, childValue] of Object.entries(value as Record<string, unknown>)) {
-      out[childKey] = sanitizeAgentVisibleValue(childKey, childValue);
-    }
-    return out;
-  }
-  return synthesizePlaceholder(key, value);
-}
-
-export function redactSecrets(obj: unknown, parentKey = ""): unknown {
-  if (obj === null || obj === undefined) return obj;
-  if (typeof obj === "string") {
-    return looksLikeSecret(parentKey, obj) ? "[REDACTED]" : obj;
-  }
-  if (Array.isArray(obj)) {
-    return obj.map((item) => redactSecrets(item, parentKey));
-  }
-  if (typeof obj === "object") {
-    const result: Record<string, unknown> = {};
-    for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
-      result[key] = redactSecrets(value, key);
-    }
-    return result;
-  }
-  return obj;
-}
-
-function synthesizePlaceholder(key: string, value: unknown): unknown {
-  if (value === null || value === undefined) return value;
-  if (typeof value === "number") return Number.isInteger(value) ? 12345 : 99.99;
-  if (typeof value === "boolean") return value;
-  if (typeof value === "string") {
-    if (looksLikeSecret(key, value)) return "[REDACTED]";
-    if (/@/.test(value)) return "user@example.com";
-    if (/^https?:\/\//.test(value)) return "https://example.com/item/123";
-    if (/^[0-9a-f]{8}-[0-9a-f]{4}/.test(value)) return "a1b2c3d4-e5f6-7890-abcd-ef1234567890";
-    if (/^\d+$/.test(value)) return "12345";
-    if (/^\d{4}-\d{2}-\d{2}/.test(value)) return "2026-01-15T00:00:00Z";
-    if (value.length <= 8) return "abc123";
-    if (value.length > 100) return "Example description text for this item.";
-    return "example-value";
-  }
-  if (Array.isArray(value)) {
-    return value.length > 0 ? [synthesizeExample(value[0], 0)] : [];
-  }
-  if (typeof value === "object") {
-    return synthesizeExample(value, 0);
-  }
-  return value;
-}
-
-export function synthesizeExample(obj: unknown, depth = 0): unknown {
-  if (depth > 5) return null;
-  if (obj === null || obj === undefined) return obj;
-  if (typeof obj !== "object") return synthesizePlaceholder("", obj);
-  if (Array.isArray(obj)) {
-    return obj.slice(0, 2).map((item) => synthesizeExample(item, depth + 1));
-  }
-  const result: Record<string, unknown> = {};
-  for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
-    result[key] = typeof value === "object" && value !== null
-      ? synthesizeExample(value, depth + 1)
-      : synthesizePlaceholder(key, value);
-  }
-  return result;
-}
-
-export function sanitizeForPublish(endpoints: EndpointDescriptor[]): EndpointDescriptor[] {
-  return endpoints.map((endpoint) => {
-    const clean = { ...endpoint };
-
-    if (clean.headers_template) {
-      clean.headers_template = redactSecrets(clean.headers_template) as Record<string, string>;
-    }
-    if (clean.query) {
-      clean.query = redactSecrets(clean.query) as Record<string, unknown>;
-    }
-    if (clean.body) {
-      clean.body = redactSecrets(clean.body) as Record<string, unknown>;
-    }
-    if (clean.query) {
-      clean.query = Object.fromEntries(
-        Object.entries(clean.query).map(([key, value]) => [key, typeof value === "string" ? "example" : value]),
-      );
-    }
-    if (clean.path_params) {
-      clean.path_params = Object.fromEntries(
-        Object.keys(clean.path_params).map((key) => [key, "example"]),
-      );
-    }
-    if (clean.body) clean.body = synthesizeExample(clean.body) as Record<string, unknown>;
-    if (clean.body_params) clean.body_params = synthesizeExample(clean.body_params) as Record<string, unknown>;
-
-    if (clean.headers_template) {
-      clean.headers_template = Object.fromEntries(
-        Object.keys(clean.headers_template).map((key) => [key, ""]),
-      );
-    }
-
-    if (clean.trigger_url) {
-      try {
-        const parsed = new URL(clean.trigger_url);
-        clean.trigger_url = parsed.origin + parsed.pathname;
-      } catch {
-        /* keep original */
-      }
-    }
-
-    if (clean.semantic) {
-      const semantic = { ...clean.semantic };
-      if (semantic.example_response_compact) {
-        semantic.example_response_compact = synthesizeExample(semantic.example_response_compact);
-      }
-      if (semantic.example_request) {
-        semantic.example_request = synthesizeExample(semantic.example_request);
-      }
-      if (semantic.sample_request_url) {
-        try {
-          const parsed = new URL(semantic.sample_request_url);
-          for (const key of parsed.searchParams.keys()) parsed.searchParams.set(key, "example");
-          semantic.sample_request_url = parsed.toString();
-        } catch {
-          delete semantic.sample_request_url;
-        }
-      }
-      if (semantic.requires) {
-        semantic.requires = semantic.requires.map((binding) => {
-          const { example_value: _exampleValue, ...rest } = binding;
-          return rest;
-        });
-      }
-      if (semantic.provides) {
-        semantic.provides = semantic.provides.map((binding) => {
-          const { example_value: _exampleValue, ...rest } = binding;
-          return rest;
-        });
-      }
-      clean.semantic = semantic;
-    }
-
-    return clean;
-  });
-}