unbrowse 3.1.0 → 3.2.0

package/runtime-src/publish/schema-review.ts

@@ -1,192 +0,0 @@
-import { sanitizeAgentVisibleValue } from "./sanitize.js";
-import type {
-  EndpointParameterReview,
-  EndpointResponseFieldReview,
-  EndpointReviewPayload,
-  OperationBinding,
-  ResponseSchema,
-  WorkflowArtifact,
-  WorkflowParameterConstraint,
-  WorkflowParameterSpec,
-} from "../types/index.js";
-
-function tokenizeSchemaPath(path: string): string[] {
-  const tokens: string[] = [];
-  for (const segment of path.split(".").filter(Boolean)) {
-    const matches = segment.match(/([^[\]]+)|\[\]/g);
-    if (matches) tokens.push(...matches);
-  }
-  return tokens;
-}
-
-function getSchemaNode(schema: ResponseSchema | undefined, path: string): ResponseSchema | null {
-  if (!schema) return null;
-  const tokens = tokenizeSchemaPath(path);
-  let node: ResponseSchema | undefined = schema;
-  for (const token of tokens) {
-    if (!node) return null;
-    node = token === "[]"
-      ? node.items
-      : node.properties?.[token];
-  }
-  return node ?? null;
-}
-
-export function flattenResponseSchemaFields(
-  schema?: ResponseSchema,
-  prefix = "",
-  required = false,
-): Array<{
-  path: string;
-  type: string;
-  required: boolean;
-  description?: string;
-  enum_values?: Array<string | number | boolean>;
-}> {
-  if (!schema) return [];
-  const rows: Array<{
-    path: string;
-    type: string;
-    required: boolean;
-    description?: string;
-    enum_values?: Array<string | number | boolean>;
-  }> = [];
-
-  if (prefix) {
-    rows.push({
-      path: prefix,
-      type: schema.type,
-      required,
-      ...(schema.description ? { description: schema.description } : {}),
-      ...(schema.enum_values?.length ? { enum_values: schema.enum_values } : {}),
-    });
-  }
-
-  for (const [key, child] of Object.entries(schema.properties ?? {})) {
-    const childPath = prefix ? `${prefix}.${key}` : key;
-    rows.push(...flattenResponseSchemaFields(child, childPath, (schema.required ?? []).includes(key)));
-  }
-  if (schema.items) {
-    const childPath = prefix ? `${prefix}[]` : "[]";
-    rows.push(...flattenResponseSchemaFields(schema.items, childPath, true));
-  }
-  return rows;
-}
-
-export function applyResponseSchemaReviews(
-  schema: ResponseSchema | undefined,
-  reviews: EndpointResponseFieldReview[] | undefined,
-): ResponseSchema | undefined {
-  if (!schema || !reviews?.length) return schema;
-  const next = structuredClone(schema);
-  for (const review of reviews) {
-    const node = getSchemaNode(next, review.path);
-    if (!node) continue;
-    if (review.description !== undefined) node.description = review.description;
-    if (review.type !== undefined) node.type = review.type;
-    if (review.enum_values !== undefined) node.enum_values = review.enum_values;
-  }
-  return next;
-}
-
-function mergeConstraint(
-  constraints: WorkflowParameterConstraint[] | undefined,
-  nextConstraint: WorkflowParameterConstraint,
-): WorkflowParameterConstraint[] {
-  const out = [...(constraints ?? []).filter((entry) => entry.kind !== nextConstraint.kind)];
-  out.push(nextConstraint);
-  return out;
-}
-
-export function applyParameterReviews(
-  specs: WorkflowParameterSpec[],
-  reviews: EndpointParameterReview[] | undefined,
-): WorkflowParameterSpec[] {
-  if (!reviews?.length) return specs;
-  const reviewMap = new Map(reviews.map((review) => [`${review.location}:${review.name}`, review]));
-  return specs.map((spec) => {
-    const review = reviewMap.get(`${spec.location}:${spec.name}`);
-    if (!review) return spec;
-    let constraints = spec.constraints;
-    if (review.format !== undefined) {
-      constraints = mergeConstraint(constraints, {
-        kind: "format",
-        value: review.format,
-      });
-    }
-    if (review.enum_values !== undefined && review.enum_values.length > 1) {
-      constraints = mergeConstraint(constraints, {
-        kind: "enum",
-        description: `Reviewed allowed values for ${spec.name}.`,
-      });
-    }
-    return {
-      ...spec,
-      ...(review.description !== undefined ? { description: review.description } : {}),
-      ...(review.type !== undefined ? { type: review.type } : {}),
-      ...(review.required !== undefined ? { required: review.required } : {}),
-      ...(review.user_supplied !== undefined ? { user_supplied: review.user_supplied } : {}),
-      ...(review.format !== undefined ? { format: review.format } : {}),
-      ...(review.enum_values !== undefined ? { enum_values: review.enum_values } : {}),
-      ...(review.derived_from !== undefined ? { derived_from: review.derived_from } : {}),
-      ...(constraints?.length ? { constraints } : {}),
-    };
-  });
-}
-
-export function applyBindingReviews(
-  bindings: OperationBinding[] | undefined,
-  reviews: EndpointParameterReview[] | undefined,
-): OperationBinding[] | undefined {
-  if (!bindings || !reviews?.length) return bindings;
-  const reviewMap = new Map(reviews.map((review) => [review.name, review]));
-  return bindings.map((binding) => {
-    const review = reviewMap.get(binding.key);
-    if (!review) return binding;
-    return {
-      ...binding,
-      ...(review.description !== undefined ? { description: review.description } : {}),
-      ...(review.type !== undefined ? { type: review.type } : {}),
-      ...(review.required !== undefined ? { required: review.required } : {}),
-    };
-  });
-}
-
-export function applyWorkflowSchemaReviews(
-  artifact: WorkflowArtifact | null,
-  reviews: EndpointReviewPayload[],
-): WorkflowArtifact | null {
-  if (!artifact) return artifact;
-  const reviewMap = new Map(reviews.map((review) => [review.endpoint_id, review]));
-  return {
-    ...artifact,
-    recipes: artifact.recipes.map((recipe) => {
-      const review = reviewMap.get(recipe.endpoint_id);
-      if (!review?.parameter_reviews?.length) return recipe;
-      return {
-        ...recipe,
-        replay_contract: {
-          ...recipe.replay_contract,
-          parameter_specs: applyParameterReviews(recipe.replay_contract.parameter_specs, review.parameter_reviews),
-        },
-      };
-    }),
-  };
-}
-
-export function buildSafeParameterSpecView(spec: WorkflowParameterSpec): Record<string, unknown> {
-  return {
-    name: spec.name,
-    location: spec.location,
-    type: spec.type,
-    required: spec.required,
-    user_supplied: spec.user_supplied,
-    ...(spec.description ? { description: spec.description } : {}),
-    ...(spec.format ? { format: spec.format } : {}),
-    ...(spec.enum_values?.length ? { enum_values: spec.enum_values } : {}),
-    ...(spec.derived_from?.length ? { derived_from: spec.derived_from } : {}),
-    ...(spec.default_value !== undefined ? { default_value: sanitizeAgentVisibleValue(spec.name, spec.default_value) } : {}),
-    ...(spec.example_value !== undefined ? { example_value: sanitizeAgentVisibleValue(spec.name, spec.example_value) } : {}),
-    source_hints: spec.source_hints,
-  };
-}

package/runtime-src/publish-admission.ts

@@ -1,388 +0,0 @@
-import { getRegistrableDomain } from "./domain.js";
-import type { EndpointDescriptor, SkillManifest } from "./types/index.js";
-
-export type PublishAdmissionReason =
-  | "ws"
-  | "non_actionable_method"
-  | "verification_failed"
-  | "low_reliability"
-  | "off_domain"
-  | "noise"
-  | "fragile_graphql"
-  | "no_durable_signal"
-  | "family_dedup"
-  | "over_limit";
-
-export type MarketplacePublishSelection = {
-  endpoints: EndpointDescriptor[];
-  stats: {
-    total: number;
-    kept: number;
-    by_reason: Record<PublishAdmissionReason, number>;
-  };
-};
-
-export type MarketplacePublishClosure = MarketplacePublishSelection & {
-  root_endpoints: EndpointDescriptor[];
-  root_endpoint_ids: string[];
-  closure_endpoint_ids: string[];
-  closure_operation_ids: string[];
-  closure_edge_count: number;
-};
-
-type PublishSelectionOptions = {
-  limit?: number;
-};
-
-type ScoredCandidate = {
-  endpoint: EndpointDescriptor;
-  familyKey: string;
-  score: number;
-};
-
-const DEFAULT_PUBLISH_ENDPOINT_LIMIT = readPositiveIntEnv("UNBROWSE_PUBLISH_ENDPOINT_LIMIT", 12);
-const MIN_PUBLISH_RELIABILITY = readProbabilityEnv("UNBROWSE_PUBLISH_MIN_RELIABILITY", 0.2);
-const CLOSURE_EDGE_KINDS = new Set(["dependency", "auth", "parent_child", "pagination"]);
-
-function readPositiveIntEnv(name: string, fallback: number): number {
-  const parsed = Number.parseInt(process.env[name] ?? "", 10);
-  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
-}
-
-function readProbabilityEnv(name: string, fallback: number): number {
-  const parsed = Number.parseFloat(process.env[name] ?? "");
-  return Number.isFinite(parsed) && parsed >= 0 && parsed <= 1 ? parsed : fallback;
-}
-
-function freshReasonCounts(): Record<PublishAdmissionReason, number> {
-  return {
-    ws: 0,
-    non_actionable_method: 0,
-    verification_failed: 0,
-    low_reliability: 0,
-    off_domain: 0,
-    noise: 0,
-    fragile_graphql: 0,
-    no_durable_signal: 0,
-    family_dedup: 0,
-    over_limit: 0,
-  };
-}
-
-function isApiLikePath(pathname: string): boolean {
-  return /\/api\/|graphql|\/rest\/|\/rpc\/|voyager/i.test(pathname);
-}
-
-function looksOpaqueIdentifier(value: string): boolean {
-  if (!value || /^\{[^}]+\}$/.test(value)) return false;
-  if (/^[0-9a-f]{12,}$/i.test(value)) return true;
-  if (value.length < 16) return false;
-  const hasAlpha = /[A-Za-z]/.test(value);
-  const hasDigit = /\d/.test(value);
-  return hasAlpha && hasDigit;
-}
-
-function isCanonicalDocumentReplay(endpoint: EndpointDescriptor): boolean {
-  if (/captured page artifact/i.test(endpoint.description ?? "")) return true;
-  if (!endpoint.trigger_url) return false;
-  try {
-    const endpointUrl = new URL(endpoint.url_template);
-    const triggerUrl = new URL(endpoint.trigger_url);
-    return (
-      endpointUrl.origin === triggerUrl.origin &&
-      endpointUrl.pathname === triggerUrl.pathname &&
-      !isApiLikePath(endpointUrl.pathname)
-    );
-  } catch {
-    return false;
-  }
-}
-
-function hasRichSemanticEvidence(endpoint: EndpointDescriptor): boolean {
-  return !!(
-    endpoint.semantic?.example_response_compact ||
-    endpoint.semantic?.response_summary ||
-    (endpoint.semantic?.example_fields?.length ?? 0) > 0
-  );
-}
-
-function hasDurableSignal(endpoint: EndpointDescriptor): boolean {
-  return !!(
-    endpoint.response_schema ||
-    endpoint.dom_extraction ||
-    endpoint.search_form ||
-    hasRichSemanticEvidence(endpoint) ||
-    isCanonicalDocumentReplay(endpoint)
-  );
-}
-
-function isVerifiedDurable(endpoint: EndpointDescriptor): boolean {
-  return (
-    endpoint.verification_status === "verified" &&
-    endpoint.reliability_score >= 0.9 &&
-    !!endpoint.response_schema
-  );
-}
-
-function isSkillDomainEndpoint(skill: SkillManifest, endpoint: EndpointDescriptor): boolean {
-  try {
-    const endpointHost = new URL(endpoint.url_template).hostname;
-    const skillDomain = getRegistrableDomain(skill.domain);
-    const endpointDomain = getRegistrableDomain(endpointHost);
-    return endpointDomain === skillDomain;
-  } catch {
-    return false;
-  }
-}
-
-function isLikelyNoiseEndpoint(endpoint: EndpointDescriptor): boolean {
-  const haystack = [endpoint.url_template, endpoint.description ?? ""].join(" ").toLowerCase();
-  return /(recaptcha|captcha|csrf|telemetry|analytics|tracking|logging|metrics|beacon|consent|cookie[-_ ]?banner|feature[-_ ]?flag|oauth|session[-_/ ]?refresh|auth[-_/ ]?refresh|header-action|badge)/i.test(
-    haystack,
-  );
-}
-
-function isFragileGraphqlEndpoint(endpoint: EndpointDescriptor): boolean {
-  try {
-    const url = new URL(endpoint.url_template);
-    const pathSegments = url.pathname.split("/").filter(Boolean);
-    const isGraphLike = pathSegments.some((segment) => /graphql|voyager/i.test(segment));
-    if (!isGraphLike) return false;
-
-    const queryId = url.searchParams.get("queryId") ?? "";
-    if (queryId) {
-      const parts = queryId.split(/[.:]/g).filter(Boolean);
-      if (parts.some(looksOpaqueIdentifier)) return true;
-    }
-
-    const opaqueGraphSegment = pathSegments.some((segment, index) => {
-      if (!/graphql|voyager/i.test(pathSegments[index - 1] ?? "")) return false;
-      return looksOpaqueIdentifier(segment);
-    });
-    if (opaqueGraphSegment) return true;
-
-    return pathSegments.some((segment) => looksOpaqueIdentifier(segment) && /graphql|voyager/i.test(url.pathname));
-  } catch {
-    return /queryId=/i.test(endpoint.url_template) && /graphql|voyager/i.test(endpoint.url_template);
-  }
-}
-
-function normalizedPathname(pathname: string): string {
-  return pathname
-    .split("/")
-    .filter(Boolean)
-    .map((segment) => {
-      if (/^\{[^}]+\}$/.test(segment)) return "{param}";
-      if (looksOpaqueIdentifier(segment)) return ":opaque";
-      return segment.toLowerCase();
-    })
-    .join("/");
-}
-
-function endpointFamilyKey(endpoint: EndpointDescriptor): string {
-  try {
-    const url = new URL(endpoint.url_template);
-    const queryKeys = new Set<string>();
-    for (const key of url.searchParams.keys()) queryKeys.add(key);
-    for (const key of Object.keys(endpoint.query ?? {})) queryKeys.add(key);
-    for (const key of Object.keys(endpoint.body_params ?? {})) queryKeys.add(`body:${key}`);
-    return [
-      endpoint.method,
-      url.hostname.toLowerCase(),
-      normalizedPathname(url.pathname),
-      [...queryKeys].sort().join(","),
-      endpoint.dom_extraction ? "dom" : "http",
-    ].join("|");
-  } catch {
-    return `${endpoint.method}|${endpoint.url_template.toLowerCase()}`;
-  }
-}
-
-function scoreEndpoint(endpoint: EndpointDescriptor): number {
-  let score = endpoint.reliability_score * 100;
-  if (endpoint.verification_status === "verified") score += 30;
-  if (endpoint.response_schema) score += 25;
-  if (endpoint.dom_extraction) score += 8;
-  if (endpoint.search_form) score += 10;
-  if (endpoint.semantic?.action_kind) score += 8;
-  if (endpoint.semantic?.resource_kind) score += 4;
-  if (endpoint.method === "GET") score += 8;
-  if (endpoint.method === "POST") score += 3;
-  if (endpoint.trigger_url) score -= 6;
-  if (isCanonicalDocumentReplay(endpoint)) score += 18;
-  if (isFragileGraphqlEndpoint(endpoint)) score -= 25;
-  return score;
-}
-
-function rejectionReason(skill: SkillManifest, endpoint: EndpointDescriptor): PublishAdmissionReason | null {
-  if (endpoint.method === "WS") return "ws";
-  if (endpoint.method === "HEAD" || endpoint.method === "OPTIONS") return "non_actionable_method";
-  if (endpoint.verification_status === "failed" || endpoint.verification_status === "disabled") {
-    return "verification_failed";
-  }
-  if (endpoint.reliability_score < MIN_PUBLISH_RELIABILITY) return "low_reliability";
-  if (!isSkillDomainEndpoint(skill, endpoint)) return "off_domain";
-  if (isLikelyNoiseEndpoint(endpoint)) return "noise";
-  if (isFragileGraphqlEndpoint(endpoint) && !isVerifiedDurable(endpoint)) return "fragile_graphql";
-  if (!hasDurableSignal(endpoint)) return "no_durable_signal";
-  return null;
-}
-
-export function selectMarketplacePublishEndpoints(
-  skill: SkillManifest,
-  options: PublishSelectionOptions = {},
-): MarketplacePublishSelection {
-  const reasons = freshReasonCounts();
-  const candidates: ScoredCandidate[] = [];
-
-  for (const endpoint of skill.endpoints ?? []) {
-    const reason = rejectionReason(skill, endpoint);
-    if (reason) {
-      reasons[reason] += 1;
-      continue;
-    }
-    candidates.push({
-      endpoint,
-      familyKey: endpointFamilyKey(endpoint),
-      score: scoreEndpoint(endpoint),
-    });
-  }
-
-  candidates.sort((a, b) => {
-    if (b.score !== a.score) return b.score - a.score;
-    return a.endpoint.endpoint_id.localeCompare(b.endpoint.endpoint_id);
-  });
-
-  const selected: ScoredCandidate[] = [];
-  const seenFamilies = new Set<string>();
-  for (const candidate of candidates) {
-    if (seenFamilies.has(candidate.familyKey)) {
-      reasons.family_dedup += 1;
-      continue;
-    }
-    seenFamilies.add(candidate.familyKey);
-    selected.push(candidate);
-  }
-
-  const limit = options.limit ?? DEFAULT_PUBLISH_ENDPOINT_LIMIT;
-  if (selected.length > limit) {
-    reasons.over_limit += selected.length - limit;
-  }
-
-  const endpoints = selected.slice(0, limit).map((candidate) => candidate.endpoint);
-  return {
-    endpoints,
-    stats: {
-      total: skill.endpoints?.length ?? 0,
-      kept: endpoints.length,
-      by_reason: reasons,
-    },
-  };
-}
-
-export function selectMarketplacePublishClosure(
-  skill: SkillManifest,
-  options: PublishSelectionOptions = {},
-): MarketplacePublishClosure {
-  const roots = selectMarketplacePublishEndpoints(skill, options);
-  const rootEndpointIds = roots.endpoints.map((endpoint) => endpoint.endpoint_id);
-
-  if (rootEndpointIds.length === 0) {
-    return {
-      ...roots,
-      root_endpoints: roots.endpoints,
-      root_endpoint_ids: [],
-      closure_endpoint_ids: [],
-      closure_operation_ids: [],
-      closure_edge_count: 0,
-    };
-  }
-
-  const graph = skill.operation_graph;
-  if (!graph?.operations?.length || !graph.edges?.length) {
-    return {
-      ...roots,
-      root_endpoints: roots.endpoints,
-      root_endpoint_ids: rootEndpointIds,
-      closure_endpoint_ids: rootEndpointIds,
-      closure_operation_ids: rootEndpointIds,
-      closure_edge_count: 0,
-    };
-  }
-
-  const endpointById = new Map(skill.endpoints.map((endpoint) => [endpoint.endpoint_id, endpoint]));
-  const operationById = new Map(graph.operations.map((operation) => [operation.operation_id, operation]));
-  const operationByEndpointId = new Map(graph.operations.map((operation) => [operation.endpoint_id, operation]));
-  const adjacency = new Map<string, Set<string>>();
-
-  for (const edge of graph.edges) {
-    if (!CLOSURE_EDGE_KINDS.has(edge.kind)) continue;
-    const fromSet = adjacency.get(edge.from_operation_id) ?? new Set<string>();
-    fromSet.add(edge.to_operation_id);
-    adjacency.set(edge.from_operation_id, fromSet);
-    const toSet = adjacency.get(edge.to_operation_id) ?? new Set<string>();
-    toSet.add(edge.from_operation_id);
-    adjacency.set(edge.to_operation_id, toSet);
-  }
-
-  const seenOperationIds = new Set<string>();
-  const queue: string[] = [];
-  for (const endpointId of rootEndpointIds) {
-    const operationId = operationByEndpointId.get(endpointId)?.operation_id ?? endpointId;
-    if (seenOperationIds.has(operationId)) continue;
-    seenOperationIds.add(operationId);
-    queue.push(operationId);
-  }
-
-  while (queue.length > 0) {
-    const current = queue.shift();
-    if (!current) continue;
-    for (const neighborId of adjacency.get(current) ?? []) {
-      if (seenOperationIds.has(neighborId)) continue;
-      const neighborOperation = operationById.get(neighborId);
-      if (!neighborOperation) continue;
-      const neighborEndpoint = endpointById.get(neighborOperation.endpoint_id);
-      if (!neighborEndpoint) continue;
-      if (rejectionReason(skill, neighborEndpoint)) continue;
-      seenOperationIds.add(neighborId);
-      queue.push(neighborId);
-    }
-  }
-
-  const includedEndpointIds = new Set<string>(rootEndpointIds);
-  for (const operationId of seenOperationIds) {
-    const operation = operationById.get(operationId);
-    if (!operation) continue;
-    includedEndpointIds.add(operation.endpoint_id);
-  }
-
-  const orderedEndpoints = [
-    ...roots.endpoints,
-    ...skill.endpoints.filter(
-      (endpoint) => includedEndpointIds.has(endpoint.endpoint_id) && !rootEndpointIds.includes(endpoint.endpoint_id),
-    ),
-  ];
-
-  const closureEdgeCount = graph.edges.filter((edge) =>
-    CLOSURE_EDGE_KINDS.has(edge.kind) &&
-    seenOperationIds.has(edge.from_operation_id) &&
-    seenOperationIds.has(edge.to_operation_id),
-  ).length;
-
-  return {
-    ...roots,
-    endpoints: orderedEndpoints,
-    root_endpoints: roots.endpoints,
-    root_endpoint_ids: rootEndpointIds,
-    closure_endpoint_ids: orderedEndpoints.map((endpoint) => endpoint.endpoint_id),
-    closure_operation_ids: [...seenOperationIds],
-    closure_edge_count: closureEdgeCount,
-  };
-}
-
-export function formatMarketplacePublishSelection(selection: MarketplacePublishSelection): string {
-  const parts = Object.entries(selection.stats.by_reason)
-    .filter(([, count]) => count > 0)
-    .map(([reason, count]) => `${reason}=${count}`);
-  return parts.length > 0 ? parts.join(", ") : "no drops";
-}

package/runtime-src/ratelimit/index.ts

@@ -1,23 +0,0 @@
-import type { FastifyInstance } from "fastify";
-import rateLimit from "@fastify/rate-limit";
-
-export async function registerRateLimiter(app: FastifyInstance): Promise<void> {
-  await app.register(rateLimit, {
-    max: 100,
-    timeWindow: "1 minute",
-  });
-}
-
-/** Per-route rate limit configs. Apply via route options in Fastify. */
-export const ROUTE_LIMITS = {
-  "/v1/intent/resolve": { max: 20, timeWindow: "1 minute" },
-  "/v1/skills/:skill_id/execute": { max: 30, timeWindow: "1 minute" },
-  "/v1/skills": { max: 5, timeWindow: "1 minute" }, // POST only
-  "/v1/auth/login": { max: 3, timeWindow: "5 minutes" },
-  "/v1/feedback": { max: 60, timeWindow: "1 minute" },
-} as const;
-
-export function routeRateLimit(path: keyof typeof ROUTE_LIMITS) {
-  const cfg = ROUTE_LIMITS[path];
-  return { config: { rateLimit: { max: cfg.max, timeWindow: cfg.timeWindow } } };
-}