unbrowse 3.1.0 → 3.2.0

package/runtime-src/telemetry-attribution.ts

@@ -1,120 +0,0 @@
-export const TELEMETRY_ATTRIBUTION_KEYS = [
-  "utm_source",
-  "utm_medium",
-  "utm_campaign",
-  "utm_content",
-  "utm_term",
-  "utm_id",
-  "gclid",
-  "wbraid",
-  "gbraid",
-  "fbclid",
-  "twclid",
-  "ttclid",
-  "msclkid",
-  "li_fat_id",
-  "referrer_host",
-  "channel",
-  "campaign_id",
-  "campaign_name",
-  "content_id",
-  "content_type",
-  "creative_id",
-  "ad_id",
-  "adset_id",
-  "inferred_icp",
-  "variant_id",
-  "experiment_id",
-  "icp",
-] as const;
-
-export type TelemetryAttributionKey = (typeof TELEMETRY_ATTRIBUTION_KEYS)[number];
-
-export interface TelemetryAttribution {
-  utm_source?: string;
-  utm_medium?: string;
-  utm_campaign?: string;
-  utm_content?: string;
-  utm_term?: string;
-  utm_id?: string;
-  gclid?: string;
-  wbraid?: string;
-  gbraid?: string;
-  fbclid?: string;
-  twclid?: string;
-  ttclid?: string;
-  msclkid?: string;
-  li_fat_id?: string;
-  referrer_host?: string;
-  channel?: string;
-  campaign_id?: string;
-  campaign_name?: string;
-  content_id?: string;
-  content_type?: string;
-  creative_id?: string;
-  ad_id?: string;
-  adset_id?: string;
-  inferred_icp?: string;
-  variant_id?: string;
-  experiment_id?: string;
-  icp?: string;
-}
-
-const MAX_ATTRIBUTION_VALUE_LENGTH = 160;
-
-export function sanitizeAttributionValue(value: unknown): string | undefined {
-  if (typeof value !== "string") return undefined;
-  const trimmed = value.trim();
-  if (!trimmed) return undefined;
-  return trimmed.slice(0, MAX_ATTRIBUTION_VALUE_LENGTH);
-}
-
-export function hasTelemetryAttribution(value: Partial<TelemetryAttribution> | null | undefined): boolean {
-  if (!value) return false;
-  return TELEMETRY_ATTRIBUTION_KEYS.some((key) => Boolean(sanitizeAttributionValue(value[key])));
-}
-
-export function sanitizeTelemetryAttribution(
-  raw: Partial<Record<TelemetryAttributionKey, unknown>> | null | undefined,
-): TelemetryAttribution | undefined {
-  if (!raw) return undefined;
-  const cleaned: TelemetryAttribution = {};
-  for (const key of TELEMETRY_ATTRIBUTION_KEYS) {
-    const value = sanitizeAttributionValue(raw[key]);
-    if (value) cleaned[key] = value;
-  }
-  return hasTelemetryAttribution(cleaned) ? cleaned : undefined;
-}
-
-export function mergeTelemetryAttribution(
-  base?: Partial<TelemetryAttribution> | null,
-  incoming?: Partial<TelemetryAttribution> | null,
-): TelemetryAttribution | undefined {
-  const merged = sanitizeTelemetryAttribution({
-    ...(base ?? {}),
-    ...(incoming ?? {}),
-  });
-  return merged;
-}
-
-export function mergeTelemetryProperties(
-  properties?: Record<string, unknown>,
-  attribution?: Partial<TelemetryAttribution> | null,
-): Record<string, unknown> | undefined {
-  const cleanedAttribution = sanitizeTelemetryAttribution(attribution);
-  if (!cleanedAttribution) return properties;
-  return {
-    ...cleanedAttribution,
-    ...(properties ?? {}),
-  };
-}
-
-export function decodeTelemetryAttribution(value: string | undefined): TelemetryAttribution | undefined {
-  if (!value) return undefined;
-  try {
-    const decoded = Buffer.from(value, "base64").toString("utf8");
-    return sanitizeTelemetryAttribution(JSON.parse(decoded) as Partial<Record<TelemetryAttributionKey, unknown>>);
-  } catch {
-    return undefined;
-  }
-}

package/runtime-src/telemetry.ts

@@ -1,253 +0,0 @@
-/**
- * Route trace telemetry — issue #28
- *
- * Persists anonymized routing artifacts to ~/.unbrowse/traces/ for future
- * ML training (RAG over traces, contextual bandits, KGE extraction).
- *
- * Anonymization rules (per spec):
- *   NEVER store: raw cookies, auth tokens, CSRF tokens, full request/response
- *                bodies, user-entered secrets, raw chain-of-thought, PII.
- *   DO store:    normalized binding names, hashed binding values, response
- *                schema shape, response hashes, route fingerprints,
- *                error class / failure reason taxonomy.
- */
-
-import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import type { RouteTraceArtifact, TraceFailureReason } from "./types/index.js";
-import { TRACE_VERSION } from "./version.js";
-
-function getTraceDir(): string {
-  return process.env.UNBROWSE_TRACES_DIR ??
-    join(process.env.HOME ?? "/tmp", ".unbrowse", "traces");
-}
-
-/** Whether trace emission is enabled (opt-out via env). */
-export function isTracingEnabled(): boolean {
-  return process.env.UNBROWSE_DISABLE_TRACES !== "1";
-}
-
-/**
- * SHA-256 hex digest of a string value.
- * Used to hash binding values so dependency matching works without exposing
- * the actual value.
- */
-export function hashValue(value: string): string {
-  return createHash("sha256").update(value).digest("hex").slice(0, 16);
-}
-
-/** Sensitive field name patterns — values of matching keys are stripped. */
-const SENSITIVE_PATTERNS = [
-  /cookie/i, /token/i, /auth/i, /key/i, /secret/i, /password/i,
-  /csrf/i, /session/i, /bearer/i, /credential/i, /apikey/i,
-];
-
-function isSensitiveName(name: string): boolean {
-  return SENSITIVE_PATTERNS.some((re) => re.test(name));
-}
-
-/**
- * Return only the binding *names* that are safe to surface in a trace.
- * Values are never included; sensitive names are omitted entirely.
- */
-export function safeBindingNames(bindings: Record<string, unknown>): string[] {
-  return Object.keys(bindings).filter((k) => !isSensitiveName(k));
-}
-
-/**
- * Strip the query string and fragment from a URL so traces never contain
- * user-entered search terms or personal identifiers embedded in URLs.
- */
-export function anonymizeUrl(url: string): string {
-  try {
-    const u = new URL(url);
-    return `${u.protocol}//${u.host}${u.pathname}`;
-  } catch {
-    return url.replace(/\?.*$/, "").replace(/#.*$/, "");
-  }
-}
-
-/**
- * Produce a SHA-256 hex digest of the serialized result body.
- * This lets the backend detect duplicate responses without storing content.
- */
-export function hashResponseBody(result: unknown): string {
-  const str = typeof result === "string" ? result : JSON.stringify(result ?? "");
-  return createHash("sha256").update(str).digest("hex").slice(0, 32);
-}
-
-/**
- * Classify an error message into a `TraceFailureReason` taxonomy value so
- * negative traces carry structured labels instead of free-form strings.
- */
-export function classifyFailure(error: string | undefined): TraceFailureReason {
-  if (!error) return "unknown";
-  const e = error.toLowerCase();
-  if (e.includes("auth") || e.includes("401") || e.includes("403")) return "auth_failed";
-  if (e.includes("schema") || e.includes("mismatch")) return "schema_mismatch";
-  if (e.includes("timeout") || e.includes("timed out")) return "timeout";
-  if (e.includes("empty") || e.includes("no result")) return "empty_response";
-  if (e.includes("missing") || e.includes("dependency")) return "dependency_missing";
-  if (e.includes("robots") || e.includes("disallowed")) return "robots_disallowed";
-  return "unknown";
-}
-
-export interface EmitTraceParams {
-  trace_id: string;
-  session_scope: string;
-  goal: string;
-  domain: string;
-  started_at: string;
-  skill_id?: string;
-  endpoint_id?: string;
-  source: RouteTraceArtifact["source"];
-  status_code?: number;
-  response_bytes: number;
-  result?: unknown;
-  schema_match?: boolean;
-  candidates_considered: number;
-  bindings_before: Record<string, unknown>;
-  bindings_resolved: Record<string, unknown>;
-  bindings_missing: string[];
-  outcome: RouteTraceArtifact["outcome"];
-  error?: string;
-  rationale?: string;
-}
-
-/**
- * Write an anonymized `RouteTraceArtifact` to the local trace directory.
- * Returns the file path written, or null if tracing is disabled or the write fails.
- */
-export function emitRouteTrace(params: EmitTraceParams): string | null {
-  if (!isTracingEnabled()) return null;
-
-  const now = new Date().toISOString();
-  const artifact: RouteTraceArtifact = {
-    trace_id: params.trace_id,
-    session_scope: params.session_scope,
-    goal: params.goal,
-    domain: params.domain,
-    started_at: params.started_at,
-    completed_at: now,
-    latency_ms: Date.now() - new Date(params.started_at).getTime(),
-    skill_id: params.skill_id,
-    endpoint_id: params.endpoint_id,
-    source: params.source,
-    status_code: params.status_code,
-    response_bytes: params.response_bytes,
-    response_hash: params.result != null ? hashResponseBody(params.result) : undefined,
-    schema_match: params.schema_match,
-    candidates_considered: params.candidates_considered,
-    bindings_before: safeBindingNames(params.bindings_before),
-    bindings_resolved: safeBindingNames(params.bindings_resolved),
-    bindings_missing: params.bindings_missing.filter((k) => !isSensitiveName(k)),
-    outcome: params.outcome,
-    failure_reason: params.outcome !== "success"
-      ? classifyFailure(params.error)
-      : undefined,
-    rationale: params.rationale,
-    trace_version: TRACE_VERSION,
-  };
-
-  try {
-    const traceDir = getTraceDir();
-    if (!existsSync(traceDir)) mkdirSync(traceDir, { recursive: true });
-    const stamp = params.started_at.replace(/[:.]/g, "-");
-    const file = join(traceDir, `${stamp}-${params.outcome}-${params.trace_id}.json`);
-    writeFileSync(file, JSON.stringify(artifact, null, 2), "utf-8");
-    return file;
-  } catch {
-    return null;
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Error accumulator — auto-file GitHub issues on repeated failures
-// ---------------------------------------------------------------------------
-
-interface AccumulatedError {
-  count: number;
-  firstSeen: string;
-  lastSeen: string;
-  error: string;
-  intent: string;
-  url?: string;
-  domain: string;
-  skillId?: string;
-  endpointId?: string;
-}
-
-const errorAccumulator = new Map<string, AccumulatedError>();
-const AUTO_FILE_THRESHOLD = 3;
-const autoFiledKeys = new Set<string>();
-
-export interface FailureContext {
-  skillId?: string;
-  endpointId?: string;
-  domain: string;
-  intent: string;
-  url?: string;
-  error: string;
-}
-
-/**
- * Record a failure. After AUTO_FILE_THRESHOLD consecutive failures for the
- * same skill+endpoint, auto-file a GitHub issue via the backend.
- */
-export function recordFailure(ctx: FailureContext): void {
-  const key = `${ctx.skillId ?? "unknown"}:${ctx.endpointId ?? "unknown"}`;
-  const existing = errorAccumulator.get(key);
-  const now = new Date().toISOString();
-
-  if (existing) {
-    existing.count++;
-    existing.lastSeen = now;
-    existing.error = ctx.error;
-  } else {
-    errorAccumulator.set(key, {
-      count: 1,
-      firstSeen: now,
-      lastSeen: now,
-      error: ctx.error,
-      intent: ctx.intent,
-      url: ctx.url,
-      domain: ctx.domain,
-      skillId: ctx.skillId,
-      endpointId: ctx.endpointId,
-    });
-  }
-
-  const entry = errorAccumulator.get(key)!;
-  if (entry.count >= AUTO_FILE_THRESHOLD && !autoFiledKeys.has(key)) {
-    autoFiledKeys.add(key);
-    // Fire-and-forget auto-file
-    import("./client/index.js")
-      .then((client) =>
-        client.autoFileIssue({
-          skill_id: ctx.skillId ?? "unknown",
-          endpoint_id: ctx.endpointId ?? "unknown",
-          domain: ctx.domain,
-          intent: ctx.intent,
-          url: ctx.url,
-          error: ctx.error,
-          failure_count: entry.count,
-          first_seen: entry.firstSeen,
-          last_seen: entry.lastSeen,
-          kuri_version: process.env.KURI_VERSION ?? "unknown",
-        }),
-      )
-      .catch((err) =>
-        console.warn(`[telemetry] auto-file failed: ${(err as Error).message}`),
-      );
-  }
-}
-
-export function getAccumulatedErrors(): Map<string, AccumulatedError> {
-  return new Map(errorAccumulator);
-}
-
-export function resetErrorAccumulatorForTests(): void {
-  errorAccumulator.clear();
-  autoFiledKeys.clear();
-}

package/runtime-src/template-params.ts

@@ -1,141 +0,0 @@
-function sanitizeBindingKey(key: string): string {
-  return key
-    .replace(/\[\]/g, "_item")
-    .replace(/\[([^\]]+)\]/g, (_m, inner: string) => `_${inner}`)
-    .replace(/[{}]/g, "")
-    .replace(/[^a-zA-Z0-9_]+/g, "_")
-    .replace(/_+/g, "_")
-    .replace(/^_+|_+$/g, "");
-}
-
-export function normalizeQueryBindingKey(key: string): string {
-  const normalized = sanitizeBindingKey(key);
-  if (!normalized) return "query";
-  return /^[0-9]/.test(normalized) ? `query_${normalized}` : normalized;
-}
-
-export function buildQueryBindingMap(keys: Iterable<string>): Record<string, string> {
-  const out: Record<string, string> = {};
-  const used = new Set<string>();
-  for (const rawKey of keys) {
-    if (!rawKey || out[rawKey]) continue;
-    const base = normalizeQueryBindingKey(rawKey);
-    let next = base;
-    let suffix = 2;
-    while (used.has(next)) next = `${base}_${suffix++}`;
-    used.add(next);
-    out[rawKey] = next;
-  }
-  return out;
-}
-
-export function extractTemplateQueryBindings(urlTemplate: string): Record<string, string> {
-  try {
-    const templateUrl = new URL(urlTemplate);
-    const out: Record<string, string> = {};
-    for (const [key, value] of templateUrl.searchParams.entries()) {
-      const placeholder = value.match(/^\{([^}]+)\}$/)?.[1];
-      if (placeholder) out[key] = placeholder;
-    }
-    return out;
-  } catch {
-    return {};
-  }
-}
-
-export function deriveTemplateParamsFromContextUrl(
-  urlTemplate: string,
-  contextUrl?: string,
-): Record<string, string> {
-  if (!contextUrl) return {};
-  try {
-    const templateUrl = new URL(urlTemplate);
-    const actualUrl = new URL(contextUrl);
-    const out: Record<string, string> = {};
-
-    const templateParts = templateUrl.pathname.split("/").filter(Boolean).map((part) => decodeURIComponent(part));
-    const actualParts = actualUrl.pathname.split("/").filter(Boolean);
-    if (templateParts.length === actualParts.length) {
-      let compatible = true;
-      for (let i = 0; i < templateParts.length; i++) {
-        const templatePart = templateParts[i];
-        const actualPart = actualParts[i];
-        const placeholder = templatePart.match(/^\{([^}]+)\}$/)?.[1];
-        if (placeholder) {
-          out[placeholder] = decodeURIComponent(actualPart);
-          continue;
-        }
-        if (templatePart !== actualPart) {
-          compatible = false;
-          break;
-        }
-      }
-      if (!compatible) {
-        for (const key of Object.keys(out)) delete out[key];
-      }
-    }
-
-    for (const [key, value] of templateUrl.searchParams.entries()) {
-      const placeholder = value.match(/^\{([^}]+)\}$/)?.[1];
-      if (!placeholder) continue;
-      const actualValue = actualUrl.searchParams.get(key);
-      if (actualValue != null && actualValue !== "") {
-        out[placeholder] = actualValue;
-      }
-    }
-    return out;
-  } catch {
-    return {};
-  }
-}
-
-function deriveSemanticTemplateParams(
-  urlTemplate: string,
-  contextUrl?: string,
-): Record<string, string> {
-  if (!contextUrl) return {};
-  try {
-    const templateNames = [...urlTemplate.matchAll(/\{([^}]+)\}/g)].map((m) => m[1]).filter(Boolean);
-    if (templateNames.length === 0) return {};
-    const actualUrl = new URL(contextUrl);
-    const actualParts = actualUrl.pathname.split("/").filter(Boolean).map((part) => decodeURIComponent(part));
-    const lowerParts = actualParts.map((part) => part.toLowerCase());
-    const out: Record<string, string> = {};
-
-    for (const name of templateNames) {
-      const lower = name.toLowerCase();
-      if (lower === "tag" || lower === "tags") {
-        const prefixIndex = lowerParts.findIndex((part) => part === "t" || part === "tag" || part === "tags");
-        if (prefixIndex >= 0 && actualParts[prefixIndex + 1]) out[name] = actualParts[prefixIndex + 1]!;
-      }
-    }
-
-    return out;
-  } catch {
-    return {};
-  }
-}
-
-export function mergeContextTemplateParams(
-  params: Record<string, unknown>,
-  urlTemplate: string,
-  contextUrl?: string,
-): Record<string, unknown> {
-  const merged: Record<string, unknown> = { ...params };
-  const inferred = deriveTemplateParamsFromContextUrl(urlTemplate, contextUrl);
-  for (const [key, value] of Object.entries(inferred)) {
-    if (merged[key] == null || merged[key] === "") merged[key] = value;
-  }
-  for (const [key, value] of Object.entries(deriveSemanticTemplateParams(urlTemplate, contextUrl))) {
-    if (merged[key] == null || merged[key] === "") merged[key] = value;
-  }
-  for (const [rawKey, placeholder] of Object.entries(extractTemplateQueryBindings(urlTemplate))) {
-    if ((merged[placeholder] == null || merged[placeholder] === "") && merged[rawKey] != null && merged[rawKey] !== "") {
-      merged[placeholder] = merged[rawKey];
-    }
-    if ((merged[rawKey] == null || merged[rawKey] === "") && merged[placeholder] != null && merged[placeholder] !== "") {
-      merged[rawKey] = merged[placeholder];
-    }
-  }
-  return merged;
-}

package/runtime-src/transform/drift.ts

@@ -1,60 +0,0 @@
-import type { DriftResult, ResponseSchema } from "../types/index.js";
-import { inferSchema } from "./index.js";
-
-/**
- * Collect all field paths from a schema (dot-notation).
- */
-function collectPaths(schema: ResponseSchema, prefix = ""): Map<string, string> {
-  const paths = new Map<string, string>();
-  if (schema.type === "object" && schema.properties) {
-    for (const [key, val] of Object.entries(schema.properties)) {
-      const fullPath = prefix ? `${prefix}.${key}` : key;
-      paths.set(fullPath, val.type);
-      if (val.type === "object" && val.properties) {
-        for (const [p, t] of collectPaths(val, fullPath)) {
-          paths.set(p, t);
-        }
-      }
-      if (val.type === "array" && val.items?.type === "object") {
-        for (const [p, t] of collectPaths(val.items, `${fullPath}[]`)) {
-          paths.set(p, t);
-        }
-      }
-    }
-  }
-  return paths;
-}
-
-/**
- * Compare stored schema against a fresh response sample.
- * Returns added/removed fields and type changes.
- */
-export function detectSchemaDrift(existing: ResponseSchema, newSample: unknown): DriftResult {
-  const newSchema = inferSchema([newSample]);
-  const oldPaths = collectPaths(existing);
-  const newPaths = collectPaths(newSchema);
-
-  const added_fields: string[] = [];
-  const removed_fields: string[] = [];
-  const type_changes: Array<{ path: string; was: string; now: string }> = [];
-
-  for (const [path, type] of newPaths) {
-    if (!oldPaths.has(path)) {
-      added_fields.push(path);
-    } else if (oldPaths.get(path) !== type) {
-      type_changes.push({ path, was: oldPaths.get(path)!, now: type });
-    }
-  }
-  for (const path of oldPaths.keys()) {
-    if (!newPaths.has(path)) {
-      removed_fields.push(path);
-    }
-  }
-
-  return {
-    drifted: added_fields.length > 0 || removed_fields.length > 0 || type_changes.length > 0,
-    added_fields,
-    removed_fields,
-    type_changes,
-  };
-}