teleportation-cli 1.3.0 → 1.4.1

package/lib/intelligence/rebuild-policies.js

@@ -0,0 +1,169 @@
+import { aggregateBenchmarksByTaskClass, scoreHarnessModelBenchmark } from './benchmark.js';
+import { mineRequestPatterns } from './transcript-mine.js';
+
+function collectOperationalTelemetry(events) {
+  const telemetry = {
+    request_interrupted: 0,
+    request_interrupted_tool_use: 0,
+    local_command_caveat: 0,
+    local_command_stdout: 0,
+    local_command_stderr: 0,
+  };
+
+  if (!Array.isArray(events) || events.length === 0) {
+    return {
+      ...telemetry,
+      interrupted_total: 0,
+      interruption_rate: 0,
+      wrapper_marker_total: 0,
+    };
+  }
+
+  for (const event of events) {
+    const metadata = event?.metadata || {};
+    const fields = [
+      metadata.user_message,
+      metadata.user_prompt,
+      metadata.prompt,
+      metadata.message,
+      metadata.instruction,
+      metadata.query,
+      metadata.task_prompt,
+    ];
+
+    for (const value of fields) {
+      if (typeof value !== 'string') continue;
+      const text = value.toLowerCase();
+      if (text.includes('[request interrupted by user for tool use]')) {
+        telemetry.request_interrupted_tool_use += 1;
+      } else if (text.includes('[request interrupted by user]')) {
+        telemetry.request_interrupted += 1;
+      }
+      if (text.includes('<local-command-caveat>')) telemetry.local_command_caveat += 1;
+      if (text.includes('<local-command-stdout>')) telemetry.local_command_stdout += 1;
+      if (text.includes('<local-command-stderr>')) telemetry.local_command_stderr += 1;
+    }
+  }
+
+  const interruptedTotal = telemetry.request_interrupted + telemetry.request_interrupted_tool_use;
+  const wrapperTotal =
+    telemetry.local_command_caveat +
+    telemetry.local_command_stdout +
+    telemetry.local_command_stderr;
+
+  return {
+    ...telemetry,
+    interrupted_total: interruptedTotal,
+    interruption_rate: Number((interruptedTotal / events.length).toFixed(4)),
+    wrapper_marker_total: wrapperTotal,
+  };
+}
+
+function buildLeaderboardAndNoRouteSet(events, slices, options = {}) {
+  const minSessionsForRouting = Number.isInteger(options.minSessionsForRouting)
+    ? options.minSessionsForRouting
+    : 30;
+  const minRouteScore = typeof options.minRouteScore === 'number' ? options.minRouteScore : 0.45;
+  const maxEscalationRate = typeof options.maxEscalationRate === 'number' ? options.maxEscalationRate : 0.5;
+
+  const scored = slices.map((slice) => {
+    const result = scoreHarnessModelBenchmark(events, {
+      harness: slice.provider,
+      model: slice.model,
+      taskCategory: slice.task_category,
+      minSessions: minSessionsForRouting,
+    });
+    return { slice, result };
+  });
+
+  const noRouteSet = scored
+    .filter(({ slice, result }) => {
+      if (!result.eligible) return true;
+      if (result.score < minRouteScore) return true;
+      if (slice.escalation_rate > maxEscalationRate) return true;
+      return false;
+    })
+    .map(({ slice, result }) => ({
+      task_category: slice.task_category,
+      provider: slice.provider,
+      model: slice.model,
+      reason: !result.eligible
+        ? result.reason
+        : result.score < minRouteScore
+          ? 'low_route_score'
+          : 'high_escalation_rate',
+      sample_size: slice.sample_size,
+      score: result.score ?? null,
+    }));
+
+  const eligible = scored
+    .filter(({ slice, result }) =>
+      result.eligible &&
+      result.score >= minRouteScore &&
+      slice.escalation_rate <= maxEscalationRate,
+    )
+    .map(({ slice, result }) => ({
+      task_category: slice.task_category,
+      provider: slice.provider,
+      model: slice.model,
+      score: result.score,
+      sample_size: result.sample_size,
+      confidence_interval: result.confidence_interval,
+    }));
+
+  const byTaskCategory = new Map();
+  for (const row of eligible) {
+    const existing = byTaskCategory.get(row.task_category);
+    if (!existing || row.score > existing.score) {
+      byTaskCategory.set(row.task_category, row);
+    }
+  }
+
+  return {
+    leaderboard: Array.from(byTaskCategory.values()).sort((a, b) => b.score - a.score),
+    no_route_set: noRouteSet.sort((a, b) => a.task_category.localeCompare(b.task_category)),
+  };
+}
+
+export function rebuildPolicyArtifacts(events, options = {}) {
+  const benchmarkMinSessions = Number.isInteger(options.benchmarkMinSessions)
+    ? options.benchmarkMinSessions
+    : 1;
+  const patternMinOccurrences = Number.isInteger(options.patternMinOccurrences)
+    ? options.patternMinOccurrences
+    : 2;
+  const topPatterns = Number.isInteger(options.topPatterns) ? options.topPatterns : 25;
+
+  const patterns = mineRequestPatterns(events, {
+    minOccurrences: patternMinOccurrences,
+    topK: topPatterns,
+  });
+
+  const benchmarkSlices = aggregateBenchmarksByTaskClass(events, {
+    minSessions: benchmarkMinSessions,
+  });
+  const { leaderboard, no_route_set } = buildLeaderboardAndNoRouteSet(events, benchmarkSlices, options);
+  const operationalTelemetry = collectOperationalTelemetry(events);
+
+  return {
+    generated_at: new Date().toISOString(),
+    scope: options.scope || 'global',
+    summary: {
+      events_processed: Array.isArray(events) ? events.length : 0,
+      pattern_count: patterns.length,
+      benchmark_slice_count: benchmarkSlices.length,
+      leaderboard_count: leaderboard.length,
+      no_route_count: no_route_set.length,
+      interrupted_total: operationalTelemetry.interrupted_total,
+      interruption_rate: operationalTelemetry.interruption_rate,
+      wrapper_marker_total: operationalTelemetry.wrapper_marker_total,
+    },
+    artifacts: {
+      request_patterns: patterns,
+      benchmark_slices: benchmarkSlices,
+      benchmark_leaderboard: leaderboard,
+      no_route_set,
+      operational_telemetry: operationalTelemetry,
+    },
+  };
+}

package/lib/intelligence/schema.js

@@ -0,0 +1,259 @@
+/**
+ * Transcript Intelligence Schema
+ *
+ * Canonical event normalization for cross-provider transcript data.
+ */
+
+const TASK_CATEGORIES = new Set([
+  'codebase-modification',
+  'api-orchestration',
+  'data-processing',
+  'web-research',
+  'code-review',
+  'multi-tool-workflow',
+  'unknown',
+]);
+
+const EVENT_TYPES = new Set([
+  'task_start',
+  'task_end',
+  'tool_call',
+  'approval_request',
+  'approval_decision',
+  'override',
+  'route_decision',
+]);
+
+const APPROVAL_DECISIONS = new Set(['approve', 'reject', 'escalate', 'deny', 'none']);
+const APPROVAL_SOURCES = new Set(['user', 'policy', 'fallback']);
+
+function toIsoTimestamp(value) {
+  if (!value) return new Date().toISOString();
+  if (typeof value === 'number') return new Date(value).toISOString();
+  if (typeof value === 'string' && /^\d+$/.test(value)) {
+    return new Date(Number(value)).toISOString();
+  }
+  const parsed = new Date(value);
+  return Number.isNaN(parsed.getTime()) ? new Date().toISOString() : parsed.toISOString();
+}
+
+function normalizeTaskCategory(value) {
+  if (typeof value !== 'string') return 'unknown';
+  const lowered = value.toLowerCase();
+  return TASK_CATEGORIES.has(lowered) ? lowered : 'unknown';
+}
+
+function normalizeApprovalDecision(value) {
+  if (typeof value !== 'string') return 'none';
+  const lowered = value.toLowerCase();
+  return APPROVAL_DECISIONS.has(lowered) ? lowered : 'none';
+}
+
+function normalizeApprovalSource(value) {
+  if (typeof value !== 'string') return 'fallback';
+  const lowered = value.toLowerCase();
+  return APPROVAL_SOURCES.has(lowered) ? lowered : 'fallback';
+}
+
+function mapTimelineTypeToEventType(type) {
+  switch (type) {
+    case 'tool_use':
+    case 'tool_executed':
+    case 'tool_completed':
+    case 'tool_failed':
+      return 'tool_call';
+    case 'approval_requested':
+      return 'approval_request';
+    case 'approval_decided':
+    case 'approval_decision':
+      return 'approval_decision';
+    case 'route_decision':
+      return 'route_decision';
+    case 'override':
+      return 'override';
+    case 'task_started':
+      return 'task_start';
+    case 'task_completed':
+      return 'task_end';
+    case 'assistant_response':
+    case 'session_registered':
+    case 'compact_summary':
+      // These relay event types do not map to a canonical intelligence event type.
+      return null;
+    default:
+      // Unknown types: return null so callers can explicitly filter or flag them.
+      return null;
+  }
+}
+
+function mapActionClass(event) {
+  if (event.type === 'tool_use') return 'tool_execute';
+  if (event.type === 'tool_failed') return 'tool_failed';
+  if (event.type === 'tool_completed') return 'tool_completed';
+  if (event.type === 'assistant_response') return 'assistant_response';
+  if (event.type === 'approval_requested') return 'approval_gate';
+  if (event.type === 'approval_decision') return 'approval_gate';
+  return 'unknown';
+}
+
+function toNumberOrNull(value) {
+  if (value == null) return null;
+  const numeric = Number(value);
+  return Number.isFinite(numeric) ? numeric : null;
+}
+
+export function normalizeTranscriptEvent(event, context = {}) {
+  const eventType = mapTimelineTypeToEventType(event.type);
+  const normalized = {
+    event_id: event.id || null,
+    occurred_at: toIsoTimestamp(event.timestamp ?? event.occurred_at),
+    user_id: context.user_id ?? null,
+    project_id: context.project_id ?? null,
+    session_id: context.session_id ?? null,
+    task_id: context.task_id ?? null,
+    task_category: normalizeTaskCategory(context.task_category),
+    provider: context.provider ?? null,
+    model: context.model ?? null,
+    event_type: eventType,
+    action_class: mapActionClass(event),
+    tool_name: event.meta?.tool_name ?? null,
+    approval: {
+      requested: eventType === 'approval_request' || eventType === 'approval_decision',
+      decision: normalizeApprovalDecision(event.meta?.decision),
+      decision_source: normalizeApprovalSource(event.meta?.decision_source),
+      confidence: toNumberOrNull(event.meta?.confidence),
+    },
+    execution: {
+      success: typeof event.meta?.success === 'boolean' ? event.meta.success : null,
+      duration_ms: toNumberOrNull(event.meta?.duration_ms),
+      tokens_used: toNumberOrNull(event.meta?.tokens_used),
+      cost_usd: toNumberOrNull(event.meta?.cost_usd),
+      loop_detected: typeof event.meta?.loop_detected === 'boolean' ? event.meta.loop_detected : null,
+    },
+    metadata: {
+      // Only include safe, non-sensitive fields from event.meta.
+      // Do NOT spread event.meta directly — it may contain tool arguments,
+      // file contents, or credentials that must not leave the ingestion layer.
+      source: event.source ?? null,
+      raw_type: event.type ?? null,
+    },
+  };
+
+  return normalized;
+}
+
+export function normalizeTranscriptEvents(events, context = {}) {
+  if (!Array.isArray(events)) return [];
+  return events.map((event) => normalizeTranscriptEvent(event, context));
+}
+
+export function isValidNormalizedEvent(event) {
+  if (!event || typeof event !== 'object') return false;
+  if (!EVENT_TYPES.has(event.event_type)) return false;
+  if (!TASK_CATEGORIES.has(event.task_category)) return false;
+  if (!event.occurred_at || Number.isNaN(new Date(event.occurred_at).getTime())) return false;
+  if (!event.approval || !APPROVAL_DECISIONS.has(event.approval.decision)) return false;
+  if (!APPROVAL_SOURCES.has(event.approval.decision_source)) return false;
+  return true;
+}
+
+// ============================================================================
+// Raw Transcript Entry Normalizer (per-message, pre-timeline-event)
+// ============================================================================
+
+const ENTRY_SCHEMA_VERSION = 'v1';
+const DEFAULT_HARNESS = 'claude-code';
+const TEXT_EXCERPT_LIMIT = 500;
+
+function toTimestampMs(value) {
+  if (typeof value === 'number' && Number.isFinite(value)) return value;
+  if (typeof value === 'string') {
+    if (/^\d+$/.test(value)) return Number(value);
+    const parsed = Date.parse(value);
+    if (!Number.isNaN(parsed)) return parsed;
+  }
+  return Date.now();
+}
+
+function toMessage(entry) {
+  if (!entry || typeof entry !== 'object') return {};
+  if (entry.message && typeof entry.message === 'object') return entry.message;
+  return entry;
+}
+
+function extractTextContent(content) {
+  if (typeof content === 'string') return content.trim();
+  if (!Array.isArray(content)) return '';
+  return content
+    .filter(block => block?.type === 'text' && typeof block.text === 'string')
+    .map(block => block.text.trim())
+    .filter(Boolean)
+    .join('\n')
+    .trim();
+}
+
+/**
+ * Normalize a raw transcript entry (message) into a stable canonical shape
+ * for the intelligence pipeline. Operates on the raw transcript format, not
+ * on timeline events.
+ */
+export function normalizeTranscriptEntry(entry, options = {}) {
+  const message = toMessage(entry);
+  const content = message.content;
+  const blocks = Array.isArray(content) ? content : [];
+  const textContent = extractTextContent(content);
+
+  const toolUses = blocks.filter(block => block?.type === 'tool_use');
+  const toolResults = blocks.filter(block => block?.type === 'tool_result');
+
+  const sessionId = typeof options.sessionId === 'string' ? options.sessionId : '';
+  const messageIndex = Number.isInteger(options.messageIndex) ? options.messageIndex : 0;
+  const harness = typeof options.harness === 'string' && options.harness
+    ? options.harness
+    : DEFAULT_HARNESS;
+
+  return {
+    schema_version: ENTRY_SCHEMA_VERSION,
+    session_id: sessionId,
+    message_index: messageIndex,
+    harness,
+    message_id: entry?.uuid || message?.uuid || `${sessionId}:${messageIndex}`,
+    role: typeof message.role === 'string' ? message.role : 'unknown',
+    model: typeof message.model === 'string' ? message.model : null,
+    timestamp_ms: toTimestampMs(entry?.timestamp ?? message?.timestamp),
+    has_content: Boolean(content),
+    text_excerpt: textContent ? textContent.slice(0, TEXT_EXCERPT_LIMIT) : null,
+    tool_uses_count: toolUses.length,
+    tool_results_count: toolResults.length,
+    approval_candidate: toolUses.length > 0,
+  };
+}
+
+/**
+ * Validate a normalized transcript entry shape.
+ */
+export function validateNormalizedTranscriptEntry(entry) {
+  const errors = [];
+  if (!entry || typeof entry !== 'object') return { valid: false, errors: ['entry must be an object'] };
+  if (entry.schema_version !== ENTRY_SCHEMA_VERSION) errors.push(`schema_version must be ${ENTRY_SCHEMA_VERSION}`);
+  if (typeof entry.session_id !== 'string') errors.push('session_id must be a string');
+  if (!Number.isInteger(entry.message_index) || entry.message_index < 0) errors.push('message_index must be a non-negative integer');
+  if (typeof entry.harness !== 'string' || !entry.harness) errors.push('harness must be a non-empty string');
+  if (typeof entry.message_id !== 'string' || !entry.message_id) errors.push('message_id must be a non-empty string');
+  if (typeof entry.role !== 'string' || !entry.role) errors.push('role must be a non-empty string');
+  if (typeof entry.timestamp_ms !== 'number' || !Number.isFinite(entry.timestamp_ms)) errors.push('timestamp_ms must be a finite number');
+  if (typeof entry.tool_uses_count !== 'number' || entry.tool_uses_count < 0) errors.push('tool_uses_count must be a non-negative number');
+  if (typeof entry.tool_results_count !== 'number' || entry.tool_results_count < 0) errors.push('tool_results_count must be a non-negative number');
+  if (typeof entry.approval_candidate !== 'boolean') errors.push('approval_candidate must be a boolean');
+  return { valid: errors.length === 0, errors };
+}
+
+export {
+  TASK_CATEGORIES,
+  EVENT_TYPES,
+  APPROVAL_DECISIONS,
+  APPROVAL_SOURCES,
+  ENTRY_SCHEMA_VERSION,
+  DEFAULT_HARNESS,
+  TEXT_EXCERPT_LIMIT,
+};