thevoidforge 21.0.10 → 21.0.12

package/dist/docs/patterns/ai-eval.ts

@@ -0,0 +1,272 @@
+/**
+ * Pattern: AI Eval
+ *
+ * Key principles:
+ * - Every AI feature needs a golden dataset — input/expected-output pairs
+ * - Automated eval runs catch regressions before they reach production
+ * - Compare scores across prompt versions — never ship a prompt that scores lower
+ * - Scoring functions are pluggable — exact match, semantic similarity, custom
+ * - Eval results are stored, not printed — you need history to detect drift
+ *
+ * Agents: Batman (testing/validation), Picard (architecture), L (monitoring)
+ *
+ * Provider note: Eval runs use the same model call patterns from ai-orchestrator.ts.
+ * The eval framework itself is provider-agnostic.
+ */
+
+// --- Core types ---
+
+/** A single test case in a golden dataset. */
+export interface EvalCase<TInput = string, TExpected = string> {
+  id: string // Stable ID for tracking across runs
+  input: TInput
+  expected: TExpected
+  tags?: string[] // e.g., ['edge-case', 'billing', 'multi-language']
+}
+
+/** Result of evaluating a single case. */
+export interface CaseResult {
+  caseId: string
+  passed: boolean
+  score: number // 0.0 - 1.0
+  actual: string // What the model returned
+  expected: string // What we wanted
+  latencyMs: number
+  error?: string // If the model call failed
+}
+
+/** Aggregate result of an eval run. */
+export interface EvalResult {
+  runId: string
+  promptVersion: string
+  model: string
+  timestamp: string
+  totalCases: number
+  passedCases: number
+  averageScore: number
+  averageLatencyMs: number
+  caseResults: CaseResult[]
+  tags: Record<string, { count: number; avgScore: number }> // Per-tag breakdown
+}
+
+/** Comparison between two eval runs. */
+export interface VersionComparison {
+  baseVersion: string
+  candidateVersion: string
+  baseScore: number
+  candidateScore: number
+  delta: number // Positive = improvement, negative = regression
+  regressions: CaseResult[] // Cases that got worse
+  improvements: CaseResult[] // Cases that got better
+  verdict: 'pass' | 'fail' | 'review' // Based on regression threshold
+}
+
+// --- Scoring functions ---
+
+/** Exact string match (case-insensitive). */
+export function exactMatch(actual: string, expected: string): number {
+  return actual.trim().toLowerCase() === expected.trim().toLowerCase() ? 1.0 : 0.0
+}
+
+/** Check if expected value is contained in actual output. */
+export function containsMatch(actual: string, expected: string): number {
+  return actual.toLowerCase().includes(expected.toLowerCase()) ? 1.0 : 0.0
+}
+
+/** JSON field match — compare specific fields in JSON outputs. */
+export function jsonFieldMatch(
+  actual: string,
+  expected: string,
+  fields: string[]
+): number {
+  try {
+    const actualObj = JSON.parse(actual)
+    const expectedObj = JSON.parse(expected)
+    let matches = 0
+    for (const field of fields) {
+      if (actualObj[field] === expectedObj[field]) matches++
+    }
+    return matches / fields.length
+  } catch {
+    return 0.0 // Parse failure = score 0
+  }
+}
+
+// --- EvalSuite ---
+
+type ModelRunner = (input: string) => Promise<string>
+type ScoringFunction = (actual: string, expected: string) => number
+
+export class EvalSuite<TInput = string> {
+  private cases: EvalCase<TInput, string>[] = []
+  private scoreFn: ScoringFunction = exactMatch
+  private passThreshold = 0.8 // Case passes if score >= this
+
+  constructor(private name: string) {}
+
+  /** Add a test case to the suite. */
+  addCase(testCase: EvalCase<TInput, string>): this {
+    this.cases.push(testCase)
+    return this
+  }
+
+  /** Add multiple test cases. */
+  addCases(cases: EvalCase<TInput, string>[]): this {
+    this.cases.push(...cases)
+    return this
+  }
+
+  /** Set the scoring function (default: exactMatch). */
+  withScoring(fn: ScoringFunction): this {
+    this.scoreFn = fn
+    return this
+  }
+
+  /** Set the pass threshold (default: 0.8). */
+  withPassThreshold(threshold: number): this {
+    this.passThreshold = threshold
+    return this
+  }
+
+  /** Run the eval suite against a model runner function. */
+  async run(
+    runner: ModelRunner,
+    promptVersion: string,
+    model: string
+  ): Promise<EvalResult> {
+    const runId = `eval-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
+    const caseResults: CaseResult[] = []
+
+    // Run cases sequentially to avoid rate limits. For large suites,
+    // batch with concurrency limit (e.g., p-limit with concurrency 5).
+    for (const testCase of this.cases) {
+      const start = Date.now()
+      let actual = ''
+      let error: string | undefined
+
+      try {
+        actual = await runner(testCase.input as string)
+      } catch (e) {
+        error = e instanceof Error ? e.message : 'Unknown error'
+      }
+
+      const score = error ? 0 : this.scoreFn(actual, testCase.expected)
+
+      caseResults.push({
+        caseId: testCase.id,
+        passed: score >= this.passThreshold,
+        score,
+        actual,
+        expected: testCase.expected,
+        latencyMs: Date.now() - start,
+        error,
+      })
+    }
+
+    // Compute per-tag breakdowns
+    const tags: Record<string, { count: number; avgScore: number }> = {}
+    for (const testCase of this.cases) {
+      for (const tag of testCase.tags ?? []) {
+        if (!tags[tag]) tags[tag] = { count: 0, avgScore: 0 }
+        const result = caseResults.find((r) => r.caseId === testCase.id)
+        if (result) {
+          tags[tag].count++
+          tags[tag].avgScore += result.score
+        }
+      }
+    }
+    for (const tag of Object.values(tags)) {
+      tag.avgScore = tag.avgScore / tag.count
+    }
+
+    return {
+      runId,
+      promptVersion,
+      model,
+      timestamp: new Date().toISOString(),
+      totalCases: this.cases.length,
+      passedCases: caseResults.filter((r) => r.passed).length,
+      averageScore: caseResults.reduce((sum, r) => sum + r.score, 0) / caseResults.length,
+      averageLatencyMs: caseResults.reduce((sum, r) => sum + r.latencyMs, 0) / caseResults.length,
+      caseResults,
+      tags,
+    }
+  }
+}
+
+// --- Version comparison ---
+
+const REGRESSION_THRESHOLD = 0.02 // 2% drop triggers review
+
+export function compareVersions(
+  base: EvalResult,
+  candidate: EvalResult
+): VersionComparison {
+  const delta = candidate.averageScore - base.averageScore
+
+  // Find regressions: cases that scored lower in the candidate
+  const regressions: CaseResult[] = []
+  const improvements: CaseResult[] = []
+
+  for (const candidateCase of candidate.caseResults) {
+    const baseCase = base.caseResults.find((b) => b.caseId === candidateCase.caseId)
+    if (!baseCase) continue
+
+    if (candidateCase.score < baseCase.score) regressions.push(candidateCase)
+    if (candidateCase.score > baseCase.score) improvements.push(candidateCase)
+  }
+
+  let verdict: VersionComparison['verdict'] = 'pass'
+  if (delta < -REGRESSION_THRESHOLD) verdict = 'fail'
+  else if (regressions.length > 0) verdict = 'review'
+
+  return {
+    baseVersion: base.promptVersion,
+    candidateVersion: candidate.promptVersion,
+    baseScore: base.averageScore,
+    candidateScore: candidate.averageScore,
+    delta,
+    regressions,
+    improvements,
+    verdict,
+  }
+}
+
+// --- Usage example ---
+
+// const suite = new EvalSuite('ticket-classifier')
+//   .withScoring(jsonFieldMatch)
+//   .addCases([
+//     { id: 'billing-1', input: 'I was charged twice', expected: '{"label":"billing"}', tags: ['billing'] },
+//     { id: 'tech-1', input: 'App crashes on login', expected: '{"label":"technical"}', tags: ['technical'] },
+//   ])
+//
+// const baseResult = await suite.run(classifyV1, '2024.01.01', 'claude-sonnet-4-20250514')
+// const candidateResult = await suite.run(classifyV2, '2024.01.15', 'claude-sonnet-4-20250514')
+// const comparison = compareVersions(baseResult, candidateResult)
+//
+// if (comparison.verdict === 'fail') {
+//   console.error(`Regression detected: ${comparison.delta.toFixed(3)} score drop`)
+//   process.exit(1) // Fail CI
+// }
+
+/**
+ * Framework adaptations:
+ *
+ * Express:
+ *   - Run evals in CI (GitHub Actions) on prompt file changes
+ *   - Store EvalResult in S3/database for historical comparison
+ *   - Endpoint to trigger eval: POST /api/admin/eval (admin-only)
+ *
+ * FastAPI:
+ *   - Same EvalSuite shape in Python with pytest fixtures
+ *   - Use pytest-benchmark for latency tracking
+ *   - Store results in PostgreSQL with SQLAlchemy models
+ *   - CI: run eval suite in GitHub Actions, compare with previous run
+ *
+ * Django:
+ *   - Management command: python manage.py run_eval --suite ticket-classifier
+ *   - EvalResult and CaseResult as Django models for admin dashboard
+ *   - Compare versions in admin: side-by-side eval result view
+ *   - Celery task for large eval suites (100+ cases)
+ */

package/dist/docs/patterns/ai-orchestrator.ts

@@ -0,0 +1,341 @@
+/**
+ * Pattern: AI Orchestrator
+ *
+ * Three patterns for coordinating AI model calls:
+ * 1. Simple completion — single call with structured output
+ * 2. Agent loop — model calls tools in a loop until done
+ * 3. Circuit breaker — prevent cascading AI failures
+ *
+ * Key principles:
+ * - Always set MAX_ITERATIONS on agent loops — unbounded loops burn tokens
+ * - Retry with exponential backoff on transient failures (429, 5xx)
+ * - Circuit breaker protects downstream when a provider is degraded
+ * - Parse and validate model output with Zod — never trust raw JSON
+ * - Log every model call with latency, tokens, and model version
+ *
+ * Agents: Picard (architecture), Stark (backend), Kenobi (rate limits)
+ *
+ * Provider note: Primary examples use Anthropic SDK (@anthropic-ai/sdk).
+ * OpenAI adaptation is noted inline. The shapes are provider-agnostic.
+ */
+
+import Anthropic from '@anthropic-ai/sdk'
+import { z } from 'zod'
+
+// --- 1. Simple Completion — single call → structured output ---
+
+const SummarySchema = z.object({
+  title: z.string(),
+  bullets: z.array(z.string()).min(1).max(5),
+  sentiment: z.enum(['positive', 'negative', 'neutral']),
+})
+
+type Summary = z.infer<typeof SummarySchema>
+
+/** Single model call with retry and structured output parsing. */
+export async function executeWithRetry<T>(
+  fn: () => Promise<T>,
+  maxRetries = 3,
+  baseDelayMs = 1000
+): Promise<T> {
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn()
+    } catch (error: unknown) {
+      const isRetryable =
+        error instanceof Error &&
+        ('status' in error && [429, 500, 502, 503].includes((error as { status: number }).status))
+
+      if (!isRetryable || attempt === maxRetries) throw error
+
+      // Exponential backoff with jitter
+      const delay = baseDelayMs * 2 ** attempt + Math.random() * baseDelayMs
+      await new Promise((r) => setTimeout(r, delay))
+    }
+  }
+  throw new Error('Unreachable') // TypeScript needs this
+}
+
+export async function summarize(client: Anthropic, text: string): Promise<Summary> {
+  const response = await executeWithRetry(() =>
+    client.messages.create({
+      model: 'claude-sonnet-4-20250514',
+      max_tokens: 512,
+      messages: [{ role: 'user', content: `Summarize as JSON: ${text}` }],
+      // System prompt enforces output shape
+      system: 'Respond with JSON matching: { title, bullets[], sentiment }',
+    })
+  )
+
+  // Extract text content, parse with Zod — never trust raw model output
+  const content = response.content[0]
+  if (content.type !== 'text') throw new Error('Expected text response')
+  const parsed = JSON.parse(content.text)
+  return SummarySchema.parse(parsed) // Throws ZodError on invalid shape
+}
+
+// OpenAI adaptation:
+//   const response = await openai.chat.completions.create({
+//     model: 'gpt-4o', messages: [...],
+//     response_format: { type: 'json_object' }, // OpenAI JSON mode
+//   })
+//   const parsed = JSON.parse(response.choices[0].message.content)
+
+// --- 2. Agent Loop — model calls tools until done ---
+
+const MAX_ITERATIONS = 10 // Hard bound — never remove this
+
+interface ToolDefinition {
+  name: string
+  description: string
+  input_schema: Record<string, unknown>
+}
+
+interface AgentResult {
+  finalResponse: string
+  toolCallCount: number
+  iterations: number
+}
+
+/** Agent loop: model decides which tools to call, iterates until done. */
+export async function runAgentLoop(
+  client: Anthropic,
+  prompt: string,
+  tools: ToolDefinition[],
+  executeTool: (name: string, input: Record<string, unknown>) => Promise<string>
+): Promise<AgentResult> {
+  const messages: Anthropic.MessageParam[] = [{ role: 'user', content: prompt }]
+  let toolCallCount = 0
+
+  for (let i = 0; i < MAX_ITERATIONS; i++) {
+    const response = await executeWithRetry(() =>
+      client.messages.create({
+        model: 'claude-sonnet-4-20250514',
+        max_tokens: 4096,
+        messages,
+        tools: tools as Anthropic.Tool[],
+      })
+    )
+
+    // If model responds with text only (no tool calls), we're done
+    if (response.stop_reason === 'end_turn') {
+      const text = response.content.find((c) => c.type === 'text')
+      return { finalResponse: text?.text ?? '', toolCallCount, iterations: i + 1 }
+    }
+
+    // Process tool calls
+    const toolUseBlocks = response.content.filter((c) => c.type === 'tool_use')
+    const toolResults: Anthropic.ToolResultBlockParam[] = []
+
+    for (const block of toolUseBlocks) {
+      if (block.type !== 'tool_use') continue
+      toolCallCount++
+      try {
+        const result = await executeTool(block.name, block.input as Record<string, unknown>)
+        toolResults.push({ type: 'tool_result', tool_use_id: block.id, content: result })
+      } catch (error) {
+        // Send error back to model — it can recover or try a different approach
+        toolResults.push({
+          type: 'tool_result',
+          tool_use_id: block.id,
+          content: `Error: ${error instanceof Error ? error.message : 'Unknown error'}`,
+          is_error: true,
+        })
+      }
+    }
+
+    // Feed assistant response + tool results back for next iteration
+    messages.push({ role: 'assistant', content: response.content })
+    messages.push({ role: 'user', content: toolResults })
+  }
+
+  throw new Error(`Agent loop exceeded MAX_ITERATIONS (${MAX_ITERATIONS})`)
+}
+
+// --- 3. Circuit Breaker — prevent cascading AI failures ---
+
+type CircuitState = 'closed' | 'open' | 'half-open'
+
+/** Circuit breaker for AI provider calls. Opens after threshold failures. */
+export class CircuitBreaker {
+  private state: CircuitState = 'closed'
+  private failureCount = 0
+  private lastFailureTime = 0
+
+  constructor(
+    private readonly failureThreshold: number = 5,
+    private readonly resetTimeoutMs: number = 60_000
+  ) {}
+
+  async execute<T>(fn: () => Promise<T>, fallback: () => Promise<T>): Promise<T> {
+    if (this.state === 'open') {
+      // Check if enough time has passed to try again
+      if (Date.now() - this.lastFailureTime > this.resetTimeoutMs) {
+        this.state = 'half-open'
+      } else {
+        return fallback() // Circuit open — use fallback immediately
+      }
+    }
+
+    try {
+      const result = await fn()
+      // Success — reset if we were testing
+      if (this.state === 'half-open') this.state = 'closed'
+      this.failureCount = 0
+      return result
+    } catch (error) {
+      this.failureCount++
+      this.lastFailureTime = Date.now()
+
+      if (this.failureCount >= this.failureThreshold) {
+        this.state = 'open'
+      }
+      return fallback()
+    }
+  }
+
+  getState(): CircuitState {
+    return this.state
+  }
+}
+
+// Usage:
+//   const breaker = new CircuitBreaker(5, 60_000)
+//   const result = await breaker.execute(
+//     () => summarize(anthropicClient, text),
+//     () => ruleBased.summarize(text) // Fallback: no AI, just rules
+//   )
+
+/**
+ * Framework adaptations:
+ *
+ * Express:
+ *   - Wrap agent loop in an Express route with req.setTimeout() for long-running calls
+ *   - Circuit breaker as singleton middleware: app.use(aiCircuitBreaker)
+ *   - Stream partial results via res.write() for SSE (see sse-endpoint.ts)
+ *
+ * FastAPI:
+ *   - executeWithRetry → tenacity.retry(wait=wait_exponential(), stop=stop_after_attempt(3))
+ *   - Agent loop: same shape, use httpx.AsyncClient for provider calls
+ *   - Circuit breaker: pybreaker library or roll your own with same state machine
+ *   - Background agent loops: FastAPI BackgroundTasks or Celery
+ *
+ * Django:
+ *   - Services layer (services.py) holds orchestration logic — never in views
+ *   - Circuit breaker state in Django cache (Redis) for multi-process
+ *   - Agent loops in Celery tasks with soft_time_limit for MAX_ITERATIONS equivalent
+ *   - Use django-ratelimit on the view to protect upstream AI spend
+ */
+
+// --- 4. Multi-Tenant AI — per-org isolation, keys, cost tracking ---
+
+/** Per-Tenant Circuit Breakers — scoped by provider+orgId, not just provider.
+ *  One org's invalid API key must not trip the breaker for all orgs. */
+const tenantBreakers = new Map<string, CircuitBreaker>()
+
+function getTenantBreaker(provider: string, orgId: string): CircuitBreaker {
+  const key = `${provider}:${orgId}`
+  let breaker = tenantBreakers.get(key)
+  if (!breaker) {
+    breaker = new CircuitBreaker(5, 60_000)
+    tenantBreakers.set(key, breaker)
+  }
+  return breaker
+}
+
+// Usage:
+//   const breaker = getTenantBreaker('anthropic', org.id)
+//   await breaker.execute(() => summarize(client, text), fallback)
+
+/** Shared Transport with Per-Tenant Keys — one connection pool, per-org auth.
+ *  Avoids N connection pools for N orgs (~100 bytes overhead per org). */
+const sharedTransport = new Anthropic() // Base client — shared pool
+
+function getOrgClient(orgApiKey: string): Anthropic {
+  // Reuses the underlying transport; only overrides auth header
+  return new Anthropic({ apiKey: orgApiKey })
+  // OpenAI: new OpenAI({ apiKey: orgApiKey })
+  // Note: Anthropic SDK creates lightweight client instances.
+  // For providers supporting .withOptions(), prefer that to avoid any pool duplication.
+}
+
+/** API Key Fallback Chain — 3-tier resolution for provider credentials.
+ *  (1) Org-specific from encrypted store → (2) Default org key → (3) Env var. */
+interface CredentialStore {
+  get(orgId: string, provider: string): Promise<string | null>
+  getDefault(provider: string): Promise<string | null>
+}
+
+async function resolveApiKey(
+  orgId: string,
+  provider: string,
+  store: CredentialStore
+): Promise<string> {
+  // Tier 1: org-specific credential
+  const orgKey = await store.get(orgId, provider)
+  if (orgKey) return orgKey
+
+  // Tier 2: default org credential (shared across orgs without their own key)
+  const defaultKey = await store.getDefault(provider)
+  if (defaultKey) return defaultKey
+
+  // Tier 3: environment variable
+  const envMap: Record<string, string> = {
+    anthropic: 'ANTHROPIC_API_KEY',
+    openai: 'OPENAI_API_KEY',
+  }
+  const envKey = process.env[envMap[provider] ?? '']
+  if (envKey) return envKey
+
+  throw new Error(`No API key found for provider=${provider}, orgId=${orgId}`)
+}
+
+/** Credential Verification Probe — validate key before storing.
+ *  Makes a lightweight API call (list models) to confirm the key works. */
+async function verifyCredential(provider: string, apiKey: string): Promise<boolean> {
+  try {
+    if (provider === 'anthropic') {
+      const probe = new Anthropic({ apiKey })
+      // Minimal call — small max_tokens to burn near-zero quota
+      await probe.messages.create({
+        model: 'claude-sonnet-4-20250514',
+        max_tokens: 1,
+        messages: [{ role: 'user', content: 'ping' }],
+      })
+    }
+    // OpenAI: await new OpenAI({ apiKey }).models.list()
+    return true
+  } catch {
+    return false // Invalid key — do not store
+  }
+}
+
+/** Per-Tenant Cost Attribution — thread orgId through all usage recording. */
+interface AiUsageRecord {
+  orgId: string
+  provider: string
+  model: string
+  inputTokens: number
+  outputTokens: number
+  costCents: number
+  timestamp: number
+}
+
+interface UsageSink {
+  record(entry: AiUsageRecord): void
+}
+
+function recordUsage(
+  sink: UsageSink,
+  orgId: string,
+  provider: string,
+  model: string,
+  inputTokens: number,
+  outputTokens: number,
+  costCents: number
+): void {
+  sink.record({ orgId, provider, model, inputTokens, outputTokens, costCents, timestamp: Date.now() })
+}
+
+// Usage:
+//   recordUsage(sink, org.id, 'anthropic', 'claude-sonnet-4-20250514', 320, 150, 2)