spine-framework 0.3.108 → 0.3.110

package/.framework/functions/eval-harness.ts

@@ -0,0 +1,353 @@
+/**
+ * @module eval-harness
+ * @audience core-contributor
+ * @layer api-handler
+ * @stability stable
+ *
+ * Evaluation infrastructure for the AI support loop. Provides CRUD for
+ * `eval_test_case` items and a `run_eval` action that exercises the real
+ * retrieval + inference pipeline and scores the results.
+ *
+ * Build evals before you scale automation. A support AI without evals is a
+ * demo. With evals it is a product.
+ *
+ * **Routed by:** `GET/POST /.netlify/functions/eval-harness`
+ *
+ * **Actions:**
+ * | method | ?action          | handler           |
+ * |--------|------------------|-------------------|
+ * | POST   | create_test_case | createTestCase    |
+ * | GET    | list_test_cases  | listTestCases     |
+ * | POST   | run_eval         | runEval           |
+ * | GET    | get_results      | getResults        |
+ * | GET    | get_coverage     | getCoverage       |
+ *
+ * INVARIANT: `run_eval` fires the actual retrieval pipeline — not a mock.
+ *   Only run against a non-production Supabase project when making schema changes.
+ *
+ * INVARIANT: Eval results are stored as `pipeline_executions` rows so they
+ *   are auditable and time-series comparable.
+ *
+ * @seeAlso agent-runner.ts (retrieveKnowledge — what is being tested)
+ * @seeAlso embeddings.ts (match_embeddings — the retrieval function under test)
+ */
+
+import { createHandler } from './_shared/middleware'
+import { adminDb } from './_shared/db'
+import { resolveTypeId } from './_shared/resolve-ids'
+import { create } from './admin-data'
+import { emitLog } from './_shared/audit'
+
+// ─── TEST CATEGORIES ─────────────────────────────────────────────────────────
+
+const TEST_TYPES = ['easy', 'medium', 'hard', 'security', 'hallucination', 'stale', 'account_gating'] as const
+type TestType = typeof TEST_TYPES[number]
+
+const CONFIDENCE_TIERS = {
+  high:   { min: 0.85, max: 1.00 },
+  medium: { min: 0.60, max: 0.849 },
+  low:    { min: 0.00, max: 0.599 },
+}
+
+// ─── CREATE TEST CASE ─────────────────────────────────────────────────────────
+/**
+ * Creates an eval_test_case item.
+ *
+ * Body: question (required), expected_tier ('high'|'medium'|'low'),
+ *   expected_escalate (bool), expected_source_ids (string[]),
+ *   excluded_source_ids (string[]), test_type
+ */
+export const createTestCase = createHandler(async (ctx, body) => {
+  const {
+    question, expected_tier = 'medium', expected_escalate = false,
+    expected_source_ids = [], excluded_source_ids = [],
+    test_type = 'medium',
+  } = body || {}
+
+  if (!question) throw new Error('question is required')
+
+  let typeId: string | null = null
+  try { typeId = await resolveTypeId('item', 'eval_test_case') } catch { /* not yet seeded */ }
+  if (!typeId) throw new Error('eval_test_case type not found — run app install first')
+
+  return await create(ctx, {
+    entity: 'items',
+    type_id: typeId,
+    title: question.slice(0, 120),
+    status: 'active',
+    data: {
+      question,
+      expected_tier,
+      expected_escalate,
+      expected_source_ids,
+      excluded_source_ids,
+      test_type,
+    },
+  })
+})
+
+// ─── LIST TEST CASES ──────────────────────────────────────────────────────────
+/**
+ * Lists eval_test_case items. Optionally filter by test_type.
+ * Query params: test_type, limit (default 200), offset
+ */
+export const listTestCases = createHandler(async (ctx, _body) => {
+  const { test_type, limit = '200', offset = '0' } = ctx.query || {}
+
+  const { data: typeRow } = await adminDb
+    .from('types').select('id').eq('slug', 'eval_test_case').eq('kind', 'item').maybeSingle()
+
+  let q = ctx.db
+    .from('items')
+    .select('*')
+    .eq('status', 'active')
+    .order('created_at', { ascending: true })
+    .range(Number(offset), Number(offset) + Number(limit) - 1)
+
+  if (typeRow?.id) q = q.eq('type_id', typeRow.id)
+  if (test_type) q = q.eq('data->>test_type', test_type)
+
+  const { data, error } = await q
+  if (error) throw error
+  return data || []
+})
+
+// ─── RUN EVAL ─────────────────────────────────────────────────────────────────
+/**
+ * Runs the retrieval pipeline against all active eval_test_case items and
+ * scores the results. Stores the run summary as a pipeline_executions row.
+ *
+ * Scoring per test case:
+ *   - retrieval_recall: expected_source_ids found in top-N results?
+ *   - boundary_safe: excluded_source_ids NOT found in results?
+ *   - tier_correct: mean similarity maps to expected_tier?
+ *
+ * Body: test_type (optional, filter), similarity_threshold (default 0.15),
+ *       match_count (default 8)
+ *
+ * Returns a summary + per-case results.
+ *
+ * NOTE: Does not call the LLM — retrieval only. Full inference eval is a
+ * future extension.
+ */
+export const runEval = createHandler(async (ctx, body) => {
+  const { test_type, similarity_threshold = 0.15, match_count = 8 } = body || {}
+  if (!ctx.accountId) throw new Error('Account context required')
+
+  const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
+  if (!apiKey) throw new Error('No OPENAI_API_KEY — cannot run eval (needs embedding API)')
+
+  const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
+
+  // Load test cases
+  const { data: typeRow } = await adminDb
+    .from('types').select('id').eq('slug', 'eval_test_case').eq('kind', 'item').maybeSingle()
+
+  let q = adminDb.from('items').select('*').eq('status', 'active').limit(500)
+  if (typeRow?.id) q = q.eq('type_id', typeRow.id)
+  if (test_type)   q = q.eq('data->>test_type', test_type)
+  const { data: testCases, error: tcErr } = await q
+  if (tcErr) throw tcErr
+  if (!testCases?.length) return { run_id: null, message: 'No test cases found', total: 0 }
+
+  // Resolve platform account for cross-account retrieval
+  const { data: platformAccount } = await adminDb
+    .from('accounts').select('id').eq('slug', 'spine-system').maybeSingle()
+  const accountIds = [ctx.accountId]
+  if (platformAccount?.id && platformAccount.id !== ctx.accountId) accountIds.push(platformAccount.id)
+
+  // Run retrieval for each test case
+  const results: any[] = []
+  let passed = 0; let failed = 0; let skipped = 0
+
+  for (const tc of testCases) {
+    const question     = tc.data?.question || tc.title
+    const expectedSrcs = tc.data?.expected_source_ids || []
+    const excludedSrcs = tc.data?.excluded_source_ids || []
+    const expectedTier = tc.data?.expected_tier || 'medium'
+
+    try {
+      // Generate embedding for the question
+      const embedRes = await fetch(`${baseUrl}/embeddings`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+        body: JSON.stringify({ model: 'text-embedding-3-small', input: question }),
+      })
+      if (!embedRes.ok) { skipped++; results.push({ id: tc.id, status: 'skipped', reason: 'embed_api_error' }); continue }
+      const embedJson: any = await embedRes.json()
+      const queryEmbedding: number[] = embedJson.data?.[0]?.embedding || []
+
+      // Run retrieval
+      const { data: matches } = await adminDb.rpc('match_embeddings', {
+        query_embedding: `[${queryEmbedding.join(',')}]`,
+        match_count: match_count * 2,
+        filter_account_ids: accountIds,
+        filter_vector_type: 'semantic',
+        similarity_threshold,
+      })
+
+      const retrievedIds = new Set((matches || []).map((m: any) => m.document_id))
+      const meanSim = matches?.length
+        ? (matches as any[]).reduce((s: number, m: any) => s + m.similarity, 0) / matches.length
+        : 0
+
+      // Score
+      const recallHits     = expectedSrcs.filter((id: string) => retrievedIds.has(id)).length
+      const retrievalRecall = expectedSrcs.length > 0 ? recallHits / expectedSrcs.length : null
+
+      const boundaryViolations = excludedSrcs.filter((id: string) => retrievedIds.has(id))
+      const boundarySafe = boundaryViolations.length === 0
+
+      // Tier based on mean similarity (proxy — full LLM confidence would be better)
+      let inferredTier: string
+      if (meanSim >= CONFIDENCE_TIERS.high.min) inferredTier = 'high'
+      else if (meanSim >= CONFIDENCE_TIERS.medium.min) inferredTier = 'medium'
+      else inferredTier = 'low'
+      const tierCorrect = inferredTier === expectedTier
+
+      const casePassed = boundarySafe &&
+        (retrievalRecall === null || retrievalRecall >= 0.5) &&
+        tierCorrect
+
+      if (casePassed) passed++; else failed++
+
+      results.push({
+        id:                  tc.id,
+        question:            question.slice(0, 80),
+        test_type:           tc.data?.test_type,
+        status:              casePassed ? 'pass' : 'fail',
+        retrieval_recall:    retrievalRecall,
+        boundary_safe:       boundarySafe,
+        boundary_violations: boundaryViolations,
+        tier_correct:        tierCorrect,
+        inferred_tier:       inferredTier,
+        expected_tier:       expectedTier,
+        mean_similarity:     Math.round(meanSim * 1000) / 1000,
+        retrieved_count:     retrievedIds.size,
+      })
+    } catch (err: any) {
+      skipped++
+      results.push({ id: tc.id, status: 'skipped', reason: err.message })
+    }
+  }
+
+  const summary = {
+    total: testCases.length,
+    passed,
+    failed,
+    skipped,
+    pass_rate: testCases.length > 0 ? Math.round((passed / testCases.length) * 100) : 0,
+    run_at: new Date().toISOString(),
+    account_id: ctx.accountId,
+  }
+
+  // Store as pipeline_executions row for audit trail
+  const { data: execRow } = await adminDb
+    .from('pipeline_executions')
+    .insert({
+      pipeline_id: '00000000-0000-0000-0000-000000000000', // sentinel UUID for eval runs
+      account_id:  ctx.accountId,
+      status:      failed === 0 ? 'completed' : 'failed',
+      trigger_data: { eval_run: true, test_type: test_type || 'all' },
+      result:      { summary, results },
+      started_at:  summary.run_at,
+      completed_at: new Date().toISOString(),
+      duration_ms: 0,
+    })
+    .select('id')
+    .single()
+
+  await emitLog(ctx, 'eval.run_completed',
+    { type: 'system', id: execRow?.id || 'eval' },
+    { after: summary }
+  )
+
+  return { run_id: execRow?.id || null, summary, results }
+})
+
+// ─── GET RESULTS ─────────────────────────────────────────────────────────────
+/**
+ * Returns the most recent eval run summary for this account.
+ * Query params: limit (default 5) — number of recent runs to return
+ */
+export const getResults = createHandler(async (ctx, _body) => {
+  if (!ctx.accountId) throw new Error('Account context required')
+
+  const limit = Number(ctx.query?.limit || 5)
+
+  const { data, error } = await adminDb
+    .from('pipeline_executions')
+    .select('id, status, result, started_at, completed_at, trigger_data')
+    .eq('account_id', ctx.accountId)
+    .eq('trigger_data->>eval_run', 'true')
+    .order('started_at', { ascending: false })
+    .limit(limit)
+
+  if (error) throw error
+  return data || []
+})
+
+// ─── GET COVERAGE ────────────────────────────────────────────────────────────
+/**
+ * Returns coverage summary: which test categories have cases, which are empty,
+ * and the most recent pass rate per category if eval runs exist.
+ */
+export const getCoverage = createHandler(async (ctx, _body) => {
+  const { data: typeRow } = await adminDb
+    .from('types').select('id').eq('slug', 'eval_test_case').eq('kind', 'item').maybeSingle()
+
+  const { data: testCases } = await adminDb
+    .from('items')
+    .select('data')
+    .eq('status', 'active')
+    .eq('type_id', typeRow?.id || '00000000-0000-0000-0000-000000000000')
+    .limit(1000)
+
+  const countByType: Record<string, number> = {}
+  for (const tt of TEST_TYPES) countByType[tt] = 0
+  for (const tc of testCases || []) {
+    const t = tc.data?.test_type || 'medium'
+    countByType[t] = (countByType[t] || 0) + 1
+  }
+
+  const coverage = TEST_TYPES.map(tt => ({
+    test_type:   tt,
+    case_count:  countByType[tt] || 0,
+    has_cases:   (countByType[tt] || 0) > 0,
+  }))
+
+  return {
+    total_cases: testCases?.length || 0,
+    coverage,
+    recommendation: coverage.filter(c => !c.has_cases).map(c => c.test_type),
+  }
+})
+
+// ─── HANDLER ─────────────────────────────────────────────────────────────────
+
+export const handler = createHandler(async (ctx, body) => {
+  const { action } = ctx.query || {}
+  const method = ctx.query?.method || 'GET'
+
+  switch (action) {
+    case 'create_test_case':
+      if (method === 'POST') return await createTestCase(ctx, body)
+      break
+    case 'list_test_cases':
+      if (method === 'GET') return await listTestCases(ctx, body)
+      break
+    case 'run_eval':
+      if (method === 'POST') return await runEval(ctx, body)
+      break
+    case 'get_results':
+      if (method === 'GET') return await getResults(ctx, body)
+      break
+    case 'get_coverage':
+      if (method === 'GET') return await getCoverage(ctx, body)
+      break
+    default:
+      if (method === 'GET') return await listTestCases(ctx, body)
+  }
+
+  throw new Error('Invalid action or method')
+})

package/.framework/functions/observability.ts

@@ -18,6 +18,7 @@
  * | latency_percentiles   | getLatencyPercentiles  |
  * | pipeline_stats        | getPipelineStats       |
  * | top_actors            | getTopActors           |
+ * | ai_support_metrics    | getAiSupportMetrics    |
  * | cleanup               | cleanupOldLogs (POST)  |
  *
  * **Authorization:** All read endpoints require account context (RLS-scoped
@@ -32,6 +33,7 @@
  */
 
 import { createHandler } from './_shared/middleware'
+import { adminDb } from './_shared/db'
 
 // ─── HANDLERS ─────────────────────────────────────────────────────────────────
 
@@ -245,6 +247,147 @@ export const cleanupOldLogs = createHandler(async (ctx, body) => {
   return { deleted_count: data?.[0]?.deleted_count || 0 }
 })
 
+// ─── AI SUPPORT METRICS ──────────────────────────────────────────────────────
+
+/**
+ * Returns the AI support loop health metrics.
+ * All values are derived from existing tables — no new table required.
+ *
+ * Metrics:
+ *   - ai_attempted:        AI agent messages sent
+ *   - ai_resolved:         Tickets resolved without escalation
+ *   - human_escalations:   Tickets assigned to humans
+ *   - thumbs_down_rate:    Ratio of not_helpful / total AI messages (0–1)
+ *   - medium_confidence:   AI messages with confidence 0.60–0.85
+ *   - kb_articles_created: Postmortem KB articles
+ *   - kb_reuse_count:      Total reuse_count across ai_feedback_record items
+ *   - knowledge_gaps:      Interactions with kb_candidate=false + not_helpful feedback
+ *
+ * Query params: from, to (ISO dates, optional — defaults to last 30 days)
+ *
+ * @sideEffects DB reads: items, messages tables
+ * @calledBy handler (?action=ai_support_metrics)
+ */
+export const getAiSupportMetrics = createHandler(async (ctx, _body) => {
+  if (!ctx.accountId) throw new Error('Account context required')
+
+  const now = new Date()
+  const thirtyDaysAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString()
+  const from = ctx.query?.from || thirtyDaysAgo
+  const to   = ctx.query?.to   || now.toISOString()
+
+  // Run all queries in parallel for performance
+  const [
+    aiMessagesResult,
+    resolvedResult,
+    escalatedResult,
+    notHelpfulResult,
+    mediumConfResult,
+    kbCreatedResult,
+    feedbackRecordsResult,
+  ] = await Promise.allSettled([
+    // AI agent messages
+    adminDb
+      .from('messages')
+      .select('id, data', { count: 'exact', head: false })
+      .eq('account_id', ctx.accountId)
+      .eq('data->>sender_type', 'agent')
+      .gte('created_at', from)
+      .lte('created_at', to),
+
+    // Tickets resolved without AI escalation
+    adminDb
+      .from('items')
+      .select('id', { count: 'exact', head: true })
+      .eq('account_id', ctx.accountId)
+      .eq('status', 'resolved')
+      .eq('data->>aim_escalation_reason', 'none')
+      .gte('created_at', from)
+      .lte('created_at', to),
+
+    // Tickets escalated to humans
+    adminDb
+      .from('items')
+      .select('id', { count: 'exact', head: true })
+      .eq('account_id', ctx.accountId)
+      .eq('status', 'human_assigned')
+      .gte('created_at', from)
+      .lte('created_at', to),
+
+    // Not-helpful AI messages
+    adminDb
+      .from('messages')
+      .select('id', { count: 'exact', head: true })
+      .eq('account_id', ctx.accountId)
+      .eq('data->>sender_type', 'agent')
+      .eq('data->>feedback', 'not_helpful')
+      .gte('created_at', from)
+      .lte('created_at', to),
+
+    // Medium-confidence AI messages
+    adminDb
+      .from('messages')
+      .select('id', { count: 'exact', head: true })
+      .eq('account_id', ctx.accountId)
+      .eq('data->>sender_type', 'agent')
+      .gte('data->>confidence', '0.60')
+      .lte('data->>confidence', '0.85')
+      .gte('created_at', from)
+      .lte('created_at', to),
+
+    // Postmortem KB articles created
+    adminDb
+      .from('items')
+      .select('id', { count: 'exact', head: true })
+      .eq('account_id', ctx.accountId)
+      .eq('data->>kb_type', 'postmortem')
+      .gte('created_at', from)
+      .lte('created_at', to),
+
+    // AI feedback records for reuse + gaps
+    adminDb
+      .from('items')
+      .select('data')
+      .eq('account_id', ctx.accountId)
+      .gte('created_at', from)
+      .lte('created_at', to)
+      .limit(500),
+  ])
+
+  // Extract values safely
+  const aiMessages = aiMessagesResult.status === 'fulfilled' ? (aiMessagesResult.value.data || []) : []
+  const aiAttempted = aiMessages.length
+
+  const aiResolved   = resolvedResult.status === 'fulfilled'   ? (resolvedResult.value.count   || 0) : 0
+  const escalated    = escalatedResult.status === 'fulfilled'  ? (escalatedResult.value.count  || 0) : 0
+  const notHelpful   = notHelpfulResult.status === 'fulfilled' ? (notHelpfulResult.value.count || 0) : 0
+  const mediumConf   = mediumConfResult.status === 'fulfilled' ? (mediumConfResult.value.count || 0) : 0
+  const kbCreated    = kbCreatedResult.status === 'fulfilled'  ? (kbCreatedResult.value.count  || 0) : 0
+
+  const feedbackRows = feedbackRecordsResult.status === 'fulfilled'
+    ? (feedbackRecordsResult.value.data || [])
+    : []
+
+  const kbReuseCount = feedbackRows.reduce((sum: number, r: any) => sum + (Number(r.data?.reuse_count) || 0), 0)
+  const knowledgeGaps = feedbackRows.filter((r: any) =>
+    r.data?.kb_candidate === false && r.data?.customer_feedback === 'not_helpful'
+  ).length
+
+  const thumbsDownRate = aiAttempted > 0 ? notHelpful / aiAttempted : 0
+
+  return {
+    period: { from, to },
+    ai_attempted:       aiAttempted,
+    ai_resolved:        aiResolved,
+    human_escalations:  escalated,
+    thumbs_down_rate:   Math.round(thumbsDownRate * 1000) / 1000,
+    medium_confidence:  mediumConf,
+    kb_articles_created: kbCreated,
+    kb_reuse_count:     kbReuseCount,
+    knowledge_gaps:     knowledgeGaps,
+  }
+})
+
 // ─── MAIN HANDLER ────────────────────────────────────────────────────────────
 
 /**
@@ -267,9 +410,11 @@ export const handler = createHandler(async (ctx, body) => {
       return await getPipelineStats(ctx, body)
     case 'top_actors':
       return await getTopActors(ctx, body)
+    case 'ai_support_metrics':
+      return await getAiSupportMetrics(ctx, body)
     case 'cleanup':
       return await cleanupOldLogs(ctx, body)
     default:
-      throw new Error(`Unknown action: ${action}. Valid actions: event_volume, error_rate, latency_percentiles, pipeline_stats, top_actors, cleanup`)
+      throw new Error(`Unknown action: ${action}. Valid actions: event_volume, error_rate, latency_percentiles, pipeline_stats, top_actors, ai_support_metrics, cleanup`)
   }
 })