spine-framework 0.3.124 → 0.3.126

package/.framework/functions/eval-harness.ts

@@ -1,45 +1,61 @@
-/**
- * @module eval-harness
- * @audience core-contributor
- * @layer api-handler
- * @stability stable
- *
- * Evaluation infrastructure for the AI support loop. Provides CRUD for
- * `eval_test_case` items and a `run_eval` action that exercises the real
- * retrieval + inference pipeline and scores the results.
- *
- * Build evals before you scale automation. A support AI without evals is a
- * demo. With evals it is a product.
- *
- * **Routed by:** `GET/POST /.netlify/functions/eval-harness`
- *
- * **Actions:**
- * | method | ?action          | handler           |
- * |--------|------------------|-------------------|
- * | POST   | create_test_case | createTestCase    |
- * | GET    | list_test_cases  | listTestCases     |
- * | POST   | run_eval         | runEval           |
- * | GET    | get_results      | getResults        |
- * | GET    | get_coverage     | getCoverage       |
- *
- * INVARIANT: `run_eval` fires the actual retrieval pipeline — not a mock.
- *   Only run against a non-production Supabase project when making schema changes.
- *
- * INVARIANT: Eval results are stored as `pipeline_executions` rows so they
- *   are auditable and time-series comparable.
- *
- * @seeAlso agent-runner.ts (retrieveKnowledge — what is being tested)
- * @seeAlso embeddings.ts (match_embeddings — the retrieval function under test)
- */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+// CIL[BEGIN chunk_id="eval-harness_moduleheader_1783969696" type="module_header" identity="imports" parent="none"]
+// CIL[DOC-MACRO purpose="Import dependencies for eval harness module"]
+// CIL[DOC-RESPONSIBILITY responsibility="Import middleware, database, utilities, and audit functions"]
+// CIL[DOC-CONTEXT module="eval-harness" layer="backend_functions" related_files="[\"custom/apps/cortex/pages/operations/OperationsDashboard.tsx\",\".framework/migrations/015_simplify_apps_table.sql\",\".framework/src/pages/admin/TypeDetailPage.tsx\",\".framework/functions/_shared/principal.ts\",\".framework/functions/_shared/schema-utils.ts\",\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="imported_modules" errors="none" side_effects="none"]
+// CIL[DOC-MICRO logic_summary="Import required dependencies for evaluation test harness" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-AUDIENCE audience="developer"]
+// CIL[DOC-VECTOR type="context" content="Layer: backend_functions. Direct callers: 0. Direct calls: 0. Related files: custom/apps/cortex/pages/operations/OperationsDashboard.tsx, .framework/migrations/015_simplify_apps_table.sql, .framework/src/pages/admin/TypeDetailPage.tsx, .framework/functions/_shared/principal.ts, .framework/functions/_shared/schema-utils.ts, \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 
 import { createHandler } from './_shared/middleware'
 import { adminDb } from './_shared/db'
 import { resolveTypeId } from './_shared/resolve-ids'
 import { create } from './admin-data'
 import { emitLog } from './_shared/audit'
-
-// ─── TEST CATEGORIES ─────────────────────────────────────────────────────────
-
+import { callOpenAIEmbeddings } from './_shared/llm'
+// CIL[END chunk_id="eval-harness_moduleheader_1783969696"]
+// CIL[BEGIN chunk_id="eval-harness_constants_1783969696" type="function" identity="constants" parent="none"]
+// CIL[DOC-MACRO purpose="Define evaluation constants and types"]
+// CIL[DOC-RESPONSIBILITY responsibility="Configure test types and confidence tier thresholds"]
+// CIL[DOC-CONTEXT module="eval-harness" layer="backend_functions" related_files="[\"custom/apps/portal/pages/ContentPage.tsx\",\"custom/apps/cortex/pages/operations/OperationsDashboard.tsx\",\".framework/src/pages/admin/IntegrationsPage.tsx\",\".framework/functions/_shared/principal.ts\",\".framework/functions/_shared/schema-utils.ts\",\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="constants" errors="none" side_effects="none"]
+// CIL[DOC-MICRO logic_summary="Define test categories and confidence score ranges for evaluation" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: backend_functions. Direct callers: 0. Direct calls: 0. Related files: custom/apps/portal/pages/ContentPage.tsx, custom/apps/cortex/pages/operations/OperationsDashboard.tsx, .framework/src/pages/admin/IntegrationsPage.tsx, .framework/functions/_shared/principal.ts, .framework/functions/_shared/schema-utils.ts, \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 const TEST_TYPES = ['easy', 'medium', 'hard', 'security', 'hallucination', 'stale', 'account_gating'] as const
 type TestType = typeof TEST_TYPES[number]
 
@@ -48,15 +64,21 @@ const CONFIDENCE_TIERS = {
   medium: { min: 0.60, max: 0.849 },
   low:    { min: 0.00, max: 0.599 },
 }
-
-// ─── CREATE TEST CASE ─────────────────────────────────────────────────────────
-/**
- * Creates an eval_test_case item.
- *
- * Body: question (required), expected_tier ('high'|'medium'|'low'),
- *   expected_escalate (bool), expected_source_ids (string[]),
- *   excluded_source_ids (string[]), test_type
- */
+// CIL[END chunk_id="eval-harness_constants_1783969696"]
+// CIL[BEGIN chunk_id="eval-harness_createtestcase_1783969696" type="function" identity="createTestCase" parent="none"]
+// CIL[DOC-MACRO purpose="Create evaluation test case"]
+// CIL[DOC-RESPONSIBILITY responsibility="Create test case with question and expected outcomes"]
+// CIL[DOC-CONTEXT module="eval-harness" layer="backend_functions" related_files="[\".framework/src/pages/admin/TypeDetailPage.tsx\",\".framework/src/components/ui/dialog.tsx\",\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="created_test_case" errors="validation_errors,db_errors" side_effects="db_write"]
+// CIL[DOC-MICRO logic_summary="Validate input, resolve type ID, create test case item" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: backend_functions. Direct callers: 1. Direct calls: 0. Related files: .framework/src/pages/admin/TypeDetailPage.tsx, .framework/src/components/ui/dialog.tsx, \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 1. Total affected: 2."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 export const createTestCase = createHandler(async (ctx, body) => {
   const {
     question, expected_tier = 'medium', expected_escalate = false,
@@ -85,12 +107,21 @@ export const createTestCase = createHandler(async (ctx, body) => {
     },
   })
 })
-
-// ─── LIST TEST CASES ──────────────────────────────────────────────────────────
-/**
- * Lists eval_test_case items. Optionally filter by test_type.
- * Query params: test_type, limit (default 200), offset
- */
+// CIL[END chunk_id="eval-harness_createtestcase_1783969696"]
+// CIL[BEGIN chunk_id="eval-harness_listtestcases_1783969696" type="function" identity="listTestCases" parent="none"]
+// CIL[DOC-MACRO purpose="List evaluation test cases"]
+// CIL[DOC-RESPONSIBILITY responsibility="Retrieve test cases with optional filtering"]
+// CIL[DOC-CONTEXT module="eval-harness" layer="backend_functions" related_files="[\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="test_cases" errors="db_errors" side_effects="none"]
+// CIL[DOC-MICRO logic_summary="Query test cases by type and test type, return paginated results" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: backend_functions. Direct callers: 0. Direct calls: 0. Related files: \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 export const listTestCases = createHandler(async (ctx, _body) => {
   const { test_type, limit = '200', offset = '0' } = ctx.query || {}
 
@@ -111,35 +142,25 @@ export const listTestCases = createHandler(async (ctx, _body) => {
   if (error) throw error
   return data || []
 })
-
-// ─── RUN EVAL ─────────────────────────────────────────────────────────────────
-/**
- * Runs the retrieval pipeline against all active eval_test_case items and
- * scores the results. Stores the run summary as a pipeline_executions row.
- *
- * Scoring per test case:
- *   - retrieval_recall: expected_source_ids found in top-N results?
- *   - boundary_safe: excluded_source_ids NOT found in results?
- *   - tier_correct: mean similarity maps to expected_tier?
- *
- * Body: test_type (optional, filter), similarity_threshold (default 0.15),
- *       match_count (default 8)
- *
- * Returns a summary + per-case results.
- *
- * NOTE: Does not call the LLM — retrieval only. Full inference eval is a
- * future extension.
- */
+// CIL[END chunk_id="eval-harness_listtestcases_1783969696"]
+// CIL[BEGIN chunk_id="eval-harness_runeval_1783969696" type="function" identity="runEval" parent="none"]
+// CIL[DOC-MACRO purpose="Run evaluation test suite"]
+// CIL[DOC-RESPONSIBILITY responsibility="Execute test cases and measure retrieval performance"]
+// CIL[DOC-CONTEXT module="eval-harness" layer="backend_functions" related_files="[\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="eval_results" errors="api_errors,db_errors" side_effects="api_calls,db_write,audit_log"]
+// CIL[DOC-MICRO logic_summary="Run embeddings search, evaluate recall and boundary violations, store results" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="complex_logic,external_api"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: backend_functions. Direct callers: 0. Direct calls: 0. Related files: \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 export const runEval = createHandler(async (ctx, body) => {
   const { test_type, similarity_threshold = 0.15, match_count = 8 } = body || {}
   if (!ctx.accountId) throw new Error('Account context required')
 
-  const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
-  if (!apiKey) throw new Error('No OPENAI_API_KEY — cannot run eval (needs embedding API)')
-
-  const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
-
-  // Load test cases
   const { data: typeRow } = await adminDb
     .from('types').select('id').eq('slug', 'eval_test_case').eq('kind', 'item').maybeSingle()
 
@@ -150,13 +171,13 @@ export const runEval = createHandler(async (ctx, body) => {
   if (tcErr) throw tcErr
   if (!testCases?.length) return { run_id: null, message: 'No test cases found', total: 0 }
 
-  // Resolve platform account for cross-account retrieval
+  
   const { data: platformAccount } = await adminDb
     .from('accounts').select('id').eq('slug', 'spine-system').maybeSingle()
   const accountIds = [ctx.accountId]
   if (platformAccount?.id && platformAccount.id !== ctx.accountId) accountIds.push(platformAccount.id)
 
-  // Run retrieval for each test case
+  
   const results: any[] = []
   let passed = 0; let failed = 0; let skipped = 0
 
@@ -167,23 +188,23 @@ export const runEval = createHandler(async (ctx, body) => {
     const expectedTier = tc.data?.expected_tier || 'medium'
 
     try {
-      // Generate embedding for the question
-      const embedRes = await fetch(`${baseUrl}/embeddings`, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
-        body: JSON.stringify({ model: 'text-embedding-3-small', input: question }),
-      })
-      if (!embedRes.ok) { skipped++; results.push({ id: tc.id, status: 'skipped', reason: 'embed_api_error' }); continue }
-      const embedJson: any = await embedRes.json()
+      
+      let embedJson: any
+      try {
+        embedJson = await callOpenAIEmbeddings(ctx, [question], 'text-embedding-3-small')
+      } catch {
+        skipped++; results.push({ id: tc.id, status: 'skipped', reason: 'embed_api_error' }); continue
+      }
       const queryEmbedding: number[] = embedJson.data?.[0]?.embedding || []
 
-      // Run retrieval
+      
       const { data: matches } = await adminDb.rpc('match_embeddings', {
         query_embedding: `[${queryEmbedding.join(',')}]`,
         match_count: match_count * 2,
         filter_account_ids: accountIds,
         filter_vector_type: 'semantic',
         similarity_threshold,
+        filter_item_types: null,
       })
 
       const retrievedIds = new Set((matches || []).map((m: any) => m.document_id))
@@ -191,14 +212,14 @@ export const runEval = createHandler(async (ctx, body) => {
         ? (matches as any[]).reduce((s: number, m: any) => s + m.similarity, 0) / matches.length
         : 0
 
-      // Score
+      
       const recallHits     = expectedSrcs.filter((id: string) => retrievedIds.has(id)).length
       const retrievalRecall = expectedSrcs.length > 0 ? recallHits / expectedSrcs.length : null
 
       const boundaryViolations = excludedSrcs.filter((id: string) => retrievedIds.has(id))
       const boundarySafe = boundaryViolations.length === 0
 
-      // Tier based on mean similarity (proxy — full LLM confidence would be better)
+      
       let inferredTier: string
       if (meanSim >= CONFIDENCE_TIERS.high.min) inferredTier = 'high'
       else if (meanSim >= CONFIDENCE_TIERS.medium.min) inferredTier = 'medium'
@@ -241,7 +262,7 @@ export const runEval = createHandler(async (ctx, body) => {
     account_id: ctx.accountId,
   }
 
-  // Store as pipeline_executions row for audit trail
+  
   const { data: execRow } = await adminDb
     .from('pipeline_executions')
     .insert({
@@ -250,7 +271,7 @@ export const runEval = createHandler(async (ctx, body) => {
       status:      failed === 0 ? 'completed' : 'failed',
       trigger_data: { eval_run: true, test_type: test_type || 'all' },
       result:      { summary, results },
-      started_at:  summary.run_at,
+      started_at: summary.run_at,
       completed_at: new Date().toISOString(),
       duration_ms: 0,
     })
@@ -264,12 +285,21 @@ export const runEval = createHandler(async (ctx, body) => {
 
   return { run_id: execRow?.id || null, summary, results }
 })
-
-// ─── GET RESULTS ─────────────────────────────────────────────────────────────
-/**
- * Returns the most recent eval run summary for this account.
- * Query params: limit (default 5) — number of recent runs to return
- */
+// CIL[END chunk_id="eval-harness_runeval_1783969696"]
+// CIL[BEGIN chunk_id="eval-harness_getresults_1783969696" type="function" identity="getResults" parent="none"]
+// CIL[DOC-MACRO purpose="Get evaluation run results"]
+// CIL[DOC-RESPONSIBILITY responsibility="Retrieve recent evaluation execution results"]
+// CIL[DOC-CONTEXT module="eval-harness" layer="backend_functions" related_files="[]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="eval_results" errors="auth_errors,db_errors" side_effects="none"]
+// CIL[DOC-MICRO logic_summary="Query pipeline executions for eval runs, return summary and results" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: backend_functions. Direct callers: 0. Direct calls: 0. Related files: none."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 export const getResults = createHandler(async (ctx, _body) => {
   if (!ctx.accountId) throw new Error('Account context required')
 
@@ -286,12 +316,21 @@ export const getResults = createHandler(async (ctx, _body) => {
   if (error) throw error
   return data || []
 })
-
-// ─── GET COVERAGE ────────────────────────────────────────────────────────────
-/**
- * Returns coverage summary: which test categories have cases, which are empty,
- * and the most recent pass rate per category if eval runs exist.
- */
+// CIL[END chunk_id="eval-harness_getresults_1783969696"]
+// CIL[BEGIN chunk_id="eval-harness_getcoverage_1783969696" type="function" identity="getCoverage" parent="none"]
+// CIL[DOC-MACRO purpose="Get test case coverage analysis"]
+// CIL[DOC-RESPONSIBILITY responsibility="Analyze test case distribution across types"]
+// CIL[DOC-CONTEXT module="eval-harness" layer="backend_functions" related_files="[]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="coverage_report" errors="db_errors" side_effects="none"]
+// CIL[DOC-MICRO logic_summary="Count test cases by type, identify coverage gaps" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: backend_functions. Direct callers: 0. Direct calls: 0. Related files: none."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 export const getCoverage = createHandler(async (ctx, _body) => {
   const { data: typeRow } = await adminDb
     .from('types').select('id').eq('slug', 'eval_test_case').eq('kind', 'item').maybeSingle()
@@ -322,9 +361,21 @@ export const getCoverage = createHandler(async (ctx, _body) => {
     recommendation: coverage.filter(c => !c.has_cases).map(c => c.test_type),
   }
 })
-
-// ─── HANDLER ─────────────────────────────────────────────────────────────────
-
+// CIL[END chunk_id="eval-harness_getcoverage_1783969696"]
+// CIL[BEGIN chunk_id="eval-harness_handler_1783969696" type="function" identity="handler" parent="none"]
+// CIL[DOC-MACRO purpose="Route eval harness HTTP methods"]
+// CIL[DOC-RESPONSIBILITY responsibility="Dispatch requests to evaluation operations"]
+// CIL[DOC-CONTEXT module="eval-harness" layer="backend_functions" related_files="[]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="function_responses" errors="method_errors" side_effects="none"]
+// CIL[DOC-MICRO logic_summary="Route based on action and method to specific eval functions" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: backend_functions. Direct callers: 0. Direct calls: 0. Related files: none."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 export const handler = createHandler(async (ctx, body) => {
   const { action } = ctx.query || {}
   const method = ctx.query?.method || 'GET'
@@ -351,3 +402,4 @@ export const handler = createHandler(async (ctx, body) => {
 
   throw new Error('Invalid action or method')
 })
+// CIL[END chunk_id="eval-harness_handler_1783969696"]