spine-framework 0.3.107 → 0.3.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.framework/functions/_shared/agent-runner.ts +146 -88
- package/.framework/functions/_shared/permissions.ts +4 -16
- package/.framework/functions/_shared/pipeline-runner.ts +31 -1
- package/.framework/functions/admin-data.ts +52 -29
- package/.framework/functions/ai-interactions.ts +240 -0
- package/.framework/functions/embeddings.ts +426 -10
- package/.framework/functions/eval-harness.ts +353 -0
- package/.framework/functions/item-progress.ts +12 -5
- package/.framework/functions/observability.ts +146 -1
- package/.framework/migrations/002_rls_and_functions.sql +30 -11
- package/.framework/migrations/026_fix_rls_policies.sql +47 -0
- package/.framework/migrations/027_embedding_lifecycle.sql +176 -0
- package/config/tsconfig.node.tsbuildinfo +1 -1
- package/package.json +3 -1
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module eval-harness
|
|
3
|
+
* @audience core-contributor
|
|
4
|
+
* @layer api-handler
|
|
5
|
+
* @stability stable
|
|
6
|
+
*
|
|
7
|
+
* Evaluation infrastructure for the AI support loop. Provides CRUD for
|
|
8
|
+
* `eval_test_case` items and a `run_eval` action that exercises the real
|
|
9
|
+
* retrieval + inference pipeline and scores the results.
|
|
10
|
+
*
|
|
11
|
+
* Build evals before you scale automation. A support AI without evals is a
|
|
12
|
+
* demo. With evals it is a product.
|
|
13
|
+
*
|
|
14
|
+
* **Routed by:** `GET/POST /.netlify/functions/eval-harness`
|
|
15
|
+
*
|
|
16
|
+
* **Actions:**
|
|
17
|
+
* | method | ?action | handler |
|
|
18
|
+
* |--------|------------------|-------------------|
|
|
19
|
+
* | POST | create_test_case | createTestCase |
|
|
20
|
+
* | GET | list_test_cases | listTestCases |
|
|
21
|
+
* | POST | run_eval | runEval |
|
|
22
|
+
* | GET | get_results | getResults |
|
|
23
|
+
* | GET | get_coverage | getCoverage |
|
|
24
|
+
*
|
|
25
|
+
* INVARIANT: `run_eval` fires the actual retrieval pipeline — not a mock.
|
|
26
|
+
* Only run against a non-production Supabase project when making schema changes.
|
|
27
|
+
*
|
|
28
|
+
* INVARIANT: Eval results are stored as `pipeline_executions` rows so they
|
|
29
|
+
* are auditable and time-series comparable.
|
|
30
|
+
*
|
|
31
|
+
* @seeAlso agent-runner.ts (retrieveKnowledge — what is being tested)
|
|
32
|
+
* @seeAlso embeddings.ts (match_embeddings — the retrieval function under test)
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { createHandler } from './_shared/middleware'
|
|
36
|
+
import { adminDb } from './_shared/db'
|
|
37
|
+
import { resolveTypeId } from './_shared/resolve-ids'
|
|
38
|
+
import { create } from './admin-data'
|
|
39
|
+
import { emitLog } from './_shared/audit'
|
|
40
|
+
|
|
41
|
+
// ─── TEST CATEGORIES ─────────────────────────────────────────────────────────
|
|
42
|
+
|
|
43
|
+
const TEST_TYPES = ['easy', 'medium', 'hard', 'security', 'hallucination', 'stale', 'account_gating'] as const
|
|
44
|
+
type TestType = typeof TEST_TYPES[number]
|
|
45
|
+
|
|
46
|
+
const CONFIDENCE_TIERS = {
|
|
47
|
+
high: { min: 0.85, max: 1.00 },
|
|
48
|
+
medium: { min: 0.60, max: 0.849 },
|
|
49
|
+
low: { min: 0.00, max: 0.599 },
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ─── CREATE TEST CASE ─────────────────────────────────────────────────────────
|
|
53
|
+
/**
|
|
54
|
+
* Creates an eval_test_case item.
|
|
55
|
+
*
|
|
56
|
+
* Body: question (required), expected_tier ('high'|'medium'|'low'),
|
|
57
|
+
* expected_escalate (bool), expected_source_ids (string[]),
|
|
58
|
+
* excluded_source_ids (string[]), test_type
|
|
59
|
+
*/
|
|
60
|
+
export const createTestCase = createHandler(async (ctx, body) => {
|
|
61
|
+
const {
|
|
62
|
+
question, expected_tier = 'medium', expected_escalate = false,
|
|
63
|
+
expected_source_ids = [], excluded_source_ids = [],
|
|
64
|
+
test_type = 'medium',
|
|
65
|
+
} = body || {}
|
|
66
|
+
|
|
67
|
+
if (!question) throw new Error('question is required')
|
|
68
|
+
|
|
69
|
+
let typeId: string | null = null
|
|
70
|
+
try { typeId = await resolveTypeId('item', 'eval_test_case') } catch { /* not yet seeded */ }
|
|
71
|
+
if (!typeId) throw new Error('eval_test_case type not found — run app install first')
|
|
72
|
+
|
|
73
|
+
return await create(ctx, {
|
|
74
|
+
entity: 'items',
|
|
75
|
+
type_id: typeId,
|
|
76
|
+
title: question.slice(0, 120),
|
|
77
|
+
status: 'active',
|
|
78
|
+
data: {
|
|
79
|
+
question,
|
|
80
|
+
expected_tier,
|
|
81
|
+
expected_escalate,
|
|
82
|
+
expected_source_ids,
|
|
83
|
+
excluded_source_ids,
|
|
84
|
+
test_type,
|
|
85
|
+
},
|
|
86
|
+
})
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
// ─── LIST TEST CASES ──────────────────────────────────────────────────────────
|
|
90
|
+
/**
|
|
91
|
+
* Lists eval_test_case items. Optionally filter by test_type.
|
|
92
|
+
* Query params: test_type, limit (default 200), offset
|
|
93
|
+
*/
|
|
94
|
+
export const listTestCases = createHandler(async (ctx, _body) => {
|
|
95
|
+
const { test_type, limit = '200', offset = '0' } = ctx.query || {}
|
|
96
|
+
|
|
97
|
+
const { data: typeRow } = await adminDb
|
|
98
|
+
.from('types').select('id').eq('slug', 'eval_test_case').eq('kind', 'item').maybeSingle()
|
|
99
|
+
|
|
100
|
+
let q = ctx.db
|
|
101
|
+
.from('items')
|
|
102
|
+
.select('*')
|
|
103
|
+
.eq('status', 'active')
|
|
104
|
+
.order('created_at', { ascending: true })
|
|
105
|
+
.range(Number(offset), Number(offset) + Number(limit) - 1)
|
|
106
|
+
|
|
107
|
+
if (typeRow?.id) q = q.eq('type_id', typeRow.id)
|
|
108
|
+
if (test_type) q = q.eq('data->>test_type', test_type)
|
|
109
|
+
|
|
110
|
+
const { data, error } = await q
|
|
111
|
+
if (error) throw error
|
|
112
|
+
return data || []
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
// ─── RUN EVAL ─────────────────────────────────────────────────────────────────
|
|
116
|
+
/**
|
|
117
|
+
* Runs the retrieval pipeline against all active eval_test_case items and
|
|
118
|
+
* scores the results. Stores the run summary as a pipeline_executions row.
|
|
119
|
+
*
|
|
120
|
+
* Scoring per test case:
|
|
121
|
+
* - retrieval_recall: expected_source_ids found in top-N results?
|
|
122
|
+
* - boundary_safe: excluded_source_ids NOT found in results?
|
|
123
|
+
* - tier_correct: mean similarity maps to expected_tier?
|
|
124
|
+
*
|
|
125
|
+
* Body: test_type (optional, filter), similarity_threshold (default 0.15),
|
|
126
|
+
* match_count (default 8)
|
|
127
|
+
*
|
|
128
|
+
* Returns a summary + per-case results.
|
|
129
|
+
*
|
|
130
|
+
* NOTE: Does not call the LLM — retrieval only. Full inference eval is a
|
|
131
|
+
* future extension.
|
|
132
|
+
*/
|
|
133
|
+
export const runEval = createHandler(async (ctx, body) => {
|
|
134
|
+
const { test_type, similarity_threshold = 0.15, match_count = 8 } = body || {}
|
|
135
|
+
if (!ctx.accountId) throw new Error('Account context required')
|
|
136
|
+
|
|
137
|
+
const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
|
|
138
|
+
if (!apiKey) throw new Error('No OPENAI_API_KEY — cannot run eval (needs embedding API)')
|
|
139
|
+
|
|
140
|
+
const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
|
|
141
|
+
|
|
142
|
+
// Load test cases
|
|
143
|
+
const { data: typeRow } = await adminDb
|
|
144
|
+
.from('types').select('id').eq('slug', 'eval_test_case').eq('kind', 'item').maybeSingle()
|
|
145
|
+
|
|
146
|
+
let q = adminDb.from('items').select('*').eq('status', 'active').limit(500)
|
|
147
|
+
if (typeRow?.id) q = q.eq('type_id', typeRow.id)
|
|
148
|
+
if (test_type) q = q.eq('data->>test_type', test_type)
|
|
149
|
+
const { data: testCases, error: tcErr } = await q
|
|
150
|
+
if (tcErr) throw tcErr
|
|
151
|
+
if (!testCases?.length) return { run_id: null, message: 'No test cases found', total: 0 }
|
|
152
|
+
|
|
153
|
+
// Resolve platform account for cross-account retrieval
|
|
154
|
+
const { data: platformAccount } = await adminDb
|
|
155
|
+
.from('accounts').select('id').eq('slug', 'spine-system').maybeSingle()
|
|
156
|
+
const accountIds = [ctx.accountId]
|
|
157
|
+
if (platformAccount?.id && platformAccount.id !== ctx.accountId) accountIds.push(platformAccount.id)
|
|
158
|
+
|
|
159
|
+
// Run retrieval for each test case
|
|
160
|
+
const results: any[] = []
|
|
161
|
+
let passed = 0; let failed = 0; let skipped = 0
|
|
162
|
+
|
|
163
|
+
for (const tc of testCases) {
|
|
164
|
+
const question = tc.data?.question || tc.title
|
|
165
|
+
const expectedSrcs = tc.data?.expected_source_ids || []
|
|
166
|
+
const excludedSrcs = tc.data?.excluded_source_ids || []
|
|
167
|
+
const expectedTier = tc.data?.expected_tier || 'medium'
|
|
168
|
+
|
|
169
|
+
try {
|
|
170
|
+
// Generate embedding for the question
|
|
171
|
+
const embedRes = await fetch(`${baseUrl}/embeddings`, {
|
|
172
|
+
method: 'POST',
|
|
173
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
|
|
174
|
+
body: JSON.stringify({ model: 'text-embedding-3-small', input: question }),
|
|
175
|
+
})
|
|
176
|
+
if (!embedRes.ok) { skipped++; results.push({ id: tc.id, status: 'skipped', reason: 'embed_api_error' }); continue }
|
|
177
|
+
const embedJson: any = await embedRes.json()
|
|
178
|
+
const queryEmbedding: number[] = embedJson.data?.[0]?.embedding || []
|
|
179
|
+
|
|
180
|
+
// Run retrieval
|
|
181
|
+
const { data: matches } = await adminDb.rpc('match_embeddings', {
|
|
182
|
+
query_embedding: `[${queryEmbedding.join(',')}]`,
|
|
183
|
+
match_count: match_count * 2,
|
|
184
|
+
filter_account_ids: accountIds,
|
|
185
|
+
filter_vector_type: 'semantic',
|
|
186
|
+
similarity_threshold,
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
const retrievedIds = new Set((matches || []).map((m: any) => m.document_id))
|
|
190
|
+
const meanSim = matches?.length
|
|
191
|
+
? (matches as any[]).reduce((s: number, m: any) => s + m.similarity, 0) / matches.length
|
|
192
|
+
: 0
|
|
193
|
+
|
|
194
|
+
// Score
|
|
195
|
+
const recallHits = expectedSrcs.filter((id: string) => retrievedIds.has(id)).length
|
|
196
|
+
const retrievalRecall = expectedSrcs.length > 0 ? recallHits / expectedSrcs.length : null
|
|
197
|
+
|
|
198
|
+
const boundaryViolations = excludedSrcs.filter((id: string) => retrievedIds.has(id))
|
|
199
|
+
const boundarySafe = boundaryViolations.length === 0
|
|
200
|
+
|
|
201
|
+
// Tier based on mean similarity (proxy — full LLM confidence would be better)
|
|
202
|
+
let inferredTier: string
|
|
203
|
+
if (meanSim >= CONFIDENCE_TIERS.high.min) inferredTier = 'high'
|
|
204
|
+
else if (meanSim >= CONFIDENCE_TIERS.medium.min) inferredTier = 'medium'
|
|
205
|
+
else inferredTier = 'low'
|
|
206
|
+
const tierCorrect = inferredTier === expectedTier
|
|
207
|
+
|
|
208
|
+
const casePassed = boundarySafe &&
|
|
209
|
+
(retrievalRecall === null || retrievalRecall >= 0.5) &&
|
|
210
|
+
tierCorrect
|
|
211
|
+
|
|
212
|
+
if (casePassed) passed++; else failed++
|
|
213
|
+
|
|
214
|
+
results.push({
|
|
215
|
+
id: tc.id,
|
|
216
|
+
question: question.slice(0, 80),
|
|
217
|
+
test_type: tc.data?.test_type,
|
|
218
|
+
status: casePassed ? 'pass' : 'fail',
|
|
219
|
+
retrieval_recall: retrievalRecall,
|
|
220
|
+
boundary_safe: boundarySafe,
|
|
221
|
+
boundary_violations: boundaryViolations,
|
|
222
|
+
tier_correct: tierCorrect,
|
|
223
|
+
inferred_tier: inferredTier,
|
|
224
|
+
expected_tier: expectedTier,
|
|
225
|
+
mean_similarity: Math.round(meanSim * 1000) / 1000,
|
|
226
|
+
retrieved_count: retrievedIds.size,
|
|
227
|
+
})
|
|
228
|
+
} catch (err: any) {
|
|
229
|
+
skipped++
|
|
230
|
+
results.push({ id: tc.id, status: 'skipped', reason: err.message })
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const summary = {
|
|
235
|
+
total: testCases.length,
|
|
236
|
+
passed,
|
|
237
|
+
failed,
|
|
238
|
+
skipped,
|
|
239
|
+
pass_rate: testCases.length > 0 ? Math.round((passed / testCases.length) * 100) : 0,
|
|
240
|
+
run_at: new Date().toISOString(),
|
|
241
|
+
account_id: ctx.accountId,
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Store as pipeline_executions row for audit trail
|
|
245
|
+
const { data: execRow } = await adminDb
|
|
246
|
+
.from('pipeline_executions')
|
|
247
|
+
.insert({
|
|
248
|
+
pipeline_id: '00000000-0000-0000-0000-000000000000', // sentinel UUID for eval runs
|
|
249
|
+
account_id: ctx.accountId,
|
|
250
|
+
status: failed === 0 ? 'completed' : 'failed',
|
|
251
|
+
trigger_data: { eval_run: true, test_type: test_type || 'all' },
|
|
252
|
+
result: { summary, results },
|
|
253
|
+
started_at: summary.run_at,
|
|
254
|
+
completed_at: new Date().toISOString(),
|
|
255
|
+
duration_ms: 0,
|
|
256
|
+
})
|
|
257
|
+
.select('id')
|
|
258
|
+
.single()
|
|
259
|
+
|
|
260
|
+
await emitLog(ctx, 'eval.run_completed',
|
|
261
|
+
{ type: 'system', id: execRow?.id || 'eval' },
|
|
262
|
+
{ after: summary }
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
return { run_id: execRow?.id || null, summary, results }
|
|
266
|
+
})
|
|
267
|
+
|
|
268
|
+
// ─── GET RESULTS ─────────────────────────────────────────────────────────────
|
|
269
|
+
/**
|
|
270
|
+
* Returns the most recent eval run summary for this account.
|
|
271
|
+
* Query params: limit (default 5) — number of recent runs to return
|
|
272
|
+
*/
|
|
273
|
+
export const getResults = createHandler(async (ctx, _body) => {
|
|
274
|
+
if (!ctx.accountId) throw new Error('Account context required')
|
|
275
|
+
|
|
276
|
+
const limit = Number(ctx.query?.limit || 5)
|
|
277
|
+
|
|
278
|
+
const { data, error } = await adminDb
|
|
279
|
+
.from('pipeline_executions')
|
|
280
|
+
.select('id, status, result, started_at, completed_at, trigger_data')
|
|
281
|
+
.eq('account_id', ctx.accountId)
|
|
282
|
+
.eq('trigger_data->>eval_run', 'true')
|
|
283
|
+
.order('started_at', { ascending: false })
|
|
284
|
+
.limit(limit)
|
|
285
|
+
|
|
286
|
+
if (error) throw error
|
|
287
|
+
return data || []
|
|
288
|
+
})
|
|
289
|
+
|
|
290
|
+
// ─── GET COVERAGE ────────────────────────────────────────────────────────────
|
|
291
|
+
/**
|
|
292
|
+
* Returns coverage summary: which test categories have cases, which are empty,
|
|
293
|
+
* and the most recent pass rate per category if eval runs exist.
|
|
294
|
+
*/
|
|
295
|
+
export const getCoverage = createHandler(async (ctx, _body) => {
|
|
296
|
+
const { data: typeRow } = await adminDb
|
|
297
|
+
.from('types').select('id').eq('slug', 'eval_test_case').eq('kind', 'item').maybeSingle()
|
|
298
|
+
|
|
299
|
+
const { data: testCases } = await adminDb
|
|
300
|
+
.from('items')
|
|
301
|
+
.select('data')
|
|
302
|
+
.eq('status', 'active')
|
|
303
|
+
.eq('type_id', typeRow?.id || '00000000-0000-0000-0000-000000000000')
|
|
304
|
+
.limit(1000)
|
|
305
|
+
|
|
306
|
+
const countByType: Record<string, number> = {}
|
|
307
|
+
for (const tt of TEST_TYPES) countByType[tt] = 0
|
|
308
|
+
for (const tc of testCases || []) {
|
|
309
|
+
const t = tc.data?.test_type || 'medium'
|
|
310
|
+
countByType[t] = (countByType[t] || 0) + 1
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const coverage = TEST_TYPES.map(tt => ({
|
|
314
|
+
test_type: tt,
|
|
315
|
+
case_count: countByType[tt] || 0,
|
|
316
|
+
has_cases: (countByType[tt] || 0) > 0,
|
|
317
|
+
}))
|
|
318
|
+
|
|
319
|
+
return {
|
|
320
|
+
total_cases: testCases?.length || 0,
|
|
321
|
+
coverage,
|
|
322
|
+
recommendation: coverage.filter(c => !c.has_cases).map(c => c.test_type),
|
|
323
|
+
}
|
|
324
|
+
})
|
|
325
|
+
|
|
326
|
+
// ─── HANDLER ─────────────────────────────────────────────────────────────────
|
|
327
|
+
|
|
328
|
+
export const handler = createHandler(async (ctx, body) => {
|
|
329
|
+
const { action } = ctx.query || {}
|
|
330
|
+
const method = ctx.query?.method || 'GET'
|
|
331
|
+
|
|
332
|
+
switch (action) {
|
|
333
|
+
case 'create_test_case':
|
|
334
|
+
if (method === 'POST') return await createTestCase(ctx, body)
|
|
335
|
+
break
|
|
336
|
+
case 'list_test_cases':
|
|
337
|
+
if (method === 'GET') return await listTestCases(ctx, body)
|
|
338
|
+
break
|
|
339
|
+
case 'run_eval':
|
|
340
|
+
if (method === 'POST') return await runEval(ctx, body)
|
|
341
|
+
break
|
|
342
|
+
case 'get_results':
|
|
343
|
+
if (method === 'GET') return await getResults(ctx, body)
|
|
344
|
+
break
|
|
345
|
+
case 'get_coverage':
|
|
346
|
+
if (method === 'GET') return await getCoverage(ctx, body)
|
|
347
|
+
break
|
|
348
|
+
default:
|
|
349
|
+
if (method === 'GET') return await listTestCases(ctx, body)
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
throw new Error('Invalid action or method')
|
|
353
|
+
})
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import { createHandler } from './_shared/middleware'
|
|
18
18
|
import { adminDb } from './_shared/db'
|
|
19
|
+
import { resolveTypeId } from './_shared/resolve-ids'
|
|
19
20
|
import { emitLog } from './_shared/audit'
|
|
20
21
|
|
|
21
22
|
// ─── HELPERS ──────────────────────────────────────────────────────────────────
|
|
@@ -101,18 +102,24 @@ export const get = createHandler(async (ctx) => {
|
|
|
101
102
|
* title, description, app_id, force (bypass direction check)
|
|
102
103
|
*/
|
|
103
104
|
export const upsert = createHandler(async (ctx, body) => {
|
|
104
|
-
const { person_id, item_id, type_id, account_id, app_id, force } = body || {}
|
|
105
|
+
const { person_id, item_id, type_id, type_slug, account_id, app_id, force } = body || {}
|
|
105
106
|
let { status, score, data: dataPayload, title, description } = body || {}
|
|
106
107
|
|
|
107
|
-
|
|
108
|
-
|
|
108
|
+
// Resolve type_id from type_slug if provided; otherwise require it directly
|
|
109
|
+
let resolvedTypeId = type_id
|
|
110
|
+
if (!resolvedTypeId && type_slug) {
|
|
111
|
+
resolvedTypeId = await resolveTypeId('item_progress', type_slug)
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (!person_id || !item_id || !resolvedTypeId || !account_id) {
|
|
115
|
+
throw new Error('person_id, item_id, type_id (or type_slug), and account_id are required')
|
|
109
116
|
}
|
|
110
117
|
|
|
111
118
|
// Fetch type record for pipeline
|
|
112
119
|
const { data: typeRecord } = await adminDb
|
|
113
120
|
.from('types')
|
|
114
121
|
.select('design_schema')
|
|
115
|
-
.eq('id',
|
|
122
|
+
.eq('id', resolvedTypeId)
|
|
116
123
|
.single()
|
|
117
124
|
|
|
118
125
|
const pipeline = getPipeline(typeRecord)
|
|
@@ -158,7 +165,7 @@ export const upsert = createHandler(async (ctx, body) => {
|
|
|
158
165
|
}
|
|
159
166
|
|
|
160
167
|
const payload: any = {
|
|
161
|
-
type_id,
|
|
168
|
+
type_id: resolvedTypeId,
|
|
162
169
|
account_id,
|
|
163
170
|
app_id: app_id ?? existing?.app_id ?? null,
|
|
164
171
|
person_id,
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
* | latency_percentiles | getLatencyPercentiles |
|
|
19
19
|
* | pipeline_stats | getPipelineStats |
|
|
20
20
|
* | top_actors | getTopActors |
|
|
21
|
+
* | ai_support_metrics | getAiSupportMetrics |
|
|
21
22
|
* | cleanup | cleanupOldLogs (POST) |
|
|
22
23
|
*
|
|
23
24
|
* **Authorization:** All read endpoints require account context (RLS-scoped
|
|
@@ -32,6 +33,7 @@
|
|
|
32
33
|
*/
|
|
33
34
|
|
|
34
35
|
import { createHandler } from './_shared/middleware'
|
|
36
|
+
import { adminDb } from './_shared/db'
|
|
35
37
|
|
|
36
38
|
// ─── HANDLERS ─────────────────────────────────────────────────────────────────
|
|
37
39
|
|
|
@@ -245,6 +247,147 @@ export const cleanupOldLogs = createHandler(async (ctx, body) => {
|
|
|
245
247
|
return { deleted_count: data?.[0]?.deleted_count || 0 }
|
|
246
248
|
})
|
|
247
249
|
|
|
250
|
+
// ─── AI SUPPORT METRICS ──────────────────────────────────────────────────────
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Returns the AI support loop health metrics.
|
|
254
|
+
* All values are derived from existing tables — no new table required.
|
|
255
|
+
*
|
|
256
|
+
* Metrics:
|
|
257
|
+
* - ai_attempted: AI agent messages sent
|
|
258
|
+
* - ai_resolved: Tickets resolved without escalation
|
|
259
|
+
* - human_escalations: Tickets assigned to humans
|
|
260
|
+
* - thumbs_down_rate: Ratio of not_helpful / total AI messages (0–1)
|
|
261
|
+
* - medium_confidence: AI messages with confidence 0.60–0.85
|
|
262
|
+
* - kb_articles_created: Postmortem KB articles
|
|
263
|
+
* - kb_reuse_count: Total reuse_count across ai_feedback_record items
|
|
264
|
+
* - knowledge_gaps: Interactions with kb_candidate=false + not_helpful feedback
|
|
265
|
+
*
|
|
266
|
+
* Query params: from, to (ISO dates, optional — defaults to last 30 days)
|
|
267
|
+
*
|
|
268
|
+
* @sideEffects DB reads: items, messages tables
|
|
269
|
+
* @calledBy handler (?action=ai_support_metrics)
|
|
270
|
+
*/
|
|
271
|
+
export const getAiSupportMetrics = createHandler(async (ctx, _body) => {
|
|
272
|
+
if (!ctx.accountId) throw new Error('Account context required')
|
|
273
|
+
|
|
274
|
+
const now = new Date()
|
|
275
|
+
const thirtyDaysAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString()
|
|
276
|
+
const from = ctx.query?.from || thirtyDaysAgo
|
|
277
|
+
const to = ctx.query?.to || now.toISOString()
|
|
278
|
+
|
|
279
|
+
// Run all queries in parallel for performance
|
|
280
|
+
const [
|
|
281
|
+
aiMessagesResult,
|
|
282
|
+
resolvedResult,
|
|
283
|
+
escalatedResult,
|
|
284
|
+
notHelpfulResult,
|
|
285
|
+
mediumConfResult,
|
|
286
|
+
kbCreatedResult,
|
|
287
|
+
feedbackRecordsResult,
|
|
288
|
+
] = await Promise.allSettled([
|
|
289
|
+
// AI agent messages
|
|
290
|
+
adminDb
|
|
291
|
+
.from('messages')
|
|
292
|
+
.select('id, data', { count: 'exact', head: false })
|
|
293
|
+
.eq('account_id', ctx.accountId)
|
|
294
|
+
.eq('data->>sender_type', 'agent')
|
|
295
|
+
.gte('created_at', from)
|
|
296
|
+
.lte('created_at', to),
|
|
297
|
+
|
|
298
|
+
// Tickets resolved without AI escalation
|
|
299
|
+
adminDb
|
|
300
|
+
.from('items')
|
|
301
|
+
.select('id', { count: 'exact', head: true })
|
|
302
|
+
.eq('account_id', ctx.accountId)
|
|
303
|
+
.eq('status', 'resolved')
|
|
304
|
+
.eq('data->>aim_escalation_reason', 'none')
|
|
305
|
+
.gte('created_at', from)
|
|
306
|
+
.lte('created_at', to),
|
|
307
|
+
|
|
308
|
+
// Tickets escalated to humans
|
|
309
|
+
adminDb
|
|
310
|
+
.from('items')
|
|
311
|
+
.select('id', { count: 'exact', head: true })
|
|
312
|
+
.eq('account_id', ctx.accountId)
|
|
313
|
+
.eq('status', 'human_assigned')
|
|
314
|
+
.gte('created_at', from)
|
|
315
|
+
.lte('created_at', to),
|
|
316
|
+
|
|
317
|
+
// Not-helpful AI messages
|
|
318
|
+
adminDb
|
|
319
|
+
.from('messages')
|
|
320
|
+
.select('id', { count: 'exact', head: true })
|
|
321
|
+
.eq('account_id', ctx.accountId)
|
|
322
|
+
.eq('data->>sender_type', 'agent')
|
|
323
|
+
.eq('data->>feedback', 'not_helpful')
|
|
324
|
+
.gte('created_at', from)
|
|
325
|
+
.lte('created_at', to),
|
|
326
|
+
|
|
327
|
+
// Medium-confidence AI messages
|
|
328
|
+
adminDb
|
|
329
|
+
.from('messages')
|
|
330
|
+
.select('id', { count: 'exact', head: true })
|
|
331
|
+
.eq('account_id', ctx.accountId)
|
|
332
|
+
.eq('data->>sender_type', 'agent')
|
|
333
|
+
.gte('data->>confidence', '0.60')
|
|
334
|
+
.lte('data->>confidence', '0.85')
|
|
335
|
+
.gte('created_at', from)
|
|
336
|
+
.lte('created_at', to),
|
|
337
|
+
|
|
338
|
+
// Postmortem KB articles created
|
|
339
|
+
adminDb
|
|
340
|
+
.from('items')
|
|
341
|
+
.select('id', { count: 'exact', head: true })
|
|
342
|
+
.eq('account_id', ctx.accountId)
|
|
343
|
+
.eq('data->>kb_type', 'postmortem')
|
|
344
|
+
.gte('created_at', from)
|
|
345
|
+
.lte('created_at', to),
|
|
346
|
+
|
|
347
|
+
// AI feedback records for reuse + gaps
|
|
348
|
+
adminDb
|
|
349
|
+
.from('items')
|
|
350
|
+
.select('data')
|
|
351
|
+
.eq('account_id', ctx.accountId)
|
|
352
|
+
.gte('created_at', from)
|
|
353
|
+
.lte('created_at', to)
|
|
354
|
+
.limit(500),
|
|
355
|
+
])
|
|
356
|
+
|
|
357
|
+
// Extract values safely
|
|
358
|
+
const aiMessages = aiMessagesResult.status === 'fulfilled' ? (aiMessagesResult.value.data || []) : []
|
|
359
|
+
const aiAttempted = aiMessages.length
|
|
360
|
+
|
|
361
|
+
const aiResolved = resolvedResult.status === 'fulfilled' ? (resolvedResult.value.count || 0) : 0
|
|
362
|
+
const escalated = escalatedResult.status === 'fulfilled' ? (escalatedResult.value.count || 0) : 0
|
|
363
|
+
const notHelpful = notHelpfulResult.status === 'fulfilled' ? (notHelpfulResult.value.count || 0) : 0
|
|
364
|
+
const mediumConf = mediumConfResult.status === 'fulfilled' ? (mediumConfResult.value.count || 0) : 0
|
|
365
|
+
const kbCreated = kbCreatedResult.status === 'fulfilled' ? (kbCreatedResult.value.count || 0) : 0
|
|
366
|
+
|
|
367
|
+
const feedbackRows = feedbackRecordsResult.status === 'fulfilled'
|
|
368
|
+
? (feedbackRecordsResult.value.data || [])
|
|
369
|
+
: []
|
|
370
|
+
|
|
371
|
+
const kbReuseCount = feedbackRows.reduce((sum: number, r: any) => sum + (Number(r.data?.reuse_count) || 0), 0)
|
|
372
|
+
const knowledgeGaps = feedbackRows.filter((r: any) =>
|
|
373
|
+
r.data?.kb_candidate === false && r.data?.customer_feedback === 'not_helpful'
|
|
374
|
+
).length
|
|
375
|
+
|
|
376
|
+
const thumbsDownRate = aiAttempted > 0 ? notHelpful / aiAttempted : 0
|
|
377
|
+
|
|
378
|
+
return {
|
|
379
|
+
period: { from, to },
|
|
380
|
+
ai_attempted: aiAttempted,
|
|
381
|
+
ai_resolved: aiResolved,
|
|
382
|
+
human_escalations: escalated,
|
|
383
|
+
thumbs_down_rate: Math.round(thumbsDownRate * 1000) / 1000,
|
|
384
|
+
medium_confidence: mediumConf,
|
|
385
|
+
kb_articles_created: kbCreated,
|
|
386
|
+
kb_reuse_count: kbReuseCount,
|
|
387
|
+
knowledge_gaps: knowledgeGaps,
|
|
388
|
+
}
|
|
389
|
+
})
|
|
390
|
+
|
|
248
391
|
// ─── MAIN HANDLER ────────────────────────────────────────────────────────────
|
|
249
392
|
|
|
250
393
|
/**
|
|
@@ -267,9 +410,11 @@ export const handler = createHandler(async (ctx, body) => {
|
|
|
267
410
|
return await getPipelineStats(ctx, body)
|
|
268
411
|
case 'top_actors':
|
|
269
412
|
return await getTopActors(ctx, body)
|
|
413
|
+
case 'ai_support_metrics':
|
|
414
|
+
return await getAiSupportMetrics(ctx, body)
|
|
270
415
|
case 'cleanup':
|
|
271
416
|
return await cleanupOldLogs(ctx, body)
|
|
272
417
|
default:
|
|
273
|
-
throw new Error(`Unknown action: ${action}. Valid actions: event_volume, error_rate, latency_percentiles, pipeline_stats, top_actors, cleanup`)
|
|
418
|
+
throw new Error(`Unknown action: ${action}. Valid actions: event_volume, error_rate, latency_percentiles, pipeline_stats, top_actors, ai_support_metrics, cleanup`)
|
|
274
419
|
}
|
|
275
420
|
})
|
|
@@ -563,24 +563,43 @@ CREATE POLICY items_access ON public.items FOR ALL
|
|
|
563
563
|
OR public.current_actor_id() IS NULL
|
|
564
564
|
);
|
|
565
565
|
|
|
566
|
-
-- threads (
|
|
566
|
+
-- threads (runtime record design_schema is the source of truth for scope)
|
|
567
567
|
DROP POLICY IF EXISTS threads_access ON public.threads;
|
|
568
568
|
CREATE POLICY threads_access ON public.threads FOR ALL
|
|
569
569
|
USING (
|
|
570
|
-
|
|
571
|
-
OR
|
|
572
|
-
OR
|
|
570
|
+
design_schema->>'scope' = 'platform' AND public.current_actor_id() IS NOT NULL
|
|
571
|
+
OR account_id IN (SELECT public.get_accessible_accounts(public.current_actor_id()))
|
|
572
|
+
OR public.current_actor_id() IS NULL
|
|
573
573
|
);
|
|
574
574
|
|
|
575
|
-
-- messages (inherits
|
|
576
|
-
--
|
|
575
|
+
-- messages (inherits thread access from the runtime record design_schema; visibility
|
|
576
|
+
-- filters internal notes to support/support_admin roles)
|
|
577
577
|
DROP POLICY IF EXISTS messages_access ON public.messages;
|
|
578
578
|
CREATE POLICY messages_access ON public.messages FOR ALL
|
|
579
|
-
USING (
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
579
|
+
USING (
|
|
580
|
+
EXISTS (
|
|
581
|
+
SELECT 1 FROM public.threads t
|
|
582
|
+
WHERE t.id = messages.thread_id
|
|
583
|
+
AND (
|
|
584
|
+
t.design_schema->>'scope' = 'platform' AND public.current_actor_id() IS NOT NULL
|
|
585
|
+
OR t.account_id IN (SELECT public.get_accessible_accounts(public.current_actor_id()))
|
|
586
|
+
)
|
|
587
|
+
)
|
|
588
|
+
AND (
|
|
589
|
+
messages.visibility IS NULL
|
|
590
|
+
OR messages.visibility = 'external'
|
|
591
|
+
OR (
|
|
592
|
+
messages.visibility = 'internal'
|
|
593
|
+
AND EXISTS (
|
|
594
|
+
SELECT 1 FROM public.people p
|
|
595
|
+
JOIN public.roles r ON p.role_id = r.id
|
|
596
|
+
WHERE p.id = public.current_actor_id()
|
|
597
|
+
AND (r.slug IN ('support', 'support_admin') OR r.is_system = true)
|
|
598
|
+
)
|
|
599
|
+
)
|
|
600
|
+
)
|
|
601
|
+
OR public.current_actor_id() IS NULL
|
|
602
|
+
);
|
|
584
603
|
|
|
585
604
|
-- links (with platform scope visibility from 012_rls_scope.sql)
|
|
586
605
|
DROP POLICY IF EXISTS links_access ON public.links;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
-- Migration 026: Fix RLS policies to use current_actor_id() and runtime design_schema scope
|
|
2
|
+
--
|
|
3
|
+
-- The policies for threads and messages were using auth.uid() inside get_accessible_accounts(),
|
|
4
|
+
-- but that function expects the internal person_id (from current_actor_id()). This broke account
|
|
5
|
+
-- hierarchy checks for regular users. Additionally, the policies hardcoded the 'platform' scope
|
|
6
|
+
-- check. This migration makes them scope-driven: any non-platform scope is treated as account-scoped.
|
|
7
|
+
--
|
|
8
|
+
-- Threads and messages now look at the runtime record's stamped design_schema for access, and
|
|
9
|
+
-- messages also filter by visibility so internal support notes stay internal.
|
|
10
|
+
|
|
11
|
+
-- threads (runtime record design_schema is the source of truth for scope)
|
|
12
|
+
DROP POLICY IF EXISTS threads_access ON public.threads;
|
|
13
|
+
CREATE POLICY threads_access ON public.threads FOR ALL
|
|
14
|
+
USING (
|
|
15
|
+
design_schema->>'scope' = 'platform' AND public.current_actor_id() IS NOT NULL
|
|
16
|
+
OR account_id IN (SELECT public.get_accessible_accounts(public.current_actor_id()))
|
|
17
|
+
OR public.current_actor_id() IS NULL
|
|
18
|
+
);
|
|
19
|
+
|
|
20
|
+
-- messages (inherits thread access from the runtime record design_schema; visibility
|
|
21
|
+
-- filters internal notes to support/support_admin roles)
|
|
22
|
+
DROP POLICY IF EXISTS messages_access ON public.messages;
|
|
23
|
+
CREATE POLICY messages_access ON public.messages FOR ALL
|
|
24
|
+
USING (
|
|
25
|
+
EXISTS (
|
|
26
|
+
SELECT 1 FROM public.threads t
|
|
27
|
+
WHERE t.id = messages.thread_id
|
|
28
|
+
AND (
|
|
29
|
+
t.design_schema->>'scope' = 'platform' AND public.current_actor_id() IS NOT NULL
|
|
30
|
+
OR t.account_id IN (SELECT public.get_accessible_accounts(public.current_actor_id()))
|
|
31
|
+
)
|
|
32
|
+
)
|
|
33
|
+
AND (
|
|
34
|
+
messages.visibility IS NULL
|
|
35
|
+
OR messages.visibility = 'external'
|
|
36
|
+
OR (
|
|
37
|
+
messages.visibility = 'internal'
|
|
38
|
+
AND EXISTS (
|
|
39
|
+
SELECT 1 FROM public.people p
|
|
40
|
+
JOIN public.roles r ON p.role_id = r.id
|
|
41
|
+
WHERE p.id = public.current_actor_id()
|
|
42
|
+
AND (r.slug IN ('support', 'support_admin') OR r.is_system = true)
|
|
43
|
+
)
|
|
44
|
+
)
|
|
45
|
+
)
|
|
46
|
+
OR public.current_actor_id() IS NULL
|
|
47
|
+
);
|