spine-framework-cortex 0.2.20 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/custom_case_analysis.ts +292 -0
- package/functions/custom_cortex-chunks.ts +52 -0
- package/functions/custom_kb-chunker.ts +576 -0
- package/functions/custom_kb-embeddings.ts +579 -0
- package/functions/custom_support-redaction.ts +115 -0
- package/functions/custom_support-solution.ts +104 -0
- package/manifest.json +10 -5
- package/package.json +1 -1
- package/pages/support/RedactionReview.tsx +36 -18
- package/pages/support/TicketDetailPage.tsx +127 -129
- package/seed/ai-agents.json +98 -0
- package/seed/pipelines.json +29 -0
- package/seed/prompt-configs.json +84 -0
- package/LICENSE.md +0 -193
- package/README.md +0 -46
|
@@ -0,0 +1,579 @@
|
|
|
1
|
+
import { createHandler } from './_shared/middleware'
|
|
2
|
+
import { adminDb } from './_shared/db'
|
|
3
|
+
import { chunkArticle, htmlToPlainText } from './custom_kb-chunker'
|
|
4
|
+
|
|
5
|
+
interface KBEmbeddingRequest {
|
|
6
|
+
item_id: string
|
|
7
|
+
vector_types: ('semantic' | 'structure' | 'summary' | 'code')[]
|
|
8
|
+
force_regenerate?: boolean
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
interface KBEmbeddingResponse {
|
|
12
|
+
success: boolean
|
|
13
|
+
embeddings_created: number
|
|
14
|
+
embeddings_updated: number
|
|
15
|
+
errors: string[]
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// Generate semantic embedding (full content)
|
|
19
|
+
function decodeHtmlContent(html: string): string {
|
|
20
|
+
return html
|
|
21
|
+
.replace(/<[^>]+>/g, ' ')
|
|
22
|
+
.replace(/</g, '<')
|
|
23
|
+
.replace(/>/g, '>')
|
|
24
|
+
.replace(/&/g, '&')
|
|
25
|
+
.replace(/"/g, '"')
|
|
26
|
+
.replace(/'/g, "'")
|
|
27
|
+
.replace(/\s+/g, ' ')
|
|
28
|
+
.trim()
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Build semantic content from structured code_metadata — developer understanding focus
|
|
32
|
+
function buildSemanticContent(item: any): string {
|
|
33
|
+
const cm = item.data?.code_metadata || {}
|
|
34
|
+
const inputs = cm.inputs && Object.keys(cm.inputs).length > 0
|
|
35
|
+
? 'Parameters: ' + Object.entries(cm.inputs).map(([k, v]) => `${k} (${v})`).join(', ')
|
|
36
|
+
: ''
|
|
37
|
+
const deps = cm.depends_on && cm.depends_on.length > 0
|
|
38
|
+
? 'Depends on: ' + cm.depends_on.join(', ')
|
|
39
|
+
: ''
|
|
40
|
+
const sideEffects = cm.side_effects && cm.side_effects.length > 0
|
|
41
|
+
? 'Side effects: ' + cm.side_effects.join(', ')
|
|
42
|
+
: ''
|
|
43
|
+
|
|
44
|
+
return [
|
|
45
|
+
cm.identifier || item.title || '',
|
|
46
|
+
cm.macro || '',
|
|
47
|
+
cm.micro || '',
|
|
48
|
+
inputs,
|
|
49
|
+
cm.outputs ? `Returns: ${cm.outputs}` : '',
|
|
50
|
+
deps,
|
|
51
|
+
sideEffects,
|
|
52
|
+
`Tags: ${(item.data?.tags || []).join(', ')}`,
|
|
53
|
+
cm.file_path ? `File: ${cm.file_path} lines ${cm.line_start}-${cm.line_end}` : ''
|
|
54
|
+
].filter(Boolean).join('\n')
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Build structure content from metadata — for filtering/faceted search
|
|
58
|
+
function buildStructureContent(item: any): string {
|
|
59
|
+
const cm = item.data?.code_metadata || {}
|
|
60
|
+
return JSON.stringify({
|
|
61
|
+
identifier: cm.identifier || '',
|
|
62
|
+
chunk_type: cm.chunk_type || '',
|
|
63
|
+
file_path: cm.file_path || '',
|
|
64
|
+
line_start: cm.line_start,
|
|
65
|
+
line_end: cm.line_end,
|
|
66
|
+
version: cm.version || '',
|
|
67
|
+
tags: item.data?.tags || [],
|
|
68
|
+
depends_on: cm.depends_on || [],
|
|
69
|
+
depended_by: cm.depended_by || [],
|
|
70
|
+
inputs: Object.keys(cm.inputs || {}),
|
|
71
|
+
has_side_effects: (cm.side_effects || []).length > 0
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Generate real embedding vector via OpenAI text-embedding-3-small (single input)
|
|
76
|
+
async function generateEmbeddingVector(content: string): Promise<string> {
|
|
77
|
+
const vectors = await generateEmbeddingBatch([content])
|
|
78
|
+
return vectors[0]
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Batch-generate embedding vectors — one API call for multiple inputs
|
|
82
|
+
async function generateEmbeddingBatch(inputs: string[]): Promise<string[]> {
|
|
83
|
+
const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
|
|
84
|
+
const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
|
|
85
|
+
|
|
86
|
+
if (!apiKey) {
|
|
87
|
+
throw new Error('No OPENAI_API_KEY configured — cannot generate embeddings')
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (inputs.length === 0) return []
|
|
91
|
+
|
|
92
|
+
const res = await fetch(`${baseUrl}/embeddings`, {
|
|
93
|
+
method: 'POST',
|
|
94
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
|
|
95
|
+
body: JSON.stringify({
|
|
96
|
+
model: 'text-embedding-3-small',
|
|
97
|
+
input: inputs,
|
|
98
|
+
}),
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
if (!res.ok) {
|
|
102
|
+
const err = await res.text()
|
|
103
|
+
throw new Error(`OpenAI embeddings error ${res.status}: ${err.slice(0, 200)}`)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const result: any = await res.json()
|
|
107
|
+
// OpenAI returns embeddings sorted by index
|
|
108
|
+
const sorted = (result.data || []).sort((a: any, b: any) => a.index - b.index)
|
|
109
|
+
return sorted.map((d: any) => `[${d.embedding.join(',')}]`)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Resolve platform account ID at runtime (slug: 'spine-system')
|
|
113
|
+
let _platformAccountId: string | null = null
|
|
114
|
+
async function getPlatformAccountId(): Promise<string | null> {
|
|
115
|
+
if (_platformAccountId) return _platformAccountId
|
|
116
|
+
const { data } = await adminDb.from('accounts').select('id').eq('slug', 'spine-system').maybeSingle()
|
|
117
|
+
_platformAccountId = data?.id ?? null
|
|
118
|
+
return _platformAccountId
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Insert an embedding record (used during batch writes)
|
|
122
|
+
async function insertEmbedding(
|
|
123
|
+
itemId: string,
|
|
124
|
+
vectorType: string,
|
|
125
|
+
chunkIndex: number,
|
|
126
|
+
content: string,
|
|
127
|
+
embedding: string,
|
|
128
|
+
metadata: any,
|
|
129
|
+
accountId?: string
|
|
130
|
+
): Promise<void> {
|
|
131
|
+
const resolvedAccountId = accountId ?? await getPlatformAccountId()
|
|
132
|
+
await adminDb
|
|
133
|
+
.from('embeddings')
|
|
134
|
+
.insert({
|
|
135
|
+
account_id: resolvedAccountId,
|
|
136
|
+
model_id: 'text-embedding-3-small',
|
|
137
|
+
document_id: itemId,
|
|
138
|
+
chunk_index: chunkIndex,
|
|
139
|
+
content: content,
|
|
140
|
+
embedding: embedding,
|
|
141
|
+
metadata: {
|
|
142
|
+
vector_type: vectorType,
|
|
143
|
+
item_type: 'kb_article',
|
|
144
|
+
...metadata
|
|
145
|
+
}
|
|
146
|
+
})
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Generate a one-paragraph AI summary for top-level retrieval matching
|
|
150
|
+
async function generateArticleSummary(item: any): Promise<string> {
|
|
151
|
+
const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
|
|
152
|
+
const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
|
|
153
|
+
if (!apiKey) return `${item.title || ''}: ${(item.description || '').slice(0, 300)}`
|
|
154
|
+
|
|
155
|
+
const plainText = (item.description || '')
|
|
156
|
+
.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 3000)
|
|
157
|
+
|
|
158
|
+
const res = await fetch(`${baseUrl}/chat/completions`, {
|
|
159
|
+
method: 'POST',
|
|
160
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
|
|
161
|
+
body: JSON.stringify({
|
|
162
|
+
model: process.env.LLM_DEFAULT_MODEL || 'gpt-4o-mini',
|
|
163
|
+
temperature: 0.2,
|
|
164
|
+
max_tokens: 120,
|
|
165
|
+
messages: [
|
|
166
|
+
{ role: 'system', content: 'Write a single concise paragraph (max 100 words) summarising this knowledge base article for search indexing. Output only the paragraph.' },
|
|
167
|
+
{ role: 'user', content: `Title: ${item.title}\n\n${plainText}` }
|
|
168
|
+
]
|
|
169
|
+
})
|
|
170
|
+
})
|
|
171
|
+
if (!res.ok) return `${item.title}: ${plainText.slice(0, 200)}`
|
|
172
|
+
const result: any = await res.json()
|
|
173
|
+
return result.choices?.[0]?.message?.content?.trim() || `${item.title}: ${plainText.slice(0, 200)}`
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Delete all existing embeddings for a document (called before re-embedding)
|
|
177
|
+
async function deleteExistingEmbeddings(itemId: string): Promise<number> {
|
|
178
|
+
const { data } = await adminDb
|
|
179
|
+
.from('embeddings')
|
|
180
|
+
.delete()
|
|
181
|
+
.eq('document_id', itemId)
|
|
182
|
+
.eq('metadata->>item_type', 'kb_article')
|
|
183
|
+
.select('id')
|
|
184
|
+
|
|
185
|
+
return data?.length || 0
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Build article-level structure content for non-code articles
|
|
189
|
+
function buildArticleStructureContent(item: any): string {
|
|
190
|
+
return JSON.stringify({
|
|
191
|
+
title: item.title || '',
|
|
192
|
+
kb_type: item.data?.kb_type || '',
|
|
193
|
+
category: item.data?.category || '',
|
|
194
|
+
tags: item.data?.tags || [],
|
|
195
|
+
audience: item.data?.audience || [],
|
|
196
|
+
security_level: item.data?.security_level || '',
|
|
197
|
+
priority: item.data?.priority || '',
|
|
198
|
+
})
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Determine if an item is a code chunk (ingested) vs a manual article
|
|
202
|
+
function isCodeChunk(item: any): boolean {
|
|
203
|
+
return !!(item.data?.code_metadata?.chunk_id || item.data?.code_metadata?.file_path)
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Main handler for generating KB embeddings
|
|
207
|
+
async function handleGenerateEmbeddings(
|
|
208
|
+
itemId: string,
|
|
209
|
+
vectorTypes: string[],
|
|
210
|
+
forceRegenerate: boolean = false,
|
|
211
|
+
overrideAccountId?: string
|
|
212
|
+
): Promise<KBEmbeddingResponse> {
|
|
213
|
+
const response: KBEmbeddingResponse = {
|
|
214
|
+
success: true,
|
|
215
|
+
embeddings_created: 0,
|
|
216
|
+
embeddings_updated: 0,
|
|
217
|
+
errors: []
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
// Get the KB item — allow any type when called from case resolution path
|
|
222
|
+
const { data: item, error: itemError } = await adminDb
|
|
223
|
+
.from('items')
|
|
224
|
+
.select('*')
|
|
225
|
+
.eq('id', itemId)
|
|
226
|
+
.single()
|
|
227
|
+
|
|
228
|
+
if (itemError || !item) {
|
|
229
|
+
throw new Error('Item not found for embedding')
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Resolve account: caller override > item's own account > platform account
|
|
233
|
+
const accountId = overrideAccountId || item.account_id || await getPlatformAccountId()
|
|
234
|
+
|
|
235
|
+
// Check if embeddings already exist
|
|
236
|
+
if (!forceRegenerate) {
|
|
237
|
+
const { data: existing } = await adminDb
|
|
238
|
+
.from('embeddings')
|
|
239
|
+
.select('id')
|
|
240
|
+
.eq('document_id', itemId)
|
|
241
|
+
.eq('metadata->>item_type', 'kb_article')
|
|
242
|
+
.limit(1)
|
|
243
|
+
|
|
244
|
+
if (existing && existing.length > 0) {
|
|
245
|
+
response.errors.push('Embeddings already exist (use force_regenerate to override)')
|
|
246
|
+
response.success = false
|
|
247
|
+
return response
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// Delete existing embeddings before regenerating
|
|
252
|
+
const deleted = await deleteExistingEmbeddings(itemId)
|
|
253
|
+
if (deleted > 0) {
|
|
254
|
+
response.embeddings_updated = deleted
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// ── Code chunks: legacy path (semantic + structure) ──
|
|
258
|
+
if (isCodeChunk(item)) {
|
|
259
|
+
const semanticContent = buildSemanticContent(item)
|
|
260
|
+
const structureContent = buildStructureContent(item)
|
|
261
|
+
|
|
262
|
+
const vectors = await generateEmbeddingBatch([semanticContent, structureContent])
|
|
263
|
+
|
|
264
|
+
await Promise.all([
|
|
265
|
+
insertEmbedding(itemId, 'semantic', 0, semanticContent, vectors[0], {
|
|
266
|
+
kb_type: item.data?.kb_type,
|
|
267
|
+
chunk_id: item.data?.code_metadata?.chunk_id,
|
|
268
|
+
chunk_type: item.data?.code_metadata?.chunk_type,
|
|
269
|
+
file_path: item.data?.code_metadata?.file_path,
|
|
270
|
+
}, accountId),
|
|
271
|
+
insertEmbedding(itemId, 'structure', 0, structureContent, vectors[1], {
|
|
272
|
+
kb_type: item.data?.kb_type,
|
|
273
|
+
tags: item.data?.tags,
|
|
274
|
+
file_path: item.data?.code_metadata?.file_path,
|
|
275
|
+
depends_on: item.data?.code_metadata?.depends_on,
|
|
276
|
+
}, accountId),
|
|
277
|
+
])
|
|
278
|
+
|
|
279
|
+
response.embeddings_created = 2
|
|
280
|
+
return response
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// ── Article path: semantic chunks + structure + summary ──
|
|
284
|
+
const articleContent = item.description || ''
|
|
285
|
+
const chunks = chunkArticle(articleContent, { articleTitle: item.title || 'Untitled' })
|
|
286
|
+
|
|
287
|
+
const structureContent = buildArticleStructureContent(item)
|
|
288
|
+
|
|
289
|
+
// Generate AI summary for top-level retrieval matching
|
|
290
|
+
const summaryContent = await generateArticleSummary(item)
|
|
291
|
+
|
|
292
|
+
// Single batched embed: N semantic chunks + 1 structure + 1 summary
|
|
293
|
+
const allTexts = [...chunks.map(c => c.content), structureContent, summaryContent]
|
|
294
|
+
const allVectors = await generateEmbeddingBatch(allTexts)
|
|
295
|
+
|
|
296
|
+
const writePromises: Promise<void>[] = []
|
|
297
|
+
|
|
298
|
+
// Semantic embeddings — one per chunk
|
|
299
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
300
|
+
writePromises.push(
|
|
301
|
+
insertEmbedding(itemId, 'semantic', i, chunks[i].content, allVectors[i], {
|
|
302
|
+
kb_type: item.data?.kb_type,
|
|
303
|
+
chunk_index: i,
|
|
304
|
+
chunk_total: chunks.length,
|
|
305
|
+
section_path: chunks[i].sectionPath,
|
|
306
|
+
security_level: item.data?.security_level,
|
|
307
|
+
}, accountId)
|
|
308
|
+
)
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// Structure embedding — facets for filtering
|
|
312
|
+
writePromises.push(
|
|
313
|
+
insertEmbedding(itemId, 'structure', 0, structureContent, allVectors[chunks.length], {
|
|
314
|
+
kb_type: item.data?.kb_type,
|
|
315
|
+
tags: item.data?.tags,
|
|
316
|
+
category: item.data?.category,
|
|
317
|
+
audience: item.data?.audience,
|
|
318
|
+
security_level: item.data?.security_level,
|
|
319
|
+
}, accountId)
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
// Summary embedding — one-paragraph AI summary for top-level matching
|
|
323
|
+
writePromises.push(
|
|
324
|
+
insertEmbedding(itemId, 'summary', 0, summaryContent, allVectors[chunks.length + 1], {
|
|
325
|
+
kb_type: item.data?.kb_type,
|
|
326
|
+
security_level: item.data?.security_level,
|
|
327
|
+
}, accountId)
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
await Promise.all(writePromises)
|
|
331
|
+
response.embeddings_created = chunks.length + 2 // +2 for structure + summary
|
|
332
|
+
|
|
333
|
+
} catch (error) {
|
|
334
|
+
response.success = false
|
|
335
|
+
response.errors = [error instanceof Error ? error.message : String(error)]
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
return response
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Embeds a resolved ticket's case data as a knowledge source for future triage RAG.
|
|
343
|
+
* Called automatically when a ticket is resolved (via case analysis completion).
|
|
344
|
+
*
|
|
345
|
+
* Stores two vectors:
|
|
346
|
+
* - 'semantic': ca_true_problem + ca_final_solution (the actual answer)
|
|
347
|
+
* - 'summary': ca_reported_issue (how the customer described it — improves matching)
|
|
348
|
+
*
|
|
349
|
+
* These are stored with item_type: 'resolved_case' so they can be included in
|
|
350
|
+
* triage agent knowledge_sources without polluting the main KB article pool.
|
|
351
|
+
*/
|
|
352
|
+
export async function embedResolvedCase(ticketId: string, accountId: string): Promise<void> {
|
|
353
|
+
// Load ticket data fields
|
|
354
|
+
const { data: ticket } = await adminDb
|
|
355
|
+
.from('items')
|
|
356
|
+
.select('id, title, data')
|
|
357
|
+
.eq('id', ticketId)
|
|
358
|
+
.single()
|
|
359
|
+
|
|
360
|
+
if (!ticket) return
|
|
361
|
+
|
|
362
|
+
const trueProb = ticket.data?.ca_true_problem || ''
|
|
363
|
+
const finalSol = ticket.data?.ca_final_solution || ''
|
|
364
|
+
const reportedIssue = ticket.data?.ca_reported_issue || ticket.title || ''
|
|
365
|
+
|
|
366
|
+
if (!trueProb && !finalSol) return // nothing to embed
|
|
367
|
+
|
|
368
|
+
// Delete any pre-existing resolved_case embeddings for this ticket
|
|
369
|
+
await adminDb
|
|
370
|
+
.from('embeddings')
|
|
371
|
+
.delete()
|
|
372
|
+
.eq('document_id', ticketId)
|
|
373
|
+
.eq('metadata->>item_type', 'resolved_case')
|
|
374
|
+
|
|
375
|
+
const semanticContent = `Problem: ${trueProb}\n\nSolution: ${finalSol}`
|
|
376
|
+
const summaryContent = `Customer reported: ${reportedIssue}`
|
|
377
|
+
|
|
378
|
+
try {
|
|
379
|
+
const vectors = await generateEmbeddingBatch([semanticContent, summaryContent])
|
|
380
|
+
await Promise.all([
|
|
381
|
+
insertEmbedding(ticketId, 'semantic', 0, semanticContent, vectors[0], {
|
|
382
|
+
item_type: 'resolved_case',
|
|
383
|
+
security_level: 'internal',
|
|
384
|
+
kb_candidate: ticket.data?.ca_kb_candidate ?? false,
|
|
385
|
+
automation_potential: ticket.data?.ca_automation_potential ?? 'unknown',
|
|
386
|
+
}, accountId),
|
|
387
|
+
insertEmbedding(ticketId, 'summary', 0, summaryContent, vectors[1], {
|
|
388
|
+
item_type: 'resolved_case',
|
|
389
|
+
security_level: 'internal',
|
|
390
|
+
}, accountId),
|
|
391
|
+
])
|
|
392
|
+
} catch (err) {
|
|
393
|
+
// Best-effort: don't fail ticket resolution if embedding fails
|
|
394
|
+
console.error(`embedResolvedCase failed for ticket ${ticketId}:`, err)
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// Generate embedding vector for a query string via OpenAI
|
|
399
|
+
async function generateQueryEmbedding(text: string): Promise<number[]> {
|
|
400
|
+
const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
|
|
401
|
+
const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
|
|
402
|
+
|
|
403
|
+
if (!apiKey) {
|
|
404
|
+
throw new Error('No OPENAI_API_KEY configured — vector search unavailable')
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
const res = await fetch(`${baseUrl}/embeddings`, {
|
|
408
|
+
method: 'POST',
|
|
409
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
|
|
410
|
+
body: JSON.stringify({
|
|
411
|
+
model: 'text-embedding-3-small',
|
|
412
|
+
input: text,
|
|
413
|
+
}),
|
|
414
|
+
})
|
|
415
|
+
|
|
416
|
+
if (!res.ok) {
|
|
417
|
+
const err = await res.text()
|
|
418
|
+
throw new Error(`OpenAI embeddings error ${res.status}: ${err.slice(0, 200)}`)
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
const result: any = await res.json()
|
|
422
|
+
return result.data?.[0]?.embedding || []
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// Platform KB account
|
|
426
|
+
const KB_PLATFORM_ACCOUNT_ID = '12acec9b-8451-40e7-80d5-e80c4e2fc0de'
|
|
427
|
+
|
|
428
|
+
// Search embeddings via vector similarity
|
|
429
|
+
async function handleSearchEmbeddings(
|
|
430
|
+
query: string,
|
|
431
|
+
accountId: string | null,
|
|
432
|
+
vectorType: string = 'semantic',
|
|
433
|
+
limit: number = 8
|
|
434
|
+
): Promise<any[]> {
|
|
435
|
+
if (!query || query.trim().length < 2) return []
|
|
436
|
+
|
|
437
|
+
// Generate embedding for the search query
|
|
438
|
+
const queryEmbedding = await generateQueryEmbedding(query.trim())
|
|
439
|
+
|
|
440
|
+
// Build account filter: user's account + platform KB
|
|
441
|
+
const accountIds = accountId
|
|
442
|
+
? [accountId, KB_PLATFORM_ACCOUNT_ID]
|
|
443
|
+
: [KB_PLATFORM_ACCOUNT_ID]
|
|
444
|
+
|
|
445
|
+
// Call the match_embeddings RPC — fetch extra to account for chunk deduplication
|
|
446
|
+
const { data: matches, error } = await adminDb.rpc('match_embeddings', {
|
|
447
|
+
query_embedding: queryEmbedding,
|
|
448
|
+
match_count: limit * 3,
|
|
449
|
+
filter_account_ids: accountIds,
|
|
450
|
+
filter_vector_type: vectorType,
|
|
451
|
+
similarity_threshold: 0.15,
|
|
452
|
+
})
|
|
453
|
+
|
|
454
|
+
if (error) throw new Error(`Vector search failed: ${error.message}`)
|
|
455
|
+
|
|
456
|
+
// Filter by similarity threshold + restricted cross-account results
|
|
457
|
+
const SIMILARITY_THRESHOLD = 0.15
|
|
458
|
+
const filtered = (matches || []).filter((r: any) => {
|
|
459
|
+
if (r.similarity < SIMILARITY_THRESHOLD) return false
|
|
460
|
+
if (r.account_id === accountId) return true
|
|
461
|
+
return r.metadata?.security_level !== 'restricted'
|
|
462
|
+
})
|
|
463
|
+
|
|
464
|
+
// Deduplicate by document_id — keep only the best-matching chunk per article
|
|
465
|
+
const bestByDoc = new Map<string, any>()
|
|
466
|
+
for (const r of filtered) {
|
|
467
|
+
const existing = bestByDoc.get(r.document_id)
|
|
468
|
+
if (!existing || r.similarity > existing.similarity) {
|
|
469
|
+
bestByDoc.set(r.document_id, r)
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
const deduped = [...bestByDoc.values()]
|
|
473
|
+
.sort((a, b) => b.similarity - a.similarity)
|
|
474
|
+
.slice(0, limit)
|
|
475
|
+
|
|
476
|
+
// Enrich with item title from the items table
|
|
477
|
+
const docIds = deduped.map((r: any) => r.document_id)
|
|
478
|
+
if (docIds.length === 0) return []
|
|
479
|
+
|
|
480
|
+
const { data: items } = await adminDb
|
|
481
|
+
.from('items')
|
|
482
|
+
.select('id, title, description, status, data')
|
|
483
|
+
.in('id', docIds)
|
|
484
|
+
|
|
485
|
+
const itemMap = new Map((items || []).map((i: any) => [i.id, i]))
|
|
486
|
+
|
|
487
|
+
return deduped.map((r: any) => {
|
|
488
|
+
const item = itemMap.get(r.document_id)
|
|
489
|
+
return {
|
|
490
|
+
id: r.document_id,
|
|
491
|
+
title: item?.title || '',
|
|
492
|
+
description: item?.description || '',
|
|
493
|
+
status: item?.status || '',
|
|
494
|
+
data: item?.data || {},
|
|
495
|
+
similarity: r.similarity,
|
|
496
|
+
matched_section: r.metadata?.section_path || null,
|
|
497
|
+
}
|
|
498
|
+
})
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// Delete embeddings for an item
|
|
502
|
+
async function handleDeleteEmbeddings(itemId: string): Promise<{ deleted: number }> {
|
|
503
|
+
const { data, error } = await adminDb
|
|
504
|
+
.from('embeddings')
|
|
505
|
+
.delete()
|
|
506
|
+
.eq('document_id', itemId)
|
|
507
|
+
.eq('metadata->>item_type', 'kb_article')
|
|
508
|
+
|
|
509
|
+
if (error) throw error
|
|
510
|
+
|
|
511
|
+
return { deleted: data?.length || 0 }
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
export const handler = createHandler(async (ctx: any, body: any) => {
|
|
515
|
+
const { action } = ctx.query || {}
|
|
516
|
+
const method = ctx.query?.method || 'POST'
|
|
517
|
+
|
|
518
|
+
switch (action) {
|
|
519
|
+
case 'generate':
|
|
520
|
+
if (method === 'POST') {
|
|
521
|
+
const ids: string[] = body.item_ids || (body.item_id ? [body.item_id] : [])
|
|
522
|
+
let totalCreated = 0, totalUpdated = 0
|
|
523
|
+
const allErrors: string[] = []
|
|
524
|
+
for (const id of ids) {
|
|
525
|
+
const r = await handleGenerateEmbeddings(id, body.vector_types, body.force_regenerate)
|
|
526
|
+
totalCreated += r.embeddings_created
|
|
527
|
+
totalUpdated += r.embeddings_updated
|
|
528
|
+
allErrors.push(...r.errors)
|
|
529
|
+
}
|
|
530
|
+
return {
|
|
531
|
+
success: allErrors.length === 0 || totalCreated > 0 || totalUpdated > 0,
|
|
532
|
+
embeddings_created: totalCreated,
|
|
533
|
+
embeddings_updated: totalUpdated,
|
|
534
|
+
errors: allErrors
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
break
|
|
538
|
+
|
|
539
|
+
case 'search':
|
|
540
|
+
if (method === 'GET' || method === 'POST') {
|
|
541
|
+
const q = body.query || ctx.query?.q || ''
|
|
542
|
+
const acctId = body.account_id || ctx.principal?.account_id || null
|
|
543
|
+
return await handleSearchEmbeddings(
|
|
544
|
+
q,
|
|
545
|
+
acctId,
|
|
546
|
+
body.vector_type || 'semantic',
|
|
547
|
+
body.limit || 8
|
|
548
|
+
)
|
|
549
|
+
}
|
|
550
|
+
break
|
|
551
|
+
|
|
552
|
+
case 'delete':
|
|
553
|
+
if (method === 'DELETE' || method === 'POST') {
|
|
554
|
+
return await handleDeleteEmbeddings(body.item_id)
|
|
555
|
+
}
|
|
556
|
+
break
|
|
557
|
+
|
|
558
|
+
default:
|
|
559
|
+
if (method === 'POST') {
|
|
560
|
+
const ids: string[] = body.item_ids || (body.item_id ? [body.item_id] : [])
|
|
561
|
+
let totalCreated = 0, totalUpdated = 0
|
|
562
|
+
const allErrors: string[] = []
|
|
563
|
+
for (const id of ids) {
|
|
564
|
+
const r = await handleGenerateEmbeddings(id, body.vector_types, body.force_regenerate)
|
|
565
|
+
totalCreated += r.embeddings_created
|
|
566
|
+
totalUpdated += r.embeddings_updated
|
|
567
|
+
allErrors.push(...r.errors)
|
|
568
|
+
}
|
|
569
|
+
return {
|
|
570
|
+
success: allErrors.length === 0 || totalCreated > 0 || totalUpdated > 0,
|
|
571
|
+
embeddings_created: totalCreated,
|
|
572
|
+
embeddings_updated: totalUpdated,
|
|
573
|
+
errors: allErrors
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
throw new Error('Invalid action or method')
|
|
579
|
+
})
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { createHandler } from './_shared/middleware'
|
|
2
|
+
import { adminDb } from './_shared/db'
|
|
3
|
+
import { resolveTypeId, resolveAgentId, resolvePromptConfigId } from './_shared/resolve-ids'
|
|
4
|
+
import { runAgent } from './_shared/index'
|
|
5
|
+
import { create } from './admin-data'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @module custom_support-redaction
|
|
9
|
+
* @layer custom/cortex
|
|
10
|
+
* @stability custom
|
|
11
|
+
*
|
|
12
|
+
* KB Redaction Analysis — drives the RedactionReview page.
|
|
13
|
+
* Runs content through the KB Redaction Agent and returns structured
|
|
14
|
+
* redaction suggestions + a generalised rewrite ready for human review.
|
|
15
|
+
*
|
|
16
|
+
* Action: POST ?action=analyze
|
|
17
|
+
* body: { ticket_id: string, content: string }
|
|
18
|
+
*
|
|
19
|
+
* Response: { analysis: RedactionAnalysis }
|
|
20
|
+
* RedactionAnalysis matches the shape expected by RedactionReview.tsx:
|
|
21
|
+
* {
|
|
22
|
+
* original_content: string
|
|
23
|
+
* redacted_content: string
|
|
24
|
+
* suggestions: RedactionSuggestion[]
|
|
25
|
+
* confidence_score: number
|
|
26
|
+
* processing_metadata: { model_used, temperature, tokens_consumed }
|
|
27
|
+
* }
|
|
28
|
+
*
|
|
29
|
+
* The agent message is saved in an internal analysis thread on the ticket
|
|
30
|
+
* (thread_purpose: 'redaction_analysis') as an audit trail.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
async function handleAnalyze(ctx: any, body: any) {
|
|
34
|
+
const { ticket_id, content } = body
|
|
35
|
+
if (!ticket_id) throw new Error('ticket_id is required')
|
|
36
|
+
if (!content) throw new Error('content is required')
|
|
37
|
+
|
|
38
|
+
const [threadTypeId, agentId, promptConfigId] = await Promise.all([
|
|
39
|
+
resolveTypeId('thread', 'thread'),
|
|
40
|
+
resolveAgentId('KB Redaction Agent'),
|
|
41
|
+
resolvePromptConfigId('kb_generator'),
|
|
42
|
+
])
|
|
43
|
+
|
|
44
|
+
// Reuse existing redaction thread if one exists (idempotent re-analysis)
|
|
45
|
+
const { data: existing } = await adminDb
|
|
46
|
+
.from('threads')
|
|
47
|
+
.select('id')
|
|
48
|
+
.eq('target_type', 'items')
|
|
49
|
+
.eq('target_id', ticket_id)
|
|
50
|
+
.eq('visibility', 'internal')
|
|
51
|
+
.eq('data->>thread_purpose', 'redaction_analysis')
|
|
52
|
+
.maybeSingle()
|
|
53
|
+
|
|
54
|
+
let threadId = existing?.id
|
|
55
|
+
|
|
56
|
+
if (!threadId) {
|
|
57
|
+
const thread = await create(ctx, {
|
|
58
|
+
entity: 'threads',
|
|
59
|
+
type_id: threadTypeId,
|
|
60
|
+
target_type: 'items',
|
|
61
|
+
target_id: ticket_id,
|
|
62
|
+
visibility: 'internal',
|
|
63
|
+
status: 'active',
|
|
64
|
+
data: {
|
|
65
|
+
thread_purpose: 'redaction_analysis',
|
|
66
|
+
agent_id: agentId,
|
|
67
|
+
prompt_config_id: promptConfigId,
|
|
68
|
+
}
|
|
69
|
+
})
|
|
70
|
+
if (!thread?.id) throw new Error('Failed to create redaction thread')
|
|
71
|
+
threadId = thread.id
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Run the KB Redaction Agent — returns structured JSON in message content
|
|
75
|
+
const agentMsg = await runAgent(threadId, content, ctx)
|
|
76
|
+
|
|
77
|
+
// Parse JSON from agent response
|
|
78
|
+
let analysis: any = null
|
|
79
|
+
try {
|
|
80
|
+
const raw = agentMsg?.content || '{}'
|
|
81
|
+
const cleaned = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '').trim()
|
|
82
|
+
analysis = JSON.parse(cleaned)
|
|
83
|
+
} catch {
|
|
84
|
+
// If the agent didn't return valid JSON, return a minimal analysis
|
|
85
|
+
// so the UI can still proceed with the original content unredacted
|
|
86
|
+
analysis = {
|
|
87
|
+
original_content: content,
|
|
88
|
+
redacted_content: content,
|
|
89
|
+
suggestions: [],
|
|
90
|
+
confidence_score: 0.0,
|
|
91
|
+
processing_metadata: {
|
|
92
|
+
model_used: 'unknown',
|
|
93
|
+
temperature: 0.1,
|
|
94
|
+
tokens_consumed: 0,
|
|
95
|
+
error: 'Agent response was not valid JSON',
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Ensure required fields are present for RedactionReview.tsx
|
|
101
|
+
if (!analysis.original_content) analysis.original_content = content
|
|
102
|
+
if (!analysis.suggestions) analysis.suggestions = []
|
|
103
|
+
if (typeof analysis.confidence_score !== 'number') analysis.confidence_score = 0.5
|
|
104
|
+
if (!analysis.processing_metadata) analysis.processing_metadata = { model_used: 'unknown', temperature: 0.1, tokens_consumed: 0 }
|
|
105
|
+
|
|
106
|
+
return { analysis, analysisThreadId: threadId }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export const handler = createHandler(async (ctx, body) => {
|
|
110
|
+
const action = (ctx as any).query?.action
|
|
111
|
+
switch (action) {
|
|
112
|
+
case 'analyze': return handleAnalyze(ctx, body)
|
|
113
|
+
default: throw new Error(`Unknown action: ${action}. Use 'analyze'.`)
|
|
114
|
+
}
|
|
115
|
+
})
|