spine-framework 0.3.108 → 0.3.109

package/.framework/functions/embeddings.ts

@@ -4,16 +4,17 @@
  * @layer api-handler
  * @stability stable
  *
- * CRUD and search API for the `embeddings` table. Embeddings store vector
- * representations of text chunks for RAG (retrieval-augmented generation)
- * workloads. Vector similarity search is performed via the
- * `search_similar_embeddings` Postgres RPC function.
+ * CRUD, lifecycle, and search API for the `embeddings` table. Embeddings store
+ * vector representations of text chunks for RAG workloads.
  *
  * **Routed by:** `GET/POST/PATCH /.netlify/functions/embeddings`
  *
  * **Actions:**
  * | method | ?action         | handler           |
  * |--------|-----------------|-------------------|
+ * | POST   | generate        | generate          |
+ * | POST   | revoke          | revoke            |
+ * | POST   | approve         | approve           |
  * | GET    | stats           | getStats          |
  * | POST   | batch-create    | batchCreate       |
  * | POST   | delete-document | deleteByDocument  |
@@ -26,22 +27,208 @@
  * | PATCH  | —               | update            |
  *
  * **Authorization:** All operations use `ctx.db` (RLS-scoped). Account context
- * required for writes. No hard-delete endpoint — use `delete-document` or
- * `cleanup` for bulk removal.
+ * required for writes.
  *
+ * INVARIANT: `generate` is the single entry point for AI-powered embedding
+ *   creation. Custom code must call `generate` — never write to the embeddings
+ *   table directly.
  * INVARIANT: `update` only patches `content` and `metadata`.
  * INVARIANT: `searchSimilar` requires a pre-computed `query_embedding` vector.
- *   Use `searchSemantic` for text-based fallback.
  *
- * @seeAlso agent-runner.ts (executeSearchKnowledge calls search_similar_embeddings)
- * @seeAlso pipeline-runner.ts (search_knowledge stage type)
- * @seeAlso audit.ts (emitLog for embedding.* / embeddings.* events)
+ * @seeAlso agent-runner.ts (retrieveKnowledge calls match_embeddings RPC)
+ * @seeAlso pipeline-runner.ts (generate_embeddings stage type)
+ * @seeAlso audit.ts (emitLog for embedding.* events)
  */
 
+import { createHash } from 'crypto'
 import { createHandler } from './_shared/middleware'
+import { adminDb } from './_shared/db'
 import { emitLog } from './_shared/audit'
 import { sanitizeRecordData } from './_shared/permissions'
 
+// ─── UNIVERSAL CONTENT BUILDERS ──────────────────────────────────────────────
+// Each builder accepts an item record and returns a plain-text string to embed.
+// Custom apps populate item.data with the named fields below; the builder picks
+// the first non-empty value from a priority chain.
+
+function stripHtml(html: string): string {
+  return html
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&amp;/g, '&')
+    .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ')
+    .replace(/\s+/g, ' ').trim()
+}
+
+function firstNonEmpty(...values: (string | undefined | null)[]): string {
+  for (const v of values) {
+    if (v && String(v).trim()) return String(v).trim()
+  }
+  return ''
+}
+
+function buildSemanticContent(item: any): string {
+  const title = item.title || ''
+  const body = stripHtml(item.description || '')
+  return [title, body].filter(Boolean).join('\n')
+}
+
+function buildStructureContent(item: any, embeddingFields: string[] = []): string {
+  const base: Record<string, any> = {
+    title: item.title || '',
+    status: item.status || '',
+    kb_type: item.data?.kb_type || '',
+    category: item.data?.category || '',
+    tags: item.data?.tags || [],
+    audience: item.data?.audience || [],
+    security_level: item.data?.security_level || '',
+    product_area: item.data?.product_area || '',
+  }
+  for (const f of embeddingFields) {
+    if (item.data?.[f] !== undefined) base[f] = item.data[f]
+  }
+  return JSON.stringify(base)
+}
+
+function buildIntentContent(item: any): string {
+  return firstNonEmpty(
+    item.data?.intent,
+    item.data?.learning_objectives,
+    item.data?.ca_reported_issue,
+    item.data?.use_case,
+  )
+}
+
+function buildCauseContent(item: any): string {
+  return firstNonEmpty(
+    item.data?.cause,
+    item.data?.ca_true_problem,
+    item.data?.concept,
+    item.data?.root_cause,
+  )
+}
+
+function buildProcedureContent(item: any): string {
+  const steps = item.data?.procedure || item.data?.ca_solution_steps || item.data?.steps
+  if (Array.isArray(steps) && steps.length) {
+    return steps.map((s: any, i: number) => `${i + 1}. ${s}`).join('\n')
+  }
+  return firstNonEmpty(item.data?.procedure, item.data?.resolution)
+}
+
+function buildOutcomeContent(item: any): string {
+  return firstNonEmpty(
+    item.data?.outcome,
+    item.data?.ca_final_solution,
+    item.data?.outcomes,
+    item.data?.result,
+  )
+}
+
+function buildContextContent(item: any): string {
+  const steps = item.data?.ca_diagnostic_steps
+  if (Array.isArray(steps) && steps.length) {
+    return steps.map((s: any, i: number) => `${i + 1}. ${s}`).join('\n')
+  }
+  return firstNonEmpty(
+    item.data?.context,
+    item.data?.prerequisites,
+    item.data?.when_to_use,
+    item.data?.product_area,
+  )
+}
+
+function buildErrorSignalContent(item: any): string {
+  const sigs = item.data?.error_signatures
+  if (Array.isArray(sigs) && sigs.length) return sigs.join('\n')
+  const tags = item.data?.ca_analysis_tags
+  if (Array.isArray(tags) && tags.length) return tags.join('\n')
+  // Extract code-like patterns from description
+  const desc = item.description || ''
+  const codeBlocks = desc.match(/```[\s\S]*?```/g) || []
+  return codeBlocks.map((b: string) => b.replace(/```\w*\n?/g, '')).join('\n').trim()
+}
+
+function buildSummaryContentSync(item: any): string {
+  // Sync fallback — AI-generated summary handled asynchronously in generateSummary()
+  const desc = stripHtml(item.description || '')
+  return `${item.title || ''}: ${desc.slice(0, 300)}`
+}
+
+async function generateAiSummary(item: any): Promise<string> {
+  const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
+  const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
+  if (!apiKey) return buildSummaryContentSync(item)
+  const plain = stripHtml(item.description || '').slice(0, 3000)
+  try {
+    const res = await fetch(`${baseUrl}/chat/completions`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+      body: JSON.stringify({
+        model: process.env.LLM_DEFAULT_MODEL || 'gpt-4o-mini',
+        temperature: 0.2,
+        max_tokens: 120,
+        messages: [
+          { role: 'system', content: 'Write one concise paragraph (max 100 words) summarising this knowledge base article for search indexing. Output only the paragraph.' },
+          { role: 'user', content: `Title: ${item.title}\n\n${plain}` },
+        ],
+      }),
+    })
+    if (!res.ok) return buildSummaryContentSync(item)
+    const j: any = await res.json()
+    return j.choices?.[0]?.message?.content?.trim() || buildSummaryContentSync(item)
+  } catch {
+    return buildSummaryContentSync(item)
+  }
+}
+
+async function callEmbeddingApi(inputs: string[]): Promise<string[]> {
+  const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
+  const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
+  if (!apiKey || !inputs.length) return inputs.map(() => '')
+  const res = await fetch(`${baseUrl}/embeddings`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+    body: JSON.stringify({ model: 'text-embedding-3-small', input: inputs }),
+  })
+  if (!res.ok) throw new Error(`Embedding API error ${res.status}`)
+  const j: any = await res.json()
+  const sorted = (j.data || []).sort((a: any, b: any) => a.index - b.index)
+  return sorted.map((d: any) => `[${d.embedding.join(',')}]`)
+}
+
+function contentHash(text: string): string {
+  return createHash('sha256').update(text).digest('hex').slice(0, 16)
+}
+
+// Resolve platform account ID (spine-system) — memoised per process
+let _platformAccountId: string | null = null
+async function getPlatformAccountId(): Promise<string | null> {
+  if (_platformAccountId) return _platformAccountId
+  const { data } = await adminDb.from('accounts').select('id').eq('slug', 'spine-system').maybeSingle()
+  _platformAccountId = data?.id ?? null
+  return _platformAccountId
+}
+
+// ─── VECTOR TYPE → CONTENT BUILDER MAP ───────────────────────────────────────
+// Maps a vector_type string to a function that produces embeddable text from
+// an item record. `summary` is handled separately due to async LLM call.
+
+type ContentBuilder = (item: any, meta?: any) => string
+
+const CONTENT_BUILDERS: Record<string, ContentBuilder> = {
+  semantic:     buildSemanticContent,
+  structure:    (item, meta) => buildStructureContent(item, meta?.embedding_fields ?? []),
+  intent:       buildIntentContent,
+  cause:        buildCauseContent,
+  procedure:    buildProcedureContent,
+  outcome:      buildOutcomeContent,
+  context:      buildContextContent,
+  error_signal: buildErrorSignalContent,
+  // summary handled separately
+}
+
+// ─── GENERATE HANDLER ────────────────────────────────────────────────────────
+
 // ─── HANDLERS ─────────────────────────────────────────────────────────────────
 
 // ─── CHUNK_START: EMBEDDINGS_LIST ──────────────────────────────────────────────
@@ -485,6 +672,220 @@ export const searchSemantic = createHandler(async (ctx, body) => {
 })
 // ─── CHUNK_END: EMBEDDINGS_SEARCH_SEMANTIC ────────────────────────────────────────────────
 
+// ─── GENERATE ────────────────────────────────────────────────────────────────
+/**
+ * Generates embeddings for an item using the universal vector vocabulary.
+ * Reads design_schema.vector_types from the item's type record to determine
+ * which vectors to produce. Skips chunks whose content_hash is unchanged.
+ * Sets approved_at = now() for published items, NULL for drafts.
+ *
+ * Body: item_id (required), force_regenerate (bool), account_id_override (uuid)
+ *
+ * This is the ONLY sanctioned way for custom code to write to the embeddings
+ * table. Direct writes from custom functions are a hard-rule violation.
+ */
+export const generate = createHandler(async (ctx, body) => {
+  const { item_id, force_regenerate = false, account_id_override } = body || {}
+  if (!item_id) throw new Error('item_id is required')
+
+  // Load item + type
+  const { data: item, error: itemErr } = await adminDb
+    .from('items')
+    .select('*, type:types(design_schema)')
+    .eq('id', item_id)
+    .single()
+
+  if (itemErr || !item) throw new Error(`Item ${item_id} not found`)
+
+  const typeDesign = item.type?.design_schema || {}
+  const kbType = item.data?.kb_type || 'article'
+
+  // Resolve which vector types to generate
+  // design_schema.vector_types can be a plain array (all kb_types share the same list)
+  // or a keyed object { postmortem: [...], how_to: [...] } to vary by kb_type sub-type
+  let vectorTypes: string[] = []
+  const rawVT = typeDesign.vector_types
+  if (Array.isArray(rawVT)) {
+    vectorTypes = rawVT
+  } else if (rawVT && typeof rawVT === 'object') {
+    vectorTypes = rawVT[kbType] || rawVT['default'] || ['semantic', 'structure']
+  } else {
+    vectorTypes = ['semantic', 'structure']
+  }
+
+  const embeddingFields: string[] = typeDesign.embedding_fields?.structure || []
+  const resolvedAccountId = account_id_override || item.account_id || await getPlatformAccountId()
+  const isPublished = item.status === 'published'
+
+  // Load existing embeddings for hash comparison
+  const { data: existing } = await adminDb
+    .from('embeddings')
+    .select('id, content_hash, chunk_index, metadata')
+    .eq('document_id', item_id)
+
+  const existingMap = new Map<string, any>()
+  for (const row of existing || []) {
+    const vt = row.metadata?.vector_type || ''
+    const ci = row.chunk_index ?? 0
+    existingMap.set(`${vt}:${ci}`, row)
+  }
+
+  // If force_regenerate, delete all current embeddings first
+  if (force_regenerate) {
+    await adminDb.from('embeddings').delete().eq('document_id', item_id)
+    existingMap.clear()
+  }
+
+  const created: string[] = []
+  const skipped: string[] = []
+  const errors: string[] = []
+
+  // Build content for non-summary vectors synchronously
+  const syncVectors = vectorTypes.filter(vt => vt !== 'summary')
+  const contents: Array<{ vt: string; text: string; chunkIndex: number }> = []
+
+  for (const vt of syncVectors) {
+    const builder = CONTENT_BUILDERS[vt]
+    if (!builder) { errors.push(`Unknown vector type: ${vt}`); continue }
+    const text = builder(item, { embedding_fields: embeddingFields })
+    if (!text) { skipped.push(`${vt}:empty`); continue }
+    contents.push({ vt, text, chunkIndex: 0 })
+  }
+
+  // Summary vector — async LLM call
+  if (vectorTypes.includes('summary')) {
+    const summaryText = await generateAiSummary(item)
+    if (summaryText) contents.push({ vt: 'summary', text: summaryText, chunkIndex: 0 })
+  }
+
+  // Filter out unchanged chunks (by content hash)
+  const toEmbed = contents.filter(({ vt, text, chunkIndex }) => {
+    const hash = contentHash(text)
+    const key = `${vt}:${chunkIndex}`
+    const row = existingMap.get(key)
+    if (row && row.content_hash === hash) { skipped.push(key); return false }
+    return true
+  })
+
+  if (!toEmbed.length) {
+    return { item_id, embeddings_created: 0, embeddings_skipped: skipped.length, errors }
+  }
+
+  // Single batched API call for all vectors that need updating
+  let vectors: string[] = []
+  try {
+    vectors = await callEmbeddingApi(toEmbed.map(c => c.text))
+  } catch (e: any) {
+    throw new Error(`Embedding API call failed: ${e.message}`)
+  }
+
+  const now = new Date().toISOString()
+
+  // Upsert each vector
+  for (let i = 0; i < toEmbed.length; i++) {
+    const { vt, text, chunkIndex } = toEmbed[i]
+    const hash = contentHash(text)
+    const key = `${vt}:${chunkIndex}`
+    const existingRow = existingMap.get(key)
+
+    const row = {
+      account_id:   resolvedAccountId,
+      model_id:     'text-embedding-3-small',
+      document_id:  item_id,
+      chunk_index:  chunkIndex,
+      content:      text,
+      embedding:    vectors[i],
+      content_hash: hash,
+      approved_at:  isPublished ? now : null,
+      revoked_at:   null,
+      metadata: {
+        vector_type:    vt,
+        item_type:      item.type?.design_schema ? (item.data?.kb_type ? 'kb_article' : 'item') : 'item',
+        kb_type:        kbType,
+        security_level: item.data?.security_level || 'internal',
+        source_item_id: item_id,
+      },
+    }
+
+    if (existingRow) {
+      await adminDb.from('embeddings').update(row).eq('id', existingRow.id)
+    } else {
+      await adminDb.from('embeddings').insert(row)
+    }
+    created.push(key)
+  }
+
+  await emitLog(ctx, 'embeddings.generated',
+    { type: 'item', id: item_id },
+    { after: { vectors_created: created.length, vectors_skipped: skipped.length, is_published: isPublished } }
+  )
+
+  return {
+    item_id,
+    embeddings_created: created.length,
+    embeddings_skipped: skipped.length,
+    errors,
+  }
+})
+
+// ─── REVOKE ───────────────────────────────────────────────────────────────────
+/**
+ * Sets revoked_at = now() on all embeddings for a document_id.
+ * Embeddings are NOT deleted — the row stays for audit trail.
+ * match_embeddings will not return them (revoked_at IS NULL is a hard filter).
+ *
+ * Body: document_id (required)
+ */
+export const revoke = createHandler(async (ctx, body) => {
+  const { document_id } = body || {}
+  if (!document_id) throw new Error('document_id is required')
+
+  const { data, error } = await adminDb
+    .from('embeddings')
+    .update({ revoked_at: new Date().toISOString() })
+    .eq('document_id', document_id)
+    .is('revoked_at', null)
+    .select('id')
+
+  if (error) throw error
+
+  await emitLog(ctx, 'embeddings.revoked',
+    { type: 'system', id: document_id },
+    { after: { revoked_count: data?.length || 0 } }
+  )
+
+  return { document_id, revoked_count: data?.length || 0 }
+})
+
+// ─── APPROVE ─────────────────────────────────────────────────────────────────
+/**
+ * Clears revoked_at and refreshes approved_at on all embeddings for a document_id.
+ * Called when an item re-enters published state (also handled automatically by
+ * the items_embedding_sync DB trigger, but available as an explicit API action).
+ *
+ * Body: document_id (required)
+ */
+export const approve = createHandler(async (ctx, body) => {
+  const { document_id } = body || {}
+  if (!document_id) throw new Error('document_id is required')
+
+  const now = new Date().toISOString()
+  const { data, error } = await adminDb
+    .from('embeddings')
+    .update({ approved_at: now, revoked_at: null })
+    .eq('document_id', document_id)
+    .select('id')
+
+  if (error) throw error
+
+  await emitLog(ctx, 'embeddings.approved',
+    { type: 'system', id: document_id },
+    { after: { approved_count: data?.length || 0 } }
+  )
+
+  return { document_id, approved_count: data?.length || 0 }
+})
+
 // ─── MAIN HANDLER ────────────────────────────────────────────────────────────
 
 // ─── CHUNK_START: EMBEDDINGS_HANDLER ──────────────────────────────────────────────
@@ -507,6 +908,21 @@ export const handler = createHandler(async (ctx, body) => {
   const method = ctx.query?.method || 'GET'
 
   switch (action) {
+    case 'generate':
+      if (method === 'POST') {
+        return await generate(ctx, body)
+      }
+      break
+    case 'revoke':
+      if (method === 'POST') {
+        return await revoke(ctx, body)
+      }
+      break
+    case 'approve':
+      if (method === 'POST') {
+        return await approve(ctx, body)
+      }
+      break
     case 'stats':
       if (method === 'GET') {
         return await getStats(ctx, body)