spine-framework 0.3.105 → 0.3.107

package/.framework/functions/_shared/agent-runner.ts

@@ -45,6 +45,7 @@
 import { CoreContext } from './middleware'
 import { adminDb } from './db'
 import { emitAudit } from './audit'
+import { create as createRecord } from '../admin-data'
 
 // ─── TYPES ───────────────────────────────────────────────────────────────
 
@@ -159,8 +160,12 @@ export async function runAgent(
 
     const { agent, promptConfig, thread, threadType } = config
 
+    // Resolve the message type from the thread type so support/community threads
+    // save replies with the correct type (and therefore correct permissions).
+    const replyTypeSlug = getReplyTypeSlug(threadType?.slug)
+
     // 2. Save user message to thread
-    const userMsg = await saveMessage(threadId, userMessage, 'human', null, ctx)
+    const userMsg = await saveMessage(threadId, userMessage, 'human', null, ctx, replyTypeSlug)
 
     // 3. Execute agent with full context
     const agentResponse = await executeAgentInference(
@@ -190,7 +195,8 @@ export async function runAgent(
         agent_id: agent.id,
         prompt_config_id: promptConfig.id
       },
-      ctx
+      ctx,
+      replyTypeSlug
     )
 
     // 6. Emit audit log
@@ -472,16 +478,28 @@ async function buildContext(
 /**
  * Retrieves relevant documents from the `embeddings` table for RAG.
  *
- * Uses Supabase full-text search (`textSearch`) on `content`. Falls back
- * to returning an empty array on error (never throws).
- * Future: replace with vector similarity search when embedding service is wired.
+ * Strategy (tiered):
+ *   1. Generate a query embedding via the OpenAI embeddings API.
+ *   2. Call the `match_embeddings` Postgres RPC (vector cosine similarity).
+ *   3. Deduplicate by document_id — keep best-matching chunk per article.
+ *   4. Enrich with item titles from the `items` table.
+ *   5. Fall back to full-text search if the embedding API is unavailable or
+ *      the RPC call fails (guarantees graceful degradation in dev/test).
  *
- * @param query - User message text to search against
- * @param knowledgeSources - Array of document UUIDs to filter the search
- * @param accountId - Account scope for the embeddings lookup
- * @returns Promise<any[]> — array of embedding rows (content, metadata)
- * @throws never — returns [] on failure
- * @sideEffects DB read: embeddings table
+ * Account scoping:
+ *   - Always includes the requesting account's embeddings.
+ *   - Always includes the platform KB account (spine-system), excluding
+ *     rows where metadata.security_level = 'restricted'.
+ *   - `knowledgeSources` optionally narrows to specific document UUIDs
+ *     (prompt_config.knowledge_sources); omit or pass [] for full scope.
+ *
+ * @param query - User message text to embed and search
+ * @param knowledgeSources - Optional document UUIDs to restrict search scope
+ * @param accountId - Requesting account for scoped access
+ * @returns Promise<any[]> — enriched result rows (id, title, content, similarity)
+ * @throws never — falls back to text search or [] on any failure
+ * @sideEffects HTTP call to OpenAI embeddings API (if API key present)
+ * @sideEffects DB read: match_embeddings RPC + items table
  * @calledBy buildContext (if promptConfig.knowledge_sources is non-empty)
  */
 async function retrieveKnowledge(
@@ -489,27 +507,113 @@ async function retrieveKnowledge(
   knowledgeSources: string[],
   accountId: string
 ): Promise<any[]> {
-  // For now, we'll need the query embedding - this requires calling an embedding service
-  // In production, this would call the same embedding model used to create embeddings
-  // For now, we'll do a text search fallback
+  const SIMILARITY_THRESHOLD = 0.15
+  const MATCH_COUNT = 8
 
-  const { data: docs, error } = await adminDb
-    .from('embeddings')
-    .select('*')
-    .eq('account_id', accountId)
-    .in('document_id', knowledgeSources)
-    .textSearch('content', query, {
-      type: 'websearch',
-      config: 'english'
+  // ── Tier 1: vector similarity via match_embeddings RPC ──────────────────
+  try {
+    const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
+    const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
+
+    if (!apiKey) throw new Error('no_api_key')
+
+    // Resolve platform account ID at runtime — no hardcoded UUIDs
+    const { data: platformAccount } = await adminDb
+      .from('accounts')
+      .select('id')
+      .eq('slug', 'spine-system')
+      .maybeSingle()
+
+    const accountIds: string[] = [accountId]
+    if (platformAccount?.id && platformAccount.id !== accountId) {
+      accountIds.push(platformAccount.id)
+    }
+
+    // Generate query embedding
+    const embedRes = await fetch(`${baseUrl}/embeddings`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+      body: JSON.stringify({ model: 'text-embedding-3-small', input: query }),
+    })
+    if (!embedRes.ok) throw new Error(`embed_api_${embedRes.status}`)
+    const embedJson: any = await embedRes.json()
+    const queryEmbedding: number[] = embedJson.data?.[0]?.embedding || []
+    if (!queryEmbedding.length) throw new Error('empty_embedding')
+
+    // Call vector RPC
+    const { data: matches, error: rpcError } = await adminDb.rpc('match_embeddings', {
+      query_embedding: `[${queryEmbedding.join(',')}]`,
+      match_count: MATCH_COUNT * 3,
+      filter_account_ids: accountIds,
+      filter_vector_type: 'semantic',
+      similarity_threshold: SIMILARITY_THRESHOLD,
     })
-    .limit(5)
+    if (rpcError) throw new Error(`rpc_${rpcError.message}`)
 
-  if (error) {
-    console.error('Knowledge retrieval failed:', error)
-    return []
-  }
+    // Filter: security_level restriction for cross-account results
+    const filtered = (matches || []).filter((r: any) => {
+      if (r.similarity < SIMILARITY_THRESHOLD) return false
+      if (r.account_id === accountId) return true
+      return r.metadata?.security_level !== 'restricted'
+    })
+
+    // Optionally narrow to knowledgeSources doc IDs
+    const scoped = knowledgeSources.length > 0
+      ? filtered.filter((r: any) => knowledgeSources.includes(r.document_id))
+      : filtered
+
+    // Deduplicate — best chunk per document
+    const bestByDoc = new Map<string, any>()
+    for (const r of scoped) {
+      const existing = bestByDoc.get(r.document_id)
+      if (!existing || r.similarity > existing.similarity) bestByDoc.set(r.document_id, r)
+    }
+    const deduped = [...bestByDoc.values()]
+      .sort((a, b) => b.similarity - a.similarity)
+      .slice(0, MATCH_COUNT)
+
+    if (deduped.length === 0) return []
+
+    // Enrich with item titles
+    const { data: items } = await adminDb
+      .from('items')
+      .select('id, title, description')
+      .in('id', deduped.map((r: any) => r.document_id))
+
+    const itemMap = new Map((items || []).map((i: any) => [i.id, i]))
+    return deduped.map((r: any) => ({
+      document_id: r.document_id,
+      content: r.content,
+      metadata: r.metadata,
+      similarity: r.similarity,
+      title: itemMap.get(r.document_id)?.title || '',
+    }))
+
+  } catch (vectorErr: any) {
+    // ── Tier 2: full-text fallback ──────────────────────────────────────
+    if (vectorErr.message !== 'no_api_key') {
+      console.warn('retrieveKnowledge vector search failed, falling back to text search:', vectorErr.message)
+    }
 
-  return docs || []
+    const baseQuery = adminDb
+      .from('embeddings')
+      .select('document_id, content, metadata, account_id')
+      .eq('account_id', accountId)
+      .eq('metadata->>vector_type', 'semantic')
+      .textSearch('content', query, { type: 'websearch', config: 'english' })
+      .limit(5)
+
+    const q = knowledgeSources.length > 0
+      ? baseQuery.in('document_id', knowledgeSources)
+      : baseQuery
+
+    const { data: docs, error } = await q
+    if (error) {
+      console.error('Knowledge retrieval fallback failed:', error)
+      return []
+    }
+    return docs || []
+  }
 }
 
 /**
@@ -1068,30 +1172,58 @@ async function handleEscalation(
  * @sideEffects DB write: messages table
  * @calledBy runAgent (user message + agent response)
  */
+// Module-level cache so we only look up message type_ids once per cold-start
+const _messageTypeIds: Record<string, string> = {}
+
+async function getMessageTypeId(slug: string): Promise<string> {
+  if (_messageTypeIds[slug]) return _messageTypeIds[slug]
+  const { data } = await adminDb.from('types').select('id').eq('slug', slug).maybeSingle()
+  if (!data?.id) throw new Error(`Type "${slug}" not found in types table`)
+  _messageTypeIds[slug] = data.id
+  return data.id
+}
+
+// Map a thread type slug to the corresponding message/reply type slug.
+function getReplyTypeSlug(threadTypeSlug: string): string {
+  if (threadTypeSlug === 'support_thread') return 'support_reply'
+  if (threadTypeSlug === 'community_thread') return 'community_reply'
+  return 'message'
+}
+
 async function saveMessage(
   threadId: string,
   content: string,
   messageType: 'human' | 'agent' | 'system' | 'tool_call' | 'tool_result',
   data: any,
-  ctx: CoreContext
+  ctx: CoreContext,
+  messageTypeSlug: string = 'message'
 ): Promise<any> {
-  const { data: message, error } = await adminDb
-    .from('messages')
-    .insert({
-      thread_id: threadId,
-      account_id: ctx.accountId,
-      person_id: messageType === 'human' ? ctx.principal?.id : null,
-      content,
-      content_format: 'text',
-      data: {
-        message_type: messageType,
-        ...data
-      },
-      created_at: new Date().toISOString()
-    })
-    .select()
-    .single()
+  const [typeId, { count }] = await Promise.all([
+    getMessageTypeId(messageTypeSlug),
+    adminDb.from('messages').select('id', { count: 'exact', head: true }).eq('thread_id', threadId),
+  ])
+  const sequence = (count ?? 0) + 1
+  const direction = messageType === 'human' ? 'inbound' : 'outbound'
+  // human + agent messages are customer-visible; tool/system messages are internal
+  const visibility = (messageType === 'human' || messageType === 'agent') ? 'external' : 'internal'
+
+  // Use admin-data's create() so design_schema + validation_schema are stamped
+  // from the type record — ensuring the permission engine returns full rows.
+  const message = await createRecord(ctx, {
+    entity: 'messages',
+    type_id: typeId,
+    thread_id: threadId,
+    person_id: messageType === 'human' ? ctx.principal?.id : null,
+    content,
+    direction,
+    sequence,
+    visibility,
+    data: {
+      message_type: messageType,
+      ...data
+    },
+  })
 
-  if (error) throw new Error(`Failed to save message: ${error.message}`)
+  if (!message?.id) throw new Error('Failed to save message')
   return message
 }

package/.framework/migrations/001_schema.sql

@@ -105,6 +105,7 @@ CREATE TABLE IF NOT EXISTS public.threads (
 CREATE TABLE IF NOT EXISTS public.messages (
   id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
   type_id uuid NOT NULL, thread_id uuid NOT NULL,
+  account_id uuid REFERENCES public.accounts(id),
   content text NOT NULL, direction text DEFAULT 'outbound',
   sequence integer NOT NULL, visibility text DEFAULT 'internal',
   data jsonb DEFAULT '{}', is_active boolean DEFAULT true,

package/.framework/migrations/002_rls_and_functions.sql

@@ -572,12 +572,14 @@ CREATE POLICY threads_access ON public.threads FOR ALL
     OR auth.uid() IS NULL
   );
 
--- messages
+-- messages (inherits visibility from parent thread; account-scoped threads use the
+-- account hierarchy, platform-scoped threads are visible to all authenticated users)
 DROP POLICY IF EXISTS messages_access ON public.messages;
 CREATE POLICY messages_access ON public.messages FOR ALL
   USING (EXISTS (SELECT 1 FROM public.threads t
                  WHERE t.id = messages.thread_id
-                 AND t.account_id IN (SELECT public.get_accessible_accounts(auth.uid())))
+                 AND (t.account_id IN (SELECT public.get_accessible_accounts(auth.uid()))
+                      OR t.design_schema->>'scope' = 'platform'))
          OR auth.uid() IS NULL);
 
 -- links (with platform scope visibility from 012_rls_scope.sql)

package/.framework/migrations/003_system_data.sql

@@ -249,7 +249,13 @@ BEGIN
     'Discussion thread for items or accounts',
     jsonb_build_object(
       'scope', 'account',
-      'record_permissions', jsonb_build_object('system_admin', ARRAY['create', 'read', 'update', 'delete']),
+      'record_permissions', jsonb_build_object(
+        'system_admin', ARRAY['create', 'read', 'update', 'delete'],
+        'support_admin', ARRAY['create', 'read', 'update'],
+        'support', ARRAY['create', 'read', 'update'],
+        'member_admin', ARRAY['create', 'read', 'update'],
+        'member', ARRAY['create', 'read', 'update']
+      ),
       'fields', jsonb_build_object(
         'title', jsonb_build_object('data_type', 'text', 'label', 'Title', 'required', false, 'system', true, 'validation', null),
         'target_type', jsonb_build_object('data_type', 'text', 'label', 'Target Type', 'required', true, 'system', true, 'validation', null),
@@ -295,7 +301,13 @@ BEGIN
     'Individual message within a thread',
     jsonb_build_object(
       'scope', 'account',
-      'record_permissions', jsonb_build_object('system_admin', ARRAY['create', 'read', 'update', 'delete']),
+      'record_permissions', jsonb_build_object(
+        'system_admin', ARRAY['create', 'read', 'update', 'delete'],
+        'support_admin', ARRAY['create', 'read', 'update'],
+        'support', ARRAY['create', 'read', 'update'],
+        'member_admin', ARRAY['create', 'read', 'update'],
+        'member', ARRAY['create', 'read', 'update']
+      ),
       'fields', jsonb_build_object(
         'content', jsonb_build_object('data_type', 'textarea', 'label', 'Content', 'required', true, 'system', true, 'validation', null),
         'direction', jsonb_build_object('data_type', 'text', 'label', 'Direction', 'required', true, 'system', true, 'validation', null),

package/.framework/migrations/022_route_prefix_domain_unique.sql

@@ -0,0 +1,21 @@
+-- Migration 022: Allow multiple apps to share a route_prefix when they have
+-- different prod_domains (i.e. they are served from different subdomains).
+--
+-- Before: UNIQUE(route_prefix) WHERE is_active AND route_prefix IS NOT NULL
+--   → only one app could claim "/" across the entire database
+--
+-- After:  UNIQUE(route_prefix, COALESCE(prod_domain, '')) WHERE is_active AND route_prefix IS NOT NULL
+--   → two apps can both use "/" as long as they have different prod_domains
+--   → apps with no prod_domain still get a unique slot (COALESCE maps NULL → '')
+--
+-- This is required to support multiple domain-scoped apps (e.g. Cortex on
+-- cortex.spine-framework.com and Portal on portal.spine-framework.com) that
+-- each serve from the root path "/" of their respective domains.
+
+-- Drop the old single-column unique index
+DROP INDEX IF EXISTS idx_apps_route_prefix;
+
+-- Create the new composite unique index
+CREATE UNIQUE INDEX IF NOT EXISTS idx_apps_route_prefix_domain
+  ON public.apps(route_prefix, COALESCE(prod_domain, ''))
+  WHERE is_active = true AND route_prefix IS NOT NULL;

package/.framework/migrations/023_match_embeddings_rpc.sql

@@ -0,0 +1,78 @@
+-- Migration 023: match_embeddings RPC for vector similarity search
+--
+-- Provides the `match_embeddings` Postgres function used by:
+--   - custom_kb-embeddings.ts (handleSearchEmbeddings)
+--   - agent-runner.ts (retrieveKnowledge — after upgrade)
+--
+-- Requires the `vector` extension (enabled in migration 002).
+-- Uses cosine distance (<=>). Returns rows ordered best-first.
+
+-- ============================================
+-- HNSW INDEX (idempotent)
+-- ============================================
+-- Create HNSW index on the embedding column for fast approximate nearest-neighbour search.
+-- Skip if it already exists (Postgres 15+ supports IF NOT EXISTS for indexes).
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_indexes
+    WHERE schemaname = 'public'
+      AND tablename  = 'embeddings'
+      AND indexname  = 'embeddings_embedding_hnsw_idx'
+  ) THEN
+    EXECUTE 'CREATE INDEX embeddings_embedding_hnsw_idx
+             ON public.embeddings
+             USING hnsw (embedding vector_cosine_ops)
+             WITH (m = 16, ef_construction = 64)';
+  END IF;
+END
+$$;
+
+-- ============================================
+-- match_embeddings RPC
+-- ============================================
+CREATE OR REPLACE FUNCTION public.match_embeddings(
+  query_embedding     vector(1536),
+  match_count         int            DEFAULT 10,
+  filter_account_ids  uuid[]         DEFAULT NULL,
+  filter_vector_type  text           DEFAULT NULL,
+  similarity_threshold float         DEFAULT 0.15
+)
+RETURNS TABLE (
+  id              uuid,
+  document_id     text,
+  content         text,
+  metadata        jsonb,
+  account_id      uuid,
+  similarity      float
+)
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = public
+AS $$
+BEGIN
+  RETURN QUERY
+  SELECT
+    e.id,
+    e.document_id,
+    e.content,
+    e.metadata,
+    e.account_id,
+    1 - (e.embedding <=> query_embedding) AS similarity
+  FROM public.embeddings e
+  WHERE
+    -- account scope filter (NULL = no filter — platform-wide search)
+    (filter_account_ids IS NULL OR e.account_id = ANY(filter_account_ids))
+    -- vector type filter (NULL = all types)
+    AND (filter_vector_type IS NULL OR e.metadata->>'vector_type' = filter_vector_type)
+    -- similarity gate (cosine similarity = 1 - cosine distance)
+    AND (1 - (e.embedding <=> query_embedding)) >= similarity_threshold
+  ORDER BY e.embedding <=> query_embedding   -- ascending distance = descending similarity
+  LIMIT match_count;
+END;
+$$;
+
+-- Grant execute to authenticated and service roles
+GRANT EXECUTE ON FUNCTION public.match_embeddings(vector, int, uuid[], text, float) TO authenticated;
+GRANT EXECUTE ON FUNCTION public.match_embeddings(vector, int, uuid[], text, float) TO service_role;
+GRANT EXECUTE ON FUNCTION public.match_embeddings(vector, int, uuid[], text, float) TO anon;

package/.framework/migrations/024_messages_account_id.sql

@@ -0,0 +1,9 @@
+-- Migration 024: Add account_id to messages table
+-- messages was the only runtime table missing account_id.
+-- Required by admin-data's create() (which stamps account_id on all runtime records)
+-- and by RLS policies that scope rows to an account.
+
+ALTER TABLE public.messages
+  ADD COLUMN IF NOT EXISTS account_id uuid REFERENCES public.accounts(id);
+
+CREATE INDEX IF NOT EXISTS idx_messages_account_id ON public.messages(account_id);