spine-framework 0.3.108 → 0.3.109

package/.framework/migrations/027_embedding_lifecycle.sql

@@ -0,0 +1,176 @@
+-- Migration 027: Embedding lifecycle — approved_at, revoked_at, content_hash
+--
+-- Adds three columns to the embeddings table so that:
+--   1. Only approved embeddings are returned by match_embeddings (hard filter, not prompt-based).
+--   2. Embeddings are invalidated automatically when their source item is unpublished or deleted.
+--   3. Re-embedding is skipped when content has not changed (content_hash deduplication).
+--
+-- Adds filter_item_types parameter to match_embeddings RPC so agents can restrict
+-- retrieval to specific content types (kb_article, code_chunk, db_schema, etc.) at
+-- the DB layer rather than relying on post-processing.
+--
+-- Adds helper functions revoke_embeddings_for_item and approve_embeddings_for_item
+-- called by the items_embedding_sync trigger.
+--
+-- IMPORTANT: approved_at defaults to now() so all existing embeddings are treated
+-- as approved. revoked_at defaults to NULL (not revoked).
+
+-- ============================================
+-- COLUMNS
+-- ============================================
+
+ALTER TABLE public.embeddings
+  ADD COLUMN IF NOT EXISTS approved_at  timestamptz DEFAULT now(),
+  ADD COLUMN IF NOT EXISTS revoked_at   timestamptz DEFAULT NULL,
+  ADD COLUMN IF NOT EXISTS content_hash text        DEFAULT NULL;
+
+-- Partial index for fast active-embedding lookups (approved and not revoked)
+CREATE INDEX IF NOT EXISTS embeddings_active_idx
+  ON public.embeddings (account_id, document_id)
+  WHERE approved_at IS NOT NULL AND revoked_at IS NULL;
+
+-- ============================================
+-- HELPER FUNCTIONS
+-- ============================================
+
+-- Revoke all embeddings for a given item (called when item leaves published state)
+CREATE OR REPLACE FUNCTION public.revoke_embeddings_for_item(p_item_id uuid)
+RETURNS integer
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = public
+AS $$
+DECLARE
+  v_count integer;
+BEGIN
+  UPDATE public.embeddings
+  SET    revoked_at = now()
+  WHERE  document_id = p_item_id::text
+  AND    revoked_at  IS NULL;
+
+  GET DIAGNOSTICS v_count = ROW_COUNT;
+  RETURN v_count;
+END;
+$$;
+
+-- Approve all embeddings for a given item (called when item re-enters published state)
+CREATE OR REPLACE FUNCTION public.approve_embeddings_for_item(p_item_id uuid)
+RETURNS integer
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = public
+AS $$
+DECLARE
+  v_count integer;
+BEGIN
+  UPDATE public.embeddings
+  SET    approved_at = now(),
+         revoked_at  = NULL
+  WHERE  document_id = p_item_id::text;
+
+  GET DIAGNOSTICS v_count = ROW_COUNT;
+  RETURN v_count;
+END;
+$$;
+
+GRANT EXECUTE ON FUNCTION public.revoke_embeddings_for_item(uuid) TO service_role;
+GRANT EXECUTE ON FUNCTION public.approve_embeddings_for_item(uuid) TO service_role;
+
+-- ============================================
+-- ITEMS → EMBEDDINGS SYNC TRIGGER
+-- ============================================
+-- When an item's status changes, keep its embeddings in sync automatically.
+-- published  → approve (clear revoked_at, set approved_at)
+-- anything   → revoke  (set revoked_at, leave approved_at intact for audit)
+
+CREATE OR REPLACE FUNCTION public.sync_item_embeddings()
+RETURNS TRIGGER
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = public
+AS $$
+BEGIN
+  -- Only act on status changes
+  IF OLD.status IS NOT DISTINCT FROM NEW.status THEN
+    RETURN NEW;
+  END IF;
+
+  IF NEW.status = 'published' THEN
+    PERFORM public.approve_embeddings_for_item(NEW.id);
+  ELSE
+    PERFORM public.revoke_embeddings_for_item(NEW.id);
+  END IF;
+
+  RETURN NEW;
+END;
+$$;
+
+DROP TRIGGER IF EXISTS items_embedding_sync ON public.items;
+CREATE TRIGGER items_embedding_sync
+  AFTER UPDATE OF status ON public.items
+  FOR EACH ROW
+  EXECUTE FUNCTION public.sync_item_embeddings();
+
+-- ============================================
+-- UPDATED match_embeddings RPC
+-- ============================================
+-- Adds filter_item_types parameter and hard approved_at / revoked_at filter.
+-- Backward compatible: existing callers that omit filter_item_types get the same
+-- behaviour as before, except that revoked / unapproved embeddings are now excluded.
+
+CREATE OR REPLACE FUNCTION public.match_embeddings(
+  query_embedding      vector(1536),
+  match_count          int            DEFAULT 10,
+  filter_account_ids   uuid[]         DEFAULT NULL,
+  filter_vector_type   text           DEFAULT NULL,
+  similarity_threshold float          DEFAULT 0.15,
+  filter_item_types    text[]         DEFAULT NULL
+)
+RETURNS TABLE (
+  id           uuid,
+  document_id  text,
+  content      text,
+  metadata     jsonb,
+  account_id   uuid,
+  similarity   float
+)
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = public
+AS $$
+BEGIN
+  RETURN QUERY
+  SELECT
+    e.id,
+    e.document_id,
+    e.content,
+    e.metadata,
+    e.account_id,
+    1 - (e.embedding <=> query_embedding) AS similarity
+  FROM public.embeddings e
+  WHERE
+    -- ── Hard lifecycle filter — never negotiable ──────────────────────────
+    e.approved_at IS NOT NULL
+    AND e.revoked_at  IS NULL
+
+    -- ── Account scope (NULL = no account filter — platform-wide search) ───
+    AND (filter_account_ids IS NULL OR e.account_id = ANY(filter_account_ids))
+
+    -- ── Vector type (NULL = all types) ────────────────────────────────────
+    AND (filter_vector_type IS NULL OR e.metadata->>'vector_type' = filter_vector_type)
+
+    -- ── Item type (NULL = all item types) ─────────────────────────────────
+    AND (filter_item_types IS NULL OR e.metadata->>'item_type' = ANY(filter_item_types))
+
+    -- ── Similarity gate ───────────────────────────────────────────────────
+    AND (1 - (e.embedding <=> query_embedding)) >= similarity_threshold
+
+  ORDER BY e.embedding <=> query_embedding
+  LIMIT match_count;
+END;
+$$;
+
+-- Re-grant execute to all roles (replace = drop+create resets grants)
+GRANT EXECUTE ON FUNCTION public.match_embeddings(vector, int, uuid[], text, float, text[]) TO authenticated;
+GRANT EXECUTE ON FUNCTION public.match_embeddings(vector, int, uuid[], text, float, text[]) TO service_role;
+GRANT EXECUTE ON FUNCTION public.match_embeddings(vector, int, uuid[], text, float, text[]) TO anon;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spine-framework",
-  "version": "0.3.108",
+  "version": "0.3.109",
   "description": "Multi-tenant, modular application platform for modern SaaS systems",
   "type": "module",
   "bin": {
@@ -33,6 +33,7 @@
     ".framework/migrations/024_messages_account_id.sql",
     ".framework/migrations/025_retype_threads_messages.sql",
     ".framework/migrations/026_fix_rls_policies.sql",
+    ".framework/migrations/027_embedding_lifecycle.sql",
     ".framework/public/",
     ".framework/scripts/",
     ".framework/seeds/",