realtimex-crm 0.9.1 → 0.9.2

package/supabase/migrations/20251221000003_fix_email_matching.sql

@@ -0,0 +1,70 @@
+-- Migration: Fix email matching to be more robust with jsonb arrays
+
+CREATE OR REPLACE FUNCTION auto_link_contact()
+RETURNS TRIGGER
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  matched_contact_id bigint;
+BEGIN
+  -- Only attempt matching if contact_id is not already set
+  IF NEW.contact_id IS NOT NULL THEN
+    RETURN NEW;
+  END IF;
+
+  -- Strategy 1: Try exact email match
+  -- Note: email is stored as email_jsonb (array of email objects)
+  -- Use jsonb_array_elements to check each email in the array
+  IF NEW.metadata ? 'from' AND NEW.metadata->>'from' LIKE '%@%' THEN
+    SELECT c.id INTO matched_contact_id
+    FROM contacts c,
+         jsonb_array_elements(c.email_jsonb) AS email
+    WHERE lower(email->>'email') = lower(NEW.metadata->>'from')
+    LIMIT 1;
+
+    IF matched_contact_id IS NOT NULL THEN
+      NEW.contact_id := matched_contact_id;
+      RETURN NEW;
+    END IF;
+  END IF;
+
+  -- Strategy 2: Try exact phone match (E.164 normalized)
+  -- Note: phones are stored as phone_jsonb (array of phone objects)
+  IF NEW.metadata ? 'from' AND NEW.metadata->>'from' LIKE '+%' THEN
+    SELECT c.id INTO matched_contact_id
+    FROM contacts c,
+         jsonb_array_elements(c.phone_jsonb) AS phone
+    WHERE normalize_phone(phone->>'number') = normalize_phone(NEW.metadata->>'from')
+    LIMIT 1;
+
+    IF matched_contact_id IS NOT NULL THEN
+      NEW.contact_id := matched_contact_id;
+      RETURN NEW;
+    END IF;
+  END IF;
+
+  -- Strategy 3: Try fuzzy phone match (last 10 digits for US numbers)
+  IF NEW.metadata ? 'from' AND length(regexp_replace(NEW.metadata->>'from', '[^0-9]', '', 'g')) >= 10 THEN
+    SELECT c.id INTO matched_contact_id
+    FROM contacts c,
+         jsonb_array_elements(c.phone_jsonb) AS phone
+    WHERE right(regexp_replace(phone->>'number', '[^0-9]', '', 'g'), 10) =
+          right(regexp_replace(NEW.metadata->>'from', '[^0-9]', '', 'g'), 10)
+    LIMIT 1;
+
+    IF matched_contact_id IS NOT NULL THEN
+      NEW.contact_id := matched_contact_id;
+      RETURN NEW;
+    END IF;
+  END IF;
+
+  -- No match found - activity will be created as "orphan" with contact_id = NULL
+  RETURN NEW;
+END;
+$$;
+
+COMMENT ON FUNCTION auto_link_contact() IS
+'Automatically links activities to contacts based on email or phone number in metadata.from field.
+Uses case-insensitive email matching and handles jsonb array structures correctly.
+Priority: Email match > Phone E.164 match > Fuzzy phone match (last 10 digits).
+Activities without matches become orphans (contact_id = NULL).';

package/supabase/migrations/20251221000004_time_based_work_stealing.sql

@@ -0,0 +1,99 @@
+-- Migration: Time-Based Work Stealing (No Coordination Table)
+-- Removes the need for crm_processing_nodes by using activity age as prioritization signal
+
+-- Function: Unlock stale locks
+-- Activities that have been locked for >5 minutes get unlocked (crashed/offline agents)
+CREATE OR REPLACE FUNCTION unlock_stale_locks()
+RETURNS INTEGER
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  unlocked_count INTEGER;
+BEGIN
+  UPDATE activities
+  SET
+    locked_by = NULL,
+    locked_at = NULL,
+    processing_status = 'raw'
+  WHERE
+    processing_status = 'processing'
+    AND locked_at < NOW() - INTERVAL '5 minutes';
+
+  GET DIAGNOSTICS unlocked_count = ROW_COUNT;
+  RETURN unlocked_count;
+END;
+$$;
+
+COMMENT ON FUNCTION unlock_stale_locks() IS
+'Unlocks activities that have been in processing state for >5 minutes.
+Should be called periodically (e.g., every minute via pg_cron or application logic).';
+
+-- Function: Claim stale activity from any user
+-- Returns activities that are >5 minutes old (timeout fallback for offline users)
+CREATE OR REPLACE FUNCTION claim_stale_activity()
+RETURNS TABLE (
+  id uuid,
+  type text,
+  direction text,
+  sales_id bigint,
+  contact_id bigint,
+  company_id bigint,
+  deal_id bigint,
+  raw_data jsonb,
+  processing_status text,
+  processed_data jsonb,
+  metadata jsonb,
+  locked_by text,
+  locked_at timestamptz,
+  created_at timestamptz,
+  updated_at timestamptz
+)
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  RETURN QUERY
+  UPDATE activities
+  SET
+    locked_by = 'agent',
+    locked_at = NOW(),
+    processing_status = 'processing'
+  WHERE activities.id = (
+    SELECT a.id
+    FROM activities a
+    WHERE a.processing_status = 'raw'
+      AND a.created_at < NOW() - INTERVAL '5 minutes'  -- Only stale activities
+      AND a.locked_by IS NULL
+    ORDER BY a.created_at ASC
+    FOR UPDATE SKIP LOCKED
+    LIMIT 1
+  )
+  RETURNING
+    activities.id,
+    activities.type,
+    activities.direction,
+    activities.sales_id,
+    activities.contact_id,
+    activities.company_id,
+    activities.deal_id,
+    activities.raw_data,
+    activities.processing_status,
+    activities.processed_data,
+    activities.metadata,
+    activities.locked_by,
+    activities.locked_at,
+    activities.created_at,
+    activities.updated_at;
+END;
+$$;
+
+COMMENT ON FUNCTION claim_stale_activity() IS
+'Claims the oldest raw activity that has been waiting for >5 minutes.
+Used by agents as fallback when no fresh work is available from their own queue.
+Implements time-based work stealing without coordination table.';
+
+-- Note: The existing claim_next_pending_activity(p_agent_sales_id) function
+-- continues to work for claiming fresh work from a specific user's queue.
+-- Agent prioritization logic:
+--   1. Try claim_next_pending_activity(my_sales_id) for my own fresh work
+--   2. Try claim_stale_activity() for anyone's stale work (>5 min old)
+--   3. Repeat every 5 seconds

package/supabase/migrations/20251221000005_realtime_functions.sql

@@ -0,0 +1,73 @@
+-- Migration: Realtime Helper Functions
+-- Purpose: Allow programmatic enabling of Realtime for tables (creating channels)
+-- This supports the RealTimeX App integration which requires Realtime subscriptions.
+
+-- 1. Create a function to safely add tables to the supabase_realtime publication
+create or replace function enable_realtime_for_table(target_table text, target_schema text default 'public')
+returns void
+language plpgsql
+security definer -- Required: Regular users can't ALTER PUBLICATION
+set search_path = public, pg_catalog -- Security: Prevent search_path attacks
+as $$
+declare
+  publication_name text := 'supabase_realtime';
+begin
+  -- Validate inputs
+  if target_table is null or target_table = '' then
+    raise exception 'target_table cannot be null or empty';
+  end if;
+
+  -- Ensure the publication exists (idempotent)
+  if not exists (select 1 from pg_publication where pubname = publication_name) then
+    execute format('create publication %I', publication_name);
+    raise notice 'Created publication %', publication_name;
+  end if;
+
+  -- Check if table is already in publication
+  if not exists (
+    select 1
+    from pg_publication_tables
+    where pubname = publication_name
+      and schemaname = target_schema
+      and tablename = target_table
+  ) then
+    execute format('alter publication %I add table %I.%I', publication_name, target_schema, target_table);
+    raise notice 'Added table %.% to publication %', target_schema, target_table, publication_name;
+  else
+    raise notice 'Table %.% is already in publication %', target_schema, target_table, publication_name;
+  end if;
+end;
+$$;
+
+comment on function enable_realtime_for_table(text, text) is
+'Safely adds tables to supabase_realtime publication. Realtime respects RLS policies - clients only receive changes for rows they can SELECT.';
+
+-- 2. Enable Realtime for core CRM tables
+-- Note: All realtime subscriptions respect RLS policies - users only receive changes for rows they can SELECT
+
+-- Activities: Real-time ingestion, activity feed updates
+select enable_realtime_for_table('activities');
+
+-- Tasks: Task assignments, status changes, team collaboration
+select enable_realtime_for_table('tasks');
+
+-- Contacts: Contact profile updates, new contact notifications
+select enable_realtime_for_table('contacts');
+
+-- Companies: Company data changes, relationship updates
+select enable_realtime_for_table('companies');
+
+-- Deals: Pipeline movement, stage changes, deal value updates (critical for sales teams!)
+select enable_realtime_for_table('deals');
+
+-- Contact Notes: Real-time note additions/updates
+select enable_realtime_for_table('contactNotes');
+
+-- Deal Notes: Deal discussion updates
+select enable_realtime_for_table('dealNotes');
+
+-- Sales: Team member changes, user status updates
+select enable_realtime_for_table('sales');
+
+-- Ingestion Providers: Channel configuration changes
+select enable_realtime_for_table('ingestion_providers');

package/supabase/migrations/20251221000006_enable_pg_net.sql

@@ -0,0 +1,3 @@
+-- Enable pg_net extension for webhook dispatcher
+-- This extension provides the net.http_post function used by the cron job
+create extension if not exists pg_net with schema extensions;

package/supabase/migrations/20251222075019_enable_extensions_and_configure_cron.sql

@@ -0,0 +1,86 @@
+-- Enable Extensions and Configure Cron for Webhook System
+-- This migration ensures all required extensions are enabled for the webhook dispatcher
+
+-- 1. Enable required extensions
+CREATE EXTENSION IF NOT EXISTS pg_cron WITH SCHEMA extensions;
+CREATE EXTENSION IF NOT EXISTS pg_net WITH SCHEMA extensions;
+
+-- 2. Create a helper function to configure cron settings
+CREATE OR REPLACE FUNCTION public.configure_webhook_cron_settings(
+    p_supabase_url text,
+    p_service_role_key text
+) RETURNS text
+LANGUAGE plpgsql
+SECURITY DEFINER
+AS $$
+BEGIN
+    -- Set the configuration at database level
+    EXECUTE format('ALTER DATABASE %I SET app.settings.supabase_url = %L',
+                   current_database(), p_supabase_url);
+    EXECUTE format('ALTER DATABASE %I SET app.settings.service_role_key = %L',
+                   current_database(), p_service_role_key);
+
+    -- Reload configuration for current session
+    PERFORM pg_reload_conf();
+
+    RETURN format('Configuration updated successfully. Please reconnect to see changes. URL: %s', p_supabase_url);
+END;
+$$;
+
+-- Grant execute permission to authenticated users
+GRANT EXECUTE ON FUNCTION public.configure_webhook_cron_settings(text, text) TO authenticated;
+GRANT EXECUTE ON FUNCTION public.configure_webhook_cron_settings(text, text) TO service_role;
+
+-- 3. Add a comment with usage instructions
+COMMENT ON FUNCTION public.configure_webhook_cron_settings(text, text) IS
+'Configures the webhook cron dispatcher settings.
+
+Usage:
+  SELECT configure_webhook_cron_settings(
+    ''your-project-ref.supabase.co'',
+    ''your-service-role-key-here''
+  );
+
+You can find these values in your Supabase Dashboard:
+- Project URL: Settings → API → Project URL (without https://)
+- Service Role Key: Settings → API → service_role key
+
+After running this function, reconnect to the database for changes to take effect.
+The cron job scheduled in 20251219120200_webhook_cron.sql will then work properly.';
+
+-- 4. Check if settings are already configured
+DO $$
+DECLARE
+    current_url text;
+    current_key text;
+BEGIN
+    -- Try to read current settings
+    BEGIN
+        current_url := current_setting('app.settings.supabase_url', true);
+        current_key := current_setting('app.settings.service_role_key', true);
+
+        IF current_url IS NOT NULL AND current_key IS NOT NULL THEN
+            RAISE NOTICE 'Webhook cron settings already configured:';
+            RAISE NOTICE '  URL: %', current_url;
+            RAISE NOTICE '  Key: %', left(current_key, 20) || '...';
+        ELSE
+            RAISE NOTICE '=================================================================';
+            RAISE NOTICE 'IMPORTANT: Webhook cron settings NOT configured!';
+            RAISE NOTICE '=================================================================';
+            RAISE NOTICE 'The webhook dispatcher cron job requires configuration.';
+            RAISE NOTICE '';
+            RAISE NOTICE 'Run this SQL to configure (replace with your actual values):';
+            RAISE NOTICE '';
+            RAISE NOTICE '  SELECT configure_webhook_cron_settings(';
+            RAISE NOTICE '    ''your-project-ref.supabase.co'',';
+            RAISE NOTICE '    ''your-service-role-key-here''';
+            RAISE NOTICE '  );';
+            RAISE NOTICE '';
+            RAISE NOTICE 'Find these values in: Supabase Dashboard → Settings → API';
+            RAISE NOTICE '=================================================================';
+        END IF;
+    EXCEPTION WHEN OTHERS THEN
+        RAISE NOTICE 'Could not check webhook cron settings. This is normal for fresh deployments.';
+    END;
+END;
+$$;

package/supabase/migrations/20251222094036_large_payload_storage.sql

@@ -0,0 +1,213 @@
+-- Large Payload Storage System
+-- Automatically moves large raw_data payloads to Supabase Storage to prevent table bloat
+
+-- 1. Create storage bucket for activity payloads
+INSERT INTO storage.buckets (id, name, public, file_size_limit, allowed_mime_types)
+VALUES (
+  'activity-payloads',
+  'activity-payloads',
+  false, -- Private bucket
+  NULL, -- No file size limit (supports large payloads of any size)
+  NULL  -- Allow all MIME types (PDFs, images, videos, etc.)
+)
+ON CONFLICT (id) DO NOTHING;
+
+-- 2. Create storage policy for service role access
+CREATE POLICY "Service role can manage activity payloads"
+ON storage.objects FOR ALL
+TO service_role
+USING (bucket_id = 'activity-payloads')
+WITH CHECK (bucket_id = 'activity-payloads');
+
+-- Allow authenticated users to read their own activity payloads
+CREATE POLICY "Users can read their activity payloads"
+ON storage.objects FOR SELECT
+TO authenticated
+USING (
+  bucket_id = 'activity-payloads' AND
+  (storage.foldername(name))[1] IN (
+    SELECT id::text FROM activities WHERE sales_id = (
+      SELECT id FROM sales WHERE user_id = auth.uid()
+    )
+  )
+);
+
+-- 3. Add columns to activities table for payload storage tracking
+ALTER TABLE activities
+  ADD COLUMN IF NOT EXISTS payload_size_bytes INTEGER,
+  ADD COLUMN IF NOT EXISTS payload_storage_status TEXT DEFAULT 'inline' CHECK (payload_storage_status IN ('inline', 'pending_move', 'in_storage')),
+  ADD COLUMN IF NOT EXISTS storage_path TEXT;
+
+-- Create index for finding activities that need storage migration
+CREATE INDEX IF NOT EXISTS idx_activities_pending_storage
+  ON activities(payload_storage_status, created_at)
+  WHERE payload_storage_status = 'pending_move';
+
+-- 4. Function to calculate payload size
+CREATE OR REPLACE FUNCTION calculate_payload_size(data JSONB)
+RETURNS INTEGER
+LANGUAGE plpgsql
+IMMUTABLE
+AS $$
+BEGIN
+  RETURN octet_length(data::text);
+END;
+$$;
+
+-- 5. Trigger function to detect large payloads
+CREATE OR REPLACE FUNCTION check_payload_size()
+RETURNS TRIGGER
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  payload_size INTEGER;
+  size_threshold INTEGER := 102400; -- 100KB threshold
+BEGIN
+  -- Calculate payload size
+  payload_size := calculate_payload_size(NEW.raw_data);
+  NEW.payload_size_bytes := payload_size;
+
+  -- If payload exceeds threshold, mark for storage migration
+  IF payload_size > size_threshold THEN
+    NEW.payload_storage_status := 'pending_move';
+
+    RAISE NOTICE 'Large payload detected (% bytes) for activity %. Will be moved to storage asynchronously.',
+      payload_size, NEW.id;
+  ELSE
+    NEW.payload_storage_status := 'inline';
+  END IF;
+
+  RETURN NEW;
+END;
+$$;
+
+-- 6. Create trigger on activities table
+DROP TRIGGER IF EXISTS trigger_check_payload_size ON activities;
+CREATE TRIGGER trigger_check_payload_size
+  BEFORE INSERT OR UPDATE OF raw_data ON activities
+  FOR EACH ROW
+  EXECUTE FUNCTION check_payload_size();
+
+-- 7. Function to move payload to storage (called by Edge Function)
+CREATE OR REPLACE FUNCTION move_payload_to_storage(
+  p_activity_id BIGINT,
+  p_storage_path TEXT
+)
+RETURNS JSONB
+LANGUAGE plpgsql
+SECURITY DEFINER
+AS $$
+DECLARE
+  original_payload JSONB;
+  payload_metadata JSONB;
+  result JSONB;
+BEGIN
+  -- Get the original payload
+  SELECT raw_data INTO original_payload
+  FROM activities
+  WHERE id = p_activity_id
+  FOR UPDATE;
+
+  IF original_payload IS NULL THEN
+    RETURN jsonb_build_object(
+      'success', false,
+      'error', 'Activity not found'
+    );
+  END IF;
+
+  -- Create metadata object to replace raw_data
+  payload_metadata := jsonb_build_object(
+    'payload_type', 'storage_ref',
+    'storage_path', p_storage_path,
+    'size_bytes', calculate_payload_size(original_payload),
+    'moved_at', now()
+  );
+
+  -- Update the activity record
+  UPDATE activities
+  SET
+    raw_data = payload_metadata,
+    payload_storage_status = 'in_storage',
+    storage_path = p_storage_path,
+    updated_at = now()
+  WHERE id = p_activity_id;
+
+  -- Return the original payload for upload
+  result := jsonb_build_object(
+    'success', true,
+    'activity_id', p_activity_id,
+    'storage_path', p_storage_path,
+    'payload', original_payload
+  );
+
+  RETURN result;
+END;
+$$;
+
+-- Grant permissions
+GRANT EXECUTE ON FUNCTION calculate_payload_size(JSONB) TO authenticated, service_role;
+GRANT EXECUTE ON FUNCTION move_payload_to_storage(BIGINT, TEXT) TO service_role;
+
+-- 8. Helper function to retrieve full payload (handles both inline and storage)
+CREATE OR REPLACE FUNCTION get_activity_payload(p_activity_id BIGINT)
+RETURNS JSONB
+LANGUAGE plpgsql
+SECURITY DEFINER
+AS $$
+DECLARE
+  activity_record RECORD;
+BEGIN
+  SELECT
+    raw_data,
+    payload_storage_status,
+    storage_path
+  INTO activity_record
+  FROM activities
+  WHERE id = p_activity_id;
+
+  IF activity_record IS NULL THEN
+    RETURN jsonb_build_object('error', 'Activity not found');
+  END IF;
+
+  -- If payload is inline or pending move, return raw_data directly
+  IF activity_record.payload_storage_status IN ('inline', 'pending_move') THEN
+    RETURN activity_record.raw_data;
+  END IF;
+
+  -- If in storage, return metadata with instructions
+  -- (Actual retrieval from storage must be done via Storage API)
+  RETURN jsonb_build_object(
+    'payload_type', 'storage_ref',
+    'storage_path', activity_record.storage_path,
+    'message', 'Payload is in storage. Use Supabase Storage API to retrieve.',
+    'url', '/storage/v1/object/activity-payloads/' || activity_record.storage_path
+  );
+END;
+$$;
+
+GRANT EXECUTE ON FUNCTION get_activity_payload(BIGINT) TO authenticated, service_role;
+
+-- 9. Add comment with usage instructions
+COMMENT ON COLUMN activities.payload_storage_status IS
+'Tracks where the activity payload is stored:
+- inline: Payload is in raw_data column (< 100KB)
+- pending_move: Payload is large and queued for storage migration
+- in_storage: Payload has been moved to Supabase Storage, raw_data contains reference';
+
+COMMENT ON FUNCTION move_payload_to_storage(BIGINT, TEXT) IS
+'Moves a large activity payload to Supabase Storage.
+Called by the process-large-payloads Edge Function.
+
+Usage:
+  SELECT move_payload_to_storage(12345, ''12345/1234567890.json'');
+
+Returns the original payload for upload to storage.';
+
+COMMENT ON FUNCTION get_activity_payload(BIGINT) IS
+'Retrieves activity payload, handling both inline and storage-based payloads.
+
+Usage:
+  SELECT get_activity_payload(12345);
+
+For storage-based payloads, returns metadata with storage path.
+Use Supabase Storage API to download the actual content.';

package/supabase/migrations/20251222094247_large_payload_cron.sql

@@ -0,0 +1,28 @@
+-- Schedule cron job to process large payloads
+-- Runs every 5 minutes to move pending large payloads to storage
+
+-- Schedule the large payload processor
+SELECT cron.schedule(
+    'process-large-payloads',
+    '*/5 * * * *', -- Every 5 minutes
+    $$
+    SELECT
+      net.http_post(
+          url:='https://' || current_setting('app.settings.supabase_url', true) || '/functions/v1/process-large-payloads',
+          headers:=jsonb_build_object(
+              'Content-Type','application/json',
+              'Authorization', 'Bearer ' || current_setting('app.settings.service_role_key', true)
+          ),
+          body:=jsonb_build_object(
+              'limit', 20,
+              'max_age_minutes', 1
+          )
+      ) as request_id;
+    $$
+);
+
+-- Add comment
+COMMENT ON EXTENSION pg_cron IS
+'Cron jobs:
+- webhook-dispatcher: Runs every minute to dispatch webhooks
+- process-large-payloads: Runs every 5 minutes to move large activity payloads to storage';

package/supabase/migrations/20251222220000_fix_large_payload_types.sql

@@ -0,0 +1,72 @@
+-- Fix type mismatch in large payload storage system
+-- activities.id is UUID but move_payload_to_storage expected BIGINT
+
+-- Drop the old function
+DROP FUNCTION IF EXISTS move_payload_to_storage(BIGINT, TEXT);
+
+-- Recreate with correct UUID type
+CREATE OR REPLACE FUNCTION move_payload_to_storage(
+  p_activity_id UUID,  -- Changed from BIGINT to UUID
+  p_storage_path TEXT
+)
+RETURNS JSONB
+LANGUAGE plpgsql
+SECURITY DEFINER
+AS $$
+DECLARE
+  original_payload JSONB;
+  payload_metadata JSONB;
+  result JSONB;
+BEGIN
+  -- Get the original payload
+  SELECT raw_data INTO original_payload
+  FROM activities
+  WHERE id = p_activity_id
+  FOR UPDATE;
+
+  IF original_payload IS NULL THEN
+    RETURN jsonb_build_object(
+      'success', false,
+      'error', 'Activity not found'
+    );
+  END IF;
+
+  -- Create metadata object to replace raw_data
+  payload_metadata := jsonb_build_object(
+    'payload_type', 'storage_ref',
+    'storage_path', p_storage_path,
+    'moved_at', NOW()
+  );
+
+  -- Update activity: replace raw_data with reference, update status
+  UPDATE activities
+  SET
+    raw_data = payload_metadata,
+    payload_storage_status = 'in_storage',
+    storage_path = p_storage_path
+  WHERE id = p_activity_id;
+
+  -- Return the original payload (to be uploaded by Edge Function)
+  result := jsonb_build_object(
+    'success', true,
+    'activity_id', p_activity_id,
+    'storage_path', p_storage_path,
+    'payload', original_payload
+  );
+
+  RETURN result;
+END;
+$$;
+
+-- Update permissions
+GRANT EXECUTE ON FUNCTION move_payload_to_storage(UUID, TEXT) TO service_role;
+
+-- Update comment
+COMMENT ON FUNCTION move_payload_to_storage(UUID, TEXT) IS
+'Moves a large activity payload to Supabase Storage.
+Called by the process-large-payloads Edge Function.
+
+Usage:
+  SELECT move_payload_to_storage(''25f160be-0e93-48c0-8a49-8eeda2e25762''::uuid, ''25f160be-0e93-48c0-8a49-8eeda2e25762/1234567890.json'');
+
+Returns the original payload for upload to storage.';