realtimex-crm 0.8.1 → 0.9.2

package/supabase/functions/ingest-activity/index.ts

@@ -0,0 +1,261 @@
+import "jsr:@supabase/functions-js/edge-runtime.d.ts";
+import { supabaseAdmin } from "../_shared/supabaseAdmin.ts";
+import { createErrorResponse, corsHeaders } from "../_shared/utils.ts";
+import { validateIngestionRequest, validateTwilioWebhook } from "../_shared/ingestionGuard.ts";
+
+/**
+ * Sanitize filename to remove special characters that break storage paths
+ * Preserves extension and basic readability while keeping original in metadata
+ */
+function sanitizeFilename(filename: string): string {
+  // Get file extension
+  const lastDotIndex = filename.lastIndexOf('.');
+  const name = lastDotIndex > 0 ? filename.substring(0, lastDotIndex) : filename;
+  const ext = lastDotIndex > 0 ? filename.substring(lastDotIndex) : '';
+
+  // Replace special characters with underscores, keep alphanumeric and basic punctuation
+  const safeName = name
+    .replace(/[^a-zA-Z0-9._-]/g, '_')  // Replace unsafe chars with underscore
+    .replace(/_+/g, '_')                // Collapse multiple underscores
+    .replace(/^_|_$/g, '');             // Remove leading/trailing underscores
+
+  // Limit length to avoid path issues (200 chars for name + extension)
+  const maxLength = 200 - ext.length;
+  const truncatedName = safeName.length > maxLength
+    ? safeName.substring(0, maxLength)
+    : safeName;
+
+  return truncatedName + ext;
+}
+
+Deno.serve(async (req) => {
+  // Handle CORS
+  if (req.method === "OPTIONS") {
+    return new Response(null, { status: 204, headers: corsHeaders });
+  }
+
+  try {
+    // 1. Validate & Identify Provider
+    const validation = await validateIngestionRequest(req);
+    if (validation.error) return validation.error;
+
+    const { provider } = validation;
+
+    // 2. Parse Body based on Content-Type
+    const contentType = req.headers.get("content-type") || "";
+    let rawBody: any;
+    const uploadedFiles: Array<{ fieldName: string; storagePath: string; size: number; type: string }> = [];
+
+    if (contentType.includes("application/json")) {
+      rawBody = await req.json();
+    } else if (contentType.includes("application/x-www-form-urlencoded") || contentType.includes("multipart/form-data")) {
+      const formData = await req.formData();
+      rawBody = {};
+
+      // Process FormData entries (files + fields)
+      for (const [key, value] of formData.entries()) {
+        if (value instanceof File) {
+          // Handle file upload: upload to storage immediately
+          const file = value as File;
+          const timestamp = Date.now();
+
+          // Sanitize filename to remove special characters that break storage paths
+          // Keep original filename in metadata, use safe version for storage path
+          const sanitizedName = sanitizeFilename(file.name);
+          const storagePath = `incoming/${timestamp}_${sanitizedName}`;
+
+          console.log(`Uploading file: ${file.name} → ${sanitizedName} (${file.size} bytes, ${file.type})`);
+
+          const { error: uploadError } = await supabaseAdmin.storage
+            .from('activity-payloads')
+            .upload(storagePath, file, {
+              contentType: file.type || 'application/octet-stream',
+              upsert: false,
+            });
+
+          if (uploadError) {
+            console.error(`Failed to upload file ${file.name}:`, uploadError);
+            return createErrorResponse(500, `File upload failed: ${uploadError.message}`);
+          }
+
+          console.log(`File uploaded successfully: ${storagePath}`);
+
+          // Store file reference instead of file content
+          uploadedFiles.push({
+            fieldName: key,
+            storagePath,
+            size: file.size,
+            type: file.type || 'application/octet-stream',
+          });
+
+          // Add storage reference to rawBody
+          rawBody[key] = {
+            _type: 'file_ref',
+            storage_path: storagePath,
+            filename: file.name,
+            size: file.size,
+            mime_type: file.type,
+          };
+        } else {
+          // Regular form field
+          rawBody[key] = value;
+        }
+      }
+    } else {
+      rawBody = await req.text();
+    }
+
+    // 2.5. Validate Twilio Signature (after body is parsed)
+    if (provider.provider_code === "twilio" && provider.config?.auth_token) {
+      const isValid = await validateTwilioWebhook(req, provider.config.auth_token, rawBody);
+      if (!isValid) {
+        console.error("Invalid Twilio signature");
+        return createErrorResponse(401, "Invalid Twilio Signature");
+      }
+    }
+
+    // 3. Normalize Data (The "Normalizer" Pattern)
+    const normalized = normalizeActivity(provider.provider_code, rawBody);
+
+    // 4. Add file upload metadata if files were uploaded
+    const activityMetadata = {
+      ...normalized.metadata,
+      provider_code: provider.provider_code,
+    };
+
+    if (uploadedFiles.length > 0) {
+      activityMetadata.uploaded_files = uploadedFiles;
+      activityMetadata.has_attachments = true;
+    }
+
+    // 5. Insert into 'activities' table
+    const { data, error } = await supabaseAdmin
+      .from("activities")
+      .insert({
+        type: normalized.type,
+        direction: "inbound",
+        processing_status: "raw", // Ready for Local Agent to steal
+        raw_data: normalized.raw_data,
+        metadata: activityMetadata,
+        provider_id: provider.id,
+        sales_id: provider.sales_id, // Auto-assign if provider has an owner
+        payload_storage_status: uploadedFiles.length > 0 ? 'in_storage' : undefined, // Files uploaded directly to storage
+      })
+      .select()
+      .single();
+
+    if (error) {
+      console.error("DB Insert Error:", error);
+      console.error("Error details:", JSON.stringify(error, null, 2));
+      return createErrorResponse(500, `Failed to persist activity: ${error.message || JSON.stringify(error)}`);
+    }
+
+    return new Response(JSON.stringify({ success: true, id: data.id }), {
+      status: 202, // Accepted
+      headers: { ...corsHeaders, "Content-Type": "application/json" },
+    });
+
+  } catch (err) {
+    console.error("Ingestion Error:", err);
+    return createErrorResponse(500, "Internal Server Error");
+  }
+});
+
+/**
+ * Normalizes provider-specific payloads into the standard RealTimeX schema.
+ */
+function normalizeActivity(providerCode: string, payload: any) {
+  let type = "other";
+  let raw_data = {};
+  let metadata = {};
+
+  if (providerCode === "postmark") {
+    type = "email";
+    raw_data = {
+      source_type: "text",
+      content: payload.TextBody || payload.HtmlBody || "",
+      subject: payload.Subject,
+      sender: payload.From,
+    };
+    metadata = {
+      message_id: payload.MessageID,
+      to: payload.To,
+    };
+  } else if (providerCode === "twilio") {
+    // Detect SMS vs Voice
+    if (payload.RecordingUrl) {
+      type = "call";
+      raw_data = {
+        source_type: "url",
+        content: payload.RecordingUrl, // The Local Agent will download this
+        format: "audio/wav",
+      };
+      metadata = {
+        call_sid: payload.CallSid,
+        from: payload.From,
+        duration: payload.CallDuration,
+      };
+    } else {
+      type = "sms";
+      raw_data = {
+        source_type: "text",
+        content: payload.Body,
+      };
+      metadata = {
+        message_sid: payload.MessageSid,
+        from: payload.From,
+      };
+    }
+  } else {
+    // Generic / Manual / Multipart
+    type = payload.type || "note";
+
+    // Check if payload contains file references from multipart upload
+    const fileFields = Object.entries(payload).filter(([_, value]) =>
+      typeof value === 'object' && value !== null && value._type === 'file_ref'
+    );
+
+    if (fileFields.length > 0) {
+      // Handle uploaded files
+      if (fileFields.length === 1) {
+        // Single file - store in raw_data
+        const [_fieldName, fileRef] = fileFields[0];
+        raw_data = {
+          source_type: "storage_ref",
+          storage_path: (fileRef as any).storage_path,
+          filename: (fileRef as any).filename,
+          format: (fileRef as any).mime_type,
+          size: (fileRef as any).size,
+        };
+      } else {
+        // Multiple files - store array in raw_data
+        raw_data = {
+          source_type: "storage_refs",
+          files: fileFields.map(([fieldName, fileRef]) => ({
+            storage_path: (fileRef as any).storage_path,
+            filename: (fileRef as any).filename,
+            format: (fileRef as any).mime_type,
+            size: (fileRef as any).size,
+            field_name: fieldName,
+          })),
+        };
+      }
+
+      // Include other form fields in metadata
+      metadata = {
+        ...payload.metadata,
+        form_data: Object.fromEntries(
+          Object.entries(payload).filter(([key]) =>
+            !key.startsWith('_') && typeof payload[key] !== 'object'
+          )
+        ),
+      };
+    } else {
+      // Regular JSON payload
+      raw_data = payload.raw_data || { source_type: "text", content: JSON.stringify(payload) };
+      metadata = payload.metadata || {};
+    }
+  }
+
+  return { type, raw_data, metadata };
+}

package/supabase/functions/webhook-dispatcher/index.ts

@@ -0,0 +1,133 @@
+import "jsr:@supabase/functions-js/edge-runtime.d.ts";
+import { supabaseAdmin } from "../_shared/supabaseAdmin.ts";
+import { generateWebhookSignature } from "../_shared/webhookSignature.ts";
+
+Deno.serve(async () => {
+  console.log("Webhook dispatcher running...");
+
+  // Get pending webhook deliveries
+  const { data: queueItems } = await supabaseAdmin
+    .from("webhook_queue")
+    .select("*, webhooks(*)")
+    .eq("status", "pending")
+    .lte("next_retry_at", new Date().toISOString())
+    .limit(50);
+
+  if (!queueItems || queueItems.length === 0) {
+    return new Response(JSON.stringify({ processed: 0 }), {
+      headers: { "Content-Type": "application/json" },
+    });
+  }
+
+  console.log(`Processing ${queueItems.length} webhook deliveries`);
+
+  const results = await Promise.allSettled(
+    queueItems.map((item) => deliverWebhook(item))
+  );
+
+  const successful = results.filter((r) => r.status === "fulfilled").length;
+  const failed = results.filter((r) => r.status === "rejected").length;
+
+  return new Response(
+    JSON.stringify({ processed: queueItems.length, successful, failed }),
+    { headers: { "Content-Type": "application/json" } }
+  );
+});
+
+async function deliverWebhook(queueItem: any) {
+  const webhook = queueItem.webhooks;
+
+  // Mark as processing
+  await supabaseAdmin
+    .from("webhook_queue")
+    .update({ status: "processing" })
+    .eq("id", queueItem.id);
+
+  try {
+    const payloadString = JSON.stringify(queueItem.payload);
+    const signature = await generateWebhookSignature(
+      webhook.secret,
+      payloadString
+    );
+
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+      "X-Webhook-Signature": signature,
+      "X-Webhook-Event": queueItem.event_type,
+      "User-Agent": "AtomicCRM-Webhooks/1.0",
+    };
+
+    // Add custom headers from webhook config
+    if (webhook.headers) {
+      Object.assign(headers, webhook.headers);
+    }
+
+    const response = await fetch(webhook.url, {
+      method: "POST",
+      headers,
+      body: payloadString,
+      signal: AbortSignal.timeout(30000), // 30 second timeout
+    });
+
+    if (response.ok) {
+      // Success
+      await supabaseAdmin
+        .from("webhook_queue")
+        .update({
+          status: "delivered",
+          delivered_at: new Date().toISOString(),
+        })
+        .eq("id", queueItem.id);
+
+      await supabaseAdmin
+        .from("webhooks")
+        .update({
+          last_triggered_at: new Date().toISOString(),
+          failure_count: 0,
+        })
+        .eq("id", webhook.id);
+
+      console.log(`Webhook ${queueItem.id} delivered successfully`);
+    } else {
+      throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+    }
+  } catch (error) {
+    console.error(`Webhook ${queueItem.id} delivery failed:`, error);
+
+    const newAttempts = queueItem.attempts + 1;
+    const shouldRetry = newAttempts < queueItem.max_attempts;
+
+    if (shouldRetry) {
+      // Exponential backoff: 1min, 5min, 15min
+      const retryDelays = [60, 300, 900];
+      const delaySeconds = retryDelays[newAttempts - 1] || 900;
+      const nextRetry = new Date(Date.now() + delaySeconds * 1000)
+        .toISOString();
+
+      await supabaseAdmin
+        .from("webhook_queue")
+        .update({
+          status: "pending",
+          attempts: newAttempts,
+          next_retry_at: nextRetry,
+          error_message: error.message,
+        })
+        .eq("id", queueItem.id);
+    } else {
+      // Max attempts reached
+      await supabaseAdmin
+        .from("webhook_queue")
+        .update({
+          status: "failed",
+          attempts: newAttempts,
+          error_message: error.message,
+        })
+        .eq("id", queueItem.id);
+
+      await supabaseAdmin
+        .from("webhooks")
+        .update({ failure_count: webhook.failure_count + 1 })
+        .eq("id", webhook.id);
+    }
+  }
+}

package/supabase/migrations/20251219120000_api_integrations.sql

@@ -0,0 +1,133 @@
+-- API Keys table
+create table "public"."api_keys" (
+    "id" bigint generated by default as identity not null,
+    "created_at" timestamp with time zone not null default now(),
+    "sales_id" bigint not null,
+    "name" text not null,
+    "key_hash" text not null,
+    "key_prefix" text not null,
+    "scopes" text[] not null default '{}',
+    "is_active" boolean not null default true,
+    "expires_at" timestamp with time zone,
+    "last_used_at" timestamp with time zone,
+    "created_by_sales_id" bigint not null
+);
+
+alter table "public"."api_keys" enable row level security;
+
+CREATE UNIQUE INDEX api_keys_pkey ON public.api_keys USING btree (id);
+CREATE UNIQUE INDEX api_keys_key_hash_unique ON public.api_keys USING btree (key_hash);
+CREATE INDEX api_keys_sales_id_idx ON public.api_keys USING btree (sales_id);
+
+alter table "public"."api_keys" add constraint "api_keys_pkey" PRIMARY KEY using index "api_keys_pkey";
+alter table "public"."api_keys" add constraint "api_keys_sales_id_fkey"
+    FOREIGN KEY (sales_id) REFERENCES sales(id) ON UPDATE CASCADE ON DELETE CASCADE not valid;
+alter table "public"."api_keys" validate constraint "api_keys_sales_id_fkey";
+alter table "public"."api_keys" add constraint "api_keys_created_by_sales_id_fkey"
+    FOREIGN KEY (created_by_sales_id) REFERENCES sales(id) ON UPDATE CASCADE ON DELETE CASCADE not valid;
+alter table "public"."api_keys" validate constraint "api_keys_created_by_sales_id_fkey";
+
+-- Webhooks table
+create table "public"."webhooks" (
+    "id" bigint generated by default as identity not null,
+    "created_at" timestamp with time zone not null default now(),
+    "sales_id" bigint not null,
+    "name" text not null,
+    "url" text not null,
+    "events" text[] not null default '{}',
+    "headers" jsonb,
+    "is_active" boolean not null default true,
+    "secret" text not null,
+    "last_triggered_at" timestamp with time zone,
+    "failure_count" int not null default 0,
+    "created_by_sales_id" bigint not null
+);
+
+alter table "public"."webhooks" enable row level security;
+
+CREATE UNIQUE INDEX webhooks_pkey ON public.webhooks USING btree (id);
+CREATE INDEX webhooks_sales_id_idx ON public.webhooks USING btree (sales_id);
+CREATE INDEX webhooks_events_idx ON public.webhooks USING gin (events);
+
+alter table "public"."webhooks" add constraint "webhooks_pkey" PRIMARY KEY using index "webhooks_pkey";
+alter table "public"."webhooks" add constraint "webhooks_sales_id_fkey"
+    FOREIGN KEY (sales_id) REFERENCES sales(id) ON UPDATE CASCADE ON DELETE CASCADE not valid;
+alter table "public"."webhooks" validate constraint "webhooks_sales_id_fkey";
+alter table "public"."webhooks" add constraint "webhooks_created_by_sales_id_fkey"
+    FOREIGN KEY (created_by_sales_id) REFERENCES sales(id) ON UPDATE CASCADE ON DELETE CASCADE not valid;
+alter table "public"."webhooks" validate constraint "webhooks_created_by_sales_id_fkey";
+
+-- API Request Logs table
+create table "public"."api_logs" (
+    "id" bigint generated by default as identity not null,
+    "created_at" timestamp with time zone not null default now(),
+    "api_key_id" bigint not null,
+    "endpoint" text not null,
+    "method" text not null,
+    "status_code" int not null,
+    "response_time_ms" int,
+    "ip_address" text,
+    "user_agent" text,
+    "error_message" text
+);
+
+alter table "public"."api_logs" enable row level security;
+
+CREATE UNIQUE INDEX api_logs_pkey ON public.api_logs USING btree (id);
+CREATE INDEX api_logs_api_key_id_idx ON public.api_logs USING btree (api_key_id);
+CREATE INDEX api_logs_created_at_idx ON public.api_logs USING btree (created_at DESC);
+
+alter table "public"."api_logs" add constraint "api_logs_pkey" PRIMARY KEY using index "api_logs_pkey";
+alter table "public"."api_logs" add constraint "api_logs_api_key_id_fkey"
+    FOREIGN KEY (api_key_id) REFERENCES api_keys(id) ON UPDATE CASCADE ON DELETE CASCADE not valid;
+alter table "public"."api_logs" validate constraint "api_logs_api_key_id_fkey";
+
+-- Webhook Queue table (for async delivery)
+create table "public"."webhook_queue" (
+    "id" bigint generated by default as identity not null,
+    "created_at" timestamp with time zone not null default now(),
+    "webhook_id" bigint not null,
+    "event_type" text not null,
+    "payload" jsonb not null,
+    "status" text not null default 'pending',
+    "attempts" int not null default 0,
+    "max_attempts" int not null default 3,
+    "next_retry_at" timestamp with time zone,
+    "delivered_at" timestamp with time zone,
+    "error_message" text
+);
+
+alter table "public"."webhook_queue" enable row level security;
+
+CREATE UNIQUE INDEX webhook_queue_pkey ON public.webhook_queue USING btree (id);
+CREATE INDEX webhook_queue_status_idx ON public.webhook_queue USING btree (status, next_retry_at);
+CREATE INDEX webhook_queue_webhook_id_idx ON public.webhook_queue USING btree (webhook_id);
+
+alter table "public"."webhook_queue" add constraint "webhook_queue_pkey" PRIMARY KEY using index "webhook_queue_pkey";
+alter table "public"."webhook_queue" add constraint "webhook_queue_webhook_id_fkey"
+    FOREIGN KEY (webhook_id) REFERENCES webhooks(id) ON UPDATE CASCADE ON DELETE CASCADE not valid;
+alter table "public"."webhook_queue" validate constraint "webhook_queue_webhook_id_fkey";
+
+-- Grant permissions
+grant select, insert, update, delete on table "public"."api_keys" to "authenticated";
+grant select, insert, update, delete on table "public"."webhooks" to "authenticated";
+grant select on table "public"."api_logs" to "authenticated";
+grant select on table "public"."webhook_queue" to "authenticated";
+
+grant all on table "public"."api_keys" to "service_role";
+grant all on table "public"."webhooks" to "service_role";
+grant all on table "public"."api_logs" to "service_role";
+grant all on table "public"."webhook_queue" to "service_role";
+
+-- RLS Policies (permissive for authenticated users)
+create policy "Enable all for authenticated users" on "public"."api_keys"
+    for all to authenticated using (true) with check (true);
+
+create policy "Enable all for authenticated users" on "public"."webhooks"
+    for all to authenticated using (true) with check (true);
+
+create policy "Enable read for authenticated users" on "public"."api_logs"
+    for select to authenticated using (true);
+
+create policy "Enable read for authenticated users" on "public"."webhook_queue"
+    for select to authenticated using (true);