django-webhook-subscriber 1.0.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

django_webhook_subscriber/delivery.py

@@ -1,208 +1,463 @@
-"""Delivery functions for Django Webhook Subscriber."""
+"""Deliveries for Django Webhook Subscriber."""
 
-import requests
+import logging
+from collections import defaultdict
 
+from django.core.cache import cache
 from django.utils import timezone
-
-from django_webhook_subscriber.conf import rest_webhook_settings
-from django_webhook_subscriber.tasks import (
-    async_deliver_webhook,
-    process_webhook_batch,
+from django.utils.module_loading import import_string
+
+from .conf import rest_webhook_settings as settings
+from .serializers import serialize_webhook_instance
+from .tasks import process_webhook_delivery_batch
+from .utils import (
+    clear_content_type_cache,
+    get_content_type_id,
+    webhooks_disabled,
 )
 
+logger = logging.getLogger(__name__)
 
-def prepare_headers(webhook):
-    """Prepare headers for the webhook request."""
 
-    headers = webhook.headers.copy() if webhook.headers else {}
+class WebhookDeliveryProcessor:
+    def __init__(self):
+        self.cache_ttl = getattr(settings, "WEBHOOK_CACHE_TTL", 300)
 
-    # Add content type header if not present
-    if 'Content-Type' not in headers:
-        headers['Content-Type'] = 'application/json'
+    def send_webhook(self, instance, event_name, context=None, **kwargs):
+        """Send webhooks for a given model instance and event."""
 
-    # Add secret key authentication header
-    headers['X-Secret'] = webhook.secret
+        try:
+            # Check if webhooks are globally disabled
+            if webhooks_disabled():
+                logger.info(
+                    "Webhooks are disabled, skipping delivery.",
+                    extra={"instance": instance, "event_name": event_name},
+                )
+                return {"skipped": "Webhooks disabled"}
 
-    return headers
+            # Get active subscriptions for this model and event
+            subscriptions = self._get_subscriptions_cached(
+                instance, event_name
+            )
 
+            if not subscriptions:
+                logger.info(
+                    "No subscriptions found, skipping delivery.",
+                    extra={"instance": instance, "event_name": event_name},
+                )
+                return {"skipped": "No subscriptions"}
 
-def deliver_webhook(webhook, payload, event_signal):
-    """Deliver the webhook to the specified endpoint."""
+            # Optimize payload generation by grouping serializer
+            subscription_groups = self._group_subscriptions_by_serializer(
+                subscriptions
+            )
 
-    from django_webhook_subscriber.models import WebhookDeliveryLog
+            # Generate payloads for each serializer group
+            for serializer_class, subs in subscription_groups.items():
+                payload = self._generate_payload(
+                    instance, event_name, serializer_class
+                )
 
-    # Create delivery log
-    delivery_log = WebhookDeliveryLog.objects.create(
-        webhook=webhook,
-        event_signal=event_signal,
-        payload=payload,
-    )
+                # Assign payload to all subscriptions in this group
+                for sub in subs:
+                    sub["payload"] = payload
+
+            # Deliver the webhooks
+            result = self._deliver_webhooks(subscriptions)
+
+            logger.info(
+                f"Queued {len(subscriptions)} webhooks for {event_name}",
+                extra={"instance": instance, "event_name": event_name},
+            )
+
+            return result
 
-    try:
-        # Prepare request parameters
-        headers = prepare_headers(webhook)
-        timeout = getattr(rest_webhook_settings, 'REQUEST_TIMEOUT')
-
-        # Send the request
-        response = requests.post(
-            webhook.endpoint,
-            json=payload,
-            headers=headers,
-            timeout=timeout,
+        except Exception as e:
+            logger.error(
+                f"Error sending webhook: {e=}",
+                extra={"instance": instance, "event_name": event_name},
+                exc_info=True,
+            )
+            return {"error": f"Error sending webhook: {e=}"}
+
+    def _group_subscriptions_by_serializer(self, subscriptions):
+        """Group subscriptions by their serializer class."""
+
+        # Using defaultdict to avoid key errors
+        groups = defaultdict(list)
+        for sub in subscriptions:
+            serializer_key = sub.get("serializer_class") or None
+            groups[serializer_key].append(sub)
+
+        return groups
+
+    def _get_subscriptions_cached(self, instance, event_name):
+        """Retrieve active subscriptions with optimized caching."""
+
+        content_type_id = get_content_type_id(
+            instance._meta.app_label,
+            instance._meta.model_name,
         )
 
-        # Update delivery log with response
-        delivery_log.response_status = response.status_code
-        delivery_log.response_body = response.content
-        delivery_log.save()
+        if content_type_id is None:
+            logger.error(
+                f"Content type not found for {instance._meta.label}",
+                extra={"instance": instance},
+            )
+            return []
+
+        cache_key = f"webhook_subscriptions:{content_type_id}:{event_name}"
+
+        # Try to get from cache
+        subscriptions = cache.get(cache_key)
+
+        if subscriptions is None:
+            # Cache miss - fetch from database
+            subscriptions = self._fetch_subscriptions_from_db(
+                content_type_id, event_name
+            )
 
-        # Update webhook with response info
-        if webhook.keep_last_response:
-            webhook.last_response = response.content
+            # Cache the result
+            cache.set(cache_key, subscriptions, self.cache_ttl)
 
-        # Update success/failure timestamps
-        now = timezone.now()
-        if 200 <= response.status_code < 300:
-            # if response.ok:
-            webhook.last_success = now
+            logger.debug(
+                f"Cached {len(subscriptions)} subscriptions for {cache_key} "
+                "key"
+            )
         else:
-            webhook.last_failure = now
+            logger.debug(
+                f"Cache hit for {cache_key}: {len(subscriptions)} "
+                "subscriptions"
+            )
+
+        return subscriptions
+
+    def _fetch_subscriptions_from_db(self, content_type_id, event_name):
+        """Fetch subscriptions from database with optimizations."""
+
+        from .models import WebhookSubscription
+
+        qs = (
+            WebhookSubscription.objects.select_related("subscriber")
+            .filter(
+                is_active=True,
+                event_name=event_name,
+                subscriber__is_active=True,
+                subscriber__content_type_id=content_type_id,
+            )
+            .only(
+                # Only load fields we need
+                "id",
+                "custom_endpoint",
+                "subscriber_id",
+                "subscriber__target_url",
+                "subscriber__serializer_class",
+            )
+        )
 
-        webhook.save()
+        # Converting to lightweight dict format for caching
+        subscriptions = []
+        for sub in qs:
+            subscription_data = {
+                "id": sub.id,
+                "subscriber_id": sub.subscriber_id,
+                "url": sub.endpoint,  # This property handles URL logic
+                "serializer_class": sub.subscriber.serializer_class,
+            }
+            subscriptions.append(subscription_data)
 
-    except Exception as e:
-        # Handle errors
-        delivery_log.error_message = str(e)
-        delivery_log.save()
+        return subscriptions
 
-        # Update webhook failure timestamp
-        webhook.last_failure = timezone.now()
-        webhook.save()
+    def _generate_payload(self, instance, event_name, serializer_class_path):
+        """Generate webhook payload for a specific serializer."""
 
-    return delivery_log
+        try:
+            serializer_class = None
+
+            # Import serializer class if specified
+            if serializer_class_path:
+                serializer_class = import_string(serializer_class_path)
+
+            # Serialize the instance
+            fields_data = serialize_webhook_instance(
+                instance, serializer_class
+            )
+
+            # Build standard payload structure
+            payload = {
+                "pk": instance.pk,
+                "event_signal": event_name,
+                "source": f"{instance._meta.app_label}."
+                f"{instance._meta.model_name}",
+                "timestamp": timezone.now().isoformat(),
+                "fields": fields_data,
+            }
+
+            return payload
+
+        except Exception as e:
+            logger.error(
+                f"Error generating payload: {e=}",
+                extra={"instance": instance, "event_name": event_name},
+                exc_info=True,
+            )
+
+            return {
+                "pk": getattr(instance, "pk", None),
+                "event_signal": event_name,
+                "source": f"{instance._meta.app_label}."
+                f"{instance._meta.model_name}",
+                "timestamp": timezone.now().isoformat(),
+                "error": f"Serialization failed: {e=}",
+                "fields": {},
+            }
+
+    def _deliver_webhooks(self, subscriptions):
+        """Deliver webhooks with batch processing and size limits."""
+
+        if not subscriptions:
+            return {"processed": 0}
+
+        # Splitting large batches to avoid overwhelming Celery
+        max_batch_size = getattr(settings, "MAX_BATCH_SIZE")
+
+        if len(subscriptions) <= max_batch_size:
+            # Single batch
+            return self._process_single_batch(subscriptions)
 
+        else:
+            # Multiple batches
+            return self._process_multiple_batches(
+                subscriptions, max_batch_size
+            )
 
-def get_webhook_for_model(model_instance):
-    """Get all active webhooks for a model instance."""
+    def _process_single_batch(self, subscriptions):
+        """Process a single batch of webhook deliveries."""
 
-    from django.contrib.contenttypes.models import ContentType
+        try:
+            result = process_webhook_delivery_batch.delay(subscriptions)
+            return {
+                "processed": len(subscriptions),
+                "batches": 1,
+                "task_id": result.id,
+            }
+
+        except Exception as e:
+            logger.error(f"Failed to queue webhook batch: {e=}", exc_info=True)
+            return {"error": f"{e=}", "processed": 0}
+
+    def _process_multiple_batches(self, subscriptions, batch_size):
+        """Process multiple batches of webhook deliveries."""
+
+        batches = []
+        task_ids = []
+        total_processed = 0
+
+        # Splitting into chunks
+        for i in range(0, len(subscriptions), batch_size):
+            batch = subscriptions[i:i + batch_size]  # fmt: skip
+
+            try:
+                result = process_webhook_delivery_batch.delay(batch)
+                batches.append({"size": len(batch), "task_id": result.id})
+                task_ids.append(result.id)
+                total_processed += len(batch)
+
+            except Exception as e:
+                logger.error(
+                    f"Failed to queue webhook batch {i//batch_size + 1}: {e=}"
+                )
+                batches.append({"size": len(batch), "error": f"{e=}"})
+        logger.info(
+            f"Queued {len(batches)} batches for {total_processed} webhook "
+            "deliveries"
+        )
 
-    model_class = model_instance.__class__
-    content_type = ContentType.objects.get_for_model(model_class)
+        return {
+            "processed": total_processed,
+            "batches": len(batches),
+            "batch_details": batches,
+            "task_ids": task_ids,
+        }
 
-    from django_webhook_subscriber.models import WebhookRegistry
+    def clear_webhook_cache(self, content_type=None, event_name=None):
+        """Clear cached webhook subscription data."""
 
-    # Get al active webhooks for this content type
-    return WebhookRegistry.objects.filter(
-        content_type=content_type,
-        is_active=True,
-    )
+        from .models import WebhookSubscription
 
+        if content_type and event_name:
+            # Clear specific cache key
+            cache_key = f"webhook_subscriptions:{content_type.id}:{event_name}"
+            cache.delete(cache_key)
+            logger.debug(f"Cleared cache for {cache_key}")
 
-def get_async_setting(webhook, system_default):
-    """Determine if the webhook should use async delivery."""
+        elif content_type:
+            # Clear all cache keys for this content type
+            event_names = (
+                WebhookSubscription.objects.filter(
+                    subscriber__content_type=content_type,
+                )
+                .values_list("event_name", flat=True)
+                .distinct()
+            )
+
+            cleared_count = 0
+            for event_name in event_names:
+                cache_key = (
+                    f"webhook_subscriptions:{content_type.id}:{event_name}"
+                )
+                cache.delete(cache_key)
+                cleared_count += 1
+            logger.debug(
+                f"Cleared {cleared_count} cache keys for content type "
+                f"{content_type}"
+            )
 
-    # If webhook has specific setting, use it
-    if webhook.use_async is not None:
-        return webhook.use_async
+        else:
+            # Clear all webhook caches (use pattern-based deletion if
+            # available)
+            if hasattr(cache, "delete_pattern"):
+                # Redis backend supports pattern deletion
+                deleted = cache.delete_pattern("webhook_subscriptions:*")
+                logger.debug(
+                    f"Cleared {deleted} webhook cache keys using pattern"
+                )
+            else:
+                # Fallback: clear entire cache
+                cache.clear()
+                logger.warning(
+                    "Cleared entire cache (pattern deletion not supported)"
+                )
 
-    # Otherwise, use system default
-    return system_default
+    def get_cache_stats(self):
+        """Get webhook cache statistics."""
 
+        # Get all possible cache keys
+        cache_keys = self._get_all_cache_keys()
 
-def process_and_deliver_webhook(
-    instance,
-    event_signal,
-    serialized_payload=None,
-    async_delivery=False,
-    serializer=None,
-):
-    """Process and deliver the webhook for a given instance and event type.
+        stats = {
+            "total_possible_keys": len(cache_keys),
+            "cached_keys": 0,
+            "total_cached_subscriptions": 0,
+            "cache_hit_ratio": 0.0,
+            "key_details": [],
+        }
 
-    This function checks the event type and the model class of the instance in
-    the registry. If the event type is registered, it serializes the instance
-    using the specified serializer (if any) and calls the delivery function to
-    send the webhook.
-    If the webhook is configured for async delivery, it will be processed
-    asynchronously using Celery tasks. If the webhook is configured for
-    synchronous delivery, it will be processed immediately.
-    """
+        for key in cache_keys:
+            cached_data = cache.get(key)
 
-    # Determine system default for async delivery
-    system_default = getattr(rest_webhook_settings, 'DEFAULT_USE_ASYNC')
+            key_info = {
+                "key": key,
+                "is_cached": cached_data is not None,
+                "subscription_count": len(cached_data) if cached_data else 0,
+            }
 
-    # If specific async_delivery is specified, override the system default
-    if async_delivery is not None:
-        system_default = async_delivery
+            if cached_data is not None:
+                stats["cached_keys"] += 1
+                stats["total_cached_subscriptions"] += len(cached_data)
 
-    # If no payload is provided, serialize the instance
-    if serialized_payload is None:
-        from django_webhook_subscriber.serializers import serialize_instance
+            stats["key_details"].append(key_info)
 
-        serialized_payload = serialize_instance(
-            instance,
-            event_signal,
-            field_serializer=serializer,
-        )
+        # Calculate hit ratio
+        if stats["total_possible_keys"] > 0:
+            stats["cache_hit_ratio"] = (
+                stats["cached_keys"] / stats["total_possible_keys"]
+            ) * 100
 
-    # Get all active webhooks for this model
-    webhooks = get_webhook_for_model(instance)
-    delivery_logs = []
+        return stats
 
-    # Filter webhooks for this model
-    sync_webhooks = []
-    async_webhooks = []
+    def _get_all_cache_keys(self):
+        """Get all possible webhook cache keys."""
 
-    for webhook in webhooks:
-        # Map event type to registry format
-        event_mapping = {
-            'created': 'CREATE',
-            'updated': 'UPDATE',
-            'deleted': 'DELETE',
-        }
-        registry_event = event_mapping.get(event_signal, event_signal)
+        # Query all unique content_type + event_name combinations
+        from .models import WebhookSubscription
 
-        # Skip if this webhook isn't configured for this event type
-        if registry_event not in webhook.event_signals:
-            continue
+        combinations = WebhookSubscription.objects.values_list(
+            "subscriber__content_type_id", "event_name"
+        ).distinct()
 
-        # Check if this webhook should use async delivery
-        if get_async_setting(webhook, system_default):
-            async_webhooks.append(webhook)
-        else:
-            sync_webhooks.append(webhook)
+        cache_keys = []
+        for content_type_id, event_name in combinations:
+            cache_key = f"webhook_subscriptions:{content_type_id}:{event_name}"
+            cache_keys.append(cache_key)
 
-    # Process async webhooks
-    tasks_ids = []
-    if async_webhooks:
-        try:
-            # If we only have one webhook, we'll process it directly
-            if len(async_webhooks) == 1:
-                webhook = async_webhooks[0]
-                task_id = async_deliver_webhook.delay(
-                    webhook.id,
-                    serialized_payload,
-                    event_signal,
-                )
-                tasks_ids.append(task_id.id)
-            else:
-                # Otherwise, we'll process them in a batch
-                webhook_ids = [w.id for w in async_webhooks]
-                batch_task = process_webhook_batch.delay(
-                    webhook_ids,
-                    serialized_payload,
-                    event_signal,
-                )
-                tasks_ids.append(batch_task.id)
-        except ImportError:
-            # Celery is not installed, fall back to synchronous delivery
-            sync_webhooks.extend(async_webhooks)
-
-    # Process sync webhooks
-    for webhook in sync_webhooks:
-        delivery_log = deliver_webhook(
-            webhook, serialized_payload, event_signal
+        return cache_keys
+
+
+# Global instance of the delivery processor
+webhook_delivery_processor = WebhookDeliveryProcessor()
+
+
+def send_webhooks(instance, event_name, context=None, **kwargs):
+    """Send webhooks for a given model instance and event."""
+
+    return webhook_delivery_processor.send_webhook(
+        instance,
+        event_name,
+        context=context,
+        **kwargs,
+    )
+
+
+def clear_webhook_cache(content_type=None, event_name=None):
+    """Clear all cached data from the delivery process."""
+
+    webhook_delivery_processor.clear_webhook_cache(
+        content_type=content_type, event_name=event_name
+    )
+    # Clear cached data from get_content_type_id function
+    clear_content_type_cache()
+
+
+def get_webhook_cache_stats():
+    """Get statistics about the webhook subscription cache."""
+
+    return webhook_delivery_processor.get_cache_stats()
+
+
+def warm_webhook_cache():
+    """Pre-fill the webhook cache by loading all active subscription
+    combinations."""
+
+    processor = webhook_delivery_processor
+    warmed_count = 0
+
+    from django.contrib.contenttypes.models import ContentType
+
+    from .models import WebhookSubscription
+
+    # Get all unique combinations of active subscriptions
+    combinations = (
+        WebhookSubscription.objects.filter(
+            is_active=True,
+            subscriber__is_active=True,
         )
-        delivery_logs.append(delivery_log)
+        .values_list("subscriber__content_type_id", "event_name")
+        .distinct()
+    )
+
+    for content_type_id, event_name in combinations:
+        try:
+
+            content_type = ContentType.objects.get(id=content_type_id)
+            model_class = content_type.model_class()
 
-    return delivery_logs + tasks_ids
+            if model_class:
+                # Create a minimal dummy instance for cache warming
+                dummy_instance = model_class()
+                dummy_instance._meta = model_class._meta
+
+                # This will populate the cache
+                processor._get_subscriptions_cached(dummy_instance, event_name)
+                warmed_count += 1
+
+        except Exception as e:
+            logger.warning(
+                f"Could not warm cache for {content_type_id}:{event_name}: "
+                f"{e=}"
+            )
+
+    logger.info(
+        f"Warmed webhook cache for {warmed_count} subscription combinations"
+    )
+    return {"warmed": warmed_count}

django_webhook_subscriber/http.py

@@ -0,0 +1,51 @@
+"""HTTP session management for webhook delivery."""
+
+from contextlib import contextmanager
+
+import requests
+from requests.adapters import HTTPAdapter
+
+
+def create_webhook_session():
+    """Create an optimized session for webhook delivery."""
+
+    session = requests.Session()
+
+    # Configure adapter for connection pooling
+    adapter = HTTPAdapter(
+        pool_connections=2,
+        pool_maxsize=5,  # Max connections per host
+        max_retries=0,  # We handle the retries
+    )
+
+    # Mount adapter for both HTTP and HTTPS
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+
+    # Set default handlers
+    session.headers.update(
+        {
+            "User-Agent": "Django-Webhook-Subscriber/2.0",
+            "Content-Type": "application/json",
+        }
+    )
+
+    return session
+
+
+@contextmanager
+def webhook_session():
+    """Context manager for webhook session.
+
+    Usage:
+        with webhook_session() as session:
+            # Use session to send requests like
+            response = session.post(url, json=payload)
+            ...
+    """
+
+    session = create_webhook_session()
+    try:
+        yield session
+    finally:
+        session.close()