devs-webhook 0.1.0__py3-none-any.whl

devs_webhook/sources/sqs_source.py

@@ -0,0 +1,306 @@
+"""SQS task source for receiving webhook events from AWS SQS.
+
+This module provides the SQS task source that polls an AWS SQS queue for
+webhook events and forwards them to the task processor.
+"""
+
+import asyncio
+import json
+import uuid
+from typing import Optional, Dict, Any
+import structlog
+
+from .base import TaskSource
+from ..core.task_processor import TaskProcessor
+from ..config import get_config
+from ..utils.github import verify_github_signature
+
+logger = structlog.get_logger()
+
+
+class SQSTaskSource(TaskSource):
+    """Task source that polls AWS SQS for GitHub webhook events.
+
+    This allows decoupling the webhook receiver (which can be a separate
+    service) from the task processor. The webhook receiver puts messages
+    into SQS, and this source polls them for processing.
+
+    Expected SQS message format:
+    {
+        "headers": {
+            "x-github-event": "issues",
+            "x-github-delivery": "...",
+            ...
+        },
+        "payload": "<base64-encoded-or-raw-json>"
+    }
+    """
+
+    def __init__(self, task_processor: Optional[TaskProcessor] = None):
+        """Initialize SQS task source.
+
+        Args:
+            task_processor: Optional task processor instance. If not provided,
+                          a new one will be created.
+        """
+        self.task_processor = task_processor or TaskProcessor()
+        self.config = get_config()
+        self._running = False
+        self._poll_task: Optional[asyncio.Task] = None
+
+        # Import boto3 lazily to avoid requiring it for webhook-only deployments
+        try:
+            import boto3
+            self.sqs_client = boto3.client(
+                'sqs',
+                region_name=self.config.aws_region
+            )
+        except ImportError:
+            logger.error("boto3 not installed - required for SQS task source")
+            raise ImportError(
+                "boto3 is required for SQS task source. "
+                "Install with: pip install boto3"
+            )
+
+        logger.info(
+            "SQS task source initialized",
+            queue_url=self.config.aws_sqs_queue_url,
+            region=self.config.aws_region,
+        )
+
+    async def start(self) -> None:
+        """Start polling SQS for webhook events.
+
+        This method blocks until the source is stopped.
+        """
+        logger.info("Starting SQS task source")
+        self._running = True
+
+        try:
+            while self._running:
+                await self._poll_and_process_messages()
+        except asyncio.CancelledError:
+            logger.info("SQS polling cancelled")
+            raise
+        except Exception as e:
+            logger.error("SQS polling error", error=str(e), exc_info=True)
+            raise
+        finally:
+            self._running = False
+
+    async def stop(self) -> None:
+        """Stop polling SQS."""
+        logger.info("Stopping SQS task source")
+        self._running = False
+
+        if self._poll_task and not self._poll_task.done():
+            self._poll_task.cancel()
+            try:
+                await self._poll_task
+            except asyncio.CancelledError:
+                pass
+
+        logger.info("SQS task source stopped")
+
+    async def _poll_and_process_messages(self) -> None:
+        """Poll SQS and process any available messages.
+
+        This uses long polling to efficiently wait for messages.
+        """
+        try:
+            # Run SQS receive in executor to avoid blocking
+            loop = asyncio.get_event_loop()
+            response = await loop.run_in_executor(
+                None,
+                lambda: self.sqs_client.receive_message(
+                    QueueUrl=self.config.aws_sqs_queue_url,
+                    MaxNumberOfMessages=1,  # Process one at a time
+                    WaitTimeSeconds=self.config.sqs_wait_time_seconds,
+                    AttributeNames=['All'],
+                    MessageAttributeNames=['All']
+                )
+            )
+
+            messages = response.get('Messages', [])
+
+            if not messages:
+                # No messages available, will poll again
+                return
+
+            for message in messages:
+                await self._process_message(message)
+
+        except Exception as e:
+            logger.error("Error polling SQS", error=str(e), exc_info=True)
+            # Wait a bit before retrying on error
+            await asyncio.sleep(5)
+
+    async def _process_message(self, message: Dict[str, Any]) -> None:
+        """Process a single SQS message.
+
+        Args:
+            message: SQS message containing webhook event
+        """
+        receipt_handle = message['ReceiptHandle']
+        message_id = message['MessageId']
+
+        try:
+            # Parse message body
+            body = json.loads(message['Body'])
+
+            # Extract headers and payload
+            headers = body.get('headers', {})
+            payload_data = body.get('payload')
+
+            # Convert payload to bytes if needed
+            if isinstance(payload_data, str):
+                # Check if it's base64 encoded
+                if payload_data.startswith('{') or payload_data.startswith('['):
+                    # Raw JSON string
+                    payload = payload_data.encode('utf-8')
+                else:
+                    # Assume base64 encoded
+                    import base64
+                    payload = base64.b64decode(payload_data)
+            elif isinstance(payload_data, dict):
+                # Already parsed JSON, re-encode to bytes
+                payload = json.dumps(payload_data).encode('utf-8')
+            else:
+                payload = payload_data
+
+            # Generate delivery ID if not present
+            delivery_id = headers.get('x-github-delivery', f'sqs-{message_id}')
+
+            # Verify GitHub webhook signature (defense in depth)
+            signature = headers.get('x-hub-signature-256', '')
+            if not verify_github_signature(payload, signature, self.config.github_webhook_secret):
+                error_msg = "Invalid GitHub webhook signature - possible security breach or misconfiguration"
+                logger.error(
+                    "Invalid webhook signature from SQS message",
+                    message_id=message_id,
+                    delivery_id=delivery_id,
+                    signature_present=bool(signature),
+                )
+                # Send to DLQ for investigation
+                if self.config.aws_sqs_dlq_url:
+                    await self._send_to_dlq(message, error_msg)
+                # Delete from main queue (don't retry invalid signatures)
+                loop = asyncio.get_event_loop()
+                await loop.run_in_executor(
+                    None,
+                    lambda: self.sqs_client.delete_message(
+                        QueueUrl=self.config.aws_sqs_queue_url,
+                        ReceiptHandle=receipt_handle
+                    )
+                )
+                logger.warning(
+                    "Rejected SQS message with invalid signature",
+                    message_id=message_id,
+                )
+                return
+
+            logger.info(
+                "Processing SQS message",
+                message_id=message_id,
+                delivery_id=delivery_id,
+                event_type=headers.get('x-github-event', 'unknown'),
+            )
+
+            # Process the webhook event
+            await self.task_processor.process_webhook(
+                headers=headers,
+                payload=payload,
+                delivery_id=delivery_id
+            )
+
+            # Delete message from queue on success
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(
+                None,
+                lambda: self.sqs_client.delete_message(
+                    QueueUrl=self.config.aws_sqs_queue_url,
+                    ReceiptHandle=receipt_handle
+                )
+            )
+
+            logger.info(
+                "SQS message processed successfully",
+                message_id=message_id,
+                delivery_id=delivery_id,
+            )
+
+        except Exception as e:
+            error_msg = f"Failed to process SQS message: {str(e)}"
+            logger.error(
+                "Error processing SQS message",
+                message_id=message_id,
+                error=error_msg,
+                exc_info=True,
+            )
+
+            # Send to DLQ if configured
+            if self.config.aws_sqs_dlq_url:
+                await self._send_to_dlq(message, error_msg)
+
+            # Delete message from main queue to prevent reprocessing
+            # (it's already in DLQ or we've logged the error)
+            try:
+                loop = asyncio.get_event_loop()
+                await loop.run_in_executor(
+                    None,
+                    lambda: self.sqs_client.delete_message(
+                        QueueUrl=self.config.aws_sqs_queue_url,
+                        ReceiptHandle=receipt_handle
+                    )
+                )
+                logger.info(
+                    "Deleted failed message from queue",
+                    message_id=message_id,
+                )
+            except Exception as delete_error:
+                logger.error(
+                    "Failed to delete message after error",
+                    message_id=message_id,
+                    error=str(delete_error),
+                )
+
+    async def _send_to_dlq(self, message: Dict[str, Any], error_msg: str) -> None:
+        """Send a failed message to the dead-letter queue.
+
+        Args:
+            message: Original SQS message
+            error_msg: Error message describing the failure
+        """
+        try:
+            message_id = message['MessageId']
+
+            # Add error information to the message
+            dlq_body = {
+                'original_message': message['Body'],
+                'error': error_msg,
+                'failed_at': asyncio.get_event_loop().time(),
+                'original_message_id': message_id,
+            }
+
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(
+                None,
+                lambda: self.sqs_client.send_message(
+                    QueueUrl=self.config.aws_sqs_dlq_url,
+                    MessageBody=json.dumps(dlq_body)
+                )
+            )
+
+            logger.info(
+                "Sent failed message to DLQ",
+                message_id=message_id,
+                dlq_url=self.config.aws_sqs_dlq_url,
+            )
+
+        except Exception as e:
+            logger.error(
+                "Failed to send message to DLQ",
+                message_id=message.get('MessageId', 'unknown'),
+                error=str(e),
+                exc_info=True,
+            )

devs_webhook/sources/webhook_source.py

@@ -0,0 +1,82 @@
+"""Webhook task source using FastAPI.
+
+This module provides the webhook task source that receives GitHub webhooks
+via a FastAPI HTTP endpoint and forwards them to the task processor.
+"""
+
+import asyncio
+import structlog
+from typing import Optional
+
+from .base import TaskSource
+from ..core.task_processor import TaskProcessor
+
+logger = structlog.get_logger()
+
+
+class WebhookTaskSource(TaskSource):
+    """Task source that receives GitHub webhooks via FastAPI.
+
+    This is the traditional webhook endpoint approach that receives
+    HTTP POST requests from GitHub and processes them.
+    """
+
+    def __init__(self, task_processor: Optional[TaskProcessor] = None):
+        """Initialize webhook task source.
+
+        Args:
+            task_processor: Optional task processor instance. If not provided,
+                          a new one will be created.
+        """
+        self.task_processor = task_processor or TaskProcessor()
+        self._server_task: Optional[asyncio.Task] = None
+        logger.info("Webhook task source initialized")
+
+    async def start(self) -> None:
+        """Start the FastAPI webhook server.
+
+        This method blocks until the server is stopped.
+        """
+        logger.info("Starting webhook task source (FastAPI)")
+
+        # Import uvicorn here to avoid import issues
+        import uvicorn
+        from ..config import get_config
+
+        config = get_config()
+
+        # Create uvicorn config
+        uvicorn_config = uvicorn.Config(
+            "devs_webhook.app:app",
+            host=config.webhook_host,
+            port=config.webhook_port,
+            log_config=None,  # Use our structlog config
+        )
+
+        # Create and start server
+        server = uvicorn.Server(uvicorn_config)
+
+        logger.info(
+            "Webhook server starting",
+            host=config.webhook_host,
+            port=config.webhook_port,
+        )
+
+        # Run the server (blocks until stopped)
+        await server.serve()
+
+    async def stop(self) -> None:
+        """Stop the webhook server."""
+        logger.info("Stopping webhook task source")
+
+        # The uvicorn server handles its own shutdown
+        # We just need to cancel any running tasks
+
+        if self._server_task and not self._server_task.done():
+            self._server_task.cancel()
+            try:
+                await self._server_task
+            except asyncio.CancelledError:
+                pass
+
+        logger.info("Webhook task source stopped")

devs_webhook/utils/__init__.py

@@ -0,0 +1 @@
+"""Utility modules."""

devs_webhook/utils/async_utils.py

@@ -0,0 +1,86 @@
+"""Async utilities for non-blocking operations."""
+
+import asyncio
+import subprocess
+from typing import Optional, Tuple
+import structlog
+
+logger = structlog.get_logger()
+
+
+async def run_subprocess_async(
+    cmd: list[str],
+    cwd: Optional[str] = None,
+    timeout: Optional[float] = None
+) -> Tuple[int, str, str]:
+    """Run a subprocess asynchronously without blocking the event loop.
+    
+    Args:
+        cmd: Command and arguments to run
+        cwd: Working directory
+        timeout: Timeout in seconds
+        
+    Returns:
+        Tuple of (return_code, stdout, stderr)
+    """
+    try:
+        # Create subprocess
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            cwd=cwd
+        )
+        
+        # Wait for completion with optional timeout
+        try:
+            stdout, stderr = await asyncio.wait_for(
+                process.communicate(),
+                timeout=timeout
+            )
+        except asyncio.TimeoutError:
+            process.kill()
+            await process.wait()
+            return -1, "", f"Command timed out after {timeout} seconds"
+        
+        return (
+            process.returncode or 0,
+            stdout.decode('utf-8', errors='replace') if stdout else "",
+            stderr.decode('utf-8', errors='replace') if stderr else ""
+        )
+        
+    except Exception as e:
+        logger.error("Subprocess execution failed",
+                    cmd=cmd,
+                    error=str(e))
+        return -1, "", str(e)
+
+
+async def run_git_async(
+    args: list[str],
+    cwd: str,
+    timeout: float = 30.0
+) -> Tuple[bool, str, str]:
+    """Run a git command asynchronously.
+    
+    Args:
+        args: Git command arguments (without 'git' prefix)
+        cwd: Working directory
+        timeout: Timeout in seconds
+        
+    Returns:
+        Tuple of (success, stdout, stderr)
+    """
+    cmd = ["git"] + args
+    returncode, stdout, stderr = await run_subprocess_async(cmd, cwd, timeout)
+    
+    success = returncode == 0
+    
+    if not success:
+        logger.debug("Git command failed",
+                    args=args,
+                    cwd=cwd,
+                    returncode=returncode,
+                    stderr=stderr)
+    
+    return success, stdout, stderr

devs_webhook/utils/github.py

@@ -0,0 +1,43 @@
+"""GitHub webhook utilities."""
+
+import hmac
+import hashlib
+
+
+def verify_github_signature(payload: bytes, signature: str, secret: str) -> bool:
+    """Verify GitHub webhook signature using HMAC-SHA256.
+
+    GitHub signs webhook payloads with HMAC-SHA256 using the webhook secret.
+    The signature is sent in the X-Hub-Signature-256 header as 'sha256=<hex>'.
+
+    Args:
+        payload: Raw webhook payload bytes (must be the exact bytes received)
+        signature: GitHub signature header (e.g., 'sha256=abc123...')
+        secret: Webhook secret configured in GitHub
+
+    Returns:
+        True if signature is valid, False otherwise
+
+    Example:
+        >>> payload = b'{"action": "opened", ...}'
+        >>> signature = request.headers.get('X-Hub-Signature-256')
+        >>> secret = os.environ['GITHUB_WEBHOOK_SECRET']
+        >>> if verify_github_signature(payload, signature, secret):
+        ...     # Process webhook
+        ...     pass
+    """
+    if not signature:
+        return False
+
+    if not signature.startswith("sha256="):
+        return False
+
+    # Compute expected signature
+    expected_signature = "sha256=" + hmac.new(
+        secret.encode("utf-8"),
+        payload,
+        hashlib.sha256
+    ).hexdigest()
+
+    # Use constant-time comparison to prevent timing attacks
+    return hmac.compare_digest(signature, expected_signature)

devs_webhook/utils/logging.py

@@ -0,0 +1,34 @@
+"""Logging configuration."""
+
+import sys
+import logging
+import structlog
+from typing import Any, Dict
+
+from ..config import get_config
+
+
+def setup_logging() -> None:
+    """Set up structured logging."""
+    config = get_config()
+    
+    # Configure structlog
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.processors.add_log_level,
+            structlog.processors.TimeStamper(fmt="ISO"),
+            structlog.dev.ConsoleRenderer() if config.log_format == "console" 
+            else structlog.processors.JSONRenderer()
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(
+            getattr(logging, config.log_level.upper(), logging.INFO)
+        ),
+        logger_factory=structlog.WriteLoggerFactory(sys.stdout),
+        cache_logger_on_first_use=True,
+    )
+
+
+def get_logger(**context: Any) -> structlog.BoundLogger:
+    """Get a logger with context."""
+    return structlog.get_logger().bind(**context)

devs_webhook/utils/serialization.py

@@ -0,0 +1,102 @@
+"""JSON serialization utilities for webhook objects."""
+
+import json
+import base64
+from typing import Any, Dict
+from pathlib import Path
+from datetime import datetime
+
+from ..github.models import WebhookEvent, DevsOptions, IssueEvent, PullRequestEvent, CommentEvent
+
+
+class WebhookEventEncoder(json.JSONEncoder):
+    """Custom JSON encoder for WebhookEvent objects."""
+    
+    def default(self, obj):
+        # Handle datetime objects
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        # Handle pydantic BaseModel instances
+        elif hasattr(obj, 'model_dump'):
+            return obj.model_dump()
+        elif isinstance(obj, (IssueEvent, PullRequestEvent, CommentEvent)):
+            return {
+                '__type__': obj.__class__.__name__,
+                '__data__': obj.model_dump()
+            }
+        elif isinstance(obj, Path):
+            return str(obj)
+        return super().default(obj)
+
+
+def serialize_webhook_event(event: WebhookEvent) -> str:
+    """Serialize WebhookEvent to base64-encoded JSON string.
+    
+    Args:
+        event: WebhookEvent to serialize
+        
+    Returns:
+        Base64-encoded JSON string
+    """
+    json_str = json.dumps(event, cls=WebhookEventEncoder, ensure_ascii=True)
+    return base64.b64encode(json_str.encode()).decode('ascii')
+
+
+def deserialize_webhook_event(data: str) -> WebhookEvent:
+    """Deserialize WebhookEvent from base64-encoded JSON string.
+    
+    Args:
+        data: Base64-encoded JSON string
+        
+    Returns:
+        WebhookEvent instance
+    """
+    json_str = base64.b64decode(data.encode('ascii')).decode()
+    raw_data = json.loads(json_str)
+    
+    # Reconstruct the appropriate event type
+    event_type = raw_data.get('__type__')
+    event_data = raw_data.get('__data__', raw_data)
+    
+    if event_type == 'IssueEvent':
+        return IssueEvent.model_validate(event_data)
+    elif event_type == 'PullRequestEvent':
+        return PullRequestEvent.model_validate(event_data)
+    elif event_type == 'CommentEvent':
+        return CommentEvent.model_validate(event_data)
+    else:
+        # Fallback - try to determine from data structure
+        if 'issue' in event_data:
+            return IssueEvent.model_validate(event_data)
+        elif 'pull_request' in event_data:
+            return PullRequestEvent.model_validate(event_data)
+        else:
+            return CommentEvent.model_validate(event_data)
+
+
+def serialize_devs_options(options: DevsOptions) -> str:
+    """Serialize DevsOptions to base64-encoded JSON string.
+    
+    Args:
+        options: DevsOptions to serialize
+        
+    Returns:
+        Base64-encoded JSON string
+    """
+    json_str = json.dumps(options.model_dump(), ensure_ascii=True)
+    return base64.b64encode(json_str.encode()).decode('ascii')
+
+
+def deserialize_devs_options(data: str) -> DevsOptions:
+    """Deserialize DevsOptions from base64-encoded JSON string.
+    
+    Args:
+        data: Base64-encoded JSON string
+        
+    Returns:
+        DevsOptions instance
+    """
+    json_str = base64.b64decode(data.encode('ascii')).decode()
+    raw_data = json.loads(json_str)
+    
+    return DevsOptions.model_validate(raw_data)