devs-webhook 0.1.0__py3-none-any.whl

devs_webhook/core/deduplication.py

@@ -0,0 +1,113 @@
+"""Content-based deduplication cache."""
+
+import time
+from typing import Dict, Tuple
+import structlog
+
+logger = structlog.get_logger()
+
+
+class DeduplicationCache:
+    """In-memory cache for content-based deduplication with TTL."""
+    
+    def __init__(self, ttl_seconds: int = 300):  # 5 minutes default
+        """Initialize deduplication cache.
+        
+        Args:
+            ttl_seconds: Time-to-live for cache entries in seconds
+        """
+        self.ttl_seconds = ttl_seconds
+        self._cache: Dict[str, Tuple[float, str]] = {}  # hash -> (timestamp, description)
+        
+    def is_duplicate(self, content_hash: str, description: str = "") -> bool:
+        """Check if content hash was recently processed.
+        
+        Args:
+            content_hash: Hash of the content to check
+            description: Optional description for logging
+            
+        Returns:
+            True if this is a duplicate within the TTL window
+        """
+        current_time = time.time()
+        
+        # Clean expired entries
+        self._cleanup_expired(current_time)
+        
+        if content_hash in self._cache:
+            cached_time, cached_desc = self._cache[content_hash]
+            age_seconds = current_time - cached_time
+            
+            logger.info("Duplicate content detected",
+                       content_hash=content_hash,
+                       age_seconds=round(age_seconds, 1),
+                       ttl_seconds=self.ttl_seconds,
+                       description=description,
+                       cached_description=cached_desc,
+                       is_duplicate=True)
+            return True
+        
+        # Not a duplicate - add to cache
+        self._cache[content_hash] = (current_time, description)
+        
+        logger.info("New content hash cached",
+                   content_hash=content_hash,
+                   cache_size=len(self._cache),
+                   description=description,
+                   is_duplicate=False)
+        
+        return False
+    
+    def _cleanup_expired(self, current_time: float) -> None:
+        """Remove expired entries from cache."""
+        expired_keys = [
+            key for key, (timestamp, _) in self._cache.items()
+            if current_time - timestamp > self.ttl_seconds
+        ]
+        
+        for key in expired_keys:
+            del self._cache[key]
+        
+        if expired_keys:
+            logger.debug("Cleaned up expired cache entries",
+                        expired_count=len(expired_keys),
+                        remaining_count=len(self._cache))
+    
+    def get_stats(self) -> Dict[str, int]:
+        """Get cache statistics."""
+        current_time = time.time()
+        valid_entries = sum(
+            1 for timestamp, _ in self._cache.values()
+            if current_time - timestamp <= self.ttl_seconds
+        )
+        
+        return {
+            "total_entries": len(self._cache),
+            "valid_entries": valid_entries,
+            "ttl_seconds": self.ttl_seconds
+        }
+    
+    def clear(self) -> None:
+        """Clear all cache entries."""
+        cleared_count = len(self._cache)
+        self._cache.clear()
+        logger.info("Deduplication cache cleared", cleared_count=cleared_count)
+
+
+# Global cache instance
+_global_cache = DeduplicationCache()
+
+
+def is_duplicate_content(content_hash: str, description: str = "") -> bool:
+    """Check if content hash is a duplicate using global cache."""
+    return _global_cache.is_duplicate(content_hash, description)
+
+
+def get_cache_stats() -> Dict[str, int]:
+    """Get global cache statistics."""
+    return _global_cache.get_stats()
+
+
+def clear_cache() -> None:
+    """Clear global cache."""
+    _global_cache.clear()

devs_webhook/core/repository_manager.py

@@ -0,0 +1,197 @@
+"""Repository management for webhook handler."""
+
+import asyncio
+from pathlib import Path
+from typing import Optional, Dict
+import structlog
+
+from ..config import get_config
+from ..github.client import GitHubClient
+from ..utils.async_utils import run_git_async
+
+logger = structlog.get_logger()
+
+
+class RepositoryManager:
+    """Manages repository cloning and caching for webhook tasks."""
+    
+    def __init__(self):
+        """Initialize repository manager."""
+        self.config = get_config()
+        
+        self.github_client = GitHubClient(self.config)
+        
+        # Track repository status
+        self.repo_locks: Dict[str, asyncio.Lock] = {}
+        
+        logger.info("Repository manager initialized",
+                   cache_dir=str(self.config.repo_cache_dir))
+    
+    async def ensure_repository(
+        self, 
+        repo_name: str, 
+        clone_url: str
+    ) -> Optional[Path]:
+        """Ensure repository is available locally and up to date.
+        
+        Args:
+            repo_name: Repository name in format "owner/repo"
+            clone_url: Repository clone URL
+            
+        Returns:
+            Path to local repository or None if failed
+        """
+        # Get or create lock for this repository
+        if repo_name not in self.repo_locks:
+            self.repo_locks[repo_name] = asyncio.Lock()
+        
+        async with self.repo_locks[repo_name]:
+            return await self._ensure_repository_locked(repo_name, clone_url)
+    
+    async def _ensure_repository_locked(
+        self, 
+        repo_name: str, 
+        clone_url: str
+    ) -> Optional[Path]:
+        """Ensure repository is available (called with lock held).
+        
+        Args:
+            repo_name: Repository name
+            clone_url: Repository clone URL
+            
+        Returns:
+            Path to local repository or None if failed
+        """
+        # Calculate local path
+        repo_dir = self.config.repo_cache_dir / repo_name.replace("/", "-")
+        
+        try:
+            if repo_dir.exists():
+                # Repository exists, update it
+                logger.info("Updating existing repository", 
+                           repo=repo_name, path=str(repo_dir))
+                
+                success = await self._update_repository(repo_dir)
+                if success:
+                    return repo_dir
+                else:
+                    # Update failed, try to reclone
+                    logger.warning("Update failed, recloning repository",
+                                  repo=repo_name)
+                    await self._remove_repository(repo_dir)
+            
+            # Clone repository
+            logger.info("Cloning repository", 
+                       repo=repo_name, path=str(repo_dir))
+            
+            success = await self.github_client.clone_repository(
+                repo_name, repo_dir
+            )
+            
+            if success:
+                return repo_dir
+            else:
+                return None
+                
+        except Exception as e:
+            logger.error("Failed to ensure repository",
+                        repo=repo_name,
+                        error=str(e))
+            return None
+    
+    async def _update_repository(self, repo_dir: Path) -> bool:
+        """Update an existing repository.
+        
+        Args:
+            repo_dir: Path to repository directory
+            
+        Returns:
+            True if update successful
+        """
+        try:
+            # Fetch all remotes using async git
+            success, _, stderr = await run_git_async(
+                ["fetch", "--all"],
+                str(repo_dir)
+            )
+            
+            if not success:
+                logger.warning("Git fetch failed",
+                              path=str(repo_dir),
+                              error=stderr)
+                return False
+            
+            # Reset to origin/main or origin/master
+            for branch in ["main", "master"]:
+                success, _, _ = await run_git_async(
+                    ["reset", "--hard", f"origin/{branch}"],
+                    str(repo_dir)
+                )
+                
+                if success:
+                    logger.info("Repository updated",
+                               path=str(repo_dir),
+                               branch=branch)
+                    return True
+            
+            logger.warning("Could not reset to main or master branch",
+                          path=str(repo_dir))
+            return False
+            
+        except Exception as e:
+            logger.error("Error updating repository",
+                        path=str(repo_dir),
+                        error=str(e))
+            return False
+    
+    async def _remove_repository(self, repo_dir: Path) -> None:
+        """Remove a repository directory.
+        
+        Args:
+            repo_dir: Path to repository directory
+        """
+        try:
+            import shutil
+            shutil.rmtree(repo_dir)
+            logger.info("Repository removed", path=str(repo_dir))
+        except Exception as e:
+            logger.error("Failed to remove repository",
+                        path=str(repo_dir),
+                        error=str(e))
+    
+    async def get_repository_info(self, repo_name: str) -> Optional[Dict]:
+        """Get information about a repository.
+        
+        Args:
+            repo_name: Repository name
+            
+        Returns:
+            Repository info dict or None if not found
+        """
+        return await self.github_client.get_repository_info(repo_name)
+    
+    async def cleanup_old_repositories(self, max_age_days: int = 7) -> None:
+        """Clean up old repository caches.
+        
+        Args:
+            max_age_days: Maximum age in days before cleanup
+        """
+        try:
+            import time
+            from datetime import datetime, timedelta
+            
+            cutoff_time = time.time() - (max_age_days * 24 * 60 * 60)
+            
+            for repo_dir in self.config.repo_cache_dir.iterdir():
+                if repo_dir.is_dir():
+                    # Check last modification time
+                    mtime = repo_dir.stat().st_mtime
+                    
+                    if mtime < cutoff_time:
+                        logger.info("Cleaning up old repository cache",
+                                   repo=repo_dir.name,
+                                   age_days=(time.time() - mtime) / (24 * 60 * 60))
+                        await self._remove_repository(repo_dir)
+                        
+        except Exception as e:
+            logger.error("Error during repository cleanup", error=str(e))

devs_webhook/core/task_processor.py

@@ -0,0 +1,286 @@
+"""Core task processor that handles webhook event processing.
+
+This module provides the TaskProcessor class which is decoupled from the task source
+(webhook, SQS, etc.) and handles all the business logic of processing GitHub events.
+"""
+
+from typing import Dict, Any
+import structlog
+
+from ..config import get_config
+from ..github.parser import WebhookParser
+from ..github.client import GitHubClient
+from ..github.models import IssueEvent, PullRequestEvent, CommentEvent
+from .container_pool import ContainerPool
+from .deduplication import is_duplicate_content, get_cache_stats
+
+logger = structlog.get_logger()
+
+
+class TaskProcessor:
+    """Core task processor that handles webhook event processing.
+
+    This class is decoupled from the task source (webhook endpoint, SQS, etc.)
+    and contains all the business logic for processing GitHub webhook events.
+    """
+
+    def __init__(self, container_pool: ContainerPool = None):
+        """Initialize task processor.
+
+        Args:
+            container_pool: Optional container pool instance. If not provided,
+                          a new one will be created.
+        """
+        self.config = get_config()
+        self.container_pool = container_pool or ContainerPool()
+        self.github_client = GitHubClient(self.config)
+
+        logger.info("Task processor initialized",
+                   mentioned_user=self.config.github_mentioned_user,
+                   container_pool=self.config.get_container_pool_list())
+
+    async def _add_eyes_reaction(self, event: Any, repo_name: str) -> None:
+        """Add an eyes reaction to indicate we're processing the event.
+
+        Args:
+            event: The webhook event object
+            repo_name: Repository in format "owner/repo"
+        """
+        try:
+            reaction_added = False
+
+            # Determine what to react to based on event type
+            if isinstance(event, CommentEvent):
+                # React to the comment itself
+                reaction_added = await self.github_client.add_reaction_to_comment(
+                    repo=repo_name,
+                    comment_id=event.comment.id,
+                    reaction="eyes"
+                )
+                logger.info("Attempting to add reaction to comment",
+                           comment_id=event.comment.id,
+                           repo=repo_name)
+            elif isinstance(event, IssueEvent):
+                # React to the issue
+                reaction_added = await self.github_client.add_reaction_to_issue(
+                    repo=repo_name,
+                    issue_number=event.issue.number,
+                    reaction="eyes"
+                )
+                logger.info("Attempting to add reaction to issue",
+                           issue_number=event.issue.number,
+                           repo=repo_name)
+            elif isinstance(event, PullRequestEvent):
+                # React to the PR (PRs are issues in GitHub API)
+                reaction_added = await self.github_client.add_reaction_to_pr(
+                    repo=repo_name,
+                    pr_number=event.pull_request.number,
+                    reaction="eyes"
+                )
+                logger.info("Attempting to add reaction to PR",
+                           pr_number=event.pull_request.number,
+                           repo=repo_name)
+
+            if reaction_added:
+                logger.info("Successfully added eyes reaction",
+                           event_type=type(event).__name__,
+                           repo=repo_name)
+            else:
+                logger.warning("Could not add eyes reaction",
+                              event_type=type(event).__name__,
+                              repo=repo_name)
+
+        except Exception as e:
+            # Log the error but don't fail the webhook processing
+            logger.error("Error adding reaction to event - continuing anyway",
+                        error=str(e),
+                        event_type=type(event).__name__,
+                        repo=repo_name,
+                        exc_info=True)
+
+    async def process_webhook(
+        self,
+        headers: Dict[str, str],
+        payload: bytes,
+        delivery_id: str
+    ) -> None:
+        """Process a GitHub webhook event.
+
+        This is the main entry point for processing webhook events, regardless
+        of the source (FastAPI endpoint, SQS, etc.).
+
+        Args:
+            headers: HTTP headers from webhook
+            payload: Raw webhook payload
+            delivery_id: Unique delivery ID for tracking
+        """
+        try:
+            # Parse webhook event
+            event = WebhookParser.parse_webhook(headers, payload)
+
+            if event is None:
+                logger.info("Unsupported webhook event type",
+                           event_type=headers.get("x-github-event"),
+                           delivery_id=delivery_id)
+                return
+
+            # Check if the user who triggered the event is authorized
+            trigger_user = event.sender.login
+            if not self.config.is_user_authorized_to_trigger(trigger_user):
+                logger.warning("User not authorized to trigger webhook processing",
+                              user=trigger_user,
+                              repo=event.repository.full_name,
+                              delivery_id=delivery_id,
+                              event_type=type(event).__name__)
+                return
+
+            # Check if repository is allowed
+            repo_owner = event.repository.owner.login
+            if not self.config.is_repository_allowed(event.repository.full_name, repo_owner):
+                logger.warning("Repository not in allowlist - rejecting webhook",
+                              repo=event.repository.full_name,
+                              owner=repo_owner,
+                              delivery_id=delivery_id,
+                              event_type=type(event).__name__)
+                return
+
+            # Load repository configuration to check for CI mode
+            devs_options = await self.container_pool.ensure_repo_config(event.repository.full_name)
+
+            # Check if we should process this event for CI
+            process_for_ci = WebhookParser.should_process_event_for_ci(event, devs_options)
+
+            # Check if we should process this event for mentions
+            process_for_mentions = WebhookParser.should_process_event(event, self.config.github_mentioned_user)
+
+            # Skip if neither CI nor mentions apply
+            if not process_for_ci and not process_for_mentions:
+                logger.info("Event does not trigger CI or contain target mentions",
+                           event_type=type(event).__name__,
+                           mentioned_user=self.config.github_mentioned_user,
+                           ci_enabled=devs_options.ci_enabled if devs_options else False,
+                           delivery_id=delivery_id)
+                return
+
+            logger.info("Event processing mode determined",
+                       event_type=type(event).__name__,
+                       process_for_ci=process_for_ci,
+                       process_for_mentions=process_for_mentions,
+                       delivery_id=delivery_id)
+
+            # Check for duplicate content
+            content_hash = event.get_content_hash()
+            if content_hash:
+                event_description = f"{type(event).__name__}({event.action}) {event.repository.full_name}"
+                if hasattr(event, 'issue'):
+                    event_description += f" issue#{event.issue.number}"
+                elif hasattr(event, 'pull_request'):
+                    event_description += f" pr#{event.pull_request.number}"
+
+                if is_duplicate_content(content_hash, event_description):
+                    logger.info("Duplicate content detected, skipping processing",
+                               event_type=type(event).__name__,
+                               action=event.action,
+                               content_hash=content_hash,
+                               event_description=event_description,
+                               delivery_id=delivery_id)
+                    return
+
+            logger.info("Processing webhook event",
+                       event_type=type(event).__name__,
+                       repo=event.repository.full_name,
+                       action=event.action,
+                       delivery_id=delivery_id)
+
+            tasks_queued = []
+
+            # Queue CI task if applicable
+            if process_for_ci:
+                ci_task_id = f"{delivery_id}-ci"
+                ci_success = await self.container_pool.queue_task(
+                    task_id=ci_task_id,
+                    repo_name=event.repository.full_name,
+                    task_description="",  # Not used for CI tasks
+                    event=event,
+                    task_type='tests'
+                )
+
+                if ci_success:
+                    tasks_queued.append("CI tests")
+                    logger.info("CI task queued successfully",
+                               delivery_id=ci_task_id,
+                               repo=event.repository.full_name)
+                else:
+                    logger.error("Failed to queue CI task",
+                                delivery_id=ci_task_id,
+                                repo=event.repository.full_name)
+
+            # Queue mention-based task if applicable
+            if process_for_mentions:
+                # Get context from the event for Claude
+                task_description = event.get_context_for_claude()
+
+                mention_task_id = f"{delivery_id}-claude" if process_for_ci else delivery_id
+                mention_success = await self.container_pool.queue_task(
+                    task_id=mention_task_id,
+                    repo_name=event.repository.full_name,
+                    task_description=task_description,
+                    event=event,
+                    task_type='claude'
+                )
+
+                if mention_success:
+                    tasks_queued.append("Claude processing")
+                    logger.info("Claude task queued successfully",
+                               delivery_id=mention_task_id,
+                               repo=event.repository.full_name)
+
+                    # Try to add "eyes" reaction to indicate we're looking into it
+                    await self._add_eyes_reaction(event, event.repository.full_name)
+                else:
+                    logger.error("Failed to queue Claude task",
+                                delivery_id=mention_task_id,
+                                repo=event.repository.full_name)
+
+            if tasks_queued:
+                logger.info("Tasks queued successfully",
+                           delivery_id=delivery_id,
+                           repo=event.repository.full_name,
+                           tasks=tasks_queued)
+            else:
+                logger.error("Failed to queue any tasks",
+                            delivery_id=delivery_id,
+                            repo=event.repository.full_name)
+
+        except Exception as e:
+            logger.error("Error processing webhook",
+                        error=str(e),
+                        delivery_id=delivery_id,
+                        exc_info=True)
+
+    async def get_status(self) -> Dict[str, Any]:
+        """Get current processor status."""
+        container_status = await self.container_pool.get_status()
+
+        # Calculate total queued tasks across all containers
+        total_queued = sum(
+            self.container_pool.container_queues[container].qsize()
+            for container in self.config.get_container_pool_list()
+        )
+
+        return {
+            "queued_tasks": total_queued,
+            "container_pool_size": len(self.config.get_container_pool_list()),
+            "containers": container_status,
+            "mentioned_user": self.config.github_mentioned_user,
+            "authorized_trigger_users": self.config.get_authorized_trigger_users_list(),
+            "deduplication_cache": get_cache_stats(),
+        }
+
+    async def stop_container(self, container_name: str) -> bool:
+        """Manually stop a container."""
+        return await self.container_pool.force_stop_container(container_name)
+
+    async def list_containers(self) -> Dict[str, Any]:
+        """List all managed containers."""
+        return await self.container_pool.get_status()