local-deep-research 0.1.0__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_github.py

@@ -0,0 +1,663 @@
+import requests
+import logging
+import base64
+import os
+import time
+from typing import Dict, List, Any, Optional, Union
+from langchain_core.language_models import BaseLLM
+
+from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
+from local_deep_research import config
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class GitHubSearchEngine(BaseSearchEngine):
+    """
+    GitHub search engine implementation.
+    Provides search across GitHub repositories, code, issues, and users.
+    """
+    
+    def __init__(self, 
+                max_results: int = 15, 
+                api_key: Optional[str] = None,
+                search_type: str = "repositories",
+                include_readme: bool = True,
+                include_issues: bool = False,
+                llm: Optional[BaseLLM] = None,
+                max_filtered_results: Optional[int] = None):
+        """
+        Initialize the GitHub search engine.
+        
+        Args:
+            max_results: Maximum number of search results
+            api_key: GitHub API token (can also be set in GITHUB_API_KEY env)
+            search_type: Type of GitHub search ("repositories", "code", "issues", "users")
+            include_readme: Whether to include README content for repositories
+            include_issues: Whether to include recent issues for repositories
+            llm: Language model for relevance filtering
+            max_filtered_results: Maximum number of results to keep after filtering
+        """
+        # Initialize the BaseSearchEngine with the LLM and max_filtered_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results)
+        
+        self.max_results = max_results
+        self.api_key = api_key or os.getenv("GITHUB_API_KEY")
+        self.search_type = search_type
+        self.include_readme = include_readme
+        self.include_issues = include_issues
+        
+        # API endpoints
+        self.api_base = "https://api.github.com"
+        self.search_endpoint = f"{self.api_base}/search/{search_type}"
+        
+        # Set up API headers
+        self.headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "User-Agent": "Local-Deep-Research-Agent"
+        }
+        
+        # Add authentication if API key provided
+        if self.api_key:
+            self.headers["Authorization"] = f"token {self.api_key}"
+            logger.info("Using authenticated GitHub API requests")
+        else:
+            logger.warning("No GitHub API key provided. Rate limits will be restricted.")
+    
+    def _handle_rate_limits(self, response):
+        """Handle GitHub API rate limits by logging warnings and sleeping if necessary"""
+        remaining = int(response.headers.get("X-RateLimit-Remaining", 60))
+        reset_time = int(response.headers.get("X-RateLimit-Reset", 0))
+        
+        if remaining < 5:
+            current_time = time.time()
+            wait_time = max(reset_time - current_time, 0)
+            logger.warning(f"GitHub API rate limit almost reached. {remaining} requests remaining.")
+            
+            if wait_time > 0 and remaining == 0:
+                logger.warning(f"GitHub API rate limit exceeded. Waiting {wait_time:.0f} seconds.")
+                time.sleep(min(wait_time, 60))  # Wait at most 60 seconds
+    
+    def _search_github(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Perform a GitHub search based on the configured search type.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of GitHub search result items
+        """
+        results = []
+        
+        try:
+            # Optimize GitHub query format
+            github_query = query
+            
+            # For long queries, focus on keywords and add filters for better results
+            if len(query) > 80:
+                # Extract key terms if it's a recommendation request
+                if "recommend" in query.lower() or "looking for" in query.lower():
+                    github_query = "stars:>100 " + " ".join([
+                        word for word in query.split() 
+                        if len(word) > 3 and word.lower() not in 
+                        ["recommend", "recommended", "github", "repositories", "looking", "developers"]
+                    ])
+            
+            logger.info(f"Optimized GitHub query: {github_query}")
+            
+            # Construct search parameters
+            params = {
+                "q": github_query,
+                "per_page": min(self.max_results, 100),  # GitHub API max is 100 per page
+                "page": 1
+            }
+            
+            # Add sort parameters based on search type
+            if self.search_type == "repositories":
+                params["sort"] = "stars"
+                params["order"] = "desc"
+            elif self.search_type == "code":
+                params["sort"] = "indexed"
+                params["order"] = "desc"
+            elif self.search_type == "issues":
+                params["sort"] = "updated"
+                params["order"] = "desc"
+            elif self.search_type == "users":
+                params["sort"] = "followers"
+                params["order"] = "desc"
+            
+            # Execute the API request
+            response = requests.get(
+                self.search_endpoint,
+                headers=self.headers,
+                params=params
+            )
+            
+            # Check for rate limiting
+            self._handle_rate_limits(response)
+            
+            # Handle response with detailed logging
+            if response.status_code == 200:
+                data = response.json()
+                total_count = data.get("total_count", 0)
+                results = data.get("items", [])
+                logger.info(f"GitHub search returned {len(results)} results (total available: {total_count})")
+                
+                # Log the rate limit information
+                rate_limit_remaining = response.headers.get("X-RateLimit-Remaining", "unknown")
+                rate_limit_reset = response.headers.get("X-RateLimit-Reset", "unknown")
+                logger.info(f"GitHub API rate limit: {rate_limit_remaining} requests remaining")
+                
+                # If no results, try to provide more guidance
+                if not results:
+                    logger.warning(f"No results found. Consider these search tips:")
+                    logger.warning(f"1. Use shorter, more specific queries")
+                    logger.warning(f"2. For repositories, try adding 'stars:>100' or 'language:python'")
+                    logger.warning(f"3. For contribution opportunities, search for 'good-first-issue' or 'help-wanted'")
+            else:
+                logger.error(f"GitHub API error: {response.status_code} - {response.text}")
+            
+        except Exception as e:
+            logger.error(f"Error searching GitHub: {e}")
+        
+        return results
+    
+    def _get_readme_content(self, repo_full_name: str) -> str:
+        """
+        Get README content for a repository.
+        
+        Args:
+            repo_full_name: Full name of the repository (owner/repo)
+            
+        Returns:
+            Decoded README content or empty string if not found
+        """
+        try:
+            # Get README
+            response = requests.get(
+                f"{self.api_base}/repos/{repo_full_name}/readme",
+                headers=self.headers
+            )
+            
+            # Check for rate limiting
+            self._handle_rate_limits(response)
+            
+            if response.status_code == 200:
+                data = response.json()
+                content = data.get("content", "")
+                encoding = data.get("encoding", "")
+                
+                if encoding == "base64" and content:
+                    return base64.b64decode(content).decode('utf-8', errors='replace')
+                return content
+            else:
+                logger.warning(f"Could not get README for {repo_full_name}: {response.status_code}")
+                return ""
+                
+        except Exception as e:
+            logger.error(f"Error getting README for {repo_full_name}: {e}")
+            return ""
+    
+    def _get_recent_issues(self, repo_full_name: str, limit: int = 5) -> List[Dict[str, Any]]:
+        """
+        Get recent issues for a repository.
+        
+        Args:
+            repo_full_name: Full name of the repository (owner/repo)
+            limit: Maximum number of issues to return
+            
+        Returns:
+            List of recent issues
+        """
+        issues = []
+        
+        try:
+            # Get recent issues
+            response = requests.get(
+                f"{self.api_base}/repos/{repo_full_name}/issues",
+                headers=self.headers,
+                params={
+                    "state": "all",
+                    "per_page": limit,
+                    "sort": "updated",
+                    "direction": "desc"
+                }
+            )
+            
+            # Check for rate limiting
+            self._handle_rate_limits(response)
+            
+            if response.status_code == 200:
+                issues = response.json()
+                logger.info(f"Got {len(issues)} recent issues for {repo_full_name}")
+            else:
+                logger.warning(f"Could not get issues for {repo_full_name}: {response.status_code}")
+                
+        except Exception as e:
+            logger.error(f"Error getting issues for {repo_full_name}: {e}")
+        
+        return issues
+    
+    def _get_file_content(self, file_url: str) -> str:
+        """
+        Get content of a file from GitHub.
+        
+        Args:
+            file_url: API URL for the file
+            
+        Returns:
+            Decoded file content or empty string if not found
+        """
+        try:
+            # Get file content
+            response = requests.get(
+                file_url,
+                headers=self.headers
+            )
+            
+            # Check for rate limiting
+            self._handle_rate_limits(response)
+            
+            if response.status_code == 200:
+                data = response.json()
+                content = data.get("content", "")
+                encoding = data.get("encoding", "")
+                
+                if encoding == "base64" and content:
+                    return base64.b64decode(content).decode('utf-8', errors='replace')
+                return content
+            else:
+                logger.warning(f"Could not get file content: {response.status_code}")
+                return ""
+                
+        except Exception as e:
+            logger.error(f"Error getting file content: {e}")
+            return ""
+    
+    def _format_repository_preview(self, repo: Dict[str, Any]) -> Dict[str, Any]:
+        """Format repository search result as preview"""
+        return {
+            "id": str(repo.get("id", "")),
+            "title": repo.get("full_name", ""),
+            "link": repo.get("html_url", ""),
+            "snippet": repo.get("description", "No description provided"),
+            "stars": repo.get("stargazers_count", 0),
+            "forks": repo.get("forks_count", 0),
+            "language": repo.get("language", ""),
+            "updated_at": repo.get("updated_at", ""),
+            "created_at": repo.get("created_at", ""),
+            "topics": repo.get("topics", []),
+            "owner": repo.get("owner", {}).get("login", ""),
+            "is_fork": repo.get("fork", False),
+            "search_type": "repository",
+            "repo_full_name": repo.get("full_name", "")
+        }
+    
+    def _format_code_preview(self, code: Dict[str, Any]) -> Dict[str, Any]:
+        """Format code search result as preview"""
+        repo = code.get("repository", {})
+        return {
+            "id": f"code_{code.get('sha', '')}",
+            "title": f"{code.get('name', '')} in {repo.get('full_name', '')}",
+            "link": code.get("html_url", ""),
+            "snippet": f"Match in {code.get('path', '')}",
+            "path": code.get("path", ""),
+            "repo_name": repo.get("full_name", ""),
+            "repo_url": repo.get("html_url", ""),
+            "search_type": "code",
+            "file_url": code.get("url", "")
+        }
+    
+    def _format_issue_preview(self, issue: Dict[str, Any]) -> Dict[str, Any]:
+        """Format issue search result as preview"""
+        repo = issue.get("repository", {}) if "repository" in issue else {"full_name": ""}
+        return {
+            "id": f"issue_{issue.get('number', '')}",
+            "title": issue.get("title", ""),
+            "link": issue.get("html_url", ""),
+            "snippet": issue.get("body", "")[:200] + "..." if len(issue.get("body", "")) > 200 else issue.get("body", ""),
+            "state": issue.get("state", ""),
+            "created_at": issue.get("created_at", ""),
+            "updated_at": issue.get("updated_at", ""),
+            "user": issue.get("user", {}).get("login", ""),
+            "comments": issue.get("comments", 0),
+            "search_type": "issue",
+            "repo_name": repo.get("full_name", "")
+        }
+    
+    def _format_user_preview(self, user: Dict[str, Any]) -> Dict[str, Any]:
+        """Format user search result as preview"""
+        return {
+            "id": f"user_{user.get('id', '')}",
+            "title": user.get("login", ""),
+            "link": user.get("html_url", ""),
+            "snippet": user.get("bio", "No bio provided"),
+            "name": user.get("name", ""),
+            "followers": user.get("followers", 0),
+            "public_repos": user.get("public_repos", 0),
+            "location": user.get("location", ""),
+            "search_type": "user",
+            "user_login": user.get("login", "")
+        }
+    
+    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Get preview information for GitHub search results.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of preview dictionaries
+        """
+        logger.info(f"Getting GitHub previews for query: {query}")
+        
+        # For contribution-focused queries, automatically adjust search type and add filters
+        if any(term in query.lower() for term in ["contribute", "contributing", "contribution", "beginner", "newcomer"]):
+            # Use repositories search with help-wanted or good-first-issue labels
+            original_search_type = self.search_type
+            self.search_type = "repositories"
+            self.search_endpoint = f"{self.api_base}/search/repositories"
+            
+            # Create a specialized query for finding beginner-friendly projects
+            specialized_query = "good-first-issues:>5 is:public archived:false"
+            
+            # Extract language preferences if present
+            languages = []
+            for lang in ["python", "javascript", "java", "rust", "go", "typescript", "c#", "c++", "ruby"]:
+                if lang in query.lower():
+                    languages.append(lang)
+            
+            if languages:
+                specialized_query += f" language:{' language:'.join(languages)}"
+            
+            # Extract keywords
+            keywords = [word for word in query.split() if len(word) > 3 and word.lower() not in 
+                       ["recommend", "recommended", "github", "repositories", "looking", 
+                        "developers", "contribute", "contributing", "beginner", "newcomer"]]
+            
+            if keywords:
+                specialized_query += " " + " ".join(keywords[:5])  # Add up to 5 keywords
+                
+            logger.info(f"Using specialized contribution query: {specialized_query}")
+            
+            # Perform GitHub search with specialized query
+            results = self._search_github(specialized_query)
+            
+            # Restore original search type
+            self.search_type = original_search_type
+            self.search_endpoint = f"{self.api_base}/search/{self.search_type}"
+        else:
+            # Perform standard GitHub search
+            results = self._search_github(query)
+        
+        if not results:
+            logger.warning(f"No GitHub results found for query: {query}")
+            return []
+        
+        # Format results as previews
+        previews = []
+        for result in results:
+            # Format based on search type
+            if self.search_type == "repositories":
+                preview = self._format_repository_preview(result)
+            elif self.search_type == "code":
+                preview = self._format_code_preview(result)
+            elif self.search_type == "issues":
+                preview = self._format_issue_preview(result)
+            elif self.search_type == "users":
+                preview = self._format_user_preview(result)
+            else:
+                logger.warning(f"Unknown search type: {self.search_type}")
+                continue
+            
+            previews.append(preview)
+        
+        logger.info(f"Formatted {len(previews)} GitHub preview results")
+        return previews
+    
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Get full content for the relevant GitHub search results.
+        
+        Args:
+            relevant_items: List of relevant preview dictionaries
+            
+        Returns:
+            List of result dictionaries with full content
+        """
+        # Check if we should add full content
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            logger.info("Snippet-only mode, skipping full content retrieval")
+            return relevant_items
+        
+        logger.info(f"Getting full content for {len(relevant_items)} GitHub results")
+        
+        results = []
+        for item in relevant_items:
+            result = item.copy()
+            search_type = item.get("search_type", "")
+            
+            # Add content based on search type
+            if search_type == "repository" and self.include_readme:
+                repo_full_name = item.get("repo_full_name", "")
+                if repo_full_name:
+                    # Get README content
+                    readme_content = self._get_readme_content(repo_full_name)
+                    result["full_content"] = readme_content
+                    result["content_type"] = "readme"
+                    
+                    # Get recent issues if requested
+                    if self.include_issues:
+                        issues = self._get_recent_issues(repo_full_name)
+                        result["recent_issues"] = issues
+                
+            elif search_type == "code":
+                file_url = item.get("file_url", "")
+                if file_url:
+                    # Get file content
+                    file_content = self._get_file_content(file_url)
+                    result["full_content"] = file_content
+                    result["content_type"] = "file"
+            
+            elif search_type == "issue":
+                # For issues, the snippet usually contains a summary already
+                # We'll just keep it as is
+                result["full_content"] = item.get("snippet", "")
+                result["content_type"] = "issue"
+            
+            elif search_type == "user":
+                # For users, construct a profile summary
+                user_login = item.get("user_login", "")
+                profile_summary = f"GitHub user: {item.get('title', '')}\n"
+                
+                if item.get("name"):
+                    profile_summary += f"Name: {item.get('name')}\n"
+                    
+                if item.get("location"):
+                    profile_summary += f"Location: {item.get('location')}\n"
+                    
+                profile_summary += f"Followers: {item.get('followers', 0)}\n"
+                profile_summary += f"Public repositories: {item.get('public_repos', 0)}\n"
+                
+                if item.get("snippet") and item.get("snippet") != "No bio provided":
+                    profile_summary += f"\nBio: {item.get('snippet')}\n"
+                
+                result["full_content"] = profile_summary
+                result["content_type"] = "user_profile"
+            
+            results.append(result)
+        
+        return results
+    
+    def search_repository(self, repo_owner: str, repo_name: str) -> Dict[str, Any]:
+        """
+        Get detailed information about a specific repository.
+        
+        Args:
+            repo_owner: Owner of the repository
+            repo_name: Name of the repository
+            
+        Returns:
+            Dictionary with repository information
+        """
+        repo_full_name = f"{repo_owner}/{repo_name}"
+        logger.info(f"Getting details for repository: {repo_full_name}")
+        
+        try:
+            # Get repository details
+            response = requests.get(
+                f"{self.api_base}/repos/{repo_full_name}",
+                headers=self.headers
+            )
+            
+            # Check for rate limiting
+            self._handle_rate_limits(response)
+            
+            if response.status_code == 200:
+                repo = response.json()
+                
+                # Format as repository preview
+                result = self._format_repository_preview(repo)
+                
+                # Add README content if requested
+                if self.include_readme:
+                    readme_content = self._get_readme_content(repo_full_name)
+                    result["full_content"] = readme_content
+                    result["content_type"] = "readme"
+                
+                # Add recent issues if requested
+                if self.include_issues:
+                    issues = self._get_recent_issues(repo_full_name)
+                    result["recent_issues"] = issues
+                
+                return result
+            else:
+                logger.error(f"Error getting repository details: {response.status_code} - {response.text}")
+                return {}
+                
+        except Exception as e:
+            logger.error(f"Error getting repository details: {e}")
+            return {}
+    
+    def search_code(self, query: str, language: Optional[str] = None, user: Optional[str] = None) -> List[Dict[str, Any]]:
+        """
+        Search for code with more specific parameters.
+        
+        Args:
+            query: Code search query
+            language: Filter by programming language
+            user: Filter by GitHub username/organization
+            
+        Returns:
+            List of code search results
+        """
+        # Build advanced query
+        advanced_query = query
+        
+        if language:
+            advanced_query += f" language:{language}"
+            
+        if user:
+            advanced_query += f" user:{user}"
+        
+        # Save current search type
+        original_search_type = self.search_type
+        
+        try:
+            # Set search type to code
+            self.search_type = "code"
+            self.search_endpoint = f"{self.api_base}/search/code"
+            
+            # Perform search
+            results = self._search_github(advanced_query)
+            
+            # Format results
+            previews = [self._format_code_preview(result) for result in results]
+            
+            # Get full content if requested
+            if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and not config.SEARCH_SNIPPETS_ONLY:
+                return self._get_full_content(previews)
+            
+            return previews
+            
+        finally:
+            # Restore original search type
+            self.search_type = original_search_type
+            self.search_endpoint = f"{self.api_base}/search/{self.search_type}"
+    
+    def search_issues(self, query: str, state: str = "open", sort: str = "updated") -> List[Dict[str, Any]]:
+        """
+        Search for issues with more specific parameters.
+        
+        Args:
+            query: Issue search query
+            state: Filter by issue state ("open", "closed", "all")
+            sort: Sort order ("updated", "created", "comments")
+            
+        Returns:
+            List of issue search results
+        """
+        # Build advanced query
+        advanced_query = query + f" state:{state}"
+        
+        # Save current search type
+        original_search_type = self.search_type
+        
+        try:
+            # Set search type to issues
+            self.search_type = "issues"
+            self.search_endpoint = f"{self.api_base}/search/issues"
+            
+            # Set sort parameter
+            params = {
+                "q": advanced_query,
+                "per_page": min(self.max_results, 100),
+                "page": 1,
+                "sort": sort,
+                "order": "desc"
+            }
+            
+            # Perform search
+            response = requests.get(
+                self.search_endpoint,
+                headers=self.headers,
+                params=params
+            )
+            
+            # Check for rate limiting
+            self._handle_rate_limits(response)
+            
+            if response.status_code == 200:
+                data = response.json()
+                results = data.get("items", [])
+                
+                # Format results
+                previews = [self._format_issue_preview(result) for result in results]
+                
+                # For issues, we don't need to get full content
+                return previews
+            else:
+                logger.error(f"GitHub API error: {response.status_code} - {response.text}")
+                return []
+                
+        finally:
+            # Restore original search type
+            self.search_type = original_search_type
+            self.search_endpoint = f"{self.api_base}/search/{self.search_type}"
+    
+    def set_search_type(self, search_type: str):
+        """
+        Set the search type for subsequent searches.
+        
+        Args:
+            search_type: Type of GitHub search ("repositories", "code", "issues", "users")
+        """
+        if search_type in ["repositories", "code", "issues", "users"]:
+            self.search_type = search_type
+            self.search_endpoint = f"{self.api_base}/search/{search_type}"
+            logger.info(f"Set GitHub search type to: {search_type}")
+        else:
+            logger.error(f"Invalid GitHub search type: {search_type}")