local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_github.py

@@ -1,35 +1,40 @@
-import requests
-import logging
 import base64
+import json
+import logging
 import os
 import time
-from typing import Dict, List, Any, Optional, Union
+from typing import Any, Dict, List, Optional
+
+import requests
 from langchain_core.language_models import BaseLLM
 
-from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
-from local_deep_research import config
+from ...config import llm_config, search_config
+from ..search_engine_base import BaseSearchEngine
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+
 class GitHubSearchEngine(BaseSearchEngine):
     """
     GitHub search engine implementation.
     Provides search across GitHub repositories, code, issues, and users.
     """
-    
-    def __init__(self, 
-                max_results: int = 15, 
-                api_key: Optional[str] = None,
-                search_type: str = "repositories",
-                include_readme: bool = True,
-                include_issues: bool = False,
-                llm: Optional[BaseLLM] = None,
-                max_filtered_results: Optional[int] = None):
+
+    def __init__(
+        self,
+        max_results: int = 15,
+        api_key: Optional[str] = None,
+        search_type: str = "repositories",
+        include_readme: bool = True,
+        include_issues: bool = False,
+        llm: Optional[BaseLLM] = None,
+        max_filtered_results: Optional[int] = None,
+    ):
         """
         Initialize the GitHub search engine.
-        
+
         Args:
             max_results: Maximum number of search results
             api_key: GitHub API token (can also be set in GITHUB_API_KEY env)
@@ -40,78 +45,144 @@ class GitHubSearchEngine(BaseSearchEngine):
             max_filtered_results: Maximum number of results to keep after filtering
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
+        super().__init__(
+            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+        )
         self.api_key = api_key or os.getenv("GITHUB_API_KEY")
         self.search_type = search_type
         self.include_readme = include_readme
         self.include_issues = include_issues
-        
+
         # API endpoints
         self.api_base = "https://api.github.com"
         self.search_endpoint = f"{self.api_base}/search/{search_type}"
-        
+
         # Set up API headers
         self.headers = {
             "Accept": "application/vnd.github.v3+json",
-            "User-Agent": "Local-Deep-Research-Agent"
+            "User-Agent": "Local-Deep-Research-Agent",
         }
-        
+
         # Add authentication if API key provided
         if self.api_key:
             self.headers["Authorization"] = f"token {self.api_key}"
             logger.info("Using authenticated GitHub API requests")
         else:
-            logger.warning("No GitHub API key provided. Rate limits will be restricted.")
-    
+            logger.warning(
+                "No GitHub API key provided. Rate limits will be restricted."
+            )
+
     def _handle_rate_limits(self, response):
         """Handle GitHub API rate limits by logging warnings and sleeping if necessary"""
         remaining = int(response.headers.get("X-RateLimit-Remaining", 60))
         reset_time = int(response.headers.get("X-RateLimit-Reset", 0))
-        
+
         if remaining < 5:
             current_time = time.time()
             wait_time = max(reset_time - current_time, 0)
-            logger.warning(f"GitHub API rate limit almost reached. {remaining} requests remaining.")
-            
+            logger.warning(
+                f"GitHub API rate limit almost reached. {remaining} requests remaining."
+            )
+
             if wait_time > 0 and remaining == 0:
-                logger.warning(f"GitHub API rate limit exceeded. Waiting {wait_time:.0f} seconds.")
+                logger.warning(
+                    f"GitHub API rate limit exceeded. Waiting {wait_time:.0f} seconds."
+                )
                 time.sleep(min(wait_time, 60))  # Wait at most 60 seconds
-    
+
+    def _optimize_github_query(self, query: str) -> str:
+        """
+        Optimize the GitHub search query using LLM to improve search results.
+
+        Args:
+            query: Original search query
+
+        Returns:
+            Optimized GitHub search query
+        """
+        # Get LLM from config if not already set
+        if not self.llm:
+            try:
+                self.llm = llm_config.get_llm()
+                if not self.llm:
+                    logger.warning("No LLM available for query optimization")
+                    return query
+            except Exception as e:
+                logger.error(f"Error getting LLM from config: {e}")
+                return query
+
+        prompt = f"""Transform this GitHub search query into an optimized version for the GitHub search API. Follow these steps:
+                        1. Strip question words (e.g., 'what', 'are', 'is'), stop words (e.g., 'and', 'as', 'of', 'on'), and redundant terms (e.g., 'repositories', 'repos', 'github') since they're implied by the search context.
+                        2. Keep only domain-specific keywords and avoid using "-related" terms.
+                        3. Add GitHub-specific filters with dynamic thresholds based on query context:
+                           - For stars: Use higher threshold (e.g., 'stars:>1000') for mainstream topics, lower (e.g., 'stars:>50') for specialized topics
+                           - For language: Detect programming language from query or omit if unclear
+                           - For search scope: Use 'in:name,description,readme' for general queries, 'in:file' for code-specific queries
+                        4. For date ranges, adapt based on query context:
+                           - For emerging: Use 'created:>2024-01-01'
+                           - For mature: Use 'pushed:>2023-01-01'
+                           - For historical research: Use 'created:2020-01-01..2024-01-01'
+                        5. For excluding results, adapt based on query:
+                           - Exclude irrelevant languages based on context
+                           - Use 'NOT' to exclude competing terms
+                        6. Ensure the output is a concise, space-separated string with no punctuation or extra text beyond keywords and filters.
+
+
+                        Original query: "{query}"
+
+                        Return ONLY the optimized query, ready for GitHub's search API. Do not include explanations or additional text."""
+
+        try:
+            response = self.llm.invoke(prompt)
+
+            # Handle different response formats (string or object with content attribute)
+            if hasattr(response, "content"):
+                optimized_query = response.content.strip()
+            else:
+                # Handle string responses
+                optimized_query = str(response).strip()
+
+            # Validate the optimized query
+            if optimized_query and len(optimized_query) > 0:
+                logger.info(
+                    f"LLM optimized query from '{query}' to '{optimized_query}'"
+                )
+                return optimized_query
+            else:
+                logger.warning("LLM returned empty query, using original")
+                return query
+
+        except Exception as e:
+            logger.error(f"Error optimizing query with LLM: {e}")
+            return query
+
     def _search_github(self, query: str) -> List[Dict[str, Any]]:
         """
         Perform a GitHub search based on the configured search type.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of GitHub search result items
         """
         results = []
-        
+
         try:
-            # Optimize GitHub query format
-            github_query = query
-            
-            # For long queries, focus on keywords and add filters for better results
-            if len(query) > 80:
-                # Extract key terms if it's a recommendation request
-                if "recommend" in query.lower() or "looking for" in query.lower():
-                    github_query = "stars:>100 " + " ".join([
-                        word for word in query.split() 
-                        if len(word) > 3 and word.lower() not in 
-                        ["recommend", "recommended", "github", "repositories", "looking", "developers"]
-                    ])
-            
-            logger.info(f"Optimized GitHub query: {github_query}")
-            
+            # Optimize GitHub query using LLM
+            github_query = self._optimize_github_query(query)
+
+            logger.info(f"Final GitHub query: {github_query}")
+
             # Construct search parameters
             params = {
                 "q": github_query,
-                "per_page": min(self.max_results, 100),  # GitHub API max is 100 per page
-                "page": 1
+                "per_page": min(
+                    self.max_results, 100
+                ),  # GitHub API max is 100 per page
+                "page": 1,
             }
-            
+
             # Add sort parameters based on search type
             if self.search_type == "repositories":
                 params["sort"] = "stars"
@@ -125,92 +196,104 @@ class GitHubSearchEngine(BaseSearchEngine):
             elif self.search_type == "users":
                 params["sort"] = "followers"
                 params["order"] = "desc"
-            
+
             # Execute the API request
             response = requests.get(
-                self.search_endpoint,
-                headers=self.headers,
-                params=params
+                self.search_endpoint, headers=self.headers, params=params
             )
-            
+
             # Check for rate limiting
             self._handle_rate_limits(response)
-            
+
             # Handle response with detailed logging
             if response.status_code == 200:
                 data = response.json()
                 total_count = data.get("total_count", 0)
                 results = data.get("items", [])
-                logger.info(f"GitHub search returned {len(results)} results (total available: {total_count})")
-                
+                logger.info(
+                    f"GitHub search returned {len(results)} results (total available: {total_count})"
+                )
+
                 # Log the rate limit information
-                rate_limit_remaining = response.headers.get("X-RateLimit-Remaining", "unknown")
-                rate_limit_reset = response.headers.get("X-RateLimit-Reset", "unknown")
-                logger.info(f"GitHub API rate limit: {rate_limit_remaining} requests remaining")
-                
+                rate_limit_remaining = response.headers.get(
+                    "X-RateLimit-Remaining", "unknown"
+                )
+                logger.info(
+                    f"GitHub API rate limit: {rate_limit_remaining} requests remaining"
+                )
+
                 # If no results, try to provide more guidance
                 if not results:
-                    logger.warning(f"No results found. Consider these search tips:")
-                    logger.warning(f"1. Use shorter, more specific queries")
-                    logger.warning(f"2. For repositories, try adding 'stars:>100' or 'language:python'")
-                    logger.warning(f"3. For contribution opportunities, search for 'good-first-issue' or 'help-wanted'")
+                    logger.warning("No results found. Consider these search tips:")
+                    logger.warning("1. Use shorter, more specific queries")
+                    logger.warning(
+                        "2. For repositories, try adding 'stars:>100' or 'language:python'"
+                    )
+                    logger.warning(
+                        "3. For contribution opportunities, search for 'good-first-issue' or 'help-wanted'"
+                    )
             else:
-                logger.error(f"GitHub API error: {response.status_code} - {response.text}")
-            
+                logger.error(
+                    f"GitHub API error: {response.status_code} - {response.text}"
+                )
+
         except Exception as e:
             logger.error(f"Error searching GitHub: {e}")
-        
+
         return results
-    
+
     def _get_readme_content(self, repo_full_name: str) -> str:
         """
         Get README content for a repository.
-        
+
         Args:
             repo_full_name: Full name of the repository (owner/repo)
-            
+
         Returns:
             Decoded README content or empty string if not found
         """
         try:
             # Get README
             response = requests.get(
-                f"{self.api_base}/repos/{repo_full_name}/readme",
-                headers=self.headers
+                f"{self.api_base}/repos/{repo_full_name}/readme", headers=self.headers
             )
-            
+
             # Check for rate limiting
             self._handle_rate_limits(response)
-            
+
             if response.status_code == 200:
                 data = response.json()
                 content = data.get("content", "")
                 encoding = data.get("encoding", "")
-                
+
                 if encoding == "base64" and content:
-                    return base64.b64decode(content).decode('utf-8', errors='replace')
+                    return base64.b64decode(content).decode("utf-8", errors="replace")
                 return content
             else:
-                logger.warning(f"Could not get README for {repo_full_name}: {response.status_code}")
+                logger.warning(
+                    f"Could not get README for {repo_full_name}: {response.status_code}"
+                )
                 return ""
-                
+
         except Exception as e:
             logger.error(f"Error getting README for {repo_full_name}: {e}")
             return ""
-    
-    def _get_recent_issues(self, repo_full_name: str, limit: int = 5) -> List[Dict[str, Any]]:
+
+    def _get_recent_issues(
+        self, repo_full_name: str, limit: int = 5
+    ) -> List[Dict[str, Any]]:
         """
         Get recent issues for a repository.
-        
+
         Args:
             repo_full_name: Full name of the repository (owner/repo)
             limit: Maximum number of issues to return
-            
+
         Returns:
             List of recent issues
         """
         issues = []
-        
+
         try:
             # Get recent issues
             response = requests.get(
@@ -220,60 +303,59 @@ class GitHubSearchEngine(BaseSearchEngine):
                     "state": "all",
                     "per_page": limit,
                     "sort": "updated",
-                    "direction": "desc"
-                }
+                    "direction": "desc",
+                },
             )
-            
+
             # Check for rate limiting
             self._handle_rate_limits(response)
-            
+
             if response.status_code == 200:
                 issues = response.json()
                 logger.info(f"Got {len(issues)} recent issues for {repo_full_name}")
             else:
-                logger.warning(f"Could not get issues for {repo_full_name}: {response.status_code}")
-                
+                logger.warning(
+                    f"Could not get issues for {repo_full_name}: {response.status_code}"
+                )
+
         except Exception as e:
             logger.error(f"Error getting issues for {repo_full_name}: {e}")
-        
+
         return issues
-    
+
     def _get_file_content(self, file_url: str) -> str:
         """
         Get content of a file from GitHub.
-        
+
         Args:
             file_url: API URL for the file
-            
+
         Returns:
             Decoded file content or empty string if not found
         """
         try:
             # Get file content
-            response = requests.get(
-                file_url,
-                headers=self.headers
-            )
-            
+            response = requests.get(file_url, headers=self.headers)
+
             # Check for rate limiting
             self._handle_rate_limits(response)
-            
+
             if response.status_code == 200:
                 data = response.json()
                 content = data.get("content", "")
                 encoding = data.get("encoding", "")
-                
+
                 if encoding == "base64" and content:
-                    return base64.b64decode(content).decode('utf-8', errors='replace')
+                    return base64.b64decode(content).decode("utf-8", errors="replace")
                 return content
             else:
                 logger.warning(f"Could not get file content: {response.status_code}")
                 return ""
-                
+
         except Exception as e:
             logger.error(f"Error getting file content: {e}")
             return ""
-    
+
     def _format_repository_preview(self, repo: Dict[str, Any]) -> Dict[str, Any]:
         """Format repository search result as preview"""
         return {
@@ -290,9 +372,9 @@ class GitHubSearchEngine(BaseSearchEngine):
             "owner": repo.get("owner", {}).get("login", ""),
             "is_fork": repo.get("fork", False),
             "search_type": "repository",
-            "repo_full_name": repo.get("full_name", "")
+            "repo_full_name": repo.get("full_name", ""),
         }
-    
+
     def _format_code_preview(self, code: Dict[str, Any]) -> Dict[str, Any]:
         """Format code search result as preview"""
         repo = code.get("repository", {})
@@ -305,26 +387,32 @@ class GitHubSearchEngine(BaseSearchEngine):
             "repo_name": repo.get("full_name", ""),
             "repo_url": repo.get("html_url", ""),
             "search_type": "code",
-            "file_url": code.get("url", "")
+            "file_url": code.get("url", ""),
         }
-    
+
     def _format_issue_preview(self, issue: Dict[str, Any]) -> Dict[str, Any]:
         """Format issue search result as preview"""
-        repo = issue.get("repository", {}) if "repository" in issue else {"full_name": ""}
+        repo = (
+            issue.get("repository", {}) if "repository" in issue else {"full_name": ""}
+        )
         return {
             "id": f"issue_{issue.get('number', '')}",
             "title": issue.get("title", ""),
             "link": issue.get("html_url", ""),
-            "snippet": issue.get("body", "")[:200] + "..." if len(issue.get("body", "")) > 200 else issue.get("body", ""),
+            "snippet": (
+                issue.get("body", "")[:200] + "..."
+                if len(issue.get("body", "")) > 200
+                else issue.get("body", "")
+            ),
             "state": issue.get("state", ""),
             "created_at": issue.get("created_at", ""),
             "updated_at": issue.get("updated_at", ""),
             "user": issue.get("user", {}).get("login", ""),
             "comments": issue.get("comments", 0),
             "search_type": "issue",
-            "repo_name": repo.get("full_name", "")
+            "repo_name": repo.get("full_name", ""),
         }
-    
+
     def _format_user_preview(self, user: Dict[str, Any]) -> Dict[str, Any]:
         """Format user search result as preview"""
         return {
@@ -337,64 +425,100 @@ class GitHubSearchEngine(BaseSearchEngine):
             "public_repos": user.get("public_repos", 0),
             "location": user.get("location", ""),
             "search_type": "user",
-            "user_login": user.get("login", "")
+            "user_login": user.get("login", ""),
         }
-    
+
     def _get_previews(self, query: str) -> List[Dict[str, Any]]:
         """
         Get preview information for GitHub search results.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of preview dictionaries
         """
         logger.info(f"Getting GitHub previews for query: {query}")
-        
+
         # For contribution-focused queries, automatically adjust search type and add filters
-        if any(term in query.lower() for term in ["contribute", "contributing", "contribution", "beginner", "newcomer"]):
+        if any(
+            term in query.lower()
+            for term in [
+                "contribute",
+                "contributing",
+                "contribution",
+                "beginner",
+                "newcomer",
+            ]
+        ):
             # Use repositories search with help-wanted or good-first-issue labels
             original_search_type = self.search_type
             self.search_type = "repositories"
             self.search_endpoint = f"{self.api_base}/search/repositories"
-            
+
             # Create a specialized query for finding beginner-friendly projects
             specialized_query = "good-first-issues:>5 is:public archived:false"
-            
+
             # Extract language preferences if present
             languages = []
-            for lang in ["python", "javascript", "java", "rust", "go", "typescript", "c#", "c++", "ruby"]:
+            for lang in [
+                "python",
+                "javascript",
+                "java",
+                "rust",
+                "go",
+                "typescript",
+                "c#",
+                "c++",
+                "ruby",
+            ]:
                 if lang in query.lower():
                     languages.append(lang)
-            
+
             if languages:
                 specialized_query += f" language:{' language:'.join(languages)}"
-            
+
             # Extract keywords
-            keywords = [word for word in query.split() if len(word) > 3 and word.lower() not in 
-                       ["recommend", "recommended", "github", "repositories", "looking", 
-                        "developers", "contribute", "contributing", "beginner", "newcomer"]]
-            
+            keywords = [
+                word
+                for word in query.split()
+                if len(word) > 3
+                and word.lower()
+                not in [
+                    "recommend",
+                    "recommended",
+                    "github",
+                    "repositories",
+                    "looking",
+                    "developers",
+                    "contribute",
+                    "contributing",
+                    "beginner",
+                    "newcomer",
+                ]
+            ]
+
             if keywords:
-                specialized_query += " " + " ".join(keywords[:5])  # Add up to 5 keywords
-                
+                specialized_query += " " + " ".join(
+                    keywords[:5]
+                )  # Add up to 5 keywords
+
             logger.info(f"Using specialized contribution query: {specialized_query}")
-            
+
             # Perform GitHub search with specialized query
             results = self._search_github(specialized_query)
-            
+
             # Restore original search type
             self.search_type = original_search_type
             self.search_endpoint = f"{self.api_base}/search/{self.search_type}"
         else:
             # Perform standard GitHub search
             results = self._search_github(query)
-        
+
         if not results:
             logger.warning(f"No GitHub results found for query: {query}")
             return []
-        
+
         # Format results as previews
         previews = []
         for result in results:
@@ -410,34 +534,39 @@ class GitHubSearchEngine(BaseSearchEngine):
             else:
                 logger.warning(f"Unknown search type: {self.search_type}")
                 continue
-            
+
             previews.append(preview)
-        
+
         logger.info(f"Formatted {len(previews)} GitHub preview results")
         return previews
-    
-    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+
+    def _get_full_content(
+        self, relevant_items: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         """
         Get full content for the relevant GitHub search results.
-        
+
         Args:
             relevant_items: List of relevant preview dictionaries
-            
+
         Returns:
             List of result dictionaries with full content
         """
         # Check if we should add full content
-        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+        if (
+            hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
+            and search_config.SEARCH_SNIPPETS_ONLY
+        ):
             logger.info("Snippet-only mode, skipping full content retrieval")
             return relevant_items
-        
+
         logger.info(f"Getting full content for {len(relevant_items)} GitHub results")
-        
+
         results = []
         for item in relevant_items:
             result = item.copy()
             search_type = item.get("search_type", "")
-            
+
             # Add content based on search type
             if search_type == "repository" and self.include_readme:
                 repo_full_name = item.get("repo_full_name", "")
@@ -446,12 +575,12 @@ class GitHubSearchEngine(BaseSearchEngine):
                     readme_content = self._get_readme_content(repo_full_name)
                     result["full_content"] = readme_content
                     result["content_type"] = "readme"
-                    
+
                     # Get recent issues if requested
                     if self.include_issues:
                         issues = self._get_recent_issues(repo_full_name)
                         result["recent_issues"] = issues
-                
+
             elif search_type == "code":
                 file_url = item.get("file_url", "")
                 if file_url:
@@ -459,197 +588,206 @@ class GitHubSearchEngine(BaseSearchEngine):
                     file_content = self._get_file_content(file_url)
                     result["full_content"] = file_content
                     result["content_type"] = "file"
-            
+
             elif search_type == "issue":
                 # For issues, the snippet usually contains a summary already
                 # We'll just keep it as is
                 result["full_content"] = item.get("snippet", "")
                 result["content_type"] = "issue"
-            
+
             elif search_type == "user":
                 # For users, construct a profile summary
-                user_login = item.get("user_login", "")
                 profile_summary = f"GitHub user: {item.get('title', '')}\n"
-                
+
                 if item.get("name"):
                     profile_summary += f"Name: {item.get('name')}\n"
-                    
+
                 if item.get("location"):
                     profile_summary += f"Location: {item.get('location')}\n"
-                    
+
                 profile_summary += f"Followers: {item.get('followers', 0)}\n"
-                profile_summary += f"Public repositories: {item.get('public_repos', 0)}\n"
-                
+                profile_summary += (
+                    f"Public repositories: {item.get('public_repos', 0)}\n"
+                )
+
                 if item.get("snippet") and item.get("snippet") != "No bio provided":
                     profile_summary += f"\nBio: {item.get('snippet')}\n"
-                
+
                 result["full_content"] = profile_summary
                 result["content_type"] = "user_profile"
-            
+
             results.append(result)
-        
+
         return results
-    
+
     def search_repository(self, repo_owner: str, repo_name: str) -> Dict[str, Any]:
         """
         Get detailed information about a specific repository.
-        
+
         Args:
             repo_owner: Owner of the repository
             repo_name: Name of the repository
-            
+
         Returns:
             Dictionary with repository information
         """
         repo_full_name = f"{repo_owner}/{repo_name}"
         logger.info(f"Getting details for repository: {repo_full_name}")
-        
+
         try:
             # Get repository details
             response = requests.get(
-                f"{self.api_base}/repos/{repo_full_name}",
-                headers=self.headers
+                f"{self.api_base}/repos/{repo_full_name}", headers=self.headers
             )
-            
+
             # Check for rate limiting
             self._handle_rate_limits(response)
-            
+
             if response.status_code == 200:
                 repo = response.json()
-                
+
                 # Format as repository preview
                 result = self._format_repository_preview(repo)
-                
+
                 # Add README content if requested
                 if self.include_readme:
                     readme_content = self._get_readme_content(repo_full_name)
                     result["full_content"] = readme_content
                     result["content_type"] = "readme"
-                
+
                 # Add recent issues if requested
                 if self.include_issues:
                     issues = self._get_recent_issues(repo_full_name)
                     result["recent_issues"] = issues
-                
+
                 return result
             else:
-                logger.error(f"Error getting repository details: {response.status_code} - {response.text}")
+                logger.error(
+                    f"Error getting repository details: {response.status_code} - {response.text}"
+                )
                 return {}
-                
+
         except Exception as e:
             logger.error(f"Error getting repository details: {e}")
             return {}
-    
-    def search_code(self, query: str, language: Optional[str] = None, user: Optional[str] = None) -> List[Dict[str, Any]]:
+
+    def search_code(
+        self, query: str, language: Optional[str] = None, user: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
         """
         Search for code with more specific parameters.
-        
+
         Args:
             query: Code search query
             language: Filter by programming language
             user: Filter by GitHub username/organization
-            
+
         Returns:
             List of code search results
         """
         # Build advanced query
         advanced_query = query
-        
+
         if language:
             advanced_query += f" language:{language}"
-            
+
         if user:
             advanced_query += f" user:{user}"
-        
+
         # Save current search type
         original_search_type = self.search_type
-        
+
         try:
             # Set search type to code
             self.search_type = "code"
             self.search_endpoint = f"{self.api_base}/search/code"
-            
+
             # Perform search
             results = self._search_github(advanced_query)
-            
+
             # Format results
             previews = [self._format_code_preview(result) for result in results]
-            
+
             # Get full content if requested
-            if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and not config.SEARCH_SNIPPETS_ONLY:
+            if (
+                hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
+                and not search_config.SEARCH_SNIPPETS_ONLY
+            ):
                 return self._get_full_content(previews)
-            
+
             return previews
-            
+
         finally:
             # Restore original search type
             self.search_type = original_search_type
             self.search_endpoint = f"{self.api_base}/search/{self.search_type}"
-    
-    def search_issues(self, query: str, state: str = "open", sort: str = "updated") -> List[Dict[str, Any]]:
+
+    def search_issues(
+        self, query: str, state: str = "open", sort: str = "updated"
+    ) -> List[Dict[str, Any]]:
         """
         Search for issues with more specific parameters.
-        
+
         Args:
             query: Issue search query
             state: Filter by issue state ("open", "closed", "all")
             sort: Sort order ("updated", "created", "comments")
-            
+
         Returns:
             List of issue search results
         """
         # Build advanced query
         advanced_query = query + f" state:{state}"
-        
+
         # Save current search type
         original_search_type = self.search_type
-        
+
         try:
             # Set search type to issues
             self.search_type = "issues"
             self.search_endpoint = f"{self.api_base}/search/issues"
-            
+
             # Set sort parameter
             params = {
                 "q": advanced_query,
                 "per_page": min(self.max_results, 100),
                 "page": 1,
                 "sort": sort,
-                "order": "desc"
+                "order": "desc",
             }
-            
+
             # Perform search
             response = requests.get(
-                self.search_endpoint,
-                headers=self.headers,
-                params=params
+                self.search_endpoint, headers=self.headers, params=params
             )
-            
+
             # Check for rate limiting
             self._handle_rate_limits(response)
-            
+
             if response.status_code == 200:
                 data = response.json()
                 results = data.get("items", [])
-                
+
                 # Format results
                 previews = [self._format_issue_preview(result) for result in results]
-                
+
                 # For issues, we don't need to get full content
                 return previews
             else:
-                logger.error(f"GitHub API error: {response.status_code} - {response.text}")
+                logger.error(
+                    f"GitHub API error: {response.status_code} - {response.text}"
+                )
                 return []
-                
+
         finally:
             # Restore original search type
             self.search_type = original_search_type
             self.search_endpoint = f"{self.api_base}/search/{self.search_type}"
-    
+
     def set_search_type(self, search_type: str):
         """
         Set the search type for subsequent searches.
-        
+
         Args:
             search_type: Type of GitHub search ("repositories", "code", "issues", "users")
         """
@@ -658,4 +796,77 @@ class GitHubSearchEngine(BaseSearchEngine):
             self.search_endpoint = f"{self.api_base}/search/{search_type}"
             logger.info(f"Set GitHub search type to: {search_type}")
         else:
-            logger.error(f"Invalid GitHub search type: {search_type}")
+            logger.error(f"Invalid GitHub search type: {search_type}")
+
+    def _filter_for_relevance(
+        self, previews: List[Dict[str, Any]], query: str
+    ) -> List[Dict[str, Any]]:
+        """
+        Filter GitHub search results for relevance using LLM.
+
+        Args:
+            previews: List of preview dictionaries
+            query: Original search query
+
+        Returns:
+            List of relevant preview dictionaries
+        """
+        if not self.llm or not previews:
+            return previews
+
+        # Create a specialized prompt for GitHub results
+        prompt = f"""Analyze these GitHub search results and rank them by relevance to the query.
+Consider:
+1. Repository stars and activity (higher is better)
+2. Match between query intent and repository description
+3. Repository language and topics
+4. Last update time (more recent is better)
+5. Whether it's a fork (original repositories are preferred)
+
+Query: "{query}"
+
+Results:
+{json.dumps(previews, indent=2)}
+
+Return ONLY a JSON array of indices in order of relevance (most relevant first).
+Example: [0, 2, 1, 3]
+Do not include any other text or explanation."""
+
+        try:
+            response = self.llm.invoke(prompt)
+            response_text = response.content.strip()
+
+            # Extract JSON array from response
+            start_idx = response_text.find("[")
+            end_idx = response_text.rfind("]")
+
+            if start_idx >= 0 and end_idx > start_idx:
+                array_text = response_text[start_idx : end_idx + 1]
+                ranked_indices = json.loads(array_text)
+
+                # Return the results in ranked order
+                ranked_results = []
+                for idx in ranked_indices:
+                    if idx < len(previews):
+                        ranked_results.append(previews[idx])
+
+                # Limit to max_filtered_results if specified
+                if (
+                    self.max_filtered_results
+                    and len(ranked_results) > self.max_filtered_results
+                ):
+                    logger.info(
+                        f"Limiting filtered results to top {self.max_filtered_results}"
+                    )
+                    return ranked_results[: self.max_filtered_results]
+
+                return ranked_results
+            else:
+                logger.info(
+                    "Could not find JSON array in response, returning no previews"
+                )
+                return []
+
+        except Exception as e:
+            logger.error(f"Error filtering GitHub results: {e}")
+            return []