llama-github 0.1.0__py3-none-any.whl

llama_github/__init__.py

@@ -0,0 +1,3 @@
+from .version import __version__
+from .github_rag import GithubRAG
+from .logger import configure_logging

llama_github/config/config.json

@@ -0,0 +1,20 @@
+{
+    "general_prompt": "You are a highly intelligent assistant with expertise in GitHub repositories and coding practices. Your task is to analyze questions related to GitHub projects, coding issues, or programming concepts. Using your extensive knowledge base, you will provide detailed, accurate, and contextually relevant answers. You have the ability to understand complex coding queries, retrieve pertinent information from GitHub repositories, and augment this data with your advanced reasoning capabilities. Your responses should guide developers towards solving their problems, understanding new concepts, or finding the information they seek related to GitHub projects and software development.",
+    "always_answer_prompt": "**Instructions:**\nAs an advanced AI assistant with deep expertise in GitHub repositories, coding practices, and programming concepts, your primary goal is to provide concise, accurate, and contextually relevant answers to complex coding queries. When presented with a question, your first step is to analyze the query and generate a succinct abstraction that captures its core essence by using only one sentence, especially if the original question is lengthy or convoluted.\n\nNext, leverage your extensive knowledge base and reasoning capabilities to craft a coherent and informative response. If possible, enhance your answer with sample code snippets that demonstrate the practical application of the concepts discussed. Remember, your responses should guide developers towards solving their problems, understanding new concepts, or finding the information they seek related to GitHub projects and software development. Please keep your responses concise and to the point, focusing on the most essential information needed to address the query. Avoid generating long articles or overly detailed explanations.\n\nIn addition to the answer itself, provide a brief analysis of how you would approach searching for relevant code and issues within GitHub repositories. This analysis should outline your thought process and the key factors you would consider when conducting these searches. However, keep this analysis concise and focused on the high-level logic rather than delving into specific search criteria or keywords.\n\nThroughout your responses, prioritize clarity and brevity. Focus on delivering the most essential information needed to address the query effectively. Even if certain details are unknown, ensure that your answers are plausible, useful, and serve as a foundation for further exploration and context generation.\n\nRemember, your ultimate aim is to empower developers with the knowledge and guidance they need to overcome challenges, expand their understanding, and navigate the vast landscape of GitHub repositories and software development practices.",
+    "code_search_criteria_prompt": "**Instructions:**\n- **Expertise-Driven Github Code Search Criteria Generation:** Generate GitHub code search criteria strings based on the provided question and its draft answer. Analyze both the question and answer to identify key concepts, technologies, and coding practices that can help locate relevant code snippets on GitHub. Always include the `language:` qualifier to focus your search on language-related content.\n\n**Output Format:** Present each search criteria string on a new line, formatted for immediate use in GitHub's code search, without additional explanations or commentary.\n\n**Optimization Considerations:**\n- **Keyword Relevance:** Extract keywords and phrases tightly related to the question from the question and answer that are likely to appear in relevant code and code comments. Prioritize terms that reflect specific coding concepts, libraries, or techniques. Avoid generic terms like \"example\" or \"integration\" that may not be present in actual code.\n- **Contextual Understanding:** Use the provided answer as additional context to inform your keyword selection. Identify key insights, technologies, or approaches mentioned in the answer tightly related to the question that can help refine the search criteria.\n- **Language and Platform Specificity:** If the question is specific to a certain programming language or platform, ensure to include relevant language or platform-specific keywords, libraries, or frameworks in the search criteria. This helps filter out irrelevant results from other languages or platforms.\n- **Simplicity and Effectiveness:** Craft search criteria with simple and limited keywords which could lead to precise search results to relevant code snippets tightly related to original question. Strike a balance between specificity and breadth to ensure the criteria capture the essential aspects of the question and answer. The search criteria should be neither too narrow that no results are returned, nor too broad that many irrelevant results are included.\n- **Multiple Perspectives:** Generate multiple search criteria strings that approach the question from different angles or emphasize different aspects mentioned in the question and answer. This increases the chances of finding relevant code snippets.",
+    "issue_search_criteria_prompt": "**Instructions:**\n- **Question-Driven GitHub Issue Search Criteria Generation:** Generate GitHub issue search criteria strings based on the provided question. Analyze the question to identify key concepts, technologies, and problem-solving approaches that can help locate relevant issues on GitHub. Consider using relevant `label:` or `is:` qualifiers when applicable.\n\n**Output Format:** Present each search criteria string on a new line, formatted for immediate use in GitHub's issue search, without additional explanations or commentary.\n\n**Optimization Considerations:**\n- **Keyword Relevance:** Extract keywords and phrases tightly related to the question that are likely to appear in issue titles, descriptions, and discussions. Prioritize terms that reflect specific problems, error messages, or technologies. Avoid generic terms like \"help\" or \"problem\" that may not effectively narrow down the search results.\n- **Contextual Understanding:** Use the question's draft answer to inform your keyword selection. Identify key aspects, technologies, or potential troubleshooting areas tightly related to the question but not only specific aspects of answers that can help refine the search criteria.\n- **Simplicity and Effectiveness:** Craft search criteria with simple and limited keywords which could lead to precise search results relevant to the original question. Strike a balance between specificity and breadth to ensure the criteria capture the essential aspects of the question without being overly restrictive.\n- **Multiple Perspectives:** Generate multiple search criteria strings that approach the question from different angles or emphasize different aspects mentioned in the question. This increases the chances of finding relevant issues that discuss similar problems or solutions.\n- **Leveraging Labels:** When appropriate, include relevant `label:` qualifiers in the search criteria to narrow down the results to issues with specific labels, such as \"bug,\" \"enhancement,\" or \"documentation.\" This can help focus the search on issues that align with the nature of the question.\n- **Considering Issue Discussions:** Keep in mind that issue discussions often contain valuable information, experiences, and workarounds shared by other developers. Craft search criteria that not only match the issue title and description but also consider the likelihood of the keywords appearing in the issue's comments and discussions.",
+    "repo_search_criteria_prompt": "**Instructions:**\n- **Expertise-Driven Github Repository Search Criteria Generation:** Generate GitHub repo search criteria strings based on the provided question. Analyze the question leverage your expertise for related key concepts, technologies, and problem-solving approaches that can help locate relevant repositories on GitHub. Focus on practical keywords and phrases likely to be present in repository names, descriptions, and topics. Use the `language:` qualifier to direct your search toward repositories written in a specific language, keeping the criteria simple and effective.\n- **Necessity Score Determination:** Evaluate the necessity of conducting a GitHub repository search based on the difficulty of question. Determine if repository-level information is essential to comprehensively address the question. Assign a necessity score indicating the importance of performing a repository search.\n\n**Output Format:**\n- **Necessity Score:** Begin your output with a necessity score (0-100) indicating the importance of performing a separate GitHub repository search. Use the following scale:\n  - 0-59: Low necessity - Only code and issue search results is sufficient.\n  - 60-79: Medium necessity - One repository search may offer additional insights and context.\n  - 80-100: High necessity - Two repository searches are crucial to gather comprehensive information, such as project structure, documentation, or community engagement, to thoroughly address the question.\n\n- **Search Criteria:** Present each search criteria string on a new line, formatted for immediate use in GitHub's repository search, without additional explanations or commentary.\n**Optimization Considerations:**\n- **Keyword Relevance:** Generate search criteria keywords and phrases from the question that are uniquely relevant to repository names, descriptions, and topics. Prioritize terms that reflect the broader context, expertise, and strategic thinking required to address the question effectively. Avoid generic terms that may lead to irrelevant search results.\n- **Simplicity and Effectiveness:** Craft search criteria that are simple yet effective in narrowing down the repository search results to the most relevant and informative ones. Strike a balance between specificity and breadth, ensuring that the criteria capture the essential aspects of the question without being overly restrictive. Aim for criteria that yield a manageable number of high-quality repository results.\n- **Language and Platform Specificity:** If the question pertains to a specific programming language or platform, incorporate relevant language or platform-specific keywords in the search criteria. Use the `language:` qualifier to filter repositories based on the language of interest. This helps focus the search on repositories that are more likely to contain relevant code, documentation, and community expertise.\n- **Multiple Criteria Flexibility:** Generate multiple search criteria strings that approach the question from different angles or emphasize different aspects mentioned in the question. This flexibility allows for a more comprehensive repository search, increasing the chances of discovering relevant repositories that may offer valuable insights, code samples, or best practices related to the question at hand.",
+    "scoring_context_prompt": "You are an expert in evaluating the relevance of coding-related contexts to given questions. Your primary function is to analyze the provided context and question, and output a single integer score between 0 and 100, indicating how well the context supports answering the question.\n\nScoring criteria:\n0-20: The context is completely irrelevant to the question and provides no useful information to answer it.\n21-40: The context is slightly relevant to the question but lacks crucial information to provide a complete answer.\n41-60: The context is somewhat relevant to the question and provides some useful information, but it may not be sufficient to fully answer the question.\n61-80: The context is highly relevant to the question and provides most of the necessary information to answer it, but some minor details may be missing.\n81-100: The context is extremely relevant to the question and provides all the necessary information to comprehensively answer it.\n\nRemember, your output should consist of only a single integer score without any additional text or explanation. Analyze the context and question carefully, and provide a score that accurately reflects the relevance of the context in answering the question.",
+    "default_embedding": "jinaai/jina-embeddings-v2-base-code",
+    "default_reranker": "jinaai/jina-reranker-v1-turbo-en",
+    "min_stars_to_keep_result": 20,
+    "max_workers": 8,
+    "code_search_max_hits": 30,
+    "issue_search_max_hits": 30,
+    "repo_search_max_hits": 10,
+    "chunk_size": 2000,
+    "issue_chunk_size": 7000,
+    "repo_chunk_size": 7000,
+    "google_chunk_size": 7000,
+    "top_n_contexts": 5
+}

llama_github/config/config.py

@@ -0,0 +1,30 @@
+# config.py
+import json
+from importlib import resources
+from llama_github.logger import logger
+
+# utils.py
+class SingletonMeta(type):
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+class Config(metaclass=SingletonMeta):
+    _config = None
+
+    def __init__(self):
+        if Config._config is None:
+            with resources.open_text('llama_github.config', 'config.json') as file:
+                Config._config = json.load(file)
+
+    @classmethod
+    def get(cls, key, default=None):
+        # Ensure the singleton instance is created
+        if cls._config is None:
+            cls()
+        return cls._config.get(key, default)
+
+config = Config()

llama_github/data_retrieval/github_api.py

@@ -0,0 +1,296 @@
+from github import GithubException
+from .github_entities import Repository, RepositoryPool
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from llama_github.logger import logger
+from llama_github.github_integration.github_auth_manager import ExtendedGithub
+from llama_github.config.config import config
+import re
+
+
+class GitHubAPIHandler:
+    def __init__(self, github_instance: ExtendedGithub):
+        """
+        Initializes the GitHubAPIHandler with a GitHub instance.
+
+        :param github_instance: Authenticated instance of a Github client.
+        """
+        self._github = github_instance
+        self.pool = RepositoryPool(github_instance)
+
+    def search_repositories(self, query, sort="best match", order="desc"):
+        """
+        Searches for repositories on GitHub based on a query.
+
+        :param query: The search query string.
+        :param sort: The field to sort the results by. Default is 'stars'.
+        :param order: The order of sorting, 'asc' or 'desc'. Default is 'desc'.
+        :return: A list of Repository objects or None if an error occurs.
+        """
+        try:
+            if sort not in ['stars', 'forks', 'updated']:
+                repositories = self._github.search_repositories(
+                    query=query, order=order)
+            else:
+                repositories = self._github.search_repositories(
+                    query=query, sort=sort, order=order)
+            result = []
+            for i, repo in enumerate(repositories):
+                if i >= config.get("repo_search_max_hits"):
+                    break
+                result.append(
+                    Repository(
+                        repo.full_name,
+                        self._github,
+                        **{
+                            'id': repo.id,
+                            'name': repo.name,
+                            'description': repo.description,
+                            'html_url': repo.html_url,
+                            'stargazers_count': repo.stargazers_count,
+                            'language': repo.language,
+                            'default_branch': repo.default_branch,
+                            'updated_at': repo.updated_at,
+                        }
+                    )
+                )
+            return result
+        except GithubException as e:
+            logger.exception(
+                f"Error searching repositories with query '{query}':")
+            return None
+
+    def get_repository(self, full_repo_name):
+        """
+        Retrieves a single repository by its full name.
+
+        :param full_repo_name: The full name of the repository (e.g., 'octocat/Hello-World').
+        :return: A Repository object or None if an error occurs.
+        """
+        return self.pool.get_repository(full_repo_name)
+
+    def _get_file_content_through_repository(self, code_search_result):
+        """
+        Helper method to get file content through a Repository object.
+
+        :param code_search_result: A single code search result.
+        :return: Tuple containing the Repository object and the file content.
+        """
+        # Assuming RepositoryPool is accessible and initialized somewhere in this class
+        repository_obj = self.get_repository(
+            code_search_result['repository']['full_name'])
+        file_content = repository_obj.get_file_content(
+            code_search_result['path'])
+        return repository_obj, file_content
+
+    def search_code(self, query, repo_full_name=None):
+        """
+        Searches for code on GitHub based on a query, optionally within a specific repository.
+
+        :param query: The search query string.
+        :param repo_full_name: Optional. The full name of the repository (e.g., 'octocat/Hello-World') to restrict the search to.
+        :return: A list of code search results or None if an error occurs.
+        """
+        try:
+            logger.debug(f"Searching code with query '{query}'...")
+            # If a repository full name is provided, include it in the query
+            if repo_full_name:
+                query = f"{query} repo:{repo_full_name}"
+
+            # Perform the search
+            code_results = self._github.search_code(
+                query=query, per_page=config.get("code_search_max_hits"))
+
+            results_with_index = []
+            with ThreadPoolExecutor(max_workers=config.get("max_workers")) as executor:
+                # Concurrently fetch the file content for each code search result
+                future_to_index = {executor.submit(
+                    self._get_file_content_through_repository, code_result): index for index, code_result in enumerate(code_results)}
+                for future in as_completed(future_to_index):
+                    index = future_to_index[future]
+                    code_result = code_results[index]
+                    try:
+                        repository_obj, file_content = future.result()
+                        if repository_obj and file_content:
+                            results_with_index.append({
+                                'index': index,
+                                'name': code_result['name'],
+                                'path': code_result['path'],
+                                'repository_full_name': code_result['repository']['full_name'],
+                                'url': code_result['html_url'],
+                                'content': file_content,
+                                'stargazers_count': repository_obj.stargazers_count,
+                                'watchers_count': repository_obj.watchers_count,
+                                'language': repository_obj.language,
+                                'description': repository_obj.description,
+                                'updated_at': repository_obj.updated_at,
+                            })
+                    except Exception as e:
+                        logger.exception(
+                            f"{code_result['name']} generated an exception:")
+
+            # Sort the results by index to maintain the original order
+            sorted_results = sorted(
+                results_with_index, key=lambda x: x['index'])
+            logger.debug(
+                f"Code search retrieved successfully with {len(sorted_results)} results.")
+            return sorted_results
+        except GithubException as e:
+            logger.exception(f"Error searching code with query '{query}':")
+            return None
+
+    def _get_issue_content_through_repository(self, issue):
+        """
+        Helper method to get issue content through issue url.
+
+        :param code_result: A single code search result.
+        :return: Tuple containing the Repository object and the file content.
+        """
+        # Assuming RepositoryPool is accessible and initialized somewhere in this clas
+        issue_content = ''
+        issue_url = issue['url']
+        # Use regular expressions to extract repo_full_name and issue_number
+        match = re.search(
+            r'https://api.github.com/repos/([^/]+/[^/]+)/issues/(\d+)', issue_url)
+        if match:
+            repo_full_name = match.group(1)
+            issue_number = int(match.group(2))
+            repository_obj = self.get_repository(repo_full_name)
+            issue_content = repository_obj.get_issue_content(
+                number=issue_number, issue=issue)
+        else:
+            logger.warning(
+                f"Failed to extract repo_full_name and issue_number from issue url: {issue_url}")
+        return issue_content
+
+    def search_issues(self, query, repo_full_name=None):
+        """
+        Searches for issues on GitHub based on a query, optionally within a specific repository.
+
+        :param query: The search query string.
+        :param repo_full_name: Optional. The full name of the repository (e.g., 'octocat/Hello-World') to restrict the search to.
+        :return: A list of issue search results or None if an error occurs.
+        """
+        try:
+            logger.debug(f"Searching issue with query '{query}'...")
+            # If a repository full name is provided, include it in the query
+            if repo_full_name:
+                query = f"{query} repo:{repo_full_name}"
+
+            # Perform the search
+            issue_results = self._github.search_issues(
+                query=query, per_page=config.get("issue_search_max_hits"))
+
+            issue_results = [issue for issue in issue_results if issue['body']
+                             is not None and issue['body'] != 'null']
+
+            results_with_index = []
+            with ThreadPoolExecutor(max_workers=config.get("max_workers")) as executor:
+                # Concurrently fetch the issue content for each issue search result
+                future_to_index = {executor.submit(
+                    self._get_issue_content_through_repository, issue): index for index, issue in enumerate(issue_results)}
+                for future in as_completed(future_to_index):
+                    index = future_to_index[future]
+                    issue_result = issue_results[index]
+                    try:
+                        issue_content = future.result()
+                        results_with_index.append({
+                            'index': index,
+                            'url': issue_result['url'],
+                            'created_at': issue_result['created_at'],
+                            'updated_at': issue_result['updated_at'],
+                            'issue_content': issue_content,
+                        })
+                    except Exception as e:
+                        logger.exception(
+                            f"{issue_result['url']} generated an exception:")
+
+            # Sort the results by index to maintain the original order
+            sorted_results = sorted(
+                results_with_index, key=lambda x: x['index'])
+            logger.debug(
+                f"Issue search retrieved successfully with {len(sorted_results)} results.")
+            return sorted_results
+        except GithubException as e:
+            logger.exception(f"Error searching issue with query '{query}':")
+            return None
+
+    def _categorize_github_url(url):
+        repo_pattern = r'^https://github\.com/[^/]+/[^/]+$'
+        issue_pattern = r'^https://github\.com/[^/]+/[^/]+/issues/\d+$'
+        repo_file_pattern = r'^https://github\.com/[^/]+/[^/]+/(?:blob|tree)/[^/]+/.+$'
+        readme_pattern = r'^https://github\.com/[^/]+/[^/]+#readme$'
+
+        if re.match(repo_pattern, url):
+            return "repo"
+        elif re.match(issue_pattern, url):
+            return "issue"
+        elif re.match(repo_file_pattern, url):
+            return "file"
+        elif re.match(readme_pattern, url):
+            return "readme"
+        else:
+            return "other"
+    
+    def get_github_url_content(self, url):
+        """
+        Retrieves the content of a GitHub URL.
+
+        :param url: The GitHub URL to retrieve content from.
+        :return: The content of the URL or None if an error occurs.
+        """
+        try:
+            logger.debug(f"Retrieving content from GitHub URL '{url}'...")
+            content = None
+            category = GitHubAPIHandler._categorize_github_url(url)
+            if category == "repo":
+                # Extract the repository full name from the URL
+                match = re.search(r'https://github\.com/([^/]+/[^/]+)', url)
+                if match:
+                    repo_full_name = match.group(1)
+                    repository_obj = self.get_repository(repo_full_name)
+                    content = repository_obj.get_readme()
+                else:
+                    logger.warning(
+                        f"Failed to extract repository full name from URL: {url}")
+            elif category == "issue":
+                # Use regular expressions to extract repo_full_name and issue_number
+                match = re.search(
+                    r'https://github\.com/([^/]+/[^/]+)/issues/(\d+)', url)
+                if match:
+                    repo_full_name = match.group(1)
+                    issue_number = int(match.group(2))
+                    repository_obj = self.get_repository(repo_full_name)
+                    content = repository_obj.get_issue_content(
+                        number=issue_number)
+                else:
+                    logger.warning(
+                        f"Failed to extract repo_full_name and issue_number from URL: {url}")
+            elif category == "file":
+                # Extract the repository full name and file path from the URL
+                match = re.search(
+                    r'https://github\.com/([^/]+/[^/]+)/(?:blob|tree)/([^/]+)/(.+)', url)
+                if match:
+                    repo_full_name = match.group(1)
+                    file_path = match.group(3)
+                    repository_obj = self.get_repository(repo_full_name)
+                    content = repository_obj.get_file_content(
+                        file_path)
+                else:
+                    logger.warning(
+                        f"Failed to extract repository full name and file path from URL: {url}")
+            elif category == "readme":
+                # Extract the repository full name from the URL
+                match = re.search(r'https://github\.com/([^/]+/[^/]+)#readme', url)
+                if match:
+                    repo_full_name = match.group(1)
+                    repository_obj = self.get_repository(repo_full_name)
+                    content = repository_obj.get_readme()
+                else:
+                    logger.warning(
+                        f"Failed to extract repository full name from URL: {url}")
+            else:
+                logger.warning(f"Unsupported GitHub URL category: {category}")
+            return content
+        except GithubException as e:
+            logger.exception(f"Error retrieving content from GitHub URL '{url}':")
+            return None