llama-github 0.1.0__py3-none-any.whl

llama_github/data_retrieval/github_entities.py

@@ -0,0 +1,301 @@
+# To do list:
+# 1. Add issues to the Repository class.
+# 2. Add a method to get issues for a repository.
+
+from github import GithubException
+from threading import Lock, Event, Thread
+from datetime import datetime, timezone
+import time
+from llama_github.logger import logger
+from llama_github.github_integration.github_auth_manager import ExtendedGithub
+import re
+
+
+class Repository:
+    def __init__(self, full_name, github_instance: ExtendedGithub, **kwargs):
+        """
+        Initializes a Repository instance with details and a GitHub instance for API calls.
+
+        :param id: The ID of the repository.
+        :param name: The name of the repository.
+        :param full_name: The full name of the repository (e.g., 'octocat/Hello-World').
+        :param description: The description of the repository.
+        :param html_url: The HTML URL of the repository.
+        :param stargazers_count: The count of stargazers of the repository.
+        :param watchers_count: The count of watchers of the repository.
+        :param language: The primary language of the repository.
+        :param forks_count: The count of forks of the repository.
+        :param github_instance: Authenticated instance of a Github client.
+        """
+
+        self._github = github_instance
+        self.full_name = full_name
+
+        self.id = kwargs.get("id")
+        self.name = kwargs.get("name")
+        self.description = kwargs.get("description")
+        self.html_url = kwargs.get("html_url")
+        self.stargazers_count = kwargs.get("stargazers_count")
+        self.language = kwargs.get("language")
+        self.default_branch = kwargs.get("default_branch")
+        self.updated_at = kwargs.get("updated_at")
+        self.watchers_count = None
+
+        self.creation_time = datetime.now(timezone.utc)
+        self.last_read_time = datetime.now(timezone.utc)
+
+        self._repo = None
+
+        # Locking for write action
+        self._structure_lock = Lock()
+        self._file_contents_lock = Lock()
+        self._issue_lock = Lock()
+        self._readme_lock = Lock()
+        self._repo_lock = Lock()
+
+        if (self.id is None) or \
+            (self.name is None) or \
+            (self.full_name is None) or \
+            (self.html_url is None) or \
+            (self.stargazers_count is None) or \
+            (self.default_branch is None) or \
+                (self.updated_at is None):
+            self.get_repo()
+
+        self._structure = None  # Singleton pattern for repository structure
+        self._file_contents = {}  # Singleton pattern for file contents
+        self._issues = {}  # Singleton pattern for file contents
+        self._readme = None  # Singleton pattern for README content
+
+    def update_last_read_time(self):
+        self.last_read_time = datetime.now(timezone.utc)
+
+    def get_readme(self) -> str:
+        """
+        Retrieves the README content of the repository using a singleton design pattern.
+        """
+        if self._readme is None:  # Check if README content has already been fetched
+            with self._readme_lock:  # Locking for write action
+                if self._readme is None:  # Check if README content has already been fetched after get lock
+                    try:
+                        readme = self.repo.get_readme()
+                        self._readme = readme.decoded_content.decode("utf-8")
+                    except GithubException as e:
+                        logger.exception(
+                            f"Error getting README for repository {self.full_name}:")
+                        self._readme = None
+        self.update_last_read_time()
+        return self._readme
+    
+    @property
+    def repo(self):
+        return self.get_repo()
+
+    def get_repo(self):
+        """
+        Retrieves the Github Repo object of the repository using a singleton design pattern.
+        """
+        if self._repo is None:  # Check if repo object has already been fetched
+            with self._repo_lock:  # Locking for write action
+                if self._repo is None:
+                    try:
+                        self._repo = self._github.get_repo(self.full_name)
+                        self.id = self._repo.id
+                        self.name = self._repo.name
+                        self.description = self._repo.description
+                        self.html_url = self._repo.html_url
+                        self.stargazers_count = self._repo.stargazers_count
+                        self.watchers_count = self._repo.subscribers_count
+                        self.language = self._repo.language
+                        self.default_branch = self._repo.default_branch
+                        self.updated_at = self._repo.updated_at
+                    except GithubException as e:
+                        logger.exception(
+                            f"Error retrieving repository '{self.full_name}':")
+                        return None
+        self.update_last_read_time()
+        return self._repo
+
+    def get_structure(self, path="/") -> dict:
+        """
+        Retrieves the structure of the repository using a singleton design pattern.
+        """
+        if self._structure is None:  # Check if structure has already been fetched
+            with self._structure_lock:  # Locking for write action
+                if self._structure is None:  # Check if structure has already been fetched after get lock
+                    try:
+                        self._structure = self._github.get_repo_structure(
+                            self.full_name, self.default_branch)
+                    except GithubException as e:
+                        logger.exception(
+                            f"Error getting structure for repository {self.full_name}:")
+                        self._structure = None
+        self.update_last_read_time()
+        return self._structure
+
+    def get_file_content(self, file_path) -> str:
+        """
+        Retrieves the content of a file using a singleton design pattern.
+        """
+        if file_path not in self._file_contents:  # Check if file content has already been fetched
+            with self._file_contents_lock:  # Locking for write action
+                if file_path not in self._file_contents:  # Check if file content has already been fetched after get lock
+                    try:
+                        file_content = self.repo.get_contents(file_path)
+                        self._file_contents[file_path] = file_content.decoded_content.decode(
+                            "utf-8")
+                    except GithubException as e:
+                        logger.exception(
+                            f"Error getting file content for {file_path} in repository {self.full_name}:")
+                        return None
+        self.update_last_read_time()
+        return self._file_contents[file_path]
+
+    def get_issue_content(self, number, issue=None) -> str:
+        """
+        Retrieves the content of a issue using a singleton design pattern.
+        """
+        if number not in self._issues:  # Check if issue has already been fetched
+            with self._issue_lock:  # Locking for write action
+                if number not in self._issues:  # Check if issue has already been fetched after get lock
+                    try:
+                        if issue is None:
+                            issue = self.repo.get_issue(number=number)
+                            comments_amount = issue.comments
+                            body_content = "This is a Github Issue related to repo \"" + (self.full_name or "") + "\". Repo description:" + (self.description or "") +\
+                                "\n\nIssue Title: "+issue.title+"\nIssue state: "+issue.state + \
+                                "\nIssue last updated at: "+str(issue.updated_at) + \
+                                "\nTotal favour: " + \
+                                str(issue.reactions['total_count']) + \
+                                "\nComment amount: " + \
+                                str(comments_amount) + \
+                                "\nIssue body:\n" + \
+                                re.sub(r'\n+\s*\n+', '\n',
+                                       issue.body.replace('\r', '\n').strip())
+                        else:
+                            comments_amount = issue['comments']
+                            body_content = "This is a Github Issue related to repo \"" + (self.full_name or "") + "\". Repo description:" + (self.description or "") +\
+                                "\n\nIssue title: "+issue['title']+"\nIssue state: "+issue['state'] + \
+                                "\nIssue last updated at: "+str(issue['updated_at']) + \
+                                "\nTotal favour: " + \
+                                str(issue['reactions']['total_count']) + \
+                                "\nComment amount: " + \
+                                str(comments_amount) + \
+                                "\nIssue body:\n" + \
+                                re.sub(
+                                    r'\n+\s*\n+', '\n', issue['body'].replace('\r', '\n').strip())
+                        if comments_amount > 0:
+                            comments = self._github.get_issue_comments(
+                                repo_full_name=self.full_name, issue_number=number)
+                            comments_text_list = []
+                            for index, comment in enumerate(comments, start=1):
+                                cleaned_body = re.sub(
+                                    r'\n+\s*\n+', '\n', comment['body'].replace('\r', '\n').strip())
+                                comment_text = (
+                                    f"Comment No.{index} - comment author type: {comment['user']['type']}; "
+                                    f"comment total favour: {comment['reactions']['total_count']}\n"
+                                    f"Comment body:\n{cleaned_body}\n"
+                                )
+                                comments_text_list.append(comment_text)
+                            comments_text = "\n".join(comments_text_list)
+                            issue_content = body_content + "\n\nIssue Comments:\n" + comments_text
+                        else:
+                            issue_content = body_content
+                        self._issues[number] = issue_content.strip()
+                    except GithubException as e:
+                        logger.exception(
+                            f"Error getting issue content for {number} in repository {self.full_name}:")
+                        return None
+        self.update_last_read_time()
+        return self._issues[number]
+
+    def __str__(self):
+        """
+        String representation of the Repository instance.
+
+        :return: A string describing the repository.
+        """
+        self.update_last_read_time()
+        return f"{self.full_name} - {self.description}"
+
+    def clear_cache(self):
+        """
+        Clears the cached data of the repository, including structure, file contents, and README content.
+        """
+        with self._structure_lock:
+            self._structure = None  # Reset the repository structure cache
+        with self._file_contents_lock:
+            self._file_contents = {}  # Reset the file contents cache
+        with self._issue_lock:
+            self._issues = {}  # Reset the issue contents cache
+        with self._readme_lock:
+            self._readme = None  # Reset the README content cache
+        with self._repo_lock:
+            self._repo = None  # Reset the Repo cache
+
+
+class RepositoryPool:
+    _instance_lock = Lock()
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:  # First check (unlocked)
+            with cls._instance_lock:  # Acquire lock
+                if cls._instance is None:  # Second check (locked)
+                    cls._instance = super(RepositoryPool, cls).__new__(cls)
+        return cls._instance
+
+    def __init__(self, github_instance, cleanup_interval=3600, max_idle_time=86400):
+        if not hasattr(self, 'initialized'):
+            with self._instance_lock:   # Prevent re-initialization
+                if not hasattr(self, 'initialized'):
+                    self.initialized = True
+                    self._locks_registry = {}  # A registry for repository-specific locks
+                    self._registry_lock = Lock()  # A lock to protect the locks registry
+                    self._pool = {}  # The repository pool
+                    self.github_instance = github_instance
+                    self.cleanup_interval = cleanup_interval  # How often to run cleanup in seconds
+                    self.max_idle_time = max_idle_time  # Maximum idle time in seconds
+                    self._cleanup_thread = Thread(
+                        target=self._cleanup, daemon=True)
+                    self._stop_event = Event()
+                    self._cleanup_thread.start()
+
+    def _cleanup(self):  # Internal method for cleaning up idle repository objects' cache content, not real delete repository objects
+        """Periodically checks and removes idle repository objects."""
+        while not self._stop_event.is_set():
+            with self._registry_lock:
+                current_time = datetime.now(timezone.utc)
+                for full_name in list(self._pool.keys()):
+                    repo = self._pool[full_name]
+                    if ((current_time - repo.last_read_time).total_seconds() > self.max_idle_time) or \
+                        ((current_time - repo.creation_time).total_seconds() > 7 * self.max_idle_time and
+                         (current_time - repo.last_read_time).total_seconds() > (self.max_idle_time // 4) + 1):
+                        # release the lock due to object already created
+                        del self._locks_registry[full_name]
+                        # clear the cache content of repository object
+                        self._pool[full_name].clear_cache()
+            time.sleep(self.cleanup_interval)
+
+    def stop_cleanup(self):
+        """Stops the cleanup thread."""
+        self._stop_event.set()
+
+    def _get_repo_lock(self, full_name):
+        """Retrieve or create a lock for a specific repository."""
+        with self._registry_lock:
+            if full_name not in self._locks_registry:
+                # Create a new lock for the repository creation
+                self._locks_registry[full_name] = Lock()
+            return self._locks_registry[full_name]
+
+    def get_repository(self, full_name, **kwargs) -> Repository:
+        """Retrieve a repository from the pool or create a new one if it doesn't exist."""
+        if full_name in self._pool:
+            return self._pool[full_name]
+        repo_lock = self._get_repo_lock(full_name)
+        with repo_lock:
+            if full_name not in self._pool:
+                self._pool[full_name] = Repository(
+                    full_name, self.github_instance, **kwargs)
+        return self._pool[full_name]

llama_github/github_integration/__init__.py

@@ -0,0 +1,7 @@
+# Import the AuthManager for easy access
+from .github_auth_manager import GitHubAuthManager
+
+# Define what is available for import
+__all__ = ["GitHubAuthManager"]
+
+# Any initialization code specific to GitHub integration can go here

llama_github/github_integration/github_auth_manager.py

@@ -0,0 +1,266 @@
+# To do list:
+# 1. add the mechanism for installation_access_token and github_instance refreshment in authenticate_with_app model
+# 2. add re-try mechanism for the API calls
+# 3. add the mechanism for the rate limit handling
+# 4. add the mechanism for the error handling
+# 5. add the mechanism for the logging
+# 6. add search issues functionality
+# 7. add search discussions functionality through Github GraphQL API
+
+from github import Github, GithubIntegration
+import requests
+from requests.adapters import HTTPAdapter
+from requests.exceptions import HTTPError, RequestException
+from urllib3.util.retry import Retry
+from llama_github.logger import logger
+
+
+class GitHubAuthManager:
+    def __init__(self):
+        self.github_instance = None
+        self.access_token = None
+        self.app_id = None
+        self.private_key = None
+        self.installation_id = None
+
+    def authenticate_with_token(self, access_token):
+        """
+        Authenticate using a personal access token or an OAuth token.
+        Suitable for individual developers and applications using OAuth for authorization.
+        """
+        self.access_token = access_token
+        self.github_instance = ExtendedGithub(login_or_token=access_token)
+        return self.github_instance
+
+    def authenticate_with_app(self, app_id, private_key, installation_id):
+        """
+        Authenticate using a GitHub App.
+        Suitable for integrations in organizational or enterprise environments.
+        """
+        self.app_id = app_id
+        self.private_key = private_key
+        self.installation_id = installation_id
+        integration = GithubIntegration(app_id, private_key)
+        installation_access_token = integration.get_access_token(
+            installation_id).token
+        self.access_token = installation_access_token
+        self.github_instance = ExtendedGithub(
+            login_or_token=installation_access_token)
+        return self.github_instance
+
+    def close_connection(self):
+        """
+        Close the connection to GitHub to free up resources.
+        """
+        if self.github_instance:
+            self.github_instance = None
+
+# Extended Github Class for powerful API calls - e.g. recursive call to get repo structure
+
+
+class ExtendedGithub(Github):
+    def __init__(self, login_or_token):
+        self.access_token = login_or_token
+        super().__init__(login_or_token=login_or_token)
+
+    def get_repo_structure(self, repo_full_name, branch='main') -> dict:
+        """
+        Get the structure of a repository (files and directories) recursively.
+        """
+        owner, repo_name = repo_full_name.split('/')
+        headers = {'Authorization': f'token {self.access_token}'}
+
+        # Function to convert the flat list to a hierarchical structure
+        def list_to_tree(items):
+            """
+            Convert the flat list to a hierarchical structure with full paths.
+            Include size metadata for files and remove 'type' attributes.
+            """
+            tree = {}
+            for item in items:
+                path_parts = item['path'].split('/')
+                current_level = tree
+                for part in path_parts[:-1]:
+                    # Ensure 'children' dictionary exists for directories without explicitly adding 'type'
+                    current_level = current_level.setdefault(
+                        part, {'children': {}})
+                    # Ensure we don't inadvertently create a 'type' key for directories
+                    current_level = current_level.get('children')
+
+                # For the last part of the path, decide if it's a file or directory and add appropriate information
+                if item['type'] == 'blob':  # It's a file
+                    current_level[path_parts[-1]] = {
+                        'path': item['path'],  # Include full path
+                        # Include size if available
+                        'size': item.get('size', 0)
+                    }
+                else:  # It's a directory
+                    # Initialize the directory if not already present, without adding 'type'
+                    if path_parts[-1] not in current_level:
+                        current_level[path_parts[-1]] = {'children': {}}
+            return tree
+
+        # Directly use the Trees API to get the full directory structure of the "main" branch
+        tree_url = f'https://api.github.com/repos/{owner}/{repo_name}/git/trees/{branch}?recursive=1'
+        tree_response = requests.get(tree_url, headers=headers)
+
+        # Check if the request was successful
+        if tree_response.status_code == 200:
+            tree_data = tree_response.json()
+            # Convert the flat list of items to a hierarchical tree structure
+            repo_structure = list_to_tree(tree_data['tree'])
+            return repo_structure
+        else:
+            print(
+                f"Error fetching tree structure: {tree_response.status_code}")
+            print("Details:", tree_response.json())
+
+    def search_code(self, query: str, per_page: int = 30) -> dict:
+        """
+        Search for code on GitHub using the GitHub API.
+
+        Parameters:
+        query (str): The search query.
+        per_page (int): The number of results per page.
+
+        Returns:
+        dict: The search result in dict format.
+        """
+        url = 'https://api.github.com/search/code'
+        headers = {
+            'Accept': 'application/vnd.github.v3+json',
+            'Authorization': f'token {self.access_token}'
+        }
+        params = {
+            'q': query,
+            'per_page': per_page
+        }
+
+        # Retry strategy
+        retry_strategy = Retry(
+            total=3,  # Total number of retries
+            # Retry on these HTTP status codes
+            status_forcelist=[429, 500, 502, 503, 504],
+            # Retry on these HTTP methods
+            allowed_methods=["HEAD", "GET", "OPTIONS"],
+            backoff_factor=1  # Exponential backoff factor
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        http = requests.Session()
+        http.mount("https://", adapter)
+
+        try:
+            response = http.get(url, headers=headers, params=params)
+            response.raise_for_status()  # Raise HTTPError for bad responses
+            return response.json().get('items', [])
+        except HTTPError as http_err:
+            logger.error(f"HTTP error occurred: {http_err}")
+        except RequestException as req_err:
+            logger.error(f"Request error occurred: {req_err}")
+        except Exception as err:
+            logger.error(f"An error occurred: {err}")
+
+    def search_issues(self, query: str, per_page: int = 30) -> dict:
+        """
+        Search for code on GitHub using the GitHub API.
+
+        Parameters:
+        query (str): The search query.
+        per_page (int): The number of results per page.
+
+        Returns:
+        dict: The search result in dict format.
+        """
+        url = 'https://api.github.com/search/issues'
+        headers = {
+            'Accept': 'application/vnd.github.v3+json',
+            'Authorization': f'token {self.access_token}'
+        }
+        params = {
+            'q': query,
+            'per_page': per_page
+        }
+
+        # Retry strategy
+        retry_strategy = Retry(
+            total=3,  # Total number of retries
+            # Retry on these HTTP status codes
+            status_forcelist=[429, 500, 502, 503, 504],
+            # Retry on these HTTP methods
+            allowed_methods=["HEAD", "GET", "OPTIONS"],
+            backoff_factor=1  # Exponential backoff factor
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        http = requests.Session()
+        http.mount("https://", adapter)
+
+        try:
+            response = http.get(url, headers=headers, params=params)
+            response.raise_for_status()  # Raise HTTPError for bad responses
+            return response.json().get('items', [])
+        except HTTPError as http_err:
+            logger.error(f"HTTP error occurred: {http_err}")
+        except RequestException as req_err:
+            logger.error(f"Request error occurred: {req_err}")
+        except Exception as err:
+            logger.error(f"An error occurred: {err}")
+
+    def get_issue_comments(self, repo_full_name: str, issue_number: int) -> dict:
+        """
+        Get comments of an issue on GitHub using the GitHub API.
+
+        Parameters:
+        repo_full_name (str): The full name of the repository (e.g., 'octocat/Hello-World').
+        issue_number (int): The issue number.
+
+        Returns:
+        dict: The comments of the issue in dict format.
+        """
+        url = f'https://api.github.com/repos/{repo_full_name}/issues/{issue_number}/comments'
+        headers = {
+            'Accept': 'application/vnd.github.v3+json',
+            'Authorization': f'token {self.access_token}'
+        }
+        # Retry strategy
+        retry_strategy = Retry(
+            total=3,  # Total number of retries
+            # Retry on these HTTP status codes
+            status_forcelist=[429, 500, 502, 503, 504],
+            # Retry on these HTTP methods
+            allowed_methods=["HEAD", "GET", "OPTIONS"],
+            backoff_factor=1  # Exponential backoff factor
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        http = requests.Session()
+        http.mount("https://", adapter)
+
+        try:
+            response = http.get(url, headers=headers)
+            response.raise_for_status()  # Raise HTTPError for bad responses
+            return response.json()
+        except HTTPError as http_err:
+            logger.error(f"HTTP error occurred: {http_err}")
+        except RequestException as req_err:
+            logger.error(f"Request error occurred: {req_err}")
+        except Exception as err:
+            logger.error(f"An error occurred: {err}")
+
+
+# Example usage:
+if __name__ == "__main__":
+    auth_manager = GitHubAuthManager()
+
+    # For developers using a personal access token or an OAuth token
+    github_instance = auth_manager.authenticate_with_token(
+        "your_personal_access_token_or_oauth_token_here")
+
+    # For organizational or enterprise environments using GitHub App
+    # github_instance = auth_manager.authenticate_with_app("app_id", "private_key", "installation_id")
+
+    # Example action: List all repositories for the authenticated user
+    if github_instance:
+        for repo in github_instance.get_user().get_repos():
+            print(repo.name)
+
+    # Close the connection when done
+    auth_manager.close_connection()