llama-github 0.1.5__tar.gz → 0.1.7__tar.gz

{llama_github-0.1.5 → llama_github-0.1.7}/CHANGELOG.md

@@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.7] - 2024-10-31
+
+### Optimized
+- fix bugs for get pr content
+
+## [0.1.6] - 2024-10-30
+
+### New Features
+- Enhanced PR content analysis with detailed commit information extraction
+- Improved issue linking detection with support for multiple reference formats
+  - Full GitHub URLs, #references, and keyword-based references
+  - Added validation for issue numbers
+
+### Improvements
+- Added detailed commit metadata extraction including stats and file changes
+- Enhanced error handling for commit fetching
+
 ## [0.1.5] - 2024-10-14
 
 ### Optimized

{llama_github-0.1.5/llama_github.egg-info → llama_github-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama-github
-Version: 0.1.5
+Version: 0.1.7
 Summary: Llama-github is an open-source Python library that empowers LLM Chatbots, AI Agents, and Auto-dev Agents to conduct Retrieval from actively selected GitHub public projects. It Augments through LLMs and Generates context for any coding question, in order to streamline the development of sophisticated AI-driven applications.
 Home-page: https://github.com/JetXu-LLM/llama-github
 Author: Jet Xu
@@ -19,9 +19,11 @@ Requires-Dist: langchain
 Requires-Dist: numpy
 Requires-Dist: pydantic
 Requires-Dist: requests
-Requires-Dist: torch
 Requires-Dist: transformers
 Requires-Dist: urllib3
+Requires-Dist: sys
+Requires-Dist: platform
+Requires-Dist: subprocess
 Requires-Dist: pydantic_core
 Requires-Dist: langchain_openai
 Requires-Dist: langchain_mistralai
@@ -138,6 +140,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.7] - 2024-10-31
+
+### Optimized
+- fix bugs for get pr content
+
+## [0.1.6] - 2024-10-30
+
+### New Features
+- Enhanced PR content analysis with detailed commit information extraction
+- Improved issue linking detection with support for multiple reference formats
+  - Full GitHub URLs, #references, and keyword-based references
+  - Added validation for issue numbers
+
+### Improvements
+- Added detailed commit metadata extraction including stats and file changes
+- Enhanced error handling for commit fetching
+
 ## [0.1.5] - 2024-10-14
 
 ### Optimized

{llama_github-0.1.5 → llama_github-0.1.7}/llama_github/data_retrieval/github_entities.py

@@ -13,6 +13,7 @@ import json
 from dateutil import parser
 from datetime import timezone
 import base64
+import requests
 
 from llama_github.utils import DiffGenerator, CodeAnalyzer
 
@@ -153,6 +154,16 @@ class Repository:
         """
         file_key = f"{file_path}/{sha}" if sha is not None else file_path
 
+        # Skip binary and system files
+        if any(file_path.endswith(ext) for ext in [
+            '.exe', '.dll', '.so', '.dylib', '.bin', 
+            '.png', '.jpg', '.jpeg', '.gif', '.ico',
+            '.mp3', '.mp4', '.avi', '.mov', '.pdf',
+            '.DS_Store'
+        ]) or '/.' in file_path:
+            logger.debug(f"Skipping binary or system file: {file_path}")
+            return None
+
         if file_key not in self._file_contents:  # Check if file content has already been fetched
             with self._file_contents_lock:  # Locking for thread-safe write action
                 if file_key not in self._file_contents:  # Double-check after acquiring the lock
@@ -162,9 +173,26 @@ class Repository:
                         else:
                             file_content = self.repo.get_contents(file_path)
                         
+                        # Handle directory case
+                        if isinstance(file_content, list):
+                            logger.debug(f"Path {file_path} is a directory")
+                            return None
+
                         # Improved encoding handling
                         if file_content.encoding == 'base64':
                             decoded_content = base64.b64decode(file_content.content).decode('utf-8')
+                        elif (file_content.encoding is None or file_content.encoding == 'none') and hasattr(file_content, 'download_url') and file_content.download_url:
+                            try:
+                                response = requests.get(
+                                    file_content.download_url,
+                                    timeout=30,
+                                    headers={'Accept': 'application/vnd.github.v3.raw'}
+                                )
+                                response.raise_for_status()
+                                decoded_content = response.text
+                            except requests.RequestException as e:
+                                logger.error(f"Failed to download file {file_path}: {str(e)}")
+                                return None
                         else:
                             decoded_content = file_content.decoded_content.decode('utf-8')
                         
@@ -180,7 +208,8 @@ class Repository:
 
         self.update_last_read_time()
         return self._file_contents.get(file_key)
-    
+
+
     def get_issue_content(self, number, issue=None) -> str:
         """
         Retrieves the content of a issue using a singleton design pattern.
@@ -200,8 +229,8 @@ class Repository:
                                 "\nComment amount: " + \
                                 str(comments_amount) + \
                                 "\nIssue body:\n" + \
-                                re.sub(r'\n+\s*\n+', '\n',
-                                       issue.body.replace('\r', '\n').strip())
+                                (re.sub(r'\n+\s*\n+', '\n',
+                                    (issue.body or "").replace('\r', '\n').strip()) if issue.body else "No issue body provided")
                         else:
                             comments_amount = issue['comments']
                             body_content = "This is a Github Issue related to repo \"" + (self.full_name or "") + "\". Repo description:" + (self.description or "") +\
@@ -236,30 +265,87 @@ class Repository:
                         logger.exception(
                             f"Error getting issue content for {number} in repository {self.full_name}:")
                         return None
+                    except Exception as e:
+                        logger.error(f"Unexpected error processing : {str(e)}")
+                        return None
         self.update_last_read_time()
         return self._issues[number]
     
     def extract_related_issues(self, pr_data: Dict[str, Any]) -> List[int]:
         """
-        Extracts related issue numbers from the PR description and other fields.
-
-        :param pr_data: The pull request data dictionary.
-        :return: A list of related issue numbers.
+        Extracts related issue numbers from all PR data following GitHub's reference syntax.
+        
+        This function implements GitHub's official autolink reference formats to find:
+        1. Full GitHub issue/PR URLs
+        2. Numeric references (#123)
+        3. Keywords + issue references (fixes #123)
+        4. Repository cross-references (owner/repo#123)
+        
+        See: https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls
+        
+        Args:
+            pr_data: Dict[str, Any] - The complete pull request data dictionary
+            
+        Returns:
+            List[int] - A sorted list of unique issue numbers found in the PR data
         """
+        # GitHub's official closing keywords
+        closing_keywords = (
+            'close', 'closes', 'closed',
+            'fix', 'fixes', 'fixed',
+            'resolve', 'resolves', 'resolved'
+        )
+
+        # Regex patterns for GitHub issue references
         patterns = [
-            rf'https://github\.com/{re.escape(self.full_name)}/issues/(\d+)',
-            r'(?:^|\s)#(\d+)',
-            r'(?:^|\s)(\d+)(?:\s|$)',
+            # Full GitHub issue/PR URL pattern
+            rf'(?:https?://)?github\.com/{re.escape(self.full_name)}/(?:issues|pull)/(\d+)',
+            
+            # Standard #123 reference with proper boundaries
+            r'(?:^|[^\w/])#(\d+)(?=[^\w/]|$)',
+            
+            # Closing keywords (fixes #123)
+            fr'(?:^|[^\w/])(?:{"|".join(closing_keywords)}):?\s+#(\d+)(?=[^\w/]|$)',
+            
+            # Cross-repo reference (owner/repo#123)
+            rf'{re.escape(self.full_name)}#(\d+)',
+            
+            # Issue keyword reference (issue #123 or issue: #123)
+            r'(?:^|[^\w/])(?:issue|bug|ticket|todo|task)s?:?\s+#?(\d+)(?=[^\w/]|$)'
         ]
+
         issues = set()
-        # Convert PR data to JSON string for pattern matching
-        pr_description = json.dumps(pr_data, default=str)
         
-        for pattern in patterns:
-            matches = re.findall(pattern, pr_description)
-            issues.update(int(match) for match in matches)
+        def extract_from_text(text: str) -> None:
+            """Helper function to extract issue numbers from text"""
+            if not isinstance(text, str):
+                return
+                
+            for pattern in patterns:
+                matches = re.findall(pattern, text, re.IGNORECASE | re.MULTILINE)
+                # Validate issue numbers (reasonable length and positive values)
+                valid_matches = [
+                    int(match) for match in matches 
+                    if match.isdigit() and len(match) <= 7 and int(match) > 0
+                ]
+                issues.update(valid_matches)
+
+        def process_value(value: Any) -> None:
+            """Recursively process dictionary values and extract issue numbers"""
+            if isinstance(value, dict):
+                for v in value.values():
+                    process_value(v)
+            elif isinstance(value, (list, tuple)):
+                for item in value:
+                    process_value(item)
+            elif isinstance(value, str):
+                extract_from_text(value)
+
+        # Process all data in pr_data recursively
+        process_value(pr_data)
         
-        return list(issues)
+        return sorted(list(issues))
+
 
     def get_issue_contents(self, issue_numbers: List[int], pr_number: int) -> List[Dict[str, Any]]:
         """
@@ -324,11 +410,16 @@ class Repository:
                                 "head_branch": pr.head.ref,
                             },
                             "related_issues": [],
+                            "commits": [],
                             "file_changes": [],
                             "ci_cd_results": [],
                             "interactions": []
                         }
 
+                        # Extract related issues from PR description and comments
+                        related_issues = self.extract_related_issues(pr_data)
+                        pr_data['related_issues'] = self.get_issue_contents(related_issues, pr.number)
+
                         # Fetch CI/CD results
                         try:
                             last_commit = pr.get_commits().reversed[0]
@@ -412,9 +503,27 @@ class Repository:
                         pr_interactions.sort(key=lambda x: parser.isoparse(x["created_at"]))
                         pr_data["interactions"] = pr_interactions
 
-                        # Extract related issues from PR description and comments
-                        related_issues = self.extract_related_issues(pr_data)
-                        pr_data['related_issues'] = self.get_issue_contents(related_issues, pr.number)
+                        # Fetch and process commits
+                        try:
+                            commits = pr.get_commits()
+                            for commit in commits:
+                                commit_data = {
+                                    "sha": commit.sha,
+                                    "message": commit.commit.message,
+                                    "author": commit.commit.author.name,
+                                    "date": self.to_isoformat(commit.commit.author.date),
+                                    "stats": {
+                                        "additions": commit.stats.additions,
+                                        "deletions": commit.stats.deletions,
+                                        "total": commit.stats.total
+                                    },
+                                    "files": [f.filename for f in commit.files]  # Just keep changed file names
+                                }
+                                pr_data["commits"].append(commit_data)
+                        except GithubException as e:
+                            logger.exception(f"Error fetching commits for PR #{number}")
+                            pr_data["commits"] = []
+                            pr_data["commit_stats"] = {}
 
                         # Process file changes
                         dependency_files = ['requirements.txt', 'Pipfile', 'Pipfile.lock', 'setup.py']
@@ -439,7 +548,10 @@ class Repository:
                                 head_content = self.get_file_content(file_path=file_path, sha=pr.head.sha)
 
                             # Generate custom diff with specified context lines
-                            custom_diff = DiffGenerator.generate_custom_diff(base_content, head_content, context_lines)
+                            if base_content is None and head_content is None:
+                                custom_diff = ''
+                            else:
+                                custom_diff = DiffGenerator.generate_custom_diff(base_content, head_content, context_lines)
 
                             # Categorize code changes
                             change_categories = CodeAnalyzer.categorize_change(custom_diff)

{llama_github-0.1.5 → llama_github-0.1.7}/llama_github/llm_integration/initial_load.py

@@ -8,6 +8,11 @@ from llama_github.config.config import config
 from llama_github.logger import logger
 
 class LLMManager:
+    """
+    Singleton class for managing Language Models and related components.
+    This class handles initialization and access to various models including LLMs,
+    embedding models, and reranking models.
+    """
     _instance_lock = Lock()
     _instance = None
     llm = None
@@ -18,6 +23,9 @@ class LLMManager:
     embedding_model = None
 
     def __new__(cls, *args, **kwargs):
+        """
+        Ensure only one instance of LLMManager is created (Singleton pattern).
+        """
         if cls._instance is None:  # First check (unlocked)
             with cls._instance_lock:  # Acquire lock
                 if cls._instance is None:  # Second check (locked)
@@ -34,6 +42,19 @@ class LLMManager:
                  rerank_model: Optional[str] = config.get("default_reranker"),
                  llm: Any = None,
                  simple_mode: bool = False):
+        """
+        Initialize the LLMManager with specified models and API keys.
+
+        Args:
+            openai_api_key (Optional[str]): API key for OpenAI.
+            mistral_api_key (Optional[str]): API key for Mistral AI.
+            huggingface_token (Optional[str]): Token for Hugging Face.
+            open_source_models_hg_dir (Optional[str]): Directory for open-source models.
+            embedding_model (Optional[str]): Name or path of the embedding model.
+            rerank_model (Optional[str]): Name or path of the reranking model.
+            llm (Any): Custom LLM instance if provided.
+            simple_mode (bool): If True, skip initialization of embedding and reranking models.
+        """
         with self._instance_lock:   # Prevent re-initialization
             if self._initialized:
                 return
@@ -41,7 +62,7 @@ class LLMManager:
 
         self.simple_mode = simple_mode
 
-        # Initialize for OpenAI GPT-4
+        # Initialize LLM based on provided API keys or custom LLM
         if llm is not None:
             self.llm = llm
             self.model_type = "Custom_langchain_llm"
@@ -59,33 +80,47 @@ class LLMManager:
         # Initialize for Open Source Models
         elif open_source_models_hg_dir is not None and open_source_models_hg_dir != "" and self.llm is None:
             logger.info(f"Initializing {open_source_models_hg_dir}...")
-            # load hugingface models
+            # load huggingface models
             self.model_type = "Hubgingface"
         elif self.llm is None:
             # default model is phi3_mini_128k
             self.model_type = "Hubgingface"
             
         if not self.simple_mode:
-            import torch
+            import sys
+            import platform
+            import subprocess
+
+            def get_device():
+                if sys.platform.startswith('darwin'):  # macOS
+                    # Check for Apple Silicon (M1/M2)
+                    if platform.machine() == 'arm64':
+                        return 'mps'
+                elif sys.platform.startswith('linux') or sys.platform.startswith('win'):
+                    # Check for NVIDIA GPU
+                    try:
+                        subprocess.run(['nvidia-smi'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                        return 'cuda'
+                    except (subprocess.CalledProcessError, FileNotFoundError):
+                        pass
+                
+                # Default to CPU
+                return 'cpu'
+
+            # Usage
+            self.device = get_device()
+
             from transformers import AutoModel
             from transformers import AutoModelForSequenceClassification
             from transformers import AutoTokenizer
-            # initial model_kwargs
-            if torch.cuda.is_available():
-                self.device = torch.device('cuda')
-            elif torch.backends.mps.is_available():
-                self.device = torch.device('mps')
-            else:
-                self.device = torch.device('cpu')
-
-            # initial embedding_model
+            # Initialize embedding model
             if self.tokenizer is None:
                 logger.info(f"Initializing {embedding_model}...")
                 self.tokenizer = AutoTokenizer.from_pretrained(embedding_model)
                 self.embedding_model = AutoModel.from_pretrained(
                     embedding_model, trust_remote_code=True).to(self.device)
 
-            # initial rerank_model
+            # Initialize reranking model
             if self.rerank_model is None:
                 logger.info(f"Initializing {rerank_model}...")
                 self.rerank_model = AutoModelForSequenceClassification.from_pretrained(
@@ -95,16 +130,46 @@ class LLMManager:
             logger.info("Simple mode enabled. Skipping embedding and rerank model initialization.")
 
     def get_llm(self):
+        """
+        Get the main Language Model.
+
+        Returns:
+            The initialized Language Model.
+        """
         return self.llm
 
     def get_llm_simple(self):
+        """
+        Get the simplified Language Model.
+
+        Returns:
+            The initialized simplified Language Model.
+        """
         return self.llm_simple
 
     def get_tokenizer(self):
+        """
+        Get the tokenizer for the embedding model.
+
+        Returns:
+            The initialized tokenizer.
+        """
         return self.tokenizer
 
     def get_rerank_model(self):
+        """
+        Get the reranking model.
+
+        Returns:
+            The initialized reranking model.
+        """
         return self.rerank_model
 
     def get_embedding_model(self):
+        """
+        Get the embedding model.
+
+        Returns:
+            The initialized embedding model.
+        """
         return self.embedding_model

llama_github-0.1.7/llama_github/version.py

@@ -0,0 +1 @@
+__version__ = '0.1.7'

{llama_github-0.1.5 → llama_github-0.1.7/llama_github.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama-github
-Version: 0.1.5
+Version: 0.1.7
 Summary: Llama-github is an open-source Python library that empowers LLM Chatbots, AI Agents, and Auto-dev Agents to conduct Retrieval from actively selected GitHub public projects. It Augments through LLMs and Generates context for any coding question, in order to streamline the development of sophisticated AI-driven applications.
 Home-page: https://github.com/JetXu-LLM/llama-github
 Author: Jet Xu
@@ -19,9 +19,11 @@ Requires-Dist: langchain
 Requires-Dist: numpy
 Requires-Dist: pydantic
 Requires-Dist: requests
-Requires-Dist: torch
 Requires-Dist: transformers
 Requires-Dist: urllib3
+Requires-Dist: sys
+Requires-Dist: platform
+Requires-Dist: subprocess
 Requires-Dist: pydantic_core
 Requires-Dist: langchain_openai
 Requires-Dist: langchain_mistralai
@@ -138,6 +140,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.7] - 2024-10-31
+
+### Optimized
+- fix bugs for get pr content
+
+## [0.1.6] - 2024-10-30
+
+### New Features
+- Enhanced PR content analysis with detailed commit information extraction
+- Improved issue linking detection with support for multiple reference formats
+  - Full GitHub URLs, #references, and keyword-based references
+  - Added validation for issue numbers
+
+### Improvements
+- Added detailed commit metadata extraction including stats and file changes
+- Enhanced error handling for commit fetching
+
 ## [0.1.5] - 2024-10-14
 
 ### Optimized

{llama_github-0.1.5 → llama_github-0.1.7}/llama_github.egg-info/requires.txt

@@ -6,9 +6,11 @@ langchain
 numpy
 pydantic
 requests
-torch
 transformers
 urllib3
+sys
+platform
+subprocess
 pydantic_core
 langchain_openai
 langchain_mistralai

{llama_github-0.1.5 → llama_github-0.1.7}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = llama-github
-version = 0.1.5
+version = 0.1.7
 author = Jet Xu
 author_email = Voldemort.xu@foxmail.com
 description = Llama-github is an open-source Python library that empowers LLM Chatbots, AI Agents, and Auto-dev Agents to conduct Retrieval from actively selected GitHub public projects. It Augments through LLMs and Generates context for any coding question, in order to streamline the development of sophisticated AI-driven applications.

llama_github-0.1.5/llama_github/version.py

@@ -1 +0,0 @@
-__version__ = '0.1.5'