jarvis-ai-assistant 0.1.104__py3-none-any.whl → 0.1.106__py3-none-any.whl

jarvis/jarvis_code_agent/relevant_files.py

@@ -6,11 +6,170 @@ import yaml
 from jarvis.agent import Agent
 from jarvis.jarvis_code_agent.file_select import select_files
 from jarvis.jarvis_codebase.main import CodeBase
-from jarvis.models.registry import PlatformRegistry
-from jarvis.tools.registry import ToolRegistry
+from jarvis.jarvis_platform.registry import PlatformRegistry
+from jarvis.jarvis_tools.registry import ToolRegistry
 from jarvis.utils import OutputType, PrettyOutput, is_disable_codebase
 
 
+def find_relevant_files_from_agent(user_input: str, files_from_codebase: List[str]) -> List[str]:
+    find_file_tool_registry = ToolRegistry()
+    find_file_tool_registry.use_tools(["read_code", 
+                                     "execute_shell", 
+                                     "lsp_get_document_symbols", 
+                                     "lsp_get_diagnostics", 
+                                     "lsp_find_references", 
+                                     "lsp_find_definition", 
+                                     "lsp_prepare_rename", 
+                                     "lsp_validate_edit"])
+    find_file_agent = Agent(
+        system_prompt="""You are a file agent, you are responsible for finding files related to the user's requirement.
+
+THINKING PROCESS:
+1. Initial File Verification
+   ```
+   Thought: Let me examine the suggested files...
+   Action: For each suggested file:
+     - Use read_code to check content
+     - Use LSP tools to analyze structure
+   Observation: Found that...
+   
+   Thought: Evaluate actual relevance...
+   Action: For each file:
+     - Check direct relationship to requirement
+     - Verify functionality matches
+     - Look for clear evidence of relevance
+   Observation: After analysis:
+     - Relevant files: [list with reasons]
+     - Removed files: [list with reasons]
+   
+   Thought: Verify removal decisions...
+   Action: Double-check each removed file
+   Observation: Removal justification:
+     - File X: [specific reason for removal]
+     - File Y: [specific reason for removal]
+   ```
+
+2. Additional File Search
+   ```
+   Thought: Plan search strategy for missing aspects...
+   Action: Use combination of tools:
+     - git grep for key terms
+     - LSP tools for references
+     - Dependency analysis
+   Observation: Found additional files...
+   
+   Thought: Validate new files...
+   Action: For each new file:
+     - Verify direct relevance
+     - Check for false positives
+     - Document clear evidence
+   Observation: After validation:
+     - Confirmed relevant: [list with evidence]
+     - Excluded: [list with reasons]
+   ```
+
+3. Comprehensive Analysis
+   ```
+   Thought: Final relevance check...
+   Action: For each remaining file:
+     - Verify essential to requirement
+     - Check for indirect inclusions
+     - Validate necessity
+   Observation: Final cleanup:
+     - Core files: [list with roles]
+     - Removed borderline cases: [list with reasons]
+   
+   Thought: Ensure minimal complete set...
+   Action: Review final file list
+   Observation: Confirmed each file is:
+     - Directly relevant
+     - Essential for requirement
+     - Supported by evidence
+   ```
+
+FILE READING GUIDELINES:
+1. For Large Files (>200 lines):
+   ```
+   Thought: This file is large, need targeted reading...
+   Action: 
+     - First: execute_shell("grep -n 'key_term' path/to/file")
+     - Then: read_code("path/to/file", start_line=x-10, end_line=x+20)
+   Observation: Relevance analysis:
+     - Relevant sections: [details]
+     - Irrelevant sections: [reasons to ignore]
+   ```
+
+2. For Small Files:
+   ```
+   Thought: This is a small file, can read entirely...
+   Action: read_code("path/to/file")
+   Observation: Relevance analysis:
+     - Key evidence: [details]
+     - Irrelevant aspects: [what to ignore]
+   ```
+
+VERIFICATION RULES:
+- Remove files without clear relevance evidence
+- Exclude files with only tangential relationships
+- Delete files that only contain imports/references
+- Remove files if relevance is uncertain
+- Document specific reasons for each removal
+- Keep only files essential to requirement
+- Maintain minimal complete set
+
+OUTPUT FORMAT:
+<FILE_PATH>
+- file_path1  # KEEP: [specific evidence of relevance]
+- file_path2  # KEEP: [clear relationship to requirement]
+</FILE_PATH>
+""", 
+        name="FindFileAgent", 
+        is_sub_agent=True,
+        tool_registry=find_file_tool_registry, 
+        platform=PlatformRegistry().get_normal_platform(),
+        auto_complete=True,
+        summary_prompt="""Please provide only the verified essential files with evidence:
+<FILE_PATH>
+- file_path1  # KEEP: [concrete evidence of necessity]
+- file_path2  # KEEP: [specific relevance proof]
+</FILE_PATH>
+""")
+
+    prompt = f"Find files related to: '{user_input}'\n"
+    if files_from_codebase:
+        prompt += f"""
+Potentially related files: {files_from_codebase}
+
+ANALYSIS REQUIRED:
+1. Verify each suggested file:
+   - Document relevance evidence
+   - Identify actual relationships
+   - Note any missing aspects
+
+2. Search for additional files:
+   - Fill coverage gaps
+   - Find related components
+   - Locate test files
+
+3. Provide reasoning:
+   - Explain why each file is included
+   - Document verification process
+   - Note any uncertainties
+"""
+    output = find_file_agent.run(prompt)
+
+    rsp_from_agent = re.findall(r'<FILE_PATH>(.*?)</FILE_PATH>', output, re.DOTALL)
+    files_from_agent = []
+    if rsp_from_agent:
+        try:
+            files_from_agent = yaml.safe_load(rsp_from_agent[0])
+        except Exception as e:
+            files_from_agent = []
+    else:
+        files_from_agent = []
+    return files_from_agent
+
+
 def find_relevant_files(user_input: str, root_dir: str) -> List[str]:
     try:
         files_from_codebase = []
@@ -20,76 +179,8 @@ def find_relevant_files(user_input: str, root_dir: str) -> List[str]:
             files_from_codebase = codebase.search_similar(user_input)
 
         PrettyOutput.print("Find files by agent...", OutputType.INFO)
-        find_file_tool_registry = ToolRegistry()
-        find_file_tool_registry.use_tools(["read_code", "execute_shell"])
-        find_file_agent = Agent(
-            system_prompt="""You are a file agent, you are responsible for finding files related to the user's requirement.
-
-        SEARCH STRATEGY:
-        1. First Pass - Quick Search:
-           - Use `execute_shell` with git grep/find to locate potential files
-           - Search for key terms, function names, and relevant patterns
-           - Example: execute_shell("git grep -l 'search_term'")
-
-        2. Content Analysis:
-           - For each potential file, analyze its content
-           - Follow the file reading guidelines for large files
-           - Look for:
-             * Direct matches to requirement terms
-             * Related functionality
-             * Imported/referenced files
-             * Test files for modified code
-
-        FILE READING GUIDELINES:
-        1. For Large Files (>200 lines):
-           - Do NOT read the entire file at once
-           - First use grep/ctags to locate relevant sections
-           - Then read specific sections with context
-           - Example:
-             * execute_shell("grep -n 'function_name' path/to/file")
-             * read_code("path/to/file", start_line=found_line-10, end_line=found_line+20)
-
-        2. For Small Files:
-           - Can read entire file directly
-
-        IMPORTANT RULES:
-        - Only return files that are DIRECTLY related to the requirement
-        - Exclude false positives and loosely related files
-        - If a file only contains imports/references, don't include it
-        - Include both implementation and test files when relevant
-        - If unsure about a file, use grep/read_code to verify relevance
-        - Return empty list if no truly relevant files are found
-        - Do NOT modify any code, only find files
-
-        OUTPUT FORMAT:
-        - Only provide file paths in the specified YAML format
-        - No additional explanations or comments
-        """, 
-            name="FindFileAgent", 
-            is_sub_agent=True,
-            tool_registry=find_file_tool_registry, 
-            platform=PlatformRegistry().get_normal_platform(),
-            auto_complete=True,
-            summary_prompt="""Please provide the file paths as YAML list:
-            <FILE_PATH>
-            - file_path1
-            - file_path2
-            </FILE_PATH>
-            """)
-        prompt = f"Find files related about '{user_input}'\n"
-        if files_from_codebase:
-            prompt += f"\n\nFiles maybe related: {files_from_codebase}\n\n Please read above files first"
-        output = find_file_agent.run(prompt)
-
-        rsp_from_agent = re.findall(r'<FILE_PATH>(.*?)</FILE_PATH>', output, re.DOTALL)
-        files_from_agent = []
-        if rsp_from_agent:
-            try:
-                files_from_agent = yaml.safe_load(rsp_from_agent[0])
-            except Exception as e:
-                files_from_agent = []
-        else:
-            files_from_agent = []
+
+        files_from_agent = find_relevant_files_from_agent(user_input, files_from_codebase)
 
         selected_files = select_files(files_from_agent, os.getcwd())
         return selected_files

jarvis/jarvis_codebase/main.py

@@ -4,12 +4,10 @@ import numpy as np
 import faiss
 from typing import List, Tuple, Optional, Dict
 
-import yaml
-from jarvis.models.registry import PlatformRegistry
+from jarvis.jarvis_platform.registry import PlatformRegistry
 import concurrent.futures
-from threading import Lock
 from concurrent.futures import ThreadPoolExecutor
-from jarvis.utils import OutputType, PrettyOutput, find_git_root, get_file_md5, get_max_context_length, get_single_line_input, get_thread_count, load_embedding_model, load_rerank_model, user_confirm
+from jarvis.utils import OutputType, PrettyOutput, find_git_root, get_file_md5, get_max_context_length, get_thread_count, load_embedding_model, user_confirm
 from jarvis.utils import init_env
 import argparse
 import pickle
@@ -62,12 +60,11 @@ class CodeBase:
         return [f for f in files if not f.startswith(".jarvis")]
 
     def is_text_file(self, file_path: str):
-        with open(file_path, "r", encoding="utf-8") as f:
-            try:
-                f.read()
-                return True
-            except UnicodeDecodeError:
-                return False
+        try:
+            open(file_path, "r", encoding="utf-8").read()
+            return True
+        except Exception:
+            return False
 
     def make_description(self, file_path: str, content: str) -> str:
         model = PlatformRegistry.get_global_platform_registry().get_cheap_platform()
@@ -386,6 +383,24 @@ Content: {content}
             force: Whether to force rebuild the index, without asking the user
         """
         try:
+            # Clean up cache for non-existent files
+            files_to_delete = []
+            for cached_file in list(self.vector_cache.keys()):
+                if not os.path.exists(cached_file) or not self.is_text_file(cached_file):
+                    files_to_delete.append(cached_file)
+                    cache_path = self._get_cache_path(cached_file)
+                    try:
+                        os.remove(cache_path)
+                    except Exception as e:
+                        PrettyOutput.print(f"Failed to delete cache file for {cached_file}: {str(e)}", 
+                                         output_type=OutputType.WARNING)
+            
+            if files_to_delete:
+                for file_path in files_to_delete:
+                    del self.vector_cache[file_path]
+                PrettyOutput.print(f"Cleaned cache for {len(files_to_delete)} non-existent files", 
+                                 output_type=OutputType.INFO)
+            
             # Update the git file list
             self.git_file_list = self.get_git_file_list()
             
@@ -642,7 +657,7 @@ Note: Only include files that have a strong connection to the query."""
             List[str]: The query variants list
         """
         model = PlatformRegistry.get_global_platform_registry().get_normal_platform()
-        prompt = f"""Please generate 3 different expressions based on the following query, each expression should fully convey the meaning of the original query. These expressions will be used for code search, maintain professionalism and accuracy.
+        prompt = f"""Please generate 3 different expressions based on the following query in English, each expression should fully convey the meaning of the original query. These expressions will be used for code search, maintain professionalism and accuracy.
 Original query: {query}
 
 Please output 3 expressions directly, separated by two line breaks, without numbering or other markers.
@@ -722,19 +737,23 @@ Please output 3 expressions directly, separated by two line breaks, without numb
 
     def ask_codebase(self, query: str, top_k: int=20) -> str:
         """Query the codebase"""
-        results = self.search_similar(query, top_k)
-        if not results:
+        reuslts_from_codebase = self.search_similar(query, top_k)
+        
+        from jarvis.jarvis_code_agent.relevant_files import find_relevant_files_from_agent
+        results_from_agent = find_relevant_files_from_agent(query, reuslts_from_codebase)
+
+        if not results_from_agent:
             PrettyOutput.print("No related files found", output_type=OutputType.WARNING)
             return ""
         
         message = "Found related files:\n"
-        for path in results:
+        for path in results_from_agent:
             message += f"File: {path}\n"
         PrettyOutput.print(message.rstrip(), output_type=OutputType.SUCCESS, lang="markdown")
         
         prompt = f"""You are a code expert, please answer the user's question based on the following file information:
 """
-        for path in results:
+        for path in results_from_agent:
             try:
                 if len(prompt) > self.max_context_length:
                     PrettyOutput.print(f"Avoid context overflow, discard low-related file: {path}", OutputType.WARNING)
@@ -755,6 +774,8 @@ File content:
 User question: {query}
 
 Please answer the user's question in Chinese using professional language. If the provided file content is insufficient to answer the user's question, please inform the user. Never make up information.
+
+Add reference files and code snippets at the end of the answer.
 """
         model = PlatformRegistry.get_global_platform_registry().get_codegen_platform()
         response = model.chat_until_success(prompt)

jarvis/jarvis_lsp/base.py

@@ -0,0 +1,143 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Optional, Tuple, Any
+
+class BaseLSP(ABC):
+    """Base class for Language Server Protocol integration.
+    
+    Core LSP features needed for LLM-based code editing:
+    1. Code navigation and analysis
+    2. Code modification validation
+    3. Diagnostic information
+    4. Symbol analysis
+    """
+    
+    language: str = ""  # Language identifier, should be overridden by subclasses
+    
+    @abstractmethod
+    def initialize(self, workspace_path: str) -> bool:
+        """Initialize LSP server for the workspace.
+        
+        Args:
+            workspace_path: Root path of the workspace
+            
+        Returns:
+            bool: True if initialization successful
+        """
+        return False
+    
+    @abstractmethod
+    def find_references(self, file_path: str, position: Tuple[int, int]) -> List[Dict[str, Any]]:
+        """Find all references of symbol at position.
+        
+        Args:
+            file_path: Path to the file
+            position: (line, character) tuple
+            
+        Returns:
+            List of references with location info:
+            [
+                {
+                    "uri": "file path",
+                    "range": {
+                        "start": {"line": int, "character": int},
+                        "end": {"line": int, "character": int}
+                    }
+                }
+            ]
+        """
+        return []
+    
+    @abstractmethod
+    def find_definition(self, file_path: str, position: Tuple[int, int]) -> Optional[Dict[str, Any]]:
+        """Find definition of symbol at position.
+        
+        Args:
+            file_path: Path to the file
+            position: (line, character) tuple
+            
+        Returns:
+            Location of definition:
+            {
+                "uri": "file path",
+                "range": {
+                    "start": {"line": int, "character": int},
+                    "end": {"line": int, "character": int}
+                }
+            }
+        """
+        return None
+    
+    @abstractmethod
+    def get_document_symbols(self, file_path: str) -> List[Dict[str, Any]]:
+        """Get all symbols in document.
+        
+        Args:
+            file_path: Path to the file
+            
+        Returns:
+            List of symbols with their locations and types
+        """
+        return []
+    
+    @abstractmethod
+    def get_diagnostics(self, file_path: str) -> List[Dict[str, Any]]:
+        """Get diagnostics (errors, warnings) for file.
+        
+        Args:
+            file_path: Path to the file
+            
+        Returns:
+            List of diagnostic items:
+            [
+                {
+                    "range": {
+                        "start": {"line": int, "character": int},
+                        "end": {"line": int, "character": int}
+                    },
+                    "severity": 1 | 2 | 3 | 4,  # Error=1, Warning=2, Info=3, Hint=4
+                    "code": str,                # Error code if any
+                    "source": str,             # Source of diagnostic (e.g. "pylint")
+                    "message": str,            # Diagnostic message
+                    "relatedInformation": [    # Optional related info
+                        {
+                            "location": {
+                                "uri": str,
+                                "range": {...}
+                            },
+                            "message": str
+                        }
+                    ]
+                }
+            ]
+        """
+        return []
+    
+    @abstractmethod
+    def prepare_rename(self, file_path: str, position: Tuple[int, int]) -> Optional[Dict[str, Any]]:
+        """Check if symbol at position can be renamed.
+        
+        Args:
+            file_path: Path to the file
+            position: Symbol position
+            
+        Returns:
+            Range that would be renamed or None if rename not allowed
+        """
+        return None
+    
+    @abstractmethod
+    def validate_edit(self, file_path: str, edit: Dict[str, Any]) -> bool:
+        """Validate if proposed edit is syntactically correct.
+        
+        Args:
+            file_path: Path to the file
+            edit: Edit operation in LSP format
+            
+        Returns:
+            bool: True if edit is valid
+        """
+        return False
+    
+    def shutdown(self):
+        """Shutdown LSP server cleanly."""
+        pass

jarvis/jarvis_lsp/cpp.py

@@ -0,0 +1,134 @@
+import os
+import shutil
+import subprocess
+from typing import List, Dict, Optional, Tuple, Any
+import json
+from jarvis.jarvis_lsp.base import BaseLSP
+from jarvis.utils import PrettyOutput, OutputType
+
+class CPPLSP(BaseLSP):
+    """C++ LSP implementation using clangd."""
+    
+    language = "cpp"
+
+    @staticmethod
+    def check() -> bool:
+        """Check if clangd is installed."""
+        return shutil.which("clangd") is not None
+    
+    def __init__(self):
+        self.workspace_path = ""
+        self.clangd_process = None
+        self.request_id = 0
+    
+    def initialize(self, workspace_path: str) -> bool:
+        try:
+            self.workspace_path = workspace_path
+            # Start clangd process
+            self.clangd_process = subprocess.Popen(
+                ["clangd", "--background-index"],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+            
+            # Send initialize request
+            self._send_request("initialize", {
+                "processId": os.getpid(),
+                "rootUri": f"file://{workspace_path}",
+                "capabilities": {}
+            })
+            
+            return True
+        except Exception as e:
+            PrettyOutput.print(f"C++ LSP initialization failed: {str(e)}", OutputType.ERROR)
+            return False
+    
+    def _send_request(self, method: str, params: Dict) -> Optional[Dict]:
+        """Send JSON-RPC request to clangd."""
+        if not self.clangd_process:
+            return None
+            
+        try:
+            self.request_id += 1
+            request = {
+                "jsonrpc": "2.0",
+                "id": self.request_id,
+                "method": method,
+                "params": params
+            }
+            
+            self.clangd_process.stdin.write(json.dumps(request).encode() + b"\n") # type: ignore
+            self.clangd_process.stdin.flush() # type: ignore
+            
+            response = json.loads(self.clangd_process.stdout.readline().decode()) # type: ignore
+            return response.get("result")
+        except Exception:
+            return None
+    
+    def find_references(self, file_path: str, position: Tuple[int, int]) -> List[Dict[str, Any]]:
+        result = self._send_request("textDocument/references", {
+            "textDocument": {"uri": f"file://{file_path}"},
+            "position": {"line": position[0], "character": position[1]},
+            "context": {"includeDeclaration": True}
+        })
+        return result or [] # type: ignore
+    
+    def find_definition(self, file_path: str, position: Tuple[int, int]) -> Optional[Dict[str, Any]]:
+        result = self._send_request("textDocument/definition", {
+            "textDocument": {"uri": f"file://{file_path}"},
+            "position": {"line": position[0], "character": position[1]}
+        })
+        return result[0] if result else None
+    
+    def get_document_symbols(self, file_path: str) -> List[Dict[str, Any]]:
+        result = self._send_request("textDocument/documentSymbol", {
+            "textDocument": {"uri": f"file://{file_path}"}
+        })
+        return result or [] # type: ignore
+    
+    def get_diagnostics(self, file_path: str) -> List[Dict[str, Any]]:
+        # Send didOpen notification to trigger diagnostics
+        self._send_request("textDocument/didOpen", {
+            "textDocument": {
+                "uri": f"file://{file_path}",
+                "languageId": "cpp",
+                "version": 1,
+                "text": open(file_path).read()
+            }
+        })
+        
+        # Wait for diagnostic notification
+        try:
+            response = json.loads(self.clangd_process.stdout.readline().decode()) # type: ignore
+            if response.get("method") == "textDocument/publishDiagnostics":
+                return response.get("params", {}).get("diagnostics", [])
+        except Exception:
+            pass
+        return []
+    
+    def prepare_rename(self, file_path: str, position: Tuple[int, int]) -> Optional[Dict[str, Any]]:
+        result = self._send_request("textDocument/prepareRename", {
+            "textDocument": {"uri": f"file://{file_path}"},
+            "position": {"line": position[0], "character": position[1]}
+        })
+        return result
+    
+    def validate_edit(self, file_path: str, edit: Dict[str, Any]) -> bool:
+        # Send workspace/willRenameFiles request to check validity
+        result = self._send_request("workspace/willRenameFiles", {
+            "files": [{
+                "oldUri": f"file://{file_path}",
+                "newUri": f"file://{file_path}.tmp"
+            }]
+        })
+        return bool(result)
+    
+    def shutdown(self):
+        if self.clangd_process:
+            try:
+                self._send_request("shutdown", {})
+                self.clangd_process.terminate()
+                self.clangd_process = None
+            except Exception:
+                pass