auto-coder 0.1.352__py3-none-any.whl → 0.1.354__py3-none-any.whl

autocoder/common/v2/agent/agentic_edit_tools/test_search_files_tool_resolver.py

@@ -0,0 +1,163 @@
+import pytest
+import os
+import tempfile
+import shutil
+from unittest.mock import patch, MagicMock
+
+from autocoder.common.v2.agent.agentic_edit_tools.search_files_tool_resolver import SearchFilesToolResolver
+from autocoder.common.v2.agent.agentic_edit_types import SearchFilesTool, ToolResult
+from autocoder.common import AutoCoderArgs
+
+# Helper function to create a directory structure with files for testing
+def create_test_files(base_dir, structure):
+    """
+    Creates a directory structure with files based on the provided dictionary.
+    Keys are filenames (relative to base_dir), values are file contents.
+    Directories are created automatically.
+    """
+    for path, content in structure.items():
+        full_path = os.path.join(base_dir, path)
+        os.makedirs(os.path.dirname(full_path), exist_ok=True)
+        with open(full_path, 'w') as f:
+            f.write(content)
+
+@pytest.fixture
+def search_tool_resolver(temp_search_dir):
+    """Fixture to provide an instance of SearchFilesToolResolver."""
+    # Create AutoCoderArgs with the temp directory as source_dir to allow the security check to pass
+    args = AutoCoderArgs()
+    args.source_dir = temp_search_dir  # Set the source_dir to our temp directory
+    return SearchFilesToolResolver(None, SearchFilesTool(path="", regex=""), args)
+
+@pytest.fixture(scope="function")
+def temp_search_dir():
+    """Fixture to create a temporary directory with test files for searching."""
+    temp_dir = tempfile.mkdtemp()
+    test_structure = {
+        "file1.txt": "Hello world\nThis is a test file.",
+        "subdir/file2.py": "import sys\n\ndef main():\n    print('Python script')\n",
+        "subdir/another.txt": "Another text file with world.",
+        ".hiddenfile": "This should be ignored by default",
+        "no_match.md": "Markdown file."
+    }
+    create_test_files(temp_dir, test_structure)
+    yield temp_dir # Provide the path to the test function
+    shutil.rmtree(temp_dir) # Cleanup after test
+
+# --- Test Cases ---
+
+def test_resolve_finds_matches(search_tool_resolver, temp_search_dir):
+    """Test that resolve finds matches correctly."""
+    # Set up the tool with the pattern we want to search for
+    tool = SearchFilesTool(
+        path="",  # Use empty path to search in the source_dir itself
+        regex="world",
+        file_pattern="*.txt"
+    )
+    search_tool_resolver.tool = tool
+
+    # Call the resolve method directly
+    response = search_tool_resolver.resolve()
+
+    # Check the response
+    assert isinstance(response, ToolResult)
+    assert response.success
+    assert "Search completed. Found 2 matches" in response.message
+    
+    # Check that the correct files were found
+    assert len(response.content) == 2
+    paths = [result["path"] for result in response.content]
+    assert any("file1.txt" in path for path in paths)
+    assert any("another.txt" in path for path in paths)
+    
+    # Check that the match lines contain our search pattern
+    for result in response.content:
+        assert "world" in result["match_line"]
+
+def test_resolve_no_matches(search_tool_resolver, temp_search_dir):
+    """Test that resolve handles no matches correctly."""
+    tool = SearchFilesTool(
+        path="",  # Use empty path to search in the source_dir itself
+        regex="nonexistent_pattern",
+        file_pattern="*"
+    )
+    search_tool_resolver.tool = tool
+
+    response = search_tool_resolver.resolve()
+
+    assert isinstance(response, ToolResult)
+    assert response.success  # Still success, just no results
+    assert "Search completed. Found 0 matches" in response.message
+    assert len(response.content) == 0
+
+def test_resolve_file_pattern(search_tool_resolver, temp_search_dir):
+    """Test that the file_pattern is correctly applied."""
+    # Test .txt pattern
+    tool_txt = SearchFilesTool(
+        path="",  # Use empty path to search in the source_dir itself
+        regex="world",
+        file_pattern="*.txt"  # Only search .txt files
+    )
+    search_tool_resolver.tool = tool_txt
+    
+    response_txt = search_tool_resolver.resolve()
+    
+    assert isinstance(response_txt, ToolResult)
+    assert response_txt.success
+    assert "Search completed. Found 2 matches" in response_txt.message
+    # Ensure only .txt files were matched
+    for result in response_txt.content:
+        assert result["path"].endswith(".txt")
+    
+    # Test .py pattern
+    tool_py = SearchFilesTool(
+        path="",  # Use empty path to search in the source_dir itself
+        regex="print",
+        file_pattern="*.py"  # Only search .py files
+    )
+    search_tool_resolver.tool = tool_py
+    
+    response_py = search_tool_resolver.resolve()
+    
+    assert isinstance(response_py, ToolResult)
+    assert response_py.success
+    assert "Search completed. Found 1 matches" in response_py.message
+    # Ensure only .py files were matched
+    for result in response_py.content:
+        assert result["path"].endswith(".py")
+
+def test_invalid_regex(search_tool_resolver, temp_search_dir):
+    """Test that an invalid regex pattern is properly handled."""
+    tool = SearchFilesTool(
+        path="",  # Use empty path to search in the source_dir itself
+        regex="[invalid regex",  # Invalid regex pattern
+        file_pattern="*"
+    )
+    search_tool_resolver.tool = tool
+
+    response = search_tool_resolver.resolve()
+
+    assert isinstance(response, ToolResult)
+    assert not response.success
+    assert "Invalid regex pattern" in response.message
+
+def test_nonexistent_path(search_tool_resolver, temp_search_dir):
+    """Test that a nonexistent path is properly handled."""
+    # Create a path that we know doesn't exist under temp_search_dir
+    nonexistent_path = "nonexistent_subdirectory"
+    
+    tool = SearchFilesTool(
+        path=nonexistent_path,  # This path doesn't exist in our temp directory
+        regex="pattern",
+        file_pattern="*"
+    )
+    search_tool_resolver.tool = tool
+
+    response = search_tool_resolver.resolve()
+
+    assert isinstance(response, ToolResult)
+    assert not response.success
+    assert "Error: Search path not found" in response.message
+
+# Add more tests as needed
+

autocoder/common/v2/code_editblock_manager.py

@@ -396,24 +396,36 @@ class CodeEditBlockManager:
         def _format_blocks(merge: MergeCodeWithoutEffect) -> Tuple[str, str]:
             unmerged_formatted_text = ""
             for file_path, head, update in merge.failed_blocks:
-                unmerged_formatted_text += "```lang\n"
-                unmerged_formatted_text += f"##File: {file_path}\n"
-                unmerged_formatted_text += "<<<<<<< SEARCH\n"
+                unmerged_formatted_text += "```lang"
+                unmerged_formatted_text += "\n"
+                unmerged_formatted_text += f"##File: {file_path}"
+                unmerged_formatted_text += "\n"
+                unmerged_formatted_text += "<<<<<<< SEARCH"
+                unmerged_formatted_text += "\n"
                 unmerged_formatted_text += head
-                unmerged_formatted_text += "=======\n"
+                unmerged_formatted_text += "\n"
+                unmerged_formatted_text += "======="
+                unmerged_formatted_text += "\n"
                 unmerged_formatted_text += update
-                unmerged_formatted_text += ">>>>>>> REPLACE\n"
+                unmerged_formatted_text += "\n"
+                unmerged_formatted_text += ">>>>>>> REPLACE"
+                unmerged_formatted_text += "\n"
                 unmerged_formatted_text += "```"
                 unmerged_formatted_text += "\n"
 
             merged_formatted_text = ""
             if merge.merged_blocks:
                 for file_path, head, update in merge.merged_blocks:
-                    merged_formatted_text += "```lang\n"
-                    merged_formatted_text += f"##File: {file_path}\n"
+                    merged_formatted_text += "```lang"
+                    merged_formatted_text += "\n"
+                    merged_formatted_text += f"##File: {file_path}"
+                    merged_formatted_text += "\n"
                     merged_formatted_text += head
-                    merged_formatted_text += "=======\n"
+                    merged_formatted_text += "\n"
+                    merged_formatted_text += "======="
+                    merged_formatted_text += "\n"
                     merged_formatted_text += update
+                    merged_formatted_text += "\n"
                     merged_formatted_text += "```"
                     merged_formatted_text += "\n"
 

autocoder/index/index.py

@@ -462,7 +462,7 @@ class IndexManager:
     def filter_exclude_files(self, file_path, exclude_patterns):
         # 增加 ignore_file_utils 的过滤
         try:
-            from src.autocoder.ignorefiles import ignore_file_utils
+            from autocoder.common.ignorefiles import ignore_file_utils
             if ignore_file_utils.should_ignore(file_path):
                 return True
         except Exception:

autocoder/models.py

@@ -60,22 +60,35 @@ default_models_list = [
         "max_output_tokens": 8096
     },    
     {
-        "name": "openrouter/quasar-alpha",
+        "name": "openai/gpt-4.1-mini",
         "description": "",
-        "model_name": "openrouter/quasar-alpha",
+        "model_name": "openai/gpt-4.1-mini",
         "model_type": "saas/openai",
         "base_url": "https://openrouter.ai/api/v1",
         "api_key_path": "",
         "is_reasoning": False,
-        "input_price": 0.0,
-        "output_price": 0.0,
+        "input_price": 2.8,
+        "output_price": 11.2,
         "average_speed": 0.0,
-        "max_output_tokens": 8096*2
+        "max_output_tokens": 8096*3
     },
     {
-        "name": "openrouter/optimus-alpha",
+        "name": "openai/gpt-4.1",
         "description": "",
-        "model_name": "openrouter/optimus-alpha",
+        "model_name": "openai/gpt-4.1",
+        "model_type": "saas/openai",
+        "base_url": "https://openrouter.ai/api/v1",
+        "api_key_path": "",
+        "is_reasoning": False,
+        "input_price": 14.0,
+        "output_price": 42.0,
+        "average_speed": 0.0,
+        "max_output_tokens": 8096*3
+    },   
+    {
+        "name": "openai/gpt-4.1-nano",
+        "description": "",
+        "model_name": "openai/gpt-4.1-nano",
         "model_type": "saas/openai",
         "base_url": "https://openrouter.ai/api/v1",
         "api_key_path": "",
@@ -83,8 +96,8 @@ default_models_list = [
         "input_price": 0.0,
         "output_price": 0.0,
         "average_speed": 0.0,
-        "max_output_tokens": 8096*2
-    },     
+        "max_output_tokens": 8096*3
+    },  
     {
         "name": "openrouter/google/gemini-2.5-pro-preview-03-25",
         "description": "",

autocoder/rag/api_server.py

@@ -187,9 +187,16 @@ async def serve_static_file(full_path: str, request: Request):
         # 直接使用规范化的路径
         file_path = os.path.join("/", os.path.normpath(unquote(full_path)))
         
+        # 获取允许的静态文件目录
+        allowed_static_abs = request.app.state.allowed_static_abs
+        logger.info(f"==allowed_static_abs==: {allowed_static_abs}")
+        
+        if file_path.startswith(("/_images","_images")):            
+            file_path = os.path.join(allowed_static_abs, file_path)  
+
         # 检查文件是否存在
         if not os.path.exists(file_path):
-            raise FileNotFoundError(f"File not found: {file_path}")
+            raise FileNotFoundError(f"File not found: {file_path}")          
         
         # 如果启用了Nginx X-Accel-Redirect，使用X-Accel特性
         if hasattr(request.app.state, "enable_nginx_x_accel") and request.app.state.enable_nginx_x_accel:
@@ -273,6 +280,9 @@ def serve(llm:ByzerLLM, args: ServerArgs):
     allowed_static_abs = os.path.abspath(allowed_static_dir)
     logger.info(f"Static files root directory: {allowed_static_abs}")
     
+    # 将允许的静态文件目录存储到应用状态中
+    router_app.state.allowed_static_abs = allowed_static_abs
+    
     router_app.add_middleware(
         CORSMiddleware,
         allow_origins=args.allowed_origins,
@@ -309,9 +319,11 @@ def serve(llm:ByzerLLM, args: ServerArgs):
                 
                 # Check if path is in allowed directory
                 abs_path = os.path.abspath(os.path.join("/", normalized_path))
+                if abs_path.startswith("/_images"):
+                    return await call_next(request)
                 
                 # 使用预先计算好的allowed_static_abs
-                is_allowed = abs_path.startswith(allowed_static_abs)
+                is_allowed = abs_path.startswith(request.app.state.allowed_static_abs)
                 
                 if not is_allowed:
                     logger.warning(f"Unauthorized path access: {abs_path}")

autocoder/rag/cache/simple_cache.py

@@ -24,6 +24,7 @@ from .failed_files_utils import load_failed_files, save_failed_files
 from autocoder.common import AutoCoderArgs
 from byzerllm import SimpleByzerLLM, ByzerLLM
 from autocoder.utils.llms import get_llm_names
+from autocoder.common.file_monitor.monitor import get_file_monitor, Change
 
 
 default_ignore_dirs = [
@@ -50,7 +51,7 @@ def generate_content_md5(content: Union[str, bytes]) -> str:
 
 
 class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
-    def __init__(self, path: str, ignore_spec, required_exts: list, update_interval: int = 5, args: Optional[AutoCoderArgs] = None, llm: Optional[Union[ByzerLLM, SimpleByzerLLM, str]] = None):
+    def __init__(self, path: str, ignore_spec, required_exts: list, args: Optional[AutoCoderArgs] = None, llm: Optional[Union[ByzerLLM, SimpleByzerLLM, str]] = None):
         """
         初始化异步更新队列，用于管理代码文件的缓存。
 
@@ -58,7 +59,8 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
             path: 需要索引的代码库根目录
             ignore_spec: 指定哪些文件/目录应被忽略的规则
             required_exts: 需要处理的文件扩展名列表
-            update_interval: 自动触发更新的时间间隔（秒），默认为5秒
+            args: AutoCoderArgs 对象，包含配置信息
+            llm: 用于代码分析的 LLM 实例
 
         缓存结构 (self.cache):
             self.cache 是一个字典，其结构如下:
@@ -99,7 +101,6 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
         self.args = args
         self.llm = llm
         self.product_mode = args.product_mode or "lite"
-        self.update_interval = update_interval
         self.queue = []
         self.cache = {}  # 初始化为空字典，稍后通过 read_cache() 填充
         self.lock = threading.Lock()
@@ -115,10 +116,16 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
         self.queue_thread.daemon = True
         self.queue_thread.start()
 
-        # 启动定时触发更新的线程
-        self.update_thread = threading.Thread(target=self._periodic_update)
-        self.update_thread.daemon = True
-        self.update_thread.start()
+        # 注册文件监控回调
+        self.file_monitor = get_file_monitor(self.path)
+        # 注册根目录的监控，这样可以捕获所有子目录和文件的变化
+        self.file_monitor.register(self.path, self._on_file_change)
+        # 确保监控器已启动
+        if not self.file_monitor.is_running():
+            self.file_monitor.start()
+            logger.info(f"Started file monitor for {self.path}")
+        else:
+            logger.info(f"File monitor already running for {self.path}")
 
         self.cache = self.read_cache()
 
@@ -130,37 +137,57 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                 logger.error(f"Error in process_queue: {e}")
             time.sleep(1)  # 避免过于频繁的检查
 
-    def _periodic_update(self):
-        """定时触发文件更新检查"""
-        while not self.stop_event.is_set():
-            try:
-                logger.debug(
-                    f"Periodic update triggered (every {self.update_interval}s)")
-                # 如果没有被初始化过，不会增量触发
-                if not self.cache:
-                    time.sleep(self.update_interval)
-                    continue
-                self.trigger_update()
-            except Exception as e:
-                logger.error(f"Error in periodic update: {e}")
-            time.sleep(self.update_interval)
+    def _on_file_change(self, change_type: Change, file_path: str):
+        """
+        文件监控回调函数，当文件发生变化时触发更新
+        
+        参数:
+            change_type: 变化类型 (Change.added, Change.modified, Change.deleted)
+            file_path: 发生变化的文件路径
+        """
+        try:
+            # 如果缓存还没有初始化，跳过触发
+            if not self.cache:
+                return
+                
+            # 检查文件扩展名，如果不在需要处理的扩展名列表中，跳过
+            if self.required_exts and not any(file_path.endswith(ext) for ext in self.required_exts):
+                return
+                
+            # 检查是否在忽略规则中
+            if self.ignore_spec and self.ignore_spec.match_file(os.path.relpath(file_path, self.path)):
+                return
+                
+            logger.info(f"File change detected: {change_type} - {file_path}")
+            self.trigger_update()
+        except Exception as e:
+            logger.error(f"Error in file change handler: {e}")
+            logger.exception(e)
 
     def stop(self):
         self.stop_event.set()
-        self.queue_thread.join()
-        self.update_thread.join()
+        # 取消注册文件监控回调
+        try:
+            self.file_monitor.unregister(self.path, self._on_file_change)
+            logger.info(f"Unregistered file monitor callback for {self.path}")
+        except Exception as e:
+            logger.error(f"Error unregistering file monitor callback: {e}")
+        # 只等待队列处理线程结束
+        if hasattr(self, 'queue_thread') and self.queue_thread.is_alive():
+            self.queue_thread.join(timeout=2.0)
 
     def fileinfo_to_tuple(self, file_info: FileInfo) -> Tuple[str, str, float, str]:
         return (file_info.file_path, file_info.relative_path, file_info.modify_time, file_info.file_md5)
 
     def __del__(self):
+        # 确保在对象被销毁时停止监控并清理资源
         self.stop()
 
     def load_first(self):
         with self.lock:
             if self.cache:
                 return
-            files_to_process = []
+            files_to_process = []            
             for file_info in self.get_all_files():
                 file_path, _, modify_time, file_md5 = file_info
                 if (
@@ -175,7 +202,7 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
             #     [process_file.remote(file_info) for file_info in files_to_process]
             # )
             from autocoder.rag.token_counter import initialize_tokenizer
-            llm_name = get_llm_names(self.llm)[0] if self.llm else None
+            llm_name = get_llm_names(self.llm)[0] if self.llm else None            
             with Pool(
                 processes=os.cpu_count(),
                 initializer=initialize_tokenizer,
@@ -184,8 +211,8 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                 
                 worker_func = functools.partial(
                     process_file_in_multi_process, llm=llm_name, product_mode=self.product_mode)
-                results = pool.map(worker_func, files_to_process)
-
+                results = pool.map(worker_func, files_to_process)            
+            
             for file_info, result in zip(files_to_process, results):
                 if result:  # 只有当result不为空时才更新缓存
                     self.update_cache(file_info, result)
@@ -203,16 +230,15 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
             file_path, relative_path, modify_time, file_md5 = file_info
             current_files.add(file_path)
             # 如果文件曾经解析失败，跳过本次增量更新
-            if file_path in self.failed_files:
-                # logger.info(f"文件 {file_path} 之前解析失败，跳过此次更新")
+            if file_path in self.failed_files:                
                 continue
-            # 变更检测
+            # 变更检测            
             if (
                 file_path not in self.cache
                 or self.cache[file_path].get("md5", "") != file_md5
             ):
                 files_to_process.append(
-                    (file_path, relative_path, modify_time, file_md5))
+                    (file_path, relative_path, modify_time, file_md5))            
 
         deleted_files = set(self.cache.keys()) - current_files
         logger.info(f"files_to_process: {files_to_process}")
@@ -289,6 +315,8 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                 for line in f:
                     data = json.loads(line)
                     cache[data["file_path"]] = data
+        else:
+            self.load_first()            
         return cache
 
     def write_cache(self):
@@ -366,6 +394,9 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
             dirs[:] = [d for d in dirs if not d.startswith(
                 ".") and d not in default_ignore_dirs]
 
+            # Filter out files that start with a dot
+            files[:] = [f for f in files if not f.startswith(".")]
+
             if self.ignore_spec:
                 relative_root = os.path.relpath(root, self.path)
                 dirs[:] = [
@@ -390,6 +421,5 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                 modify_time = os.path.getmtime(file_path)
                 file_md5 = generate_file_md5(file_path)
                 all_files.append(
-                    (file_path, relative_path, modify_time, file_md5))
-
+                    (file_path, relative_path, modify_time, file_md5))        
         return all_files

autocoder/rag/loaders/docx_loader.py

@@ -1,5 +1,4 @@
 from io import BytesIO
-from autocoder.utils._markitdown import MarkItDown
 import traceback
 
 def extract_text_from_docx_old(docx_path):
@@ -13,6 +12,7 @@ def extract_text_from_docx_old(docx_path):
 
 def extract_text_from_docx(docx_path):
     try:
+        from autocoder.utils._markitdown import MarkItDown
         md_converter = MarkItDown()
         result = md_converter.convert(docx_path)
         return result.text_content