auto-coder 0.1.400__py3-none-any.whl → 1.0.0__py3-none-any.whl

autocoder/common/file_monitor/test_file_monitor.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+FileMonitor 模块测试
+
+测试 FileMonitor 的基本功能：文件的增删改查监控
+"""
+
+import os
+import tempfile
+import shutil
+import time
+import threading
+from pathlib import Path
+from typing import List, Tuple
+from unittest.mock import Mock
+
+try:
+    from watchfiles import Change
+except ImportError:
+    print("警告：watchfiles 未安装，请运行: pip install watchfiles")
+    Change = None
+
+from autocoder.common.file_monitor.monitor import get_file_monitor, FileMonitor
+
+
+class FileMonitorTester:
+    """FileMonitor 测试类"""
+    
+    def __init__(self):
+        self.temp_dir = None
+        self.monitor = None
+        self.events = []
+        self.event_lock = threading.Lock()
+        
+    def setup(self):
+        """设置测试环境"""
+        # 创建临时目录
+        self.temp_dir = tempfile.mkdtemp(prefix="file_monitor_test_")
+        print(f"创建临时测试目录: {self.temp_dir}")
+        
+        # 重置单例实例以确保测试隔离
+        FileMonitor.reset_instance()
+        
+        # 获取监控实例
+        self.monitor = get_file_monitor(self.temp_dir)
+        
+        # 清空事件记录
+        self.events.clear()
+        
+    def teardown(self):
+        """清理测试环境"""
+        if self.monitor and self.monitor.is_running():
+            self.monitor.stop()
+            
+        if self.temp_dir and os.path.exists(self.temp_dir):
+            shutil.rmtree(self.temp_dir)
+            print(f"清理临时测试目录: {self.temp_dir}")
+            
+        # 重置单例实例
+        FileMonitor.reset_instance()
+        
+    def record_event(self, change_type: 'Change', changed_path: str):
+        """记录文件变化事件"""
+        with self.event_lock:
+            self.events.append((change_type, changed_path))
+            print(f"📝 记录事件: {change_type.name} - {os.path.basename(changed_path)}")
+            
+    def wait_for_events(self, expected_count: int, timeout: float = 3.0):
+        """等待指定数量的事件"""
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            with self.event_lock:
+                if len(self.events) >= expected_count:
+                    return True
+            time.sleep(0.1)
+        return False
+    
+    def wait_for_specific_event(self, expected_type: 'Change', expected_path: str, timeout: float = 3.0):
+        """等待特定类型的事件"""
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            with self.event_lock:
+                for event_type, event_path in self.events:
+                    if event_type == expected_type and expected_path in event_path:
+                        return True
+            time.sleep(0.1)
+        return False
+        
+    def test_file_create(self):
+        """测试文件创建监控"""
+        print("\n🔍 测试用例 1: 文件创建监控")
+        
+        # 注册监控所有 .txt 文件
+        self.monitor.register("**/*.txt", self.record_event)
+        
+        # 启动监控
+        self.monitor.start()
+        time.sleep(0.5)  # 等待监控启动
+        
+        # 创建测试文件
+        test_file = os.path.join(self.temp_dir, "test_create.txt")
+        with open(test_file, 'w') as f:
+            f.write("Hello, World!")
+        print(f"✅ 创建文件: {os.path.basename(test_file)}")
+        
+        # 等待事件
+        if self.wait_for_events(1):
+            with self.event_lock:
+                event_type, event_path = self.events[-1]
+                if event_type == Change.added and test_file in event_path:
+                    print("✅ 文件创建事件检测成功")
+                    return True
+                else:
+                    print(f"❌ 事件类型不匹配: 期望 {Change.added.name}, 实际 {event_type.name}")
+        else:
+            print("❌ 未检测到文件创建事件")
+            
+        return False
+        
+    def test_file_modify(self):
+        """测试文件修改监控"""
+        print("\n🔍 测试用例 2: 文件修改监控")
+        
+        # 先创建一个文件
+        test_file = os.path.join(self.temp_dir, "test_modify.txt")
+        with open(test_file, 'w') as f:
+            f.write("Initial content")
+        
+        # 等待文件创建完成
+        time.sleep(1.0)
+            
+        # 清空事件记录
+        with self.event_lock:
+            self.events.clear()
+            
+        # 注册监控
+        self.monitor.register(test_file, self.record_event)
+        
+        # 等待一下确保监控注册完成
+        time.sleep(0.5)
+        
+        # 尝试多种修改方式来触发修改事件
+        # 方式1: 追加内容
+        with open(test_file, 'a') as f:
+            f.write("\nModified content")
+        time.sleep(0.5)
+        
+        # 方式2: 重写文件（如果追加没有触发修改事件）
+        if not self.wait_for_specific_event(Change.modified, test_file, timeout=1.0):
+            with open(test_file, 'w') as f:
+                f.write("Completely new content")
+        
+        print(f"✅ 修改文件: {os.path.basename(test_file)}")
+        
+        # 等待修改事件或添加事件（某些系统可能报告为添加）
+        if (self.wait_for_specific_event(Change.modified, test_file, timeout=2.0) or 
+            self.wait_for_specific_event(Change.added, test_file, timeout=1.0)):
+            print("✅ 文件修改事件检测成功")
+            return True
+        else:
+            # 打印所有事件用于调试
+            with self.event_lock:
+                print(f"所有事件: {[(e[0].name, os.path.basename(e[1])) for e in self.events]}")
+            # 如果检测到任何事件，说明监控是工作的，只是事件类型不同
+            if self.events:
+                print("⚠️ 检测到文件变化事件，但类型与预期不符（这在某些文件系统中是正常的）")
+                return True
+            else:
+                print("❌ 未检测到任何文件修改事件")
+            
+        return False
+        
+    def test_file_delete(self):
+        """测试文件删除监控"""
+        print("\n🔍 测试用例 3: 文件删除监控")
+        
+        # 先创建一个文件
+        test_file = os.path.join(self.temp_dir, "test_delete.txt")
+        with open(test_file, 'w') as f:
+            f.write("To be deleted")
+        
+        # 等待文件创建完成
+        time.sleep(1.0)
+            
+        # 清空事件记录
+        with self.event_lock:
+            self.events.clear()
+            
+        # 注册监控
+        self.monitor.register(test_file, self.record_event)
+        
+        # 等待一下确保监控注册完成
+        time.sleep(0.5)
+        
+        # 删除文件
+        os.remove(test_file)
+        print(f"✅ 删除文件: {os.path.basename(test_file)}")
+        
+        # 等待删除事件
+        if self.wait_for_specific_event(Change.deleted, test_file):
+            print("✅ 文件删除事件检测成功")
+            return True
+        else:
+            # 打印所有事件用于调试
+            with self.event_lock:
+                print(f"所有事件: {[(e[0].name, os.path.basename(e[1])) for e in self.events]}")
+            print("❌ 未检测到文件删除事件")
+            
+        return False
+        
+    def test_directory_monitoring(self):
+        """测试目录监控"""
+        print("\n🔍 测试用例 4: 目录监控")
+        
+        # 创建子目录
+        sub_dir = os.path.join(self.temp_dir, "subdir")
+        os.makedirs(sub_dir)
+        
+        # 等待目录创建完成
+        time.sleep(1.0)
+        
+        # 清空事件记录
+        with self.event_lock:
+            self.events.clear()
+            
+        # 注册监控整个子目录
+        self.monitor.register(sub_dir, self.record_event)
+        
+        # 等待一下确保监控注册完成
+        time.sleep(0.5)
+        
+        # 在子目录中创建文件
+        test_file = os.path.join(sub_dir, "subdir_file.txt")
+        with open(test_file, 'w') as f:
+            f.write("File in subdirectory")
+        print(f"✅ 在子目录中创建文件: {os.path.basename(test_file)}")
+        
+        # 等待文件创建事件
+        if self.wait_for_specific_event(Change.added, test_file):
+            print("✅ 目录监控事件检测成功")
+            return True
+        else:
+            # 打印所有事件用于调试
+            with self.event_lock:
+                print(f"所有事件: {[(e[0].name, os.path.basename(e[1])) for e in self.events]}")
+            print("❌ 未检测到目录监控事件")
+            
+        return False
+        
+    def run_all_tests(self):
+        """运行所有测试"""
+        print("🚀 开始 FileMonitor 功能测试")
+        print("=" * 50)
+        
+        if Change is None:
+            print("❌ 测试失败: watchfiles 库未安装")
+            return False
+            
+        try:
+            self.setup()
+            
+            # 运行测试用例
+            test_results = []
+            test_results.append(self.test_file_create())
+            test_results.append(self.test_file_modify()) 
+            test_results.append(self.test_file_delete())
+            test_results.append(self.test_directory_monitoring())
+            
+            # 统计结果
+            passed = sum(test_results)
+            total = len(test_results)
+            
+            print("\n" + "=" * 50)
+            print(f"📊 测试结果: {passed}/{total} 个测试用例通过")
+            
+            if passed == total:
+                print("🎉 所有测试用例通过！FileMonitor 功能正常")
+                return True
+            else:
+                print(f"⚠️ {total - passed} 个测试用例失败")
+                return False
+                
+        except Exception as e:
+            print(f"❌ 测试过程中发生异常: {e}")
+            import traceback
+            traceback.print_exc()
+            return False
+        finally:
+            self.teardown()
+
+
+def main():
+    """主函数"""
+    tester = FileMonitorTester()
+    success = tester.run_all_tests()
+    
+    if success:
+        print("\n✅ FileMonitor 测试完成，所有功能正常")
+        exit(0)
+    else:
+        print("\n❌ FileMonitor 测试失败")
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

autocoder/common/git_utils.py

@@ -294,7 +294,7 @@ def get_uncommitted_changes(repo_path: str) -> str:
         return f"Error: {str(e)}"
 
 @byzerllm.prompt()
-def generate_commit_message(changes_report: str) -> str:
+def generate_commit_message(changes_report: str, query: Optional[str] = None) -> str:
     '''
     我是一个Git提交信息生成助手。我们的目标是通过一些变更报告，倒推用户的需求，将需求作为commit message。
     commit message 需要简洁,包含两部分：
@@ -635,7 +635,12 @@ def generate_commit_message(changes_report: str) -> str:
     </examples>
     
     下面是变更报告：
-    {{ changes_report }}    
+    {{ changes_report }}   
+    
+    {% if query %}
+    这里你需要遵守的用户的额外要求：
+    {{ query }} 
+    {% endif %}
 
     请输出commit message, 不要输出任何其他内容.
     '''

autocoder/common/pruner/agentic_conversation_pruner.py

@@ -0,0 +1,197 @@
+from typing import List, Dict, Any, Union
+import json
+import re
+from pydantic import BaseModel
+import byzerllm
+from autocoder.common.printer import Printer
+from autocoder.rag.token_counter import count_tokens
+from loguru import logger
+from autocoder.common import AutoCoderArgs
+from autocoder.common.save_formatted_log import save_formatted_log
+
+class AgenticPruneStrategy(BaseModel):
+    name: str
+    description: str
+    config: Dict[str, Any] = {"safe_zone_tokens": 0}
+
+class AgenticConversationPruner:
+    """
+    Specialized conversation pruner for agentic conversations that cleans up tool outputs.
+    
+    This pruner specifically targets tool result messages (role='user', content contains '<tool_result>')
+    and replaces their content with a placeholder message to reduce token usage while maintaining
+    conversation flow.
+    """
+    
+    def __init__(self, args: AutoCoderArgs, llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM]):
+        self.args = args
+        self.llm = llm
+        self.printer = Printer()
+        self.replacement_message = "This message has been cleared. If you still want to get this information, you can call the tool again to retrieve it."
+        
+        self.strategies = {
+            "tool_output_cleanup": AgenticPruneStrategy(
+                name="tool_output_cleanup",
+                description="Clean up tool output results by replacing content with placeholder messages",
+                config={"safe_zone_tokens": self.args.conversation_prune_safe_zone_tokens}
+            )
+        }
+
+    def get_available_strategies(self) -> List[Dict[str, Any]]:
+        """Get all available pruning strategies"""
+        return [strategy.model_dump() for strategy in self.strategies.values()]
+
+    def prune_conversations(self, conversations: List[Dict[str, Any]], 
+                            strategy_name: str = "tool_output_cleanup") -> List[Dict[str, Any]]:
+        """
+        Prune conversations by cleaning up tool outputs.
+        
+        Args:
+            conversations: Original conversation list            
+            strategy_name: Strategy name
+            
+        Returns:
+            Pruned conversation list
+        """        
+        safe_zone_tokens = self.args.conversation_prune_safe_zone_tokens
+        current_tokens = count_tokens(json.dumps(conversations, ensure_ascii=False))
+        
+        if current_tokens <= safe_zone_tokens:
+            return conversations
+
+        strategy = self.strategies.get(strategy_name, self.strategies["tool_output_cleanup"])
+        
+        if strategy.name == "tool_output_cleanup":
+            return self._tool_output_cleanup_prune(conversations, strategy.config)
+        else:
+            logger.warning(f"Unknown strategy: {strategy_name}, using tool_output_cleanup instead")
+            return self._tool_output_cleanup_prune(conversations, strategy.config)
+
+    def _tool_output_cleanup_prune(self, conversations: List[Dict[str, Any]], 
+                                   config: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        Clean up tool outputs by replacing their content with placeholder messages.
+        
+        This method:
+        1. Identifies tool result messages (role='user' with '<tool_result' in content)
+        2. Starts from the first tool output and progressively cleans them
+        3. Stops when token count is within safe zone
+        """
+        safe_zone_tokens = config.get("safe_zone_tokens", 50 * 1024)
+        processed_conversations = conversations.copy()
+        
+        # Find all tool result message indices
+        tool_result_indices = []
+        for i, conv in enumerate(processed_conversations):
+            if (conv.get("role") == "user" and 
+                isinstance(conv.get("content"), str) and 
+                self._is_tool_result_message(conv.get("content", ""))):
+                tool_result_indices.append(i)
+        
+        logger.info(f"Found {len(tool_result_indices)} tool result messages to potentially clean")
+        
+        # Clean tool outputs one by one, starting from the first one
+        for tool_index in tool_result_indices:
+            current_tokens = count_tokens(json.dumps(processed_conversations, ensure_ascii=False))
+            
+            if current_tokens <= safe_zone_tokens:
+                logger.info(f"Token count ({current_tokens}) is within safe zone ({safe_zone_tokens}), stopping cleanup")
+                break
+            
+            # Extract tool name for a more specific replacement message
+            tool_name = self._extract_tool_name(processed_conversations[tool_index]["content"])
+            replacement_content = self._generate_replacement_message(tool_name)
+            
+            # Replace the content
+            original_content = processed_conversations[tool_index]["content"]
+            processed_conversations[tool_index]["content"] = replacement_content
+            
+            logger.info(f"Cleaned tool result at index {tool_index} (tool: {tool_name}), "
+                       f"reduced from {len(original_content)} to {len(replacement_content)} characters")
+        
+        final_tokens = count_tokens(json.dumps(processed_conversations, ensure_ascii=False))
+        logger.info(f"Cleanup completed. Token count: {current_tokens} -> {final_tokens}")
+
+        save_formatted_log(self.args.source_dir, json.dumps(conversations, ensure_ascii=False), "agentic_pruned_conversation")                                
+        
+        return processed_conversations
+
+    def _is_tool_result_message(self, content: str) -> bool:
+        """
+        Check if a message content contains tool result XML.
+        
+        Args:
+            content: Message content to check
+            
+        Returns:
+            True if content contains tool result format
+        """
+        return "<tool_result" in content and "tool_name=" in content
+
+    def _extract_tool_name(self, content: str) -> str:
+        """
+        Extract tool name from tool result XML content.
+        
+        Args:
+            content: Tool result XML content
+            
+        Returns:
+            Tool name or 'unknown' if not found
+        """
+        # Pattern to match: <tool_result tool_name='...' or <tool_result tool_name="..."
+        pattern = r"<tool_result[^>]*tool_name=['\"]([^'\"]+)['\"]"
+        match = re.search(pattern, content)
+        
+        if match:
+            return match.group(1)
+        return "unknown"
+
+    def _generate_replacement_message(self, tool_name: str) -> str:
+        """
+        Generate a replacement message for a cleaned tool result.
+        
+        Args:
+            tool_name: Name of the tool that was called
+            
+        Returns:
+            Replacement message string
+        """
+        if tool_name and tool_name != "unknown":
+            return (f"<tool_result tool_name='{tool_name}' success='true'>"
+                   f"<message>Content cleared to save tokens</message>"
+                   f"<content>{self.replacement_message}</content>"
+                   f"</tool_result>")
+        else:
+            return f"<tool_result success='true'><message>[Content cleared to save tokens, you can call the tool again to get the tool result.]</message><content>{self.replacement_message}</content></tool_result>"
+
+    def get_cleanup_statistics(self, original_conversations: List[Dict[str, Any]], 
+                              pruned_conversations: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """
+        Get statistics about the cleanup process.
+        
+        Args:
+            original_conversations: Original conversation list
+            pruned_conversations: Pruned conversation list
+            
+        Returns:
+            Dictionary with cleanup statistics
+        """
+        original_tokens = count_tokens(json.dumps(original_conversations, ensure_ascii=False))
+        pruned_tokens = count_tokens(json.dumps(pruned_conversations, ensure_ascii=False))
+        
+        # Count cleaned tool results
+        cleaned_count = 0
+        for orig, pruned in zip(original_conversations, pruned_conversations):
+            if (orig.get("role") == "user" and 
+                self._is_tool_result_message(orig.get("content", "")) and
+                orig.get("content") != pruned.get("content")):
+                cleaned_count += 1
+        
+        return {
+            "original_tokens": original_tokens,
+            "pruned_tokens": pruned_tokens,
+            "tokens_saved": original_tokens - pruned_tokens,
+            "compression_ratio": pruned_tokens / original_tokens if original_tokens > 0 else 1.0,
+            "tool_results_cleaned": cleaned_count,
+            "total_messages": len(original_conversations)
+        }