auto-coder 0.1.263__py3-none-any.whl → 0.1.265__py3-none-any.whl

autocoder/common/context_pruner.py

@@ -0,0 +1,305 @@
+from typing import List, Dict, Any, Union
+from pathlib import Path
+import json
+from loguru import logger
+from autocoder.rag.token_counter import count_tokens
+from autocoder.common import AutoCoderArgs,SourceCode
+from byzerllm.utils.client.code_utils import extract_code
+from autocoder.index.types import VerifyFileRelevance
+import byzerllm
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+class PruneContext:
+    def __init__(self, max_tokens: int, args: AutoCoderArgs, llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM]):
+        self.max_tokens = max_tokens
+        self.args = args
+        self.llm = llm
+
+    def _delete_overflow_files(self, file_paths: List[str]) -> List[SourceCode]:
+        """直接删除超出 token 限制的文件"""
+        total_tokens = 0
+        selected_files = []
+        token_count = 0
+        for file_path in file_paths:
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    content = f.read()
+                    token_count = count_tokens(content)
+                    if total_tokens + token_count <= self.max_tokens:
+                        total_tokens += token_count
+                        print(f"{file_path} {token_count} {content}")
+                        selected_files.append(SourceCode(module_name=file_path,source_code=content,tokens=token_count))
+                    else:
+                        break
+            except Exception as e:
+                logger.error(f"Failed to read file {file_path}: {e}")
+                selected_files.append(SourceCode(module_name=file_path,source_code=content,tokens=token_count))
+
+        return selected_files
+
+    def _extract_code_snippets(self, file_paths: List[str], conversations: List[Dict[str, str]]) -> List[SourceCode]:
+        """抽取关键代码片段策略"""
+        token_count = 0
+        selected_files = []
+        full_file_tokens = int(self.max_tokens * 0.8)
+
+        @byzerllm.prompt()
+        def extract_code_snippets(conversations: List[Dict[str, str]], content: str) -> str:
+            """
+            根据提供的代码文件和对话历史提取相关代码片段。            
+
+            处理示例：
+            <examples>
+            1.  代码文件：
+            <code_file>
+                1 def add(a, b):
+                2     return a + b
+                3 def sub(a, b):
+                4     return a - b
+            </code_file>
+            <conversation_history>
+                <user>: 如何实现加法？                
+            </conversation_history>
+
+            输出：
+            ```json
+            [
+                {"start_line": 1, "end_line": 2}                
+            ]
+            ```
+
+            2.  代码文件：
+                1 class User:
+                2     def __init__(self, name):
+                3         self.name = name
+                4     def greet(self):
+                5         return f"Hello, {self.name}"
+            </code_file>
+            <conversation_history>
+                <user>: 如何创建一个User对象？                
+            </conversation_history>
+
+            输出：
+            ```json
+            [
+                {"start_line": 1, "end_line": 3}
+            ]
+            ```
+
+            3.  代码文件：
+            <code_file>
+                1 def foo():
+                2     pass
+            </code_file>
+            <conversation_history>
+                <user>: 如何实现减法？                
+            </conversation_history>
+
+            输出：
+            ```json
+            []
+            ```
+            </examples>
+
+            输入:
+            1. 代码文件内容:
+            <code_file>
+            {{ content }}
+            </code_file>
+
+            2. 对话历史:
+            <conversation_history>
+            {% for msg in conversations %}
+            <{{ msg.role }}>: {{ msg.content }}
+            {% endfor %}
+            </conversation_history>
+
+            任务:
+            1. 分析最后一个用户问题及其上下文。
+            2. 在代码文件中找出与问题相关的一个或多个重要代码段。
+            3. 对每个相关代码段，确定其起始行号(start_line)和结束行号(end_line)。
+            4. 代码段数量不超过4个。
+
+            输出要求:
+            1. 返回一个JSON数组，每个元素包含"start_line"和"end_line"。
+            2. start_line和end_line必须是整数，表示代码文件中的行号。
+            3. 行号从1开始计数。
+            4. 如果没有相关代码段，返回空数组[]。
+
+            输出格式:
+            严格的JSON数组，不包含其他文字或解释。
+
+            ```json
+            [
+                {"start_line": 第一个代码段的起始行号, "end_line": 第一个代码段的结束行号},
+                {"start_line": 第二个代码段的起始行号, "end_line": 第二个代码段的结束行号}
+            ]
+            ```            
+            """
+
+        for file_path in file_paths:
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    content = f.read()
+
+                    # 完整文件优先
+                    tokens = count_tokens(content)
+                    if token_count + tokens <= full_file_tokens:
+                        selected_files.append(SourceCode(module_name=file_path,source_code=content,tokens=tokens))
+                        token_count += tokens
+                        continue
+
+                    # 抽取关键片段
+                    extracted = extract_code_snippets.with_llm(self.llm).run(
+                        conversations=conversations, 
+                        content=content
+                    )                
+
+                    if extracted:
+                        json_str = extract_code(extracted)[0][1]
+                        snippets = json.loads(json_str)
+                        new_content = self._build_snippet_content(file_path, content, snippets)
+
+                        snippet_tokens = count_tokens(new_content)
+                        if token_count + snippet_tokens <= self.max_tokens:
+                            selected_files.append(SourceCode(module_name=file_path,source_code=new_content,tokens=snippet_tokens))
+                            token_count += snippet_tokens
+                        else:
+                            break
+            except Exception as e:
+                logger.error(f"Failed to process {file_path}: {e}")
+                continue
+
+        return selected_files
+
+    def _build_snippet_content(self, file_path: str, full_content: str, snippets: List[dict]) -> str:
+        """构建包含代码片段的文件内容"""
+        lines = full_content.split("\n")
+        header = f"Snippets:\n"
+
+        content = []
+        for snippet in snippets:
+            start = max(0, snippet["start_line"] - 1)
+            end = min(len(lines), snippet["end_line"])
+            content.append(f"# Lines {start+1}-{end} ({snippet.get('reason','')})")
+            content.extend(lines[start:end])
+
+        return header + "\n".join(content)
+
+    def handle_overflow(
+        self,
+        file_paths: List[str],
+        conversations: List[Dict[str, str]],
+        strategy: str = "score"        
+    ) -> List[SourceCode]:
+        """
+        处理超出 token 限制的文件
+        :param file_paths: 要处理的文件路径列表
+        :param conversations: 对话上下文（用于提取策略）
+        :param strategy: 处理策略 (delete/extract/score)        
+        """
+        total_tokens,sources = self._count_tokens(file_paths)
+        if total_tokens <= self.max_tokens:
+            return sources
+
+        if strategy == "score":            
+            return self._score_and_filter_files(file_paths, conversations)
+        if strategy == "delete":
+            return self._delete_overflow_files(file_paths)
+        elif strategy == "extract":
+            return self._extract_code_snippets(file_paths, conversations)
+        else:
+            raise ValueError(f"无效策略: {strategy}. 可选值: delete/extract/score")
+    
+    def _count_tokens(self, file_paths: List[str]) -> int:
+        """计算文件总token数"""
+        total_tokens = 0
+        sources = []
+        for file_path in file_paths:
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    content = f.read()
+                    sources.append(SourceCode(module_name=file_path,source_code=content,tokens=count_tokens(content)))
+                    total_tokens += count_tokens(content)
+            except Exception as e:
+                logger.error(f"Failed to read file {file_path}: {e}")
+                total_tokens += 0
+        return total_tokens,sources
+
+    def _score_and_filter_files(self, file_paths: List[str], conversations: List[Dict[str, str]]) -> List[SourceCode]:
+        """根据文件相关性评分过滤文件，直到token数大于max_tokens 停止追加"""
+        selected_files = []
+        total_tokens = 0
+        scored_files = []        
+
+        @byzerllm.prompt()
+        def verify_file_relevance(file_content: str, conversations: List[Dict[str, str]]) -> str:
+            """
+            请验证下面的文件内容是否与用户对话相关:
+
+            文件内容:
+            {{ file_content }}
+
+            历史对话:
+            <conversation_history>
+            {% for msg in conversations %}
+            <{{ msg.role }}>: {{ msg.content }}
+            {% endfor %}
+            </conversation_history>
+
+            相关是指，需要依赖这个文件提供上下文，或者需要修改这个文件才能解决用户的问题。
+            请给出相应的可能性分数：0-10，并结合用户问题，理由控制在50字以内。格式如下:
+
+            ```json
+            {
+                "relevant_score": 0-10,
+                "reason": "这是相关的原因（不超过10个中文字符）..."
+            }
+            ```
+            """
+
+        def _score_file(file_path: str) -> dict:
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    content = f.read()
+                    tokens = count_tokens(content)
+                    result = verify_file_relevance.with_llm(self.llm).with_return_type(VerifyFileRelevance).run(
+                        file_content=content,
+                        conversations=conversations
+                    )
+                    return {
+                        "file_path": file_path,
+                        "score": result.relevant_score,
+                        "tokens": tokens,
+                        "content": content
+                    }
+            except Exception as e:
+                logger.error(f"Failed to score file {file_path}: {e}")
+                return None
+
+        # 使用线程池并行打分
+        with ThreadPoolExecutor() as executor:
+            futures = [executor.submit(_score_file, file_path) for file_path in file_paths]
+            for future in as_completed(futures):
+                result = future.result()
+                print(f"score file {result['file_path']} {result['score']}")
+                if result:
+                    scored_files.append(result)
+
+        # 第二步：按分数从高到低排序
+        scored_files.sort(key=lambda x: x["score"], reverse=True)
+
+        # 第三步：从高分开始过滤，直到token数大于max_tokens 停止追加
+        for file_info in scored_files:
+            if total_tokens + file_info["tokens"] <= self.max_tokens:
+                selected_files.append(SourceCode(
+                    module_name=file_info["file_path"],
+                    source_code=file_info["content"],
+                    tokens=file_info["tokens"]
+                ))
+                total_tokens += file_info["tokens"]
+            else:
+                break
+
+        return selected_files
+

autocoder/common/files.py

@@ -1,5 +1,5 @@
 from autocoder.common.auto_coder_lang import get_message_with_format
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Generator, Tuple
 
 def read_file(file_path):
     """Read a file with automatic encoding detection.
@@ -30,6 +30,45 @@ def read_file(file_path):
         file_path=file_path, 
         encodings=", ".join(encodings)))
 
+def read_lines(file_path:str):
+    encodings = ['utf-8', 'gbk', 'utf-16', 'latin-1']
+    for encoding in encodings:
+        try:
+            with open(file_path, 'r', encoding=encoding) as f:
+                return f.readlines()
+        except UnicodeDecodeError:
+            continue
+    raise ValueError(get_message_with_format("file_decode_error", 
+        file_path=file_path, 
+        encodings=", ".join(encodings)))
+
+
+
+def read_file_with_line_numbers(file_path: str,line_number_start:int=0) -> Generator[Tuple[int, str], None, None]:
+    """Read a file and return its content with line numbers.
+
+    Args:
+        file_path (str): Path to the file to read
+
+    Returns:
+        List[str]: A list of strings where each string is in the format "line_number:line_content"
+
+    Raises:
+        ValueError: If the file cannot be decoded with any of the tried encodings
+    """
+    encodings = ['utf-8', 'gbk', 'utf-16', 'latin-1']
+    
+    for encoding in encodings:
+        try:
+            with open(file_path, 'r', encoding=encoding) as file:
+                for line_number, line in enumerate(file, start=line_number_start):
+                    yield (line_number, line)                
+        except UnicodeDecodeError:
+            continue
+
+    raise ValueError(get_message_with_format("file_decode_error", 
+        file_path=file_path, 
+        encodings=", ".join(encodings)))
 
 
 def save_file(file_path: str, content: Union[str, List[str]]) -> None:
@@ -55,4 +94,4 @@ def save_file(file_path: str, content: Union[str, List[str]]) -> None:
     except IOError as e:
         raise IOError(get_message_with_format("file_write_error",
             file_path=file_path,
-            error=str(e)))
+            error=str(e)))

autocoder/common/image_to_page.py

@@ -122,7 +122,7 @@ class ImageToPageDirectly:
 
         counter = 1 
         target_html_path = os.path.join(html_dir,f"{html_file_name}-{counter}.html") 
-        with open(target_html_path, "w") as f:
+        with open(target_html_path, "w",encoding="utf-8") as f:
             f.write(html)               
         
         while counter < max_iter:
@@ -137,11 +137,11 @@ class ImageToPageDirectly:
             
             target_html_path = os.path.join(html_dir,f"{html_file_name}-{counter}.html")
             logger.info(f"generate html: {target_html_path}")                
-            with open(target_html_path, "w") as f:
+            with open(target_html_path, "w",encoding="utf-8") as f:
                 f.write(html)            
         
         logger.info(f"finally generate html: {html_path}")
-        with open(html_path, "w") as f:
+        with open(html_path, "w",encoding="utf-8") as f:
             f.write(html)    
              
 
@@ -248,7 +248,7 @@ class ImageToPage:
                 file_path = block.path
                 os.makedirs(os.path.dirname(file_path), exist_ok=True)
 
-                with open(file_path, "w") as f:
+                with open(file_path, "w",encoding="utf-8") as f:
                     logger.info(f"Upsert path: {file_path}")                                       
                     f.write(block.content)
                     file_modified_num += 1
@@ -268,7 +268,7 @@ class ImageToPage:
         ## generate html by image description        
         content_contains_html_prompt = self.generate_html.prompt(desc,html_path) 
 
-        with open(self.args.target_file, "w") as f:
+        with open(self.args.target_file, "w",encoding="utf-8") as f:
             f.write(content_contains_html_prompt) 
 
         t = self.llm.chat_oai(conversations=[{
@@ -278,7 +278,7 @@ class ImageToPage:
 
         content_contains_html = t[0].output 
 
-        with open(self.args.target_file, "w") as f:
+        with open(self.args.target_file, "w",encoding="utf-8") as f:
             f.write(content_contains_html)
         
         
@@ -296,7 +296,7 @@ class ImageToPage:
 
         for i in range(max_iter):
             logger.info(f"iterate  {i}")
-            with open(html_path,"r") as f:
+            with open(html_path,"r",encoding="utf-8") as f:
                 prev_html = f.read()
 
             gen_screenshots(url=html_path,image_dir=new_image_dir)        
@@ -309,7 +309,7 @@ class ImageToPage:
             ## get new description prompt by comparing old and new image
             new_desc_prompt = self.get_optimize(self.score(origin_image,new_image))
 
-            with open(self.args.target_file, "w") as f:
+            with open(self.args.target_file, "w",encoding="utf-8") as f:
                 f.write(new_desc_prompt) 
 
             t = self.llm.chat_oai(conversations=[{
@@ -319,7 +319,7 @@ class ImageToPage:
 
             new_desc = t[0].output 
 
-            with open(self.args.target_file, "w") as f:
+            with open(self.args.target_file, "w",encoding="utf-8") as f:
                 f.write(new_desc)            
 
             logger.info(f"score old/new image: {new_desc}")
@@ -327,7 +327,7 @@ class ImageToPage:
             ## generate new html by new description
             optimze_html_prompt =  self.optimize_html.prompt(desc=new_desc,html=prev_html,html_path=html_path)                                    
             
-            with open(self.args.target_file, "w") as f:
+            with open(self.args.target_file, "w",encoding="utf-8") as f:
                 f.write(optimze_html_prompt) 
 
             t = self.llm.chat_oai(conversations=[{
@@ -336,7 +336,7 @@ class ImageToPage:
             }],llm_config={**extra_llm_config}) 
             new_code = t[0].output
             
-            with open(self.args.target_file, "w") as f:
+            with open(self.args.target_file, "w",encoding="utf-8") as f:
                 f.write(new_code) 
 
             self.write_code(new_code,html_path)

autocoder/common/index_import_export.py

@@ -3,6 +3,9 @@ import json
 import shutil
 from loguru import logger
 from autocoder.common.printer import Printer
+from autocoder.common.result_manager import ResultManager
+
+result_manager = ResultManager()
 
 
 def export_index(project_root: str, export_path: str) -> bool:
@@ -22,11 +25,11 @@ def export_index(project_root: str, export_path: str) -> bool:
         if not os.path.exists(index_path):
             printer.print_in_terminal("index_not_found", path=index_path)
             return False
-            
+
         # Read and convert paths
-        with open(index_path, "r") as f:
+        with open(index_path, "r",encoding="utf-8") as f:
             index_data = json.load(f)
-            
+
         # Convert absolute paths to relative
         converted_data = {}
         for abs_path, data in index_data.items():
@@ -35,21 +38,29 @@ def export_index(project_root: str, export_path: str) -> bool:
                 data["module_name"] = rel_path
                 converted_data[rel_path] = data
             except ValueError:
-                printer.print_in_terminal("index_convert_path_fail", path=abs_path)
+                printer.print_in_terminal(
+                    "index_convert_path_fail", path=abs_path)
                 converted_data[abs_path] = data
-        
+
         # Write to export location
         export_file = os.path.join(export_path, "index.json")
         os.makedirs(export_path, exist_ok=True)
-        with open(export_file, "w") as f:
+        with open(export_file, "w",encoding="utf-8") as f:
             json.dump(converted_data, f, indent=2)
-            
+        printer.print_in_terminal("index_export_success", path=export_file)
+        result_manager.add_result(content=printer.get_message_from_key_with_format("index_export_success", path=export_file), meta={"action": "index_export", "input": {
+            "path": export_file
+        }})
         return True
-        
+
     except Exception as e:
         printer.print_in_terminal("index_error", error=str(e))
+        result_manager.add_result(content=printer.get_message_from_key_with_format("index_error", error=str(e)), meta={"action": "index_export", "input": {
+            "path": export_file
+        }})
         return False
 
+
 def import_index(project_root: str, import_path: str) -> bool:
     printer = Printer()
     """
@@ -67,11 +78,11 @@ def import_index(project_root: str, import_path: str) -> bool:
         if not os.path.exists(import_file):
             printer.print_in_terminal("index_not_found", path=import_file)
             return False
-            
+
         # Read and convert paths
-        with open(import_file, "r") as f:
+        with open(import_file, "r",encoding="utf-8") as f:
             index_data = json.load(f)
-            
+
         # Convert relative paths to absolute
         converted_data = {}
         for rel_path, data in index_data.items():
@@ -80,22 +91,31 @@ def import_index(project_root: str, import_path: str) -> bool:
                 data["module_name"] = abs_path
                 converted_data[abs_path] = data
             except Exception:
-                printer.print_in_terminal("index_convert_path_fail", path=rel_path)
+                printer.print_in_terminal(
+                    "index_convert_path_fail", path=rel_path)
                 converted_data[rel_path] = data
-        
+
         # Backup existing index
         index_path = os.path.join(project_root, ".auto-coder", "index.json")
         if os.path.exists(index_path):
             backup_path = index_path + ".bak"
             shutil.copy2(index_path, backup_path)
             printer.print_in_terminal("index_backup_success", path=backup_path)
-            
+
         # Write new index
-        with open(index_path, "w") as f:
+        with open(index_path, "w",encoding="utf-8") as f:
             json.dump(converted_data, f, indent=2)
-            
-        return True
         
+        printer.print_in_terminal("index_import_success", path=index_path)
+        result_manager.add_result(content=printer.get_message_from_key_with_format("index_import_success", path=index_path), meta={"action": "index_import", "input": {
+            "path": index_path
+        }})
+
+        return True
+
     except Exception as e:
         printer.print_in_terminal("index_error", error=str(e))
-        return False
+        result_manager.add_result(content=printer.get_message_from_key_with_format("index_error", error=str(e)), meta={"action": "index_import", "input": {
+            "path": index_path
+        }})
+        return False

autocoder/common/mcp_hub.py

@@ -116,7 +116,7 @@ class McpHub:
     def _write_default_settings(self):
         """Write default MCP settings file"""
         default_settings = {"mcpServers": {}}
-        with open(self.settings_path, "w") as f:
+        with open(self.settings_path, "w",encoding="utf-8") as f:
             json.dump(default_settings, f, indent=2)
 
     async def add_server_config(self, name: str, config:Dict[str,Any]) -> None:
@@ -129,7 +129,7 @@ class McpHub:
         try:
             settings = self._read_settings()            
             settings["mcpServers"][name] = config
-            with open(self.settings_path, "w") as f:
+            with open(self.settings_path, "w",encoding="utf-8") as f:
                 json.dump(settings, f, indent=2, ensure_ascii=False)
             await self.initialize()
             logger.info(f"Added/updated MCP server config: {name}")
@@ -148,7 +148,7 @@ class McpHub:
             settings = self._read_settings()
             if name in settings["mcpServers"]:
                 del settings["mcpServers"][name]
-                with open(self.settings_path, "w") as f:
+                with open(self.settings_path, "w",encoding="utf-8") as f:
                     json.dump(settings, f, indent=2, ensure_ascii=False)
                 logger.info(f"Removed MCP server config: {name}")
                 await self.initialize()

autocoder/common/mcp_server.py

@@ -80,7 +80,7 @@ def get_mcp_external_servers() -> List[McpExternalServer]:
     if os.path.exists(cache_file):
         cache_time = os.path.getmtime(cache_file)
         if time.time() - cache_time < 3600:  # 1 hour cache
-            with open(cache_file, "r") as f:
+            with open(cache_file, "r",encoding="utf-8") as f:
                 raw_data = json.load(f)
                 return [McpExternalServer(**item) for item in raw_data]
 
@@ -91,7 +91,7 @@ def get_mcp_external_servers() -> List[McpExternalServer]:
         response = requests.get(url)
         if response.status_code == 200:
             raw_data = response.json()
-            with open(cache_file, "w") as f:
+            with open(cache_file, "w",encoding="utf-8") as f:
                 json.dump(raw_data, f)
             return [McpExternalServer(**item) for item in raw_data]
         return []