auto-coder 0.1.263__py3-none-any.whl → 0.1.265__py3-none-any.whl

autocoder/index/filter/quick_filter.py

@@ -20,6 +20,8 @@ from byzerllm import MetaHolder
 
 from autocoder.utils.llms import get_llm_names, get_model_info
 from loguru import logger
+from byzerllm.utils.client.code_utils import extract_code
+import json
 
 
 def get_file_path(file_path):
@@ -32,6 +34,15 @@ class QuickFilterResult(BaseModel):
     files: Dict[str, TargetFile]
     has_error: bool
     error_message: Optional[str] = None
+    file_positions: Optional[Dict[str, int]] = {}
+
+    def get_sorted_file_positions(self) -> List[str]:
+        """
+        返回按 value 排序的文件列表
+        """
+        if not self.file_positions:
+            return []
+        return [file_path for file_path, _ in sorted(self.file_positions.items(), key=lambda x: x[1])]
 
 
 class QuickFilter():
@@ -82,6 +93,7 @@ class QuickFilter():
                     self.index_manager.index_filter_llm)
                 model_name = ",".join(model_names)
                 files: Dict[str, TargetFile] = {}
+                file_positions: Dict[str, int] = {}
 
                 # 获取模型价格信息
                 model_info_map = {}
@@ -166,7 +178,7 @@ class QuickFilter():
                     )
 
                 if file_number_list:
-                    for file_number in file_number_list.file_list:
+                    for index,file_number in enumerate(file_number_list.file_list):
                         if file_number < 0 or file_number >= len(chunk):
                             self.printer.print_in_terminal(
                                 "invalid_file_number",
@@ -182,9 +194,11 @@ class QuickFilter():
                             reason=self.printer.get_message_from_key(
                                 "quick_filter_reason")
                         )
+                        file_positions[file_path] = index
                 return QuickFilterResult(
                     files=files,
-                    has_error=False
+                    has_error=False,
+                    file_positions=file_positions
                 )
 
             except Exception as e:
@@ -212,6 +226,7 @@ class QuickFilter():
 
         # 合并所有结果
         final_files: Dict[str, TargetFile] = {}
+        final_file_positions: Dict[str, int] = {}
         has_error = False
         error_messages: List[str] = []
 
@@ -222,16 +237,57 @@ class QuickFilter():
                     error_messages.append(result.error_message)
             final_files.update(result.files)
 
+
+        for result in results:
+            if result.has_error:
+                has_error = True
+                if result.error_message:
+                    error_messages.append(result.error_message)
+            ## 实现多个 result.file_positions 交织排序
+            # 比如第一个是 {file_path_1_0: 0, file_path_1_1: 1, file_path_1_2: 2}    
+            # 第二个是 {file_path_2_0: 0, file_path_2_1: 1}
+            # 第三个是 {file_path_3_0: 0, file_path_3_1: 1, file_path_3_2: 2, file_path_3_3: 3}
+            # 收集逻辑为所以 0 的为一组，然后序号为 0,1,2, 所有1 的为一组，序号是 3,4,5,依次往下推
+            # {file_path_1_0: 0, file_path_2_0: 1, file_path_3_0: 2, file_path_1_1: 3, file_path_2_1: 4, file_path_3_1: 5}            
+            # 
+            # 获取所有结果的最大 position 值
+            max_position = max([max(pos.values()) for pos in [result.file_positions for result in results if result.file_positions]] + [0])
+
+            # 创建一个映射表，用于记录每个 position 对应的文件路径
+            position_map = {}
+            for result in results:
+                if result.file_positions:
+                    for file_path, position in result.file_positions.items():
+                        if position not in position_map:
+                            position_map[position] = []
+                        position_map[position].append(file_path)
+
+            # 重新排序文件路径
+            new_file_positions = {}
+            current_index = 0
+            for position in range(max_position + 1):
+                if position in position_map:
+                    for file_path in position_map[position]:
+                        new_file_positions[file_path] = current_index
+                        current_index += 1
+
+            # 更新 final_file_positions
+            final_file_positions.update(new_file_positions)
+
         return QuickFilterResult(
             files=final_files,
             has_error=has_error,
             error_message="\n".join(error_messages) if error_messages else None
         )
+    
 
     @byzerllm.prompt()
     def quick_filter_files(self, file_meta_list: List[IndexItem], query: str) -> str:
         '''
-        当用户提一个需求的时候，我们需要找到相关的文件，然后阅读这些文件，并且修改其中部分文件。
+        当用户提一个需求的时候，我们要找到两种类型的源码文件：
+        1. 根据需求需要被修改的文件，我们叫 edited_files
+        2. 为了能够完成修改这些文件，还需要的一些额外参考文件, 我们叫 reference_files
+
         现在，给定下面的索引文件：
 
         <index>
@@ -258,12 +314,13 @@ class QuickFilter():
         }
         ```
 
-        特别注意    
-        1. 如果用户的query里 @文件 或者 @@符号，那么被@的文件或者@@的符号必须要返回，并且尝试通过索引文件诸如导入语句等信息找到这些文件依赖的其他文件，再分析这些文件是否需要提供才能满足后续编码。
-        2. 如果 query 里是一段历史对话，那么对话里的内容提及的文件路径必须要返回。
-        3. 想想，如果是你需要修改代码，然后满足这个需求，根据索引文件，你希望查看哪些文件，修改哪些文件，然后返回这些文件。
-        4. 如果用户需求为空，则直接返回空列表即可。
-        5. 返回的 json格式数据不允许有注释
+        特别注意:    
+        1. 如果用户的query里有 @文件 或者 @@符号，那么被@的文件或者@@的符号必须要返回。
+        2. 根据需求以及根据 @文件 或者 @@符号 找到的文件，猜测需要被修改的edited_files文件，然后尝试通过索引文件诸如导入语句等信息找到这些文件依赖的其他文件得到 reference_files。
+        3. file_list 里的文件序号，按被 @ 或者 @@ 文件，edited_files文件，reference_files文件的顺序排列。注意，reference_files 你要根据需求来猜测是否需要，过滤掉不相关的，避免返回文件数过多。
+        4. 如果 query 里是一段历史对话，那么对话里的内容提及的文件路径必须要返回。        
+        5. 如果用户需求为空，则直接返回空列表即可。        
+        6. 返回的 json格式数据不允许有注释
         '''
 
         file_meta_str = "\n".join(
@@ -273,9 +330,58 @@ class QuickFilter():
             "query": query            
         }
         return context
+    
+
+    def _extract_code_snippets_from_overflow_files(self, validated_file_numbers: List[int],index_items: List[IndexItem], conversations: List[Dict[str, str]]):
+        token_count = 0        
+        selected_files = []
+        selected_file_contents = []
+        full_file_tokens = int(self.max_tokens * 0.8)
+        for file_number in validated_file_numbers:
+            file_path = get_file_path(index_items[file_number].module_name)
+            with open(file_path, "r", encoding="utf-8") as f:
+                content = f.read()
+            tokens = count_tokens(content)
+            if token_count + tokens <= full_file_tokens:
+                selected_files.append(file_number)
+                selected_file_contents.append(content)
+                token_count += tokens
+            else:
+                # 对超出部分抽取代码片段
+                try:
+                    extracted_info = (
+                        self.extract_code_snippets_from_files.options(
+                            {"llm_config": {"max_length": 100}}
+                        )
+                        .with_llm(self.index_manager.index_filter_llm)
+                        .run(conversations, [content])
+                    )
+                    json_str = extract_code(extracted_info)[0][1]
+                    json_objs = json.loads(json_str)
+
+                    new_content = ""
+
+                    if json_objs:                        
+                        for json_obj in json_objs:
+                            start_line = json_obj["start_line"] - 1
+                            end_line = json_obj["end_line"]
+                            chunk = "\n".join(content.split("\n")[start_line:end_line])
+                            new_content += chunk + "\n"
+                      
+                        token_count += count_tokens(new_content)
+                        if token_count >= self.max_tokens:
+                            break
+                        else:
+                            selected_files.append(file_number)
+                            selected_file_contents.append(new_content)
+                except Exception as e:
+                    logger.error(f"Failed to extract code snippets from {file_path}: {e}")
+        return selected_files
+
 
     def filter(self, index_items: List[IndexItem], query: str) -> QuickFilterResult:
         final_files: Dict[str, TargetFile] = {}
+        final_file_positions: Dict[str, int] = {}
         start_time = time.monotonic()
 
         prompt_str = self.quick_filter_files.prompt(index_items, query)
@@ -385,6 +491,7 @@ class QuickFilter():
             )
 
         if file_number_list:
+            validated_file_numbers = []
             for file_number in file_number_list.file_list:
                 if file_number < 0 or file_number >= len(index_items):
                     self.printer.print_in_terminal(
@@ -394,14 +501,21 @@ class QuickFilter():
                         total_files=len(index_items)
                     )
                     continue
-                final_files[get_file_path(index_items[file_number].module_name)] = TargetFile(
+                validated_file_numbers.append(file_number)
+                                
+            # 将最终选中的文件加入final_files
+            for index,file_number in enumerate(validated_file_numbers):
+                file_path = get_file_path(index_items[file_number].module_name)
+                final_files[file_path] = TargetFile(
                     file_path=index_items[file_number].module_name,
-                    reason=self.printer.get_message_from_key(
-                        "quick_filter_reason")
+                    reason=self.printer.get_message_from_key("quick_filter_reason")
                 )
+                final_file_positions[file_path] = index
+
         end_time = time.monotonic()
         self.stats["timings"]["quick_filter"] = end_time - start_time
         return QuickFilterResult(
             files=final_files,
-            has_error=False
+            has_error=False,
+            file_positions=final_file_positions
         )

autocoder/index/index.py

@@ -26,6 +26,7 @@ from autocoder.index.types import (
 )
 from autocoder.common.global_cancel import global_cancel
 from autocoder.utils.llms import get_llm_names
+from autocoder.rag.token_counter import count_tokens
 class IndexManager:
     def __init__(
         self, llm: byzerllm.ByzerLLM, sources: List[SourceCode], args: AutoCoderArgs
@@ -257,13 +258,13 @@ class IndexManager:
             total_input_cost = 0.0
             total_output_cost = 0.0
             
-            if len(source.source_code) > self.max_input_length:
+            if count_tokens(source.source_code) > self.args.conversation_prune_safe_zone_tokens:
                 self.printer.print_in_terminal(
                     "index_file_too_large",
                     style="yellow",
                     file_path=source.module_name,
                     file_size=len(source.source_code),
-                    max_length=self.max_input_length
+                    max_length=self.args.conversation_prune_safe_zone_tokens
                 )
                 chunks = self.split_text_into_chunks(
                     source_code, self.max_input_length - 1000
@@ -341,7 +342,7 @@ class IndexManager:
 
     def build_index(self):
         if os.path.exists(self.index_file):
-            with open(self.index_file, "r") as file:
+            with open(self.index_file, "r",encoding="utf-8") as file:
                 index_data = json.load(file)
         else:
             index_data = {}
@@ -432,13 +433,13 @@ class IndexManager:
                     index_data[module_name] = result
                     updated_sources.append(module_name)
                     if len(updated_sources) > 5:
-                        with open(self.index_file, "w") as file:
+                        with open(self.index_file, "w",encoding="utf-8") as file:
                             json.dump(index_data, file, ensure_ascii=False, indent=2)
                         updated_sources = []
         
         # 如果 updated_sources 或 keys_to_remove 有值，则保存索引文件
         if updated_sources or keys_to_remove:
-            with open(self.index_file, "w") as file:
+            with open(self.index_file, "w",encoding="utf-8") as file:
                 json.dump(index_data, file, ensure_ascii=False, indent=2)
 
             print("")    
@@ -460,14 +461,14 @@ class IndexManager:
         if not os.path.exists(self.index_file):
             return []
 
-        with open(self.index_file, "r") as file:
+        with open(self.index_file, "r",encoding="utf-8") as file:
             return file.read()
 
     def read_index(self) -> List[IndexItem]:
         if not os.path.exists(self.index_file):
             return []
 
-        with open(self.index_file, "r") as file:
+        with open(self.index_file, "r",encoding="utf-8") as file:
             index_data = json.load(file)
 
         index_items = []

autocoder/models.py

@@ -97,7 +97,7 @@ def load_models() -> List[Dict]:
         if model.get("api_key_path",""):           
             api_key_file = os.path.join(api_key_dir, model["api_key_path"])
             if os.path.exists(api_key_file):
-                with open(api_key_file, "r") as f:
+                with open(api_key_file, "r",encoding="utf-8") as f:
                     model["api_key"] = f.read()                   
     return target_models
 
@@ -269,7 +269,7 @@ def update_model_with_api_key(name: str, api_key: str) -> Dict:
         api_key_dir = os.path.expanduser("~/.auto-coder/keys")
         os.makedirs(api_key_dir, exist_ok=True)
         api_key_file = os.path.join(api_key_dir, api_key_path)
-        with open(api_key_file, "w") as f:
+        with open(api_key_file, "w",encoding="utf-8") as f:
             f.write(api_key.strip())
         
         # 如果是新模型，添加到模型列表中

autocoder/pyproject/__init__.py

@@ -174,7 +174,8 @@ class PyProject:
         return False
 
     def output(self):
-        return open(self.target_file, "r").read()
+        with open(self.target_file, "r",encoding="utf-8") as file:
+            return file.read()
 
     def is_python_file(self, file_path):
         return file_path.endswith(".py")
@@ -182,9 +183,8 @@ class PyProject:
     def read_file_content(self, file_path):
         if self.args.auto_merge == "strict_diff":
             result = []
-            with open(file_path, "r") as file:
-                for line_number, line in enumerate(file, start=1):
-                    result.append(f"{line_number}:{line}")
+            for line_number, line in FileUtils.read_file_with_line_numbers(file_path,line_number_start=1):
+                result.append(f"{line_number}:{line}")
             return "\n".join(result)
 
         return FileUtils.read_file(file_path)
@@ -357,7 +357,7 @@ class PyProject:
             self.clone_repository()
 
         if self.target_file:
-            with open(self.target_file, "w") as file:
+            with open(self.target_file, "w",encoding="utf-8") as file:
 
                 for code in self.get_rest_source_codes():
                     self.sources.append(code)                    

autocoder/rag/cache/byzer_storage_cache.py

@@ -126,7 +126,7 @@ class ByzerStorageCache(BaseCacheManager):
         """Load cache from file"""
         if os.path.exists(self.cache_file):
             try:
-                with open(self.cache_file, "r") as f:
+                with open(self.cache_file, "r",encoding="utf-8") as f:
                     lines = f.readlines()
                     cache = {}
                     for line in lines:
@@ -147,7 +147,7 @@ class ByzerStorageCache(BaseCacheManager):
 
         if not fcntl:
             try:
-                with open(cache_file, "w") as f:
+                with open(cache_file, "w",encoding="utf-8") as f:
                     for data in self.cache.values():
                         json.dump(data, f, ensure_ascii=False)
                         f.write("\n")
@@ -155,12 +155,12 @@ class ByzerStorageCache(BaseCacheManager):
                 logger.error(f"Error writing cache file: {str(e)}")
         else:
             lock_file = cache_file + ".lock"
-            with open(lock_file, "w") as lockf:
+            with open(lock_file, "w",encoding="utf-8") as lockf:
                 try:
                     # 获取文件锁
                     fcntl.flock(lockf, fcntl.LOCK_EX | fcntl.LOCK_NB)
                     # 写入缓存文件
-                    with open(cache_file, "w") as f:
+                    with open(cache_file, "w",encoding="utf-8") as f:
                         for data in self.cache.values():
                             json.dump(data, f, ensure_ascii=False)
                             f.write("\n")

autocoder/rag/cache/file_monitor_cache.py

@@ -106,11 +106,11 @@ class AutoCoderRAGDocListener(BaseCacheManager):
         gitignore_path = os.path.join(self.path, ".gitignore")
 
         if os.path.exists(serveignore_path):
-            with open(serveignore_path, "r") as ignore_file:
+            with open(serveignore_path, "r",encoding="utf-8") as ignore_file:
                 patterns = ignore_file.readlines()
                 return [pattern.strip() for pattern in patterns]
         elif os.path.exists(gitignore_path):
-            with open(gitignore_path, "r") as ignore_file:
+            with open(gitignore_path, "r",encoding="utf-8") as ignore_file:
                 patterns = ignore_file.readlines()
                 return [pattern.strip() for pattern in patterns]
         return []

autocoder/rag/cache/simple_cache.py

@@ -160,7 +160,7 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
 
         cache = {}
         if os.path.exists(cache_file):
-            with open(cache_file, "r") as f:
+            with open(cache_file, "r",encoding="utf-8") as f:
                 for line in f:
                     data = json.loads(line)
                     cache[data["file_path"]] = data
@@ -171,7 +171,7 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
         cache_file = os.path.join(cache_dir, "cache.jsonl")
 
         if not fcntl:
-            with open(cache_file, "w") as f:
+            with open(cache_file, "w",encoding="utf-8") as f:
                 for data in self.cache.values():
                     try:
                         json.dump(data, f, ensure_ascii=False)
@@ -181,12 +181,12 @@ class AutoCoderRAGAsyncUpdateQueue(BaseCacheManager):
                             f"Failed to write {data['file_path']} to .cache/cache.jsonl: {e}")
         else:
             lock_file = cache_file + ".lock"
-            with open(lock_file, "w") as lockf:
+            with open(lock_file, "w",encoding="utf-8") as lockf:
                 try:
                     # 获取文件锁
                     fcntl.flock(lockf, fcntl.LOCK_EX | fcntl.LOCK_NB)
                     # 写入缓存文件
-                    with open(cache_file, "w") as f:
+                    with open(cache_file, "w",encoding="utf-8") as f:
                         for data in self.cache.values():
                             try:
                                 json.dump(data, f, ensure_ascii=False)

autocoder/rag/long_context_rag.py

@@ -245,10 +245,10 @@ class LongContextRAG:
         gitignore_path = os.path.join(self.path, ".gitignore")
 
         if os.path.exists(serveignore_path):
-            with open(serveignore_path, "r") as ignore_file:
+            with open(serveignore_path, "r",encoding="utf-8") as ignore_file:
                 return pathspec.PathSpec.from_lines("gitwildmatch", ignore_file)
         elif os.path.exists(gitignore_path):
-            with open(gitignore_path, "r") as ignore_file:
+            with open(gitignore_path, "r",encoding="utf-8") as ignore_file:
                 return pathspec.PathSpec.from_lines("gitwildmatch", ignore_file)
         return None
 

autocoder/regexproject/__init__.py

@@ -78,7 +78,8 @@ class RegexProject:
             raise ValueError("Invalid project_type format. Expected 'regex//<pattern>'")
 
     def output(self):
-        return open(self.target_file, "r").read()
+        with open(self.target_file, "r",encoding="utf-8") as file:
+            return file.read()
 
     def is_regex_match(self, file_path):
         return re.search(self.regex_pattern, file_path) is not None
@@ -231,7 +232,7 @@ class RegexProject:
             self.clone_repository()
 
         if self.target_file:
-            with open(self.target_file, "w") as file:
+            with open(self.target_file, "w",encoding="utf-8") as file:
                 for code in self.get_source_codes():
                     self.sources.append(code)
                     file.write(f"##File: {code.module_name}\n")

autocoder/suffixproject/__init__.py

@@ -114,7 +114,8 @@ class SuffixProject:
         return False
 
     def output(self):
-        return open(self.target_file, "r").read()
+        with open(self.target_file, "r",encoding="utf-8") as file:
+            return file.read()
 
     def is_suffix_file(self, file_path):
         return any([file_path.endswith(suffix) for suffix in self.suffixs])
@@ -273,7 +274,7 @@ class SuffixProject:
             self.clone_repository()
 
         if self.target_file:
-            with open(self.target_file, "w") as file:
+            with open(self.target_file, "w",encoding="utf-8") as file:
                 for code in self.get_source_codes():
                     self.sources.append(code)
                     file.write(f"##File: {code.module_name}\n")

autocoder/tsproject/__init__.py

@@ -106,7 +106,8 @@ class TSProject:
         return False
 
     def output(self):
-        return open(self.target_file, "r").read()
+        with open(self.target_file, "r",encoding="utf-8") as file:
+            return file.read()
 
     def read_file_content(self, file_path):
         return FileUtils.read_file(file_path)        
@@ -308,7 +309,7 @@ class TSProject:
             self.clone_repository()
 
         if self.target_file:
-            with open(self.target_file, "w") as file:
+            with open(self.target_file, "w",encoding="utf-8") as file:
                 for code in self.get_rest_source_codes():
                     self.sources.append(code)
                     file.write(f"##File: {code.module_name}\n")

autocoder/utils/conversation_store.py

@@ -34,7 +34,7 @@ def load_code_model_conversation_from_store(args: AutoCoderArgs):
         return []
 
     conversations = []
-    with open(conversation_file, "r") as f:
+    with open(conversation_file, "r",encoding="utf-8") as f:
         for line in f:
             conversations.append(json.loads(line))
 

autocoder/utils/operate_config_api.py

@@ -15,7 +15,7 @@ import hashlib
 def convert_yaml_to_config(yaml_file: str):
 
     args = AutoCoderArgs()
-    with open(yaml_file, "r") as f:
+    with open(yaml_file, "r",encoding="utf-8") as f:
         config = yaml.safe_load(f)
         config = load_include_files(config, yaml_file)
         for key, value in config.items():
@@ -75,7 +75,7 @@ def get_llm_friendly_package_docs(memory,
                                         if return_paths:
                                             docs.append(file_path)
                                         else:
-                                            with open(file_path, "r") as f:
+                                            with open(file_path, "r",encoding="utf-8") as f:
                                                 docs.append(f.read())
 
     return docs
@@ -130,7 +130,7 @@ def get_llm(memory, model:Optional[str]=None):
             # 临时保存yaml文件，然后读取yaml文件，转换为args
             temp_yaml = os.path.join("actions", f"{uuid.uuid4()}.yml")
             try:
-                with open(temp_yaml, "w") as f:
+                with open(temp_yaml, "w",encoding="utf-8") as f:
                     f.write(convert_yaml_config_to_str(
                         yaml_config=yaml_config))
                 args = convert_yaml_to_config(temp_yaml)