auto-coder 0.1.268__py3-none-any.whl → 0.1.270__py3-none-any.whl

autocoder/common/context_pruner.py

@@ -1,4 +1,5 @@
 from typing import List, Dict, Any, Union
+from typing import Tuple
 from pathlib import Path
 import json
 from loguru import logger
@@ -19,6 +20,100 @@ class PruneContext:
         self.llm = llm
         self.printer = Printer()
 
+    def _split_content_with_sliding_window(self, content: str, window_size=100, overlap=20) -> List[Tuple[int, int, str]]:
+        """使用滑动窗口分割大文件内容，返回包含行号信息的文本块
+        
+        Args:
+            content: 要分割的文件内容
+            window_size: 每个窗口包含的行数
+            overlap: 相邻窗口的重叠行数
+            
+        Returns:
+            List[Tuple[int, int, str]]: 返回元组列表，每个元组包含:
+                - 起始行号(从1开始)，在原始文件的绝对行号
+                - 结束行号，在原始文件的绝对行号
+                - 带行号的内容文本
+        """
+        # 按行分割内容
+        lines = content.splitlines()
+        chunks = []
+        start = 0
+        
+        while start < len(lines):
+            # 计算当前窗口的结束位置
+            end = min(start + window_size, len(lines))
+            
+            # 计算实际的起始位置(考虑重叠)
+            actual_start = max(0, start - overlap)
+            
+            # 提取当前窗口的行
+            chunk_lines = lines[actual_start:end]
+            
+            # 为每一行添加行号
+            # 行号从actual_start+1开始，保持与原文件的绝对行号一致
+            chunk_content = "\n".join([
+                f"{i+1} {line}" for i, line in enumerate(chunk_lines, start=actual_start)
+            ])
+            
+            # 保存分块信息：(起始行号, 结束行号, 带行号的内容)
+            # 行号从1开始计数
+            chunks.append((actual_start + 1, end, chunk_content))
+            
+            # 移动到下一个窗口的起始位置
+            # 减去overlap确保窗口重叠
+            start += (window_size - overlap)
+            
+        return chunks
+
+    def _merge_overlapping_snippets(self, snippets: List[dict]) -> List[dict]:
+        """合并重叠或相邻的代码片段
+        
+        Args:
+            snippets: 代码片段列表，每个片段是包含start_line和end_line的字典
+            
+        Returns:
+            List[dict]: 合并后的代码片段列表
+            
+        示例:
+            输入: [
+                {"start_line": 1, "end_line": 5},
+                {"start_line": 4, "end_line": 8},
+                {"start_line": 10, "end_line": 12}
+            ]
+            输出: [
+                {"start_line": 1, "end_line": 8},
+                {"start_line": 10, "end_line": 12}
+            ]
+        """
+        if not snippets:
+            return []
+
+        # 按起始行排序
+        sorted_snippets = sorted(snippets, key=lambda x: x["start_line"])
+
+        merged = [sorted_snippets[0]]
+        
+        for current in sorted_snippets[1:]:
+            last = merged[-1]
+            
+            # 判断是否需要合并:
+            # 1. 如果当前片段的起始行小于等于上一个片段的结束行+1
+            # 2. +1是为了合并相邻的片段，比如1-5和6-8应该合并为1-8
+            if current["start_line"] <= last["end_line"] + 1:
+                # 合并区间:
+                # - 起始行取两者最小值
+                # - 结束行取两者最大值
+                merged[-1] = {
+                    "start_line": min(last["start_line"], current["start_line"]),
+                    "end_line": max(last["end_line"], current["end_line"])
+                }
+            else:
+                # 如果不重叠且不相邻，则作为新片段添加
+                merged.append(current)
+
+        return merged
+     
+
     def _delete_overflow_files(self, file_paths: List[str]) -> List[SourceCode]:
         """直接删除超出 token 限制的文件"""
         total_tokens = 0
@@ -40,6 +135,8 @@ class PruneContext:
                 selected_files.append(SourceCode(module_name=file_path,source_code=content,tokens=token_count))
 
         return selected_files
+    
+    
 
     def _extract_code_snippets(self, file_paths: List[str], conversations: List[Dict[str, str]]) -> List[SourceCode]:
         """抽取关键代码片段策略"""
@@ -48,7 +145,7 @@ class PruneContext:
         full_file_tokens = int(self.max_tokens * 0.8)
 
         @byzerllm.prompt()
-        def extract_code_snippets(conversations: List[Dict[str, str]], content: str) -> str:
+        def extract_code_snippets(conversations: List[Dict[str, str]], content: str, is_partial_content: bool = False) -> str:
             """
             根据提供的代码文件和对话历史提取相关代码片段。            
 
@@ -111,6 +208,13 @@ class PruneContext:
             {{ content }}
             </code_file>
 
+            <% if is_partial_content: %>
+            <partial_content_process_note>
+            当前处理的是文件的局部内容（行号{start_line}-{end_line}），
+            请仅基于当前可见内容判断相关性，返回标注的行号区间。            
+            </partial_content_process_note>
+            <% endif %>
+
             2. 对话历史:
             <conversation_history>
             {% for msg in conversations %}
@@ -131,15 +235,17 @@ class PruneContext:
             4. 如果没有相关代码段，返回空数组[]。
 
             输出格式:
-            严格的JSON数组，不包含其他文字或解释。
-
+            严格的JSON数组，不包含其他文字或解释。           
+            
             ```json
             [
                 {"start_line": 第一个代码段的起始行号, "end_line": 第一个代码段的结束行号},
                 {"start_line": 第二个代码段的起始行号, "end_line": 第二个代码段的结束行号}
             ]
-            ```            
+            ``` 
+                       
             """
+            
 
         for file_path in file_paths:
             try:
@@ -152,22 +258,76 @@ class PruneContext:
                         selected_files.append(SourceCode(module_name=file_path,source_code=content,tokens=tokens))
                         token_count += tokens
                         continue
-
+                    
+                    ## 如果单个文件太大，那么先按滑动窗口分割，然后对窗口抽取代码片段
+                    if tokens > self.max_tokens:
+                        self.printer.print_in_terminal("file_sliding_window_processing", file_path=file_path, tokens=tokens)
+                        chunks = self._split_content_with_sliding_window(content, 
+                                                                         self.args.context_prune_sliding_window_size, 
+                                                                         self.args.context_prune_sliding_window_overlap)                        
+                        all_snippets = [] 
+                        for chunk_start, chunk_end, chunk_content in chunks: 
+                            extracted = extract_code_snippets.with_llm(self.llm).run( 
+                                conversations=conversations, 
+                                content=chunk_content, 
+                                is_partial_content=True 
+                            )
+                            if extracted:
+                                json_str = extract_code(extracted)[0][1]
+                                snippets = json.loads(json_str)  
+
+                                # 获取到的本来就是在原始文件里的绝对行号  
+                                # 后续在构建代码片段内容时，会为了适配数组操作修改行号，这里无需处理
+                                adjusted_snippets = [{
+                                    "start_line": snippet["start_line"],
+                                    "end_line": snippet["end_line"]
+                                } for snippet in snippets]                                
+                                all_snippets.extend(adjusted_snippets)                                                                
+                        merged_snippets = self._merge_overlapping_snippets(all_snippets)                        
+                        content_snippets = self._build_snippet_content(file_path, content, merged_snippets)
+                        snippet_tokens = count_tokens(content_snippets)
+                        if token_count + snippet_tokens <= self.max_tokens:
+                            selected_files.append(SourceCode(module_name=file_path,source_code=content_snippets,tokens=snippet_tokens))                            
+                            token_count += snippet_tokens
+                            self.printer.print_in_terminal("file_snippet_procesed", file_path=file_path, 
+                                                           total_tokens=token_count, 
+                                                           tokens=tokens, 
+                                                           snippet_tokens=snippet_tokens)
+                            continue
+                        else:
+                            break
+                        
                     # 抽取关键片段
+                    lines = content.splitlines()
+                    new_content = ""
+
+                    ## 将文件内容按行编号
+                    for index,line in enumerate(lines):                        
+                        new_content += f"{index+1} {line}\n"
+
+                    ## 抽取代码片段
+                    self.printer.print_in_terminal("file_snippet_processing", file_path=file_path)
                     extracted = extract_code_snippets.with_llm(self.llm).run(
                         conversations=conversations, 
-                        content=content
+                        content=new_content
                     )                
 
+                    ## 构建代码片段内容
                     if extracted:
                         json_str = extract_code(extracted)[0][1]
                         snippets = json.loads(json_str)
-                        new_content = self._build_snippet_content(file_path, content, snippets)
+                        content_snippets = self._build_snippet_content(file_path, content, snippets)
 
-                        snippet_tokens = count_tokens(new_content)
-                        if token_count + snippet_tokens <= self.max_tokens:
-                            selected_files.append(SourceCode(module_name=file_path,source_code=new_content,tokens=snippet_tokens))
+                        snippet_tokens = count_tokens(content_snippets)
+                        if token_count + snippet_tokens <= self.max_tokens:                            
+                            selected_files.append(SourceCode(module_name=file_path,
+                                                             source_code=content_snippets,
+                                                             tokens=snippet_tokens))
                             token_count += snippet_tokens
+                            self.printer.print_in_terminal("file_snippet_procesed", file_path=file_path, 
+                                                           total_tokens = token_count,
+                                                           tokens=tokens, 
+                                                           snippet_tokens=snippet_tokens)
                         else:
                             break
             except Exception as e:
@@ -175,10 +335,32 @@ class PruneContext:
                 continue
 
         return selected_files
+    
+
+    def _merge_overlapping_snippets(self, snippets: List[dict]) -> List[dict]: 
+        if not snippets:
+            return []
+
+        # 按起始行排序
+        sorted_snippets = sorted(snippets, key=lambda x: x["start_line"])
+
+        merged = [sorted_snippets[0]]
+        for current in sorted_snippets[1:]:
+            last = merged[-1]
+            if current["start_line"] <= last["end_line"] + 1:  # 允许1行间隔
+                # 合并区间
+                merged[-1] = {
+                    "start_line": min(last["start_line"], current["start_line"]),
+                    "end_line": max(last["end_line"], current["end_line"])
+                }
+            else:
+                merged.append(current)
+
+        return merged
 
     def _build_snippet_content(self, file_path: str, full_content: str, snippets: List[dict]) -> str:
         """构建包含代码片段的文件内容"""
-        lines = full_content.split("\n")
+        lines = full_content.splitlines()
         header = f"Snippets:\n"
 
         content = []
@@ -205,7 +387,26 @@ class PruneContext:
         total_tokens,sources = self._count_tokens(file_paths)
         if total_tokens <= self.max_tokens:
             return sources
-        # print(f"total_tokens: {total_tokens} {self.max_tokens}, 进行策略: {strategy}")
+        
+        self.printer.print_in_terminal(
+            "context_pruning_reason",
+            total_tokens=total_tokens,
+            max_tokens=self.max_tokens,
+            style="yellow"
+        )
+
+        self.printer.print_in_terminal(
+                "sorted_files_message", 
+                files=file_paths
+            )
+
+        self.printer.print_in_terminal(
+            "context_pruning_start",
+            total_tokens=total_tokens,
+            max_tokens=self.max_tokens,
+            strategy=strategy
+        )
+
         if strategy == "score":            
             return self._score_and_filter_files(file_paths, conversations)
         if strategy == "delete":
@@ -214,7 +415,7 @@ class PruneContext:
             return self._extract_code_snippets(file_paths, conversations)
         else:
             raise ValueError(f"无效策略: {strategy}. 可选值: delete/extract/score")
-    
+
     def _count_tokens(self, file_paths: List[str]) -> int:
         """计算文件总token数"""
         total_tokens = 0
@@ -312,4 +513,3 @@ class PruneContext:
                 break
 
         return selected_files
-

autocoder/common/conversation_pruner.py

@@ -3,8 +3,9 @@ import json
 from pydantic import BaseModel
 import byzerllm
 from autocoder.common.printer import Printer
-from autocoder.utils.llms import count_tokens
+from autocoder.rag.token_counter import count_tokens
 from loguru import logger
+from autocoder.common import AutoCoderArgs
 
 class PruneStrategy(BaseModel):
     name: str
@@ -12,25 +13,25 @@ class PruneStrategy(BaseModel):
     config: Dict[str, Any] = {"safe_zone_tokens": 0, "group_size": 4}
 
 class ConversationPruner:
-    def __init__(self, llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM], 
-                 safe_zone_tokens: int = 500, group_size: int = 4):
+    def __init__(self, args: AutoCoderArgs, llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM]):
+        self.args = args
         self.llm = llm
         self.printer = Printer()
         self.strategies = {
             "summarize": PruneStrategy(
                 name="summarize",
                 description="对早期对话进行分组摘要，保留关键信息",
-                config={"safe_zone_tokens": safe_zone_tokens, "group_size": group_size}
+                config={"safe_zone_tokens": self.args.conversation_prune_safe_zone_tokens, "group_size": self.args.conversation_prune_group_size}
             ),
             "truncate": PruneStrategy(
                 name="truncate",
                 description="分组截断最早的部分对话",
-                config={"safe_zone_tokens": safe_zone_tokens, "group_size": group_size}
+                config={"safe_zone_tokens": self.args.conversation_prune_safe_zone_tokens, "group_size": self.args.conversation_prune_group_size}
             ),
             "hybrid": PruneStrategy(
                 name="hybrid",
                 description="先尝试分组摘要，如果仍超限则分组截断",
-                config={"safe_zone_tokens": safe_zone_tokens, "group_size": group_size}
+                config={"safe_zone_tokens": self.args.conversation_prune_safe_zone_tokens, "group_size": self.args.conversation_prune_group_size}
             )
         }
 
@@ -57,7 +58,7 @@ class ConversationPruner:
         if strategy.name == "summarize":
             return self._summarize_prune(conversations, strategy.config)
         elif strategy.name == "truncate":
-            return self._truncate_prune.with_llm(self.llm).run(conversations)
+            return self._truncate_prune(conversations)
         elif strategy.name == "hybrid":
             pruned = self._summarize_prune(conversations, strategy.config)
             if count_tokens(json.dumps(pruned, ensure_ascii=False)) > self.args.conversation_prune_safe_zone_tokens:
@@ -80,8 +81,8 @@ class ConversationPruner:
                 break
                 
             # 找到要处理的对话组
-            early_conversations = processed_conversations[:-group_size]
-            recent_conversations = processed_conversations[-group_size:]
+            early_conversations = processed_conversations[:group_size]
+            recent_conversations = processed_conversations[group_size:]
             
             if not early_conversations:
                 break
@@ -90,7 +91,7 @@ class ConversationPruner:
             group_summary = self._generate_summary.with_llm(self.llm).run(early_conversations[-group_size:])
             
             # 更新对话历史
-            processed_conversations = early_conversations[:-group_size] + [
+            processed_conversations =  [
                 {"role": "user", "content": f"历史对话摘要：\n{group_summary}"},
                 {"role": "assistant", "content": f"收到"}
             ] + recent_conversations

autocoder/index/entry.py

@@ -58,8 +58,12 @@ def build_index_and_filter_files(
             return file_path.strip()[2:]
         return file_path
 
+    # 文件路径 -> TargetFile 
     final_files: Dict[str, TargetFile] = {}
 
+    # 文件路径 -> 文件在文件列表中的位置（越前面表示越相关）
+    file_positions:Dict[str,int] = {}
+
     # Phase 1: Process REST/RAG/Search sources
     printer = Printer()
     printer.print_in_terminal("phase1_processing_sources")
@@ -102,25 +106,20 @@ def build_index_and_filter_files(
                     })
                 )
             )
-
+        
+        
         if not args.skip_filter_index and args.index_filter_model:
             model_name = getattr(index_manager.index_filter_llm, 'default_model_name', None)
             if not model_name:
                 model_name = "unknown(without default model name)"    
             printer.print_in_terminal("quick_filter_start", style="blue", model_name=model_name)
             quick_filter = QuickFilter(index_manager,stats,sources)        
-            quick_filter_result = quick_filter.filter(index_manager.read_index(),args.query)
-            # if quick_filter_result.has_error:                
-            #     raise KeyboardInterrupt(printer.get_message_from_key_with_format("quick_filter_failed",error=quick_filter_result.error_message))
-
-            # Merge quick filter results into final_files
-            if args.context_prune:
-                context_pruner = PruneContext(max_tokens=args.conversation_prune_safe_zone_tokens, args=args, llm=llm)
-                pruned_files = context_pruner.handle_overflow(quick_filter_result.files, [{"role":"user","content":args.query}], args.context_prune_strategy)
-                for source_file in pruned_files:
-                    final_files[source_file.module_name] = quick_filter_result.files[source_file.module_name]
-            else:
-                final_files.update(quick_filter_result.files)                                     
+            quick_filter_result = quick_filter.filter(index_manager.read_index(),args.query)            
+            
+            final_files.update(quick_filter_result.files)
+            
+            if quick_filter_result.file_positions:
+                file_positions.update(quick_filter_result.file_positions)
         
         if not args.skip_filter_index and not args.index_filter_model:
             model_name = getattr(index_manager.llm, 'default_model_name', None)
@@ -261,32 +260,55 @@ def build_index_and_filter_files(
         for file in final_filenames:
             print(f"{file} - {final_files[file].reason}")
 
-    source_code = ""  
+    # source_code = ""  
     source_code_list = SourceCodeList(sources=[])
     depulicated_sources = set()
-
+    
+    ## 先去重
+    temp_sources = []
     for file in sources:
         if file.module_name in final_filenames:
             if file.module_name in depulicated_sources:
                 continue
             depulicated_sources.add(file.module_name)
-            source_code += f"##File: {file.module_name}\n"
-            source_code += f"{file.source_code}\n\n"
-            source_code_list.sources.append(file)
+            # source_code += f"##File: {file.module_name}\n"
+            # source_code += f"{file.source_code}\n\n"
+            temp_sources.append(file)
+
+    ## 开启了裁剪，则需要做裁剪，不过目前只针对 quick filter 生效
+    if args.context_prune:
+        context_pruner = PruneContext(max_tokens=args.conversation_prune_safe_zone_tokens, args=args, llm=llm)
+        # 如果 file_positions 不为空，则通过 file_positions 来获取文件
+        if file_positions:
+            ## 拿到位置列表，然后根据位置排序 得到 [(pos,file_path)]
+            ## 将 [(pos,file_path)] 转换为 [file_path]
+            ## 通过 [file_path] 顺序调整 temp_sources 的顺序             
+            ## MARK 
+             # 将 file_positions 转换为 [(pos, file_path)] 的列表
+            position_file_pairs = [(pos, file_path) for file_path, pos in file_positions.items()]
+            # 按位置排序
+            position_file_pairs.sort(key=lambda x: x[0])
+            # 提取排序后的文件路径列表
+            sorted_file_paths = [file_path for _, file_path in position_file_pairs]
+            # 根据 sorted_file_paths 重新排序 temp_sources
+            temp_sources.sort(key=lambda x: sorted_file_paths.index(x.module_name) if x.module_name in sorted_file_paths else len(sorted_file_paths))                       
+            
+        pruned_files = context_pruner.handle_overflow([source.module_name for source in temp_sources], [{"role":"user","content":args.query}], args.context_prune_strategy)        
+        source_code_list.sources = pruned_files
+
+
     if args.request_id and not args.skip_events:
         queue_communicate.send_event(
             request_id=args.request_id,
             event=CommunicateEvent(
                 event_type=CommunicateEventType.CODE_INDEX_FILTER_FILE_SELECTED.value,
                 data=json.dumps([
-                    (file["file_path"], file.reason)
-                    for file in final_files.values()
-                    if file.file_path in depulicated_sources
+                    (file.module_name, "") for file in source_code_list.sources
                 ])
             )
         )        
 
-    stats["final_files"] = len(depulicated_sources)
+    stats["final_files"] = len(source_code_list.sources)
     phase_end = time.monotonic()
     stats["timings"]["prepare_output"] = phase_end - phase_start
 

autocoder/index/index.py

@@ -400,7 +400,7 @@ class IndexManager:
 
         # 删除被排除的文件
         try:
-            exclude_patterns = self.parse_exclude_files(self.args.exclude_files)
+            exclude_patterns = self.parse_exclude_files(self.args.exclude_files)            
             for file_path in index_data:
                 if self.filter_exclude_files(file_path, exclude_patterns):
                     keys_to_remove.append(file_path)

autocoder/utils/auto_project_type.py

@@ -0,0 +1,120 @@
+import os
+import json
+from collections import defaultdict
+from typing import Dict, List, Set, Tuple
+from pathlib import Path
+from loguru import logger
+import byzerllm
+from autocoder.common import AutoCoderArgs
+from autocoder.common.printer import Printer
+from typing import Union
+import pydantic
+from autocoder.common.result_manager import ResultManager
+
+class ExtensionClassifyResult(pydantic.BaseModel):
+    code: List[str] = []
+    config: List[str] = []
+    data: List[str] = []
+    document: List[str] = []
+    other: List[str] = []
+    framework: List[str] = []
+    
+class ProjectTypeAnalyzer:
+    def __init__(self, args: AutoCoderArgs, llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM]):
+        self.args = args
+        self.llm = llm
+        self.printer = Printer()
+        self.default_exclude_dirs = [
+            ".git", ".svn", ".hg", "build", "dist", "__pycache__", 
+            "node_modules", ".auto-coder", ".vscode", ".idea", "venv",
+            ".next", ".nuxt", ".svelte-kit", "out", "cache", "logs",
+            "temp", "tmp", "coverage", ".DS_Store", "public", "static"
+        ]
+        self.extension_counts = defaultdict(int)
+        self.stats_file = Path(args.source_dir) / ".auto-coder" / "project_type_stats.json"
+        self.result_manager = ResultManager()
+
+    def traverse_project(self) -> None:
+        """遍历项目目录，统计文件后缀"""
+        for root, dirs, files in os.walk(self.args.source_dir):
+            # 过滤掉默认排除的目录
+            dirs[:] = [d for d in dirs if d not in self.default_exclude_dirs]
+            
+            for file in files:
+                _, ext = os.path.splitext(file)
+                if ext:  # 只统计有后缀的文件
+                    self.extension_counts[ext.lower()] += 1
+
+    def count_extensions(self) -> Dict[str, int]:
+        """返回文件后缀统计结果"""
+        return dict(sorted(self.extension_counts.items(), key=lambda x: x[1], reverse=True))
+
+    @byzerllm.prompt()
+    def classify_extensions(self, extensions: str) -> str:
+        """
+        根据文件后缀列表，将后缀分类为代码、配置、数据、文档等类型。
+
+        文件后缀列表：
+        {{ extensions }}
+
+        请返回如下JSON格式：
+        {
+            "code": ["后缀1", "后缀2"],
+            "config": ["后缀3", "后缀4"],
+            "data": ["后缀5", "后缀6"],
+            "document": ["后缀7", "后缀8"],
+            "other": ["后缀9", "后缀10"],
+            "framework": ["后缀11", "后缀12"]
+        }
+        """
+        return {
+            "extensions": extensions
+        }
+
+    def save_stats(self) -> None:
+        """保存统计结果到文件"""
+        stats = {
+            "extension_counts": self.extension_counts,
+            "project_type": self.detect_project_type()
+        }
+        
+        # 确保目录存在
+        self.stats_file.parent.mkdir(parents=True, exist_ok=True)
+        
+        with open(self.stats_file, "w", encoding="utf-8") as f:
+            json.dump(stats, f, indent=2)
+        
+        self.printer.print_in_terminal("stats_saved", path=str(self.stats_file))
+
+    def load_stats(self) -> Dict[str, any]:
+        """从文件加载统计结果"""
+        if not self.stats_file.exists():
+            self.printer.print_in_terminal("stats_not_found", path=str(self.stats_file))
+            return {}
+            
+        with open(self.stats_file, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def detect_project_type(self) -> str:
+        """根据后缀统计结果推断项目类型"""
+        # 获取统计结果
+        ext_counts = self.count_extensions()        
+        # 将后缀分类
+        classification = self.classify_extensions.with_llm(self.llm).with_return_type(ExtensionClassifyResult).run(json.dumps(ext_counts,ensure_ascii=False))        
+        return ",".join(classification.code)
+
+    def analyze(self) -> Dict[str, any]:
+        """执行完整的项目类型分析流程"""
+        # 遍历项目目录
+        self.traverse_project()
+        
+        # 检测项目类型
+        project_type = self.detect_project_type()  
+
+        self.result_manager.add_result(content=project_type, meta={
+                    "action": "get_project_type",
+                    "input": {
+                       
+                    }
+                })      
+        return project_type