auto-coder 0.1.316__py3-none-any.whl → 0.1.318__py3-none-any.whl

autocoder/common/v2/code_auto_merge_strict_diff.py

@@ -0,0 +1,259 @@
+import os
+import difflib
+import diff_match_patch as dmp_module
+from pathlib import Path
+from typing import List, Dict, Tuple
+import pydantic
+import byzerllm
+from autocoder.common import AutoCoderArgs, git_utils
+from autocoder.common.types import CodeGenerateResult, MergeCodeWithoutEffect
+from autocoder.common.v2.code_auto_merge import CodeAutoMerge
+from autocoder.common import files as FileUtils
+
+class PathAndCode(pydantic.BaseModel):
+    path: str
+    content: str
+
+def safe_abs_path(res):
+    "Gives an abs path, which safely returns a full (not 8.3) windows path"
+    res = Path(res).resolve()
+    return str(res)
+
+def apply_hunk(content, hunk):
+    before, after = hunk_to_before_after(hunk)
+    
+    # Get line numbers from @@ ... @@ markers
+    line_info = hunk[0].split("@@")[1].strip()
+    s_line_num = int(line_info.split(" ")[1].lstrip("+"))
+
+    # Split content into lines
+    content_lines = content.splitlines()
+    
+    # Merge changes using difflib
+    merged_lines = list(difflib.ndiff(before.splitlines(), after.splitlines()))
+    
+    # Apply changes to original content
+    j = 0
+    content_out = content_lines[:s_line_num]
+    for line in merged_lines:
+        if line.startswith("- "):
+            continue
+        elif line.startswith("+ "):
+            content_out.append(line[2:])
+        elif line.startswith("  "):
+            if j < len(content_lines):
+                content_out.append(content_lines[s_line_num+j])
+            j += 1
+    
+    content_out.extend(content_lines[s_line_num+j:])
+
+    return "\n".join(content_out)
+
+def hunk_to_before_after(hunk, lines=False):
+    before = []
+    after = []
+    op = " "
+    for line in hunk:
+        if len(line) < 2:
+            op = " "
+            line = line
+        else:
+            op = line[0]
+            line = line[1:]
+
+        if op == " ":
+            before.append(line)
+            after.append(line)
+        elif op == "-":
+            before.append(line)
+        elif op == "+":
+            after.append(line)
+
+    if lines:
+        return before, after
+
+    before = "".join(before)
+    after = "".join(after)
+
+    return before, after
+
+class CodeAutoMergeStrictDiff(CodeAutoMerge):
+    def parse_diff_block(self,text: str) -> List[PathAndCode]:
+        lines = text.split('\n')
+        lines_len = len(lines)    
+        start_marker_count = 0       
+        inline_start_marker_count = 0 
+        block = []
+        path_and_code_list = []
+
+        def guard(index):
+            return index+1 < lines_len 
+
+        def start_marker(line,index):
+            return line.startswith('```diff') and guard(index)
+        
+        def inline_start_marker(line,index):
+            return line.startswith('```') and not line.startswith('```diff') and line.strip() != '```'
+
+        def end_marker(line,index):
+            return line.startswith('```') and line.strip() == '```'
+        
+
+        for (index,line) in enumerate(lines):
+            if start_marker(line,index) and start_marker_count == 0:
+                start_marker_count += 1        
+            elif (start_marker(line,index) or inline_start_marker(line,index)) and start_marker_count > 0:
+                inline_start_marker_count += 1     
+                block.append(line)    
+            elif end_marker(line,index) and start_marker_count == 1 and inline_start_marker_count == 0:
+                start_marker_count -= 1            
+                if block:
+                    # ori_path = block[0][4:0].strip()                       
+                    new_path = block[1][4:].strip()                    
+                    content = '\n'.join(block)                                
+                    block = []
+                    path_and_code_list.append(PathAndCode(path=new_path,content=content))       
+            elif end_marker(line,index) and inline_start_marker_count > 0:
+                inline_start_marker_count -= 1   
+                block.append(line)                 
+            elif start_marker_count > 0:
+                block.append(line)                
+
+        return path_and_code_list
+
+    def abs_root_path(self, path):
+        if path.startswith(self.args.source_dir):
+            return safe_abs_path(Path(path))
+        res = Path(self.args.source_dir) / path
+        return safe_abs_path(res)            
+
+    def _merge_code_without_effect(self, content: str) -> MergeCodeWithoutEffect:
+        """Merge code without any side effects like git operations or file writing.
+        Returns a tuple of:
+        - list of (file_path, new_content) tuples for successfully merged blocks
+        - list of (file_path, content) tuples for failed to merge blocks"""
+        diff_blocks = self.parse_diff_block(content)
+        file_content_mapping = {}
+        failed_blocks = []
+        
+        for block in diff_blocks:
+            path = block.path
+            content = block.content
+            full_path = self.abs_root_path(path)
+            
+            if not os.path.exists(full_path):
+                file_content_mapping[full_path] = content
+                continue
+                
+            if full_path not in file_content_mapping:
+                file_content_mapping[full_path] = FileUtils.read_file(full_path)
+            
+            try:
+                import patch
+                patch_obj = patch.fromstring(content.encode('utf-8'))
+                root_path = None
+                if not path.startswith(self.args.source_dir):
+                    root_path = self.args.source_dir
+
+                # Create a copy of the content to apply patch
+                temp_content = file_content_mapping[full_path]
+                success = patch_obj.apply(root=root_path, content=temp_content)
+                if success:
+                    file_content_mapping[full_path] = temp_content
+                else:
+                    failed_blocks.append((full_path, content))
+            except Exception as e:
+                self.printer.print_in_terminal("merge_failed", style="yellow", path=full_path, error=str(e))
+                failed_blocks.append((full_path, content))
+                
+        return MergeCodeWithoutEffect(
+            success_blocks=[(path, content) for path, content in file_content_mapping.items()],
+            failed_blocks=failed_blocks
+        )
+
+    def print_diff_blocks(self, diff_blocks: List[PathAndCode]):
+        """Print diff blocks for user review using rich library"""
+        from rich.syntax import Syntax
+        from rich.panel import Panel
+
+        # Group blocks by file path
+        file_blocks = {}
+        for block in diff_blocks:
+            if block.path not in file_blocks:
+                file_blocks[block.path] = []
+            file_blocks[block.path].append(block.content)
+
+        # Generate formatted text for each file
+        formatted_text = ""
+        for path, contents in file_blocks.items():
+            formatted_text += f"##File: {path}\n"
+            for content in contents:
+                formatted_text += content + "\n"
+            formatted_text += "\n"
+
+        # Print with rich panel
+        self.printer.print_in_terminal("diff_blocks_title", style="bold green")
+        self.printer.console.print(
+            Panel(
+                Syntax(formatted_text, "diff", theme="monokai"),
+                title="Diff Blocks",
+                border_style="green",
+                expand=False
+            )
+        )
+
+    def _merge_code(self, content: str, force_skip_git: bool = False):        
+        total = 0
+        
+        file_content = FileUtils.read_file(self.args.file)
+        md5 = hashlib.md5(file_content.encode('utf-8')).hexdigest()
+        file_name = os.path.basename(self.args.file)
+        
+        if not force_skip_git and not self.args.skip_commit:
+            try:
+                git_utils.commit_changes(self.args.source_dir, f"auto_coder_pre_{file_name}_{md5}")
+            except Exception as e:            
+                self.printer.print_in_terminal("git_init_required", style="red", source_dir=self.args.source_dir, error=str(e))
+                return            
+       
+        diff_blocks = self.parse_diff_block(content)        
+
+        for diff_blocks in diff_blocks:
+            path = diff_blocks.path
+            content = diff_blocks.content          
+
+            import patch
+            patch_obj = patch.fromstring(content.encode('utf-8'))
+            root_path = None
+            if not path.startswith(self.args.source_dir):
+                root_path = self.args.source_dir
+
+            success = patch_obj.apply(root=root_path)
+            if not success:
+                raise Exception("Error applying diff to file: " + path)
+                            
+        self.printer.print_in_terminal("files_merged_total", total=total)
+        if not force_skip_git and not self.args.skip_commit:
+            commit_result = git_utils.commit_changes(
+                self.args.source_dir, f"{self.args.query}\nauto_coder_{file_name}"
+            )
+            action_yml_file_manager = ActionYmlFileManager(self.args.source_dir)
+            action_file_name = os.path.basename(self.args.file)
+            add_updated_urls = []
+            commit_result.changed_files
+            for file in commit_result.changed_files:
+                add_updated_urls.append(os.path.join(self.args.source_dir, file))
+
+            self.args.add_updated_urls = add_updated_urls
+            update_yaml_success = action_yml_file_manager.update_yaml_field(action_file_name, "add_updated_urls", add_updated_urls)
+            if not update_yaml_success:                        
+                self.printer.print_in_terminal("yaml_save_error", style="red", yaml_file=action_file_name)  
+            
+            if self.args.enable_active_context:
+                active_context_manager = ActiveContextManager(self.llm, self.args.source_dir)
+                active_context_manager.process_changes(self.args)
+            
+            git_utils.print_commit_info(commit_result=commit_result)
+        else:
+            # Print diff blocks for review
+            self.print_diff_blocks(diff_blocks) 

autocoder/common/v2/code_diff_manager.py

@@ -0,0 +1,266 @@
+from typing import List, Dict, Tuple, Optional, Any
+import os
+import json
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import byzerllm
+from byzerllm.utils.client import code_utils
+
+from autocoder.common.types import Mode, CodeGenerateResult, MergeCodeWithoutEffect
+from autocoder.common import AutoCoderArgs, git_utils, SourceCodeList
+from autocoder.common import sys_prompt
+from autocoder.privacy.model_filter import ModelPathFilter
+from autocoder.common.utils_code_auto_generate import chat_with_continue, stream_chat_with_continue, ChatWithContinueResult
+from autocoder.utils.auto_coder_utils.chat_stream_out import stream_out
+from autocoder.common.stream_out_type import CodeGenerateStreamOutType
+from autocoder.common.auto_coder_lang import get_message_with_format
+from autocoder.common.printer import Printer
+from autocoder.rag.token_counter import count_tokens
+from autocoder.utils import llms as llm_utils
+from autocoder.memory.active_context_manager import ActiveContextManager
+from autocoder.common.v2.code_auto_generate_diff import CodeAutoGenerateDiff
+from autocoder.common.v2.code_auto_merge_diff import CodeAutoMergeDiff
+from autocoder.shadows.shadow_manager import ShadowManager
+from autocoder.linters.shadow_linter import ShadowLinter
+from autocoder.linters.models import IssueSeverity
+from loguru import logger
+from autocoder.common.global_cancel import global_cancel
+from autocoder.linters.models import ProjectLintResult
+from autocoder.common.token_cost_caculate import TokenCostCalculator
+
+
+class CodeDiffManager:
+    """
+    A class that combines code generation, linting, and merging with automatic error correction.
+    It generates code using diff format, lints it, and if there are errors, regenerates the code up to 5 times
+    before merging the final result.
+    """
+    
+    def __init__(
+        self,
+        llm: byzerllm.ByzerLLM,
+        args: AutoCoderArgs,
+        action=None,
+    ) -> None:
+        self.llm = llm
+        self.args = args
+        self.action = action
+        self.generate_times_same_model = args.generate_times_same_model
+        self.max_correction_attempts = args.auto_fix_lint_max_attempts
+        self.printer = Printer()
+        
+        # Initialize sub-components
+        self.code_generator = CodeAutoGenerateDiff(llm, args, action)
+        self.code_merger = CodeAutoMergeDiff(llm, args)
+        
+        # Create shadow manager for linting
+        self.shadow_manager = ShadowManager(args.source_dir)
+        self.shadow_linter = ShadowLinter(self.shadow_manager, verbose=False)
+
+    @byzerllm.prompt()
+    def fix_linter_errors(self, query: str, lint_issues: str) -> str:
+        """        
+        Linter 检测到的问题:
+        <lint_issues>
+        {{ lint_issues }}
+        </lint_issues>
+        
+        用户原始需求:
+        <user_query_wrapper>
+        {{ query }}
+        </user_query_wrapper>
+
+        修复上述问题，请确保代码质量问题被解决，同时保持代码的原有功能。
+        请使用 unified diff 格式输出修改。
+        """
+        
+    def _create_shadow_files_from_edits(self, generation_result: CodeGenerateResult) -> Dict[str, str]:
+        """
+        从编辑块内容中提取代码并创建临时影子文件用于检查。
+        
+        参数:
+            generation_result (CodeGenerateResult): 包含diff格式的内容
+            
+        返回:
+            Dict[str, str]: 映射 {影子文件路径: 内容}
+        """
+        result = self.code_merger.choose_best_choice(generation_result)
+        merge = self.code_merger._merge_code_without_effect(result.contents[0])        
+        shadow_files = {}
+        for file_path, new_content in merge.success_blocks:
+            self.shadow_manager.update_file(file_path, new_content)
+            shadow_files[self.shadow_manager.to_shadow_path(file_path)] = new_content
+
+        return shadow_files
+
+    def _format_lint_issues(self, lint_results:ProjectLintResult,level:IssueSeverity) -> str:
+        """
+        将linter结果格式化为字符串供模型使用
+        
+        参数:
+            lint_results: Linter结果对象
+            level: 过滤问题的级别
+            
+        返回:
+            str: 格式化的问题描述
+        """
+        formatted_issues = []
+        
+        for file_path, result in lint_results.file_results.items():                        
+            file_has_issues = False
+            file_issues = []
+            
+            for issue in result.issues:                
+                if issue.severity.value != level.value:
+                    continue
+                    
+                if not file_has_issues:
+                    file_has_issues = True
+                    file_issues.append(f"文件: {file_path}")
+                
+                severity = "错误" if issue.severity == IssueSeverity.ERROR else "警告" if issue.severity == IssueSeverity.WARNING else "信息"
+                line_info = f"第{issue.position.line}行"
+                if issue.position.column:
+                    line_info += f", 第{issue.position.column}列"
+                
+                file_issues.append(
+                    f"  - [{severity}] {line_info}: {issue.message} (规则: {issue.code})"
+                )
+            
+            if file_has_issues:
+                formatted_issues.extend(file_issues)
+                formatted_issues.append("")  # 空行分隔不同文件
+            
+        return "\n".join(formatted_issues)
+
+    def _count_errors(self, lint_results:ProjectLintResult) -> int:
+        """
+        计算lint结果中的错误数量
+        
+        参数:
+            lint_results: Linter结果对象
+            
+        返回:
+            int: 错误数量
+        """
+        error_count = 0
+        
+        for _, result in lint_results.file_results.items():                
+            error_count += result.error_count
+                
+        return error_count
+
+    def generate_and_fix(self, query: str, source_code_list: SourceCodeList) -> CodeGenerateResult:
+        """
+        生成代码，运行linter，修复错误，最多尝试指定次数
+        
+        参数:
+            query (str): 用户查询
+            source_code_list (SourceCodeList): 源代码列表
+            
+        返回:
+            CodeGenerateResult: 生成的代码结果
+        """
+        # 初始代码生成
+        self.printer.print_in_terminal("generating_initial_code")
+        start_time = time.time()
+        generation_result = self.code_generator.single_round_run(query, source_code_list)
+        
+        token_cost_calculator = TokenCostCalculator(args=self.args)
+        token_cost_calculator.track_token_usage_by_generate(
+            llm=self.llm,
+            generate=generation_result,
+            operation_name="code_generation_complete",
+            start_time=start_time,
+            end_time=time.time()
+        )
+        
+        # 确保结果非空
+        if not generation_result.contents:
+            self.printer.print_in_terminal("generation_failed", style="red")
+            return generation_result
+                            
+        
+        # 最多尝试修复5次
+        for attempt in range(self.max_correction_attempts):
+            global_cancel.check_and_raise()
+            # 代码生成结果更新到影子文件里去
+            shadow_files = self._create_shadow_files_from_edits(generation_result)
+            
+            if not shadow_files:
+                self.printer.print_in_terminal("no_files_to_lint", style="yellow")
+                break
+                
+            # 运行linter
+            lint_results = self.shadow_linter.lint_all_shadow_files()
+            error_count = self._count_errors(lint_results)
+            
+            # 如果没有错误则完成
+            if error_count == 0:
+                self.printer.print_in_terminal("no_lint_errors_found", style="green")
+                break
+
+            # 格式化lint问题
+            formatted_issues = self._format_lint_issues(lint_results, IssueSeverity.ERROR)            
+
+            # 打印当前错误
+            self.printer.print_in_terminal(
+                "lint_attempt_status", 
+                style="yellow", 
+                attempt=(attempt + 1), 
+                max_correction_attempts=self.max_correction_attempts,
+                error_count=error_count,
+                formatted_issues=formatted_issues
+            )
+            
+            if attempt == self.max_correction_attempts - 1:
+                self.printer.print_in_terminal("max_attempts_reached", style="yellow")
+                break
+                
+            # 准备修复提示
+            fix_prompt = self.fix_linter_errors.prompt(
+                query=query,
+                lint_issues=formatted_issues
+            )
+            
+            for source in source_code_list.sources:
+                print(f"file_path: {source.module_name}")                
+            print(f"fix_prompt: {fix_prompt}")
+            
+            # 将 shadow_files 转化为 source_code_list
+            source_code_list = self.code_merger.get_source_code_list_from_shadow_files(shadow_files)
+            start_time = time.time()
+            generation_result = self.code_generator.single_round_run(fix_prompt, source_code_list)            
+            token_cost_calculator.track_token_usage_by_generate(
+                llm=self.llm,
+                generate=generation_result,
+                operation_name="code_generation_complete",
+                start_time=start_time,
+                end_time=time.time()
+            )
+        
+        # 清理临时影子文件
+        self.shadow_manager.clean_shadows()
+        
+        # 返回最终结果
+        return generation_result
+
+    def run(self, query: str, source_code_list: SourceCodeList) -> CodeGenerateResult:
+        """
+        执行完整的代码生成、修复、合并流程
+        
+        参数:
+            query (str): 用户查询
+            source_code_list (SourceCodeList): 源代码列表
+            
+        返回:
+            CodeGenerateResult: 生成和修复的代码结果
+        """
+        # 生成代码并自动修复lint错误
+        generation_result = self.generate_and_fix(query, source_code_list)
+        global_cancel.check_and_raise()
+        # 合并代码        
+        self.code_merger.merge_code(generation_result)
+        
+        return generation_result