auto-coder 0.1.207__py3-none-any.whl → 0.1.209__py3-none-any.whl

autocoder/common/code_auto_merge.py

@@ -2,10 +2,12 @@
 import os
 from byzerllm.utils.client import code_utils
 from autocoder.common import AutoCoderArgs,git_utils
-from typing import List
+from typing import List,Union,Tuple
 import pydantic
 import byzerllm
 from loguru import logger
+from autocoder.common.types import CodeGenerateResult, MergeCodeWithoutEffect
+from autocoder.common.code_modification_ranker import CodeModificationRanker
 import hashlib
 
 class PathAndCode(pydantic.BaseModel):
@@ -58,7 +60,26 @@ class CodeAutoMerge:
             elif start_marker_count > 0:
                 block.append(line)                
 
-        return path_and_code_list     
+        return path_and_code_list
+
+    def merge_code(self, generate_result: CodeGenerateResult, force_skip_git: bool = False):
+        result = self.choose_best_choice(generate_result)
+        self._merge_code(result.contents[0], force_skip_git)
+        return result
+
+    def choose_best_choice(self, generate_result: CodeGenerateResult) -> CodeGenerateResult:
+        if len(generate_result.contents) == 1:
+            return generate_result
+
+        ranker = CodeModificationRanker(self.llm, self.args)
+        ranked_result = ranker.rank_modifications(generate_result)
+        # Filter out contents with failed blocks
+        for content,conversations in zip(ranked_result.contents,ranked_result.conversations):
+            merge_result = self._merge_code_without_effect(content)
+            if not merge_result.failed_blocks:
+                return CodeGenerateResult(contents=[content], conversations=[conversations])
+        # If all have failed blocks, return the first one
+        return CodeGenerateResult(contents=[ranked_result.contents[0]], conversations=[ranked_result.conversations[0]])
 
 
     def parse_text(self, text: str) -> List[PathAndCode]:
@@ -99,7 +120,34 @@ class CodeAutoMerge:
         Error: {{ error }}
         '''
 
-    def merge_code(self, content: str,force_skip_git:bool=False):        
+    def _merge_code_without_effect(self, content: str) -> MergeCodeWithoutEffect:
+        """Merge code without any side effects like git operations or file writing.
+        Returns a tuple of:
+        - list of (file_path, new_content) tuples for successfully merged blocks
+        - list of (file_path, content) tuples for failed to merge blocks"""
+        codes = self.parse_whole_text_v2(content)
+        file_content_mapping = {}
+        failed_blocks = []
+        
+        for block in codes:
+            file_path = block.path
+            if not os.path.exists(file_path):
+                file_content_mapping[file_path] = block.content
+            else:
+                if file_path not in file_content_mapping:
+                    with open(file_path, "r") as f:
+                        file_content_mapping[file_path] = f.read()
+                if file_content_mapping[file_path] != block.content:
+                    file_content_mapping[file_path] = block.content
+                else:
+                    failed_blocks.append((file_path, block.content))
+                
+        return MergeCodeWithoutEffect(
+            success_blocks=[(path, content) for path, content in file_content_mapping.items()],
+            failed_blocks=failed_blocks
+        )
+
+    def _merge_code(self, content: str,force_skip_git:bool=False):        
         total = 0
         
         file_content = open(self.args.file).read()
@@ -114,18 +162,6 @@ class CodeAutoMerge:
                 logger.error(self.git_require_msg(source_dir=self.args.source_dir,error=str(e)))
                 return            
 
-        # codes =  code_utils.extract_code(content)
-        # for (lang,code) in codes:            
-        #     parsed_blocks = self.parse_text(code)
-
-        #     for block in parsed_blocks:
-        #         file_path = block.path
-        #         os.makedirs(os.path.dirname(file_path), exist_ok=True)
-
-        #         with open(file_path, "w") as f:
-        #             logger.info(f"Upsert path: {file_path}")
-        #             total += 1
-        #             f.write(block.content)
         codes = self.parse_whole_text_v2(content)
         for block in codes:
             file_path = block.path

autocoder/common/code_auto_merge_diff.py

@@ -1,7 +1,7 @@
 import os
 import difflib
 from autocoder.common import AutoCoderArgs,git_utils
-from typing import List
+from typing import List,Union,Tuple
 import pydantic
 import byzerllm
 from loguru import logger
@@ -15,6 +15,8 @@ from autocoder.common.search_replace import (
     flexible_search_and_replace,
     search_and_replace,
 )
+from autocoder.common.types import CodeGenerateResult, MergeCodeWithoutEffect
+from autocoder.common.code_modification_ranker import CodeModificationRanker
 
 class PathAndCode(pydantic.BaseModel):
     path: str
@@ -374,6 +376,25 @@ class CodeAutoMergeDiff:
             edits.append((path, hunk))
 
         return edits
+
+    def merge_code(self, generate_result: CodeGenerateResult, force_skip_git: bool = False):
+        result = self.choose_best_choice(generate_result)
+        self._merge_code(result.contents[0], force_skip_git)
+        return result
+
+    def choose_best_choice(self, generate_result: CodeGenerateResult) -> CodeGenerateResult:
+        if len(generate_result.contents) == 1:
+            return generate_result
+
+        ranker = CodeModificationRanker(self.llm, self.args)
+        ranked_result = ranker.rank_modifications(generate_result)
+        # Filter out contents with failed blocks
+        for content,conversations in zip(ranked_result.contents,ranked_result.conversations):
+            merge_result = self._merge_code_without_effect(content)
+            if not merge_result.failed_blocks:
+                return CodeGenerateResult(contents=[content], conversations=[conversations])
+        # If all have failed blocks, return the first one
+        return CodeGenerateResult(contents=[ranked_result.contents[0]], conversations=[ranked_result.conversations[0]])
     
     @byzerllm.prompt(render="jinja2")
     def git_require_msg(self,source_dir:str,error:str)->str:
@@ -450,7 +471,39 @@ class CodeAutoMergeDiff:
                 errors += other_hunks_applied
             raise ValueError(errors)    
 
-    def merge_code(self, content: str,force_skip_git:bool=False):        
+    def _merge_code_without_effect(self, content: str) -> MergeCodeWithoutEffect:
+        """Merge code without any side effects like git operations or file writing.
+        Returns a tuple of:
+        - list of (file_path, new_content) tuples for successfully merged blocks
+        - list of (file_path, hunk) tuples for failed to merge blocks"""
+        edits = self.get_edits(content)
+        file_content_mapping = {}
+        failed_blocks = []
+        
+        for path, hunk in edits:
+            full_path = self.abs_root_path(path)
+            if not os.path.exists(full_path):
+                _, after = hunk_to_before_after(hunk)
+                file_content_mapping[full_path] = after
+                continue
+                
+            if full_path not in file_content_mapping:
+                with open(full_path, "r") as f:
+                    file_content_mapping[full_path] = f.read()
+            
+            content = file_content_mapping[full_path]
+            new_content = do_replace(full_path, content, hunk)
+            if new_content:
+                file_content_mapping[full_path] = new_content
+            else:
+                failed_blocks.append((full_path, "\n".join(hunk)))
+                
+        return MergeCodeWithoutEffect(
+            success_blocks=[(path, content) for path, content in file_content_mapping.items()],
+            failed_blocks=failed_blocks
+        )
+
+    def _merge_code(self, content: str,force_skip_git:bool=False):        
         total = 0
         
         file_content = open(self.args.file).read()

autocoder/common/code_auto_merge_editblock.py

@@ -7,7 +7,6 @@ from autocoder.utils.queue_communicate import (
     CommunicateEvent,
     CommunicateEventType,
 )
-from typing import List
 import pydantic
 import byzerllm
 from loguru import logger
@@ -18,6 +17,9 @@ from rich.console import Console
 from rich.panel import Panel
 from rich.syntax import Syntax
 import json
+from typing import Union, List, Tuple
+from autocoder.common.types import CodeGenerateResult, MergeCodeWithoutEffect
+from autocoder.common.code_modification_ranker import CodeModificationRanker
 
 
 class PathAndCode(pydantic.BaseModel):
@@ -138,7 +140,7 @@ class CodeAutoMergeEditBlock:
             elif end_marker(line, index) and start_marker_count == 1:
                 start_marker_count -= 1
                 if block:
-                    if current_editblock_mode == "two_line_mode":                        
+                    if current_editblock_mode == "two_line_mode":
                         path = block[0].split(":", 1)[1].strip()
                         content = "\n".join(block[1:])
                     else:
@@ -152,6 +154,25 @@ class CodeAutoMergeEditBlock:
 
         return path_and_code_list
 
+    def merge_code(self, generate_result: CodeGenerateResult, force_skip_git: bool = False):
+        result = self.choose_best_choice(generate_result)
+        self._merge_code(result.contents[0], force_skip_git)
+        return result
+
+    def choose_best_choice(self, generate_result: CodeGenerateResult) -> CodeGenerateResult:
+        if len(generate_result.contents) == 1:
+            return generate_result
+
+        ranker = CodeModificationRanker(self.llm, self.args)
+        ranked_result = ranker.rank_modifications(generate_result)
+        # Filter out contents with failed blocks
+        for content,conversations in zip(ranked_result.contents,ranked_result.conversations):
+            merge_result = self._merge_code_without_effect(content)
+            if not merge_result.failed_blocks:
+                return CodeGenerateResult(contents=[content], conversations=[conversations])
+        # If all have failed blocks, return the first one
+        return CodeGenerateResult(contents=[ranked_result.contents[0]], conversations=[ranked_result.conversations[0]])
+
     @byzerllm.prompt()
     def git_require_msg(self, source_dir: str, error: str) -> str:
         """
@@ -197,9 +218,57 @@ class CodeAutoMergeEditBlock:
                 if in_updated:
                     updates.append(line)
             result.append((edit.path, "\n".join(heads), "\n".join(updates)))
-        return result
+        return result        
+
+    def _merge_code_without_effect(self, content: str) -> MergeCodeWithoutEffect:
+        """Merge code without any side effects like git operations, linting or file writing.
+        Returns a tuple of:
+        - list of (file_path, new_content) tuples for successfully merged blocks
+        - list of (file_path, head, update) tuples for failed to merge blocks"""
+        codes = self.get_edits(content)
+        file_content_mapping = {}
+        failed_blocks = []
+
+        for block in codes:
+            file_path, head, update = block
+            if not os.path.exists(file_path):
+                file_content_mapping[file_path] = update
+            else:
+                if file_path not in file_content_mapping:
+                    with open(file_path, "r") as f:
+                        temp = f.read()
+                        file_content_mapping[file_path] = temp
+                existing_content = file_content_mapping[file_path]
+
+                # First try exact match
+                new_content = (
+                    existing_content.replace(head, update, 1)
+                    if head
+                    else existing_content + "\n" + update
+                )
+
+                # If exact match fails, try similarity match
+                if new_content == existing_content and head:
+                    similarity, best_window = TextSimilarity(
+                        head, existing_content
+                    ).get_best_matching_window()
+                    if similarity > self.args.editblock_similarity:
+                        new_content = existing_content.replace(
+                            best_window, update, 1
+                        )
 
-    def merge_code(self, content: str, force_skip_git: bool = False):
+                if new_content != existing_content:
+                    file_content_mapping[file_path] = new_content
+                else:
+                    failed_blocks.append((file_path, head, update))
+
+        return MergeCodeWithoutEffect(
+            success_blocks=[(path, content)
+                            for path, content in file_content_mapping.items()],
+            failed_blocks=failed_blocks
+        )
+
+    def _merge_code(self, content: str, force_skip_git: bool = False):
         file_content = open(self.args.file).read()
         md5 = hashlib.md5(file_content.encode("utf-8")).hexdigest()
         file_name = os.path.basename(self.args.file)
@@ -207,15 +276,15 @@ class CodeAutoMergeEditBlock:
         codes = self.get_edits(content)
         changes_to_make = []
         changes_made = False
-        unmerged_blocks = []  
-        merged_blocks = []      
+        unmerged_blocks = []
+        merged_blocks = []
 
         # First, check if there are any changes to be made
         file_content_mapping = {}
         for block in codes:
             file_path, head, update = block
             if not os.path.exists(file_path):
-                changes_to_make.append((file_path, None, update))                
+                changes_to_make.append((file_path, None, update))
                 file_content_mapping[file_path] = update
                 merged_blocks.append((file_path, "", update, 1))
                 changes_made = True
@@ -235,7 +304,7 @@ class CodeAutoMergeEditBlock:
                         (file_path, existing_content, new_content))
                     file_content_mapping[file_path] = new_content
                     merged_blocks.append((file_path, head, update, 1))
-                    changes_made = True                    
+                    changes_made = True
                 else:
                     # If the SEARCH BLOCK is not found exactly, then try to use
                     # the similarity ratio to find the best matching block
@@ -250,8 +319,9 @@ class CodeAutoMergeEditBlock:
                                 (file_path, existing_content, new_content)
                             )
                             file_content_mapping[file_path] = new_content
-                            merged_blocks.append((file_path, head, update, similarity))
-                            changes_made = True                            
+                            merged_blocks.append(
+                                (file_path, head, update, similarity))
+                            changes_made = True
                     else:
                         unmerged_blocks.append(
                             (file_path, head, update, similarity))
@@ -317,10 +387,10 @@ class CodeAutoMergeEditBlock:
                 file_path, head, update, similarity = code
                 event_data.append(
                     {
-                            "file_path": file_path,
-                            "head": head,
-                            "update": update,
-                            "similarity": similarity,
+                        "file_path": file_path,
+                        "head": head,
+                        "update": update,
+                        "similarity": similarity,
                     }
                 )
 

autocoder/common/code_auto_merge_strict_diff.py

@@ -2,12 +2,14 @@ import os
 import difflib
 import diff_match_patch as dmp_module
 from autocoder.common import AutoCoderArgs, git_utils
-from typing import List
+from typing import List,Tuple
 import pydantic
 import byzerllm
 from loguru import logger
 import hashlib
 from pathlib import Path
+from autocoder.common.types import CodeGenerateResult, MergeCodeWithoutEffect
+from autocoder.common.code_modification_ranker import CodeModificationRanker
 
 class PathAndCode(pydantic.BaseModel):
     path: str
@@ -125,7 +127,26 @@ class CodeAutoMergeStrictDiff:
             elif start_marker_count > 0:
                 block.append(line)                
 
-        return path_and_code_list    
+        return path_and_code_list
+
+    def merge_code(self, generate_result: CodeGenerateResult, force_skip_git: bool = False):
+        result = self.choose_best_choice(generate_result)
+        self._merge_code(result.contents[0], force_skip_git)
+        return result
+
+    def choose_best_choice(self, generate_result: CodeGenerateResult) -> CodeGenerateResult:
+        if len(generate_result.contents) == 1:
+            return generate_result
+
+        ranker = CodeModificationRanker(self.llm, self.args)
+        ranked_result = ranker.rank_modifications(generate_result)
+        # Filter out contents with failed blocks
+        for content,conversations in zip(ranked_result.contents,ranked_result.conversations):
+            merge_result = self._merge_code_without_effect(content)
+            if not merge_result.failed_blocks:
+                return CodeGenerateResult(contents=[content], conversations=[conversations])
+        # If all have failed blocks, return the first one
+        return CodeGenerateResult(contents=[ranked_result.contents[0]], conversations=[ranked_result.conversations[0]])
     
 
     def abs_root_path(self, path):
@@ -134,7 +155,52 @@ class CodeAutoMergeStrictDiff:
         res = Path(self.args.source_dir) / path
         return safe_abs_path(res)            
 
-    def merge_code(self, content: str, force_skip_git: bool = False):        
+    def _merge_code_without_effect(self, content: str) -> MergeCodeWithoutEffect:
+        """Merge code without any side effects like git operations or file writing.
+        Returns a tuple of:
+        - list of (file_path, new_content) tuples for successfully merged blocks
+        - list of (file_path, content) tuples for failed to merge blocks"""
+        diff_blocks = self.parse_diff_block(content)
+        file_content_mapping = {}
+        failed_blocks = []
+        
+        for block in diff_blocks:
+            path = block.path
+            content = block.content
+            full_path = self.abs_root_path(path)
+            
+            if not os.path.exists(full_path):
+                file_content_mapping[full_path] = content
+                continue
+                
+            if full_path not in file_content_mapping:
+                with open(full_path, "r") as f:
+                    file_content_mapping[full_path] = f.read()
+            
+            try:
+                import patch
+                patch_obj = patch.fromstring(content.encode('utf-8'))
+                root_path = None
+                if not path.startswith(self.args.source_dir):
+                    root_path = self.args.source_dir
+
+                # Create a copy of the content to apply patch
+                temp_content = file_content_mapping[full_path]
+                success = patch_obj.apply(root=root_path, content=temp_content)
+                if success:
+                    file_content_mapping[full_path] = temp_content
+                else:
+                    failed_blocks.append((full_path, content))
+            except Exception as e:
+                logger.warning(f"Failed to apply patch to {full_path}: {str(e)}")
+                failed_blocks.append((full_path, content))
+                
+        return MergeCodeWithoutEffect(
+            success_blocks=[(path, content) for path, content in file_content_mapping.items()],
+            failed_blocks=failed_blocks
+        )
+
+    def _merge_code(self, content: str, force_skip_git: bool = False):        
         total = 0
         
         file_content = open(self.args.file).read()
@@ -154,35 +220,6 @@ class CodeAutoMergeStrictDiff:
             path = diff_blocks.path
             content = diff_blocks.content          
 
-            # unidiff_patch = unidiff.PatchSet(content)
-            # dmp_patches = []
-            # for patched_file in unidiff_patch:
-            #     diffs = []
-            #     start_line = 0
-            #     for hunk in patched_file:
-            #         start_line = hunk.target_start - 1  # 获取hunk的起始位置
-            #         for line in hunk:
-            #             if line.is_added:
-            #                 diffs.append(dmp_module.diff('', line.value.strip(), start_line))
-            #                 start_line += 1
-            #             elif line.is_removed:
-            #                 diffs.append(dmp_module.diff(line.value.strip(), '', start_line))
-            #             else:
-            #                 start_line += 1
-            #     patch_text = dmp.patch_make(diffs)
-            #     dmp_patches.extend(patch_text)
-            
-            # with open(path, 'r') as f:
-            #     original_content = f.read()
-                
-            # dmp = dmp_module.diff_match_patch()            
-            # new_text, results = dmp.patch_apply(dmp_patches, original_content)            
-            # if any(results) is False:
-            #     raise Exception("Error applying diff to file: " + path)
-            # with open(self.abs_root_path(path), 'w') as f:
-            #     f.write(new_text)
-            # total += 1 
-
             import patch
             patch_obj = patch.fromstring(content.encode('utf-8'))
             root_path = None

autocoder/common/code_modification_ranker.py

@@ -0,0 +1,100 @@
+import byzerllm
+from typing import List,Union
+from autocoder.common import AutoCoderArgs
+from autocoder.common.types import CodeGenerateResult
+from pydantic import BaseModel
+from loguru import logger
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import traceback
+
+class RankResult(BaseModel):
+    rank_result:List[int]
+
+class CodeModificationRanker:
+    def __init__(self, llm: byzerllm.ByzerLLM, args: AutoCoderArgs):
+        self.llm = llm
+        self.args = args     
+        if self.llm.get_sub_client("generate_rerank_model"):
+            self.rerank_llm = self.llm.get_sub_client("generate_rerank_model")
+        else:
+            self.rerank_llm = self.llm
+    
+    @byzerllm.prompt()
+    def _rank_modifications(self, s:CodeGenerateResult) -> str:
+        '''
+        对一组代码修改进行质量评估并排序。
+
+        下面是修改需求：
+        
+        <edit_requirement>
+        {{ s.conversations[0][-2]["content"] }}
+        </edit_requirement>
+        
+        下面是相应的代码修改：
+        {% for content in s.contents %}
+        <edit_block id="{{ loop.index0 }}">
+        {{content}}
+        </edit_block>
+        {% endfor %}
+        
+        请输出如下格式的评估结果,只包含 JSON 数据:
+
+        ```json
+        {
+            "rank_result": [id1, id2, id3]  // id 为 edit_block 的 id,按质量从高到低排序
+        }
+        ```
+
+        注意，只输出前面要求的 Json 格式就好，不要输出其他内容，Json 需要使用 ```json ```包裹。
+        '''
+        
+
+    def rank_modifications(self, generate_result: CodeGenerateResult) -> CodeGenerateResult:
+        import time
+        start_time = time.time()
+        
+        # 如果只有一个候选，直接返回
+        if len(generate_result.contents) == 1:
+            logger.info("Only 1 candidate, skip ranking")
+            return generate_result
+            
+        logger.info(f"Start ranking {len(generate_result.contents)} candidates")
+        generate_times = self.args.generate_times_same_model
+        
+        try:
+            # Create a thread pool with generate_times workers
+            with ThreadPoolExecutor(max_workers=generate_times) as executor:
+                # Submit tasks
+                futures = [
+                    executor.submit(
+                        self._rank_modifications.with_llm(self.rerank_llm).with_return_type(RankResult).run,
+                        generate_result
+                    ) for _ in range(generate_times)
+                ]
+                
+                # Process results as they complete
+                for future in as_completed(futures):
+                    try:
+                        v = future.result()
+                        # If we get a valid result, use it and cancel other tasks
+                        for f in futures:
+                            f.cancel()
+                            
+                        elapsed = time.time() - start_time
+                        logger.info(f"Ranking completed in {elapsed:.2f}s, best candidate index: {v.rank_result[0]}")
+                        
+                        rerank_contents = [generate_result.contents[i] for i in v.rank_result]
+                        rerank_conversations = [generate_result.conversations[i] for i in v.rank_result]
+                        return CodeGenerateResult(contents=rerank_contents,conversations=rerank_conversations)
+                    except Exception as e:
+                        logger.warning(f"Ranking request failed: {str(e)}")
+                        logger.debug(traceback.format_exc())
+                        continue
+        except Exception as e:
+            logger.error(f"Ranking process failed: {str(e)}")
+            logger.debug(traceback.format_exc())
+        
+        # If all requests failed, use the original codes
+        elapsed = time.time() - start_time
+        logger.warning(f"All ranking requests failed in {elapsed:.2f}s, using original order")
+        return generate_result

autocoder/common/command_completer.py

@@ -12,7 +12,7 @@ COMMANDS = {
         "/svg": {},
         "/sd": {},
     },
-    "/coding": {},
+    "/coding": {"/apply": {}, "/next": {}},
     "/chat": {"/new": {}, "/review": {}, "/no_context": {}},
     "/lib": {
         "/add": "",
@@ -147,7 +147,8 @@ class CommandTextParser:
                 current_word += v
         self.is_extracted = True
         self.current_word_end_pos = self.pos + 1
-        self.current_word_start_pos = self.current_word_end_pos - len(current_word)
+        self.current_word_start_pos = self.current_word_end_pos - \
+            len(current_word)
 
     def previous(self) -> str:
         if self.pos > 1:
@@ -227,10 +228,11 @@ class CommandTextParser:
             self.is_extracted = True
 
         self.current_word_end_pos = self.pos + 1
-        self.current_word_start_pos = self.current_word_end_pos - len(current_word)
+        self.current_word_start_pos = self.current_word_end_pos - \
+            len(current_word)
 
     def current_word(self) -> str:
-        return self.text[self.current_word_start_pos : self.current_word_end_pos]
+        return self.text[self.current_word_start_pos: self.current_word_end_pos]
 
     def get_current_word(self) -> str:
         return self.current_word()

autocoder/common/types.py

@@ -1,10 +1,18 @@
 from enum import Enum
 import pydantic
-
+from typing import List, Dict, Tuple,Any
 class Mode(Enum):
     MULTI_ROUND = "multi_round"
     SINGLE_ROUND = "single_round"
 
 class StepNum(pydantic.BaseModel):
     step_num:int= pydantic.Field(1,description="总共步骤数")
-    content:int= pydantic.Field(1,description="详细的执行步骤，每个步骤需要包含一个shell/python 代码块")    
+    content:int= pydantic.Field(1,description="详细的执行步骤，每个步骤需要包含一个shell/python 代码块")    
+
+class CodeGenerateResult(pydantic.BaseModel):
+    contents:List[str]
+    conversations:List[List[Dict[str, Any]]]
+
+class MergeCodeWithoutEffect(pydantic.BaseModel):
+    success_blocks: List[Tuple[str, str]]
+    failed_blocks: List[Any]