autocoder-nano 0.1.25__py3-none-any.whl → 0.1.27__py3-none-any.whl

autocoder_nano/edit/code/merge_editblock.py

@@ -0,0 +1,418 @@
+import hashlib
+import os
+import subprocess
+import tempfile
+from typing import List
+
+from loguru import logger
+from rich.console import Console
+from rich.panel import Panel
+from rich.syntax import Syntax
+from rich.table import Table
+
+from autocoder_nano.edit.code.modification_ranker import CodeModificationRanker
+from autocoder_nano.edit.text import TextSimilarity
+from autocoder_nano.git_utils import commit_changes
+from autocoder_nano.llm_client import AutoLLM
+from autocoder_nano.llm_prompt import prompt
+from autocoder_nano.llm_types import AutoCoderArgs, PathAndCode, MergeCodeWithoutEffect, CodeGenerateResult, \
+    CommitResult
+
+
+console = Console()
+
+
+def git_print_commit_info(commit_result: CommitResult):
+    table = Table(
+        title="Commit Information (Use /revert to revert this commit)", show_header=True, header_style="bold magenta"
+    )
+    table.add_column("Attribute", style="cyan", no_wrap=True)
+    table.add_column("Value", style="green")
+
+    table.add_row("Commit Hash", commit_result.commit_hash)
+    table.add_row("Commit Message", commit_result.commit_message)
+    table.add_row("Changed Files", "\n".join(commit_result.changed_files))
+
+    console.print(
+        Panel(table, expand=False, border_style="green", title="Git Commit Summary")
+    )
+
+    if commit_result.diffs:
+        for file, diff in commit_result.diffs.items():
+            console.print(f"\n[bold blue]File: {file}[/bold blue]")
+            syntax = Syntax(diff, "diff", theme="monokai", line_numbers=True)
+            console.print(
+                Panel(syntax, expand=False, border_style="yellow", title="File Diff")
+            )
+
+
+class CodeAutoMergeEditBlock:
+    def __init__(self, args: AutoCoderArgs, llm: AutoLLM, fence_0: str = "```", fence_1: str = "```"):
+        self.llm = llm
+        self.llm.setup_default_model_name(args.code_model)
+        self.args = args
+        self.fence_0 = fence_0
+        self.fence_1 = fence_1
+
+    @staticmethod
+    def run_pylint(code: str) -> tuple[bool, str]:
+        """
+        --disable=all 禁用所有 Pylint 的检查规则
+        --enable=E0001,W0311,W0312 启用指定的 Pylint 检查规则,
+        E0001：语法错误(Syntax Error),
+        W0311：代码缩进使用了 Tab 而不是空格(Bad indentation)
+        W0312：代码缩进不一致(Mixed indentation)
+        :param code:
+        :return:
+        """
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as temp_file:
+            temp_file.write(code)
+            temp_file_path = temp_file.name
+
+        try:
+            result = subprocess.run(
+                ["pylint", "--disable=all", "--enable=E0001,W0311,W0312", temp_file_path,],
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            os.unlink(temp_file_path)
+            if result.returncode != 0:
+                error_message = result.stdout.strip() or result.stderr.strip()
+                logger.warning(f"Pylint 检查代码失败: {error_message}")
+                return False, error_message
+            return True, ""
+        except subprocess.CalledProcessError as e:
+            error_message = f"运行 Pylint 时发生错误: {str(e)}"
+            logger.error(error_message)
+            os.unlink(temp_file_path)
+            return False, error_message
+
+    def parse_whole_text(self, text: str) -> List[PathAndCode]:
+        """
+        从文本中抽取如下格式代码(two_line_mode)：
+
+        ```python
+        ##File: /project/path/src/autocoder/index/index.py
+        <<<<<<< SEARCH
+        =======
+        >>>>>>> REPLACE
+        ```
+
+        或者 (one_line_mode)
+
+        ```python:/project/path/src/autocoder/index/index.py
+        <<<<<<< SEARCH
+        =======
+        >>>>>>> REPLACE
+        ```
+        """
+        HEAD = "<<<<<<< SEARCH"
+        DIVIDER = "======="
+        UPDATED = ">>>>>>> REPLACE"
+        lines = text.split("\n")
+        lines_len = len(lines)
+        start_marker_count = 0
+        block = []
+        path_and_code_list = []
+        # two_line_mode or one_line_mode
+        current_editblock_mode = "two_line_mode"
+        current_editblock_path = None
+
+        def guard(_index):
+            return _index + 1 < lines_len
+
+        def start_marker(_line, _index):
+            nonlocal current_editblock_mode
+            nonlocal current_editblock_path
+            if _line.startswith(self.fence_0) and guard(_index) and ":" in _line and lines[_index + 1].startswith(HEAD):
+                current_editblock_mode = "one_line_mode"
+                current_editblock_path = _line.split(":", 1)[1].strip()
+                return True
+            if _line.startswith(self.fence_0) and guard(_index) and lines[_index + 1].startswith("##File:"):
+                current_editblock_mode = "two_line_mode"
+                current_editblock_path = None
+                return True
+            return False
+
+        def end_marker(_line, _index):
+            return _line.startswith(self.fence_1) and UPDATED in lines[_index - 1]
+
+        for index, line in enumerate(lines):
+            if start_marker(line, index) and start_marker_count == 0:
+                start_marker_count += 1
+            elif end_marker(line, index) and start_marker_count == 1:
+                start_marker_count -= 1
+                if block:
+                    if current_editblock_mode == "two_line_mode":
+                        path = block[0].split(":", 1)[1].strip()
+                        content = "\n".join(block[1:])
+                    else:
+                        path = current_editblock_path
+                        content = "\n".join(block)
+                    block = []
+                    path_and_code_list.append(PathAndCode(path=path, content=content))
+            elif start_marker_count > 0:
+                block.append(line)
+
+        return path_and_code_list
+
+    def get_edits(self, content: str):
+        edits = self.parse_whole_text(content)
+        HEAD = "<<<<<<< SEARCH"
+        DIVIDER = "======="
+        UPDATED = ">>>>>>> REPLACE"
+        result = []
+        for edit in edits:
+            heads = []
+            updates = []
+            c = edit.content
+            in_head = False
+            in_updated = False
+            for line in c.splitlines():
+                if line.strip() == HEAD:
+                    in_head = True
+                    continue
+                if line.strip() == DIVIDER:
+                    in_head = False
+                    in_updated = True
+                    continue
+                if line.strip() == UPDATED:
+                    in_head = False
+                    in_updated = False
+                    continue
+                if in_head:
+                    heads.append(line)
+                if in_updated:
+                    updates.append(line)
+            result.append((edit.path, "\n".join(heads), "\n".join(updates)))
+        return result
+
+    @prompt()
+    def git_require_msg(self, source_dir: str, error: str) -> str:
+        """
+        auto_merge only works for git repositories.
+
+        Try to use git init in the source directory.
+
+        ```shell
+        cd {{ source_dir }}
+        git init .
+        ```
+
+        Then try to run auto-coder again.
+        Error: {{ error }}
+        """
+
+    def _merge_code_without_effect(self, content: str) -> MergeCodeWithoutEffect:
+        """
+        合并代码时不会产生任何副作用，例如 Git 操作、代码检查或文件写入。
+        返回一个元组，包含：
+        - 成功合并的代码块的列表，每个元素是一个 (file_path, new_content) 元组，
+          其中 file_path 是文件路径，new_content 是合并后的新内容。
+        - 合并失败的代码块的列表，每个元素是一个 (file_path, head, update) 元组，
+          其中：file_path 是文件路径，head 是原始内容，update 是尝试合并的内容。
+        """
+        codes = self.get_edits(content)
+        file_content_mapping = {}
+        failed_blocks = []
+
+        for block in codes:
+            file_path, head, update = block
+            if not os.path.exists(file_path):
+                file_content_mapping[file_path] = update
+            else:
+                if file_path not in file_content_mapping:
+                    with open(file_path, "r") as f:
+                        temp = f.read()
+                        file_content_mapping[file_path] = temp
+                existing_content = file_content_mapping[file_path]
+
+                # First try exact match
+                new_content = (
+                    existing_content.replace(head, update, 1)
+                    if head
+                    else existing_content + "\n" + update
+                )
+
+                # If exact match fails, try similarity match
+                if new_content == existing_content and head:
+                    similarity, best_window = TextSimilarity(
+                        head, existing_content
+                    ).get_best_matching_window()
+                    if similarity > self.args.editblock_similarity:
+                        new_content = existing_content.replace(
+                            best_window, update, 1
+                        )
+
+                if new_content != existing_content:
+                    file_content_mapping[file_path] = new_content
+                else:
+                    failed_blocks.append((file_path, head, update))
+        return MergeCodeWithoutEffect(
+            success_blocks=[(path, content) for path, content in file_content_mapping.items()],
+            failed_blocks=failed_blocks
+        )
+
+    def choose_best_choice(self, generate_result: CodeGenerateResult) -> CodeGenerateResult:
+        """ 选择最佳代码 """
+        if len(generate_result.contents) == 1:  # 仅一份代码立即返回
+            logger.info("仅有一个候选结果，跳过排序")
+            return generate_result
+
+        ranker = CodeModificationRanker(args=self.args, llm=self.llm)
+        ranked_result = ranker.rank_modifications(generate_result)
+        # 过滤掉包含失败块的内容
+        for content, conversations in zip(ranked_result.contents, ranked_result.conversations):
+            merge_result = self._merge_code_without_effect(content)
+            if not merge_result.failed_blocks:
+                return CodeGenerateResult(contents=[content], conversations=[conversations])
+        # 如果所有内容都包含失败块，则返回第一个
+        return CodeGenerateResult(contents=[ranked_result.contents[0]], conversations=[ranked_result.conversations[0]])
+
+    def _merge_code(self, content: str, force_skip_git: bool = False):
+        file_content = open(self.args.file).read()
+        md5 = hashlib.md5(file_content.encode("utf-8")).hexdigest()
+        file_name = os.path.basename(self.args.file)
+
+        codes = self.get_edits(content)
+        changes_to_make = []
+        changes_made = False
+        unmerged_blocks = []
+        merged_blocks = []
+
+        # First, check if there are any changes to be made
+        file_content_mapping = {}
+        for block in codes:
+            file_path, head, update = block
+            if not os.path.exists(file_path):
+                changes_to_make.append((file_path, None, update))
+                file_content_mapping[file_path] = update
+                merged_blocks.append((file_path, "", update, 1))
+                changes_made = True
+            else:
+                if file_path not in file_content_mapping:
+                    with open(file_path, "r") as f:
+                        temp = f.read()
+                        file_content_mapping[file_path] = temp
+                existing_content = file_content_mapping[file_path]
+                new_content = (
+                    existing_content.replace(head, update, 1)
+                    if head
+                    else existing_content + "\n" + update
+                )
+                if new_content != existing_content:
+                    changes_to_make.append(
+                        (file_path, existing_content, new_content))
+                    file_content_mapping[file_path] = new_content
+                    merged_blocks.append((file_path, head, update, 1))
+                    changes_made = True
+                else:
+                    # If the SEARCH BLOCK is not found exactly, then try to use
+                    # the similarity ratio to find the best matching block
+                    similarity, best_window = TextSimilarity(head, existing_content).get_best_matching_window()
+                    if similarity > self.args.editblock_similarity:  # 相似性比较
+                        new_content = existing_content.replace(
+                            best_window, update, 1)
+                        if new_content != existing_content:
+                            changes_to_make.append(
+                                (file_path, existing_content, new_content)
+                            )
+                            file_content_mapping[file_path] = new_content
+                            merged_blocks.append(
+                                (file_path, head, update, similarity))
+                            changes_made = True
+                    else:
+                        unmerged_blocks.append((file_path, head, update, similarity))
+
+        if unmerged_blocks:
+            if self.args.request_id and not self.args.skip_events:
+                # collect unmerged blocks
+                event_data = []
+                for file_path, head, update, similarity in unmerged_blocks:
+                    event_data.append(
+                        {
+                            "file_path": file_path,
+                            "head": head,
+                            "update": update,
+                            "similarity": similarity,
+                        }
+                    )
+                return
+            logger.warning(f"发现 {len(unmerged_blocks)} 个未合并的代码块，更改将不会应用，请手动检查这些代码块后重试。")
+            self._print_unmerged_blocks(unmerged_blocks)
+            return
+
+        # lint check
+        for file_path, new_content in file_content_mapping.items():
+            if file_path.endswith(".py"):
+                pylint_passed, error_message = self.run_pylint(new_content)
+                if not pylint_passed:
+                    logger.warning(f"代码文件 {file_path} 的 Pylint 检查未通过，本次更改未应用。错误信息: {error_message}")
+
+        if changes_made and not force_skip_git and not self.args.skip_commit:
+            try:
+                commit_changes(self.args.source_dir, f"auto_coder_pre_{file_name}_{md5}")
+            except Exception as e:
+                logger.error(
+                    self.git_require_msg(
+                        source_dir=self.args.source_dir, error=str(e))
+                )
+                return
+        # Now, apply the changes
+        for file_path, new_content in file_content_mapping.items():
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            with open(file_path, "w") as f:
+                f.write(new_content)
+
+        if self.args.request_id and not self.args.skip_events:
+            # collect modified files
+            event_data = []
+            for code in merged_blocks:
+                file_path, head, update, similarity = code
+                event_data.append(
+                    {
+                        "file_path": file_path,
+                        "head": head,
+                        "update": update,
+                        "similarity": similarity,
+                    }
+                )
+
+        if changes_made:
+            if not force_skip_git and not self.args.skip_commit:
+                try:
+                    commit_result = commit_changes(self.args.source_dir, f"auto_coder_{file_name}_{md5}")
+                    git_print_commit_info(commit_result=commit_result)
+                except Exception as e:
+                    logger.error(
+                        self.git_require_msg(
+                            source_dir=self.args.source_dir, error=str(e)
+                        )
+                    )
+            logger.info(
+                f"已在 {len(file_content_mapping.keys())} 个文件中合并更改，"
+                f"完成 {len(changes_to_make)}/{len(codes)} 个代码块。"
+            )
+        else:
+            logger.warning("未对任何文件进行更改。")
+
+    def merge_code(self, generate_result: CodeGenerateResult, force_skip_git: bool = False):
+        result = self.choose_best_choice(generate_result)
+        self._merge_code(result.contents[0], force_skip_git)
+        return result
+
+    @staticmethod
+    def _print_unmerged_blocks(unmerged_blocks: List[tuple]):
+        console.print(f"\n[bold red]未合并的代码块:[/bold red]")
+        for file_path, head, update, similarity in unmerged_blocks:
+            console.print(f"\n[bold blue]文件:[/bold blue] {file_path}")
+            console.print(
+                f"\n[bold green]搜索代码块（相似度：{similarity}）:[/bold green]")
+            syntax = Syntax(head, "python", theme="monokai", line_numbers=True)
+            console.print(Panel(syntax, expand=False))
+            console.print("\n[bold yellow]替换代码块:[/bold yellow]")
+            syntax = Syntax(update, "python", theme="monokai",
+                            line_numbers=True)
+            console.print(Panel(syntax, expand=False))
+        console.print(f"\n[bold red]未合并的代码块总数: {len(unmerged_blocks)}[/bold red]")

autocoder_nano/edit/code/modification_ranker.py

@@ -0,0 +1,90 @@
+import traceback
+
+from loguru import logger
+
+from autocoder_nano.llm_client import AutoLLM
+from autocoder_nano.llm_prompt import prompt
+from autocoder_nano.llm_types import AutoCoderArgs, CodeGenerateResult, RankResult
+
+
+class CodeModificationRanker:
+    def __init__(self, args: AutoCoderArgs, llm: AutoLLM):
+        self.args = args
+        self.llm = llm
+        self.llm.setup_default_model_name(args.code_model)
+        self.llms = [self.llm]
+
+    @prompt()
+    def _rank_modifications(self, s: CodeGenerateResult) -> str:
+        """
+        对一组代码修改进行质量评估并排序。
+
+        下面是修改需求：
+
+        <edit_requirement>
+        {{ s.conversations[0][-2]["content"] }}
+        </edit_requirement>
+
+        下面是相应的代码修改：
+        {% for content in s.contents %}
+        <edit_block id="{{ loop.index0 }}">
+        {{content}}
+        </edit_block>
+        {% endfor %}
+
+        请输出如下格式的评估结果,只包含 JSON 数据:
+
+        ```json
+        {
+            "rank_result": [id1, id2, id3]  // id 为 edit_block 的 id,按质量从高到低排序
+        }
+        ```
+
+        注意：
+        1. 只输出前面要求的 Json 格式就好，不要输出其他内容，Json 需要使用 ```json ```包裹
+        """
+
+    def rank_modifications(self, generate_result: CodeGenerateResult) -> CodeGenerateResult:
+        import time
+        from collections import defaultdict
+
+        start_time = time.time()
+        logger.info(f"开始对 {len(generate_result.contents)} 个候选结果进行排序")
+
+        try:
+            results = []
+            for llm in self.llms:
+                v = self._rank_modifications.with_llm(llm).with_return_type(RankResult).run(generate_result)
+                results.append(v.rank_result)
+
+            if not results:
+                raise Exception("All ranking requests failed")
+
+            # 计算每个候选人的分数
+            candidate_scores = defaultdict(float)
+            for rank_result in results:
+                for idx, candidate_id in enumerate(rank_result):
+                    # Score is 1/(position + 1) since position starts from 0
+                    candidate_scores[candidate_id] += 1.0 / (idx + 1)
+            # 按分数降序对候选人进行排序
+            sorted_candidates = sorted(candidate_scores.keys(),
+                                       key=lambda x: candidate_scores[x],
+                                       reverse=True)
+
+            elapsed = time.time() - start_time
+            score_details = ", ".join([f"candidate {i}: {candidate_scores[i]:.2f}" for i in sorted_candidates])
+            logger.info(
+                f"排序完成，耗时 {elapsed:.2f} 秒，最佳候选索引: {sorted_candidates[0]}，评分详情: {score_details}"
+            )
+
+            rerank_contents = [generate_result.contents[i] for i in sorted_candidates]
+            rerank_conversations = [generate_result.conversations[i] for i in sorted_candidates]
+
+            return CodeGenerateResult(contents=rerank_contents, conversations=rerank_conversations)
+
+        except Exception as e:
+            logger.error(f"排序过程失败: {str(e)}")
+            logger.debug(traceback.format_exc())
+            elapsed = time.time() - start_time
+            logger.warning(f"排序失败，耗时 {elapsed:.2f} 秒，将使用原始顺序")
+            return generate_result

autocoder_nano/edit/text.py

@@ -0,0 +1,38 @@
+from difflib import SequenceMatcher
+
+
+class TextSimilarity:
+    """
+    找到 text_b 中与 text_a 最相似的部分(滑动窗口)
+    返回相似度分数和最相似的文本片段
+    """
+
+    def __init__(self, text_a, text_b):
+        self.text_a = text_a
+        self.text_b = text_b
+        self.lines_a = self._split_into_lines(text_a)
+        self.lines_b = self._split_into_lines(text_b)
+        self.m = len(self.lines_a)
+        self.n = len(self.lines_b)
+
+    @staticmethod
+    def _split_into_lines(text):
+        return text.splitlines()
+
+    @staticmethod
+    def _levenshtein_ratio(s1, s2):
+        return SequenceMatcher(None, s1, s2).ratio()
+
+    def get_best_matching_window(self):
+        best_similarity = 0
+        best_window = []
+
+        for i in range(self.n - self.m + 1):  # 滑动窗口
+            window_b = self.lines_b[i:i + self.m]
+            similarity = self._levenshtein_ratio("\n".join(self.lines_a), "\n".join(window_b))
+
+            if similarity > best_similarity:
+                best_similarity = similarity
+                best_window = window_b
+
+        return best_similarity, "\n".join(best_window)