auto-coder 0.1.74__tar.gz → 0.1.76__tar.gz

{auto-coder-0.1.74 → auto-coder-0.1.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.74
+Version: 0.1.76
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto-coder-0.1.74 → auto-coder-0.1.76}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.74
+Version: 0.1.76
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto-coder-0.1.74 → auto-coder-0.1.76}/src/auto_coder.egg-info/SOURCES.txt

@@ -22,9 +22,11 @@ src/autocoder/common/cleaner.py
 src/autocoder/common/code_auto_execute.py
 src/autocoder/common/code_auto_generate.py
 src/autocoder/common/code_auto_generate_diff.py
+src/autocoder/common/code_auto_generate_editblock.py
 src/autocoder/common/code_auto_generate_strict_diff.py
 src/autocoder/common/code_auto_merge.py
 src/autocoder/common/code_auto_merge_diff.py
+src/autocoder/common/code_auto_merge_editblock.py
 src/autocoder/common/code_auto_merge_strict_diff.py
 src/autocoder/common/command_templates.py
 src/autocoder/common/const.py

{auto-coder-0.1.74 → auto-coder-0.1.76}/src/autocoder/auto_coder.py

@@ -20,6 +20,8 @@ from byzerllm.apps.byzer_storage.env import get_latest_byzer_retrieval_lib
 from autocoder.command_args import parse_args
 from autocoder.rag.api_server import serve,ServerArgs
 from loguru import logger
+import shutil
+import subprocess
 
 def resolve_include_path(base_path, include_path):
    if include_path.startswith('.') or include_path.startswith('..'):        
@@ -117,6 +119,45 @@ def main():
         gen_screenshots(args.urls, args.output)
         print(f"Successfully captured screenshot of {args.urls} and saved to {args.output}")
         return
+    
+    if raw_args.command == "next":
+        actions_dir = os.path.join(os.getcwd(), "actions")
+        if not os.path.exists(actions_dir):
+            print("Current directory does not have an actions directory")
+            return
+        
+        action_files = [f for f in os.listdir(actions_dir) if f[:3].isdigit() and f.endswith(".yml") and f[:3] != "101"]
+        if not action_files:
+            max_seq = 0
+        else:
+            seqs = [int(f[:3]) for f in action_files]
+            max_seq = max(seqs)
+        
+        new_seq = str(max_seq + 1).zfill(3)
+        prev_files = [f for f in action_files if int(f[:3]) < int(new_seq)]
+        
+        if not prev_files:
+            new_file = os.path.join(actions_dir, f"{new_seq}_{raw_args.name}.yml")
+            with open(new_file, "w") as f:
+                pass
+        else:
+            prev_file = sorted(prev_files)[-1]  # 取序号最大的文件
+            with open(os.path.join(actions_dir, prev_file), "r") as f:
+                content = f.read()
+            new_file = os.path.join(actions_dir, f"{new_seq}_{raw_args.name}.yml") 
+            with open(new_file, "w") as f:
+                f.write(content)
+        
+        print(f"Successfully created new action file: {new_file}")        
+        
+        if os.environ.get("TERMINAL_EMULATOR") == "JetBrains-JediTerm":            
+            subprocess.run(["idea", new_file])
+        else:            
+            if shutil.which("code"):
+                subprocess.run(["code", "-r", new_file])
+            elif shutil.which("idea"):
+                subprocess.run(["idea", new_file])                
+        return        
         
     if args.model:
         

{auto-coder-0.1.74 → auto-coder-0.1.76}/src/autocoder/command_args.py

@@ -160,6 +160,10 @@ def parse_args() -> AutoCoderArgs:
     screenshot_parser.add_argument("--output", required=True, help=desc["screenshot_output"])
     screenshot_parser.add_argument("--source_dir", default=".", help="")
 
+    next_parser = subparsers.add_parser("next", help="Create a new action file based on the previous one")
+    next_parser.add_argument("name", help="Name for the new action file")
+    next_parser.add_argument("--source_dir", default=".", help="")
+
     doc2html_parser = subparsers.add_parser("doc2html", help="Convert word/pdf document to html")
     doc2html_parser.add_argument("--file", default="", help=desc["file"])
     doc2html_parser.add_argument("--model", default="", help=desc["model"])

auto-coder-0.1.76/src/autocoder/common/code_auto_generate_editblock.py

@@ -0,0 +1,413 @@
+from typing import List, Dict, Tuple
+from autocoder.common.types import Mode
+from autocoder.common import AutoCoderArgs
+import byzerllm
+
+
+class CodeAutoGenerateEditBlock:
+    def __init__(
+        self,
+        llm: byzerllm.ByzerLLM,
+        args: AutoCoderArgs,
+        action=None,
+        fence_0: str = "```",
+        fence_1: str = "```",
+    ) -> None:
+        self.llm = llm
+        self.args = args
+        self.action = action
+        self.fence_0 = fence_0
+        self.fence_1 = fence_1
+        if not self.llm:
+            raise ValueError(
+                "Please provide a valid model instance to use for code generation."
+            )
+        if self.llm.get_sub_client("code_model"):
+            self.llm = self.llm.get_sub_client("code_model")
+
+    @byzerllm.prompt(llm=lambda self: self.llm)
+    def auto_implement_function(self, instruction: str, content: str) -> str:
+        """
+        下面是一些文件路径以及每个文件对应的源码：
+
+        {{ content }}
+
+        请参考上面的内容，重新实现所有文件下方法体标记了如下内容的方法：
+
+        ```python
+        raise NotImplementedError("This function should be implemented by the model.")
+        ```
+
+        {{ instruction }}
+
+        """
+
+    @byzerllm.prompt(llm=lambda self: self.llm)
+    def multi_round_instruction(self, instruction: str, content: str) -> str:
+        """
+        如果你需要生成代码，对于每个需要更改的文件,你需要按 *SEARCH/REPLACE block* 的格式进行生成。
+
+        # *SEARCH/REPLACE block* Rules:
+
+        Every *SEARCH/REPLACE block* must use this format:
+        1. The opening fence and code language, eg: {{ fence_0 }}python
+        2. The file path alone on a line, starting with "##File:" and verbatim. No bold asterisks, no quotes around it, no escaping of characters, etc.
+        3. The start of search block: <<<<<<< SEARCH
+        4. A contiguous chunk of lines to search for in the existing source code
+        5. The dividing line: =======
+        6. The lines to replace into the source code
+        7. The end of the replace block: >>>>>>> REPLACE
+        8. The closing fence: {{ fence_1 }}
+
+        Every *SEARCH* section must *EXACTLY MATCH* the existing source code, character for character, including all comments, docstrings, etc.
+
+        *SEARCH/REPLACE* blocks will replace *all* matching occurrences.
+        Include enough lines to make the SEARCH blocks unique.
+
+        Include *ALL* the code being searched and replaced!
+
+        To move code within a file, use 2 *SEARCH/REPLACE* blocks: 1 to delete it from its current location, 1 to insert it in the new location.
+
+        If you want to put code in a new file, use a *SEARCH/REPLACE block* with:
+        - A new file path, including dir name if needed
+        - An empty `SEARCH` section
+        - The new file's contents in the `REPLACE` section
+
+        ONLY EVER RETURN CODE IN A *SEARCH/REPLACE BLOCK*!
+
+        下面我们来看一个例子：
+
+        当前项目目录结构：
+        1. 项目根目录： /tmp/projects/mathweb
+        2. 项目子目录/文件列表(类似tree 命令输出)
+        flask/
+            app.py
+            templates/
+                index.html
+            static/
+                style.css
+
+        用户需求： Change get_factorial() to use math.factorial
+
+        回答： To make this change we need to modify `/tmp/projects/mathweb/flask/app.py` to:
+
+        1. Import the math package.
+        2. Remove the existing factorial() function.
+        3. Update get_factorial() to call math.factorial instead.
+
+        Here are the *SEARCH/REPLACE* blocks:
+
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/flask/app.py
+        <<<<<<< SEARCH
+        from flask import Flask
+        =======
+        import math
+        from flask import Flask
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/flask/app.py
+        <<<<<<< SEARCH
+        def factorial(n):
+            "compute factorial"
+
+            if n == 0:
+                return 1
+            else:
+                return n * factorial(n-1)
+
+        =======
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/flask/app.py
+        <<<<<<< SEARCH
+            return str(factorial(n))
+        =======
+            return str(math.factorial(n))
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+
+        用户需求： Refactor hello() into its own file.
+
+        回答：To make this change we need to modify `main.py` and make a new file `hello.py`:
+
+        1. Make a new hello.py file with hello() in it.
+        2. Remove hello() from main.py and replace it with an import.
+
+        Here are the *SEARCH/REPLACE* blocks:
+
+        
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/hello.py
+        <<<<<<< SEARCH
+        =======
+        def hello():
+            "print a greeting"
+
+            print("hello")
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+        
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/main.py
+        <<<<<<< SEARCH
+        def hello():
+            "print a greeting"
+
+            print("hello")
+        =======
+        from hello import hello
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+
+        现在让我们开始一个新的任务:
+
+        {%- if structure %}
+        {{ structure }}
+        {%- endif %}
+
+        {%- if content %}
+        下面是一些文件路径以及每个文件对应的源码：
+
+        {{ content }}
+        {%- endif %}
+
+        下面是用户的需求：
+
+        {{ instruction }}
+
+        每次生成一个文件的*SEARCH/REPLACE* blocks，然后询问我是否继续，当我回复继续，继续生成下一个文件的*SEARCH/REPLACE* blocks。当没有后续任务时，请回复 "__完成__" 或者 "__EOF__"。
+        """
+        return {
+            "structure": (
+                self.action.pp.get_tree_like_directory_structure()
+                if self.action
+                else ""
+            ),
+            "fence_0": self.fence_0,
+            "fence_1": self.fence_1,
+        }
+
+    @byzerllm.prompt(llm=lambda self: self.llm)
+    def single_round_instruction(self, instruction: str, content: str) -> str:
+        """
+        如果你需要生成代码，对于每个需要更改的文件,你需要按 *SEARCH/REPLACE block* 的格式进行生成。
+
+        # *SEARCH/REPLACE block* Rules:
+
+        Every *SEARCH/REPLACE block* must use this format:
+        1. The opening fence and code language, eg: {{ fence_0 }}python
+        2. The file path alone on a line, starting with "##File:" and verbatim. No bold asterisks, no quotes around it, no escaping of characters, etc.
+        3. The start of search block: <<<<<<< SEARCH
+        4. A contiguous chunk of lines to search for in the existing source code
+        5. The dividing line: =======
+        6. The lines to replace into the source code
+        7. The end of the replace block: >>>>>>> REPLACE
+        8. The closing fence: {{ fence_1 }}
+
+        Every *SEARCH* section must *EXACTLY MATCH* the existing source code, character for character, including all comments, docstrings, etc.
+
+        *SEARCH/REPLACE* blocks will replace *all* matching occurrences.
+        Include enough lines to make the SEARCH blocks unique.
+
+        Include *ALL* the code being searched and replaced!
+
+        To move code within a file, use 2 *SEARCH/REPLACE* blocks: 1 to delete it from its current location, 1 to insert it in the new location.
+
+        If you want to put code in a new file, use a *SEARCH/REPLACE block* with:
+        - A new file path, including dir name if needed
+        - An empty `SEARCH` section
+        - The new file's contents in the `REPLACE` section
+
+        ONLY EVER RETURN CODE IN A *SEARCH/REPLACE BLOCK*!
+
+        下面我们来看一个例子：
+
+        当前项目目录结构：
+        1. 项目根目录： /tmp/projects/mathweb
+        2. 项目子目录/文件列表(类似tree 命令输出)
+        flask/
+            app.py
+            templates/
+                index.html
+            static/
+                style.css
+
+        用户需求： Change get_factorial() to use math.factorial
+
+        回答： To make this change we need to modify `/tmp/projects/mathweb/flask/app.py` to:
+
+        1. Import the math package.
+        2. Remove the existing factorial() function.
+        3. Update get_factorial() to call math.factorial instead.
+
+        Here are the *SEARCH/REPLACE* blocks:
+
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/flask/app.py
+        <<<<<<< SEARCH
+        from flask import Flask
+        =======
+        import math
+        from flask import Flask
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/flask/app.py
+        <<<<<<< SEARCH
+        def factorial(n):
+            "compute factorial"
+
+            if n == 0:
+                return 1
+            else:
+                return n * factorial(n-1)
+
+        =======
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/flask/app.py
+        <<<<<<< SEARCH
+            return str(factorial(n))
+        =======
+            return str(math.factorial(n))
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+
+        用户需求： Refactor hello() into its own file.
+
+        回答：To make this change we need to modify `main.py` and make a new file `hello.py`:
+
+        1. Make a new hello.py file with hello() in it.
+        2. Remove hello() from main.py and replace it with an import.
+
+        Here are the *SEARCH/REPLACE* blocks:
+        
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/hello.py
+        <<<<<<< SEARCH
+        =======
+        def hello():
+            "print a greeting"
+
+            print("hello")
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+        
+        {{ fence_0 }}python
+        ##File: /tmp/projects/mathweb/main.py
+        <<<<<<< SEARCH
+        def hello():
+            "print a greeting"
+
+            print("hello")
+        =======
+        from hello import hello
+        >>>>>>> REPLACE
+        {{ fence_1 }}
+
+        现在让我们开始一个新的任务:
+
+        {%- if structure %}
+        {{ structure }}
+        {%- endif %}
+
+        {%- if content %}
+        下面是一些文件路径以及每个文件对应的源码：
+
+        {{ content }}
+        {%- endif %}
+
+        下面是用户的需求：
+
+        {{ instruction }}
+
+        """
+        return {
+            "structure": (
+                self.action.pp.get_tree_like_directory_structure()
+                if self.action
+                else ""
+            ),
+            "fence_0": self.fence_0,
+            "fence_1": self.fence_1,
+        }
+
+    def single_round_run(
+        self, query: str, source_content: str
+    ) -> Tuple[str, Dict[str, str]]:
+        llm_config = {"human_as_model": self.args.human_as_model}
+
+        if self.args.template == "common":
+            init_prompt = self.single_round_instruction.prompt(
+                instruction=query, content=source_content
+            )
+        elif self.args.template == "auto_implement":
+            init_prompt = self.auto_implement_function.prompt(
+                instruction=query, content=source_content
+            )
+
+        with open(self.args.target_file, "w") as file:
+            file.write(init_prompt)
+
+        conversations = [{"role": "user", "content": init_prompt}]
+
+        t = self.llm.chat_oai(conversations=conversations, llm_config=llm_config)
+        conversations.append({"role": "assistant", "content": t[0].output})
+        return [t[0].output], conversations
+
+    def multi_round_run(
+        self, query: str, source_content: str, max_steps: int = 10
+    ) -> Tuple[List[str], List[Dict[str, str]]]:
+        llm_config = {"human_as_model": self.args.human_as_model}
+        result = []
+
+        if self.args.template == "common":
+            init_prompt = self.multi_round_instruction.prompt(
+                instruction=query, content=source_content
+            )
+        elif self.args.template == "auto_implement":
+            init_prompt = self.auto_implement_function.prompt(
+                instruction=query, content=source_content
+            )
+
+        conversations = [{"role": "user", "content": init_prompt}]
+
+        with open(self.args.target_file, "w") as file:
+            file.write(init_prompt)
+
+        t = self.llm.chat_oai(conversations=conversations, llm_config=llm_config)
+
+        result.append(t[0].output)
+
+        conversations.append({"role": "assistant", "content": t[0].output})
+
+        if "__完成__" in t[0].output:
+            return result, conversations
+
+        current_step = 0
+
+        while current_step < max_steps:
+
+            conversations.append({"role": "user", "content": "继续"})
+
+            with open(self.args.target_file, "w") as file:
+                file.write("继续")
+
+            t = self.llm.chat_oai(conversations=conversations, llm_config=llm_config)
+
+            result.append(t[0].output)
+            conversations.append({"role": "assistant", "content": t[0].output})
+            current_step += 1
+
+            if "__完成__" in t[0].output or "__EOF__" in t[0].output:
+                return result, conversations
+
+        return result, conversations

auto-coder-0.1.76/src/autocoder/common/code_auto_merge_editblock.py

@@ -0,0 +1,171 @@
+import os
+from byzerllm.utils.client import code_utils
+from autocoder.common import AutoCoderArgs, git_utils
+from typing import List
+import pydantic
+import byzerllm
+from loguru import logger
+import hashlib
+
+
+class PathAndCode(pydantic.BaseModel):
+    path: str
+    content: str
+
+
+class CodeAutoMergeEditBlock:
+    def __init__(
+        self,
+        llm: byzerllm.ByzerLLM,
+        args: AutoCoderArgs,
+        fence_0: str = "```",
+        fence_1: str = "```",
+    ):
+        self.llm = llm
+        self.args = args
+        self.fence_0 = fence_0
+        self.fence_1 = fence_1
+
+    def parse_whole_text(self, text: str) -> List[PathAndCode]:
+        lines = text.split("\n")
+        lines_len = len(lines)
+        start_marker_count = 0
+        inline_start_marker_count = 0
+        block = []
+        path_and_code_list = []
+
+        def guard(index):
+            return index + 1 < lines_len
+
+        def start_marker(line, index):
+            return (
+                line.startswith(self.fence_0)
+                and guard(index)
+                and lines[index + 1].startswith("##File:")
+            )
+
+        def inline_start_marker(line, index):
+            return line.startswith(self.fence_0) and line.strip() != self.fence_0
+
+        def end_marker(line, index):
+            return line.startswith(self.fence_1) and line.strip() == self.fence_1
+
+        for index, line in enumerate(lines):
+            if start_marker(line, index) and start_marker_count == 0:
+                start_marker_count += 1
+            elif (
+                start_marker(line, index) or inline_start_marker(line, index)
+            ) and start_marker_count > 0:
+                inline_start_marker_count += 1
+                block.append(line)
+            elif (
+                end_marker(line, index)
+                and start_marker_count == 1
+                and inline_start_marker_count == 0
+            ):
+                start_marker_count -= 1
+                if block:
+                    path = block[0].split(":", 1)[1].strip()
+                    content = "\n".join(block[1:])
+                    block = []
+                    path_and_code_list.append(PathAndCode(path=path, content=content))
+            elif end_marker(line, index) and inline_start_marker_count > 0:
+                inline_start_marker_count -= 1
+                block.append(line)
+            elif start_marker_count > 0:
+                block.append(line)
+
+        return path_and_code_list   
+
+    @byzerllm.prompt()
+    def git_require_msg(self, source_dir: str, error: str) -> str:
+        """
+        auto_merge only works for git repositories.
+
+        Try to use git init in the source directory.
+
+        ```shell
+        cd {{ source_dir }}
+        git init .
+        ```
+
+        Then try to run auto-coder again.
+        Error: {{ error }}
+        """
+
+    def get_edits(self,content: str):                
+        edits = self.parse_whole_text(content)
+        HEAD = "<<<<<<< SEARCH"
+        DIVIDER = "======="
+        UPDATED = ">>>>>>> REPLACE"
+        result = []
+        for edit in edits:
+            heads = []
+            updates = []
+            c = edit.content
+            in_head = False
+            in_updated = False
+            for line in c.splitlines():
+                if line.strip()==HEAD:
+                    in_head = True
+                    continue
+                if line.strip()==DIVIDER:
+                    in_head= False
+                    in_updated = True
+                    continue
+                if line.strip()==UPDATED:
+                    in_head = False
+                    in_updated = False
+                    continue
+                if in_head:
+                    heads.append(line)
+                if in_updated:
+                    updates.append(line)
+            result.append((edit.path,"\n".join(heads),"\n".join(updates)))
+        return result        
+
+    def merge_code(self, content: str, force_skip_git: bool = False):        
+        file_content = open(self.args.file).read()
+        md5 = hashlib.md5(file_content.encode("utf-8")).hexdigest()
+        # get the file name
+        file_name = os.path.basename(self.args.file)
+
+        if not force_skip_git:
+            try:
+                git_utils.commit_changes(
+                    self.args.source_dir, f"auto_coder_pre_{file_name}_{md5}"
+                )
+            except Exception as e:
+                logger.error(
+                    self.git_require_msg(source_dir=self.args.source_dir, error=str(e))
+                )
+                return
+      
+        codes = self.get_edits(content)
+        updated_files  = []
+        for block in codes:
+            file_path = block[0]
+            updated_files.append(file_path)
+            head = block[1]
+            update = block[2]
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            if not os.path.exists(file_path):                
+                with open(file_path, "w") as f:
+                    logger.info(f"Upsert path: {file_path}")                    
+                    f.write(update)
+                continue
+            with open(file_path, "r") as f:
+                existing_content = f.read()
+            logger.info(f'''in:\n {file_path}
+replace:\n{head}
+with:\n{update}''')    
+            existing_content = existing_content.replace(head,update,1)
+            logger.info(f"Upsert Result: {existing_content}")            
+            with open(file_path, "w") as f:
+                f.write(existing_content)                    
+         
+        logger.info(f"Merged {len(set(updated_files))} files into the project.")
+        if not force_skip_git:
+            git_utils.commit_changes(
+                self.args.source_dir, f"auto_coder_{file_name}_{md5}"
+            )