bioguider 0.2.3__py3-none-any.whl

bioguider/agents/dockergeneration_observe_step.py

@@ -0,0 +1,153 @@
+
+import os
+from langchain.prompts import ChatPromptTemplate
+from pydantic import BaseModel, Field
+
+from bioguider.utils.constants import DEFAULT_TOKEN_USAGE
+from bioguider.agents.agent_utils import run_command, read_file
+from bioguider.agents.dockergeneration_task_utils import DockerGenerationWorkflowState
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.agents.peo_common_step import PEOCommonStep
+
+DOCKERGENERATION_OBSERVE_SYSTEM_PROMPT = """You are an expert in software containerization and reproducibility engineering.
+We have a generated **Dockerfile**, here is its content:
+{dockerfile_content}
+
+Here is the output of docker image building with command "docker build":
+```{docker_build_output}```
+
+Here is the output of running docker image with command "docker run":
+```{docker_run_output}```
+
+### **Instructions**
+1. Carefully review **Dockerfile**, output of building docker image and output of running docker image, give your
+thoughts and advice as the following format:
+```
+**Thoughts**: you thoughts here
+``` 
+2. Be precise and support your reasoning with evidence from the input.
+
+### **Notes**
+- We are generating Dockerfile over multiple rounds, your thoughts and the output of this step will be persisted, 
+we'll continue with the next round accordingly
+"""
+
+class DockerGenerationObserveResult(BaseModel):
+    thoughts: str = Field(description="thoughts on input")
+
+MAX_TIMEOUT = 900 # 15 mins
+MAX_ERROR_OUTPTU_LENGTH = 2048 # 2k
+class DockerGenerationObserveStep(PEOCommonStep):
+    def __init__(self, llm, repo_path: str):
+        super().__init__(llm)
+        self.step_name = "Docker Generation Observe"
+        self.repo_path = repo_path
+
+    def _build_system_prompt(
+        self, 
+        state: DockerGenerationWorkflowState,
+        build_error: str,
+        run_error: str,
+    ):
+        dockerfile=state["dockerfile"]
+        dockerfile_path = os.path.join(self.repo_path, dockerfile)
+        dockerfile_content = read_file(dockerfile_path)
+        return ChatPromptTemplate.from_template(DOCKERGENERATION_OBSERVE_SYSTEM_PROMPT).format(
+            dockerfile_content=dockerfile_content,
+            docker_build_output=build_error,
+            docker_run_output=run_error,
+        )
+    
+    @staticmethod
+    def _extract_error_message(output: str):
+        if isinstance(output, bytes):
+            output = output.decode('utf-8')
+        extracted_msg = ""
+        output_lower = output.lower()
+        if "error:" in output_lower:
+            ix = output_lower.find("error:")
+            extracted_msg = output[ix:]
+        elif "error" in output_lower:
+            ix = output_lower.find("error")
+            extracted_msg = output[ix:]
+        else:
+            extracted_msg = output
+        if len(extracted_msg) > MAX_ERROR_OUTPTU_LENGTH:
+            extracted_msg = extracted_msg[((-1) * MAX_ERROR_OUTPTU_LENGTH):]
+        return extracted_msg
+
+    def _execute_directly(self, state: DockerGenerationWorkflowState):
+        token_usage = {**DEFAULT_TOKEN_USAGE}
+        if "dockerfile" in state and len(state["dockerfile"]) > 0:
+            dockerfile=state["dockerfile"]
+            dockerfile_path = os.path.join(self.repo_path, dockerfile)
+            docker_image_name: str = os.path.splitext(dockerfile)[0]
+            docker_image_name = docker_image_name.lower()
+            
+            out, error, code = run_command([
+                "docker", "build", 
+                "-t", docker_image_name, 
+                "-f", dockerfile_path,
+                self.repo_path
+            ], timeout=MAX_TIMEOUT)
+            if code != 0:
+                error_msg = DockerGenerationObserveStep._extract_error_message(error)
+                system_prompt = self._build_system_prompt(state, error_msg, "N/A")
+                agent = CommonAgentTwoSteps(llm=self.llm)
+                res, _, token_usage, reasoning = agent.go(
+                    system_prompt=system_prompt,
+                    instruction_prompt="Now, let's begin observing.",
+                    schema=DockerGenerationObserveResult,
+                )
+                state["step_dockerfile_content"] = read_file(dockerfile_path)
+                state["step_output"] = error_msg
+                state["step_thoughts"] = res.thoughts
+                self._print_step(
+                    state,
+                    step_output=f"**Observation Reasoning Process**\n{reasoning}"
+                )
+                return state, token_usage
+            out, error, code = run_command([
+                "docker", "run",
+                "--name", "bioguider_demo",
+                docker_image_name
+            ], timeout=MAX_TIMEOUT)
+            run_command([
+                "docker", "rm", "-f",
+                "bioguider_demo"
+            ], timeout=MAX_TIMEOUT)
+            run_command([
+                "docker", "rmi", docker_image_name
+            ], timeout=MAX_TIMEOUT)
+            if code != 0:
+                system_prompt = self._build_system_prompt(
+                    state, 
+                    "docker build successfully.",
+                    error,
+                )
+                agent = CommonAgentTwoSteps(llm=self.llm)
+                res, _, token_usage, reasoning = agent.go(
+                    system_prompt=system_prompt,
+                    instruction_prompt="Now, let's begin observing.",
+                    schema=DockerGenerationObserveResult,
+                )
+                state["step_dockerfile_content"] = read_file(dockerfile_path)
+                state["step_output"] = error
+                state["step_thoughts"] = res.thoughts
+                self._print_step(
+                    state,
+                    step_output=f"**Observation Reasoning Process**\n{reasoning}",
+                )
+                return state, token_usage
+            
+            state["final_answer"] = read_file(dockerfile_path)
+            return state, token_usage
+        
+        state["step_thoughts"] = "No Dockerfile is generated."
+        return state, token_usage
+
+
+        
+        
+
+

bioguider/agents/dockergeneration_plan_step.py

@@ -0,0 +1,158 @@
+
+import os
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain.tools import BaseTool
+from langchain_core.prompts import ChatPromptTemplate
+from nanoid import generate
+
+from bioguider.agents.agent_utils import (
+    convert_plan_to_string, 
+    get_tool_names_and_descriptions,
+    PlanAgentResult,
+    PlanAgentResultJsonSchema,
+)
+from bioguider.agents.peo_common_step import PEOCommonStep
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.agents.dockergeneration_task_utils import (
+    DockerGenerationWorkflowState, 
+    prepare_provided_files_string,
+)
+
+DOCKERGENERATION_PLAN_SYSTEM_PROMPT = ChatPromptTemplate.from_template("""
+You are an expert in software containerization and reproducibility engineering.
+Your task is to generate a **Dockerfile** that prepares the environment and runs a simple get-started example based on the provided files from a GitHub repository.
+---
+
+### Repository File Structure
+Below is the 2-level file structure of the repository (`f` = file, `d` = directory, `l` - symlink, `u` - unknown):  
+{repo_structure}
+
+### **Input Files:**
+
+You are given the contents of the following files extracted from the repository:
+
+{extracted_files}
+---
+
+### **Intermediate Dockerfile**
+Here is the Dockerfile you generated before.
+{intermediate_dockerfile_content}
+
+---
+
+### **Intermediate Error**
+Here is the error when building or running the Dockerfile
+{intermediate_error}
+
+## ** Intermediate Thoughts **
+Here is the thoughts you need to take into consideration.
+{intermediate_thoughts}
+---
+                                                                       
+### **Function Tools**
+You have access to the following function tools:
+{tools}
+---
+
+### Instructions:
+1. We will iterate through multiple **Plan -> Execution -> Observation** loops as needed.
+   - Plan stage(current stage) will make a plan based on provided **tools**, **intermediate output** and **repo structure**
+   - Execution stage will execute the planned actions to generate Dockerfile
+   - Observation stage will observe the Dockerfile that is generated in execution step and provide advice in **intermediate thoughts**
+2. Your current task is to make a plan to achieve the goal.
+   You can start by `write_file_tool` to prepare script files, then use `generate_Dockerfile_tool` to generate **Dockerfile**
+3. When using `write_file_tool`, you must specify both the **file name** and **file content** as input.
+   - Use `write_file_tool` to create new files, such as a minimal demo script.
+   - You may also use it to **overwrite existing files** if **needed**.
+   - If no update, **do not** use `write_file_tool` to overwrite existed file.
+   - Always provide **complete and concrete file content**—do **not** include suggestions, placeholders, abstract descriptions, or part of content.
+4. You can use `extract_python_file_from_notebook_tool` to extract python code from python notebook and save to a python file to avoid running python notebook with jupyter.
+5. You may use the `python_repl` tool to execute Python code, but this should **also be avoided in the first step**.
+6. The Dockerfile will be placed at the root of the repository.
+   Therefore, in the Dockerfile, you can assume all repository files are accessible and can be copied as needed.
+7. If you are given **Intermediate Error** and **Intermediate Dockerfile**, you need to analyze them carefully, and try to fix them with new generated Dockerfile.
+   You need to provide concrete resolution in your reasoning process.
+8. When using `generate_Dockerfile_tool` to generate a Dockerfile, please use `demo-bioguider-{docker_id}.Dockerfile` as file name.
+9. Always use `generate_Dockerfile_tool` as the **final action step** in your plan to ensure the Dockerfile is generated at the end of the process.
+---
+
+### **Output Format**  
+Your plan should be returned as a sequence of step actions in the following format:
+
+Step: <tool name>   # Tool name must be one of {tool_names}  
+Step Input: <file or directory name>
+
+Step: <tool name>  
+Step Input: <file or directory name>
+...
+""")
+
+class DockerGenerationPlanStep(PEOCommonStep):
+    def __init__(
+         self, 
+         llm: BaseChatOpenAI,
+         repo_path: str,
+         repo_structure: str,
+         gitignore_path: str,
+         custom_tools: list[BaseTool] | None = None,
+      ):
+        super().__init__(llm)
+        self.step_name = "Dockerfile Generation Plan Step"
+        self.repo_path = repo_path
+        self.repo_structure = repo_structure
+        self.gitignore_path = gitignore_path
+        self.custom_tools = custom_tools
+
+    def _prepare_intermediate_steps(self, state: DockerGenerationWorkflowState):
+        _, intermediate_thoughts = super()._build_intermediate_analysis_and_thoughts(state)
+        intermediate_dockerfile_content = state["step_dockerfile_content"] if "step_dockerfile_content" in state else "N/A"
+        intermediate_error = state["step_output"] if "step_output" in state else "N/A"
+        intermediate_error = intermediate_error.replace("{", "(").replace("}", ")")
+
+        return intermediate_dockerfile_content, intermediate_error, intermediate_thoughts
+
+    def _prepare_system_prompt(self, state: DockerGenerationWorkflowState) -> str:
+        docker_id = generate('1234567890abcdefhijklmnopqrstuvwxyz', size=10)
+        tool_names, tools_desc = get_tool_names_and_descriptions(self.custom_tools)
+        provided_files = state["provided_files"]
+        str_provided_files = prepare_provided_files_string(self.repo_path, provided_files)
+         
+        intermediate_dockerfile_content, intermediate_error, intermediate_thoughts = self._prepare_intermediate_steps(state)
+        system_prompt = DOCKERGENERATION_PLAN_SYSTEM_PROMPT.format(
+            repo_structure=self.repo_structure,
+            tools=tools_desc,
+            tool_names=tool_names,
+            extracted_files=str_provided_files,
+            intermediate_dockerfile_content=intermediate_dockerfile_content,
+            intermediate_error=intermediate_error,
+            intermediate_thoughts=intermediate_thoughts,
+            docker_id=docker_id,
+        )
+        self._print_step(
+            state,
+            step_output="**Intermediate Step Output**\n" + intermediate_error
+        )
+        self._print_step(
+            state,
+            step_output="**Intermediate Step Thoughts**\n" + intermediate_thoughts
+        )
+        return system_prompt         
+
+    def _execute_directly(self, state: DockerGenerationWorkflowState):
+        system_prompt = self._prepare_system_prompt(state)
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin to make a plan",
+            schema=PlanAgentResultJsonSchema,
+        )
+        res = PlanAgentResult(**res)
+        self._print_step(state, step_output=f"**Reasoning Process**\n{reasoning}")
+        self._print_step(state, step_output=f"**Plan**\n{str(res.actions)}")
+        state["plan_thoughts"] = reasoning
+        state["plan_actions"] = convert_plan_to_string(res)
+
+        return state, token_usage
+        
+
+

bioguider/agents/dockergeneration_task.py

@@ -0,0 +1,158 @@
+
+import os
+import re
+from pydantic import BaseModel, Field
+from typing import Callable, List, Optional, TypedDict, Union
+from langchain_core.prompts import ChatPromptTemplate, StringPromptTemplate
+from langchain_core.messages import SystemMessage, HumanMessage
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain.tools import StructuredTool, Tool, tool, BaseTool
+from langchain.agents import (
+    initialize_agent, 
+    AgentType, 
+    AgentOutputParser,
+    create_react_agent,
+    AgentExecutor,
+)
+from langchain.schema import (
+    AgentFinish,
+    AgentAction,
+)
+from langgraph.graph import StateGraph, START, END
+
+from bioguider.database.summarized_file_db import SummarizedFilesDb
+from bioguider.agents.peo_common_step import PEOCommonStep
+from bioguider.utils.file_utils import get_file_type
+from bioguider.agents.agent_utils import read_directory, read_file
+from bioguider.agents.collection_task_utils import (
+    RELATED_FILE_GOAL_ITEM,
+    CollectionWorkflowState, 
+    check_file_related_tool,
+)
+from bioguider.agents.common_agent import CommonAgent
+from bioguider.agents.dockergeneration_task_utils import (
+    generate_Dockerfile_tool,
+    prepare_provided_files_string,
+    write_file_tool,
+    extract_python_file_from_notebook_tool,
+)
+from bioguider.agents.python_ast_repl_tool import CustomPythonAstREPLTool
+from bioguider.agents.dockergeneration_plan_step import DockerGenerationPlanStep
+from bioguider.agents.dockergeneration_execute_step import DockerGenerationExecuteStep
+from bioguider.agents.dockergeneration_observe_step import DockerGenerationObserveStep
+from bioguider.agents.dockergeneration_task_utils import DockerGenerationWorkflowState
+from bioguider.agents.agent_task import AgentTask
+
+class DockerGenerationTask(AgentTask):
+    def __init__(
+        self, 
+        llm, 
+        step_callback = None
+    ):
+        super().__init__(llm, step_callback)
+        self.repo_path: str | None = None
+        self.gitignore_path: str | None = None
+        self.repo_structure: str | None = None
+        self.steps: list[PEOCommonStep] = []
+        self.tools: list[any] | None = None
+        self.provided_files: list[str] | None = None
+
+    def _initialize(self):
+        # initialize the 2-level file structure of the repo
+        if not os.path.exists(self.repo_path):
+            raise ValueError(f"Repository path {self.repo_path} does not exist.")
+        files = read_directory(self.repo_path, os.path.join(self.repo_path, ".gitignore"))
+        file_pairs = [(f, get_file_type(os.path.join(self.repo_path, f)).value) for f in files]
+        self.repo_structure = ""
+        for f, f_type in file_pairs:
+            self.repo_structure += f"{f} - {f_type}\n"
+
+        # initialize extracted files string
+        if self.provided_files is not None:
+            self.str_extracted_files = prepare_provided_files_string(
+                self.repo_path, self.provided_files
+            )
+        write_tool = write_file_tool(self.repo_path)
+        generate_tool = generate_Dockerfile_tool(
+            llm=self.llm,
+            repo_path=self.repo_path,
+            extracted_files=self.str_extracted_files,
+            repo_structure=self.repo_structure,
+            output_callback=self.step_callback,
+        )
+        extract_tool = extract_python_file_from_notebook_tool(
+            repo_path=self.repo_path,
+        )
+        self.tools = [
+            write_tool, generate_tool, extract_tool,
+        ]
+        self.custom_tools = [
+            StructuredTool.from_function(
+                write_tool.run,
+                description=write_tool.__class__.__doc__,
+                name=write_tool.__class__.__name__,
+            ),
+            Tool(
+                func=generate_tool.run,
+                description=generate_tool.__class__.__doc__,
+                name=generate_tool.__class__.__name__,
+            ),
+            StructuredTool.from_function(
+                extract_tool.run,
+                description=extract_tool.__class__.__doc__,
+                name=extract_tool.__class__.__name__,
+            )
+        ]
+        self.custom_tools.append(CustomPythonAstREPLTool())
+        plan_step = DockerGenerationPlanStep(
+            llm=self.llm,
+            repo_path=self.repo_path,
+            repo_structure=self.repo_structure,
+            gitignore_path=self.gitignore_path,
+            custom_tools=self.custom_tools,
+        )
+        execute_step = DockerGenerationExecuteStep(
+            llm=self.llm,
+            repo_path=self.repo_path,
+            repo_structure=self.repo_structure,
+            gitignore_path=self.gitignore_path,
+            custom_tools=self.custom_tools,
+        )
+        observe_step = DockerGenerationObserveStep(
+            llm=self.llm,
+            repo_path=self.repo_path,
+        )
+        self.steps = [
+            plan_step, execute_step, observe_step,
+        ]
+        # pass generate_Dockerfile_tool to execute step
+        execute_step.set_generate_Dockerfile_tool(generate_tool)
+
+    def _compile(self, repo_path, gitignore_path, **kwargs):
+        self.repo_path = repo_path
+        self.gitignore_path = gitignore_path
+        self.provided_files = kwargs.get("provided_files")
+        self._initialize()
+
+        def check_observe_step(state: DockerGenerationWorkflowState):
+            if "final_answer" in state and state["final_answer"] is not None:
+                self._print_step(step_name="Final Answer")
+                self._print_step(step_output=state["final_answer"])
+                return END
+            return "plan_step"
+        
+        graph = StateGraph(DockerGenerationWorkflowState)
+        graph.add_node("plan_step", self.steps[0].execute)
+        graph.add_node("execute_step", self.steps[1].execute)
+        graph.add_node("observe_step", self.steps[2].execute)
+        graph.add_edge(START, "plan_step")
+        graph.add_edge("plan_step", "execute_step")
+        graph.add_edge("execute_step", "observe_step")
+        graph.add_conditional_edges("observe_step", check_observe_step, {"plan_step", END})
+
+        self.graph = graph.compile()
+
+    def generate(self):
+        s = self._go_graph({"provided_files": self.provided_files})
+        return s
+

bioguider/agents/dockergeneration_task_utils.py

@@ -0,0 +1,220 @@
+
+import os
+import logging
+from typing import Callable, Optional, TypedDict
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from pydantic import BaseModel, Field
+
+from bioguider.agents.agent_tools import agent_tool
+from bioguider.agents.agent_utils import read_file, write_file
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.utils.file_utils import extract_code_from_notebook
+
+logger = logging.getLogger(__name__)
+
+class DockerGenerationPlanResult(BaseModel):
+    Dockerfile: str = Field(description="Dockerfile content")
+    
+class DockerGenerationWorkflowState(TypedDict):
+    llm: Optional[BaseChatOpenAI]
+    step_output_callback: Optional[Callable]
+    provided_files: Optional[list[str]]
+    intermediate_steps: Optional[str]
+    step_dockerfile_content: Optional[str]
+    step_output: Optional[str]
+    step_thoughts: Optional[str]
+    plan_thoughts: Optional[str]
+    plan_actions: Optional[str]
+    dockerfile: Optional[str]
+    final_answer: Optional[str]
+
+def extract_dockergeneration_related_content(filename: str):
+    pass
+
+DOCKERGENERATION_SYSTEM_PROMPT = ChatPromptTemplate.from_template("""
+You are an expert in software containerization and reproducibility engineering.
+Your task is to generate a **Dockerfile** that prepares the environment and runs a simple get-started example based on the provided files from a GitHub repository.
+---
+### Repository File Structure
+Below is the 2-level file structure of the repository (`f` = file, `d` = directory, `l` - symlink, `u` - unknown):  
+{repo_structure}
+
+---
+### **Input Files:**
+
+You are given the contents of the following files extracted from the repository:
+
+{extracted_files}
+---
+                                                                  
+### **plan thoughts**
+Here is the plan thoughts, you are in **generate_Dockerfile_tool** action:
+{plan_thoughts}
+
+---
+
+### **Intermediate Output**
+Here is the Dockerfile you generate before.
+{step_dockerfile_content}
+
+---
+
+### **Intermediate Error**
+Here is the error occurred in building or running the above generated Dockerfile:
+{step_error}
+                                                                  
+### **Requirements:**
+1. **Environment Setup**
+   * When generating the Dockerfile, prioritize using the base image provided in the repository. If no base image is specified, select an appropriate one based on the project's context.
+   * Use the relevant installation and configuration details from the input files (e.g., `requirements.txt`, `environment.yml`, `setup.py`, etc.).
+   * Choose an appropriate base image (e.g., `python:3.10`, `r-base`, etc.) based on the language and setup instructions.
+2. **Dependency Installation**
+   * Include all commands necessary to install packages, tools, or dependencies as specified in the input files.
+   * Make sure to always install common system utilities and development tools such as gcc, g++, build-essential, curl, wget, and similar essential packages.
+3. **Running a Get-Started Example**
+   * Identify a minimal executable script or command (e.g., `python example.py`, `Rscript demo.R`, `jupyter nbconvert --execute`) that demonstrates the basic functionality of the repository.
+4. **Keep the Dockerfile Minimal and Reproducible**
+   * Use best practices such as specifying exact versions where possible, minimizing layers, and using `COPY`, `WORKDIR`, and `CMD` appropriately.
+5. The Dockerfile will be placed at the root of the repository.
+   Therefore, in the Dockerfile, you can assume all repository files are accessible and can be copied as needed.
+6. If the **Intermediate Output** and **Intermediate Error** are provided, you need to analyze them carefully, and try to fix them in the generated Dockerfile.
+---
+### **Output Format:**
+Return only the Dockerfile content enclosed in triple backticks:
+```dockerfile
+# Dockerfile
+<your generated Dockerfile content here>
+```
+Do not include any explanation, comments, or extra output outside the code block.
+""")
+
+class generate_Dockerfile_tool(agent_tool):
+    """ Generate Dockerfile for provided repository
+Args:
+    output_path str: the output path to save Dockerfile
+Returns:
+    boolean: if Dockerfile is saved successfully
+    """
+    def __init__(
+        self,
+        llm: BaseChatOpenAI,
+        repo_path: str,
+        extracted_files: str,
+        repo_structure: str,
+        output_callback: Callable | None = None,
+    ):
+        super().__init__(llm, output_callback=output_callback)
+        self.repo_path = repo_path
+        self.repo_struture = repo_structure
+        self.extracted_files = extracted_files
+        self.plan_thoughts = None
+        self.step_error: str = None
+        self.step_dockerfile_content: str = None
+
+    def set_intermediate_output(self, plan_thoughts: str, step_error: str, step_dockerfile_content: str):
+        plan_thoughts = plan_thoughts.replace("{", "(").replace("}", ")")
+        step_error = step_error.replace("{", "(").replace("}", ")")
+        self.plan_thoughts = plan_thoughts
+        self.step_error = step_error
+        self.step_dockerfile_content = step_dockerfile_content
+
+    def run(self, output_path: str):
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        system_prompt = DOCKERGENERATION_SYSTEM_PROMPT.format(
+            repo_structure = self.repo_struture,
+            extracted_files = self.extracted_files,
+            plan_thoughts=self.plan_thoughts,
+            step_error=self.step_error,
+            step_dockerfile_content=self.step_dockerfile_content
+        )
+        res, _, token_usage, reasoning = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's start to generate Dockerfile.",
+            schema=DockerGenerationPlanResult,
+        )
+        res: DockerGenerationPlanResult = res
+        self._print_step_output(step_output=reasoning) 
+        self._print_token_usage(token_usage)
+        if self.repo_path not in output_path:
+            output_path = os.path.join(self.repo_path, output_path)
+        content = res.Dockerfile
+        if content.startswith("```dockerfile"):
+            content = content[13:]
+        content = content.strip().strip("```")
+        write_file(output_path, content)
+
+        return True
+
+class write_file_tool():
+    """write file tool
+Args:
+    file_name str: a string specifies file path that will be written to.
+    file_content str: a string speifies file content.
+Returns:
+    bool, True if it is succeeded to write to file, otherwise False
+    """
+    def __init__(self, repo_path: str):
+        self.repo_path = repo_path
+
+    def run(self, file_name: str, file_content: str):
+        if file_name is None or file_content is None:
+            return False
+        file_name = file_name
+        content = file_content
+        file_name = file_name.strip()
+        if self.repo_path is not None and self.repo_path not in file_name:
+            file_name = os.path.join(self.repo_path, file_name)
+        try:
+            with open(file_name, "w") as fobj:
+                fobj.write(content)
+                return True
+        except Exception as e:
+            logger.error(e)
+            return False
+        
+class extract_python_file_from_notebook_tool:
+    """extract code in a notebook to a python file
+Args:
+    notebook_path str: a string speicifies notebook path to extract.
+    output_path str: a string specifies output python file path.
+Returns:
+    bool True if it is succeeded to extract to python file, otherwise False
+    """
+    def __init__(self, repo_path: str):
+        self.repo_path = repo_path
+
+    def run(self, notebook_path: str, output_path: str):
+        # notebook_path = notebook_path_and_output_path[0]
+        # output_path = notebook_path_and_output_path[1]
+        if notebook_path is None or output_path is None:
+            return False
+        if self.repo_path not in notebook_path:
+            notebook_path = os.path.join(self.repo_path, notebook_path)
+        if self.repo_path not in output_path:
+            output_path = os.path.join(self.repo_path, output_path)
+        content = extract_code_from_notebook(notebook_path)
+        try:
+            with open(output_path, "w") as fobj:
+                fobj.write(content)
+                return True
+        except FileNotFoundError as e:
+            logger.error(str(e))
+            return f"False, {output_path} does not exist."
+        
+
+def prepare_provided_files_string(repo_path: str, provided_files: list[str]):
+    if provided_files is None or len(provided_files) == 0:
+        return "N/A"
+    str_provided_files = ""
+    for fn in provided_files:
+        file_path = os.path.join(repo_path, fn)
+        if fn.endswith(".ipynb"): # python notebook
+            content = extract_code_from_notebook(file_path)
+        else:
+            content = read_file(file_path)
+        content = content.replace("{", "{{").replace("}", "}}")
+        str_provided_files += f"""**{fn}**:\n{content}\n"""
+
+    return str_provided_files
+