bioguider 0.2.52__py3-none-any.whl

bioguider/agents/dockergeneration_task.py

@@ -0,0 +1,158 @@
+
+import os
+import re
+from pydantic import BaseModel, Field
+from typing import Callable, List, Optional, TypedDict, Union
+from langchain_core.prompts import ChatPromptTemplate, StringPromptTemplate
+from langchain_core.messages import SystemMessage, HumanMessage
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain.tools import StructuredTool, Tool, tool, BaseTool
+from langchain.agents import (
+    initialize_agent, 
+    AgentType, 
+    AgentOutputParser,
+    create_react_agent,
+    AgentExecutor,
+)
+from langchain.schema import (
+    AgentFinish,
+    AgentAction,
+)
+from langgraph.graph import StateGraph, START, END
+
+from bioguider.database.summarized_file_db import SummarizedFilesDb
+from bioguider.agents.peo_common_step import PEOCommonStep
+from bioguider.utils.file_utils import get_file_type
+from bioguider.agents.agent_utils import read_directory, read_file
+from bioguider.agents.collection_task_utils import (
+    RELATED_FILE_GOAL_ITEM,
+    CollectionWorkflowState, 
+    check_file_related_tool,
+)
+from bioguider.agents.common_agent import CommonAgent
+from bioguider.agents.dockergeneration_task_utils import (
+    generate_Dockerfile_tool,
+    prepare_provided_files_string,
+    write_file_tool,
+    extract_python_file_from_notebook_tool,
+)
+from bioguider.agents.python_ast_repl_tool import CustomPythonAstREPLTool
+from bioguider.agents.dockergeneration_plan_step import DockerGenerationPlanStep
+from bioguider.agents.dockergeneration_execute_step import DockerGenerationExecuteStep
+from bioguider.agents.dockergeneration_observe_step import DockerGenerationObserveStep
+from bioguider.agents.dockergeneration_task_utils import DockerGenerationWorkflowState
+from bioguider.agents.agent_task import AgentTask
+
+class DockerGenerationTask(AgentTask):
+    def __init__(
+        self, 
+        llm, 
+        step_callback = None,
+    ):
+        super().__init__(llm, step_callback)
+        self.repo_path: str | None = None
+        self.gitignore_path: str | None = None
+        self.repo_structure: str | None = None
+        self.steps: list[PEOCommonStep] = []
+        self.tools: list[any] | None = None
+        self.provided_files: list[str] | None = None
+
+    def _initialize(self):
+        # initialize the 2-level file structure of the repo
+        if not os.path.exists(self.repo_path):
+            raise ValueError(f"Repository path {self.repo_path} does not exist.")
+        files = read_directory(self.repo_path, os.path.join(self.repo_path, ".gitignore"))
+        file_pairs = [(f, get_file_type(os.path.join(self.repo_path, f)).value) for f in files]
+        self.repo_structure = ""
+        for f, f_type in file_pairs:
+            self.repo_structure += f"{f} - {f_type}\n"
+
+        # initialize extracted files string
+        if self.provided_files is not None:
+            self.str_extracted_files = prepare_provided_files_string(
+                self.repo_path, self.provided_files
+            )
+        write_tool = write_file_tool(self.repo_path)
+        generate_tool = generate_Dockerfile_tool(
+            llm=self.llm,
+            repo_path=self.repo_path,
+            extracted_files=self.str_extracted_files,
+            repo_structure=self.repo_structure,
+            output_callback=self.step_callback,
+        )
+        extract_tool = extract_python_file_from_notebook_tool(
+            repo_path=self.repo_path,
+        )
+        self.tools = [
+            write_tool, generate_tool, extract_tool,
+        ]
+        self.custom_tools = [
+            StructuredTool.from_function(
+                write_tool.run,
+                description=write_tool.__class__.__doc__,
+                name=write_tool.__class__.__name__,
+            ),
+            Tool(
+                func=generate_tool.run,
+                description=generate_tool.__class__.__doc__,
+                name=generate_tool.__class__.__name__,
+            ),
+            StructuredTool.from_function(
+                extract_tool.run,
+                description=extract_tool.__class__.__doc__,
+                name=extract_tool.__class__.__name__,
+            )
+        ]
+        self.custom_tools.append(CustomPythonAstREPLTool())
+        plan_step = DockerGenerationPlanStep(
+            llm=self.llm,
+            repo_path=self.repo_path,
+            repo_structure=self.repo_structure,
+            gitignore_path=self.gitignore_path,
+            custom_tools=self.custom_tools,
+        )
+        execute_step = DockerGenerationExecuteStep(
+            llm=self.llm,
+            repo_path=self.repo_path,
+            repo_structure=self.repo_structure,
+            gitignore_path=self.gitignore_path,
+            custom_tools=self.custom_tools,
+        )
+        observe_step = DockerGenerationObserveStep(
+            llm=self.llm,
+            repo_path=self.repo_path,
+        )
+        self.steps = [
+            plan_step, execute_step, observe_step,
+        ]
+        # pass generate_Dockerfile_tool to execute step
+        execute_step.set_generate_Dockerfile_tool(generate_tool)
+
+    def _compile(self, repo_path, gitignore_path, **kwargs):
+        self.repo_path = repo_path
+        self.gitignore_path = gitignore_path
+        self.provided_files = kwargs.get("provided_files")
+        self._initialize()
+
+        def check_observe_step(state: DockerGenerationWorkflowState):
+            if "final_answer" in state and state["final_answer"] is not None:
+                self._print_step(step_name="Final Answer")
+                self._print_step(step_output=state["final_answer"])
+                return END
+            return "plan_step"
+        
+        graph = StateGraph(DockerGenerationWorkflowState)
+        graph.add_node("plan_step", self.steps[0].execute)
+        graph.add_node("execute_step", self.steps[1].execute)
+        graph.add_node("observe_step", self.steps[2].execute)
+        graph.add_edge(START, "plan_step")
+        graph.add_edge("plan_step", "execute_step")
+        graph.add_edge("execute_step", "observe_step")
+        graph.add_conditional_edges("observe_step", check_observe_step, {"plan_step", END})
+
+        self.graph = graph.compile()
+
+    def generate(self):
+        s = self._go_graph({"provided_files": self.provided_files})
+        return s
+

bioguider/agents/dockergeneration_task_utils.py

@@ -0,0 +1,220 @@
+
+import os
+import logging
+from typing import Callable, Optional, TypedDict
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from pydantic import BaseModel, Field
+
+from bioguider.agents.agent_tools import agent_tool
+from bioguider.agents.agent_utils import read_file, write_file
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.utils.file_utils import extract_code_from_notebook
+
+logger = logging.getLogger(__name__)
+
+class DockerGenerationPlanResult(BaseModel):
+    Dockerfile: str = Field(description="Dockerfile content")
+    
+class DockerGenerationWorkflowState(TypedDict):
+    llm: Optional[BaseChatOpenAI]
+    step_output_callback: Optional[Callable]
+    provided_files: Optional[list[str]]
+    intermediate_steps: Optional[str]
+    step_dockerfile_content: Optional[str]
+    step_output: Optional[str]
+    step_thoughts: Optional[str]
+    plan_thoughts: Optional[str]
+    plan_actions: Optional[str]
+    dockerfile: Optional[str]
+    final_answer: Optional[str]
+
+def extract_dockergeneration_related_content(filename: str):
+    pass
+
+DOCKERGENERATION_SYSTEM_PROMPT = ChatPromptTemplate.from_template("""
+You are an expert in software containerization and reproducibility engineering.
+Your task is to generate a **Dockerfile** that prepares the environment and runs a simple get-started example based on the provided files from a GitHub repository.
+---
+### Repository File Structure
+Below is the 2-level file structure of the repository (`f` = file, `d` = directory, `l` - symlink, `u` - unknown):  
+{repo_structure}
+
+---
+### **Input Files:**
+
+You are given the contents of the following files extracted from the repository:
+
+{extracted_files}
+---
+                                                                  
+### **plan thoughts**
+Here is the plan thoughts, you are in **generate_Dockerfile_tool** action:
+{plan_thoughts}
+
+---
+
+### **Intermediate Output**
+Here is the Dockerfile you generate before.
+{step_dockerfile_content}
+
+---
+
+### **Intermediate Error**
+Here is the error occurred in building or running the above generated Dockerfile:
+{step_error}
+                                                                  
+### **Requirements:**
+1. **Environment Setup**
+   * When generating the Dockerfile, prioritize using the base image provided in the repository. If no base image is specified, select an appropriate one based on the project's context.
+   * Use the relevant installation and configuration details from the input files (e.g., `requirements.txt`, `environment.yml`, `setup.py`, etc.).
+   * Choose an appropriate base image (e.g., `python:3.10`, `r-base`, etc.) based on the language and setup instructions.
+2. **Dependency Installation**
+   * Include all commands necessary to install packages, tools, or dependencies as specified in the input files.
+   * Make sure to always install common system utilities and development tools such as gcc, g++, build-essential, curl, wget, and similar essential packages.
+3. **Running a Get-Started Example**
+   * Identify a minimal executable script or command (e.g., `python example.py`, `Rscript demo.R`, `jupyter nbconvert --execute`) that demonstrates the basic functionality of the repository.
+4. **Keep the Dockerfile Minimal and Reproducible**
+   * Use best practices such as specifying exact versions where possible, minimizing layers, and using `COPY`, `WORKDIR`, and `CMD` appropriately.
+5. The Dockerfile will be placed at the root of the repository.
+   Therefore, in the Dockerfile, you can assume all repository files are accessible and can be copied as needed.
+6. If the **Intermediate Output** and **Intermediate Error** are provided, you need to analyze them carefully, and try to fix them in the generated Dockerfile.
+---
+### **Output Format:**
+Return only the Dockerfile content enclosed in triple backticks:
+```dockerfile
+# Dockerfile
+<your generated Dockerfile content here>
+```
+Do not include any explanation, comments, or extra output outside the code block.
+""")
+
+class generate_Dockerfile_tool(agent_tool):
+    """ Generate Dockerfile for provided repository
+Args:
+    output_path str: the output path to save Dockerfile
+Returns:
+    boolean: if Dockerfile is saved successfully
+    """
+    def __init__(
+        self,
+        llm: BaseChatOpenAI,
+        repo_path: str,
+        extracted_files: str,
+        repo_structure: str,
+        output_callback: Callable | None = None,
+    ):
+        super().__init__(llm, output_callback=output_callback)
+        self.repo_path = repo_path
+        self.repo_struture = repo_structure
+        self.extracted_files = extracted_files
+        self.plan_thoughts = None
+        self.step_error: str = None
+        self.step_dockerfile_content: str = None
+
+    def set_intermediate_output(self, plan_thoughts: str, step_error: str, step_dockerfile_content: str):
+        plan_thoughts = plan_thoughts.replace("{", "(").replace("}", ")")
+        step_error = step_error.replace("{", "(").replace("}", ")")
+        self.plan_thoughts = plan_thoughts
+        self.step_error = step_error
+        self.step_dockerfile_content = step_dockerfile_content
+
+    def run(self, output_path: str):
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        system_prompt = DOCKERGENERATION_SYSTEM_PROMPT.format(
+            repo_structure = self.repo_struture,
+            extracted_files = self.extracted_files,
+            plan_thoughts=self.plan_thoughts,
+            step_error=self.step_error,
+            step_dockerfile_content=self.step_dockerfile_content
+        )
+        res, _, token_usage, reasoning = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's start to generate Dockerfile.",
+            schema=DockerGenerationPlanResult,
+        )
+        res: DockerGenerationPlanResult = res
+        self._print_step_output(step_output=reasoning) 
+        self._print_token_usage(token_usage)
+        if self.repo_path not in output_path:
+            output_path = os.path.join(self.repo_path, output_path)
+        content = res.Dockerfile
+        if content.startswith("```dockerfile"):
+            content = content[13:]
+        content = content.strip().strip("```")
+        write_file(output_path, content)
+
+        return True
+
+class write_file_tool():
+    """write file tool
+Args:
+    file_name str: a string specifies file path that will be written to.
+    file_content str: a string speifies file content.
+Returns:
+    bool, True if it is succeeded to write to file, otherwise False
+    """
+    def __init__(self, repo_path: str):
+        self.repo_path = repo_path
+
+    def run(self, file_name: str, file_content: str):
+        if file_name is None or file_content is None:
+            return False
+        file_name = file_name
+        content = file_content
+        file_name = file_name.strip()
+        if self.repo_path is not None and self.repo_path not in file_name:
+            file_name = os.path.join(self.repo_path, file_name)
+        try:
+            with open(file_name, "w") as fobj:
+                fobj.write(content)
+                return True
+        except Exception as e:
+            logger.error(e)
+            return False
+        
+class extract_python_file_from_notebook_tool:
+    """extract code in a notebook to a python file
+Args:
+    notebook_path str: a string speicifies notebook path to extract.
+    output_path str: a string specifies output python file path.
+Returns:
+    bool True if it is succeeded to extract to python file, otherwise False
+    """
+    def __init__(self, repo_path: str):
+        self.repo_path = repo_path
+
+    def run(self, notebook_path: str, output_path: str):
+        # notebook_path = notebook_path_and_output_path[0]
+        # output_path = notebook_path_and_output_path[1]
+        if notebook_path is None or output_path is None:
+            return False
+        if self.repo_path not in notebook_path:
+            notebook_path = os.path.join(self.repo_path, notebook_path)
+        if self.repo_path not in output_path:
+            output_path = os.path.join(self.repo_path, output_path)
+        content = extract_code_from_notebook(notebook_path)
+        try:
+            with open(output_path, "w") as fobj:
+                fobj.write(content)
+                return True
+        except FileNotFoundError as e:
+            logger.error(str(e))
+            return f"False, {output_path} does not exist."
+        
+
+def prepare_provided_files_string(repo_path: str, provided_files: list[str]):
+    if provided_files is None or len(provided_files) == 0:
+        return "N/A"
+    str_provided_files = ""
+    for fn in provided_files:
+        file_path = os.path.join(repo_path, fn)
+        if fn.endswith(".ipynb"): # python notebook
+            content = extract_code_from_notebook(file_path)
+        else:
+            content = read_file(file_path)
+        content = content.replace("{", "{{").replace("}", "}}")
+        str_provided_files += f"""**{fn}**:\n{content}\n"""
+
+    return str_provided_files
+

bioguider/agents/evaluation_installation_task.py

@@ -0,0 +1,270 @@
+import os
+from pathlib import Path
+import logging
+from langchain.prompts import ChatPromptTemplate
+from markdownify import markdownify as md
+
+from bioguider.agents.agent_utils import read_file
+from bioguider.agents.collection_task import CollectionTask
+from bioguider.agents.prompt_utils import EVALUATION_INSTRUCTION, CollectionGoalItemEnum
+from bioguider.utils.constants import (
+    DEFAULT_TOKEN_USAGE, 
+    ProjectMetadata,
+    StructuredEvaluationInstallationResult,
+    FreeEvaluationInstallationResult,
+    EvaluationInstallationResult,
+)
+from bioguider.rag.data_pipeline import count_tokens
+from .evaluation_utils import run_llm_evaluation
+
+from .evaluation_task import EvaluationTask
+from bioguider.utils.utils import get_overall_score, increase_token_usage
+
+
+logger = logging.getLogger(__name__)
+
+STRUCTURED_EVALUATION_INSTALLATION_SYSTEM_PROMPT = """
+You are an expert in evaluating the quality of installation information in software repositories. 
+Your task is to analyze the provided files related to installation and generate a structured quality assessment based on the following criteria.
+---
+
+### **Evaluation Criteria**
+
+1. **Installation Available**: Is the installation section in document (like README.md or INSTALLATION)?
+   * Output: `Yes` or `No`
+
+2. **Installation Tutorial**: Is the step-by-step installation tutorial provided?
+   * Ouput: `Yes` or `No`
+
+3. **Number of required Dependencies Installation**: The number of dependencies that are required to install
+   * Output: Number
+   * Suggest specific improvements if necessary, such as missing dependencies
+
+4. **Compatible Operating System**: Is the compatible operating system described?
+   * Output: `Yes` or `No`
+
+5. **Hardware Requirements**: Is the hardware requirements described?
+   * Output: `Yes` or `No`
+
+6. **Overall Score**: Give an overall quality rating of the Installation information.
+   * Output: a number between 0 and 100 representing the overall quality rating.
+   * **Grade Level**:
+     - **85-100**: The installation information is exceptionally clear, polished, and engaging. It reads smoothly, with minimal effort required from the reader.
+     - **65-84**: The installation information is clear and easy to understand, with a natural flow and minimal jargon.
+     - **45-64**: The installation information is somewhat clear, but could benefit from more polish and consistency.
+     - **0-44**: The installation information is difficult to understand, with unclear language, jargon, or overly complex sentences.
+
+### Installation Files Provided:
+{installation_files_content}
+
+"""
+
+
+FREE_EVALUATION_INSTALLATION_SYSTEM_PROMPT = """
+You are an expert in evaluating the quality of **installation instructions** in software repositories.
+Your task is to analyze the provided content of installation-related files and generate a **comprehensive, structured quality report**.
+You will be given:
+1. The content of installation-related files.
+2. A structured evaluation of the installation-related files and its reasoning process.
+
+---
+
+### **Instructions**
+1. Based on the provided structured evaluation and its reasoning process, generate a free evaluation of the installation-related files.
+2. Focus on the explanation of assessment in structured evaluation and how to improve the installation-related files based on the structured evaluation and its reasoning process.
+   * For each suggestion to improve the installation-related files, you **must provide some examples** of the original text snippet and the improving comments.
+3. For each item in the structured evaluation, provide a detailed assessment followed by specific, actionable comments for improvement.
+4. Your improvement suggestions must also include the original text snippet and the improving comments.
+5. Your improvement suggestions must also include suggestions to improve readability.
+6. If you think the it is good enough, you can say so.
+7. In each section output, please first give a detailed explanation of the assessment, and then provide the detailed suggestion for improvement. If you think the it is good enough, you can say so.
+  The following is an example of the output format:
+  **Ease of Access:**
+    Detailed explanation of the assessment. Such as: The INSTALLATION file is present in the repository. The content of the file has been shared completely and is accessible. This confirms the availability of the installation documentation for evaluation. There's no issue with availability.
+    Detailed suggestion for improvement. Such as: No need to improve accessibility of the installation documentation.
+  **Clarity of Dependency Specification:**
+    Detailed explanation of the assessment. Such as: The installation section provides a clear list of dependencies. No need to improve.
+    Detailed suggestion for improvement. Such as: No need to improve.
+  **Hardware Requirements:**
+    Detailed explanation of the assessment. Such as: No hardware requirements are provided in the installation section.
+    Detailed suggestion for improvement. Such as: Add a section to describe the hardware requirements for the installation.
+  **Installation Guide:**
+    Detailed explanation of the assessment. Such as: The installation guide is present in the INSTALLATION file, but lacks a clear and concise guide for the installation process.
+    Detailed suggestion for improvement. Such as: 
+    - Add a clear and concise guide for the installation process.
+    - Improve the readability of the installation guide, like:
+      - <original text snippet> - <improving comments>
+      - <original text snippet> - <improving comments>
+      - ...
+  **Compatible Operating System:**
+    Detailed explanation of the assessment. Such as: No compatible operating system is provided in the installation section.
+    Detailed suggestion for improvement. Such as: 
+    - Add a section to describe the compatible operating systems for the installation.
+    - For example, Ubuntu 22.04 LTS+, CentOS 7+, Mac OS X 10.15+, Windows 10+ are supported.
+    - ...
+  **Overall Score:**
+    Detailed explanation of the assessment. Such as: The installation section provides a clear list of installation steps. No need to improve.
+    Detailed suggestion for improvement. Such as: Add a section to describe the compatible operating systems for the installation.
+
+---
+
+### **Output Format**
+Your output must **exactly match** the following format. Do not add or omit any sections. In the output, your detailed assessment and detailed suggestion must include the original text snippet and the improving comments, and include the analysis and explanation.
+
+**FinalAnswer**
+**Ease of Access:** 
+  <Your assessment and suggestion here>
+**Clarity of Dependency Specification:**
+  <Your assessment and suggestion here>
+**Hardware Requirements:**
+  <Your assessment and suggestion here>
+**Installation Guide:**
+  <Your assessment and suggestion here>  
+**Compatible Operating System:**
+  <Your assessment and suggestion here>
+**Overall Score:**
+  <Your assessment and suggestion here>
+
+---
+
+### **Structured Evaluation and Reasoning Process**
+{structured_evaluation_and_reasoning_process}
+
+---
+
+### Installation Files Provided:
+{installation_files_content}
+
+"""
+
+class EvaluationInstallationTask(EvaluationTask):
+    def __init__(
+        self, 
+        llm, 
+        repo_path,
+        gitignore_path, 
+        meta_data = None, 
+        step_callback = None,
+        summarized_files_db = None,
+        collected_files: list[str] | None = None,
+    ):
+        super().__init__(llm, repo_path, gitignore_path, meta_data, step_callback, summarized_files_db)
+        self.evaluation_name = "Installation Evaluation"
+        self.collected_files = collected_files
+
+    def _collect_install_files_content(self, files: list[str] | None=None) -> str:
+        if files is None or len(files) == 0:
+            return "N/A"
+        files_content = ""
+        MAX_TOKENS = os.environ.get("OPENAI_MAX_INPUT_TOKENS", 102400)
+        for f in files:
+            if f.endswith(".html") or f.endswith(".htm"):
+                html = read_file(os.path.join(self.repo_path, f))
+                content = md(html, escape_underscores=False)
+            else:
+                content = read_file(os.path.join(self.repo_path, f))
+            if count_tokens(content) > int(MAX_TOKENS):
+                content = content[:100000]
+            files_content += f"""
+{f} content:
+{content}
+
+"""
+        return files_content
+    
+    def _structured_evaluate(self, files: list[str] | None = None) -> tuple[dict|None, dict]:
+        if files is None or len(files) == 0:
+            return None, {**DEFAULT_TOKEN_USAGE}
+        
+        files_content = self._collect_install_files_content(files)
+        system_prompt = ChatPromptTemplate.from_template(
+            STRUCTURED_EVALUATION_INSTALLATION_SYSTEM_PROMPT,
+        ).format(
+            installation_files_content=files_content,
+        )
+        res, token_usage, reasoning_process = run_llm_evaluation(
+            llm=self.llm,
+            system_prompt=system_prompt,
+            instruction_prompt=EVALUATION_INSTRUCTION,
+            schema=StructuredEvaluationInstallationResult,
+            chain=True,
+        )
+        res: StructuredEvaluationInstallationResult = res
+        res.overall_score = get_overall_score([
+            res.install_available, 
+            res.install_tutorial, 
+            res.compatible_os, 
+            res.hardware_requirements,
+        ], [3, 3, 1, 1])
+        res.dependency_number = 0 if res.dependency_number is None else res.dependency_number
+        self.print_step(step_output=reasoning_process)
+        self.print_step(token_usage=token_usage)
+
+        return {
+            "evaluation": res,
+            "reasoning_process": reasoning_process,
+        }, token_usage
+    
+    def _free_evaluate(
+        self, 
+        files: list[str] | None=None,
+        structured_evaluation_and_reasoning_process: str | None=None,
+    ) -> tuple[dict|None, dict]:
+        if files is None or len(files) == 0:
+            return None, {**DEFAULT_TOKEN_USAGE}
+        
+        structured_evaluation_and_reasoning_process = structured_evaluation_and_reasoning_process or "N/A"
+        files_content = self._collect_install_files_content(files)
+        system_prompt = ChatPromptTemplate.from_template(FREE_EVALUATION_INSTALLATION_SYSTEM_PROMPT).format(
+            installation_files_content=files_content,
+            structured_evaluation_and_reasoning_process=structured_evaluation_and_reasoning_process,
+        )
+        res, token_usage, reasoning_process = run_llm_evaluation(
+            llm=self.llm,
+            system_prompt=system_prompt,
+            instruction_prompt=EVALUATION_INSTRUCTION,
+            schema=FreeEvaluationInstallationResult,
+        )
+        self.print_step(step_output=reasoning_process)
+        self.print_step(token_usage=token_usage)
+        evaluation = {
+            "evaluation": res,
+            "reasoning_process": reasoning_process,
+        }
+        return evaluation, token_usage
+    
+    def _evaluate(self, files: list[str] | None = None) -> tuple[EvaluationInstallationResult | None, dict, list[str]]:
+        total_token_usage = {**DEFAULT_TOKEN_USAGE}
+
+        structured_evaluation, structured_token_usage = self._structured_evaluate(files)
+        total_token_usage = increase_token_usage(total_token_usage, structured_token_usage)
+        evaluation, token_usage = self._free_evaluate(files, structured_evaluation["reasoning_process"])
+        total_token_usage = increase_token_usage(total_token_usage, token_usage)
+
+        combined_evaluation = EvaluationInstallationResult(
+            structured_evaluation=structured_evaluation["evaluation"],
+            free_evaluation=evaluation["evaluation"],
+            structured_reasoning_process=structured_evaluation["reasoning_process"],
+            free_reasoning_process=evaluation["reasoning_process"],
+        )
+
+        return combined_evaluation, total_token_usage, files
+
+    def _collect_files(self):
+        if self.collected_files is not None:
+            return self.collected_files
+        
+        task = CollectionTask(
+            llm=self.llm,
+            step_callback=self.step_callback,
+        )
+        task.compile(
+            repo_path=self.repo_path,
+            gitignore_path=Path(self.repo_path, ".gitignore"),
+            db=self.summarized_files_db,
+            goal_item=CollectionGoalItemEnum.Installation.name,
+        )
+        files = task.collect()
+        if files is None:
+            return []
+        return files