bioguider 0.2.52__py3-none-any.whl

bioguider/agents/collection_observe_step.py

@@ -0,0 +1,125 @@
+
+from typing import Callable
+from langchain.tools import BaseTool
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from bioguider.agents.agent_utils import ObservationResult
+from bioguider.agents.collection_task_utils import CollectionWorkflowState
+from bioguider.agents.common_agent_2step import CommonAgentTwoChainSteps, CommonAgentTwoSteps
+from bioguider.agents.peo_common_step import PEOCommonStep
+from bioguider.agents.prompt_utils import COLLECTION_GOAL, COLLECTION_PROMPTS
+from bioguider.utils.constants import MAX_STEP_COUNT
+
+
+COLLECTION_OBSERVE_SYSTEM_PROMPT = """You are an expert software developer and technical documentation analyst.
+{goal_item_desc}
+
+{related_file_description}
+---
+
+### **Repository Structure**
+Here is the 2-level file structure of the repository (`f` = file, `d` = directory, `l` - symlink, `u` - unknown):
+{repo_structure}
+---
+
+### **Intermediate Output**
+{intermediate_output}
+---
+
+### **Instructions**
+1. Your goal is to identify files that are relevant to the **goal item**.
+2. Carefully review the **Goal**, **Repository Structure**, and **Intermediate Output**.
+3. If you believe **all relevant files** have been collected:
+
+* Proceed with the following format:
+
+  * Provide your reasoning under **Analysis**
+  * Then list all relevant files and folders under **FinalAnswer**
+  * **FinalAnswer** format must exactly match this format:
+    **FinalAnswer**: {{"final_answer": [<file path>, <file path>, <file path>, ...]}}
+  * Be sure to include the **full relative paths** with respect to the repository root.
+  * Your answer **must exactly match the follwing format** (note: no JSON code block, no additional comments), **do not** make up anything:
+
+  ```
+  **Analysis**: your analysis here 
+  **FinalAnswer**: {{"final_answer": ["path/to/file1", "path/to/file2", ...]}}
+  ```
+4. If you believe **more files still need to be collected**:
+* Provide your reasoning under **Thoughts**:
+
+  ```
+  **Thoughts**: your explanation here
+  ```
+  
+5. Important instructions:
+  {important_instructions}
+Be precise and support your reasoning with evidence from the input.
+---
+
+### Notes
+- We are collecting information over multiple rounds, your thoughts and the output of this step will be persisted, so please **do not rush to provide a Final Answer**.  
+  If you find the current information insufficient, share your thoughts instead—we’ll continue with the next round accordingly.
+"""
+
+class CollectionObserveStep(PEOCommonStep):
+    def __init__(
+        self,
+        llm: BaseChatOpenAI,
+        repo_path: str,
+        repo_structure: str,
+        gitignore_path: str,
+    ):
+        super().__init__(llm=llm)
+        self.repo_path = repo_path
+        self.repo_structure = repo_structure
+        self.gitignore_path = gitignore_path
+        self.step_name = "Collection Observation Step"
+
+    def _build_prompt(self, state):
+        str_goal_item = state["goal_item"]
+        collection_item = COLLECTION_PROMPTS[str_goal_item]
+        goal_item_desc = \
+            ChatPromptTemplate.from_template(COLLECTION_GOAL).format(goal_item=collection_item["goal_item"])
+        repo_structure = self.repo_structure
+        intermediate_steps = self._build_intermediate_steps(state)
+        prompt = ChatPromptTemplate.from_template(COLLECTION_OBSERVE_SYSTEM_PROMPT)
+        important_instructions = "N/A" if "observe_important_instructions" not in collection_item or len(collection_item["observe_important_instructions"]) == 0 \
+            else collection_item["observe_important_instructions"]
+        return prompt.format(
+            goal_item_desc=goal_item_desc,
+            related_file_description=collection_item["related_file_description"],
+            repo_structure=repo_structure,
+            intermediate_output=intermediate_steps,
+            important_instructions=important_instructions,
+        )
+    def _execute_directly(self, state: CollectionWorkflowState):
+        step_count = state["step_count"]
+        plan = state["plan_actions"]
+        plan = plan.strip()
+        if len(plan) == 0:
+            instruction = "No plan provided, please let's generate the final answer based on the current information."
+        else:
+            instruction = "Now, we have reached max recursion limit, please give me the **final answer** based on the current information" \
+                if step_count == MAX_STEP_COUNT/3 - 2 else "Let's begin thinking."
+        system_prompt = self._build_prompt(state)
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt=instruction,
+            schema=ObservationResult,
+        )
+        state["final_answer"] = res.FinalAnswer
+        analysis = res.Analysis
+        thoughts = res.Thoughts
+        state["step_analysis"] = analysis
+        state["step_thoughts"] = thoughts
+        state["step_count"] += 1
+        self._print_step(
+            state,
+            step_output=f"**Observation Reasoning Process: {state['step_count']}**\n{reasoning_process}"
+        )
+        self._print_step(
+            state,
+            step_output=f"Final Answer: {res.FinalAnswer if res.FinalAnswer else None}\nAnalysis: {analysis}\nThoughts: {thoughts}",
+        )
+        return state, token_usage

bioguider/agents/collection_plan_step.py

@@ -0,0 +1,156 @@
+
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain.tools import BaseTool
+from langchain_core.prompts import ChatPromptTemplate, StringPromptTemplate
+from bioguider.agents.agent_utils import (
+    convert_plan_to_string, 
+    get_tool_names_and_descriptions,
+    PlanAgentResultJsonSchema,
+    PlanAgentResult,
+)
+from bioguider.agents.common_agent_2step import CommonAgentTwoChainSteps, CommonAgentTwoSteps
+from bioguider.agents.peo_common_step import PEOCommonStep
+from bioguider.agents.collection_task_utils import CollectionWorkflowState
+from bioguider.agents.prompt_utils import COLLECTION_GOAL, COLLECTION_PROMPTS
+
+COLLECTION_PLAN_SYSTEM_PROMPT = ChatPromptTemplate.from_template("""### **Goal**  
+You are an expert developer specializing in the biomedical domain. 
+**{goal}**
+
+{related_file_description}
+---
+
+### **Repository File Structure**  
+Below is the 2-level file structure of the repository (`f` = file, `d` = directory, `l` - symlink, `u` - unknown):  
+{repo_structure}
+
+---
+
+### **Function Tools**  
+You have access to the following function tools:  
+{tools}
+
+---
+
+### **Intermediate Steps**  
+Here are the results from previous steps:  
+{intermediate_steps}
+
+---
+
+### **Intermediate Thoughts**  
+- **Analysis**: {intermediate_analysis}  
+- **Thoughts**: {intermediate_thoughts}
+
+---
+
+### **Instructions**
+
+1. We will iterate through multiple **Plan -> Execution -> Observation** loops as needed.  
+   - All variables and tool outputs are **persisted across rounds**, so you can build on prior results.  
+   - Develop your plan **incrementally**, and reflect on intermediate observations before proceeding.  
+   - Limit each step to **one or two actions** — avoid trying to complete everything in a single step.
+
+2. Your task is to collect all files that are relevant to the goal.  
+   - Start by using the `summarize_file` tool to inspect file content quickly.  
+   - If needed, follow up with the `read_file` tool for full content extraction.
+
+3. You may use the `read_directory` tool to explore directory contents, but avoid using it in the first step unless necessary.
+
+4. Your plan can only use the above tools, **do not** make up any tools not in the above tools list.
+
+5. Your planned step input file or input directory must come from the above repository files structure, **do not** make up file name or directory name.
+
+---
+
+### **Important Instructions**
+{important_instructions}
+
+### **Output Format**  
+Your plan **must exactly match** a sequence of steps in the following format, **do not** make up anything:
+
+Step: <tool name>   # Tool name **must be one** of {tool_names}  
+Step Input: <file or directory name>
+
+Step: <tool name>  # Tool name **must be one** of {tool_names}  
+Step Input: <file or directory name>
+...
+""")
+
+class CollectionPlanStep(PEOCommonStep):
+    """
+    CollectionPlanStep is a step in the collection plan process.
+    It is responsible for initializing the tools and compiling the step.
+    """
+    
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI,
+        repo_path: str,
+        repo_structure: str,
+        gitignore_path: str,
+        custom_tools: list[BaseTool] | None = None,
+    ):
+        super().__init__(llm)
+        self.step_name = "Collection Plan Step"
+        self.repo_path = repo_path
+        self.repo_structure = repo_structure
+        self.gitignore_path = gitignore_path
+        self.custom_tools = custom_tools if custom_tools is not None else []
+    
+        
+    def _prepare_system_prompt(self, state: CollectionWorkflowState) -> str:
+        collection_state = state
+        goal_item = collection_state["goal_item"]
+        collection_item = COLLECTION_PROMPTS[goal_item]
+        intermediate_steps = self._build_intermediate_steps(state)
+        step_analysis, step_thoughts = self._build_intermediate_analysis_and_thoughts(state)
+        goal = ChatPromptTemplate.from_template(COLLECTION_GOAL).format(goal_item=collection_item["goal_item"])
+        related_file_description = collection_item["related_file_description"]
+        important_instructions="N/A" if "plan_important_instructions" not in collection_item or len(collection_item["plan_important_instructions"]) == 0 \
+            else collection_item["plan_important_instructions"]
+        tool_names, tools_desc = get_tool_names_and_descriptions(self.custom_tools)
+        system_prompt = COLLECTION_PLAN_SYSTEM_PROMPT.format(
+            goal=goal,
+            related_file_description=related_file_description,
+            repo_structure=self.repo_structure,
+            tools=tools_desc,
+            intermediate_steps=intermediate_steps,
+            intermediate_analysis=step_analysis,
+            intermediate_thoughts=step_thoughts,
+            tool_names=tool_names,
+            important_instructions=important_instructions,
+        )
+        self._print_step(
+            state,
+            step_output="**Intermediate Step Output**\n" + intermediate_steps
+        )
+        self._print_step(
+            state,
+            step_output="**Intermediate Step Analysis**\n{step_analysis}\n**Intermediate Step Thoughts**\n{step_thoughts}",
+        )
+        return system_prompt
+
+    def _execute_directly(self, state: CollectionWorkflowState):
+        system_prompt = self._prepare_system_prompt(state)
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin the collection plan step.",
+            schema=PlanAgentResultJsonSchema,
+        )
+        PEOCommonStep._reset_step_state(state)
+        res = PlanAgentResult(**res)
+        self._print_step(state, step_output=f"**Reasoning Process**\n{reasoning_process}")
+        self._print_step(state, step_output=f"**Plan**\n{str(res.actions)}")
+        state["plan_actions"] = convert_plan_to_string(res)
+
+        return state, token_usage
+        
+        
+        
+
+
+
+
+

bioguider/agents/collection_task.py

@@ -0,0 +1,184 @@
+
+import os
+import logging
+from typing import Callable
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain.tools import StructuredTool, Tool
+from langgraph.graph import StateGraph, START, END
+
+from bioguider.database.summarized_file_db import SummarizedFilesDb
+from bioguider.utils.file_utils import flatten_files, get_file_type
+from bioguider.agents.agent_utils import parse_final_answer, read_directory
+from bioguider.agents.collection_task_utils import (
+    RELATED_FILE_GOAL_ITEM,
+    CollectionWorkflowState, 
+    check_file_related_tool,
+)
+from bioguider.agents.agent_tools import (
+    read_directory_tool, 
+    summarize_file_tool, 
+    read_file_tool,
+)
+from bioguider.agents.peo_common_step import PEOCommonStep
+from bioguider.agents.prompt_utils import COLLECTION_PROMPTS
+from bioguider.agents.agent_task import AgentTask
+from bioguider.agents.collection_plan_step import CollectionPlanStep
+from bioguider.agents.collection_execute_step import CollectionExecuteStep
+from bioguider.agents.collection_observe_step import CollectionObserveStep
+
+logger = logging.getLogger(__name__)
+
+class CollectionTask(AgentTask):
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI, 
+        step_callback: Callable | None = None,
+        summarize_instruction: str | None = "N/A",
+        summarized_files_db: SummarizedFilesDb | None = None,
+        provided_files: list[str] | None = None,
+    ):
+        super().__init__(llm, step_callback, summarized_files_db=summarized_files_db)
+        self.repo_path: str | None = None
+        self.gitignore_path: str | None = None
+        self.repo_structure: str | None = None
+        self.goal_item: str | None = None
+        self.steps: list[PEOCommonStep] = []
+        self.tools: list[any] | None = None
+        self.custom_tools: list[Tool] | None = None
+        self.summarize_instruction = summarize_instruction
+        self.provided_files = provided_files
+
+    def _prepare_tools(self, related_file_goal_item_desc):
+        tool_rd = read_directory_tool(repo_path=self.repo_path)
+        tool_sum = summarize_file_tool(
+            llm=self.llm,
+            repo_path=self.repo_path,
+            output_callback=self.step_callback,
+            db=self.summarized_files_db,
+            summaize_instruction=self.summarize_instruction,
+        )
+        tool_rf = read_file_tool(repo_path=self.repo_path)
+        tool_cf = check_file_related_tool(
+            llm=self.llm,
+            repo_path=self.repo_path,
+            goal_item_desc=related_file_goal_item_desc,
+            output_callback=self.step_callback,
+            summarize_instruction=self.summarize_instruction,
+            summarized_files_db=self.summarized_files_db,
+        )
+        self.tools = [tool_rd, tool_sum, tool_rf, tool_cf]
+        self.custom_tools = [
+            Tool(
+                name = tool_rd.__class__.__name__,
+                func = tool_rd.run,
+                description=tool_rd.__class__.__doc__,
+            ),
+            StructuredTool.from_function(
+                tool_sum.run,
+                description=tool_sum.__class__.__doc__,
+                name=tool_sum.__class__.__name__,
+            ),
+            Tool(
+                name = tool_rf.__class__.__name__,
+                func = tool_rf.run,
+                description=tool_rf.__class__.__doc__,
+            ),
+            Tool(
+                name = tool_cf.__class__.__name__,
+                func = tool_cf.run,
+                description=tool_cf.__class__.__doc__,
+            ),
+        ]
+        # self.custom_tools.append(CustomPythonAstREPLTool())
+
+    def _initialize(self):
+        # initialize the 2-level file structure of the repo
+        if not os.path.exists(self.repo_path):
+            raise ValueError(f"Repository path {self.repo_path} does not exist.")
+        files = self.provided_files
+        if files is None:
+            files = read_directory(self.repo_path, os.path.join(self.repo_path, ".gitignore"))
+        file_pairs = [(f, get_file_type(os.path.join(self.repo_path, f)).value) for f in files]
+        self.repo_structure = ""
+        for f, f_type in file_pairs:
+            self.repo_structure += f"{f} - {f_type}\n"
+            
+        collection_item = COLLECTION_PROMPTS[self.goal_item]
+        related_file_goal_item_desc = ChatPromptTemplate.from_template(RELATED_FILE_GOAL_ITEM).format(
+            goal_item=collection_item["goal_item"],
+            related_file_description=collection_item["related_file_description"],
+        )
+        
+        self._prepare_tools(related_file_goal_item_desc)
+        self.steps = [
+            CollectionPlanStep(
+                llm=self.llm,
+                repo_path=self.repo_path,
+                repo_structure=self.repo_structure,
+                gitignore_path=self.gitignore_path,
+                custom_tools=self.custom_tools,
+            ),
+            CollectionExecuteStep(
+                llm=self.llm,
+                repo_path=self.repo_path,
+                repo_structure=self.repo_structure,
+                gitignore_path=self.gitignore_path,
+                custom_tools=self.custom_tools,
+            ),
+            CollectionObserveStep(
+                llm=self.llm,
+                repo_path=self.repo_path,
+                repo_structure=self.repo_structure,
+                gitignore_path=self.gitignore_path,
+            ),
+        ]
+
+    def _compile(self, repo_path: str, gitignore_path: str, **kwargs):
+        self.repo_path = repo_path
+        self.gitignore_path = gitignore_path
+        self.goal_item = kwargs.get("goal_item")
+        self._initialize()
+
+        def check_observe_step(state):
+            if "final_answer" in state and state["final_answer"] is not None:
+                self._print_step(step_name="Final Answer")
+                self._print_step(step_output=state["final_answer"])
+                return END
+            return "plan_step"
+
+        graph = StateGraph(CollectionWorkflowState)
+        graph.add_node("plan_step", self.steps[0].execute)
+        graph.add_node("execute_step", self.steps[1].execute)
+        graph.add_node("observe_step", self.steps[2].execute)
+        graph.add_edge(START, "plan_step")
+        graph.add_edge("plan_step", "execute_step")
+        graph.add_edge("execute_step", "observe_step")
+        graph.add_conditional_edges("observe_step", check_observe_step, {"plan_step", END})
+
+        self.graph = graph.compile()
+
+    def collect(self) -> list[str] | None:
+        s = self._go_graph({"goal_item": self.goal_item, "step_count": 0})
+        if s is None or 'final_answer' not in s:
+            return None
+        if s["final_answer"] is None:
+            return None
+        result = s["final_answer"].strip()
+        the_obj = parse_final_answer(result)
+        if the_obj is None or "final_answer" not in the_obj:
+            logger.error(f"Final answer is not a valid JSON: {result}")
+            return None
+        final_result = the_obj["final_answer"]
+        files = None
+        if isinstance(final_result, str):
+            final_result = final_result.strip()
+            files = [final_result]
+        elif isinstance(final_result, list):
+            files = final_result
+        else:
+            logger.error(f"Final answer is not a valid JSON list or string: {result}")
+            return None
+
+        files = flatten_files(self.repo_path, files)
+        return files

bioguider/agents/collection_task_utils.py

@@ -0,0 +1,142 @@
+import os
+from typing import Callable, Optional, TypedDict
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain_core.messages import AIMessage
+from pydantic import BaseModel, Field
+import logging
+
+from bioguider.agents.agent_tools import agent_tool
+from bioguider.agents.agent_utils import read_file, summarize_file
+from bioguider.agents.peo_common_step import PEOWorkflowState
+from bioguider.agents.common_agent import CommonAgent
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.database.summarized_file_db import SummarizedFilesDb
+from bioguider.utils.constants import MAX_FILE_LENGTH
+
+logger = logging.getLogger(__name__)
+
+class CollectionWorkflowState(TypedDict):
+    llm: Optional[BaseChatOpenAI]
+    step_output_callback: Optional[Callable]
+    
+    intermediate_steps: Optional[str]
+    step_output: Optional[str]
+    step_analysis: Optional[str]
+    step_thoughts: Optional[str]
+    plan_actions: Optional[list[dict]]
+
+    goal_item: Optional[str]
+    final_answer: Optional[str]
+    step_count: Optional[int]
+
+RELATED_FILE_GOAL_ITEM = """
+Your task is to determine whether the file is related to **{goal_item}**.
+
+{related_file_description} 
+"""
+
+CHECK_FILE_RELATED_USER_PROMPT = ChatPromptTemplate.from_template("""
+You are given a summary of a file’s content.  
+
+{goal_item_desc}
+
+Here is the file summary:  
+```
+{summarized_file_content}
+```
+
+### **Question:**  
+Does this file appear to contain related information?
+
+---
+
+### **Output Format:** 
+Respond with exactly two parts:
+1. A single word: Yes or No (indicating if the file meets the goal criteria)
+2. One brief explanatory sentence.
+For example: Yes. This file is a compiled binary file, so, it is related to the compiled standalone file (goal item).
+""")
+
+class CheckFileRelatedResult(BaseModel):
+    is_related: str = Field(description="A string conclusion specify if the provided file is related. The string value contains two parts:\n 1. A single word: Yes or No (indicating if the file meets the goal criteria).\n 2. One brief explanatory sentence.")
+
+class check_file_related_tool(agent_tool):
+    """ Check if the file is related to the goal item
+Args:
+    file_path str: file path
+Returns:
+    str: A string conclusion. The string conclusion contains two parts:\n 1. A single word: Yes or No (indicating if the file meets the goal criteria).\n 2. One brief explanatory sentence.
+    """ 
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI, 
+        repo_path: str,
+        goal_item_desc: str,
+        output_callback: Callable | None = None,
+        summarize_instruction: str | None = None,
+        summarize_level: int | None = 6,
+        summarized_files_db: SummarizedFilesDb | None = None,
+    ):
+        super().__init__(llm=llm, output_callback=output_callback)
+        self.repo_path = repo_path
+        self.goal_item_desc = goal_item_desc
+        self.summarize_instruction = summarize_instruction \
+            if summarize_instruction is not None else "N/A"
+        self.summarize_level = summarize_level
+        self.summarized_files_db = summarized_files_db
+
+    def run(self, file_path: str) -> str:
+        if not self.repo_path in file_path:
+            file_path = os.path.join(self.repo_path, file_path)
+        file_path = file_path.strip()
+        if not os.path.isfile(file_path):
+            return "Can't read file"
+        
+        check_prompts = None
+        try:
+            file_content = read_file(file_path)
+        except UnicodeDecodeError as e:
+            logger.error(str(e))
+            check_prompts = "Can't summarize binary file, please decide according to file name and extension."
+        except Exception as e:
+            logger.error(str(e))
+            check_prompts = "Failed to summarize file, please decide according to file name and extension."
+        if check_prompts is None and file_content is None:
+            return "Failed to read file"
+        if check_prompts is not None:
+            summarized_content = check_prompts
+        else:
+            if len(file_content) > MAX_FILE_LENGTH:
+                file_content = file_content[:MAX_FILE_LENGTH]
+            summarized_content, token_usage = summarize_file(
+                llm=self.llm, 
+                name=file_path, 
+                content=file_content, 
+                level=self.summarize_level,
+                summary_instructions=self.summarize_instruction,
+                db=self.summarized_files_db,
+            )
+            if summarized_content is None:
+                return "Failed to summarize file"
+            self._print_token_usage(token_usage)
+        
+        prompt = CHECK_FILE_RELATED_USER_PROMPT.format(
+            goal_item_desc=self.goal_item_desc,
+            summarized_file_content=summarized_content,
+        )
+
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning = agent.go(
+            system_prompt=prompt,
+            instruction_prompt="Now, please check if the file is related to the goal item.",
+            schema=CheckFileRelatedResult,
+        )
+        # res: AIMessage = self.llm.invoke([("human", prompt)])
+        res: CheckFileRelatedResult = res
+        out = res.is_related
+        
+        self._print_step_output(step_output=reasoning)
+        self._print_token_usage(token_usage)
+        return res.is_related
+