bioguider 0.2.3__py3-none-any.whl

bioguider/agents/evaluation_task.py

@@ -0,0 +1,269 @@
+
+import os
+from pathlib import Path
+import logging
+from typing import Callable
+from abc import ABC, abstractmethod
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+
+from bioguider.agents.agent_utils import read_file
+from bioguider.utils.constants import ProjectMetadata
+from .common_agent_2step import CommonAgentTwoSteps
+from .common_agent import CommonConversation
+from ..utils.pyphen_utils import PyphenReadability
+from ..utils.gitignore_checker import GitignoreChecker
+
+logger = logging.getLogger(__name__)
+
+EVALUATION_README_SYSTEM_PROMPT = """
+You are an expert in evaluating the quality of README files in software repositories. Your task is to analyze the provided README file and generate a comprehensive quality report.
+
+### **README Quality Report**
+
+For each criterion below, provide a brief assessment followed by specific, actionable comments for improvement.
+
+---
+
+**1. Project Clarity & Purpose**
+ * **Assessment**: [Your evaluation of whether the project's purpose is clear.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote a specific line/section from the README.]
+    * **Improving comments:** [Provide your suggestions to improve clarity.]
+
+**2. Installation Instructions**
+ * **Assessment**: [Your evaluation of the installation instructions.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to installation.]
+    * **Improving comments:** [Provide your suggestions.]
+
+**3. Usage Instructions**
+ * **Assessment**: [Your evaluation of the usage instructions.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to usage.]
+    * **Improving comments:** [Provide your suggestions.]
+
+**4. Contributing Guidelines**
+ * **Assessment**: [Your evaluation of the contributing guidelines.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to contributions.]
+    * **Improving comments:** [Provide your suggestions.]
+
+**5. License Information**
+ * **Assessment**: [Your evaluation of the license information.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to the license.]
+    * **Improving comments:** [Provide your suggestions.]
+
+**6. Readability Analysis**
+ * **Flesch Reading Ease**: `{flesch_reading_ease}` (A higher score is better, with 60-70 being easily understood by most adults).
+ * **Flesch-Kincaid Grade Level**: `{flesch_kincaid_grade}` (Represents the US school-grade level needed to understand the text).
+ * **Gunning Fog Index**: `{gunning_fog_index}` (A score above 12 is generally considered too hard for most people).
+ * **SMOG Index**: `{smog_index}` (Estimates the years of education needed to understand the text).
+ * **Assessment**: Based on these scores, evaluate the overall readability and technical complexity of the language used.
+
+**7. Overall Quality Summary**
+ * Provide a final, overall assessment of the README file's quality, summarizing the key strengths and areas for improvement.
+
+---
+
+### **README Content:**
+{readme_content}
+"""
+
+class EvaluationTask(ABC):
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI, 
+        repo_path: str, 
+        gitignore_path: str,
+        meta_data: ProjectMetadata | None = None,
+        step_callback: Callable | None = None
+    ):
+        self.evaluation_name = ""
+        self.llm = llm
+        self.repo_path = repo_path
+        self.gitignore_path = gitignore_path
+        self.step_callback = step_callback
+        self.metadata = meta_data
+    def print_step(
+        self,
+        step_name: str | None = None,
+        step_output: str | None = None,
+        token_usage: dict | None = None,
+    ):
+        if self.step_callback is None:
+            return
+        self.step_callback(
+            step_name=step_name,
+            step_output=step_output,
+            token_usage=token_usage,
+        )
+
+    def evaluate(self, files: list[str] | None = None):
+        self._enter_evaluation()
+        evaluation, token_usage = self._evaluate(files)
+        self._leave_evaluation(token_usage)
+        return evaluation
+    
+    def _enter_evaluation(self):
+        self.print_step(step_name=self.evaluation_name)
+
+    def _leave_evaluation(self, token_usage):
+        self.print_step(token_usage=token_usage)
+
+    @abstractmethod
+    def _evaluate(self, files: list[str]):
+        pass
+
+class EvaluationREADMETask(EvaluationTask):
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI, 
+        repo_path: str, 
+        gitignore_path: str,
+        meta_data: ProjectMetadata | None = None,
+        step_callback: Callable | None = None
+    ):
+        super().__init__(llm, repo_path, gitignore_path, meta_data, step_callback)
+        self.evaluation_name = "README Evaluation"
+            
+    def _evaluate(self, files: list[str]):
+        readme_files = files
+        if readme_files is None or len(readme_files) == 0:
+            return None
+        
+        readme_evaluations = {}
+        for readme_file in readme_files:
+            readme_path = Path(self.repo_path, readme_file)
+            readme_content = read_file(readme_path)
+            if readme_content is None:
+                logger.error(f"Error in reading file {readme_file}")
+                continue
+
+            readability = PyphenReadability()
+            flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, smog_index, \
+                _, _, _, _, _ = readability.readability_metrics(readme_content)
+            system_prompt = ChatPromptTemplate.from_template(
+                EVALUATION_README_SYSTEM_PROMPT
+            ).format(
+                readme_content=readme_content,
+                flesch_reading_ease=flesch_reading_ease,
+                flesch_kincaid_grade=flesch_kincaid_grade,
+                gunning_fog_index=gunning_fog_index,
+                smog_index=smog_index,
+            )
+            conversation = CommonConversation(llm=self.llm)
+            response, token_usage = conversation.generate(
+                system_prompt=system_prompt,
+                instruction_prompt="Before arriving at the conclusion, clearly explain your reasoning step by step. Now, let's begin the evaluation."
+            )
+            self.print_step(step_output=f"README: {readme_file}")
+            self.print_step(step_output=response)
+            readme_evaluations[readme_file] = response
+        return readme_evaluations, token_usage
+        
+EVALUATION_TUTORIAL_SYSTEM_PROMPT="""
+You are an expert in software documentation and developer education.
+You are given the content of a tutorial file from a GitHub repository. Your task is to **critically evaluate** the quality of this tutorial based on best practices in technical writing and developer onboarding.
+Please assess the tutorial using the following criteria. Provide your evaluation in structured sections:
+
+---
+
+### **Evaluation Criteria:**
+1. **Readability**: You are provided the following metrics scores calculated with pyphen, please evaluate readability based on the scores:
+   * Flesch Reading Ease: {flesch_reading_ease} (206.835 - 1.015(words/sentences) - 84.6(syllables/words))
+   * Flesch-Kincaid Grade Level: {flesch_kincaid_grade} (0.39(words/sentences) + 11.8(syllables/words) - 15.59)
+   * Gunning Fog Index: {gunning_fog_index} (0.4[(words/sentences) + 100(complex words/words)])
+   * SMOG Index: {smog_index} (1.043*sqrt(polysyllables * (30/sentences)) + 3.1291)
+2. **Coverage**
+   * Does the tutorial cover all major steps needed to get started?
+   * Are dependencies, prerequisites, setup steps, and example usage included?
+3. **Structure & Organization**
+   * Is the content logically structured (e.g., introduction → setup → examples → summary)?
+   * Are sections well-labeled and easy to navigate?
+4. **Balance Between Code and Explanation**
+   * Is there a good balance between code snippets and narrative explanation?
+   * Are code blocks properly annotated or explained?
+5. **Terminology Consistency**
+   * Is technical terminology used consistently and accurately?
+   * Are key terms introduced and reused correctly?
+6. **Example Quality**
+   * Are the examples relevant, correct, and representative of real usage?
+   * Are edge cases or typical user pitfalls addressed?
+7. **Formatting and Style**
+   * Are headings, bullet points, code formatting, and markdown style used effectively?
+   * Are there any formatting issues that hurt clarity?
+---
+
+### **Output Format:**
+Please respond in the following format:
+
+```
+**FinalAnswer**
+**Readability**: Your comments here  
+**Coverage**: Your comments here  
+**Structure & Organization**: Your comments here  
+**Code vs. Explanation Balance**: Your comments here  
+**Terminology Consistency**: Your comments here  
+**Example Quality**: Your comments here  
+**Formatting and Style**: Your comments here  
+**Overall Rating**: [Poor / Fair / Good / Excellent]  
+```
+
+---
+
+### **Tutorial File Content:**
+
+```
+{tutorial_file_content}
+```
+
+---
+"""
+class EvaluationTutorialTask(EvaluationTask):
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI, 
+        repo_path: str, 
+        gitignore_path: str,
+        meta_data: ProjectMetadata | None = None,
+        step_callback: Callable | None = None
+    ):
+        super().__init__(llm, repo_path, gitignore_path, meta_data, step_callback)
+        self.evaluation_name = "Tutorial Evaluation"
+
+    def _evaluate(self, files: list[str]):
+        if len(files) == 0:
+            return None
+        
+        evaluations = {}
+        for file in files:
+            tutorial_path = Path(self.repo_path, file)
+            tutorial_content = read_file(tutorial_path)
+            if tutorial_content is None:
+                logging.error(f"Error in reading file {file}")
+                continue
+
+            readability = PyphenReadability()
+            flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, smog_index, \
+                _, _, _, _, _ = readability.readability_metrics(tutorial_content)
+            system_prompt = ChatPromptTemplate.from_template(
+                EVALUATION_TUTORIAL_SYSTEM_PROMPT
+            ).format(
+                tutorial_file_content=tutorial_content,
+                flesch_reading_ease=flesch_reading_ease,
+                flesch_kincaid_grade=flesch_kincaid_grade,
+                gunning_fog_index=gunning_fog_index,
+                smog_index=smog_index,
+            )
+            conversation = CommonConversation(llm=self.llm)
+            response, token_usage = conversation.generate(
+                system_prompt=system_prompt,
+                instruction_prompt="Before arriving at the conclusion, clearly explain your reasoning step by step. Now, let's begin the evaluation."
+            )
+            self.print_step(step_output=f"Tutorial: {file}")
+            self.print_step(step_output=response)
+            evaluations[file] = response
+        return evaluations, token_usage
+

bioguider/agents/identification_execute_step.py

@@ -0,0 +1,179 @@
+
+import logging
+
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain.tools import BaseTool
+from langchain.agents import AgentExecutor, create_react_agent
+from langchain_community.callbacks.openai_info import OpenAICallbackHandler
+
+from bioguider.utils.constants import DEFAULT_TOKEN_USAGE
+from bioguider.agents.agent_utils import CustomOutputParser, CustomPromptTemplate
+from bioguider.agents.peo_common_step import ( 
+    PEOCommonStep,
+)
+
+logger = logging.getLogger(__name__)
+
+## execution system prompt
+IDENTIFICATION_EXECUTION_SYSTEM_PROMPT = """You are an expert Python developer.
+
+You are given a **plan** and are expected to complete it using Python code and the available tools.
+
+---
+
+### **Available Tools**
+{tools}
+
+---
+
+### **Your Task**
+
+Execute the plan step by step using the format below:
+
+```
+Thought: Describe what you are thinking or planning to do next.  
+Action: The tool you are going to use (must be one of: {tool_names})  
+Action Input: The input to the selected action  
+Observation: The result returned by the action  
+```
+
+You may repeat the **Thought → Action → Action Input → Observation** loop as many times as needed.
+
+Once the plan is fully completed, output the result in the following format:
+```
+Thought: I have completed the plan.
+Final Answer:
+Action: {{tool_name}}
+Action Input: {{file_name1}}
+Action Observation: {{Observation1}}
+---
+Action: {{tool_name}}
+Action Input: {{file_name2}}
+Action Observation: {{Observation2}}
+---
+...
+```
+
+---
+
+### **Example**
+```
+Action: summarize_file_tool  
+Action Input: README.md  
+Observation: # BioGuider\nBioGuider is a Python package for bioinformatics.\n...
+...
+Final Answer:
+Action: summarize_file_tool
+Action Input: README.md
+Action Observation: # BioGuider\nBioGuider is a Python package for bioinformatics.\n...
+---
+Action: check_file_related_tool
+Action Input: pyproject.toml
+Action Observation: Yes, the file is related to the project.
+---
+...
+```
+
+---
+
+### **Important Notes**
+
+- You must strictly follow the provided plan.  
+- **Do not take any additional or alternative actions**, even if:  
+  - No relevant result is found  
+  - The file content is missing, empty, or irrelevant  
+- If no information is found in a step, simply proceed to the next action in the plan without improvising.  
+- Only use the tools specified in the plan actions. No independent decisions or extra steps are allowed.
+
+### **Plan**
+{plan_actions}
+
+### **Actions Already Taken**
+{agent_scratchpad}
+
+---
+
+{input}
+"""
+
+class IdentificationExecuteStep(PEOCommonStep):
+    """
+    This class is a placeholder for common step functionality in the PEO agent.
+    It is currently empty and can be extended in the future.
+    """
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI,
+        repo_path: str,
+        repo_structure: str,
+        gitignore_path: str,
+        custom_tools: list[BaseTool] | None = None,
+    ):
+        super().__init__(llm=llm)
+        self.llm = llm
+        self.step_name = "Identification Execution Step"
+        self.repo_path = repo_path
+        self.repo_structure = repo_structure
+        self.gitignore_path = gitignore_path
+        self.custom_tools = custom_tools if custom_tools is not None else []
+
+    def _execute_directly(self, state):
+        plan_actions = state["plan_actions"]
+        prompt = CustomPromptTemplate(
+            template=IDENTIFICATION_EXECUTION_SYSTEM_PROMPT,
+            tools=self.custom_tools,
+            plan_actions=plan_actions,
+            input_variables=[
+                "tools", 
+                "tool_names", 
+                "agent_scratchpad", 
+                "intermediate_steps", 
+                "plan_actions",
+            ],
+        )
+        output_parser = CustomOutputParser()
+        agent = create_react_agent(
+            llm=self.llm,
+            tools=self.custom_tools,
+            prompt=prompt,
+            output_parser=output_parser,
+            stop_sequence=["\nObservation:"],
+        )
+        callback_handler = OpenAICallbackHandler()
+        agent_executor = AgentExecutor(
+            agent=agent,
+            tools=self.custom_tools,
+            max_iterations=10,
+        )
+        response = agent_executor.invoke(
+            input={"plan_actions": plan_actions, "input": "Now, let's begin."},
+            callbacks=[callback_handler],
+        )
+        # parse the response
+        if "output" in response:
+            output = response["output"]
+            if "**Final Answer**" in output:
+                final_answer = output.split("**Final Answer:**")[-1].strip().strip(":")
+                step_output = final_answer
+            elif "Final Answer" in output:
+                final_answer = output.split("Final Answer")[-1].strip().strip(":")
+                step_output = final_answer
+            else:
+                step_output = output
+            step_output = step_output.strip().strip("```").strip('"""')
+            self._print_step(state, step_output=step_output)
+            state["step_output"] = step_output
+        else:
+            logger.error("No output found in the response.")
+            self._print_step(
+                state,
+                step_output="Error: No output found in the response.",
+            )
+            state["step_output"] = "Error: No output found in the response."
+        
+        token_usage = vars(callback_handler)
+        token_usage = {**DEFAULT_TOKEN_USAGE, **token_usage}
+            
+        return state, token_usage
+
+

bioguider/agents/identification_observe_step.py

@@ -0,0 +1,92 @@
+
+from langchain.prompts import ChatPromptTemplate
+
+from bioguider.agents.agent_utils import ObservationResult
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.agents.identification_task_utils import IdentificationWorkflowState
+from bioguider.agents.peo_common_step import PEOWorkflowState, PEOCommonStep
+
+
+## observation system prompt
+IDENTIFICATION_OBSERVATION_SYSTEM_PROMPT = """Your goal is:
+{goal}
+
+### **Repository File Structure**
+Here is the 2-level file structure of the repository (f - file, d - directory, l - symlink, u - unknown):
+{repo_structure}
+
+### **Intermediate Output**
+{intermediate_output}
+
+### **Instructions**
+Carefully review the **Goal**, **Repository File Structure**, and **Intermediate Output**.
+- If you believe the goal **can be achieved**, proceed as follows:  
+  - Provide your reasoning under **Analysis**  
+  - Then provide your result under **FinalAnswer**  
+  ```
+  **Analysis**: your analysis here  
+  **FinalAnswer**: your final answer here, in json format **without** json fence (```json ... ```), like {final_answer_example}
+  ```
+- If the information is **not sufficient** to achieve the goal, simply explain why under **Thoughts**:  
+  ```
+  **Thoughts**: your thoughts here
+  ```
+Be precise and support your reasoning with evidence from the input.
+
+### Notes
+We are collecting information over multiple rounds, your thoughts and the output of this step will be persisted, so please **do not rush to provide a Final Answer**.  
+If you find the current information insufficient, share your reasoning or thoughts instead—we’ll continue with the next round accordingly.
+"""
+
+
+class IdentificationObserveStep(PEOCommonStep):
+    def __init__(
+        self,
+        llm,
+        repo_path: str,
+        repo_structure: str,
+        gitignore_path: str,
+        custom_tools: list = None,
+    ):
+        super().__init__(llm)
+        self.step_name = "Identification Observe Step"
+        self.repo_path = repo_path
+        self.repo_structure = repo_structure
+        self.gitignore_path = gitignore_path
+        self.custom_tools = custom_tools if custom_tools is not None else []
+
+    def _prepare_system_prompt(self, state: IdentificationWorkflowState): 
+        goal = state["goal"]
+        final_answer_example = state["final_answer_example"]
+        intermediate_output = self._build_intermediate_steps(state)
+        prompt = ChatPromptTemplate.from_template(IDENTIFICATION_OBSERVATION_SYSTEM_PROMPT)
+
+        return prompt.format(
+            goal=goal,
+            repo_structure=self.repo_structure,
+            intermediate_output=intermediate_output,
+            final_answer_example=final_answer_example,
+        )
+
+    def _execute_directly(self, state: IdentificationWorkflowState):
+        system_prompt = self._prepare_system_prompt(state)
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin.",
+            schema=ObservationResult,
+        )
+        state["final_answer"] = res.FinalAnswer
+        analysis = res.Analysis
+        thoughts = res.Thoughts
+        state["step_analysis"] = analysis
+        state["step_thoughts"] = thoughts
+        self._print_step(
+            state,
+            step_output=f"**Observation Reasoning Process**\n{reasoning_process}"
+        )
+        self._print_step(
+            state,
+            step_output=f"Final Answer: {res.FinalAnswer if res.FinalAnswer else None}\nAnalysis: {analysis}\nThoughts: {thoughts}",
+        )
+        return state, token_usage

bioguider/agents/identification_plan_step.py

@@ -0,0 +1,135 @@
+
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain.tools import BaseTool
+from pydantic import BaseModel, Field
+
+from bioguider.agents.agent_utils import get_tool_names_and_descriptions
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.agents.identification_task_utils import IdentificationWorkflowState
+from bioguider.agents.peo_common_step import PEOCommonStep
+
+## plan system prompt
+IDENTIFICATION_PLAN_SYSTEM_PROMPT = ChatPromptTemplate.from_template("""### **Goal**
+You are an expert developer in the field of biomedical domain. Your goal is:
+{goal}
+
+### **Repository File Structure**
+Here is the 2-level file structure of the repository (f - file, d - directory, l - symlink, u - unknown):
+{repo_structure}
+
+### **Function Tools**
+You are provided the following function tools:
+{tools}
+
+### Intermediate Steps
+Hers are the intermediate steps results:
+{intermediate_steps}
+
+### Intermediate Thoughts
+Analysis: {intermediate_analysis}
+Thoughts: {intermediate_thoughts}
+
+### **Instruction**
+We will repeat **Plan - Execution - Observation** loops as many times as needed. All the results in each round will be persisted, 
+meaning that states and variables will persisted through multiple rounds of plan execution. Be sure to take advantage of this by 
+developing your collection plan incrementally and reflect on the intermediate observations at each round, instead of coding up 
+everything in one go. Be sure to take only one or two actions in each step.
+
+### **Output**
+You plan should follow this format:
+Step: tool name, should be one of {tool_names}
+Step Input: file name or directory name
+Step: tool name, should be one of {tool_names}
+Step Input: file name or directory name
+""")
+
+class IdentificationPlanResult(BaseModel):
+    """ Identification Plan Result """
+    actions: list[dict] = Field(description="a list of action dictionary, e.g. [{'name': 'read_file', 'input': 'README.md'}, ...]")
+
+IdentificationPlanResultJsonSchema = {
+    "title": "identification_plan_result",
+    "description": "plan result",
+    "type": "object",
+    "properties": {
+        "actions": {
+            "type": "array",
+            "description": """a list of action dictionary, e.g. [{'name': 'read_file', 'input': 'README.md'}, ...]""",
+            "title": "Actions",
+            "items": {"type": "object"}
+        },
+    },
+    "required": ["actions"],
+}
+
+class IdentificationPlanStep(PEOCommonStep):
+    def __init__(
+        self,
+        llm: BaseChatOpenAI,
+        repo_path: str,
+        repo_structure: str,
+        gitignore_path: str,
+        custom_tools: list[BaseTool] | None = None,
+    ): 
+        super().__init__(llm)
+        self.step_name = "Identification Plan Step"
+        self.repo_path = repo_path
+        self.repo_structure = repo_structure
+        self.gitignore_path = gitignore_path
+        self.custom_tools = custom_tools if custom_tools is not None else []
+
+    def _prepare_system_prompt(self, state: IdentificationWorkflowState) -> str:
+        goal = state["goal"]
+        repo_structure = self.repo_structure
+        intermdediate_steps = self._build_intermediate_steps(state)
+        step_analysis, step_thoughts = self._build_intermediate_analysis_and_thoughts(state)
+        self._print_step(
+            state,
+            step_output="**Intermediate Step Output**\n" + intermdediate_steps
+        )
+        self._print_step(
+            state,
+            step_output="**Intermediate Step Analysis**\n{step_analysis}\n**Intermediate Step Thoughts**\n{step_thoughts}",
+        )
+        tool_names, tools_desc = get_tool_names_and_descriptions(self.custom_tools)
+        return IDENTIFICATION_PLAN_SYSTEM_PROMPT.format(
+            goal=goal,
+            repo_structure=repo_structure,
+            tools=tools_desc,
+            intermediate_steps=intermdediate_steps,
+            intermediate_analysis=step_analysis,
+            intermediate_thoughts=step_thoughts,
+            tool_names=tool_names,
+        )
+
+    def _convert_to_plan_actions_text(self, actions: list[dict]) -> str:
+        plan_str = ""
+        for action in actions:
+            action_str = f"Step: {action['name']}\n"
+            action_str += f"Step Input: {action['input']}\n"
+            plan_str += action_str
+        return plan_str
+
+    def _execute_directly(self, state: IdentificationWorkflowState):
+        system_prompt = self._prepare_system_prompt(state)
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin.",
+            schema=IdentificationPlanResultJsonSchema,
+        )
+        PEOCommonStep._reset_step_state(state)
+        res = IdentificationPlanResult(**res)
+        self._print_step(
+            state,
+            step_output="**Reasoning Process**\n" + reasoning_process,
+        )
+        self._print_step(
+            state,
+            step_output=f"**Plan**\n{res.actions}"
+        )
+        state["plan_actions"] = self._convert_to_plan_actions_text(res.actions)
+
+        return state, token_usage
+