bioguider 0.2.20__py3-none-any.whl → 0.2.22__py3-none-any.whl

bioguider/agents/agent_utils.py

@@ -2,6 +2,7 @@
 import json
 from json import JSONDecodeError
 import os
+from pathlib import Path
 import re
 from typing import List, Optional, Tuple, Union
 from langchain_openai import AzureChatOpenAI
@@ -22,6 +23,7 @@ from pydantic import BaseModel, Field
 
 from bioguider.utils.constants import DEFAULT_TOKEN_USAGE, MAX_FILE_LENGTH, MAX_SENTENCE_NUM
 from bioguider.utils.file_utils import get_file_type
+from bioguider.utils.utils import clean_action_input
 from ..utils.gitignore_checker import GitignoreChecker
 from ..database.summarized_file_db import SummarizedFilesDb
 from bioguider.agents.common_conversation import CommonConversation
@@ -122,16 +124,18 @@ def pretty_print(message, printout = True):
 HUGE_FILE_LENGTH = 10 * 1024 # 10K
 
 def read_file(
-    file_path: str,    
+    file_path: str | Path,
 ) -> str | None:
+    file_path = str(file_path).strip()
     if not os.path.isfile(file_path):
         return None
     with open(file_path, 'r') as f:
         content = f.read()
         return content
 
-def write_file(file_path: str, content: str):
+def write_file(file_path: str | Path, content: str):
     try:
+        file_path = str(file_path).strip()
         with open(file_path, "w") as fobj:
             fobj.write(content)
             return True
@@ -140,10 +144,11 @@ def write_file(file_path: str, content: str):
         return False
 
 def read_directory(
-    dir_path: str,
+    dir_path: str | Path,
     gitignore_path: str,
     level: int=1,
 ) -> list[str] | None:
+    dir_path = str(dir_path).strip()
     if not os.path.isdir(dir_path):
         return None
     gitignore_checker = GitignoreChecker(
@@ -182,15 +187,16 @@ Now, let's start to summarize.
 
 def summarize_file(
     llm: BaseChatOpenAI, 
-    name: str, 
+    name: str | Path, 
     content: str | None = None, 
     level: int = 3,
     summary_instructions: str | None = None,
     summarize_prompt: str = "N/A",
     db: SummarizedFilesDb | None = None,
 ) -> Tuple[str, dict]:
+    name = str(name).strip()
     if content is None:
-        try:
+        try:            
             with open(name, "r") as fobj:
                 content = fobj.read()
         except Exception as e:
@@ -289,9 +295,7 @@ class CustomOutputParser(AgentOutputParser):
         action_input = match.group(2)
         # Return the action and action input
         action_dict = None
-        action_input_replaced = action_input.strip().strip(" ").strip('"').strip('`').strip()
-        action_input_replaced = action_input_replaced.replace("'", '"')
-        action_input_replaced = action_input_replaced.replace("`", '"')
+        action_input_replaced = clean_action_input(action_input)
         try:
             action_dict = json.loads(action_input_replaced)
         except json.JSONDecodeError:
@@ -410,8 +414,10 @@ def read_license_file(repo_path: str) -> tuple[str | None, str|None]:
     ]
     license_files = []
     for file in hardcoded_license_files:
-        if os.path.exists(os.path.join(repo_path, file)):
-            with open(os.path.join(repo_path, file), "r") as f:
+        file_path = os.path.join(str(repo_path), file)
+        file_path = file_path.strip()
+        if os.path.exists(file_path):
+            with open(file_path, "r") as f:
                 license_files.append((f.read(), os.path.join(repo_path, file)))
     
     max_item = max(license_files, key=lambda x: len(x[0])) if len(license_files) > 0 else (None, None)

bioguider/agents/collection_observe_step.py

@@ -94,8 +94,13 @@ class CollectionObserveStep(PEOCommonStep):
         )
     def _execute_directly(self, state: CollectionWorkflowState):
         step_count = state["step_count"]
-        instruction = "Now, we have reached max recursion limit, please give me the **final answer** based on the current information" \
-            if step_count == MAX_STEP_COUNT/3 - 2 else "Let's begin thinking."
+        plan = state["plan_actions"]
+        plan = plan.strip()
+        if len(plan) == 0:
+            instruction = "No plan provided, please let's generate the final answer based on the current information."
+        else:
+            instruction = "Now, we have reached max recursion limit, please give me the **final answer** based on the current information" \
+                if step_count == MAX_STEP_COUNT/3 - 2 else "Let's begin thinking."
         system_prompt = self._build_prompt(state)
         agent = CommonAgentTwoSteps(llm=self.llm)
         res, _, token_usage, reasoning_process = agent.go(

bioguider/agents/collection_task_utils.py

@@ -89,6 +89,7 @@ Returns:
     def run(self, file_path: str) -> str:
         if not self.repo_path in file_path:
             file_path = os.path.join(self.repo_path, file_path)
+        file_path = file_path.strip()
         if not os.path.isfile(file_path):
             return "Can't read file"
         

bioguider/agents/consistency_collection_step.py

@@ -0,0 +1,102 @@
+
+
+
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from pydantic import BaseModel, Field
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.agents.consistency_evaluation_task_utils import ConsistencyEvaluationState
+from bioguider.agents.peo_common_step import PEOCommonStep
+
+
+CONSISTANCY_COLLECTION_SYSTEM_PROMPT = """
+### **Goal**  
+You are an expert developer specializing in the biomedical domain.
+You will be given a {domain} documentation. Your task is to collect all the functions, classes, and methods that the {domain} documentation mentions.
+
+---
+
+### **Input {domain} Documentation**
+{documentation}
+
+### **Output Format**
+The collected functions, classes, and methods **must exactly match** the following format, **do not** make up anything:
+
+```
+name: <function/class/method name>
+file_path: <file path, if not sure, just put "N/A">
+parameters: <parameters, if not sure, just put "N/A">
+parent: <parent name, if it is a class method, put the class name as the parent name, if not sure, just put "N/A">
+
+...
+
+```
+
+---
+
+### **Output Example**
+```
+name: __init__
+file_path: src/agents/common_agent.py
+parameters: llm, step_output_callback, summarized_files_db
+parent: CommonAgent
+
+name: _invoke_agent
+file_path: src/agents/common_agent.py
+parameters: system_prompt, instruction_prompt, schema, post_process
+parent: CommonAgent
+
+...
+```
+
+"""
+
+class ConsistencyCollectionResult(BaseModel):
+    functions_and_classes: list[dict] = Field(description="A list of functions and classes that the documentation mentions")
+
+ConsistencyCollectionResultJsonSchema = {
+  "properties": {
+    "functions_and_classes": {
+      "description": "A list of functions and classes that the documentation mentions",
+      "items": {
+        "type": "object"
+      },
+      "title": "Functions And Classes",
+      "type": "array"
+    }
+  },
+  "required": [
+    "functions_and_classes"
+  ],
+  "title": "ConsistencyCollectionResult",
+  "type": "object"
+}
+
+class ConsistencyCollectionStep(PEOCommonStep):
+    def __init__(self, llm: BaseChatOpenAI):
+        super().__init__(llm)
+        self.step_name = "Consistency Collection Step"
+
+    def _prepare_system_prompt(self, state: ConsistencyEvaluationState) -> str:
+        documentation = state["documentation"]
+        domain = state["domain"]
+        return ChatPromptTemplate.from_template(CONSISTANCY_COLLECTION_SYSTEM_PROMPT).format(
+            domain=domain,
+            documentation=documentation,
+        )
+
+    def _execute_directly(self, state: ConsistencyEvaluationState) -> tuple[dict, dict[str, int]]:
+        system_prompt = self._prepare_system_prompt(state)
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin the consistency collection step.",
+            schema=ConsistencyCollectionResultJsonSchema,
+        )
+        res: ConsistencyCollectionResult = ConsistencyCollectionResult.model_validate(res)
+        state["functions_and_classes"] = res.functions_and_classes
+        self._print_step(state, step_output=f"Consistency Collection Result: {res.functions_and_classes}")
+        self._print_step(state, step_output=f"Consistency Collection Reasoning Process: {reasoning_process}")
+        
+        return state, token_usage
+

bioguider/agents/consistency_evaluation_task.py

@@ -0,0 +1,57 @@
+
+
+
+from typing import Callable
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from pydantic import BaseModel
+
+from bioguider.agents.consistency_evaluation_task_utils import ConsistencyEvaluationState
+from bioguider.database.code_structure_db import CodeStructureDb
+from .consistency_collection_step import ConsistencyCollectionStep
+from .consistency_query_step import ConsistencyQueryStep
+from .consistency_observe_step import ConsistencyObserveStep
+
+class ConsistencyEvaluationResult(BaseModel):
+    score: str
+    assessment: str
+    development: list[str]
+    strengths: list[str]
+
+class ConsistencyEvaluationTask:
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI, 
+        code_structure_db: CodeStructureDb, 
+        step_callback: Callable | None = None
+    ):
+        self.llm = llm
+        self.code_structure_db = code_structure_db
+        self.step_callback = step_callback
+
+    def evaluate(self, domain: str, documentation: str) -> ConsistencyEvaluationResult:
+        collection_step = ConsistencyCollectionStep(llm=self.llm)
+        query_step = ConsistencyQueryStep(code_structure_db=self.code_structure_db)
+        observe_step = ConsistencyObserveStep(llm=self.llm)
+
+        state = ConsistencyEvaluationState(
+            domain=domain,
+            documentation=documentation,
+            step_output_callback=self.step_callback,
+        )
+
+        state = collection_step.execute(state)
+        state = query_step.execute(state)
+        state = observe_step.execute(state)
+
+        score = state["consistency_score"]
+        assessment = state["consistency_assessment"]
+        development = state["consistency_development"]
+        strengths = state["consistency_strengths"]
+
+        return ConsistencyEvaluationResult(
+            score=score,
+            assessment=assessment,
+            development=development,
+            strengths=strengths,
+        )
+

bioguider/agents/consistency_evaluation_task_utils.py

@@ -0,0 +1,14 @@
+
+from typing import Callable, Optional, TypedDict
+
+
+class ConsistencyEvaluationState(TypedDict):
+    domain: str
+    documentation: str
+    step_output_callback: Optional[Callable]
+    functions_and_classes: Optional[list[dict]]
+    all_query_rows: Optional[list[any]]
+    consistency_score: Optional[str]
+    consistency_assessment: Optional[str]
+    consistency_development: Optional[list[str]]
+    consistency_strengths: Optional[list[str]]

bioguider/agents/consistency_observe_step.py

@@ -0,0 +1,109 @@
+
+
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from pydantic import BaseModel, Field
+from bioguider.agents.common_agent_2step import CommonAgentTwoSteps
+from bioguider.agents.consistency_evaluation_task_utils import ConsistencyEvaluationState
+from bioguider.agents.peo_common_step import PEOCommonStep
+
+CONSISTENCY_OBSERVE_SYSTEM_PROMPT = """
+You are an expert developer specializing in the biomedical domain.
+Your task is to analyze both:
+1. the provided file related to {domain} documentation,
+2. the code definitions related to the {domain} documentation
+and generate a structured consistency assessment based on the following criteria.
+
+---
+
+### **Evaluation Criteria**
+
+**Consistency**:
+  * **Score**: [Poor / Fair / Good / Excellent]
+  * **Assessment**: [Your evaluation of whether the {domain} documentation is consistent with the code definitions]
+  * **Development**: [A list of inconsistent function/class/method name and inconsistent docstring, and describe how they are inconsistent]
+  * **Strengths**: [A list of strengths of the {domain} documentation on consistency]
+
+---
+
+### **Output Format**
+Your output **must exactly match** the following format:
+```
+**Consistency**:
+  * **Score**: [Poor / Fair / Good / Excellent]
+  * **Assessment**: [Your evaluation of whether the {domain} documentation is consistent with the code definitions]
+  * **Development**: [A list of inconsistent function/class/method name and inconsistent docstring, and describe how they are inconsistent]
+  * **Strengths**: [A list of strengths of the {domain} documentation on consistency]
+```
+
+### **Output Example**
+
+```
+**Consistency**:
+  * **Assessment**: [Your evaluation of whether the {domain} documentation is consistent with the code definitions]
+  * **Development**:
+    - Inconsistent function/class/method name 1
+    - Inconsistent docstring 1
+    - Inconsistent function/class/method name 2
+    - Inconsistent docstring 2
+    - ...
+  * **Strengths**: 
+    - Strengths 1
+    - Strengths 2
+    - ...
+```
+
+---
+
+### **Input {domain} Documentation**
+{documentation}
+
+### **Code Definitions**
+{code_definitions}
+
+
+"""
+
+class ConsistencyEvaluationObserveResult(BaseModel):
+    consistency_score: str=Field(description="A string value, could be `Poor`, `Fair`, `Good`, or `Excellent`")
+    consistency_assessment: str=Field(description="Your evaluation of whether the documentation is consistent with the code definitions")
+    consistency_development: list[str]=Field(description="A list of inconsistent function/class/method name and inconsistent docstring")
+    consistency_strengths: list[str]=Field(description="A list of strengths of the documentation on consistency")
+
+
+class ConsistencyObserveStep(PEOCommonStep):
+    def __init__(self, llm: BaseChatOpenAI):
+        super().__init__(llm)
+        self.step_name = "Consistency Observe Step"
+
+    def _prepare_system_prompt(self, state: ConsistencyEvaluationState):
+        all_query_rows = state["all_query_rows"]
+        documentation = state["documentation"]
+        domain = state["domain"]
+        code_definition = ""
+        for row in all_query_rows:
+            content = f"name: {row['name']}\nfile_path: {row['path']}\nparent: {row['parent']}\nparameters: {row['params']}\ndoc_string: {row['doc_string']}"
+            code_definition += content
+            code_definition += "\n\n\n"
+        return ChatPromptTemplate.from_template(CONSISTENCY_OBSERVE_SYSTEM_PROMPT).format(
+            code_definitions=code_definition,
+            documentation=documentation,
+            domain=domain,
+        )
+
+    def _execute_directly(self, state: ConsistencyEvaluationState):
+        system_prompt = self._prepare_system_prompt(state)
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin the consistency evaluation step.",
+            schema=ConsistencyEvaluationObserveResult,
+        )
+        res: ConsistencyEvaluationObserveResult = res
+        state["consistency_score"] = res.consistency_score
+        state["consistency_assessment"] = res.consistency_assessment
+        state["consistency_development"] = res.consistency_development
+        state["consistency_strengths"] = res.consistency_strengths
+        return state, token_usage
+
+

bioguider/agents/consistency_query_step.py

@@ -0,0 +1,74 @@
+
+
+from bioguider.agents.common_step import CommonStep
+from bioguider.agents.consistency_evaluation_task_utils import ConsistencyEvaluationState
+from bioguider.database.code_structure_db import CodeStructureDb
+from bioguider.utils.constants import DEFAULT_TOKEN_USAGE
+
+
+class ConsistencyQueryStep(CommonStep):
+    def __init__(self, code_structure_db: CodeStructureDb):
+        super().__init__()
+        self.step_name = "Consistency Query Step"
+        self.code_structure_db = code_structure_db
+
+    def _execute_directly(self, state: ConsistencyEvaluationState):
+        functions_and_classes = state["functions_and_classes"]
+        all_rows: list[any] = []
+        for function_or_class in functions_and_classes:
+            function_or_class_name = function_or_class["name"]
+            function_or_class_file_path = function_or_class["file_path"]
+            function_or_class_parameters = function_or_class["parameters"]
+            function_or_class_parent = function_or_class["parent"]
+            self._print_step(state, step_output=(
+                f"Consistency Query Step: \n{function_or_class_name},\n"
+                f" {function_or_class_file_path},\n"
+                f" {function_or_class_parameters},\n"
+                f" {function_or_class_parent}"
+            ))
+            file_path = None
+            parent = None
+            name = None
+            if "file_path" in function_or_class and function_or_class["file_path"] != "N/A":
+                file_path = function_or_class["file_path"]            
+            if "parent" in function_or_class and function_or_class["parent"] != "N/A":
+                parent = function_or_class["parent"]
+            if "name" in function_or_class and function_or_class["name"] != "N/A":
+                name = function_or_class["name"]
+            
+            rows: list[any] | None = None
+            if name is None:
+                if file_path is not None:
+                    rows = self.code_structure_db.select_by_path(file_path)
+                elif parent is not None:
+                    rows = self.code_structure_db.select_by_parent(parent)
+            else:
+                if file_path is not None and parent is not None:
+                    rows = self.code_structure_db.select_by_name_and_parent_and_path(name, parent, file_path)
+                    if rows is None or len(rows) == 0:
+                        rows = self.code_structure_db.select_by_name_and_path(name, file_path)
+                    if rows is None or len(rows) == 0:
+                        rows = self.code_structure_db.select_by_name_and_parent(name, parent)
+                    if rows is None or len(rows) == 0:
+                        rows = self.code_structure_db.select_by_name(name)
+                elif file_path is not None:
+                    rows = self.code_structure_db.select_by_name_and_path(name, file_path)
+                    if rows is None or len(rows) == 0:
+                        rows = self.code_structure_db.select_by_name(name)
+                elif parent is not None:
+                    rows = self.code_structure_db.select_by_name_and_parent(name, parent)
+                    if rows is None or len(rows) == 0:
+                        rows = self.code_structure_db.select_by_name(name)
+                else:
+                    rows = self.code_structure_db.select_by_name(name)
+            if rows is None or len(rows) == 0:
+                self._print_step(state, step_output=f"No such function or class {name}")
+                continue
+            all_rows.extend(rows)
+
+        state["all_query_rows"] = all_rows
+
+        return state, {**DEFAULT_TOKEN_USAGE}
+           
+            
+

bioguider/agents/evaluation_task.py

@@ -204,113 +204,3 @@ class EvaluationTask(ABC):
     @abstractmethod
     def _collect_files(self) -> list[str]:
         pass
-
-       
-EVALUATION_TUTORIAL_SYSTEM_PROMPT="""
-You are an expert in software documentation and developer education.
-You are given the content of a tutorial file from a GitHub repository. Your task is to **critically evaluate** the quality of this tutorial based on best practices in technical writing and developer onboarding.
-Please assess the tutorial using the following criteria. Provide your evaluation in structured sections:
-
----
-
-### **Evaluation Criteria:**
-1. **Readability**: You are provided the following metrics scores calculated with pyphen, please evaluate readability based on the scores:
-   * Flesch Reading Ease: {flesch_reading_ease} (206.835 - 1.015(words/sentences) - 84.6(syllables/words))
-   * Flesch-Kincaid Grade Level: {flesch_kincaid_grade} (0.39(words/sentences) + 11.8(syllables/words) - 15.59)
-   * Gunning Fog Index: {gunning_fog_index} (0.4[(words/sentences) + 100(complex words/words)])
-   * SMOG Index: {smog_index} (1.043*sqrt(polysyllables * (30/sentences)) + 3.1291)
-2. **Coverage**
-   * Does the tutorial cover all major steps needed to get started?
-   * Are dependencies, prerequisites, setup steps, and example usage included?
-3. **Structure & Organization**
-   * Is the content logically structured (e.g., introduction → setup → examples → summary)?
-   * Are sections well-labeled and easy to navigate?
-4. **Balance Between Code and Explanation**
-   * Is there a good balance between code snippets and narrative explanation?
-   * Are code blocks properly annotated or explained?
-5. **Terminology Consistency**
-   * Is technical terminology used consistently and accurately?
-   * Are key terms introduced and reused correctly?
-6. **Example Quality**
-   * Are the examples relevant, correct, and representative of real usage?
-   * Are edge cases or typical user pitfalls addressed?
-7. **Formatting and Style**
-   * Are headings, bullet points, code formatting, and markdown style used effectively?
-   * Are there any formatting issues that hurt clarity?
----
-
-### **Output Format:**
-Please respond in the following format:
-
-```
-**FinalAnswer**
-**Readability**: Your comments here  
-**Coverage**: Your comments here  
-**Structure & Organization**: Your comments here  
-**Code vs. Explanation Balance**: Your comments here  
-**Terminology Consistency**: Your comments here  
-**Example Quality**: Your comments here  
-**Formatting and Style**: Your comments here  
-**Overall Rating**: [Poor / Fair / Good / Excellent]  
-```
-
----
-
-### **Tutorial File Content:**
-
-```
-{tutorial_file_content}
-```
-
----
-"""
-class EvaluationTutorialTask(EvaluationTask):
-    def __init__(
-        self, 
-        llm: BaseChatOpenAI, 
-        repo_path: str, 
-        gitignore_path: str,
-        meta_data: ProjectMetadata | None = None,
-        step_callback: Callable | None = None,
-        summarized_files_db = None,
-    ):
-        super().__init__(llm, repo_path, gitignore_path, meta_data, step_callback, summarized_files_db)
-        self.evaluation_name = "Tutorial Evaluation"
-
-    def _evaluate(self, files: list[str]) -> tuple[dict, dict]:
-        if len(files) == 0:
-            return {}, {**DEFAULT_TOKEN_USAGE}
-        
-        evaluations = {}
-        for file in files:
-            tutorial_path = Path(self.repo_path, file)
-            tutorial_content = read_file(tutorial_path)
-            if tutorial_content is None:
-                logging.error(f"Error in reading file {file}")
-                continue
-
-            readability = PyphenReadability()
-            flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, smog_index, \
-                _, _, _, _, _ = readability.readability_metrics(tutorial_content)
-            system_prompt = ChatPromptTemplate.from_template(
-                EVALUATION_TUTORIAL_SYSTEM_PROMPT
-            ).format(
-                tutorial_file_content=tutorial_content,
-                flesch_reading_ease=flesch_reading_ease,
-                flesch_kincaid_grade=flesch_kincaid_grade,
-                gunning_fog_index=gunning_fog_index,
-                smog_index=smog_index,
-            )
-            conversation = CommonConversation(llm=self.llm)
-            response, token_usage = conversation.generate(
-                system_prompt=system_prompt,
-                instruction_prompt=EVALUATION_INSTRUCTION,
-            )
-            self.print_step(step_output=f"Tutorial: {file}")
-            self.print_step(step_output=response)
-            evaluations[file] = response
-        return evaluations, token_usage
-    
-    def _collect_files(self):
-        return []
-