bioguider 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl

bioguider/agents/consistency_collection_task_utils.py

@@ -0,0 +1,137 @@
+import os
+from pathlib import Path
+from typing import Callable, Optional, TypedDict
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from langchain_core.messages import AIMessage
+from pydantic import BaseModel, Field
+import logging
+
+from bioguider.agents.agent_tools import agent_tool
+from bioguider.database.code_structure_db import CodeStructureDb
+
+logger = logging.getLogger(__name__)
+
+class ConsistencyCollectionWorkflowState(TypedDict):
+    user_guide_api_documentation: str
+    step_output_callback: Optional[Callable]
+    intermediate_steps: Optional[str]
+    step_output: Optional[str]
+    step_analysis: Optional[str]
+    step_thoughts: Optional[str]
+    plan_actions: Optional[list[dict]]
+
+    final_answer: Optional[str]
+    final_assembly_result: Optional[str]
+    step_count: Optional[int]
+
+class retrieve_method_definition_and_docstring_tool:
+    """ Retrieve the method definition and docstring.
+    If the method is a method of a class, you **must** put the class name as the parent name and better to put the file path as the file path of the class.
+Args:
+    method_name str: the name of the method
+    class_name str: the name of the class that the method is in.
+    file_path str: the path of the file that the method is in. If not sure, just put "N/A"
+Returns:
+    str: the method definition and docstring
+    """
+    def __init__(self, llm: BaseChatOpenAI, code_structure_db: CodeStructureDb):
+        self.llm = llm
+        self.code_structure_db = code_structure_db
+
+    def run(self, method_name: str, class_name: str, file_path: str) -> str:
+        if file_path != "N/A":
+            row = self.code_structure_db.select_by_name_and_parent_and_path(method_name, class_name, file_path)
+            if row is None:
+                return "Can't retrieve method definition and docstring"
+            return f"Method: {row['name']}\nDocstring: {row['doc_string']}\nParams: {row['params']}"
+        else:
+            rows = self.code_structure_db.select_by_name_and_parent(method_name, class_name)
+            if rows is None or len(rows) == 0:
+                return "Can't retrieve method definition and docstring"
+            return f"Method: {rows[0]['name']}\nDocstring: {rows[0]['doc_string']}\nParams: {rows[0]['params']}"
+
+class retrieve_function_definition_and_docstring_tool:
+    """ Retrieve the function definition and docstring
+Args:
+    function_name str: the name of the function
+    file_path str: the path of the file that the function is in. If not sure, just put "N/A"
+Returns:
+    str: the function definition and docstring
+    """
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI,
+        code_structure_db: CodeStructureDb,
+    ):
+        self.llm = llm
+        self.code_structure_db = code_structure_db
+
+    def run(self, function_name: str, file_path: str) -> str:
+        if file_path != "N/A":
+            row = self.code_structure_db.select_by_name_and_path(function_name, file_path)
+            if row is None:
+                return f"No such function {function_name}"
+            return f"Function: {row['name']}\nDocstring: {row['doc_string']}\nParams: {row['params']}"
+        else:
+            rows = self.code_structure_db.select_by_name(function_name)
+            if rows is None or len(rows) == 0:
+                return f"No such function {function_name}"
+            return f"Function: {rows[0]['name']}\nDocstring: {rows[0]['doc_string']}\nParams: {rows[0]['params']}"
+
+class retrieve_class_definition_and_docstring_tool:
+    """ Retrieve the class definition and docstring
+Args:
+    class_name str: the name of the class
+    file_path str: the path of the file that the class is in. If not sure, just put "N/A"
+Returns:
+    str: the class definition and docstring
+    """
+    def __init__(self, llm: BaseChatOpenAI, code_structure_db: CodeStructureDb):
+        self.llm = llm
+        self.code_structure_db = code_structure_db
+
+    def run(self, class_name: str, file_path: str) -> str:
+        if file_path != "N/A":
+            row = self.code_structure_db.select_by_name_and_path(class_name, file_path)
+            if row is None:
+                return f"No such class {class_name}"
+            return f"Class: {row['name']}\nDocstring: {row['doc_string']}\nParams: {row['params']}"
+        else:
+            rows = self.code_structure_db.select_by_name(class_name)
+            if rows is None or len(rows) == 0:
+                return f"No such class {class_name}"
+            return f"Class: {rows[0]['name']}\nDocstring: {rows[0]['doc_string']}\nParams: {rows[0]['params']}"
+
+class retrieve_class_and_method_definition_and_docstring_tool:
+    """ Retrieve the class and all methods definition and docstring
+Args:
+    class_name str: the name of the class
+    file_path str: the path of the file that the class is in. If not sure, just put "N/A"
+Returns:
+    str: the class and method definition and docstring
+    """
+    def __init__(self, llm: BaseChatOpenAI, code_structure_db: CodeStructureDb):
+        self.llm = llm
+        self.code_structure_db = code_structure_db
+
+    def run(self, class_name: str, file_path: str) -> str:
+        if file_path != "N/A":
+            row = self.code_structure_db.select_by_name_and_path(class_name, file_path)
+            if row is None:
+                return f"No such class {class_name}"
+        else:
+            rows = self.code_structure_db.select_by_name(class_name)
+            if rows is None or len(rows) == 0:
+                return f"No such class {class_name}"
+            row = rows[0]
+        
+        parent_path = file_path if file_path is not None and file_path.lower() != "n/a" else row["path"]
+        methods = self.code_structure_db.select_by_parent(
+            class_name, 
+            parent_path
+        )
+        method_definitions = []
+        for method in methods:
+            method_definitions.append(f"Method: {method['name']}\nDocstring: {method['doc_string']}\nParams: {method['params']}\n\n")
+        return f"Class: {row['name']}\nDocstring: {row['doc_string']}\nParams: {row['params']}\nMethods: {method_definitions}"

bioguider/agents/evaluation_readme_task.py

@@ -13,7 +13,10 @@ from bioguider.agents.agent_utils import (
     read_file, read_license_file, 
     summarize_file
 )
-from bioguider.agents.common_agent_2step import CommonAgentTwoChainSteps
+from bioguider.agents.common_agent_2step import (
+    CommonAgentTwoChainSteps, 
+    CommonAgentTwoSteps,
+)
 from bioguider.agents.evaluation_task import EvaluationTask
 from bioguider.utils.constants import (
     DEFAULT_TOKEN_USAGE, 
@@ -168,6 +171,28 @@ You will be given:
 
 ---
 
+### **Output Format**
+Your output must **exactly match** the following format. Do not add or omit any sections. 
+
+**FinalAnswer**
+**Available:**
+  <Your assessment and suggestion here>
+**Readability:** 
+  <Your assessment and suggestion here>
+**Project Purpose:** 
+  <Your assessment and suggestion here>
+**Hardware and software spec and compatibility description:**
+  <Your assessment and suggestion here>
+**Dependencies clearly stated:** 
+  <Your assessment and suggestion here>
+**License Information Included:** 
+  <Your assessment and suggestion here>
+**Code contributor / Author information included
+  <Your assessment and suggestion here>
+**Overall Score:**
+  <Your assessment and suggestion here>
+---
+
 ### **Instructions**
 1. Based on the provided structured evaluation and its reasoning process, generate a free evaluation of the README file.
 2. Focus on the explanation of assessment in structured evaluation and how to improve the README file based on the structured evaluation and its reasoning process.
@@ -175,8 +200,9 @@ You will be given:
 3. For each item in the structured evaluation, provide a detailed assessment followed by specific, actionable comments for improvement.
 4. Your improvement suggestions must also include the original text snippet and the improving comments.
 5. Your improvement suggestions must also include suggestions to improve readability.
-6. In each section output, please first give a detailed explanation of the assessment, and then provide the detailed suggestion for improvement. If you think the it is good enough, you can say so.
+6. In the **FinalAnswer** of output, in each section output, please first give a detailed explanation of the assessment, and then provide the detailed suggestion for improvement. If you think the it is good enough, you can say so.
   The following is an example of the output format:
+  **FinalAnswer**
   **Available:**
     Detailed explanation of the assessment. Such as: The README file is present in the repository. The content of the file has been shared completely and is accessible. This confirms the availability of the README documentation for evaluation. There's no issue with availability.
     Detailed suggestion for improvement. Such as: Add a brief introductory section summarizing the project and its main purpose would help orient readers.
@@ -222,28 +248,7 @@ You will be given:
     - <original text snippet> - <improving comments>
     - ...
     - Break down long instructions into smaller bullet points.
----
-
-### **Output Format**
-Your output must **exactly match** the following format. Do not add or omit any sections. 
 
-**FinalAnswer**
-**Available:**
-  <Your detailed assessment and detailed suggestion here, including the original text snippet and the improving comments>
-**Readability:** 
-  <Your detailed assessment and detailed suggestion here, including the original text snippet and the improving comments>
-**Project Purpose:** 
-  <Your detailed assessment and detailed suggestion here, including the original text snippet and the improving comments>
-**Hardware and software spec and compatibility description:**
-  <Your detailed assessment and detailed suggestion here, including the original text snippet and the improving comments>
-**Dependencies clearly stated:** 
-  <Your detailed assessment and detailed suggestion here, including the original text snippet and the improving comments>
-**License Information Included:** 
-  <Your detailed assessment and detailed suggestion here, including the original text snippet and the improving comments>
-**Code contributor / Author information included
-  <Your detailed assessment and detailed suggestion here, including the original text snippet and the improving comments>
-**Overall Score:**
-  <Your detailed assessment and detailed suggestion here, including the original text snippet and the improving comments>
 ---
 
 ### **Structured Evaluation and Reasoning Process**
@@ -492,7 +497,7 @@ class EvaluationREADMETask(EvaluationTask):
             readme_content=readme_content,
             structured_evaluation=structured_reasoning_process,
         )
-        agent = CommonAgentTwoChainSteps(llm=self.llm)
+        agent = CommonAgentTwoSteps(llm=self.llm)
         response, _, token_usage, reasoning_process = agent.go(
             system_prompt=system_prompt,
             instruction_prompt=EVALUATION_INSTRUCTION,

bioguider/agents/evaluation_task.py

@@ -184,7 +184,7 @@ class EvaluationTask(ABC):
             token_usage=token_usage,
         )
 
-    def evaluate(self) -> dict:
+    def evaluate(self) -> tuple[dict, list[str]]:
         self._enter_evaluation()
         files = self._collect_files()
         evaluations, token_usage, files = self._evaluate(files)
@@ -198,7 +198,7 @@ class EvaluationTask(ABC):
         self.print_step(token_usage=token_usage)
 
     @abstractmethod
-    def _evaluate(self, files: list[str]) -> tuple[dict, dict]:
+    def _evaluate(self, files: list[str]) -> tuple[dict, dict, list[str]]:
         pass
 
     @abstractmethod

bioguider/agents/evaluation_userguide_prompts.py

@@ -0,0 +1,162 @@
+
+INDIVIDUAL_USERGUIDE_EVALUATION_SYSTEM_PROMPT = """
+You are an expert in evaluating the quality of user guide in software repositories. 
+Your task is to analyze the provided files related to user guide and generate a structured quality assessment based on the following criteria.
+---
+
+### **Evaluation Criteria**
+
+1. **Readability**:
+   * **Flesch Reading Ease**: `{flesch_reading_ease}` (A higher score is better, with 60-70 being easily understood by most adults).
+   * **Flesch-Kincaid Grade Level**: `{flesch_kincaid_grade}` (Represents the US school-grade level needed to understand the text).
+   * **Gunning Fog Index**: `{gunning_fog_index}` (A score above 12 is generally considered too hard for most people).
+   * **SMOG Index**: `{smog_index}` (Estimates the years of education needed to understand the text).
+   * **Assessment**: Based on these scores, evaluate the overall readability and technical complexity of the language used.
+
+2. **Arguments and Clarity**:
+   * **Assessment**: [Your evaluation of whether it provides a clear **description** of arguments and their usage]
+   * **Improvement Suggestions**:
+      * **Original text:** [Quote a specific line/section from the user guide.]
+      * **Improving comments:** [Provide your suggestions to improve clarity.]
+
+3. **Return Value and Clarity**:
+   * **Assessment**: [Your evaluation of whether it provides a clear **description** of return value and its meaning]
+   * **Improvement Suggestions**:
+      * **Original text:** [Quote a specific line/section from the user guide.]
+      * **Improving comments:** [Provide your suggestions to improve clarity.]
+
+4. **Context and Purpose**:
+   * **Assessment**: [Your evaluation of whether it provides a clear **description** of the context and purpose of the module]
+   * **Improvement Suggestions**:
+      * **Original text:** [Quote a specific line/section from the user guide.]
+      * **Improving comments:** [Provide your suggestions to improve clarity.]
+
+5. **Error Handling**:
+   * **Assessment**: [Your evaluation of whether it provides a clear **description** of error handling]
+   * **Improvement Suggestions**:
+      * **Original text:** [Quote a specific line/section from the user guide.]
+      * **Improving comments:** [Provide your suggestions to improve clarity.]
+
+6. **Usage Examples**:
+   * **Assessment**: [Your evaluation of whether it provides a clear **description** of usage examples]
+   * **Improvement Suggestions**:
+      * **Original text:** [Quote a specific line/section from the user guide.]
+      * **Improving comments:** [Provide your suggestions to improve clarity.]
+
+7. **Overall Score**: Give an overall quality rating of the User Guide information.
+   * Output: `Poor`, `Fair`, `Good`, or `Excellent`
+
+---
+
+### **Final Report Ouput**
+Your final report must **exactly match** the following format. Do not add or omit any sections.
+
+**FinalAnswer**
+* **Overall Score:** [Poor / Fair / Good / Excellent]
+* **Overall Key Strengths**: <brief summary of the User Guide's strongest points in 2-3 sentences> 
+* **Overall Improvement Suggestions:**
+  - "Original text snippet 1" - Improving comment 1  
+  - "Original text snippet 2" - Improving comment 2  
+  - ...
+* **Readability Analysis Score:** [Poor / Fair / Good / Excellent]
+* **Readability Analysis Key Strengths**: <brief summary of the User Guide's strongest points in 2-3 sentences> 
+* **Readability Analysis Improvement Suggestions:**
+  - "Original text snippet 1" - Improving comment 1  
+  - "Original text snippet 2" - Improving comment 2  
+  - ...
+* **Arguments and Clarity Score:** [Poor / Fair / Good / Excellent]
+* **Arguments and Clarity Key Strengths**: <brief summary of the User Guide's strongest points in 2-3 sentences> 
+* **Arguments and Clarity Improvement Suggestions:**
+  - "Original text snippet 1" - Improving comment 1  
+  - "Original text snippet 2" - Improving comment 2  
+  - ...
+* **Return Value and Clarity Score:** [Poor / Fair / Good / Excellent]
+* **Return Value and Clarity Key Strengths**: <brief summary of the User Guide's strongest points in 2-3 sentences> 
+* **Return Value and Clarity Improvement Suggestions:**
+  - "Original text snippet 1" - Improving comment 1  
+  - "Original text snippet 2" - Improving comment 2  
+  - ...
+* **Context and Purpose Score:** [Poor / Fair / Good / Excellent]
+* **Context and Purpose Key Strengths**: <brief summary of the User Guide's strongest points in 2-3 sentences> 
+* **Context and Purpose Improvement Suggestions:**
+  - "Original text snippet 1" - Improving comment 1  
+  - "Original text snippet 2" - Improving comment 2  
+  - ...
+* **Error Handling Score:** [Poor / Fair / Good / Excellent]
+* **Error Handling Key Strengths**: <brief summary of the User Guide's strongest points in 2-3 sentences> 
+* **Error Handling Improvement Suggestions:**
+  - "Original text snippet 1" - Improving comment 1  
+  - "Original text snippet 2" - Improving comment 2  
+  - ...
+* **Usage Examples Score:** [Poor / Fair / Good / Excellent]
+* **Usage Examples Key Strengths**: <brief summary of the User Guide's strongest points in 2-3 sentences> 
+* **Usage Examples Improvement Suggestions:**
+  - "Original text snippet 1" - Improving comment 1  
+  - "Original text snippet 2" - Improving comment 2  
+  - ...
+...
+
+---
+
+### **User Guide Content:**
+{userguide_content}
+
+---
+
+"""
+
+CONSISTENCY_EVAL_SYSTEM_PROMPT = """
+You are an expert in evaluating the consistency of user guide in software repositories.
+Your task is to analyze both:
+1. the provided file related to user guide/API documentation,
+2. the code definitions related to the user guide/API documentation
+and generate a structured consistency assessment based on the following criteria.
+
+---
+
+### **Evaluation Criteria**
+
+**Consistency**:
+  * **Score**: [Poor / Fair / Good / Excellent]
+  * **Assessment**: [Your evaluation of whether the user guide/API documentation is consistent with the code definitions]
+  * **Development**: [A list of inconsistent function/class/method name and inconsistent docstring]
+  * **Strengths**: [A list of strengths of the user guide/API documentation on consistency]
+
+### **Output Format**
+Your output **must exactly match** the following format:
+```
+**Consistency**:
+  * **Score**: [Poor / Fair / Good / Excellent]
+  * **Assessment**: [Your evaluation of whether the user guide/API documentation is consistent with the code definitions]
+  * **Development**: [A list of inconsistent function/class/method name and inconsistent docstring]
+  * **Strengths**: [A list of strengths of the user guide/API documentation on consistency]
+```
+
+### **Output Example**
+
+```
+**Consistency**:
+  * **Assessment**: [Your evaluation of whether the user guide/API documentation is consistent with the code definitions]
+  * **Development**:
+    - Inconsistent function/class/method name 1
+    - Inconsistent docstring 1
+    - Inconsistent function/class/method name 2
+    - Inconsistent docstring 2
+    - ...
+  * **Strengths**: 
+    - Strengths 1
+    - Strengths 2
+    - ...
+```
+
+---
+
+### **Input User Guide/API Documentation**
+{user_guide_api_documentation}
+
+### **Code Definitions**
+{code_definitions}
+
+---
+
+"""

bioguider/agents/evaluation_userguide_task.py

@@ -0,0 +1,164 @@
+
+import os
+from pathlib import Path
+import logging
+from langchain.prompts import ChatPromptTemplate
+from markdownify import markdownify as md
+from pydantic import BaseModel, Field
+
+from bioguider.agents.agent_utils import read_file
+from bioguider.agents.collection_task import CollectionTask
+from bioguider.agents.prompt_utils import EVALUATION_INSTRUCTION, CollectionGoalItemEnum
+from bioguider.utils.constants import (
+    DEFAULT_TOKEN_USAGE, 
+    ProjectMetadata,
+    StructuredEvaluationInstallationResult,
+    FreeEvaluationInstallationResult,
+    EvaluationInstallationResult,
+)
+from bioguider.rag.data_pipeline import count_tokens
+from .common_agent_2step import CommonAgentTwoSteps, CommonAgentTwoChainSteps
+from ..utils.pyphen_utils import PyphenReadability
+
+from .evaluation_task import EvaluationTask
+from .agent_utils import read_file
+from bioguider.utils.utils import increase_token_usage
+from .evaluation_userguide_prompts import CONSISTENCY_EVAL_SYSTEM_PROMPT, INDIVIDUAL_USERGUIDE_EVALUATION_SYSTEM_PROMPT
+from .consistency_collection_task import ConsistencyCollectionTask
+
+class ConsistencyEvaluationResult(BaseModel):
+    consistency_score: str=Field(description="A string value, could be `Poor`, `Fair`, `Good`, or `Excellent`")
+    consistency_assessment: str=Field(description="Your evaluation of whether the user guide/API documentation is consistent with the code definitions")
+    consistency_development: list[str]=Field(description="A list of inconsistent function/class/method name and inconsistent docstring")
+    consistency_strengths: list[str]=Field(description="A list of strengths of the user guide/API documentation on consistency")
+
+class UserGuideEvaluationResult(BaseModel):
+    overall_score: str=Field(description="A string value, could be `Poor`, `Fair`, `Good`, or `Excellent`")
+    overall_key_strengths: str=Field(description="A string value, the key strengths of the user guide")
+    overall_improvement_suggestions: str=Field(description="Suggestions to improve the overall score if necessary")
+    readability_score: str=Field(description="A string value, could be `Poor`, `Fair`, `Good`, or `Excellent`")
+    readability_suggestions: str=Field(description="Suggestions to improve readability if necessary")
+    context_and_purpose_score: str=Field(description="A string value, could be `Poor`, `Fair`, `Good`, or `Excellent`")
+    context_and_purpose_suggestions: str=Field(description="Suggestions to improve context and purpose if necessary")
+    error_handling_score: str=Field(description="A string value, could be `Poor`, `Fair`, `Good`, or `Excellent`")
+    error_handling_suggestions: str=Field(description="Suggestions to improve error handling if necessary")
+
+class IndividualUserGuideEvaluationResult(BaseModel):
+    user_guide_evaluation: UserGuideEvaluationResult | None=Field(description="The evaluation result of the user guide")
+    consistency_evaluation: ConsistencyEvaluationResult | None=Field(description="The evaluation result of the consistency of the user guide")
+
+logger = logging.getLogger(__name__)
+
+class EvaluationUserGuideTask(EvaluationTask):
+    def __init__(
+        self, 
+        llm, 
+        repo_path, 
+        gitignore_path, 
+        meta_data = None, 
+        step_callback = None,
+        summarized_files_db = None,
+        code_structure_db = None,
+    ):
+        super().__init__(llm, repo_path, gitignore_path, meta_data, step_callback, summarized_files_db)
+        self.evaluation_name = "User Guide Evaluation"
+        self.code_structure_db = code_structure_db
+
+    def _collect_files(self):
+        task = CollectionTask(
+            llm=self.llm,
+            step_callback=self.step_callback,
+            summarized_files_db=self.summarized_files_db,
+        )
+        task.compile(
+            repo_path=self.repo_path,
+            gitignore_path=Path(self.repo_path, ".gitignore"),
+            goal_item=CollectionGoalItemEnum.UserGuide.name,
+        )
+        files = task.collect()
+        return files
+
+    def _evaluate_consistency(self, file: str) -> tuple[EvaluationInstallationResult | None, dict, list[str]]:
+        consistency_collect_task = ConsistencyCollectionTask(
+            llm=self.llm,
+            code_structure_db=self.code_structure_db,
+            step_callback=self.step_callback,
+        )
+        consistency_collect_task.compile(repo_path=self.repo_path, gitignore_path=Path(self.repo_path, ".gitignore"))
+        with open(Path(self.repo_path, file), "r") as f:
+            user_guide_api_documentation = f.read()
+        res, code_definitions = consistency_collect_task.collect(user_guide_api_documentation)
+
+        if not res:
+            # No sufficient information to evaluate the consistency of the user guide/API documentation
+            return None, {**DEFAULT_TOKEN_USAGE}
+
+        system_prompt = ChatPromptTemplate.from_template(
+            CONSISTENCY_EVAL_SYSTEM_PROMPT
+        ).format(
+            user_guide_api_documentation=user_guide_api_documentation,
+            code_definitions=code_definitions,
+        )
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin the consistency evaluation step.",
+            schema=ConsistencyEvaluationResult,
+        )
+        res: ConsistencyEvaluationResult = res
+        self.print_step(step_output=f"Consistency Evaluation Result: {res}")
+        self.print_step(step_output=f"Consistency Evaluation Reasoning Process: {reasoning_process}")
+        self.print_step(token_usage=token_usage)
+
+        return res, token_usage
+
+    def _evaluate_individual_userguide(self, file: str) -> tuple[IndividualUserGuideEvaluationResult | None, dict]:
+        content = read_file(Path(self.repo_path, file))
+        
+        if content is None:
+            logger.error(f"Error in reading file {file}")
+            return None, {**DEFAULT_TOKEN_USAGE}
+
+        readability = PyphenReadability()
+        flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, smog_index, \
+                _, _, _, _, _ = readability.readability_metrics(content)
+        system_prompt = ChatPromptTemplate.from_template(
+            INDIVIDUAL_USERGUIDE_EVALUATION_SYSTEM_PROMPT
+        ).format(
+            flesch_reading_ease=flesch_reading_ease,
+            flesch_kincaid_grade=flesch_kincaid_grade,
+            gunning_fog_index=gunning_fog_index,
+            smog_index=smog_index,
+            userguide_content=content,
+        )
+        agent = CommonAgentTwoSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin the user guide/API documentation evaluation.",
+            schema=UserGuideEvaluationResult,
+        )
+        res: UserGuideEvaluationResult = res
+
+        consistency_evaluation_result, _temp_token_usage = self._evaluate_consistency(file)
+        if consistency_evaluation_result is None:
+            # No sufficient information to evaluate the consistency of the user guide/API documentation
+            consistency_evaluation_result = ConsistencyEvaluationResult(
+                consistency_score="N/A",
+                consistency_assessment="No sufficient information to evaluate the consistency of the user guide/API documentation",
+                consistency_development=[],
+                consistency_strengths=[],
+            )
+        return IndividualUserGuideEvaluationResult(
+            user_guide_evaluation=res,
+            consistency_evaluation=consistency_evaluation_result,
+        ), token_usage
+
+    def _evaluate(self, files: list[str] | None = None) -> tuple[dict[str, IndividualUserGuideEvaluationResult] | None, dict, list[str]]:
+        total_token_usage = {**DEFAULT_TOKEN_USAGE}
+        user_guide_evaluation_results = {}
+        for file in files:
+            user_guide_evaluation_result, token_usage = self._evaluate_individual_userguide(file)
+            total_token_usage = increase_token_usage(total_token_usage, token_usage)
+            user_guide_evaluation_results[file] = user_guide_evaluation_result
+
+        return user_guide_evaluation_results, total_token_usage, files

bioguider/agents/prompt_utils.py

@@ -104,19 +104,22 @@ COLLECTION_PROMPTS = {
         "goal_item": "User Guide",
         "related_file_description": """A document qualifies as a **User Guide** if it includes **at least one** of the following elements.
 If **any one** of these is present, the document should be classified as a User Guide — full coverage is **not required**:
- - Overview: A brief introduction to the software, its purpose, and its intended audience.
- - Installation Instructions: Step-by-step setup procedures.
- - Input/Output Specifications: Detailed information on the data the software accepts and produces.
- - Configuration Options: Descriptions of settings and parameters that can be adjusted.
- - Function/Interface Listings: Comprehensive lists of available functions or interfaces, including their descriptions, parameters, and return values.
- - Mathematical Equations/Numerical Methods: Embedded documentation explaining the underlying mathematical concepts or algorithms.
- - Developer Guidance: Instructions on how to extend the software or contribute to its development.
- **Do not** classify the document as a User Guide if it primarily serves as a Tutorial or Example. Such documents typically include:
+ - Document **functions, methods, or classes**
+ - Describe **input parameters, return values**, and **usage syntax**
+ - Include **technical guidance** for using specific APIs
+ - Are often found in folders such as
+   * `man/` (for `.Rd` files in R)
+   * `docs/reference/`, `docs/api/`, `docs/dev/` (for Python) or similar
+   * Standalone files with names like `api.md`, `reference.md`, `user_guide.md`
+**Do not** classify the document as a User Guide if it primarily serves as a Tutorial or Example. Such documents typically include:
  - Sample Datasets: Example data used to illustrate functionality.
  - Narrative Explanations: Story-like descriptions guiding the user through examples.
  - Code Walkthroughs: Detailed explanations of code snippets in a tutorial format.
 **Do not** classify the document as a User Guide if it is souce code or a script (*.py, *.R) that is not intended for end-user interaction.
  - You can include directory names if all files in the directory are relevant to the goal item.""",
+        "plan_important_instructions": """ - **Do not** classify the document as a User Guide if it is source code or a script (*.py, *.R) that is not intended for end-user interaction.
+ - **Do not** classify the document as a User Guide if it is a notebook (*.ipynb, *.Rmd) that is not intended for end-user interaction.
+ - You plan **must not** include any source code or script (*.py, *.R) or notebook (*.ipynb, *.Rmd) that is not intended for end-user interaction."""
     },
     "Tutorial": {
         "goal_item": "Tutorials & Vignettes",