bioguider 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

bioguider/agents/evaluation_installation_task.py

@@ -0,0 +1,151 @@
+import os
+from pathlib import Path
+import logging
+from typing import Callable, Optional
+from abc import ABC, abstractmethod
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from pydantic import BaseModel, Field
+
+from bioguider.agents.agent_utils import read_file
+from bioguider.utils.constants import DEFAULT_TOKEN_USAGE, ProjectMetadata
+from .common_agent_2step import CommonAgentTwoSteps, CommonAgentTwoChainSteps
+from .common_agent import CommonConversation
+from ..utils.pyphen_utils import PyphenReadability
+from ..utils.gitignore_checker import GitignoreChecker
+from .evaluation_task import EvaluationTask
+from .agent_utils import read_file
+
+
+logger = logging.getLogger(__name__)
+
+EVALUATION_INSTALLATION_SYSTEM_PROMPT = """
+You are an expert in evaluating the quality of **installation instructions** in software repositories.
+Your task is to analyze the provided content of installation-related files and generate a **comprehensive, structured quality report**.
+
+---
+
+### Evaluation Criteria
+
+Please assess the installation information using the following criteria. For each, provide a concise evaluation and specific feedback:
+
+1. **Ease of Access**
+   * Is the installation information clearly presented and easy to locate within the repository?
+   * Is it included in a top-level README, a dedicated INSTALL.md file, or other accessible locations?
+
+2. **Clarity of Dependency Specification**
+   * Are all software and library dependencies clearly listed?
+   * Are installation methods (e.g., `pip`, `conda`, `apt`) for those dependencies explicitly provided?
+
+3. **Hardware Requirements**
+   * Does the documentation specify hardware needs (e.g., GPU, memory, OS) if relevant?
+
+4. **Step-by-Step Installation Guide**
+   * Is there a clear, ordered set of instructions for installing the software?
+   * Are example commands or configuration steps provided to help users follow along?
+
+---
+
+### Output Format
+
+Your response **must exactly follow** the structure below:
+
+```
+**FinalAnswer**
+**Overall Score:** [Poor / Fair / Good / Excellent]  
+**Ease of Access:** <your comments>  
+**Clarity of Dependency Specification:** <your comments>  
+**Hardware Requirements:** <your comments>  
+**Installation Guide:** <your comments>  
+```
+
+---
+
+### Installation Files Provided:
+{installation_file_contents}
+
+"""
+
+class EvaluationInstallationResult(BaseModel):
+    ease_of_access: Optional[str]=Field(description="Is the installation information easy to access")
+    score: Optional[str]=Field(description="An overall score, could be Poor, Fair, Good or Excellent")
+    clarity_of_dependency: Optional[str]=Field(description="Are all dependencies clearly listed")
+    hardware_requirements: Optional[str]=Field(description="Are all hardware requirements clearly specified")
+    installation_guide: Optional[str]=Field(description="Is there a clear, ordered set of instructions for installing the software")
+
+EvaluationInstallationResultSchema = {
+    "title": "EvaluationREADMEResult",
+    "type": "object",
+    "properties": {
+        "ease_of_access": {
+            "anyOf": [{"type": "string"}, {"type": "null"}],
+            "description": "Is the installation information easy to access",
+            "title": "Ease of Access"
+        },
+        "score": {
+            "anyOf": [{"type": "string"}, {"type": "null"}],
+            "description": "An overall score, could be Poor, Fair, Good or Excellent",
+            "title": "Score"
+        },
+        "clarity_of_dependency": {
+            "anyOf": [{"type": "string"}, {"type": "null"}],
+            "description": "Are all dependencies clearly listed",
+            "title": "Clarity of Dependency",
+        },
+        "hardware_requirements": {
+            "anyOf": [{"type": "string"}, {"type": "null"}],
+            "description": "Are all hardware requirements clearly specified",
+            "title": "Hardware Requirements"
+        },
+        "installation_guide": {
+            "anyOf": [{"type": "string"}, {"type": "null"}],
+            "description": "Is there a clear, ordered set of instructions for installing the software",
+            "title": "Installation Guide"
+        }
+    },
+    "required": ["ease_of_access", "score", "clarity_of_dependency", "hardware_requirements", "installation_guide"]
+}
+
+class EvaluationInstallationTask(EvaluationTask):
+    def __init__(
+        self, 
+        llm, 
+        repo_path,
+        gitignore_path, 
+        meta_data = None, 
+        step_callback = None,
+    ):
+        super().__init__(llm, repo_path, gitignore_path, meta_data, step_callback)
+
+    def _evaluate(self, files: list[str] | None = None):
+        if files is None or len(files) == 0:
+            return None
+        
+        files_content = ""
+        for f in files:
+            content = read_file(os.path.join(self.repo_path, f))
+            files_content += f"""
+{f} content:
+{content}
+
+"""
+        system_prompt = ChatPromptTemplate.from_template(EVALUATION_INSTALLATION_SYSTEM_PROMPT).format(
+            installation_file_contents=files_content
+        )
+        agent = CommonAgentTwoChainSteps(llm=self.llm)
+        res, _, token_usage, reasoning_process = agent.go(
+            system_prompt=system_prompt,
+            instruction_prompt="Before arriving at the conclusion, clearly explain your reasoning step by step. Now, let's begin the evaluation.",
+            schema=EvaluationInstallationResultSchema,
+        )
+        res = EvaluationInstallationResult(**res)
+        evaluation = {
+            "score": res.score,
+            "ease_of_access": res.ease_of_access,
+            "hardware_requirements": res.hardware_requirements,
+            "clarity_of_dependency": res.clarity_of_dependency,
+            "installation_guide": res.installation_guide,
+            "reasoning_process": reasoning_process,
+        }
+        return evaluation, token_usage
+        

bioguider/agents/evaluation_task.py

@@ -71,8 +71,22 @@ For each criterion below, provide a brief assessment followed by specific, actio
  * **Assessment**: Based on these scores, evaluate the overall readability and technical complexity of the language used.
 
 **Final Answer**
- * Project-Level README: Yes / No
- * Provide a final, overall assessment of the README file's quality, summarizing the key strengths and areas for improvement.
+ The final answer **must exactly match** the following format:
+```
+  * Project-Level README: Yes / No
+  * **Score:** <number from 0 to 100>  
+  * **Key Strengths**: <brief summary of the README's strongest points in 2-3 sentences> 
+  * **Overall Improvement Suggestions:**
+    - "Original text snippet 1" - Improving comment 1  
+    - "Original text snippet 2" - Improving comment 2  
+    - ...
+```
+
+  * **Project-Level README**: Indicate “Yes” if the README is project-level, otherwise “No.”
+  * **Score**: Provide an overall quality score (100 = perfect).
+  * **Key Strengths**: Provide the README's strongest points in 2-3 sentences
+  * **Overall Improvement Suggestions**:
+    * List each original text snippet that needs improvement, followed by your suggestion.
 
 ---
 
@@ -108,8 +122,14 @@ For each criterion below, provide a brief assessment followed by specific, actio
  * **Assessment**: Based on these scores, evaluate the overall readability and technical complexity of the language used.
 
 **Final Answer**
+  The final answer **must exactly match** the following format:
  * Project-Level README: Yes / No
- * Provide a final, overall assessment of the README file's quality, summarizing the key strengths and areas for improvement.
+ * **Score:** <number from 0 to 100>  
+  * **Key Strengths**: <brief summary of the README's strongest points in 2-3 sentences> 
+  * **Overall Improvement Suggestions:**
+    - "Original text snippet 1" - Improving comment 1  
+    - "Original text snippet 2" - Improving comment 2  
+    - ...
 ---
 
 ### **README path:**
@@ -167,8 +187,38 @@ class EvaluationTask(ABC):
         pass
 
 class EvaluationREADMEResult(BaseModel):
-    project_level: Optional[bool]=Field(description="a boolean value specifying if the README file is **project-level** README. TRUE: project-level, FALSE, folder-level")
-    overall_assessment: Optional[str]=Field(description="an overall assessment")
+    project_level: Optional[bool]=Field(description="A boolean value specifying if the README file is **project-level** README. TRUE: project-level, FALSE, folder-level")
+    score: Optional[float]=Field(description="An overall score")
+    key_strengths: Optional[str]=Field(description="A string specifying the key strengths of README file.")
+    overall_improvement_suggestions: Optional[list[str]]=Field(description="A list of overall improvement suggestions")
+
+EvaluationREADMEResultSchema = {
+    "title": "EvaluationREADMEResult",
+    "type": "object",
+    "properties": {
+        "project_level": {
+            "anyOf": [{"type": "boolean"}, {"type": "null"}],
+            "description": "A boolean value specifying if the README file is **project-level** README. TRUE: project-level, FALSE: folder-level.",
+            "title": "Project Level"
+        },
+        "score": {
+            "anyOf": [{"type": "number"}, {"type": "null"}],
+            "description": "An overall score",
+            "title": "Score"
+        },
+        "key_strengths": {
+            "anyOf": [{"type": "string"}, {"type": "null"}],
+            "description": "A string specifying the key strengths of README file.",
+            "title": "Key Strengths",
+        },
+        "overall_improvement_suggestions": {
+            "anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}],
+            "description": "A list of improvement suggestions",
+            "title": "Overall Improvement Suggestions"
+        }
+    },
+    "required": ["project_level", "score", "key_strengths", "overall_improvement_suggestions"]
+}
 
 class EvaluationREADMETask(EvaluationTask):
     def __init__(
@@ -195,7 +245,16 @@ class EvaluationREADMETask(EvaluationTask):
                 logger.error(f"Error in reading file {readme_file}")
                 continue
             if len(readme_content.strip()) == 0:
-                readme_content = "empty file"
+                readme_evaluations[readme_file] = {
+                    "evaluation": {
+                        "project_level": "/" in readme_file,
+                        "score": 0,
+                        "key_strengths": f"{readme_file} is an empty file.",
+                        "overall_improvement_suggestions": f"{readme_file} is an empty file.",
+                    },
+                    "reasoning_process": f"{readme_file} is an empty file.",
+                }
+                continue
 
             readability = PyphenReadability()
             flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, smog_index, \
@@ -215,14 +274,17 @@ class EvaluationREADMETask(EvaluationTask):
             response, _, token_usage, reasoning_process = agent.go(
                 system_prompt=system_prompt,
                 instruction_prompt="Before arriving at the conclusion, clearly explain your reasoning step by step. Now, let's begin the evaluation.",
-                schema=EvaluationREADMEResult,
+                schema=EvaluationREADMEResultSchema,
             )
+            response = EvaluationREADMEResult(**response)
             self.print_step(step_output=f"README: {readme_file}")
             self.print_step(step_output=reasoning_process)
             readme_evaluations[readme_file] = {
                 "evaluation": {
                     "project_level": response.project_level,
-                    "overall_assessment": response.overall_assessment,
+                    "score": response.score,
+                    "key_strengths": response.key_strengths,
+                    "overall_improvement_suggestions": response.overall_improvement_suggestions,
                 }, 
                 "reasoning_process": reasoning_process
             }

bioguider/agents/prompt_utils.py

@@ -165,7 +165,7 @@ If **any one** of these is present, the document should be classified as Install
         "important_instructions": """- Give priority to analyzing README file that contain installation instructions and the files whose names include **"install"** or **"setup"**.
 - If multiple files are found, select the most comprehensive one that covers the installation process.
 - The total number of collected files should **not exceed 3**.
-- Make sure to include **only one installation instruction file**, selecting the most comprehensive and representative one.
+- Identify and select **no more than three** installation instruction files — choose the most comprehensive and representative ones.
 """
     },
     "License": {

bioguider/managers/evaluation_manager.py

@@ -10,6 +10,7 @@ from ..rag.rag import RAG
 from ..utils.file_utils import parse_repo_url
 from ..database.summarized_file_db import SummarizedFilesDb
 from ..agents.evaluation_task import EvaluationREADMETask
+from ..agents.evaluation_installation_task import EvaluationInstallationTask
 from ..agents.collection_task import CollectionTask
 
 class EvaluationManager:
@@ -84,6 +85,30 @@ class EvaluationManager:
         if s is None or 'final_answer' not in s:
             return None
         
+    def evaluate_installation(self):
+        task = CollectionTask(
+            llm=self.llm,
+            step_callback=self.step_callback,
+        )
+        task.compile(
+            repo_path=self.rag.repo_dir,
+            gitignore_path=Path(self.rag.repo_dir, ".gitignore"),
+            db=self.summary_file_db,
+            goal_item=CollectionGoalItemEnum.Installation.name,
+        )
+        files = task.collect()
+        if files is None or len(files) == 0:
+            return None
+        evaluation_task = EvaluationInstallationTask(
+            llm=self.llm,
+            repo_path=self.rag.repo_dir,
+            gitignore_path=Path(self.rag.repo_dir, ".gitignore"),
+            meta_data=self.project_metadata,
+            step_callback=self.step_callback,
+        )
+        evaluation = evaluation_task.evaluate(files)
+        return evaluation, files
+        
     def _find_readme_files(self) -> list[str]:
         """
         Search for a README file in the repository directory.

{bioguider-0.2.4.dist-info → bioguider-0.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bioguider
-Version: 0.2.4
+Version: 0.2.6
 Summary: An AI-Powered package to help biomedical developers to generate clear documentation
 License: MIT
 Author: Cankun Wang

{bioguider-0.2.4.dist-info → bioguider-0.2.6.dist-info}/RECORD

@@ -16,19 +16,20 @@ bioguider/agents/dockergeneration_observe_step.py,sha256=93PO_Y4YyUShVTKRt0nErcj
 bioguider/agents/dockergeneration_plan_step.py,sha256=SB8tQM9PkIKsD2o1DFD7bedcxz6r6hSy8n_EVK60Fz0,7235
 bioguider/agents/dockergeneration_task.py,sha256=ezsweVHJsFpOyOI6rYMt1DZ3PE19dcq4J3Lm-d0IA8M,6220
 bioguider/agents/dockergeneration_task_utils.py,sha256=v7emqrJlVW-A5ZdLmPSdiaMSKCR8uzy9UYzx_1cgzyo,9041
-bioguider/agents/evaluation_task.py,sha256=RRcbw4qTUPElZuiBAMj7heGsDV3t93l_IrioudMTGpc,14316
+bioguider/agents/evaluation_installation_task.py,sha256=ULxlw5VvW_w-tH7mVWbvOFjKYzjpUz2dVTCHKqzU_8w,5922
+bioguider/agents/evaluation_task.py,sha256=8tjRPGsRQ9qDxVgYYDo-GhhEiwW71zSS-iyLZUPRbBA,17178
 bioguider/agents/identification_execute_step.py,sha256=_9x_KITVtNJjQlgjfq-LazoUl46Tctaj_W_AVxz5q-w,5488
 bioguider/agents/identification_observe_step.py,sha256=OENwf9XyOSIHvJMp7eoyQOYGjjtPnPT2S29xf1rCATk,3667
 bioguider/agents/identification_plan_step.py,sha256=p0BKziXdB4ph4D_T9FU5bH8CbHD5Gv0YuszMds_xh-Y,5224
 bioguider/agents/identification_task.py,sha256=-4bnw39OSkhu8RkDVQPRpmu0g8K79dji3HXTybch5As,8329
 bioguider/agents/identification_task_utils.py,sha256=5gevknha9hJiiQN5L7Yp9-pyhAlbR_j31aGRK5j0D_w,522
 bioguider/agents/peo_common_step.py,sha256=iw2c1h7X11WJzSE2tSRg0UAoXH0QOlQDxW9CCzSVMOY,2677
-bioguider/agents/prompt_utils.py,sha256=KBNWsUJMjZAczD_y7nFMnjfJv4dVBPI3cAlQDMbFHec,12379
+bioguider/agents/prompt_utils.py,sha256=udl4PSTZtAc6vBRYJJq4ZGB2iy3ihRE4i9afFJLT5kM,12390
 bioguider/agents/python_ast_repl_tool.py,sha256=o7-4P1h8jS8ikhGSA4CI_OWQ2a0Eg5tEdmuAp_qrO-0,2519
 bioguider/agents/rag_collection_task.py,sha256=r_jPAMjQcC7dIydKxX77UuMqjJ3MiVKswNZ-yNw7yx8,5199
 bioguider/conversation.py,sha256=DIvk_d7pz_guuORByK1eaaF09FAK-8shcNTrbSUHz9Y,1779
 bioguider/database/summarized_file_db.py,sha256=9Iyin5BChlFjDJlFamTwmzkYZvO3BzFu7M8AbikyPXw,4441
-bioguider/managers/evaluation_manager.py,sha256=h-I-ot41pvXsy1Q8vXfJAS_Znb4-68vnwrdIC-OBXeA,3954
+bioguider/managers/evaluation_manager.py,sha256=93XOE2Q2a-mRa8DMF3IZC7mhE2CxxqOZZ5MLbWlPsjo,4904
 bioguider/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bioguider/rag/config.py,sha256=5g4IqTzgyfZfax9Af9CTkXShgItPOt4_9TEMSekCPik,4602
 bioguider/rag/data_pipeline.py,sha256=OXnsqETVytHBMXHerg9gACtNhwpWSODYWvzxVDTP_So,27767
@@ -41,7 +42,7 @@ bioguider/utils/file_utils.py,sha256=9VfAHsz1UkFPtzAmvWZvPl1TMaKIYNjNlLgsfB8tNjg
 bioguider/utils/gitignore_checker.py,sha256=pOYUwsS9D5014LxcZb0cj3s2CAYaD2uF_pYJpaNKcho,6532
 bioguider/utils/pyphen_utils.py,sha256=cdZc3qphkvMDeL5NiZ8Xou13M_uVNP7ifJ-FwxO-0BE,2680
 bioguider/utils/utils.py,sha256=YP3HXgU_rvYDWkEcTzWGiYZw-mlfVrqGhUGSc0_4Pms,900
-bioguider-0.2.4.dist-info/LICENSE,sha256=qzkvZcKwwA5DuSuhXMOm2LcO6BdEr4V7jwFZVL2-jL4,1065
-bioguider-0.2.4.dist-info/METADATA,sha256=VD_LpYetLIYKw2y-JH7_OZZT8D5oIbmzSCx8t5kgyJY,1823
-bioguider-0.2.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-bioguider-0.2.4.dist-info/RECORD,,
+bioguider-0.2.6.dist-info/LICENSE,sha256=qzkvZcKwwA5DuSuhXMOm2LcO6BdEr4V7jwFZVL2-jL4,1065
+bioguider-0.2.6.dist-info/METADATA,sha256=MBcrwDU4pJKKEkd1jBvtBYd0WDUrbIZjKJuwOk_TQ3Y,1823
+bioguider-0.2.6.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+bioguider-0.2.6.dist-info/RECORD,,