bioguider 0.2.52__py3-none-any.whl

bioguider/agents/evaluation_submission_requirements_task.py

@@ -0,0 +1,172 @@
+
+
+from bioguider.agents.agent_utils import try_parse_json_object, try_parse_with_llm
+from bioguider.agents.evaluation_task import EvaluationTask
+from bioguider.agents.collection_task import CollectionTask
+from bioguider.agents.identification_task import IdentificationTask
+from bioguider.agents.prompt_utils import CollectionGoalItemEnum
+from bioguider.utils.constants import (
+    DEFAULT_TOKEN_USAGE, 
+    EvaluationInstallationResult, 
+    EvaluationREADMEResult,
+    SoftwarePackageContentResult,
+    DemoInstructionsResult,
+    EvaluationSubmissionRequirementsResult,
+)
+
+DEMO_INSTRUCTION_GOAL = """
+1. Identify if it provides the instructions to run on provided data
+2. Identify if it provides the instructions to run on custom data
+3. Identify if it provides the expected output
+"""
+
+DEMO_INSTRUCTION_FINAL_ANSWER = \
+'{{"run_on_data_instruction": <True or False>, "run_on_custom_instruction": <True or False>, "expected_output_description": <True Or False>}}'
+
+class EvaluationSubmissionRequirementsTask(EvaluationTask):
+    def __init__(
+        self, 
+        llm, 
+        repo_path, 
+        gitignore_path, 
+        meta_data = None, 
+        step_callback = None, 
+        summarized_files_db = None,
+        readme_files_evaluation: dict[str, EvaluationREADMEResult] | None = None,
+        installation_evaluation: EvaluationInstallationResult | None = None,
+        installation_files: list[str] | None = None
+    ):
+        super().__init__(llm, repo_path, gitignore_path, meta_data, step_callback, summarized_files_db)
+        self.evaluation_name = "Submission Requirements Evaluation"
+        self.readme_files_evaluation = readme_files_evaluation
+        self.installation_evaluation = installation_evaluation
+        self.installation_files = installation_files
+
+    def _collect_software_package_content(self):
+        collection_task = CollectionTask(
+            llm = self.llm,
+            step_callback=self.step_callback,
+            summarize_instruction="We are collecting compiled standalone files, source code files and example data files.",
+            summarized_files_db=self.summarized_files_db,
+        )
+        collection_task.compile(
+            repo_path=self.repo_path,
+            gitignore_path=self.gitignore_path,
+            db=self.summarized_files_db,
+            goal_item=CollectionGoalItemEnum.SoftwarePackageContent.name,
+        )
+        files = collection_task.collect()
+
+        return files
+    
+    def _evaluate_software_package_content(self) -> tuple[SoftwarePackageContentResult, list[str]]:
+        files = self._collect_software_package_content()
+        if len(files) == 3:
+            return SoftwarePackageContentResult(
+                compiled_standalone_software=files[0].strip().lower() != "n/a",
+                source_code=files[1].strip().lower() != "n/a",
+                demo_dataset=files[2].strip().lower() != "n/a",
+            ), files
+        else:
+            return SoftwarePackageContentResult(
+                compiled_standalone_software=False,
+                source_code=False,
+                demo_dataset=False,
+            ), files
+    
+    def _evaluatie_demo_instructions(self) -> tuple[DemoInstructionsResult | None, list[str]]:
+        readme_files = [f for f in self.readme_files_evaluation.keys() \
+                        if self.readme_files_evaluation[f].project_level]
+        installation_files = self.installation_files if self.installation_files is not None else []
+        provided_files = readme_files + installation_files
+        provided_files = provided_files if len(provided_files) > 0 else None
+        identify_task = IdentificationTask(
+            llm=self.llm,
+            step_callback=self.step_callback,
+            summarized_files_db=self.summarized_files_db,
+            provided_files=provided_files
+        )
+        identify_task.compile(
+            repo_path=self.repo_path,
+            gitignore_path=self.gitignore_path,
+        )
+        final_answer = identify_task.identify_customize_goal(
+            goal="demo instructions",
+            final_answer_example=DEMO_INSTRUCTION_FINAL_ANSWER,
+        )
+        final_answer = final_answer["final_answer"] \
+            if final_answer is not None and "final_answer" in final_answer else final_answer
+        parsed_obj = self._parse_demo_instruction_result(final_answer)
+        return parsed_obj, provided_files
+
+    def _parse_demo_instruction_result(self, result: str | dict) -> DemoInstructionsResult:
+        parsed_obj = None
+        if isinstance(result, dict):
+            parsed_obj = result
+        else:
+            parsed_obj = try_parse_json_object(result)
+            if parsed_obj is None:
+                parsed_obj, token_usage = try_parse_with_llm(
+                    llm=self.llm,
+                    input_text=result,
+                    schema=DemoInstructionsResult,
+                )
+                parsed_obj = vars(parsed_obj) if parsed_obj is not None else parsed_obj
+                self.print_step(token_usage=token_usage)
+                self.print_step(step_output=str(parsed_obj))
+
+        return DemoInstructionsResult(
+            run_on_data_instruction = parsed_obj["run_on_data_instruction"] \
+                if "run_on_data_instruction" in parsed_obj else False,
+            run_on_custom_instruction = parsed_obj["run_on_custom_instruction"] \
+                if "run_on_custom_instruction" in parsed_obj else False,
+            expected_output_description = parsed_obj["expected_output_description"] \
+                if "expected_output_description" in parsed_obj else False,
+        )
+
+    def _combine_evaluation(
+        self,
+        software_evaluation: SoftwarePackageContentResult,
+        demo_evaluation: DemoInstructionsResult,
+    ) -> EvaluationSubmissionRequirementsResult:
+        readme_files = [f for f in self.readme_files_evaluation.keys() \
+                        if self.readme_files_evaluation[f].project_level]
+        structured_install_evaluation = self.installation_evaluation.structured_evaluation
+        software_dependency = structured_install_evaluation.dependency_number > 0
+        install_tutorial = structured_install_evaluation.install_tutorial
+        hardware_requirements = structured_install_evaluation.hardware_requirements
+        compatible_os = structured_install_evaluation.compatible_os
+        license = any([
+            self.readme_files_evaluation[f].structured_evaluation.license_score \
+                if self.readme_files_evaluation[f].structured_evaluation is not None \
+                else False for f in readme_files
+        ])
+        return EvaluationSubmissionRequirementsResult(
+            compiled_standalone_software=software_evaluation.compiled_standalone_software,
+            source_code=software_evaluation.source_code,
+            demo_dataset=software_evaluation.demo_dataset,
+            run_on_data_instruction=demo_evaluation.run_on_data_instruction,
+            run_on_custom_instruction=demo_evaluation.run_on_custom_instruction,
+            expected_output_description=demo_evaluation.expected_output_description,
+            complete_readme=len(readme_files) > 0,
+            software_dependency=software_dependency,
+            install_tutorial=install_tutorial,
+            license=license,
+            hardware_requirements=hardware_requirements,
+            compatible_os=compatible_os,
+        )
+
+    def _evaluate(self, files):
+        
+        software_evaluation, software_files = self._evaluate_software_package_content()
+        demo_evaluation, demo_files = self._evaluatie_demo_instructions()
+        files = list(set(software_files + demo_files))
+
+        return self._combine_evaluation(software_evaluation, demo_evaluation), {**DEFAULT_TOKEN_USAGE}, files
+
+
+    def _collect_files(self):
+        return []
+        
+
+

bioguider/agents/evaluation_task.py

@@ -0,0 +1,206 @@
+
+import os
+from pathlib import Path
+import logging
+from typing import Callable
+from abc import ABC, abstractmethod
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+
+from bioguider.agents.agent_utils import read_file
+from bioguider.agents.prompt_utils import EVALUATION_INSTRUCTION
+from bioguider.database.summarized_file_db import SummarizedFilesDb
+from bioguider.utils.constants import DEFAULT_TOKEN_USAGE, ProjectMetadata
+from .common_conversation import CommonConversation
+from ..utils.pyphen_utils import PyphenReadability
+
+logger = logging.getLogger(__name__)
+
+EVALUATION_README_SYSTEM_PROMPT = """
+You are an expert in evaluating the quality of README files in software repositories. 
+Your task is to analyze the provided README file and generate a comprehensive quality report.
+
+---
+
+### **Step 1:  Identify README type
+
+First, determine whether the provided README is a **project-level README** (typically at the root of a repository) or a **folder-level README** (typically inside subdirectories).
+
+---
+
+### **Evaluation Criteria**
+
+#### If the README is a **project-level** file, evaluate it using the following criteria.
+
+For each criterion below, provide a brief assessment followed by specific, actionable comments for improvement.
+
+**1. Project Clarity & Purpose**
+ * **Assessment**: [Your evaluation of whether the project's purpose is clear.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote a specific line/section from the README.]
+    * **Improving comments:** [Provide your suggestions to improve clarity.]
+    * **Original text:** [Quote a specific line/section from the README.]
+    * **Improving comments:** [Provide your suggestions to improve clarity.]
+    ...
+
+**2. Installation Instructions**
+ * **Assessment**: [Your evaluation of the installation instructions.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to installation.]
+    * **Improving comments:** [Provide your suggestions.]
+    * **Original text:** [Quote text related to installation.]
+    * **Improving comments:** [Provide your suggestions.]
+    ...
+
+**3. Usage Instructions**
+ * **Assessment**: [Your evaluation of the usage instructions.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to usage.]
+    * **Improving comments:** [Provide your suggestions.]
+    * **Original text:** [Quote text related to usage.]
+    * **Improving comments:** [Provide your suggestions.]
+    ...
+
+**4. Contributing Guidelines**
+ * **Assessment**: [Your evaluation of the contributing guidelines.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to contributions.]
+    * **Improving comments:** [Provide your suggestions.]
+    * **Original text:** [Quote text related to contributions.]
+    * **Improving comments:** [Provide your suggestions.]
+    ...
+
+**5. License Information**
+ * **Assessment**: [Your evaluation of the license information.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to the license.]
+    * **Improving comments:** [Provide your suggestions.]
+    * **Original text:** [Quote text related to the license.]
+    * **Improving comments:** [Provide your suggestions.]
+    ...
+
+**6. Readability Analysis**
+ * **Flesch Reading Ease**: `{flesch_reading_ease}` (A higher score is better, with 60-70 being easily understood by most adults).
+ * **Flesch-Kincaid Grade Level**: `{flesch_kincaid_grade}` (Represents the US school-grade level needed to understand the text).
+ * **Gunning Fog Index**: `{gunning_fog_index}` (A score above 12 is generally considered too hard for most people).
+ * **SMOG Index**: `{smog_index}` (Estimates the years of education needed to understand the text).
+ * **Assessment**: Based on these scores, evaluate the overall readability and technical complexity of the language used.
+
+---
+
+#### If if is a **folder-level** file, use the following criteria instead.
+
+For each criterion below, provide a brief assessment followed by specific, actionable comments for improvement.
+
+**1. Folder Description**
+ * **Assessment**: [Your evaluation of whether it Provides a clear **description** of what the folder contains (e.g., modules, scripts, data).]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote a specific line/section from the README.]
+    * **Improving comments:** [Provide your suggestions to improve clarity.]
+
+**2. Folder Purpose**
+ * **Assessment**: [Your evaluation of whether it explains the **purpose** or **role** of the components inside this subfolder.]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to purpose.]
+    * **Improving comments:** [Provide your suggestions.]
+
+**3. Usage**
+ * **Assessment**: [Your evaluation of whether it includes **usage instructions** specific to this folder (e.g., commands, import paths, input/output files).]
+ * **Improvement Suggestions**:
+    * **Original text:** [Quote text related to usage.]
+    * **Improving comments:** [Provide your suggestions.]
+
+**4. Readability Analysis**
+ * **Flesch Reading Ease**: `{flesch_reading_ease}` (A higher score is better, with 60-70 being easily understood by most adults).
+ * **Flesch-Kincaid Grade Level**: `{flesch_kincaid_grade}` (Represents the US school-grade level needed to understand the text).
+ * **Gunning Fog Index**: `{gunning_fog_index}` (A score above 12 is generally considered too hard for most people).
+ * **SMOG Index**: `{smog_index}` (Estimates the years of education needed to understand the text).
+ * **Assessment**: Based on these scores, evaluate the overall readability and technical complexity of the language used.
+
+---
+
+### Final Report Format
+
+#### Your output **must exactly match**  the following template:
+
+**FinalAnswer**
+
+ * Project-Level README: Yes / No
+ * **Score:** [Poor / Fair / Good / Excellent]
+  * **Key Strengths**: <brief summary of the README's strongest points in 2-3 sentences> 
+  * **Overall Improvement Suggestions:**
+    - "Original text snippet 1" - Improving comment 1  
+    - "Original text snippet 2" - Improving comment 2  
+    - ...
+
+#### Notes
+
+* **Project-Level README**: "Yes" if root-level; "No" if folder-level.
+* **Score**: Overall quality rating, could be Poor / Fair / Good / Excellent.
+* **Key Strengths**: Briefly highlight the README's strongest aspects.
+* **Improvement Suggestions**: Provide concrete snippets and suggested improvements.
+
+
+---
+
+### **README path:**
+{readme_path}
+
+---
+
+### **README Content:**
+{readme_content}
+"""
+
+class EvaluationTask(ABC):
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI, 
+        repo_path: str, 
+        gitignore_path: str,
+        meta_data: ProjectMetadata | None = None,
+        step_callback: Callable | None = None,
+        summarized_files_db: SummarizedFilesDb | None=None,
+    ):
+        self.evaluation_name = ""
+        self.llm = llm
+        self.repo_path = repo_path
+        self.gitignore_path = gitignore_path
+        self.step_callback = step_callback
+        self.metadata = meta_data
+        self.summarized_files_db = summarized_files_db
+
+    def print_step(
+        self,
+        step_name: str | None = None,
+        step_output: str | None = None,
+        token_usage: dict | None = None,
+    ):
+        if self.step_callback is None:
+            return
+        self.step_callback(
+            step_name=step_name,
+            step_output=step_output,
+            token_usage=token_usage,
+        )
+
+    def evaluate(self) -> tuple[dict, list[str]]:
+        self._enter_evaluation()
+        files = self._collect_files()
+        evaluations, token_usage, files = self._evaluate(files)
+        self._leave_evaluation(token_usage)
+        return evaluations, files
+    
+    def _enter_evaluation(self):
+        self.print_step(step_name=self.evaluation_name)
+
+    def _leave_evaluation(self, token_usage):
+        self.print_step(token_usage=token_usage)
+
+    @abstractmethod
+    def _evaluate(self, files: list[str]) -> tuple[dict, dict, list[str]]:
+        pass
+
+    @abstractmethod
+    def _collect_files(self) -> list[str]:
+        pass

bioguider/agents/evaluation_tutorial_task.py

@@ -0,0 +1,169 @@
+
+
+from pathlib import Path
+from typing import Callable, Optional
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai.chat_models.base import BaseChatOpenAI
+from pydantic import BaseModel, Field
+import logging
+
+from bioguider.agents.consistency_evaluation_task import ConsistencyEvaluationResult
+from bioguider.agents.evaluation_task import EvaluationTask
+from bioguider.agents.collection_task import CollectionTask
+from bioguider.agents.evaluation_tutorial_task_prompts import INDIVIDUAL_TUTORIAL_EVALUATION_SYSTEM_PROMPT
+from bioguider.agents.prompt_utils import CollectionGoalItemEnum
+from bioguider.utils.constants import DEFAULT_TOKEN_USAGE, ProjectMetadata
+from bioguider.utils.file_utils import flatten_files
+from bioguider.utils.utils import increase_token_usage, get_overall_score
+from .evaluation_utils import (
+    compute_readability_metrics,
+    default_consistency_result,
+    evaluate_consistency_on_content,
+    normalize_evaluation_content,
+    run_llm_evaluation,
+    sanitize_files,
+)
+
+logger = logging.getLogger(__name__)
+
+MAX_FILE_SIZE = 1024 * 100 # 100K
+
+class TutorialEvaluationResult(BaseModel):
+    overall_score: int=Field(description="A number between 0 and 100 representing the overall quality rating.")
+    overall_key_strengths: str=Field(description="A string value, the key strengths of the tutorial")
+    # overall_improvement_suggestions: str=Field(description="Suggestions to improve the overall score if necessary")
+    readability_score: int=Field(description="A number between 0 and 100 representing the readability quality rating.")
+    readability_error_count: Optional[int]=Field(default=0, description="Total number of ERROR INSTANCES found (count every occurrence, not types)")
+    readability_errors_found: list[str]=Field(default_factory=list, description="List of ALL individual error instances with format: 'ERROR_TYPE: original → corrected - location'")
+    readability_suggestions: list[str]=Field(default_factory=list, description="General readability improvement suggestions (non-error related)")
+    setup_and_dependencies_score: int=Field(description="A number between 0 and 100 representing the setup and dependencies quality rating.")
+    setup_and_dependencies_suggestions: list[str]=Field(description="A list of string values, suggestions to improve setup and dependencies if necessary")
+    reproducibility_score: int=Field(description="A number between 0 and 100 representing the reproducibility quality rating.")
+    reproducibility_suggestions: list[str]=Field(description="A list of string values, suggestions to improve reproducibility if necessary")
+    structure_and_navigation_score: int=Field(description="A number between 0 and 100 representing the structure and navigation quality rating.")
+    structure_and_navigation_suggestions: list[str]=Field(description="A list of string values, suggestions to improve structure and navigation if necessary")
+    executable_code_quality_score: int=Field(description="A number between 0 and 100 representing the executable code quality rating.")
+    executable_code_quality_suggestions: list[str]=Field(description="A list of string values, suggestions to improve executable code quality if necessary")
+    result_verification_score: int=Field(description="A number between 0 and 100 representing the result verification quality rating.")
+    result_verification_suggestions: list[str]=Field(description="A list of string values, suggestions to improve result verification if necessary")
+    performance_and_resource_notes_score: int=Field(description="A number between 0 and 100 representing the performance and resource notes quality rating.")
+    performance_and_resource_notes_suggestions: list[str]=Field(description="A list of string values, suggestions to improve performance and resource notes if necessary")
+    
+class IndividualTutorialEvaluationResult(BaseModel):
+    tutorial_evaluation: TutorialEvaluationResult | None=Field(description="The evaluation result of the tutorial")
+    consistency_evaluation: ConsistencyEvaluationResult | None=Field(description="The evaluation result of the consistency of the tutorial")
+
+class EvaluationTutorialTask(EvaluationTask):
+    def __init__(
+        self, 
+        llm: BaseChatOpenAI, 
+        repo_path: str, 
+        gitignore_path: str,
+        meta_data: ProjectMetadata | None = None,
+        step_callback: Callable | None = None,
+        summarized_files_db = None,
+        code_structure_db = None,
+        collected_files: list[str] | None = None,
+    ):
+        super().__init__(llm, repo_path, gitignore_path, meta_data, step_callback, summarized_files_db)
+        self.evaluation_name = "Tutorial Evaluation"
+        self.code_structure_db = code_structure_db
+        self.collected_files = collected_files
+
+    def _sanitize_files(self, files: list[str]) -> list[str]:
+        return sanitize_files(
+            self.repo_path,
+            files,
+            max_size_bytes=MAX_FILE_SIZE,
+            disallowed_exts={".svg"},
+            check_ipynb_size=False,
+        )
+
+    def _collect_files(self):
+        if self.collected_files is not None:
+            return self.collected_files
+        
+        task = CollectionTask(
+            llm=self.llm,
+            step_callback=self.step_callback,
+            summarized_files_db=self.summarized_files_db,
+        )
+        task.compile(
+            repo_path=self.repo_path,
+            gitignore_path=Path(self.repo_path, ".gitignore"),
+            goal_item=CollectionGoalItemEnum.Tutorial.name,
+        )
+        files = task.collect()
+        files = flatten_files(self.repo_path, files)
+        files = self._sanitize_files(files)
+        return files
+
+    def _evaluate_individual_tutorial(self, file: str) -> tuple[IndividualTutorialEvaluationResult | None, dict]:
+        content, readability_content = normalize_evaluation_content(
+            self.repo_path, file
+        )
+        if content is None or readability_content is None:
+            logger.error(f"Error in sanitizing file {file} - {Path(self.repo_path, file).resolve()}")
+            return None, {**DEFAULT_TOKEN_USAGE}
+            
+        # evaluate general criteria
+        flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, smog_index = \
+            compute_readability_metrics(readability_content)
+        system_prompt = ChatPromptTemplate.from_template(
+            INDIVIDUAL_TUTORIAL_EVALUATION_SYSTEM_PROMPT
+        ).format(
+            flesch_reading_ease=flesch_reading_ease,
+            flesch_kincaid_grade=flesch_kincaid_grade,
+            gunning_fog_index=gunning_fog_index,
+            smog_index=smog_index,
+            tutorial_file_content=readability_content,
+        )
+                
+        res, token_usage, reasoning_process = run_llm_evaluation(
+            llm=self.llm,
+            system_prompt=system_prompt,
+            instruction_prompt="Now, let's begin the tutorial evaluation.",
+            schema=TutorialEvaluationResult,
+        )
+        res: TutorialEvaluationResult = res
+
+        # evaluate consistency
+        consistency_evaluation_result, _temp_token_usage = evaluate_consistency_on_content(
+            llm=self.llm,
+            code_structure_db=self.code_structure_db,
+            step_callback=self.step_callback,
+            domain="tutorial/vignette",
+            content=content,
+        )
+        if consistency_evaluation_result is None:
+            # No sufficient information to evaluate the consistency of the tutorial
+            consistency_evaluation_result = default_consistency_result("tutorial/vignette")
+
+        # calculate overall score
+        res.overall_score = get_overall_score(
+            [
+                consistency_evaluation_result.score,
+                res.readability_score, 
+                res.setup_and_dependencies_score, 
+                res.reproducibility_score, 
+                res.structure_and_navigation_score, 
+                res.executable_code_quality_score, 
+                res.result_verification_score, 
+                res.performance_and_resource_notes_score,
+            ],
+            [3, 3, 3, 1, 1, 2, 1, 1],
+        )
+        
+        return IndividualTutorialEvaluationResult(
+            tutorial_evaluation=res,
+            consistency_evaluation=consistency_evaluation_result,
+        ), token_usage
+
+    def _evaluate(self, files: list[str] | None = None) -> tuple[dict[str, IndividualTutorialEvaluationResult] | None, dict, list[str]]:
+        total_token_usage = {**DEFAULT_TOKEN_USAGE}
+        tutorial_evaluation_results = {}
+        for file in files:
+            tutorial_evaluation_result, token_usage = self._evaluate_individual_tutorial(file)
+            total_token_usage = increase_token_usage(total_token_usage, token_usage)
+            tutorial_evaluation_results[file] = tutorial_evaluation_result
+        return tutorial_evaluation_results, total_token_usage, files