bioguider 0.2.52__py3-none-any.whl

bioguider/rag/rag.py

@@ -0,0 +1,138 @@
+import os
+from typing import Any, List, Tuple, Optional, Dict
+from uuid import uuid4
+import logging
+import re
+import adalflow as adal
+from adalflow.core.types import (
+    Conversation,
+    DialogTurn,
+    UserQuery,
+    AssistantResponse,
+)
+from adalflow.components.retriever.faiss_retriever import FAISSRetriever
+from adalflow.components.model_client.openai_client import OpenAIClient
+from adalflow.components.model_client.azureai_client import AzureAIClient
+from .config import configs, create_model_client, create_model_kwargs
+from .data_pipeline import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+# Maximum token limit for embedding models
+MAX_INPUT_TOKENS = 7500  # Safe threshold below 8192 token limit
+
+class RAG(adal.Component):
+    """RAG with one repo.
+    If you want to load a new repos, call prepare_retriever(repo_url_or_path) first."""
+
+    def __init__(self, use_s3: bool = False):
+        """
+        Initialize the RAG component.
+
+        Args:
+            use_s3: Whether to use S3 for database storage (default: False)
+        """
+        super().__init__()
+
+        self.embedder = adal.Embedder(
+            model_client=create_model_client(),
+            model_kwargs=create_model_kwargs(),
+        )
+
+        self.initialize_db_manager()
+
+    @property
+    def repo_dir(self):
+        if self.db_manager:
+            return self.db_manager.repo_dir
+        return None
+
+    def initialize_db_manager(self):
+        """Initialize the database manager with local storage"""
+        self.db_manager = DatabaseManager()
+        self.transformed_doc_documents: list | None = None
+        self.transformed_code_documents: list | None = None
+        self.access_token: str | None = None
+
+    def initialize_repo(self, repo_url_or_path: str, access_token: str = None):
+        self.repo_url_or_path = repo_url_or_path
+        self.access_token = access_token
+        self.db_manager.reset_database_and_create_repo(repo_url_or_path, access_token)
+
+    def _prepare_retriever(self):
+        """
+        Prepare the retriever for a repository.
+        Will load database from local storage if available.
+        """
+        if self.transformed_code_documents is not None and self.transformed_doc_documents is not None:
+            # retrievers have been prepared
+            return
+        self.transformed_doc_documents, self.transformed_code_documents \
+            = self.db_manager.prepare_database()
+        logger.info(f"Loaded {len(self.transformed_doc_documents)} doc documents for retrieval")
+        logger.info(f"Loaded {len(self.transformed_code_documents)} code documents for retrieval")
+        self.doc_retriever = FAISSRetriever(
+            **configs["retriever"],
+            embedder=self.embedder,
+            documents=self.transformed_doc_documents,
+            document_map_func=lambda doc: doc.vector,
+            dimensions=256,
+        )
+        self.code_retriever = FAISSRetriever(
+            **configs["retriever"],
+            embedder=self.embedder,
+            documents=self.transformed_code_documents,
+            document_map_func=lambda doc: doc.vector,
+            dimensions=256,
+        )
+
+    def query_doc(self, query: str) -> List:
+        """
+        Process a query using RAG.
+
+        Args:
+            query: The user's query
+
+        Returns:
+            retrieved_documents: List of documents retrieved based on the query
+        """
+        self._prepare_retriever()
+        retrieved_documents = self.doc_retriever(query)
+        # Fill in the documents
+        retrieved_documents[0].documents = [
+            self.transformed_doc_documents[doc_index]
+            for doc_index in retrieved_documents[0].doc_indices
+        ]
+        return retrieved_documents
+    
+    def query_code(self, query: str) -> List:
+        """
+        Process a code query using RAG.
+
+        Args:
+            query: The user's code query
+
+        Returns:
+            retrieved_documents: List of code documents retrieved based on the query
+        """
+        try:
+            retrieved_documents = self.code_retriever(query)
+            # Fill in the documents
+            retrieved_documents[0].documents = [
+                self.transformed_code_documents[doc_index]
+                for doc_index in retrieved_documents[0].doc_indices
+            ]
+        except Exception as e:
+            logger.error(e)
+            raise e
+        return retrieved_documents
+        
+    @property
+    def save_repo_dir(self) -> str:
+        """
+        Get the directory where the repository is saved.
+
+        Returns:
+            str: The path to the repository directory
+        """
+        return self.db_manager.repo_paths["save_repo_dir"]

bioguider/settings.py

@@ -0,0 +1,103 @@
+from enum import StrEnum
+from typing import Optional
+
+from iso639 import Language, LanguageNotFoundError
+from pydantic import (
+    DirectoryPath,
+    Field,
+    HttpUrl,
+    PositiveFloat,
+    PositiveInt,
+    SecretStr,
+    field_validator,
+)
+from pydantic_settings import BaseSettings
+from pathlib import Path
+
+class ProjectSettings(BaseSettings):
+    target_repo: DirectoryPath = ""  # type: ignore
+    hierarchy_name: str = ".project_doc_record"
+    markdown_docs_name: str = "markdown_docs"
+    ignore_list: list[str] = []
+    language: str = "English"
+    max_thread_count: PositiveInt = 4
+
+    @field_validator("language")
+    @classmethod
+    def validate_language_code(cls, v: str) -> str:
+        try:
+            language_name = Language.match(v).name
+            return language_name  # Returning the resolved language name
+        except LanguageNotFoundError:
+            raise ValueError(
+                "Invalid language input. Please enter a valid ISO 639 code or language name."
+            )
+
+class ChatCompletionSettings(BaseSettings):
+    model: str = "gpt-4o-mini"  # NOTE: No model restrictions for user flexibility, but it's recommended to use models with a larger context window.
+    temperature: PositiveFloat = 0.2
+    request_timeout: PositiveInt = 60
+    openai_base_url: str = "https://api.openai.com/v1"
+    openai_api_key: SecretStr = Field("", exclude=True)
+
+    @field_validator("openai_base_url", mode="before")
+    @classmethod
+    def convert_base_url_to_str(cls, openai_base_url: HttpUrl) -> str:
+        return str(openai_base_url)
+
+
+class Setting(BaseSettings):
+    project: ProjectSettings = {}  # type: ignore
+    chat_completion: ChatCompletionSettings = {}  # type: ignore
+
+
+class SettingsManager:
+    _setting_instance: Optional[Setting] = (
+        None  # Private class attribute, initially None
+    )
+
+    @classmethod
+    def get_setting(cls):
+        if cls._setting_instance is None:
+            cls._setting_instance = Setting()
+        return cls._setting_instance
+
+    @classmethod
+    def initialize_with_params(
+        cls,
+        target_repo: Path,
+        markdown_docs_name: str,
+        hierarchy_name: str,
+        ignore_list: list[str],
+        language: str,
+        max_thread_count: int,
+        model: str,
+        temperature: float,
+        request_timeout: int,
+        openai_base_url: str,
+    ):
+        project_settings = ProjectSettings(
+            target_repo=target_repo,
+            hierarchy_name=hierarchy_name,
+            markdown_docs_name=markdown_docs_name,
+            ignore_list=ignore_list,
+            language=language,
+            max_thread_count=max_thread_count,
+        )
+
+        chat_completion_settings = ChatCompletionSettings(
+            model=model,
+            temperature=temperature,
+            request_timeout=request_timeout,
+            openai_base_url=openai_base_url,
+        )
+
+        cls._setting_instance = Setting(
+            project=project_settings,
+            chat_completion=chat_completion_settings,
+        )
+
+
+if __name__ == "__main__":
+    setting = SettingsManager.get_setting()
+    print(setting.model_dump())

bioguider/utils/code_structure_builder.py

@@ -0,0 +1,59 @@
+from pathlib import Path
+import logging
+
+from bioguider.utils.r_file_handler import RFileHandler
+
+from .gitignore_checker import GitignoreChecker
+from .python_file_handler import PythonFileHandler
+from ..database.code_structure_db import CodeStructureDb
+from ..rag.config import configs
+
+logger = logging.getLogger(__name__)
+
+class CodeStructureBuilder:
+    def __init__(
+        self, 
+        repo_path: str | Path, 
+        gitignore_path: str | Path,
+        code_structure_db: CodeStructureDb,
+    ):
+        self.repo_path = str(repo_path)
+        self.gitignore_checker = GitignoreChecker(
+            directory=repo_path,
+            gitignore_path=str(gitignore_path),
+            exclude_dir_patterns=configs["file_filters"]["excluded_dirs"],
+            exclude_file_patterns=configs["file_filters"]["excluded_files"],
+        )
+        self.file_handler = PythonFileHandler(repo_path)
+        self.code_structure_db = code_structure_db
+
+    def build_code_structure(self):
+        if self.code_structure_db.is_database_built():
+            return
+        files = self.gitignore_checker.check_files_and_folders()
+        for file in files:
+            if not file.endswith(".py") and not file.endswith(".R"):
+                continue
+            logger.info(f"Building code structure for {file}")
+            if file.endswith(".py"):
+                file_handler = PythonFileHandler(Path(self.repo_path) / file)
+            else:
+                file_handler = RFileHandler(Path(self.repo_path) / file)
+            try:
+                functions_and_classes = file_handler.get_functions_and_classes()
+            except Exception as e:
+                logger.error(f"Error getting functions and classes for {file}: {e}")
+                continue
+            # fixme: currently, we don't extract reference graph for each function or class
+            for function_or_class in functions_and_classes:
+                self.code_structure_db.insert_code_structure(
+                    function_or_class[0], # name
+                    file,
+                    function_or_class[2], # start line number
+                    function_or_class[3], # end line number
+                    function_or_class[1], # parent name
+                    function_or_class[4], # doc string
+                    function_or_class[5], # params
+                )
+        
+

bioguider/utils/constants.py

@@ -0,0 +1,135 @@
+
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+DEFAULT_TOKEN_USAGE = {
+    "total_tokens": 0,
+    "completion_tokens": 0,
+    "prompt_tokens": 0,
+}
+
+class ProjectTypeEnum(Enum):
+    application="application"
+    package="package"
+    pipeline="pipeline"
+    unknown="unknown type"
+
+class PrimaryLanguageEnum(Enum):
+    python="python"
+    R="R"
+    unknown="unknown type"
+
+class ProjectMetadata:
+    def __init__(
+        self,
+        url: str,
+        project_type: ProjectTypeEnum,
+        primary_language: PrimaryLanguageEnum,
+        repo_name: str=None,
+        owner: Optional[str]=None,
+        description: Optional[str]=None,
+        license: Optional[str]=None,
+    ):
+        self.url = url
+        self.project_type = project_type
+        self.primary_language = primary_language
+        self.repo_name = repo_name
+        self.owner = owner
+        self.description = description
+        self.license = license
+
+MAX_FILE_LENGTH=10 *1024 # 10K
+MAX_SENTENCE_NUM=20
+MAX_STEP_COUNT=3*10
+
+class ProjectLevelEvaluationREADMEResult(BaseModel):
+    project_level: Optional[bool]=Field(description="A boolean value specifying if the README file is **project-level** README. TRUE: project-level, FALSE, folder-level")
+
+class StructuredEvaluationREADMEResult(BaseModel):
+    available_score: Optional[bool]=Field(description="A boolean value, Is the README accessible and present?")
+    readability_score: Optional[int]=Field(description="A number between 0 and 100 representing the readability quality rating.")
+    readability_error_count: Optional[int]=Field(default=0, description="Total number of errors found (typos + links + markdown + bio_terms + grammar)")
+    readability_errors_found: Optional[list[str]]=Field(default_factory=list, description="List of ALL errors found with format: 'ERROR_TYPE | original text snippet | suggested fix'")
+    readability_suggestions: Optional[str]=Field(description="General suggestions to improve readability if necessary")
+    project_purpose_score: Optional[bool]=Field(description="A boolean value. Is the project's goal or function clearly stated?")
+    project_purpose_suggestions: Optional[str]=Field(description="Suggestions if not clear")
+    hardware_and_software_spec_score: Optional[int]=Field(description="A number between 0 and 100 representing the hardware and software spec and compatibility description quality rating.")
+    hardware_and_software_spec_suggestions: Optional[str]=Field(description="Suggestions if not clear")
+    dependency_score: Optional[int]=Field(description="A number between 0 and 100 representing the dependencies quality rating.")
+    dependency_suggestions: Optional[str]=Field(description="Suggestions if dependencies are not clearly stated")
+    license_score: Optional[bool]=Field(description="A boolean value, Are contributor or maintainer details provided?")
+    license_suggestions: Optional[str]=Field(description="Suggestions to improve license information")
+    contributor_author_score: Optional[bool]=Field(description="A boolean value. are contributors or author included?")
+    overall_score: int=Field(description="A number between 0 and 100 representing the overall quality rating.")
+
+class FreeProjectLevelEvaluationREADMEResult(BaseModel):
+    available: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestion for the availability of the README file")
+    readability: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestion for the readability of the README file")
+    project_purpose: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestion for the project purpose of the README file")
+    hardware_and_software_spec: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestion for the hardware and software spec and compatibility description of the README file")
+    dependency: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestion for the dependencies of the README file")
+    license: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestion for the license information of the README file")
+    contributor_author: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestion for the contributor and author information of the README file")
+    overall_score: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestion for the overall score of the README file")
+
+class FreeFolderLevelEvaluationREADMEResult(BaseModel):
+    score: Optional[str]=Field(description="An overall score")
+    key_strengths: Optional[str]=Field(description="A string specifying the key strengths of README file.")
+    overall_improvement_suggestions: Optional[list[str]]=Field(description="A list of overall improvement suggestions")
+
+class EvaluationREADMEResult(BaseModel):
+    project_level: bool
+    structured_evaluation: StructuredEvaluationREADMEResult | None
+    free_evaluation: FreeProjectLevelEvaluationREADMEResult | FreeFolderLevelEvaluationREADMEResult | None
+    structured_reasoning_process: str | None
+    free_reasoning_process: str | None
+
+
+class StructuredEvaluationInstallationResult(BaseModel):
+    install_available: Optional[bool]=Field(description="A boolean value. Is the installation documents accessible and present?")
+    install_tutorial: Optional[bool]=Field(description="A boolean value. Is the installation tutorial provided?")
+    dependency_number: Optional[int]=Field(description="A number. It is the number of dependencies that are required to install.")
+    dependency_suggestions: Optional[str]=Field(description="A string value. It is the specific improvements if necessary, such as missing dependencies")
+    compatible_os: Optional[bool]=Field(description="A boolean value. Is compatible operating system described?")
+    overall_score: Optional[int]=Field(description="A number between 0 and 100 representing the overall quality rating.")
+    hardware_requirements: Optional[bool]=Field(description="A boolean value. Is the hardware requirements described?")
+
+class FreeEvaluationInstallationResult(BaseModel):
+    ease_of_access: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestions for the ease of access of the installation information")
+    clarity_of_dependency: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestions for the clarity of dependency specification")
+    hardware_requirements: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestions for the hardware requirements")
+    installation_guide: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestions for the installation guide")
+    compatible_os: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestions for the compatible operating system")
+    overall_score: Optional[list[str]]=Field(description="markdown texts including newlines that contains detailed assessment and detailed suggestions for the overall score of the installation")
+
+class EvaluationInstallationResult(BaseModel):
+    structured_evaluation: StructuredEvaluationInstallationResult | None
+    free_evaluation: FreeEvaluationInstallationResult | None
+    structured_reasoning_process: str | None
+    free_reasoning_process: str | None
+
+class SoftwarePackageContentResult(BaseModel):
+    compiled_standalone_software: Optional[bool] = Field(description="A boolean value. Does it provide the compiled standalone software?")
+    source_code: Optional[bool] = Field(description="A boolean value. Does it provide the source code?")
+    demo_dataset: Optional[bool] = Field(description="A boolean value. Does it provide the demo dataset?")
+
+class DemoInstructionsResult(BaseModel):
+    run_on_data_instruction: Optional[bool] = Field(description="A boolean value. Does it provide instructions on how to run on provided data?")
+    run_on_custom_instruction: Optional[bool] = Field(description="A boolean value. Does it provide instructions on how to run on custom data?")
+    expected_output_description: Optional[bool] = Field(description="A boolean value. Does it provide the description of expected output?")
+
+class EvaluationSubmissionRequirementsResult(BaseModel):
+    compiled_standalone_software: bool | None
+    source_code: bool | None
+    demo_dataset: bool | None
+    run_on_data_instruction: bool | None
+    run_on_custom_instruction: bool | None
+    expected_output_description: bool | None
+    complete_readme: bool | None
+    software_dependency: bool | None
+    install_tutorial: bool | None
+    license: bool | None
+    hardware_requirements: bool | None
+    compatible_os: bool | None

bioguider/utils/default.gitignore

@@ -0,0 +1,140 @@
+# History files
+.Rhistory
+.Rapp.history
+
+# R session data
+.RData
+.RDataTmp
+
+# User-specific files
+.Rproj.user/
+.Ruserdata/
+
+# RStudio files
+.rsituational.user
+.Renviron
+*proj.user
+
+# Example and temporary files
+*.Rhistory
+*.RData
+*.RData~
+*.Rproj.user
+*~
+.DS_Store # macOS specific
+Thumbs.db # Windows specific
+
+# Knitr and R Markdown cache and intermediate files
+*_cache/
+/cache/
+*.utf8.md
+*.knit.md
+*.log
+*.ind
+*.aux
+*.synctex.gz
+*.toc
+*.out
+*.nav
+*.snm
+
+# Package and dependency management
+# renv, packrat, etc.
+renv/
+packrat/
+.renv/
+.Rprofile
+
+# Compiled code from packages like Rcpp
+/src/*.so
+/src/*.o
+/src/*.dll
+
+# Shiny app logs
+*.log
+*.log.*
+
+
+# Byte-compiled / optimized / C files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are in 'dist' folder, but they may appear in the root directory
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Virtual Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDE and editor configuration files
+# VSCode
+.vscode/
+# PyCharm
+.idea/
+# Sublime Text
+*.sublime-project
+*.sublime-workspace
+# Atom
+.atom/
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Testing
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.tox/
+
+# Scrapy stuff
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# OS-generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db