bioguider 0.2.3__py3-none-any.whl

bioguider/rag/rag.py

@@ -0,0 +1,134 @@
+import os
+from typing import Any, List, Tuple, Optional, Dict
+from uuid import uuid4
+import logging
+import re
+import adalflow as adal
+from adalflow.core.types import (
+    Conversation,
+    DialogTurn,
+    UserQuery,
+    AssistantResponse,
+)
+from adalflow.components.retriever.faiss_retriever import FAISSRetriever
+from adalflow.components.model_client.openai_client import OpenAIClient
+from adalflow.components.model_client.azureai_client import AzureAIClient
+from .config import configs, create_model_client, create_model_kwargs
+from .data_pipeline import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+# Maximum token limit for embedding models
+MAX_INPUT_TOKENS = 7500  # Safe threshold below 8192 token limit
+
+class RAG(adal.Component):
+    """RAG with one repo.
+    If you want to load a new repos, call prepare_retriever(repo_url_or_path) first."""
+
+    def __init__(self, use_s3: bool = False):
+        """
+        Initialize the RAG component.
+
+        Args:
+            use_s3: Whether to use S3 for database storage (default: False)
+        """
+        super().__init__()
+
+        self.embedder = adal.Embedder(
+            model_client=create_model_client(),
+            model_kwargs=create_model_kwargs(),
+        )
+
+        self.initialize_db_manager()
+
+    @property
+    def repo_dir(self):
+        if self.db_manager:
+            return self.db_manager.repo_dir
+        return None
+
+    def initialize_db_manager(self):
+        """Initialize the database manager with local storage"""
+        self.db_manager = DatabaseManager()
+        self.transformed_doc_documents = []
+        self.transformed_code_documents = []
+
+    def prepare_retriever(self, repo_url_or_path: str, access_token: str = None):
+        """
+        Prepare the retriever for a repository.
+        Will load database from local storage if available.
+
+        Args:
+            repo_url_or_path: URL or local path to the repository
+            access_token: Optional access token for private repositories
+        """
+        self.initialize_db_manager()
+        self.repo_url_or_path = repo_url_or_path
+        self.transformed_doc_documents, self.transformed_code_documents \
+            = self.db_manager.prepare_database(repo_url_or_path, access_token)
+        logger.info(f"Loaded {len(self.transformed_doc_documents)} doc documents for retrieval")
+        logger.info(f"Loaded {len(self.transformed_code_documents)} code documents for retrieval")
+        self.doc_retriever = FAISSRetriever(
+            **configs["retriever"],
+            embedder=self.embedder,
+            documents=self.transformed_doc_documents,
+            document_map_func=lambda doc: doc.vector,
+            dimensions=256,
+        )
+        self.code_retriever = FAISSRetriever(
+            **configs["retriever"],
+            embedder=self.embedder,
+            documents=self.transformed_code_documents,
+            document_map_func=lambda doc: doc.vector,
+            dimensions=256,
+        )
+
+    def query_doc(self, query: str) -> List:
+        """
+        Process a query using RAG.
+
+        Args:
+            query: The user's query
+
+        Returns:
+            retrieved_documents: List of documents retrieved based on the query
+        """
+        retrieved_documents = self.doc_retriever(query)
+        # Fill in the documents
+        retrieved_documents[0].documents = [
+            self.transformed_doc_documents[doc_index]
+            for doc_index in retrieved_documents[0].doc_indices
+        ]
+        return retrieved_documents
+    
+    def query_code(self, query: str) -> List:
+        """
+        Process a code query using RAG.
+
+        Args:
+            query: The user's code query
+
+        Returns:
+            retrieved_documents: List of code documents retrieved based on the query
+        """
+        try:
+            retrieved_documents = self.code_retriever(query)
+            # Fill in the documents
+            retrieved_documents[0].documents = [
+                self.transformed_code_documents[doc_index]
+                for doc_index in retrieved_documents[0].doc_indices
+            ]
+        except Exception as e:
+            logger.error(e)
+            raise e
+        return retrieved_documents
+        
+    @property
+    def save_repo_dir(self) -> str:
+        """
+        Get the directory where the repository is saved.
+
+        Returns:
+            str: The path to the repository directory
+        """
+        return self.db_manager.repo_paths["save_repo_dir"]

bioguider/settings.py

@@ -0,0 +1,103 @@
+from enum import StrEnum
+from typing import Optional
+
+from iso639 import Language, LanguageNotFoundError
+from pydantic import (
+    DirectoryPath,
+    Field,
+    HttpUrl,
+    PositiveFloat,
+    PositiveInt,
+    SecretStr,
+    field_validator,
+)
+from pydantic_settings import BaseSettings
+from pathlib import Path
+
+class ProjectSettings(BaseSettings):
+    target_repo: DirectoryPath = ""  # type: ignore
+    hierarchy_name: str = ".project_doc_record"
+    markdown_docs_name: str = "markdown_docs"
+    ignore_list: list[str] = []
+    language: str = "English"
+    max_thread_count: PositiveInt = 4
+
+    @field_validator("language")
+    @classmethod
+    def validate_language_code(cls, v: str) -> str:
+        try:
+            language_name = Language.match(v).name
+            return language_name  # Returning the resolved language name
+        except LanguageNotFoundError:
+            raise ValueError(
+                "Invalid language input. Please enter a valid ISO 639 code or language name."
+            )
+
+class ChatCompletionSettings(BaseSettings):
+    model: str = "gpt-4o-mini"  # NOTE: No model restrictions for user flexibility, but it's recommended to use models with a larger context window.
+    temperature: PositiveFloat = 0.2
+    request_timeout: PositiveInt = 60
+    openai_base_url: str = "https://api.openai.com/v1"
+    openai_api_key: SecretStr = Field("", exclude=True)
+
+    @field_validator("openai_base_url", mode="before")
+    @classmethod
+    def convert_base_url_to_str(cls, openai_base_url: HttpUrl) -> str:
+        return str(openai_base_url)
+
+
+class Setting(BaseSettings):
+    project: ProjectSettings = {}  # type: ignore
+    chat_completion: ChatCompletionSettings = {}  # type: ignore
+
+
+class SettingsManager:
+    _setting_instance: Optional[Setting] = (
+        None  # Private class attribute, initially None
+    )
+
+    @classmethod
+    def get_setting(cls):
+        if cls._setting_instance is None:
+            cls._setting_instance = Setting()
+        return cls._setting_instance
+
+    @classmethod
+    def initialize_with_params(
+        cls,
+        target_repo: Path,
+        markdown_docs_name: str,
+        hierarchy_name: str,
+        ignore_list: list[str],
+        language: str,
+        max_thread_count: int,
+        model: str,
+        temperature: float,
+        request_timeout: int,
+        openai_base_url: str,
+    ):
+        project_settings = ProjectSettings(
+            target_repo=target_repo,
+            hierarchy_name=hierarchy_name,
+            markdown_docs_name=markdown_docs_name,
+            ignore_list=ignore_list,
+            language=language,
+            max_thread_count=max_thread_count,
+        )
+
+        chat_completion_settings = ChatCompletionSettings(
+            model=model,
+            temperature=temperature,
+            request_timeout=request_timeout,
+            openai_base_url=openai_base_url,
+        )
+
+        cls._setting_instance = Setting(
+            project=project_settings,
+            chat_completion=chat_completion_settings,
+        )
+
+
+if __name__ == "__main__":
+    setting = SettingsManager.get_setting()
+    print(setting.model_dump())

bioguider/utils/constants.py

@@ -0,0 +1,40 @@
+
+from enum import Enum
+from typing import Optional
+
+DEFAULT_TOKEN_USAGE = {
+    "total_tokens": 0,
+    "completion_tokens": 0,
+    "prompt_tokens": 0,
+}
+
+class ProjectTypeEnum(Enum):
+    application="application"
+    package="package"
+    pipeline="pipeline"
+    unknown="unknown type"
+
+class PrimaryLanguageEnum(Enum):
+    python="python"
+    R="R"
+    unknown="unknown type"
+
+class ProjectMetadata:
+    def __init__(
+        self,
+        url: str,
+        project_type: ProjectTypeEnum,
+        primary_language: PrimaryLanguageEnum,
+        repo_name: str=None,
+        owner: Optional[str]=None,
+        description: Optional[str]=None,
+        license: Optional[str]=None,
+    ):
+        self.url = url
+        self.project_type = project_type
+        self.primary_language = primary_language
+        self.repo_name = repo_name
+        self.owner = owner
+        self.description = description
+        self.license = license
+

bioguider/utils/default.gitignore

@@ -0,0 +1,140 @@
+# History files
+.Rhistory
+.Rapp.history
+
+# R session data
+.RData
+.RDataTmp
+
+# User-specific files
+.Rproj.user/
+.Ruserdata/
+
+# RStudio files
+.rsituational.user
+.Renviron
+*proj.user
+
+# Example and temporary files
+*.Rhistory
+*.RData
+*.RData~
+*.Rproj.user
+*~
+.DS_Store # macOS specific
+Thumbs.db # Windows specific
+
+# Knitr and R Markdown cache and intermediate files
+*_cache/
+/cache/
+*.utf8.md
+*.knit.md
+*.log
+*.ind
+*.aux
+*.synctex.gz
+*.toc
+*.out
+*.nav
+*.snm
+
+# Package and dependency management
+# renv, packrat, etc.
+renv/
+packrat/
+.renv/
+.Rprofile
+
+# Compiled code from packages like Rcpp
+/src/*.so
+/src/*.o
+/src/*.dll
+
+# Shiny app logs
+*.log
+*.log.*
+
+
+# Byte-compiled / optimized / C files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are in 'dist' folder, but they may appear in the root directory
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Virtual Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDE and editor configuration files
+# VSCode
+.vscode/
+# PyCharm
+.idea/
+# Sublime Text
+*.sublime-project
+*.sublime-workspace
+# Atom
+.atom/
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Testing
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.tox/
+
+# Scrapy stuff
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# OS-generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db

bioguider/utils/file_utils.py

@@ -0,0 +1,126 @@
+import os
+from enum import Enum
+import json
+# from adalflow.utils import get_adalflow_default_root_path
+from pathlib import Path
+
+class FileType(Enum):
+    unknown = "u"
+    file = "f"
+    directory = "d"
+    symlink = "l"
+    broken_symlink = "broken symlink"
+
+def get_file_type(file_path: str) -> FileType:
+    """
+    Get the file type of a given file path.
+    
+    Args:
+        file_path (str): The path to the file or directory.
+    
+    Returns:
+        FileType: The type of the file (file, directory, or symlink).
+    """
+    if os.path.isfile(file_path):
+        return FileType.file
+    elif os.path.isdir(file_path):
+        return FileType.directory
+    elif os.path.islink(file_path):
+        try:
+            os.stat(file_path)
+            return FileType.symlink
+        except FileNotFoundError:
+            return FileType.broken_symlink
+        except Exception:
+            return FileType.unknown
+    else:
+        # raise ValueError(f"Unknown file type for path: {file_path}")
+        return FileType.unknown
+
+def remove_output_cells(notebook_path: str) -> str:
+    """
+    Remove output cells from a Jupyter notebook to reduce its size.
+
+    Args:
+        notebook_path (str): Path to the input Jupyter notebook file.
+        output_path (str): Path to save the modified notebook file.
+    """
+    with open(notebook_path, 'r', encoding='utf-8') as nb_file:
+        notebook = json.load(nb_file)
+
+    notebook['cells'] = [
+        cell for cell in notebook.get('cells', []) 
+        if cell.get('cell_type') != 'markdown'
+    ]
+    for cell in notebook.get('cells'):
+        if cell.get('cell_type') == 'code':
+            cell['outputs'] = []
+            cell['execution_count'] = None
+        
+
+    return json.dumps(notebook)
+
+def extract_code_from_notebook(notebook_path: str) -> str:
+    """
+    Extract all code from a Jupyter notebook.
+
+    Args:
+        notebook_path (str): Path to the input Jupyter notebook file.
+
+    Returns:
+        str: A concatenated string of all code cells.
+    """
+    with open(notebook_path, 'r', encoding='utf-8') as nb_file:
+        notebook = json.load(nb_file)
+
+    # Extract code from cells of type 'code'
+    code_cells = [
+        '\n'.join(cell['source']) for cell in notebook.get('cells', [])
+        if cell.get('cell_type') == 'code'
+    ]
+    code_cells = [
+        cell.replace("\n\n", "\n") for cell in code_cells
+    ]
+
+    # Combine all code cells into a single string
+    return '\n\n'.join(code_cells)
+
+def parse_repo_url(url: str) -> tuple[str | None, str | None]:
+    """
+    Parses a git repository URL to extract the author/organization and repository name.
+
+    Args:
+        url: The repository URL (e.g., HTTPS or SSH).
+
+    Returns:
+        A tuple containing (author_or_org, repo_name), or (None, None) if parsing fails.
+    """
+    try:
+        # Handle SSH format first (e.g., git@github.com:user/repo.git)
+        if '@' in url and ':' in url:
+            path_part = url.split(':')[-1]
+        # Handle HTTPS format (e.g., https://github.com/user/repo.git)
+        else:
+            path_part = url.split('://')[-1].split('/', 1)[-1]
+
+        # Clean up the path
+        if path_part.endswith('.git'):
+            path_part = path_part[:-4]
+
+        parts = path_part.split('/')
+        if len(parts) >= 2:
+            author = parts[-2]
+            repo_name = parts[-1]
+            return author, repo_name
+        else:
+            return None, None
+    except Exception:
+        return None, None
+
+def retrieve_data_root_path():
+    data_folder = os.environ.get("DATA_FOLDER", "./data")
+    return data_folder
+
+
+
+