bioguider 0.2.52__py3-none-any.whl

bioguider/utils/file_utils.py

@@ -0,0 +1,215 @@
+import os
+from enum import Enum
+import json
+# from adalflow.utils import get_adalflow_default_root_path
+from pathlib import Path
+from typing import Union, List, Optional, Tuple
+
+import os
+import string
+
+try:
+    import magic  # optional: pip install python-magic
+    HAS_MAGIC = True
+except ImportError:
+    HAS_MAGIC = False
+
+class FileType(Enum):
+    unknown = "u"
+    file = "f"
+    directory = "d"
+    symlink = "l"
+    broken_symlink = "broken symlink"
+
+def get_file_type(file_path: str) -> FileType:
+    """
+    Get the file type of a given file path.
+    
+    Args:
+        file_path (str): The path to the file or directory.
+    
+    Returns:
+        FileType: The type of the file (file, directory, or symlink).
+    """
+    if os.path.isfile(file_path):
+        return FileType.file
+    elif os.path.isdir(file_path):
+        return FileType.directory
+    elif os.path.islink(file_path):
+        try:
+            os.stat(file_path)
+            return FileType.symlink
+        except FileNotFoundError:
+            return FileType.broken_symlink
+        except Exception:
+            return FileType.unknown
+    else:
+        # raise ValueError(f"Unknown file type for path: {file_path}")
+        return FileType.unknown
+
+def remove_output_cells(notebook_path: str) -> str:
+    """
+    Remove output cells from a Jupyter notebook to reduce its size.
+
+    Args:
+        notebook_path (str): Path to the input Jupyter notebook file.
+        output_path (str): Path to save the modified notebook file.
+    """
+    with open(notebook_path, 'r', encoding='utf-8') as nb_file:
+        notebook = json.load(nb_file)
+
+    notebook['cells'] = [
+        cell for cell in notebook.get('cells', []) 
+        if cell.get('cell_type') != 'markdown'
+    ]
+    for cell in notebook.get('cells'):
+        if cell.get('cell_type') == 'code':
+            cell['outputs'] = []
+            cell['execution_count'] = None
+        
+
+    return json.dumps(notebook)
+
+def extract_code_from_notebook(notebook_path: str) -> str:
+    """
+    Extract all code from a Jupyter notebook.
+
+    Args:
+        notebook_path (str): Path to the input Jupyter notebook file.
+
+    Returns:
+        str: A concatenated string of all code cells.
+    """
+    with open(notebook_path, 'r', encoding='utf-8') as nb_file:
+        notebook = json.load(nb_file)
+
+    # Extract code from cells of type 'code'
+    code_cells = [
+        '\n'.join(cell['source']) for cell in notebook.get('cells', [])
+        if cell.get('cell_type') == 'code'
+    ]
+    code_cells = [
+        cell.replace("\n\n", "\n") for cell in code_cells
+    ]
+
+    # Combine all code cells into a single string
+    return '\n\n'.join(code_cells)
+
+def parse_repo_url(url: str) -> Tuple[Optional[str], Optional[str]]:
+    """
+    Parses a git repository URL to extract the author/organization and repository name.
+
+    Args:
+        url: The repository URL (e.g., HTTPS or SSH).
+
+    Returns:
+        A tuple containing (author_or_org, repo_name), or (None, None) if parsing fails.
+    """
+    try:
+        # Handle SSH format first (e.g., git@github.com:user/repo.git)
+        if '@' in url and ':' in url:
+            path_part = url.split(':')[-1]
+        # Handle HTTPS format (e.g., https://github.com/user/repo.git)
+        else:
+            path_part = url.split('://')[-1].split('/', 1)[-1]
+
+        # Clean up the path
+        if path_part.endswith('.git'):
+            path_part = path_part[:-4]
+
+        parts = path_part.split('/')
+        if len(parts) >= 2:
+            author = parts[-2]
+            repo_name = parts[-1]
+            return author, repo_name
+        else:
+            return None, None
+    except Exception:
+        return None, None
+
+def parse_refined_repo_path(refined_repo_path: str) -> Tuple[Optional[str], Optional[str]]:
+    repo_path = refined_repo_path.split("/")[-1]
+    arr = repo_path.split("_")
+    repo_name = arr[-1] if len(arr) > 1 else repo_path
+    author = arr[0] if len(arr) > 1 else ""
+    return author, repo_name
+
+def retrieve_data_root_path() -> Path:
+    data_folder = os.environ.get("DATA_FOLDER", "./data")
+    root_folder = Path(data_folder, ".adalflow")
+    return root_folder.absolute()
+    
+def detect_file_type(filepath, blocksize=2048, use_magic=True):
+    """
+    Detect if a file is text or binary.
+
+    Args:
+        filepath (str): Path to file.
+        blocksize (int): Number of bytes to read for inspection.
+        use_magic (bool): Use python-magic if available.
+
+    Returns:
+        str: "text" or "binary"
+    """
+    # Option 1: Use python-magic if available and requested
+    if use_magic and HAS_MAGIC:
+        try:
+            mime = magic.from_file(filepath, mime=True)
+            if mime and mime.startswith("text/"):
+                return "text"
+            return "binary"
+        except Exception:
+            pass  # fallback to heuristic
+
+    # Option 2: Heuristic detection
+    with open(filepath, "rb") as f:
+        chunk = f.read(blocksize)
+        if not chunk:  # empty file → treat as text
+            return "text"
+
+        # Null byte check
+        if b"\0" in chunk:
+            return "binary"
+
+        # Check ratio of non-printable characters
+        text_chars = bytearray(string.printable, "ascii")
+        nontext = chunk.translate(None, text_chars)
+        if float(len(nontext)) / len(chunk) > 0.30:
+            return "binary"
+
+    return "text"
+
+def flatten_files(repo_path: Union[str, Path], files: Optional[List[str]]) -> List[str]:
+    """
+    Flatten directories into individual files.
+    
+    Args:
+        repo_path (Union[str, Path]): The root path of the repository
+        files (Optional[List[str]]): List of file/directory paths to flatten
+        
+    Returns:
+        List[str]: List of individual file paths (directories are expanded to their contents)
+    """
+    if files is None:
+        return []
+    
+    flattened = []
+    repo_path = Path(repo_path)
+    
+    for file_path in files:
+        full_path = repo_path / file_path
+        
+        if full_path.is_dir():
+            # If it's a directory, recursively get all files in it
+            for item in full_path.rglob("*"):
+                if item.is_file():
+                    # Get relative path from repo_path
+                    rel_path = item.relative_to(repo_path)
+                    flattened.append(str(rel_path))
+        elif full_path.is_file():
+            # If it's already a file, just add it
+            flattened.append(file_path)
+        # Skip if path doesn't exist
+    
+    return flattened
+

bioguider/utils/gitignore_checker.py

@@ -0,0 +1,175 @@
+import fnmatch
+import os
+from pathlib import Path
+from typing import Callable
+
+class GitignoreChecker:
+    def __init__(
+        self, 
+        directory: str, 
+        gitignore_path: str,
+        exclude_dir_patterns: list[str] | None = None,
+        exclude_file_patterns: list[str] | None = None
+    ):
+        """
+        Initialize the GitignoreChecker with a specific directory and the path to a .gitignore file.
+
+        Args:
+            directory (str): The directory to be checked.
+            gitignore_path (str): The path to the .gitignore file.
+        """
+        self.directory = directory
+        self.gitignore_path = gitignore_path
+        self.folder_patterns, self.file_patterns = self._load_gitignore_patterns()
+        self.exclude_dir_patterns = exclude_dir_patterns
+        self.exclude_file_patterns = exclude_file_patterns
+
+    def _load_gitignore_patterns(self) -> tuple:
+        """
+        Load and parse the .gitignore file, then split the patterns into folder and file patterns.
+
+        If the specified .gitignore file is not found, fall back to the default path.
+
+        Returns:
+            tuple: A tuple containing two lists - one for folder patterns and one for file patterns.
+        """
+        try:
+            with open(self.gitignore_path, "r", encoding="utf-8") as file:
+                gitignore_content = file.read()
+        except FileNotFoundError:
+            # Fallback to the default .gitignore path if the specified file is not found
+            default_path = os.path.join(
+                os.path.dirname(__file__), "default.gitignore"
+            )
+            with open(default_path, "r", encoding="utf-8") as file:
+                gitignore_content = file.read()
+
+        patterns = self._parse_gitignore(gitignore_content)
+        return self._split_gitignore_patterns(patterns)
+
+    @staticmethod
+    def _parse_gitignore(gitignore_content: str) -> list:
+        """
+        Parse the .gitignore content and return patterns as a list.
+
+        Args:
+            gitignore_content (str): The content of the .gitignore file.
+
+        Returns:
+            list: A list of patterns extracted from the .gitignore content.
+        """
+        patterns = []
+        for line in gitignore_content.splitlines():
+            line = line.strip()
+            if line and not line.startswith("#"):
+                patterns.append(line)
+        return patterns
+
+    @staticmethod
+    def _split_gitignore_patterns(gitignore_patterns: list) -> tuple:
+        """
+        Split the .gitignore patterns into folder patterns and file patterns.
+
+        Args:
+            gitignore_patterns (list): A list of patterns from the .gitignore file.
+
+        Returns:
+            tuple: Two lists, one for folder patterns and one for file patterns.
+        """
+        folder_patterns = []
+        file_patterns = []
+        for pattern in gitignore_patterns:
+            if pattern.endswith("/"):
+                folder_patterns.append(pattern.rstrip("/"))
+            else:
+                file_patterns.append(pattern)
+        return folder_patterns, file_patterns
+
+    @staticmethod
+    def _is_ignored(path: str, patterns: list, is_dir: bool = False) -> bool:
+        """
+        Check if the given path matches any of the patterns.
+
+        Args:
+            path (str): The path to check.
+            patterns (list): A list of patterns to check against.
+            is_dir (bool): True if the path is a directory, False otherwise.
+
+        Returns:
+            bool: True if the path matches any pattern, False otherwise.
+        """
+        for pattern in patterns:
+            if fnmatch.fnmatch(path, pattern):
+                return True
+            if is_dir and pattern.endswith("/") and fnmatch.fnmatch(path, pattern[:-1]):
+                return True
+        return False
+    
+    @staticmethod
+    def _is_ignored_by_default(path: str, is_dir: bool=False) -> bool:
+        return is_dir and path.startswith(".") # path == ".git" 
+
+
+    def _is_ignored_by_exclude_file_patterns(self, f: str):
+        if self.exclude_file_patterns is None:
+            return False
+        return True if self._is_ignored(f, self.exclude_file_patterns) else False
+
+
+    def check_files_and_folders(
+        self, 
+        level=-1, 
+        check_file_cb: Callable[[str, str], bool] | None = None
+    ) -> list:
+        """
+        Check all files and folders in the given directory against the split gitignore patterns.
+        Return a list of files that are not ignored.
+        The returned file paths are relative to the self.directory.
+
+        Returns:
+            list: A list of paths to files that are not ignored.
+        """
+        not_ignored_files = []
+        root_path = Path(self.directory)
+        for root, dirs, files in os.walk(self.directory):
+            current_root_path = Path(root)
+            current_levels = len(current_root_path.relative_to(root_path).parts)
+            if level >= 0 and current_levels > level:
+                continue
+            dirs[:] = [
+                d
+                for d in dirs
+                if not self._is_ignored(d, self.folder_patterns, is_dir=True) \
+                and not self._is_ignored_by_default(d, True)
+            ]
+            if self.exclude_dir_patterns:
+                dirs[:] = [
+                    d
+                    for d in dirs
+                    if not self._is_ignored(d, self.exclude_dir_patterns, is_dir=True)
+                ]
+
+            for file in files:
+                file_path = os.path.join(root, file)
+                relative_path = os.path.relpath(file_path, self.directory)
+                if not self._is_ignored(
+                    file, self.file_patterns
+                ) and not self._is_ignored_by_exclude_file_patterns(file):
+                    if check_file_cb is None:
+                        not_ignored_files.append(relative_path)
+                    else:
+                        if check_file_cb(self.directory, relative_path):
+                            not_ignored_files.append(relative_path)
+            
+            if level >= 0 and current_levels == level:
+                not_ignored_files = \
+                    not_ignored_files + \
+                    [os.path.relpath(os.path.join(root, d), self.directory) for d in dirs]
+
+        return not_ignored_files
+
+
+# Example usage:
+# gitignore_checker = GitignoreChecker('path_to_directory', 'path_to_gitignore_file')
+# not_ignored_files = gitignore_checker.check_files_and_folders()
+# print(not_ignored_files)

bioguider/utils/notebook_utils.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+from pathlib import Path
+from typing import Union, Dict, Any, List
+import json
+
+def extract_markdown_from_notebook(
+    ipynb_path: Union[str, Path],
+    out_path: Union[str, Path, None] = None,
+) -> Dict[str, Any]:
+    """
+    Extract markdown from a Jupyter notebook.
+    """
+    ipynb_path = Path(ipynb_path)
+    if not ipynb_path.exists():
+        raise FileNotFoundError(f"File {ipynb_path} does not exist")
+    try:
+        with ipynb_path.open("r", encoding="utf-8") as f:
+            nb = json.load(f)
+    except json.JSONDecodeError:
+        raise ValueError(f"File {ipynb_path} is not a valid JSON file")
+
+    markdown_txts = [
+        "\n".join(cell.get("source")) if isinstance(cell.get("source"), list) else cell.get("source") for cell in nb.get("cells", [])
+        if cell.get("cell_type") == "markdown"
+    ]
+    text = "\n".join(markdown_txts)
+    if out_path is not None:
+        with open(out_path, "w", encoding="utf-8") as f:
+            f.write(text)
+    return text
+
+def strip_notebook_to_code_and_markdown(
+    ipynb_path: Union[str, Path],
+    out_path: Union[str, Path, None] = None,
+    keep_top_metadata: bool = True,
+) -> Dict[str, Any]:
+    """
+    Load a .ipynb and return a new notebook that:
+      - keeps ONLY 'code' and 'markdown' cells
+      - empties outputs and execution_count for code cells
+      - drops all other cell types (e.g., 'raw')
+      - preserves attachments on markdown cells
+      - optionally preserves top-level metadata (kernelspec, language_info, etc.)
+
+    Parameters
+    ----------
+    ipynb_path : str | Path
+        Path to the input .ipynb file.
+    out_path : str | Path | None, default None
+        If provided, write the cleaned notebook to this path.
+    keep_top_metadata : bool, default True
+        If True, copy top-level metadata as-is (useful for re-running).
+        If False, keep only minimal metadata.
+
+    Returns
+    -------
+    dict
+        The cleaned notebook (nbformat v4-style dict).
+    """
+    ipynb_path = Path(ipynb_path)
+    if not ipynb_path.exists():
+        raise FileNotFoundError(f"File {ipynb_path} does not exist")
+    try:
+        with ipynb_path.open("r", encoding="utf-8") as f:
+            nb = json.load(f)
+    except json.JSONDecodeError:
+        raise ValueError(f"File {ipynb_path} is not a valid JSON file")
+
+    nbformat = nb.get("nbformat", 4)
+    nbformat_minor = nb.get("nbformat_minor", 5)
+
+    def _to_text(src) -> str:
+        # nbformat allows str or list of lines
+        if isinstance(src, list):
+            return "".join(src)
+        return src or ""
+
+    new_cells: List[Dict[str, Any]] = []
+    for cell in nb.get("cells", []):
+        ctype = cell.get("cell_type")
+        if ctype == "markdown":
+            new_cell = {
+                "cell_type": "markdown",
+                "metadata": cell.get("metadata", {}),
+                "source": _to_text(cell.get("source", "")),
+            }
+            if "attachments" in cell:
+                new_cell["attachments"] = cell["attachments"]
+            new_cells.append(new_cell)
+
+        elif ctype == "code":
+            new_cells.append({
+                "cell_type": "code",
+                "metadata": cell.get("metadata", {}),
+                "source": _to_text(cell.get("source", "")),
+                "execution_count": None,   # clear execution count
+                "outputs": [],             # strip ALL outputs
+            })
+
+        # else: drop 'raw' and any other unknown cell types
+
+    # Build new notebook object
+    new_nb: Dict[str, Any] = {
+        "nbformat": nbformat,
+        "nbformat_minor": nbformat_minor,
+        "metadata": nb.get("metadata", {}) if keep_top_metadata else {},
+        "cells": new_cells,
+    }
+
+    if out_path is not None:
+        out_path = Path(out_path)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        with out_path.open("w", encoding="utf-8") as f:
+            json.dump(new_nb, f, ensure_ascii=False, indent=1)
+
+    return new_nb
+

bioguider/utils/pyphen_utils.py

@@ -0,0 +1,73 @@
+
+import os
+import re
+import pyphen
+import math
+
+class PyphenReadability:
+    def __init__(self, lang='en'):
+        self.dic = pyphen.Pyphen(lang=lang)
+
+    def count_syllables(self, word):
+        return self.dic.inserted(word).count('-') + 1 if word.isalpha() else 0
+    
+    def extract_urls(self, text):
+        """Find all URLs in the text."""
+        url_pattern = r'https?://\S+|www\.\S+'
+        return re.findall(url_pattern, text)
+    
+    def remove_urls(self, text):
+        """Remove URLs from text for clean sentence splitting."""
+        url_pattern = r'https?://\S+|www\.\S+'
+        return re.sub(url_pattern, '', text)
+    
+    def split_sentences(self, text):
+        """Split into sentences using punctuation."""
+        return re.split(r'[.!?]+', text)
+    
+    def split_words(self, text):
+        """Extract words."""
+        return re.findall(r'\b\w+\b', text)
+    
+    def is_polysyllabic(self, word):
+        return self.count_syllables(word) >= 3
+    
+    def is_complex(self, word):
+        return self.is_polysyllabic(word)
+    
+    def readability_metrics(self, text):
+        # Extract and remove URLs
+        urls = self.extract_urls(text)
+        url_count = len(urls)
+        text_without_urls = self.remove_urls(text)
+    
+        # Split and count
+        sentences = [s for s in self.split_sentences(text_without_urls) if s.strip()]
+        sentence_count = len(sentences) + url_count
+    
+        words = self.split_words(text) # split_words(text_without_urls)
+        word_count = len(words)
+    
+        syllable_count = sum(self.count_syllables(w) for w in words)
+        polysyllables = sum(1 for w in words if self.is_polysyllabic(w))
+        complex_words = sum(1 for w in words if self.is_complex(w))
+    
+        # Avoid division by zero
+        words_per_sentence = word_count / sentence_count if sentence_count > 0 else 0
+        syllables_per_word = syllable_count / word_count if word_count > 0 else 0
+        complex_per_word = complex_words / word_count if word_count > 0 else 0
+    
+        # Readability formulas
+        flesch_reading_ease = 206.835 - 1.015 * words_per_sentence - 84.6 * syllables_per_word
+        flesch_kincaid_grade = 0.39 * words_per_sentence + 11.8 * syllables_per_word - 15.59
+        gunning_fog_index = 0.4 * (words_per_sentence + 100 * complex_per_word)
+        smog_index = (
+            1.043 * math.sqrt(polysyllables * (30 / sentence_count)) + 3.1291
+            if sentence_count >= 1 else 0
+        )
+    
+        return flesch_reading_ease, flesch_kincaid_grade, gunning_fog_index, smog_index,\
+             sentence_count, word_count, syllable_count, polysyllables, complex_words
+    
+
+    

bioguider/utils/python_file_handler.py

@@ -0,0 +1,65 @@
+import ast
+import os
+
+class PythonFileHandler:
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+
+    def get_functions_and_classes(self) -> list[str]:
+        """
+        Get the functions and classes in a given file.
+        Returns a list of tuples, each containing:
+        1. the function or class name,
+        2. parent name,
+        3. start line number,
+        4. end line number,
+        5. doc string,
+        6. params.
+        """
+        with open(self.file_path, 'r') as f:
+            tree = ast.parse(f.read())
+            functions_and_classes = []
+            for node in tree.body:
+                if isinstance(node, ast.FunctionDef) or isinstance(node, ast.ClassDef):
+                    start_lineno = node.lineno
+                    end_lineno = self.get_end_lineno(node)
+                    doc_string = ast.get_docstring(node)
+                    params = (
+                        [arg.arg for arg in node.args.args] if "args" in dir(node) else []
+                    )
+                    parent = None
+                    functions_and_classes.append((node.name, parent, start_lineno, end_lineno, doc_string, params))
+                    for child in node.body:
+                        if isinstance(child, ast.FunctionDef):
+                            start_lineno = child.lineno
+                            end_lineno = self.get_end_lineno(child)
+                            doc_string = ast.get_docstring(child)
+                            params = (
+                                [arg.arg for arg in child.args.args] if "args" in dir(child) else []
+                            )
+                            parent = node.name
+                            functions_and_classes.append((child.name, parent, start_lineno, end_lineno, doc_string, params))
+            return functions_and_classes
+    
+    def get_imports(self) -> list[str]:
+        pass
+
+    def get_end_lineno(self, node):
+        """
+        Get the end line number of a given node.
+
+        Args:
+            node: The node for which to find the end line number.
+
+        Returns:
+            int: The end line number of the node. Returns -1 if the node does not have a line number.
+        """
+        if not hasattr(node, "lineno"):
+            return -1  # 返回-1表示此节点没有行号
+
+        end_lineno = node.lineno
+        for child in ast.iter_child_nodes(node):
+            child_end = getattr(child, "end_lineno", None) or self.get_end_lineno(child)
+            if child_end > -1:  # 只更新当子节点有有效行号时
+                end_lineno = max(end_lineno, child_end)
+        return end_lineno