aurelian 0.1.0__py3-none-any.whl

aurelian/utils/doi_fetcher.py

@@ -0,0 +1,238 @@
+import os
+import re
+from tempfile import NamedTemporaryFile
+from typing import Any, Dict, List, Optional
+
+import logfire
+import requests
+import requests_cache
+from bs4 import BeautifulSoup
+from markitdown import MarkItDown
+from openai import BaseModel
+from pydantic import Field
+
+
+class FullTextInfo(BaseModel):
+    """Data model for full text information."""
+
+    success: bool = True
+    abstract: Optional[str] = Field(None, description="Abstract of the article")
+    text: Optional[str] = Field(None, description="Full text of the article")
+    source: Optional[str] = Field(None, description="Source of the full text")
+    metadata: Optional[Dict[str, Any]] = Field(None, description="Metadata of the article")
+    pdf_url: Optional[str] = Field(None, description="URL to the PDF version of the article")
+
+
+class DOIFetcher:
+    """Fetch metadata and full text for a DOI using various APIs."""
+
+    def __init__(self, email: Optional[str] = None, url_prefixes: Optional[List[str]] = None):
+        """Initialize the DOI fetcher with a contact email (required by some APIs).
+
+        Args:
+            email (str): Contact email for API access
+            url_prefixes (List[str]): List of URL prefixes to check for full text
+
+        """
+        self.email = email or os.getenv("EMAIL") or "test@example.com"
+        self.url_prefixes = url_prefixes or os.getenv("DOI_FULL_TEXT_URLS", "").split(",")
+        self.headers = {"User-Agent": f"DOIFetcher/1.0 (mailto:{email})", "Accept": "application/json"}
+
+    def clean_text(self, text: str) -> str:
+        """Clean extracted text by removing extra whitespace and normalized characters.
+
+        Args:
+            text:
+
+        Returns:
+            str: The cleaned text
+
+        """
+        # Remove extra whitespace
+        text = re.sub(r"\s+", " ", text)
+        # Remove non-printable characters
+        text = "".join(char for char in text if char.isprintable())
+        return text.strip()
+
+    def get_metadata(self, doi: str, strict=False) -> Optional[Dict[str, Any]]:
+        """Fetch metadata for a DOI using the Crossref API.
+
+        Args:
+            doi (str): The DOI to look up
+            strict (bool): Raise exceptions if API call fails
+
+        Returns:
+            Optional[Dict[str, Any]]: Metadata dictionary if successful, None otherwise
+
+        """
+        base_url = "https://api.crossref.org/works/"
+        try:
+            response = requests.get(f"{base_url}{doi}", headers=self.headers)
+            response.raise_for_status()
+            return response.json()["message"]
+        except Exception as e:
+            if strict:
+                raise e
+            logfire.warn(f"Error fetching metadata: {e}")
+            return None
+
+    def get_unpaywall_info(self, doi: str, strict=False) -> Optional[Dict[str, Any]]:
+        """Check Unpaywall for open access versions.
+
+        Example:
+            >>> fetcher = DOIFetcher()
+            >>> doi = "10.1038/nature12373"
+            >>> unpaywall_data = fetcher.get_unpaywall_info(doi)
+            >>> assert unpaywall_data["doi"] == doi
+            >>> unpaywall_data["best_oa_location"]["url_for_pdf"]
+            'https://europepmc.org/articles/pmc4221854?pdf=render'
+
+        Args:
+            doi (str): The DOI to look up
+            strict (bool): Raise exceptions if API call fails
+
+        Returns:
+            Optional[Dict[str, Any]]: Unpaywall data if successful, None otherwise
+
+        """
+        base_url = f"https://api.unpaywall.org/v2/{doi}"
+        try:
+            response = requests.get(f"{base_url}?email={self.email}")
+            response.raise_for_status()
+            return response.json()
+        except Exception as e:
+            if strict:
+                raise e
+            logfire.warn(f"Error fetching Unpaywall data: {e}")
+            return None
+
+    def get_full_text(self, doi: str, fallback_to_abstract=True) -> Optional[str]:
+        """Get the full text of a paper using various methods.
+
+        Example:
+            >>> fetcher = DOIFetcher()
+            >>> doi = "10.1128/msystems.00045-18"
+            >>> full_text = fetcher.get_full_text(doi)
+            >>> assert "Populus Microbiome" in full_text
+
+        Args:
+            doi:
+            fallback_to_abstract:
+
+        Returns:
+            str: The full text if available, else abstract text if fallback_to_abstract,
+              else None
+
+        """
+        info = self.get_full_text_info(doi)
+        if not info:
+            return None
+        text = info.text
+        if text:
+            return self.clean_text(text)
+        if info.pdf_url:
+            text = self.text_from_pdf_url(info.pdf_url)
+            if text:
+                return self.clean_text(text)
+        message = "FULL TEXT NOT AVAILABLE"
+        if fallback_to_abstract:
+            metadata = info.metadata or {}
+            abstract = metadata.get("abstract")
+            if abstract:
+                return self.clean_text(abstract) + f"\n\n{message}"
+        return message
+
+    def get_full_text_info(self, doi: str) -> Optional[FullTextInfo]:
+        """Attempt to get the full text of a paper using various methods.
+
+            >>> fetcher = DOIFetcher()
+            >>> doi = "10.1128/msystems.00045-18"
+            >>> info = fetcher.get_full_text_info(doi)
+            >>> metadata = info.metadata
+            >>> metadata["type"]
+            'journal-article'
+            >>> metadata["title"][0][0:20]
+            'Exploration of the B'
+            >>> assert info.pdf_url is not None
+            >>> info.pdf_url
+            'https://europepmc.org/articles/pmc6172771?pdf=render'
+
+        Args:
+            doi (str): The DOI to fetch
+
+        Returns:
+            FullTextInfo: Full text information
+
+        """
+        # Get metadata
+        metadata = self.get_metadata(doi)
+
+        # Check Unpaywall
+        unpaywall_data = self.get_unpaywall_info(doi)
+        if unpaywall_data and unpaywall_data.get("is_oa"):
+            locations = unpaywall_data.get("oa_locations", [])
+            if unpaywall_data.get("best_oa_location"):
+                best_oa_location = unpaywall_data.get("best_oa_location")
+                locations = [best_oa_location] + locations
+
+            # Find best open access location
+            for location in locations:
+                pdf_url = location.get("url_for_pdf")
+                if pdf_url:
+                    return FullTextInfo(text=None, pdf_url=pdf_url, source="unpaywall", metadata=metadata)
+
+        # Fallback
+        url_prefixes = os.getenv("DOI_FULL_TEXT_URLS", "").split(",")
+
+        for url_prefix in url_prefixes:
+            url_prefix.rstrip("/")
+            url = f"{url_prefix}/{doi}"
+            try:
+                response = requests.get(url)
+                if response.status_code == 200:
+                    soup = BeautifulSoup(response.text, "html.parser")
+                    pdf_embed = soup.find("embed", id="pdf")
+                    if pdf_embed and pdf_embed.get("src"):
+                        pdf_url = pdf_embed["src"]
+                        # Remove any URL parameters after #
+                        pdf_url = pdf_url.split("#")[0]
+                        if not pdf_url.startswith("http"):
+                            pdf_url = "https:" + pdf_url
+                        return FullTextInfo(
+                            pdf_url=pdf_url,
+                            source=url,
+                            metadata=metadata,
+                        )
+            except Exception:
+                continue
+
+    def text_from_pdf_url(self, pdf_url: str, raise_for_status=False) -> Optional[str]:
+        """Extract text from a PDF URL.
+
+        Example:
+            >>> fetcher = DOIFetcher()
+            >>> pdf_url = "https://ceur-ws.org/Vol-1747/IT201_ICBO2016.pdf"
+            >>> text = fetcher.text_from_pdf_url(pdf_url)
+            >>> assert "biosphere" in text
+
+        Args:
+            pdf_url:
+            raise_for_status:
+
+        Returns:
+
+        """
+        session = requests_cache.CachedSession("pdf_cache")
+        # Download the PDF
+        response = session.get(pdf_url)
+        if raise_for_status:
+            response.raise_for_status()
+        if response.status_code != 200:
+            return None
+        with NamedTemporaryFile(delete=False) as tmpf:
+            tmpf.write(response.content)
+            tmp_name = tmpf.name
+            with open(tmp_name, "wb") as f:
+                f.write(response.content)
+            md = MarkItDown()
+            return md.convert(tmpf.name).text_content

aurelian/utils/ontology_utils.py

@@ -0,0 +1,68 @@
+import logfire
+import pystow
+from cachetools.func import lru_cache
+from linkml_store.api import Collection
+from linkml_store.api.stores.duckdb import DuckDBDatabase
+from linkml_store.index import LLMIndexer
+from oaklib import BasicOntologyInterface, get_adapter
+
+llm_indexer = LLMIndexer()
+
+
+@lru_cache
+def get_collection_for_adapter(handle: str, name: str) -> Collection:
+    """
+    Retrieve or create a cached ontology collection.
+
+    Args:
+        handle (str): The ontology handle (e.g., `sqlite:obo:uberon`).
+        name (str): The name of the ontology (e.g., `uberon`).
+
+    Returns:
+        Collection: The indexed ontology collection.
+    """
+    adapter = get_adapter(handle)
+    cache_dir = pystow.join("aurelian", "indexes")
+    duckdb_path = str(cache_dir / f"{name}.duckdb")
+    database = DuckDBDatabase(duckdb_path)
+    collection = database.get_collection(name, create_if_not_exists=True)
+
+    if collection.size() > 0:
+        return collection
+
+    objs = [{"id": id, "label": lbl} for id, lbl in adapter.labels(adapter.entities())]
+    collection.insert(objs)
+    return collection
+
+
+def search_ontology(adapter: BasicOntologyInterface, query: str, limit=10):
+    """
+    Search the ontology for the given query term.
+
+    Example:
+        >>> from oaklib import get_adapter
+        >>> adapter = get_adapter("sqlite:obo:uberon")
+        >>> terms = search_ontology(adapter, "manus")
+        >>> assert len(terms) > 1
+        >>> terms = search_ontology(adapter, "l~digit", limit=5)
+        >>> assert len(terms) == 5
+
+    Args:
+        adapter (BasicOntologyInterface): The ontology adapter.
+        query (str): The query term.
+        limit (int): The maximum number of search results to return.
+
+    Returns:
+        List[Tuple[str, str]]: A list of tuples containing ontology term IDs and labels.
+    """
+    scheme = adapter.resource.scheme
+    name = adapter.resource.slug
+    local_name = name.split(":")[-1]
+    handle = f"{scheme}:{name}"
+
+    collection = get_collection_for_adapter(handle, local_name)
+    with logfire.span("search_ontology {name} {query}", name=name, query=query):
+        print(f"Searching {scheme}:{name} for {query}")
+        qr = collection.search(query, limit=limit, index_name="llm")
+        objs = [(obj["id"], obj["label"]) for obj in qr.rows]
+    return objs

aurelian/utils/pdf_fetcher.py

@@ -0,0 +1,23 @@
+import tempfile
+import requests
+from pdfminer.high_level import extract_text
+
+
+def extract_text_from_pdf(pdf_url: str) -> str:
+    """
+    Download and extract text from a PDF given its URL, using a temporary file.
+    """
+    response = requests.get(pdf_url)
+    if response.status_code != 200:
+        return "Error: Unable to retrieve PDF."
+
+    try:
+        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=True) as temp_pdf:
+            temp_pdf.write(response.content)
+            temp_pdf.flush()  # Ensure all data is written before reading
+
+            text = extract_text(temp_pdf.name)
+            return text.strip() if text else "Error: No text extracted from PDF."
+
+    except Exception as e:
+        return f"Error extracting PDF text: {e}"

aurelian/utils/process_logs.py

@@ -0,0 +1,100 @@
+import json
+from pathlib import Path
+from collections import defaultdict
+import re
+
+
+def parse_reportlog(log_path: str):
+    """Parse pytest-reportlog output into structured format."""
+    tests = defaultdict(dict)
+
+    with open(log_path) as f:
+        for line in f:
+            entry = json.loads(line)
+
+            # Only process TestReport entries
+            if entry.get('$report_type') != 'TestReport':
+                continue
+
+            nodeid = entry['nodeid']
+
+            # Store test outcome
+            if 'outcome' in entry:
+                tests[nodeid]['outcome'] = entry['outcome']
+
+            # Store duration
+            if 'duration' in entry:
+                tests[nodeid]['duration'] = entry['duration']
+
+            # Convert user_properties to dict
+            if 'user_properties' in entry:
+                props = dict(entry['user_properties'])
+                tests[nodeid]['properties'] = props
+
+            # Store parameters from nodeid
+            # Extract from something like: test_search_ontology[sqlite:obo:bfo-3D spatial-10-expected0]
+            if '[' in nodeid:
+                param_str = nodeid[nodeid.index('[') + 1:nodeid.rindex(']')]
+                # You might want to customize this parsing based on your parameter format
+                tests[nodeid]['parameters'] = param_str
+
+    return tests
+
+
+def generate_markdown(tests):
+    """Convert test results to markdown documentation."""
+    md = []
+    md.append("# Test Results Documentation\n")
+
+    # Group tests by their base function name
+    test_groups = defaultdict(list)
+    for nodeid, data in tests.items():
+        # Split nodeid into parts: path::function[params]
+        base_name = nodeid.split('::')[1].split('[')[0] if '[' in nodeid else nodeid.split('::')[1]
+        test_groups[base_name].append((nodeid, data))
+
+    for base_name, group in test_groups.items():
+        md.append(f"## {base_name}\n")
+
+        # Create table for all test runs
+        md.append("### Test Runs\n")
+
+        # Headers: Parameters, Properties, Duration, Outcome
+        md.append('| Parameters | Properties | Duration (s) | Outcome |')
+        md.append('|------------|------------|-------------|---------|')
+
+        for nodeid, data in group:
+            # Extract parameters from nodeid
+            params = nodeid.split('[')[1].rstrip(']') if '[' in nodeid else ''
+
+            # Format properties
+            props = data.get('properties', {})
+            props_str = '; '.join(f"{k}: {v}" for k, v in props.items())
+
+            # Format duration
+            duration = f"{data.get('duration', 0):.3f}"
+
+            row = [
+                params,
+                props_str,
+                duration,
+                data.get('outcome', '')
+            ]
+
+            md.append('| ' + ' | '.join(str(cell) for cell in row) + ' |')
+
+        md.append('')
+    return '\n'.join(md)
+
+# Example usage:
+if __name__ == '__main__':
+    # Assume report.jsonl exists from running:
+    # pytest test_examples.py --report-log=report.jsonl
+
+    log_path = Path('report.jsonl')
+    tests = parse_reportlog(log_path)
+    markdown = generate_markdown(tests)
+
+    # Write markdown to file
+    with open('docs/unit_tests.md', 'w') as f:
+        f.write(markdown)

aurelian/utils/pubmed_utils.py

@@ -0,0 +1,238 @@
+import re
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+from aurelian.utils.doi_fetcher import DOIFetcher
+
+BIOC_URL = "https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_xml/{pmid}/ascii"
+PUBMED_EUTILS_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id={pmid}&retmode=xml"
+EFETCH_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={pmid}&retmode=xml"
+
+DOI_PATTERN = r"/(10\.\d{4,9}/[\w\-.]+)"
+
+doi_fetcher = DOIFetcher()
+
+
+def extract_doi_from_url(url: str) -> Optional[str]:
+    """Extracts the DOI from a given journal URL.
+
+    Args:
+        url (str): The URL of the article.
+
+    Returns:
+        str: The extracted DOI if found, otherwise an empty string.
+
+    """
+    doi_match = re.search(DOI_PATTERN, url)
+    return doi_match.group(1) if doi_match else None
+
+
+def doi_to_pmid(doi: str) -> Optional[str]:
+    """Converts a DOI to a PMID using the NCBI ID Converter API.
+
+    Args:
+        doi (str): The DOI to be converted.
+
+    Returns:
+        str: The corresponding PMID if found, otherwise an empty string.
+
+    """
+    API_URL = f"https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids={doi}&format=json"
+    response = requests.get(API_URL).json()
+    records = response.get("records", [])
+    pmid = records[0].get("pmid", None) if records else None
+    return pmid
+
+
+def get_doi_text(doi: str) -> str:
+    """Fetch the full text of an article using a DOI.
+
+    TODO: non pubmed sources
+
+    Example:
+        >>> doi = "10.1128/msystems.00045-18"
+        >>> full_text = get_doi_text(doi)
+        >>> assert "Populus Microbiome" in full_text
+
+    Args:
+        doi: The DOI of the article.
+
+    Returns:
+        The full text of the article if available, otherwise an empty string.
+
+    """
+    pmid = doi_to_pmid(doi)
+    if not pmid:
+        info = doi_fetcher.get_full_text(doi)
+        if info:
+            return info
+        else:
+            return f"PMID not found for {doi} and not available via unpaywall"
+    return get_pmid_text(pmid)
+
+
+def get_pmid_from_pmcid(pmcid):
+    """Fetch the PMID from a PMC ID using the Entrez E-utilities `esummary`.
+
+    Example:
+        >>> pmcid = "PMC5048378"
+        >>> pmid = get_pmid_from_pmcid(pmcid)
+        >>> print(pmid)
+        27629041
+
+    Args:
+        pmcid:
+
+    Returns:
+
+    """
+    if ":" in pmcid:
+        pmcid = pmcid.split(":")[1]
+    url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"
+    params = {"db": "pmc", "id": pmcid.replace("PMC", ""), "retmode": "json"}  # Remove "PMC" prefix if included
+
+    response = requests.get(url, params=params)
+    data = response.json()
+
+    # Extract PMID
+    try:
+        uid = data["result"]["uids"][0]  # Extract the UID
+        article_ids = data["result"][uid]["articleids"]  # Get article IDs
+        for item in article_ids:
+            if item["idtype"] == "pmid":
+                return item["value"]
+    except KeyError:
+        return "PMID not found"
+
+
+def get_pmcid_text(pmcid: str) -> str:
+    """Fetch full text from PubMed Central Open Access BioC XML.
+
+    Example:
+        >>> pmcid = "PMC5048378"
+        >>> full_text = get_pmcid_text(pmcid)
+        >>> assert "integrated stress response (ISR)" in full_text
+
+    Args:
+        pmcid:
+
+    Returns:
+
+    """
+    pmid = get_pmid_from_pmcid(pmcid)
+    return get_pmid_text(pmid)
+
+
+def get_pmid_text(pmid: str) -> str:
+    """Fetch full text from PubMed Central Open Access BioC XML.
+    If full text is not available, fallback to fetching the abstract from PubMed.
+
+    Example:
+        >>> pmid = "11"
+        >>> full_text = get_pmid_text(pmid)
+        >>> print(full_text)
+        Identification of adenylate cyclase-coupled beta-adrenergic receptors with radiolabeled beta-adrenergic antagonists.
+        <BLANKLINE>
+        No abstract available
+
+    Args:
+        pmid: PubMed ID of the article.
+
+    Returns:
+        The full text of the article if available, otherwise the abstract.
+
+    """
+    if ":" in pmid:
+        pmid = pmid.split(":")[1]
+    text = get_full_text_from_bioc(pmid)
+    if not text:
+        doi = pmid_to_doi(pmid)
+        if doi:
+            text = doi_fetcher.get_full_text(doi)
+    if not text:
+        text = get_abstract_from_pubmed(pmid)
+    return text
+
+def pmid_to_doi(pmid: str) -> Optional[str]:
+    if ":" in pmid:
+        pmid = pmid.split(":")[1]
+    url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id={pmid}&retmode=json"
+    response = requests.get(url)
+    data = response.json()
+
+    try:
+        article_info = data["result"][str(pmid)]
+        for aid in article_info["articleids"]:
+            if aid["idtype"] == "doi":
+                return aid["value"]
+        elocationid = article_info.get("elocationid", "")
+        if elocationid.startswith("10."):  # DOI starts with "10."
+            return elocationid
+        else:
+            return None
+    except KeyError:
+        return None
+
+
+def get_full_text_from_bioc(pmid: str) -> str:
+    """Fetch full text from PubMed Central Open Access BioC XML.
+
+    Example:
+        >>> pmid = "17299597"
+        >>> full_text = get_full_text_from_bioc(pmid)
+        >>> assert "Evolution of biological complexity." in full_text
+
+    Args:
+        pmid: PubMed ID of the article.
+
+    Returns:
+        The full text of the article if available, otherwise an empty string.
+
+    """
+    response = requests.get(BIOC_URL.format(pmid=pmid))
+
+    if response.status_code != 200:
+        return ""  # Return empty string if request fails
+
+    soup = BeautifulSoup(response.text, "xml")
+
+    # Extract ONLY text from <text> tags within <passage>
+    text_sections = [text_tag.get_text() for text_tag in soup.find_all("text")]
+
+    full_text = "\n".join(text_sections).strip()
+    return full_text
+
+
+def get_abstract_from_pubmed(pmid: str) -> str:
+    """Fetch the title and abstract of an article from PubMed using Entrez E-utilities `efetch`.
+
+    Example:
+        >>> pmid = "31653696"
+        >>> abstract = get_abstract_from_pubmed(pmid)
+        >>> assert "The apparent deglycase activity of DJ-1" in abstract
+
+    Args:
+        pmid: PubMed ID of the article.
+
+    Returns:
+        The title and abstract text if available, otherwise an empty string.
+
+    """
+    response = requests.get(EFETCH_URL.format(pmid=pmid))
+
+    if response.status_code != 200:
+        return ""
+
+    soup = BeautifulSoup(response.text, "xml")
+
+    # Extract title
+    title_tag = soup.find("ArticleTitle")
+    title = title_tag.get_text().strip() if title_tag else "No title available"
+
+    # Extract abstract (may contain multiple sections)
+    abstract_tags = soup.find_all("AbstractText")
+    abstract = "\n".join(tag.get_text().strip() for tag in abstract_tags) if abstract_tags else "No abstract available"
+
+    return f"{title}\n\n{abstract}"