aurelian 0.3.2__py3-none-any.whl

aurelian/utils/ontology_utils.py

@@ -0,0 +1,68 @@
+import logfire
+import pystow
+from cachetools.func import lru_cache
+from linkml_store.api import Collection
+from linkml_store.api.stores.duckdb import DuckDBDatabase
+from linkml_store.index import LLMIndexer
+from oaklib import BasicOntologyInterface, get_adapter
+
+llm_indexer = LLMIndexer()
+
+
+@lru_cache
+def get_collection_for_adapter(handle: str, name: str) -> Collection:
+    """
+    Retrieve or create a cached ontology collection.
+
+    Args:
+        handle (str): The ontology handle (e.g., `sqlite:obo:uberon`).
+        name (str): The name of the ontology (e.g., `uberon`).
+
+    Returns:
+        Collection: The indexed ontology collection.
+    """
+    adapter = get_adapter(handle)
+    cache_dir = pystow.join("aurelian", "indexes")
+    duckdb_path = str(cache_dir / f"{name}.duckdb")
+    database = DuckDBDatabase(duckdb_path)
+    collection = database.get_collection(name, create_if_not_exists=True)
+
+    if collection.size() > 0:
+        return collection
+
+    objs = [{"id": id, "label": lbl} for id, lbl in adapter.labels(adapter.entities())]
+    collection.insert(objs)
+    return collection
+
+
+def search_ontology(adapter: BasicOntologyInterface, query: str, limit=10):
+    """
+    Search the ontology for the given query term.
+
+    Example:
+        >>> from oaklib import get_adapter
+        >>> adapter = get_adapter("sqlite:obo:uberon")
+        >>> terms = search_ontology(adapter, "manus")
+        >>> assert len(terms) > 1
+        >>> terms = search_ontology(adapter, "l~digit", limit=5)
+        >>> assert len(terms) == 5
+
+    Args:
+        adapter (BasicOntologyInterface): The ontology adapter.
+        query (str): The query term.
+        limit (int): The maximum number of search results to return.
+
+    Returns:
+        List[Tuple[str, str]]: A list of tuples containing ontology term IDs and labels.
+    """
+    scheme = adapter.resource.scheme
+    name = adapter.resource.slug
+    local_name = name.split(":")[-1]
+    handle = f"{scheme}:{name}"
+
+    collection = get_collection_for_adapter(handle, local_name)
+    with logfire.span("search_ontology {name} {query}", name=name, query=query):
+        print(f"Searching {scheme}:{name} for {query}")
+        qr = collection.search(query, limit=limit, index_name="llm")
+        objs = [(obj["id"], obj["label"]) for obj in qr.rows]
+    return objs

aurelian/utils/pdf_fetcher.py

@@ -0,0 +1,23 @@
+import tempfile
+import requests
+from pdfminer.high_level import extract_text
+
+
+def extract_text_from_pdf(pdf_url: str) -> str:
+    """
+    Download and extract text from a PDF given its URL, using a temporary file.
+    """
+    response = requests.get(pdf_url)
+    if response.status_code != 200:
+        return "Error: Unable to retrieve PDF."
+
+    try:
+        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=True) as temp_pdf:
+            temp_pdf.write(response.content)
+            temp_pdf.flush()  # Ensure all data is written before reading
+
+            text = extract_text(temp_pdf.name)
+            return text.strip() if text else "Error: No text extracted from PDF."
+
+    except Exception as e:
+        return f"Error extracting PDF text: {e}"

aurelian/utils/process_logs.py

@@ -0,0 +1,100 @@
+import json
+from pathlib import Path
+from collections import defaultdict
+import re
+
+
+def parse_reportlog(log_path: str):
+    """Parse pytest-reportlog output into structured format."""
+    tests = defaultdict(dict)
+
+    with open(log_path) as f:
+        for line in f:
+            entry = json.loads(line)
+
+            # Only process TestReport entries
+            if entry.get('$report_type') != 'TestReport':
+                continue
+
+            nodeid = entry['nodeid']
+
+            # Store test outcome
+            if 'outcome' in entry:
+                tests[nodeid]['outcome'] = entry['outcome']
+
+            # Store duration
+            if 'duration' in entry:
+                tests[nodeid]['duration'] = entry['duration']
+
+            # Convert user_properties to dict
+            if 'user_properties' in entry:
+                props = dict(entry['user_properties'])
+                tests[nodeid]['properties'] = props
+
+            # Store parameters from nodeid
+            # Extract from something like: test_search_ontology[sqlite:obo:bfo-3D spatial-10-expected0]
+            if '[' in nodeid:
+                param_str = nodeid[nodeid.index('[') + 1:nodeid.rindex(']')]
+                # You might want to customize this parsing based on your parameter format
+                tests[nodeid]['parameters'] = param_str
+
+    return tests
+
+
+def generate_markdown(tests):
+    """Convert test results to markdown documentation."""
+    md = []
+    md.append("# Test Results Documentation\n")
+
+    # Group tests by their base function name
+    test_groups = defaultdict(list)
+    for nodeid, data in tests.items():
+        # Split nodeid into parts: path::function[params]
+        base_name = nodeid.split('::')[1].split('[')[0] if '[' in nodeid else nodeid.split('::')[1]
+        test_groups[base_name].append((nodeid, data))
+
+    for base_name, group in test_groups.items():
+        md.append(f"## {base_name}\n")
+
+        # Create table for all test runs
+        md.append("### Test Runs\n")
+
+        # Headers: Parameters, Properties, Duration, Outcome
+        md.append('| Parameters | Properties | Duration (s) | Outcome |')
+        md.append('|------------|------------|-------------|---------|')
+
+        for nodeid, data in group:
+            # Extract parameters from nodeid
+            params = nodeid.split('[')[1].rstrip(']') if '[' in nodeid else ''
+
+            # Format properties
+            props = data.get('properties', {})
+            props_str = '; '.join(f"{k}: {v}" for k, v in props.items())
+
+            # Format duration
+            duration = f"{data.get('duration', 0):.3f}"
+
+            row = [
+                params,
+                props_str,
+                duration,
+                data.get('outcome', '')
+            ]
+
+            md.append('| ' + ' | '.join(str(cell) for cell in row) + ' |')
+
+        md.append('')
+    return '\n'.join(md)
+
+# Example usage:
+if __name__ == '__main__':
+    # Assume report.jsonl exists from running:
+    # pytest test_examples.py --report-log=report.jsonl
+
+    log_path = Path('report.jsonl')
+    tests = parse_reportlog(log_path)
+    markdown = generate_markdown(tests)
+
+    # Write markdown to file
+    with open('docs/unit_tests.md', 'w') as f:
+        f.write(markdown)

aurelian/utils/pubmed_utils.py

@@ -0,0 +1,238 @@
+import re
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+from aurelian.utils.doi_fetcher import DOIFetcher
+
+BIOC_URL = "https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_xml/{pmid}/ascii"
+PUBMED_EUTILS_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id={pmid}&retmode=xml"
+EFETCH_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={pmid}&retmode=xml"
+
+DOI_PATTERN = r"/(10\.\d{4,9}/[\w\-.]+)"
+
+doi_fetcher = DOIFetcher()
+
+
+def extract_doi_from_url(url: str) -> Optional[str]:
+    """Extracts the DOI from a given journal URL.
+
+    Args:
+        url (str): The URL of the article.
+
+    Returns:
+        str: The extracted DOI if found, otherwise an empty string.
+
+    """
+    doi_match = re.search(DOI_PATTERN, url)
+    return doi_match.group(1) if doi_match else None
+
+
+def doi_to_pmid(doi: str) -> Optional[str]:
+    """Converts a DOI to a PMID using the NCBI ID Converter API.
+
+    Args:
+        doi (str): The DOI to be converted.
+
+    Returns:
+        str: The corresponding PMID if found, otherwise an empty string.
+
+    """
+    API_URL = f"https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids={doi}&format=json"
+    response = requests.get(API_URL).json()
+    records = response.get("records", [])
+    pmid = records[0].get("pmid", None) if records else None
+    return pmid
+
+
+def get_doi_text(doi: str) -> str:
+    """Fetch the full text of an article using a DOI.
+
+    TODO: non pubmed sources
+
+    Example:
+        >>> doi = "10.1128/msystems.00045-18"
+        >>> full_text = get_doi_text(doi)
+        >>> assert "Populus Microbiome" in full_text
+
+    Args:
+        doi: The DOI of the article.
+
+    Returns:
+        The full text of the article if available, otherwise an empty string.
+
+    """
+    pmid = doi_to_pmid(doi)
+    if not pmid:
+        info = doi_fetcher.get_full_text(doi)
+        if info:
+            return info
+        else:
+            return f"PMID not found for {doi} and not available via unpaywall"
+    return get_pmid_text(pmid)
+
+
+def get_pmid_from_pmcid(pmcid):
+    """Fetch the PMID from a PMC ID using the Entrez E-utilities `esummary`.
+
+    Example:
+        >>> pmcid = "PMC5048378"
+        >>> pmid = get_pmid_from_pmcid(pmcid)
+        >>> print(pmid)
+        27629041
+
+    Args:
+        pmcid:
+
+    Returns:
+
+    """
+    if ":" in pmcid:
+        pmcid = pmcid.split(":")[1]
+    url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"
+    params = {"db": "pmc", "id": pmcid.replace("PMC", ""), "retmode": "json"}  # Remove "PMC" prefix if included
+
+    response = requests.get(url, params=params)
+    data = response.json()
+
+    # Extract PMID
+    try:
+        uid = data["result"]["uids"][0]  # Extract the UID
+        article_ids = data["result"][uid]["articleids"]  # Get article IDs
+        for item in article_ids:
+            if item["idtype"] == "pmid":
+                return item["value"]
+    except KeyError:
+        return "PMID not found"
+
+
+def get_pmcid_text(pmcid: str) -> str:
+    """Fetch full text from PubMed Central Open Access BioC XML.
+
+    Example:
+        >>> pmcid = "PMC5048378"
+        >>> full_text = get_pmcid_text(pmcid)
+        >>> assert "integrated stress response (ISR)" in full_text
+
+    Args:
+        pmcid:
+
+    Returns:
+
+    """
+    pmid = get_pmid_from_pmcid(pmcid)
+    return get_pmid_text(pmid)
+
+
+def get_pmid_text(pmid: str) -> str:
+    """Fetch full text from PubMed Central Open Access BioC XML.
+    If full text is not available, fallback to fetching the abstract from PubMed.
+
+    Example:
+        >>> pmid = "11"
+        >>> full_text = get_pmid_text(pmid)
+        >>> print(full_text)
+        Identification of adenylate cyclase-coupled beta-adrenergic receptors with radiolabeled beta-adrenergic antagonists.
+        <BLANKLINE>
+        No abstract available
+
+    Args:
+        pmid: PubMed ID of the article.
+
+    Returns:
+        The full text of the article if available, otherwise the abstract.
+
+    """
+    if ":" in pmid:
+        pmid = pmid.split(":")[1]
+    text = get_full_text_from_bioc(pmid)
+    if not text:
+        doi = pmid_to_doi(pmid)
+        if doi:
+            text = doi_fetcher.get_full_text(doi)
+    if not text:
+        text = get_abstract_from_pubmed(pmid)
+    return text
+
+def pmid_to_doi(pmid: str) -> Optional[str]:
+    if ":" in pmid:
+        pmid = pmid.split(":")[1]
+    url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id={pmid}&retmode=json"
+    response = requests.get(url)
+    data = response.json()
+
+    try:
+        article_info = data["result"][str(pmid)]
+        for aid in article_info["articleids"]:
+            if aid["idtype"] == "doi":
+                return aid["value"]
+        elocationid = article_info.get("elocationid", "")
+        if elocationid.startswith("10."):  # DOI starts with "10."
+            return elocationid
+        else:
+            return None
+    except KeyError:
+        return None
+
+
+def get_full_text_from_bioc(pmid: str) -> str:
+    """Fetch full text from PubMed Central Open Access BioC XML.
+
+    Example:
+        >>> pmid = "17299597"
+        >>> full_text = get_full_text_from_bioc(pmid)
+        >>> assert "Evolution of biological complexity." in full_text
+
+    Args:
+        pmid: PubMed ID of the article.
+
+    Returns:
+        The full text of the article if available, otherwise an empty string.
+
+    """
+    response = requests.get(BIOC_URL.format(pmid=pmid))
+
+    if response.status_code != 200:
+        return ""  # Return empty string if request fails
+
+    soup = BeautifulSoup(response.text, "xml")
+
+    # Extract ONLY text from <text> tags within <passage>
+    text_sections = [text_tag.get_text() for text_tag in soup.find_all("text")]
+
+    full_text = "\n".join(text_sections).strip()
+    return full_text
+
+
+def get_abstract_from_pubmed(pmid: str) -> str:
+    """Fetch the title and abstract of an article from PubMed using Entrez E-utilities `efetch`.
+
+    Example:
+        >>> pmid = "31653696"
+        >>> abstract = get_abstract_from_pubmed(pmid)
+        >>> assert "The apparent deglycase activity of DJ-1" in abstract
+
+    Args:
+        pmid: PubMed ID of the article.
+
+    Returns:
+        The title and abstract text if available, otherwise an empty string.
+
+    """
+    response = requests.get(EFETCH_URL.format(pmid=pmid))
+
+    if response.status_code != 200:
+        return ""
+
+    soup = BeautifulSoup(response.text, "xml")
+
+    # Extract title
+    title_tag = soup.find("ArticleTitle")
+    title = title_tag.get_text().strip() if title_tag else "No title available"
+
+    # Extract abstract (may contain multiple sections)
+    abstract_tags = soup.find_all("AbstractText")
+    abstract = "\n".join(tag.get_text().strip() for tag in abstract_tags) if abstract_tags else "No abstract available"
+
+    return f"{title}\n\n{abstract}"

aurelian/utils/pytest_report_to_markdown.py

@@ -0,0 +1,67 @@
+import json
+from pathlib import Path
+from collections import defaultdict
+import re
+from typing import Iterator
+
+import click
+
+
+def report_md(log_path: str) -> str:
+    return '\n'.join(list(report_md_iter(log_path)))
+
+def report_md_iter(log_path: str) -> Iterator[str]:
+    """
+    Parse pytest-reportlog output into structured format.
+
+    Args:
+        log_path:
+
+    Returns:
+
+    """
+
+    with open(log_path) as f:
+        outcome = None
+        duration = None
+        for line in f:
+            entry = json.loads(line)
+
+            # Only process TestReport entries
+            if entry.get('$report_type') != 'TestReport':
+                continue
+
+            nodeid = entry['nodeid']
+            outcome = entry.get('outcome')
+            duration = entry.get('duration')
+
+            if not outcome:
+                continue
+
+            yield f"## {nodeid}\n"
+
+
+
+            for p in entry.get('user_properties', []):
+                k = p[0]
+                v = p[1]
+
+                yield f"### {k}\n\n"
+                yield f"{v}\n"
+
+        yield "## Stats\n\n"
+        if outcome:
+            yield f"* Outcome: {outcome}\n"
+        if duration:
+            yield f"* Duration: {duration}\n"
+
+
+
+@click.command()
+@click.argument("log_path", type=click.Path(exists=True))
+def main(log_path: str):
+    markdown = report_md(log_path)
+    print(markdown)
+
+if __name__ == "__main__":
+    main()

aurelian/utils/robot_ontology_utils.py

@@ -0,0 +1,112 @@
+from typing import Dict, Optional, List, Tuple
+
+from aurelian.dependencies.workdir import WorkDir
+
+MERGED_IMPORT_PATH = "_imports_.owl"
+
+def run(cmd: str):
+    """
+    Run a command, raising an error if the command fails,
+    returning stdout
+
+    Args:
+        cmd:
+
+    Returns:
+
+    """
+    import subprocess
+    result = subprocess.run(cmd, shell=True, capture_output=True)
+    if result.returncode != 0:
+        stdout = result.stdout.decode()
+        stderr = result.stderr.decode()
+        raise Exception(f"Command failed: {cmd}\nError: {stderr}\nOutput: {stdout}")
+    return result.stdout.decode()
+
+def parse_component_name(name: str) -> Tuple[str, Optional[str]]:
+    """
+    Parse file name
+
+    Example:
+
+        >>> parse_component_name("foo.owl")
+        ('foo', 'owl')
+        >>> parse_component_name("foo")
+        ('foo', None)
+
+
+    Args:
+        name:
+
+    Returns:
+
+    """
+    parts = name.split(".")
+    if len(parts) == 1:
+        return name, None
+    return ".".join(parts[:-1]), parts[-1]
+
+
+def depends_on_csv(workdir: WorkDir, name: str) -> Optional[str]:
+    base, suffix = parse_component_name(name)
+    if not suffix:
+        suffix = "owl"
+        base = name
+    if suffix == "owl":
+        for d_suffix in ("tsv", "csv"):
+            d_name = f"{base}.{d_suffix}"
+            if workdir.check_file_exists(d_name):
+                return d_name
+    return None
+
+def run_robot_template_command(workdir: WorkDir, template_path: str, prefix_map: Dict[str, str], output_path: Optional[str] = None, import_ontologies: Optional[List[str]] = None) -> str:
+    """
+    Generate a robot template command
+
+    Args:
+        workdir:
+        template_path:
+        prefix_map:
+        output_path:
+        import_ontologies:
+
+    Returns:
+
+    """
+    if output_path is None:
+        output_path = template_path.replace(".csv", ".owl")
+    prefixes = " ".join([f"--prefix '{k}: {v}'" for k, v in prefix_map.items()])
+    if not import_ontologies:
+        import_ontologies = []
+    import_owls = []
+    for import_ontology in import_ontologies:
+        local_name, suffix = parse_component_name(import_ontology)
+        if suffix == "owl":
+            import_ontology_owl = import_ontology
+            if not workdir.check_file_exists(import_ontology_owl):
+                depends_on = depends_on_csv(workdir, import_ontology_owl)
+                if not workdir.check_file_exists(depends_on):
+                    raise Exception(f"Cannot make owl file {import_ontology_owl} as no {depends_on}")
+                run_robot_template_command(
+                    workdir,
+                    depends_on,
+                    prefix_map=prefix_map,
+                    output_path=import_ontology_owl,
+                )
+        else:
+            if suffix:
+                import_ontology_owl = import_ontology.replace(suffix, "owl")
+            else:
+                import_ontology_owl = import_ontology + ".owl"
+            run_robot_template_command(workdir, import_ontology, prefix_map=prefix_map, output_path=import_ontology_owl)
+        import_owls.append(import_ontology_owl)
+    if import_owls:
+        input_opts = [f"--input {owl}" for owl in import_owls]
+        cmd = f"cd {workdir.location} && robot merge {' '.join(input_opts)} --output {MERGED_IMPORT_PATH}"
+        run(cmd)
+        import_ontology_opt = f"--input {MERGED_IMPORT_PATH}"
+    else:
+        import_ontology_opt = ""
+    cmd = f"cd {workdir.location} && robot template {import_ontology_opt} --template {template_path} {prefixes} reason --output {output_path}"
+    run(cmd)
+    return output_path