pyobo 0.11.2__py3-none-any.whl → 0.12.0__py3-none-any.whl

pyobo/sources/{icd_utils.py → icd/icd_utils.py}

@@ -1,17 +1,20 @@
 """Utilities or interacting with the ICD API.
 
-Want to get your own API cliend ID and client secret?
+Want to get your own API client ID and client secret?
 
 1. Register at https://icdapihome.azurewebsites.net/icdapi/Account/Register
 2. Sell your soul to the American government
+
+.. note::
+
+    If web requests are stalling, try deleting the ``~/.cachier`` directory.
 """
 
 import datetime
 import json
-import os
-from collections.abc import Iterable, Mapping
+from collections.abc import Callable, Iterable, Mapping
 from pathlib import Path
-from typing import Any, Callable, Union
+from typing import Any
 
 import pystow
 import requests
@@ -19,27 +22,66 @@ from cachier import cachier
 from pystow.config_api import ConfigError
 from tqdm.auto import tqdm
 
-from ..getters import NoBuildError
-from ..struct import Term
+from ...getters import NoBuildError
+from ...struct import Term
 
 TOKEN_URL = "https://icdaccessmanagement.who.int/connect/token"  # noqa:S105
 
 ICD_BASE_URL = "https://id.who.int/icd"
 
 ICD11_TOP_LEVEL_URL = f"{ICD_BASE_URL}/entity"
+ICD_11_MMS_URL = f"{ICD_BASE_URL}/release/11/2024-01/mms"
 ICD10_TOP_LEVEL_URL = f"{ICD_BASE_URL}/release/10/2016"
 
 
 def get_icd(url: str) -> requests.Response:
     """Get an ICD API endpoint."""
-    return requests.get(url, headers=get_icd_api_headers())
+    headers = get_icd_api_headers()
+    return requests.get(url, headers=headers, timeout=5)
+
+
+def get_icd_10_top(version: str, path: Path) -> dict[str, Any]:
+    """Get from the ICD10 top."""
+    if path.is_file():
+        return json.loads(path.read_text())
+    rv = get_icd(ICD10_TOP_LEVEL_URL).json()
+    path.write_text(json.dumps(rv, indent=2))
+    return rv
+
+
+def get_icd_11(identifier: str) -> dict[str, Any]:
+    """Get from ICD11."""
+    return get_icd_entity(ICD11_TOP_LEVEL_URL, identifier)
+
+
+def get_icd_11_mms(identifier: str) -> dict[str, Any]:
+    """Get from ICD11 MMS."""
+    return get_icd_entity(ICD_11_MMS_URL, identifier)
 
 
-def _get_entity(endpoint: str, identifier: str):
+class ICDError(ValueError):
+    """An error on getting data from ICD."""
+
+    def __init__(self, identifier: str, url: str, text: str) -> None:
+        """Instantiate an ICD error."""
+        self.identifier = identifier
+        self.url = url
+        self.text = text
+
+    def __str__(self) -> str:
+        """Make a string for the ICD error."""
+        return f"[icd11:{self.identifier}] Error getting {self.url} - {self.text}. Try {ICD11_TOP_LEVEL_URL}/{self.identifier}"
+
+
+def get_icd_entity(endpoint: str, identifier: str) -> dict[str, Any]:
+    """Query a given endpoint at ICD."""
     url = f"{endpoint}/{identifier}"
-    # tqdm.write(f'query {identifier} at {url}')
     res = get_icd(url)
-    return res.json()
+    try:
+        rv = res.json()
+    except OSError:
+        raise ICDError(identifier, url, res.text) from None
+    return rv
 
 
 def get_child_identifiers(endpoint: str, res_json: Mapping[str, Any]) -> list[str]:
@@ -47,9 +89,13 @@ def get_child_identifiers(endpoint: str, res_json: Mapping[str, Any]) -> list[st
     return [url[len(endpoint) :].lstrip("/") for url in res_json.get("child", [])]
 
 
-@cachier(stale_after=datetime.timedelta(minutes=45))
+DELAY = 45
+
+
+@cachier(stale_after=datetime.timedelta(minutes=DELAY))
 def get_icd_api_headers() -> Mapping[str, str]:
     """Get the headers, and refresh every hour."""
+    tqdm.write("Getting ICD credentials w/ PyStow")
     try:
         icd_client_id = pystow.get_config("pyobo", "icd_client_id", raise_on_missing=True)
         icd_client_secret = pystow.get_config("pyobo", "icd_client_secret", raise_on_missing=True)
@@ -58,8 +104,10 @@ def get_icd_api_headers() -> Mapping[str, str]:
 
     grant_type = "client_credentials"
     body_params = {"grant_type": grant_type}
-    tqdm.write("getting ICD API token")
-    res = requests.post(TOKEN_URL, data=body_params, auth=(icd_client_id, icd_client_secret))
+    tqdm.write(f"getting ICD API token, good for {DELAY} minutes")
+    res = requests.post(
+        TOKEN_URL, data=body_params, auth=(icd_client_id, icd_client_secret), timeout=10
+    )
     res_json = res.json()
     access_type = res_json["token_type"]
     access_token = res_json["access_token"]
@@ -73,7 +121,7 @@ def get_icd_api_headers() -> Mapping[str, str]:
 def visiter(
     identifier: str,
     visited_identifiers: set[str],
-    directory: Union[str, Path],
+    directory: str | Path,
     *,
     endpoint: str,
     converter: Callable[[Mapping[str, Any]], Term],
@@ -84,13 +132,11 @@ def visiter(
         return
     visited_identifiers.add(identifier)
 
-    if os.path.exists(path):
-        with open(path) as file:
-            res_json = json.load(file)
+    if path.is_file():
+        res_json = json.loads(path.read_text())
     else:
-        res_json = _get_entity(endpoint, identifier)
-        with open(path, "w") as file:
-            json.dump(res_json, file, indent=2)
+        res_json = get_icd_entity(endpoint, identifier)
+        path.write_text(json.dumps(res_json, indent=2))
 
     yield converter(res_json)
     for identifier in get_child_identifiers(endpoint, res_json):

pyobo/sources/interpro.py

@@ -5,7 +5,7 @@ from collections.abc import Iterable, Mapping
 
 from .utils import get_go_mapping
 from ..struct import Obo, Reference, Term
-from ..struct.typedef import enables, has_member
+from ..struct.typedef import enables, has_category, has_member
 from ..utils.io import multisetdict
 from ..utils.path import ensure_df, ensure_path
 
@@ -30,18 +30,13 @@ class InterProGetter(Obo):
     """An ontology representation of InterPro."""
 
     ontology = bioversions_key = PREFIX
-    typedefs = [enables, has_member]
+    typedefs = [enables, has_member, has_category]
 
     def iter_terms(self, force: bool = False) -> Iterable[Term]:
         """Iterate over InterPro terms."""
         return iter_terms(version=self._version_or_raise, force=force)
 
 
-def get_obo(force: bool = False) -> Obo:
-    """Get InterPro as OBO."""
-    return InterProGetter(force=force)
-
-
 def iter_terms(*, version: str, proteins: bool = False, force: bool = False) -> Iterable[Term]:
     """Get InterPro terms."""
     parents = get_interpro_tree(version=version, force=force)
@@ -74,7 +69,7 @@ def iter_terms(*, version: str, proteins: bool = False, force: bool = False) ->
             term.append_relationship(
                 enables, Reference(prefix="go", identifier=go_id, name=go_name)
             )
-        term.append_property("type", entry_type)
+        term.annotate_string(has_category, entry_type)
         for uniprot_id in interpro_to_proteins.get(identifier, []):
             term.append_relationship(has_member, Reference(prefix="uniprot", identifier=uniprot_id))
         yield term
@@ -91,7 +86,7 @@ def get_interpro_tree(version: str, force: bool = False):
     """Get InterPro Data source."""
     url = f"https://ftp.ebi.ac.uk/pub/databases/interpro/releases/{version}/ParentChildTreeFile.txt"
     path = ensure_path(PREFIX, url=url, version=version, force=force)
-    with open(path) as f:
+    with path.open() as f:
         return _parse_tree_helper(f)
 
 

pyobo/sources/itis.py

@@ -43,11 +43,6 @@ class ITISGetter(Obo):
         return iter_terms(force=force, version=self._version_or_raise)
 
 
-def get_obo() -> Obo:
-    """Get ITIS as OBO."""
-    return ITISGetter()
-
-
 def iter_terms(version: str, force: bool = False) -> Iterable[Term]:
     """Get ITIS terms."""
     zip_path = ensure_path(PREFIX, url=URL, force=force, version=version)

pyobo/sources/kegg/api.py

@@ -3,7 +3,7 @@
 import urllib.error
 from collections.abc import Mapping
 from dataclasses import dataclass
-from typing import Optional
+from pathlib import Path
 
 from pyobo import Reference, Term, ensure_path
 from pyobo.struct import from_species
@@ -30,9 +30,9 @@ class KEGGGenome:
 
     identifier: str
     name: str
-    code: Optional[str]
-    long_code: Optional[str]
-    taxonomy_id: Optional[str]
+    code: str | None
+    long_code: str | None
+    taxonomy_id: str | None
 
     def annotate_term(self, term: Term) -> None:
         """Annotate the term with the species represented by this object."""
@@ -52,7 +52,7 @@ class KEGGGenome:
         )
 
 
-def ensure_list_genomes(version: str) -> str:
+def ensure_list_genomes(version: str) -> Path:
     """Ensure the KEGG Genome file is downloaded."""
     return ensure_path(
         KEGG_GENOME_PREFIX,
@@ -76,7 +76,7 @@ def ensure_list_pathways(version: str) -> Mapping[str, str]:
 """GENOME SPECIFIC"""
 
 
-def ensure_list_genome(kegg_genome_id: str, *, version: str) -> str:
+def ensure_list_genome(kegg_genome_id: str, *, version: str) -> Path:
     """Get the list of genes for the given organism."""
     return ensure_path(
         KEGG_GENES_PREFIX,
@@ -87,22 +87,14 @@ def ensure_list_genome(kegg_genome_id: str, *, version: str) -> str:
     )
 
 
-def ensure_conv_genome_uniprot(
-    kegg_genome_id: str, *, version: str, error_on_missing: bool = False
-) -> Optional[str]:
+def ensure_conv_genome_uniprot(kegg_genome_id: str, *, version: str) -> Path | None:
     """Get the KEGG-UniProt protein map for the given organism."""
-    return _ensure_conv_genome_helper(
-        kegg_genome_id, "uniprot", version=version, error_on_missing=error_on_missing
-    )
+    return _ensure_conv_genome_helper(kegg_genome_id, "uniprot", version=version)
 
 
-def ensure_conv_genome_ncbigene(
-    kegg_genome_id: str, *, version: str, error_on_missing: bool = False
-) -> Optional[str]:
+def ensure_conv_genome_ncbigene(kegg_genome_id: str, *, version: str) -> Path | None:
     """Get the KEGG-NCBIGENE protein map for the given organism."""
-    return _ensure_conv_genome_helper(
-        kegg_genome_id, "ncbi-geneid", version=version, error_on_missing=error_on_missing
-    )
+    return _ensure_conv_genome_helper(kegg_genome_id, "ncbi-geneid", version=version)
 
 
 def _ensure_conv_genome_helper(
@@ -110,8 +102,7 @@ def _ensure_conv_genome_helper(
     target_database: str,
     *,
     version: str,
-    error_on_missing: bool = False,
-) -> Optional[str]:
+) -> Path | None:
     """Get the KEGG-external protein map for the given organism/database."""
     name = f"{kegg_genome_id}.tsv"
     try:
@@ -120,7 +111,6 @@ def _ensure_conv_genome_helper(
             f"conv_{target_database}",
             url=f"{BASE}/conv/{target_database}/{kegg_genome_id}",
             name=name,
-            error_on_missing=error_on_missing,
             version=version,
         )
     except urllib.error.HTTPError:
@@ -132,42 +122,30 @@ def _ensure_conv_genome_helper(
         )
         with path_rv.open("w") as file:
             print(file=file)
-        return path_rv.as_posix()
+        return path_rv
     except FileNotFoundError:
         return None
     else:
         return rv
 
 
-def ensure_link_pathway_genome(
-    kegg_genome_id: str, *, version: str, error_on_missing: bool = False
-) -> str:
-    """Get the protein-pathway links for the given organism.
-
-    :raises: FileNotFoundError
-    """
+def ensure_link_pathway_genome(kegg_genome_id: str, *, version: str) -> Path:
+    """Get the protein-pathway links for the given organism."""
     return ensure_path(
         KEGG_PATHWAY_PREFIX,
         "link_pathway",
         url=f"{BASE}/link/pathway/{kegg_genome_id}",
         name=f"{kegg_genome_id}.tsv",
-        error_on_missing=error_on_missing,
         version=version,
     )
 
 
-def ensure_list_pathway_genome(
-    kegg_genome_id: str, *, version: str, error_on_missing: bool = False
-) -> str:
-    """Get the list of pathways for the given organism.
-
-    :raises: FileNotFoundError
-    """
+def ensure_list_pathway_genome(kegg_genome_id: str, *, version: str) -> Path:
+    """Get the list of pathways for the given organism."""
     return ensure_path(
         KEGG_PATHWAY_PREFIX,
         "pathways",
         url=f"{BASE}/list/pathway/{kegg_genome_id}",
         name=f"{kegg_genome_id}.tsv",
-        error_on_missing=error_on_missing,
         version=version,
     )

pyobo/sources/kegg/genes.py

@@ -5,10 +5,8 @@ Run with ``python -m pyobo.sources.kegg.genes``
 
 import logging
 from collections.abc import Iterable
-from typing import Optional
+from pathlib import Path
 
-import click
-from more_click import verbose_option
 from tqdm.auto import tqdm
 
 from .api import (
@@ -42,11 +40,6 @@ class KEGGGeneGetter(Obo):
         return iter_terms(version=self._version_or_raise)
 
 
-def get_obo() -> Obo:
-    """Get KEGG Genes as OBO."""
-    return KEGGGeneGetter()
-
-
 def iter_terms(version: str) -> Iterable[Term]:
     """Iterate over terms for KEGG Genome."""
     for kegg_genome in iter_kegg_genomes(version=version, desc="KEGG Genes"):
@@ -72,9 +65,9 @@ def iter_terms(version: str) -> Iterable[Term]:
 
 def _make_terms(
     kegg_genome: KEGGGenome,
-    list_genome_path: str,
-    conv_uniprot_path: Optional[str] = None,
-    conv_ncbigene_path: Optional[str] = None,
+    list_genome_path: Path,
+    conv_uniprot_path: Path | None = None,
+    conv_ncbigene_path: Path | None = None,
 ) -> Iterable[Term]:
     uniprot_conv = _load_conv(conv_uniprot_path, "up:") if conv_uniprot_path else {}
     ncbigene_conv = _load_conv(conv_ncbigene_path, "ncbi-geneid:") if conv_ncbigene_path else {}
@@ -101,7 +94,9 @@ def _make_terms(
 
             uniprot_xref = uniprot_conv.get(identifier)
             if uniprot_xref is not None:
-                term.append_relationship(has_gene_product, Reference("uniprot", uniprot_xref))
+                term.annotate_object(
+                    has_gene_product, Reference(prefix="uniprot", identifier=uniprot_xref)
+                )
 
             ncbigene_xref = ncbigene_conv.get(identifier)
             if ncbigene_xref is not None:
@@ -111,17 +106,11 @@ def _make_terms(
             yield term
 
 
-def _load_conv(path, value_prefix):
+def _load_conv(path: Path, value_prefix):
     m = open_map_tsv(path)
     m = {k: v[len(value_prefix) :] for k, v in m.items()}
     return m
 
 
-@click.command()
-@verbose_option
-def _main():
-    get_obo().write_default()
-
-
 if __name__ == "__main__":
-    _main()
+    KEGGGeneGetter.cli()

pyobo/sources/kegg/genome.py

@@ -42,12 +42,6 @@ class KEGGGenomeGetter(Obo):
         return iter_terms(version=self._version_or_raise)
 
 
-def get_obo() -> Obo:
-    """Get KEGG Genome as OBO."""
-    # since old kegg versions go away forever, do NOT add a force option
-    return KEGGGenomeGetter()
-
-
 def parse_genome_line(line: str) -> KEGGGenome | None:
     """Parse a line from the KEGG Genome database."""
     if not line.startswith("T"):
@@ -94,7 +88,7 @@ def iter_kegg_genomes(version: str, desc: str) -> Iterable[KEGGGenome]:
     """Iterate over all KEGG genomes."""
     # since old kegg versions go away forever, do NOT add a force option
     path = ensure_list_genomes(version=version)
-    with open(path) as file:
+    with path.open() as file:
         lines = [line.strip() for line in file]
     it = tqdm(lines, desc=desc, unit_scale=True, unit="genome")
     for line in it:

pyobo/sources/kegg/pathway.py

@@ -8,7 +8,6 @@ import urllib.error
 from collections import defaultdict
 from collections.abc import Iterable, Mapping
 from functools import partial
-from typing import Union
 
 from tqdm.auto import tqdm
 from tqdm.contrib.concurrent import thread_map
@@ -39,6 +38,7 @@ __all__ = [
 logger = logging.getLogger(__name__)
 
 
+# FIXME KEGG API is not usable anymore
 class KEGGPathwayGetter(Obo):
     """An ontology representation of KEGG Pathways."""
 
@@ -51,12 +51,6 @@ class KEGGPathwayGetter(Obo):
         return iter_terms(version=self._version_or_raise)
 
 
-def get_obo() -> Obo:
-    """Get KEGG Pathways as OBO."""
-    # since old kegg versions go away forever, do NOT add a force option
-    return KEGGPathwayGetter()
-
-
 def iter_terms(version: str, skip_missing: bool = True) -> Iterable[Term]:
     """Iterate over terms for KEGG Pathway."""
     # since old kegg versions go away forever, do NOT add a force option
@@ -135,7 +129,7 @@ def _iter_genome_terms(
             tqdm.write(f"could not find kegg.pathway:{pathway_id} for {kegg_genome.name}")
             continue
         for protein_id in protein_ids:
-            pathway_term.append_relationship(
+            pathway_term.annotate_object(
                 has_participant,
                 Reference(
                     prefix=KEGG_GENES_PREFIX,
@@ -148,26 +142,20 @@ def _iter_genome_terms(
 
 def iter_kegg_pathway_paths(
     version: str, skip_missing: bool = True
-) -> Iterable[Union[tuple[KEGGGenome, str, str], tuple[None, None, None]]]:
+) -> Iterable[tuple[KEGGGenome, str, str] | tuple[None, None, None]]:
     """Get paths for the KEGG Pathway files."""
-    genomes = list(iter_kegg_genomes(version=version, desc="KEGG Pathways"))
+    genomes = sorted(
+        iter_kegg_genomes(version=version, desc="KEGG Pathways"), key=lambda x: int(x.identifier)
+    )
     func = partial(_process_genome, version=version, skip_missing=skip_missing)
     return thread_map(func, genomes, unit="pathway", unit_scale=True)
 
 
-def _process_genome(kegg_genome, version, skip_missing):
+def _process_genome(kegg_genome: KEGGGenome, version: str, skip_missing: bool):
     with logging_redirect_tqdm():
         try:
-            list_pathway_path = ensure_list_pathway_genome(
-                kegg_genome.identifier,
-                version=version,
-                error_on_missing=not skip_missing,
-            )
-            link_pathway_path = ensure_link_pathway_genome(
-                kegg_genome.identifier,
-                version=version,
-                error_on_missing=not skip_missing,
-            )
+            list_pathway_path = ensure_list_pathway_genome(kegg_genome.identifier, version=version)
+            link_pathway_path = ensure_link_pathway_genome(kegg_genome.identifier, version=version)
         except urllib.error.HTTPError as e:
             code = e.getcode()
             if code != 404: