pyobo 0.11.1__py3-none-any.whl → 0.12.0__py3-none-any.whl

pyobo/sources/{hgnc.py → hgnc/hgnc.py}

@@ -6,8 +6,6 @@ import logging
 import typing
 from collections import Counter, defaultdict
 from collections.abc import Iterable
-from operator import attrgetter
-from typing import Optional
 
 from tabulate import tabulate
 from tqdm.auto import tqdm
@@ -17,11 +15,13 @@ from pyobo.resources.so import get_so_name
 from pyobo.struct import (
     Obo,
     Reference,
-    Synonym,
     SynonymTypeDef,
     Term,
+    TypeDef,
+    default_reference,
     from_species,
     gene_product_member_of,
+    has_citation,
     has_gene_product,
     member_of,
     orthologous,
@@ -42,10 +42,27 @@ DEFINITIONS_URL_FMT = (
     "hgnc_complete_set_{version}.json"
 )
 
-previous_symbol_type = SynonymTypeDef.from_text("previous_symbol")
-alias_symbol_type = SynonymTypeDef.from_text("alias_symbol")
-previous_name_type = SynonymTypeDef.from_text("previous_name")
-alias_name_type = SynonymTypeDef.from_text("alias_name")
+previous_symbol_type = SynonymTypeDef(
+    reference=default_reference(PREFIX, "previous_symbol", name="previous symbol")
+)
+alias_symbol_type = SynonymTypeDef(
+    reference=default_reference(PREFIX, "alias_symbol", name="alias symbol")
+)
+previous_name_type = SynonymTypeDef(
+    reference=default_reference(PREFIX, "previous_name", name="previous name")
+)
+alias_name_type = SynonymTypeDef(
+    reference=default_reference(PREFIX, "alias_name", name="alias name")
+)
+HAS_LOCUS_TYPE = TypeDef(
+    reference=default_reference(PREFIX, "locus_type", name="has locus type"), is_metadata_tag=True
+)
+HAS_LOCUS_GROUP = TypeDef(
+    reference=default_reference(PREFIX, "locus_group", name="has locus group"), is_metadata_tag=True
+)
+HAS_LOCATION = TypeDef(
+    reference=default_reference(PREFIX, "location", name="has location"), is_metadata_tag=True
+)
 
 #: First column is MIRIAM prefix, second column is HGNC key
 gene_xrefs = [
@@ -129,6 +146,7 @@ SKIP_KEYS = {
     "cd",  # symbol
     "homeodb",  # TODO add to bioregistry, though this is defunct
     "mamit-trnadb",  # TODO add to bioregistry, though this is defunct
+    "mane_select",  # TODO
 }
 
 #: A mapping from HGNC's locus_type annotations to sequence ontology identifiers
@@ -167,38 +185,8 @@ LOCUS_TYPE_TO_SO = {
     None: "0000704",  # gene
 }
 
-IDSPACES = {
-    prefix: f"https://bioregistry.io/{prefix}:"
-    for prefix in {
-        "rgd",
-        "mgi",
-        "eccode",
-        "rnacentral",
-        "pubmed",
-        "uniprot",
-        "mirbase",
-        "snornabase",
-        "hgnc",
-        "hgnc.genegroup",
-        "debio",
-        "ensembl",
-        "NCBIGene",
-        "vega",
-        "ucsc",
-        "ena",
-        "ccds",
-        "omim",
-        "cosmic",
-        "merops",
-        "orphanet",
-        "pseudogene",
-        "lncipedia",
-        "refseq",
-    }
-}
-IDSPACES.update(
-    NCBITaxon="http://purl.obolibrary.org/obo/NCBITaxon_",
-    SO="http://purl.obolibrary.org/obo/SO_",
+PUBLICATION_TERM = Term(
+    reference=Reference(prefix="IAO", identifier="0000013", name="journal article")
 )
 
 
@@ -214,8 +202,11 @@ class HGNCGetter(Obo):
         orthologous,
         member_of,
         exact_match,
+        has_citation,
+        HAS_LOCUS_GROUP,
+        HAS_LOCUS_TYPE,
+        HAS_LOCATION,
     ]
-    idspaces = IDSPACES
     synonym_typedefs = [
         previous_name_type,
         previous_symbol_type,
@@ -233,12 +224,7 @@ class HGNCGetter(Obo):
         return get_terms(force=force, version=self.data_version)
 
 
-def get_obo(*, force: bool = False) -> Obo:
-    """Get HGNC as OBO."""
-    return HGNCGetter(force=force)
-
-
-def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Term]:
+def get_terms(version: str | None = None, force: bool = False) -> Iterable[Term]:
     """Get HGNC terms."""
     if version is None:
         version = get_version("hgnc")
@@ -251,18 +237,15 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
         version=version,
         name="hgnc_complete_set.json",
     )
-    with open(path) as file:
+    with path.open() as file:
         entries = json.load(file)["response"]["docs"]
 
     yield Term.from_triple("NCBITaxon", "9606", "Homo sapiens")
-    yield from sorted(
-        {
-            Term(reference=Reference(prefix="SO", identifier=so_id, name=get_so_name(so_id)))
-            for so_id in sorted(LOCUS_TYPE_TO_SO.values())
-            if so_id
-        },
-        key=attrgetter("identifier"),
-    )
+    _so_ids: set[str] = {s for s in LOCUS_TYPE_TO_SO.values() if s}
+    yield from [
+        Term(reference=Reference(prefix="SO", identifier=so_id, name=get_so_name(so_id)))
+        for so_id in sorted(_so_ids)
+    ]
 
     statuses = set()
     for entry in tqdm(entries, desc=f"Mapping {PREFIX}", unit="gene", unit_scale=True):
@@ -273,7 +256,7 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
         )
         status = entry.pop("status")
         if status == "Approved":
-            is_obsolete = False
+            is_obsolete = None
         elif status not in statuses:
             statuses.add(status)
             tqdm.write(f"[{PREFIX}] unhandled {status}")
@@ -297,7 +280,7 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
                 continue  # only add concrete annotations
             term.append_relationship(
                 gene_product_member_of,
-                Reference(prefix="eccode", identifier=ec_code),
+                Reference(prefix="ec", identifier=ec_code),
             )
         for rna_central_ids in entry.pop("rna_central_id", []):
             for rna_central_id in rna_central_ids.split(","):
@@ -364,7 +347,7 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
             xref_identifiers = entry.pop(key, None)
             if xref_identifiers is None:
                 continue
-            if isinstance(xref_identifiers, (str, int)):
+            if isinstance(xref_identifiers, str | int):
                 xref_identifiers = [str(xref_identifiers)]
 
             if xref_prefix == "merops.entry":
@@ -389,7 +372,7 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
 
         gene_group_ids = entry.pop("gene_group_id", [])
         gene_groups = entry.pop("gene_group", [])
-        for gene_group_id, gene_group_label in zip(gene_group_ids, gene_groups):
+        for gene_group_id, gene_group_label in zip(gene_group_ids, gene_groups, strict=False):
             term.append_relationship(
                 member_of,
                 Reference(
@@ -400,20 +383,20 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
             )
 
         for alias_symbol in entry.pop("alias_symbol", []):
-            term.append_synonym(Synonym(name=alias_symbol, type=alias_symbol_type))
+            term.append_synonym(alias_symbol, type=alias_symbol_type)
         for alias_name in entry.pop("alias_name", []):
-            term.append_synonym(Synonym(name=alias_name, type=alias_name_type))
+            term.append_synonym(alias_name, type=alias_name_type)
         for previous_symbol in itt.chain(
             entry.pop("previous_symbol", []), entry.pop("prev_symbol", [])
         ):
-            term.append_synonym(Synonym(name=previous_symbol, type=previous_symbol_type))
+            term.append_synonym(previous_symbol, type=previous_symbol_type)
         for previous_name in entry.pop("prev_name", []):
-            term.append_synonym(Synonym(name=previous_name, type=previous_name_type))
+            term.append_synonym(previous_name, type=previous_name_type)
 
-        for prop in ["location"]:
+        for prop, td in [("location", HAS_LOCATION)]:
             value = entry.pop(prop, None)
             if value:
-                term.append_property(prop, value)
+                term.annotate_string(td, value)
 
         locus_type = entry.pop("locus_type")
         locus_group = entry.pop("locus_group")
@@ -425,8 +408,8 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
                 Reference(prefix="SO", identifier="0000704", name=get_so_name("0000704"))
             )  # gene
             unhandle_locus_types[locus_type][identifier] = term
-            term.append_property("locus_type", locus_type)
-            term.append_property("locus_group", locus_group)
+            term.annotate_string(HAS_LOCUS_TYPE, locus_type)
+            term.annotate_string(HAS_LOCUS_GROUP, locus_group)
 
         term.set_species(identifier="9606", name="Homo sapiens")
 
@@ -453,9 +436,11 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
                         hgnc_id,
                         term.name,
                         term.is_obsolete,
-                        term.bioregistry_link,
+                        f"https://bioregistry.io/{term.curie}",
                         ", ".join(
-                            p.bioregistry_link for p in term.provenance if p.bioregistry_link
+                            f"https://bioregistry.io/{p.curie}"
+                            for p in term.provenance
+                            if isinstance(p, Reference)
                         ),
                     )
                     for hgnc_id, term in sorted(v.items())
@@ -472,7 +457,8 @@ def get_terms(version: Optional[str] = None, force: bool = False) -> Iterable[Te
     logger.warning(
         "Unhandled locus types:\n%s", tabulate(unhandle_locus_type_counter.most_common())
     )
-    logger.warning("Unhandled keys:\n%s", tabulate(unhandled_entry_keys.most_common()))
+    if unhandled_entry_keys:
+        logger.warning("Unhandled keys:\n%s", tabulate(unhandled_entry_keys.most_common()))
 
 
 if __name__ == "__main__":

pyobo/sources/{hgncgenefamily.py → hgnc/hgncgenefamily.py}

@@ -5,16 +5,9 @@ from collections.abc import Iterable, Mapping
 
 import pandas as pd
 
-from ..struct import (
-    Obo,
-    Reference,
-    Synonym,
-    SynonymTypeDef,
-    Term,
-    enables,
-    from_species,
-)
-from ..utils.path import ensure_path
+from ...struct import Obo, Reference, SynonymTypeDef, Term, has_citation
+from ...struct.typedef import enables, exact_match, from_species
+from ...utils.path import ensure_path
 
 __all__ = [
     "HGNCGroupGetter",
@@ -36,18 +29,13 @@ class HGNCGroupGetter(Obo):
     ontology = PREFIX
     bioversions_key = "hgnc"
     synonym_typedefs = [symbol_type]
-    typedefs = [from_species, enables]
+    typedefs = [from_species, enables, exact_match, has_citation]
 
     def iter_terms(self, force: bool = False) -> Iterable[Term]:
         """Iterate over terms in the ontology."""
         return get_terms(force=force)
 
 
-def get_obo(force: bool = False) -> Obo:
-    """Get HGNC Gene Groups as OBO."""
-    return HGNCGroupGetter(force=force)
-
-
 def get_hierarchy(force: bool = False) -> Mapping[str, list[str]]:
     """Get the HGNC Gene Families hierarchy as a dictionary."""
     path = ensure_path(PREFIX, url=HIERARCHY_URL, force=force)
@@ -99,12 +87,14 @@ def _get_terms_helper(force: bool = False) -> Iterable[Term]:
         )
         if pubmed_ids and pd.notna(pubmed_ids):
             for s in pubmed_ids.replace(" ", ",").split(","):
-                term.append_provenance(Reference(prefix="pubmed", identifier=s.strip()))
+                s = s.strip()
+                if s:
+                    term.append_provenance(Reference(prefix="pubmed", identifier=s))
         if desc_go and pd.notna(desc_go):
             go_id = desc_go[len("http://purl.uniprot.org/go/") :]
             term.append_relationship(enables, Reference(prefix="GO", identifier=go_id))
         if symbol and pd.notna(symbol):
-            term.append_synonym(Synonym(name=symbol, type=symbol_type))
+            term.append_synonym(symbol, type=symbol_type)
         term.set_species(identifier="9606", name="Homo sapiens")
         yield term
 

pyobo/sources/icd/__init__.py

@@ -0,0 +1,9 @@
+"""Resources from ICD."""
+
+from .icd10 import ICD10Getter
+from .icd11 import ICD11Getter
+
+__all__ = [
+    "ICD10Getter",
+    "ICD11Getter",
+]

pyobo/sources/{icd10.py → icd/icd10.py}

@@ -1,24 +1,27 @@
 """Convert ICD-10 to OBO.
 
-Run with python -m pyobo.sources.icd10 -v
+Run with ``python -m pyobo.sources.icd10 -v``.
+
+.. note::
+
+    If web requests are stalling, try deleting the ``~/.cachier`` directory.
 """
 
 import logging
 from collections.abc import Iterable, Mapping
+from pathlib import Path
 from typing import Any
 
-import click
-from more_click import verbose_option
 from tqdm.auto import tqdm
 
-from ..sources.icd_utils import (
+from .icd_utils import (
     ICD10_TOP_LEVEL_URL,
     get_child_identifiers,
-    get_icd,
+    get_icd_10_top,
     visiter,
 )
-from ..struct import Obo, Reference, Synonym, Term
-from ..utils.path import prefix_directory_join
+from ...struct import Obo, Reference, Synonym, Term, has_category
+from ...utils.path import prefix_directory_join
 
 __all__ = [
     "ICD10Getter",
@@ -34,37 +37,39 @@ class ICD10Getter(Obo):
     """An ontology representation of ICD-10."""
 
     ontology = PREFIX
-    dynamic_version = True
+    static_version = VERSION
+    typedefs = [has_category]
 
     def iter_terms(self, force: bool = False) -> Iterable[Term]:
         """Iterate over terms in the ontology."""
-        return iter_terms()
+        return iter_terms(self._version_or_raise)
 
 
-def get_obo() -> Obo:
-    """Get ICD-10 as OBO."""
-    return ICD10Getter()
+def _get_chapters(version: str, path: Path):
+    res_json = get_icd_10_top(version=version, path=path)
+    chapter_urls = res_json["child"]
+    tqdm.write(f"there are {len(chapter_urls)} chapters")
+    identifiers = get_child_identifiers(ICD10_TOP_LEVEL_URL, res_json)
+    return identifiers
 
 
-def iter_terms() -> Iterable[Term]:
+def iter_terms(version: str) -> Iterable[Term]:
     """Iterate over ICD-10 terms."""
-    r = get_icd(ICD10_TOP_LEVEL_URL)
-    res_json = r.json()
-
-    directory = prefix_directory_join(PREFIX, version=VERSION)
-
-    chapter_urls = res_json["child"]
-    tqdm.write(f"there are {len(chapter_urls)} chapters")
+    directory = prefix_directory_join(PREFIX, version=version)
+    identifiers = _get_chapters(version=version, path=directory.joinpath("top.json"))
 
     visited_identifiers: set[str] = set()
-    for identifier in get_child_identifiers(ICD10_TOP_LEVEL_URL, res_json):
-        yield from visiter(
-            identifier,
-            visited_identifiers,
-            directory,
-            endpoint=ICD10_TOP_LEVEL_URL,
-            converter=_extract_icd10,
-        )
+    with tqdm(desc=f"[{PREFIX}]") as pbar:
+        for identifier in identifiers:
+            for term in visiter(
+                identifier,
+                visited_identifiers,
+                directory,
+                endpoint=ICD10_TOP_LEVEL_URL,
+                converter=_extract_icd10,
+            ):
+                pbar.update(1)
+                yield term
 
 
 def _extract_icd10(res_json: Mapping[str, Any]) -> Term:
@@ -81,17 +86,10 @@ def _extract_icd10(res_json: Mapping[str, Any]) -> Term:
         synonyms=synonyms,
         parents=parents,
     )
-
-    rv.append_property("class_kind", res_json["classKind"])
+    rv.annotate_string(has_category, res_json["classKind"])
 
     return rv
 
 
-@click.command()
-@verbose_option
-def _main():
-    get_obo().write_default(use_tqdm=True)
-
-
 if __name__ == "__main__":
-    _main()
+    ICD10Getter.cli()

pyobo/sources/icd/icd11.py

@@ -0,0 +1,148 @@
+"""Convert ICD11 to OBO.
+
+Run with ``python -m pyobo.sources.icd11 -v``.
+
+.. note::
+
+    If web requests are stalling, try deleting the ``~/.cachier`` directory.
+"""
+
+import json
+import logging
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+from tqdm.auto import tqdm
+
+from .icd_utils import (
+    ICD11_TOP_LEVEL_URL,
+    ICDError,
+    get_child_identifiers,
+    get_icd,
+    get_icd_11_mms,
+    visiter,
+)
+from ...struct import Obo, Reference, Synonym, Term, TypeDef, default_reference
+from ...utils.path import prefix_directory_join
+
+__all__ = [
+    "ICD11Getter",
+]
+
+logger = logging.getLogger(__name__)
+
+PREFIX = "icd11"
+CODE_PREFIX = "icd11.code"
+
+CODE_PROP = TypeDef(reference=default_reference(PREFIX, "icd_mms_code"), is_metadata_tag=True)
+
+
+class ICD11Getter(Obo):
+    """An ontology representation of ICD-11."""
+
+    ontology = PREFIX
+    typedefs = [CODE_PROP]
+    dynamic_version = True
+
+    def iter_terms(self, force: bool = False) -> Iterable[Term]:
+        """Iterate over terms in the ontology."""
+        return iterate_icd11()
+
+
+def iterate_icd11(version: str | None = None) -> Iterable[Term]:
+    """Iterate over the terms in ICD11 and enrich them with MMS."""
+    # Get all terms from the ICD foundation API
+    version_strict, terms = _get_icd11_terms_helper(version=version)
+
+    # prepare a directory for enriching from MMS
+    mms_directory = prefix_directory_join(PREFIX, "mms", version=version_strict)
+
+    # this takes a bit more than 2 hours
+    for term in tqdm(terms, desc="Getting MMS", unit_scale=True):
+        path = mms_directory.joinpath(term.identifier).with_suffix(".json")
+        if path.exists():
+            mms_data = json.loads(path.read_text())
+        else:
+            try:
+                mms_data = get_icd_11_mms(term.identifier)
+            except ICDError:
+                # writing this isn't necessary since not all terms have MMS entries
+                # tqdm.write(str(e))
+                mms_data = {}
+            path.write_text(json.dumps(mms_data))
+
+        if code := mms_data.get("code"):
+            term.append_exact_match(Reference(prefix=CODE_PREFIX, identifier=code))
+
+        yield term
+
+
+def _get_icd11_terms_helper(version: str | None = None) -> tuple[str, list[Term]]:
+    """Iterate over the terms in ICD11.
+
+    The API doesn't seem to have a rate limit, but returns pretty slow. This means that
+    it only gets results at at about 5 calls/second. Get ready to be patient - the API
+    token expires every hour so there's a caching mechanism with :mod:`cachier` that
+    gets a new one every hour.
+    """
+    if version is not None:
+        directory = prefix_directory_join(PREFIX, "base", version=version)
+        top_path = directory.joinpath("top.json")
+        if top_path.is_file():
+            res_json = json.loads(top_path.read_text())
+        else:
+            res_json = get_icd(ICD11_TOP_LEVEL_URL).json()
+            top_path.write_text(json.dumps(res_json, indent=2))
+    else:
+        tqdm.write("No version passed, looking up version from ICD11")
+        res_json = get_icd(ICD11_TOP_LEVEL_URL).json()
+        version = res_json["releaseId"]
+        directory = prefix_directory_join(PREFIX, "base", version=version)
+        top_path = directory.joinpath("top.json")
+        with top_path.open("w") as file:
+            json.dump(res_json, file, indent=2)
+
+    tqdm.write(f"There are {len(res_json['child'])} top level entities")
+
+    visited_identifiers: set[str] = set()
+    rv: list[Term] = []
+    for identifier in get_child_identifiers(ICD11_TOP_LEVEL_URL, res_json):
+        rv.extend(
+            visiter(
+                identifier,
+                visited_identifiers,
+                directory,
+                endpoint=ICD11_TOP_LEVEL_URL,
+                converter=_extract_icd11,
+            )
+        )
+
+    return version, rv
+
+
+def _extract_icd11(res_json: Mapping[str, Any]) -> Term:
+    identifier = res_json["@id"][len(ICD11_TOP_LEVEL_URL) :].lstrip("/")
+    if "definition" in res_json:
+        definition = res_json["definition"]["@value"]
+        definition = definition.strip().replace("\r\n", " ")
+        definition = definition.strip().replace("\\n", " ")
+        definition = definition.strip().replace("\n", " ")
+    else:
+        definition = None
+    name = res_json["title"]["@value"]
+    synonyms = [Synonym(synonym["label"]["@value"]) for synonym in res_json.get("synonym", [])]
+    parents = [
+        Reference(prefix=PREFIX, identifier=url[len("http://id.who.int/icd/entity/") :])
+        for url in res_json["parent"]
+        if url[len("http://id.who.int/icd/entity/") :]
+    ]
+    return Term(
+        reference=Reference(prefix=PREFIX, identifier=identifier, name=name),
+        definition=definition,
+        synonyms=synonyms,
+        parents=parents,
+    )
+
+
+if __name__ == "__main__":
+    ICD11Getter.cli()