pyobo 0.10.5__py3-none-any.whl → 0.10.7__py3-none-any.whl

pyobo/__init__.py

@@ -25,6 +25,7 @@ from .api import (  # noqa: F401
     get_name,
     get_name_by_curie,
     get_name_id_mapping,
+    get_obsolete,
     get_primary_curie,
     get_primary_identifier,
     get_properties,

pyobo/api/__init__.py

@@ -27,6 +27,7 @@ from .names import (  # noqa: F401
     get_name,
     get_name_by_curie,
     get_name_id_mapping,
+    get_obsolete,
     get_synonyms,
 )
 from .properties import (  # noqa: F401

pyobo/api/names.py

@@ -24,6 +24,7 @@ __all__ = [
     "get_id_definition_mapping",
     "get_synonyms",
     "get_id_synonyms_mapping",
+    "get_obsolete",
 ]
 
 logger = logging.getLogger(__name__)
@@ -184,6 +185,26 @@ def get_id_definition_mapping(
     return _get_mapping()
 
 
+def get_obsolete(
+    prefix: str,
+    *,
+    force: bool = False,
+    strict: bool = False,
+    version: Optional[str] = None,
+) -> Set[str]:
+    """Get the set of obsolete local unique identifiers."""
+    if version is None:
+        version = get_version(prefix)
+    path = prefix_cache_join(prefix, name="obsolete.tsv", version=version)
+
+    @cached_collection(path=path, force=force)
+    def _get_obsolete() -> Set[str]:
+        ontology = get_ontology(prefix, force=force, strict=strict, version=version)
+        return ontology.get_obsolete()
+
+    return set(_get_obsolete())
+
+
 @wrap_norm_prefix
 def get_synonyms(prefix: str, identifier: str) -> Optional[List[str]]:
     """Get the synonyms for an entity."""

pyobo/gilda_utils.py

@@ -2,7 +2,6 @@
 
 """PyOBO's Gilda utilities."""
 
-import itertools as itt
 import logging
 from typing import Iterable, List, Optional, Tuple, Type, Union
 
@@ -11,6 +10,7 @@ import gilda.api
 import gilda.term
 from gilda.grounder import Grounder
 from gilda.process import normalize
+from gilda.term import filter_out_duplicates
 from tqdm.auto import tqdm
 
 from pyobo import (
@@ -18,6 +18,7 @@ from pyobo import (
     get_id_species_mapping,
     get_id_synonyms_mapping,
     get_ids,
+    get_obsolete,
 )
 from pyobo.getters import NoBuild
 from pyobo.utils.io import multidict
@@ -31,32 +32,6 @@ __all__ = [
 logger = logging.getLogger(__name__)
 
 
-_STATUSES = {"curated": 1, "name": 2, "synonym": 3, "former_name": 4}
-
-
-def filter_out_duplicates(terms: List[gilda.term.Term]) -> List[gilda.term.Term]:
-    """Filter out duplicates."""
-    # TODO import from gilda.term import filter_out_duplicates when it gets moved,
-    #  see https://github.com/indralab/gilda/pull/103
-    logger.debug("filtering %d terms for uniqueness", len(terms))
-    new_terms: List[gilda.term.Term] = [
-        min(terms_group, key=_status_key)
-        for _, terms_group in itt.groupby(sorted(terms, key=_term_key), key=_term_key)
-    ]
-    # Re-sort the terms
-    new_terms = sorted(new_terms, key=lambda x: (x.text, x.db, x.id))
-    logger.debug("got %d unique terms.", len(new_terms))
-    return new_terms
-
-
-def _status_key(term: gilda.term.Term) -> int:
-    return _STATUSES[term.status]
-
-
-def _term_key(term: gilda.term.Term) -> Tuple[str, str, str]:
-    return term.db, term.id, term.text
-
-
 def iter_gilda_prediction_tuples(
     prefix: str,
     relation: str = "skos:exactMatch",
@@ -115,10 +90,12 @@ def normalize_identifier(prefix: str, identifier: str) -> str:
 
 def get_grounder(
     prefixes: Union[str, Iterable[str]],
+    *,
     unnamed: Optional[Iterable[str]] = None,
     grounder_cls: Optional[Type[Grounder]] = None,
     versions: Union[None, str, Iterable[Union[str, None]]] = None,
     strict: bool = True,
+    skip_obsolete: bool = False,
 ) -> Grounder:
     """Get a Gilda grounder for the given prefix(es)."""
     unnamed = set() if unnamed is None else set(unnamed)
@@ -140,7 +117,11 @@ def get_grounder(
         try:
             p_terms = list(
                 get_gilda_terms(
-                    prefix, identifiers_are_names=prefix in unnamed, version=version, strict=strict
+                    prefix,
+                    identifiers_are_names=prefix in unnamed,
+                    version=version,
+                    strict=strict,
+                    skip_obsolete=skip_obsolete,
                 )
             )
         except NoBuild:
@@ -155,26 +136,50 @@ def get_grounder(
         return grounder_cls(terms_dict)
 
 
+def _fast_term(
+    *,
+    text: str,
+    prefix: str,
+    identifier: str,
+    name: str,
+    status: str,
+    organism: Optional[str] = None,
+) -> gilda.term.Term:
+    return gilda.term.Term(
+        norm_text=normalize(text),
+        text=text,
+        db=prefix,
+        id=identifier,
+        entry_name=name,
+        status=status,
+        source=prefix,
+        organism=organism,
+    )
+
+
 def get_gilda_terms(
     prefix: str,
+    *,
     identifiers_are_names: bool = False,
     version: Optional[str] = None,
     strict: bool = True,
+    skip_obsolete: bool = False,
 ) -> Iterable[gilda.term.Term]:
     """Get gilda terms for the given namespace."""
     id_to_name = get_id_name_mapping(prefix, version=version, strict=strict)
     id_to_species = get_id_species_mapping(prefix, version=version, strict=strict)
+    obsoletes = get_obsolete(prefix, version=version, strict=strict) if skip_obsolete else set()
 
     it = tqdm(id_to_name.items(), desc=f"[{prefix}] mapping", unit_scale=True, unit="name")
     for identifier, name in it:
-        yield gilda.term.Term(
-            norm_text=normalize(name),
+        if identifier in obsoletes:
+            continue
+        yield _fast_term(
             text=name,
-            db=prefix,
-            id=identifier,
-            entry_name=name,
+            prefix=prefix,
+            identifier=identifier,
+            name=name,
             status="name",
-            source=prefix,
             organism=id_to_species.get(identifier),
         )
 
@@ -184,29 +189,31 @@ def get_gilda_terms(
             id_to_synonyms.items(), desc=f"[{prefix}] mapping", unit_scale=True, unit="synonym"
         )
         for identifier, synonyms in it:
+            if identifier in obsoletes:
+                continue
             name = id_to_name[identifier]
             for synonym in synonyms:
-                yield gilda.term.Term(
-                    norm_text=normalize(synonym),
+                if not synonym:
+                    continue
+                yield _fast_term(
                     text=synonym,
-                    db=prefix,
-                    id=identifier,
-                    entry_name=name,
+                    prefix=prefix,
+                    identifier=identifier,
+                    name=name,
                     status="synonym",
-                    source=prefix,
                     organism=id_to_species.get(identifier),
                 )
 
     if identifiers_are_names:
         it = tqdm(get_ids(prefix), desc=f"[{prefix}] mapping", unit_scale=True, unit="id")
         for identifier in it:
-            yield gilda.term.Term(
-                norm_text=normalize(identifier),
+            if identifier in obsoletes:
+                continue
+            yield _fast_term(
                 text=identifier,
-                db=prefix,
-                id=identifier,
-                entry_name=None,
-                status="identifier",
-                source=prefix,
+                prefix=prefix,
+                identifier=identifier,
+                name=identifier,
+                status="name",
                 organism=id_to_species.get(identifier),
             )

pyobo/identifier_utils.py

@@ -119,7 +119,7 @@ def wrap_norm_prefix(f):
 
 def standardize_ec(ec: str) -> str:
     """Standardize an EC code identifier by removing all trailing dashes and dots."""
-    ec = ec.strip()
+    ec = ec.strip().replace(" ", "")
     for _ in range(4):
         ec = ec.rstrip("-").rstrip(".")
     return ec

pyobo/obographs.py

@@ -19,7 +19,7 @@ from bioontologies.obograph import (
 from bioontologies.robot import ParseResults
 
 from pyobo.struct import Obo, Reference, Term
-from pyobo.struct.typedef import is_a
+from pyobo.struct.typedef import definition_source, is_a
 
 __all__ = [
     "graph_from_obo",
@@ -61,12 +61,12 @@ def _rewire(r: Reference) -> curies.Reference:
 
 
 def _get_class_node(term: Term) -> Node:
-    if not term.definition:
-        definition = None
-    else:
+    if term.definition or term.provenance:
         definition = Definition.from_parsed(
             value=term.definition, references=[_rewire(p) for p in term.provenance]
         )
+    else:
+        definition = None
 
     if term.xrefs:
         if not term.xref_types:
@@ -126,3 +126,11 @@ def _iter_edges(term: Term) -> Iterable[Edge]:
                 _rewire(typedef.reference),
                 _rewire(target),
             )
+
+    for provenance_reference in term.provenance:
+        yield Edge.from_parsed(
+            _rewire(term.reference),
+            _rewire(definition_source.reference),
+            _rewire(provenance_reference),
+        )
+    # TODO also look through xrefs and seealso to get provenance xrefs?

pyobo/reader.py

@@ -444,20 +444,23 @@ def _extract_synonym(
             break
 
     stype: Optional[SynonymTypeDef] = None
-    if specificity is not None:  # go fishing for a synonym type definition
-        for _stype in synonym_typedefs.values():
-            # Since there aren't a lot of carefully defined synonym definitions, it
-            # can appear as a string or curie. Therefore, we might see temporary prefixes
-            # get added, so we should check against full curies as well as local unique
-            # identifiers
-            if rest.startswith(_stype.curie):
-                rest = rest[len(_stype.curie) :].strip()
-                stype = _stype
-                break
-            elif rest.startswith(_stype.identifier):
-                rest = rest[len(_stype.identifier) :].strip()
-                stype = _stype
-                break
+    for _stype in synonym_typedefs.values():
+        # Since there aren't a lot of carefully defined synonym definitions, it
+        # can appear as a string or curie. Therefore, we might see temporary prefixes
+        # get added, so we should check against full curies as well as local unique
+        # identifiers
+        if rest.startswith(_stype.curie):
+            rest = rest[len(_stype.curie) :].strip()
+            stype = _stype
+            break
+        elif rest.startswith(_stype.preferred_curie):
+            rest = rest[len(_stype.preferred_curie) :].strip()
+            stype = _stype
+            break
+        elif rest.startswith(_stype.identifier):
+            rest = rest[len(_stype.identifier) :].strip()
+            stype = _stype
+            break
 
     if not rest.startswith("[") or not rest.endswith("]"):
         logger.warning("[%s:%s] problem with synonym: %s", prefix, identifier, s)

pyobo/sources/__init__.py

@@ -20,6 +20,7 @@ from .drugcentral import DrugCentralGetter
 from .expasy import ExpasyGetter
 from .famplex import FamPlexGetter
 from .flybase import FlyBaseGetter
+from .geonames import GeonamesGetter
 from .gwascentral_phenotype import GWASCentralPhenotypeGetter
 from .gwascentral_study import GWASCentralStudyGetter
 from .hgnc import HGNCGetter
@@ -46,6 +47,7 @@ from .pubchem import PubChemCompoundGetter
 from .reactome import ReactomeGetter
 from .rgd import RGDGetter
 from .rhea import RheaGetter
+from .ror import RORGetter
 from .selventa import SCHEMGetter, SCOMPGetter, SDISGetter, SFAMGetter
 from .sgd import SGDGetter
 from .slm import SLMGetter
@@ -74,6 +76,7 @@ __all__ = [
     "FlyBaseGetter",
     "GWASCentralPhenotypeGetter",
     "GWASCentralStudyGetter",
+    "GeonamesGetter",
     "HGNCGetter",
     "HGNCGroupGetter",
     "ICD10Getter",
@@ -98,6 +101,7 @@ __all__ = [
     "PomBaseGetter",
     "PubChemCompoundGetter",
     "RGDGetter",
+    "RORGetter",
     "ReactomeGetter",
     "RheaGetter",
     "SCHEMGetter",

pyobo/sources/cgnc.py

@@ -8,6 +8,7 @@ from typing import Iterable
 import pandas as pd
 
 from pyobo.struct import Obo, Reference, Term, from_species
+from pyobo.struct.typedef import exact_match
 from pyobo.utils.path import ensure_df
 
 __all__ = [
@@ -25,7 +26,7 @@ class CGNCGetter(Obo):
 
     ontology = PREFIX
     dynamic_version = True
-    typedefs = [from_species]
+    typedefs = [from_species, exact_match]
 
     def iter_terms(self, force: bool = False) -> Iterable[Term]:
         """Iterate over terms in the ontology."""

pyobo/sources/chembl.py

@@ -12,7 +12,7 @@ from typing import Iterable
 import chembl_downloader
 
 from pyobo.struct import Obo, Reference, Term
-from pyobo.struct.typedef import has_inchi, has_smiles
+from pyobo.struct.typedef import exact_match, has_inchi, has_smiles
 
 __all__ = [
     "ChEMBLCompoundGetter",
@@ -45,6 +45,7 @@ class ChEMBLCompoundGetter(Obo):
 
     ontology = "chembl.compound"
     bioversions_key = "chembl"
+    typedefs = [exact_match]
 
     def iter_terms(self, force: bool = False) -> Iterable[Term]:
         """Iterate over terms in the ontology."""

pyobo/sources/complexportal.py

@@ -82,6 +82,17 @@ def _parse_xrefs(s) -> List[Tuple[Reference, str]]:
             logger.warning("xref missing (: %s", xref)
             continue
         note = note.rstrip(")")
+        note.replace("rhea:rhea ", "rhea:")
+        note.replace("rhea:Rhea ", "rhea:")
+        note.replace("eccode::", "eccode:")
+        note.replace("eccode:EC:", "eccode:")
+        note.replace("eccode:RHEA:", "rhea:")
+        if note.lower().startswith("rhea "):
+            note = note[len("Rhea ") :]
+            if note.lower().startswith("rhea:rhea "):
+                note = note[len("rhea:rhea ") :]
+        if note.lower().startswith("EC:"):
+            note = note[len("EC:") :]
         try:
             reference = Reference.from_curie(xref_curie)
         except ValueError:

pyobo/sources/depmap.py

@@ -8,6 +8,7 @@ import pandas as pd
 import pystow
 
 from pyobo import Obo, Reference, Term
+from pyobo.struct.typedef import exact_match
 
 __all__ = [
     "get_obo",
@@ -23,6 +24,7 @@ class DepMapGetter(Obo):
 
     ontology = bioversions_key = PREFIX
     data_version = VERSION
+    typedefs = [exact_match]
 
     def iter_terms(self, force: bool = False) -> Iterable[Term]:
         """Iterate over terms in the ontology."""

pyobo/sources/drugcentral.py

@@ -12,7 +12,7 @@ import psycopg2
 from tqdm.auto import tqdm
 
 from pyobo.struct import Obo, Reference, Synonym, Term
-from pyobo.struct.typedef import has_inchi, has_smiles
+from pyobo.struct.typedef import exact_match, has_inchi, has_smiles
 
 __all__ = [
     "DrugCentralGetter",
@@ -34,6 +34,7 @@ class DrugCentralGetter(Obo):
     """An ontology representation of the DrugCentral database."""
 
     ontology = bioversions_key = PREFIX
+    typedefs = [exact_match]
 
     def iter_terms(self, force: bool = False) -> Iterable[Term]:
         """Iterate over terms in the ontology."""

pyobo/sources/geonames.py

@@ -0,0 +1,239 @@
+"""Get terms from geonames."""
+
+import logging
+from typing import Collection, Iterable, Mapping
+
+import pandas as pd
+from pystow.utils import read_zipfile_csv
+from tqdm import tqdm
+
+from pyobo import Obo, Term
+from pyobo.struct import Reference, part_of
+from pyobo.utils.path import ensure_df, ensure_path
+
+__all__ = ["GeonamesGetter"]
+
+logger = logging.getLogger(__name__)
+
+PREFIX = "geonames"
+COUNTRIES_URL = "https://download.geonames.org/export/dump/countryInfo.txt"
+ADMIN1_URL = "https://download.geonames.org/export/dump/admin1CodesASCII.txt"
+ADMIN2_URL = "https://download.geonames.org/export/dump/admin2Codes.txt"
+CITIES_URL = "https://download.geonames.org/export/dump/cities15000.zip"
+
+
+class GeonamesGetter(Obo):
+    """An ontology representation of GeoNames."""
+
+    ontology = PREFIX
+    dynamic_version = True
+    typedefs = [part_of]
+
+    def iter_terms(self, force: bool = False) -> Iterable[Term]:
+        """Iterate over terms in the ontology."""
+        return get_terms(force=force)
+
+
+def get_terms(*, force: bool = False) -> Collection[Term]:
+    """Get terms."""
+    code_to_country = get_code_to_country(force=force)
+    code_to_admin1 = get_code_to_admin1(code_to_country, force=force)
+    code_to_admin2 = get_code_to_admin2(
+        code_to_country=code_to_country, code_to_admin1=code_to_admin1, force=force
+    )
+    id_to_term = get_cities(
+        code_to_country=code_to_country,
+        code_to_admin1=code_to_admin1,
+        code_to_admin2=code_to_admin2,
+        force=force,
+    )
+    return id_to_term.values()
+
+
+def get_code_to_country(*, force: bool = False) -> Mapping[str, Term]:
+    """Get a mapping from country code to country term."""
+    countries_df = ensure_df(
+        PREFIX,
+        url=COUNTRIES_URL,
+        force=force,
+        skiprows=49,
+        keep_default_na=False,  # NA is a country code
+        dtype=str,
+    )
+    logger.info(f"got {len(countries_df.index):,} countries")
+    reorder = ["geonameid", *(c for c in countries_df.columns if c != "geonameid")]
+    countries_df = countries_df[reorder]
+    code_to_country = {}
+    cols = ["geonameid", "Country", "#ISO", "fips", "ISO3"]
+    for identifier, name, code, fips, iso3 in countries_df[cols].values:
+        if pd.isna(code):
+            continue
+        term = Term.from_triple(
+            "geonames", identifier, name if pd.notna(name) else None, type="Instance"
+        )
+        term.append_synonym(code)
+        if name.startswith("The "):
+            term.append_synonym(name.removeprefix("The "))
+        if pd.notna(fips):
+            term.append_synonym(fips)
+        if pd.notna(iso3):
+            term.append_synonym(iso3)
+        term.append_property("code", code)
+        code_to_country[code] = term
+    logger.info(f"got {len(code_to_country):,} country records")
+    return code_to_country
+
+
+def get_code_to_admin1(
+    code_to_country: Mapping[str, Term], *, force: bool = False
+) -> Mapping[str, Term]:
+    """Get a mapping from admin1 code to term."""
+    admin1_df = ensure_df(
+        PREFIX,
+        url=ADMIN1_URL,
+        header=None,
+        names=["code", "name", "asciiname", "geonames_id"],
+        dtype=str,
+        force=force,
+    )
+    code_to_admin1 = {}
+    for code, name, asciiname, identifier in admin1_df.values:
+        if pd.isna(identifier) or pd.isna(code):
+            tqdm.write(f"Missing info for  {name} / {asciiname} / {code=} / {identifier=}")
+            continue
+
+        term = Term.from_triple(
+            "geonames", identifier, name if pd.notna(name) else None, type="Instance"
+        )
+        term.append_property("code", code)
+        code_to_admin1[code] = term
+
+        country_code = code.split(".")[0]
+        country_term = code_to_country[country_code]
+        term.append_relationship(part_of, country_term)
+    return code_to_admin1
+
+
+def get_code_to_admin2(
+    *, code_to_country: Mapping[str, Term], code_to_admin1: Mapping[str, Term], force: bool = False
+) -> Mapping[str, Term]:
+    """Get a mapping from admin2 code to term."""
+    admin2_df = ensure_df(
+        PREFIX,
+        url=ADMIN2_URL,
+        header=None,
+        names=["code", "name", "asciiname", "geonames_id"],
+        dtype=str,
+        force=force,
+    )
+    code_to_admin2 = {}
+    for identifier, name, code in admin2_df[["geonames_id", "name", "code"]].values:
+        if pd.isna(identifier) or pd.isna(code):
+            continue
+        term = Term.from_triple(
+            "geonames", identifier, name if pd.notna(name) else None, type="Instance"
+        )
+        term.append_property("code", code)
+        code_to_admin2[code] = term
+        admin1_code = code.rsplit(".", 1)[0]
+        admin1_term = code_to_admin1.get(admin1_code)
+        if admin1_term:
+            term.append_relationship(part_of, admin1_term)
+        else:
+            country_code = admin1_code.split(".", 1)[0]
+            country_term = code_to_country[country_code]
+            term.append_relationship(part_of, country_term)
+    return code_to_admin2
+
+
+def get_cities(
+    code_to_country,
+    code_to_admin1,
+    code_to_admin2,
+    *,
+    minimum_population: int = 100_000,
+    force: bool = False,
+) -> Mapping[str, Term]:
+    """Get a mapping from city code to term."""
+    columns = [
+        "geonames_id",
+        "name",
+        "asciiname",
+        "synonyms",
+        "latitude",
+        "longitude",
+        "feature_class",
+        "feature_code",
+        "country_code",
+        "cc2",
+        "admin1",
+        "admin2",
+        "admin3",
+        "admin4",
+        "population",
+        "elevation",
+        "dem",
+        "timezone",
+        "date_modified",
+    ]
+    path = ensure_path(PREFIX, url=CITIES_URL, force=force)
+    cities_df = read_zipfile_csv(
+        path=path,
+        inner_path="cities15000.txt",
+        header=None,
+        names=columns,
+        dtype=str,
+    )
+
+    cities_df = cities_df[cities_df.population.astype(int) > minimum_population]
+    cities_df.synonyms = cities_df.synonyms.str.split(",")
+
+    terms = {}
+    for term in code_to_country.values():
+        terms[term.identifier] = term
+
+    cols = ["geonames_id", "name", "synonyms", "country_code", "admin1", "admin2", "feature_code"]
+    for identifier, name, synonyms, country, admin1, admin2, feature_code in cities_df[cols].values:
+        terms[identifier] = term = Term.from_triple(
+            "geonames", identifier, name if pd.notna(name) else None, type="Instance"
+        )
+        term.append_parent(Reference(prefix="geonames.feature", identifier=feature_code))
+        if synonyms and not isinstance(synonyms, float):
+            for synonym in synonyms:
+                if pd.notna(synonym):
+                    term.append_synonym(synonym)
+
+        if pd.isna(admin1):
+            # TODO try to annotate these directly onto countries
+            tqdm.write(
+                f"[geonames:{identifier}] {name}, a city in {country}, is missing admin 1 code"
+            )
+            continue
+
+        admin1_full = f"{country}.{admin1}"
+        admin1_term = code_to_admin1.get(admin1_full)
+        if admin1_term is None:
+            logger.info(f"could not find admin1 {admin1_full}")
+            continue
+
+        terms[admin1_term.identifier] = admin1_term
+
+        if pd.notna(admin2):
+            admin2_full = f"{country}.{admin1}.{admin2}"
+            admin2_term = code_to_admin2.get(admin2_full)
+            if admin2_term is None or admin1_term is None:
+                pass
+                # print("could not find admin2", admin2_full)
+            else:
+                term.append_relationship(part_of, admin2_term)
+                terms[admin2_term.identifier] = admin2_term
+
+        else:  # pd.notna(admin1):
+            # If there's no admin 2, just annotate directly onto admin 1
+            term.append_relationship(part_of, admin1_term)
+
+    return terms
+
+
+if __name__ == "__main__":
+    GeonamesGetter().write_default(write_obo=True, force=True)