pyobo 0.11.2__py3-none-any.whl → 0.12.0__py3-none-any.whl

pyobo/{xrefdb/xrefs_pipeline.py → cli/database_utils.py}

@@ -1,84 +1,55 @@
 """Pipeline for extracting all xrefs from OBO documents available."""
 
+from __future__ import annotations
+
 import gzip
-import itertools as itt
 import logging
+import warnings
 from collections.abc import Iterable
-from typing import Optional, cast
+from functools import partial
+from typing import cast
 
 import bioregistry
-import networkx as nx
-import pandas as pd
 from tqdm.auto import tqdm
+from typing_extensions import Unpack
 
-from .sources import iter_xref_plugins
-from .. import get_xrefs_df
 from ..api import (
+    get_edges_df,
     get_id_definition_mapping,
     get_id_name_mapping,
     get_id_species_mapping,
     get_id_synonyms_mapping,
     get_id_to_alts,
+    get_mappings_df,
     get_metadata,
     get_properties_df,
     get_relations_df,
     get_typedef_df,
+    get_xrefs_df,
 )
-from ..constants import SOURCE_ID, SOURCE_PREFIX, TARGET_ID, TARGET_PREFIX
-from ..getters import iter_helper, iter_helper_helper
-from ..sources import ncbigene, pubchem
+from ..getters import IterHelperHelperDict, iter_helper, iter_helper_helper
+from ..sources import pubchem
+from ..sources.ncbi import ncbigene
 from ..utils.path import ensure_path
 
 logger = logging.getLogger(__name__)
 
 
-# TODO a normal graph can easily be turned into a directed graph where each
-#  edge points from low priority to higher priority, then the graph can
-#  be reduced to a set of star graphs and ultimately to a single dictionary
-
-
-def get_graph_from_xref_df(df: pd.DataFrame) -> nx.Graph:
-    """Generate a graph from the mappings dataframe."""
-    rv = nx.Graph()
-
-    it = itt.chain(
-        df[[SOURCE_PREFIX, SOURCE_ID]].drop_duplicates().values,
-        df[[TARGET_PREFIX, TARGET_ID]].drop_duplicates().values,
-    )
-    it = tqdm(it, desc="loading curies", unit_scale=True)
-    for prefix, identifier in it:
-        rv.add_node(_to_curie(prefix, identifier), prefix=prefix, identifier=identifier)
-
-    it = tqdm(df.values, total=len(df.index), desc="loading xrefs", unit_scale=True)
-    for source_ns, source_id, target_ns, target_id, provenance in it:
-        rv.add_edge(
-            _to_curie(source_ns, source_id),
-            _to_curie(target_ns, target_id),
-            provenance=provenance,
-        )
-
-    return rv
-
-
-def _to_curie(prefix: str, identifier: str) -> str:
-    return f"{prefix}:{identifier}"
-
-
-def _iter_ncbigene(left, right):
+def _iter_ncbigene(left: int, right: int) -> Iterable[tuple[str, str, str]]:
     ncbi_path = ensure_path(ncbigene.PREFIX, url=ncbigene.GENE_INFO_URL)
     with gzip.open(ncbi_path, "rt") as file:
         next(file)  # throw away the header
         for line in tqdm(
             file, desc=f"extracting {ncbigene.PREFIX}", unit_scale=True, total=27_000_000
         ):
-            line = line.strip().split("\t")
-            yield ncbigene.PREFIX, line[left], line[right]
+            parts = line.strip().split("\t")
+            yield ncbigene.PREFIX, parts[left], parts[right]
 
 
-def _iter_metadata(**kwargs):
+def _iter_metadata(**kwargs: Unpack[IterHelperHelperDict]):
     for prefix, data in iter_helper_helper(get_metadata, **kwargs):
         version = data["version"]
-        tqdm.write(f"[{prefix}] using version {version}")
+        logger.debug(f"[{prefix}] using version {version}")
         yield prefix, version, data["date"], bioregistry.is_deprecated(prefix)
 
 
@@ -99,29 +70,33 @@ def _iter_names(leave: bool = False, **kwargs) -> Iterable[tuple[str, str, str]]
             yield pubchem.PREFIX, identifier, name
 
 
-def _iter_species(leave: bool = False, **kwargs) -> Iterable[tuple[str, str, str]]:
+def _iter_species(
+    leave: bool = False, **kwargs: Unpack[IterHelperHelperDict]
+) -> Iterable[tuple[str, str, str]]:
     """Iterate over all prefix-identifier-species triples we can get."""
     yield from iter_helper(get_id_species_mapping, leave=leave, **kwargs)
     # TODO ncbigene
 
 
-def _iter_definitions(leave: bool = False, **kwargs) -> Iterable[tuple[str, str, str]]:
+def _iter_definitions(
+    leave: bool = False, **kwargs: Unpack[IterHelperHelperDict]
+) -> Iterable[tuple[str, str, str]]:
     """Iterate over all prefix-identifier-descriptions triples we can get."""
     yield from iter_helper(get_id_definition_mapping, leave=leave, **kwargs)
     yield from _iter_ncbigene(1, 8)
 
 
 def _iter_alts(
-    leave: bool = False, strict: bool = True, **kwargs
+    leave: bool = False, **kwargs: Unpack[IterHelperHelperDict]
 ) -> Iterable[tuple[str, str, str]]:
-    for prefix, identifier, alts in iter_helper(
-        get_id_to_alts, leave=leave, strict=strict, **kwargs
-    ):
+    for prefix, identifier, alts in iter_helper(get_id_to_alts, leave=leave, **kwargs):
         for alt in alts:
             yield prefix, identifier, alt
 
 
-def _iter_synonyms(leave: bool = False, **kwargs) -> Iterable[tuple[str, str, str]]:
+def _iter_synonyms(
+    leave: bool = False, **kwargs: Unpack[IterHelperHelperDict]
+) -> Iterable[tuple[str, str, str]]:
     """Iterate over all prefix-identifier-synonym triples we can get.
 
     :param leave: should the tqdm be left behind?
@@ -131,7 +106,7 @@ def _iter_synonyms(leave: bool = False, **kwargs) -> Iterable[tuple[str, str, st
             yield prefix, identifier, synonym
 
 
-def _iter_typedefs(**kwargs) -> Iterable[tuple[str, str, str, str]]:
+def _iter_typedefs(**kwargs: Unpack[IterHelperHelperDict]) -> Iterable[tuple[str, str, str, str]]:
     """Iterate over all prefix-identifier-name triples we can get."""
     for prefix, df in iter_helper_helper(get_typedef_df, **kwargs):
         for t in df.values:
@@ -139,14 +114,22 @@ def _iter_typedefs(**kwargs) -> Iterable[tuple[str, str, str, str]]:
                 yield cast(tuple[str, str, str, str], (prefix, *t))
 
 
-def _iter_relations(**kwargs) -> Iterable[tuple[str, str, str, str, str, str]]:
+def _iter_relations(
+    **kwargs: Unpack[IterHelperHelperDict],
+) -> Iterable[tuple[str, str, str, str, str, str]]:
     for prefix, df in iter_helper_helper(get_relations_df, **kwargs):
         for t in df.values:
             if all(t):
                 yield cast(tuple[str, str, str, str, str, str], (prefix, *t))
 
 
-def _iter_properties(**kwargs) -> Iterable[tuple[str, str, str, str]]:
+def _iter_edges(**kwargs: Unpack[IterHelperHelperDict]) -> Iterable[tuple[str, str, str, str]]:
+    for prefix, df in iter_helper_helper(get_edges_df, **kwargs):
+        for row in df.values:
+            yield cast(tuple[str, str, str, str], (*row, prefix))
+
+
+def _iter_properties(**kwargs: Unpack[IterHelperHelperDict]) -> Iterable[tuple[str, str, str, str]]:
     for prefix, df in iter_helper_helper(get_properties_df, **kwargs):
         for t in df.values:
             if all(t):
@@ -154,27 +137,25 @@ def _iter_properties(**kwargs) -> Iterable[tuple[str, str, str, str]]:
 
 
 def _iter_xrefs(
-    *,
-    force: bool = False,
-    use_tqdm: bool = True,
-    skip_below: Optional[str] = None,
-    strict: bool = True,
-    **kwargs,
+    **kwargs: Unpack[IterHelperHelperDict],
 ) -> Iterable[tuple[str, str, str, str, str]]:
-    it = iter_helper_helper(
-        get_xrefs_df,
-        use_tqdm=use_tqdm,
-        force=force,
-        skip_below=skip_below,
-        strict=strict,
-        **kwargs,
-    )
+    warnings.warn(f"use {_iter_mappings.__name__} instead", DeprecationWarning, stacklevel=2)
+    it = iter_helper_helper(get_xrefs_df, **kwargs)
     for prefix, df in it:
         df.dropna(inplace=True)
         for row in df.values:
             if any(not element for element in row):
                 continue
             yield cast(tuple[str, str, str, str, str], (prefix, *row, prefix))
-    for df in iter_xref_plugins(skip_below=skip_below):
-        df.dropna(inplace=True)
-        yield from tqdm(df.values, leave=False, total=len(df.index), unit_scale=True)
+
+
+def _iter_mappings(
+    **kwargs: Unpack[IterHelperHelperDict],
+) -> Iterable[tuple[str, str, str, str, str]]:
+    f = partial(get_mappings_df, names=False, include_mapping_source_column=True)
+    # hack in a name to the partial function object since
+    # it's used for the tqdm description in iter_helper_helper
+    f.__name__ = "get_mappings_df"  # type:ignore
+    it = iter_helper_helper(f, **kwargs)
+    for _prefix, df in it:
+        yield from df.values