pyobo 0.11.2__py3-none-any.whl → 0.12.1__py3-none-any.whl

pyobo/cli/database.py

@@ -1,14 +1,40 @@
 """CLI for PyOBO Database Generation."""
 
 import logging
-from typing import Optional
+import warnings
+from collections.abc import Iterable
+from pathlib import Path
 
+import bioregistry
 import click
 from more_click import verbose_option
 from tqdm.contrib.logging import logging_redirect_tqdm
+from typing_extensions import Unpack
 from zenodo_client import update_zenodo
 
-from .utils import directory_option, force_option, no_strict_option, zenodo_option
+from .database_utils import (
+    IterHelperHelperDict,
+    _iter_alts,
+    _iter_definitions,
+    _iter_edges,
+    _iter_mappings,
+    _iter_names,
+    _iter_properties,
+    _iter_relations,
+    _iter_species,
+    _iter_synonyms,
+    _iter_typedefs,
+    _iter_xrefs,
+    iter_helper_helper,
+)
+from .utils import (
+    Clickable,
+    directory_option,
+    force_option,
+    force_process_option,
+    strict_option,
+    zenodo_option,
+)
 from ..constants import (
     ALTS_DATA_RECORD,
     DEFINITIONS_RECORD,
@@ -19,39 +45,60 @@ from ..constants import (
     SPECIES_RECORD,
     SYNONYMS_RECORD,
     TYPEDEFS_RECORD,
+    DatabaseKwargs,
 )
-from ..getters import db_output_helper
-from ..xrefdb.xrefs_pipeline import (
-    _iter_alts,
-    _iter_definitions,
-    _iter_metadata,
-    _iter_names,
-    _iter_properties,
-    _iter_relations,
-    _iter_species,
-    _iter_synonyms,
-    _iter_typedefs,
-    _iter_xrefs,
-)
+from ..getters import db_output_helper, get_ontology
 
 __all__ = [
     "main",
 ]
 
+logger = logging.getLogger(__name__)
+
 
 @click.group(name="database")
 def main():
     """Build the PyOBO Database."""
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
+skip_pyobo_option = click.option(
+    "--skip-pyobo",
+    is_flag=True,
+    help="Skip prefixes whose ontologies are implemented as PyOBO sources",
+)
+skip_below_option = click.option(
+    "--skip-below", help="Skip prefixes lexically sorted below the given one"
+)
+
+
+def database_annotate(f: Clickable) -> Clickable:
+    """Add appropriate decorators to database CLI functions."""
+    decorators = [
+        main.command(),
+        zenodo_option,
+        verbose_option,
+        directory_option,
+        force_option,
+        force_process_option,
+        strict_option,
+        skip_pyobo_option,
+        skip_below_option,
+    ]
+    for decorator in decorators:
+        f = decorator(f)
+    return f
+
+
+def _update_database_kwargs(kwargs: DatabaseKwargs) -> DatabaseKwargs:
+    updated_kwargs = dict(kwargs)
+    updated_kwargs.update(force=False, force_process=False)
+    # FIXME get typing right on next line
+    return updated_kwargs  # type:ignore
+
+
+@database_annotate
 @click.pass_context
-def build(ctx: click.Context, directory: str, zenodo: bool, no_strict: bool, force: bool):
+def build(ctx: click.Context, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Build all databases."""
     # if no_strict and zenodo:
     #    click.secho("Must be strict before uploading", fg="red")
@@ -59,103 +106,97 @@ def build(ctx: click.Context, directory: str, zenodo: bool, no_strict: bool, for
     with logging_redirect_tqdm():
         click.secho("Collecting metadata and building", fg="cyan", bold=True)
         # note that this is the only one that needs a force=force
-        ctx.invoke(metadata, directory=directory, no_strict=no_strict, force=force)
+        ctx.invoke(metadata, **kwargs)
+
+        # After running once, we don't want to force or re-process.
+        # All the other arguments come along for the ride!
+        updated_kwargs = _update_database_kwargs(kwargs)
+
         click.secho("Alternate Identifiers", fg="cyan", bold=True)
-        ctx.invoke(alts, directory=directory, zenodo=zenodo, no_strict=no_strict)
+        ctx.invoke(alts, **updated_kwargs)
         click.secho("Synonyms", fg="cyan", bold=True)
-        ctx.invoke(synonyms, directory=directory, zenodo=zenodo, no_strict=no_strict)
-        click.secho("Xrefs", fg="cyan", bold=True)
-        ctx.invoke(xrefs, directory=directory, zenodo=zenodo, no_strict=no_strict)
+        ctx.invoke(synonyms, **updated_kwargs)
+        click.secho("Mappings", fg="cyan", bold=True)
+        ctx.invoke(mappings, **updated_kwargs)
         click.secho("Names", fg="cyan", bold=True)
-        ctx.invoke(names, directory=directory, zenodo=zenodo, no_strict=no_strict)
+        ctx.invoke(names, **updated_kwargs)
         click.secho("Definitions", fg="cyan", bold=True)
-        ctx.invoke(definitions, directory=directory, zenodo=zenodo, no_strict=no_strict)
+        ctx.invoke(definitions, **updated_kwargs)
         click.secho("Properties", fg="cyan", bold=True)
-        ctx.invoke(properties, directory=directory, zenodo=zenodo, no_strict=no_strict)
+        ctx.invoke(properties, **updated_kwargs)
         click.secho("Relations", fg="cyan", bold=True)
-        ctx.invoke(relations, directory=directory, zenodo=zenodo, no_strict=no_strict)
+        ctx.invoke(relations, **updated_kwargs)
+        click.secho("Edges", fg="cyan", bold=True)
+        ctx.invoke(edges, **updated_kwargs)
         click.secho("Typedefs", fg="cyan", bold=True)
-        ctx.invoke(typedefs, directory=directory, zenodo=zenodo, no_strict=no_strict)
+        ctx.invoke(typedefs, **updated_kwargs)
         click.secho("Species", fg="cyan", bold=True)
-        ctx.invoke(species, directory=directory, zenodo=zenodo, no_strict=no_strict)
+        ctx.invoke(species, **updated_kwargs)
+
 
+@database_annotate
+def cache(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
+    """Cache all things."""
+    if zenodo:
+        click.echo("no zenodo for caching")
 
-skip_below_option = click.option("--skip-below")
-skip_below_exclusive_option = click.option("--skip-below-exclusive", is_flag=True)
+    kwargs["force_process"] = True
+    with logging_redirect_tqdm():
+        for _ in iter_helper_helper(get_ontology, **kwargs):
+            # this pass intentional to consume the iterable
+            pass
 
 
-@main.command()
-@verbose_option
-@directory_option
-@force_option
-@no_strict_option
-@skip_below_option
-@click.option("--skip-pyobo")
-def metadata(
-    directory: str, no_strict: bool, force: bool, skip_below: Optional[str], skip_pyobo: bool
-):
+@database_annotate
+def metadata(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the prefix-metadata dump."""
+    from ..api import get_metadata
+
+    def _iter_metadata(
+        **kwargs: Unpack[IterHelperHelperDict],
+    ) -> Iterable[tuple[str, str, str, bool]]:
+        for prefix, data in iter_helper_helper(get_metadata, **kwargs):
+            version = data["version"]
+            logger.debug(f"[{prefix}] using version {version}")
+            yield prefix, version, data["date"], bioregistry.is_deprecated(prefix)
+
+    it = _iter_metadata(**kwargs)
     db_output_helper(
-        _iter_metadata,
+        it,
         "metadata",
         ("prefix", "version", "date", "deprecated"),
-        strict=not no_strict,
-        force=force,
-        directory=directory,
         use_gzip=False,
-        skip_below=skip_below,
-        skip_pyobo=skip_pyobo,
+        directory=directory,
     )
+    if zenodo:
+        click.secho("No Zenodo record for metadata", fg="red")
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-@skip_below_option
-@skip_below_exclusive_option
-def names(
-    directory: str,
-    zenodo: bool,
-    no_strict: bool,
-    force: bool,
-    skip_below: Optional[str],
-    skip_below_exclusive: bool,
-):
+@database_annotate
+def names(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the prefix-identifier-name dump."""
+    it = _iter_names(**kwargs)
     with logging_redirect_tqdm():
         paths = db_output_helper(
-            _iter_names,
+            it,
             "names",
             ("prefix", "identifier", "name"),
-            strict=not no_strict,
-            force=force,
             directory=directory,
-            skip_below=skip_below,
-            skip_below_inclusive=not skip_below_exclusive,
         )
     if zenodo:
         # see https://zenodo.org/record/4020486
         update_zenodo(OOH_NA_NA_RECORD, paths)
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-def species(directory: str, zenodo: bool, no_strict: bool, force: bool):
+@database_annotate
+def species(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the prefix-identifier-species dump."""
     with logging_redirect_tqdm():
+        it = _iter_species(**kwargs)
         paths = db_output_helper(
-            _iter_species,
+            it,
             "species",
             ("prefix", "identifier", "species"),
-            strict=not no_strict,
-            force=force,
             directory=directory,
         )
     if zenodo:
@@ -163,110 +204,90 @@ def species(directory: str, zenodo: bool, no_strict: bool, force: bool):
         update_zenodo(SPECIES_RECORD, paths)
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-def definitions(directory: str, zenodo: bool, no_strict: bool, force: bool):
+def _extend_skip_set(kwargs: DatabaseKwargs, skip_set: set[str]) -> None:
+    ss = kwargs.get("skip_set")
+    if ss is None:
+        kwargs["skip_set"] = skip_set
+    else:
+        ss.update(skip_set)
+
+
+@database_annotate
+def definitions(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the prefix-identifier-definition dump."""
     with logging_redirect_tqdm():
+        _extend_skip_set(kwargs, {"kegg.pathway", "kegg.genes", "kegg.genome", "umls"})
+        it = _iter_definitions(**kwargs)
         paths = db_output_helper(
-            _iter_definitions,
+            it,
             "definitions",
             ("prefix", "identifier", "definition"),
-            strict=not no_strict,
-            force=force,
             directory=directory,
-            skip_set={"kegg.pathway", "kegg.genes", "kegg.genome", "umls"},
         )
     if zenodo:
         # see https://zenodo.org/record/4637061
         update_zenodo(DEFINITIONS_RECORD, paths)
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-def typedefs(directory: str, zenodo: bool, no_strict: bool, force: bool):
+@database_annotate
+def typedefs(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the typedef prefix-identifier-name dump."""
     with logging_redirect_tqdm():
+        _extend_skip_set(kwargs, {"ncbigene", "kegg.pathway", "kegg.genes", "kegg.genome"})
+        it = _iter_typedefs(**kwargs)
         paths = db_output_helper(
-            _iter_typedefs,
+            it,
             "typedefs",
             ("prefix", "typedef_prefix", "identifier", "name"),
-            strict=not no_strict,
-            force=force,
-            directory=directory,
             use_gzip=False,
-            skip_set={"ncbigene", "kegg.pathway", "kegg.genes", "kegg.genome"},
+            directory=directory,
         )
     if zenodo:
         # see https://zenodo.org/record/4644013
         update_zenodo(TYPEDEFS_RECORD, paths)
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-def alts(directory: str, zenodo: bool, force: bool, no_strict: bool):
+@database_annotate
+def alts(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the prefix-alt-id dump."""
     with logging_redirect_tqdm():
+        _extend_skip_set(kwargs, {"kegg.pathway", "kegg.genes", "kegg.genome", "umls"})
+        it = _iter_alts(**kwargs)
         paths = db_output_helper(
-            _iter_alts,
+            it,
             "alts",
             ("prefix", "identifier", "alt"),
             directory=directory,
-            force=force,
-            strict=not no_strict,
-            skip_set={"kegg.pathway", "kegg.genes", "kegg.genome", "umls"},
         )
     if zenodo:
         # see https://zenodo.org/record/4021476
         update_zenodo(ALTS_DATA_RECORD, paths)
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-def synonyms(directory: str, zenodo: bool, force: bool, no_strict: bool):
+@database_annotate
+def synonyms(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the prefix-identifier-synonym dump."""
     with logging_redirect_tqdm():
+        _extend_skip_set(kwargs, {"kegg.pathway", "kegg.genes", "kegg.genome"})
+        it = _iter_synonyms(**kwargs)
         paths = db_output_helper(
-            _iter_synonyms,
+            it,
             "synonyms",
             ("prefix", "identifier", "synonym"),
             directory=directory,
-            force=force,
-            strict=not no_strict,
-            skip_set={"kegg.pathway", "kegg.genes", "kegg.genome"},
         )
     if zenodo:
         # see https://zenodo.org/record/4021482
         update_zenodo(SYNONYMS_RECORD, paths)
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-def relations(directory: str, zenodo: bool, force: bool, no_strict: bool):
+@database_annotate
+def relations(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the relation dump."""
     with logging_redirect_tqdm():
+        it = _iter_relations(**kwargs)
         paths = db_output_helper(
-            _iter_relations,
+            it,
             "relations",
             (
                 "source_prefix",
@@ -276,62 +297,91 @@ def relations(directory: str, zenodo: bool, force: bool, no_strict: bool):
                 "target_prefix",
                 "target_identifier",
             ),
-            directory=directory,
-            force=force,
-            strict=not no_strict,
             summary_detailed=(0, 2, 3),  # second column corresponds to relation type
+            directory=directory,
         )
     if zenodo:
         # see https://zenodo.org/record/4625167
         update_zenodo(RELATIONS_RECORD, paths)
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-def properties(directory: str, zenodo: bool, force: bool, no_strict: bool):
+@database_annotate
+def edges(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
+    """Make the edges dump."""
+    with logging_redirect_tqdm():
+        it = _iter_edges(**kwargs)
+        db_output_helper(
+            it,
+            "edges",
+            (
+                ":START_ID",
+                ":TYPE",
+                ":END_ID",
+                "provenance",
+            ),
+            directory=directory,
+        )
+    if zenodo:
+        raise NotImplementedError
+
+
+@database_annotate
+def properties(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the properties dump."""
     with logging_redirect_tqdm():
+        it = _iter_properties(**kwargs)
         paths = db_output_helper(
-            _iter_properties,
+            it,
             "properties",
             ("prefix", "identifier", "property", "value"),
-            directory=directory,
-            force=force,
-            strict=not no_strict,
             summary_detailed=(0, 2),  # second column corresponds to property type
+            directory=directory,
         )
     if zenodo:
         # see https://zenodo.org/record/4625172
         update_zenodo(PROPERTIES_RECORD, paths)
 
 
-@main.command()
-@verbose_option
-@directory_option
-@zenodo_option
-@force_option
-@no_strict_option
-def xrefs(directory: str, zenodo: bool, force: bool, no_strict: bool):
+@database_annotate
+def xrefs(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
     """Make the prefix-identifier-xref dump."""
+    warnings.warn("Use pyobo.database.mappings instead", DeprecationWarning, stacklevel=2)
     with logging_redirect_tqdm():
+        it = _iter_xrefs(**kwargs)
         paths = db_output_helper(
-            _iter_xrefs,
+            it,
             "xrefs",
             ("prefix", "identifier", "xref_prefix", "xref_identifier", "provenance"),
-            directory=directory,
-            force=force,
-            strict=not no_strict,
             summary_detailed=(0, 2),  # second column corresponds to xref prefix
+            directory=directory,
         )
     if zenodo:
         # see https://zenodo.org/record/4021477
         update_zenodo(JAVERT_RECORD, paths)
 
 
+@database_annotate
+def mappings(zenodo: bool, directory: Path, **kwargs: Unpack[DatabaseKwargs]) -> None:
+    """Make the SSSOM dump."""
+    columns = [
+        "subject_id",
+        "object_id",
+        "predicate_id",
+        "mapping_justification",
+        "mapping_source",
+    ]
+    with logging_redirect_tqdm():
+        it = _iter_mappings(**kwargs)
+        db_output_helper(
+            it,
+            "mappings",
+            columns,
+            directory=directory,
+        )
+    if zenodo:
+        raise NotImplementedError("need to do initial manual upload of SSSOM build")
+
+
 if __name__ == "__main__":
     logging.captureWarnings(True)
     with logging_redirect_tqdm():

pyobo/cli/database_utils.py

@@ -0,0 +1,155 @@
+"""Pipeline for extracting all xrefs from OBO documents available."""
+
+from __future__ import annotations
+
+import gzip
+import logging
+import warnings
+from collections.abc import Iterable
+from functools import partial
+from typing import cast
+
+from tqdm.auto import tqdm
+from typing_extensions import Unpack
+
+from ..api import (
+    get_edges_df,
+    get_id_definition_mapping,
+    get_id_name_mapping,
+    get_id_species_mapping,
+    get_id_synonyms_mapping,
+    get_id_to_alts,
+    get_mappings_df,
+    get_properties_df,
+    get_relations_df,
+    get_typedef_df,
+    get_xrefs_df,
+)
+from ..getters import IterHelperHelperDict, iter_helper, iter_helper_helper
+from ..sources import pubchem
+from ..sources.ncbi import ncbigene
+from ..utils.path import ensure_path
+
+logger = logging.getLogger(__name__)
+
+
+def _iter_ncbigene(left: int, right: int) -> Iterable[tuple[str, str, str]]:
+    ncbi_path = ensure_path(ncbigene.PREFIX, url=ncbigene.GENE_INFO_URL)
+    with gzip.open(ncbi_path, "rt") as file:
+        next(file)  # throw away the header
+        for line in tqdm(
+            file, desc=f"[{ncbigene.PREFIX}] extracting names", unit_scale=True, total=56_700_000
+        ):
+            parts = line.strip().split("\t")
+            yield ncbigene.PREFIX, parts[left], parts[right]
+
+
+def _iter_names(leave: bool = False, **kwargs) -> Iterable[tuple[str, str, str]]:
+    """Iterate over all prefix-identifier-name triples we can get.
+
+    :param leave: should the tqdm be left behind?
+    """
+    yield from iter_helper(get_id_name_mapping, leave=leave, **kwargs)
+    yield from _iter_ncbigene(1, 2)
+    yield from _iter_pubchem_compound()
+
+
+def _iter_pubchem_compound():
+    pcc_path = pubchem._ensure_cid_name_path()
+    with gzip.open(pcc_path, mode="rt", encoding="ISO-8859-1") as file:
+        for line in tqdm(
+            file, desc=f"[{pubchem.PREFIX}] extracting names", unit_scale=True, total=119_000_000
+        ):
+            identifier, name = line.strip().split("\t", 1)
+            yield pubchem.PREFIX, identifier, name
+
+
+def _iter_species(
+    leave: bool = False, **kwargs: Unpack[IterHelperHelperDict]
+) -> Iterable[tuple[str, str, str]]:
+    """Iterate over all prefix-identifier-species triples we can get."""
+    yield from iter_helper(get_id_species_mapping, leave=leave, **kwargs)
+    # TODO ncbigene
+
+
+def _iter_definitions(
+    leave: bool = False, **kwargs: Unpack[IterHelperHelperDict]
+) -> Iterable[tuple[str, str, str]]:
+    """Iterate over all prefix-identifier-descriptions triples we can get."""
+    yield from iter_helper(get_id_definition_mapping, leave=leave, **kwargs)
+    yield from _iter_ncbigene(1, 8)
+
+
+def _iter_alts(
+    leave: bool = False, **kwargs: Unpack[IterHelperHelperDict]
+) -> Iterable[tuple[str, str, str]]:
+    for prefix, identifier, alts in iter_helper(get_id_to_alts, leave=leave, **kwargs):
+        for alt in alts:
+            yield prefix, identifier, alt
+
+
+def _iter_synonyms(
+    leave: bool = False, **kwargs: Unpack[IterHelperHelperDict]
+) -> Iterable[tuple[str, str, str]]:
+    """Iterate over all prefix-identifier-synonym triples we can get.
+
+    :param leave: should the tqdm be left behind?
+    """
+    for prefix, identifier, synonyms in iter_helper(get_id_synonyms_mapping, leave=leave, **kwargs):
+        for synonym in synonyms:
+            yield prefix, identifier, synonym
+
+
+def _iter_typedefs(**kwargs: Unpack[IterHelperHelperDict]) -> Iterable[tuple[str, str, str, str]]:
+    """Iterate over all prefix-identifier-name triples we can get."""
+    for prefix, df in iter_helper_helper(get_typedef_df, **kwargs):
+        for t in df.values:
+            if all(t):
+                yield cast(tuple[str, str, str, str], (prefix, *t))
+
+
+def _iter_relations(
+    **kwargs: Unpack[IterHelperHelperDict],
+) -> Iterable[tuple[str, str, str, str, str, str]]:
+    for prefix, df in iter_helper_helper(get_relations_df, **kwargs):
+        for t in df.values:
+            if all(t):
+                yield cast(tuple[str, str, str, str, str, str], (prefix, *t))
+
+
+def _iter_edges(**kwargs: Unpack[IterHelperHelperDict]) -> Iterable[tuple[str, str, str, str]]:
+    for prefix, df in iter_helper_helper(get_edges_df, **kwargs):
+        for row in df.values:
+            yield cast(tuple[str, str, str, str], (*row, prefix))
+
+
+def _iter_properties(**kwargs: Unpack[IterHelperHelperDict]) -> Iterable[tuple[str, str, str, str]]:
+    for prefix, df in iter_helper_helper(get_properties_df, **kwargs):
+        for t in df.values:
+            if all(t):
+                yield cast(tuple[str, str, str, str], (prefix, *t))
+
+
+def _iter_xrefs(
+    **kwargs: Unpack[IterHelperHelperDict],
+) -> Iterable[tuple[str, str, str, str, str]]:
+    warnings.warn(f"use {_iter_mappings.__name__} instead", DeprecationWarning, stacklevel=2)
+    it = iter_helper_helper(get_xrefs_df, **kwargs)
+    for prefix, df in it:
+        df.dropna(inplace=True)
+        for row in df.values:
+            if any(not element for element in row):
+                continue
+            yield cast(tuple[str, str, str, str, str], (prefix, *row, prefix))
+
+
+def _iter_mappings(
+    **kwargs: Unpack[IterHelperHelperDict],
+) -> Iterable[tuple[str, str, str, str, str]]:
+    f = partial(get_mappings_df, names=False, include_mapping_source_column=True)
+    # hack in a name to the partial function object since
+    # it's used for the tqdm description in iter_helper_helper
+    f.__name__ = "get_mappings_df"  # type:ignore
+    it = iter_helper_helper(f, **kwargs)
+    for _prefix, df in it:
+        yield from df.values