pyobo 0.11.2__py3-none-any.whl → 0.12.1__py3-none-any.whl

pyobo/utils/cache.py

@@ -1,12 +1,10 @@
 """Utilities for caching files."""
 
-import gzip
 import json
 import logging
-import os
 from collections.abc import Iterable, Mapping
 from pathlib import Path
-from typing import Generic, TypeVar, Union
+from typing import Generic, TypeVar
 
 import networkx as nx
 from pystow.cache import Cached
@@ -15,18 +13,18 @@ from pystow.cache import CachedDataFrame as cached_df  # noqa:N813
 from pystow.cache import CachedJSON as cached_json  # noqa:N813
 from pystow.cache import CachedPickle as cached_pickle  # noqa:N813
 
-from .io import open_map_tsv, open_multimap_tsv, write_map_tsv, write_multimap_tsv
+from .io import open_map_tsv, open_multimap_tsv, safe_open, write_map_tsv, write_multimap_tsv
 
 __all__ = [
-    # from pystow
-    "cached_json",
     "cached_collection",
     "cached_df",
-    "cached_pickle",
     # implemented here
     "cached_graph",
+    # from pystow
+    "cached_json",
     "cached_mapping",
     "cached_multidict",
+    "cached_pickle",
 ]
 
 logger = logging.getLogger(__name__)
@@ -39,14 +37,15 @@ class _CachedMapping(Cached[X], Generic[X]):
 
     def __init__(
         self,
-        path: Union[str, Path, os.PathLike],
+        path: str | Path,
         header: Iterable[str],
         *,
         use_tqdm: bool = False,
         force: bool = False,
+        cache: bool = True,
     ):
         """Initialize the mapping cache."""
-        super().__init__(path=path, force=force)
+        super().__init__(path=path, cache=cache, force=force)
         self.header = header
         self.use_tqdm = use_tqdm
 
@@ -65,17 +64,19 @@ class CachedMapping(_CachedMapping[Mapping[str, str]]):
 
 cached_mapping = CachedMapping
 
+NODE_LINK_STYLE = "links"  # TODO update to "edges"
+
 
-def get_gzipped_graph(path: Union[str, Path]) -> nx.MultiDiGraph:
+def get_gzipped_graph(path: str | Path) -> nx.MultiDiGraph:
     """Read a graph that's gzipped nodelink."""
-    with gzip.open(path, "rt") as file:
-        return nx.node_link_graph(json.load(file))
+    with safe_open(path, read=True) as file:
+        return nx.node_link_graph(json.load(file), edges=NODE_LINK_STYLE)
 
 
-def write_gzipped_graph(graph: nx.MultiDiGraph, path: Union[str, Path]) -> None:
+def write_gzipped_graph(graph: nx.MultiDiGraph, path: str | Path) -> None:
     """Write a graph as gzipped nodelink."""
-    with gzip.open(path, "wt") as file:
-        json.dump(nx.node_link_data(graph), file)
+    with safe_open(path, read=False) as file:
+        json.dump(nx.node_link_data(graph, edges=NODE_LINK_STYLE), file)
 
 
 class CachedGraph(Cached[nx.MultiDiGraph]):

pyobo/utils/io.py

@@ -1,33 +1,31 @@
 """I/O utilities."""
 
 import collections.abc
+import contextlib
 import csv
 import gzip
 import logging
-import time
 from collections import defaultdict
-from collections.abc import Iterable, Mapping
+from collections.abc import Generator, Iterable, Mapping
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Optional, TypeVar, Union
-from xml.etree.ElementTree import Element
+from typing import Literal, TextIO, TypeVar
 
 import pandas as pd
-from lxml import etree
 from tqdm.auto import tqdm
 
 __all__ = [
-    "open_map_tsv",
-    "open_multimap_tsv",
+    "get_reader",
     "multidict",
     "multisetdict",
+    "open_map_tsv",
+    "open_multimap_tsv",
+    "open_reader",
+    "safe_open",
+    "safe_open_writer",
+    "write_iterable_tsv",
     "write_map_tsv",
     "write_multimap_tsv",
-    "write_iterable_tsv",
-    "parse_xml_gz",
-    "get_writer",
-    "open_reader",
-    "get_reader",
 ]
 
 logger = logging.getLogger(__name__)
@@ -37,10 +35,10 @@ Y = TypeVar("Y")
 
 
 @contextmanager
-def open_reader(path: Union[str, Path], sep: str = "\t"):
+def open_reader(path: str | Path, sep: str = "\t"):
     """Open a file and get a reader for it."""
     path = Path(path)
-    with gzip.open(path, "rt") if path.suffix == ".gz" else open(path) as file:
+    with safe_open(path, read=True) as file:
         yield get_reader(file, sep=sep)
 
 
@@ -49,16 +47,11 @@ def get_reader(x, sep: str = "\t"):
     return csv.reader(x, delimiter=sep, quoting=csv.QUOTE_MINIMAL)
 
 
-def get_writer(x, sep: str = "\t"):
-    """Get a :func:`csv.writer` with PyOBO default settings."""
-    return csv.writer(x, delimiter=sep, quoting=csv.QUOTE_MINIMAL)
-
-
 def open_map_tsv(
-    path: Union[str, Path], *, use_tqdm: bool = False, has_header: bool = True
+    path: str | Path, *, use_tqdm: bool = False, has_header: bool = True
 ) -> Mapping[str, str]:
     """Load a mapping TSV file into a dictionary."""
-    with open(path) as file:
+    with safe_open(path, read=True) as file:
         if has_header:
             next(file)  # throw away header
         if use_tqdm:
@@ -73,7 +66,7 @@ def open_map_tsv(
 
 
 def open_multimap_tsv(
-    path: Union[str, Path],
+    path: str | Path,
     *,
     use_tqdm: bool = False,
     has_header: bool = True,
@@ -83,14 +76,17 @@ def open_multimap_tsv(
 
 
 def _help_multimap_tsv(
-    path: Union[str, Path],
+    path: str | Path,
     *,
     use_tqdm: bool = False,
     has_header: bool = True,
 ) -> Iterable[tuple[str, str]]:
-    with open(path) as file:
+    with safe_open(path, read=True) as file:
         if has_header:
-            next(file)  # throw away header
+            try:
+                next(file)  # throw away header
+            except gzip.BadGzipFile as e:
+                raise ValueError(f"could not open file {path}") from e
         if use_tqdm:
             file = tqdm(file, desc=f"loading TSV from {path}")
         yield from get_reader(file)
@@ -115,9 +111,9 @@ def multisetdict(pairs: Iterable[tuple[X, Y]]) -> dict[X, set[Y]]:
 
 def write_map_tsv(
     *,
-    path: Union[str, Path],
-    header: Optional[Iterable[str]] = None,
-    rv: Union[Iterable[tuple[str, str]], Mapping[str, str]],
+    path: str | Path,
+    header: Iterable[str] | None = None,
+    rv: Iterable[tuple[str, str]] | Mapping[str, str],
     sep: str = "\t",
 ) -> None:
     """Write a mapping dictionary to a TSV file."""
@@ -129,7 +125,7 @@ def write_map_tsv(
 
 def write_multimap_tsv(
     *,
-    path: Union[str, Path],
+    path: str | Path,
     header: Iterable[str],
     rv: Mapping[str, list[str]],
     sep: str = "\t",
@@ -141,26 +137,40 @@ def write_multimap_tsv(
 
 def write_iterable_tsv(
     *,
-    path: Union[str, Path],
-    header: Optional[Iterable[str]] = None,
+    path: str | Path,
+    header: Iterable[str] | None = None,
     it: Iterable[tuple[str, ...]],
     sep: str = "\t",
 ) -> None:
     """Write a mapping dictionary to a TSV file."""
     it = (row for row in it if all(cell is not None for cell in row))
     it = sorted(it)
-    with open(path, "w") as file:
-        writer = get_writer(file, sep=sep)
+    with safe_open_writer(path, delimiter=sep) as writer:
         if header is not None:
             writer.writerow(header)
         writer.writerows(it)
 
 
-def parse_xml_gz(path: Union[str, Path]) -> Element:
-    """Parse an XML file from a path to a GZIP file."""
-    path = Path(path).resolve()
-    t = time.time()
-    logger.info("parsing xml from %s", path)
-    tree = etree.parse(path.as_posix())  # type:ignore
-    logger.info("parsed xml in %.2f seconds", time.time() - t)
-    return tree.getroot()
+@contextlib.contextmanager
+def safe_open(
+    path: str | Path, read: bool, encoding: str | None = None
+) -> Generator[TextIO, None, None]:
+    """Safely open a file for reading or writing text."""
+    path = Path(path).expanduser().resolve()
+    mode: Literal["rt", "wt"] = "rt" if read else "wt"
+    if path.suffix.endswith(".gz"):
+        with gzip.open(path, mode=mode, encoding=encoding) as file:
+            yield file
+    else:
+        with open(path, mode=mode) as file:
+            yield file
+
+
+@contextlib.contextmanager
+def safe_open_writer(f: str | Path | TextIO, *, delimiter: str = "\t"):  # type:ignore
+    """Open a CSV writer, wrapping :func:`csv.writer`."""
+    if isinstance(f, str | Path):
+        with safe_open(f, read=False) as file:
+            yield csv.writer(file, delimiter=delimiter)
+    else:
+        yield csv.writer(f, delimiter=delimiter)

pyobo/utils/iter.py

@@ -8,8 +8,8 @@ from typing import TypeVar
 from more_itertools import peekable
 
 __all__ = [
-    "iterate_together",
     "iterate_gzips_together",
+    "iterate_together",
 ]
 
 X = TypeVar("X")
@@ -20,9 +20,9 @@ Y = TypeVar("Y")
 def iterate_gzips_together(a_path, b_path) -> Iterable[tuple[str, str, list[str]]]:
     """Iterate over two gzipped files together."""
     with gzip.open(a_path, mode="rt", errors="ignore") as a, gzip.open(b_path, mode="rt") as b:
-        a = csv.reader(a, delimiter="\t", quoting=csv.QUOTE_MINIMAL)
-        b = csv.reader(b, delimiter="\t", quoting=csv.QUOTE_MINIMAL)
-        yield from iterate_together(a, b)
+        a_reader = csv.reader(a, delimiter="\t", quoting=csv.QUOTE_MINIMAL)
+        b_reader = csv.reader(b, delimiter="\t", quoting=csv.QUOTE_MINIMAL)
+        yield from iterate_together(a_reader, b_reader)  # type:ignore
 
 
 def iterate_together(
@@ -38,7 +38,7 @@ def iterate_together(
     - Each key in the index is present within both files
     """
     b_peekable = peekable(b)
-    b_index, _ = b_peekable.peek()
+    b_index: X | type[_Done] = b_peekable.peek()[0]
 
     for a_index, a_value in a:
         zs = []

pyobo/utils/misc.py

@@ -1,79 +1,67 @@
 """Miscellaneous utilities."""
 
-import gzip
 import logging
-import os
 from datetime import datetime
-from subprocess import check_output
-from typing import Optional
 
 __all__ = [
-    "obo_to_obograph",
-    "obo_to_owl",
     "cleanup_version",
 ]
 
-
 logger = logging.getLogger(__name__)
 
-
-def obo_to_obograph(obo_path, obograph_path) -> None:
-    """Convert an OBO file to OBO Graph file with pronto."""
-    import pronto
-
-    ontology = pronto.Ontology(obo_path)
-    with gzip.open(obograph_path, "wb") as file:
-        ontology.dump(file, format="json")
-
-
-def obo_to_owl(obo_path, owl_path, owl_format: str = "ofn"):
-    """Convert an OBO file to an OWL file with ROBOT."""
-    args = ["robot", "convert", "-i", obo_path, "-o", owl_path, "--format", owl_format]
-    ret = check_output(  # noqa:S603
-        args,
-        cwd=os.path.dirname(__file__),
-    )
-    return ret.decode()
-
-
 BIZARRE_LOGGED = set()
 
+#: Rewrites for mostly static resources that have weird quirks
+VERSION_REWRITES = {
+    "$Date: 2009/11/15 10:54:12 $": "2009-11-15",  # for owl
+    "http://www.w3.org/2006/time#2016": "2016",  # for time
+}
+STATIC_VERSION_REWRITES = {"orth": "2"}
+VERSION_PREFIXES = [
+    "http://www.orpha.net/version",
+    "https://www.orphadata.com/data/ontologies/ordo/last_version/ORDO_en_",
+    "http://humanbehaviourchange.org/ontology/bcio.owl/",
+    "http://purl.org/pav/",
+    "http://identifiers.org/combine.specifications/teddy.rel-",
+    "https://purl.dataone.org/odo/MOSAIC/",
+    "http://purl.dataone.org/odo/SASAP/",  # like in http://purl.dataone.org/odo/SASAP/0.3.1
+    "http://purl.dataone.org/odo/SENSO/",  # like in http://purl.dataone.org/odo/SENSO/0.1.0
+    "https://purl.dataone.org/odo/ADCAD/",
+]
+VERSION_PREFIX_SPLITS = [
+    "http://www.ebi.ac.uk/efo/releases/v",
+    "http://www.ebi.ac.uk/swo/swo.owl/",
+    "http://semanticscience.org/ontology/sio/v",
+    "http://ontology.neuinfo.org/NIF/ttl/nif/version/",
+]
+
 
-def cleanup_version(data_version: str, prefix: str) -> Optional[str]:
+def cleanup_version(data_version: str, prefix: str) -> str:
     """Clean the version information."""
-    if data_version.endswith(".owl"):
-        data_version = data_version[: -len(".owl")]
+    if data_version in VERSION_REWRITES:
+        return VERSION_REWRITES[data_version]
+
+    data_version = data_version.removesuffix(".owl")
     if data_version.endswith(prefix):
         data_version = data_version[: -len(prefix)]
-    if data_version.startswith("releases/"):
-        data_version = data_version[len("releases/") :]
-    if prefix == "orth":
-        # TODO add bioversions for this
-        return "2"
+    data_version = data_version.removesuffix("/")
+
+    data_version = data_version.removeprefix("releases/")
+    data_version = data_version.removeprefix("release/")
 
-    version_prefixes = [
-        "http://www.orpha.net/version",
-        "https://www.orphadata.com/data/ontologies/ordo/last_version/ORDO_en_",
-        "http://humanbehaviourchange.org/ontology/bcio.owl/",
-        "http://purl.org/pav/",
-        "http://identifiers.org/combine.specifications/teddy.rel-",
-    ]
-    for version_prefix in version_prefixes:
+    for version_prefix in VERSION_PREFIXES:
         if data_version.startswith(version_prefix):
-            return data_version[len(version_prefix) :]
+            return data_version.removeprefix(version_prefix)
 
-    version_prefixes_split = [
-        "http://www.ebi.ac.uk/efo/releases/v",
-        "http://www.ebi.ac.uk/swo/swo.owl/",
-        "http://semanticscience.org/ontology/sio/v",
-        "http://ontology.neuinfo.org/NIF/ttl/nif/version/",
-    ]
-    for version_prefix_split in version_prefixes_split:
+    for version_prefix_split in VERSION_PREFIX_SPLITS:
         if data_version.startswith(version_prefix_split):
-            return data_version[len(version_prefix_split) :].split("/")[0]
+            return data_version.removeprefix(version_prefix_split).split("/")[0]
 
+    # use a heuristic to determine if the version is one of
+    # consecutive, major.minor, or semantic versioning (i.e., major.minor.patch)
     if data_version.replace(".", "").isnumeric():
-        return data_version  # consecutive, major.minor, or semantic versioning
+        return data_version
+
     for v in reversed(data_version.split("/")):
         v = v.strip()
         try:

pyobo/utils/path.py

@@ -1,60 +1,42 @@
 """Utilities for building paths."""
 
+import enum
 import logging
 from pathlib import Path
-from typing import Any, Callable, Literal, Optional, Union
+from typing import Any, Literal
 
 import pandas as pd
-import requests_ftp
-from pystow.utils import download, name_from_url, read_tarfile_csv
+from curies import Reference
+from pystow import VersionHint
 
-from .misc import cleanup_version
-from ..constants import RAW_MODULE
+from ..constants import CACHE_SUBDIRECTORY_NAME, RAW_MODULE, RELATION_SUBDIRECTORY_NAME
 
 __all__ = [
-    "prefix_directory_join",
-    "prefix_directory_join",
-    "prefix_cache_join",
-    "get_prefix_obo_path",
-    "ensure_path",
+    "CacheArtifact",
     "ensure_df",
-    "ensure_tar_df",
+    "ensure_path",
+    "get_cache_path",
+    "get_relation_cache_path",
+    "prefix_directory_join",
 ]
 
 logger = logging.getLogger(__name__)
 
-VersionHint = Union[None, str, Callable[[], Optional[str]]]
-
-requests_ftp.monkeypatch_session()
-
 
 def prefix_directory_join(
     prefix: str,
     *parts: str,
-    name: Optional[str] = None,
+    name: str | None = None,
     version: VersionHint = None,
     ensure_exists: bool = True,
 ) -> Path:
     """Join in the prefix directory."""
-    if version is None:
-        return RAW_MODULE.join(prefix, *parts, name=name, ensure_exists=ensure_exists)
-    if callable(version):
-        logger.info("[%s] looking up version", prefix)
-        version = version()
-        logger.info("[%s] got version %s", prefix, version)
-    elif not isinstance(version, str):
-        raise TypeError(f"Invalid type: {version} ({type(version)})")
-    if version is None:
-        raise AssertionError
-    version = cleanup_version(version, prefix=prefix)
-    if version is not None and "/" in version:
-        raise ValueError(f"[{prefix}] Can not have slash in version: {version}")
-    return RAW_MODULE.join(prefix, version, *parts, name=name, ensure_exists=ensure_exists)
-
-
-def get_prefix_obo_path(prefix: str, version: VersionHint = None, ext: str = "obo") -> Path:
-    """Get the canonical path to the OBO file."""
-    return prefix_directory_join(prefix, name=f"{prefix}.{ext}", version=version)
+    return RAW_MODULE.module(prefix).join(
+        *parts,
+        name=name,
+        ensure_exists=ensure_exists,
+        version=version,
+    )
 
 
 def ensure_path(
@@ -62,36 +44,29 @@ def ensure_path(
     *parts: str,
     url: str,
     version: VersionHint = None,
-    name: Optional[str] = None,
+    name: str | None = None,
     force: bool = False,
-    error_on_missing: bool = False,
     backend: Literal["requests", "urllib"] = "urllib",
     verify: bool = True,
-) -> str:
+    **download_kwargs: Any,
+) -> Path:
     """Download a file if it doesn't exist."""
-    if name is None:
-        name = name_from_url(url)
-
-    path = prefix_directory_join(prefix, *parts, name=name, version=version)
-
-    if not path.exists() and error_on_missing:
-        raise FileNotFoundError
-
-    kwargs: dict[str, Any]
     if verify:
-        kwargs = {"backend": backend}
+        download_kwargs = {"backend": backend}
     else:
         if backend != "requests":
             logger.warning("using requests since verify=False")
-        kwargs = {"backend": "requests", "verify": False}
+        download_kwargs = {"backend": "requests", "verify": False}
 
-    download(
+    path = RAW_MODULE.module(prefix).ensure(
+        *parts,
         url=url,
-        path=path,
+        name=name,
         force=force,
-        **kwargs,
+        version=version,
+        download_kwargs=download_kwargs,
     )
-    return path.as_posix()
+    return path
 
 
 def ensure_df(
@@ -99,7 +74,7 @@ def ensure_df(
     *parts: str,
     url: str,
     version: VersionHint = None,
-    name: Optional[str] = None,
+    name: str | None = None,
     force: bool = False,
     sep: str = "\t",
     dtype=str,
@@ -121,21 +96,49 @@ def ensure_df(
     return pd.read_csv(_path, sep=sep, dtype=dtype, **kwargs)
 
 
-def ensure_tar_df(
-    prefix: str,
-    *parts: str,
-    url: str,
-    inner_path: str,
-    version: VersionHint = None,
-    path: Optional[str] = None,
-    force: bool = False,
-    **kwargs,
-) -> pd.DataFrame:
-    """Download a tar file and open as a dataframe."""
-    path = ensure_path(prefix, *parts, url=url, version=version, name=path, force=force)
-    return read_tarfile_csv(path, inner_path=inner_path, **kwargs)
+class CacheArtifact(enum.Enum):
+    """An enumeration for."""
+
+    names = "names.tsv.gz"
+    definitions = "definitions.tsv.gz"
+    species = "species.tsv.gz"
+    mappings = "mappings.tsv.gz"
+    relations = "relations.tsv.gz"
+    alts = "alt_ids.tsv.gz"
+    typedefs = "typedefs.tsv.gz"
+    literal_mappings = "literal_mappings.tsv.gz"
+    references = "references.tsv.gz"
+    obsoletes = "obsolete.tsv.gz"
+
+    literal_properties = "literal_properties.tsv.gz"
+    object_properties = "object_properties.tsv.gz"
 
+    nodes = "nodes.tsv.gz"
+    edges = "edges.tsv.gz"
 
-def prefix_cache_join(prefix: str, *parts, name: Optional[str], version: VersionHint) -> Path:
-    """Ensure the prefix cache is available."""
-    return prefix_directory_join(prefix, "cache", *parts, name=name, version=version)
+    prefixes = "prefixes.json"
+    metadata = "metadata.json"
+
+
+def get_cache_path(
+    ontology: str,
+    name: CacheArtifact,
+    *,
+    version: str | None = None,
+) -> Path:
+    """Get a cache path."""
+    return prefix_directory_join(
+        ontology, CACHE_SUBDIRECTORY_NAME, name=name.value, version=version
+    )
+
+
+def get_relation_cache_path(
+    ontology: str,
+    reference: Reference,
+    *,
+    version: str | None = None,
+) -> Path:
+    """Get a relation cache path."""
+    return prefix_directory_join(
+        ontology, RELATION_SUBDIRECTORY_NAME, name=f"{reference.curie}.tsv", version=version
+    )

pyobo/version.py

@@ -8,11 +8,11 @@ from subprocess import CalledProcessError, check_output
 
 __all__ = [
     "VERSION",
-    "get_version",
     "get_git_hash",
+    "get_version",
 ]
 
-VERSION = "0.11.2"
+VERSION = "0.12.1"
 
 
 def get_git_hash() -> str:
@@ -30,7 +30,7 @@ def get_git_hash() -> str:
             return ret.strip().decode("utf-8")[:8]
 
 
-def get_version(with_git_hash: bool = False):
+def get_version(with_git_hash: bool = False) -> str:
     """Get the PyOBO version string, including a git hash."""
     return f"{VERSION}-{get_git_hash()}" if with_git_hash else VERSION