biocypher 0.5.19__py3-none-any.whl → 0.5.21__py3-none-any.whl

biocypher/_get.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python
+
+#
+# Copyright 2021, Heidelberg University Clinic
+#
+# File author(s): Sebastian Lobentanzer
+#                 ...
+#
+# Distributed under MIT licence, see the file `LICENSE`.
+#
+"""
+BioCypher get module. Used to download and cache data from external sources.
+"""
+
+from __future__ import annotations
+
+from ._logger import logger
+
+logger.debug(f"Loading module {__name__}.")
+
+from datetime import datetime, timedelta
+from tempfile import TemporaryDirectory
+import os
+import json
+import ftplib
+
+import pooch
+
+from ._misc import to_list
+
+
+class Resource:
+    def __init__(
+        self,
+        name: str,
+        url_s: str | list[str],
+        lifetime: int = 0,
+        is_dir: bool = False,
+    ):
+        """
+        A resource is a file that can be downloaded from a URL and cached
+        locally. This class implements checks of the minimum requirements for
+        a resource, to be implemented by a biocypher adapter.
+
+        Args:
+            name (str): The name of the resource.
+
+            url_s (str | list[str]): The URL or URLs of the resource.
+
+            lifetime (int): The lifetime of the resource in days. If 0, the
+                resource is considered to be permanent.
+        """
+        self.name = name
+        self.url_s = url_s
+        self.lifetime = lifetime
+        self.is_dir = is_dir
+
+
+class Downloader:
+    def __init__(self, cache_dir: str):
+        """
+        A downloader is a collection of resources that can be downloaded
+        and cached locally. It manages the lifetime of downloaded resources by
+        keeping a JSON record of the download date of each resource.
+
+        Args:
+            cache_dir (str): The directory where the resources are cached. If
+                not given, a temporary directory is created.
+        """
+        self.cache_dir = cache_dir or TemporaryDirectory().name
+        self.cache_file = os.path.join(self.cache_dir, "cache.json")
+        self.cache_dict = self._load_cache_dict()
+
+    # download function that accepts a resource or a list of resources
+    def download(self, *resources: Resource):
+        """
+        Download one or multiple resources.
+
+        Args:
+            resources (Resource): The resource or resources to download.
+
+        Returns:
+            str or list: The path or paths to the downloaded resource(s).
+        """
+        paths = []
+        for resource in resources:
+            paths.append(self._download_or_cache(resource))
+
+        # flatten list if it is nested
+        if is_nested(paths):
+            paths = [path for sublist in paths for path in sublist]
+
+        return paths
+
+    def _download_or_cache(self, resource: Resource, cache: bool = True):
+        """
+        Download a resource if it is not cached or exceeded its lifetime.
+
+        Args:
+            resource (Resource): The resource to download.
+
+        Returns:
+            str or list: The path or paths to the downloaded resource(s).
+        """
+        # check if resource is cached
+        cache_record = self._get_cache_record(resource)
+
+        if cache_record:
+            # check if resource is expired (formatted in days)
+            dl = cache_record.get("date_downloaded")
+            lt = timedelta(days=resource.lifetime)
+            expired = dl + lt < datetime.now()
+        else:
+            expired = True
+
+        # download resource
+        if expired or not cache:
+            logger.info(f"Downloading resource {resource.name}.")
+
+            if resource.is_dir:
+                files = self._get_files(resource)
+                resource.url_s = [resource.url_s + "/" + file for file in files]
+                resource.is_dir = False
+                paths = self._download_or_cache(resource, cache)
+            elif isinstance(resource.url_s, list):
+                paths = []
+                for url in resource.url_s:
+                    fname = url[url.rfind("/") + 1 :]
+                    paths.append(
+                        self._retrieve(
+                            url=url,
+                            fname=fname,
+                            path=os.path.join(self.cache_dir, resource.name),
+                        )
+                    )
+            else:
+                fname = resource.url_s[resource.url_s.rfind("/") + 1 :]
+                paths = self._retrieve(
+                    url=resource.url_s,
+                    fname=fname,
+                    path=os.path.join(self.cache_dir, resource.name),
+                )
+
+            # sometimes a compressed file contains multiple files
+            # TODO ask for a list of files in the archive to be used from the
+            # adapter
+
+            # update cache record
+            self._update_cache_record(resource)
+
+            return paths
+
+    def _retrieve(
+        self,
+        url: str,
+        fname: str,
+        path: str,
+        known_hash: str = None,
+    ):
+        """
+        Retrieve a file from a URL using Pooch. Infer type of file from
+        extension and use appropriate processor.
+
+        Args:
+            url (str): The URL to retrieve the file from.
+
+            fname (str): The name of the file.
+
+            path (str): The path to the file.
+        """
+        if fname.endswith(".zip"):
+            return pooch.retrieve(
+                url=url,
+                known_hash=known_hash,
+                fname=fname,
+                path=path,
+                processor=pooch.Unzip(),
+                progressbar=True,
+            )
+
+        elif fname.endswith(".tar.gz"):
+            return pooch.retrieve(
+                url=url,
+                known_hash=known_hash,
+                fname=fname,
+                path=path,
+                processor=pooch.Untar(),
+                progressbar=True,
+            )
+
+        elif fname.endswith(".gz"):
+            return pooch.retrieve(
+                url=url,
+                known_hash=known_hash,
+                fname=fname,
+                path=path,
+                processor=pooch.Decompress(),
+                progressbar=True,
+            )
+
+        else:
+            return pooch.retrieve(
+                url=url,
+                known_hash=known_hash,
+                fname=fname,
+                path=path,
+                progressbar=True,
+            )
+
+    def _get_files(self, resource: Resource):
+        """
+        Get the files contained in a directory resource.
+
+        Args:
+            resource (Resource): The directory resource.
+
+        Returns:
+            list: The files contained in the directory.
+        """
+        if resource.url_s.startswith("ftp://"):
+            # remove protocol
+            url = resource.url_s[6:]
+            # get base url
+            url = url[: url.find("/")]
+            # get directory (remove initial slash as well)
+            dir = resource.url_s[7 + len(url) :]
+            # get files
+            ftp = ftplib.FTP(url)
+            ftp.login()
+            ftp.cwd(dir)
+            files = ftp.nlst()
+            ftp.quit()
+        else:
+            raise NotImplementedError(
+                "Only FTP directories are supported at the moment."
+            )
+
+        return files
+
+    def _load_cache_dict(self):
+        """
+        Load the cache dictionary from the cache file. Create an empty cache
+        file if it does not exist.
+        """
+        if not os.path.exists(self.cache_dir):
+            logger.info(f"Creating cache directory {self.cache_dir}.")
+            os.makedirs(self.cache_dir)
+
+        if not os.path.exists(self.cache_file):
+            logger.info(f"Creating cache file {self.cache_file}.")
+            with open(self.cache_file, "w") as f:
+                json.dump({}, f)
+
+        with open(self.cache_file, "r") as f:
+            logger.info(f"Loading cache file {self.cache_file}.")
+            return json.load(f)
+
+    def _get_cache_record(self, resource: Resource):
+        """
+        Get the cache record of a resource.
+
+        Args:
+            resource (Resource): The resource to get the cache record of.
+
+        Returns:
+            The cache record of the resource.
+        """
+        return self.cache_dict.get(resource.name, {})
+
+    def _update_cache_record(self, resource: Resource):
+        """
+        Update the cache record of a resource.
+
+        Args:
+            resource (Resource): The resource to update the cache record of.
+        """
+        cache_record = {}
+        cache_record["url"] = to_list(resource.url_s)
+        cache_record["date_downloaded"] = datetime.now()
+        cache_record["lifetime"] = resource.lifetime
+        self.cache_dict[resource.name] = cache_record
+        with open(self.cache_file, "w") as f:
+            json.dump(self.cache_dict, f, default=str)
+
+
+def is_nested(lst):
+    """
+    Check if a list is nested.
+
+    Args:
+        lst (list): The list to check.
+
+    Returns:
+        bool: True if the list is nested, False otherwise.
+    """
+    for item in lst:
+        if isinstance(item, list):
+            return True
+    return False

biocypher/_metadata.py

@@ -19,7 +19,7 @@ import importlib.metadata
 
 import toml
 
-_VERSION = "0.5.19"
+_VERSION = "0.5.21"
 
 
 def get_metadata():

biocypher/_ontology.py

@@ -269,7 +269,7 @@ class Ontology:
         """
 
         self._head_ontology_meta = head_ontology
-        self.extended_schema = ontology_mapping.extended_schema
+        self.mapping = ontology_mapping
         self._tail_ontology_meta = tail_ontologies
 
         self._tail_ontologies = None
@@ -403,7 +403,7 @@ class Ontology:
         if not self._nx_graph:
             self._nx_graph = self._head_ontology.get_nx_graph().copy()
 
-        for key, value in self.extended_schema.items():
+        for key, value in self.mapping.extended_schema.items():
             if not value.get("is_a"):
                 if self._nx_graph.has_node(value.get("synonym_for")):
                     continue
@@ -485,7 +485,7 @@ class Ontology:
         setting the synonym as the primary node label.
         """
 
-        for key, value in self.extended_schema.items():
+        for key, value in self.mapping.extended_schema.items():
             if key in self._nx_graph.nodes:
                 self._nx_graph.nodes[key].update(value)
 
@@ -541,9 +541,9 @@ class Ontology:
 
         if not full:
             # set of leaves and their intermediate parents up to the root
-            filter_nodes = set(self.extended_schema.keys())
+            filter_nodes = set(self.mapping.extended_schema.keys())
 
-            for node in self.extended_schema.keys():
+            for node in self.mapping.extended_schema.keys():
                 filter_nodes.update(self.get_ancestors(node).nodes)
 
             # filter graph
@@ -557,11 +557,13 @@ class Ontology:
             tree = _misc.create_tree_visualisation(G)
 
             # add synonym information
-            for node in self.extended_schema:
-                if self.extended_schema[node].get("synonym_for"):
+            for node in self.mapping.extended_schema:
+                if not isinstance(self.mapping.extended_schema[node], dict):
+                    continue
+                if self.mapping.extended_schema[node].get("synonym_for"):
                     tree.nodes[node].tag = (
                         f"{node} = "
-                        f"{self.extended_schema[node].get('synonym_for')}"
+                        f"{self.mapping.extended_schema[node].get('synonym_for')}"
                     )
 
             tree.show()
@@ -602,7 +604,7 @@ class Ontology:
             "node_id": self._get_current_id(),
             "node_label": "BioCypher",
             "properties": {
-                "schema": "self.extended_schema",
+                "schema": "self.ontology_mapping.extended_schema",
             },
         }
 

biocypher/_pandas.py

@@ -1,11 +1,10 @@
 import pandas as pd
 
-from ._create import BioCypherEdge, BioCypherNode
+from ._create import BioCypherEdge, BioCypherNode, BioCypherRelAsNode
 
 
 class Pandas:
-    def __init__(self, ontology, translator, deduplicator):
-        self.ontology = ontology
+    def __init__(self, translator, deduplicator):
         self.translator = translator
         self.deduplicator = deduplicator
 
@@ -18,22 +17,48 @@ class Pandas:
         """
         lists = {}
         for entity in entities:
-            if not isinstance(entity, BioCypherNode) and not isinstance(
-                entity, BioCypherEdge
+            if (
+                not isinstance(entity, BioCypherNode)
+                and not isinstance(entity, BioCypherEdge)
+                and not isinstance(entity, BioCypherRelAsNode)
             ):
                 raise TypeError(
-                    f"Expected a BioCypherNode or BioCypherEdge, got {type(entity)}."
+                    "Expected a BioCypherNode / BioCypherEdge / "
+                    f"BioCypherRelAsNode, got {type(entity)}."
                 )
 
             if isinstance(entity, BioCypherNode):
                 seen = self.deduplicator.node_seen(entity)
             elif isinstance(entity, BioCypherEdge):
                 seen = self.deduplicator.edge_seen(entity)
+            elif isinstance(entity, BioCypherRelAsNode):
+                seen = self.deduplicator.rel_as_node_seen(entity)
 
             if seen:
                 continue
 
-            _type = entity.get_label()
+            if isinstance(entity, BioCypherRelAsNode):
+                node = entity.get_node()
+                source_edge = entity.get_source_edge()
+                target_edge = entity.get_target_edge()
+
+                _type = node.get_type()
+                if not _type in lists:
+                    lists[_type] = []
+                lists[_type].append(node)
+
+                _source_type = source_edge.get_type()
+                if not _source_type in lists:
+                    lists[_source_type] = []
+                lists[_source_type].append(source_edge)
+
+                _target_type = target_edge.get_type()
+                if not _target_type in lists:
+                    lists[_target_type] = []
+                lists[_target_type].append(target_edge)
+                continue
+
+            _type = entity.get_type()
             if not _type in lists:
                 lists[_type] = []
             lists[_type].append(entity)

biocypher/_translate.py

@@ -23,7 +23,7 @@ from more_itertools import peekable
 
 from . import _misc
 from ._create import BioCypherEdge, BioCypherNode, BioCypherRelAsNode
-from ._mapping import OntologyMapping
+from ._ontology import Ontology
 
 __all__ = ["BiolinkAdapter", "Translator"]
 
@@ -41,9 +41,7 @@ class Translator:
     and cypher queries.
     """
 
-    def __init__(
-        self, ontology_mapping: "OntologyMapping", strict_mode: bool = False
-    ):
+    def __init__(self, ontology: "Ontology", strict_mode: bool = False):
         """
         Args:
             leaves:
@@ -57,7 +55,7 @@ class Translator:
                 carry source, licence, and version information.
         """
 
-        self.extended_schema = ontology_mapping.extended_schema
+        self.ontology = ontology
         self.strict_mode = strict_mode
 
         # record nodes without biolink type configured in schema_config.yaml
@@ -71,7 +69,7 @@ class Translator:
 
     def translate_nodes(
         self,
-        id_type_prop_tuples: Iterable,
+        node_tuples: Iterable,
     ) -> Generator[BioCypherNode, None, None]:
         """
         Translates input node representation to a representation that
@@ -79,16 +77,16 @@ class Translator:
         requires explicit statement of node type on pass.
 
         Args:
-            id_type_tuples (list of tuples): collection of tuples
+            node_tuples (list of tuples): collection of tuples
                 representing individual nodes by their unique id and a type
                 that is translated from the original database notation to
                 the corresponding BioCypher notation.
 
         """
 
-        self._log_begin_translate(id_type_prop_tuples, "nodes")
+        self._log_begin_translate(node_tuples, "nodes")
 
-        for _id, _type, _props in id_type_prop_tuples:
+        for _id, _type, _props in node_tuples:
             # check for strict mode requirements
             required_props = ["source", "licence", "version"]
 
@@ -132,8 +130,9 @@ class Translator:
         """
 
         return (
-            self.extended_schema[_bl_type]["preferred_id"]
-            if "preferred_id" in self.extended_schema.get(_bl_type, {})
+            self.ontology.mapping.extended_schema[_bl_type]["preferred_id"]
+            if "preferred_id"
+            in self.ontology.mapping.extended_schema.get(_bl_type, {})
             else "id"
         )
 
@@ -142,7 +141,9 @@ class Translator:
         Filters properties for those specified in schema_config if any.
         """
 
-        filter_props = self.extended_schema[bl_type].get("properties", {})
+        filter_props = self.ontology.mapping.extended_schema[bl_type].get(
+            "properties", {}
+        )
 
         # strict mode: add required properties (only if there is a whitelist)
         if self.strict_mode and filter_props:
@@ -150,7 +151,7 @@ class Translator:
                 {"source": "str", "licence": "str", "version": "str"},
             )
 
-        exclude_props = self.extended_schema[bl_type].get(
+        exclude_props = self.ontology.mapping.extended_schema[bl_type].get(
             "exclude_properties", []
         )
 
@@ -188,7 +189,7 @@ class Translator:
 
     def translate_edges(
         self,
-        id_src_tar_type_prop_tuples: Iterable,
+        edge_tuples: Iterable,
     ) -> Generator[Union[BioCypherEdge, BioCypherRelAsNode], None, None]:
         """
         Translates input edge representation to a representation that
@@ -197,7 +198,7 @@ class Translator:
 
         Args:
 
-            id_src_tar_type_prop_tuples (list of tuples):
+            edge_tuples (list of tuples):
 
                 collection of tuples representing source and target of
                 an interaction via their unique ids as well as the type
@@ -206,18 +207,18 @@ class Translator:
                 Can optionally possess its own ID.
         """
 
-        self._log_begin_translate(id_src_tar_type_prop_tuples, "edges")
+        self._log_begin_translate(edge_tuples, "edges")
 
         # legacy: deal with 4-tuples (no edge id)
         # TODO remove for performance reasons once safe
-        id_src_tar_type_prop_tuples = peekable(id_src_tar_type_prop_tuples)
-        if len(id_src_tar_type_prop_tuples.peek()) == 4:
-            id_src_tar_type_prop_tuples = [
+        edge_tuples = peekable(edge_tuples)
+        if len(edge_tuples.peek()) == 4:
+            edge_tuples = [
                 (None, src, tar, typ, props)
-                for src, tar, typ, props in id_src_tar_type_prop_tuples
+                for src, tar, typ, props in edge_tuples
             ]
 
-        for _id, _src, _tar, _type, _props in id_src_tar_type_prop_tuples:
+        for _id, _src, _tar, _type, _props in edge_tuples:
             # check for strict mode requirements
             if self.strict_mode:
                 if not "source" in _props:
@@ -239,7 +240,9 @@ class Translator:
                 # filter properties for those specified in schema_config if any
                 _filtered_props = self._filter_props(bl_type, _props)
 
-                rep = self.extended_schema[bl_type]["represented_as"]
+                rep = self.ontology.mapping.extended_schema[bl_type][
+                    "represented_as"
+                ]
 
                 if rep == "node":
                     if _id:
@@ -295,9 +298,9 @@ class Translator:
                     yield BioCypherRelAsNode(n, e_s, e_t)
 
                 else:
-                    edge_label = self.extended_schema[bl_type].get(
-                        "label_as_edge"
-                    )
+                    edge_label = self.ontology.mapping.extended_schema[
+                        bl_type
+                    ].get("label_as_edge")
 
                     if edge_label is None:
                         edge_label = bl_type
@@ -356,7 +359,7 @@ class Translator:
 
         self._ontology_mapping = {}
 
-        for key, value in self.extended_schema.items():
+        for key, value in self.ontology.mapping.extended_schema.items():
             labels = value.get("input_label") or value.get("label_in_input")
 
             if isinstance(labels, str):