siibra 1.0a1__1-py3-none-any.whl

siibra/retrieval/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2018-2024
+# Institute of Neuroscience and Medicine (INM-1), Forschungszentrum Jülich GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""File digestion and caching."""
+
+from .repositories import (
+    GitlabConnector,
+    OwncloudConnector,
+    EbrainsHdgConnector,
+    EbrainsPublicDatasetConnector,
+    LocalFileRepository,
+    ZipfileConnector
+)
+from .requests import HttpRequest, ZipfileRequest, EbrainsRequest, SiibraHttpRequestError
+from .cache import CACHE
+from .exceptions import NoSiibraConfigMirrorsAvailableException, TagNotFoundException

siibra/retrieval/cache.py

@@ -0,0 +1,233 @@
+# Copyright 2018-2024
+# Institute of Neuroscience and Medicine (INM-1), Forschungszentrum Jülich GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Maintaining and handling caching files on disk."""
+
+import hashlib
+import os
+from appdirs import user_cache_dir
+import tempfile
+from functools import wraps
+from enum import Enum
+from typing import Callable, List, NamedTuple, Union
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from filelock import FileLock as Lock
+
+from ..commons import logger, SIIBRA_CACHEDIR, SKIP_CACHEINIT_MAINTENANCE, siibra_tqdm
+from ..exceptions import WarmupRegException
+
+
+def assert_folder(folder):
+    # make sure the folder exists and is writable, then return it.
+    # If it cannot be written, create and return
+    # a temporary folder.
+    try:
+        if not os.path.isdir(folder):
+            os.makedirs(folder)
+        if not os.access(folder, os.W_OK):
+            raise OSError
+        return folder
+    except OSError:
+        # cannot write to requested directory, create a temporary one.
+        tmpdir = tempfile.mkdtemp(prefix="siibra-cache-")
+        logger.warning(
+            f"Siibra created a temporary cache directory at {tmpdir}, as "
+            f"the requested folder ({folder}) was not usable. "
+            "Please consider to set the SIIBRA_CACHEDIR environment variable "
+            "to a suitable directory.")
+        return tmpdir
+
+
+class Cache:
+
+    _instance = None
+    folder = user_cache_dir(".".join(__name__.split(".")[:-1]), "")
+    SIZE_GIB = 2  # maintenance will delete old files to stay below this limit
+
+    def __init__(self):
+        raise RuntimeError(
+            "Call instance() to access "
+            f"{self.__class__.__name__}")
+
+    @classmethod
+    def instance(cls):
+        """
+        Return an instance of the siibra cache. Create folder if needed.
+        """
+        if cls._instance is None:
+            if SIIBRA_CACHEDIR:
+                cls.folder = SIIBRA_CACHEDIR
+            cls.folder = assert_folder(cls.folder)
+            cls._instance = cls.__new__(cls)
+            if SKIP_CACHEINIT_MAINTENANCE:
+                logger.debug("Will not run maintenance on cache as SKIP_CACHE_MAINTENANCE is set to True.")
+            else:
+                cls._instance.run_maintenance()
+        return cls._instance
+
+    def clear(self):
+        import shutil
+
+        logger.info(f"Clearing siibra cache at {self.folder}")
+        shutil.rmtree(self.folder)
+        self.folder = assert_folder(self.folder)
+
+    def run_maintenance(self):
+        """ Shrinks the cache by deleting oldest files first until the total size
+        is below cache size (Cache.SIZE) given in GiB."""
+        # build sorted list of cache files and their os attributes
+        files = [os.path.join(self.folder, fname) for fname in os.listdir(self.folder)]
+        sfiles = sorted([(fn, os.stat(fn)) for fn in files], key=lambda t: t[1].st_atime)
+
+        # determine the first n files that need to be deleted to reach the accepted cache size
+        size_gib = sum(t[1].st_size for t in sfiles) / 1024**3
+        targetsize = size_gib
+        index = 0
+        for index, (fn, st) in enumerate(sfiles):
+            if targetsize <= self.SIZE_GIB:
+                break
+            targetsize -= st.st_size / 1024**3
+
+        if index > 0:
+            logger.debug(f"Removing the {index + 1} oldest files to keep cache size below {targetsize:.2f} GiB.")
+            for fn, st in sfiles[:index + 1]:
+                if os.path.isdir(fn):
+                    import shutil
+                    size = sum(os.path.getsize(f) for f in os.listdir(fn) if os.path.isfile(f))
+                    shutil.rmtree(fn)
+                else:
+                    size = st.st_size
+                    os.remove(fn)
+                size_gib -= size / 1024**3
+
+    @property
+    def size(self):
+        """ Return size of the cache in GiB. """
+        return sum(os.path.getsize(fn) for fn in self) / 1024**3
+
+    def __iter__(self):
+        """ Iterate all element names in the cache directory. """
+        return (os.path.join(self.folder, f) for f in os.listdir(self.folder))
+
+    def build_filename(self, str_rep: str, suffix=None) -> str:
+        """
+        Generate a filename in the cache.
+
+        Parameters
+        ----------
+        str_rep: str
+            Unique string representation of the item. Will be used to compute a hash.
+        suffix: str. Default: None
+            Optional file suffix, in order to allow filetype recognition by the name.
+
+        Returns
+        -------
+            str
+        """
+        hashfile = os.path.join(
+            self.folder, str(hashlib.sha256(str_rep.encode("ascii")).hexdigest())
+        )
+        if suffix is None:
+            return hashfile
+        else:
+            if suffix.startswith("."):
+                return hashfile + suffix
+            else:
+                return hashfile + "." + suffix
+
+
+CACHE = Cache.instance()
+
+
+class WarmupLevel(int, Enum):
+    TEST = -1000
+    INSTANCE = 1
+    DATA = 5
+
+
+class WarmupParam(NamedTuple):
+    level: Union[int, WarmupLevel]
+    fn: Callable
+    is_factory: bool = False
+
+
+class Warmup:
+
+    _warmup_fns: List[WarmupParam] = []
+
+    @staticmethod
+    def fn_eql(wrapped_fn, original_fn):
+        return wrapped_fn is original_fn or wrapped_fn.__wrapped__ is original_fn
+
+    @classmethod
+    def is_registered(cls, fn):
+        return len([warmup_fn.fn
+                    for warmup_fn in cls._warmup_fns
+                    if cls.fn_eql(warmup_fn.fn, fn)]) > 0
+
+    @classmethod
+    def register_warmup_fn(cls, warmup_level: WarmupLevel = WarmupLevel.INSTANCE, *, is_factory=False):
+        def outer(fn):
+            if cls.is_registered(fn):
+                raise WarmupRegException
+
+            @wraps(fn)
+            def inner(*args, **kwargs):
+                return fn(*args, **kwargs)
+
+            cls._warmup_fns.append(WarmupParam(warmup_level, inner, is_factory))
+            return inner
+        return outer
+
+    @classmethod
+    def deregister_warmup_fn(cls, original_fn):
+        cls._warmup_fns = [
+            warmup_fn for warmup_fn in cls._warmup_fns
+            if not cls.fn_eql(warmup_fn.fn, original_fn)
+        ]
+
+    @classmethod
+    def warmup(cls, warmup_level: WarmupLevel = WarmupLevel.INSTANCE, *, max_workers=4):
+        all_fns = [warmup for warmup in cls._warmup_fns if warmup.level <= warmup_level]
+
+        def call_fn(fn: WarmupParam):
+            return_val = fn.fn()
+            if not fn.is_factory:
+                return
+            for f in return_val:
+                f()
+
+        with Lock(CACHE.build_filename("lockfile", ".warmup")):
+            with ThreadPoolExecutor(max_workers=max_workers) as ex:
+                for _ in siibra_tqdm(
+                    ex.map(
+                        call_fn,
+                        all_fns
+                    ),
+                    desc="Warming cache",
+                    total=len(all_fns),
+                ):
+                    ...
+
+
+try:
+    from joblib import Memory
+    jobmemory_path = Path(CACHE.folder) / "joblib"
+    jobmemory_path.mkdir(parents=True, exist_ok=True)
+    jobmemory = Memory(jobmemory_path, verbose=0)
+    cache_user_fn = jobmemory.cache
+except ImportError:
+    from functools import lru_cache
+    cache_user_fn = lru_cache

siibra/retrieval/datasets.py

@@ -0,0 +1,389 @@
+# Copyright 2018-2024
+# Institute of Neuroscience and Medicine (INM-1), Forschungszentrum Jülich GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Metadata connection to EBRAINS datasets."""
+
+from .requests import MultiSourcedRequest, GitlabProxy, GitlabProxyEnum
+
+import re
+from typing import Union, List
+from abc import ABC, abstractproperty
+from hashlib import md5
+
+try:
+    from typing import TypedDict
+except ImportError:
+    # support python 3.7
+    from typing_extensions import TypedDict
+
+
+class EbrainsDatasetUrl(TypedDict):
+    url: str
+
+
+EbrainsDatasetPerson = TypedDict("EbrainsDatasetPerson", {
+    "@id": str,
+    "schema.org/shortName": str,
+    "identifier": str,
+    "shortName": str,
+    "name": str,
+})
+
+EbrainsDatasetEmbargoStatus = TypedDict("EbrainsDatasetEmbargoStatus", {
+    "@id": str,
+    "name": str,
+    "identifier": List[str]
+})
+
+
+class EbrainsBaseDataset(ABC):
+    @abstractproperty
+    def id(self) -> str:
+        raise NotImplementedError
+
+    @abstractproperty
+    def name(self) -> str:
+        raise NotImplementedError
+
+    @abstractproperty
+    def urls(self) -> List[EbrainsDatasetUrl]:
+        raise NotImplementedError
+
+    @abstractproperty
+    def description(self) -> str:
+        raise NotImplementedError
+
+    @abstractproperty
+    def contributors(self) -> List[EbrainsDatasetPerson]:
+        raise NotImplementedError
+
+    @abstractproperty
+    def ebrains_page(self) -> str:
+        raise NotImplementedError
+
+    @abstractproperty
+    def custodians(self) -> List[EbrainsDatasetPerson]:
+        raise NotImplementedError
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def __eq__(self, o: object) -> bool:
+        return hasattr(o, "id") and self.id == o.id
+
+    def match(self, spec: Union[str, 'EbrainsBaseDataset']) -> bool:
+        """
+        Checks if the given specification describes this dataset.
+
+        Parameters
+        ----------
+        spec (str, EbrainsBaseDataset)
+            specification to be matched.
+        Returns
+        -------
+        bool
+        """
+        if spec is self:
+            return True
+        if isinstance(spec, str):
+            return self.id == spec
+        raise RuntimeError(
+            f"Cannot match {spec.__class__}, must be either str or EbrainsBaseDataset"
+        )
+
+    @property
+    def LICENSE(self) -> str:
+        license_ = self._detail.get("license", [])
+        if len(license_) > 0:
+            return license_ if isinstance(license_, str) else '\n'.join(license_)
+        return None
+
+
+class EbrainsDataset(EbrainsBaseDataset):
+    """Ebrains dataset v1 connection"""
+
+    def __init__(self, id, name=None, embargo_status: List[EbrainsDatasetEmbargoStatus] = None, *, cached_data=None):
+        super().__init__()
+
+        self._id = id
+        self._name = name
+        self._cached_data = cached_data
+        self.embargo_status = embargo_status
+
+        if id is None:
+            raise TypeError("Dataset id is required")
+
+        match = re.search(r"([a-f0-9-]+)$", id)
+        if not match:
+            raise ValueError(
+                f"{self.__class__.__name__} initialized with invalid id: {self.id}"
+            )
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def _detail(self):
+        if not self._cached_data:
+            match = re.search(r"([a-f0-9-]+)$", self.id)
+            instance_id = match.group(1)
+            self._cached_data = MultiSourcedRequest(
+                requests=[
+                    GitlabProxy(
+                        GitlabProxyEnum.DATASET_V1,
+                        instance_id=instance_id,
+                    ),
+                ]
+            ).data
+        return self._cached_data
+
+    @property
+    def name(self) -> str:
+        if self._name is None:
+            self._name = self._detail.get("name")
+        return self._name
+
+    @property
+    def urls(self) -> List[EbrainsDatasetUrl]:
+        return [
+            {
+                "url": f if f.startswith("http") else f"https://doi.org/{f}",
+            }
+            for f in self._detail.get("kgReference", [])
+        ]
+
+    @property
+    def description(self) -> str:
+        return self._detail.get("description")
+
+    @property
+    def contributors(self) -> List[EbrainsDatasetPerson]:
+        return self._detail.get("contributors")
+
+    @property
+    def ebrains_page(self):
+        return f"https://search.kg.ebrains.eu/instances/{self.id}"
+
+    @property
+    def custodians(self) -> EbrainsDatasetPerson:
+        return self._detail.get("custodians")
+
+
+class EbrainsV3DatasetVersion(EbrainsBaseDataset):
+    @staticmethod
+    def _parse_person(d: dict) -> EbrainsDatasetPerson:
+        assert "https://openminds.ebrains.eu/core/Person" in d.get("type"), "Cannot convert a non person to a person dict!"
+        _id = d.get("id")
+        name = f"{d.get('givenName')} {d.get('familyName')}"
+        return {
+            '@id': _id,
+            'schema.org/shortName': name,
+            'identifier': _id,
+            'shortName': name,
+            'name': name
+        }
+
+    def __init__(self, id, *, cached_data=None) -> None:
+        super().__init__()
+
+        self._id = id
+        self._cached_data = cached_data
+
+    @property
+    def _detail(self):
+        if not self._cached_data:
+            match = re.search(r"([a-f0-9-]+)$", self._id)
+            instance_id = match.group(1)
+            self._cached_data = MultiSourcedRequest(
+                requests=[
+                    GitlabProxy(
+                        GitlabProxyEnum.DATASETVERSION_V3,
+                        instance_id=instance_id,
+                    ),
+                ]
+            ).data
+        return self._cached_data
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def name(self) -> str:
+        fullname = self._detail.get("fullName")
+        if not fullname:
+            for dataset in self.is_version_of:
+                if fullname:
+                    break
+                fullname = dataset.name
+        version_id = self._detail.get("versionIdentifier")
+        return f"{fullname} ({version_id})"
+
+    @property
+    def is_version_of(self):
+        if not hasattr(self, "_is_version_of"):
+            self._is_version_of = [EbrainsV3Dataset(id=id.get("id")) for id in self._detail.get("isVersionOf", [])]
+        return self._is_version_of
+
+    @property
+    def urls(self) -> List[EbrainsDatasetUrl]:
+        return [{
+            "url": doi.get("identifier", None)
+        } for doi in self._detail.get("doi", [])]
+
+    @property
+    def description(self) -> str:
+        description = self._detail.get("description")
+        for ds in self.is_version_of:
+            if description:
+                break
+            description = ds.description
+        return description or ""
+
+    @property
+    def contributors(self) -> List[EbrainsDatasetPerson]:
+        return [EbrainsV3DatasetVersion._parse_person(d) for d in self._detail.get("author", [])]
+
+    @property
+    def ebrains_page(self) -> str:
+        if len(self.urls) > 0:
+            return self.urls[0].get("url")
+        return None
+
+    @property
+    def custodians(self) -> EbrainsDatasetPerson:
+        return [EbrainsV3DatasetVersion._parse_person(d) for d in self._detail.get("custodian", [])]
+
+    @property
+    def version_changelog(self):
+        return self._detail.get("versionInnovation", "")
+
+    @property
+    def version_identifier(self):
+        return self._detail.get("versionIdentifier", "")
+
+
+class EbrainsV3Dataset(EbrainsBaseDataset):
+    def __init__(self, id, *, cached_data=None) -> None:
+        super().__init__()
+
+        self._id = id
+        self._cached_data = cached_data
+        self._contributers = None
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def name(self) -> str:
+        return self._detail.get("fullName")
+
+    @property
+    def urls(self) -> List[EbrainsDatasetUrl]:
+        return [{
+            "url": doi.get("identifier", None)
+        } for doi in self._detail.get("doi", [])]
+
+    @property
+    def _detail(self):
+        if not self._cached_data:
+            match = re.search(r"([a-f0-9-]+)$", self._id)
+            instance_id = match.group(1)
+            self._cached_data = MultiSourcedRequest(
+                requests=[
+                    GitlabProxy(
+                        GitlabProxyEnum.DATASET_V3,
+                        instance_id=instance_id,
+                    ),
+                ]
+            ).data
+        return self._cached_data
+
+    @property
+    def description(self) -> str:
+        return self._detail.get("description", "")
+
+    @property
+    def contributors(self):
+        if self._contributers is None:
+            contributers = {}
+            for version_id in self.version_ids:
+                contributers.update(
+                    {c['@id']: c for c in EbrainsV3DatasetVersion(version_id).contributors}
+                )
+            self._contributers = list(contributers.values())
+        return self._contributers
+
+    @property
+    def ebrains_page(self) -> str:
+        if len(self.urls) > 0:
+            return self.urls[0].get("url")
+        return None
+
+    @property
+    def custodians(self) -> EbrainsDatasetPerson:
+        return [EbrainsV3DatasetVersion._parse_person(d) for d in self._detail.get("custodian", [])]
+
+    @property
+    def version_ids(self) -> List['str']:
+        return [version.get("id") for version in self._detail.get("versions", [])]
+
+
+class GenericDataset():
+
+    def __init__(
+        self,
+        name: str = None,
+        contributors: List[str] = None,
+        url: str = None,
+        description: str = None,
+        license: str = None
+    ):
+        self._name = name
+        self._contributors = contributors
+        self._url = url
+        self._description = description
+        self._license = license
+
+    @property
+    def contributors(self):
+        return [{"name": cont} for cont in self._contributors]
+
+    @property
+    def id(self) -> str:
+        return md5(self.name.encode('utf-8')).hexdigest()
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def LICENSE(self) -> str:
+        return self._license
+
+    @property
+    def urls(self) -> List[EbrainsDatasetUrl]:
+        return [{"url": self._url}]
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def __eq__(self, o: object) -> bool:
+        return hasattr(o, "id") and self.id == o.id

siibra/retrieval/exceptions/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2018-2024
+# Institute of Neuroscience and Medicine (INM-1), Forschungszentrum Jülich GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Exceptions concerning file retrieval processes."""
+
+
+class NoSiibraConfigMirrorsAvailableException(Exception):
+    pass
+
+
+class TagNotFoundException(Exception):
+    pass
+
+
+class EbrainsAuthenticationError(Exception):
+    pass