siibra 0.4a35__py3-none-any.whl → 0.4a46__py3-none-any.whl

siibra/retrieval/datasets.py

@@ -72,7 +72,7 @@ class EbrainsBaseDataset(ABC):
         raise NotImplementedError
 
     @abstractproperty
-    def custodians(self) -> EbrainsDatasetPerson:
+    def custodians(self) -> List[EbrainsDatasetPerson]:
         raise NotImplementedError
     
     def __hash__(self):
@@ -168,10 +168,80 @@ class EbrainsDataset(EbrainsBaseDataset):
     def custodians(self) -> EbrainsDatasetPerson:
         return self.detail.get("custodians")
 
+class EbrainsV3DatasetVersion(EbrainsBaseDataset):
+
+    @staticmethod
+    def parse_person(d: dict) -> EbrainsDatasetPerson:
+        assert "https://openminds.ebrains.eu/core/Person" in d.get("type"), f"Cannot convert a non person to a person dict!"
+        _id = d.get('id')
+        name = f"{d.get('givenName')} {d.get('familyName')}"
+        return {
+            '@id': _id,
+            'schema.org/shortName': name,
+            'identifier': _id,
+            "shortName": name,
+            "name": name
+        }
+
+    def __init__(self, id, *, cached_data=None) -> None:
+        super().__init__()
+
+        self._id = id
+        self._cached_data = cached_data
+    
+    @property
+    def detail(self):
+        if not self._cached_data:
+            match = re.search(r"([a-f0-9-]+)$", self._id)
+            instance_id = match.group(1)
+            self._cached_data = MultiSourcedRequest(
+                requests=[
+                    GitlabProxy(
+                        GitlabProxyEnum.DATASETVERSION_V3,
+                        instance_id=instance_id,
+                    ),
+                ]
+            ).data
+        return self._cached_data
+    
+    @property
+    def id(self) -> str :
+        return self._id
+
+    @property
+    def name(self) -> str :
+        fullname = self.detail.get("fullName")
+        version_id = self.detail.get("versionIdentifier")
+        return f"{fullname} ({version_id})"
+        
+    @property
+    def urls(self) -> List[EbrainsDatasetUrl] :
+        return [{
+            "url": doi.get("identifier", None)
+        } for doi in self.detail.get("doi", []) ]
+    
+    @property
+    def description(self) -> str :
+        return self.detail.get("description", "")
+
+    @property
+    def contributors(self) -> List[EbrainsDatasetPerson] :
+        return [EbrainsV3DatasetVersion.parse_person(d) for d in self.detail.get("author", [])]
+
+    @property
+    def ebrains_page(self) -> str :
+        if len(self.urls) > 0:
+            return self.urls[0].get("url")
+        return None
+    
+    @property
+    def custodians(self) -> EbrainsDatasetPerson:
+        return [EbrainsV3DatasetVersion.parse_person(d) for d in self.detail.get("custodian", [])]
+
 class EbrainsV3Dataset(EbrainsBaseDataset):
     # TODO finish implementing me
     # some fields are currently missing, e.g. desc, contributors etc.
-    def __init__(self, id, *, cached_data) -> None:
+    def __init__(self, id, *, cached_data=None) -> None:
         super().__init__()
 
         self._id = id
@@ -180,7 +250,7 @@ class EbrainsV3Dataset(EbrainsBaseDataset):
     @property
     def detail(self):
         if not self._cached_data:
-            match = re.search(r"([a-f0-9-]+)$", self.id)
+            match = re.search(r"([a-f0-9-]+)$", self._id)
             instance_id = match.group(1)
             self._cached_data = MultiSourcedRequest(
                 requests=[

siibra/retrieval/repositories.py

@@ -16,11 +16,10 @@
 from .requests import DECODERS, HttpRequest, EbrainsRequest, SiibraHttpRequestError
 from .cache import CACHE
 
-from .. import logger
+from ..commons import logger, siibra_tqdm
 
 from abc import ABC, abstractmethod
 from urllib.parse import quote
-from tqdm import tqdm
 import os
 from zipfile import ZipFile
 from typing import List
@@ -84,9 +83,7 @@ class RepositoryConnector(ABC):
         if progress is None or all_cached:
             return result
         else:
-            return list(tqdm(
-                result, total=len(fnames), desc=progress, disable=logger.level > 20
-            ))
+            return list(siibra_tqdm(result, total=len(fnames), desc=progress))
 
     @classmethod
     def _from_url(cls, url: str):
@@ -101,6 +98,9 @@ class RepositoryConnector(ABC):
                 f"connector from url '{url}'."
             )
 
+    def __eq__(self, other):
+        return self.base_url == other.base_url
+
 
 class LocalFileRepository(RepositoryConnector):
 
@@ -159,6 +159,9 @@ class LocalFileRepository(RepositoryConnector):
     def __str__(self):
         return f"{self.__class__.__name__} at {self._folder}"
 
+    def __eq__(self, other):
+        return self._folder == other._folder
+
 
 class GitlabConnector(RepositoryConnector):
 
@@ -270,6 +273,12 @@ class GitlabConnector(RepositoryConnector):
         with open(f"{archive_directory}/{folder}/{filename}", "rb") as fp:
             return decoder(fp.read())
 
+    def __eq__(self, other):
+        return all([
+            self.base_url == other.base_url,
+            self.reftag == other.reftag
+        ])
+
 
 class ZipfileConnector(RepositoryConnector):
 
@@ -307,6 +316,9 @@ class ZipfileConnector(RepositoryConnector):
                 result.append(fname)
         return result
 
+    def __eq__(self, other):
+        return self.url == other.url
+
     class FileLoader:
         """
         Loads a file from the zip archive, but mimics the behaviour
@@ -625,7 +637,6 @@ class EbrainsPublicDatasetConnectorMinds(RepositoryConnector):
             logger.info(f"Using title '{title}' for EBRAINS dataset search, ignoring id '{dataset_id}'")
             url = f"{self.base_url}/{self.QUERY_ID}/instances?databaseScope={stage}&title={title}"
         req = EbrainsRequest(url, DECODERS[".json"])
-        print(req.cachefile)
         response = req.get()
         self._files = {}
         results = response.get('results', [])

siibra/retrieval/requests.py

@@ -15,7 +15,7 @@
 
 from .cache import CACHE
 from .exceptions import EbrainsAuthenticationError
-from ..commons import logger, HBP_AUTH_TOKEN, KEYCLOAK_CLIENT_ID, KEYCLOAK_CLIENT_SECRET
+from ..commons import logger, HBP_AUTH_TOKEN, KEYCLOAK_CLIENT_ID, KEYCLOAK_CLIENT_SECRET, siibra_tqdm, SIIBRA_USE_LOCAL_SNAPSPOT
 from .. import __version__
 
 import json
@@ -26,15 +26,20 @@ from nibabel import Nifti1Image, GiftiImage, streamlines
 from skimage import io
 import gzip
 from io import BytesIO
-import urllib
+import urllib.parse
 import pandas as pd
 import numpy as np
-from tqdm import tqdm
 from typing import List, Callable, Any, TYPE_CHECKING
 from enum import Enum
 from functools import wraps
 from time import sleep
 import sys
+import platform
+
+if platform.system() == "Linux":
+    from filelock import FileLock as Lock
+else:
+    from filelock import SoftFileLock as Lock
 
 if TYPE_CHECKING:
     from .repositories import GitlabConnector
@@ -48,7 +53,7 @@ DECODERS = {
     ".json": lambda b: json.loads(b.decode()),
     ".tck": lambda b: streamlines.load(BytesIO(b)),
     ".csv": lambda b: pd.read_csv(BytesIO(b), delimiter=";"),
-    ".tsv": lambda b: pd.read_csv(BytesIO(b), delimiter="\t"),
+    ".tsv": lambda b: pd.read_csv(BytesIO(b), delimiter="\t").dropna(axis=0, how="all"),
     ".txt": lambda b: pd.read_csv(BytesIO(b), delimiter=" ", header=None),
     ".zip": lambda b: ZipFile(BytesIO(b)),
     ".png": lambda b: io.imread(BytesIO(b)),
@@ -66,16 +71,16 @@ class SiibraHttpRequestError(Exception):
 
     def __str__(self):
         return (
-            f"{self.msg}\n\tStatus code:{self.status_code:68.68}\n\tUrl:{self.response.url:76.76}"
+            f"{self.msg}\n\tStatus code: {self.status_code}\n\tUrl: {self.url:76.76}"
         )
 
 
 class HttpRequest:
     def __init__(
         self,
-        url,
-        func=None,
-        msg_if_not_cached=None,
+        url: str,
+        func: Callable=None,
+        msg_if_not_cached: str=None,
         refresh=False,
         post=False,
         **kwargs,
@@ -133,58 +138,55 @@ class HttpRequest:
         return os.path.isfile(self.cachefile)
 
     def _retrieve(self, block_size=1024, min_bytesize_with_no_progress_info=2e8):
-        # Loads the data from http if required.
-        # If the data is already cached, None is returned,
-        # otherwise data (as it is already in memory anyway).
-        # The caller should load the cachefile only
-        # if None is returned.
+        # Populates the file cache with the data from http if required.
+        # noop if 1/ data is already cached and 2/ refresh flag not set
+        # The caller should load the cachefile after _retrieve successfuly executes
 
         if self.cached and not self.refresh:
             return
-        else:
-            # not yet in cache, perform http request.
-            if self.msg_if_not_cached is not None:
-                logger.debug(self.msg_if_not_cached)
-            headers = self.kwargs.get('headers', {})
-            other_kwargs = {key: self.kwargs[key] for key in self.kwargs if key != "headers"}
-            if self.post:
-                r = requests.post(self.url, headers={
-                    **USER_AGENT_HEADER,
-                    **headers,
-                }, **other_kwargs, stream=True)
-            else:
-                r = requests.get(self.url, headers={
-                    **USER_AGENT_HEADER,
-                    **headers,
-                }, **other_kwargs, stream=True)
-            if r.ok:
-                size_bytes = int(r.headers.get('content-length', 0))
-                if size_bytes > min_bytesize_with_no_progress_info:
-                    progress_bar = tqdm(
-                        total=size_bytes, unit='iB', unit_scale=True,
-                        position=0, leave=True,
-                        desc=f"Downloading {os.path.split(self.url)[-1]} ({size_bytes / 1024**2:.1f} MiB)"
-                    )
-                temp_cachefile = self.cachefile + "_temp"
-                with open(temp_cachefile, "wb") as f:
-                    for data in r.iter_content(block_size):
-                        if size_bytes > min_bytesize_with_no_progress_info:
-                            progress_bar.update(len(data))
-                        f.write(data)
-                if size_bytes > min_bytesize_with_no_progress_info:
-                    progress_bar.close()
-                self.refresh = False
-                os.rename(temp_cachefile, self.cachefile)
-                with open(self.cachefile, 'rb') as f:
-                    return f.read()
-            else:
-                raise SiibraHttpRequestError(status_code=r.status_code, url=self.url)
+        
+        # not yet in cache, perform http request.
+        if self.msg_if_not_cached is not None:
+            logger.debug(self.msg_if_not_cached)
+            
+        headers = self.kwargs.get('headers', {})
+        other_kwargs = {key: self.kwargs[key] for key in self.kwargs if key != "headers"}
+
+        http_method = requests.post if self.post else requests.get
+        r = http_method(self.url, headers={
+            **USER_AGENT_HEADER,
+            **headers,
+        }, **other_kwargs, stream=True)
+
+        if not r.ok:
+            raise SiibraHttpRequestError(status_code=r.status_code, url=self.url)
+
+        size_bytes = int(r.headers.get('content-length', 0))
+        if size_bytes > min_bytesize_with_no_progress_info:
+            progress_bar = siibra_tqdm(
+                total=size_bytes, unit='iB', unit_scale=True,
+                position=0, leave=True,
+                desc=f"Downloading {os.path.split(self.url)[-1]} ({size_bytes / 1024**2:.1f} MiB)"
+            )
+        temp_cachefile = f"{self.cachefile}_temp"
+        lock = Lock(f"{temp_cachefile}.lock")
+        
+        with lock:
+            with open(temp_cachefile, "wb") as f:
+                for data in r.iter_content(block_size):
+                    if size_bytes > min_bytesize_with_no_progress_info:
+                        progress_bar.update(len(data))
+                    f.write(data)
+            if size_bytes > min_bytesize_with_no_progress_info:
+                progress_bar.close()
+            self.refresh = False
+            os.rename(temp_cachefile, self.cachefile)
+            
 
     def get(self):
-        data = self._retrieve()
-        if data is None:
-            with open(self.cachefile, "rb") as f:
-                data = f.read()
+        self._retrieve()
+        with open(self.cachefile, "rb") as f:
+            data = f.read()
         try:
             return data if self.func is None else self.func(data)
         except Exception as e:
@@ -262,13 +264,13 @@ class EbrainsRequest(HttpRequest):
             logger.debug(f"token_endpoint exists in .well-known/openid-configuration. Setting _IAM_TOKEN_ENDPOINT to {json_resp.get('token_endpoint')}")
             cls._IAM_TOKEN_ENDPOINT = json_resp.get("token_endpoint")
         else:
-            logger.warn("expect token endpoint in .well-known/openid-configuration, but was not present")
+            logger.warning("expect token endpoint in .well-known/openid-configuration, but was not present")
 
         if "device_authorization_endpoint" in json_resp:
             logger.debug(f"device_authorization_endpoint exists in .well-known/openid-configuration. setting _IAM_DEVICE_ENDPOINT to {json_resp.get('device_authorization_endpoint')}")
             cls._IAM_DEVICE_ENDPOINT = json_resp.get("device_authorization_endpoint")
         else:
-            logger.warn("expected device_authorization_endpoint in .well-known/openid-configuration, but was not present")
+            logger.warning("expected device_authorization_endpoint in .well-known/openid-configuration, but was not present")
 
     @classmethod
     def fetch_token(cls, **kwargs):
@@ -299,13 +301,13 @@ class EbrainsRequest(HttpRequest):
             if not scopes:
                 return None
             if not isinstance(scopes, list):
-                logger.warn(f"scopes needs to be a list, is but is not... skipping")
+                logger.warning(f"scopes needs to be a list, is but is not... skipping")
                 return None
             if not all(isinstance(scope, str) for scope in scopes):
-                logger.warn(f"scopes needs to be all str, but is not")
+                logger.warning(f"scopes needs to be all str, but is not")
                 return None
             if len(scopes) == 0:
-                logger.warn(f'provided empty list as scopes... skipping')
+                logger.warning(f'provided empty list as scopes... skipping')
                 return None
             return "+".join(scopes)
         
@@ -466,6 +468,7 @@ class GitlabProxyEnum(Enum):
     DATASET_V1 = "DATASET_V1"
     PARCELLATIONREGION_V1 = "PARCELLATIONREGION_V1"
     DATASET_V3 = "DATASET_V3"
+    DATASETVERSION_V3 = "DATASETVERSION_V3"
 
     @property
     def connectors(self) -> List['GitlabConnector']:
@@ -473,7 +476,10 @@ class GitlabProxyEnum(Enum):
             ("https://jugit.fz-juelich.de", 7846),
             ("https://gitlab.ebrains.eu", 421),
         ]
-        from .repositories import GitlabConnector
+        from .repositories import GitlabConnector, LocalFileRepository
+        if SIIBRA_USE_LOCAL_SNAPSPOT:
+            logger.info(f"Using localsnapshot at {SIIBRA_USE_LOCAL_SNAPSPOT}")
+            return [LocalFileRepository(SIIBRA_USE_LOCAL_SNAPSPOT)]
         return [GitlabConnector(server[0], server[1], "master", archive_mode=True) for server in servers]
 
     @try_all_connectors()
@@ -492,6 +498,7 @@ class GitlabProxy(HttpRequest):
     folder_dict = {
         GitlabProxyEnum.DATASET_V1: "ebrainsquery/v1/dataset",
         GitlabProxyEnum.DATASET_V3: "ebrainsquery/v3/Dataset",
+        GitlabProxyEnum.DATASETVERSION_V3: "ebrainsquery/v3/DatasetVersion",
         GitlabProxyEnum.PARCELLATIONREGION_V1: "ebrainsquery/v1/parcellationregions",
     }
 

siibra/volumes/neuroglancer.py

@@ -144,8 +144,8 @@ class NeuroglancerProvider(volume.VolumeProvider, srctype="neuroglancer/precompu
                 s0 = np.identity(4)
                 s0[:3, -1] = list(bbox.minpoint.transform(np.linalg.inv(img.affine)))
                 result_affine = np.dot(img.affine, s0)  # adjust global bounding box offset to get global affine
-                voxdims = np.dot(np.linalg.inv(result_affine), np.r_[bbox.shape, 1])[:3]
-                result_arr = np.zeros((voxdims + .5).astype('int'))
+                voxdims = np.asanyarray(bbox.transform(result_affine).shape, dtype="int")
+                result_arr = np.zeros(voxdims, dtype=img.dataobj.dtype)
                 result = nib.Nifti1Image(dataobj=result_arr, affine=result_affine)
 
             arr = np.asanyarray(img.dataobj)
@@ -162,7 +162,7 @@ class NeuroglancerProvider(volume.VolumeProvider, srctype="neuroglancer/precompu
 
         if num_conflicts > 0:
             num_voxels = np.count_nonzero(result_arr)
-            logger.warn(f"Merging fragments required to overwrite {num_conflicts} conflicting voxels ({num_conflicts/num_voxels*100.:2.1f}%).")
+            logger.warning(f"Merging fragments required to overwrite {num_conflicts} conflicting voxels ({num_conflicts/num_voxels*100.:2.1f}%).")
 
         return result
 
@@ -201,7 +201,7 @@ class NeuroglancerVolume:
             return self._transform_nm
 
         self._transform_nm = np.identity(1)
-        logger.warn(f"No transform.json found at {self.url}, using identity.")
+        logger.warning(f"No transform.json found at {self.url}, using identity.")
         return self._transform_nm
 
     @transform_nm.setter
@@ -278,7 +278,7 @@ class NeuroglancerVolume:
             scale = suitable[-1]
         else:
             scale = self.scales[0]
-            logger.warn(
+            logger.warning(
                 f"Requested resolution {resolution_mm} is not available. "
                 f"Falling back to the highest possible resolution of "
                 f"{', '.join(map('{:.2f}'.format, scale.res_mm))} mm."
@@ -294,7 +294,7 @@ class NeuroglancerVolume:
                     f"relative to the limit of {self.MAX_BYTES/1024**3}GiB."
                 )
         if scale_changed:
-            logger.warn(f"Resolution was reduced to {scale.res_mm} to provide a feasible volume size")
+            logger.warning(f"Resolution was reduced to {scale.res_mm} to provide a feasible volume size")
         return scale
 
 
@@ -399,7 +399,7 @@ class NeuroglancerScale:
         x1, y1, z1 = np.minimum(self.chunk_sizes + [x0, y0, z0], self.size)
         chunk_czyx = self.volume._io.read_chunk(self.key, (x0, x1, y0, y1, z0, z1))
         if not chunk_czyx.shape[0] == 1 and not self.color_warning_issued:
-            logger.warn(
+            logger.warning(
                 "Color channel data is not yet supported. Returning first channel only."
             )
             self.color_warning_issued = True
@@ -419,7 +419,7 @@ class NeuroglancerScale:
 
         for dim in range(3):
             if bbox_.shape[dim] < 1:
-                logger.warn(
+                logger.warning(
                     f"Bounding box in voxel space will be enlarged to voxel size 1 along axis {dim}."
                 )
                 bbox_.maxpoint[dim] = bbox_.maxpoint[dim] + 1
@@ -507,7 +507,7 @@ class NeuroglancerMesh(volume.VolumeProvider, srctype="neuroglancer/precompmesh"
             fragment_names = meshinfo.get('fragments')
 
             if len(fragment_names) == 0:
-                raise RuntimeError("No fragments found at {meshurl}")
+                raise RuntimeError(f"No fragments found at {meshurl}")
             elif len(self._meshes) > 1:
                 # multiple meshes were configured, so we expect only one fragment under each mesh url
                 if len(fragment_names) > 1:

siibra/volumes/nifti.py

@@ -50,7 +50,6 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
             self._img_loaders = {lbl: loader(url) for lbl, url in src.items()}
         else:
             raise ValueError(f"Invalid source specification for {self.__class__}: {src}")
-
         if not isinstance(src, nib.Nifti1Image):
             self._init_url = src
 
@@ -72,7 +71,7 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
         for loader in self._img_loaders.values():
             img = loader()
             if len(img.shape) > 3:
-                logger.warn(
+                logger.warning(
                     f"N-D NIfTI volume has shape {img.shape}, but "
                     f"bounding box considers only {img.shape[:3]}"
                 )
@@ -95,8 +94,8 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
                 s0 = np.identity(4)
                 s0[:3, -1] = list(bbox.minpoint.transform(np.linalg.inv(img.affine)))
                 result_affine = np.dot(img.affine, s0)  # adjust global bounding box offset to get global affine
-                voxdims = np.dot(np.linalg.inv(result_affine), np.r_[bbox.shape, 1])[:3]
-                result_arr = np.zeros((voxdims + .5).astype('int'))
+                voxdims = np.asanyarray(bbox.transform(result_affine).shape, dtype="int")
+                result_arr = np.zeros(voxdims, dtype=img.dataobj.dtype)
                 result = nib.Nifti1Image(dataobj=result_arr, affine=result_affine)
 
             arr = np.asanyarray(img.dataobj)
@@ -113,7 +112,7 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
 
         if num_conflicts > 0:
             num_voxels = np.count_nonzero(result_arr)
-            logger.warn(f"Merging fragments required to overwrite {num_conflicts} conflicting voxels ({num_conflicts/num_voxels*100.:2.1f}%).")
+            logger.warning(f"Merging fragments required to overwrite {num_conflicts} conflicting voxels ({num_conflicts/num_voxels*100.:2.1f}%).")
 
         return result