siibra 0.4a33__py3-none-any.whl → 0.4a46__py3-none-any.whl

siibra/retrieval/requests.py

@@ -15,7 +15,7 @@
 
 from .cache import CACHE
 from .exceptions import EbrainsAuthenticationError
-from ..commons import logger, HBP_AUTH_TOKEN, KEYCLOAK_CLIENT_ID, KEYCLOAK_CLIENT_SECRET
+from ..commons import logger, HBP_AUTH_TOKEN, KEYCLOAK_CLIENT_ID, KEYCLOAK_CLIENT_SECRET, siibra_tqdm, SIIBRA_USE_LOCAL_SNAPSPOT
 from .. import __version__
 
 import json
@@ -26,15 +26,20 @@ from nibabel import Nifti1Image, GiftiImage, streamlines
 from skimage import io
 import gzip
 from io import BytesIO
-import urllib
+import urllib.parse
 import pandas as pd
 import numpy as np
-from tqdm import tqdm
 from typing import List, Callable, Any, TYPE_CHECKING
 from enum import Enum
 from functools import wraps
 from time import sleep
 import sys
+import platform
+
+if platform.system() == "Linux":
+    from filelock import FileLock as Lock
+else:
+    from filelock import SoftFileLock as Lock
 
 if TYPE_CHECKING:
     from .repositories import GitlabConnector
@@ -48,7 +53,7 @@ DECODERS = {
     ".json": lambda b: json.loads(b.decode()),
     ".tck": lambda b: streamlines.load(BytesIO(b)),
     ".csv": lambda b: pd.read_csv(BytesIO(b), delimiter=";"),
-    ".tsv": lambda b: pd.read_csv(BytesIO(b), delimiter="\t"),
+    ".tsv": lambda b: pd.read_csv(BytesIO(b), delimiter="\t").dropna(axis=0, how="all"),
     ".txt": lambda b: pd.read_csv(BytesIO(b), delimiter=" ", header=None),
     ".zip": lambda b: ZipFile(BytesIO(b)),
     ".png": lambda b: io.imread(BytesIO(b)),
@@ -66,16 +71,16 @@ class SiibraHttpRequestError(Exception):
 
     def __str__(self):
         return (
-            f"{self.msg}\n\tStatus code:{self.status_code:68.68}\n\tUrl:{self.response.url:76.76}"
+            f"{self.msg}\n\tStatus code: {self.status_code}\n\tUrl: {self.url:76.76}"
         )
 
 
 class HttpRequest:
     def __init__(
         self,
-        url,
-        func=None,
-        msg_if_not_cached=None,
+        url: str,
+        func: Callable=None,
+        msg_if_not_cached: str=None,
         refresh=False,
         post=False,
         **kwargs,
@@ -133,58 +138,55 @@ class HttpRequest:
         return os.path.isfile(self.cachefile)
 
     def _retrieve(self, block_size=1024, min_bytesize_with_no_progress_info=2e8):
-        # Loads the data from http if required.
-        # If the data is already cached, None is returned,
-        # otherwise data (as it is already in memory anyway).
-        # The caller should load the cachefile only
-        # if None is returned.
+        # Populates the file cache with the data from http if required.
+        # noop if 1/ data is already cached and 2/ refresh flag not set
+        # The caller should load the cachefile after _retrieve successfuly executes
 
         if self.cached and not self.refresh:
             return
-        else:
-            # not yet in cache, perform http request.
-            if self.msg_if_not_cached is not None:
-                logger.debug(self.msg_if_not_cached)
-            headers = self.kwargs.get('headers', {})
-            other_kwargs = {key: self.kwargs[key] for key in self.kwargs if key != "headers"}
-            if self.post:
-                r = requests.post(self.url, headers={
-                    **USER_AGENT_HEADER,
-                    **headers,
-                }, **other_kwargs, stream=True)
-            else:
-                r = requests.get(self.url, headers={
-                    **USER_AGENT_HEADER,
-                    **headers,
-                }, **other_kwargs, stream=True)
-            if r.ok:
-                size_bytes = int(r.headers.get('content-length', 0))
-                if size_bytes > min_bytesize_with_no_progress_info:
-                    progress_bar = tqdm(
-                        total=size_bytes, unit='iB', unit_scale=True,
-                        position=0, leave=True,
-                        desc=f"Downloading {os.path.split(self.url)[-1]} ({size_bytes / 1024**2:.1f} MiB)"
-                    )
-                temp_cachefile = self.cachefile + "_temp"
-                with open(temp_cachefile, "wb") as f:
-                    for data in r.iter_content(block_size):
-                        if size_bytes > min_bytesize_with_no_progress_info:
-                            progress_bar.update(len(data))
-                        f.write(data)
-                if size_bytes > min_bytesize_with_no_progress_info:
-                    progress_bar.close()
-                self.refresh = False
-                os.rename(temp_cachefile, self.cachefile)
-                with open(self.cachefile, 'rb') as f:
-                    return f.read()
-            else:
-                raise SiibraHttpRequestError(status_code=r.status_code, url=self.url)
+        
+        # not yet in cache, perform http request.
+        if self.msg_if_not_cached is not None:
+            logger.debug(self.msg_if_not_cached)
+            
+        headers = self.kwargs.get('headers', {})
+        other_kwargs = {key: self.kwargs[key] for key in self.kwargs if key != "headers"}
+
+        http_method = requests.post if self.post else requests.get
+        r = http_method(self.url, headers={
+            **USER_AGENT_HEADER,
+            **headers,
+        }, **other_kwargs, stream=True)
+
+        if not r.ok:
+            raise SiibraHttpRequestError(status_code=r.status_code, url=self.url)
+
+        size_bytes = int(r.headers.get('content-length', 0))
+        if size_bytes > min_bytesize_with_no_progress_info:
+            progress_bar = siibra_tqdm(
+                total=size_bytes, unit='iB', unit_scale=True,
+                position=0, leave=True,
+                desc=f"Downloading {os.path.split(self.url)[-1]} ({size_bytes / 1024**2:.1f} MiB)"
+            )
+        temp_cachefile = f"{self.cachefile}_temp"
+        lock = Lock(f"{temp_cachefile}.lock")
+        
+        with lock:
+            with open(temp_cachefile, "wb") as f:
+                for data in r.iter_content(block_size):
+                    if size_bytes > min_bytesize_with_no_progress_info:
+                        progress_bar.update(len(data))
+                    f.write(data)
+            if size_bytes > min_bytesize_with_no_progress_info:
+                progress_bar.close()
+            self.refresh = False
+            os.rename(temp_cachefile, self.cachefile)
+            
 
     def get(self):
-        data = self._retrieve()
-        if data is None:
-            with open(self.cachefile, "rb") as f:
-                data = f.read()
+        self._retrieve()
+        with open(self.cachefile, "rb") as f:
+            data = f.read()
         try:
             return data if self.func is None else self.func(data)
         except Exception as e:
@@ -262,24 +264,26 @@ class EbrainsRequest(HttpRequest):
             logger.debug(f"token_endpoint exists in .well-known/openid-configuration. Setting _IAM_TOKEN_ENDPOINT to {json_resp.get('token_endpoint')}")
             cls._IAM_TOKEN_ENDPOINT = json_resp.get("token_endpoint")
         else:
-            logger.warn("expect token endpoint in .well-known/openid-configuration, but was not present")
+            logger.warning("expect token endpoint in .well-known/openid-configuration, but was not present")
 
         if "device_authorization_endpoint" in json_resp:
             logger.debug(f"device_authorization_endpoint exists in .well-known/openid-configuration. setting _IAM_DEVICE_ENDPOINT to {json_resp.get('device_authorization_endpoint')}")
             cls._IAM_DEVICE_ENDPOINT = json_resp.get("device_authorization_endpoint")
         else:
-            logger.warn("expected device_authorization_endpoint in .well-known/openid-configuration, but was not present")
+            logger.warning("expected device_authorization_endpoint in .well-known/openid-configuration, but was not present")
 
     @classmethod
-    def fetch_token(cls):
-        """Fetch an EBRAINS token using commandline-supplied username/password
+    def fetch_token(cls, **kwargs):
+        """
+        Fetch an EBRAINS token using commandline-supplied username/password
         using the data proxy endpoint.
+        
         :ref:`Details on how to access EBRAINS are here.<accessEBRAINS>`
         """
-        cls.device_flow()
+        cls.device_flow(**kwargs)
         
     @classmethod
-    def device_flow(cls):
+    def device_flow(cls, **kwargs):
         if all([
             not sys.__stdout__.isatty(),  # if is tty, do not raise
             not any(k in ['JPY_INTERRUPT_EVENT', "JPY_PARENT_PID"] for k in os.environ),  # if is notebook environment, do not raise
@@ -291,11 +295,33 @@ class EbrainsRequest(HttpRequest):
             )
 
         cls.init_oidc()
+
+        def get_scopes() -> str:
+            scopes = kwargs.get("scopes")
+            if not scopes:
+                return None
+            if not isinstance(scopes, list):
+                logger.warning(f"scopes needs to be a list, is but is not... skipping")
+                return None
+            if not all(isinstance(scope, str) for scope in scopes):
+                logger.warning(f"scopes needs to be all str, but is not")
+                return None
+            if len(scopes) == 0:
+                logger.warning(f'provided empty list as scopes... skipping')
+                return None
+            return "+".join(scopes)
+        
+        scopes = get_scopes()
+        
+        data = {
+            'client_id': cls._IAM_DEVICE_FLOW_CLIENTID
+        }
+
+        if scopes:
+            data['scopes'] = scopes
         resp = requests.post(
             url=cls._IAM_DEVICE_ENDPOINT,
-            data={
-                'client_id': cls._IAM_DEVICE_FLOW_CLIENTID
-            }
+            data=data
         )
         resp.raise_for_status()
         resp_json = resp.json()
@@ -420,55 +446,6 @@ class EbrainsRequest(HttpRequest):
         return super().get()
 
 
-class EbrainsKgQuery(EbrainsRequest):
-    """Request outputs from a knowledge graph query."""
-
-    server = "https://kg.humanbrainproject.eu"
-    org = "minds"
-    domain = "core"
-    version = "v1.0.0"
-
-    SC_MESSAGES = {
-        401: "The provided EBRAINS authentication token is not valid",
-        403: "No permission to access the given query",
-        404: "Query with this id not found",
-    }
-
-    def __init__(self, query_id, instance_id=None, schema="dataset", params={}):
-        inst_tail = "/" + instance_id if instance_id is not None else ""
-        self.schema = schema
-        url = "{}/query/{}/{}/{}/{}/{}/instances{}?databaseScope=RELEASED".format(
-            self.server,
-            self.org,
-            self.domain,
-            self.schema,
-            self.version,
-            query_id,
-            inst_tail,
-        )
-        EbrainsRequest.__init__(
-            self,
-            url,
-            decoder=DECODERS[".json"],
-            params=params,
-            msg_if_not_cached=f"Executing EBRAINS KG query {query_id}{inst_tail}",
-        )
-
-    def get(self):
-        try:
-            result = EbrainsRequest.get(self)
-        except SiibraHttpRequestError as e:
-            if e.status_code in self.SC_MESSAGES:
-                raise RuntimeError(self.SC_MESSAGES[e.status_code])
-            else:
-                raise RuntimeError(
-                    f"Could not process HTTP request (status code: "
-                    f"{e.status_code}). Message was: {e.msg}"
-                    f"URL was: {e.url}"
-                )
-        return result
-
-
 def try_all_connectors():
     def outer(fn):
         @wraps(fn)
@@ -491,6 +468,7 @@ class GitlabProxyEnum(Enum):
     DATASET_V1 = "DATASET_V1"
     PARCELLATIONREGION_V1 = "PARCELLATIONREGION_V1"
     DATASET_V3 = "DATASET_V3"
+    DATASETVERSION_V3 = "DATASETVERSION_V3"
 
     @property
     def connectors(self) -> List['GitlabConnector']:
@@ -498,7 +476,10 @@ class GitlabProxyEnum(Enum):
             ("https://jugit.fz-juelich.de", 7846),
             ("https://gitlab.ebrains.eu", 421),
         ]
-        from .repositories import GitlabConnector
+        from .repositories import GitlabConnector, LocalFileRepository
+        if SIIBRA_USE_LOCAL_SNAPSPOT:
+            logger.info(f"Using localsnapshot at {SIIBRA_USE_LOCAL_SNAPSPOT}")
+            return [LocalFileRepository(SIIBRA_USE_LOCAL_SNAPSPOT)]
         return [GitlabConnector(server[0], server[1], "master", archive_mode=True) for server in servers]
 
     @try_all_connectors()
@@ -515,8 +496,9 @@ class GitlabProxyEnum(Enum):
 class GitlabProxy(HttpRequest):
 
     folder_dict = {
-        GitlabProxyEnum.DATASET_V1: "ebrainsquery/v1/datasets",
-        GitlabProxyEnum.DATASET_V3: "ebrainsquery/v3/datasets",
+        GitlabProxyEnum.DATASET_V1: "ebrainsquery/v1/dataset",
+        GitlabProxyEnum.DATASET_V3: "ebrainsquery/v3/Dataset",
+        GitlabProxyEnum.DATASETVERSION_V3: "ebrainsquery/v3/DatasetVersion",
         GitlabProxyEnum.PARCELLATIONREGION_V1: "ebrainsquery/v1/parcellationregions",
     }
 

siibra/volumes/gifti.py

@@ -55,12 +55,24 @@ class GiftiMesh(volume.VolumeProvider, srctype="gii-mesh"):
 
     def fetch(self, fragment: str = None, **kwargs):
         """
-        Returns the mesh as a dictionary with two numpy arrays: An Nx3 array of vertex coordinates,
-        and an Mx3 array of face definitions using row indices of the vertex array.
-
-        A fragment name can be specified to choose from multiple fragments.
-        If not specified, multiple fragments will be merged into one mesh. In such a case,
-        the verts and faces arrays of different fragments are appended to one another.
+        Returns the mesh as a dictionary with two numpy arrays.
+
+        Parameters
+        ----------
+        fragment: str, default: None
+            A fragment name can be specified to choose from multiple fragments.
+
+            Note
+            ----
+            If not specified, multiple fragments will be merged into one mesh.
+            In such a case, the verts and faces arrays of different fragments
+            are appended to one another.
+
+        Returns
+        -------
+        dict
+            - 'verts': An Nx3 array of vertex coordinates,
+            - 'faces': an Mx3 array of face definitions using row indices of the vertex array
         """
         for arg in ["resolution_mm", "voi"]:
             if kwargs.get(arg):
@@ -93,11 +105,14 @@ class GiftiMesh(volume.VolumeProvider, srctype="gii-mesh"):
 
     def fetch_iter(self):
         """
-        Iterator returning all submeshes, each represented as a dictionary
-        with elements
-        - 'verts': An Nx3 array of vertex coordinates,
-        - 'faces': an Mx3 array of face definitions using row indices of the vertex array
-        - 'name': Name of the of the mesh variant
+        Iterator returning all submeshes
+
+        Returns
+        -------
+        dict
+            - 'verts': An Nx3 array of vertex coordinates,
+            - 'faces': an Mx3 array of face definitions using row indices of the vertex array
+            - 'name': Name of the of the mesh variant
         """
         return (self.fetch(v) for v in self.variants)
 

siibra/volumes/neuroglancer.py

@@ -58,14 +58,14 @@ class NeuroglancerProvider(volume.VolumeProvider, srctype="neuroglancer/precompu
 
         Parameters
         ----------
-        fragment: str
-            Optional name of a fragment volume to fetch, if any.
-            For example, some volumes are split into left and right hemisphere fragments.
-            see :func:`~siibra.volumes.Volume.fragments`
+        fragment: str, optional
+            The name of a fragment volume to fetch, if any. For example,
+            some volumes are split into left and right hemisphere fragments.
+            See :func:`~siibra.volumes.Volume.fragments`
         resolution_mm: float
-            Specify the resolugion
-        voi : BoundingBox
-            optional specification of a volume of interst to fetch.
+            Specify the resolution
+        voi: BoundingBox
+            optional specification of a volume of interest to fetch.
         """
 
         result = None
@@ -144,8 +144,8 @@ class NeuroglancerProvider(volume.VolumeProvider, srctype="neuroglancer/precompu
                 s0 = np.identity(4)
                 s0[:3, -1] = list(bbox.minpoint.transform(np.linalg.inv(img.affine)))
                 result_affine = np.dot(img.affine, s0)  # adjust global bounding box offset to get global affine
-                voxdims = np.dot(np.linalg.inv(result_affine), np.r_[bbox.shape, 1])[:3]
-                result_arr = np.zeros((voxdims + .5).astype('int'))
+                voxdims = np.asanyarray(bbox.transform(result_affine).shape, dtype="int")
+                result_arr = np.zeros(voxdims, dtype=img.dataobj.dtype)
                 result = nib.Nifti1Image(dataobj=result_arr, affine=result_affine)
 
             arr = np.asanyarray(img.dataobj)
@@ -162,7 +162,7 @@ class NeuroglancerProvider(volume.VolumeProvider, srctype="neuroglancer/precompu
 
         if num_conflicts > 0:
             num_voxels = np.count_nonzero(result_arr)
-            logger.warn(f"Merging fragments required to overwrite {num_conflicts} conflicting voxels ({num_conflicts/num_voxels*100.:2.1f}%).")
+            logger.warning(f"Merging fragments required to overwrite {num_conflicts} conflicting voxels ({num_conflicts/num_voxels*100.:2.1f}%).")
 
         return result
 
@@ -201,7 +201,7 @@ class NeuroglancerVolume:
             return self._transform_nm
 
         self._transform_nm = np.identity(1)
-        logger.warn(f"No transform.json found at {self.url}, using identity.")
+        logger.warning(f"No transform.json found at {self.url}, using identity.")
         return self._transform_nm
 
     @transform_nm.setter
@@ -278,7 +278,7 @@ class NeuroglancerVolume:
             scale = suitable[-1]
         else:
             scale = self.scales[0]
-            logger.warn(
+            logger.warning(
                 f"Requested resolution {resolution_mm} is not available. "
                 f"Falling back to the highest possible resolution of "
                 f"{', '.join(map('{:.2f}'.format, scale.res_mm))} mm."
@@ -294,7 +294,7 @@ class NeuroglancerVolume:
                     f"relative to the limit of {self.MAX_BYTES/1024**3}GiB."
                 )
         if scale_changed:
-            logger.warn(f"Resolution was reduced to {scale.res_mm} to provide a feasible volume size")
+            logger.warning(f"Resolution was reduced to {scale.res_mm} to provide a feasible volume size")
         return scale
 
 
@@ -399,7 +399,7 @@ class NeuroglancerScale:
         x1, y1, z1 = np.minimum(self.chunk_sizes + [x0, y0, z0], self.size)
         chunk_czyx = self.volume._io.read_chunk(self.key, (x0, x1, y0, y1, z0, z1))
         if not chunk_czyx.shape[0] == 1 and not self.color_warning_issued:
-            logger.warn(
+            logger.warning(
                 "Color channel data is not yet supported. Returning first channel only."
             )
             self.color_warning_issued = True
@@ -419,7 +419,7 @@ class NeuroglancerScale:
 
         for dim in range(3):
             if bbox_.shape[dim] < 1:
-                logger.warn(
+                logger.warning(
                     f"Bounding box in voxel space will be enlarged to voxel size 1 along axis {dim}."
                 )
                 bbox_.maxpoint[dim] = bbox_.maxpoint[dim] + 1
@@ -507,7 +507,7 @@ class NeuroglancerMesh(volume.VolumeProvider, srctype="neuroglancer/precompmesh"
             fragment_names = meshinfo.get('fragments')
 
             if len(fragment_names) == 0:
-                raise RuntimeError("No fragments found at {meshurl}")
+                raise RuntimeError(f"No fragments found at {meshurl}")
             elif len(self._meshes) > 1:
                 # multiple meshes were configured, so we expect only one fragment under each mesh url
                 if len(fragment_names) > 1:
@@ -535,9 +535,25 @@ class NeuroglancerMesh(volume.VolumeProvider, srctype="neuroglancer/precompmesh"
     def fetch(self, label: int, fragment: str):
         """
         Fetches a particular mesh. Each mesh is a dictionary with keys:
-
-        - verts: an Nx3 array of coordinates (in nanometer)
-        - faces: an MX3 array containing connection data of vertices
+        
+        Parameters
+        ----------
+        label: int
+            Label of the volume
+        fragment: str, default: None
+            A fragment name can be specified to choose from multiple fragments.
+
+            Note
+            ----
+            If not specified, multiple fragments will be merged into one mesh.
+            In such a case, the verts and faces arrays of different fragments
+            are appended to one another.
+        Returns
+        -------
+        dict
+            - 'verts': An Nx3 array of vertex coordinates (in nanometer)
+            - 'faces': an MX3 array containing connection data of vertices
+            - 'name': Name of the of the mesh variant
         """
 
         # extract fragment information for the requested mesh

siibra/volumes/nifti.py

@@ -50,7 +50,6 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
             self._img_loaders = {lbl: loader(url) for lbl, url in src.items()}
         else:
             raise ValueError(f"Invalid source specification for {self.__class__}: {src}")
-
         if not isinstance(src, nib.Nifti1Image):
             self._init_url = src
 
@@ -72,7 +71,7 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
         for loader in self._img_loaders.values():
             img = loader()
             if len(img.shape) > 3:
-                logger.warn(
+                logger.warning(
                     f"N-D NIfTI volume has shape {img.shape}, but "
                     f"bounding box considers only {img.shape[:3]}"
                 )
@@ -95,8 +94,8 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
                 s0 = np.identity(4)
                 s0[:3, -1] = list(bbox.minpoint.transform(np.linalg.inv(img.affine)))
                 result_affine = np.dot(img.affine, s0)  # adjust global bounding box offset to get global affine
-                voxdims = np.dot(np.linalg.inv(result_affine), np.r_[bbox.shape, 1])[:3]
-                result_arr = np.zeros((voxdims + .5).astype('int'))
+                voxdims = np.asanyarray(bbox.transform(result_affine).shape, dtype="int")
+                result_arr = np.zeros(voxdims, dtype=img.dataobj.dtype)
                 result = nib.Nifti1Image(dataobj=result_arr, affine=result_affine)
 
             arr = np.asanyarray(img.dataobj)
@@ -113,7 +112,7 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
 
         if num_conflicts > 0:
             num_voxels = np.count_nonzero(result_arr)
-            logger.warn(f"Merging fragments required to overwrite {num_conflicts} conflicting voxels ({num_conflicts/num_voxels*100.:2.1f}%).")
+            logger.warning(f"Merging fragments required to overwrite {num_conflicts} conflicting voxels ({num_conflicts/num_voxels*100.:2.1f}%).")
 
         return result
 
@@ -133,8 +132,8 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
             For example, some volumes are split into left and right hemisphere fragments.
             see :func:`~siibra.volumes.Volume.fragments`
         voi : BoundingBox
-            optional specification of a volume of interst to fetch.
-        label: int or None
+            optional specification of a volume of interest to fetch.
+        label: int, default: None
             Optional: a label index can be provided. Then the mask of the
             3D volume will be returned, where voxels matching this label
             are marked as "1".
@@ -202,14 +201,14 @@ class NiftiProvider(volume.VolumeProvider, srctype="nii"):
         """
         Find peaks in the image data.
 
-        Arguments:
+        Parameters
         ----------
         min_distance_mm : float
             Minimum distance between peaks in mm
 
         Returns:
         --------
-        peaks: PointSet
+        PointSet
         """
 
         from skimage.feature.peak import peak_local_max