siibra 1.0a1__1-py3-none-any.whl

siibra/livequeries/bigbrain.py

@@ -0,0 +1,197 @@
+# Copyright 2018-2024
+# Institute of Neuroscience and Medicine (INM-1), Forschungszentrum Jülich GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Matches BigBrain intesity profiles extracted by Wagstyl et al. to volumes."""
+
+from . import query
+
+from ..features.tabular import bigbrain_intensity_profile, layerwise_bigbrain_intensities
+from ..features import anchor as _anchor
+from ..commons import logger
+from ..locations import point, pointcloud
+from ..core import structure
+from ..retrieval import requests, cache
+from ..retrieval.datasets import GenericDataset
+
+import numpy as np
+from typing import List
+from os import path
+
+
+class WagstylProfileLoader:
+
+    REPO = "https://github.com/kwagstyl/cortical_layers_tutorial/raw/main"
+    PROFILES_FILE_LEFT = "data/profiles_left.npy"
+    THICKNESSES_FILE_LEFT = "data/thicknesses_left.npy"
+    MESH_FILE_LEFT = "data/gray_left_327680.surf.gii"
+    _profiles = None
+    _vertices = None
+    _boundary_depths = None
+    DATASET = GenericDataset(
+        name="HIBALL workshop on cortical layers",
+        contributors=[
+            'Konrad Wagstyl',
+            'Stéphanie Larocque',
+            'Guillem Cucurull',
+            'Claude Lepage',
+            'Joseph Paul Cohen',
+            'Sebastian Bludau',
+            'Nicola Palomero-Gallagher',
+            'Lindsay B. Lewis',
+            'Thomas Funck',
+            'Hannah Spitzer',
+            'Timo Dickscheid',
+            'Paul C. Fletcher',
+            'Adriana Romero',
+            'Karl Zilles',
+            'Katrin Amunts',
+            'Yoshua Bengio',
+            'Alan C. Evans'
+        ],
+        url="https://github.com/kwagstyl/cortical_layers_tutorial/",
+        description="Cortical profiles of BigBrain staining intensities computed by Konrad Wagstyl, "
+        "as described in the publication 'Wagstyl, K., et al (2020). BigBrain 3D atlas of "
+        "cortical layers: Cortical and laminar thickness gradients diverge in sensory and "
+        "motor cortices. PLoS Biology, 18(4), e3000678. "
+        "http://dx.doi.org/10.1371/journal.pbio.3000678."
+        "The data is taken from the tutorial at "
+        "https://github.com/kwagstyl/cortical_layers_tutorial. Each vertex is "
+        "assigned to the regional map when queried."
+    )
+
+    def __init__(self):
+        if self._profiles is None:
+            self.__class__._load()
+
+    @property
+    def profile_labels(self):
+        return np.arange(0., 1., 1. / self._profiles.shape[1])
+
+    @classmethod
+    def _load(cls):
+        # read thicknesses, in mm, and normalize by their last column which is the total thickness
+        thickness = requests.HttpRequest(f"{cls.REPO}/{cls.THICKNESSES_FILE_LEFT}").data.T
+        total_thickness = thickness[:, :-1].sum(1)  # last column is the computed total thickness
+        valid = np.where(total_thickness > 0)[0]
+        cls._boundary_depths = np.c_[np.zeros_like(valid), (thickness[valid, :] / total_thickness[valid, None]).cumsum(1)]
+        cls._boundary_depths[:, -1] = 1  # account for float calculation errors
+
+        # find profiles with valid thickness
+        profile_l_url = f"{cls.REPO}/{cls.PROFILES_FILE_LEFT}"
+        if not path.exists(cache.CACHE.build_filename(profile_l_url)):
+            logger.info(
+                "First request to BigBrain profiles. Preprocessing the data "
+                "now. This may take a little."
+            )
+        profiles_l_all = requests.HttpRequest(profile_l_url).data
+        cls._profiles = profiles_l_all[valid, :]
+
+        # read mesh vertices
+        mesh_left = requests.HttpRequest(f"{cls.REPO}/{cls.MESH_FILE_LEFT}").data
+        mesh_vertices = mesh_left.darrays[0].data
+        cls._vertices = mesh_vertices[valid, :]
+
+        logger.debug(f"{cls._profiles.shape[0]} BigBrain intensity profiles.")
+        assert cls._vertices.shape[0] == cls._profiles.shape[0]
+
+    def __len__(self):
+        return self._vertices.shape[0]
+
+
+cache.Warmup.register_warmup_fn()(lambda: WagstylProfileLoader._load())
+
+
+class BigBrainProfileQuery(query.LiveQuery, args=[], FeatureType=bigbrain_intensity_profile.BigBrainIntensityProfile):
+
+    def __init__(self):
+        query.LiveQuery.__init__(self)
+
+    def query(self, concept: structure.BrainStructure, **kwargs) -> List[bigbrain_intensity_profile.BigBrainIntensityProfile]:
+        loader = WagstylProfileLoader()
+        mesh_vertices = pointcloud.PointCloud(loader._vertices, space='bigbrain')
+        matched = concept.intersection(mesh_vertices)  # returns a reduced PointCloud with og indices as labels
+        if matched is None:
+            return []
+        assert isinstance(matched, pointcloud.PointCloud)
+        indices = matched.labels
+        assert indices is not None
+        features = []
+        for i in matched.labels:
+            anchor = _anchor.AnatomicalAnchor(
+                location=point.Point(loader._vertices[i], space='bigbrain'),
+                region=str(concept),
+                species='Homo sapiens'
+            )
+            prof = bigbrain_intensity_profile.BigBrainIntensityProfile(
+                anchor=anchor,
+                depths=loader.profile_labels,
+                values=loader._profiles[i],
+                boundaries=loader._boundary_depths[i]
+            )
+            prof.anchor._assignments[concept] = _anchor.AnatomicalAssignment(
+                query_structure=concept,
+                assigned_structure=concept,
+                qualification=_anchor.Qualification.CONTAINED,
+                explanation=f"Surface vertex of BigBrain cortical profile was filtered using {concept}"
+            )
+            prof.datasets = [WagstylProfileLoader.DATASET]
+            features.append(prof)
+
+        return features
+
+
+class LayerwiseBigBrainIntensityQuery(query.LiveQuery, args=[], FeatureType=layerwise_bigbrain_intensities.LayerwiseBigBrainIntensities):
+
+    def __init__(self):
+        query.LiveQuery.__init__(self)
+
+    def query(self, concept: structure.BrainStructure, **kwargs) -> List[layerwise_bigbrain_intensities.LayerwiseBigBrainIntensities]:
+
+        loader = WagstylProfileLoader()
+        mesh_vertices = pointcloud.PointCloud(loader._vertices, space='bigbrain')
+        matched = concept.intersection(mesh_vertices)  # returns a reduced PointCloud with og indices as labels
+        if matched is None:
+            return []
+        assert isinstance(matched, pointcloud.PointCloud)
+        indices = matched.labels
+        assert indices is not None
+        matched_profiles = loader._profiles[indices, :]
+        boundary_depths = loader._boundary_depths[indices, :]
+        # compute array of layer labels for all coefficients in profiles_left
+        N = matched_profiles.shape[1]
+        prange = np.arange(N)
+        layer_labels = 7 - np.array([
+            [np.array([[(prange < T) * 1] for i, T in enumerate((b * N).astype('int'))]).squeeze().sum(0)]
+            for b in boundary_depths
+        ]).reshape((-1, 200))
+
+        anchor = _anchor.AnatomicalAnchor(
+            location=pointcloud.PointCloud(loader._vertices[indices, :], space='bigbrain'),
+            region=str(concept),
+            species='Homo sapiens'
+        )
+        result = layerwise_bigbrain_intensities.LayerwiseBigBrainIntensities(
+            anchor=anchor,
+            means=[matched_profiles[layer_labels == layer].mean() for layer in range(1, 7)],
+            stds=[matched_profiles[layer_labels == layer].std() for layer in range(1, 7)],
+        )
+        result.anchor._assignments[concept] = _anchor.AnatomicalAssignment(
+            query_structure=concept,
+            assigned_structure=concept,
+            qualification=_anchor.Qualification.CONTAINED,
+            explanation=f"Surface vertices of BigBrain cortical profiles were filtered using {concept}"
+        )
+        result.datasets = [WagstylProfileLoader.DATASET]
+
+        return [result]

siibra/livequeries/ebrains.py

@@ -0,0 +1,145 @@
+# Copyright 2018-2024
+# Institute of Neuroscience and Medicine (INM-1), Forschungszentrum Jülich GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Query data features published as Ebrains datasets with AtlasConcepts"""
+
+from ..features.dataset import ebrains as _ebrains
+from . import query
+
+from ..commons import logger, siibra_tqdm
+from ..features import anchor as _anchor
+from ..retrieval import requests, datasets, cache
+from ..core import parcellation, region
+
+from collections import defaultdict
+import re
+from packaging.version import Version
+from tempfile import NamedTemporaryFile
+
+
+class EbrainsFeatureQuery(query.LiveQuery, args=[], FeatureType=_ebrains.EbrainsDataFeature):
+    # in EBRAINS knowledge graph prior to v3, versions were modelled
+    # in dataset names. Typically found formats are (v1.0) and [rat, v2.1]
+    VERSION_PATTERN = re.compile(r"^(.*?) *[\[\(][^v]*?(v[0-9].*?)[\]\)]")
+    COMPACT_FEATURE_LIST = True
+
+    # datasets whose name contains any of these strings will be ignored
+    _BLACKLIST = {
+        "Whole-brain parcellation of the Julich-Brain Cytoarchitectonic Atlas",
+        "whole-brain collections of cytoarchitectonic probabilistic maps",
+        "DiFuMo atlas",
+        "Automated Anatomical Labeling (AAL1) atlas",
+    }
+
+    loader = requests.MultiSourcedRequest(
+        requests=[
+            requests.GitlabProxy(
+                flavour=requests.GitlabProxyEnum.PARCELLATIONREGION_V1,
+            ),
+        ]
+    )
+
+    parcellation_ids = None
+
+    def __init__(self, **kwargs):
+        query.LiveQuery.__init__(self, **kwargs)
+
+        if self.__class__.parcellation_ids is None:
+            self.__class__.parcellation_ids = [
+                dset.id
+                for parc in parcellation.Parcellation.registry()
+                for dset in parc.datasets
+                if isinstance(dset, datasets.EbrainsV3DatasetVersion)
+            ]
+
+    def query(self, region: region.Region):
+        versioned_datasets = defaultdict(dict)
+        invalid_species_datasets = {}
+        results = self.loader.data.get("results", [])
+
+        for r in siibra_tqdm(results, total=len(results)):
+            regionname = r.get("name", None)
+            alias = r.get("alias", None)
+            for ds_spec in r.get("datasets", []):
+                ds_name = ds_spec.get("name")
+                ds_id = ds_spec.get("@id")
+                if "dataset" not in ds_id:
+                    continue
+
+                try:
+                    ds_species = _anchor.Species.decode(ds_spec)
+                except ValueError:
+                    logger.debug(f"Cannot decode {ds_spec}")
+                    invalid_species_datasets[ds_id] = ds_name
+                    continue
+
+                if self.COMPACT_FEATURE_LIST:
+                    if any(ds_id.endswith(i) for i in self.parcellation_ids):
+                        continue
+                    if any(e.lower() in ds_name.lower() for e in self._BLACKLIST):
+                        continue
+
+                dset = _ebrains.EbrainsDataFeature(
+                    dataset_version_id=ds_id,
+                    anchor=_anchor.AnatomicalAnchor(
+                        region=alias or regionname,
+                        species=ds_species,
+                    ),
+                )
+                if not dset.matches(region):
+                    continue
+
+                version_match = self.VERSION_PATTERN.search(ds_name)
+                if version_match is None or not self.COMPACT_FEATURE_LIST:
+                    yield dset
+                else:  # store version, add only the latest version after the loop
+                    name, version = version_match.groups()
+                    versioned_datasets[name][version] = dset
+
+        if len(invalid_species_datasets) > 0:
+            with NamedTemporaryFile(mode="w", suffix=".txt", delete=False, encoding="utf-8") as f:
+                for dsid, dsname in invalid_species_datasets.items():
+                    f.write(f"{dsid} {dsname}\n")
+                logger.warning(
+                    f"{len(invalid_species_datasets)} datasets have been ignored, "
+                    "because siibra could not decode their species. "
+                    f"See {f.name}"
+                )
+
+        # if versioned datasets have been recorded, register only
+        # the newest one with older ones linked as a version history.
+        for name, dsets in versioned_datasets.items():
+            try:  # if possible, sort by version tag
+                sorted_versions = sorted(dsets.keys(), key=Version)
+            except TypeError:  # else sort lexicographically
+                sorted_versions = sorted(dsets.keys())
+
+            # chain the dataset versions
+            prev = None
+            for version in sorted_versions:
+                curr = dsets[version]
+                curr.version = version
+                if prev is not None:
+                    curr._prev = prev
+                    prev._next = curr
+                prev = curr
+
+            logger.debug(
+                f"Registered only version {version} of {', '.join(sorted_versions)} for {name}. "
+                f"Its version history is: {curr.version_history}"
+            )
+            yield curr
+
+
+cache.Warmup.register_warmup_fn(cache.WarmupLevel.DATA)(lambda: EbrainsFeatureQuery.loader.data)

siibra/livequeries/query.py

@@ -0,0 +1,49 @@
+# Copyright 2018-2024
+# Institute of Neuroscience and Medicine (INM-1), Forschungszentrum Jülich GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Handles feature queries that rely on live or on-the-fly calculations."""
+
+from ..commons import logger
+from ..features.feature import Feature
+from ..core.concept import AtlasConcept
+
+from abc import ABC, abstractmethod
+from typing import List
+
+
+class LiveQuery(ABC):
+
+    # set of mandatory query argument names
+    _query_args = []
+
+    def __init__(self, **kwargs):
+        parstr = ", ".join(f"{k}={v}" for k, v in kwargs.items())
+        if parstr:
+            parstr = "with parameters " + parstr
+        if not all(p in kwargs for p in self._query_args):
+            logger.error(
+                f"Incomplete specification for {self.__class__.__name__} query "
+                f"(Mandatory arguments: {', '.join(self._query_args)})"
+            )
+        self._kwargs = kwargs
+
+    def __init_subclass__(cls, args: List[str], FeatureType: type):
+        cls._query_args = args
+        cls.feature_type = FeatureType
+        FeatureType._live_queries.append(cls)
+        return super().__init_subclass__()
+
+    @abstractmethod
+    def query(self, concept: AtlasConcept, **kwargs) -> List[Feature]:
+        raise NotImplementedError(f"Dervied class {self.__class__} needs to implement query()")

siibra/locations/__init__.py

@@ -0,0 +1,91 @@
+# Copyright 2018-2024
+# Institute of Neuroscience and Medicine (INM-1), Forschungszentrum Jülich GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Handles spatial concepts and spatial operation like warping between spaces."""
+
+from .location import Location
+from .point import Point
+from .pointcloud import PointCloud, from_points
+from .boundingbox import BoundingBox
+
+
+def reassign_union(loc0: 'Location', loc1: 'Location') -> 'Location':
+    """
+    Add two locations of same or diffrent type to find their union as a
+    Location object.
+    Note
+    ----
+    `loc1` will be warped to `loc0` they are not in the same space.
+    Parameters
+    ----------
+    loc0 : Location
+        _description_
+    loc1 : Location
+        _description_
+    Returns
+    -------
+    Location
+        - Point U Point = PointCloud
+        - Point U PointCloud = PointCloud
+        - PointCloud U PointCloud = PointCloud
+        - BoundingBox U BoundingBox = BoundingBox
+        - BoundingBox U PointCloud = BoundingBox
+        - BoundingBox U Point = BoundingBox
+        - WholeBrain U Location = NotImplementedError
+        (all operations are commutative)
+    """
+    if loc0 is None or loc1 is None:
+        return loc0 or loc1
+
+    # All location types should be unionable among each other and this should
+    # be implemented here to avoid code repetition. Volumes are the only type of
+    # location that has its own union method since it is not a part of locations
+    # module and to avoid importing Volume here.
+    if not all(
+        isinstance(loc, (Point, PointCloud, BoundingBox)) for loc in [loc0, loc1]
+    ):
+        try:
+            return loc1.union(loc0)
+        except Exception:
+            raise NotImplementedError(f"There are no union method for {(loc0.__class__.__name__, loc1.__class__.__name__)}")
+
+    # convert Points to PointClouds
+    loc0, loc1 = [
+        from_points([loc]) if isinstance(loc, Point) else loc
+        for loc in [loc0, loc1]
+    ]
+
+    # adopt the space of the first location
+    loc1_w = loc1.warp(loc0.space)
+
+    if isinstance(loc0, PointCloud):
+        if isinstance(loc1_w, PointCloud):
+            points = list(dict.fromkeys([*loc0, *loc1_w]))
+            return from_points(points)
+        if isinstance(loc1_w, BoundingBox):
+            return reassign_union(loc0.boundingbox, loc1_w)
+
+    if isinstance(loc0, BoundingBox) and isinstance(loc1_w, BoundingBox):
+        coordinates = [loc0.minpoint, loc0.maxpoint, loc1_w.minpoint, loc1_w.maxpoint]
+        return BoundingBox(
+            point1=[min(p[i] for p in coordinates) for i in range(3)],
+            point2=[max(p[i] for p in coordinates) for i in range(3)],
+            space=loc0.space,
+            sigma_mm=[loc0.minpoint.sigma, loc0.maxpoint.sigma]
+        )
+
+    return reassign_union(loc1_w, loc0)
+
+
+Location.union = reassign_union