siibra 1.0a9__py3-none-any.whl → 1.0a14__py3-none-any.whl

siibra/features/feature.py

@@ -24,7 +24,9 @@ from typing import Union, TYPE_CHECKING, List, Dict, Type, Tuple, BinaryIO, Any,
 from hashlib import md5
 from collections import defaultdict
 from zipfile import ZipFile
-from abc import ABC
+from abc import ABC, abstractmethod
+from re import sub
+from textwrap import wrap
 
 if TYPE_CHECKING:
     from ..retrieval.datasets import EbrainsDataset
@@ -96,7 +98,8 @@ class Feature:
         modality: str,
         description: str,
         anchor: _anchor.AnatomicalAnchor,
-        datasets: List['TypeDataset'] = []
+        datasets: List['TypeDataset'] = [],
+        id: str = None
     ):
         """
         Parameters
@@ -113,6 +116,7 @@ class Feature:
         self._description = description
         self._anchor_cached = anchor
         self.datasets = datasets
+        self._id = id
 
     @property
     def modality(self):
@@ -194,7 +198,8 @@ class Feature:
     @property
     def name(self):
         """Returns a short human-readable name of this feature."""
-        return f"{self.__class__.__name__} ({self.modality}) anchored at {self.anchor}"
+        readable_class_name = sub("([a-z])([A-Z])", r"\g<1> \g<2>", self.__class__.__name__)
+        return sub("([b,B]ig [b,B]rain)", "BigBrain", readable_class_name)
 
     @classmethod
     def _get_instances(cls, **kwargs) -> List['Feature']:
@@ -260,12 +265,17 @@ class Feature:
 
     @property
     def id(self):
+        if self._id:
+            return self._id
+
         prefix = ''
         for ds in self.datasets:
             if hasattr(ds, "id"):
                 prefix = ds.id + '--'
                 break
-        return prefix + md5(self.name.encode("utf-8")).hexdigest()
+        return prefix + md5(
+            f"{self.name} - {self.anchor}".encode("utf-8")
+        ).hexdigest()
 
     def _to_zip(self, fh: ZipFile):
         """
@@ -383,6 +393,7 @@ class Feature:
 
     @staticmethod
     def _encode_concept(concept: concept.AtlasConcept):
+        from ..locations import Location
         encoded_c = []
         if isinstance(concept, space.Space):
             encoded_c.append(f"s:{concept.id}")
@@ -393,6 +404,8 @@ class Feature:
             encoded_c.append(f"r:{concept.name}")
         elif isinstance(concept, volume.Volume):
             encoded_c.append(f"v:{concept.name}")
+        elif isinstance(concept, Location):
+            encoded_c.append(f"loc:{Location}")
 
         if len(encoded_c) == 0:
             raise EncodeLiveQueryIdException("no concept is encoded")
@@ -607,6 +620,7 @@ class Feature:
             def __init__(self, inst: Feature, fid: str):
                 self.inst = inst
                 self.fid = fid
+                self.category = inst.category
 
             def __str__(self) -> str:
                 return self.inst.__str__()
@@ -677,6 +691,24 @@ class Compoundable(ABC):
     def _merge_anchors(cls, anchors: List[_anchor.AnatomicalAnchor]):
         return sum(anchors)
 
+    @classmethod
+    @abstractmethod
+    def _merge_elements(
+        cls,
+        elements,
+        description: str,
+        modality: str,
+        anchor: _anchor.AnatomicalAnchor
+    ) -> Feature:
+        """
+        Compute the merge data and create a merged instance from a set of
+        elements of this class. This will be used by CompoundFeature to
+        create the aggegated data and plot it. For example, to compute an
+        average connectivity matrix from a set of subfeatures, we create a
+        RegionalConnectivty feature.
+        """
+        raise NotImplementedError
+
 
 class CompoundFeature(Feature):
     """
@@ -725,6 +757,7 @@ class CompoundFeature(Feature):
             datasets=list(dict.fromkeys([ds for f in elements for ds in f.datasets]))
         )
         self._queryconcept = queryconcept
+        self._merged_feature_cached = None
 
     def __getattr__(self, attr: str) -> Any:
         """Expose compounding attributes explicitly."""
@@ -743,9 +776,27 @@ class CompoundFeature(Feature):
         return super().__dir__() + list(self._compounding_attributes.keys())
 
     def plot(self, *args, **kwargs):
-        raise NotImplementedError(
-            "CompoundFeatures does not have a standardized plot. Try plotting the elements instead."
+        kwargs["title"] = "(Derived data: averaged)\n" + kwargs.get(
+            "title",
+            "\n".join(wrap(self.name, kwargs.pop("textwrap", 40)))
         )
+        return self._get_merged_feature().plot(*args, **kwargs)
+
+    def _get_merged_feature(self) -> Feature:
+        if self._merged_feature_cached is None:
+            logger.info(f"{self.__class__.__name__}.data averages the data of each element.")
+            assert issubclass(self.feature_type, Compoundable)
+            self._merged_feature_cached = self.feature_type._merge_elements(
+                elements=self.elements,
+                modality=self.modality,
+                description=self.description,
+                anchor=self.anchor
+            )
+        return self._merged_feature_cached
+
+    @property
+    def data(self):
+        return self._get_merged_feature().data
 
     @property
     def indexing_attributes(self) -> Tuple[str]:
@@ -770,14 +821,16 @@ class CompoundFeature(Feature):
     @property
     def name(self) -> str:
         """Returns a short human-readable name of this feature."""
+        readable_feature_type = sub(
+            "([b,B]ig [b,B]rain)", "BigBrain",
+            sub("([a-z])([A-Z])", r"\g<1> \g<2>", self.feature_type.__name__)
+        )
         groupby = ', '.join([
-            f"{v} {k}" for k, v in self._compounding_attributes.items()
+            f"{k}: {v}"
+            for k, v in self._compounding_attributes.items()
+            if k != 'modality'
         ])
-        return (
-            f"{self.__class__.__name__} of {len(self)} "
-            f"{self.feature_type.__name__} features grouped by ({groupby})"
-            f" anchored at {self.anchor}"
-        )
+        return f"{len(self)} {readable_feature_type} features{f' {groupby}' if groupby else ''}"
 
     @property
     def id(self) -> str:

siibra/features/image/image.py

@@ -63,13 +63,15 @@ class Image(feature.Feature, _volume.Volume):
         providers: List[provider.VolumeProvider],
         region: str = None,
         datasets: List = [],
+        id: str = None
     ):
         feature.Feature.__init__(
             self,
             modality=modality,
             description=None,  # lazy implementation below!
             anchor=None,  # lazy implementation below!
-            datasets=datasets
+            datasets=datasets,
+            id=id
         )
 
         _volume.Volume.__init__(

siibra/features/tabular/bigbrain_intensity_profile.py

@@ -30,7 +30,7 @@ class BigBrainIntensityProfile(
         "as described in the publication 'Wagstyl, K., et al (2020). BigBrain 3D atlas of "
         "cortical layers: Cortical and laminar thickness gradients diverge in sensory and "
         "motor cortices. PLoS Biology, 18(4), e3000678. "
-        "http://dx.doi.org/10.1371/journal.pbio.3000678'."
+        "http://dx.doi.org/10.1371/journal.pbio.3000678."
         "The data is taken from the tutorial at "
         "https://github.com/kwagstyl/cortical_layers_tutorial. Each vertex is "
         "assigned to the regional map when queried."

siibra/features/tabular/cell_density_profile.py

@@ -69,7 +69,8 @@ class CellDensityProfile(
         patch: int,
         url: str,
         anchor: _anchor.AnatomicalAnchor,
-        datasets: list = []
+        datasets: list = [],
+        id: str = None
     ):
         """
         Generate a cell density profile from a URL to a cloud folder
@@ -82,6 +83,7 @@ class CellDensityProfile(
             unit="cells / 0.1mm3",
             anchor=anchor,
             datasets=datasets,
+            id=id
         )
         self._step = 0.01
         self._url = url
@@ -234,7 +236,7 @@ class CellDensityProfile(
 
     @property
     def _depths(self):
-        return [d + self._step / 2 for d in np.arange(0, 1, self._step)]
+        return np.arange(0, 1, self._step) + self._step / 2
 
     @property
     def _values(self):
@@ -246,7 +248,7 @@ class CellDensityProfile(
                 densities.append(self.density_image[mask].mean())
             else:
                 densities.append(np.NaN)
-        return densities
+        return np.asanyarray(densities)
 
     @property
     def key(self):

siibra/features/tabular/cortical_profile.py

@@ -19,7 +19,7 @@ from ..feature import Compoundable
 from .. import anchor as _anchor
 
 import pandas as pd
-from typing import Union, Dict, Tuple
+from typing import Union, Dict, Tuple, List
 from textwrap import wrap
 import numpy as np
 
@@ -56,7 +56,8 @@ class CorticalProfile(tabular.Tabular, Compoundable):
         values: Union[list, np.ndarray] = None,
         unit: str = None,
         boundary_positions: Dict[Tuple[int, int], float] = None,
-        datasets: list = []
+        datasets: list = [],
+        id: str = None
     ):
         """Initialize profile.
 
@@ -96,21 +97,22 @@ class CorticalProfile(tabular.Tabular, Compoundable):
             description=description,
             anchor=anchor,
             data=None,  # lazy loader below
-            datasets=datasets
+            datasets=datasets,
+            id=id
         )
 
     def _check_sanity(self):
         # check plausibility of the profile
-        assert isinstance(self._depths, (list, np.ndarray))
-        assert isinstance(self._values, (list, np.ndarray))
-        assert len(self._values) == len(self._depths)
-        assert all(0 <= d <= 1 for d in self._depths)
+        assert isinstance(self._depths, (list, np.ndarray)), "Some depths are not valid"
+        assert isinstance(self._values, (list, np.ndarray)), "Some values are not valid"
+        assert len(self._values) == len(self._depths), "There exist uneven number of depths and values"
+        assert all(0 <= d <= 1 for d in self._depths), "Some depths is not between 0 and 1"
         if self.boundaries_mapped:
-            assert all(0 <= d <= 1 for d in self.boundary_positions.values())
+            assert all(0 <= d <= 1 for d in self.boundary_positions.values()), "Some boundary positions are not between 0 and 1"
             assert all(
                 layerpair in self.BOUNDARIES
                 for layerpair in self.boundary_positions.keys()
-            )
+            ), "Some value in BOUNDARIES are not mapped in boundary_positions"
 
     @property
     def unit(self) -> str:
@@ -159,8 +161,32 @@ class CorticalProfile(tabular.Tabular, Compoundable):
     def data(self):
         """Return a pandas Series representing the profile."""
         self._check_sanity()
-        return pd.DataFrame(
-            self._values, index=self._depths, columns=[f"{self.modality} ({self.unit})"]
+        iscompound = len(self._values.shape) > 1 and self._values.shape[1] == 2
+        if iscompound:
+            columns = [f"{self.modality} mean ({self.unit})", "std"]
+        else:
+            columns = [f"{self.modality} ({self.unit})"]
+        return pd.DataFrame(self._values, index=self._depths, columns=columns)
+
+    @classmethod
+    def _merge_elements(
+        cls,
+        elements: List["CorticalProfile"],
+        description: str,
+        modality: str,
+        anchor: _anchor.AnatomicalAnchor,
+    ):
+        assert all(np.array_equal(elements[0]._depths, f._depths) for f in elements)
+        assert len({f.unit for f in elements}) == 1
+        values_stacked = np.stack([f._values for f in elements])
+        return CorticalProfile(
+            description=description,
+            modality=modality,
+            anchor=anchor,
+            depths=np.stack([f._depths for f in elements]).mean(0),
+            values=np.stack([values_stacked.mean(0), values_stacked.std(0)]).T,
+            unit=elements[0].unit,
+            boundary_positions=None,
         )
 
     def plot(self, *args, backend="matplotlib", **kwargs):
@@ -180,12 +206,17 @@ class CorticalProfile(tabular.Tabular, Compoundable):
         kwargs["title"] = kwargs.get("title", "\n".join(wrap(self.name, wrapwidth)))
         layercolor = kwargs.pop("layercolor", "gray")
 
+        iscompound = len(self._values.shape) > 1 and self._values.shape[1] == 2
+        ymax = max(
+            0,
+            sum(self._values.max(axis=0)) if iscompound else self._values.max()
+        )
         if backend == "matplotlib":
             kwargs["xlabel"] = kwargs.get("xlabel", "Cortical depth")
             kwargs["ylabel"] = kwargs.get("ylabel", self.unit)
             kwargs["grid"] = kwargs.get("grid", True)
-            kwargs["ylim"] = kwargs.get("ylim", (0, max(self._values)))
-            axs = self.data.plot(*args, **kwargs, backend=backend)
+            axs = self.data.iloc[:, 0].plot(*args, **kwargs, backend=backend)
+            axs.set_ylim(kwargs.get("ylim", (0, ymax)))
 
             if self.boundaries_mapped:
                 bvals = list(self.boundary_positions.values())
@@ -201,14 +232,22 @@ class CorticalProfile(tabular.Tabular, Compoundable):
                         axs.axvspan(d1, d2, color=layercolor, alpha=0.3)
 
             axs.set_title(axs.get_title(), fontsize="medium")
+
+            if iscompound:
+                axs.set_ylabel(f"average {kwargs['ylabel']} \u00b1 std")
+                av = self.data.values[:, 0]
+                std = self.data.values[:, 1]
+                axs.fill_between(self.data.index.values, av - std, av + std, alpha=0.5)
+
             return axs
+
         elif backend == "plotly":
             kwargs["title"] = kwargs["title"].replace("\n", "<br>")
             kwargs["labels"] = {
                 "index": kwargs.pop("xlabel", None) or kwargs.pop("index", "Cortical depth"),
                 "value": kwargs.pop("ylabel", None) or kwargs.pop("value", self.unit)
             }
-            fig = self.data.plot(*args, **kwargs, backend=backend)
+            fig = self.data.iloc[:, 0].plot(*args, **kwargs, backend=backend)
             if self.boundaries_mapped:
                 bvals = list(self.boundary_positions.values())
                 for i, (d1, d2) in enumerate(list(zip(bvals[:-1], bvals[1:]))):
@@ -219,12 +258,29 @@ class CorticalProfile(tabular.Tabular, Compoundable):
                     )
             fig.update_layout(
                 showlegend=False,
-                yaxis_range=(0, max(self._values)),
+                yaxis_range=(0, ymax),
                 title=dict(
                     automargin=True, yref="container", xref="container",
                     pad=dict(t=40), xanchor="left", yanchor="top"
                 )
             )
+            if iscompound:
+                from plotly.graph_objects import Scatter
+                x = self.data.index.values
+                av = self.data.values[:, 0]
+                std = self.data.values[:, 1]
+                fig.update_layout(yaxis_title=f"average {kwargs['labels']['value']} &plusmn; std")
+                fig.add_traces(
+                    Scatter(
+                        x=np.concatenate((x, x[::-1])),  # x, then x reversed
+                        y=np.concatenate((av + std, (av - std)[::-1])),  # upper, then lower reversed
+                        fill='toself',
+                        fillcolor='rgba(0,100,80,0.5)',
+                        line=dict(color='rgba(255,255,255,0)'),
+                        hoverinfo="skip",
+                        showlegend=False
+                    )
+                )
             return fig
         else:
             return self.data.plot(*args, **kwargs, backend=backend)
@@ -254,3 +310,11 @@ class CorticalProfile(tabular.Tabular, Compoundable):
                 f"'_values' not available for {self.__class__.__name__}."
             )
         return self._values_cached
+
+    @property
+    def name(self):
+        if hasattr(self, "receptor"):
+            return super().name + f": {self.receptor}"
+        if hasattr(self, "location"):
+            return super().name + f": {self.location.coordinate}"
+        return super().name

siibra/features/tabular/gene_expression.py

@@ -15,6 +15,7 @@
 
 from .. import anchor as _anchor
 from . import tabular
+from ...retrieval.datasets import GenericDataset
 
 import pandas as pd
 from textwrap import wrap
@@ -46,6 +47,114 @@ class GeneExpressions(
     as specified at https://alleninstitute.org/legal/terms-use/.
     """
 
+    DATASET = GenericDataset(
+        name="An anatomically comprehensive atlas of the adult human brain transcriptome",
+        contributors=[
+            'Michael J. Hawrylycz',
+            'Ed S. Lein',
+            'Angela L. Guillozet-Bongaarts',
+            'Elaine H. Shen',
+            'Lydia Ng',
+            'Jeremy A. Miller',
+            'Louie N. van de Lagemaat',
+            'Kimberly A. Smith',
+            'Amanda Ebbert',
+            'Zackery L. Riley',
+            'Chris Abajian',
+            'Christian F. Beckmann',
+            'Amy Bernard',
+            'Darren Bertagnolli',
+            'Andrew F. Boe',
+            'Preston M. Cartagena',
+            'M. Mallar Chakravarty',
+            'Mike Chapin',
+            'Jimmy Chong',
+            'Rachel A. Dalley',
+            'Barry David Daly',
+            'Chinh Dang',
+            'Suvro Datta',
+            'Nick Dee',
+            'Tim A. Dolbeare',
+            'Vance Faber',
+            'David Feng',
+            'David R. Fowler',
+            'Jeff Goldy',
+            'Benjamin W. Gregor',
+            'Zeb Haradon',
+            'David R. Haynor',
+            'John G. Hohmann',
+            'Steve Horvath',
+            'Robert E. Howard',
+            'Andreas Jeromin',
+            'Jayson M. Jochim',
+            'Marty Kinnunen',
+            'Christopher Lau',
+            'Evan T. Lazarz',
+            'Changkyu Lee',
+            'Tracy A. Lemon',
+            'Ling Li',
+            'Yang Li',
+            'John A. Morris',
+            'Caroline C. Overly',
+            'Patrick D. Parker',
+            'Sheana E. Parry',
+            'Melissa Reding',
+            'Joshua J. Royall',
+            'Jay Schulkin',
+            'Pedro Adolfo Sequeira',
+            'Clifford R. Slaughterbeck',
+            'Simon C. Smith',
+            'Andy J. Sodt',
+            'Susan M. Sunkin',
+            'Beryl E. Swanson',
+            'Marquis P. Vawter',
+            'Derric Williams',
+            'Paul Wohnoutka',
+            'H. Ronald Zielke',
+            'Daniel H. Geschwind',
+            'Patrick R. Hof',
+            'Stephen M. Smith',
+            'Christof Koch',
+            'Seth G. N. Grant',
+            'Allan R. Jones'
+        ],
+        url="https://doi.org/10.1038%2Fnature11405",
+        description='Neuroanatomically precise, genome-wide maps of transcript '
+            'distributions are critical resources to complement genomic '
+            'sequence data and to correlate functional and genetic brain '
+            'architecture. Here we describe the generation and analysis '
+            'of a transcriptional atlas of the adult human brain, '
+            'comprising extensive histological analysis and comprehensive '
+            'microarray profiling of ~900 neuroanatomically precise '
+            'subdivisions in two individuals. Transcriptional regulation '
+            'varies enormously by anatomical location, with different '
+            'regions and their constituent cell types displaying robust '
+            'molecular signatures that are highly conserved between '
+            'individuals. Analysis of differential gene expression and '
+            'gene co-expression relationships demonstrates that brain-'
+            'wide variation strongly reflects the distributions of major '
+            'cell classes such as neurons, oligodendrocytes, astrocytes '
+            'and microglia. Local neighbourhood relationships between '
+            'fine anatomical subdivisions are associated with discrete '
+            'neuronal subtypes and genes involved with synaptic '
+            'transmission. The neocortex displays a relatively '
+            'homogeneous transcriptional pattern, but with distinct '
+            'features associated selectively with primary sensorimotor '
+            'cortices and with enriched frontal lobe expression. Notably, '
+            'the spatial topography of the neocortex is strongly '
+            'reflected in its molecular topography— the closer two '
+            'cortical regions, the more similar their transcriptomes. '
+            'This freely accessible online data resource forms a high-'
+            'resolution transcriptional baseline for neurogenetic studies '
+            'of normal and abnormal human brain function.'
+            ""
+            "For retrieving microarray data, siibra connects to the web API of "
+            "the Allen Brain Atlas (© 2015 Allen Institute for Brain Science), "
+            "available from https://brain-map.org/api/index.html. Any use of the "
+            "microarray data needs to be in accordance with their terms of use, "
+            "as specified at https://alleninstitute.org/legal/terms-use/."
+    )
+
     class _DonorDict(TypedDict):
         id: int
         name: str
@@ -66,7 +175,7 @@ class GeneExpressions(
         genes: List[str],
         additional_columns: dict,
         anchor: _anchor.AnatomicalAnchor,
-        datasets: List = []
+        datasets: List = [DATASET]
     ):
         """
         Construct gene expression table.
@@ -130,7 +239,7 @@ class GeneExpressions(
         """
         wrapwidth = kwargs.pop("textwrap") if "textwrap" in kwargs else 40
         kwargs["title"] = kwargs.pop("title", None) \
-            or "\n".join(wrap(f"{self.modality} measured in {self.anchor._regionspec}", wrapwidth))
+            or "\n".join(wrap(f"{self.modality} measured in {self.anchor._regionspec or self.anchor.location}", wrapwidth))
         kwargs["kind"] = "box"
         if backend == "matplotlib":
             for arg in ['yerr', 'y', 'ylabel', 'xlabel', 'width']:

siibra/features/tabular/layerwise_bigbrain_intensities.py

@@ -35,7 +35,7 @@ class LayerwiseBigBrainIntensities(
         "'Wagstyl, K., et al (2020). BigBrain 3D atlas of "
         "cortical layers: Cortical and laminar thickness gradients diverge in sensory and "
         "motor cortices. PLoS Biology, 18(4), e3000678. "
-        "http://dx.doi.org/10.1371/journal.pbio.3000678'."
+        "http://dx.doi.org/10.1371/journal.pbio.3000678."
         "The data is taken from the tutorial at "
         "https://github.com/kwagstyl/cortical_layers_tutorial. Each vertex is "
         "assigned to the regional map when queried."

siibra/features/tabular/layerwise_cell_density.py

@@ -56,6 +56,7 @@ class LayerwiseCellDensity(
         layerfiles: list,
         anchor: _anchor.AnatomicalAnchor,
         datasets: list = [],
+        id: str = None
     ):
         tabular.Tabular.__init__(
             self,
@@ -63,7 +64,8 @@ class LayerwiseCellDensity(
             modality="Cell body density",
             anchor=anchor,
             datasets=datasets,
-            data=None  # lazy loading below
+            data=None,  # lazy loading below
+            id=id
         )
         self.unit = "# detected cells/0.1mm3"
         self._filepairs = list(zip(segmentfiles, layerfiles))

siibra/features/tabular/receptor_density_fingerprint.py

@@ -42,7 +42,8 @@ class ReceptorDensityFingerprint(
         self,
         tsvfile: str,
         anchor: _anchor.AnatomicalAnchor,
-        datasets: list = []
+        datasets: list = [],
+        id: str = None
     ):
         """ Generate a receptor fingerprint from a URL to a .tsv file
         formatted according to the structure used by Palomero-Gallagher et al.
@@ -54,6 +55,7 @@ class ReceptorDensityFingerprint(
             anchor=anchor,
             data=None,  # lazy loading below
             datasets=datasets,
+            id=id
         )
         self._loader = requests.HttpRequest(tsvfile)
 

siibra/features/tabular/receptor_density_profile.py

@@ -41,7 +41,8 @@ class ReceptorDensityProfile(
         receptor: str,
         tsvfile: str,
         anchor: _anchor.AnatomicalAnchor,
-        datasets: list = []
+        datasets: list = [],
+        id: str = None
     ):
         """Generate a receptor density profile from a URL to a .tsv file
         formatted according to the structure used by Palomero-Gallagher et al.
@@ -52,6 +53,7 @@ class ReceptorDensityProfile(
             modality="Receptor density",
             anchor=anchor,
             datasets=datasets,
+            id=id
         )
         self.receptor = receptor
         self._data_cached = None
@@ -72,10 +74,6 @@ class ReceptorDensityProfile(
     def receptor_fullname(self):
         return vocabularies.RECEPTOR_SYMBOLS[self.receptor]['receptor']['name']
 
-    @property
-    def name(self):
-        return super().name + f" for {self.receptor}"
-
     @property
     def neurotransmitter(self):
         return "{} ({})".format(