frametree 0.11.0__py3-none-any.whl

frametree/common/__init__.py

@@ -0,0 +1,2 @@
+from .dirtree import DirTree
+from .space import Clinical, Samples

frametree/common/dirtree.py

@@ -0,0 +1,447 @@
+from __future__ import annotations
+import os
+from pathlib import Path
+import re
+import typing as ty
+import logging
+import json
+import attrs
+from fileformats.core import FileSet, Field
+from frametree.core.exceptions import FrameTreeUsageError
+from frametree.core.set.base import DataTree
+from frametree.core.row import DataRow
+from frametree.core.entry import DataEntry
+from frametree.core.store import LocalStore
+from frametree.core.utils import full_path
+
+
+logger = logging.getLogger("frametree")
+
+
+# Matches directory names used for summary rows with dunder beginning and
+# end (e.g. '__visit_01__') and hidden directories (i.e. starting with '.' or
+# '~')
+special_dir_re = re.compile(r"(__.*__$|\..*|~.*)")
+
+
+@attrs.define
+class DirTree(LocalStore):
+    """
+    A Repository class for data stored hierarchically within sub-directories
+    of a file-system directory. The depth and which layer in the data tree
+    the sub-directories correspond to is defined by the `hierarchy` argument.
+
+    Parameters
+    ----------
+    base_dir : str
+        Path to the base directory of the "store", i.e. datasets are
+        arranged by name as sub-directories of the base dir.
+
+    """
+
+    PROV_SUFFIX = ".provenance"
+    FIELDS_FNAME = "__fields__"
+    FIELDS_PROV_FNAME = "__fields_provenance__"
+
+    # Note this name will be constant, as there is only ever one store,
+    # which covers whole FS
+    name: str = "dirtree"
+
+    #################################
+    # Abstract-method implementations
+    #################################
+
+    def populate_tree(self, tree: DataTree):
+        """
+        Scans the data present in the dataset and populates the data tree with nodes
+
+        Parameters
+        ----------
+        dataset : Dataset
+            The dataset to construct the tree dimensions for
+        """
+        if not os.path.exists(tree.dataset_id):
+            raise FrameTreeUsageError(
+                f"Could not find a directory at '{tree.dataset_id}' to be the "
+                "root row of the dataset"
+            )
+        dpaths = sorted(d for d, _, _ in os.walk(tree.dataset_id))
+        for dpath in dpaths:
+            tree_path = tuple(Path(dpath).relative_to(tree.dataset_id).parts)
+            if len(tree_path) != len(tree.hierarchy):
+                continue
+            if self.ARCANA_DIR in tree_path:
+                continue
+            tree.add_leaf(tree_path)
+
+    def populate_row(self, row: DataRow):
+        """Scans the node in the data tree corresponding to the data row and populates
+        the row with data entries found in the tree node
+
+        Parameters
+        ----------
+        row : DataRow
+            the data row to populate
+        """
+
+        def filter_entry_dir(entry_dir):
+            for subpath in entry_dir.iterdir():
+                entry_name = subpath.name
+                if (
+                    not entry_name.startswith(".")
+                    and entry_name != self.ARCANA_DIR
+                    and entry_name
+                    not in (
+                        self.FIELDS_FNAME,
+                        self.FIELDS_PROV_FNAME,
+                        self.FIELDS_FNAME + self.LOCK_SUFFIX,
+                    )
+                    and not entry_name.endswith(self.PROV_SUFFIX)
+                ):
+                    yield subpath
+
+        root_dir = full_path(row.dataset.id)
+
+        # Iterate through all directories saved for the source and dataset derivatives
+        for dataset_name in self._row_dataset_names(row):
+            row_dir = root_dir / self._row_relpath(row, dataset_name=dataset_name)
+            if row_dir.exists():
+                # Filter contents of directory to omit fields JSON and provenance and
+                # add file-set entries
+                for entry_fspath in filter_entry_dir(row_dir):
+                    path = str(entry_fspath.relative_to(row_dir))
+                    if dataset_name is not None:
+                        path += "@" + dataset_name
+                    row.add_entry(
+                        path=path,
+                        datatype=FileSet,
+                        uri=str(entry_fspath.relative_to(root_dir)),
+                    )
+                # Add field entries
+                fields_json = row_dir / self.FIELDS_FNAME
+                try:
+                    with open(fields_json) as f:
+                        fields_dict = json.load(f)
+                except FileNotFoundError:
+                    pass
+                else:
+                    for name in fields_dict:
+                        path = (
+                            f"{name}@{dataset_name}"
+                            if dataset_name is not None
+                            else name
+                        )
+                        row.add_entry(
+                            path=path,
+                            datatype=Field,
+                            uri=str(fields_json.relative_to(root_dir)) + "::" + name,
+                        )
+
+    def get_field(self, entry: DataEntry, datatype: type) -> Field:
+        """Retrieve the field associated with the given entry and return it cast
+        to the specified datatype
+
+        Parameters
+        ----------
+        entry : DataEntry
+            the entry to retrieve the field for
+        datatype : type (subclass DataType)
+            the datatype to return the field as
+
+        Returns
+        -------
+        Field
+            the retrieved field
+        """
+        fspath, key = self._fields_fspath_and_key(entry)
+        return datatype(self.read_from_json(fspath, key))
+
+    def get_fileset(self, entry: DataEntry, datatype: type) -> FileSet:
+        """Retrieve the file-set associated with the given entry and return it cast
+        to the specified datatype
+
+        Parameters
+        ----------
+        entry : DataEntry
+            the entry to retrieve the file-set for
+        datatype : type (subclass DataType)
+            the datatype to return the file-set as
+
+        Returns
+        -------
+        FileSet
+            the retrieved file-set
+        """
+        return datatype(self._fileset_fspath(entry))
+
+    def put_fileset(self, fileset: FileSet, entry: DataEntry) -> FileSet:
+        """Put a file-set into the specified data entry
+
+        Parameters
+        ----------
+        fileset : FileSet
+            the file-set to store
+        entry : DataEntry
+            the entry to store the file-set in
+
+        Returns
+        -------
+        FileSet
+            the copy of the file-set that has been stored within the data entry
+        """
+        fspath = self._fileset_fspath(entry)
+        _, new_stem, new_ext = FileSet.decompose_fspath(entry.path.split("@")[0])
+        if new_ext:
+            if len(fileset.fspaths) > 1:
+                raise FrameTreeUsageError(
+                    "Cannot store file-set with multiple files in dirtree store "
+                    "when extension is specified"
+                )
+            if new_ext != FileSet.decompose_fspath(fileset.fspath)[2]:
+                raise FrameTreeUsageError(
+                    "Cannot change extension of file-set when copying to dirtree store"
+                )
+        # Create target directory if it doesn't exist already
+        copied_fileset = fileset.copy(
+            dest_dir=fspath.parent,
+            collation=fileset.CopyCollation.adjacent,
+            new_stem=new_stem,
+            make_dirs=True,
+            overwrite=True,
+        )
+        return copied_fileset
+
+    def put_field(self, field: Field, entry: DataEntry):
+        """Put a field into the specified data entry
+
+        Parameters
+        ----------
+        field : Field
+            the field to store
+        entry : DataEntry
+            the entry to store the field in
+        """
+        fspath, key = self._fields_fspath_and_key(entry)
+        self.update_json(fspath, key, field.primitive(field))
+
+    def get_fileset_provenance(
+        self, entry: DataEntry
+    ) -> ty.Union[ty.Dict[str, ty.Any], None]:
+        """Retrieves provenance associated with a file-set data entry
+
+        Parameters
+        ----------
+        entry : DataEntry
+            the entry of the file-set to retrieve the provenance for
+
+        Returns
+        -------
+        ty.Dict[str, ty.Any] or None
+            the retrieved provenance or None if it doesn't exist
+        """
+        with open(self._fileset_prov_fspath(entry)) as f:
+            provenance = json.load(f)
+        return provenance
+
+    def put_fileset_provenance(
+        self, provenance: ty.Dict[str, ty.Any], entry: DataEntry
+    ):
+        """Puts provenance associated with a file-set data entry into the store
+
+        Parameters
+        ----------
+        provenance : dict[str, ty.Any]
+            the provenance to store
+        entry : DataEntry
+            the entry to associate the provenance with
+        """
+        with open(self._fileset_prov_fspath(entry), "w") as f:
+            json.dump(provenance, f)
+
+    def get_field_provenance(
+        self, entry: DataEntry
+    ) -> ty.Union[ty.Dict[str, ty.Any], None]:
+        """Retrieves provenance associated with a field data entry
+
+        Parameters
+        ----------
+        entry : DataEntry
+            the entry of the field to retrieve the provenance for
+
+        Returns
+        -------
+        ty.Dict[str, ty.Any] or None
+            the retrieved provenance or None if it doesn't exist
+        """
+        fspath, key = self._fields_prov_fspath_and_key(entry)
+        with open(fspath) as f:
+            fields_provenance = json.load(f)
+        return fields_provenance[key]
+
+    def put_field_provenance(self, provenance: ty.Dict[str, ty.Any], entry: DataEntry):
+        """Puts provenance associated with a field data entry into the store
+
+        Parameters
+        ----------
+        provenance : dict[str, ty.Any]
+            the provenance to store
+        entry : DataEntry
+            the entry to associate the provenance with
+        """
+        fspath, key = self._fields_prov_fspath_and_key(entry)
+        self.update_json(fspath, key, provenance)
+
+    def fileset_uri(self, path: str, datatype: type, row: DataRow) -> str:
+        """Returns the "uri" (e.g. file-system path relative to root dir) of a file-set
+        entry at the given path relative to the given row
+
+        Parameters
+        ----------
+        path : str
+            path to the entry relative to the row
+        datatype : type
+            the datatype of the entry
+        row : DataRow
+            the row of the entry
+
+        Returns
+        -------
+        uri : str
+            the "uri" to the file-set entry relative to the data store
+        """
+        path, dataset_name = DataEntry.split_dataset_name_from_path(path)
+        row_dir = self._row_relpath(row, dataset_name=dataset_name)
+        return str(row_dir.joinpath(*path.split("/"))) + datatype.strext
+
+    def field_uri(self, path: str, datatype: type, row: DataRow) -> str:
+        """Returns the "uri" (e.g. file-system path relative to root dir) of a field
+        entry at the given path relative to the given row
+
+        Parameters
+        ----------
+        path : str
+            path to the entry relative to the row
+        datatype : type
+            the datatype of the entry
+        row : DataRow
+            the row of the entry
+
+        Returns
+        -------
+        uri : str
+            the "uri" to the field entry relative to the data store
+        """
+        path, dataset_name = DataEntry.split_dataset_name_from_path(path)
+        row_dir = self._row_relpath(row, dataset_name=dataset_name)
+        return str(row_dir / self.FIELDS_FNAME) + "::" + path
+
+    def create_data_tree(self, id: str, leaves: ty.List[ty.Tuple[str, ...]], **kwargs):
+        """creates a new empty dataset within in the store. Used in test routines and
+        importing/exporting datasets between stores
+
+        Parameters
+        ----------
+        id : str
+            ID for the newly created dataset
+        leaves : list[tuple[str, ...]]
+                        list of IDs for each leaf node to be added to the dataset. The IDs for each
+            leaf should be a tuple with an ID for each level in the tree's hierarchy, e.g.
+            for a hierarchy of [subject, timepoint] ->
+            [("SUBJ01", "TIMEPOINT01"), ("SUBJ01", "TIMEPOINT02"), ....]
+        hierarchy: ty.List[str]
+            the hierarchy of the dataset to be created
+        space : type(DataSpace)
+            the data space of the dataset
+        """
+        root_dir = Path(id)
+        root_dir.mkdir(parents=True)
+        # Create sub-directories corresponding to rows of the dataset
+        for ids_tuple in leaves:
+            root_dir.joinpath(*ids_tuple).mkdir(parents=True)
+
+    ##################
+    # Helper functions
+    ##################
+
+    def _row_relpath(self, row: DataRow, dataset_name=None):
+        """Get the file-system path to the dataset root for the given row, taking into
+        account non-leaf rows
+
+        Parameters
+        ----------
+        row : DataRow
+            the row to get the relative path for
+        derivatives : bool
+            whether to return the directory containing derivatives or originals
+
+        Returns
+        -------
+        relpath : Path
+            the relative path to the row directory
+        """
+        relpath = Path()
+        if row.frequency is max(row.dataset.space):  # leaf node
+            for freq in row.dataset.hierarchy:
+                relpath /= row.frequency_id(freq)
+            if dataset_name is not None:
+                relpath /= self.ARCANA_DIR
+                if dataset_name:
+                    relpath /= dataset_name
+                else:
+                    relpath /= self.EMPTY_DATASET_NAME
+        else:
+            relpath = relpath.joinpath(
+                self.ARCANA_DIR,
+                str(row.frequency),
+            )
+            if isinstance(row.id, tuple):
+                relpath /= ".".join(row.id)
+            elif row.id:
+                relpath /= row.id
+            if dataset_name is None:
+                relpath /= self.ARCANA_DIR
+            elif not dataset_name:
+                relpath /= self.EMPTY_DATASET_NAME
+            else:
+                relpath /= dataset_name
+        return relpath
+
+    def _row_dataset_names(self, row: DataRow):
+        """list all dataset names stored in the given row
+
+        Parameters
+        ----------
+        row : DataRow
+            row to return the dataset names for
+
+        Returns
+        -------
+        dataset_names : list[str]
+            list of dataset names stored in the given row
+        """
+        dataset_names = [None]  # The source data
+        derivs_dir = (
+            Path(row.dataset.id) / self._row_relpath(row, dataset_name="").parent
+        )
+        if derivs_dir.exists():
+            dataset_names.extend(
+                ("" if d.name == self.EMPTY_DATASET_NAME else d.name)
+                for d in derivs_dir.iterdir()
+                if d.name != self.ARCANA_DIR
+            )
+        return dataset_names
+
+    def _fileset_fspath(self, entry):
+        return Path(entry.row.dataset.id) / entry.uri
+
+    def _fields_fspath_and_key(self, entry):
+        relpath, key = entry.uri.split("::")
+        fspath = Path(entry.row.dataset.id) / relpath
+        return fspath, key
+
+    def _fileset_prov_fspath(self, entry):
+        return self._fileset_fspath(entry).with_suffix(self.PROV_SUFFIX)
+
+    def _fields_prov_fspath_and_key(self, entry):
+        fields_fspath, key = self._fields_fspath_and_key(entry)
+        return fields_fspath.parent / self.FIELDS_PROV_FNAME, key

frametree/common/space.py

@@ -0,0 +1,42 @@
+from frametree.core.space import DataSpace
+
+
+class Clinical(DataSpace):
+    """
+    An enum used to specify the hierarchy of data trees and the "frequencies" of items
+    within dataset typical of medimage research, i.e. subjects split into groups scanned
+    at different timepoints (in longitudinal studies).
+    """
+
+    # Root row of the dataset
+    dataset = 0b000  # singular within the dataset
+
+    # Axes of the data space
+    member = 0b001  # subjects relative to their group membership, i.e.
+    # matched pairs of test and control subjects should share
+    # the same member IDs.
+    group = 0b010  # subject groups (e.g. test & control)
+    timepoint = 0b100  # timepoints in longitudinal studies
+
+    # Combinations
+    session = 0b111  # a single session (i.e. a single timepoint of a subject)
+    subject = 0b011  # uniquely identified subject within in the dataset.
+    # As opposed to 'member', which specifies a subject in
+    # relation to its group (i.e. one subject for each member
+    # in each group). For datasets with only one study group,
+    # then subject and member are equivalent
+    batch = 0b110  # data from separate groups at separate timepoints
+    matchedpoint = 0b101  # matched members (e.g. test & control) across
+    # all groups and timepoints
+
+
+class Samples(DataSpace):
+    """
+    The most basic data space within only one dimension
+    """
+
+    # Root row of the dataset
+    dataset = 0b0  # singular within the dataset
+
+    # Axes of the data space
+    sample = 0b1

frametree/core/__init__.py

@@ -0,0 +1,6 @@
+from ._version import __version__  # noqa: F401
+
+PACKAGE_NAME = "frametree"
+CODE_URL = f"https://github.com/ArcanaFramework/{PACKAGE_NAME}"
+
+__authors__ = [("Thomas G. Close", "tom.g.close@gmail.com")]

frametree/core/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.11.0'
+__version_tuple__ = version_tuple = (0, 11, 0)

frametree/core/cell.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+import typing as ty
+import attrs
+from fileformats.core import DataType
+from frametree.core.exceptions import FrameTreeError
+
+if ty.TYPE_CHECKING:  # pragma: no cover
+    from .row import DataRow
+    from .column import DataColumn
+    from .entry import DataEntry
+
+
+@attrs.define
+class DataCell:
+    """
+    An abstract container representing the intersection between a row and a column,
+    which points to an item within a dataset.
+
+    Parameters
+    ----------
+    row : DataRow
+        the row the cell belongs to
+    column : DataColumn
+        the column the cell belongs to
+    entry : DataEntry or None
+        the entry in the row that matches the column
+    provenance : Provenance | None
+        The recorded provenance for the item stored within the data cell,
+        if applicable
+    """
+
+    row: DataRow
+    column: DataColumn
+    entry: DataEntry
+    provenance: ty.Dict[str, ty.Any] = None
+
+    @property
+    def datatype(self):
+        return self.column.datatype
+
+    @property
+    def is_empty(self):
+        return self.entry is None
+
+    @property
+    def item(self) -> DataType:
+        if self.is_empty:
+            raise FrameTreeError(f"Cannot access item of empty cell, {self}")
+        return self.entry.get_item(self.column.datatype)
+
+    @item.setter
+    def item(self, item: DataType):
+        if not self.column.is_sink:
+            raise FrameTreeError(
+                f"Cannot set data items ({item}) into source column cell {self}"
+            )
+        item = self.datatype(item)
+        if self.is_empty:
+            entry = self.row.dataset.store.post(
+                item=item, path=self.column.path, datatype=self.datatype, row=self.row
+            )
+            self.entry = entry
+        else:
+            self.entry.item = item
+
+    @classmethod
+    def intersection(
+        cls, column: DataColumn, row: DataRow, allow_empty: ty.Optional[bool] = None
+    ) -> DataCell:
+        if allow_empty is None:
+            allow_empty = column.is_sink
+        return cls(
+            row=row,
+            column=column,
+            entry=column.match_entry(row, allow_none=allow_empty),
+        )

frametree/core/cli/__init__.py

@@ -0,0 +1,6 @@
+from .base import cli
+from .ext import ext
+from .store import store
+from .dataset import dataset
+from .apply import apply
+from .derive import derive