modacor 1.0.0__py3-none-any.whl

modacor/io/csv/csv_source.py

@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Brian R. Pauw"]
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "12/12/2025"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+__all__ = ["CSVSource"]
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from attrs import define, field, validators
+
+from modacor.dataclasses.messagehandler import MessageHandler
+from modacor.io.io_source import ArraySlice
+
+from ..io_source import IoSource
+
+
+def _is_callable(_, __, value):
+    if not callable(value):
+        raise TypeError("method must be callable")
+
+
+@define(kw_only=True)
+class CSVSource(IoSource):
+    """
+    IoSource for loading columnar data from CSV-like text files using NumPy's
+    loadtxt or genfromtxt.
+
+    Expected usage
+    --------------
+    - Data is 1D per column (no multi-dimensional fields).
+    - Columns are returned as 1D arrays; each column corresponds to one data_key.
+    - for np.loadtxt, column names must be provided via dtype with field names, e.g.:
+        dtype=[("q", float), ("I", float), ("I_sigma", float)]
+    - for np.genfromtxt, column names come from the first row or are specified explicitly via the `names` parameter. Typical patterns:
+        * np.genfromtxt(..., names=True, delimiter=..., ...)  # use first row as names
+        * np.genfromtxt(..., names=["q", "I", "I_sigma"], ...)  # specify names explicitly
+      so that they can be clearly identified later.
+
+    Configuration
+    -------------
+    `iosource_method_kwargs` is passed directly to the NumPy function `method`.
+    This allows you to use all standard NumPy options, e.g.:
+
+    For np.genfromtxt:
+        delimiter=","
+        skip_header=3
+        max_rows=1000
+        usecols=(0, 1, 2)
+        names=True or names=["q", "I", "sigma"]
+        dtype=None or dtype=float
+        encoding="utf-8"
+        comments="#"
+        ...
+
+    For np.loadtxt:
+        delimiter=","
+        skiprows=3
+        max_rows=1000
+        usecols=(0, 1, 2)
+        dtype=float
+        encoding="utf-8"
+        comments="#"
+        ...
+
+    Notes
+    -----
+    - 2D arrays (no field names) are not supported in this implementation.
+      If the resulting array does not have `dtype.names`, a ValueError is raised.
+    """
+
+    # external API:
+    resource_location: Path = field(converter=Path, validator=validators.instance_of((Path)))
+    method: Callable[..., np.ndarray] = field(
+        default=np.genfromtxt, validator=_is_callable
+    )  # default to genfromtxt, better for names
+    # internal use (type hints; real values set per-instance)
+    _data_cache: np.ndarray | None = field(init=False, default=None)
+    _data_dict_cache: dict[str, np.ndarray] = field(factory=dict)
+    _file_datasets_dtypes: dict[str, np.dtype] = field(init=False)
+    _file_datasets_shapes: dict[str, tuple[int, ...]] = field(init=False)
+    logger: MessageHandler = field(init=False)
+
+    def __attrs_post_init__(self) -> None:
+        # super().__init__(source_reference=self.source_reference, iosource_method_kwargs=self.iosource_method_kwargs)
+        self.logger = MessageHandler(level=self.logging_level, name="CSVSource")
+        # Set file path
+        if not self.resource_location.is_file():
+            self.logger.error(f"CSVSource: file {self.resource_location} does not exist.")
+
+        # Bookkeeping structures for IoSource API
+        self._file_datasets_shapes: dict[str, tuple[int, ...]] = {}
+        self._file_datasets_dtypes: dict[str, np.dtype] = {}
+
+        # Load and preprocess data immediately
+        self._load_data()
+        self._preload()
+
+    # ------------------------------------------------------------------ #
+    # Internal loading / preprocessing                                    #
+    # ------------------------------------------------------------------ #
+
+    def _load_data(self) -> None:
+        """
+        Load the CSV data into a structured NumPy array using the configured
+        method (np.genfromtxt or np.loadtxt).
+
+        iosource_method_kwargs are passed directly to that method.
+        """
+        self.logger.info(
+            f"CSVSource loading data from {self.resource_location} "
+            f"using {self.method.__name__} with options: {self.iosource_method_kwargs}"
+        )
+
+        try:
+            self._data_cache = self.method(self.resource_location, **self.iosource_method_kwargs)
+        except Exception as exc:  # noqa: BLE001
+            self.logger.error(f"Error while loading CSV data from {self.resource_location}: {exc}")
+            raise
+
+        if self._data_cache is None:
+            raise ValueError(f"CSVSource: no data loaded from file {self.resource_location}.")
+        # Ensure we have a structured array with named fields
+        if self._data_cache.dtype.names is None:
+            raise ValueError(
+                "CSVSource expected a structured array with named fields, "
+                "but dtype.names is None.\n"
+                "Hint: use np.genfromtxt with 'names=True' or 'names=[...]', "
+                "or provide an appropriate 'dtype' with field names."
+            )
+
+    def _preload(self) -> None:
+        """
+        Populate dataset lists, shapes, and dtypes from the structured array.
+        """
+        assert self._data_cache is not None  # for type checkers
+
+        self._data_dict_cache = {}
+        self._file_datasets_shapes.clear()
+        self._file_datasets_dtypes.clear()
+
+        for name in self._data_cache.dtype.names:
+            column = self._data_cache[name]
+            self._data_dict_cache[name] = self._data_cache[name]
+            self._file_datasets_shapes[name] = column.shape
+            self._file_datasets_dtypes[name] = column.dtype
+
+        self.logger.info(f"CSVSource loaded datasets: {self._file_datasets_shapes.keys()}")
+
+    # ------------------------------------------------------------------ #
+    # IoSource API                                                       #
+    # ------------------------------------------------------------------ #
+
+    def get_static_metadata(self, data_key: str) -> None:
+        """
+        CSVSource does not support static metadata; always returns None.
+        """
+        self.logger.warning(
+            f"You asked for static metadata '{data_key}', but CSVSource does not support static metadata."
+        )
+        return None
+
+    def get_data(self, data_key: str, load_slice: ArraySlice = ...) -> np.ndarray:
+        """
+        Return the data column corresponding to `data_key`, cast to float, apply `load_slice`.
+
+        - data_key must match one of the field names in the structured array.
+        - `load_slice` is applied to that 1D column (e.g. ellipsis, slice, array of indices).
+        """
+        if self._data_cache is None:
+            raise RuntimeError("CSVSource data cache is empty; loading may have failed.")
+
+        try:
+            column = self._data_dict_cache[data_key]
+        except KeyError:
+            raise KeyError(
+                f"Data key '{data_key}' not found in CSV data. Available keys: {list(self._data_dict_cache.keys())}"  # noqa: E713
+            ) from None
+
+        return np.asarray(column[load_slice]).astype(float)
+
+    def get_data_shape(self, data_key: str) -> tuple[int, ...]:
+        if data_key in self._file_datasets_shapes:
+            return self._file_datasets_shapes[data_key]
+        return ()
+
+    def get_data_dtype(self, data_key: str) -> np.dtype | None:
+        if data_key in self._file_datasets_dtypes:
+            return self._file_datasets_dtypes[data_key]
+        return None
+
+    def get_data_attributes(self, data_key: str) -> dict[str, Any]:
+        """
+        CSV has no per-dataset attributes; return a dict with None.
+        """
+        self.logger.warning(
+            f"You asked for attributes of '{data_key}', but CSVSource does not support data attributes."
+        )
+        return {data_key: None}

modacor/io/hdf/__init__.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2025 MoDaCor Authors
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without
+#    specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+__license__ = "BSD-3-Clause"
+__copyright__ = "Copyright 2025 MoDaCor Authors"
+__status__ = "Alpha"

modacor/io/hdf/hdf_source.py

@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from typing import Any
+
+__coding__ = "utf-8"
+__authors__ = ["Tim Snow", "Brian R. Pauw"]
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "22/10/2025"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+__all__ = ["HDFSource"]
+
+from pathlib import Path
+
+import h5py
+import numpy as np
+from attrs import define, field, validators
+
+from modacor.dataclasses.messagehandler import MessageHandler
+
+# from modacor.dataclasses.basedata import BaseData
+from modacor.io.io_source import ArraySlice
+
+from ..io_source import IoSource
+
+
+@define(kw_only=True)
+class HDFSource(IoSource):
+    resource_location: Path | str | None = field(
+        init=True, default=None, validator=validators.optional(validators.instance_of((Path, str)))
+    )
+    _data_cache: dict[str, np.ndarray] = field(init=False, factory=dict, validator=validators.instance_of(dict))
+    _file_path: Path | None = field(
+        init=False, default=None, validator=validators.optional(validators.instance_of(Path))
+    )
+    _file_datasets_shapes: dict[str, tuple[int, ...]] = field(
+        init=False, factory=dict, validator=validators.instance_of(dict)
+    )
+    _file_datasets_dtypes: dict[str, np.dtype] = field(init=False, factory=dict, validator=validators.instance_of(dict))
+    _static_metadata_cache: dict[str, Any] = field(init=False, factory=dict, validator=validators.instance_of(dict))
+    logger: MessageHandler = field(init=False)
+
+    # source_reference comes from IoSource
+    # iosource_method_kwargs comes from IoSource
+
+    def __attrs_post_init__(self):
+        # super().__init__(source_reference=source_reference)
+        self.logger = MessageHandler(level=self.logging_level, name="HDFSource")
+        self._file_path = Path(self.resource_location) if self.resource_location is not None else None
+        # self._file_datasets = []
+        self._file_datasets_shapes = {}
+        self._file_datasets_dtypes = {}
+        self._data_cache = {}
+        self._static_metadata_cache = {}
+        self._preload()  # load the HDF5 file structure immediately so we have some information, but not the data
+
+    def _preload(self):
+        assert self._file_path.is_file(), self.logger.error(f"HDF5 file {self._file_path} does not exist.")
+        try:
+            with h5py.File(self._file_path, "r") as f:
+                f.visititems(self._find_datasets)
+        except OSError as error:
+            self.logger.log.error(error)
+            raise OSError(error)
+
+    def _find_datasets(self, path_name, path_object):
+        """
+        An internal function to be used to walk the tree of an HDF5 file and return a list of
+        the datasets within
+        """
+        if isinstance(path_object, h5py._hl.dataset.Dataset):
+            # self._file_datasets.append(path_name)
+            self._file_datasets_shapes[path_name] = path_object.shape
+            self._file_datasets_dtypes[path_name] = path_object.dtype
+
+    def get_static_metadata(self, data_key):
+        if data_key not in self._static_metadata_cache:
+            # if there's an "@" in the key, it's an attribute, we need to split it
+            if "@" in data_key:
+                dkey, akey = data_key.rsplit("@", 1)
+                self._static_metadata_cache[data_key] = self.get_data_attributes(dkey).get(akey, None)
+            else:
+                with h5py.File(self._file_path, "r") as f:
+                    value = f[data_key][()]
+                    # decode bytes to string if necessary
+                    if isinstance(value, bytes):
+                        value = value.decode("utf-8")
+                    self._static_metadata_cache[data_key] = value
+        return self._static_metadata_cache[data_key]
+
+    def get_data(self, data_key: str, load_slice: ArraySlice = ...) -> np.ndarray:
+        if data_key not in self._data_cache:
+            with h5py.File(self._file_path, "r") as f:
+                data_array = f[data_key][load_slice]  # if load_slice is not None else f[data_key][()]
+                self._data_cache[data_key] = np.array(data_array)
+        return self._data_cache[data_key]
+
+    def get_data_shape(self, data_key: str) -> tuple[int, ...]:
+        if data_key in self._file_datasets_shapes:
+            return self._file_datasets_shapes[data_key]
+        return ()
+
+    def get_data_dtype(self, data_key: str) -> np.dtype | None:
+        if data_key in self._file_datasets_dtypes:
+            return self._file_datasets_dtypes[data_key]
+        return None
+
+    def get_data_attributes(self, data_key: str) -> dict[str, Any]:
+        attributes = {}
+        with h5py.File(self._file_path, "r") as f:
+            if data_key in f:
+                dataset = f[data_key]
+                for attr_key in dataset.attrs:
+                    attributes[attr_key] = dataset.attrs[attr_key]
+        return attributes

modacor/io/io_sink.py

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Brian R. Pauw"]  # add names to the list as appropriate
+__copyright__ = "Copyright 2026, The MoDaCor team"
+__date__ = "09/01/2026"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+from logging import WARNING
+from typing import Any
+
+import attrs
+from attrs import define, field
+
+
+def default_config() -> dict[str, Any]:
+    return {}
+
+
+@define
+class IoSink:
+    """
+    Base class for IO sinks. Mirrors IoSource.
+
+    Sinks are registered with a resource_location (file/socket/etc.).
+    The routed write call passes an optional 'subpath' after '::', which may be empty.
+    """
+
+    configuration: dict[str, Any] = field(factory=default_config)
+    sink_reference: str = field(default="", converter=str, validator=attrs.validators.instance_of(str))
+    type_reference: str = "IoSink"
+    iosink_method_kwargs: dict[str, Any] = field(factory=dict, validator=attrs.validators.instance_of(dict))
+    logging_level: int = field(default=WARNING, validator=attrs.validators.instance_of(int))
+
+    def write(self, subpath: str, *args, **kwargs):
+        raise NotImplementedError("This method should be implemented in subclasses.")

modacor/io/io_sinks.py

@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Brian R. Pauw"]  # add names to the list as appropriate
+__copyright__ = "Copyright 2026, The MoDaCor team"
+__date__ = "09/01/2026"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+__all__ = ["IoSinks"]
+
+from typing import Any
+
+from attrs import define, field
+
+from modacor.io.io_sink import IoSink
+
+
+@define
+class IoSinks:
+    """
+    Registry for IoSink instances. Mirrors IoSources.
+    """
+
+    defined_sinks: dict[str, IoSink] = field(factory=dict)
+
+    def register_sink(self, sink: IoSink, sink_reference: str | None = None) -> None:
+        if not isinstance(sink, IoSink):
+            raise TypeError("sink must be an instance of IoSink")
+        if sink_reference is None:
+            sink_reference = sink.sink_reference
+        if not isinstance(sink_reference, str):
+            raise TypeError("sink_reference must be a string")
+        if sink_reference in self.defined_sinks:
+            raise ValueError(f"Sink {sink_reference} already registered.")
+        self.defined_sinks[sink_reference] = sink
+
+    def get_sink(self, sink_reference: str) -> IoSink:
+        if sink_reference not in self.defined_sinks:
+            raise KeyError(f"Sink {sink_reference} not registered.")
+        return self.defined_sinks[sink_reference]
+
+    def split_target_reference(self, target_reference: str) -> tuple[str, str]:
+        """
+        Split 'sink_ref::subpath'. Subpath may be empty (e.g. 'export_csv::').
+        """
+        _split = target_reference.split("::", 1)
+        if len(_split) != 2:
+            raise ValueError(
+                "target_reference must be in the format 'sink_ref::subpath' with a double colon separator."
+            )
+        return _split[0], _split[1]
+
+    def write_data(self, target_reference: str, *args, **kwargs) -> Any:
+        sink_ref, subpath = self.split_target_reference(target_reference)
+        sink = self.get_sink(sink_ref)
+        return sink.write(subpath, *args, **kwargs)

modacor/io/io_source.py

@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Brian R. Pauw"]  # add names to the list as appropriate
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "14/06/2025"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+from logging import WARNING
+from typing import Any, Optional, Tuple, Union
+
+import attrs
+import numpy as np
+from attrs import define, field
+
+# for type hinting of slicing:
+Index = Union[int, slice, type(Ellipsis)]
+ArraySlice = Union[Index, Tuple[Index, ...]]
+
+
+def default_config() -> dict[str, Any]:
+    """
+    Default configuration for the IoSource class.
+
+    Returns
+    -------
+    dict[str, Any] :
+        A dictionary containing the default configuration.
+    """
+    return {
+        "data_rank": 1,
+        "data_key": None,
+        "data_rank_dims": (0,),
+        "metadata_key": None,
+        "non_data_slicing": "",
+    }
+
+
+@define
+class IoSource:
+    """
+    IoSource is a base class for all IO sources in the MoDaCor framework.
+
+    It provides access to a specific IO source and its associated methods.
+
+    Required configuration keys are:
+
+        data_rank : int
+            The rank of the data.
+        data_key : str
+            The key to access the data.
+            Special note for get_static_metadata: If the key contains an @ character, the part before the @ is
+            considered the group/dataset path, and the part after the @ is considered the attribute name.
+        data_rank_dims : tuple[int]
+            The dimensions of the data rank.
+        non_data_slicing : str
+            Slicing information for non-data dimensions. This must be a
+            string that can be evaluated to a slice object. Multiple data
+            slices can be separated by double semicolon ';;'.
+    """
+
+    configuration: dict[str, Any] = field(factory=default_config)
+    source_reference: str = field(default="", converter=str, validator=attrs.validators.instance_of(str))
+    type_reference: str = "IoSource"
+    # for passing extra kwargs to the data loading method if needed (e.g. csv_source)
+    iosource_method_kwargs: dict[str, Any] = field(factory=dict, validator=attrs.validators.instance_of(dict))
+    logging_level: int = field(default=WARNING, validator=attrs.validators.instance_of(int))
+
+    def get_data(self, data_key: str, load_slice: Optional[ArraySlice] = None) -> np.ndarray:
+        """
+        Get data from the IO source using the provided data key.
+
+        Parameters
+        ----------
+        data_key : str
+            The key to access the data, e.g. '/entry1/instrument/detector00/data'.
+        load_slice : Optional[ArraySlice]
+            A slice or tuple of slices to apply to the data. If None, the entire data is returned.
+            Slicing is not yet implemented, so this will raise NotImplementedError if used.
+            Consider using the numpy.s_ or numpy.index_exp for simplifying the slicing syntax.
+
+        Returns
+        -------
+        np.ndarray :
+            The data array associated with the provided key. For scalars, this is a 0-d array.
+        """
+        if load_slice is not None:
+            raise NotImplementedError("Slicing is not yet implemented.")
+        raise NotImplementedError("This method should be implemented in subclasses.")
+
+    def get_data_shape(self, data_key: str) -> Tuple[int, ...]:
+        """
+        Get the shape of the data from the IO source if the format supports it else empty tuple.
+
+        Parameters
+        ----------
+        data_key : str
+            The key to the data for which the shape is requested.
+
+        Returns
+        -------
+        Tuple[int, ...] :
+            The shape of the data associated with the provided key.
+            Returns an empty tuple if nothing available or unsupported.
+        """
+        raise NotImplementedError("This method should be implemented in subclasses.")
+
+    def get_data_dtype(self, data_key: str) -> Optional[np.dtype]:
+        """
+        Get the data type of the data from the IO source if the format supports it else None.
+
+        Parameters
+        ----------
+        data_key : str
+            The key to the data for which the dtype is requested.
+
+        Returns
+        -------
+        Optional[np.dtype] :
+            The data type of the data associated with the provided key.
+            Returns None if nothing available or unsupported.
+        """
+        raise NotImplementedError("This method should be implemented in subclasses.")
+
+    def get_data_attributes(self, data_key: str) -> dict[str, Any]:
+        """
+        Get data attributes from the IO source if the format supports it else empty dict.
+
+        Parameters
+        ----------
+        data_key : str
+            The key to the data for which attributes are requested.
+
+        Returns
+        -------
+        dict[str, Any] :
+            The attributes associated with the data.
+            Returns an empty dictionary if nothing available or unsupported.
+        """
+        raise NotImplementedError("This method should be implemented in subclasses.")
+
+    def get_static_metadata(self, data_key: str) -> Any:
+        """
+        Get static metadata from the IO source using the provided data key.
+        If the key contains an @ character, the part before the @ is
+        considered the group/dataset path, and the part after the @ is considered the attribute name.
+        Useful, for example, to get units from HDF5 attributes.
+
+        Parameters
+        ----------
+        data_key : str
+            The key to access the metadata.
+
+        Returns
+        -------
+        Any :
+            The static metadata associated with the provided key.
+        """
+        raise NotImplementedError("This method should be implemented in subclasses.")