modacor 1.0.0__py3-none-any.whl

modacor/modules/base_modules/append_processing_data.py

@@ -0,0 +1,329 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Brian R. Pauw"]  # add names to the list as appropriate
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "30/10/2025"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+__all__ = ["AppendProcessingData"]
+__version__ = "20251030.3"
+
+from pathlib import Path
+from typing import Any
+
+from modacor import ureg
+from modacor.dataclasses.basedata import BaseData
+from modacor.dataclasses.databundle import DataBundle
+from modacor.dataclasses.helpers import basedata_from_sources
+from modacor.dataclasses.messagehandler import MessageHandler
+from modacor.dataclasses.process_step import ProcessStep
+from modacor.dataclasses.process_step_describer import ProcessStepDescriber
+from modacor.io.io_sources import IoSources
+
+# Module-level handler; facilities can swap MessageHandler implementation as needed
+logger = MessageHandler(name=__name__)
+
+
+class AppendProcessingData(ProcessStep):
+    """
+    Load signal data from ``self.io_sources`` into a processing :class:`DataBundle`
+    in ``self.processing_data``.
+
+    This step creates or updates a single :class:`DataBundle` from existing
+    :class:`IoSources` entries in ``self.io_sources``. It:
+
+    1. Loads the signal array from ``signal_location`` (string reference).
+    2. Loads units either from:
+       - ``units_location`` via :meth:`IoSources.get_static_metadata`, or
+       - ``units_override`` as a direct units string, or
+       - defaults to dimensionless if neither is provided.
+    3. Optionally loads uncertainty arrays from ``uncertainties_sources``.
+    4. Wraps everything in a :class:`BaseData` instance.
+    5. Sets ``BaseData.rank_of_data`` based on the configured ``rank_of_data``:
+       - If it is an ``int``, it is used directly.
+       - If it is a ``str``, it is interpreted as an IoSources metadata reference
+         (``'<io_source_id>::<dataset_path>'``) and read via
+         :meth:`IoSources.get_static_metadata`, then converted to ``int``.
+       - Validation and bounds checking are handled by :func:`validate_rank_of_data`
+         inside :class:`BaseData`.
+    6. Stores the resulting :class:`BaseData` under the configured
+       ``databundle_output_key`` (default: ``"signal"``) in a
+       :class:`DataBundle` at ``self.processing_data[processing_key]``. If that
+       DataBundle already exists, it is updated: existing entries are preserved
+       and the ``databundle_output_key`` entry is overwritten or added.
+
+    The resulting mapping ``{processing_key: DataBundle}`` is returned.
+    """
+
+    documentation = ProcessStepDescriber(
+        calling_name="Append Processing Data",
+        calling_id="AppendProcessingData",
+        calling_module_path=Path(__file__),
+        calling_version=__version__,
+        required_data_keys=[],  # this step creates/updates a DataBundle
+        modifies={},  # processing_key: databundle: databundle_output_key
+        arguments={
+            "processing_key": {
+                "type": str,
+                "required": True,
+                "default": "",
+                "doc": "ProcessingData key to create or update.",
+            },
+            "signal_location": {
+                "type": str,
+                "required": True,
+                "default": "",
+                "doc": "IoSources reference '<io_source_id>::<dataset_path>'.",
+            },
+            "rank_of_data": {
+                "type": (int, str),
+                "required": True,
+                "default": 2,
+                "doc": "BaseData rank as int or IoSources metadata reference.",
+            },
+            "databundle_output_key": {
+                "type": str,
+                "default": "signal",
+                "doc": "BaseData key inside the DataBundle.",
+            },
+            "units_location": {
+                "type": (str, type(None)),
+                "default": None,
+                "doc": "Optional IoSources reference for units metadata.",
+            },
+            "units_override": {
+                "type": (str, type(None)),
+                "default": None,
+                "doc": "Optional unit string that overrides loaded units.",
+            },
+            "uncertainties_sources": {
+                "type": dict,
+                "default": {},
+                "doc": "Mapping of uncertainty name to IoSources reference.",
+            },
+        },
+        step_keywords=["append", "processing", "data", "signal"],
+        step_doc="Append signal data from IoSources into a processing DataBundle.",
+        step_reference="",
+        step_note=(
+            "This step reads from existing IoSources and creates or updates a named DataBundle "
+            "with a BaseData entry (default 'signal') for use in the processing pipeline."
+        ),
+    )
+
+    # -------------------------------------------------------------------------
+    # Internal helpers
+    # -------------------------------------------------------------------------
+    def _resolve_rank_of_data(self, rank_cfg: Any, io_sources: IoSources) -> int:
+        """
+        Resolve the configured rank_of_data to an integer.
+
+        Parameters
+        ----------
+        rank_cfg :
+            Either an integer directly, or a string reference of the form
+            '<io_source_id>::<dataset_path>' pointing to metadata that contains
+            the rank as an integer-compatible value.
+
+        io_sources :
+            The IoSources object used to resolve metadata references.
+
+        Returns
+        -------
+        int
+            The resolved rank_of_data. Actual bounds checking is performed by
+            BaseData's internal validation.
+        """
+        # Direct int → use as-is (with int() for safety)
+        if isinstance(rank_cfg, int):
+            return int(rank_cfg)
+
+        # If it *looks* like an io_source reference, treat it as such
+        if isinstance(rank_cfg, str):
+            logger.debug(
+                f"AppendProcessingData: resolving rank_of_data from IoSources metadata reference '{rank_cfg}'."
+            )
+            meta_value = io_sources.get_static_metadata(rank_cfg)
+            try:
+                return int(meta_value)
+            except (TypeError, ValueError) as exc:
+                raise ValueError(
+                    f"Could not convert rank_of_data metadata from '{rank_cfg}' to int (value={meta_value!r})."
+                ) from exc
+
+        # Fallback: try to cast whatever it is to int
+        try:
+            return int(rank_cfg)
+        except (TypeError, ValueError) as exc:
+            raise TypeError(
+                "rank_of_data must be an int or an IoSources metadata reference string, "
+                f"got {rank_cfg!r} ({type(rank_cfg).__name__})."
+            ) from exc
+
+    def _load_and_validate_configuration(self) -> dict[str, Any]:
+        """
+        Load and validate configuration values from ``self.configuration``.
+
+        Returns
+        -------
+        dict[str, Any]
+            A dictionary containing the resolved configuration:
+            - processing_key (str)
+            - signal_location (str)
+            - rank_of_data (int)
+            - databundle_output_key (str)
+            - units_location (str | None)
+            - units_override (str | None)
+            - uncertainties_sources (dict[str, str])
+        """
+        cfg = self.configuration
+
+        processing_key = cfg.get("processing_key")
+        if not isinstance(processing_key, str) or not processing_key:
+            raise ValueError("AppendProcessingData requires 'processing_key' to be a non-empty string.")
+
+        signal_location = cfg.get("signal_location")
+        if not isinstance(signal_location, str) or not signal_location:
+            raise ValueError("AppendProcessingData requires 'signal_location' to be a non-empty string.")
+
+        if "rank_of_data" not in cfg:
+            raise ValueError("AppendProcessingData requires 'rank_of_data' in the configuration.")
+        rank_cfg = cfg["rank_of_data"]
+        resolved_rank = self._resolve_rank_of_data(rank_cfg, self.io_sources)
+
+        databundle_output_key = cfg.get("databundle_output_key", "signal")
+        if not isinstance(databundle_output_key, str) or not databundle_output_key:
+            raise ValueError("AppendProcessingData requires 'databundle_output_key' to be a non-empty string.")
+
+        units_location = cfg.get("units_location")
+        if units_location is not None and not isinstance(units_location, str):
+            raise TypeError("'units_location' must be a string '<source_ref>::<dataset_path>' or None.")
+
+        units_override = cfg.get("units_override")
+        if units_override is not None and not isinstance(units_override, str):
+            raise TypeError("'units_override' must be a units string if provided.")
+
+        uncertainties_sources: dict[str, str] = cfg.get("uncertainties_sources", {}) or {}
+        if not isinstance(uncertainties_sources, dict):
+            raise TypeError(
+                f"'uncertainties_sources' must be a dict[str, str], got {type(uncertainties_sources).__name__}."
+            )
+
+        return {
+            "processing_key": processing_key,
+            "signal_location": signal_location,
+            "rank_of_data": resolved_rank,
+            "databundle_output_key": databundle_output_key,
+            "units_location": units_location,
+            "units_override": units_override,
+            "uncertainties_sources": uncertainties_sources,
+        }
+
+    # -------------------------------------------------------------------------
+    # Public API used by the pipeline
+    # -------------------------------------------------------------------------
+    def calculate(self) -> dict[str, DataBundle]:
+        """
+        Create or update a DataBundle from ``self.io_sources`` and return it.
+
+        Configuration fields:
+
+        - ``processing_key`` (str):
+            Name under which the DataBundle will be stored in
+            ``self.processing_data``.
+
+        - ``signal_location`` (str):
+            Data reference in the form ``'<source_ref>::<dataset_path>'``.
+
+        - ``rank_of_data`` (int or str):
+            Desired rank for the created :class:`BaseData` object.
+            If a string, it is treated as an IoSources metadata reference and
+            resolved via :meth:`IoSources.get_static_metadata`. Validation and
+            bounds checking are handled by :class:`BaseData`.
+
+        - ``databundle_output_key`` (str, default: ``"signal"``):
+            Key under which the new :class:`BaseData` will be stored inside the
+            :class:`DataBundle`. If the DataBundle already contains an entry
+            under this key, it will be overwritten.
+
+        - ``units_location`` (str | None):
+            Data reference pointing to a static metadata entry that defines the
+            units. If provided, the value from
+            :meth:`IoSources.get_static_metadata` is passed to :func:`ureg.Unit`.
+            If omitted and ``units_override`` is None, units default to
+            dimensionless.
+
+        - ``units_override`` (str | None):
+            Direct units string (e.g. ``"counts"`` or ``"1/m"``) that overrides
+            any value loaded via ``units_location``.
+
+        - ``uncertainties_sources`` (dict[str, str]):
+            Mapping from uncertainty name (e.g. ``"poisson"``) to data reference
+            (``'<source_ref>::<dataset_path>'``).
+        """
+        cfg = self._load_and_validate_configuration()
+
+        processing_key: str = cfg["processing_key"]
+        signal_location: str = cfg["signal_location"]
+        rank_of_data: int = cfg["rank_of_data"]
+        databundle_output_key: str = cfg["databundle_output_key"]
+        units_location = cfg["units_location"]
+        units_override = cfg["units_override"]
+        uncertainties_sources: dict[str, str] = cfg["uncertainties_sources"]
+
+        io_sources: IoSources = self.io_sources
+
+        logger.info(
+            (
+                f"AppendProcessingData: creating/updating DataBundle '{processing_key}' "
+                f"from signal_location='{signal_location}' into key '{databundle_output_key}'."
+            ),
+        )
+
+        # Load BaseData via helper: signal + units + uncertainties
+        bd: BaseData = basedata_from_sources(
+            io_sources=io_sources,
+            signal_source=signal_location,
+            units_source=units_location,
+            uncertainty_sources=uncertainties_sources,
+        )
+
+        # Override units if requested
+        if units_override is not None:
+            logger.debug(
+                f"AppendProcessingData: overriding units for '{processing_key}' to '{units_override}'.",
+            )
+            bd.units = ureg.Unit(units_override)
+
+        # Set rank_of_data; BaseData's own validation handles bounds
+        bd.rank_of_data = rank_of_data
+
+        # Create or update the DataBundle in processing_data
+        existing_bundle = self.processing_data.get(processing_key)
+        if existing_bundle is None:
+            databundle = DataBundle()
+        else:
+            databundle = existing_bundle
+
+        # Update/insert the BaseData at the requested key
+        databundle[databundle_output_key] = bd
+
+        # If no default_plot is set yet, use this key as a sensible default
+        if getattr(databundle, "default_plot", None) is None:
+            databundle.default_plot = databundle_output_key
+
+        # Store back into processing_data and build output
+        self.processing_data[processing_key] = databundle
+        output: dict[str, DataBundle] = {processing_key: databundle}
+
+        logger.info(
+            f"AppendProcessingData: DataBundle '{processing_key}' now contains datasets {list(databundle.keys())}.",
+        )
+
+        return output

modacor/modules/base_modules/append_sink.py

@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Brian R. Pauw"]
+__copyright__ = "Copyright 2026, The MoDaCor team"
+__date__ = "09/01/2026"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+__all__ = ["AppendSink"]
+__version__ = "20260109.1"
+
+from importlib import import_module
+from pathlib import Path
+from typing import Any, Callable
+
+from modacor.dataclasses.databundle import DataBundle
+from modacor.dataclasses.messagehandler import MessageHandler
+from modacor.dataclasses.process_step import ProcessStep
+from modacor.dataclasses.process_step_describer import ProcessStepDescriber
+from modacor.io.io_sinks import IoSinks
+
+# Module-level handler; facilities can swap MessageHandler implementation as needed
+logger = MessageHandler(name=__name__)
+
+
+class AppendSink(ProcessStep):
+    """
+    Appends an IoSink to self.io_sinks.
+
+    This mirrors AppendSource: it augments the set of available I/O sinks but does
+    not touch the actual data bundles.
+    """
+
+    documentation = ProcessStepDescriber(
+        calling_name="Append Sink",
+        calling_id="AppendSink",
+        calling_module_path=Path(__file__),
+        calling_version=__version__,
+        required_data_keys=[],
+        modifies={},  # sinks only; no data modified
+        arguments={
+            "sink_identifier": {
+                "type": (str, list),
+                "required": True,
+                "default": "",
+                "doc": "Identifier(s) to register the ioSink(s) under.",
+            },
+            "sink_location": {
+                "type": (str, list),
+                "required": True,
+                "default": "",
+                "doc": "Resource location(s) understood by the sink.",
+            },
+            "iosink_module": {
+                "type": str,
+                "required": True,
+                "default": "",
+                "doc": "Fully qualified module path to the ioSink class.",
+            },
+            "iosink_method_kwargs": {
+                "type": dict,
+                "default": {},
+                "doc": "Keyword arguments forwarded to the ioSink constructor.",
+            },
+        },
+        step_keywords=["append", "sink"],
+        step_doc="Append an IoSink to the available data sinks",
+        step_reference="",
+        step_note="This adds an IoSink to the data sinks registry.",
+    )
+
+    # -------------------------------------------------------------------------
+    # Public API used by the pipeline
+    # -------------------------------------------------------------------------
+    def calculate(self) -> dict[str, DataBundle]:
+        output: dict[str, DataBundle] = {}
+
+        sink_ids: str | list[str] = self.configuration["sink_identifier"]
+        sink_locations: str | list[str] = self.configuration["sink_location"]
+        iosink_module: str = self.configuration["iosink_module"]
+
+        # Normalise to lists
+        if isinstance(sink_ids, str):
+            sink_ids = [sink_ids]
+        if isinstance(sink_locations, str):
+            sink_locations = [sink_locations]
+
+        if len(sink_ids) != len(sink_locations):
+            raise ValueError("If multiple sink_identifiers and sink_locations are provided, their counts must match.")
+
+        for sink_id, sink_location in zip(sink_ids, sink_locations):
+            if sink_id not in self.io_sinks.defined_sinks:
+                self._append_sink_by_name(
+                    sink_name=iosink_module,
+                    sink_location=sink_location,
+                    sink_identifier=sink_id,
+                    iosink_method_kwargs=self.configuration.get("iosink_method_kwargs", {}),
+                )
+
+        return output
+
+    # -------------------------------------------------------------------------
+    # Internal helpers
+    # -------------------------------------------------------------------------
+    def _append_sink_by_name(
+        self,
+        sink_name: str,
+        sink_location: str,
+        sink_identifier: str,
+        iosink_method_kwargs: dict[str, Any] = {},
+    ) -> None:
+        sink_callable = self._resolve_iosink_callable(sink_name)
+
+        # Ensure io_sinks exists or initialize it
+        if not hasattr(self, "io_sinks") or self.io_sinks is None:
+            self.io_sinks = IoSinks()
+            logger.info("Initialized self.io_sinks in AppendSink step.")
+
+        self.io_sinks.register_sink(
+            sink_callable(
+                sink_reference=sink_identifier,
+                resource_location=sink_location,
+                iosink_method_kwargs=iosink_method_kwargs,
+            )
+        )
+
+    def _resolve_iosink_callable(self, sink_name: str) -> Callable[..., Any]:
+        module_path, attr_name = sink_name.rsplit(".", 1)
+        module = import_module(module_path)
+        try:
+            sink_obj = getattr(module, attr_name)
+        except AttributeError as exc:
+            raise ImportError(
+                f"Could not find '{attr_name}' in module '{module_path}' for iosink_module='{sink_name}'."
+            ) from exc
+        return sink_obj

modacor/modules/base_modules/append_source.py

@@ -0,0 +1,181 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Brian R. Pauw"]  # add names to the list as appropriate
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "30/10/2025"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+__all__ = ["AppendSource"]
+__version__ = "20251030.1"
+
+from importlib import import_module
+from pathlib import Path
+from typing import Any, Callable
+
+from modacor.dataclasses.databundle import DataBundle
+from modacor.dataclasses.messagehandler import MessageHandler
+from modacor.dataclasses.process_step import ProcessStep
+from modacor.dataclasses.process_step_describer import ProcessStepDescriber
+from modacor.io.io_sources import IoSources
+
+# Module-level handler; facilities can swap MessageHandler implementation as needed
+logger = MessageHandler(name=__name__)
+
+
+class AppendSource(ProcessStep):
+    """
+    Appends an ioSource to self.io_sources.
+
+    This step is intended for pipeline-graph / provenance operations: it augments
+    the set of available I/O sources but does not touch the actual data bundles.
+    """
+
+    documentation = ProcessStepDescriber(
+        calling_name="Append Source",
+        calling_id="AppendSource",
+        calling_module_path=Path(__file__),
+        calling_version=__version__,
+        required_data_keys=[],
+        modifies={},  # sources only; no data modified
+        arguments={
+            "source_identifier": {
+                "type": (str, list),
+                "required": True,
+                "default": "",
+                "doc": "Identifier(s) to register the ioSource(s) under.",
+            },
+            "source_location": {
+                "type": (str, list),
+                "required": True,
+                "default": "",
+                "doc": "Resource location(s) to load for the ioSource(s).",
+            },
+            "iosource_module": {
+                "type": str,
+                "required": True,
+                "default": "",
+                "doc": "Fully qualified module path to the ioSource class.",
+            },
+            "iosource_method_kwargs": {
+                "type": dict,
+                "default": {},
+                "doc": "Keyword arguments forwarded to the ioSource constructor.",
+            },
+        },
+        step_keywords=["append", "source"],
+        step_doc="Append an ioSource to the available data sources",
+        step_reference="",
+        step_note="This adds an ioSource to the data sources of the databundle.",
+    )
+
+    # -------------------------------------------------------------------------
+    # Public API used by the pipeline
+    # -------------------------------------------------------------------------
+    def calculate(self) -> dict[str, DataBundle]:
+        """
+        Append one or more sources to ``self.io_sources``.
+
+        Notes
+        -----
+        - No ``DataBundle`` objects are modified or created.
+        - The pipeline can treat an empty output dict as "no-op on data",
+          while the side-effect on ``self.io_sources`` persists.
+        """
+        output: dict[str, DataBundle] = {}
+
+        source_ids: str | list[str] = self.configuration["source_identifier"]
+        source_locations: str | list[str] = self.configuration["source_location"]
+        iosource_module: str = self.configuration["iosource_module"]
+
+        # Normalise to lists
+        if isinstance(source_ids, str):
+            source_ids = [source_ids]
+        if isinstance(source_locations, str):
+            source_locations = [source_locations]
+
+        if len(source_ids) != len(source_locations):
+            raise ValueError(
+                "If multiple source_identifiers and source_locations are provided, their counts must match."
+            )
+
+        for source_id, source_location in zip(source_ids, source_locations):
+            # Only append if not already present
+            if source_id not in self.io_sources.defined_sources:
+                self._append_loader_by_name(
+                    loader_name=iosource_module,
+                    source_location=source_location,
+                    source_identifier=source_id,
+                    iosource_method_kwargs=self.configuration.get("iosource_method_kwargs", {}),
+                )
+        # No data modified – only side-effect is on self.io_sources
+        return output
+
+    # -------------------------------------------------------------------------
+    # Internal helpers
+    # -------------------------------------------------------------------------
+    def _append_loader_by_name(
+        self,
+        loader_name: str,
+        source_location: str,
+        source_identifier: str,
+        iosource_method_kwargs: dict[str, Any] = {},
+    ) -> None:
+        """
+        Resolve the requested loader and append the resulting ioSource to
+        ``self.io_sources``.
+
+        Parameters
+        ----------
+        loader_name:
+            Either a fully qualified import path
+            (e.g. ``"modacor.io.hdf.hdf_source.HDFSource"``).
+        source_location:
+            Path / URI / identifier understood by the loader.
+        source_identifier:
+            Key under which the resulting ioSource will be stored in
+            ``self.io_sources``.
+        iosource_method_kwargs:
+            Additional keyword arguments to pass to the loader callable.
+        """
+        source_callable = self._resolve_iosource_callable(loader_name)
+
+        # Ensure io_sources exists or initialize it
+        if not hasattr(self, "io_sources") or self.io_sources is None:
+            # ProcessStep normally sets this up, but be defensive.
+            self.io_sources = IoSources()
+            logger.info("Initialized self.io_sources in AppendSource step.")
+
+        self.io_sources.register_source(
+            source_callable(
+                source_reference=source_identifier,
+                resource_location=source_location,
+                iosource_method_kwargs=iosource_method_kwargs,
+            )
+        )
+
+    def _resolve_iosource_callable(self, loader_name: str) -> Callable[..., Any]:
+        """
+        Resolve the configured loader into a callable.
+
+        Strategy
+        --------
+        1. If ``loader_name`` contains a dot, treat it as a fully qualified
+           import path like ``package.module.ClassOrFunc``.
+        """
+
+        module_path, attr_name = loader_name.rsplit(".", 1)
+        module = import_module(module_path)
+        try:
+            loader_obj = getattr(module, attr_name)
+        except AttributeError as exc:
+            raise ImportError(
+                f"Could not find '{attr_name}' in module '{module_path}' for iosource_module='{loader_name}'."
+            ) from exc
+
+        return loader_obj