modacor 1.0.0__py3-none-any.whl

modacor/tests/dataclasses/test_processstep.py

@@ -0,0 +1,282 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Malte Storm", "Jérome Kieffer", "Anja Hörmann", "Brian R. Pauw"]  # add names to the list as appropriate
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "16/11/2025"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+from typing import Iterable
+
+import numpy as np
+import pytest
+
+from modacor import ureg
+from modacor.dataclasses.basedata import BaseData
+from modacor.dataclasses.databundle import DataBundle
+from modacor.dataclasses.process_step import ProcessStep
+from modacor.dataclasses.processing_data import ProcessingData
+from modacor.io import IoSources
+
+TEST_IO_SOURCES = IoSources()
+
+_TEST_KEYS = {
+    "test_str": {
+        "type": str,
+        "allow_iterable": False,
+        "allow_none": False,
+        "default": "",
+    },
+    "test_str_allow_list": {
+        "type": str,
+        "allow_iterable": True,
+        "allow_none": False,
+        "default": "",
+    },
+    "test_str_allow_none": {
+        "type": str,
+        "allow_iterable": False,
+        "allow_none": True,
+        "default": None,
+    },
+    "test_str_allow_list_none": {
+        "type": str,
+        "allow_iterable": True,
+        "allow_none": True,
+        "default": None,
+    },
+    "test_int": {
+        "type": int,
+        "allow_iterable": False,
+        "allow_none": False,
+        "default": 0,
+    },
+    "test_int_allow_list": {
+        "type": int,
+        "allow_iterable": True,
+        "allow_none": False,
+        "default": 0,
+    },
+    "test_int_allow_none": {
+        "type": int,
+        "allow_iterable": False,
+        "allow_none": True,
+        "default": None,
+    },
+    "test_int_allow_list_none": {
+        "type": int,
+        "allow_iterable": True,
+        "allow_none": True,
+        "default": None,
+    },
+}
+
+_TEST_VALUES = ["", "test", 1, 42, None]
+_TEST_LISTS = [["", "2"], ["", 12], [12, 42], [12, None], ["b", None]]
+_TEST_TUPLES = [("12", "b"), ("", 12), (12, 42), ("a", None), (12, None)]
+
+
+class TESTProcessingStep(ProcessStep):
+    CONFIG_KEYS = {_k: _v for _k, _v in _TEST_KEYS.items()}
+
+    def calculate(self) -> dict[str, DataBundle]:
+        _data = self.processing_data.get("dummy_key", DataBundle())
+        _data["new_key"] = BaseData(signal=np.arange(100).reshape(10, 10), uncertainties={"sem": 0.0}, units=ureg.meter)
+        _data2 = self.processing_data.get("bundle2", DataBundle())
+        _data2["new_key"] = BaseData(signal=np.zeros(20), uncertainties={"sem": 0.0}, units=ureg.meter)
+        return {"dummy_key": _data, "bundle2": _data2}
+
+
+@pytest.fixture
+def class_with_config_keys(request):
+    _keys = request.param
+
+    class TestClass(TESTProcessingStep):
+        CONFIG_KEYS = {_key: _TEST_KEYS[_key] for _key in _keys}
+
+    return _keys, TestClass
+
+
+@pytest.fixture
+def processing_data():
+    data = ProcessingData()
+    data["bundle1"] = DataBundle()
+    data["bundle2"] = DataBundle()
+    data["bundle1"]["key1"] = BaseData(signal=np.arange(50), uncertainties={"sem": 0.0}, units=ureg.meter)
+    data["bundle2"]["key2"] = BaseData(signal=np.ones((10, 10)), uncertainties={"sem": 0.0}, units=ureg.meter)
+    return data
+
+
+def test_process_step_default_config__generic():
+    _defaults = ProcessStep.default_config()
+    assert isinstance(_defaults, dict)
+    assert all(key in _defaults for key in ProcessStep.CONFIG_KEYS.keys())
+
+
+@pytest.mark.parametrize(
+    "class_with_config_keys", [["test_str"], ["test_int"], ["test_str", "test_int"]], indirect=True
+)
+def test_process_step_default_config__specific(class_with_config_keys):
+    _keys, _class = class_with_config_keys
+    _defaults = _class.default_config()
+    assert isinstance(_defaults, dict)
+    assert all(key in _defaults for key in _keys)
+
+
+@pytest.mark.parametrize("item", _TEST_VALUES + _TEST_LISTS + _TEST_TUPLES)
+@pytest.mark.parametrize("class_with_config_keys", [[_k] for _k in _TEST_KEYS.keys()], indirect=True)
+def test_is_process_step_dict__w_correct_key(class_with_config_keys, item):
+    _keys, _class = class_with_config_keys
+    _config = _class.CONFIG_KEYS[_keys[0]]
+    _test_dict = {_keys[0]: item}
+    if item is None:
+        assert _class.is_process_step_dict(None, None, _test_dict) == _config["allow_none"]
+    elif not _config["allow_iterable"]:
+        assert _class.is_process_step_dict(None, None, _test_dict) == isinstance(item, _config["type"])
+    elif _config["allow_iterable"]:
+        assert _class.is_process_step_dict(None, None, _test_dict) == (
+            (isinstance(item, Iterable) and not isinstance(item, str))
+            and all(isinstance(i, _config["type"]) for i in item)
+            or isinstance(item, _config["type"])
+        )
+    else:
+        assert False
+
+
+def test_is_process_step_dict__w_wrong_key():
+    test_dict = ProcessStep.default_config() | {"wrong_key": "value"}
+    assert not ProcessStep.is_process_step_dict(None, None, test_dict)
+
+
+def test_minimal_instantiation():
+    ps = ProcessStep(io_sources=TEST_IO_SOURCES)
+    assert isinstance(ps, ProcessStep)
+
+
+def test_instantiation_of_subclass():
+    instance = TESTProcessingStep(io_sources=TEST_IO_SOURCES)
+    assert all(k in instance.configuration for k in TESTProcessingStep.CONFIG_KEYS)
+    assert isinstance(instance, TESTProcessingStep)
+
+
+def test_process_step__reset():
+    ps = ProcessStep(io_sources=TEST_IO_SOURCES)
+    ps.produced_outputs = {"a": 1}
+    ps._ProcessStep__prepared = True
+    ps.executed = True
+    ps.reset()
+    assert ps.produced_outputs == {}
+    assert ps._ProcessStep__prepared is False
+    assert ps.executed is False
+
+
+def test_normalised_processing_keys_with_single_entry():
+    data = ProcessingData()
+    data["only"] = DataBundle()
+    ps = ProcessStep(io_sources=TEST_IO_SOURCES, processing_data=data)
+    ps.configuration["with_processing_keys"] = None
+    assert ps._normalised_processing_keys() == ["only"]
+
+
+def test_normalised_processing_keys_with_multiple_entries_requires_explicit_keys():
+    data = ProcessingData()
+    data["a"] = DataBundle()
+    data["b"] = DataBundle()
+    ps = ProcessStep(io_sources=TEST_IO_SOURCES, processing_data=data)
+    ps.configuration["with_processing_keys"] = None
+    with pytest.raises(ValueError):
+        ps._normalised_processing_keys()
+
+
+def test_normalised_processing_keys_accepts_string_or_list():
+    data = ProcessingData()
+    data["a"] = DataBundle()
+    ps = ProcessStep(io_sources=TEST_IO_SOURCES, processing_data=data)
+    ps.configuration["with_processing_keys"] = "a"
+    assert ps._normalised_processing_keys() == ["a"]
+
+    ps.configuration["with_processing_keys"] = ["a", "b"]
+    assert ps._normalised_processing_keys() == ["a", "b"]
+
+
+def test_normalised_processing_keys_rejects_empty_list():
+    data = ProcessingData()
+    data["a"] = DataBundle()
+    ps = ProcessStep(io_sources=TEST_IO_SOURCES, processing_data=data)
+    ps.configuration["with_processing_keys"] = []
+    with pytest.raises(ValueError):
+        ps._normalised_processing_keys()
+
+
+def test_normalised_processing_keys_requires_processing_data():
+    ps = ProcessStep(io_sources=TEST_IO_SOURCES, processing_data=None)
+    ps.configuration["with_processing_keys"] = ["a"]
+    with pytest.raises(RuntimeError):
+        ps._normalised_processing_keys()
+
+
+@pytest.mark.parametrize("class_with_config_keys", [["test_str"]], indirect=True)
+def test_modify_config__valid_key(class_with_config_keys):
+    instance = class_with_config_keys[1](io_sources=TEST_IO_SOURCES)
+    instance.modify_config_by_kwargs(test_str="new_value")
+    assert instance.configuration["test_str"] == "new_value"
+    assert not instance._ProcessStep__prepared
+
+
+@pytest.mark.parametrize("class_with_config_keys", [["test_str"]], indirect=True)
+def test_modify_config__by_dict(class_with_config_keys):
+    instance = class_with_config_keys[1](io_sources=TEST_IO_SOURCES)
+    instance.modify_config_by_dict({"test_str": "new_value"})
+    assert instance.configuration["test_str"] == "new_value"
+    assert not instance._ProcessStep__prepared
+
+
+@pytest.mark.parametrize("class_with_config_keys", [["test_str"]], indirect=True)
+def test_modify_config__invalid_key(class_with_config_keys):
+    instance = class_with_config_keys[1](io_sources=TEST_IO_SOURCES)
+    with pytest.raises(KeyError):
+        instance.modify_config_by_kwargs(silly_key="new_value")
+
+
+def test_calculate():
+    ps = TESTProcessingStep(io_sources=TEST_IO_SOURCES)
+    ps.processing_data = ProcessingData()
+    _return = ps.calculate()
+    assert isinstance(_return, dict)
+
+
+def test_calculate__abstract():
+    ps = ProcessStep(io_sources=TEST_IO_SOURCES)
+    with pytest.raises(NotImplementedError):
+        ps.calculate()
+
+
+def test_execute(processing_data):
+    ps = TESTProcessingStep(io_sources=TEST_IO_SOURCES)
+    ps.execute(processing_data)
+    assert ps.executed is True
+    assert ps._ProcessStep__prepared is True
+    assert isinstance(ps.produced_outputs, dict)
+    assert isinstance(ps.produced_outputs["dummy_key"], DataBundle)
+    assert isinstance(ps.produced_outputs["bundle2"], DataBundle)
+    assert "dummy_key" in processing_data
+    assert isinstance(processing_data["bundle2"]["key2"], BaseData)
+    assert isinstance(processing_data["bundle2"]["new_key"], BaseData)
+
+
+def test_call(processing_data):
+    ps = TESTProcessingStep(io_sources=TEST_IO_SOURCES)
+    ps(processing_data)
+    assert ps.executed is True
+    assert ps._ProcessStep__prepared is True
+    assert isinstance(ps.produced_outputs, dict)
+    assert isinstance(ps.produced_outputs["dummy_key"], DataBundle)
+    assert isinstance(ps.produced_outputs["bundle2"], DataBundle)
+    assert "dummy_key" in processing_data
+    assert isinstance(processing_data["bundle2"]["key2"], BaseData)
+    assert isinstance(processing_data["bundle2"]["new_key"], BaseData)

modacor/tests/debug/test_tracing_integration.py

@@ -0,0 +1,188 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+__coding__ = "utf-8"
+__authors__ = ["Brian R. Pauw"]  # add names to the list as appropriate
+__copyright__ = "Copyright 2025, The MoDaCor team"
+__date__ = "13/12/2025"
+__status__ = "Development"  # "Development", "Production"
+# end of header and standard imports
+
+from pathlib import Path
+
+from modacor.dataclasses.process_step import ProcessStep
+from modacor.dataclasses.process_step_describer import ProcessStepDescriber
+from modacor.dataclasses.processing_data import ProcessingData
+from modacor.debug.pipeline_tracer import PipelineTracer
+from modacor.runner.pipeline import Pipeline
+
+
+class DummyStep(ProcessStep):
+    documentation = ProcessStepDescriber(
+        calling_name="Dummy",
+        calling_id="dummy.step",
+        calling_module_path=Path(__file__),
+        calling_version="0",
+        required_data_keys=[],
+    )
+
+    def calculate(self):
+        return {}  # does not modify ProcessingData
+
+
+def test_tracer_emits_nothing_when_no_changes_and_no_empty_events():
+    tracer = PipelineTracer(
+        watch={"sample": ["signal"]},
+        record_only_on_change=True,
+        record_empty_step_events=False,  # requires the optional field; if you did not add it, remove this line
+    )
+
+    step = DummyStep(io_sources=None, step_id="A")
+    data = ProcessingData()
+
+    tracer.after_step(step, data)
+    assert tracer.events == []
+
+
+def test_tracer_can_emit_empty_step_events_when_enabled():
+    tracer = PipelineTracer(
+        watch={"sample": ["signal"]},
+        record_only_on_change=True,
+        record_empty_step_events=True,  # requires the optional field
+    )
+
+    step = DummyStep(io_sources=None, step_id="A")
+    data = ProcessingData()
+
+    tracer.after_step(step, data)
+    assert len(tracer.events) == 1
+    assert tracer.events[0]["step_id"] == "A"
+    assert tracer.events[0]["changed"] == {}
+
+
+def test_pipeline_attach_tracer_event_always_attaches_event_even_without_tracer_events():
+    step = DummyStep(io_sources=None, step_id="A")
+    pipeline = Pipeline.from_dict({step: []}, name="t")
+
+    tracer = PipelineTracer(watch={"sample": ["signal"]})
+    # No tracer.after_step() call -> tracer.events empty
+
+    ev = pipeline.attach_tracer_event(step, tracer)
+
+    assert ev.step_id == "A"
+    assert ev.module == "DummyStep"
+    assert ev.datasets == {}  # no tracer payload
+    assert "A" in pipeline.trace_events
+    assert pipeline.trace_events["A"][-1] is ev
+
+
+def test_pipeline_attach_tracer_event_includes_datasets_when_tracer_has_matching_event():
+    step = DummyStep(io_sources=None, step_id="A")
+    pipeline = Pipeline.from_dict({step: []}, name="t")
+
+    tracer = PipelineTracer(
+        watch={"sample": ["signal"]},
+        record_only_on_change=True,
+        record_empty_step_events=True,  # ensure tracer produces an event even if empty
+    )
+    data = ProcessingData()
+
+    tracer.after_step(step, data)
+    ev = pipeline.attach_tracer_event(step, tracer)
+
+    # Event exists; datasets may still be empty because watched target doesn't exist in ProcessingData
+    assert ev.step_id == "A"
+    assert isinstance(ev.datasets, dict)
+
+
+def test_pipeline_attach_tracer_event_can_embed_rendered_trace_block():
+    step = DummyStep(io_sources=None, step_id="A")
+    pipeline = Pipeline.from_dict({step: []}, name="t")
+
+    tracer = PipelineTracer(
+        watch={"sample": ["signal"]},
+        record_only_on_change=False,
+    )
+
+    # Ensure the tracer has at least one event to render:
+    tracer.after_step(step, ProcessingData())
+    assert tracer.events  # sanity
+
+    ev = pipeline.attach_tracer_event(
+        step,
+        tracer,
+        include_rendered_trace=True,
+        include_rendered_config=True,
+        rendered_format="text/plain",
+    )
+
+    assert isinstance(ev.messages, list)
+    trace_msgs = [m for m in ev.messages if m.get("kind") in {"rendered_trace", "rendered_trace_error"}]
+    assert trace_msgs, "Expected a rendered_trace (or error) block"
+    m = trace_msgs[0]
+    assert "format" in m and "content" in m
+
+
+def test_attach_tracer_event_renders_step_local_block():
+    step = DummyStep(io_sources=None, step_id="A")
+    pipeline = Pipeline.from_dict({step: []}, name="t")
+
+    tracer = PipelineTracer(
+        watch={"sample": ["signal"]},
+        record_only_on_change=False,  # ensure an event is recorded even if nothing changes
+    )
+
+    tracer.after_step(step, ProcessingData())
+    ev = pipeline.attach_tracer_event(
+        step,
+        tracer,
+        include_rendered_trace=True,
+        include_rendered_config=True,
+        rendered_format="text/plain",
+    )
+
+    trace_msgs = [m for m in ev.messages if m.get("kind") in {"rendered_trace", "rendered_trace_error"}]
+    assert trace_msgs, "Expected a rendered_trace (or error) block"
+    m = trace_msgs[0]
+
+    if m.get("kind") == "rendered_trace":
+        assert "Step A" in m.get("content", "")
+
+
+def test_attach_tracer_event_embeds_rendered_trace_and_config():
+    step = DummyStep(io_sources=None, step_id="A")
+    pipeline = Pipeline.from_dict({step: []}, name="t")
+
+    tracer = PipelineTracer(watch={"sample": ["signal"]}, record_only_on_change=False)
+    tracer.after_step(step, ProcessingData())
+
+    ev = pipeline.attach_tracer_event(
+        step,
+        tracer,
+        include_rendered_trace=True,
+        include_rendered_config=True,
+        rendered_format="text/plain",
+    )
+
+    kinds = {m.get("kind") for m in ev.messages}
+    assert "rendered_config" in kinds
+    assert "rendered_trace" in kinds  # since record_only_on_change=False guarantees a step event exists
+
+    # step-local sanity: rendered trace header contains step id
+    trace_blocks = [m for m in ev.messages if m.get("kind") == "rendered_trace"]
+    if trace_blocks:
+        assert "Step A" in trace_blocks[0].get("content", "")
+
+
+def test_attach_tracer_event_copies_duration():
+    step = DummyStep(io_sources=None, step_id="A")
+    pipeline = Pipeline.from_dict({step: []}, name="t")
+
+    tracer = PipelineTracer(watch={"sample": ["signal"]}, record_only_on_change=False)
+    tracer.after_step(step, ProcessingData(), duration_s=0.0123)
+
+    ev = pipeline.attach_tracer_event(step, tracer)
+    assert ev.duration_s == 0.0123