bec-server 4.0.0__tar.gz → 4.0.2__tar.gz

{bec_server-4.0.0 → bec_server-4.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: bec-server
-Version: 4.0.0
+Version: 4.0.2
 Summary: BEC server
 Project-URL: Bug Tracker, https://github.com/bec-project/bec/issues
 Project-URL: Homepage, https://github.com/bec-project/bec

{bec_server-4.0.0 → bec_server-4.0.2}/bec_server/file_writer/async_writer.py

@@ -65,7 +65,9 @@ class AsyncWriter(threading.Thread):
         "index": int
     }
 
-    "index" is only required for the 'add_slice' type and specifies the row index to append the data to.
+    "index" is required for 'add_slice' and must be nonnegative. It selects the row whose
+    contents are extended and allows revisiting earlier rows. Use 'add' to append new rows
+    along the first axis; 'add' ignores "index".
     "max_shape" is required for the 'add' and 'add_slice' types and specifies the maximum shape of the dataset. If the dataset is 1D, 'max_shape' should be [None].
 
     """
@@ -301,7 +303,12 @@ class AsyncWriter(threading.Thread):
                         if key == "value":
                             self.write_value_data(signal_group, value, async_update)
                         elif key == "timestamp":
-                            self.write_timestamp_data(signal_group, value)
+                            self.write_timestamp_data(
+                                signal_group,
+                                value,
+                                replace=isinstance(async_update, dict)
+                                and async_update.get("type") == "replace",
+                            )
                         else:  # pragma: no cover
                             # this should never happen as the keys are fixed in the pydantic model
                             msg = f"Unknown key: {key}. Data will not be written."
@@ -352,7 +359,7 @@ class AsyncWriter(threading.Thread):
 
         elif update_type == "replace":
             # store the data to be written after the scan is complete
-            self.device_data_replace[signal_group.name] = value
+            self.device_data_replace[signal_group.name] = np.asarray(value)
         else:
             msg = f"Unknown async update type: {update_type}. Data will not be written."
             self.connector.raise_alarm(
@@ -444,7 +451,11 @@ class AsyncWriter(threading.Thread):
 
         if len(max_shape) != 2:
             # We currently only support 2D datasets for the 'add_slice' async update type
-            msg = f"Invalid max_shape for signal group {signal_group.name} async update type 'add_slice': {max_shape}. max_shape cannot exceed two dimensions. Data will not be written."
+            msg = (
+                f"Invalid max_shape for signal group {signal_group.name} async update type "
+                f"'add_slice': {max_shape}. max_shape must have exactly two dimensions. "
+                "Data will not be written."
+            )
             self.connector.raise_alarm(
                 severity=Alarms.WARNING,
                 info=messages.ErrorInfo(
@@ -474,43 +485,41 @@ class AsyncWriter(threading.Thread):
         # if max_shape contains more than one None, we have to use a vlen_dtype
         num_undefined = sum(1 for i in max_shape if i is None)
         row_index = async_update["index"]
+        if row_index < 0 or (max_shape[0] is not None and row_index >= max_shape[0]):
+            msg = (
+                f"Invalid row index {row_index} for signal group {signal_group.name}: "
+                "'add_slice' updates require a nonnegative row index "
+                f"within max_shape {max_shape}. "
+                "Data will not be written."
+            )
+            self.connector.raise_alarm(
+                severity=Alarms.WARNING,
+                info=messages.ErrorInfo(
+                    error_message=msg,
+                    compact_error_message=msg,
+                    exception_type="ValueError",
+                    device=signal_group.name,
+                ),
+                metadata={"scan_id": self.scan_id, "scan_number": self.scan_number},
+            )
+            return
 
         if "value" not in signal_group:
             if num_undefined > 1:
-                row_index = 0
-                shape = (1,) if value.ndim < len(max_shape) else (value.shape[0],)
                 signal_group.create_dataset(
                     "value",
-                    shape=shape,
+                    shape=(row_index + 1,),
                     maxshape=(None,),
                     dtype=h5py.vlen_dtype(np.dtype(value[0].dtype)),
                 )
                 signal_group["value"][row_index] = value
             else:
-                if value.ndim < len(max_shape):
-                    value = value.reshape((1,) + value.shape)
-                if value.shape[1] > max_shape[1]:
-                    msg = f"Data for {signal_group.name} exceeds the defined max_shape {max_shape}. Data will be truncated."
-                    self.connector.raise_alarm(
-                        severity=Alarms.WARNING,
-                        info=messages.ErrorInfo(
-                            error_message=msg,
-                            compact_error_message=msg,
-                            exception_type="ValueError",
-                            device=signal_group.name,
-                        ),
-                        metadata={"scan_id": self.scan_id, "scan_number": self.scan_number},
-                    )
-                    value = value[:, : max_shape[1]]
-                create_dataset_safe(signal_group, "value", data=value, maxshape=max_shape)
-                self.cursor[signal_group.name][row_index] = value.shape[1]
+                self._write_initial_fixed_slice(signal_group, value, max_shape, row_index)
             return
 
         # add a slice to the already existing dataset
         if num_undefined > 1:
             max_index = signal_group["value"].shape[0]
-            if row_index == -1:
-                row_index = max_index + 1
             if row_index >= max_index:
                 signal_group["value"].resize((row_index + 1,))
                 signal_group["value"][row_index] = value
@@ -532,23 +541,68 @@ class AsyncWriter(threading.Thread):
                     metadata={"scan_id": self.scan_id, "scan_number": self.scan_number},
                 )
                 value = value[: max_shape[1] - col_index]
-            signal_group["value"].resize((row_index + 1, max_shape[1]))
+            signal_group["value"].resize(
+                (max(signal_group["value"].shape[0], row_index + 1), max_shape[1])
+            )
             signal_group["value"][row_index, col_index : col_index + len(value)] = value
             self.cursor[signal_group.name][row_index] = col_index + len(value)
 
-    def write_timestamp_data(self, signal_group, value):
+    def _write_initial_fixed_slice(
+        self,
+        signal_group: h5py.Group,
+        value: np.ndarray,
+        max_shape: list[int | None],
+        row_index: int,
+    ) -> None:
+        """Create a fixed-width slice dataset at the validated initial row index."""
+        value = value.reshape((1,) + value.shape)
+        if value.shape[1] > max_shape[1]:
+            msg = (
+                f"Data for {signal_group.name} exceeds the defined max_shape {max_shape}. "
+                "Data will be truncated."
+            )
+            self.connector.raise_alarm(
+                severity=Alarms.WARNING,
+                info=messages.ErrorInfo(
+                    error_message=msg,
+                    compact_error_message=msg,
+                    exception_type="ValueError",
+                    device=signal_group.name,
+                ),
+                metadata={"scan_id": self.scan_id, "scan_number": self.scan_number},
+            )
+            value = value[:, : max_shape[1]]
+        dataset = create_dataset_safe(signal_group, "value", data=value, maxshape=max_shape)
+        if row_index > 0:
+            # Keep dtype inference from the data and let HDF5 fill the skipped rows.
+            dataset.resize((0, value.shape[1]))
+            dataset.resize((row_index + 1, value.shape[1]))
+            if value.size:
+                dataset.write_direct(
+                    np.ascontiguousarray(value), dest_sel=np.s_[row_index : row_index + 1, :]
+                )
+        self.cursor[signal_group.name][row_index] = value.shape[1]
+
+    def write_timestamp_data(
+        self, signal_group: h5py.Group, value: Any, replace: bool = False
+    ) -> None:
         """
         Write the timestamp data to the file.
         Timestamp data is always written as a 1D array, irrespective of the async update type.
+        Replacement updates retain only the latest timestamps.
 
         Args:
             signal_group (h5py.Group): The group to write the data to
-            value (list): The timestamp data to write
+            value (Any): The timestamp data to write
+            replace (bool): Whether to replace existing timestamps instead of appending
         """
         if not isinstance(value, (list, np.ndarray)):
             value = [value]
         if "timestamp" not in signal_group:
             create_dataset_safe(signal_group, "timestamp", data=value, maxshape=(None,))
+        elif replace:
+            signal_group["timestamp"].resize((len(value),))
+            signal_group["timestamp"][:] = value
         else:
             signal_group["timestamp"].resize((len(signal_group["timestamp"]) + len(value),))
             signal_group["timestamp"][-len(value) :] = value

{bec_server-4.0.0 → bec_server-4.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "bec-server"
-version = "4.0.0"
+version = "4.0.2"
 description = "BEC server"
 requires-python = ">=3.11"
 classifiers = [

{bec_server-4.0.0 → bec_server-4.0.2}/tests/tests_file_writer/test_async_file_writer.py

@@ -210,6 +210,37 @@ def test_async_writer_add_slice_var_size(async_writer, data):
     assert out[1].shape == (10,)
 
 
+@pytest.mark.parametrize("initial_row", [1, 3])
+@pytest.mark.parametrize("dtype", [np.int16, np.float64])
+def test_async_writer_add_slice_var_size_starts_at_requested_row(
+    async_writer, tmp_path, initial_row, dtype
+):
+    """A sparse first vlen row retains its data when earlier rows and more slices arrive."""
+    with h5py.File(tmp_path / "initial_vlen_row.h5", "w") as file:
+        signal_group = file.create_group("signal")
+        async_writer.write_value_data(
+            signal_group,
+            np.array([7, 8], dtype=dtype),
+            {"type": "add_slice", "index": initial_row, "max_shape": [None, None]},
+        )
+        dataset = signal_group["value"]
+        assert dataset.shape == (initial_row + 1,)
+        assert h5py.check_vlen_dtype(dataset.dtype) == np.dtype(dtype)
+        assert all(row.size == 0 for row in dataset[:initial_row])
+        np.testing.assert_array_equal(dataset[initial_row], [7, 8])
+
+        for row_index, values in [(0, [1, 2]), (initial_row, [9]), (0, [3])]:
+            async_writer.write_value_data(
+                signal_group,
+                np.array(values, dtype=dtype),
+                {"type": "add_slice", "index": row_index, "max_shape": [None, None]},
+            )
+        assert dataset.shape == (initial_row + 1,)
+        np.testing.assert_array_equal(dataset[0], [1, 2, 3])
+        np.testing.assert_array_equal(dataset[initial_row], [7, 8, 9])
+        assert all(row.size == 0 for row in dataset[1:initial_row])
+
+
 def test_async_writer_add_slice_var_size_2D_data_warns(async_writer):
     """
     Test that adding a slice with 2D data when max_shape is [None, None] raises a warning and skips writing the data.
@@ -271,6 +302,153 @@ def test_async_writer_add_slice_fixed_size(async_writer, data):
     assert out.shape == (2, 20)
 
 
+@pytest.mark.parametrize("max_rows", [None, 4])
+@pytest.mark.parametrize("later_row", [1, 3])
+@pytest.mark.parametrize("earlier_slice", [[3], [3, 4, 5]])
+def test_async_writer_add_slice_fixed_size_preserves_later_rows(
+    async_writer, tmp_path, max_rows, later_row, earlier_slice
+):
+    """Interleaved slices preserve rows and append positions, including after truncation."""
+    with h5py.File(tmp_path / "interleaved.h5", "w") as file:
+        signal_group = file.create_group("signal")
+        with mock.patch.object(async_writer.connector, "raise_alarm") as raise_alarm:
+            for row_index, values in [(0, [1, 2]), (later_row, [10, 11]), (0, earlier_slice)]:
+                async_writer.write_value_data(
+                    signal_group,
+                    values,
+                    {"type": "add_slice", "index": row_index, "max_shape": [max_rows, 4]},
+                )
+
+            expected = np.zeros((later_row + 1, 4), dtype=int)
+            row_values = [1, 2] + earlier_slice[:2]
+            expected[0, : len(row_values)] = row_values
+            expected[later_row, :2] = [10, 11]
+            np.testing.assert_array_equal(signal_group["value"][:], expected)
+            assert raise_alarm.call_count == (len(earlier_slice) > 2)
+
+            async_writer.write_value_data(
+                signal_group,
+                [12, 13],
+                {"type": "add_slice", "index": later_row, "max_shape": [max_rows, 4]},
+            )
+            expected[later_row] = [10, 11, 12, 13]
+            np.testing.assert_array_equal(signal_group["value"][:], expected)
+
+
+@pytest.mark.parametrize("max_rows", [None, 4])
+@pytest.mark.parametrize("initial_values", [[7, 8], [7, 8, 9, 10, 11], []])
+@pytest.mark.parametrize("strided", [False, True])
+def test_async_writer_add_slice_fixed_size_starts_at_requested_row(
+    async_writer, tmp_path, max_rows, initial_values, strided
+):
+    """A nonzero first row keeps its data and cursor when earlier rows arrive later."""
+    with h5py.File(tmp_path / "initial_row.h5", "w") as file:
+        signal_group = file.create_group("signal")
+        values = np.asarray(initial_values, dtype=np.int16)
+        if strided:
+            values = np.repeat(values, 2)[::2]
+        async_writer.write_value_data(
+            signal_group, values, {"type": "add_slice", "index": 3, "max_shape": [max_rows, 4]}
+        )
+        expected_initial = np.zeros((4, min(len(initial_values), 4)), dtype=np.int16)
+        expected_initial[3] = initial_values[:4]
+        np.testing.assert_array_equal(signal_group["value"][:], expected_initial)
+        assert signal_group["value"].dtype == np.dtype(np.int16)
+
+        for row_index, values in [(0, [1, 2]), (3, [12, 13])]:
+            async_writer.write_value_data(
+                signal_group,
+                values,
+                {"type": "add_slice", "index": row_index, "max_shape": [max_rows, 4]},
+            )
+        expected = np.zeros((4, 4), dtype=np.int16)
+        expected[0, :2] = [1, 2]
+        row_values = (initial_values + [12, 13])[:4]
+        expected[3, : len(row_values)] = row_values
+        np.testing.assert_array_equal(signal_group["value"][:], expected)
+
+
+def test_async_writer_add_slice_fixed_size_nonzero_first_row_preserves_string_dtype(
+    async_writer, tmp_path
+):
+    """Moving the initial row preserves HDF5's inference of variable-length strings."""
+    with h5py.File(tmp_path / "initial_string_row.h5", "w") as file:
+        signal_group = file.create_group("signal")
+        async_writer.write_value_data(
+            signal_group,
+            np.array(["a", "bb"], dtype=object),
+            {"type": "add_slice", "index": 2, "max_shape": [None, 4]},
+        )
+        dataset = signal_group["value"]
+        np.testing.assert_array_equal(dataset.asstr()[:], [["", ""], ["", ""], ["a", "bb"]])
+        assert h5py.check_string_dtype(dataset.dtype).length is None
+
+
+@pytest.mark.parametrize("cells", [[[1, 2], [3]], [[1, 2], [3, 4]], [[1, 2]], [[], []], [[]]])
+def test_async_writer_add_slice_fixed_size_nonzero_first_row_preserves_vlen_dtype(
+    async_writer, tmp_path, cells
+):
+    """HDF5 initializes skipped rows correctly for variable-length numeric values."""
+    values = np.empty(len(cells), dtype=h5py.vlen_dtype(np.dtype(np.int32)))
+    for index, cell in enumerate(cells):
+        values[index] = np.array(cell, dtype=np.int32)
+    with h5py.File(tmp_path / "initial_vlen_row.h5", "w") as file:
+        signal_group = file.create_group("signal")
+        async_writer.write_value_data(
+            signal_group, values, {"type": "add_slice", "index": 2, "max_shape": [None, 4]}
+        )
+        dataset = signal_group["value"]
+        assert dataset.shape == (3, len(cells))
+        assert h5py.check_vlen_dtype(dataset.dtype) == np.dtype(np.int32)
+        assert all(cell.size == 0 for row in dataset[:2] for cell in row)
+        for index, value in enumerate(values):
+            np.testing.assert_array_equal(dataset[2, index], value)
+
+
+@pytest.mark.parametrize("populated", [False, True])
+@pytest.mark.parametrize("index, max_rows, row_width", [(-1, None, 4), (4, 4, 4), (-1, None, None)])
+def test_async_writer_add_slice_rejects_invalid_index(
+    async_writer, tmp_path, populated, index, max_rows, row_width
+):
+    """An invalid row index cannot overwrite existing rows or initialize an invalid cursor."""
+    with h5py.File(tmp_path / "invalid_index.h5", "w") as file:
+        signal_group = file.create_group("signal")
+        if populated:
+            for row_index, values in [(0, [1, 2]), (1, [10, 11])]:
+                async_writer.write_value_data(
+                    signal_group,
+                    values,
+                    {"type": "add_slice", "index": row_index, "max_shape": [max_rows, row_width]},
+                )
+            expected = signal_group["value"][:]
+        cursor_before = dict(async_writer.cursor.get(signal_group.name, {}))
+
+        with mock.patch.object(async_writer.connector, "raise_alarm") as raise_alarm:
+            async_writer.write_value_data(
+                signal_group,
+                [99],
+                {"type": "add_slice", "index": index, "max_shape": [max_rows, row_width]},
+            )
+        raise_alarm.assert_called_once()
+        assert raise_alarm.call_args.kwargs["severity"] == Alarms.WARNING
+        assert "nonnegative row index" in raise_alarm.call_args.kwargs["info"].error_message
+        assert async_writer.cursor.get(signal_group.name, {}) == cursor_before
+        if populated:
+            assert signal_group["value"].shape == expected.shape
+            for actual_row, expected_row in zip(signal_group["value"][:], expected):
+                np.testing.assert_array_equal(actual_row, expected_row)
+        else:
+            assert "value" not in signal_group
+
+        async_writer.write_value_data(
+            signal_group, [3], {"type": "add_slice", "index": 0, "max_shape": [max_rows, row_width]}
+        )
+        expected_row = [1, 2, 3] if populated else [3]
+        if populated and row_width is not None:
+            expected_row.append(0)
+        np.testing.assert_array_equal(signal_group["value"][0], expected_row)
+
+
 def test_async_writer_add_slice_fixed_size_data_consistency(async_writer):
     endpoint = MessageEndpoints.device_async_readback("scan_id", "monitor_async")
     data = [
@@ -413,9 +591,65 @@ def test_async_writer_replace(async_writer, data):
     # read the data back
     with h5py.File(async_writer.file_path, "r") as f:
         out = f[async_writer.BASE_PATH]["monitor_async"]["monitor_async"]["value"][:]
+        timestamps = f[async_writer.BASE_PATH]["monitor_async"]["monitor_async"]["timestamp"][:]
 
     assert out.shape == (10,)
     assert np.allclose(out, data[-1].signals["monitor_async"]["value"])
+    assert np.array_equal(timestamps, [data[-1].signals["monitor_async"]["timestamp"]])
+
+
+@pytest.mark.parametrize(
+    "values",
+    [
+        (0, 7),
+        (0.0, 1.2),
+        (False, True),
+        (np.int64(0), np.int64(7)),
+        (np.float64(0.0), np.float64(1.2)),
+    ],
+)
+def test_async_writer_replace_scalar(async_writer, values):
+    """Write only the latest scalar value and timestamp after scan completion."""
+    for timestamp, value in enumerate(values, start=1):
+        message = messages.DeviceMessage(
+            signals={"monitor_async": {"value": value, "timestamp": float(timestamp)}},
+            metadata={"async_update": {"type": "replace"}},
+        )
+        async_writer.write_data({"monitor_async": [message]})
+
+    group = async_writer.file_handle[f"{async_writer.BASE_PATH}/monitor_async/monitor_async"]
+    try:
+        assert "value" not in group
+        async_writer.write_data({}, write_replace=True)
+        assert group["value"].shape == ()
+        assert group["value"][()] == values[-1]
+        assert np.array_equal(group["timestamp"][:], [2.0])
+    finally:
+        async_writer.file_handle.close()
+
+
+@pytest.mark.parametrize("update_type", ["replace", "add", "add_slice"])
+@pytest.mark.parametrize("timestamps", [[1, 2, 3], [1]])
+def test_async_writer_timestamp_aggregation(async_writer, update_type, timestamps):
+    """Keep replacement timestamps together and preserve append behavior for other updates."""
+    metadata = {"type": update_type}
+    if update_type == "add":
+        metadata["max_shape"] = [None]
+    elif update_type == "add_slice":
+        metadata.update(max_shape=[None, None], index=0)
+
+    for timestamp in (timestamps, [4, 5]):
+        message = messages.DeviceMessage(
+            signals={"monitor_async": {"value": np.array([6, 7]), "timestamp": timestamp}},
+            metadata={"async_update": metadata},
+        )
+        async_writer.write_data({"monitor_async": [message]})
+    async_writer.write_data({}, write_replace=True)
+
+    group = async_writer.file_handle[f"{async_writer.BASE_PATH}/monitor_async/monitor_async"]
+    expected = [4, 5] if update_type == "replace" else timestamps + [4, 5]
+    assert np.array_equal(group["timestamp"][:], expected)
+    async_writer.file_handle.close()
 
 
 def test_async_writer_async_signal(async_writer):