pymmcore-plus 0.10.2__py3-none-any.whl → 0.11.1__py3-none-any.whl

pymmcore_plus/mda/_runner.py

@@ -210,6 +210,10 @@ class MDARunner:
         if error is not None:
             raise error
 
+    def seconds_elapsed(self) -> float:
+        """Return the number of seconds since the start of the acquisition."""
+        return time.perf_counter() - self._t0
+
     def _outputs_connected(
         self, output: SingleOutput | Sequence[SingleOutput] | None
     ) -> ContextManager:
@@ -268,6 +272,7 @@ class MDARunner:
         teardown_event = getattr(engine, "teardown_event", lambda e: None)
         event_iterator = getattr(engine, "event_iterator", iter)
         _events: Iterator[MDAEvent] = event_iterator(events)
+        self._reset_timer()
 
         for event in _events:
             # If cancelled break out of the loop
@@ -279,13 +284,17 @@ class MDARunner:
             engine.setup_event(event)
 
             try:
+                elapsed_ms = self.seconds_elapsed() * 1000
+                # this is a bit of a hack to pass the time into the engine
+                # it is used for intra-event time calculations
+                # we pop it off after the event is executed.
+                event.metadata["runner_t0"] = self._t0
                 output = engine.exec_event(event) or ()  # in case output is None
-
                 for payload in output:
                     img, event, meta = payload
-                    if "PerfCounter" in meta:
-                        meta["ElapsedTime-ms"] = (meta["PerfCounter"] - self._t0) * 1000
-                    meta["Event"] = event
+                    event.metadata.pop("runner_t0", None)
+                    if "runner_time_ms" not in meta:
+                        meta["runner_time_ms"] = elapsed_ms
                     with exceptions_logged():
                         self._signals.frameReady.emit(img, event, meta)
             finally:
@@ -308,18 +317,13 @@ class MDARunner:
         self._sequence = sequence
 
         meta = self._engine.setup_sequence(sequence)
-        logger.info("MDA Started: %s", sequence)
-
         self._signals.sequenceStarted.emit(sequence, meta or {})
-        self._reset_timer()
+        logger.info("MDA Started: %s", sequence)
         return self._engine
 
     def _reset_timer(self) -> None:
         self._t0 = time.perf_counter()  # reference time, in seconds
 
-    def _time_elapsed(self) -> float:
-        return time.perf_counter() - self._t0
-
     def _check_canceled(self) -> bool:
         """Return True if the cancel method has been called and emit relevant signals.
 
@@ -370,7 +374,7 @@ class MDARunner:
             go_at = event.min_start_time + self._paused_time
             # We need to enter a loop here checking paused and canceled.
             # otherwise you'll potentially wait a long time to cancel
-            remaining_wait_time = go_at - self._time_elapsed()
+            remaining_wait_time = go_at - self.seconds_elapsed()
             while remaining_wait_time > 0:
                 self._signals.awaitingEvent.emit(event, remaining_wait_time)
                 while self._paused and not self._canceled:
@@ -381,7 +385,7 @@ class MDARunner:
                 if self._canceled:
                     break
                 time.sleep(min(remaining_wait_time, 0.5))
-                remaining_wait_time = go_at - self._time_elapsed()
+                remaining_wait_time = go_at - self.seconds_elapsed()
 
         # check canceled again in case it was canceled
         # during the waiting loop
@@ -403,12 +407,3 @@ class MDARunner:
 
         logger.info("MDA Finished: %s", sequence)
         self._signals.sequenceFinished.emit(sequence)
-
-
-def _assert_handler(handler: Any) -> None:
-    if (
-        not hasattr(handler, "start")
-        or not hasattr(handler, "finish")
-        or not hasattr(handler, "put")
-    ):
-        raise TypeError("Handler must have start, finish, and put methods.")

pymmcore_plus/mda/events/_protocol.py

@@ -8,7 +8,11 @@ class PMDASignaler(Protocol):
     """Declares the protocol for all signals that will be emitted from [`pymmcore_plus.mda.MDARunner`][]."""  # noqa: E501
 
     sequenceStarted: PSignal
-    """Emits `(sequence: MDASequence, metadata: dict)` when an acquisition sequence is started."""  # noqa: E501
+    """Emits `(sequence: MDASequence, metadata: dict)` when an acquisition sequence is started.
+
+    For the default [`MDAEngine`][pymmcore_plus.mda.MDAEngine], the metadata `dict` will
+    be of type [`SummaryMetaV1`][pymmcore_plus.metadata.schema.SummaryMetaV1].
+    """  # noqa: E501
     sequencePauseToggled: PSignal
     """Emits `(paused: bool)` when an acquisition sequence is paused or unpaused."""
     sequenceCanceled: PSignal
@@ -16,7 +20,11 @@ class PMDASignaler(Protocol):
     sequenceFinished: PSignal
     """Emits `(sequence: MDASequence)` when an acquisition sequence is finished."""
     frameReady: PSignal
-    """Emits `(img: np.ndarray, event: MDAEvent, metadata: dict)` after an image is acquired during an acquisition sequence."""  # noqa: E501
+    """Emits `(img: np.ndarray, event: MDAEvent, metadata: dict)` after an image is acquired during an acquisition sequence.
+
+    For the default [`MDAEngine`][pymmcore_plus.mda.MDAEngine], the metadata `dict` will
+    be of type [`FrameMetaV1`][pymmcore_plus.metadata.schema.FrameMetaV1].
+    """  # noqa: E501
     awaitingEvent: PSignal
     """Emits `(event: MDAEvent, remaining_sec: float)` when the runner is waiting to start an event.
 

pymmcore_plus/mda/handlers/_5d_writer_base.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-import json
+import warnings
 from abc import abstractmethod
 from collections import defaultdict
 from typing import TYPE_CHECKING, Generic, Mapping, Protocol, TypeVar
@@ -11,10 +11,13 @@ if TYPE_CHECKING:
     import numpy as np
     import useq
 
+    from pymmcore_plus.metadata import FrameMetaV1, SummaryMetaV1
+
     class SupportsSetItem(Protocol):
         def __setitem__(self, key: tuple[int, ...], value: np.ndarray) -> None: ...
 
 
+_NULL = object()
 POS_PREFIX = "p"
 T = TypeVar("T", bound="SupportsSetItem")
 
@@ -66,7 +69,7 @@ class _5DWriterBase(Generic[T]):
 
         # storage of individual frame metadata
         # maps position key to list of frame metadata
-        self.frame_metadatas: defaultdict[str, list[dict]] = defaultdict(list)
+        self.frame_metadatas: defaultdict[str, list[FrameMetaV1]] = defaultdict(list)
 
         # set during sequenceStarted and cleared during sequenceFinished
         self.current_sequence: useq.MDASequence | None = None
@@ -95,8 +98,19 @@ class _5DWriterBase(Generic[T]):
         """
         return self._position_sizes
 
-    def sequenceStarted(self, seq: useq.MDASequence) -> None:
+    def sequenceStarted(
+        self, seq: useq.MDASequence, meta: SummaryMetaV1 | object = _NULL
+    ) -> None:
         """On sequence started, simply store the sequence."""
+        # this is here for backwards compatibility with experimental viewer widget.
+        if meta is _NULL:  # pragma: no cover
+            warnings.warn(
+                "calling `sequenceStarted` without metadata as the second argument is "
+                "deprecated and will raise an exception in the future. Please propagate"
+                " metadata from the event callback.",
+                UserWarning,
+                stacklevel=2,
+            )
         self.frame_metadatas.clear()
         self.current_sequence = seq
         if seq:
@@ -115,7 +129,9 @@ class _5DWriterBase(Generic[T]):
         """
         return f"{POS_PREFIX}{position_index}"
 
-    def frameReady(self, frame: np.ndarray, event: useq.MDAEvent, meta: dict) -> None:
+    def frameReady(
+        self, frame: np.ndarray, event: useq.MDAEvent, meta: FrameMetaV1
+    ) -> None:
         """Write frame to the zarr array for the appropriate position."""
         # get the position key to store the array in the group
         p_index = event.index.get("p", 0)
@@ -142,8 +158,8 @@ class _5DWriterBase(Generic[T]):
 
         index = tuple(event.index[k] for k in pos_sizes)
         t = event.index.get("t", 0)
-        if t >= len(self._timestamps) and "ElapsedTime-ms" in meta:
-            self._timestamps.append(meta["ElapsedTime-ms"])
+        if t >= len(self._timestamps) and "runner_time_ms" in meta:
+            self._timestamps.append(meta["runner_time_ms"])
         self.write_frame(ary, index, frame)
         self.store_frame_metadata(key, event, meta)
 
@@ -189,7 +205,9 @@ class _5DWriterBase(Generic[T]):
         # WRITE DATA TO DISK
         ary[index] = frame
 
-    def store_frame_metadata(self, key: str, event: useq.MDAEvent, meta: dict) -> None:
+    def store_frame_metadata(
+        self, key: str, event: useq.MDAEvent, meta: FrameMetaV1
+    ) -> None:
         """Called during each frameReady event to store metadata for the frame.
 
         Subclasses may override this method to customize how metadata is stored for each
@@ -208,11 +226,6 @@ class _5DWriterBase(Generic[T]):
         # needn't be re-implemented in subclasses
         # default implementation is to store the metadata in self._frame_metas
         # use finalize_metadata to write to disk at the end of the sequence.
-        if meta:
-            # fix serialization MDAEvent
-            # XXX: There is already an Event object in meta, this overwrites it.
-            event_json = event.json(exclude={"sequence"}, exclude_defaults=True)
-            meta["Event"] = json.loads(event_json)
         self.frame_metadatas[key].append(meta or {})
 
     def finalize_metadata(self) -> None:
@@ -266,7 +279,6 @@ class _5DWriterBase(Generic[T]):
             raise IndexError(
                 f"Position index {p_index} out of range for {len(self.position_sizes)}"
             ) from e
-
         data = self.position_arrays[self.get_position_key(p_index)]
         full = slice(None, None)
         index = tuple(indexers.get(k, full) for k in sizes)

pymmcore_plus/mda/handlers/_img_sequence_writer.py

@@ -7,11 +7,12 @@ provided.
 
 from __future__ import annotations
 
-import json
 from itertools import count
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, ClassVar, Mapping, Sequence, cast
 
+from pymmcore_plus.metadata.serialize import json_dumps
+
 from ._util import get_full_sequence_axes
 
 if TYPE_CHECKING:
@@ -176,11 +177,13 @@ class ImageSequenceWriter:
                 include_frame_count=self._include_frame_count,
             )
             # make directory and write metadata
-            self._seq_meta_file.write_text(seq.json(exclude_unset=True, indent=4))
+            self._seq_meta_file.write_text(
+                seq.model_dump_json(exclude_unset=True, indent=2)
+            )
 
     def sequenceFinished(self, seq: useq.MDASequence) -> None:
         # write final frame metadata to disk
-        self._frame_meta_file.write_text(json.dumps(self._frame_metadata, indent=2))
+        self._frame_meta_file.write_bytes(json_dumps(self._frame_metadata, indent=2))
 
     def frameReady(self, frame: np.ndarray, event: useq.MDAEvent, meta: dict) -> None:
         """Write a frame to disk."""
@@ -199,11 +202,12 @@ class ImageSequenceWriter:
         self._imwrite(str(self._directory / filename), frame, **self._imwrite_kwargs)
 
         # store metadata
-        meta["Event"] = json.loads(event.json(exclude={"sequence"}, exclude_unset=True))
         self._frame_metadata[filename] = meta
         # write metadata to disk every 10 frames
         if frame_idx % 10 == 0:
-            self._frame_meta_file.write_text(json.dumps(self._frame_metadata, indent=2))
+            self._frame_meta_file.write_bytes(
+                json_dumps(self._frame_metadata, indent=2)
+            )
 
     @staticmethod
     def fname_template(

pymmcore_plus/mda/handlers/_ome_tiff_writer.py

@@ -41,13 +41,15 @@ from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
-from ._5d_writer_base import _5DWriterBase
+from ._5d_writer_base import _NULL, _5DWriterBase
 
 if TYPE_CHECKING:
     from pathlib import Path
 
     import useq
 
+    from pymmcore_plus.metadata import SummaryMetaV1
+
 IMAGEJ_AXIS_ORDER = "tzcyxs"
 
 
@@ -81,8 +83,10 @@ class OMETiffWriter(_5DWriterBase[np.memmap]):
 
         super().__init__()
 
-    def sequenceStarted(self, seq: useq.MDASequence) -> None:
-        super().sequenceStarted(seq)
+    def sequenceStarted(
+        self, seq: useq.MDASequence, meta: SummaryMetaV1 | object = _NULL
+    ) -> None:
+        super().sequenceStarted(seq, meta)
         # Non-OME (ImageJ) hyperstack axes MUST be in TZCYXS order
         # so we reorder the ordered position_sizes dicts.  This will ensure
         # that the array indices created from event.index are in the correct order.

pymmcore_plus/mda/handlers/_ome_zarr_writer.py

@@ -10,6 +10,8 @@ from typing import TYPE_CHECKING, Any, Literal, MutableMapping, Protocol
 
 import numpy as np
 
+from pymmcore_plus.metadata.serialize import to_builtins
+
 from ._5d_writer_base import _5DWriterBase
 
 if TYPE_CHECKING:
@@ -193,7 +195,7 @@ class OMEZarrWriter(_5DWriterBase["zarr.Array"]):
         while self.frame_metadatas:
             key, metas = self.frame_metadatas.popitem()
             if key in self.position_arrays:
-                self.position_arrays[key].attrs["frame_meta"] = metas
+                self.position_arrays[key].attrs["frame_meta"] = to_builtins(metas)
 
         if self._minify_metadata:
             self._minify_zattrs_metadata()
@@ -209,12 +211,12 @@ class OMEZarrWriter(_5DWriterBase["zarr.Array"]):
             return
 
         sizes = {**seq.sizes}
-        px = 1
+        px: float = 1.0
         if self.frame_metadatas:
             key, metas = next(iter(self.frame_metadatas.items()))
             if key in self.position_arrays:
                 shape = self.position_arrays[key].shape
-                px = metas[-1].get("PixelSizeUm", 1)
+                px = metas[-1].get("pixel_size_um", 1)
                 with suppress(IndexError):
                     sizes.update(y=shape[-2], x=shape[-1])
 
@@ -274,7 +276,10 @@ class OMEZarrWriter(_5DWriterBase["zarr.Array"]):
         self._group.attrs["multiscales"] = scales
         ary.attrs["_ARRAY_DIMENSIONS"] = dims
         if seq := self.current_sequence:
-            ary.attrs["useq_MDASequence"] = json.loads(seq.json(exclude_unset=True))
+            ary.attrs["useq_MDASequence"] = to_builtins(
+                seq.model_dump(exclude_unset=True)
+            )
+
         return ary
 
     # # the superclass implementation is all we need

pymmcore_plus/mda/handlers/_tensorstore_handler.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import atexit
-import json
 import os
 import shutil
 import tempfile
@@ -10,14 +9,19 @@ from itertools import product
 from os import PathLike
 from typing import TYPE_CHECKING, Any, cast
 
+from pymmcore_plus.metadata.serialize import json_dumps, json_loads
+
 from ._util import position_sizes
 
 if TYPE_CHECKING:
-    from typing import Literal, Mapping, Self, Sequence, TypeAlias
+    from typing import Literal, Mapping, Sequence, TypeAlias
 
     import numpy as np
     import tensorstore as ts
     import useq
+    from typing_extensions import Self  # py311
+
+    from pymmcore_plus.metadata import FrameMetaV1, SummaryMetaV1
 
     TsDriver: TypeAlias = Literal["zarr", "zarr3", "n5", "neuroglancer_precomputed"]
     EventKey: TypeAlias = frozenset[tuple[str, int]]
@@ -107,12 +111,12 @@ class TensorStoreHandler:
 
         # storage of individual frame metadata
         # maps position key to list of frame metadata
-        self.frame_metadatas: list[tuple[useq.MDAEvent, dict]] = []
+        self.frame_metadatas: list[tuple[useq.MDAEvent, FrameMetaV1]] = []
 
         self._size_increment = 300
 
         self._store: ts.TensorStore | None = None
-        self._futures: list[ts.Future] = []
+        self._futures: list[ts.Future | ts.WriteFutures] = []
         self._frame_indices: dict[EventKey, int | ts.DimExpression] = {}
 
         # "_nd_storage" means we're greedily attempting to store the data in a
@@ -170,7 +174,7 @@ class TensorStoreHandler:
 
         return cls(path=path, **kwargs)
 
-    def sequenceStarted(self, seq: useq.MDASequence) -> None:
+    def sequenceStarted(self, seq: useq.MDASequence, meta: SummaryMetaV1) -> None:
         """On sequence started, simply store the sequence."""
         self._frame_index = 0
         self._store = None
@@ -192,7 +196,9 @@ class TensorStoreHandler:
         if self.frame_metadatas:
             self.finalize_metadata()
 
-    def frameReady(self, frame: np.ndarray, event: useq.MDAEvent, meta: dict) -> None:
+    def frameReady(
+        self, frame: np.ndarray, event: useq.MDAEvent, meta: FrameMetaV1
+    ) -> None:
         """Write frame to the zarr array for the appropriate position."""
         if self._store is None:
             self._store = self.new_store(frame, event.sequence, meta).result()
@@ -230,7 +236,7 @@ class TensorStoreHandler:
         return self._store[ts_index].read().result().squeeze()  # type: ignore
 
     def new_store(
-        self, frame: np.ndarray, seq: useq.MDASequence | None, meta: dict
+        self, frame: np.ndarray, seq: useq.MDASequence | None, meta: FrameMetaV1
     ) -> ts.Future[ts.TensorStore]:
         shape, chunks, labels = self.get_shape_chunks_labels(frame.shape, seq)
         self._nd_storage = FRAME_DIM not in labels
@@ -287,14 +293,7 @@ class TensorStoreHandler:
         if not (store := self._store) or not store.kvstore:
             return  # pragma: no cover
 
-        data = []
-        for event, meta in self.frame_metadatas:
-            # FIXME: unnecessary ser/des
-            js = event.model_dump_json(exclude={"sequence"}, exclude_defaults=True)
-            meta["Event"] = json.loads(js)
-            data.append(meta)
-
-        metadata = {"frame_metadatas": data}
+        metadata = {"frame_metadatas": [m[1] for m in self.frame_metadatas]}
         if not self._nd_storage:
             metadata["frame_indices"] = [
                 (tuple(dict(k).items()), v)  # type: ignore
@@ -302,11 +301,11 @@ class TensorStoreHandler:
             ]
 
         if self.ts_driver.startswith("zarr"):
-            store.kvstore.write(".zattrs", json.dumps(metadata))
+            store.kvstore.write(".zattrs", json_dumps(metadata).decode("utf-8"))
         elif self.ts_driver == "n5":  # pragma: no cover
-            attrs = json.loads(store.kvstore.read("attributes.json").result().value)
+            attrs = json_loads(store.kvstore.read("attributes.json").result().value)
             attrs.update(metadata)
-            store.kvstore.write("attributes.json", json.dumps(attrs))
+            store.kvstore.write("attributes.json", json_dumps(attrs).decode("utf-8"))
 
     def _expand_store(self, store: ts.TensorStore) -> ts.Future[ts.TensorStore]:
         """Grow the store by `self._size_increment` frames.
@@ -324,7 +323,8 @@ class TensorStoreHandler:
         The return value is safe to use as an index to self._store[...]
         """
         if self._nd_storage:
-            return self._ts.d[index][tuple(index.values())]
+            keys, values = zip(*index.items())
+            return self._ts.d[keys][values]
 
         if any(isinstance(v, slice) for v in index.values()):
             idx: list | int | ts.DimExpression = self._get_frame_indices(index)

pymmcore_plus/metadata/__init__.py

@@ -0,0 +1,36 @@
+from .functions import frame_metadata, summary_metadata
+from .schema import (
+    ConfigGroup,
+    ConfigPreset,
+    DeviceInfo,
+    FrameMetaV1,
+    ImageInfo,
+    PixelSizeConfigPreset,
+    Position,
+    PropertyInfo,
+    PropertyValue,
+    StagePosition,
+    SummaryMetaV1,
+    SystemInfo,
+)
+from .serialize import json_dumps, to_builtins
+
+__all__ = [
+    "ConfigGroup",
+    "ConfigPreset",
+    "ConfigPreset",
+    "DeviceInfo",
+    "frame_metadata",
+    "FrameMetaV1",
+    "ImageInfo",
+    "json_dumps",
+    "PixelSizeConfigPreset",
+    "Position",
+    "PropertyInfo",
+    "PropertyValue",
+    "StagePosition",
+    "summary_metadata",
+    "SummaryMetaV1",
+    "SystemInfo",
+    "to_builtins",
+]