ezmsg-tools 0.3.1__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/PKG-INFO +5 -5
  2. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/pyproject.toml +6 -7
  3. ezmsg_tools-0.4.0/src/ezmsg/tools/__version__.py +24 -0
  4. ezmsg_tools-0.4.0/src/ezmsg/tools/chmeta.py +87 -0
  5. ezmsg_tools-0.4.0/src/ezmsg/tools/shmem/aux_meta.py +208 -0
  6. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/shmem/shmem.py +150 -0
  7. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/shmem/shmem_mirror.py +173 -1
  8. ezmsg_tools-0.4.0/tests/test_shmem_aux_meta.py +436 -0
  9. ezmsg_tools-0.3.1/src/ezmsg/tools/__version__.py +0 -34
  10. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/.github/workflows/docs.yml +0 -0
  11. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/.github/workflows/python-publish.yml +0 -0
  12. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/.github/workflows/python-tests.yml +0 -0
  13. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/.gitignore +0 -0
  14. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/.pre-commit-config.yaml +0 -0
  15. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/LICENSE +0 -0
  16. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/README.md +0 -0
  17. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/docs/Makefile +0 -0
  18. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/docs/make.bat +0 -0
  19. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/docs/source/_templates/autosummary/module.rst +0 -0
  20. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/docs/source/api/index.rst +0 -0
  21. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/docs/source/conf.py +0 -0
  22. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/docs/source/index.md +0 -0
  23. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/scripts_nbs/generator/eeg_generator_graph.py +0 -0
  24. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/scripts_nbs/orphan/run_attach.py +0 -0
  25. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/scripts_nbs/profiler/ecog_preproc.py +0 -0
  26. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/scripts_nbs/profiler/parse_profiler.ipynb +0 -0
  27. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/scripts_nbs/profiler/profiler.py +0 -0
  28. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/__init__.py +0 -0
  29. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/dag.py +0 -0
  30. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/perfmon/__init__.py +0 -0
  31. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/perfmon/cli.py +0 -0
  32. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/shmem/__init__.py +0 -0
  33. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/sigmon/__init__.py +0 -0
  34. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/sigmon/cli.py +0 -0
  35. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/sigmon/dag_widget.py +0 -0
  36. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/viewer/__init__.py +0 -0
  37. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/src/ezmsg/tools/viewer/cli.py +0 -0
  38. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/tests/test_shmem_mirror.py +0 -0
  39. {ezmsg_tools-0.3.1 → ezmsg_tools-0.4.0}/tests/test_shmem_sink.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ezmsg-tools
3
- Version: 0.3.1
3
+ Version: 0.4.0
4
4
  Summary: ezmsg namespace package containing tools for working with ezmsg graphs.
5
5
  Author-email: Chadwick Boulay <chadwick.boulay@gmail.com>
6
6
  License-Expression: MIT
@@ -19,15 +19,15 @@ Requires-Dist: plotly>=5.24.1; extra == 'perfmon'
19
19
  Requires-Dist: pygtail>=0.14.0; extra == 'perfmon'
20
20
  Requires-Dist: typer>=0.15.1; extra == 'perfmon'
21
21
  Provides-Extra: sigmon
22
- Requires-Dist: ezmsg-qt; extra == 'sigmon'
22
+ Requires-Dist: ezmsg-qt>=0.2.1; extra == 'sigmon'
23
23
  Requires-Dist: pandas; extra == 'sigmon'
24
- Requires-Dist: phosphor>=0.4; extra == 'sigmon'
24
+ Requires-Dist: phosphor>=0.5.0; extra == 'sigmon'
25
25
  Requires-Dist: pygraphviz>=1.14; extra == 'sigmon'
26
26
  Requires-Dist: pyside6>=6.7; extra == 'sigmon'
27
27
  Requires-Dist: typer>=0.15.1; extra == 'sigmon'
28
28
  Provides-Extra: viewer
29
- Requires-Dist: ezmsg-qt; extra == 'viewer'
30
- Requires-Dist: phosphor>=0.4; extra == 'viewer'
29
+ Requires-Dist: ezmsg-qt>=0.2.1; extra == 'viewer'
30
+ Requires-Dist: phosphor>=0.5.0; extra == 'viewer'
31
31
  Requires-Dist: pyside6>=6.7; extra == 'viewer'
32
32
  Requires-Dist: typer>=0.15.1; extra == 'viewer'
33
33
  Description-Content-Type: text/markdown
@@ -18,8 +18,8 @@ dependencies = [
18
18
  dev = [
19
19
  "pre-commit>=4.0.0",
20
20
  "scipy>=1.14.1",
21
- "ezmsg-sigproc>=2.18.0",
22
- "ezmsg-simbiophys>=1.4.1",
21
+ "ezmsg-sigproc>=2.36.0",
22
+ "ezmsg-simbiophys>=1.8.0",
23
23
  {include-group = "lint"},
24
24
  {include-group = "test"},
25
25
  ]
@@ -53,15 +53,15 @@ sigmon = [
53
53
  "PySide6>=6.7",
54
54
  "pygraphviz>=1.14",
55
55
  "typer>=0.15.1",
56
- "phosphor>=0.4",
56
+ "ezmsg-qt>=0.2.1",
57
+ "phosphor>=0.5.0",
57
58
  "pandas",
58
- "ezmsg-qt",
59
59
  ]
60
60
  viewer = [
61
61
  "PySide6>=6.7",
62
62
  "typer>=0.15.1",
63
- "phosphor>=0.4",
64
- "ezmsg-qt",
63
+ "ezmsg-qt>=0.2.1",
64
+ "phosphor>=0.5.0",
65
65
  ]
66
66
 
67
67
  [project.scripts]
@@ -101,4 +101,3 @@ known-third-party = ["ezmsg"]
101
101
  [tool.uv.sources]
102
102
  # Uncomment to use development version of ezmsg from git
103
103
  #ezmsg = { git = "https://github.com/ezmsg-org/ezmsg.git", branch = "dev" }
104
- ezmsg-qt = { git = "https://github.com/ezmsg-org/ezmsg-qt.git", branch = "dynamic_subscriber" }
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '0.4.0'
22
+ __version_tuple__ = version_tuple = (0, 4, 0)
23
+
24
+ __commit_id__ = commit_id = None
@@ -0,0 +1,87 @@
1
+ """Turning a structured ``ch`` coordinate axis into per-channel display names.
2
+
3
+ An AxisArray's ``ch`` axis usually carries a structured array with one row per
4
+ channel. What is *in* those rows depends on the acquisition system: an LSL or
5
+ NWB source typically offers a ``label``; a Blackrock source additionally offers
6
+ ``bank`` and ``elec``, which is what its users actually read off the front panel.
7
+ A plot needs one string per channel, so something has to choose.
8
+
9
+ This module makes that choice explicit and configurable rather than hard-coding
10
+ one system's convention. ``label`` is the default because it is the field most
11
+ sources populate and the one most likely to be meaningful to whoever recorded
12
+ the data; a GUI that knows better can ask for other fields.
13
+ """
14
+
15
+ import typing
16
+
17
+ import numpy as np
18
+
19
+ __all__ = ["available_fields", "channel_names"]
20
+
21
+
22
+ def available_fields(ch_axis_data: typing.Optional[np.ndarray]) -> typing.List[str]:
23
+ """Field names present on a structured ``ch`` axis, for building a chooser."""
24
+ if ch_axis_data is None:
25
+ return []
26
+ return list((ch_axis_data.dtype.fields or {}).keys())
27
+
28
+
29
+ def _field_text(row_value: typing.Any) -> str:
30
+ """One field of one channel as display text, or "" if it carries nothing.
31
+
32
+ Numeric fields use 0 as their "unset" value -- a Blackrock ``elec`` of 0
33
+ means no electrode, not electrode zero -- so it is treated as absent. Bytes
34
+ arrive from numpy ``S`` dtypes and are decoded leniently: a mangled label is
35
+ still more use than an exception.
36
+ """
37
+ if isinstance(row_value, bytes):
38
+ row_value = row_value.decode("utf8", errors="replace")
39
+ if isinstance(row_value, np.generic):
40
+ row_value = row_value.item()
41
+ if isinstance(row_value, bytes):
42
+ row_value = row_value.decode("utf8", errors="replace")
43
+ if isinstance(row_value, (int, float)) and not isinstance(row_value, bool):
44
+ return "" if row_value == 0 else f"{row_value:g}"
45
+ return str(row_value).strip()
46
+
47
+
48
+ def channel_names(
49
+ ch_axis_data: typing.Optional[np.ndarray],
50
+ n_ch: typing.Optional[int] = None,
51
+ *,
52
+ fields: typing.Sequence[str] = ("label",),
53
+ sep: str = "-",
54
+ fallback: str = "ch{index}",
55
+ ) -> typing.List[str]:
56
+ """Per-channel display names from a structured ``ch`` axis.
57
+
58
+ :param ch_axis_data: The axis's structured array, or None if the source
59
+ provided no ``ch`` axis.
60
+ :param n_ch: Channel count, used only when ``ch_axis_data`` is None or
61
+ unstructured. Otherwise the axis's own length governs.
62
+ :param fields: Field names to join, in order. Fields absent from the dtype
63
+ are skipped; fields present but empty for a given channel are skipped for
64
+ that channel only, so a partially-populated column degrades per-row
65
+ rather than for the whole array.
66
+ :param sep: Joins the field values, e.g. ``("bank", "elec")`` -> ``"A-1"``.
67
+ :param fallback: Format string used when no requested field yields anything
68
+ for a channel. ``{index}`` is the channel's index.
69
+
70
+ :return: One name per channel. Never empty strings, so a caller can render
71
+ the result without further guarding.
72
+ """
73
+ if ch_axis_data is None or ch_axis_data.dtype.fields is None:
74
+ n = int(n_ch or 0) if ch_axis_data is None else int(ch_axis_data.shape[0])
75
+ return [fallback.format(index=i) for i in range(n)]
76
+
77
+ present = [f for f in fields if f in ch_axis_data.dtype.fields]
78
+ n = int(ch_axis_data.shape[0])
79
+ if not present:
80
+ return [fallback.format(index=i) for i in range(n)]
81
+
82
+ columns = [ch_axis_data[f] for f in present]
83
+ names = []
84
+ for i in range(n):
85
+ parts = [t for t in (_field_text(col[i]) for col in columns) if t]
86
+ names.append(sep.join(parts) if parts else fallback.format(index=i))
87
+ return names
@@ -0,0 +1,208 @@
1
+ """Codec for the *static* half of an AxisArray, carried alongside the shmem ring.
2
+
3
+ The circular buffer in :mod:`.shmem` transports raw samples plus the little that
4
+ fits in a fixed ctypes header: dtype, shape, sample rate, and ``key``. Everything
5
+ else an :class:`~ezmsg.util.messages.axisarray.AxisArray` knows -- the ``ch``
6
+ coordinate axis holding per-channel ``bank``/``elec``/``label``, the units, the
7
+ message ``attrs`` -- is dropped at the boundary. A consumer on the far side can
8
+ therefore plot the signal but cannot say what any of it *is*.
9
+
10
+ This module encodes that dropped metadata into a self-describing byte blob which
11
+ :class:`~.shmem.ShMemCircBuff` publishes into its own shared-memory segment, and
12
+ which :class:`~.shmem_mirror.EZShmMirror` decodes on the far side.
13
+
14
+ Wire format
15
+ -----------
16
+ A pickled ``dict`` of **plain Python and numpy types only** -- never ezmsg
17
+ classes. The two halves of a shmem link are separate processes and may be
18
+ separate environments with different ezmsg versions installed; pinning the wire
19
+ format to ezmsg's dataclass layout would make an upgrade on one side a silent
20
+ decode failure on the other. Plain dicts cost one down-conversion and buy
21
+ version independence. ``AUX_FORMAT_VERSION`` guards the shape of the dict
22
+ itself.
23
+
24
+ Axes decode to::
25
+
26
+ {"kind": "linear", "unit": str, "gain": float, "offset": float}
27
+ {"kind": "coord", "unit": str, "dims": list[str], "data": np.ndarray}
28
+
29
+ The buffered axis (normally ``time``) is a deliberate special case: its
30
+ ``offset`` advances with every message and a coordinate time axis's ``data`` is
31
+ wholly new each message, so including either would make the metadata change
32
+ continuously and defeat the point of a low-rate side channel. Only its static
33
+ descriptors are kept -- see :func:`encode_aux`.
34
+ """
35
+
36
+ import pickle
37
+ import typing
38
+
39
+ import numpy as np
40
+
41
+ AUX_FORMAT_VERSION = 1
42
+
43
+ # Values we are willing to put on the wire. Anything else in ``attrs`` is
44
+ # dropped rather than pickled: an arbitrary object would force the decoding
45
+ # process to import the class that defines it, which is exactly the coupling
46
+ # this format exists to avoid.
47
+ _PLAIN_SCALARS = (str, bytes, int, float, bool, complex, type(None))
48
+
49
+
50
+ def _is_plain(value: typing.Any) -> bool:
51
+ """Whether ``value`` is safe to pickle into the blob."""
52
+ if isinstance(value, _PLAIN_SCALARS):
53
+ return True
54
+ if isinstance(value, np.ndarray):
55
+ # Object arrays hold arbitrary picklable classes; same objection.
56
+ return value.dtype != np.dtype("O")
57
+ if isinstance(value, np.generic):
58
+ return True
59
+ if isinstance(value, (list, tuple)):
60
+ return all(_is_plain(v) for v in value)
61
+ if isinstance(value, dict):
62
+ return all(isinstance(k, str) and _is_plain(v) for k, v in value.items())
63
+ return False
64
+
65
+
66
+ def axis_to_plain(axis: typing.Any, *, static_only: bool = False) -> dict:
67
+ """Down-convert one ezmsg axis to plain types.
68
+
69
+ ``static_only`` keeps just the descriptors that do not change from message to
70
+ message -- used for the buffered axis, whose position along the stream is
71
+ carried by the ring's write index rather than by this blob.
72
+ """
73
+ unit = getattr(axis, "unit", "")
74
+ if hasattr(axis, "data"): # CoordinateAxis
75
+ if static_only:
76
+ return {"kind": "coord", "unit": unit, "dims": list(axis.dims)}
77
+ return {
78
+ "kind": "coord",
79
+ "unit": unit,
80
+ "dims": list(axis.dims),
81
+ "data": np.array(axis.data, copy=True),
82
+ }
83
+ out = {"kind": "linear", "unit": unit, "gain": float(axis.gain)}
84
+ if not static_only:
85
+ out["offset"] = float(axis.offset)
86
+ return out
87
+
88
+
89
+ def encode_aux(
90
+ dims: typing.Sequence[str],
91
+ axes: typing.Mapping[str, typing.Any],
92
+ attrs: typing.Mapping[str, typing.Any],
93
+ key: str,
94
+ buffered_axis: str,
95
+ ) -> tuple[bytes, list[str]]:
96
+ """Serialize an AxisArray's static metadata.
97
+
98
+ Returns the blob and the list of ``attrs`` keys that were dropped for not
99
+ being plain types, so the caller can log them once rather than per message.
100
+ """
101
+ plain_axes = {name: axis_to_plain(ax, static_only=(name == buffered_axis)) for name, ax in axes.items()}
102
+ plain_attrs = {}
103
+ dropped = []
104
+ for name, value in attrs.items():
105
+ if isinstance(name, str) and _is_plain(value):
106
+ plain_attrs[name] = value
107
+ else:
108
+ dropped.append(str(name))
109
+ payload = {
110
+ "version": AUX_FORMAT_VERSION,
111
+ "dims": list(dims),
112
+ "axes": plain_axes,
113
+ "attrs": plain_attrs,
114
+ "key": key,
115
+ "buffered_axis": buffered_axis,
116
+ }
117
+ return pickle.dumps(payload, protocol=pickle.HIGHEST_PROTOCOL), dropped
118
+
119
+
120
+ def decode_aux(blob: bytes) -> dict:
121
+ """Inverse of :func:`encode_aux`.
122
+
123
+ :raises ValueError: if the blob is unreadable or was written by a format
124
+ version this build does not understand.
125
+ """
126
+ try:
127
+ payload = pickle.loads(blob)
128
+ except Exception as exc: # noqa: BLE001 - any unpickling failure is the same failure to us
129
+ raise ValueError(f"could not decode shmem metadata blob: {exc}") from exc
130
+ if not isinstance(payload, dict):
131
+ raise ValueError(f"shmem metadata blob decoded to {type(payload).__name__}, expected dict")
132
+ version = payload.get("version")
133
+ if version != AUX_FORMAT_VERSION:
134
+ raise ValueError(
135
+ f"shmem metadata blob is format version {version!r}, this build understands {AUX_FORMAT_VERSION}"
136
+ )
137
+ return payload
138
+
139
+
140
+ def _axis_equal(a: typing.Any, b: typing.Any) -> bool:
141
+ """Value equality for one axis, compared field by field.
142
+
143
+ Deliberately does not use ``==``. As of ezmsg 3.6, ``CoordinateAxis.__eq__``
144
+ resolves through the MRO to the dataclass-generated ``AxisBase.__eq__``,
145
+ which compares ``unit`` and nothing else -- ``ArrayWithNamedDims.__eq__``,
146
+ written to compare ``dims`` and ``data``, is shadowed and never runs. Two
147
+ coordinate axes with different data, or even different lengths, therefore
148
+ compare equal. Relying on that would mean a channel relabelling silently
149
+ never reaching the far side of the shmem link, which is the one thing this
150
+ module exists to deliver. Comparing explicitly also keeps the check correct
151
+ across ezmsg versions, which matters given the two halves of a link need not
152
+ share one.
153
+ """
154
+ a_data = getattr(a, "data", None)
155
+ b_data = getattr(b, "data", None)
156
+ if (a_data is None) != (b_data is None):
157
+ return False
158
+ if getattr(a, "unit", "") != getattr(b, "unit", ""):
159
+ return False
160
+ if a_data is None:
161
+ return a.gain == b.gain and a.offset == b.offset
162
+ if list(a.dims) != list(b.dims):
163
+ return False
164
+ if a_data is b_data:
165
+ return True
166
+ if a_data.shape != b_data.shape or a_data.dtype != b_data.dtype:
167
+ return False
168
+ return bool(np.array_equal(a_data, b_data))
169
+
170
+
171
+ def axes_equal(a: typing.Mapping[str, typing.Any], b: typing.Mapping[str, typing.Any]) -> bool:
172
+ """Cheap "have the axes changed?" test, for the per-message hot path.
173
+
174
+ Identity is checked before value at every level, which is what makes this
175
+ affordable at kHz rates: an ezmsg processor that leaves an axis alone passes
176
+ the *same object* through, so the common case costs one pointer comparison
177
+ per axis. An element-wise comparison happens only when a producer rebuilt an
178
+ axis -- rare, and precisely the case we must not get wrong.
179
+ """
180
+ if a is b:
181
+ return True
182
+ if a.keys() != b.keys():
183
+ return False
184
+ for name, av in a.items():
185
+ bv = b[name]
186
+ if av is bv:
187
+ continue
188
+ if type(av) is not type(bv):
189
+ return False
190
+ if not _axis_equal(av, bv):
191
+ return False
192
+ return True
193
+
194
+
195
+ def attrs_equal(a: typing.Mapping[str, typing.Any], b: typing.Mapping[str, typing.Any]) -> bool:
196
+ """Cheap "have the attrs changed?" test.
197
+
198
+ Identity only. ``attrs`` values are arbitrary -- a numpy array's ``==``
199
+ returns an array, and a user class's ``__eq__`` could be arbitrarily
200
+ expensive -- so value equality is deliberately not attempted here. A producer
201
+ that rebuilds equal attrs every message trips this check; the caller absorbs
202
+ that by comparing the encoded blob before it republishes anything.
203
+ """
204
+ if a is b:
205
+ return True
206
+ if a.keys() != b.keys():
207
+ return False
208
+ return all(a[name] is b[name] for name in a)
@@ -19,6 +19,12 @@ Upon receiving a data message, its metadata is checked, and if it does not match
19
19
 
20
20
  The other half must monitor the metadata shared memory to see if it changes, and if it does then it must recreate
21
21
  the data shared memory buffer reader at the new location.
22
+
23
+ Finally, there is a third piece of shared memory carrying everything about the AxisArray that does not fit in the
24
+ fixed-size metadata header: the non-buffered coordinate axes (e.g. a `ch` axis holding per-channel bank/elec/label),
25
+ axis units, and the message `attrs`. It lives at shorten_shmem_name(f"{shmem_name}/meta{meta_generation}") and is
26
+ republished -- under a fresh generation, following the same pattern as the data buffer -- only when that metadata
27
+ actually changes, which for a typical stream means once per session. See the .aux_meta module for the wire format.
22
28
  """
23
29
 
24
30
  import asyncio
@@ -35,6 +41,8 @@ import numpy as np
35
41
  import numpy.typing as npt
36
42
  from ezmsg.util.messages.axisarray import AxisArray, AxisBase
37
43
 
44
+ from .aux_meta import attrs_equal, axes_equal, encode_aux
45
+
38
46
  UINT64_SIZE = 8
39
47
  BYTEORDER = "little"
40
48
 
@@ -70,6 +78,28 @@ def shorten_shmem_name(long_name: str) -> str:
70
78
 
71
79
  MAXKEYLEN = 1024
72
80
 
81
+ # Sentinel at offset 0 of every metadata segment ("EZMS"). Distinguishes one of
82
+ # our headers from an unrelated segment that happens to collide on a name, and
83
+ # from a header written by a build old enough to predate this check.
84
+ SHMEM_META_MAGIC = 0x455A4D53
85
+
86
+ # Bumped on any change to ShmemArrMeta._fields_ or to the .aux_meta wire format.
87
+ #
88
+ # The two halves of a shmem link must be the same version -- there is no
89
+ # compatibility shim, by choice: the layouts are an internal detail between two
90
+ # processes we deploy together, and carrying forward every past field shape would
91
+ # cost more than it is worth. What we do owe is a loud failure rather than a
92
+ # quiet one, so the reader validates the magic and version up front and raises
93
+ # instead of misreading a header it does not understand.
94
+ SHMEM_META_STRUCT_VERSION = 1
95
+
96
+
97
+ class ShmemVersionError(RuntimeError):
98
+ """A shmem segment was written by an incompatible build.
99
+
100
+ Not recoverable and not transient: upgrade both ends together.
101
+ """
102
+
73
103
 
74
104
  class ShmemArrMeta(ctypes.Structure):
75
105
  """
@@ -91,6 +121,10 @@ class ShmemArrMeta(ctypes.Structure):
91
121
 
92
122
  _pack_ = 1
93
123
  _fields_ = [
124
+ # magic and struct_version lead so a reader can validate the layout
125
+ # before it trusts a single field that follows.
126
+ ("magic", ctypes.c_uint32),
127
+ ("struct_version", ctypes.c_uint32),
94
128
  ("bvalid", ctypes.c_bool),
95
129
  ("dtype", ctypes.c_char),
96
130
  ("srate", ctypes.c_double),
@@ -101,6 +135,11 @@ class ShmemArrMeta(ctypes.Structure):
101
135
  ("_key_bytes", ctypes.c_byte * MAXKEYLEN),
102
136
  ("_key_len", ctypes.c_uint32),
103
137
  ("write_index", ctypes.c_uint64),
138
+ # 0 = no metadata blob published yet. Otherwise names the segment at
139
+ # shorten_shmem_name(f"{shmem_name}/meta{meta_generation}").
140
+ ("meta_generation", ctypes.c_uint32),
141
+ # Exact length of the blob; the segment itself is page-rounded.
142
+ ("aux_nbytes", ctypes.c_uint32),
104
143
  ]
105
144
 
106
145
  @property
@@ -127,6 +166,16 @@ class ShMemCircBuffState(ez.State):
127
166
  buffer_shmem: typing.Optional[SharedMemory] = None
128
167
  buffer_arr: typing.Optional[npt.NDArray] = None
129
168
  meta_hash: int = -1
169
+ # Segment holding the serialized static metadata (see .aux_meta).
170
+ aux_shmem: typing.Optional[SharedMemory] = None
171
+ # The (dims, axes, attrs, key) we last encoded, held by reference for the
172
+ # per-message identity check in _update_aux_if_needed.
173
+ last_aux_src: typing.Optional[tuple] = None
174
+ # ...and the bytes they encoded to, so a producer that rebuilds equal
175
+ # metadata every message cannot cause a republish.
176
+ last_aux_blob: typing.Optional[bytes] = None
177
+ # attrs keys dropped as non-plain, remembered so we warn once, not per message.
178
+ warned_dropped_attrs: typing.Optional[frozenset] = None
130
179
 
131
180
 
132
181
  def _persist_create_shmem(name: str, size: int) -> SharedMemory:
@@ -201,6 +250,7 @@ class ShMemCircBuff(ez.Unit):
201
250
 
202
251
  async def shutdown(self) -> None:
203
252
  self._cleanup_buffer()
253
+ self._cleanup_aux()
204
254
  self._cleanup_meta()
205
255
  if self.SETTINGS.conn is not None:
206
256
  self.SETTINGS.conn.send("close")
@@ -215,6 +265,7 @@ class ShMemCircBuff(ez.Unit):
215
265
  if self.SETTINGS.conn is not None:
216
266
  self.SETTINGS.conn.send("meta cleanup")
217
267
 
268
+ self._cleanup_aux()
218
269
  self.STATE.meta_struct = None
219
270
 
220
271
  if self.STATE.meta_shmem is not None:
@@ -226,6 +277,21 @@ class ShMemCircBuff(ez.Unit):
226
277
  del self.STATE.meta_shmem
227
278
  self.STATE.meta_shmem = None
228
279
 
280
+ def _cleanup_aux(self):
281
+ """
282
+ Release the static-metadata segment, if one is published.
283
+
284
+ Also forgets the change-detection state, so the next message republishes
285
+ from scratch -- which is what we want after a name change or a shutdown /
286
+ restart, where a reader may be starting fresh too.
287
+ """
288
+ self._cleanup_aux_segment()
289
+ self.STATE.last_aux_src = None
290
+ self.STATE.last_aux_blob = None
291
+ if self.STATE.meta_struct is not None:
292
+ self.STATE.meta_struct.meta_generation = 0
293
+ self.STATE.meta_struct.aux_nbytes = 0
294
+
229
295
  def _cleanup_buffer(self):
230
296
  """
231
297
  Destroy the data buffer and the shared memory object.
@@ -274,10 +340,88 @@ class ShMemCircBuff(ez.Unit):
274
340
  # Build the metadata structure.
275
341
  self.STATE.meta_struct = ShmemArrMeta.from_buffer(self.STATE.meta_shmem.buf)
276
342
  self.STATE.meta_struct.bvalid = False
343
+ self.STATE.meta_struct.magic = SHMEM_META_MAGIC
344
+ self.STATE.meta_struct.struct_version = SHMEM_META_STRUCT_VERSION
345
+ self.STATE.meta_struct.meta_generation = 0
346
+ self.STATE.meta_struct.aux_nbytes = 0
277
347
  if reset_generation:
278
348
  self.STATE.meta_struct.buffer_generation = -1
279
349
  # We will wait for a data packet before we modify the remaining fields.
280
350
 
351
+ def _update_aux_if_needed(self, msg: AxisArray) -> bool:
352
+ """
353
+ Republish the static metadata segment if this message's metadata differs
354
+ from what is currently published.
355
+
356
+ Runs on every message, so the common path must be cheap. It is three
357
+ tiers, each only reached when the one before it is inconclusive:
358
+
359
+ 1. Identity/value comparison of (dims, axes, attrs, key) against what we
360
+ last encoded. Costs a handful of pointer comparisons when the producer
361
+ passes its axes through untouched, which is the normal case.
362
+ 2. Encode, and compare the bytes to what is published. This absorbs
363
+ producers that rebuild equal metadata every message -- they cost an
364
+ encode, but never a republish, so a reader is never woken for nothing.
365
+ 3. Allocate a new generation's segment and point the header at it.
366
+
367
+ Returns True if a new generation was published.
368
+ """
369
+ src = (msg.dims, msg.axes, msg.attrs, msg.key)
370
+ last = self.STATE.last_aux_src
371
+ if last is not None:
372
+ last_dims, last_axes, last_attrs, last_key = last
373
+ if (
374
+ msg.key == last_key
375
+ and msg.dims == last_dims
376
+ and axes_equal(msg.axes, last_axes)
377
+ and attrs_equal(msg.attrs, last_attrs)
378
+ ):
379
+ return False
380
+
381
+ blob, dropped = encode_aux(msg.dims, msg.axes, msg.attrs, msg.key, self.SETTINGS.axis)
382
+ if dropped:
383
+ dropped_set = frozenset(dropped)
384
+ if self.STATE.warned_dropped_attrs != dropped_set:
385
+ self.STATE.warned_dropped_attrs = dropped_set
386
+ ez.logger.warning(
387
+ f"ShMemCircBuff dropped non-plain attrs from the shmem metadata: {sorted(dropped_set)}. "
388
+ "Only str/bytes/number/bool/None, non-object ndarrays, and containers of those are transported."
389
+ )
390
+
391
+ # Hold the references that produced this blob whether or not we go on to
392
+ # publish it, so an unchanged-but-rebuilt message is only encoded once.
393
+ self.STATE.last_aux_src = src
394
+ if blob == self.STATE.last_aux_blob:
395
+ return False
396
+ self.STATE.last_aux_blob = blob
397
+
398
+ self._cleanup_aux_segment()
399
+ # 0 means "nothing published", so skip it when the uint32 wraps.
400
+ generation = (self.STATE.meta_struct.meta_generation + 1) % (2**32) or 1
401
+ aux_name = shorten_shmem_name(self.SETTINGS.shmem_name + "/meta" + str(generation))
402
+ self.STATE.aux_shmem = _persist_create_shmem(aux_name, len(blob))
403
+ self.STATE.aux_shmem.buf[: len(blob)] = blob
404
+
405
+ # Order matters: the segment is fully written before the header names it,
406
+ # so a reader never sees a generation it cannot completely read.
407
+ self.STATE.meta_struct.aux_nbytes = len(blob)
408
+ self.STATE.meta_struct.meta_generation = generation
409
+
410
+ if self.SETTINGS.conn is not None:
411
+ self.SETTINGS.conn.send("aux updated")
412
+ return True
413
+
414
+ def _cleanup_aux_segment(self) -> None:
415
+ """Release just the published segment, keeping change-detection state."""
416
+ if self.STATE.aux_shmem is not None:
417
+ self.STATE.aux_shmem.close()
418
+ try:
419
+ self.STATE.aux_shmem.unlink()
420
+ except FileNotFoundError:
421
+ pass
422
+ del self.STATE.aux_shmem
423
+ self.STATE.aux_shmem = None
424
+
281
425
  def _n_frames_for_axis(self, axis: AxisBase) -> int:
282
426
  """
283
427
  Utility function to calculate the number of frames to allocate for the buffer.
@@ -408,6 +552,12 @@ class ShMemCircBuff(ez.Unit):
408
552
  if self._update_meta_if_needed(msg):
409
553
  self._reset_buffer(msg)
410
554
 
555
+ # Independently of the buffer: republish the static metadata if it moved.
556
+ # The two are deliberately not coupled -- a `ch` axis can gain labels
557
+ # without the buffer's shape changing, and the buffer can be rebuilt
558
+ # (e.g. dtype change) with the channel identities untouched.
559
+ self._update_aux_if_needed(msg)
560
+
411
561
  n_samples = data.shape[0]
412
562
  write_stop = self.STATE.meta_struct.write_index + n_samples
413
563