npc-sync 0.2.5__tar.gz → 0.2.7__tar.gz

{npc_sync-0.2.5/src/npc_sync.egg-info → npc_sync-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: npc-sync
-Version: 0.2.5
+Version: 0.2.7
 Summary: Repackaging of AllenSDK.SyncDataset with extra functionality, compatible with data in the cloud.
 Author-email: Ben Hardcastle <ben.hardcastle@alleninstitue.org>
 License: MIT
@@ -24,7 +24,7 @@ Requires-Dist: typing-extensions>=4.9.0
 Requires-Dist: npc-io>=0.1.26
 Requires-Dist: npc-session>=0.1.33
 Requires-Dist: scipy>=1.13.1
-Requires-Dist: npc-stim>=0.1.14
+Requires-Dist: npc-stim>=0.1.23
 Dynamic: license-file
 
 # npc_sync

{npc_sync-0.2.5 → npc_sync-0.2.7}/pyproject.toml

@@ -14,9 +14,9 @@ dependencies = [
     "npc-io>=0.1.26",
     "npc-session>=0.1.33",
     "scipy>=1.13.1",
-    "npc-stim>=0.1.14",
+    "npc-stim>=0.1.23",
 ]
-version = "0.2.5"
+version = "0.2.7"
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Programming Language :: Python :: 3",

{npc_sync-0.2.5 → npc_sync-0.2.7}/src/npc_sync/sync.py

@@ -63,6 +63,11 @@ MIN_VSYNC_DIODE_FLIP_SEPARATION_SEC = 0.015  # 0.022 is typical
 FRAME_RATE = 59.95
 FRAME_INTERVAL = 1 / FRAME_RATE
 # we've seen apparently real flip intervals of 0.29 * 59.95, and the line between anomalous and real isn't clear-cut
+REPLAY_SLOT_SAMPLES = 10_000
+"""Slot width, in samples, used to fingerprint sync events when looking for replayed
+acquisition buffers. Buffer sizes seen so far (50,000 and 10,000 samples) are multiples
+of this. Buffer size is not recorded in the sync file metadata."""
+
 SUSPICIOUS_INTERVAL_THRESHOLD = 0.3 / FRAME_RATE
 DEFINITE_SHORT_INTERVAL_THRESHOLD = 0.05 / FRAME_RATE
 # at 60 fps we should never have diode-flip intervals this small: consider them anomalous
@@ -154,6 +159,106 @@ def get_bit(uint_array: npt.NDArray, bit: int) -> npt.NDArray[np.uint8]:
     return np.bitwise_and(uint_array, 2**bit).astype(bool).astype(np.uint8)
 
 
+def _splitmix64(x: npt.NDArray[np.uint64]) -> npt.NDArray[np.uint64]:
+    """Vectorized splitmix64 finalizer: scrambles 64-bit integers."""
+    x = x.astype(np.uint64, copy=True)
+    with np.errstate(over="ignore"):
+        x ^= x >> np.uint64(30)
+        x *= np.uint64(0xBF58476D1CE4E5B9)
+        x ^= x >> np.uint64(27)
+        x *= np.uint64(0x94D049BB133111EB)
+        x ^= x >> np.uint64(31)
+    return x
+
+
+def find_replayed_buffers(
+    times: npt.ArrayLike,
+    bits: npt.ArrayLike,
+    slot_samples: int = REPLAY_SLOT_SAMPLES,
+) -> tuple[tuple[tuple[int, int], ...], int | None]:
+    """Find acquisition buffers in a sync event list that are replayed copies of a later
+    buffer.
+
+    Some sync files contain runs of consecutive buffers with identical event content on
+    every line (same sample times relative to the buffer start, same bit states). The
+    copies overwrite real data; the last buffer of each run is assumed to be genuine.
+
+    Method: split events into slots of `slot_samples`, fingerprint each non-empty slot
+    by summing a hash of each event's (time relative to slot start, bit state), then find
+    slots with identical fingerprints. The most common slot separation between identical
+    slots gives the buffer size, so any buffer size that is a multiple of `slot_samples`
+    is found. Buffers are assumed to be aligned to sample 0.
+
+    Parameters
+    ----------
+    times : array of int
+        Sample index of each event.
+    bits : array of int
+        Bit state (all lines) of each event.
+    slot_samples : int
+        Fingerprint slot width in samples. Must divide the buffer size.
+
+    Returns
+    -------
+    intervals : tuple of (start, stop) sample indices
+        Half-open intervals of replayed copies. `stop` is the start of the genuine buffer.
+    buffer_size : int or None
+        Inferred buffer size in samples, or None if no replay was found.
+
+    Examples
+    --------
+    >>> rng = np.random.default_rng(0)
+    >>> buffer, n_buffers = 50_000, 20
+    >>> rel = [np.sort(rng.choice(buffer, 100, replace=False)) for _ in range(n_buffers)]
+    >>> state = [rng.integers(0, 2**32, 100) for _ in range(n_buffers)]
+    >>> for i in (5, 6, 7):  # buffers 5-7 are copies of buffer 8
+    ...     rel[i], state[i] = rel[8], state[8]
+    >>> times = np.concatenate([r + i * buffer for i, r in enumerate(rel)])
+    >>> find_replayed_buffers(times, np.concatenate(state))
+    (((250000, 400000),), 50000)
+    >>> find_replayed_buffers(times[times < 250_000], np.concatenate(state)[times < 250_000])
+    ((), None)
+    """
+    times = np.asarray(times).astype(np.int64)
+    bits = np.asarray(bits).astype(np.uint64)
+    if times.size == 0:
+        return (), None
+    slot = times // slot_samples
+    starts = np.flatnonzero(np.r_[True, slot[1:] != slot[:-1]])  # non-empty slots only
+    tokens = ((times - slot * slot_samples).astype(np.uint64) << np.uint64(32)) | (
+        bits & np.uint64(0xFFFFFFFF)
+    )
+    with np.errstate(over="ignore"):
+        fingerprints = np.add.reduceat(_splitmix64(tokens), starts)
+    slots = slot[starts]
+    order = np.lexsort((slots, fingerprints))  # by fingerprint, then slot
+    fingerprints, slots = fingerprints[order], slots[order]
+    same = fingerprints[1:] == fingerprints[:-1]
+    if not same.any():
+        return (), None
+    earlier = slots[:-1][same]
+    lags = slots[1:][same] - earlier
+    values, counts = np.unique(lags, return_counts=True)
+    lag = int(values[np.argmax(counts)])
+    if len(values) > 1:
+        logger.warning(
+            "Identical sync slots found at more than one separation "
+            f"(in units of {slot_samples} samples): "
+            f"{dict(zip(values.tolist(), counts.tolist()))}. Using the most common ({lag})."
+        )
+    buffer_size = lag * slot_samples
+    copies = np.unique(
+        earlier[lags == lag] // lag
+    )  # buffer index of each replayed copy
+    breaks = np.flatnonzero(np.diff(copies) != 1)
+    first = copies[np.r_[0, breaks + 1]]
+    last = copies[np.r_[breaks, len(copies) - 1]]
+    intervals = tuple(
+        (int(a * buffer_size), int((z + 1) * buffer_size)) for a, z in zip(first, last)
+    )
+    return intervals, buffer_size
+
+
 def get_sync_line_for_stim_onset(
     waveform_type: (
         str | Literal["sound", "audio", "opto", "galvo", "laser_488", "laser_633"]
@@ -1006,6 +1111,32 @@ class SyncDataset:
 
         return tuple(blocks)
 
+    @npc_io.cached_property
+    def invalid_intervals(self) -> tuple[tuple[float, float], ...]:
+        """(start, stop) times in seconds of each interval of sync data that is a replayed
+        copy of a later acquisition buffer. Empty if none are found.
+
+        Events in [start, stop) are not real on any line. The last buffer of each run of
+        identical buffers is assumed to be genuine and is not included. See
+        `find_replayed_buffers`.
+        """
+        events = self.get_all_events()
+        if self.meta_data["ni_daq"]["counter_bits"] == 32:
+            times = events[:, 0]
+        else:
+            times = self.times
+        intervals, buffer_size = find_replayed_buffers(times, events[:, -1])
+        result = tuple(
+            (a / self.sample_freq, z / self.sample_freq) for a, z in intervals
+        )
+        if result:
+            logger.warning(
+                f"{self}: {len(result)} interval(s) of replayed sync data with "
+                f"{buffer_size}-sample buffers (all lines invalid): "
+                f"{[(round(a, 3), round(z, 3)) for a, z in result]}"
+            )
+        return result
+
     @property
     def total_seconds(self) -> float:
         return self.meta_data["total_samples"] / self.sample_freq
@@ -1104,9 +1235,44 @@ class SyncDataset:
             result[block_idx] = npc_stim.get_total_stim_frames(stim_data)
         return result
 
+    def _get_diode_box_frame_interval(self, block_idx: int) -> int:
+        """Return the number of frames between diode-box flips for a block.
+
+        Stim files created before ``diodeBoxFrameInterval`` was added always
+        flipped the diode box every frame, so missing metadata defaults to 1.
+        """
+        path = self.block_index_to_stim_path[block_idx]
+        if path is None:
+            return 1
+        stim_data = None
+        try:
+            stim_data = npc_stim.get_stim_data(path)
+            value = stim_data.get("diodeBoxFrameInterval", 1)
+            if isinstance(value, h5py.Dataset):
+                value = value[()]
+            return int(value)
+        except Exception as exc:
+            logger.warning(
+                f"Failed to load diodeBoxFrameInterval for block {block_idx} at {path}: {exc}"
+            )
+            return 1
+        finally:
+            if isinstance(stim_data, h5py.File):
+                stim_data.close()
+
     @npc_io.cached_property
     def frame_display_time_blocks(self) -> tuple[npt.NDArray[np.floating], ...]:
-        """Blocks of adjusted diode times: one block per stimulus."""
+        """Blocks of adjusted diode times: one block per stimulus.
+
+        Raises `AssertionError` if the sync file contains replayed data (see
+        `invalid_intervals`), since vsync and diode edges can't be trusted there.
+        """
+        if self.invalid_intervals:
+            raise AssertionError(
+                f"{self} contains replayed sync data (all lines invalid) at "
+                + ", ".join(f"{a:.3f}-{z:.3f} s" for a, z in self.invalid_intervals)
+                + ": frame display times can't be computed reliably"
+            )
         vsync_times_in_blocks = self.vsync_times_in_blocks
 
         diode_rising_edges = self.get_rising_edges("stim_photodiode", units="seconds")
@@ -1252,12 +1418,7 @@ class SyncDataset:
                 """Concatenate the rising and falling edges into a single array."""
                 return np.sort(np.concatenate((rising, falling)))
 
-            # July 2025 we updated the photodiode interval from every frame to every 3 frames
-            vsyncs_per_diode_flip: int = (
-                1
-                if self.start_time < datetime.datetime(2025, 7, 15)
-                else round(np.mean(np.diff(concat_flips(rising, falling))) * FRAME_RATE)
-            )
+            vsyncs_per_diode_flip = self._get_diode_box_frame_interval(block_idx)
 
             if (
                 vsyncs_per_diode_flip == 1

{npc_sync-0.2.5 → npc_sync-0.2.7/src/npc_sync.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: npc-sync
-Version: 0.2.5
+Version: 0.2.7
 Summary: Repackaging of AllenSDK.SyncDataset with extra functionality, compatible with data in the cloud.
 Author-email: Ben Hardcastle <ben.hardcastle@alleninstitue.org>
 License: MIT
@@ -24,7 +24,7 @@ Requires-Dist: typing-extensions>=4.9.0
 Requires-Dist: npc-io>=0.1.26
 Requires-Dist: npc-session>=0.1.33
 Requires-Dist: scipy>=1.13.1
-Requires-Dist: npc-stim>=0.1.14
+Requires-Dist: npc-stim>=0.1.23
 Dynamic: license-file
 
 # npc_sync

{npc_sync-0.2.5 → npc_sync-0.2.7}/src/npc_sync.egg-info/SOURCES.txt

@@ -12,4 +12,5 @@ src/npc_sync.egg-info/requires.txt
 src/npc_sync.egg-info/top_level.txt
 tests/test_core.py
 tests/test_known_files.py
-tests/test_plots.py
+tests/test_plots.py
+tests/test_replay.py

{npc_sync-0.2.5 → npc_sync-0.2.7}/src/npc_sync.egg-info/requires.txt

@@ -5,4 +5,4 @@ typing-extensions>=4.9.0
 npc-io>=0.1.26
 npc-session>=0.1.33
 scipy>=1.13.1
-npc-stim>=0.1.14
+npc-stim>=0.1.23

{npc_sync-0.2.5 → npc_sync-0.2.7}/tests/test_core.py

@@ -1,3 +1,7 @@
+import pathlib
+
+import h5py
+
 import npc_sync
 
 
@@ -28,3 +32,26 @@ def test_init_from_instance_and_meta_data() -> None:
     meta = s2.meta_data
     assert isinstance(meta, dict)
     assert "line_labels" in meta
+
+
+def test_diode_box_frame_interval_is_read_from_stim_file(
+    tmp_path: pathlib.Path,
+) -> None:
+    current_stim = tmp_path / "current.hdf5"
+    with h5py.File(current_stim, "w") as stim_data:
+        stim_data["diodeBoxFrameInterval"] = 3
+
+    old_stim = tmp_path / "old.hdf5"
+    with h5py.File(old_stim, "w"):
+        pass
+
+    sync = object.__new__(npc_sync.SyncDataset)
+    sync.block_index_to_stim_path = {
+        0: current_stim,
+        1: old_stim,
+        2: None,
+    }
+
+    assert sync._get_diode_box_frame_interval(0) == 3
+    assert sync._get_diode_box_frame_interval(1) == 1
+    assert sync._get_diode_box_frame_interval(2) == 1

npc_sync-0.2.7/tests/test_replay.py

@@ -0,0 +1,96 @@
+"""Detection of replayed sync acquisition buffers.
+
+See outputs/vsync-frame-count-discrepancies.md.
+"""
+
+import pathlib
+
+import numpy as np
+import pytest
+
+import npc_sync
+
+
+def _make_events(
+    buffer_size: int,
+    n_buffers: int,
+    copies: dict[int, int],
+    events_per_buffer: int = 200,
+    seed: int = 0,
+) -> tuple[np.ndarray, np.ndarray]:
+    """Random events in each buffer; `copies` maps copy buffer index -> source buffer index."""
+    rng = np.random.default_rng(seed)
+    rel = [
+        np.sort(rng.choice(buffer_size, events_per_buffer, replace=False))
+        for _ in range(n_buffers)
+    ]
+    state = [rng.integers(0, 2**32, events_per_buffer) for _ in range(n_buffers)]
+    for copy, source in copies.items():
+        rel[copy], state[copy] = rel[source].copy(), state[source].copy()
+    times = np.concatenate([r + i * buffer_size for i, r in enumerate(rel)])
+    return times, np.concatenate(state)
+
+
+@pytest.mark.parametrize("buffer_size", [10_000, 50_000, 30_000])
+def test_buffer_size_inferred(buffer_size: int) -> None:
+    times, bits = _make_events(buffer_size, 40, {10: 13, 11: 13, 12: 13})
+    intervals, size = npc_sync.find_replayed_buffers(times, bits)
+    assert size == buffer_size
+    assert intervals == ((10 * buffer_size, 13 * buffer_size),)
+
+
+def test_multiple_runs() -> None:
+    times, bits = _make_events(50_000, 40, {5: 6, 20: 22, 21: 22})
+    intervals, size = npc_sync.find_replayed_buffers(times, bits)
+    assert size == 50_000
+    assert intervals == ((250_000, 300_000), (1_000_000, 1_100_000))
+
+
+def test_no_replay() -> None:
+    times, bits = _make_events(50_000, 40, {})
+    assert npc_sync.find_replayed_buffers(times, bits) == ((), None)
+    assert npc_sync.find_replayed_buffers([], []) == ((), None)
+
+
+def test_trailing_event_difference() -> None:
+    """The genuine buffer can have an extra trailing event that the copies lack
+    (DRpilot_847958_20260413): its last slot doesn't match, but the copy buffers are
+    still reported in full."""
+    buffer_size = 50_000
+    times, bits = _make_events(buffer_size, 40, {10: 12, 11: 12})
+    genuine = (times >= 12 * buffer_size) & (times < 13 * buffer_size)
+    extra_time = times[genuine].max() + 1
+    i = np.searchsorted(times, extra_time)
+    times = np.insert(times, i, extra_time)
+    bits = np.insert(bits, i, 12345)
+    intervals, size = npc_sync.find_replayed_buffers(times, bits)
+    assert size == buffer_size
+    assert intervals == ((10 * buffer_size, 12 * buffer_size),)
+
+
+def test_frame_display_time_blocks_fails_fast_on_replay() -> None:
+    sync = object.__new__(npc_sync.SyncDataset)
+    sync.path = pathlib.Path("replayed.h5")
+    sync.invalid_intervals = ((5247.5, 5257.0),)
+    with pytest.raises(AssertionError, match="5247.500-5257.000 s"):
+        _ = sync.frame_display_time_blocks
+
+
+DATA = pathlib.Path(__file__).parents[1] / "experiments" / "vsync-frame-count" / "data"
+KNOWN = {
+    "DRpilot_836629_20260430": ((5247.5, 5257.0),),
+    "DRpilot_839146_20260518": ((5622.0, 5629.0),),
+    "DRpilot_856847_20260602": ((801.5, 805.0),),
+    "DRpilot_856847_20260604": ((2743.0, 2744.5),),
+    "DRpilot_862025_20260805": ((705.0, 706.5),),
+    "DRpilot_847958_20260413": ((5706.0, 5707.5),),
+    "DRpilot_840160_20260428": (),  # timebase drop, not a replay
+}
+
+
+@pytest.mark.parametrize("session", sorted(KNOWN))
+def test_known_sessions(session: str) -> None:
+    paths = sorted((DATA / session).glob("*.h5"))
+    if not paths:
+        pytest.skip(f"local sync file not available for {session}")
+    assert npc_sync.SyncDataset(paths[0]).invalid_intervals == KNOWN[session]