oep-client-python 0.0.47__tar.gz → 0.0.49__tar.gz

{oep_client_python-0.0.47 → oep_client_python-0.0.49}/CHANGELOG.md

@@ -2,6 +2,40 @@
 
 ## Unreleased
 
+## 0.0.49
+- (EN) Changed (virtual bench): a stream follows the probe's clock as a real probe's does. Logic samples are made at once
+  with numpy (a second of 1 MS/s on 2 lines: format 0 0.04 s instead of 4.6 s, format 1 about 0.2 to 0.3 s), sample by
+  sample only where a track needs it (the glitch filter, the external clock, analog, a test's own value()). The samples
+  the clock has taken and the bench has not made yet count as held in its stream buffer: a tick makes a bounded number
+  and catches up over ticks; stop makes every sample up to the stop; behind by more than the stream buffer holds, the
+  track stops in error as a probe whose buffer overflowed (state 6, error storage). Before, on a busy host the stream ran
+  late and ended before the stop: a heading placed by the probe's clock fell past its end (4 cores with 8 busy
+  processes, during=steps at 1 MS/s on 2 lines: 8 of 8 by 0.5 to 1.1 s; now 0 of 8, none lost).
+- (JA) 変更（仮想ベンチ）: stream は、実機と同じくプローブの時計に従う。ロジックの標本は numpy でまとめて作る（2 本・
+  1 MS/s の 1 秒分が、format 0 で 4.6 s から 0.04 s、format 1 で約 0.2〜0.3 s）。1 標本ずつ作るのは、そうしか作れない track
+  だけ（グリッチフィルタ、外部クロック、アナログ、試験が差し替えた value()）。時計が取ったのにまだ作っていない標本は、
+  ストリーム用のバッファに持っている分とみなす。tick は上限つきで作り、何回かで追いつく。stop は止めた時刻までの標本を
+  全部作る。遅れがストリーム用のバッファを超えたら、バッファのあふれた実機と同じく track を error で止める（state 6、
+  error storage）。これまでは、混んだ host で stream が遅れたまま止めた時刻より前で終わり、プローブの時計で置いた見出しが
+  その後になった（4 コアに 8 本の空回り、during=steps で 2 本・1 MS/s: 8 回中 8 回、0.5〜1.1 s。直した後は 8 回中 0 回、
+  欠損なし）。
+- (EN) Fixed (0.0.48): a format 1 stream put together on disk failed with a KeyError (a segment's position): the loop
+  joining a segment's lines bit by bit reused the name of the loop over the segments, so the next segment taken was the
+  wrong one. A format 1 stream on disk is tested now.
+- (JA) 修正（0.0.48）: ディスクで組み立てる format 1 の stream が KeyError（区画の位置）で落ちていた。区画の線を 1 bit ずつ
+  つなぐループが、区画を回すループと同じ変数名を使っていて、次に取る区画を取り違えていた。ディスクで組み立てる format 1 の
+  stream の試験を足した。
+
+## 0.0.48
+- (EN) Fixed: a stream's segments join bit by bit, so a stream is one part unless data was lost. A segment whose
+  samples did not fill whole bytes of each line (the ESP32-P4's 65332 samples of 2 lines at 5 and 10 MS/s) started a new
+  part every time: 300 to 550 parts, none lost. A format 1 segment whose samples are not a multiple of every D still
+  starts a new part (its ticks restart, capture §1.3).
+- (JA) 修正: stream の区画を 1 bit ずつつなぐ。欠損がなければ、stream は 1 つの部分になる。線ごとに標本がバイトの切れ目で
+  終わらない区画（ESP32-P4 の 2 本の線の 65332 標本、5 MS/s と 10 MS/s）では、そのたびに新しい部分を始めていて、欠損なしで
+  300〜550 の部分に分かれていた。format 1 で標本数がどの D の倍数でもない区画は、今までどおり新しい部分を始める
+  （その刻みが始め直しになるため、capture §1.3）。
+
 ## 0.0.47
 - (EN) Added: `acquire.Request(during=steps)` for a stream in place of `seconds`: it streams until `steps(wait)`
   returns, `wait(seconds)` collecting while it waits, so a script marks sections and sends ops or console commands on

{oep_client_python-0.0.47 → oep_client_python-0.0.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: oep-client-python
-Version: 0.0.47
+Version: 0.0.49
 Summary: Open Embedded Probe (OEP) v1 host: serial / USB / TCP transports, the session rules, the standard interfaces, and a virtual bench
 Project-URL: Homepage, https://github.com/Open-Embedded-Probe/oep-client-python
 Project-URL: Repository, https://github.com/Open-Embedded-Probe/oep-client-python

{oep_client_python-0.0.47 → oep_client_python-0.0.49}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "oep-client-python"
-version = "0.0.47"
+version = "0.0.49"
 description = "Open Embedded Probe (OEP) v1 host: serial / USB / TCP transports, the session rules, the standard interfaces, and a virtual bench"
 readme = "README.md"
 license = "MIT"

{oep_client_python-0.0.47 → oep_client_python-0.0.49}/src/oep_client/__init__.py

@@ -2,4 +2,4 @@
 
 __all__ = ["__version__"]
 
-__version__ = "0.0.47"
+__version__ = "0.0.49"

{oep_client_python-0.0.47 → oep_client_python-0.0.49}/src/oep_client/acquire.py

@@ -1655,14 +1655,14 @@ class _Spool:
             p, nxt = ready[k][0], ready[j][1]
             x = self.segs[p]
             samples = sum(self.segs.pop(q).samples for q, _ in ready[k:j + 1])
-            if self.cur is not None and (self.cur["samples"] % 8 or any(self.cur["samples"] % d for d in self.ds)):
+            if self.cur is not None and any(self.cur["samples"] % d for d in self.ds):
                 self._close(follows=True)      # a grid restart (format 1) is not a loss: the next part follows on
             self._open(p, x)
             data = bytes(self.pending[p - self.at:nxt - self.at])
             cols = _segment_columns(self.cap, data, samples, len(self.names), self.cfg)
-            for col, f, ev in zip(cols, self.cur["files"], self.cur["events"]):
-                if isinstance(col, bytes):
-                    f.write(col[:(samples + 7) // 8])
+            for c, (col, f, ev) in enumerate(zip(cols, self.cur["files"], self.cur["events"])):
+                if isinstance(col, bytes):           # joined bit by bit: a segment need not end on a whole byte
+                    self.cur["carry"][c] = _write_bits(f, self.cur["carry"][c], col, samples)
                 else:
                     ev[0].append(col[0] + self.cur["samples"] // col[4])
                     ev[1].append(col[1])
@@ -1693,7 +1693,7 @@ class _Spool:
         k = len(self.parts)
         files = [open(os.path.join(self.dir, f"p{k}-{j}.bits"), "wb") for j in range(len(self.names))]
         self.cur = {"tick0": tick0, "meta": _seg_meta(seg) if seg is not None else {}, "samples": 0, "files": files,
-                    "events": [[[], [], [], None] for _ in self.names]}
+                    "events": [[[], [], [], None] for _ in self.names], "carry": [None] * len(self.names)}
         self._follows = False
 
     _last_samples, _last_end, _follows = 0, None, False
@@ -1705,6 +1705,7 @@ class _Spool:
         cur, n = self.cur, self.cur["samples"]
         chans = []
         for role, (name, ch, f, ev) in enumerate(zip(self.names, self.ids, cur["files"], cur["events"])):
+            _flush_bits(f, cur["carry"][role])
             f.close()
             d = self.cfg.event_d.get(role) if self.blocks else None
             if d is not None:
@@ -1791,7 +1792,7 @@ def _stream_parts(cap, cfg, got, names, ids, req: Request) -> tuple[list, list]:
                 chans = from_interleaved(data, names, cfg.width, cfg.positions, samples)
             else:
                 chans = _from_blocks(cap, data, names, samples, cfg)
-            restart = acc is not None and (acc_samples % 8 or any(acc_samples % d for d in ds))
+            restart = acc is not None and any(acc_samples % d for d in ds)
             if acc is None or restart:           # a grid restart (format 1) is not a loss: the next part follows on
                 if acc is not None:
                     parts.append(_part(_built(acc, acc_samples, ids), acc_meta, tick0))
@@ -1865,10 +1866,41 @@ def _samples_at_once(run, names, ids, cfg, samples: int) -> list:
             for name, ch, bits in zip(names, ids, packed)]
 
 
+def _write_bits(f, carry, col: bytes, samples: int):
+    """Append `samples` bits of `col` (sample i at bit i % 8 of byte i // 8) to file `f` after the `carry` bits not
+    yet written (None: none): whole bytes go to the file, the bits left over (fewer than 8) come back as the carry."""
+    if carry is None and samples % 8 == 0:
+        f.write(col[:samples // 8])
+        return None
+    import numpy as np
+    bits = np.unpackbits(np.frombuffer(col, np.uint8, (samples + 7) // 8), bitorder="little")[:samples]
+    if carry is not None:
+        bits = np.concatenate([carry, bits])
+    whole = len(bits) // 8 * 8
+    f.write(np.packbits(bits[:whole], bitorder="little").tobytes())
+    return bits[whole:] if whole < len(bits) else None
+
+
+def _flush_bits(f, carry) -> None:
+    """The last carry of _write_bits: its bits in one byte, the rest 0."""
+    if carry is not None and len(carry):
+        import numpy as np
+        f.write(np.packbits(carry, bitorder="little").tobytes())
+
+
+def _join_bits(pieces) -> bytes:
+    """Pieces (bits, n) joined bit by bit: n need not be a multiple of 8."""
+    if all(n % 8 == 0 for _, n in pieces[:-1]):
+        return b"".join(b[:(n + 7) // 8] for b, n in pieces)
+    import numpy as np
+    return np.packbits(np.concatenate([np.unpackbits(np.frombuffer(b, np.uint8, (n + 7) // 8), bitorder="little")[:n]
+                                       for b, n in pieces]), bitorder="little").tobytes()
+
+
 class _Acc:
-    """One channel of a part being joined from its segments (bits: whole bytes, the part's samples so far a multiple
-    of 8; events: quanta offset by the samples so far, a multiple of D). A segment that follows on carries the level
-    before it as its start (§1.3), so its changes join the part's as they are."""
+    """One channel of a part being joined from its segments (bits: joined bit by bit; events: quanta offset by the
+    samples so far, a multiple of D). A segment that follows on carries the level before it as its start (§1.3), so
+    its changes join the part's as they are."""
     def __init__(self, c):
         self.first, self.pieces = c, []
         self.start = getattr(c, "start_level", None)
@@ -1894,8 +1926,7 @@ def _built(acc: list, samples: int, ids) -> list:
                 ct += c_
             out.append(changes(c.name, ks, lv, ct, -(-samples // c.step), c.step, 0, a.start, pin=ch))
         else:
-            out.append(Bits(c.name, b"".join(b for b, _ in a.pieces)[:(samples + 7) // 8], samples, 1, 0,
-                                          {"pin": ch}))
+            out.append(Bits(c.name, _join_bits(a.pieces)[:(samples + 7) // 8], samples, 1, 0, {"pin": ch}))
     return out
 
 

{oep_client_python-0.0.47 → oep_client_python-0.0.49}/src/oep_client/virtual_bench_capture.py

@@ -82,11 +82,11 @@ BITS = 12                              # the analog value's b (layout)
 BLOCK_TOP = 4096                       # its own: the longest block L it makes (§1.3)
 STREAM_BUFFER_BYTES = 1 << 20          # its own: what a stream holds unsent, said in the answer (stream_buffer, §3.3)
 UNKNOWN_END = 1 << 62                  # streaming format 1: a segment dropped whole whose end is not taken yet
-BLOCKS_A_TICK = 1 << 14                # streaming: the most samples one tick makes - Python makes far below a fast rate,
-                                       # and catching up on all the time passed fell further behind every tick until the
-                                       # bench answered nothing (wireskein: format 1 at 2 MHz x 4 lines; format 0 at 1 MHz
-                                       # x 2 lines, stop answered after 12.8 s): it makes what it can and the stream runs
-                                       # late instead
+FAST_A_TICK = 1 << 17                  # streaming: the most one tick makes when numpy makes them at once (a_tick)
+BLOCKS_A_TICK = 1 << 14                # streaming: the most samples one tick makes sample by sample - catching up on all
+                                       # the time passed at once fell further behind every tick until the bench answered
+                                       # nothing (wireskein: format 1 at 2 MHz x 4 lines; format 0 at 1 MHz x 2 lines,
+                                       # stop answered after 12.8 s): it catches up over ticks (held_most)
 
 
 TIMEBASE_HZ = 100_000_000             # its own: format 3's delta tick (the answer's timebase, §3.8)
@@ -206,6 +206,38 @@ def logic_block(raw: list[list[int]], events: list[list[int]], ds: list[int], be
     return fallback + start + struct.pack("<I", len(ev)) + body + b"".join(ev)
 
 
+def logic_block_np(np, raw: list, events: list, ds: list[int], before: list[int], L: int) -> bytes:
+    """logic_block with each channel's levels a numpy array: the same bytes, made at once."""
+    r = len((raw + events)[0])
+    E = len(events)
+    fb, ev = [], []
+    for e, lv in enumerate(events):
+        d = ds[e]
+        q = -(-r // d)
+        prev = np.concatenate([np.array([before[e]], np.uint8), lv[:-1]])
+        changed = np.zeros(q * d, np.int64)
+        changed[:r] = lv != prev
+        counts = changed.reshape(q, d).sum(axis=1)
+        last = lv[np.minimum((np.arange(q) + 1) * d, r) - 1].astype(np.int64)
+        t = np.nonzero(counts)[0]
+        rec = np.zeros(len(t), dtype=[("t", "<u2"), ("e", "u1"), ("v", "u1")])
+        rec["t"], rec["e"], rec["v"] = t, e, (last[t] << 7) | np.minimum(counts[t], 127)
+        if len(t) * 4 > L // 8:
+            fb.append(e)
+        else:
+            ev.append(rec.tobytes())
+    nb = (E + 7) // 8
+    fallback = sum(1 << e for e in fb).to_bytes(nb, "little")
+    start = sum(before[e] << e for e in range(E)).to_bytes(nb, "little")
+    cols = raw + [events[e] for e in fb]
+    body = b""
+    if cols:
+        bits = np.stack(cols, axis=1).astype(np.uint8)
+        body = np.packbits(bits.reshape(-1), bitorder="little").tobytes()
+    n_ev = sum(len(x) // 4 for x in ev)
+    return fallback + start + struct.pack("<I", n_ev) + body + b"".join(ev)
+
+
 def analog_part(values: list[int], kind: int, d: int, b: int) -> bytes:
     """One channel's part of a format 2 block (§1.4): packed the r values of b bits; else for every interval of d
     (the last one partial) min and max (b bits each) and / or the sum (b + ceil(log2 d) bits), in that order - bits by
@@ -230,6 +262,15 @@ def analog_part(values: list[int], kind: int, d: int, b: int) -> bytes:
     return acc.to_bytes((at + 7) // 8, "little")
 
 
+class _wave:
+    """A logic wave: wave(i, rate) for one sample, wave.many(np, samples, rate) for an array of them (numpy)."""
+    def __init__(self, one, many):
+        self.one, self.many = one, many
+
+    def __call__(self, i, rate):
+        return self.one(i, rate)
+
+
 def logic_wave(spec: str):
     """A logic channel's waveform instead of the counter's bit: `counter`; `quiet[:LEVEL]` (never changes, default
     high); `square:PERIOD` (samples); `uart:BAUD[:TEXT]` - 8N1 characters of TEXT (default "OEP\\n") one after another
@@ -239,10 +280,10 @@ def logic_wave(spec: str):
         return None
     if kind == "quiet":
         level = int(args[0]) if args else 1
-        return lambda i, rate: level
+        return _wave(lambda i, rate: level, lambda np, i, rate: np.full(len(i), level, np.uint8))
     if kind == "square":
         half = max(1, int(args[0]) // 2)
-        return lambda i, rate: (i // half) & 1
+        return _wave(lambda i, rate: (i // half) & 1, lambda np, i, rate: ((i // half) & 1).astype(np.uint8))
     if kind == "uart":
         baud = int(args[0])
         text = (args[1] if len(args) > 1 else "OEP\n").encode()
@@ -255,7 +296,14 @@ def logic_wave(spec: str):
             if at <= 8:
                 return text[char % len(text)] >> (at - 1) & 1      # LSB first
             return 1                                               # stop, idle
-        return uart
+
+        def many(np, i, rate):                                     # the same, for an array of samples
+            r = Fraction(rate)                                     # the track's rate is exact: so is the bit time
+            bit = (i * (baud * r.denominator)) // r.numerator
+            char, at = np.divmod(bit, 20)
+            data = (np.frombuffer(text, np.uint8)[char % len(text)] >> np.clip(at - 1, 0, 7)) & 1
+            return np.where(at == 0, 0, np.where(at <= 8, data, 1)).astype(np.uint8)
+        return _wave(uart, many)
     raise ValueError(f"logic wave {spec!r}: counter, quiet[:LEVEL], square:PERIOD or uart:BAUD[:TEXT]")
 
 
@@ -702,8 +750,39 @@ class VirtualCapture:
             slot = "H" if self.width == 2 * self.channels else "I"
             return b"".join(struct.pack(f"<{self.channels}{slot}", *(self.value(k, first + i) for k in self.analog_roles))
                             for i in range(n))
+        levels = self._levels(first, n)
+        if levels is not None:                                     # numpy: the samples at once
+            import numpy as np
+            bits = np.zeros((n, self.width), np.uint8)
+            bits[:, :] = np.stack(levels[:self.width], axis=1)
+            return np.packbits(bits.reshape(-1), bitorder="little").tobytes()
         return pack_samples([sum(self.value(k, first + i) << k for k in range(self.width)) for i in range(n)], self.width)
 
+    def _levels(self, first: int, n: int):
+        """Logic channels 0 .. max(width, channels) - 1 at samples first .. first + n - 1 as numpy arrays, as value() gives them; None
+        where only value() can (numpy missing, the glitch filter, the external clock, a wave without its array form, a
+        value() a test put on the track).
+        Python made a sample a microsecond or more: a bench behind 1 MS/s on a busy host."""
+        ns = max(self.glitch_ns.values(), default=0)
+        if self.analog or self.clock is not None or n <= 0 or (ns and -(-ns * self.rate // 1_000_000_000) > 1) or \
+                "value" in vars(self):                             # a test's own value(): only it says
+            return None
+        try:
+            import numpy as np
+        except ImportError:
+            return None
+        i = np.arange(first, first + n, dtype=np.int64)
+        out = []
+        for k in range(max(self.width, self.channels)):            # format 1: the channels, its w apart (0 or less)
+            wave = self.waves.get(self.logic_roles[k]) if k < len(self.logic_roles) else None
+            if wave is None:
+                out.append(((i >> k) & 1).astype(np.uint8))
+            elif isinstance(wave, _wave):
+                out.append(wave.many(np, i, self.rate))
+            else:
+                return None
+        return out
+
     def _blocks(self, first: int, n: int, contiguous: bool, segment_start: bool = True) -> bytes:
         """A format 1 segment (§1.3): blocks of L from its sample 0; start_level the sample before when it is
         contiguous in the generation, else the block's first. Format 2: _analog_blocks; format 3: _timed."""
@@ -715,10 +794,18 @@ class VirtualCapture:
         raw = [k for k in roles if self.carriages.get(k, (CARRIAGE["raw"], 1))[0] == CARRIAGE["raw"]]
         ev = [k for k in roles if k not in raw]
         ds = [self.carriages[k][1] for k in ev]
+        levels = self._levels(first, n)                            # numpy: the samples and the blocks at once
         out = bytearray()
         for b in range(0, n, self.block):
             r = min(self.block, n - b)
             at = first + b
+            if levels is not None:
+                before = [int(levels[k][b - 1]) if b else (self.value(k, at - 1) if contiguous else int(levels[k][0]))
+                          for k in ev]
+                import numpy as np
+                out += logic_block_np(np, [levels[k][b:b + r] for k in raw], [levels[k][b:b + r] for k in ev], ds,
+                                      before, self.block)
+                continue
             before = [self.value(k, at - 1) if (b or contiguous) else self.value(k, at) for k in ev]
             out += logic_block([[self.value(k, at + i) for i in range(r)] for k in raw],
                                [[self.value(k, at + i) for i in range(r)] for k in ev], ds, before, self.block)
@@ -1018,22 +1105,43 @@ class VirtualCapture:
                 if len(self.segs) >= self.segments_max:
                     self.state = STATE["paused"]
                     break
+        elif self.mode == MODE["streaming"] and due - self.produced > self.held_most():
+            events += self.overflow()                              # behind by more than its stream holds (below)
         elif self.mode == MODE["streaming"] and self.format in PIECES:
             self._stream_blocks(due, whole=True)
         elif self.mode == MODE["streaming"]:
-            per_byte = 1 if self.analog else 8 // math.gcd(self.width, 8)   # whole bytes only (§2.2)
-            n = min((due - self.produced) // per_byte * per_byte, BLOCKS_A_TICK)
-            if n > 0:
-                self.data += self._pack(self.produced, n)
-                self.produced += n
-                taken = self.produced - self.capture_begin
-                first = self.first_segment_samples or self.samples
-                complete = 0 if taken < first else 1 + (taken - first) // self.samples
-                self.serial_done = complete & 0xFFFFFFFF
-                done_samples = 0 if not complete else first + (complete - 1) * self.samples
-                self.stream_committed_bytes = self.bytes_of(done_samples)
+            self._stream_samples(due)
         return events
 
+    # Streaming follows the clock as a probe's does: sample i is the one taken at t0 + i / rate, and the samples the
+    # clock has taken that the bench has not made yet are as a probe's samples held unsent in its stream buffer. A tick
+    # makes a bounded number (the bench keeps answering), so it may be behind for a while and catch up; stop makes every
+    # sample up to the stop (a probe sends all it holds); behind by more than the stream buffer holds, the track stops
+    # in error as a probe whose buffer overflowed (state 6, error storage) - never a stream that silently ran late
+    # (a section placed by the probe's clock past the stream's end, wireskein on a busy host).
+    def held_most(self) -> int:
+        """The samples its stream buffer holds (stream_buffer, §3.3), at format 0's bytes a sample."""
+        return STREAM_BUFFER_BYTES // max(1, self.width) if self.analog else STREAM_BUFFER_BYTES * 8 // max(1, self.width)
+
+    def a_tick(self) -> int:
+        """The most samples one tick makes: more when numpy makes them at once (a sample a microsecond or more in
+        Python, about a hundredth of that at once)."""
+        return FAST_A_TICK if self._levels(self.produced, 1) is not None else BLOCKS_A_TICK
+
+    def _stream_samples(self, due: int, most: int | None = None) -> None:
+        """Format 0 streaming: the samples up to `due`, at most `most` (a_tick() by default), whole bytes only."""
+        per_byte = 1 if self.analog else 8 // math.gcd(self.width, 8)   # whole bytes only (§2.2)
+        n = min((due - self.produced) // per_byte * per_byte, self.a_tick() if most is None else most)
+        if n > 0:
+            self.data += self._pack(self.produced, n)
+            self.produced += n
+            taken = self.produced - self.capture_begin
+            first = self.first_segment_samples or self.samples
+            complete = 0 if taken < first else 1 + (taken - first) // self.samples
+            self.serial_done = complete & 0xFFFFFFFF
+            done_samples = 0 if not complete else first + (complete - 1) * self.samples
+            self.stream_committed_bytes = self.bytes_of(done_samples)
+
     def _stream_geometry(self, rel: int) -> tuple[int, int, int]:
         """Streaming: the segment holding sample `rel` (from the retained start) -> (its number, first sample, size)."""
         first = self.first_segment_samples or self.samples
@@ -1046,7 +1154,7 @@ class VirtualCapture:
         """Format 1 streaming: the blocks the clock has taken - each whole (a block counts its events first, §1.3), at
         most BLOCKS_A_TICK samples a call; `whole` False at stop: the block being taken ends there, short."""
         if whole:
-            due = min(due, self.produced + BLOCKS_A_TICK)
+            due = min(due, self.produced + self.a_tick())
         else:                                                      # stop: only the block begun, not all the time behind
             rel = self.produced - self.capture_begin
             _, at, size = self._stream_geometry(rel)
@@ -1104,6 +1212,18 @@ class VirtualCapture:
                 index = self.first_trigger_index if self.first_trigger_index < partial else NONE
                 segment = self._segment(partial, index, flags=FLAG["short"])
                 events.append(bytes([EVENT["segment"]]) + segment.pack())
+        if self.state == STATE["capturing"] and self.mode == MODE["streaming"] and now_ms is not None:
+            due = self.kept(int((now_ms - self.started_ms) * self.rate / 1000))   # every sample up to the stop: the
+            if due - self.produced > self.held_most():             # stream holds them (tick's rules, above)
+                return events + self.overflow()
+            while self.state == STATE["capturing"]:
+                before = self.produced
+                if self.format in PIECES:
+                    self._stream_blocks(due, whole=True)
+                else:
+                    self._stream_samples(due, most=due)
+                if self.produced == before:
+                    break
         if self.state == STATE["capturing"] and self.mode == MODE["streaming"] and self.format in PIECES:
             if now_ms is not None:                                 # the taken samples, a short last block
                 self._stream_blocks(self.kept(int((now_ms - self.started_ms) * self.rate / 1000)), whole=False)

{oep_client_python-0.0.47 → oep_client_python-0.0.49}/tests/test_acquire.py

@@ -115,6 +115,15 @@ def test_during_is_for_a_stream_in_place_of_seconds():
             acquire.Request([("A", "GPIO10")], 1_000_000, 100, during=lambda wait: None, **kw)
 
 
+def test_a_format_1_stream_on_disk(probe, tmp_path):
+    """Streamed in format 1, raw and changes lines, put together on disk as it comes: every segment joined in order."""
+    for resolution in ({"B": 1}, {"A": 4, "B": 1}):
+        req = acquire.Request([("A", "GPIO10"), ("B", "GPIO11")], 1_000_000, 0, mode="stream", seconds=0.3,
+                              resolution=resolution, spool_dir=str(tmp_path))
+        res = acquire.capture_at(probe(), req)
+        assert res.complete and len(res.parts) == 1, resolution
+
+
 def test_format_1_lines_as_their_changes(probe):
     req = acquire.Request([("A", "GPIO10"), ("B", "GPIO11")], 20_000_000, 1000, resolution={"B": 1})
     res = acquire.capture_at(probe(), req)
@@ -165,3 +174,43 @@ def test_lines_are_named_by_channel_name_or_label(probe):
         acquire.capture_at(probe(), acquire.Request([("A", "10")], 20_000_000, 100))     # a number is not a name
     with pytest.raises(ValueError, match="no channel carries that label"):
         acquire.capture_at(probe(), acquire.Request([("A", "x035.nrst")], 20_000_000, 100))
+
+
+def test_stream_segments_join_bit_by_bit(tmp_path):
+    """A stream's segments need not end on a whole byte of each line (2 lines in 2-bit samples, 65332 samples a
+    segment on the P4): they join into one part, bit by bit - parts are cut only where data was lost."""
+    import random
+    import types
+    from oep_client.capture import SegmentInfo
+    rnd = random.Random(1)
+    n_seg, per = 3, 12                                  # 12 samples: 3 bytes of the stream, 1.5 bytes a line
+    a = [rnd.randrange(2) for _ in range(n_seg * per)]
+    b = [rnd.randrange(2) for _ in range(n_seg * per)]
+    data = bytearray()
+    for i in range(0, len(a), 4):                       # sample i: bits 2i (A) and 2i + 1 (B) of the stream
+        v = 0
+        for j in range(4):
+            v |= (a[i + j] | b[i + j] << 1) << (2 * j)
+        data.append(v)
+    segs = {k * 3: SegmentInfo(k * 3, per, 1_000_000 + k * per * 200, 50) for k in range(n_seg)}
+    got = types.SimpleNamespace(data=data, segments=segs, gaps=[], start=0)
+    cfg = types.SimpleNamespace(rate=5_000_000, width=2, positions=None, event_d={})
+    req = acquire.Request([("A", "x"), ("B", "y")], 5_000_000, 0, mode="stream", seconds=1)
+    parts, lost = acquire._stream_parts(None, cfg, got, ["A", "B"], [10, 11], req)
+    assert not lost and len(parts) == 1
+    got_a, got_b = (list(levels(c)) for c in parts[0].channels)
+    assert got_a == a and got_b == b
+
+
+def test_bits_written_across_bytes(tmp_path):
+    import io
+    rnd = __import__("random").Random(2)
+    want, f, carry = [], io.BytesIO(), None
+    for n in (5, 12, 3, 16, 7):
+        bits = [rnd.randrange(2) for _ in range(n)]
+        want += bits
+        col = bytes(sum(bits[i + j] << j for j in range(min(8, n - i))) for i in range(0, n, 8))
+        carry = acquire._write_bits(f, carry, col, n)
+    acquire._flush_bits(f, carry)
+    out = np.unpackbits(np.frombuffer(f.getvalue(), np.uint8), bitorder="little")[: len(want)]
+    assert list(out) == want and len(f.getvalue()) == (len(want) + 7) // 8

{oep_client_python-0.0.47 → oep_client_python-0.0.49}/tests/test_virtual_bench_capture_formats.py

@@ -297,10 +297,10 @@ def test_streaming_answer_says_its_stream_buffer():
     assert lc.configure(rate=10_000_000, samples=100).stream_buffer is None    # one-shot: not said
 
 
-def test_format_0_streaming_makes_a_tick_at_most_and_stop_answers():
-    """Streaming format 0: a tick makes at most BLOCKS_A_TICK samples, so a bench far behind a fast rate still answers
-    (the stream runs late) - catching up on all the time passed at once answered stop after 12.8 s (wireskein: 1 MHz x
-    2 lines)."""
+def test_format_0_streaming_catches_up_over_ticks_and_stop_makes_every_sample():
+    """Streaming format 0: a tick makes a bounded number of samples (a_tick), so a bench behind a fast rate still
+    answers and catches up over ticks; stop makes every sample up to the stop (a probe sends what its stream buffer
+    holds) - before, the stream ran late and ended before the stop (wireskein: a section past its end on a busy host)."""
     from oep_client import virtual_bench_capture as vbc
     ep, hst, lc, clock = bench()
     planned(hst, lc, 2)
@@ -310,12 +310,30 @@ def test_format_0_streaming_makes_a_tick_at_most_and_stop_answers():
     cap = ep.captures[lc.fn]
     clock.t += 2_000                                           # 2 s (within the lease): two million samples due
     cap.tick(clock.t)
-    assert cap.produced == vbc.BLOCKS_A_TICK
+    assert cap.produced == cap.a_tick() < 2_000_000
     lc.stop()
-    assert cap.produced <= 4 * vbc.BLOCKS_A_TICK < 2_000_000   # a tick or a few (the requests tick too), not 2 s
+    assert cap.produced == 2_000_000                           # all of them, at the stop
     assert lc.status().state == c.STATE["configured"]
 
 
+def test_streaming_behind_more_than_its_buffer_stops_in_error(monkeypatch):
+    """Behind by more than its stream buffer holds, the track stops in error as a probe whose buffer overflowed: never a
+    stream that silently ran late."""
+    from oep_client import virtual_bench_capture as vbc
+    monkeypatch.setattr(vbc, "STREAM_BUFFER_BYTES", 1 << 12)  # 16384 samples of 2 lines: 16 ms at 1 MS/s
+    ep, hst, lc, clock = bench()
+    planned(hst, lc, 2)
+    lc.configure(rate=1_000_000, mode=c.STREAMING)
+    lc.subscribe(16384, 20)
+    lc.start()
+    cap = ep.captures[lc.fn]
+    assert cap.held_most() == 16384
+    clock.t += 30                                              # more than the buffer's time, in one go
+    cap.tick(clock.t)
+    st = lc.status()
+    assert st.state == c.STATE["error"]
+
+
 def test_format_1_with_every_channel_events_has_w_and_c_0():
     ep, hst, lc, clock = bench()
     roles = planned(hst, lc, 3)

oep_client_python-0.0.49/tests/test_virtual_bench_fast_samples.py

@@ -0,0 +1,64 @@
+"""The virtual bench's logic samples made at once with numpy are the ones value() gives, sample by sample: the counter,
+the logic waves, every w, any first sample and length; and a second of 1 MS/s on 2 lines takes well under a second."""
+
+import random
+import time
+from fractions import Fraction
+
+import pytest
+
+from oep_client import virtual_bench_capture as vbc
+
+pytest.importorskip("numpy")
+
+WAVES = ["quiet", "quiet:0", "square:6", "square:7", "uart:115200", "uart:9600:Hi", "uart:1000000:OEP\n"]
+
+
+def track(width, channels, rate, waves):
+    cap = vbc.VirtualCapture({0, 3}, {width}, 1, 1 << 30)
+    cap.width, cap.channels, cap.rate, cap.format = width, channels, Fraction(rate), vbc.FORMAT["raw"] \
+        if "raw" in vbc.FORMAT else 0
+    cap.logic_roles = tuple(range(channels))
+    cap.waves = {k: vbc.logic_wave(w) for k, w in waves.items()}
+    return cap
+
+
+def slow(cap, first, n):
+    return vbc.pack_samples([sum(cap.value(k, first + i) << k for k in range(cap.width)) for i in range(n)], cap.width)
+
+
+def test_at_once_is_value_sample_by_sample():
+    rnd = random.Random(5)
+    for _ in range(200):
+        width = rnd.choice([1, 2, 3, 4, 5, 8, 12, 16])
+        channels = rnd.randint(1, width)
+        rate = rnd.choice([1_000_000, 3_000_000, Fraction(80_000_000, 3), 48_000])
+        waves = {k: rnd.choice(WAVES) for k in range(channels) if rnd.random() < 0.6}
+        cap = track(width, channels, rate, waves)
+        first, n = rnd.randrange(0, 1 << 24), rnd.randint(1, 300)
+        assert cap._levels(first, n) is not None
+        assert cap._pack(first, n) == slow(cap, first, n), (width, channels, rate, waves, first, n)
+
+
+def test_a_second_of_1_ms_s_on_2_lines_is_fast():
+    cap = track(2, 2, 1_000_000, {1: "uart:115200"})
+    t = time.perf_counter()
+    for at in range(0, 1_000_000, vbc.BLOCKS_A_TICK):
+        cap._pack(at, vbc.BLOCKS_A_TICK)
+    assert time.perf_counter() - t < 0.5
+
+
+def test_format_1_blocks_at_once_are_the_same():
+    rnd = random.Random(6)
+    for _ in range(150):
+        channels = rnd.randint(1, 4)
+        cap = track(8, channels, 1_000_000, {k: rnd.choice(WAVES) for k in range(channels) if rnd.random() < 0.6})
+        cap.block = rnd.choice([64, 256])
+        cap.carriages = {k: (vbc.CARRIAGE["events"], rnd.choice([1, 4])) for k in range(channels) if rnd.random() < 0.5}
+        first, n = rnd.randrange(1, 1 << 20), rnd.randint(1, 700)
+        contiguous = rnd.random() < 0.5
+        fast = cap._blocks(first, n, contiguous=contiguous)
+        levels = cap._levels
+        cap._levels = lambda first, n: None
+        assert fast == cap._blocks(first, n, contiguous=contiguous)
+        cap._levels = levels