oep-client-python 0.0.47__tar.gz → 0.0.48__tar.gz

{oep_client_python-0.0.47 → oep_client_python-0.0.48}/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## Unreleased
 
+## 0.0.48
+- (EN) Fixed: a stream's segments join bit by bit, so a stream is one part unless data was lost. A segment whose
+  samples did not fill whole bytes of each line (the ESP32-P4's 65332 samples of 2 lines at 5 and 10 MS/s) started a new
+  part every time: 300 to 550 parts, none lost. A format 1 segment whose samples are not a multiple of every D still
+  starts a new part (its ticks restart, capture §1.3).
+- (JA) 修正: stream の区画を 1 bit ずつつなぐ。欠損がなければ、stream は 1 つの部分になる。線ごとに標本がバイトの切れ目で
+  終わらない区画（ESP32-P4 の 2 本の線の 65332 標本、5 MS/s と 10 MS/s）では、そのたびに新しい部分を始めていて、欠損なしで
+  300〜550 の部分に分かれていた。format 1 で標本数がどの D の倍数でもない区画は、今までどおり新しい部分を始める
+  （その刻みが始め直しになるため、capture §1.3）。
+
 ## 0.0.47
 - (EN) Added: `acquire.Request(during=steps)` for a stream in place of `seconds`: it streams until `steps(wait)`
   returns, `wait(seconds)` collecting while it waits, so a script marks sections and sends ops or console commands on

{oep_client_python-0.0.47 → oep_client_python-0.0.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: oep-client-python
-Version: 0.0.47
+Version: 0.0.48
 Summary: Open Embedded Probe (OEP) v1 host: serial / USB / TCP transports, the session rules, the standard interfaces, and a virtual bench
 Project-URL: Homepage, https://github.com/Open-Embedded-Probe/oep-client-python
 Project-URL: Repository, https://github.com/Open-Embedded-Probe/oep-client-python

{oep_client_python-0.0.47 → oep_client_python-0.0.48}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "oep-client-python"
-version = "0.0.47"
+version = "0.0.48"
 description = "Open Embedded Probe (OEP) v1 host: serial / USB / TCP transports, the session rules, the standard interfaces, and a virtual bench"
 readme = "README.md"
 license = "MIT"

{oep_client_python-0.0.47 → oep_client_python-0.0.48}/src/oep_client/__init__.py

@@ -2,4 +2,4 @@
 
 __all__ = ["__version__"]
 
-__version__ = "0.0.47"
+__version__ = "0.0.48"

{oep_client_python-0.0.47 → oep_client_python-0.0.48}/src/oep_client/acquire.py

@@ -1655,14 +1655,14 @@ class _Spool:
             p, nxt = ready[k][0], ready[j][1]
             x = self.segs[p]
             samples = sum(self.segs.pop(q).samples for q, _ in ready[k:j + 1])
-            if self.cur is not None and (self.cur["samples"] % 8 or any(self.cur["samples"] % d for d in self.ds)):
+            if self.cur is not None and any(self.cur["samples"] % d for d in self.ds):
                 self._close(follows=True)      # a grid restart (format 1) is not a loss: the next part follows on
             self._open(p, x)
             data = bytes(self.pending[p - self.at:nxt - self.at])
             cols = _segment_columns(self.cap, data, samples, len(self.names), self.cfg)
-            for col, f, ev in zip(cols, self.cur["files"], self.cur["events"]):
-                if isinstance(col, bytes):
-                    f.write(col[:(samples + 7) // 8])
+            for j, (col, f, ev) in enumerate(zip(cols, self.cur["files"], self.cur["events"])):
+                if isinstance(col, bytes):           # joined bit by bit: a segment need not end on a whole byte
+                    self.cur["carry"][j] = _write_bits(f, self.cur["carry"][j], col, samples)
                 else:
                     ev[0].append(col[0] + self.cur["samples"] // col[4])
                     ev[1].append(col[1])
@@ -1693,7 +1693,7 @@ class _Spool:
         k = len(self.parts)
         files = [open(os.path.join(self.dir, f"p{k}-{j}.bits"), "wb") for j in range(len(self.names))]
         self.cur = {"tick0": tick0, "meta": _seg_meta(seg) if seg is not None else {}, "samples": 0, "files": files,
-                    "events": [[[], [], [], None] for _ in self.names]}
+                    "events": [[[], [], [], None] for _ in self.names], "carry": [None] * len(self.names)}
         self._follows = False
 
     _last_samples, _last_end, _follows = 0, None, False
@@ -1705,6 +1705,7 @@ class _Spool:
         cur, n = self.cur, self.cur["samples"]
         chans = []
         for role, (name, ch, f, ev) in enumerate(zip(self.names, self.ids, cur["files"], cur["events"])):
+            _flush_bits(f, cur["carry"][role])
             f.close()
             d = self.cfg.event_d.get(role) if self.blocks else None
             if d is not None:
@@ -1791,7 +1792,7 @@ def _stream_parts(cap, cfg, got, names, ids, req: Request) -> tuple[list, list]:
                 chans = from_interleaved(data, names, cfg.width, cfg.positions, samples)
             else:
                 chans = _from_blocks(cap, data, names, samples, cfg)
-            restart = acc is not None and (acc_samples % 8 or any(acc_samples % d for d in ds))
+            restart = acc is not None and any(acc_samples % d for d in ds)
             if acc is None or restart:           # a grid restart (format 1) is not a loss: the next part follows on
                 if acc is not None:
                     parts.append(_part(_built(acc, acc_samples, ids), acc_meta, tick0))
@@ -1865,10 +1866,41 @@ def _samples_at_once(run, names, ids, cfg, samples: int) -> list:
             for name, ch, bits in zip(names, ids, packed)]
 
 
+def _write_bits(f, carry, col: bytes, samples: int):
+    """Append `samples` bits of `col` (sample i at bit i % 8 of byte i // 8) to file `f` after the `carry` bits not
+    yet written (None: none): whole bytes go to the file, the bits left over (fewer than 8) come back as the carry."""
+    if carry is None and samples % 8 == 0:
+        f.write(col[:samples // 8])
+        return None
+    import numpy as np
+    bits = np.unpackbits(np.frombuffer(col, np.uint8, (samples + 7) // 8), bitorder="little")[:samples]
+    if carry is not None:
+        bits = np.concatenate([carry, bits])
+    whole = len(bits) // 8 * 8
+    f.write(np.packbits(bits[:whole], bitorder="little").tobytes())
+    return bits[whole:] if whole < len(bits) else None
+
+
+def _flush_bits(f, carry) -> None:
+    """The last carry of _write_bits: its bits in one byte, the rest 0."""
+    if carry is not None and len(carry):
+        import numpy as np
+        f.write(np.packbits(carry, bitorder="little").tobytes())
+
+
+def _join_bits(pieces) -> bytes:
+    """Pieces (bits, n) joined bit by bit: n need not be a multiple of 8."""
+    if all(n % 8 == 0 for _, n in pieces[:-1]):
+        return b"".join(b[:(n + 7) // 8] for b, n in pieces)
+    import numpy as np
+    return np.packbits(np.concatenate([np.unpackbits(np.frombuffer(b, np.uint8, (n + 7) // 8), bitorder="little")[:n]
+                                       for b, n in pieces]), bitorder="little").tobytes()
+
+
 class _Acc:
-    """One channel of a part being joined from its segments (bits: whole bytes, the part's samples so far a multiple
-    of 8; events: quanta offset by the samples so far, a multiple of D). A segment that follows on carries the level
-    before it as its start (§1.3), so its changes join the part's as they are."""
+    """One channel of a part being joined from its segments (bits: joined bit by bit; events: quanta offset by the
+    samples so far, a multiple of D). A segment that follows on carries the level before it as its start (§1.3), so
+    its changes join the part's as they are."""
     def __init__(self, c):
         self.first, self.pieces = c, []
         self.start = getattr(c, "start_level", None)
@@ -1894,8 +1926,7 @@ def _built(acc: list, samples: int, ids) -> list:
                 ct += c_
             out.append(changes(c.name, ks, lv, ct, -(-samples // c.step), c.step, 0, a.start, pin=ch))
         else:
-            out.append(Bits(c.name, b"".join(b for b, _ in a.pieces)[:(samples + 7) // 8], samples, 1, 0,
-                                          {"pin": ch}))
+            out.append(Bits(c.name, _join_bits(a.pieces)[:(samples + 7) // 8], samples, 1, 0, {"pin": ch}))
     return out
 
 

{oep_client_python-0.0.47 → oep_client_python-0.0.48}/tests/test_acquire.py

@@ -165,3 +165,43 @@ def test_lines_are_named_by_channel_name_or_label(probe):
         acquire.capture_at(probe(), acquire.Request([("A", "10")], 20_000_000, 100))     # a number is not a name
     with pytest.raises(ValueError, match="no channel carries that label"):
         acquire.capture_at(probe(), acquire.Request([("A", "x035.nrst")], 20_000_000, 100))
+
+
+def test_stream_segments_join_bit_by_bit(tmp_path):
+    """A stream's segments need not end on a whole byte of each line (2 lines in 2-bit samples, 65332 samples a
+    segment on the P4): they join into one part, bit by bit - parts are cut only where data was lost."""
+    import random
+    import types
+    from oep_client.capture import SegmentInfo
+    rnd = random.Random(1)
+    n_seg, per = 3, 12                                  # 12 samples: 3 bytes of the stream, 1.5 bytes a line
+    a = [rnd.randrange(2) for _ in range(n_seg * per)]
+    b = [rnd.randrange(2) for _ in range(n_seg * per)]
+    data = bytearray()
+    for i in range(0, len(a), 4):                       # sample i: bits 2i (A) and 2i + 1 (B) of the stream
+        v = 0
+        for j in range(4):
+            v |= (a[i + j] | b[i + j] << 1) << (2 * j)
+        data.append(v)
+    segs = {k * 3: SegmentInfo(k * 3, per, 1_000_000 + k * per * 200, 50) for k in range(n_seg)}
+    got = types.SimpleNamespace(data=data, segments=segs, gaps=[], start=0)
+    cfg = types.SimpleNamespace(rate=5_000_000, width=2, positions=None, event_d={})
+    req = acquire.Request([("A", "x"), ("B", "y")], 5_000_000, 0, mode="stream", seconds=1)
+    parts, lost = acquire._stream_parts(None, cfg, got, ["A", "B"], [10, 11], req)
+    assert not lost and len(parts) == 1
+    got_a, got_b = (list(levels(c)) for c in parts[0].channels)
+    assert got_a == a and got_b == b
+
+
+def test_bits_written_across_bytes(tmp_path):
+    import io
+    rnd = __import__("random").Random(2)
+    want, f, carry = [], io.BytesIO(), None
+    for n in (5, 12, 3, 16, 7):
+        bits = [rnd.randrange(2) for _ in range(n)]
+        want += bits
+        col = bytes(sum(bits[i + j] << j for j in range(min(8, n - i))) for i in range(0, n, 8))
+        carry = acquire._write_bits(f, carry, col, n)
+    acquire._flush_bits(f, carry)
+    out = np.unpackbits(np.frombuffer(f.getvalue(), np.uint8), bitorder="little")[: len(want)]
+    assert list(out) == want and len(f.getvalue()) == (len(want) + 7) // 8