expr_tracker 0.4.0__tar.gz → 0.4.1__tar.gz

{expr_tracker-0.4.0 → expr_tracker-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: expr_tracker
-Version: 0.4.0
+Version: 0.4.1
 Summary: Local-first experiment tracking with queryable history and expression-based alerts on your training metrics
 Project-URL: Homepage, https://hspk.github.io/expr_tracker/
 Project-URL: Documentation, https://hspk.github.io/expr_tracker/
@@ -76,7 +76,8 @@ et.history(-1, output_type="pd")  # everything, as a DataFrame
 ## Why
 
 - **The file is the source of truth.** One JSON object per step, appended to
-  `metrics.jsonl`. No server, no database, no vendor.
+  `metrics.jsonl`, automatically split into retained segments at 100 MiB.
+  History queries read all segments transparently. No server, no database, no vendor.
 - **History is queryable during the run.** `et.history(n)` answers from an in-memory
   cache and touches the file only for what it evicted — 227 µs for
   `history(50)` whether the run has 1,000 steps or 100,000.

{expr_tracker-0.4.0 → expr_tracker-0.4.1}/README.md

@@ -26,7 +26,8 @@ et.history(-1, output_type="pd")  # everything, as a DataFrame
 ## Why
 
 - **The file is the source of truth.** One JSON object per step, appended to
-  `metrics.jsonl`. No server, no database, no vendor.
+  `metrics.jsonl`, automatically split into retained segments at 100 MiB.
+  History queries read all segments transparently. No server, no database, no vendor.
 - **History is queryable during the run.** `et.history(n)` answers from an in-memory
   cache and touches the file only for what it evicted — 227 µs for
   `history(50)` whether the run has 1,000 steps or 100,000.

{expr_tracker-0.4.0 → expr_tracker-0.4.1}/docs/architecture.md

@@ -123,6 +123,22 @@ plain functions over a sequence, so they hold no lock and are tested directly.
 `et.history(run=...)` bypasses the store entirely and reads through `JsonlReader`
 (`read_history`), which is what makes offline analysis and `et rules test` possible.
 
+### Rotation segments
+
+Metric and span writers default to `max_file_bytes=100 << 20`; `None` disables
+rotation. The original file is never renamed. Additional, permanently retained
+segments live in `<filename>.parts/00000001.jsonl`, incrementing numerically.
+Writes split batches only at record boundaries, and oversized records stay intact.
+On failure, rollback truncates the logical byte stream to its pre-batch size and
+removes newly written suffix segments before retrying.
+
+The sparse index and metadata use cumulative byte offsets and row counts across
+all segments. Readers capture the segment list and each file's byte boundary per
+query, then scan or reverse-tail them as one logical file. This preserves cache
+boundaries, cross-segment patch merging, and snapshots while new segments appear.
+Resume repairs the active segment's torn tail and restores the whole run's state.
+Span trace export uses the same segmented raw reader.
+
 ### Rows versus steps
 
 A step normally occupies one physical row, but an explicit commit followed by

{expr_tracker-0.4.0 → expr_tracker-0.4.1}/docs/reference/configuration.md

@@ -27,6 +27,7 @@ list of valid options rather than being silently ignored.
 | `max_pending_records` | 100,000 | cap on queued plus in-flight records |
 | `max_pending_bytes` | 64 MiB | cap on queued plus in-flight encoded bytes; `None` disables |
 | `max_record_bytes` | 8 MiB | maximum encoded size of one record; `None` disables |
+| `max_file_bytes` | 100 MiB | rotate metric and span files at record boundaries; `None` disables |
 
 ```python
 et.init(project="p", cache_bytes=256 << 20, step_policy="allow")
@@ -68,6 +69,22 @@ wait for accepted records to be flushed in either mode.
 `et.info()["history"]` reports metric/span `pending_bytes`, `span_pending_bytes`,
 `dropped_records` and `dropped_spans` for diagnosing overload and failed writes.
 
+### Automatic rotation
+
+Metrics and spans rotate by default before the next record would make a file
+exceed 100 MiB. Change the threshold with `et.init(max_file_bytes=20 << 20)`, or
+disable rotation with `et.init(max_file_bytes=None)`. The limit must be a positive
+integer or `None`. A record larger than the threshold stays intact in its own file
+(subject to the independent `max_record_bytes` limit).
+
+The original filename remains in place. Subsequent segments live in its `.parts`
+directory, for example `metrics.jsonl.parts/00000001.jsonl`. Stream and rank files
+rotate independently. All segments are retained: rotation limits individual file
+size, not total disk usage. Keep the `.parts` directory alongside the original file
+when copying a run. Live and offline history queries, resume, and span trace export
+read the complete logical history across segments, even when rotation is later
+disabled. Segment files are implementation details, not separate streams.
+
 ## Alert configuration
 
 Resolved in this order:

{expr_tracker-0.4.0 → expr_tracker-0.4.1}/src/expr_tracker/history/reader.py

@@ -16,12 +16,13 @@ from loguru import logger
 
 from .frame import project, to_output
 from .naming import RANK_PATTERN, metrics_filename
+from .rotation import segments
 
 CHUNK_SIZE = 256 * 1024
 
 
 class JsonlReader:
-    """Read-only access to one ``metrics.jsonl``.
+    """Read-only access to a JSONL file and all its rotation segments.
 
     ``index`` is a sparse list of ``(step, line, offset)`` anchors, loaded from the
     meta sidecar or rebuilt by scanning. With ``merge=True`` results are merged by
@@ -51,6 +52,7 @@ class JsonlReader:
         self._snapshot_limit: int | None = None
         self._snapshot_file_size: int | None = None
         self._cached_snapshot: tuple[int, JsonlReader] | None = None
+        self._segments: list[tuple[Path, int]] | None = None
 
     def _snapshot(self, end: int | None = None) -> JsonlReader:
         """A query-local reader whose metadata and data share one byte boundary."""
@@ -58,13 +60,25 @@ class JsonlReader:
             end is None or end >= self._snapshot_limit
         ):
             return self
-        size = self.size
+        captured = self._segments
+        if captured is None:
+            captured = [
+                (path, path.stat().st_size)
+                for path in segments(self.path)
+                if path.exists()
+            ]
+        size = (
+            self.size
+            if self._snapshot_limit is not None
+            else sum(length for _, length in captured)
+        )
         limit = size if end is None else max(0, min(end, size))
         source = self
         cached = self._cached_snapshot
         if cached is not None and cached[0] >= limit:
             source = cached[1]
         reader = JsonlReader(self.path)
+        reader._segments = captured
         reader._snapshot_limit = limit
         reader._snapshot_file_size = (
             size if self._snapshot_file_size is None else self._snapshot_file_size
@@ -83,7 +97,7 @@ class JsonlReader:
     def size(self) -> int:
         if self._snapshot_limit is not None:
             return self._snapshot_limit
-        return self.path.stat().st_size if self.path.exists() else 0
+        return sum(path.stat().st_size for path in segments(self.path) if path.exists())
 
     @property
     def merge(self) -> bool:
@@ -170,6 +184,23 @@ class JsonlReader:
 
     def _scan(self, start: int = 0, end: int | None = None):
         """Yield ``(byte offset, raw line)`` for every line, blanks included."""
+        if self._segments is not None:
+            limit = self.size if end is None else min(end, self.size)
+            base = 0
+            for path, length in self._segments:
+                if base >= limit:
+                    break
+                if base + length > start:
+                    reader = JsonlReader(path)
+                    reader._snapshot_limit = min(length, limit - base)
+                    reader._snapshot_file_size = length
+                    for offset, raw in reader._scan_file(max(0, start - base)):
+                        yield base + offset, raw
+                base += length
+            return
+        yield from self._scan_file(start, end)
+
+    def _scan_file(self, start: int = 0, end: int | None = None):
         if not self.path.exists():
             return
         limit = self.size if end is None else end
@@ -207,6 +238,17 @@ class JsonlReader:
 
     def _reverse_raw(self, end: int | None = None) -> Iterator[bytes]:
         """Yield nonblank lines backwards, joining long lines only once."""
+        if self._segments is not None:
+            limit = self.size if end is None else min(end, self.size)
+            base = sum(length for _, length in self._segments)
+            for path, length in reversed(self._segments):
+                base -= length
+                if base < limit:
+                    reader = JsonlReader(path)
+                    reader._snapshot_limit = min(length, limit - base)
+                    reader._snapshot_file_size = length
+                    yield from reader._reverse_raw()
+            return
         if not self.path.exists():
             return
         size = self.size

expr_tracker-0.4.1/src/expr_tracker/history/rotation.py

@@ -0,0 +1,29 @@
+"""Stable JSONL segments, addressed as one append-only byte stream."""
+
+from pathlib import Path
+
+DEFAULT_MAX_FILE_BYTES = 100 * 1024 * 1024
+
+
+def validate_file_limit(max_file_bytes: int | None) -> None:
+    if max_file_bytes is not None and (
+        type(max_file_bytes) is not int or max_file_bytes <= 0
+    ):
+        raise ValueError("max_file_bytes must be a positive integer or None")
+
+
+def segment_path(path: Path, number: int) -> Path:
+    return path.with_name(path.name + ".parts") / f"{number:08d}.jsonl"
+
+
+def segments(path: Path) -> list[Path]:
+    """The original file stays in place; later segments never rename older data."""
+    parts = path.with_name(path.name + ".parts")
+    numbered = []
+    if parts.exists():
+        numbered.extend(
+            part
+            for part in parts.glob("*.jsonl")
+            if part.stem.isascii() and part.stem.isdecimal()
+        )
+    return [path, *sorted(numbered, key=lambda part: int(part.stem))]

{expr_tracker-0.4.0 → expr_tracker-0.4.1}/src/expr_tracker/history/store.py

@@ -38,6 +38,7 @@ from .naming import (
     validate_stream,
 )
 from .reader import JsonlReader, merge_steps, recent_steps
+from .rotation import DEFAULT_MAX_FILE_BYTES, validate_file_limit
 from .series import DEFAULT_WINDOW, MetricSeries
 from .writer import (
     DEFAULT_BUFFER_INTERVAL,
@@ -115,6 +116,7 @@ class HistoryOptions:
     max_pending_records: int = DEFAULT_MAX_PENDING_RECORDS
     max_pending_bytes: int | None = DEFAULT_MAX_PENDING_BYTES
     max_record_bytes: int | None = DEFAULT_MAX_RECORD_BYTES
+    max_file_bytes: int | None = DEFAULT_MAX_FILE_BYTES
     # in-memory cache
     cache_bytes: int = DEFAULT_CACHE_BYTES
     cache_rows: int = DEFAULT_CACHE_ROWS
@@ -138,6 +140,7 @@ class HistoryOptions:
 
     def __post_init__(self):
         validate_byte_limits(self.max_pending_bytes, self.max_record_bytes)
+        validate_file_limit(self.max_file_bytes)
         if self.step_policy not in STEP_POLICIES:
             raise ValueError(
                 f"Unknown step_policy {self.step_policy!r}; "
@@ -304,6 +307,7 @@ class HistoryStore:
                 max_pending_records=opts.max_pending_records,
                 max_pending_bytes=opts.max_pending_bytes,
                 max_record_bytes=opts.max_record_bytes,
+                max_file_bytes=opts.max_file_bytes,
                 async_write=opts.span_async_write,
             )
             if self.spans_fp is not None
@@ -336,6 +340,7 @@ class HistoryStore:
             max_pending_records=options.max_pending_records,
             max_pending_bytes=options.max_pending_bytes,
             max_record_bytes=options.max_record_bytes,
+            max_file_bytes=options.max_file_bytes,
             meta_extra={
                 "project": self.project,
                 "name": self.name,
@@ -1095,9 +1100,12 @@ class HistoryStore:
         records = view.records()
         if view.complete:
             return records
-        return self._older(
-            lambda: self._older_tail(view, steps, records + opened, merge=merge)
-        ) + records
+        return (
+            self._older(
+                lambda: self._older_tail(view, steps, records + opened, merge=merge)
+            )
+            + records
+        )
 
     def _older(self, fetch: Callable[[], list[dict]]) -> list[dict]:
         """Rows from before the cache; an unreadable file degrades to cache-only.

{expr_tracker-0.4.0 → expr_tracker-0.4.1}/src/expr_tracker/history/writer.py

@@ -18,6 +18,12 @@ from loguru import logger
 
 from .._process import check_process, process_id, register_fork_reset
 from .reader import JsonlReader
+from .rotation import (
+    DEFAULT_MAX_FILE_BYTES,
+    segment_path,
+    segments,
+    validate_file_limit,
+)
 
 DEFAULT_BUFFER_SIZE = 50
 DEFAULT_BUFFER_INTERVAL = 1.0
@@ -140,6 +146,9 @@ class JsonlWriter:
       ``max_pending_bytes``. Both byte limits must be positive integers or ``None``.
     - ``async_write``: enqueue without producer disk I/O; ``flush`` and ``close``
       still wait for accepted writes. The default preserves synchronous flushing.
+    - ``max_file_bytes``: rotate before a record would exceed this file size.
+      All segments are retained; a larger record occupies a segment by itself.
+      ``None`` disables rotation.
     """
 
     def __init__(
@@ -152,13 +161,19 @@ class JsonlWriter:
         max_pending_records: int = DEFAULT_MAX_PENDING_RECORDS,
         max_pending_bytes: int | None = DEFAULT_MAX_PENDING_BYTES,
         max_record_bytes: int | None = DEFAULT_MAX_RECORD_BYTES,
+        max_file_bytes: int | None = DEFAULT_MAX_FILE_BYTES,
         index_every: int = DEFAULT_INDEX_EVERY,
         meta_extra: dict | None = None,
         async_write: bool = False,
     ):
         self._owner_pid = process_id()
         validate_byte_limits(max_pending_bytes, max_record_bytes)
+        validate_file_limit(max_file_bytes)
         self.path = Path(path)
+        self.max_file_bytes = max_file_bytes
+        active = segments(self.path)[-1]
+        self._active_number = 0 if active == self.path else int(active.stem)
+        self._active_size = 0
         self.meta_path = self.path.with_name(self.path.stem + ".meta.json")
         self.buffer_size = max(1, int(buffer_size))
         self.buffer_interval = (
@@ -213,12 +228,21 @@ class JsonlWriter:
 
     # ------------------------------------------------------------------ open/resume
 
+    @property
+    def _active_path(self) -> Path:
+        return (
+            self.path
+            if self._active_number == 0
+            else segment_path(self.path, self._active_number)
+        )
+
     def _load(self):
         """Restore state from an existing file, repairing a torn tail and stale meta."""
         if not self.path.exists():
             return
         self._repair_torn_tail()
-        size = self.path.stat().st_size
+        self._active_size = self._active_path.stat().st_size
+        size = JsonlReader(self.path).size
         if not self._restore_from_meta(size):
             self._rebuild_from_disk(size)
         self.size = size
@@ -228,10 +252,10 @@ class JsonlWriter:
 
     def _repair_torn_tail(self):
         """Drop a partially written final line so appends cannot glue onto it."""
-        size = self.path.stat().st_size
+        size = self._active_path.stat().st_size
         if size == 0:
             return
-        with open(self.path, "rb") as f:
+        with open(self._active_path, "rb") as f:
             f.seek(size - 1)
             if f.read(1) == b"\n":
                 return
@@ -245,7 +269,7 @@ class JsonlWriter:
                     cut = pos + index + 1
                     break
         try:
-            os.truncate(self.path, cut)
+            os.truncate(self._active_path, cut)
         except Exception as e:
             self._diagnostics.add(
                 "error", f"Could not truncate torn tail of {self.path}: {e}"
@@ -253,7 +277,7 @@ class JsonlWriter:
             return
         self._diagnostics.add(
             "warning",
-            f"Dropped {size - cut} bytes of a torn final line in {self.path}."
+            f"Dropped {size - cut} bytes of a torn final line in {self.path}.",
         )
 
     def _restore_from_meta(self, size: int) -> bool:
@@ -293,12 +317,9 @@ class JsonlWriter:
     def _rescan(self, offset: int, line_no: int):
         """Fold every line from ``offset`` onwards back into the counters and index."""
         self.lines = self._next_index_line = line_no
-        with open(self.path, "rb") as f:
-            f.seek(offset)
-            for raw in f:
-                if raw.strip():
-                    self._track_line(_peek_step(raw), offset)
-                offset += len(raw)
+        for position, raw in JsonlReader(self.path)._snapshot()._scan(offset):
+            if raw.strip():
+                self._track_line(_peek_step(raw), position)
 
     def _track_line(self, step: int | None, offset: int):
         """Account for one line that is already on disk."""
@@ -449,7 +470,8 @@ class JsonlWriter:
                 if self._first_buffered_time is None:
                     self._first_buffered_time = now
                 interval = (
-                    None if self._last_append_time is None
+                    None
+                    if self._last_append_time is None
                     else now - self._last_append_time
                 )
                 self._last_append_time = now
@@ -592,13 +614,42 @@ class JsonlWriter:
             self._save_meta()
 
     def _write(self, payload: bytes):
+        """Split a batch at record boundaries, without renaming pinned segments."""
+        if (
+            self.max_file_bytes is None
+            or self._active_size + len(payload) <= self.max_file_bytes
+        ):
+            self._append(payload)
+            self._active_size += len(payload)
+            return
+        start = 0
+        while start < len(payload):
+            end = start
+            while end < len(payload):
+                next_end = payload.find(b"\n", end) + 1
+                if next_end == 0:
+                    next_end = len(payload)
+                if self._active_size + next_end - start > self.max_file_bytes and (
+                    self._active_size or end > start
+                ):
+                    break
+                end = next_end
+            if end == start:
+                self._active_number += 1
+                self._active_size = 0
+                continue
+            self._append(payload[start:end])
+            self._active_size += end - start
+            start = end
+
+    def _append(self, payload: bytes):
         """Append, recreating the parent directory once if it disappeared."""
         try:
-            with open(self.path, "ab") as f:
+            with open(self._active_path, "ab") as f:
                 f.write(payload)
         except FileNotFoundError:
-            self.path.parent.mkdir(parents=True, exist_ok=True)
-            with open(self.path, "ab") as f:
+            self._active_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(self._active_path, "ab") as f:
                 f.write(payload)
 
     def _track_flushed(self, records: deque[tuple[int, int, bytes]], offset: int):
@@ -625,9 +676,23 @@ class JsonlWriter:
     def _truncate_partial_write(self, size_before: int) -> bool:
         """Roll back a partial write so retries cannot duplicate lines."""
         try:
-            if not self.path.is_file() or self.path.stat().st_size <= size_before:
-                return True  # nothing reached the file
-            os.truncate(self.path, size_before)
+            remaining = size_before
+            for path in segments(self.path):
+                if not path.exists():
+                    continue
+                length = path.stat().st_size
+                if remaining >= length:
+                    remaining -= length
+                elif remaining or path == self.path:
+                    os.truncate(path, remaining)
+                    remaining = 0
+                else:
+                    path.unlink()
+            active = segments(self.path)[-1]
+            self._active_number = 0 if active == self.path else int(active.stem)
+            self._active_size = (
+                self._active_path.stat().st_size if self._active_path.exists() else 0
+            )
             return True
         except Exception as e:
             self._diagnostics.add(
@@ -645,13 +710,12 @@ class JsonlWriter:
             self._first_buffered_time = time.monotonic()
 
     def _has_capacity(self, incoming_bytes: int) -> bool:
-        return (
-            len(self._buffer) + self._inflight_records < self.max_pending_records
-            and (
-                self.max_pending_bytes is None
-                or self._queued_bytes + self._inflight_bytes + incoming_bytes
-                <= self.max_pending_bytes
-            )
+        return len(
+            self._buffer
+        ) + self._inflight_records < self.max_pending_records and (
+            self.max_pending_bytes is None
+            or self._queued_bytes + self._inflight_bytes + incoming_bytes
+            <= self.max_pending_bytes
         )
 
     def _cap_buffer(self, incoming_bytes: int) -> tuple[bool, str]:
@@ -751,7 +815,7 @@ class JsonlWriter:
                                     "error",
                                     f"Closing {self.path} with {len(self._buffer)} "
                                     "unwritten records after a failed final flush "
-                                    f"({self._queued_bytes} bytes)."
+                                    f"({self._queued_bytes} bytes).",
                                 )
                     finally:
                         with self._condition:

{expr_tracker-0.4.0 → expr_tracker-0.4.1}/src/expr_tracker/trace.py

@@ -18,6 +18,7 @@ from tempfile import NamedTemporaryFile, TemporaryDirectory
 from typing import Any
 
 from .history.naming import parse_stream, spans_filename
+from .history.reader import JsonlReader
 
 MICROSECONDS = 1_000_000.0
 
@@ -90,7 +91,7 @@ def _iter_spans(
     if not path.is_file():
         return
     # Binary lines isolate invalid UTF-8 to one record, like malformed JSON.
-    with path.open("rb") as source:
+    with closing(JsonlReader(path).iter_raw()) as source:
         for line in source:
             try:
                 record = json.loads(line)

expr_tracker-0.4.1/tests/test_rotation.py

@@ -0,0 +1,217 @@
+import json
+
+import pytest
+
+import expr_tracker as et
+from expr_tracker.history import HistoryStore, JsonlReader, JsonlWriter, read_history
+from expr_tracker.history.rotation import DEFAULT_MAX_FILE_BYTES, segments
+from expr_tracker.trace import build_trace, read_spans
+
+
+def line(_step, **fields):
+    return json.dumps({"_step": _step, **fields}).encode() + b"\n"
+
+
+def write(writer, records):
+    start = writer.lines
+    for row, record in enumerate(records, start):
+        writer.enqueue(record["_step"], row, line(**record))
+    writer.flush()
+
+
+def test_default_and_validation(tmp_path):
+    writer = JsonlWriter(tmp_path / "metrics.jsonl")
+    assert writer.max_file_bytes == DEFAULT_MAX_FILE_BYTES == 100 << 20
+    writer.close()
+    for invalid in (0, -1, True, 1.5, "100"):
+        with pytest.raises(ValueError, match="max_file_bytes"):
+            JsonlWriter(tmp_path / "metrics.jsonl", max_file_bytes=invalid)
+        with pytest.raises(ValueError, match="max_file_bytes"):
+            HistoryStore().init("p", "n", dir=str(tmp_path), max_file_bytes=invalid)
+
+
+@pytest.mark.parametrize("async_write", [False, True])
+def test_batch_rotation_and_resume(tmp_path, async_write):
+    path = tmp_path / "metrics.jsonl"
+    records = [{"_step": step, "v": step} for step in range(20)]
+    writer = JsonlWriter(
+        path, max_file_bytes=80, max_buffer_seconds=None, async_write=async_write
+    )
+    write(writer, records)
+    writer.close()
+    assert len(segments(path)) > 2
+    assert all(part.stat().st_size <= 80 for part in segments(path))
+    reader = JsonlReader(path)
+    assert reader.read_all() == records
+    assert reader.tail(3) == records[-3:]
+    assert reader.read_steps(3, 8) == records[3:8]
+    assert reader.count_lines() == 20
+    for row in range(21):
+        boundary = reader.offset_of_line(row)
+        assert reader.tail_rows(3, end=boundary) == records[max(0, row - 3) : row]
+        assert list(reader.iter_raw(end=boundary)) == [
+            line(**record) for record in records[:row]
+        ]
+        if boundary:
+            assert list(reader.iter_raw(end=boundary - 1)) == [
+                line(**record) for record in records[: row - 1]
+            ]
+    assert list(reader.iter_raw(reader.offset_of_line(7))) == [
+        line(**record) for record in records[7:]
+    ]
+    resumed = JsonlWriter(path, max_file_bytes=None)
+    assert resumed.lines == 20
+    assert resumed.max_step == 19
+    write(resumed, [{"_step": 20, "v": 20}])
+    resumed.close()
+    assert JsonlReader(path).read_all() == [*records, {"_step": 20, "v": 20}]
+
+
+def test_boundary_oversize_and_disabled(tmp_path):
+    raw = line(0, v="x")
+    path = tmp_path / "metrics.jsonl"
+    writer = JsonlWriter(path, max_file_bytes=len(raw))
+    write(
+        writer,
+        [
+            {"_step": 0, "v": "x"},
+            {"_step": 1, "v": "x" * 100},
+            {"_step": 2, "v": "x"},
+        ],
+    )
+    writer.close()
+    assert [part.stat().st_size for part in segments(path)] == [
+        len(raw),
+        len(line(1, v="x" * 100)),
+        len(raw),
+    ]
+    disabled = JsonlWriter(tmp_path / "disabled.jsonl", max_file_bytes=None)
+    write(disabled, [{"_step": i} for i in range(10)])
+    disabled.close()
+    assert len(segments(disabled.path)) == 1
+
+
+def test_snapshot_and_patches_across_segments(tmp_path):
+    path = tmp_path / "metrics.jsonl"
+    writer = JsonlWriter(path, max_file_bytes=30)
+    write(writer, [{"_step": 0, "a": 1}, {"_step": 1, "a": 2}])
+    snapshot = writer.reader()._snapshot()
+    size = snapshot.size
+    write(writer, [{"_step": 0, "b": 3}, {"_step": 2, "a": 4}])
+    assert snapshot.size == size
+    assert snapshot.read_all() == [{"_step": 0, "a": 1}, {"_step": 1, "a": 2}]
+    assert writer.reader().tail(2) == [
+        {"_step": 0, "a": 1, "b": 3},
+        {"_step": 2, "a": 4},
+    ]
+    writer.close()
+    path.with_name("metrics.meta.json").unlink()
+    resumed = JsonlWriter(path, max_file_bytes=30)
+    assert resumed.lines == 4
+    assert resumed.max_step == 2
+    assert not resumed.sorted
+    resumed.close()
+
+
+def test_partial_write_rollback_across_segments(tmp_path, monkeypatch):
+    writer = JsonlWriter(
+        tmp_path / "metrics.jsonl", max_file_bytes=30, max_buffer_seconds=None
+    )
+    write(writer, [{"_step": 0}])
+    original = writer._append
+    calls = 0
+
+    def fail(payload):
+        nonlocal calls
+        calls += 1
+        if calls == 2:
+            original(payload[:5])
+            raise OSError("disk unavailable")
+        original(payload)
+
+    monkeypatch.setattr(writer, "_append", fail)
+    write(writer, [{"_step": i} for i in range(1, 5)])
+    assert writer.reader().read_all() == [{"_step": 0}]
+    monkeypatch.setattr(writer, "_append", original)
+    writer.flush()
+    writer.close()
+    assert JsonlReader(writer.path).read_all() == [{"_step": i} for i in range(5)]
+
+
+def test_live_offline_and_torn_tail(tmp_path):
+    store = HistoryStore().init(
+        "p",
+        "n",
+        dir=str(tmp_path),
+        max_file_bytes=100,
+        cache_rows=1,
+        cache_bytes=100,
+        max_buffer_seconds=None,
+        buffer_size=1,
+    )
+    for step in range(12):
+        store.log({"v": step}, step=step, commit=True)
+    store.flush()
+    assert [row["v"] for row in store.get(-1, output_type="records")] == list(range(12))
+    path = store.log_fp
+    store.finish()
+    assert [row["v"] for row in read_history(path, -1, output_type="records")] == list(
+        range(12)
+    )
+    with segments(path)[-1].open("ab") as source:
+        source.write(b'{"_step":')
+    resumed = JsonlWriter(path, max_file_bytes=100)
+    assert resumed.lines == 12
+    assert resumed.max_step == 11
+    resumed.close()
+
+
+def test_spans_include_rotated_records(tmp_path):
+    path = tmp_path / "spans.jsonl"
+    writer = JsonlWriter(path, max_file_bytes=100)
+    records = [
+        {"_step": i, "name": f"span{i}", "start": i, "dur_ms": 1} for i in range(5)
+    ]
+    write(writer, records)
+    writer.close()
+    assert [span["name"] for span in read_spans(path)] == [
+        record["name"] for record in records
+    ]
+    assert [
+        event["name"]
+        for event in build_trace(tmp_path)["traceEvents"]
+        if event["ph"] == "X"
+    ] == [record["name"] for record in records]
+
+
+@pytest.mark.parametrize("rank", [0, 2])
+def test_public_history_stream_rank_spans_and_resume(run, monkeypatch, rank):
+    monkeypatch.setenv("RANK", str(rank))
+    instance = run(
+        stream="data",
+        max_file_bytes=100,
+        cache_rows=1,
+        buffer_size=1,
+        max_buffer_seconds=None,
+    )
+    for step in range(8):
+        with et.span("work"):
+            pass
+        et.log({"v": step})
+    instance.history.flush()
+    assert instance.history.writer.max_file_bytes == 100
+    assert instance.history.span_writer.max_file_bytes == 100
+    path = instance.history.log_fp
+    span_path = instance.history.spans_fp
+    assert len(segments(path)) > 1
+    assert [row["v"] for row in et.history(-1, output_type="records")] == list(range(8))
+    et.finish()
+    assert len(read_spans(span_path)) == 8
+    assert [
+        row["v"]
+        for row in et.history(-1, run=path.parent, stream="data", output_type="records")
+    ] == list(range(8))
+    resumed = run(stream="data", max_file_bytes=100, cache_rows=1, buffer_size=1)
+    assert resumed.history.current_step == 8
+    et.log({"v": 8})
+    assert [row["v"] for row in et.history(-1, output_type="records")] == list(range(9))