errlore 0.1.0__py3-none-any.whl

errlore/io/jsonl_index.py

@@ -0,0 +1,199 @@
+"""JSONL sidecar index for O(1) access to records by byte offset.
+
+Each JSONL file can have a companion .idx file storing 8-byte big-endian
+unsigned offsets. This enables:
+- get_recent() via tail-read without scanning the entire file
+- O(1) append (add offset to end of .idx)
+- rebuild when the index is corrupted
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+import struct
+import tempfile
+import threading
+from pathlib import Path
+
+logger = logging.getLogger("errlore.io")
+
+# Each entry in .idx = 8 bytes (unsigned long long, big-endian)
+_OFFSET_FORMAT = ">Q"
+_OFFSET_SIZE = struct.calcsize(_OFFSET_FORMAT)
+
+
+class JSONLIndex:
+    """Byte-offset index for a JSONL file."""
+
+    def __init__(self, jsonl_path: Path) -> None:
+        self.jsonl_path = Path(jsonl_path)
+        self.idx_path = self.jsonl_path.with_suffix(".idx")
+        self._lock = threading.Lock()
+
+    def append_offset(self, byte_offset: int) -> None:
+        """Append an offset for a new record to the index."""
+        with self._lock:
+            self.idx_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(self.idx_path, "ab") as f:
+                f.write(struct.pack(_OFFSET_FORMAT, byte_offset))
+
+    def _read_all_offsets(self) -> list[int]:
+        """Read all valid offsets from .idx, discarding any truncated tail."""
+        if not self.idx_path.exists():
+            return []
+
+        file_size = self.idx_path.stat().st_size
+        if file_size == 0:
+            return []
+
+        aligned_size = file_size - (file_size % _OFFSET_SIZE)
+        if aligned_size != file_size:
+            logger.warning(
+                "Index file %s has truncated tail (%d bytes ignored)",
+                self.idx_path.name,
+                file_size - aligned_size,
+            )
+
+        if aligned_size == 0:
+            return []
+
+        offsets: list[int] = []
+        with open(self.idx_path, "rb") as f:
+            data = f.read(aligned_size)
+        for i in range(0, aligned_size, _OFFSET_SIZE):
+            offsets.append(struct.unpack(_OFFSET_FORMAT, data[i : i + _OFFSET_SIZE])[0])
+        return offsets
+
+    def get_recent_offsets(self, count: int) -> list[int]:
+        """Get byte-offsets of the last N records."""
+        if count <= 0:
+            return []
+        with self._lock:
+            offsets = self._read_all_offsets()
+            total_entries = len(offsets)
+            if total_entries == 0:
+                return []
+            start = max(0, total_entries - count)
+            return offsets[start:]
+
+    def read_records_at_offsets(self, offsets: list[int]) -> list[dict[str, object]]:
+        """Read records from the JSONL file at specified byte-offsets."""
+        if not self.jsonl_path.exists():
+            return []
+        records: list[dict[str, object]] = []
+        with open(self.jsonl_path, encoding="utf-8") as f:
+            for offset in offsets:
+                try:
+                    f.seek(offset)
+                    line = f.readline()
+                    if line.strip():
+                        records.append(json.loads(line))
+                except (OSError, json.JSONDecodeError, UnicodeDecodeError) as e:
+                    logger.warning("Corrupted record at offset %d: %s", offset, e)
+        return records
+
+    def get_recent_records(self, count: int) -> list[dict[str, object]]:
+        """Get the last N records via the index. O(1) per seek."""
+        offsets = self.get_recent_offsets(count)
+        if not offsets:
+            return []
+        return self.read_records_at_offsets(offsets)
+
+    def entry_count(self) -> int:
+        """Number of entries in the index."""
+        return len(self._read_all_offsets())
+
+    def rebuild(self) -> int:
+        """Rebuild the index from the JSONL file.
+
+        Returns:
+            Number of indexed entries.
+        """
+        if not self.jsonl_path.exists():
+            if self.idx_path.exists():
+                try:
+                    self.idx_path.unlink()
+                except OSError:
+                    logger.warning("Failed to remove stale index file %s", self.idx_path)
+            return 0
+
+        count = 0
+        tmp_fd, tmp_path_str = tempfile.mkstemp(dir=self.idx_path.parent, suffix=".idx.tmp")
+        try:
+            with self._lock, os.fdopen(tmp_fd, "wb") as idx_f:
+                with open(self.jsonl_path, encoding="utf-8") as jsonl_f:
+                    while True:
+                        offset = jsonl_f.tell()
+                        line = jsonl_f.readline()
+                        if not line:
+                            break
+                        line = line.strip()
+                        if not line:
+                            continue
+                        try:
+                            json.loads(line)
+                            idx_f.write(struct.pack(_OFFSET_FORMAT, offset))
+                            count += 1
+                        except json.JSONDecodeError:
+                            logger.warning(
+                                "Skipping corrupted line at offset %d during rebuild",
+                                offset,
+                            )
+                idx_f.flush()
+                os.fsync(idx_f.fileno())
+            os.replace(tmp_path_str, str(self.idx_path))
+            tmp_path_str = ""
+        finally:
+            if tmp_path_str:
+                with contextlib.suppress(OSError):
+                    os.unlink(tmp_path_str)
+        logger.info("Rebuilt index for %s: %d entries", self.jsonl_path.name, count)
+        return count
+
+    def verify_integrity(self) -> tuple[bool, list[str]]:
+        """Verify index integrity against the JSONL file.
+
+        Returns:
+            Tuple of (is_valid, list_of_error_messages).
+        """
+        errors: list[str] = []
+
+        if not self.idx_path.exists():
+            if self.jsonl_path.exists() and self.jsonl_path.stat().st_size > 0:
+                errors.append("Index file missing but JSONL has data")
+            return len(errors) == 0, errors
+
+        idx_size = self.idx_path.stat().st_size
+        if idx_size % _OFFSET_SIZE != 0:
+            errors.append(f"Index file size {idx_size} not aligned to {_OFFSET_SIZE} bytes")
+
+        if not self.jsonl_path.exists():
+            errors.append("JSONL file missing but index exists")
+            return False, errors
+
+        jsonl_size = self.jsonl_path.stat().st_size
+        offsets = self.get_recent_offsets(self.entry_count())
+        previous_offset = -1
+        for offset in offsets:
+            if offset < previous_offset:
+                errors.append("Offsets are not monotonically increasing")
+            previous_offset = offset
+            if offset >= jsonl_size:
+                errors.append(f"Offset {offset} exceeds JSONL size {jsonl_size}")
+                continue
+
+            try:
+                with open(self.jsonl_path, encoding="utf-8") as f:
+                    f.seek(offset)
+                    line = f.readline()
+                if not line.strip():
+                    errors.append(f"Offset {offset} points to empty record")
+                    continue
+                json.loads(line)
+            except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc:
+                errors.append(f"Offset {offset} is invalid: {exc}")
+
+        return len(errors) == 0, errors

errlore/io/jsonl_writer.py

@@ -0,0 +1,342 @@
+"""Cross-process durable JSONL writer with filelock.
+
+Guarantees append atomicity via:
+- filelock (cross-process locking)
+- binary mode "ab" (atomic writes below PIPE_BUF)
+- explicit fsync by default
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+import struct
+import tempfile
+import threading
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from filelock import FileLock
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Sequence
+
+logger = logging.getLogger("errlore.io")
+
+_LOCK_TIMEOUT: float = 10.0
+_REPLACE_RETRIES: int = 5
+_REPLACE_BACKOFF: float = 0.5
+
+_DEFAULT_MAX_BYTES: int = 50_000_000  # 50 MB
+_DEFAULT_MAX_ARCHIVES: int = 3
+
+
+class JSONLWriter:
+    """Cross-process JSONL writer with filelock, rotation, and atomic rewrite.
+
+    Args:
+        lock_timeout: Timeout in seconds for acquiring file locks.
+        max_bytes: Maximum file size before rotation.  ``None`` disables
+            rotation entirely -- useful for files with ID-based lookups
+            (errors, lessons, injections) where all records must remain
+            visible.  Growth is handled by future log compaction.
+        max_archives: Maximum number of rotated archive files to keep.
+    """
+
+    def __init__(
+        self,
+        lock_timeout: float = _LOCK_TIMEOUT,
+        max_bytes: int | None = _DEFAULT_MAX_BYTES,
+        max_archives: int = _DEFAULT_MAX_ARCHIVES,
+    ) -> None:
+        self._lock_timeout = lock_timeout
+        self._max_bytes = max_bytes
+        self._max_archives = max_archives
+        self._locks: dict[str, FileLock] = {}
+        self._lock_guard = threading.Lock()
+        # C1: mtime+size read cache -- avoids O(n) re-parse on hot paths.
+        # Key: str(path), value: (mtime_ns, size, records).
+        self._read_cache: dict[str, tuple[int, int, list[dict[str, object]]]] = {}
+        self._cache_lock = threading.Lock()
+
+    def _get_lock(self, path: Path) -> FileLock:
+        """Get or create a FileLock for the given path (thread-safe)."""
+        key = str(path)
+        if key not in self._locks:
+            with self._lock_guard:
+                # Double-checked: another thread may have created it.
+                if key not in self._locks:
+                    self._locks[key] = FileLock(
+                        key + ".lock", timeout=self._lock_timeout,
+                    )
+        return self._locks[key]
+
+    def lock(self, path: Path) -> FileLock:
+        """Return the FileLock for *path* (public access for callers that
+        need to wrap broader read-modify-write cycles under the same lock).
+
+        The returned ``FileLock`` is re-entrant within the same thread, so
+        nested ``append`` / ``read_all`` calls inside a ``with writer.lock(p):``
+        block will not deadlock.
+        """
+        return self._get_lock(path)
+
+    @staticmethod
+    def _rotate_versioned_files(path: Path, max_archives: int) -> None:
+        """Shift archive chain for a versioned file.
+
+        Works for both *.jsonl and *.idx: current becomes .1,
+        previous .1 becomes .2, etc.
+        """
+        for i in range(max_archives, 0, -1):
+            src = path.with_suffix(f"{path.suffix}.{i}")
+            dst = path.with_suffix(f"{path.suffix}.{i + 1}")
+            if i == max_archives and src.exists():
+                src.unlink()
+            elif src.exists():
+                src.replace(dst)
+
+        if path.exists():
+            path.replace(path.with_suffix(f"{path.suffix}.1"))
+
+    def _rotate_if_needed(self, path: Path) -> None:
+        """Rotate the JSONL file when it exceeds max_bytes.
+
+        When ``max_bytes`` is ``None``, rotation is disabled.
+        """
+        if self._max_bytes is None:
+            return
+        if not path.exists() or path.stat().st_size < self._max_bytes:
+            return
+        idx_path = path.with_suffix(".idx")
+        self._rotate_versioned_files(idx_path, self._max_archives)
+        self._rotate_versioned_files(path, self._max_archives)
+
+    def append(
+        self,
+        path: Path,
+        entry: dict[str, object],
+        *,
+        fsync: bool = True,
+    ) -> int:
+        """Append a single record to a JSONL file (cross-process safe).
+
+        Args:
+            path: Path to the JSONL file.
+            entry: Dictionary record to write.
+            fsync: Force fsync after write (default True).
+
+        Returns:
+            Byte offset of the written line's start.
+        """
+        lock = self._get_lock(path)
+        line = (json.dumps(entry, ensure_ascii=False, default=str) + "\n").encode("utf-8")
+
+        with lock:
+            self._rotate_if_needed(path)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with open(path, "ab") as f:
+                offset = f.tell()
+                f.write(line)
+                if fsync:
+                    f.flush()
+                    os.fsync(f.fileno())
+        return offset
+
+    def append_batch(
+        self,
+        path: Path,
+        entries: Sequence[dict[str, object]],
+        *,
+        fsync: bool = True,
+    ) -> int:
+        """Append multiple records under a single lock acquisition.
+
+        Args:
+            path: Path to the JSONL file.
+            entries: Sequence of dict records.
+            fsync: Force fsync after the entire batch (default True).
+
+        Returns:
+            Number of records written.
+        """
+        if not entries:
+            return 0
+
+        lines = b"".join(
+            (json.dumps(e, ensure_ascii=False, default=str) + "\n").encode("utf-8")
+            for e in entries
+        )
+        lock = self._get_lock(path)
+
+        with lock:
+            self._rotate_if_needed(path)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with open(path, "ab") as f:
+                f.write(lines)
+                if fsync:
+                    f.flush()
+                    os.fsync(f.fileno())
+        return len(entries)
+
+    def atomic_rewrite(
+        self,
+        path: Path,
+        entries: Sequence[dict[str, object]],
+    ) -> None:
+        """Atomically rewrite a JSONL file (filelock + tmp + os.replace).
+
+        Also rebuilds the sidecar .idx file atomically.
+
+        Args:
+            path: Path to the JSONL file.
+            entries: Full list of records to write.
+        """
+        from errlore.io.jsonl_index import JSONLIndex
+
+        lock = self._get_lock(path)
+        tmp_jsonl: str | None = None
+        tmp_idx: str | None = None
+        idx_path = path.with_suffix(".idx")
+
+        with lock:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            data_fd, tmp_jsonl = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
+            idx_fd, tmp_idx = tempfile.mkstemp(dir=path.parent, suffix=".idx.tmp")
+            try:
+                with os.fdopen(data_fd, "wb") as data_f, os.fdopen(idx_fd, "wb") as idx_f:
+                    offset = 0
+                    for entry in entries:
+                        line = (
+                            json.dumps(entry, ensure_ascii=False, default=str) + "\n"
+                        ).encode("utf-8")
+                        data_f.write(line)
+                        idx_f.write(struct.pack(">Q", offset))
+                        offset += len(line)
+                    data_f.flush()
+                    os.fsync(data_f.fileno())
+                    idx_f.flush()
+                    os.fsync(idx_f.fileno())
+
+                # Replace data file with retries (Windows Defender can hold handles)
+                for attempt in range(_REPLACE_RETRIES):
+                    try:
+                        if tmp_jsonl is not None:
+                            os.replace(tmp_jsonl, str(path))
+                        tmp_jsonl = None
+                        break
+                    except PermissionError:
+                        if attempt < _REPLACE_RETRIES - 1:
+                            time.sleep(_REPLACE_BACKOFF * (attempt + 1))
+                        else:
+                            raise
+
+                # Replace index file
+                try:
+                    if tmp_idx is not None:
+                        os.replace(tmp_idx, str(idx_path))
+                    tmp_idx = None
+                except PermissionError:
+                    # If sidecar locked, rebuild index from fresh JSONL
+                    JSONLIndex(path).rebuild()
+                    if tmp_idx is not None:
+                        with contextlib.suppress(OSError):
+                            os.unlink(tmp_idx)
+                    tmp_idx = None
+            finally:
+                if tmp_jsonl is not None:
+                    with contextlib.suppress(OSError):
+                        os.unlink(tmp_jsonl)
+                if tmp_idx is not None:
+                    with contextlib.suppress(OSError):
+                        os.unlink(tmp_idx)
+
+    def atomic_update(
+        self,
+        path: Path,
+        transform: Callable[[list[dict[str, object]]], list[dict[str, object]] | None],
+    ) -> list[dict[str, object]] | None:
+        """Read-modify-write a JSONL file under ONE file lock (no lost updates).
+
+        ``atomic_rewrite`` alone is not race-safe for read-modify-write cycles:
+        if a caller reads the file, computes new content, and then rewrites,
+        any record appended by another thread/process in between is silently
+        lost.  ``atomic_update`` holds the file lock across the entire
+        read -> transform -> replace sequence, so concurrent ``append`` calls
+        (which take the same lock) serialize correctly.
+
+        Args:
+            path: Path to the JSONL file.
+            transform: Callback receiving the current records; returns the new
+                full record list, or None to abort without writing.
+
+        Returns:
+            The new record list that was written, or None if aborted.
+        """
+        lock = self._get_lock(path)
+        with lock:  # FileLock is re-entrant within the same thread
+            entries = self.read_all(path)
+            new_entries = transform(entries)
+            if new_entries is None:
+                return None
+            self.atomic_rewrite(path, new_entries)
+            return new_entries
+
+    def _invalidate_cache(self, path: Path) -> None:
+        """Drop the read cache entry for *path*."""
+        with self._cache_lock:
+            self._read_cache.pop(str(path), None)
+
+    def read_all(self, path: Path) -> list[dict[str, object]]:
+        """Read all valid records from a JSONL file.
+
+        Results are cached by (mtime_ns, size).  Subsequent calls that hit the
+        cache skip JSON parsing entirely.  Callers receive shallow copies of
+        the cached records so mutations do not corrupt the cache.
+
+        Args:
+            path: Path to the JSONL file.
+
+        Returns:
+            List of parsed dict records.
+        """
+        if not path.exists():
+            return []
+
+        st = path.stat()
+        key = str(path)
+
+        with self._cache_lock:
+            cached = self._read_cache.get(key)
+            if cached is not None:
+                c_mtime, c_size, c_records = cached
+                if c_mtime == st.st_mtime_ns and c_size == st.st_size:
+                    return [dict(r) for r in c_records]
+
+        # Cache miss -- parse from disk.
+        records: list[dict[str, object]] = []
+        with open(path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    obj = json.loads(line)
+                    if isinstance(obj, dict):
+                        records.append(obj)
+                except json.JSONDecodeError:
+                    logger.warning("Skipping corrupted line in %s", path.name)
+
+        # Re-stat after parse to store a consistent snapshot.
+        try:
+            st2 = path.stat()
+        except OSError:
+            return records
+
+        with self._cache_lock:
+            self._read_cache[key] = (st2.st_mtime_ns, st2.st_size, records)
+
+        return [dict(r) for r in records]