errlore 0.1.0__py3-none-any.whl

errlore/io/repair.py

@@ -0,0 +1,199 @@
+"""Auto-repair for corrupted JSONL files.
+
+Scans JSONL files, recovers glued JSON records (missing newlines),
+drops unrecoverable lines, and rebuilds the sidecar index.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import re
+from pathlib import Path
+
+from errlore.io.jsonl_index import JSONLIndex
+from errlore.io.jsonl_writer import JSONLWriter
+
+logger = logging.getLogger("errlore.io")
+
+# Pattern for glued JSON objects: `}{` without a newline between them
+_GLUED_PATTERN = re.compile(r"\}\s*\{")
+
+
+def _try_parse(line: str) -> list[dict[str, object]]:
+    """Attempt to parse a line as JSON. Handles glued records."""
+    line = line.strip()
+    if not line:
+        return []
+
+    # Attempt 1: standard JSON
+    try:
+        obj = json.loads(line)
+        if isinstance(obj, dict):
+            return [obj]
+        return []
+    except json.JSONDecodeError:
+        pass
+
+    # Attempt 2: glued JSON (`}{` -> `}\n{`)
+    if _GLUED_PATTERN.search(line):
+        parts = _GLUED_PATTERN.sub("}\n{", line).split("\n")
+        results: list[dict[str, object]] = []
+        for part in parts:
+            part = part.strip()
+            if not part:
+                continue
+            try:
+                obj = json.loads(part)
+                if isinstance(obj, dict):
+                    results.append(obj)
+            except json.JSONDecodeError:
+                pass
+        if results:
+            return results
+
+    return []
+
+
+class RepairStats:
+    """Statistics from a JSONL repair operation."""
+
+    __slots__ = ("dropped", "fixed", "index_rebuilt", "ok", "total_lines")
+
+    def __init__(self) -> None:
+        self.ok: int = 0
+        self.fixed: int = 0
+        self.dropped: int = 0
+        self.total_lines: int = 0
+        self.index_rebuilt: int = 0
+
+    def as_dict(self) -> dict[str, int]:
+        """Return stats as a plain dict."""
+        return {
+            "ok": self.ok,
+            "fixed": self.fixed,
+            "dropped": self.dropped,
+            "total_lines": self.total_lines,
+            "index_rebuilt": self.index_rebuilt,
+        }
+
+
+def repair_file(
+    path: Path,
+    *,
+    dry_run: bool = False,
+    writer: JSONLWriter | None = None,
+) -> RepairStats:
+    """Repair a single JSONL file.
+
+    Recovers glued records, drops unrecoverable lines, then atomically
+    rewrites the file and rebuilds the sidecar index.
+
+    Args:
+        path: Path to the JSONL file.
+        dry_run: If True, only report what would happen.
+        writer: Optional JSONLWriter instance (created internally if None).
+
+    Returns:
+        RepairStats with counts of ok/fixed/dropped lines.
+    """
+    stats = RepairStats()
+    repaired: list[dict[str, object]] = []
+
+    if not path.exists():
+        # Clean up orphan index if JSONL is gone
+        idx = JSONLIndex(path)
+        if not dry_run and idx.idx_path.exists():
+            with contextlib.suppress(OSError):
+                idx.idx_path.unlink()
+        return stats
+
+    w = writer or JSONLWriter()
+
+    # A3: hold the file lock for the entire read -> parse -> rewrite cycle
+    # so that concurrent appends are not lost between read and rewrite.
+    with w.lock(path):
+        try:
+            raw = path.read_bytes()
+        except OSError as exc:
+            logger.error("Cannot read %s: %s", path, exc)
+            return stats
+
+        for line_bytes in raw.split(b"\n"):
+            stats.total_lines += 1
+            try:
+                line = line_bytes.decode("utf-8").strip()
+            except UnicodeDecodeError:
+                stats.dropped += 1
+                continue
+
+            if not line:
+                continue
+
+            parsed = _try_parse(line)
+            if not parsed:
+                stats.dropped += 1
+                continue
+
+            if len(parsed) == 1:
+                # Single JSON per line
+                try:
+                    json.loads(line)
+                    stats.ok += 1
+                except json.JSONDecodeError:
+                    stats.fixed += 1
+            else:
+                # Glued records separated
+                stats.fixed += len(parsed)
+
+            repaired.extend(parsed)
+
+        if not dry_run and (stats.fixed > 0 or stats.dropped > 0):
+            w.atomic_rewrite(path, repaired)
+
+        if not dry_run:
+            idx = JSONLIndex(path)
+            stats.index_rebuilt = idx.rebuild()
+
+    return stats
+
+
+def repair_directory(
+    directory: Path,
+    *,
+    dry_run: bool = False,
+    recursive: bool = True,
+) -> dict[str, RepairStats]:
+    """Repair all JSONL files in a directory.
+
+    Args:
+        directory: Directory to scan.
+        dry_run: If True, only report what would happen.
+        recursive: If True, scan subdirectories too.
+
+    Returns:
+        Mapping of filename to RepairStats.
+    """
+    if not directory.is_dir():
+        logger.error("Not a directory: %s", directory)
+        return {}
+
+    pattern = "**/*.jsonl" if recursive else "*.jsonl"
+    files = sorted(directory.glob(pattern))
+
+    # Deduplicate resolved paths
+    seen: set[str] = set()
+    unique: list[Path] = []
+    for f in files:
+        key = str(f.resolve())
+        if key not in seen:
+            seen.add(key)
+            unique.append(f)
+
+    writer = JSONLWriter()
+    results: dict[str, RepairStats] = {}
+    for path in unique:
+        results[str(path)] = repair_file(path, dry_run=dry_run, writer=writer)
+
+    return results

errlore/lessons/__init__.py

@@ -0,0 +1,16 @@
+"""Lesson subsystem: error tracking, lesson extraction, reinforcement, and decay.
+
+Public API:
+    ErrorRecord  -- dataclass for logged errors
+    Lesson       -- dataclass for extracted lessons
+    LessonStore  -- persistent store backed by JSONL via errlore.io
+"""
+
+from errlore.lessons.models import ErrorRecord, Lesson
+from errlore.lessons.store import LessonStore
+
+__all__ = [
+    "ErrorRecord",
+    "Lesson",
+    "LessonStore",
+]

errlore/lessons/models.py

@@ -0,0 +1,129 @@
+"""Data models for the lesson subsystem."""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any
+
+
+def _short_id() -> str:
+    """Generate a 12-char hex ID from uuid4."""
+    return uuid.uuid4().hex[:12]
+
+
+def _utc_now_iso() -> str:
+    """Return timezone-aware UTC ISO timestamp."""
+    return datetime.now(UTC).isoformat()
+
+
+@dataclass
+class ErrorRecord:
+    """A recorded error event."""
+
+    model: str
+    task_type: str
+    error_type: str
+    message: str
+    id: str = field(default_factory=_short_id)
+    timestamp: str = field(default_factory=_utc_now_iso)
+    resolved: bool = False
+    resolution: str = ""
+    context: str = ""
+    stacktrace: str = ""
+    metadata: dict[str, Any] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dict for JSONL storage."""
+        d: dict[str, Any] = {
+            "id": self.id,
+            "model": self.model,
+            "task_type": self.task_type,
+            "error_type": self.error_type,
+            "message": self.message,
+            "timestamp": self.timestamp,
+            "resolved": self.resolved,
+            "resolution": self.resolution,
+            "context": self.context,
+            "stacktrace": self.stacktrace,
+        }
+        if self.metadata is not None:
+            d["metadata"] = self.metadata
+        return d
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> ErrorRecord:
+        """Deserialize from a JSONL dict."""
+        return cls(
+            id=str(d.get("id", _short_id())),
+            model=str(d.get("model", "")),
+            task_type=str(d.get("task_type", "")),
+            error_type=str(d.get("error_type", "")),
+            message=str(d.get("message", "")),
+            timestamp=str(d.get("timestamp", _utc_now_iso())),
+            resolved=bool(d.get("resolved", False)),
+            resolution=str(d.get("resolution", "")),
+            context=str(d.get("context", "")),
+            stacktrace=str(d.get("stacktrace", "")),
+            metadata=d.get("metadata"),
+        )
+
+
+@dataclass
+class Lesson:
+    """An extracted lesson linking a problem pattern to its solution."""
+
+    pattern: str
+    solution: str
+    id: str = field(default_factory=_short_id)
+    task_type: str = ""
+    error_type: str = ""
+    confidence: float = 0.8
+    applied_count: int = 0
+    created_at: str = field(default_factory=_utc_now_iso)
+    updated_at: str = field(default_factory=_utc_now_iso)
+    source_error_id: str = ""
+    source_errors: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dict for JSONL storage."""
+        d: dict[str, Any] = {
+            "id": self.id,
+            "pattern": self.pattern,
+            "solution": self.solution,
+            "task_type": self.task_type,
+            "error_type": self.error_type,
+            "confidence": self.confidence,
+            "applied_count": self.applied_count,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "source_error_id": self.source_error_id,
+            "source_errors": self.source_errors,
+        }
+        if self.metadata is not None:
+            d["metadata"] = self.metadata
+        return d
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> Lesson:
+        """Deserialize from a JSONL dict."""
+        raw_source_errors = d.get("source_errors")
+        source_errors: list[str] = (
+            [str(e) for e in raw_source_errors] if isinstance(raw_source_errors, list) else []
+        )
+        return cls(
+            id=str(d.get("id", _short_id())),
+            pattern=str(d.get("pattern", "")),
+            solution=str(d.get("solution", "")),
+            task_type=str(d.get("task_type", "")),
+            error_type=str(d.get("error_type", "")),
+            confidence=float(d.get("confidence", 0.8)),
+            applied_count=int(d.get("applied_count", 0)),
+            created_at=str(d.get("created_at", _utc_now_iso())),
+            updated_at=str(d.get("updated_at", _utc_now_iso())),
+            source_error_id=str(d.get("source_error_id", "")),
+            source_errors=source_errors,
+            metadata=d.get("metadata"),
+        )