thread-archive 0.0.2__py3-none-any.whl

thread_archive/_importers/_line_stream.py

@@ -0,0 +1,193 @@
+"""Shared incremental-import scaffold for the line-stream providers (codex, grok,
+antigravity).
+
+All three read one JSONL transcript and differ only in provider-specific steps
+supplied as callbacks. The orchestration — the append-proving cursor (:mod:`._cursor`),
+thread resolve/create, empty-thread cleanup, watermark persist — is identical and
+lives here. Runs as one atomic transaction (events + watermark commit together)
+when no session is passed.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Callable, Optional
+
+from .._store import get_session
+from ._cursor import resolve_source_cursor
+from ._read import parse_session_lines, read_source_bytes
+from ._result import IncrementalImportResult
+from ._state import (
+    adopt_if_unwatermarked,
+    create_thread,
+    discard_new_thread,
+    get_import_state,
+    get_thread_by_source,
+    upsert_import_state,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _run(
+    session,
+    *,
+    source: str,
+    source_id: str,
+    session_path: Path,
+    all_lines: list[dict],
+    source_bytes: bytes,
+    prepare,
+    has_importable_content,
+    make_title,
+    import_lines,
+    make_source_metadata,
+) -> IncrementalImportResult:
+    import_state = get_import_state(session, source, source_id)
+    cursor = resolve_source_cursor(import_state, source_bytes, source=source, source_id=source_id)
+    current_file_size = len(source_bytes)
+
+    if cursor.unchanged and import_state:
+        # Byte-identical to the last import; stamp the digest if the watermark predates it.
+        import_state.last_content_hash = cursor.content_hash
+        return IncrementalImportResult(
+            0, 0, import_state.thread_id or 0, False, import_state.last_message_uuid
+        )
+
+    total_lines = len(all_lines)
+    start_line = cursor.start_line
+    if start_line >= total_lines:
+        # Changed bytes, no new parsed lines (a torn tail the writer hasn't finished).
+        # Carry size + digest forward so the completed line reads as an append next
+        # poll; the line cursor tracks what parsed, so that line lands then.
+        if import_state:
+            import_state.last_line_count = total_lines
+            import_state.last_file_size = current_file_size
+            import_state.last_content_hash = cursor.content_hash
+        return IncrementalImportResult(
+            0,
+            0,
+            (import_state.thread_id or 0) if import_state else 0,
+            False,
+            import_state.last_message_uuid if import_state else None,
+        )
+
+    new_lines = all_lines[start_line:]
+    ctx = prepare(all_lines, session_path) if prepare is not None else None
+
+    thread_id: Optional[int] = (
+        import_state.thread_id if (import_state and import_state.thread_id) else None
+    )
+    if thread_id is None:
+        existing = get_thread_by_source(session, source, source_id)
+        thread_id = existing.id if existing else None
+
+    if import_state is None and adopt_if_unwatermarked(
+        session, source=source, source_id=source_id,
+        thread_id=thread_id, total_lines=total_lines, file_size=current_file_size,
+        content_hash=cursor.content_hash,
+    ):
+        return IncrementalImportResult(0, 0, thread_id or 0, False, None)
+
+    is_new_thread = False
+    if thread_id is None:
+        if not has_importable_content(new_lines):
+            # Nothing worth a thread — record the watermark so the watcher doesn't
+            # re-scan, and stop.
+            upsert_import_state(
+                session,
+                source=source,
+                source_id=source_id,
+                thread_id=None,
+                last_line_count=total_lines,
+                last_file_size=current_file_size,
+                last_content_hash=cursor.content_hash,
+                last_message_uuid=None,
+            )
+            return IncrementalImportResult(len(new_lines), 0, 0, False)
+        thread_id = create_thread(
+            session,
+            source=source,
+            source_id=source_id,
+            title=make_title(all_lines, ctx),
+            source_metadata=make_source_metadata(ctx) if make_source_metadata else None,
+        )
+        is_new_thread = True
+
+    events_created, last_uuid = import_lines(session, thread_id, all_lines, new_lines, ctx)
+
+    if is_new_thread and events_created == 0:
+        # Row AND staged truth record — no ghost threads/<id>.jsonl on commit.
+        discard_new_thread(session, thread_id)
+        thread_id = 0
+        is_new_thread = False
+
+    upsert_import_state(
+        session,
+        source=source,
+        source_id=source_id,
+        thread_id=thread_id or None,
+        last_line_count=total_lines,
+        last_file_size=current_file_size,
+        last_content_hash=cursor.content_hash,
+        last_message_uuid=last_uuid,
+    )
+
+    return IncrementalImportResult(
+        lines_processed=len(new_lines),
+        events_created=events_created,
+        thread_id=thread_id or 0,
+        is_new_thread=is_new_thread,
+        last_message_uuid=last_uuid,
+    )
+
+
+def import_line_stream_session(
+    *,
+    source: str,
+    source_id: str,
+    session_path,
+    session=None,
+    not_found_msg: str,
+    has_importable_content: Callable[[list[dict]], bool],
+    make_title: Callable[[list[dict], object], str],
+    import_lines: Callable[..., tuple[int, Optional[str]]],
+    prepare: Optional[Callable[[list[dict], Path], object]] = None,
+    make_source_metadata: Optional[Callable[[object], Optional[dict]]] = None,
+) -> IncrementalImportResult:
+    """Import a single-JSONL line-stream provider transcript incrementally.
+
+    Callbacks:
+      - ``prepare(all_lines, path) -> ctx`` — read provider metadata once (or None).
+      - ``has_importable_content(new_lines) -> bool``
+      - ``make_title(all_lines, ctx) -> str``
+      - ``make_source_metadata(ctx) -> dict | None`` — optional.
+      - ``import_lines(session, thread_id, all_lines, new_lines, ctx) -> (events, last_uuid)``.
+    """
+    session_path = Path(session_path)
+    if not session_path.exists():
+        raise FileNotFoundError(not_found_msg)
+
+    source_bytes = read_source_bytes(session_path)
+    all_lines = parse_session_lines(source_bytes, session_path.name)
+
+    kwargs = dict(
+        source=source,
+        source_id=source_id,
+        session_path=session_path,
+        all_lines=all_lines,
+        source_bytes=source_bytes,
+        prepare=prepare,
+        has_importable_content=has_importable_content,
+        make_title=make_title,
+        import_lines=import_lines,
+        make_source_metadata=make_source_metadata,
+    )
+
+    if session is not None:
+        return _run(session, **kwargs)
+    with get_session() as s:
+        result = _run(s, **kwargs)
+        s.commit()
+        return result

thread_archive/_importers/_read.py

@@ -0,0 +1,82 @@
+"""Read + sanitize a provider JSONL transcript.
+
+``errors="replace"`` so a half-written multibyte tail (a poll catching a file
+mid-write) decodes to U+FFFD instead of raising; the per-line ``JSONDecodeError``
+guard means one bad line is skipped, never the whole file. Each parsed line is
+scrubbed of null bytes + lone UTF-16 surrogates at the ingest boundary — source
+JSONL is written by external (JS/TS) processes that can slice strings on UTF-16
+boundaries and leave an orphaned surrogate, which then fails ``json.dumps`` on the
+write path.
+
+The importers read the file's **bytes** once (:func:`read_source_bytes`) and parse
+from that buffer (:func:`parse_session_lines`), because the same buffer is what
+:mod:`._cursor` digests to prove the source was appended to rather than rewritten.
+Reading twice would race a live writer: the bytes the cursor verified must be the
+bytes the lines were parsed from.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def _scrub_str(s: str) -> str:
+    if "\x00" in s:
+        s = s.replace("\x00", "")
+    try:
+        s.encode("utf-8")
+    except UnicodeEncodeError:
+        # Lone surrogate(s): replace them rather than fail the whole import.
+        s = s.encode("utf-8", "replace").decode("utf-8")
+    return s
+
+
+def sanitize_payload(obj: Any) -> Any:
+    """Recursively scrub strings in a parsed JSON value (null bytes / lone surrogates)."""
+    if isinstance(obj, str):
+        return _scrub_str(obj)
+    if isinstance(obj, dict):
+        return {k: sanitize_payload(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [sanitize_payload(v) for v in obj]
+    return obj
+
+
+def read_source_bytes(session_path: Path) -> bytes:
+    """The transcript's raw bytes — the one read the importers do per poll."""
+    return Path(session_path).read_bytes()
+
+
+def parse_session_lines(data: bytes, name: str = "<transcript>") -> list[dict]:
+    """Parse every line of a JSONL transcript's bytes; skip (log) bad lines.
+
+    ``io.StringIO(..., newline=None)`` gives the same universal-newline line split
+    as ``open()`` in text mode: only ``\\n`` (post-translation) ends a line, so a
+    raw U+2028 inside a JSON string doesn't tear a valid line in half the way
+    ``str.splitlines()`` would.
+    """
+    lines: list[dict] = []
+    parse_errors = 0
+    text = data.decode("utf-8", errors="replace")
+    for line_num, line in enumerate(io.StringIO(text, newline=None), 1):
+        if line.strip():
+            try:
+                lines.append(sanitize_payload(json.loads(line)))
+            except json.JSONDecodeError as e:
+                parse_errors += 1
+                logger.warning("%s:%d — JSON parse error: %s", name, line_num, e)
+    if parse_errors:
+        logger.warning("%s: %d lines skipped due to parse errors", name, parse_errors)
+    return lines
+
+
+def read_session_lines(session_path: Path) -> list[dict]:
+    """Read + parse every line of a JSONL transcript; skip (log) bad lines."""
+    session_path = Path(session_path)
+    return parse_session_lines(read_source_bytes(session_path), session_path.name)

thread_archive/_importers/_result.py

@@ -0,0 +1,17 @@
+"""Result contract for an incremental import."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class IncrementalImportResult:
+    """Outcome of one incremental-import call."""
+
+    lines_processed: int
+    events_created: int
+    thread_id: int
+    is_new_thread: bool
+    last_message_uuid: Optional[str] = None

thread_archive/_importers/_sidecar.py

@@ -0,0 +1,113 @@
+"""Claude Code hook-context sidecar import.
+
+Claude Code writes a sibling ``<session>.context.jsonl`` of hook-context lines
+(``{"hook", "context", "ts", "prompt", "metadata"}``) — content that never reaches
+the session JSONL. Each non-empty ``context`` line becomes a ``hook_context`` event
+on the same thread, cursored independently via a derived ``{source_id}:context``
+import-state row so re-imports only carry the grown tail. Ported from canonical
+``streaming/incremental_import/_events.py:_import_sidecar_lines``.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from thread_archive._thread_import.event_builder import compute_dedup_key
+
+from .._store import Event
+from .._truth import write_events
+from ._events import _existing_dedup_keys
+from ._read import read_session_lines
+from ._state import get_import_state, upsert_import_state
+
+logger = logging.getLogger(__name__)
+
+
+def read_sidecar_lines(session_path) -> Optional[list[dict]]:
+    """The sibling ``<session>.context.jsonl`` lines, or None when absent."""
+    sidecar = Path(session_path).with_suffix(".context.jsonl")
+    if not sidecar.exists():
+        return None
+    return read_session_lines(sidecar)
+
+
+def import_sidecar_lines(
+    session: Session, thread_id: int, source_id: str, sidecar_lines: list[dict]
+) -> int:
+    """Import grown hook-context sidecar lines as ``hook_context`` events. Returns
+    the number written (idempotent: dedup_key membership + a line-count cursor)."""
+    sidecar_source_id = f"{source_id}:context"
+    state = get_import_state(session, "claude-code", sidecar_source_id)
+    start_line = state.last_line_count if state else 0
+    total = len(sidecar_lines)
+    new_lines = sidecar_lines[start_line:]
+    if not new_lines:
+        return 0
+
+    seen = _existing_dedup_keys(session, thread_id)
+    batch: list[Event] = []
+    for entry in new_lines:
+        if not isinstance(entry, dict):
+            continue
+        context = entry.get("context", "")
+        if not context:
+            continue
+        hook_name = entry.get("hook", "unknown")
+        ts_str = entry.get("ts")
+        prompt = entry.get("prompt")
+
+        occurred_at: Optional[datetime] = None
+        if ts_str:
+            try:
+                occurred_at = datetime.fromisoformat(ts_str)
+            except (ValueError, TypeError):
+                pass
+
+        payload: dict = {"hook_name": hook_name, "context": context}
+        if prompt:
+            payload["prompt"] = prompt
+        if entry.get("metadata"):
+            payload["metadata"] = entry["metadata"]
+        if occurred_at is None:
+            # Same convention as the event builder: a fabricated timestamp is
+            # flagged, never silent — and timezone-aware like every other write.
+            payload["timestamp_inferred"] = True
+            payload["timestamp_source"] = "fabricated_no_source"
+
+        # Deterministic identity over the line's stable parts so a re-import collapses.
+        content_anchor = f"{ts_str or ''}:{hook_name}:{context}"
+        dedup_key = compute_dedup_key(content_anchor, "hook_context", payload)
+        if dedup_key in seen:
+            continue
+        seen.add(dedup_key)
+        batch.append(Event(
+            thread_id=thread_id,
+            stream_id=str(uuid.uuid4()),
+            event_type="hook_context",
+            payload=payload,
+            occurred_at=occurred_at or datetime.now(timezone.utc),
+            dedup_key=dedup_key,
+        ))
+
+    if batch:
+        write_events(session, batch)
+        from .._retrieval.fts import index_events
+
+        index_events(session, batch)
+
+    upsert_import_state(
+        session,
+        source="claude-code",
+        source_id=sidecar_source_id,
+        thread_id=thread_id,
+        last_line_count=total,
+        last_file_size=0,
+        last_message_uuid=None,
+    )
+    return len(batch)

thread_archive/_importers/_state.py

@@ -0,0 +1,240 @@
+"""Thread resolution + import-state watermarks — the SQLite slice that incremental
+import needs.
+
+A thread's identity is its ``(source, source_id)`` and its ``name`` is the
+deterministic ``"{source}:{source_id}"`` (also the unique key), since we always
+resolve by source before creating.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy import delete, func, select
+from sqlalchemy.orm import Session
+
+from .._store import Event, ImportState, Thread
+
+# Placeholder model values that aren't a real model — skipped when denormalizing a
+# thread's models list (mirrors canonical NON_MODEL_VALUES).
+NON_MODEL_VALUES = ("", "unknown", "<synthetic>")
+
+
+def get_thread_by_source(session: Session, source: str, source_id: str) -> Optional[Thread]:
+    return session.execute(
+        select(Thread).where(Thread.source == source, Thread.source_id == source_id)
+    ).scalars().first()
+
+
+def lookup_parent_thread(
+    session: Session, parent_source_id: str, *, source: str = "claude-code"
+) -> Optional[int]:
+    """Resolve a parent session's thread_id by ``(source, parent_source_id)`` —
+    the import-state watermark first (authoritative even before the thread row is
+    visible), then the threads table. None when unknown."""
+    state = get_import_state(session, source, parent_source_id)
+    if state and state.thread_id:
+        return state.thread_id
+    thread = get_thread_by_source(session, source, parent_source_id)
+    return thread.id if thread else None
+
+
+def adopt_if_unwatermarked(
+    session: Session,
+    *,
+    source: str,
+    source_id: str,
+    thread_id: Optional[int],
+    total_lines: int,
+    file_size: int,
+    content_hash: Optional[str] = None,
+) -> bool:
+    """Guard against re-importing a thread whose ingest cursor was lost.
+
+    The last-resort path for a thread that exists with no ``import_state`` at all —
+    e.g. a bulk-seeded archive whose events were loaded outside the incremental
+    importers. (Reindex carries the watermarks over — from the previous index and the
+    ``import_state.jsonl`` checkpoint snapshot — so a rebuild alone no longer lands
+    here.) Re-importing such a file from line 0 would re-insert events the truth
+    already holds: their stored ``dedup_key`` need not match a fresh import's, so the
+    dedup check wouldn't catch them and every event would double. When the thread
+    already has events, we instead stamp the watermark at the file's current EOF and
+    skip — the existing events stand, and only genuinely-new appended lines import on
+    later polls. The cost is that source lines never imported before adoption are
+    skipped for good, which is why the watermarks are kept durable and this path is
+    kept rare. Returns True when it adopted (caller must not import)."""
+    if thread_id is None:
+        return False
+    has_events = session.execute(
+        select(Event.id).where(Event.thread_id == thread_id).limit(1)
+    ).first() is not None
+    if not has_events:
+        return False
+    upsert_import_state(
+        session,
+        source=source,
+        source_id=source_id,
+        thread_id=thread_id,
+        last_line_count=total_lines,
+        last_file_size=file_size,
+        last_content_hash=content_hash,
+        last_message_uuid=None,
+    )
+    return True
+
+
+def create_thread(
+    session: Session,
+    *,
+    source: str,
+    source_id: str,
+    title: Optional[str] = None,
+    thread_type: str = "conversation",
+    description: Optional[str] = None,
+    source_metadata: Optional[dict] = None,
+) -> int:
+    """Create a thread for ``(source, source_id)`` and return its id (flushed)."""
+    thread = Thread(
+        name=f"{source}:{source_id}",
+        title=title,
+        thread_type=thread_type,
+        description=description,
+        source=source,
+        source_id=source_id,
+        source_metadata=source_metadata,
+    )
+    session.add(thread)
+    session.flush()
+    # Stage the thread's metadata record so its truth file opens with it (written
+    # to threads/<id>.jsonl on commit, before the events that follow).
+    from .._truth.jsonl_log import record_thread
+
+    record_thread(session, thread)
+    return thread.id
+
+
+def discard_new_thread(session: Session, thread_id: int) -> None:
+    """Remove a just-created thread that imported nothing — row AND staged truth.
+
+    The complement of :func:`create_thread` for the empty-import cleanup path.
+    Deleting only the row is not enough: ``create_thread`` staged the thread's
+    metadata record for its truth file, and a commit would still write that
+    ``threads/<id>.jsonl`` ghost — which ``verify`` counts as drift and the next
+    reindex resurrects as an empty thread. Only for threads created in THIS
+    session's transaction (nothing else may reference them yet)."""
+    session.execute(delete(Thread).where(Thread.id == thread_id))
+    from .._truth.jsonl_log import unstage_thread
+
+    unstage_thread(session, thread_id)
+
+
+def _restage_thread(session: Session, thread: Thread) -> None:
+    """Bump ``updated_at`` and re-stage the thread's metadata record to truth.
+
+    A metadata change after creation (title resync, description/models backfill)
+    must reach the JSONL truth or a ``reindex`` would lose it. ``record_thread``
+    appends a fresh ``{"type": "thread", ...}`` record (latest wins on reindex),
+    and the bumped ``updated_at`` also lets the checkpoint backstop pick it up."""
+    thread.updated_at = datetime.now(timezone.utc)
+    session.flush()
+    from .._truth.jsonl_log import record_thread
+
+    record_thread(session, thread)
+
+
+def update_thread_title(session: Session, thread_id: int, title: str) -> bool:
+    """Set a thread's title and re-stage it to truth. No-op if unchanged/missing."""
+    thread = session.get(Thread, thread_id)
+    if not thread or thread.title == title:
+        return False
+    thread.title = title
+    _restage_thread(session, thread)
+    return True
+
+
+def update_thread_description(session: Session, thread_id: int, description: str) -> bool:
+    """Set a thread's one-line description and re-stage it to truth."""
+    thread = session.get(Thread, thread_id)
+    if not thread:
+        return False
+    thread.description = description
+    _restage_thread(session, thread)
+    return True
+
+
+def set_thread_models_from_events(session: Session, thread_id: int) -> list[str]:
+    """Denormalize a thread's models into ``source_metadata['models']`` from its
+    ``api_request_completed`` events (distinct, first-appearance order), so new
+    imports land in the sidebar's model filter. Falls back to the operative
+    ``source_metadata['model']`` when no model appears in the events. Pure
+    re-derivation (overwrites any prior value); re-stages truth only on change.
+
+    SQLite analogue of canonical ``set_thread_models_from_events`` — ``json_extract``
+    replaces the Postgres ``payload['model'].astext``."""
+    thread = session.get(Thread, thread_id)
+    if not thread:
+        return []
+    model_col = func.json_extract(Event.payload, "$.model")
+    rows = session.execute(
+        select(model_col)
+        .where(Event.thread_id == thread_id)
+        .where(Event.event_type == "api_request_completed")
+        .where(model_col.is_not(None))
+        .where(model_col.notin_(NON_MODEL_VALUES))
+        .order_by(Event.id.asc())
+    ).scalars().all()
+    models: list[str] = []
+    for m in rows:
+        if m and m not in models:
+            models.append(m)
+    if not models:
+        operative = (thread.source_metadata or {}).get("model")
+        if operative:
+            models = [operative]
+    if not models:
+        return []
+    if (thread.source_metadata or {}).get("models") == models:
+        return models
+    # Reassign a fresh dict so SQLAlchemy flags the JSON column dirty.
+    meta = dict(thread.source_metadata or {})
+    meta["models"] = models
+    thread.source_metadata = meta
+    _restage_thread(session, thread)
+    return models
+
+
+def get_import_state(session: Session, source: str, source_id: str) -> Optional[ImportState]:
+    return session.execute(
+        select(ImportState).where(ImportState.source == source, ImportState.source_id == source_id)
+    ).scalars().first()
+
+
+def upsert_import_state(
+    session: Session,
+    *,
+    source: str,
+    source_id: str,
+    thread_id: Optional[int],
+    last_line_count: int,
+    last_file_size: int,
+    last_message_uuid: Optional[str],
+    last_content_hash: Optional[str] = None,
+) -> ImportState:
+    """Insert or update the ``(source, source_id)`` watermark (no commit).
+
+    ``last_content_hash`` is the digest of the source bytes this watermark was
+    computed over — the file sources pass it so the next poll can prove the cursor
+    still points into the same content; the DB-backed sources leave it None.
+    """
+    state = get_import_state(session, source, source_id)
+    if state is None:
+        state = ImportState(source=source, source_id=source_id)
+        session.add(state)
+    state.thread_id = thread_id
+    state.last_line_count = last_line_count
+    state.last_file_size = last_file_size
+    state.last_content_hash = last_content_hash
+    state.last_message_uuid = last_message_uuid
+    state.last_import_at = datetime.now(timezone.utc)
+    return state