thread-archive 0.0.2__py3-none-any.whl

thread_archive/_scripts/denamespace_dedup_keys.py

@@ -0,0 +1,120 @@
+"""Strip the legacy ``{thread_id}:`` prefix from dedup_keys.
+
+A retired code path wrote ``dedup_key`` as ``{thread_id}:{compute_dedup_key(...)}``;
+the current importer writes the bare ``compute_dedup_key(...)`` form (dedup is already
+thread-scoped by the ``WHERE thread_id = ...`` clause, so the prefix is redundant, and
+there is no global unique index that would need it). The two formats coexist, so a
+re-import — which computes the bare form — won't recognize a prefixed row and would
+duplicate it. This one-shot pass aligns the prefixed rows to the current bare format.
+
+Safe: it only removes a leading ``{thread_id}:`` that is literally present (a bare
+key's anchor is a uuid / ``c=<hash>`` / ``queue-…`` / ``file-snapshot-N`` / ``tool=…`` —
+never the thread's own integer id, so a bare key can't be mistaken for a prefixed one).
+A stripped key can only equal another row's key when they are the same event identity.
+Dry-run by default; ``--apply`` writes per-thread with a row-level backup (id + original
+key) so it is reversible.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Optional
+
+from sqlalchemy import Text, func, select, update
+
+from .._store import Event, get_session
+
+logger = logging.getLogger(__name__)
+
+_PREFIXED = Event.dedup_key.like(func.cast(Event.thread_id, Text).concat(":%"))
+
+
+def _prefixed_thread_ids(session, limit: Optional[int]) -> list[int]:
+    """Threads holding at least one ``{thread_id}:``-prefixed dedup_key."""
+    q = select(Event.thread_id).where(Event.dedup_key.is_not(None), _PREFIXED).group_by(Event.thread_id)
+    if limit is not None:
+        q = q.limit(limit)
+    return list(session.execute(q).scalars().all())
+
+
+def plan_thread(session, thread_id: int) -> tuple[list[tuple[int, str, str]], dict]:
+    """Return (updates, stats): (event_id, original_key, bare_key) for each prefixed row."""
+    prefix = f"{thread_id}:"
+    rows = list(
+        session.execute(
+            select(Event.id, Event.dedup_key).where(
+                Event.thread_id == thread_id, Event.dedup_key.is_not(None)
+            )
+        ).all()
+    )
+    existing_bare = {k for _i, k in rows if not k.startswith(prefix)}
+    stats: dict[str, int] = defaultdict(int)
+    updates: list[tuple[int, str, str]] = []
+    for eid, key in rows:
+        if not key.startswith(prefix):
+            continue
+        bare = key[len(prefix):]
+        if bare in existing_bare:
+            # same identity already present bare — a genuine duplicate row; de-prefixing
+            # is still correct (both collapse to one identity).
+            stats["collapses_onto_existing"] += 1
+        updates.append((eid, key, bare))
+        stats["stripped"] += 1
+    return updates, stats
+
+
+def run(*, apply: bool = False, limit: Optional[int] = None, backup_path: Optional[Path] = None) -> dict:
+    totals: dict[str, Any] = defaultdict(int)
+    backup = open(backup_path, "a", encoding="utf-8") if (apply and backup_path) else None
+    try:
+        with get_session() as s:
+            thread_ids = _prefixed_thread_ids(s, limit)
+        totals["threads"] = len(thread_ids)
+        for tid in thread_ids:
+            with get_session() as s:
+                updates, stats = plan_thread(s, tid)
+                totals["stripped"] += len(updates)
+                totals["collapses"] += stats.get("collapses_onto_existing", 0)
+                if not updates:
+                    continue
+                totals["threads_changed"] += 1
+                if apply:
+                    if backup:
+                        backup.write(json.dumps({
+                            "thread_id": tid,
+                            "rows": [{"id": eid, "old": old} for eid, old, _bare in updates],
+                        }) + "\n")
+                        backup.flush()
+                    for eid, _old, bare in updates:
+                        s.execute(update(Event).where(Event.id == eid).values(dedup_key=bare))
+                    s.commit()
+    finally:
+        if backup:
+            backup.close()
+    return dict(totals)
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--apply", action="store_true", help="write (default: dry-run)")
+    ap.add_argument("--limit", type=int, default=None, help="cap threads")
+    ap.add_argument("--backup", type=Path, default=None, help="row-level backup JSONL (apply)")
+    ap.add_argument("-v", "--verbose", action="store_true")
+    args = ap.parse_args(argv)
+    logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING)
+
+    totals = run(apply=args.apply, limit=args.limit, backup_path=args.backup)
+    mode = "APPLIED" if args.apply else "DRY-RUN"
+    print(f"[{mode}] denamespace-dedup-keys")
+    print(f"  threads with prefixed keys:  {totals.get('threads', 0):,}")
+    print(f"  threads changed:             {totals.get('threads_changed', 0):,}")
+    print(f"  keys de-prefixed:            {totals.get('stripped', 0):,}")
+    print(f"  collapse-onto-existing-bare: {totals.get('collapses', 0):,}")
+
+
+if __name__ == "__main__":
+    main()

thread_archive/_scripts/recover_dropped_events.py

@@ -0,0 +1,190 @@
+"""Backfill import-dropped content the builder now preserves.
+
+The ``DefaultEventBuilder`` used to silently drop three kinds of content it didn't
+model — IDE context blocks (``<ide_selection>`` / ``<ide_opened_file>``), unmodeled
+assistant block types (``server_tool_use``, ``web_search_tool_result``, images, …),
+and messages whose role isn't user/assistant/system. The builder fix preserves them
+going forward as ``ide_context`` / ``content_block`` / ``message`` events. This
+one-shot backfill recovers the same events for **already-imported** CC-shaped threads
+(``claude-code`` + ``cloth``) by re-parsing their still-on-disk source transcripts.
+
+Why this is safe on the live memory-of-record — even though a full re-import is not:
+it only ever inserts events whose ``event_type`` is in :data:`RECOVERABLE_TYPES`.
+Those types were **never written before the fix**, so a fresh parse's ``dedup_key``
+for them cannot collide with any existing row. Existing events are never rewritten,
+so the bulk-seed dedup-key mismatch that makes a blanket re-import double events (the
+hazard ``adopt_if_unwatermarked`` guards) simply doesn't apply here. Re-running is
+idempotent: an already-recovered event's key is now present, so it's skipped.
+
+Recovered events reuse the **existing** turn's ``stream_id`` / ``api_call_id`` (looked
+up by the source message id recovered from each event's dedup-key anchor) so they land
+in the right turn; a fully-dropped turn (e.g. an ide-only turn, or an unknown-role
+turn) has no existing event to borrow from and gets a fresh stream.
+
+Scope: CC-shaped sources only — that's where ``ide_context`` originates and the bulk of
+the archive lives. Other providers (codex/grok/cursor/opencode/…) build events through
+their own paths; their unmodeled-block/role incidence is low and is a separate follow-up.
+
+Dry-run by default (reports what it *would* recover). Pass ``--apply`` to write.
+Threads are re-parsed and committed one at a time, so the run is resumable and keeps
+its lock windows short against the concurrently-writing watcher — but for a clean apply,
+pausing the archive watcher first is recommended.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import uuid as _uuid
+from pathlib import Path
+from typing import Iterator, Optional
+
+from sqlalchemy import select
+
+from thread_archive._thread_import import DefaultEventBuilder
+from thread_archive._thread_import.parsers.claude_code import ClaudeCodeParser
+
+from .._importers._events import _to_event
+from .._importers._read import read_session_lines
+from .._retrieval.fts import index_events
+from .._store import Event, ImportState, get_session
+from .._truth import write_events
+from .._watcher.sources import ClaudeCodeWatcher, cloth_watcher
+
+logger = logging.getLogger(__name__)
+
+# Event types the fix newly preserves — the only rows this backfill ever inserts.
+RECOVERABLE_TYPES = frozenset({"ide_context", "content_block", "message"})
+
+
+def _existing(session, thread_id: int) -> tuple[set[str], dict[str, tuple[str, Optional[str]]]]:
+    """Return ``(dedup_keys, anchor→(stream_id, api_call_id))`` for a thread's events.
+
+    The anchor is the source message id (dedup-key prefix); it lets a recovered event
+    borrow the ``stream_id`` / ``api_call_id`` of the turn it belongs to."""
+    rows = session.execute(select(Event).where(Event.thread_id == thread_id)).scalars().all()
+    keys = {e.dedup_key for e in rows if e.dedup_key}
+    anchor: dict[str, tuple[str, Optional[str]]] = {}
+    for e in rows:
+        if e.dedup_key:
+            anchor.setdefault(e.dedup_key.split(":", 1)[0], (e.stream_id, e.api_call_id))
+    return keys, anchor
+
+
+def plan_thread(session, thread_id: int, lines: list[dict]) -> list[Event]:
+    """Re-parse a thread's source lines and return the recoverable Event rows not yet
+    present. Pure planning — writes nothing."""
+    parser = ClaudeCodeParser()
+    builder = DefaultEventBuilder()
+    session_data = {
+        "provider": "claude-code",
+        "sessions": [{"session_id": "recover", "project": "recover", "lines": lines}],
+    }
+    messages = parser.parse_export(session_data)
+    existing_keys, anchor = _existing(session, thread_id)
+
+    out: list[Event] = []
+    prev = None
+    for msg in messages:
+        pmid = msg.get("provider_message_id") or msg.get("uuid") or ""
+        stream_id, api_call_id = anchor.get(pmid, (str(_uuid.uuid4()), None))
+        events = builder.build_events(msg, stream_id, api_call_id, prev_occurred_at=prev)
+        if events:
+            prev = events[-1].occurred_at
+        for e in events:
+            if (
+                e.event_type in RECOVERABLE_TYPES
+                and e.dedup_key
+                and e.dedup_key not in existing_keys
+            ):
+                out.append(_to_event(thread_id, e))
+                existing_keys.add(e.dedup_key)  # guard against in-file duplicates
+    return out
+
+
+def _iter_pairs() -> Iterator[tuple[str, Path, str]]:
+    """Yield ``(source, path, source_id)`` for every on-disk CC-shaped transcript."""
+    for watcher, name in ((ClaudeCodeWatcher(), "claude-code"), (cloth_watcher(), "cloth")):
+        if not watcher.is_available():
+            continue
+        for path, source_id in watcher._iter_files():
+            yield name, path, source_id
+
+
+def run(*, apply: bool = False, limit: Optional[int] = None) -> dict:
+    """Walk on-disk CC-shaped transcripts, backfilling recoverable events onto their
+    already-imported threads. Returns a summary dict."""
+    totals = {
+        "files_seen": 0,
+        "threads_mapped": 0,
+        "threads_recovered": 0,
+        "events_recovered": 0,
+        "by_type": {},
+        "read_errors": 0,
+        "write_errors": 0,
+    }
+    examined = 0
+    for name, path, source_id in _iter_pairs():
+        totals["files_seen"] += 1
+        if limit is not None and examined >= limit:
+            break
+        with get_session() as s:
+            state = s.execute(
+                select(ImportState).where(
+                    ImportState.source == name, ImportState.source_id == source_id
+                )
+            ).scalar_one_or_none()
+            if state is None or not state.thread_id:
+                continue  # never imported (or merged elsewhere) — nothing to backfill
+            totals["threads_mapped"] += 1
+            examined += 1
+            try:
+                lines = read_session_lines(path)
+            except Exception as e:  # noqa: BLE001
+                logger.warning("read failed for %s (%s): %s", source_id, path, e)
+                totals["read_errors"] += 1
+                continue
+
+            rows = plan_thread(s, state.thread_id, lines)
+            if not rows:
+                continue
+            totals["threads_recovered"] += 1
+            totals["events_recovered"] += len(rows)
+            for r in rows:
+                totals["by_type"][r.event_type] = totals["by_type"].get(r.event_type, 0) + 1
+
+            if apply:
+                try:
+                    write_events(s, rows)
+                    index_events(s, rows)
+                    s.commit()
+                except Exception as e:  # noqa: BLE001 — one bad thread must not abort the run
+                    logger.warning("write failed for thread %s (%s): %s", state.thread_id, source_id, e)
+                    totals["write_errors"] += 1
+                    s.rollback()
+    return totals
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--apply", action="store_true", help="write recovered events (default: dry-run)")
+    ap.add_argument("--limit", type=int, default=None, help="cap threads examined (sampling)")
+    ap.add_argument("-v", "--verbose", action="store_true")
+    args = ap.parse_args(argv)
+    logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING)
+
+    totals = run(apply=args.apply, limit=args.limit)
+    mode = "APPLIED" if args.apply else "DRY-RUN"
+    print(f"[{mode}] recover-dropped-events")
+    print(f"  files seen:        {totals['files_seen']}")
+    print(f"  threads mapped:    {totals['threads_mapped']}")
+    print(f"  threads w/ recovery: {totals['threads_recovered']}")
+    print(f"  events recovered:  {totals['events_recovered']}")
+    for t, n in sorted(totals["by_type"].items()):
+        print(f"    {t:<14} {n}")
+    if totals["read_errors"] or totals["write_errors"]:
+        print(f"  read errors: {totals['read_errors']}  write errors: {totals['write_errors']}")
+
+
+if __name__ == "__main__":
+    main()

thread_archive/_scripts/repair_grok_tool_names.py

@@ -0,0 +1,118 @@
+"""One-off repair for three grok threads (ids 3716011 / 3716012 / 3716013,
+sessions of 2026-06-29) whose live tail-import baked in chunking artifacts: a
+``tool_result`` processed in a later poll than its ``tool_calls`` line lost the
+id->name map and recorded ``tool_name: "unknown"``, and several thinking/text
+blocks were anchored at a stale previous-turn timestamp. The monorepo pg store
+tailed the same sessions on its own schedule and froze *different* artifacts,
+so the ingest-cutover soak's byte-level content check flags these threads as
+divergent on every rotation pass — blocking the week-of-aligned-runs cut bar.
+
+The fix aligns both stores to a canonical deterministic parse: this repo's own
+importer run over the *complete* ``chat_history.jsonl`` files (no chunk
+boundaries). This script applies the standalone half of the patch plan (21
+events); its sibling in the monorepo
+(``archive/src/archive/scripts/repair_grok_tool_names.py``) applies the pg
+half. The plan (``repair_grok_tool_names_plan_20260704.json`` beside this
+script) carries old + new values; old payloads are asserted before writing and
+the changed rows are dumped to a backup file first. Truth-file history is
+inherent: the corrected event lines append via the normal ``append_event_row``
+seam and reindex is last-wins by id, so the pre-repair lines remain in the
+per-thread JSONL as history. FTS rows for the patched events are re-indexed
+(tool_name is an FTS column); vectors are left alone (embedded text is
+unchanged).
+
+Preview (read-only): .venv/bin/python src/thread_archive/_scripts/repair_grok_tool_names.py
+Apply:               .venv/bin/python src/thread_archive/_scripts/repair_grok_tool_names.py --apply
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from sqlalchemy import delete
+from sqlalchemy import text as sa_text
+
+from thread_archive._api import open_archive
+from thread_archive._retrieval.fts import index_events
+from thread_archive._store import use_session
+from thread_archive._store.models import Event, EventFts
+from thread_archive._truth.jsonl_log import append_event_row
+
+HERE = Path(__file__).parent
+PLAN_PATH = HERE / "repair_grok_tool_names_plan_20260704.json"
+BACKUP_PATH = HERE / f"repair_grok_tool_names_backup_{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}.json"
+
+
+def canonical_json(payload) -> str:
+    obj = json.loads(payload) if isinstance(payload, (str, bytes)) else payload
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"))
+
+
+def main() -> int:
+    apply = "--apply" in sys.argv
+    patches = json.loads(PLAN_PATH.read_text())["sa"]
+    print(f"{len(patches)} standalone patches loaded from {PLAN_PATH.name}")
+
+    open_archive(None)
+    with use_session() as session:
+        events: dict[int, Event] = {}
+        for p in patches:
+            ev = session.get(Event, p["event_id"])
+            if ev is None:
+                print(f"ABORT: event {p['event_id']} not found")
+                return 1
+            if canonical_json(ev.payload) != p["old_payload"]:
+                print(f"ABORT: event {p['event_id']} payload no longer matches plan")
+                return 1
+            if ev.event_type != p["old_event_type"]:
+                print(f"ABORT: event {p['event_id']} type {ev.event_type} != plan")
+                return 1
+            events[p["event_id"]] = ev
+
+        for p in patches:
+            changes = []
+            if p["old_event_type"] != p["new_event_type"]:
+                changes.append(f"type {p['old_event_type']} -> {p['new_event_type']}")
+            if p["old_occurred_at"] != p["new_occurred_at"]:
+                changes.append(f"ts {p['old_occurred_at']} -> {p['new_occurred_at']}")
+            if p["old_payload"] != p["new_payload"]:
+                changes.append("payload")
+            print(f"  {p['event_id']}: {', '.join(changes)}")
+
+        if not apply:
+            print("\npreview only — rerun with --apply to write")
+            return 0
+
+        BACKUP_PATH.write_text(json.dumps(
+            [{"id": ev.id, "thread_id": ev.thread_id, "event_type": ev.event_type,
+              "occurred_at": str(ev.occurred_at), "payload": ev.payload,
+              "dedup_key": ev.dedup_key}
+             for ev in events.values()], indent=1, default=str))
+        print(f"\nbackup written: {BACKUP_PATH}")
+
+        for p in patches:
+            ev = events[p["event_id"]]
+            ev.event_type = p["new_event_type"]
+            ev.occurred_at = datetime.fromisoformat(p["new_occurred_at"])
+            ev.payload = json.loads(p["new_payload"])
+            if p.get("new_dedup_key"):
+                ev.dedup_key = p["new_dedup_key"]
+            append_event_row(session, ev)
+
+        ids = list(events)
+        session.execute(delete(EventFts).where(EventFts.event_id.in_(ids)))
+        for eid in ids:
+            session.execute(
+                sa_text("DELETE FROM event_search WHERE event_id = :id"), {"id": eid})
+        session.flush()
+        indexed = index_events(session, list(events.values()))
+        session.commit()
+        print(f"applied {len(patches)} standalone event repairs; re-indexed {indexed} FTS rows")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())