pcode 0.1.0__py3-none-any.whl

pcode/history_cursor.py

@@ -0,0 +1,147 @@
+"""Process-local continuations and bounded, snapshot journal reads for recall.
+
+Continuations retain parser state, not a reusable search index. They expire and
+never write transcript text to disk. Journal readers reopen at their byte offset
+on each page, so no file descriptor is held while a caller considers a result.
+"""
+
+import json
+import os
+import threading
+import time
+from collections import OrderedDict
+from collections.abc import Callable, Generator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from pcode.sessions import SessionInfo, SessionReadBudget
+
+CURSOR_TTL = 30 * 60
+MAX_CURSORS = 16
+
+
+@dataclass
+class Continuation:
+    binding: tuple
+    pages: Generator[tuple[Any, bool], None, None]
+    expires: float
+
+
+_continuations: OrderedDict[str, Continuation] = OrderedDict()
+_lock = threading.Lock()
+
+
+def advance(
+    binding: tuple,
+    after: str | None,
+    pages: Callable[[], Generator[tuple[Any, bool], None, None]],
+):
+    """Advance a single-use cursor, returning the page and its next opaque token."""
+    now = time.monotonic()
+    with _lock:
+        for token, saved in list(_continuations.items()):
+            if saved.expires <= now:
+                del _continuations[token]
+                saved.pages.close()
+        if after:
+            saved = _continuations.get(after)
+            if saved is None:
+                raise ValueError("Unknown cursor or expired continuation; restart without after.")
+            if saved.binding != binding:
+                raise ValueError("Cursor does not match this scope or read request.")
+            del _continuations[after]
+            iterator = saved.pages
+        else:
+            iterator = pages()
+    try:
+        result, more = next(iterator)
+    except BaseException:
+        iterator.close()
+        raise
+    token = None
+    if more:
+        token = uuid4().hex
+        with _lock:
+            while len(_continuations) >= MAX_CURSORS:
+                _, saved = _continuations.popitem(last=False)
+                saved.pages.close()
+            _continuations[token] = Continuation(binding, iterator, time.monotonic() + CURSOR_TTL)
+    else:
+        iterator.close()
+    return result, token
+
+
+@dataclass
+class JournalReader:
+    """Read a finite journal snapshot, even when a JSON line spans several pages."""
+
+    info: SessionInfo
+    root: Path
+    kinds: tuple[str, ...]
+    offset: int = 0
+    pending: bytearray = field(default_factory=bytearray)
+    incomplete_tail: bool = False
+
+    def __post_init__(self):
+        self.path = self.root / self.info.id / "transcript.jsonl"
+        self._check_path()
+        stat = self.path.stat()
+        self.identity = (stat.st_dev, stat.st_ino)
+        self.size = stat.st_size
+        self.mtime = stat.st_mtime_ns
+        self.markers = tuple(f'"{kind}"'.encode() for kind in self.kinds)
+
+    @property
+    def done(self) -> bool:
+        return self.offset == self.size
+
+    def _check_path(self):
+        if self.path.parent.is_symlink() or self.path.is_symlink():
+            raise OSError("symlinked session")
+        if not (self.path.parent / "session.json").is_file():
+            raise OSError("session removed")
+
+    def validate(self, stat=None):
+        # Catches a replaced, shrunk, or same-size-edited journal. Journals are
+        # assumed append-only otherwise: a rewrite that grows the file looks
+        # exactly like an append and is not detected.
+        self._check_path()
+        stat = stat or self.path.stat()
+        if (
+            (stat.st_dev, stat.st_ino) != self.identity
+            or stat.st_size < self.size
+            or (stat.st_size == self.size and stat.st_mtime_ns != self.mtime)
+        ):
+            raise ValueError("Session journal changed; restart without after.")
+
+    def records(self, budget: SessionReadBudget):
+        self.validate()
+        with self.path.open("rb") as stream:
+            self.validate(os.fstat(stream.fileno()))
+            stream.seek(self.offset)
+            while not self.done and budget.remaining > 0:
+                part = stream.readline(min(budget.remaining, self.size - self.offset))
+                if not part:
+                    raise ValueError("Session journal was truncated; restart without after.")
+                budget.remaining -= len(part)
+                self.offset += len(part)
+                self.pending.extend(part)
+                terminated = part.endswith(b"\n")
+                if not terminated and not self.done:
+                    continue
+                line, self.pending = self.pending, bytearray()
+                if terminated and not any(marker in line for marker in self.markers):
+                    continue
+                try:
+                    record = json.loads(line.decode("utf-8", errors="replace"))
+                except ValueError:
+                    # A complete final JSON object needs no newline, but a torn
+                    # live write must not be mistaken for complete coverage.
+                    if not terminated:
+                        self.incomplete_tail = True
+                    continue
+                if isinstance(record, dict) and record.get("kind") in self.kinds:
+                    yield record
+        budget.exhausted = not self.done

pcode/history_embeddings.py

@@ -0,0 +1,102 @@
+"""Opt-in semantic ranking; cache vectors locally, never transcript text."""
+
+import asyncio
+import hashlib
+import json
+import math
+import sqlite3
+from contextlib import closing
+from pathlib import Path
+
+from pcode.diagnostics import redact
+from pcode.history import Chunk
+from pcode.sessions import private_file
+
+MAX_NEW_CHUNKS = 128
+BATCH_SIZE = 32
+CACHE_NAME = ".history-embeddings.sqlite3"
+
+
+def _cache(path: Path, keys: list[str], updates: dict[str, list[float]] | None = None):
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    private_file(path)
+    with closing(sqlite3.connect(path)) as db, db:
+        db.execute(
+            "CREATE TABLE IF NOT EXISTS vectors (key TEXT PRIMARY KEY, vector TEXT NOT NULL)"
+        )
+        if updates:
+            db.executemany(
+                "INSERT OR REPLACE INTO vectors VALUES (?, ?)",
+                [(key, json.dumps(vector)) for key, vector in updates.items()],
+            )
+        result = {}
+        for key in keys:
+            row = db.execute("SELECT vector FROM vectors WHERE key = ?", (key,)).fetchone()
+            if row:
+                result[key] = json.loads(row[0])
+        return result
+
+
+def _unit(vector) -> list[float]:
+    values = [float(value) for value in vector]
+    if not values or not all(math.isfinite(value) for value in values):
+        raise ValueError("Invalid embedding vector")
+    length = math.hypot(*values)
+    if not length:
+        raise ValueError("Empty embedding vector")
+    return [value / length for value in values]
+
+
+async def semantic_ranking(
+    chunks: list[Chunk], query: str, model: str, root: Path, *, embedder=None
+) -> tuple[list[int], list[str]]:
+    """Incrementally index only this search's scope, bounded to 128 new chunks per call."""
+    if not chunks:
+        return [], []
+    if embedder is None:
+        from pydantic_ai import Embedder
+
+        embedder = Embedder(model)
+    # Include format/model in the key: changing either cannot reuse incompatible vectors.
+    keys = [hashlib.sha256(f"v1\0{model}\0{c.text}".encode()).hexdigest() for c in chunks]
+    path = root / CACHE_NAME
+    cached = await asyncio.to_thread(_cache, path, keys)
+    missing = list(dict.fromkeys(key for key in keys if key not in cached))
+    texts = {key: chunk.text for key, chunk in zip(keys, chunks)}
+    for start in range(0, min(len(missing), MAX_NEW_CHUNKS), BATCH_SIZE):
+        batch = missing[start : min(start + BATCH_SIZE, MAX_NEW_CHUNKS)]
+        result = await embedder.embed_documents([texts[key] for key in batch])
+        if len(result.embeddings) != len(batch):
+            raise ValueError("Embedding count mismatch")
+        updates = {key: _unit(vector) for key, vector in zip(batch, result.embeddings)}
+        await asyncio.to_thread(_cache, path, [], updates)
+        cached.update(updates)
+    result = await embedder.embed_query(redact(query))
+    query_vector = _unit(result.embeddings[0])
+    scored = []
+    for index, key in enumerate(keys):
+        if key not in cached:
+            continue
+        vector = _unit(cached[key])
+        if len(vector) != len(query_vector):
+            raise ValueError("Embedding dimensions changed; clear the history embedding cache")
+        similarity = sum(a * b for a, b in zip(vector, query_vector))
+        if similarity > 0:
+            scored.append((similarity, index))
+    warnings = []
+    if len(missing) > MAX_NEW_CHUNKS:
+        warnings.append(
+            f"Semantic index is partial: {len(missing) - MAX_NEW_CHUNKS} chunks remain. "
+            "Later semantic searches index more; keyword search covers the scanned corpus."
+        )
+    ranking, seen = [], set()
+    for _, index in sorted(scored, key=lambda item: (-item[0], item[1])):
+        chunk = chunks[index]
+        identity = (chunk.session.id, chunk.turn.id)
+        if identity in seen:
+            continue
+        seen.add(identity)
+        ranking.append(index)
+        if len(ranking) == 50:
+            break
+    return ranking, warnings