stillvalid 0.1.0__py3-none-any.whl

stillvalid/__init__.py

@@ -0,0 +1,24 @@
+"""stillvalid — is this information still valid?
+
+A validity layer between retrieval and action. Feed it what you know about
+a document; it answers VALID / LIKELY_VALID / VERIFY / STALE / UNKNOWN with
+evidence, via a cascade that prefers cheap deterministic checks and only
+falls back to learned probability where nothing deterministic exists.
+
+    from stillvalid import Checker, Doc
+
+    sv = Checker(history_db="observations.db")
+    v = sv.check(Doc(id="policy/refund-de",
+                     last_verified_at=1790000000,
+                     last_changed_at=1767225600))
+    if not v.usable:
+        ...  # re-fetch the source before acting on it
+
+Standard library only. Experimental (v0.1) — API may change.
+"""
+from .cascade import Checker
+from .doc import Doc
+from .verdict import Policy, State, Verdict
+
+__all__ = ["Checker", "Doc", "Policy", "State", "Verdict"]
+__version__ = "0.1.0"

stillvalid/backfill.py

@@ -0,0 +1,146 @@
+"""Backfill — import a corpus's past, instead of waiting to observe it.
+
+The learned layers need change history. Waiting weeks to accumulate it is
+the single worst thing about installing a freshness layer, and it is also
+unnecessary: most knowledge bases already keep that history. Git has it in
+the commit log; wikis keep page revisions; CMSes keep audit trails; a
+warehouse keeps `updated_at` snapshots. None of it has to be observed
+going forward — it can be read today.
+
+    from stillvalid import Checker
+    from stillvalid.backfill import from_git
+
+    with Checker(history_db="observations.db") as sv:
+        n = from_git(sv, r"C:\\work\\handbook")   # seconds, not weeks
+
+What is imported is a sighting per (document, change time) — the same
+shape `record()` writes — so the change-rate layer lights up immediately
+and the calibrated trainer has something to chew on.
+"""
+from __future__ import annotations
+
+import csv
+import subprocess
+from pathlib import Path
+from typing import Callable, Iterable, Iterator
+
+#: file suffixes worth treating as documents when scanning a repository
+DOC_SUFFIXES = (".md", ".mdx", ".rst", ".txt", ".adoc", ".html", ".htm",
+                ".yaml", ".yml", ".json", ".csv", ".org", ".tex")
+
+_SKIP_PARTS = {".git", "node_modules", "__pycache__", ".venv", "dist",
+               "build", "vendor", ".next", "target"}
+
+
+def _import(checker, rows: Iterable[tuple[str, int, str]],
+            prefix: str = "") -> int:
+    """Write (doc_id, ts, value_hash) sightings. Returns rows written."""
+    if checker.history is None:
+        raise RuntimeError("Checker was created without history_db")
+    n = 0
+    for doc_id, ts, value in rows:
+        if checker.history.record(f"{prefix}{doc_id}", int(ts), value):
+            n += 1
+    return n
+
+
+def iter_git_changes(repo: str | Path, *, suffixes: tuple[str, ...] | None
+                     = DOC_SUFFIXES, max_commits: int = 0,
+                     paths: Iterable[str] | None = None
+                     ) -> Iterator[tuple[str, int, str]]:
+    """Yield (path, commit_time, commit_sha) for every file change.
+
+    `suffixes=None` imports every tracked file. Renames break a file's
+    chain (git reports the new path), which understates history for moved
+    documents — the same limitation the GitHub API has.
+    """
+    repo = str(repo)
+    cmd = ["git", "-C", repo, "log", "--name-only", "--format=%x01%H %ct"]
+    if max_commits:
+        cmd.insert(4, f"-n{max_commits}")
+    if paths:
+        cmd += ["--", *paths]
+    out = subprocess.run(cmd, capture_output=True, timeout=600,
+                         check=True).stdout.decode("utf-8", "replace")
+    sha = ts = None
+    for line in out.splitlines():
+        line = line.rstrip()
+        if line.startswith("\x01"):
+            try:
+                sha, ts = line[1:].split()
+            except ValueError:
+                sha = ts = None
+        elif line and ts:
+            p = Path(line)
+            if _SKIP_PARTS & set(p.parts):
+                continue
+            if suffixes and p.suffix.lower() not in suffixes:
+                continue
+            yield line, int(ts), sha
+
+
+def from_git(checker, repo: str | Path, *, prefix: str = "",
+             suffixes: tuple[str, ...] | None = DOC_SUFFIXES,
+             max_commits: int = 0) -> int:
+    """Import a git repository's change history. Returns sightings written.
+
+    Fast enough to be interactive: thousands of changes in seconds, versus
+    weeks of waiting for the same signal to arrive on its own.
+    """
+    return _import(checker, iter_git_changes(
+        repo, suffixes=suffixes, max_commits=max_commits), prefix)
+
+
+def from_rows(checker, rows: Iterable[tuple[str, float, str]], *,
+              prefix: str = "") -> int:
+    """Import (doc_id, timestamp, revision) triples from anywhere.
+
+    The general entry point: a wiki's revision API, a CMS audit table, a
+    warehouse query. `revision` is any value that changes when the content
+    changes (version number, hash, etag); consecutive equal values read as
+    "still the same", which is exactly the censoring information the
+    learned layers want.
+    """
+    return _import(checker, ((d, int(t), str(r)) for d, t, r in rows), prefix)
+
+
+def from_csv(checker, path: str | Path, *, prefix: str = "") -> int:
+    """Import a CSV with columns doc_id, ts, rev (header required).
+
+    `ts` may be epoch seconds or an ISO-8601 string.
+    """
+    from .integrations.common import to_epoch
+
+    with open(path, encoding="utf-8-sig", newline="") as f:
+        reader = csv.DictReader(f)
+        missing = {"doc_id", "ts"} - set(reader.fieldnames or ())
+        if missing:
+            raise ValueError(f"CSV is missing columns: {sorted(missing)} "
+                             "(expected doc_id, ts[, rev])")
+        rows = []
+        for r in reader:
+            ts = to_epoch(r["ts"])
+            if ts is None:
+                continue
+            rows.append((r["doc_id"], ts, r.get("rev") or str(int(ts))))
+    return from_rows(checker, rows, prefix=prefix)
+
+
+def summarize(checker, doc_ids: Iterable[str] | None = None) -> dict:
+    """What did the import actually buy? Counts per readiness tier."""
+    h = checker.history
+    if h is None:
+        raise RuntimeError("Checker was created without history_db")
+    with h._lock:
+        rows = h.conn.execute(
+            "SELECT doc_id, COUNT(*) FROM observations GROUP BY doc_id"
+        ).fetchall()
+    if doc_ids is not None:
+        want = set(doc_ids)
+        rows = [r for r in rows if r[0] in want]
+    ready = sum(1 for _, c in rows if c >= 2)
+    trainable = sum(1 for _, c in rows if c >= 8)
+    return {"documents": len(rows),
+            "sightings": sum(c for _, c in rows),
+            "change_rate_ready": ready,      # layer 5 can estimate
+            "training_ready": trainable}     # enough events to train on

stillvalid/cascade.py

@@ -0,0 +1,170 @@
+"""The cascade — try cheap, certain judgments first; stop at the first
+layer that can decide; abstain honestly when none can.
+
+Order:
+  1. explicit expiry            (deterministic, zero history)
+  2. live content hash compare  (deterministic, zero history)
+  3. live Last-Modified compare (deterministic, zero history)
+  6. calibrated survival model  (needs gate-passed params covering the doc)
+  5. crude change-rate estimate (needs >= 2 observed changes; uncalibrated)
+  7. UNKNOWN                    (honest abstention)
+
+Layer 6 runs before layer 5 on purpose: when a calibrated model covers the
+document it beats the crude estimate. The numbering follows capability,
+not execution order. Cross-source verification (layer 4 in the design
+memo) is on the roadmap, not in v0.1.
+"""
+from __future__ import annotations
+
+import math
+import time
+from pathlib import Path
+from typing import Iterable
+
+from .doc import Doc
+from .history import History
+from .survival import SurvivalParams
+from .verdict import DEFAULT_ACTIONS, Policy, State, Verdict
+
+
+class Checker:
+    def __init__(self, *, survival_params: str | None = None,
+                 history_db: str | Path | None = None,
+                 policy: Policy | None = None):
+        self.params = SurvivalParams.load(survival_params) \
+            if survival_params else None
+        self.history = History(history_db) if history_db else None
+        self.policy = policy or Policy()
+
+    def close(self) -> None:
+        if self.history is not None:
+            self.history.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()
+
+    # ── public API ────────────────────────────────────────────
+
+    def check(self, doc: Doc, now: float | None = None) -> Verdict:
+        now = time.time() if now is None else now
+        for layer in (self._expiry, self._hash_compare, self._modified_compare,
+                      self._survival, self._change_rate):
+            v = layer(doc, now)
+            if v is not None:
+                return v
+        return self._verdict(doc, State.UNKNOWN, "none",
+                             "no layer could judge this document")
+
+    def check_many(self, docs: Iterable[Doc],
+                   now: float | None = None) -> list[Verdict]:
+        now = time.time() if now is None else now   # one clock per batch
+        return [self.check(d, now=now) for d in docs]
+
+    def record(self, doc_id: str, value_hash: str,
+               ts: float | None = None, dedup_s: float = 0.0) -> bool:
+        """Log a sighting — call whenever you fetch/reindex a document."""
+        if self.history is None:
+            raise RuntimeError("Checker was created without history_db")
+        return self.history.record(
+            doc_id, int(time.time() if ts is None else ts),
+            value_hash, dedup_s=dedup_s)
+
+    # ── layers ────────────────────────────────────────────────
+
+    def _expiry(self, doc: Doc, now: float) -> Verdict | None:
+        if doc.expires_at is None or math.isnan(doc.expires_at):
+            return None   # NaN compares False to everything — that is not
+                          # evidence of expiry, it is a broken input
+        if now < doc.expires_at:
+            return self._verdict(doc, State.VALID, "expiry",
+                                 "explicit expiry is "
+                                 f"{_dur(doc.expires_at - now)} away")
+        return self._verdict(doc, State.STALE, "expiry",
+                             "explicitly expired "
+                             f"{_dur(now - doc.expires_at)} ago")
+
+    def _hash_compare(self, doc: Doc, now: float) -> Verdict | None:
+        if not doc.source_hash or not doc.verified_hash:
+            return None   # empty strings are missing data, not a match
+        if doc.source_hash == doc.verified_hash:
+            return self._verdict(doc, State.VALID, "hash",
+                                 "live content hash equals verified hash")
+        return self._verdict(doc, State.STALE, "hash",
+                             "live content hash differs from verified hash")
+
+    def _modified_compare(self, doc: Doc, now: float) -> Verdict | None:
+        if doc.source_modified_at is None or doc.last_verified_at is None \
+                or math.isnan(doc.source_modified_at) \
+                or math.isnan(doc.last_verified_at):
+            return None
+        if doc.source_modified_at <= doc.last_verified_at:
+            return self._verdict(doc, State.VALID, "modified",
+                                 "source unmodified since your verification "
+                                 f"{_dur(now - doc.last_verified_at)} ago")
+        return self._verdict(doc, State.STALE, "modified",
+                             "source was modified "
+                             f"{_dur(doc.source_modified_at - doc.last_verified_at)}"
+                             " after your last verification")
+
+    def _survival(self, doc: Doc, now: float) -> Verdict | None:
+        if (self.params is None or doc.last_changed_at is None
+                or doc.last_verified_at is None):
+            return None
+        if self.params.status(doc.id) != "ok":
+            return None   # observing/unknown → let later layers try
+        p = self.params.p_valid(doc.id, doc.last_changed_at,
+                                doc.last_verified_at, now)
+        if p is None:
+            return None   # params declined to judge (broken input)
+        state = self.policy.state_for(p)
+        return self._verdict(doc, state, "survival",
+                             f"calibrated {p:.0%} probability it is unchanged "
+                             f"{_dur(now - doc.last_verified_at)} after "
+                             "verification, from its learned update habits",
+                             probability=p, calibrated=True)
+
+    def _change_rate(self, doc: Doc, now: float) -> Verdict | None:
+        if self.history is None or doc.last_verified_at is None:
+            return None
+        n, mean_gap = self.history.change_stats(doc.id)
+        if mean_gap is None or mean_gap <= 0:
+            return None
+        # Exponential (memoryless) survival since last verification —
+        # a crude, UNCALIBRATED estimate. It exists so the cascade is useful
+        # before the survival model's gate opens; treat it as a hint.
+        p = math.exp(-max(now - doc.last_verified_at, 0.0) / mean_gap)
+        state = self.policy.state_for(p)
+        return self._verdict(doc, state, "change_rate",
+                             f"rough {p:.0%} estimate (uncalibrated) - it has "
+                             f"changed {n} times, roughly every "
+                             f"{_dur(mean_gap)}, and is "
+                             f"{_dur(now - doc.last_verified_at)} unverified",
+                             probability=p, calibrated=False)
+
+    # ── helpers ───────────────────────────────────────────────
+
+    def _verdict(self, doc: Doc, state: State, layer: str, reason: str,
+                 probability: float | None = None,
+                 calibrated: bool = False) -> Verdict:
+        return Verdict(doc_id=doc.id, state=state,
+                       action=DEFAULT_ACTIONS[state], layer=layer,
+                       reason=reason, probability=probability,
+                       calibrated=calibrated,
+                       last_verified_at=doc.last_verified_at)
+
+
+def _dur(seconds: float) -> str:
+    """Humanize a duration — receipts read better than epoch math."""
+    s = abs(seconds)
+    if s < 120:
+        return f"{s:.0f}s"
+    if s < 7200:
+        return f"{s / 60:.0f}m"
+    if s < 2 * 86400:
+        return f"{s / 3600:.0f}h"
+    if s < 2 * 365.25 * 86400:
+        return f"{s / 86400:.0f}d"
+    return f"{s / (365.25 * 86400):.1f}y"

stillvalid/doc.py

@@ -0,0 +1,27 @@
+"""Input shape — what the caller knows about a retrieved document."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Doc:
+    """Everything is optional except id; each field unlocks a cascade layer.
+
+    Times are Unix epoch seconds.
+
+    last_verified_at : when YOU last confirmed the content (e.g., index time).
+    last_changed_at  : last modification time known as of that verification.
+    expires_at       : explicit expiry, if the source declares one (layer 1).
+    source_hash      : live content hash fetched just now, if you have it
+                       (layer 2 — compare against verified_hash).
+    verified_hash    : content hash recorded at verification time.
+    source_modified_at: live Last-Modified fetched just now (layer 3).
+    """
+    id: str
+    last_verified_at: float | None = None
+    last_changed_at: float | None = None
+    expires_at: float | None = None
+    source_hash: str | None = None
+    verified_hash: str | None = None
+    source_modified_at: float | None = None

stillvalid/history.py

@@ -0,0 +1,142 @@
+"""Local observation log — fuels layers 5 (change rate) and, with enough
+events, training data for layer 6.
+
+One SQLite file, standard library only. Record what you saw whenever you
+fetch or reindex a document; the cascade gets smarter as this accrues.
+"""
+from __future__ import annotations
+
+import sqlite3
+import threading
+from pathlib import Path
+
+_SCHEMA_BASE = """
+CREATE TABLE IF NOT EXISTS observations (
+    id       INTEGER PRIMARY KEY AUTOINCREMENT,
+    doc_id   TEXT    NOT NULL,
+    ts       INTEGER NOT NULL,
+    value_hash TEXT  NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_obs_doc_ts ON observations (doc_id, ts);
+"""
+
+#: One sighting per (document, instant, value): re-importing the same
+#: history is an ordinary thing to do (a second import_history call, a
+#: retried job) and must not double-count a document's change rate.
+_UNIQUE_INDEX = ("CREATE UNIQUE INDEX IF NOT EXISTS idx_obs_unique "
+                 "ON observations (doc_id, ts, value_hash)")
+
+
+class History:
+    """Thread- and process-safe observation log.
+
+    A single Checker is routinely shared across request threads (a web
+    server with the LangChain wrapper, an MCP server), so the connection
+    is opened with check_same_thread=False and every statement runs under
+    one lock. WAL lets a second process (an indexing job, another server)
+    read and write the same file without blocking.
+    """
+
+    def __init__(self, path: str | Path, timeout: float = 10.0):
+        self.conn = sqlite3.connect(str(path), check_same_thread=False,
+                                    timeout=timeout)
+        self._lock = threading.Lock()
+        with self._lock:
+            try:
+                self.conn.execute("PRAGMA journal_mode=WAL")
+                # Default autocheckpoint let the -wal file reach ~4 MB for a
+                # 0.8 MB database in a write-heavy soak; a long-lived server
+                # never closes, so pin a tighter budget.
+                self.conn.execute("PRAGMA wal_autocheckpoint=256")
+            except sqlite3.DatabaseError:
+                pass   # e.g. a network filesystem that refuses WAL
+            self.conn.executescript(_SCHEMA_BASE)
+            try:
+                self.conn.execute(_UNIQUE_INDEX)
+            except sqlite3.IntegrityError:
+                # a log written before the constraint existed may hold
+                # duplicates; collapse them, then enforce it from now on
+                self.conn.execute(
+                    "DELETE FROM observations WHERE id NOT IN ("
+                    "  SELECT MIN(id) FROM observations"
+                    "  GROUP BY doc_id, ts, value_hash)")
+                self.conn.execute(_UNIQUE_INDEX)
+            self.conn.commit()
+
+    def close(self):
+        with self._lock:
+            self.conn.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()
+
+    def record(self, doc_id: str, ts: int, value_hash: str,
+               dedup_s: float = 0.0) -> bool:
+        """Log one sighting of a document's content hash.
+
+        An identical (doc_id, ts, value_hash) row is never stored twice, so
+        re-importing a history is harmless. With dedup_s > 0, a sighting
+        with the SAME hash as the latest row is additionally skipped unless
+        that row is older than dedup_s — keeps high-traffic auto-recording
+        from flooding the log while still capturing every change
+        immediately. Returns True if a row was written.
+        """
+        with self._lock:
+            if dedup_s > 0:
+                prev = self._last(doc_id)
+                if prev is not None and prev[1] == value_hash \
+                        and ts - prev[0] < dedup_s:
+                    return False
+            cur = self.conn.execute(
+                "INSERT OR IGNORE INTO observations (doc_id, ts, value_hash) "
+                "VALUES (?,?,?)", (doc_id, int(ts), value_hash))
+            self.conn.commit()
+            return cur.rowcount > 0
+
+    def last(self, doc_id: str) -> tuple[int, str] | None:
+        """(ts, value_hash) of the most recent sighting, or None."""
+        with self._lock:
+            return self._last(doc_id)
+
+    def _last(self, doc_id: str) -> tuple[int, str] | None:
+        """Caller must hold the lock."""
+        row = self.conn.execute(
+            "SELECT ts, value_hash FROM observations WHERE doc_id=? "
+            "ORDER BY ts DESC, id DESC LIMIT 1", (doc_id,)).fetchone()
+        return (row[0], row[1]) if row else None
+
+    #: how many recent sightings the change-rate estimate looks at. Reading
+    #: a document's whole history made judging cost grow linearly with it
+    #: (12 ms at 20k sightings) — and recent behavior is the better
+    #: predictor anyway, so the window is both faster and more honest.
+    RECENT_WINDOW = 500
+
+    def change_stats(self, doc_id: str,
+                     window: int | None = None) -> tuple[int, float | None]:
+        """(number of observed changes, mean seconds between changes).
+
+        A change is a transition between consecutive distinct hashes; the
+        interval is measured between the observation times of those
+        transitions. Needs >= 2 changes for a mean. Only the most recent
+        `window` sightings are considered (RECENT_WINDOW by default).
+        """
+        with self._lock:
+            rows = self.conn.execute(
+                "SELECT ts, value_hash FROM ("
+                "  SELECT ts, value_hash, id FROM observations WHERE doc_id=?"
+                "  ORDER BY ts DESC, id DESC LIMIT ?"
+                ") ORDER BY ts ASC, id ASC",
+                (doc_id, window or self.RECENT_WINDOW)).fetchall()
+        change_ts = []
+        prev_hash = None
+        for ts, h in rows:
+            if prev_hash is not None and h != prev_hash:
+                change_ts.append(ts)
+            prev_hash = h
+        if len(change_ts) < 2:
+            return len(change_ts), None
+        gaps = [b - a for a, b in zip(change_ts, change_ts[1:])]
+        return len(change_ts), sum(gaps) / len(gaps)

stillvalid/integrations/__init__.py

@@ -0,0 +1,5 @@
+"""Framework integrations — each module requires its framework installed.
+
+    stillvalid.integrations.langchain   — document compressor (retriever stage)
+    stillvalid.integrations.llamaindex  — node postprocessor
+"""