claimkeep 0.2.0__py3-none-any.whl

claimkeep/__init__.py

@@ -0,0 +1,18 @@
+"""ClaimKeep public API."""
+
+from .brief import Brief, Claim, Supplement, make_id, normalize
+from .config import default_config
+from . import harvesters
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "__version__",
+    "Brief",
+    "Claim",
+    "Supplement",
+    "normalize",
+    "make_id",
+    "default_config",
+    "harvesters",
+]

claimkeep/__main__.py

@@ -0,0 +1,5 @@
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

claimkeep/brief.py

@@ -0,0 +1,215 @@
+"""Brief schema v1 primitives."""
+
+from __future__ import annotations
+
+import json
+import re
+import hashlib
+import unicodedata
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterable, List, Optional
+
+
+SCHEMA_VERSION = 1
+
+
+def normalize(text: str) -> str:
+    """Normalize text for deterministic id hashing only."""
+    return re.sub(r"\s+", " ", unicodedata.normalize("NFC", text).casefold().strip())
+
+
+def make_id(source_harvester: str, middle: str, text: str) -> str:
+    payload = source_harvester + "|" + middle + "|" + normalize(text)
+    return hashlib.sha1(payload.encode("utf-8")).hexdigest()[:16]
+
+
+@dataclass
+class Claim:
+    text: str
+    confidence: Optional[float]
+    topic: str
+    source_harvester: str
+    ts: Optional[str] = None
+    source_span: Optional[str] = None
+    id: Optional[str] = None
+    # Supersession chain. A later claim on the same topic does NOT delete the
+    # earlier one: the earlier one is marked superseded_by and kept. Dropping it
+    # silently makes a retraction indistinguishable from a fact never stated,
+    # and the reader cannot tell which of two conflicting facts is live.
+    superseded_by: Optional[str] = None
+    supersedes: Optional[str] = None
+
+    def __post_init__(self) -> None:
+        if self.confidence is not None:
+            self.confidence = max(0.0, min(1.0, float(self.confidence)))
+        if self.id is None:
+            self.id = make_id(self.source_harvester, self.topic, self.text)
+
+    @property
+    def is_active(self) -> bool:
+        """True when nothing later has superseded this claim."""
+        return self.superseded_by is None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "text": self.text,
+            "confidence": self.confidence,
+            "topic": self.topic,
+            "source_harvester": self.source_harvester,
+            "ts": self.ts,
+            "source_span": self.source_span,
+            "superseded_by": self.superseded_by,
+            "supersedes": self.supersedes,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Claim":
+        return cls(
+            id=str(data["id"]),
+            text=str(data["text"]),
+            confidence=None if data.get("confidence") is None else float(data["confidence"]),
+            topic=str(data["topic"]),
+            source_harvester=str(data["source_harvester"]),
+            ts=data.get("ts"),
+            source_span=data.get("source_span"),
+            superseded_by=data.get("superseded_by"),
+            supersedes=data.get("supersedes"),
+        )
+
+
+@dataclass
+class Supplement:
+    text: str
+    kind: str
+    source_harvester: str
+    id: Optional[str] = None
+
+    def __post_init__(self) -> None:
+        if self.kind not in {"id", "path", "decision"}:
+            raise ValueError("supplement kind must be one of: id, path, decision")
+        if self.id is None:
+            self.id = make_id(self.source_harvester, self.kind, self.text)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "text": self.text,
+            "kind": self.kind,
+            "source_harvester": self.source_harvester,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Supplement":
+        return cls(
+            id=str(data["id"]),
+            text=str(data["text"]),
+            kind=str(data["kind"]),
+            source_harvester=str(data["source_harvester"]),
+        )
+
+
+@dataclass
+class Brief:
+    claims: List[Claim] = field(default_factory=list)
+    supplement: List[Supplement] = field(default_factory=list)
+    created_utc: Optional[str] = None
+    source: Optional[Dict[str, Any]] = None
+    open_threads: List[str] = field(default_factory=list)
+    last_user_ask: Optional[str] = None
+    narrative: List[str] = field(default_factory=list)
+    schema_version: int = SCHEMA_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != SCHEMA_VERSION:
+            raise ValueError("unsupported schema_version")
+        self.claims = self._dedup_claims(self.claims)
+        self.supplement = self._dedup_supplement(self.supplement)
+
+    def add_claim(self, claim: Claim) -> None:
+        self.claims.append(claim)
+        self.claims = self._dedup_claims(self.claims)
+
+    def add_supplement(self, supplement: Supplement) -> None:
+        self.supplement.append(supplement)
+        self.supplement = self._dedup_supplement(self.supplement)
+
+    @property
+    def active_claims(self) -> List[Claim]:
+        return [claim for claim in self.claims if claim.is_active]
+
+    @staticmethod
+    def _dedup_claims(claims: Iterable[Claim]) -> List[Claim]:
+        """Collapse exact repeats, mark same-topic history as superseded.
+
+        Two passes. Identical ids are the same claim restated, so only the last
+        occurrence survives. Different claims on the same topic are a change of
+        position over time: the newest stays active and every earlier one is
+        kept with superseded_by pointing at it, newest carrying supersedes back.
+        """
+        latest_by_id: Dict[str, tuple[int, Claim]] = {}
+        for index, claim in enumerate(claims):
+            latest_by_id[str(claim.id)] = (index, claim)
+        ordered = [item for item in sorted(latest_by_id.values(), key=lambda item: item[0])]
+
+        by_topic: Dict[str, List[Claim]] = {}
+        for _, claim in ordered:
+            by_topic.setdefault(claim.topic, []).append(claim)
+
+        for topic_claims in by_topic.values():
+            newest = topic_claims[-1]
+            newest.superseded_by = None
+            if len(topic_claims) > 1:
+                newest.supersedes = topic_claims[-2].id
+            for earlier in topic_claims[:-1]:
+                earlier.superseded_by = newest.id
+        return [claim for _, claim in ordered]
+
+    @staticmethod
+    def _dedup_supplement(supplements: Iterable[Supplement]) -> List[Supplement]:
+        by_id: Dict[str, Supplement] = {}
+        for item in supplements:
+            by_id[str(item.id)] = item
+        return list(by_id.values())
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "created_utc": self.created_utc,
+            "source": self.source,
+            "claims": [claim.to_dict() for claim in self.claims],
+            "supplement": [item.to_dict() for item in self.supplement],
+            "open_threads": list(self.open_threads),
+            "last_user_ask": self.last_user_ask,
+            "narrative": list(self.narrative),
+        }
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, indent=2) + "\n"
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Brief":
+        if data.get("schema_version") != SCHEMA_VERSION:
+            raise ValueError("unsupported schema_version")
+        for key in ("claims", "supplement"):
+            if key not in data:
+                raise ValueError("missing required key: " + key)
+        return cls(
+            schema_version=SCHEMA_VERSION,
+            created_utc=data.get("created_utc"),
+            source=data.get("source"),
+            claims=[Claim.from_dict(item) for item in data.get("claims", [])],
+            supplement=[Supplement.from_dict(item) for item in data.get("supplement", [])],
+            open_threads=list(data.get("open_threads", [])),
+            last_user_ask=data.get("last_user_ask"),
+            narrative=list(data.get("narrative", [])),
+        )
+
+    @classmethod
+    def from_json(cls, text: str) -> "Brief":
+        return cls.from_dict(json.loads(text))
+
+    def render(self) -> str:
+        from .rehydrate import render
+
+        return render(self)

claimkeep/cli.py

@@ -0,0 +1,335 @@
+"""Command line interface for ClaimKeep."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as _datetime
+import glob
+import json
+import os
+import sys
+from typing import Any, Dict, List, Optional
+
+from . import __version__
+from .brief import Brief, Claim, Supplement
+from .config import default_config
+from .harvesters import get_harvester
+from .harvesters.lessons import LessonHarvester
+from .lessons import Lesson, LessonStore
+from .prompt import marker_instruction
+from .redact import redact
+from .rehydrate import postcompact_payload
+from .retrieve import recall
+from .select import apply_budget
+from .stats import collect as collect_stats, render as render_stats
+
+
+def _now_iso() -> str:
+    return _datetime.datetime.now(_datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def _extract_text(obj: Dict[str, Any]) -> Optional[str]:
+    for key in ("text", "content"):
+        value = obj.get(key)
+        if isinstance(value, str):
+            return value
+    message = obj.get("message")
+    if isinstance(message, str):
+        return message
+    if isinstance(message, dict):
+        content = message.get("content")
+        if isinstance(content, str):
+            return content
+        if isinstance(content, list):
+            parts: List[str] = []
+            for item in content:
+                if isinstance(item, str):
+                    parts.append(item)
+                elif isinstance(item, dict) and isinstance(item.get("text"), str):
+                    parts.append(item["text"])
+            if parts:
+                return "\n".join(parts)
+    return None
+
+
+def _read_transcript(path: str) -> List[str]:
+    units: List[str] = []
+    with open(path, "r", encoding="utf-8") as handle:
+        for line in handle:
+            try:
+                obj = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(obj, dict):
+                continue
+            text = _extract_text(obj)
+            if text:
+                units.append(text)
+    return units
+
+
+def _read_hook_stdin() -> Dict[str, Any]:
+    try:
+        raw = sys.stdin.read()
+        if not raw.strip():
+            return {}
+        data = json.loads(raw)
+        return data if isinstance(data, dict) else {}
+    except (OSError, json.JSONDecodeError):
+        return {}
+
+
+def _newest_brief(brief_dir: str) -> Optional[str]:
+    paths = glob.glob(os.path.join(brief_dir, "*.json"))
+    if not paths:
+        return None
+    return max(paths, key=os.path.getmtime)
+
+
+def _carry_lessons(
+    claims: List[Claim], config: Any, created_utc: str, source: Dict[str, Any]
+) -> List[Claim]:
+    """Persist lessons found this session, then carry the store's newest forward.
+
+    Lessons are the one thing that must not be scoped to a single brief: a rule
+    learned on Monday is worthless if it is gone by Friday. Everything else in
+    the brief describes this session; lessons describe how to work in the next.
+    Failure here is never fatal — a broken store must not cost the whole brief.
+    """
+    if not getattr(config, "lessons_enabled", True):
+        return claims
+    prefix = LessonHarvester.TOPIC + ":"
+    fresh = [claim for claim in claims if claim.topic.startswith(prefix)]
+    try:
+        store = LessonStore(config.expanded_lessons_path())
+        store.append([Lesson(text=claim.text, ts=created_utc, session=source.get("session"))
+                      for claim in fresh])
+        carried = store.recent(int(getattr(config, "lessons_in_brief", 0) or 0))
+    except OSError:
+        return claims
+
+    known = {claim.text.casefold() for claim in fresh}
+    for lesson in carried:
+        if lesson.text.casefold() in known:
+            continue
+        known.add(lesson.text.casefold())
+        claims.append(
+            Claim(
+                text=lesson.text,
+                confidence=None,
+                topic=prefix + lesson.text.casefold()[:40],
+                source_harvester=LessonHarvester.name,
+                ts=lesson.ts,
+            )
+        )
+    return claims
+
+
+def _build_brief(transcript: List[str], created_utc: str, source: Dict[str, Any]) -> Brief:
+    config = default_config()
+    if getattr(config, "redact", True):
+        transcript = [redact(unit) for unit in transcript]
+    claims: List[Claim] = []
+    supplements: List[Supplement] = []
+    # harvest_enabled=False yields an empty (naive) brief — the control arm for
+    # a control-vs-treatment evaluation.
+    if getattr(config, "harvest_enabled", True):
+        for name in config.harvesters:
+            harvester = get_harvester(name)()
+            for item in harvester.harvest(transcript, config):
+                if isinstance(item, Claim):
+                    claims.append(item)
+                elif isinstance(item, Supplement):
+                    supplements.append(item)
+        claims = _carry_lessons(claims, config, created_utc, source)
+    brief = Brief(created_utc=created_utc, source=source, claims=claims, supplement=supplements)
+    # Bound the brief before it is written. Without this the brief is the whole
+    # harvest and cannot be re-injected into the window it is meant to restore.
+    return apply_budget(brief, int(getattr(config, "budget_chars", 0) or 0))
+
+
+def _probe_log(brief: Brief, source: Dict[str, Any], created_utc: str) -> None:
+    """Append one JSONL record per PreCompact when CLAIMKEEP_PROBE_LOG is set.
+
+    Records the full reinjected brief, the harvest_enabled flag, and a
+    session/corpus/timestamp header so control and treatment runs over the same
+    corpus produce machine-distinguishable artifacts. Best-effort; never raises.
+    """
+    path = os.environ.get("CLAIMKEEP_PROBE_LOG")
+    if not path:
+        return
+    try:
+        record = {
+            "ts": created_utc,
+            "session_id": source.get("session"),
+            "corpus_id": os.environ.get("CLAIMKEEP_CORPUS_ID"),
+            "harvest_enabled": bool(getattr(default_config(), "harvest_enabled", True)),
+            "brief": brief.to_dict(),
+        }
+        parent = os.path.dirname(os.path.abspath(path))
+        if parent:
+            os.makedirs(parent, exist_ok=True)
+        with open(path, "a", encoding="utf-8") as handle:
+            handle.write(json.dumps(record, ensure_ascii=False) + "\n")
+    except Exception:
+        return
+
+
+def _cmd_version(_args: argparse.Namespace) -> int:
+    print(__version__)
+    return 0
+
+
+def _cmd_markers(_args: argparse.Namespace) -> int:
+    """Print the confidence-marker instruction for an agent's system prompt.
+
+    The calibration harvester is only as good as the markers it finds, so the
+    plugin ships the convention instead of assuming it.
+    """
+    print(marker_instruction())
+    return 0
+
+
+def _cmd_stats(args: argparse.Namespace) -> int:
+    """Report what the layer actually did, counted from the stored briefs."""
+    report = collect_stats(default_config())
+    print(json.dumps(report, ensure_ascii=False, indent=2) if args.json else render_stats(report), end="")
+    return 0
+
+
+def _cmd_recall(args: argparse.Namespace) -> int:
+    """Search every stored brief and lesson, not just the most recent one."""
+    config = default_config()
+    rows = recall(args.query, config, limit=args.limit, budget_chars=args.budget)
+    if args.json:
+        print(json.dumps(
+            [{"text": row["doc"].text, "kind": row["doc"].kind, "id": row["doc"].id,
+              "ts": row["doc"].ts, "superseded": row["doc"].superseded,
+              "score": row["score"], "bm25": row["bm25"], "source": row["doc"].source}
+             for row in rows],
+            ensure_ascii=False, indent=2))
+        return 0
+    if not rows:
+        print("no match")
+        return 0
+    for row in rows:
+        doc = row["doc"]
+        flag = " [superseded]" if doc.superseded else ""
+        print(f"- ({doc.kind}, {row['score']}){flag} {doc.text}")
+    return 0
+
+
+def _cmd_lessons(args: argparse.Namespace) -> int:
+    config = default_config()
+    store = LessonStore(config.expanded_lessons_path())
+    if args.add:
+        written = store.append([Lesson(text=args.add, ts=_now_iso())])
+        print("stored" if written else "already stored")
+        return 0
+    lessons = store.recent(args.limit)
+    if args.json:
+        print(json.dumps([lesson.to_dict() for lesson in lessons], ensure_ascii=False, indent=2))
+        return 0
+    if not lessons:
+        print("no lessons stored yet")
+        return 0
+    for lesson in lessons:
+        stamp = lesson.ts or "unknown time"
+        print(f"- [{stamp}] {lesson.text}")
+    return 0
+
+
+def _cmd_precompact(args: argparse.Namespace) -> int:
+    try:
+        hook = {} if args.transcript else _read_hook_stdin()
+        transcript_path = args.transcript or hook.get("transcript_path")
+        if not transcript_path:
+            return 0
+        transcript = _read_transcript(str(transcript_path))
+        created_utc = args.now or _now_iso()
+        source = {
+            "agent": str(hook.get("agent", "claude-code")),
+            "session": hook.get("session_id") or hook.get("sessionId") or hook.get("session"),
+        }
+        brief = _build_brief(transcript, created_utc, source)
+        out = args.out
+        if not out:
+            config = default_config()
+            brief_dir = config.expanded_brief_dir()
+            os.makedirs(brief_dir, exist_ok=True)
+            stamp = created_utc.replace(":", "").replace("-", "")
+            session = source.get("session") or "session"
+            out = os.path.join(brief_dir, f"{stamp}-{session}.json")
+        else:
+            parent = os.path.dirname(os.path.abspath(out))
+            if parent:
+                os.makedirs(parent, exist_ok=True)
+        with open(out, "w", encoding="utf-8") as handle:
+            handle.write(brief.to_json())
+        _probe_log(brief, source, created_utc)
+        print(out)
+        return 0
+    except Exception:
+        return 0
+
+
+def _cmd_postcompact(args: argparse.Namespace) -> int:
+    try:
+        brief_path = args.brief
+        if not brief_path:
+            brief_path = _newest_brief(default_config().expanded_brief_dir())
+        if not brief_path:
+            return 0
+        with open(brief_path, "r", encoding="utf-8") as handle:
+            brief = Brief.from_json(handle.read())
+        print(json.dumps(postcompact_payload(brief, args.event), ensure_ascii=False))
+        return 0
+    except Exception:
+        return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="claimkeep")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    version = sub.add_parser("version")
+    version.set_defaults(func=_cmd_version)
+
+    markers = sub.add_parser("markers", help="print the confidence-marker instruction")
+    markers.set_defaults(func=_cmd_markers)
+
+    stats_cmd = sub.add_parser("stats", help="report what the layer did across every stored brief")
+    stats_cmd.add_argument("--json", action="store_true")
+    stats_cmd.set_defaults(func=_cmd_stats)
+
+    recall_cmd = sub.add_parser("recall", help="search every stored brief and lesson")
+    recall_cmd.add_argument("query")
+    recall_cmd.add_argument("--limit", type=int, default=10)
+    recall_cmd.add_argument("--budget", type=int, default=0, help="cap the result set in characters")
+    recall_cmd.add_argument("--json", action="store_true")
+    recall_cmd.set_defaults(func=_cmd_recall)
+
+    lessons = sub.add_parser("lessons", help="list or add durable lessons")
+    lessons.add_argument("--limit", type=int, default=20)
+    lessons.add_argument("--add", help="store a lesson verbatim")
+    lessons.add_argument("--json", action="store_true")
+    lessons.set_defaults(func=_cmd_lessons)
+
+    precompact = sub.add_parser("precompact")
+    precompact.add_argument("--transcript")
+    precompact.add_argument("--out")
+    precompact.add_argument("--now")
+    precompact.set_defaults(func=_cmd_precompact)
+
+    postcompact = sub.add_parser("postcompact")
+    postcompact.add_argument("--brief")
+    postcompact.add_argument("--event", choices=("SessionStart", "PostCompact"), default="SessionStart")
+    postcompact.set_defaults(func=_cmd_postcompact)
+
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return int(args.func(args))

claimkeep/config.py

@@ -0,0 +1,86 @@
+"""Configuration for ClaimKeep harvesters and storage."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from typing import List
+
+
+DEFAULT_BRIEF_DIR = "~/.claude/plugins/data/claimkeep/briefs"
+DEFAULT_LESSONS_PATH = "~/.claude/plugins/data/claimkeep/lessons.jsonl"
+
+
+@dataclass
+class Config:
+    # `atomic` runs first and carries the load on ordinary prose: measured on
+    # LongMemEval it lifted R@10 from 0.456 to 0.926. The other three stay
+    # because they catch what prose rules cannot — markers, paths, ids, lessons —
+    # and that is most of what an agent transcript is made of.
+    harvesters: List[str] = field(
+        default_factory=lambda: ["retraction", "atomic", "calibration", "regex_floor", "lessons"]
+    )
+    # Accepts both the bare marker `[C:80%]` and the extended form that carries
+    # an evidence pointer, `[C:80%, basis: read the file]`. The extended form is
+    # what long-running agents actually emit; a regex that only matched the bare
+    # form silently harvested zero claims from real transcripts.
+    calibration_marker_regex: str = r"\[C:\s*(\d{1,3})\s*%[^\]]*\]"
+    floor_paths: bool = True
+    floor_ids: bool = True
+    floor_decisions: bool = True
+    redact: bool = True
+    harvest_enabled: bool = True
+    brief_dir: str = DEFAULT_BRIEF_DIR
+    # Character budget for the assembled brief. The brief is re-injected into
+    # the context window the compaction just freed, so it must be bounded or it
+    # defeats its own purpose. 0 disables the cap (raw harvest, evaluation only).
+    # 12000 chars is roughly the size of a native compaction summary.
+    budget_chars: int = 12000
+    # Lessons outlive the session: harvested lessons are appended to an
+    # append-only store and the most recent ones are carried into every later
+    # brief, so a rule learned on Monday is still in front of the agent on Friday.
+    lessons_enabled: bool = True
+    lessons_path: str = DEFAULT_LESSONS_PATH
+    lessons_in_brief: int = 10
+
+    @classmethod
+    def from_file(cls, path: str) -> "Config":
+        with open(path, "r", encoding="utf-8") as handle:
+            data = json.load(handle)
+        cfg = cls()
+        for key, value in data.items():
+            if hasattr(cfg, key):
+                setattr(cfg, key, value)
+        return cfg
+
+    @classmethod
+    def from_env(cls) -> "Config":
+        cfg = cls()
+        if os.environ.get("CLAIMKEEP_BRIEF_DIR"):
+            cfg.brief_dir = os.environ["CLAIMKEEP_BRIEF_DIR"]
+        harvest = os.environ.get("CLAIMKEEP_HARVEST")
+        if harvest is not None and harvest.strip().lower() in ("0", "false", "off", "no"):
+            cfg.harvest_enabled = False
+        budget = os.environ.get("CLAIMKEEP_BUDGET_CHARS")
+        if budget is not None:
+            try:
+                cfg.budget_chars = max(0, int(budget.strip()))
+            except ValueError:
+                pass
+        if os.environ.get("CLAIMKEEP_LESSONS_PATH"):
+            cfg.lessons_path = os.environ["CLAIMKEEP_LESSONS_PATH"]
+        lessons = os.environ.get("CLAIMKEEP_LESSONS")
+        if lessons is not None and lessons.strip().lower() in ("0", "false", "off", "no"):
+            cfg.lessons_enabled = False
+        return cfg
+
+    def expanded_brief_dir(self) -> str:
+        return os.path.abspath(os.path.expanduser(self.brief_dir))
+
+    def expanded_lessons_path(self) -> str:
+        return os.path.abspath(os.path.expanduser(self.lessons_path))
+
+
+def default_config() -> Config:
+    return Config.from_env()