pbuffer 0.1.0__py3-none-any.whl

patternbuffer/classify.py

@@ -0,0 +1,359 @@
+"""The durability classifier and its rebuildable sidecar (whitepaper §5).
+
+Durability is an index, not truth (P1): the class is a judgment about a
+fact, keyed by assertion id, living in a sidecar table that can be
+dropped and re-derived from the untouched log at any time. Deterministic
+guardrails run first and short-circuit; the injected model judges only
+what they leave ambiguous, under asymmetric-cost defaults.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from patternbuffer.buffer import PatternBuffer
+from patternbuffer.codec import json_default
+from patternbuffer.model import ATTR_PREFIX, META_ATTRIBUTES, Assertion
+from patternbuffer.semantics import AttributeSemantics
+
+logger = logging.getLogger(__name__)
+
+CONSTITUTIVE = "CONSTITUTIVE"
+DISPOSITIONAL = "DISPOSITIONAL"
+STATE = "STATE"
+EVENT = "EVENT"
+DURABILITIES = frozenset({CONSTITUTIVE, DISPOSITIONAL, STATE, EVENT})
+# The classes the MODEL may assign (CLASSIFIER-EVENT-SAFETY-V1). EVENT is
+# excluded: it is the one verdict that ERASES a row from every fold
+# (indexes drops EVENT before folding), so a model flip STATE->EVENT is a
+# silent read-completeness failure. Occurrence-ness is structural — assigned
+# only by the deterministic guardrails (event:/caused_by/META) or host
+# declaration — never sampled per-row by the model. The model judges only the
+# standing-durability spectrum; ambiguity defaults to STATE (visible,
+# correctable), never erased.
+_STANDING_DURABILITIES = frozenset({CONSTITUTIVE, DISPOSITIONAL, STATE})
+
+# Review threshold: low-confidence CONSTITUTIVE verdicts silently corrupt
+# every future materialization (whitepaper §5.1), so they are flagged.
+_REVIEW_FLOOR = 0.6
+
+_SIDECAR_SCHEMA = """
+CREATE TABLE IF NOT EXISTS sidecar_classification (
+  assertion_id     TEXT PRIMARY KEY,
+  durability       TEXT NOT NULL,
+  class_confidence REAL NOT NULL,
+  needs_review     INTEGER NOT NULL DEFAULT 0
+);
+CREATE TABLE IF NOT EXISTS sidecar_classifier_meta (
+  key   TEXT PRIMARY KEY,
+  value INTEGER NOT NULL
+);
+"""
+
+_MODEL_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "durability": {"enum": sorted(_STANDING_DURABILITIES)},
+        "class_confidence": {"type": "number"},
+    },
+    "required": ["durability", "class_confidence"],
+}
+
+
+@dataclass(frozen=True, slots=True)
+class Classification:
+    assertion_id: str
+    durability: str
+    class_confidence: float
+    needs_review: bool
+
+
+class Classifier:
+    """classify(assertion, world_context) -> {durability, class_confidence}.
+
+    Writes the sidecar only — never the log (role matrix §12).
+    """
+
+    def __init__(
+        self,
+        buffer: PatternBuffer,
+        model: Callable[[str, dict], Any],
+        semantics: AttributeSemantics | None = None,
+    ) -> None:
+        self._buffer = buffer
+        self._model = model
+        self._semantics = semantics or AttributeSemantics(buffer)
+        conn = buffer.raw_connection()
+        conn.executescript(_SIDECAR_SCHEMA)
+        conn.execute(
+            "INSERT OR IGNORE INTO sidecar_classifier_meta (key, value)"
+            " VALUES ('version', 0)"
+        )
+        conn.commit()
+
+    # ------------------------------------------------------------ judgment
+
+    def _guardrails(self, row: Assertion) -> tuple[str, float] | None:
+        """Deterministic short-circuits. Return None to defer to the model."""
+        if row.value_type == "delta":
+            return STATE, 1.0
+        if row.entity.startswith("event:") or row.attribute == "caused_by":
+            return EVENT, 1.0
+        if row.attribute in {"name", "alias"}:
+            return CONSTITUTIVE, 0.95  # identity anchors
+        if row.attribute in META_ATTRIBUTES:
+            # Meta-assertions ride with their subject; they never fold as
+            # world facts. Classified EVENT-like for lens purposes: immutable.
+            return EVENT, 1.0
+        if row.value_type == "unresolved":
+            return STATE, 1.0  # a thunk occupies its key like mutable state
+        if self._semantics.is_containment(row.attribute):
+            if row.attribute in {"held_by", "worn_by", "carried_by"}:
+                return STATE, 0.95  # things held by agents are movable
+            if row.entity.startswith("place:"):
+                # A place's containment is part of what the place IS — the
+                # study does not move out of the home (fixture sub-rule).
+                return CONSTITUTIVE, 0.9
+            return None  # objects: fixture vs movable is a judgment
+        if self._semantics.is_structural(row.attribute):
+            return CONSTITUTIVE, 1.0
+        return None
+
+    def classify(self, row: Assertion) -> Classification:
+        verdict = self._guardrails(row)
+        if verdict is not None:
+            durability, confidence = verdict
+        else:
+            durability, confidence = self._ask_model(row)
+        needs_review = durability == CONSTITUTIVE and confidence < _REVIEW_FLOOR
+        c = Classification(row.id, durability, confidence, needs_review)
+        self._store(c)
+        return c
+
+    def _ask_model(self, row: Assertion) -> tuple[str, float]:
+        prompt = (
+            "Classify the lifetime of this fact about a world.\n"
+            f"Subject: {row.entity}\nAttribute: {row.attribute}\n"
+            f"Value: {json.dumps(row.value, default=json_default)}\n\n"
+            "CONSTITUTIVE: what the thing IS (identity, structure, fixtures, era). "
+            "True at every moment unless the world is re-authored.\n"
+            "DISPOSITIONAL: what it TENDS to be (habits, roles, recurring behavior). "
+            "Generally true but defeasible.\n"
+            "STATE: what it is RIGHT NOW (positions of movables, moods, conditions). "
+            "One event could flip it.\n\n"
+            "The mutability test resolves most ambiguity: could one event flip this "
+            "without re-authoring the world? -> STATE. Asymmetric defaults: an "
+            "ambiguous property is STATE; ambiguous fixture containment is "
+            "CONSTITUTIVE."
+        )
+        try:
+            out = self._model(prompt, _MODEL_SCHEMA)
+            durability = out["durability"]
+            confidence = float(out["class_confidence"])
+            if durability not in _STANDING_DURABILITIES:
+                raise ValueError(durability)
+        except Exception:
+            logger.exception("classifier model call failed for %s; defaulting", row.id)
+            # Asymmetric default: ambiguous property -> STATE, except
+            # in/within containment, which defaults CONSTITUTIVE.
+            if self._semantics.is_containment(row.attribute):
+                return CONSTITUTIVE, 0.5
+            return STATE, 0.5
+        return durability, confidence
+
+    # ------------------------------------------------------------- sidecar
+
+    def _store(self, c: Classification) -> None:
+        conn = self._buffer.raw_connection()
+        conn.execute(
+            "INSERT OR REPLACE INTO sidecar_classification"
+            " (assertion_id, durability, class_confidence, needs_review)"
+            " VALUES (?, ?, ?, ?)",
+            (c.assertion_id, c.durability, c.class_confidence, int(c.needs_review)),
+        )
+        conn.execute(
+            "UPDATE sidecar_classifier_meta SET value = value + 1"
+            " WHERE key = 'version'"
+        )
+        conn.commit()
+
+    @property
+    def version(self) -> int:
+        row = self._buffer.raw_connection().execute(
+            "SELECT value FROM sidecar_classifier_meta WHERE key = 'version'"
+        ).fetchone()
+        return int(row[0]) if row else 0
+
+    def set(self, assertion_id: str, durability: str, confidence: float = 1.0) -> None:
+        """Judgment injection for components that hold the world context
+        (e.g. the resolver knows invented contents are movables). Sidecar
+        only — the log is untouched, and rebuild() re-derives."""
+        if durability not in DURABILITIES:
+            raise ValueError(durability)
+        self._store(Classification(assertion_id, durability, confidence, False))
+
+    def get(self, assertion_id: str) -> Classification | None:
+        r = self._buffer.raw_connection().execute(
+            "SELECT assertion_id, durability, class_confidence, needs_review"
+            " FROM sidecar_classification WHERE assertion_id = ?",
+            (assertion_id,),
+        ).fetchone()
+        return Classification(r[0], r[1], r[2], bool(r[3])) if r else None
+
+    def durability(self, assertion_id: str) -> str:
+        """The fold's view. Unclassified rows read as STATE (conservative:
+        a transient mistaken for structure decays; structure mistaken for
+        transient gets flagged on contradiction) — classify_all() should
+        leave none."""
+        c = self.get(assertion_id)
+        return c.durability if c else STATE
+
+    def classify_all(self, batch_size: int | None = None) -> int:
+        """Classify every row not yet in the sidecar. Returns count.
+
+        With ``batch_size``, model-judged rows go to the model in batches
+        (one call per N rows) — same judgments, fewer round trips."""
+        pending = [r for r in self._buffer.all_rows() if self.get(r.id) is None]
+        if not batch_size:
+            for row in pending:
+                self.classify(row)
+            return len(pending)
+        return self.classify_rows(pending, batch_size)
+
+    def classify_rows(self, rows: list[Assertion], batch_size: int | None = None,
+                      model: bool = True) -> int:
+        """Classify a SPECIFIC set of rows (INGEST-HARDENING-V1): guardrails
+        resolve immediately with zero model calls; model-needing rows go to one
+        batch call (or `batch_size` chunks). The shared core of `classify_all`'s
+        batch path and the opt-in batched ingest. Already-classified rows skip.
+
+        ``model=False`` (INGEST-LATENCY-V2 `classify="rules"`): the ambiguous
+        remainder gets the doctrine's primary asymmetric default STATE with NO
+        LM call — fast and deterministic. Deliberately STATE (not the
+        containment→CONSTITUTIVE except sub-rule) so a movable's containment
+        recency-supersedes; `needs_review` is never raised (STATE isn't flagged).
+        The standing-spectrum default is safe: durability is rebuildable and the
+        erasing EVENT is barred (CLASSIFIER-EVENT-SAFETY)."""
+        rows = [r for r in rows if self.get(r.id) is None]
+        deferred: list[Assertion] = []
+        for row in rows:
+            verdict = self._guardrails(row)
+            if verdict is not None:
+                durability, confidence = verdict
+                self._store(
+                    Classification(
+                        row.id, durability, confidence,
+                        durability == CONSTITUTIVE and confidence < _REVIEW_FLOOR,
+                    )
+                )
+            else:
+                deferred.append(row)
+        if deferred:
+            if not model:
+                for row in deferred:
+                    self._store(Classification(row.id, STATE, 0.5, False))
+            else:
+                step = batch_size or len(deferred)
+                for start in range(0, len(deferred), step):
+                    self._classify_batch(deferred[start : start + step])
+        return len(rows)
+
+    def _classify_batch(self, rows: list[Assertion]) -> None:
+        listing = "\n".join(
+            f"{i}. {r.entity} · {r.attribute} · {json.dumps(r.value, default=json_default)}"
+            for i, r in enumerate(rows)
+        )
+        schema = {
+            "type": "object",
+            "properties": {
+                "verdicts": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "index": {"type": "integer"},
+                            "durability": {"enum": sorted(_STANDING_DURABILITIES)},
+                            "class_confidence": {"type": "number"},
+                        },
+                        "required": ["index", "durability", "class_confidence"],
+                    },
+                }
+            },
+            "required": ["verdicts"],
+        }
+        prompt = (
+            "Classify the lifetime of each fact about a world. Classes:\n"
+            "CONSTITUTIVE: what the thing IS (identity, structure, fixtures, era); "
+            "true at every moment unless the world is re-authored.\n"
+            "DISPOSITIONAL: what it TENDS to be (habits, roles); defeasible.\n"
+            "STATE: what it is RIGHT NOW (positions of movables, moods); one event "
+            "could flip it.\n"
+            "Mutability test: could one event flip it without re-authoring the "
+            "world? -> STATE. Ambiguous property -> STATE; ambiguous fixture "
+            "containment -> CONSTITUTIVE.\n\nFacts:\n" + listing
+        )
+        verdicts: dict[int, tuple[str, float]] = {}
+        try:
+            out = self._model(prompt, schema)
+            for v in out["verdicts"]:
+                # EVENT (or any non-standing verdict) is rejected -> the row
+                # falls to the asymmetric default below, never erased.
+                if v["durability"] in _STANDING_DURABILITIES:
+                    verdicts[int(v["index"])] = (v["durability"], float(v["class_confidence"]))
+        except Exception:
+            logger.exception("batch classification failed; defaulting batch")
+        for i, row in enumerate(rows):
+            durability, confidence = verdicts.get(
+                i,
+                (CONSTITUTIVE, 0.5)
+                if self._semantics.is_containment(row.attribute)
+                else (STATE, 0.5),
+            )
+            self._store(
+                Classification(
+                    row.id, durability, confidence,
+                    durability == CONSTITUTIVE and confidence < _REVIEW_FLOOR,
+                )
+            )
+
+    def promote_accruals(self, threshold: int = 3) -> int:
+        """Accrual promotion (whitepaper §5.1): repeated STATE observations
+        of the same (entity, attribute, value) — at distinct valid times —
+        promote to DISPOSITIONAL in the sidecar. The world learns its
+        habits; the log is untouched; rebuild() re-derives. Returns the
+        number of rows promoted. Promotion frequency doubles as the
+        salience baseline's input."""
+        groups: dict[tuple, set] = {}
+        rows_by_group: dict[tuple, list] = {}
+        for row in self._buffer.visible():
+            if (
+                row.entity.startswith("a:")
+                or row.entity.startswith(ATTR_PREFIX)
+                or row.value_type in ("unresolved", "delta")
+                or self._semantics.is_accrue(row.attribute)
+            ):
+                continue  # deltas / accrue quantities are summed, never a habit
+            if self.durability(row.id) != STATE:
+                continue
+            key = (row.entity, row.attribute, row.frame, repr(row.value))
+            groups.setdefault(key, set()).add(row.valid_from)
+            rows_by_group.setdefault(key, []).append(row)
+        promoted = 0
+        for key, valid_times in groups.items():
+            if len(valid_times) >= threshold:
+                for row in rows_by_group[key]:
+                    self.set(row.id, DISPOSITIONAL, 0.8)
+                    promoted += 1
+        if promoted:
+            logger.info("accrual promotion: %d row(s) -> DISPOSITIONAL", promoted)
+        return promoted
+
+    def rebuild(self) -> int:
+        """Drop the sidecar and re-derive it from the untouched log."""
+        conn = self._buffer.raw_connection()
+        conn.execute("DELETE FROM sidecar_classification")
+        conn.commit()
+        return self.classify_all()

patternbuffer/codec.py

@@ -0,0 +1,122 @@
+"""Value codec: exact-decimal quantities as tagged JSON scalars.
+
+EXACT-DECIMAL-QUANTITIES-V1: a quantity that must fold exactly (money, any
+real ledger) is a Python ``Decimal`` in memory and the reserved tag dict
+``{"$decimal": "12.50"}`` on every JSON boundary — storage, dump, prompts,
+host payloads. ``str(Decimal)`` preserves authored scale (``12.50`` stays
+``12.50``); no float ever touches the number. Fiction floats stay floats:
+the Decimal path is entered only by values that are Decimal.
+
+Fold arithmetic (`exact_sum`/`exact_div`) runs under one explicit, fixed
+context (`MONEY_CTX`) so builds are byte-deterministic across platforms
+(P7) regardless of the process's ambient decimal context. Mixing exact
+Decimal with float in one fold is an authoring smell and raises; Decimal
+with int promotes losslessly.
+"""
+
+from __future__ import annotations
+
+import decimal
+from decimal import Decimal, localcontext
+from typing import Any, Iterable
+
+DEC_TAG = "$decimal"
+"""Reserved key in value-space (LEXICON). Only an exact one-key dict whose
+value parses as a finite Decimal is reconstructed; anything else passes
+through untouched (the collision guard)."""
+
+MONEY_CTX = decimal.Context(prec=50, rounding=decimal.ROUND_HALF_EVEN)
+"""The one context all Decimal fold arithmetic runs under. Addition is
+exact for any realistic ledger within prec=50; division (avg) rounds
+HALF_EVEN — named and deterministic."""
+
+
+def _is_finite_dec_str(s: object) -> bool:
+    if not isinstance(s, str):
+        return False
+    try:
+        return Decimal(s).is_finite()
+    except decimal.InvalidOperation:
+        return False
+
+
+def encode_value(v: Any) -> Any:
+    """One in-memory scalar -> its JSON-safe form (Decimal -> tag dict)."""
+    if isinstance(v, Decimal):
+        return {DEC_TAG: str(v)}
+    return v
+
+
+def decode_value(v: Any) -> Any:
+    """One JSON-safe scalar -> in-memory (exact tag dict -> Decimal)."""
+    if isinstance(v, dict) and set(v) == {DEC_TAG} and _is_finite_dec_str(v[DEC_TAG]):
+        return Decimal(v[DEC_TAG])
+    return v
+
+
+def json_default(o: Any) -> Any:
+    """``json.dumps(..., default=json_default)`` for internal serialization."""
+    if isinstance(o, Decimal):
+        return {DEC_TAG: str(o)}
+    raise TypeError(f"Object of type {type(o).__name__} is not JSON serializable")
+
+
+def decode_hook(d: dict) -> Any:
+    """``json.loads(..., object_hook=decode_hook)``: reconstruct tag dicts."""
+    return decode_value(d)
+
+
+def encode_out(obj: Any) -> Any:
+    """Recursively encode Decimal leaves in a host-facing payload.
+
+    Porcelain/neighborhood public returns wrap in this at the return
+    statement (one wrapper per verb) so no field — present or future —
+    can leak a raw Decimal into the plain-JSON contract.
+    """
+    if isinstance(obj, Decimal):
+        return {DEC_TAG: str(obj)}
+    if isinstance(obj, dict):
+        return {k: encode_out(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [encode_out(v) for v in obj]
+    return obj
+
+
+def check_no_mix(values: Iterable[Any], *, ids: Iterable[str] = ()) -> None:
+    """Raise if exact Decimal and float coexist in one fold's value set.
+
+    Runs before ANY rollup op (sum/avg/min/max — count validates with the
+    rest): a mixed representation on one attribute is an authoring smell
+    to surface, never to silently promote."""
+    vals = list(values)
+    if any(isinstance(v, Decimal) for v in vals) and any(
+        isinstance(v, float) for v in vals
+    ):
+        where = ", ".join(ids)
+        raise ValueError(
+            "exact-decimal and float quantities mixed in one fold"
+            + (f" ({where})" if where else "")
+            + "; author one representation per attribute"
+        )
+
+
+def exact_sum(values: Iterable[Any], *, ids: Iterable[str] = ()) -> Any:
+    """Sum a fold's values: any-Decimal => Decimal-exact under MONEY_CTX
+    (ints promote losslessly); all-float/int => the native sum, unchanged."""
+    vals = list(values)
+    check_no_mix(vals, ids=ids)
+    if any(isinstance(v, Decimal) for v in vals):
+        with localcontext(MONEY_CTX):
+            return sum(
+                (v if isinstance(v, Decimal) else Decimal(v) for v in vals),
+                Decimal(0),
+            )
+    return sum(vals)
+
+
+def exact_div(total: Any, count: int) -> Any:
+    """Division for avg: Decimal path under MONEY_CTX (HALF_EVEN), else native."""
+    if isinstance(total, Decimal):
+        with localcontext(MONEY_CTX):
+            return total / Decimal(count)
+    return total / count

patternbuffer/dump.py

@@ -0,0 +1,113 @@
+"""JSONL dump + deterministic builder (spec §10, the letter-005 seam).
+
+The dump is the canonical, diffable artifact; the SQLite file is
+disposable. ``build`` replays a dump through a builder-privileged append
+that preserves ``seq``/``id``/``asserted_at`` byte-for-byte — and it is
+not a second write authority: every constraint below aborts the build.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+from patternbuffer.buffer import PatternBuffer
+from patternbuffer.codec import decode_hook, json_default
+from patternbuffer.model import STATUSES, VALUE_TYPES, Assertion
+from patternbuffer.roles import _make_builder_role
+
+logger = logging.getLogger(__name__)
+
+_FIELDS = (
+    "seq",
+    "id",
+    "world_id",
+    "entity",
+    "attribute",
+    "value_type",
+    "value",
+    "valid_from",
+    "valid_to",
+    "frame",
+    "status",
+    "confidence",
+    "asserted_at",
+)
+
+
+class DumpError(ValueError):
+    """A dump failed the builder's validation; nothing was built."""
+
+
+def dump(buffer: PatternBuffer) -> str:
+    """The full log as JSONL, one assertion per line, in append order."""
+    lines = []
+    for row in buffer.all_rows():
+        lines.append(
+            json.dumps({f: getattr(row, f) for f in _FIELDS}, sort_keys=True,
+                       default=json_default)
+        )
+    return "\n".join(lines) + ("\n" if lines else "")
+
+
+def build(jsonl: str, path: str | Path) -> PatternBuffer:
+    """Materialize a buffer from a dump. Refuses anything but a clean replay.
+
+    Constraints (spec §10): target absent or empty; exactly one
+    ``world_id`` across all rows; ``seq`` contiguous from 1;
+    ``id == "a:<seq>"`` and ``asserted_at == seq``; statuses and value
+    types in vocabulary. No judgment runs — the dump already carries the
+    log as it was; sidecars are rebuilt by their own ``rebuild()``.
+    """
+    path = Path(path)
+    if path.exists() and path.stat().st_size > 0:
+        raise DumpError(f"build target {path} already exists; restore is not a write path")
+
+    rows = []
+    for n, line in enumerate(jsonl.splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            raw = json.loads(line, object_hook=decode_hook)
+        except json.JSONDecodeError as exc:
+            raise DumpError(f"line {n}: not valid JSON: {exc}") from exc
+        if not isinstance(raw, dict):
+            # A bare tag line ({"$decimal": ...}) decodes to a Decimal, and any
+            # other non-object line is equally not a row — abort as validation,
+            # never an AttributeError.
+            raise DumpError(f"line {n}: not an assertion row object")
+        missing = set(_FIELDS) - raw.keys()
+        if missing:
+            raise DumpError(f"line {n}: missing fields {sorted(missing)}")
+        rows.append(Assertion(**{f: raw[f] for f in _FIELDS}))
+
+    world_ids = {r.world_id for r in rows}
+    if len(world_ids) > 1:
+        raise DumpError(f"dump spans worlds {sorted(world_ids)}; a buffer holds exactly one")
+    if not rows:
+        raise DumpError("empty dump; nothing to build")
+    builder = _make_builder_role()
+    for i, row in enumerate(rows, start=1):
+        if row.seq != i:
+            raise DumpError(f"seq not contiguous from 1: expected {i}, got {row.seq}")
+        if row.id != f"a:{i}":
+            raise DumpError(f"row {i}: id {row.id!r} != 'a:{i}'")
+        if row.asserted_at != i:
+            raise DumpError(f"row {i}: asserted_at {row.asserted_at} != seq {i}")
+        if row.status not in STATUSES or row.status == "default":
+            raise DumpError(f"row {i}: status {row.status!r} cannot appear in a log")
+        if row.value_type not in VALUE_TYPES:
+            raise DumpError(f"row {i}: unknown value_type {row.value_type!r}")
+        builder.check(row.status)
+
+    buffer = PatternBuffer(path, world_id=rows[0].world_id)
+    try:
+        for row in rows:
+            buffer._insert(row)
+    except Exception:
+        buffer.close()
+        path.unlink(missing_ok=True)
+        raise
+    logger.info("built %s: %d assertions, world %s", path, len(rows), rows[0].world_id)
+    return buffer