rootmemory 0.1.0__py3-none-any.whl

rootmemory/services/independence_service.py

@@ -0,0 +1,159 @@
+"""IndependenceService: how many *independent* sources really back a set of beliefs.
+
+This is the service that answers the question the whole project exists for:
+
+    agreeing agents      != independent evidence sources
+    supporting beliefs   != independent evidence sources
+
+Both numbers are always computed and always reported (spec section 64).
+"""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+
+from sqlalchemy.orm import Session
+
+from rootmemory.models.belief import Belief
+from rootmemory.models.enums import EpistemicStatus
+from rootmemory.models.node_ref import NodeRef
+from rootmemory.models.observation import Observation
+from rootmemory.repositories.observation_repo import ObservationRepository
+from rootmemory.services.provenance_service import DEFAULT_ROOT_STATUSES, ProvenanceService
+from rootmemory.services.scoring_service import (
+    aggregate_confidence_by_root,
+    naive_average_confidence,
+)
+
+
+@dataclass(slots=True)
+class IndependenceReport:
+    """Independence analysis for one set of beliefs."""
+
+    belief_ids: list[uuid.UUID] = field(default_factory=list)
+    agreeing_agent_ids: set[uuid.UUID] = field(default_factory=set)
+    #: independence key -> the beliefs that rest on that source
+    beliefs_by_source: dict[str, list[uuid.UUID]] = field(default_factory=dict)
+    #: independence key -> the observations grouped under it
+    observations_by_source: dict[str, list[uuid.UUID]] = field(default_factory=dict)
+    #: beliefs whose ancestry contains no currently valid observation
+    unsupported_belief_ids: list[uuid.UUID] = field(default_factory=list)
+    aggregate_confidence: float = 0.0
+    naive_confidence: float = 0.0
+
+    @property
+    def belief_count(self) -> int:
+        """How many beliefs assert this - the number a naive system counts."""
+        return len(self.belief_ids)
+
+    @property
+    def agreeing_agent_count(self) -> int:
+        """How many distinct agents assert this."""
+        return len(self.agreeing_agent_ids)
+
+    @property
+    def independent_source_count(self) -> int:
+        """How many genuinely independent sources back this."""
+        return len(self.beliefs_by_source)
+
+    @property
+    def root_observation_ids(self) -> list[uuid.UUID]:
+        flat: list[uuid.UUID] = []
+        for observation_ids in self.observations_by_source.values():
+            flat.extend(observation_ids)
+        return flat
+
+    @property
+    def shares_single_source(self) -> bool:
+        """True when several beliefs all descend from one source - the failure
+        mode this system exists to catch."""
+        return self.belief_count > 1 and self.independent_source_count == 1
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "supporting_belief_count": self.belief_count,
+            "agreeing_agent_count": self.agreeing_agent_count,
+            "independent_source_count": self.independent_source_count,
+            "independent_sources": sorted(self.beliefs_by_source),
+            "root_observation_ids": [str(o) for o in self.root_observation_ids],
+            "unsupported_belief_ids": [str(b) for b in self.unsupported_belief_ids],
+            "aggregate_confidence": self.aggregate_confidence,
+            "naive_average_confidence": self.naive_confidence,
+        }
+
+
+class IndependenceService:
+    """Groups beliefs by the independent source they ultimately rest on."""
+
+    def __init__(self, session: Session) -> None:
+        self.session = session
+        self.provenance = ProvenanceService(session)
+        self.observations = ObservationRepository(session)
+
+    def analyze(
+        self,
+        beliefs: Sequence[Belief],
+        allowed_statuses: Iterable[str] = DEFAULT_ROOT_STATUSES,
+        ignore_retracted: bool = True,
+    ) -> IndependenceReport:
+        """Build the independence report for a set of beliefs.
+
+        Retracted beliefs are ignored by default: an agent that has withdrawn a
+        belief is no longer asserting it (spec section 21).
+        """
+        considered = [
+            b
+            for b in beliefs
+            if not (ignore_retracted and b.epistemic_status == EpistemicStatus.RETRACTED)
+        ]
+
+        report = IndependenceReport(
+            belief_ids=[b.id for b in considered],
+            agreeing_agent_ids={b.agent_id for b in considered},
+        )
+        if not considered:
+            return report
+
+        roots_by_belief = {
+            belief.id: self.provenance.find_evidence_roots(
+                NodeRef.belief(belief.id), allowed_statuses
+            ).roots
+            for belief in considered
+        }
+
+        all_root_ids = {root for roots in roots_by_belief.values() for root in roots}
+        observations: dict[uuid.UUID, Observation] = {
+            o.id: o for o in self.observations.get_many(all_root_ids)
+        }
+
+        for belief in considered:
+            roots = roots_by_belief[belief.id]
+            if not roots:
+                report.unsupported_belief_ids.append(belief.id)
+                continue
+            for root_id in roots:
+                observation = observations.get(root_id)
+                if observation is None:
+                    continue
+                # Observations copied from a common origin share a family id and
+                # therefore collapse into ONE independent source (spec section 20).
+                key = observation.independence_key
+                report.beliefs_by_source.setdefault(key, [])
+                if belief.id not in report.beliefs_by_source[key]:
+                    report.beliefs_by_source[key].append(belief.id)
+                report.observations_by_source.setdefault(key, [])
+                if root_id not in report.observations_by_source[key]:
+                    report.observations_by_source[key].append(root_id)
+
+        confidence_by_belief = {b.id: b.confidence for b in considered}
+        report.aggregate_confidence = aggregate_confidence_by_root(
+            report.beliefs_by_source, confidence_by_belief
+        )
+        report.naive_confidence = naive_average_confidence([b.confidence for b in considered])
+        return report
+
+    def independent_evidence_count(self, beliefs: Sequence[Belief]) -> int:
+        """Convenience wrapper: just the independent source count."""
+        return self.analyze(beliefs).independent_source_count

rootmemory/services/invalidation_service.py

@@ -0,0 +1,164 @@
+"""InvalidationService: cascade repair when a source turns out to be wrong.
+
+Observations are never edited or deleted. When one is retracted, its validity
+flips and this service walks forward through everything that was built on it:
+
+    observation -> beliefs -> claims -> decisions
+
+Beliefs that still rest on other valid evidence are downgraded to uncertain;
+beliefs left with nothing are retracted. Affected claims are re-run through the
+promotion gate, and decisions that depended on them are flagged for review.
+The whole traversal is written to a RepairReport (spec sections 24-26).
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+
+from sqlalchemy.orm import Session
+
+from rootmemory.errors import NodeNotFoundError
+from rootmemory.models.audit import RepairReport
+from rootmemory.models.enums import (
+    DecisionStatus,
+    EpistemicStatus,
+    NodeType,
+    ValidityStatus,
+    Visibility,
+)
+from rootmemory.models.node_ref import NodeRef
+from rootmemory.repositories.audit_repo import AuditRepository
+from rootmemory.repositories.belief_repo import BeliefRepository
+from rootmemory.repositories.claim_repo import ClaimRepository
+from rootmemory.repositories.decision_repo import DecisionRepository
+from rootmemory.repositories.observation_repo import ObservationRepository
+from rootmemory.services.promotion_service import PromotionPolicy, PromotionService
+from rootmemory.services.provenance_service import ProvenanceService
+
+
+@dataclass(slots=True)
+class RepairSummary:
+    """Human/JSON friendly view of one repair pass."""
+
+    report_id: uuid.UUID
+    trigger_node_id: uuid.UUID
+    trigger_reason: str
+    affected_beliefs: list[uuid.UUID] = field(default_factory=list)
+    affected_claims: list[uuid.UUID] = field(default_factory=list)
+    affected_decisions: list[uuid.UUID] = field(default_factory=list)
+    details: dict = field(default_factory=dict)
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "repair_report_id": str(self.report_id),
+            "trigger_node_id": str(self.trigger_node_id),
+            "trigger_reason": self.trigger_reason,
+            "affected_beliefs": [str(b) for b in self.affected_beliefs],
+            "affected_claims": [str(c) for c in self.affected_claims],
+            "affected_decisions": [str(d) for d in self.affected_decisions],
+            "details": self.details,
+        }
+
+
+class InvalidationService:
+    def __init__(self, session: Session, policy: PromotionPolicy | None = None) -> None:
+        self.session = session
+        self.observations = ObservationRepository(session)
+        self.beliefs = BeliefRepository(session)
+        self.claims = ClaimRepository(session)
+        self.decisions = DecisionRepository(session)
+        self.audits = AuditRepository(session)
+        self.provenance = ProvenanceService(session)
+        self.promotion = PromotionService(session, policy)
+
+    def invalidate_observation(
+        self,
+        observation_id: uuid.UUID,
+        reason: str = "",
+        status: ValidityStatus = ValidityStatus.INVALID,
+    ) -> RepairSummary:
+        """Retract a source and repair everything downstream of it."""
+        observation = self.observations.get(observation_id)
+        if observation is None:
+            raise NodeNotFoundError(f"observation {observation_id} does not exist")
+
+        self.observations.set_validity(observation, status, reason)
+
+        cascade = self.provenance.cascade_descendants(NodeRef.observation(observation_id))
+        details: dict[str, dict] = {"beliefs": {}, "claims": {}, "decisions": {}}
+
+        # 1. beliefs -------------------------------------------------------
+        for belief in self.beliefs.get_many(cascade.beliefs):
+            roots = self.provenance.find_evidence_roots(NodeRef.belief(belief.id))
+            previous = belief.epistemic_status
+            if not roots.roots:
+                belief.epistemic_status = str(EpistemicStatus.RETRACTED)
+                belief.visibility = str(Visibility.PRIVATE)
+            elif belief.epistemic_status != EpistemicStatus.RETRACTED:
+                belief.epistemic_status = str(EpistemicStatus.UNCERTAIN)
+            details["beliefs"][str(belief.id)] = {
+                "previous_status": str(previous),
+                "status": str(belief.epistemic_status),
+                "remaining_independent_roots": len(roots.roots),
+            }
+        self.session.flush()
+
+        # 2. claims --------------------------------------------------------
+        for claim_id in cascade.claims:
+            claim = self.claims.get(claim_id)
+            if claim is None:
+                continue
+            previous_status = str(claim.status)
+            evaluation = self.promotion.recompute(claim_id)
+            details["claims"][str(claim_id)] = {
+                "previous_status": previous_status,
+                "status": str(claim.status),
+                "independent_support_count": evaluation.independent_support_count,
+                "aggregate_confidence": evaluation.aggregate_confidence,
+                "decision": "promote" if evaluation.promoted else "reject",
+            }
+
+        # 3. decisions -----------------------------------------------------
+        decision_ids = list(cascade.decisions)
+        for dependent in self.decisions.decisions_for_claims(cascade.claims):
+            if dependent.id not in decision_ids:
+                decision_ids.append(dependent.id)
+
+        for decision_id in decision_ids:
+            affected = self.decisions.get(decision_id)
+            if affected is None:
+                continue
+            previous_status = str(affected.status)
+            if affected.status == DecisionStatus.ACTIVE:
+                self.decisions.set_status(affected, DecisionStatus.NEEDS_REVIEW)
+            details["decisions"][str(decision_id)] = {
+                "previous_status": previous_status,
+                "status": str(affected.status),
+            }
+
+        report = self.audits.record_repair(
+            trigger_node_id=observation_id,
+            trigger_node_type=str(NodeType.OBSERVATION),
+            trigger_reason=reason or "source retracted",
+            affected_beliefs=[str(b) for b in cascade.beliefs],
+            affected_claims=[str(c) for c in cascade.claims],
+            affected_decisions=[str(d) for d in decision_ids],
+            details=details,
+        )
+
+        return RepairSummary(
+            report_id=report.id,
+            trigger_node_id=observation_id,
+            trigger_reason=report.trigger_reason,
+            affected_beliefs=list(cascade.beliefs),
+            affected_claims=list(cascade.claims),
+            affected_decisions=decision_ids,
+            details=details,
+        )
+
+    def get_report(self, report_id: uuid.UUID) -> RepairReport:
+        report = self.audits.get_repair(report_id)
+        if report is None:
+            raise NodeNotFoundError(f"repair report {report_id} does not exist")
+        return report

rootmemory/services/normalization_service.py

@@ -0,0 +1,234 @@
+"""Claim normalization: deciding which proposition a belief is about.
+
+This is the gap between the demo and real text. "The deadline is Oct 15",
+"delivery is scheduled for October 15" and "Oct 15 is confirmed" are the same
+proposition, but unless something maps them onto one ``claim_key`` they never
+meet, every claim looks single-source, and the gate silently rejects everything.
+
+Three strategies, chosen by ``CLAIM_NORMALIZATION``:
+
+``manual``         the caller always supplies the key (V1 behaviour, the default)
+``deterministic``  derive a slug from the text - no model, fully reproducible
+``llm``            ask a model to match an existing claim, falling back to the
+                   deterministic slug when it declines or is unavailable
+
+**The boundary that matters.** A normalizer only decides *which claim a belief
+is filed under*. It never touches provenance, independence counting, confidence
+aggregation or the promotion gate - those stay deterministic (spec section 47).
+The worst a bad normalizer can do is file a belief under the wrong claim, which
+is visible and repairable. It can never make one source look like two.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Sequence
+from typing import Protocol
+
+from rootmemory.config import Settings, get_settings
+from rootmemory.logging_config import get_logger
+
+logger = get_logger("rootmemory.normalization")
+
+#: Words carrying no propositional content, dropped when building a slug.
+#:
+#: Negation words are deliberately NOT here. "the deadline is not October 15"
+#: and "the deadline is October 15" are opposite claims, and dropping "not"
+#: would collapse them onto one.
+STOPWORDS = frozenset(
+    """a an the is are was were be been being of to in on at for from by with
+    that this these those it its as and or but if then than so such may might
+    will would can could should must have has had do does did we they
+    he she you i our their his her your my""".split()
+)
+
+MAX_SLUG_WORDS = 6
+MAX_SLUG_LENGTH = 120
+
+
+class ClaimNormalizer(Protocol):
+    """Maps belief text onto a claim key."""
+
+    def normalize(self, text: str, existing_keys: Sequence[str] = ()) -> str:
+        """Return the claim key this text belongs under."""
+        ...
+
+
+def slugify(text: str) -> str:
+    """A stable, readable key derived from the text alone.
+
+    Deterministic and dependency-free, so the same sentence always produces the
+    same key on every machine and in every process.
+    """
+    words = re.findall(r"[a-z0-9]+", text.lower())
+    meaningful = [w for w in words if w not in STOPWORDS] or words
+    return "_".join(meaningful[:MAX_SLUG_WORDS])[:MAX_SLUG_LENGTH] or "unnamed_claim"
+
+
+class ManualNormalizer:
+    """No normalization: the caller is responsible for the key."""
+
+    def normalize(self, text: str, existing_keys: Sequence[str] = ()) -> str:
+        return slugify(text)
+
+
+#: Prefixes that flip a word's meaning. "stable" and "unstable" share almost
+#: all their letters and all their neighbouring words, so plain overlap scoring
+#: happily merges a proposition with its own negation.
+NEGATION_PREFIXES = ("un", "in", "im", "il", "ir", "non", "dis", "anti")
+
+#: Standalone words that flip a statement's polarity.
+NEGATION_WORDS = frozenset({"not", "no", "never", "without", "denies", "denied", "false"})
+
+
+def contradicts(left: set[str], right: set[str]) -> bool:
+    """True when two word sets look like opposite claims rather than the same one.
+
+    Two checks: a bare negation word present on one side only, and a word on one
+    side that is a negated form of a word on the other ("stable"/"unstable").
+    """
+    if bool(left & NEGATION_WORDS) != bool(right & NEGATION_WORDS):
+        return True
+
+    for a, b in ((left, right), (right, left)):
+        extra = a - b
+        # "non compliant" tokenizes to a bare prefix beside the word it negates.
+        if extra & set(NEGATION_PREFIXES) and (a - extra - {"non"}) & b:
+            return True
+        for word in extra:
+            for prefix in NEGATION_PREFIXES:
+                if word.startswith(prefix) and word[len(prefix) :] in b:
+                    return True
+    return False
+
+
+class DeterministicNormalizer:
+    """Slug-based matching with a cheap overlap check against known claims.
+
+    Not clever, but honest and free: two sentences sharing most of their
+    meaningful words land on the same claim - unless one is the negation of the
+    other, which overlap alone cannot see.
+
+    Its weakness is recall, not safety: it merges rephrasings only when the
+    wording is close, so varied phrasing starts separate claims. That is the
+    failure an LLM normalizer is for.
+    """
+
+    #: Share of meaningful words that must overlap to reuse an existing key.
+    similarity_threshold = 0.6
+
+    def normalize(self, text: str, existing_keys: Sequence[str] = ()) -> str:
+        candidate = slugify(text)
+        candidate_words = set(candidate.split("_"))
+        if not candidate_words:
+            return candidate
+
+        best_key, best_score = candidate, 0.0
+        for key in existing_keys:
+            key_words = set(key.split("_"))
+            if not key_words:
+                continue
+            # Never merge a claim with its own negation: filing a contradicting
+            # belief as support would corrupt both sides of the claim.
+            if contradicts(candidate_words, key_words):
+                continue
+            overlap = len(candidate_words & key_words) / len(candidate_words | key_words)
+            if overlap > best_score:
+                best_key, best_score = key, overlap
+
+        return best_key if best_score >= self.similarity_threshold else candidate
+
+
+class LLMClaimNormalizer:
+    """Ask a model which existing claim a belief is about.
+
+    The model is given the existing keys and may only choose one of them or
+    decline. It never invents the key itself: a decline falls through to the
+    deterministic slug, so a hallucinated or malformed answer cannot create
+    junk claims.
+    """
+
+    system_prompt = (
+        "You map a statement onto the proposition it asserts.\n"
+        "You are given existing claim keys. If the statement asserts the SAME "
+        "proposition as one of them, return that key exactly.\n"
+        "Two statements match only if one being true makes the other true. "
+        "Statements about the same topic that assert different things do NOT match "
+        "-- 'the supplier is stable' and 'the supplier is unstable' are different "
+        "propositions.\n"
+        'Reply with JSON only: {"key": "<existing key>"} or {"key": null}.'
+    )
+
+    def __init__(
+        self,
+        model: str,
+        api_key: str,
+        base_url: str | None = None,
+        fallback: ClaimNormalizer | None = None,
+    ) -> None:
+        self.model = model
+        self.api_key = api_key
+        self.base_url = base_url
+        self.fallback = fallback or DeterministicNormalizer()
+
+    def normalize(self, text: str, existing_keys: Sequence[str] = ()) -> str:
+        if not existing_keys:
+            return self.fallback.normalize(text, existing_keys)
+        try:
+            chosen = self._ask(text, list(existing_keys))
+        except Exception as exc:  # noqa: BLE001 - never fail a write on the model
+            logger.warning("normalizer_unavailable", error=str(exc))
+            return self.fallback.normalize(text, existing_keys)
+
+        # Only an exact match on an offered key is accepted, so a hallucinated
+        # or malformed answer can never create a junk claim.
+        if chosen is not None and chosen in existing_keys:
+            return chosen
+        return self.fallback.normalize(text, existing_keys)
+
+    def _ask(self, text: str, existing_keys: list[str]) -> str | None:
+        from openai import OpenAI  # noqa: PLC0415 - optional dependency
+
+        client = OpenAI(api_key=self.api_key, base_url=self.base_url)
+        response = client.chat.completions.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": self.system_prompt},
+                {
+                    "role": "user",
+                    "content": json.dumps(
+                        {"statement": text, "existing_keys": existing_keys}
+                    ),
+                },
+            ],
+            response_format={"type": "json_object"},
+            temperature=0,
+        )
+        payload = json.loads(response.choices[0].message.content or "{}")
+        key = payload.get("key")
+        return key if isinstance(key, str) else None
+
+
+def build_normalizer(settings: Settings | None = None) -> ClaimNormalizer:
+    """Pick a normalizer from configuration, degrading safely."""
+    settings = settings or get_settings()
+    mode = settings.claim_normalization
+
+    if mode == "llm":
+        if settings.openai_api_key and settings.openai_model:
+            return LLMClaimNormalizer(
+                model=settings.openai_model,
+                api_key=settings.openai_api_key,
+                base_url=settings.openai_base_url,
+            )
+        logger.warning(
+            "llm_normalizer_unconfigured",
+            detail="CLAIM_NORMALIZATION=llm needs OPENAI_API_KEY and OPENAI_MODEL; "
+            "falling back to deterministic",
+        )
+        return DeterministicNormalizer()
+
+    if mode == "deterministic":
+        return DeterministicNormalizer()
+    return ManualNormalizer()