rememberstack 0.1.0__py3-none-any.whl

rememberstack/model/operations.py

@@ -0,0 +1,153 @@
+"""Typed, bounded operational inspection and dead-letter replay values."""
+
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+from rememberstack.model.processing import ProcessingStatus
+from rememberstack.model.processing import ProcessingTarget
+from rememberstack.model.queue import PipelineStage
+from rememberstack.model.queue import ProcessingLane
+from rememberstack.model.queue import QueueRoute
+from rememberstack.model.queue import UTCDateTime
+
+
+class PipelineRouteStatus(BaseModel):
+    """One deployment route/status aggregate; the enum vocabulary bounds its size."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    stage: PipelineStage
+    lane: ProcessingLane | None
+    status: ProcessingStatus
+    count: int = Field(ge=0)
+
+
+class DeadLetterGroup(BaseModel):
+    """One dead-letter aggregate used to spot a failing version or exception."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    stage: PipelineStage
+    error_class: str
+    component_version: str
+    count: int = Field(ge=1)
+    oldest_enqueued_at: UTCDateTime
+    latest_finished_at: UTCDateTime | None
+
+
+class DeadLetterRecord(BaseModel):
+    """One bounded DLQ sample retaining the complete diagnostic and replay input."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    processing_id: UUID
+    target_kind: ProcessingTarget
+    target_id: UUID
+    stage: PipelineStage
+    component_version: str
+    content_hash: str
+    lane: ProcessingLane | None
+    attempts: int = Field(ge=0)
+    max_attempts: int = Field(ge=1)
+    error_class: str
+    last_error: str | None
+    payload: dict[str, object] | None
+    enqueued_at: UTCDateTime
+    finished_at: UTCDateTime | None
+
+
+class DeadLetterReport(BaseModel):
+    """Deployment DLQ totals plus independently bounded aggregate/detail samples."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    total: int = Field(ge=0)
+    group_total: int = Field(ge=0)
+    groups: tuple[DeadLetterGroup, ...]
+    items: tuple[DeadLetterRecord, ...]
+
+
+class PoisonTargetRecord(BaseModel):
+    """A target dead-lettered by at least two component versions for one stage."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    target_kind: ProcessingTarget
+    target_id: UUID
+    stage: PipelineStage
+    component_version_total: int = Field(ge=2)
+    component_versions: tuple[str, ...]
+    dead_letters: int = Field(ge=2)
+
+
+class PoisonTargetReport(BaseModel):
+    """Total poison targets and a bounded sample."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    total: int = Field(ge=0)
+    items: tuple[PoisonTargetRecord, ...]
+
+
+class ProjectionSnapshotState(BaseModel):
+    """The current P2 or P3 snapshot pointer, if that plane has one."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    snapshot_id: UUID
+    plane: Literal["P2_graph", "P3_corpusfs"]
+    version: str
+    store_uri: str
+    row_counts: dict[str, object] | None
+    built_at: UTCDateTime
+    published_at: UTCDateTime | None
+
+
+class CurrencyMismatch(BaseModel):
+    """A claim whose cached currency differs from the append-only ledger truth."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    claim_id: UUID
+    cached_current: bool
+    ledger_current: bool
+
+
+class CurrencyLedgerAudit(BaseModel):
+    """Deployment claim count and bounded currency-ledger mismatch evidence."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    claims: int = Field(ge=0)
+    mismatch_total: int = Field(ge=0)
+    mismatches: tuple[CurrencyMismatch, ...]
+
+
+class OperationalReport(BaseModel):
+    """One coherent, deployment-scoped operational inspection snapshot."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    deployment_id: UUID
+    generated_at: UTCDateTime
+    routes: tuple[PipelineRouteStatus, ...]
+    dead_letters: DeadLetterReport
+    poison_targets: PoisonTargetReport
+    latest_projections: tuple[ProjectionSnapshotState, ...]
+    currency: CurrencyLedgerAudit
+
+
+class DeadLetterReplayResult(BaseModel):
+    """Authoritative route and due time after reopening one dead-letter row."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    processing_id: UUID
+    route: QueueRoute
+    not_before: UTCDateTime
+    attempts: int = Field(ge=0)
+    max_attempts: int = Field(ge=1)

rememberstack/model/processing.py

@@ -0,0 +1,228 @@
+"""Typed records for the D67 work ledger: enqueue, claim, attempt, and cost rows."""
+
+from decimal import Decimal
+from enum import StrEnum
+from uuid import UUID
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+from rememberstack.model.queue import PipelineStage
+from rememberstack.model.queue import ProcessingLane
+from rememberstack.model.queue import UTCDateTime
+
+
+class ProcessingTarget(StrEnum):
+    """Exact values of the binding Postgres ``processing_target`` enum."""
+
+    DOCUMENT = "document"
+    DOCUMENT_VERSION = "document_version"
+    DOCUMENT_SECTION = "document_section"
+    CHUNK = "chunk"
+    CLAIM = "claim"
+    RELATION = "relation"
+    OBSERVATION = "observation"
+    ENTITY = "entity"
+    SNAPSHOT = "snapshot"
+    KNOWLEDGE_ARTIFACT = "knowledge_artifact"
+    KNOWLEDGE_DISPATCH = "knowledge_dispatch"
+
+
+class ProcessingStatus(StrEnum):
+    """Exact values of the binding Postgres ``processing_status`` enum."""
+
+    PENDING = "pending"
+    RUNNING = "running"
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    DEAD_LETTER = "dead_letter"
+    SKIPPED = "skipped"
+
+
+class DeferReason(StrEnum):
+    """Exact values of the binding Postgres ``processing_defer_reason`` enum (D67)."""
+
+    SCHEDULED = "scheduled"
+    RETRY_BACKOFF = "retry_backoff"
+    BUDGET = "budget"
+
+
+class EnqueueWork(BaseModel):
+    """One unit of work to insert into ``processing_state`` (D12 idempotency key)."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    deployment_id: UUID
+    target_kind: ProcessingTarget
+    target_id: UUID
+    stage: PipelineStage
+    component_version: str
+    content_hash: str
+    lane: ProcessingLane | None
+    payload: dict[str, object] | None = None
+    not_before: UTCDateTime | None = None
+
+
+class EnqueueOutcome(BaseModel):
+    """What an enqueue did: created a row, promoted its lane, or found it existing."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    processing_id: UUID
+    created: bool
+    promoted_to_steady: bool
+
+
+class BackfillSeedRequest(BaseModel):
+    """One version-bump campaign to enumerate into the backfill lane."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    deployment_id: UUID
+    stage: PipelineStage
+    component_version: str = Field(min_length=1)
+
+
+class BackfillSeedResult(BaseModel):
+    """Outcome of one bounded, restartable seeder transaction."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    selected: int = Field(ge=0)
+    created: int = Field(ge=0)
+    already_present: int = Field(ge=0)
+    complete: bool
+
+
+class ClaimedWork(BaseModel):
+    """A claimed ``processing_state`` row, running its ``attempt``-th handler execution."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    processing_id: UUID
+    deployment_id: UUID
+    target_kind: ProcessingTarget
+    target_id: UUID
+    stage: PipelineStage
+    component_version: str
+    content_hash: str
+    lane: ProcessingLane | None
+    attempt: int = Field(ge=1)
+    payload: dict[str, object] | None
+
+
+class CostBudget(BaseModel):
+    """One explicit spend ceiling for a deployment, stage, lane, and fixed window."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    deployment_id: UUID
+    stage: PipelineStage
+    lane: ProcessingLane | None
+    window_seconds: int = Field(gt=0)
+    ceiling_usd: Decimal = Field(gt=Decimal(0))
+
+
+class BudgetParked(BaseModel):
+    """A due work row parked before its next handler attempt because spend is exhausted."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    processing_id: UUID
+    resume_at: UTCDateTime
+    spent_usd: Decimal = Field(ge=Decimal(0))
+    ceiling_usd: Decimal = Field(gt=Decimal(0))
+
+
+class CostTierSpend(BaseModel):
+    """Current-window spend attributed to one recorded cascade tier."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    tier: str | None
+    cost_usd: Decimal = Field(ge=Decimal(0))
+
+
+class CostBudgetStatus(BaseModel):
+    """Admin-visible state derived from one configured ceiling and the durable ledgers."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    deployment_id: UUID
+    stage: PipelineStage
+    lane: ProcessingLane | None
+    window_seconds: int = Field(gt=0)
+    window_started_at: UTCDateTime
+    window_ends_at: UTCDateTime
+    ceiling_usd: Decimal = Field(gt=Decimal(0))
+    spent_usd: Decimal = Field(ge=Decimal(0))
+    remaining_usd: Decimal = Field(ge=Decimal(0))
+    exhausted: bool
+    parked_work: int = Field(ge=0)
+    tiers: tuple[CostTierSpend, ...]
+
+
+class RecordCall(BaseModel):
+    """One billed model/provider call to attribute to the claimed row's attempt.
+
+    Attribution fields (stage, lane, attempt) are copied from the locked
+    ``processing_state`` row by the spine and can never be supplied here (D67).
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    processing_id: UUID
+    call_key: str
+    model_name: str | None = None
+    tier: str | None = None
+    tokens_in: int | None = None
+    tokens_out: int | None = None
+    cost_usd: Decimal | None = None
+    latency_ms: int | None = None
+
+
+class RunResultOutcome(StrEnum):
+    """How one worker pass ended for the row it claimed (or that none was due)."""
+
+    NO_WORK = "no_work"
+    BUDGET_PARKED = "budget_parked"
+    SUCCEEDED = "succeeded"
+    RETRY_SCHEDULED = "retry_scheduled"
+    DEAD_LETTERED = "dead_lettered"
+
+
+class WorkLedgerError(Exception):
+    """Base error for work-ledger operations."""
+
+
+class BackfillNotDrainedError(WorkLedgerError):
+    """Search-index maintenance was requested while backfill work was unresolved."""
+
+
+class LaneRouteError(WorkLedgerError):
+    """A lane value that is illegal for the stage's route (D67 pairing rule)."""
+
+
+class WorkNotFoundError(WorkLedgerError):
+    """A ``processing_id`` that does not exist in ``processing_state``."""
+
+
+class WorkNotRunningError(WorkLedgerError):
+    """An operation that requires a running attempt hit a non-running row."""
+
+
+class WorkNotDeadLetterError(WorkLedgerError):
+    """An operation that requires a dead-letter row hit another status."""
+
+
+class NonRetryableHandlerError(Exception):
+    """A handler failure classified as permanent: the work dead-letters immediately."""
+
+
+class HandlerAlreadyRegisteredError(Exception):
+    """A second handler registration for a stage that already has one."""
+
+
+class UnknownStageHandlerError(Exception):
+    """A claimed stage with no registered handler."""

rememberstack/model/queue.py

@@ -0,0 +1,73 @@
+"""D67 queue-route snapshots and the binding Postgres stage/lane vocabulary."""
+
+from datetime import datetime
+from datetime import timedelta
+from enum import StrEnum
+from typing import Annotated
+from typing import TypeAlias
+from uuid import UUID
+
+from pydantic import AfterValidator
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+
+class PipelineStage(StrEnum):
+    """Exact values of the binding Postgres ``pipeline_stage`` enum."""
+
+    INGEST = "ingest"
+    CONVERT = "convert"
+    STRUCTURE = "structure"
+    CROSSREF = "crossref"
+    CHUNK = "chunk"
+    EMBED_CHUNK = "embed_chunk"
+    EXTRACT_CLAIMS = "extract_claims"
+    EMBED_CLAIM = "embed_claim"
+    GROUND_CLAIMS = "ground_claims"
+    RESOLVE_ENTITIES = "resolve_entities"
+    NORMALIZE_RELATIONS = "normalize_relations"
+    ADJUDICATE_SUPERSESSION = "adjudicate_supersession"
+    ADJUDICATE_OBSERVATIONS = "adjudicate_observations"
+    EMBED_RELATION = "embed_relation"
+    LABEL_RELATION = "label_relation"
+    EMBED_OBSERVATION = "embed_observation"
+    LABEL_OBSERVATION = "label_observation"
+    REFRESH_PROFILE = "refresh_profile"
+    BUILD_SNAPSHOT = "build_snapshot"
+    DETECT_COMMUNITIES = "detect_communities"
+    COMPILE_KNOWLEDGE = "compile_knowledge"
+    REFLECT_KNOWLEDGE = "reflect_knowledge"
+    LINT_KNOWLEDGE = "lint_knowledge"
+    RECONCILE = "reconcile"
+    DISPATCH_KNOWLEDGE = "dispatch_knowledge"  # appended by WP-6.6
+    HARD_FORGET = "hard_forget"
+
+
+class ProcessingLane(StrEnum):
+    """The two and only two lane values used by Plane-E work."""
+
+    STEADY = "steady"
+    BACKFILL = "backfill"
+
+
+def _require_utc(value: datetime) -> datetime:
+    """Require an aware datetime whose UTC offset is exactly zero."""
+    if value.tzinfo is None or value.utcoffset() != timedelta(0):
+        raise ValueError("datetime must be timezone-aware UTC")
+    return value
+
+
+UTCDateTime: TypeAlias = Annotated[
+    datetime, Field(strict=True), AfterValidator(_require_utc)
+]
+
+
+class QueueRoute(BaseModel):
+    """Non-authoritative D67 physical-delivery route snapshot."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    deployment_id: UUID
+    stage: PipelineStage
+    lane: ProcessingLane | None

rememberstack/model/recipes.py

@@ -0,0 +1,83 @@
+"""Recipe registry values (D50): frozen query plans as data, not code.
+
+A **recipe** is a named, versioned composition of the zero-LLM primitives
+(retrieval §4) — `relation_hybrid_rrf`, `claims_as_of`, `entity_timeline`,
+and the rest. It is a *registry row*, never code: the MCP tool list renders
+from these rows, the eval harness measures recall@k per recipe version, and
+adding a query pattern is inserting a row. The load-bearing property is that
+a recipe adds **no capability** — anything it does, an agent can compose from
+§3 — so a recipe is exactly its `chain`, and the eval harness proves it by
+replaying the chain and diffing.
+
+Two declared enums make the D41 grain bar ("claims never answer *is it true
+now*") a mechanical constraint rather than a prose judgment: `output_grain`
+(the D49 envelope grain the recipe returns) and `answer_intent` (what kind of
+question it answers). The database CHECK enforces the headline rule
+(`current_facts` ⇒ `fact` grain); the registration linter (`core`) validates
+the chain itself against the same enums.
+"""
+
+from enum import StrEnum
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+from rememberstack.model.envelope import Grain
+
+
+class RecipeAnswerIntent(StrEnum):
+    """What kind of question a recipe answers (the D50 `answer_intent`).
+
+    The intent is what the grain linter reasons over: `current_facts` may
+    only ride validity-filtered fact primitives, `assertion_history` is the
+    evidence-grain "what did sources assert" read, `audit` is the decision
+    trail, `change_feed` is the delta, and `orientation` is a pre-paid
+    synthesis (K pages, briefs).
+    """
+
+    CURRENT_FACTS = "current_facts"
+    ASSERTION_HISTORY = "assertion_history"
+    ORIENTATION = "orientation"
+    AUDIT = "audit"
+    CHANGE_FEED = "change_feed"
+
+
+class RecipeStep(BaseModel):
+    """One primitive invocation in a recipe chain (retrieval §3 op + settings).
+
+    `op` names a §3 primitive; `settings` are the FIXED arguments the recipe
+    freezes (channel sets, RRF constants, rerank weights); `bind` maps a
+    primitive keyword to the name of a recipe parameter the caller supplies;
+    and `inputs` names the prior steps (by index) whose outputs this op
+    consumes — how `fuse` references the two searches above it. A recipe with
+    one step is the common case (a recipe over a single primitive with frozen
+    settings); the chain generalizes to fusion pipelines.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    op: str
+    settings: dict[str, object] = Field(default_factory=dict)
+    bind: dict[str, str] = Field(default_factory=dict)
+    inputs: tuple[int, ...] = ()
+
+
+class Recipe(BaseModel):
+    """One recipe registry row (D50): the frozen plan and its declared grain.
+
+    `parameters` is a JSON-Schema-shaped description of the arguments a
+    caller passes (rendered into the MCP tool signature). `chain` is the
+    ordered primitive composition — the recipe's entire behavior. The two
+    enums are the contract the linter and the DB CHECK enforce.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    name: str = Field(min_length=1)
+    description: str = Field(min_length=1)
+    parameters: dict[str, object] = Field(default_factory=dict)
+    chain: tuple[RecipeStep, ...] = Field(min_length=1)
+    output_grain: Grain
+    answer_intent: RecipeAnswerIntent
+    version: int = Field(default=1, ge=1)

rememberstack/model/relations.py

@@ -0,0 +1,79 @@
+"""E3 normalization values: LLM candidates, resolution, and fact records (D2-D5, D17-D18, D43)."""
+
+from typing import Annotated
+from uuid import UUID
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+_NonEmpty = Annotated[str, Field(min_length=1)]
+
+
+class EntityRef(BaseModel):
+    """One entity as the normalizer emitted it: canonical form + registry type."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    name: _NonEmpty
+    type: _NonEmpty
+
+
+class RelationCandidate(BaseModel):
+    """One (subject, predicate, object) proposal from the normalizer call."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    subject: EntityRef
+    predicate: _NonEmpty
+    object: EntityRef
+
+
+class ObservationCandidate(BaseModel):
+    """One entity-anchored value/statement proposal (D43), incl. stances (D59)."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    subject: EntityRef
+    statement: _NonEmpty
+
+
+class ObservationAssertion(BaseModel):
+    """One resolved observation input in a document/entity adjudication batch."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    statement: _NonEmpty
+    claim_id: UUID
+    doc_id: UUID
+
+
+class NormalizationResponse(BaseModel):
+    """The normalizer call's structured output for one claim (0..n of each)."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    relations: tuple[RelationCandidate, ...] = ()
+    observations: tuple[ObservationCandidate, ...] = ()
+
+
+class ClaimForNormalization(BaseModel):
+    """One accepted claim as the normalize stage loads it."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    claim_id: UUID
+    doc_id: UUID
+    chunk_id: UUID
+    claim_text: str
+    is_attributed: bool
+
+
+class ResolvedEntity(BaseModel):
+    """A T0 resolution outcome: the canonical id and whether it was minted."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    entity_id: UUID
+    created: bool
+    entity_type: _NonEmpty

rememberstack/model/resolution.py

@@ -0,0 +1,83 @@
+"""ER cascade values (D17): candidates, bands, verdicts, and the T4 response.
+
+Block-loose / decide-tight: T1/T2 generate candidates and never decide; T0,
+T3, and T4 decide. Thresholds are per-type, golden-set-measured starting
+points versioned in `resolver_versions` — never committed constants.
+"""
+
+from typing import Annotated
+from uuid import UUID
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+_NonEmpty = Annotated[str, Field(min_length=1)]
+_Unit = Annotated[float, Field(ge=-1.0, le=1.0)]
+
+
+class ResolutionCandidate(BaseModel):
+    """One blocked candidate: which tier surfaced it and its scores."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    entity_id: UUID
+    canonical_name: _NonEmpty
+    type: _NonEmpty
+    blocking_tier: _NonEmpty  # T0 | T1 | T2
+    trigram_score: float | None = None
+    embedding_score: _Unit | None = None
+
+
+class TypeThresholds(BaseModel):
+    """One entity type's decision bands (starting points to measure, D22).
+
+    T3 cosine >= accept: match. <= reject: not this candidate. Between the
+    bands: escalate to T4 — cheap tiers never auto-reject near-misses (the
+    blocking ceiling is a recall ceiling, not a verdict).
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    t3_accept: _Unit = 0.88
+    t3_reject: _Unit = 0.60
+    t4_small_confidence_floor: Annotated[float, Field(ge=0.0, le=1.0)] = 0.75
+
+
+class ResolverConfig(BaseModel):
+    """The versioned cascade configuration (`resolver_versions` row shape)."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    resolver_version: _NonEmpty
+    trigram_floor: Annotated[float, Field(ge=0.0, le=1.0)] = 0.3
+    blocking_limit: Annotated[int, Field(ge=1)] = 10
+    t4_max_candidates: Annotated[int, Field(ge=1)] = 3
+    default_thresholds: TypeThresholds = TypeThresholds()
+    thresholds_by_type: dict[str, TypeThresholds] = {}
+
+    def thresholds_for(self, *, entity_type: str) -> TypeThresholds:
+        """The type's bands, falling back to the defaults."""
+        return self.thresholds_by_type.get(entity_type, self.default_thresholds)
+
+
+class AdjudicationVerdict(BaseModel):
+    """The T4 call's structured output: same entity or not, with confidence."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    match: bool
+    confidence: Annotated[float, Field(ge=0.0, le=1.0)]
+    rationale: str | None = None
+
+
+class P1EntityRow(BaseModel):
+    """One row of the P1 entities table: the T3 profile embedding home (D8)."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    entity_id: UUID
+    deployment_id: UUID
+    type: _NonEmpty
+    canonical_name: _NonEmpty
+    vector: Annotated[tuple[float, ...], Field(min_length=1)]