zikaron 0.1.0__py3-none-any.whl

zikaron/core/consolidation/payload.py

@@ -0,0 +1,246 @@
+"""The shapes `zikaron_memory_next_group` returns, as three types a caller pattern-matches on.
+
+`architecture.md` §"Consolidator tool surface" is normative for every field here. These are `core`'s
+values, not wire objects: the literal `{done: true}` and `{busy: true}` booleans belong to whichever
+transport serializes them, because `isinstance(outcome, RunDone)` already *is* that fact at the
+Python level and a boolean field repeating it would carry no information the type does not.
+
+Kept apart from the serve loop because the two change for different reasons — a payload field moves
+when the tool surface does, while the loop moves when the lifecycle does — and because the ladder
+that authorizes the write verbs needs the record shape without needing the loop.
+"""
+
+from dataclasses import dataclass
+
+from zikaron.core.consolidation.groups import Shard
+from zikaron.core.errors import ErrorCode, ZikaronError
+from zikaron.core.records.memory import ConflictRecord, Memory, Rewrite
+
+
+@dataclass(frozen=True, slots=True)
+class GroupRecord:
+    """One row a serve delivers: `architecture.md`'s `{uuid, expected_version, gist, content,
+    created_at}` exactly, for a member, the anchor and a candidate alike.
+
+    `expected_version` is the version whose prose is in **this** payload, re-read at serve time —
+    not the version at plan time. The two are different fields for a reason: `version_seen` is
+    change detection, while this is the version the receipt was minted at, and delivering current
+    prose under a stale version would break D26 while delivering historical prose is impossible,
+    since no history is stored.
+    """
+
+    uuid: str
+    expected_version: int
+    gist: str
+    content: str
+    created_at: str
+
+    @classmethod
+    def of(cls, row: Memory) -> "GroupRecord":
+        """This row as a payload record, at its current version."""
+        return cls(
+            uuid=row.uuid,
+            expected_version=row.version,
+            gist=row.gist,
+            content=row.content,
+            created_at=row.created_at,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class RankedRecord:
+    """One candidate and its rank in `retrieval.md`'s total order.
+
+    The rank is delivered so a run is reproducible from the payload alone, and persisted alongside
+    it so the authorization set survives a service restart.
+    """
+
+    record: GroupRecord
+    rank: int
+
+
+@dataclass(frozen=True, slots=True)
+class ServedGroup:
+    """What one delivery hands the consolidator: `zikaron_memory_next_group`'s success shape.
+
+    `anchor` is `None` for an orphan group, and also `None` with `anchor_vacated=True` when a
+    planned anchor stopped being targetable before the group was served — two different situations
+    that the payload deliberately distinguishes, because the second means "there was a record here
+    and it is gone", which is information a model deciding between `merge` and `promote` can use.
+    """
+
+    group_id: str
+    run_id: str
+    anchor: GroupRecord | None
+    anchor_vacated: bool
+    journal_entries: tuple[GroupRecord, ...]
+    candidates: tuple[RankedRecord, ...]
+    shard: Shard
+    serve_count: int
+    n_gists_used: int
+    remaining_groups: int
+
+
+@dataclass(frozen=True, slots=True)
+class RunDone:
+    """`{done: true}`: this run has nothing left to serve.
+
+    A value rather than `None`, so that "the run is finished" and "something went wrong" cannot be
+    the same return. Carries no fields: the run is closed, and every count about it is in its
+    `consolidate_run` event.
+    """
+
+
+@dataclass(frozen=True, slots=True)
+class Busy:
+    """`{busy: true, ...}`: an effectively-active run belongs to a different `(session_id, pid)`.
+
+    A defined, deterministic answer rather than an error and rather than a second plan — the skill
+    reports it and stops. `holder_pid` is returned so same-session contention is diagnosable rather
+    than silent, which matters now that two consolidators launched from one kiro session share a
+    `session_id` *and* a `client_kind`.
+    """
+
+    holder_session: str
+    holder_pid: int
+    expires_at: str
+
+
+#: `ServedGroup | RunDone | Busy`: `next_group`'s three response shapes as one return type a caller
+#: pattern-matches on, rather than three optional fields two of which are always meaningless.
+type NextGroupOutcome = ServedGroup | RunDone | Busy
+
+
+@dataclass(frozen=True, slots=True)
+class NamedRow:
+    """One `{uuid, expected_version}` pair — how every consolidator verb names a row it touches.
+
+    Targets, absorbed rows and discarded rows alike, never the merge target only. Otherwise a
+    primary agent that amends an absorbed journal row after it was planned into a group would have
+    its repair silently retired by the consolidator, which is exactly the lost update D26 exists to
+    prevent.
+    """
+
+    uuid: str
+    expected_version: int
+
+
+@dataclass(frozen=True, slots=True)
+class NamedRows:
+    """Every row one call names: its `absorb` list, and a merge target where the verb has one.
+
+    Self-validating on the one bound that is independent of store state, which is what makes it rung
+    1: **a uuid may appear at most once in `absorb`**. A repeat is malformed on its own terms — the
+    list names the rows one call dispositions, so a uuid twice would bump one row's `version` twice
+    for one logical action and report an `n_absorbed` that counts it twice. The other half of the
+    stated bound, `≤ the group's member count`, needs the store and so is rung 2's, where it follows
+    from every element having to be a member of that group. The **minimum** of one row is rung 2's
+    `not_in_group` rather than `bounds`, which the error table states directly.
+
+    Raises:
+        ZikaronError: `BOUNDS` naming `absorb` if a uuid appears twice.
+    """
+
+    absorb: tuple[NamedRow, ...]
+    target: NamedRow | None = None
+
+    def __post_init__(self) -> None:
+        seen: set[str] = set()
+        for row in self.absorb:
+            if row.uuid in seen:
+                raise ZikaronError(
+                    ErrorCode.BOUNDS,
+                    field="absorb",
+                    limit="each uuid at most once",
+                    actual=row.uuid,
+                )
+            seen.add(row.uuid)
+
+    @property
+    def every(self) -> tuple[NamedRow, ...]:
+        """Every named row, the target first where there is one.
+
+        One order, used by the existence, version and receipt rungs alike, so that a call naming
+        several offending rows reports them in a defined sequence rather than in whichever order a
+        dict happened to iterate.
+        """
+        return (*(() if self.target is None else (self.target,)), *self.absorb)
+
+
+@dataclass(frozen=True, slots=True)
+class MergeTarget:
+    """The long-term record a `merge` rewrites: which row, at which version, with what prose.
+
+    One value rather than three parameters, because the three are one instruction — and because the
+    row and the prose must not be able to arrive from different places, which is what a call site
+    threading `target`, `gist` and `content` separately makes possible.
+    """
+
+    row: NamedRow
+    rewrite: Rewrite
+
+
+@dataclass(frozen=True, slots=True)
+class GroupConflict:
+    """`{conflict: true, current: [...], remaining_uuids: [...]}` — the three verbs' shared
+    rejection.
+
+    A **list** of records, unlike the primary-agent verbs' single object: a consolidator call names
+    several rows, all versions are validated before anything mutates, and one mismatch returns the
+    current record for **every** conflicting uuid so the model can re-decide in one round trip.
+
+    Carries `remaining_uuids` because the caller learns the same remaining state whether its call
+    landed or raced. A call rejected by an *error* carries no member list at all — it changed
+    nothing, so the list the caller already holds still stands.
+    """
+
+    current: tuple[ConflictRecord, ...]
+    remaining_uuids: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class Merged:
+    """`zikaron_memory_merge`'s success shape:
+    `{uuid, version, remaining_uuids, group_complete}`."""
+
+    uuid: str
+    version: int
+    remaining_uuids: tuple[str, ...]
+    group_complete: bool
+
+
+@dataclass(frozen=True, slots=True)
+class Promoted:
+    """`zikaron_memory_promote`'s success shape — identical in shape to `Merged`, and a separate
+    type.
+
+    Separate for the reason `Amended` and `Retired` are: the two verbs are not interchangeable at
+    any call site that matches on the outcome's own type, and a promotion's `uuid` may be a row
+    that did not exist before the call while a merge's never is.
+    """
+
+    uuid: str
+    version: int
+    remaining_uuids: tuple[str, ...]
+    group_complete: bool
+
+
+@dataclass(frozen=True, slots=True)
+class Discarded:
+    """`zikaron_memory_discard`'s success shape: `{retired, remaining_uuids, group_complete}`.
+
+    `retired` is how many member rows this call actually moved, read from the disposition write
+    rather than from the length of the argument, so a number the store did not produce cannot be
+    reported.
+    """
+
+    retired: int
+    remaining_uuids: tuple[str, ...]
+    group_complete: bool
+
+
+#: The three verbs' return types. Each is its own success shape or the shared conflict, as a union a
+#: caller pattern-matches rather than a boolean flag beside fields that are sometimes meaningless.
+type MergeOutcome = Merged | GroupConflict
+type PromoteOutcome = Promoted | GroupConflict
+type DiscardOutcome = Discarded | GroupConflict

zikaron/core/consolidation/planning.py

@@ -0,0 +1,192 @@
+"""`plan_groups`: write one run's whole partition, and close whatever run preceded it.
+
+`architecture.md` §"Consolidation lifecycle" §Planning is normative. Four things happen here, in
+this order, and the order is what enforces invariant 15:
+
+1. **Close any pre-existing `active` run**, with the status two tests imply — `expired` if the lease
+   has passed, else `abandoned` for its own owner and `taken_over` for anybody else. This function
+   is the only producer of all three, which is what makes them one decision rather than several
+   rules whose conditions could all be true of one run. **An explicit call takes the run over**: no
+   evidence inside the store separates a dead consolidator from a slow one, so a human invoking the
+   skill again is the evidence, and `next_group` — the path a *model* uses — still refuses a
+   stranger.
+2. **Create the new run**, owned by `(session_id, pid)` with a fresh lease. It is only after step 1
+   that this can be done without there being two effectively-active runs for an instant.
+3. **Plan and persist the groups.** Membership is frozen here. Planning is a pure function of the
+   store plus the effective config, so a run is reproducible from its inputs.
+4. **Emit `planned`**, and — when the journal was empty — close the run `complete` immediately,
+since a run with no groups has nothing left to serve.
+
+**Replanning is never incremental.** A new run always plans from scratch, which is safe precisely
+because undispositioned members stay `tier='journal' AND active=1`: nothing was lost by the run
+being abandoned, so nothing needs carrying forward. That is D29's never-lose guard doing its work —
+the store's own state is the record of what happened, and no cooperation from the model is required.
+"""
+
+import aiosqlite
+
+from zikaron.core.consolidation import grouping, groups, runs
+from zikaron.core.consolidation.context import ConsolidationCall
+from zikaron.core.consolidation.groups import Group, GroupStatus
+from zikaron.core.consolidation.runs import Run, RunStatus
+from zikaron.core.errors import ErrorCode, IndexStage, ZikaronError
+from zikaron.core.events import RunPhase
+from zikaron.core.store import transactions
+
+
+def _closing_status(previous: Run, *, call: ConsolidationCall, at: str) -> RunStatus:
+    """Which terminal status a pre-existing `active` run gets: two tests, three answers.
+
+    A lapsed lease is `expired`, whoever asks. An unexpired one is `abandoned` when the caller
+    **is** its owner — a worker deliberately restarting its own run — and `taken_over` when it is
+    not. The third value exists so those last two stay distinguishable: they have different costs,
+    and in the likeliest case nothing else on the row separates them, since a user retrying the
+    skill in one kiro session presents the same `session_id` and only a different pid.
+    """
+    if previous.has_lapsed(at=at):
+        return RunStatus.EXPIRED
+    return RunStatus.ABANDONED if previous.owner == call.owner else RunStatus.TAKEN_OVER
+
+
+async def _close_previous(
+    db: aiosqlite.Connection, *, call: ConsolidationCall, at: str
+) -> Run | None:
+    """Close the store's one `active` run, if it has one, with the status the two tests above imply.
+
+    Returns the run that was closed, so a caller can tell "there was nothing to close" from "there
+    was, and it is closed now" — which is the difference between a first run and a takeover.
+    """
+    previous = await runs.stored_active(db)
+    if previous is None:
+        return None
+    status = _closing_status(previous, call=call, at=at)
+    await runs.close(db, run_id=previous.run_id, status=status, ctx=call.ctx)
+    return previous
+
+
+async def plan_within_transaction(
+    db: aiosqlite.Connection, *, call: ConsolidationCall, at: str
+) -> Run:
+    """`plan_groups`' work, assuming the caller already holds an open transaction.
+
+    Neither commits nor rolls back, so `next_group` can call this inside its own serve transaction —
+    which it must, because an implicit replan and the serve that follows it have to be one atomic
+    step or a second caller could serve from a run this one is still writing.
+
+    Args:
+        at: the single instant this transaction is about — the closing lease comparison, the new
+            run's `started_at`, and the lease derived from it. Passed in rather than read here so a
+            serve that replans implicitly stamps the replan and the delivery identically.
+
+    Returns:
+        The new run, `active` unless the journal was empty, in which case it is already `complete`
+        in the store and this value's own `status` is the pre-close one. Callers that need the
+        current status re-read it; the serve loop instead asks the group table what is servable,
+        which is the same question asked of the authoritative side.
+    """
+    await _close_previous(db, call=call, at=at)
+    run = await runs.create(
+        db,
+        owner=call.owner,
+        started_at=at,
+        lease_seconds=call.settings.run_lease_seconds,
+    )
+    planned = await grouping.plan(db, call=call)
+    for group in planned:
+        group_id = groups.new_group_id()
+        await groups.insert(
+            db,
+            Group(
+                group_id=group_id,
+                run_id=run.run_id,
+                anchor_uuid=group.anchor_uuid,
+                order_key=group.order_key,
+                shard=group.shard,
+                status=GroupStatus.PENDING,
+                serve_count=0,
+                served_at=None,
+            ),
+        )
+        for member in group.members:
+            await groups.insert_member(
+                db,
+                group_id=group_id,
+                memory_uuid=member.uuid,
+                version_seen=member.version,
+            )
+    await runs.log_phase(
+        db,
+        run_id=run.run_id,
+        phase=RunPhase.PLANNED,
+        run_counts=await groups.run_counts(db, run.run_id),
+        ctx=call.ctx,
+    )
+    if not planned:
+        # A run with zero groups has nothing to serve, so it is complete the moment it is planned.
+        # Invariant 17's `active → complete` cause is "whichever transaction first observes that no
+        # group of the run is pending or served", and this transaction is the first to observe it.
+        await runs.close(db, run_id=run.run_id, status=RunStatus.COMPLETE, ctx=call.ctx)
+    return run
+
+
+async def plan_groups(db: aiosqlite.Connection, *, call: ConsolidationCall) -> Run:
+    """Plan a fresh consolidation run over the whole journal, in one transaction — taking over if
+    held.
+
+    The service RPC, and it is the **one** entry point that may displace a live worker. Whoever
+    calls it wins: an unexpired run of another owner is closed `taken_over`, its own unexpired run
+    is closed `abandoned`, and a lapsed one is closed `expired`. `next_group` answers `{busy: true}`
+    to a stranger instead, and that asymmetry is what D32 buys: a model confined to a configured
+    tool surface cannot *request* a takeover, because this RPC is in neither tool set. It is not a
+    claim that nothing calls it — the consolidator's own MCP client does, immediately before the
+    first `next_group` it forwards and at most once *successfully* per process — nor a capability
+    boundary, since the socket is unauthenticated to any same-uid process.
+
+    **Why takeover rather than refusal, since the refusal is the safer-looking choice.** Nothing
+    reachable from inside the store distinguishes a dead consolidator from one that is merely slow.
+    A lease is a timer, so it cannot. A pid-liveness check is not useless — the holder's `pid` names
+    an MCP client process that normally exits with its own agent — but it cannot carry the decision:
+    it answers *is this process alive*, never *will this worker make progress*, pid reuse can return
+    a false alive and prolong the very lockout being diagnosed, and the service and the client are
+    not guaranteed a shared pid namespace. The evidence that settles it is outside the store
+    altogether: a human invoking the consolidation skill a second time. Refusing would pin the store
+    for up to `run_lease` on a worker that has stopped.
+
+    **Nothing the displaced worker does can land, and that is the ladder's property rather than a
+    hope.** Rung 2 requires a group's run to be owned by the caller *and* effectively active, so the
+    displaced worker's next `merge`/`promote`/`discard` is refused `group_expired` before touching a
+    row, and its next `next_group` finds the new run foreign and stops. It loses its reasoning in
+    progress — what the human chose to discard — and no journal row, since undispositioned members
+    stay `tier='journal' AND active=1` and are replanned.
+
+    `next_group` does not come through here: it calls `plan_within_transaction` inside its own serve
+    transaction, having already established that no effectively-active run exists.
+
+    Returns:
+        The new run, `active` unless the journal was empty.
+
+    Raises:
+        ZikaronError: `STORE_BUSY` if the store was locked, which the caller may retry, or
+            `INDEX_FAILED` at the `index_write` stage for any other driver failure — the plan is a
+            write, so `architecture.md` §Errors gives it a write's two codes and nothing is left
+            behind in either case.
+    """
+
+    async def work(connection: aiosqlite.Connection) -> Run:
+        return await plan_within_transaction(connection, call=call, at=runs.now())
+
+    return await transactions.in_one_transaction(db, work, failure=_failure_map("plan_groups"))
+
+
+def _failure_map(verb: str) -> transactions.FailureMap:
+    """Name contention `store_busy` and everything else `index_failed`, per `architecture.md`.
+
+    This layer's own decision for its own transactions, deliberately not a shared utility: what
+    varies between callers is exactly how a driver failure is named, and a consolidation transaction
+    writes rows, chunks and vectors, so it is a write by that section's own test.
+    """
+    return lambda error: (
+        ZikaronError(ErrorCode.STORE_BUSY, verb=verb)
+        if transactions.is_contention(error)
+        else ZikaronError(ErrorCode.INDEX_FAILED, stage=IndexStage.INDEX_WRITE)
+    )

zikaron/core/consolidation/rowstate.py

@@ -0,0 +1,68 @@
+"""The two row-state re-checks a serve and the ladder both run, evaluated by SQLite's own clause.
+
+`architecture.md` names them as one condition each, and names them **twice** on purpose: a member
+that fails the first at serve time is recorded `disposition='vacated'`, and a member that fails it
+at call time is *vacated in fact and not yet recorded*, which is why rung 6 answers `not_in_group`
+rather than inventing a code. Those two sites must apply the identical predicate or the second
+would reject rows the first would have delivered.
+
+**So the predicate is not re-expressed here.** The clause comes from
+`eligibility.CONSUMER_FILTERS` — `tier='journal' AND active=1` for a member, `tier='long_term' AND
+active=1` for a merge target — and is evaluated by SQLite against the row. A Python mirror
+(`row.tier is Tier.JOURNAL and row.active`) would read better and would be a second statement of a
+rule the design gives exactly one home, silently surviving a change to that table. This is the same
+choice the query constructor makes when it hands quoted terms to FTS5's own tokenizer instead of
+reimplementing `unicode61`.
+
+The cost is one indexed `EXISTS` per row on a manually-invoked, rare path, inside a transaction the
+caller already holds.
+"""
+
+from typing import Final
+
+import aiosqlite
+
+from zikaron.core.retrieval.eligibility import MEMORY_ALIAS, Consumer, narrowing
+
+#: The two consumers whose filters double as these state conditions. Named so the mapping from
+#: "what is being checked" to "whose filter says so" is written once rather than at each call site.
+_MEMBER: Final = Consumer.ORPHAN
+_TARGET: Final = Consumer.CONSOLIDATION
+
+
+async def _satisfies(db: aiosqlite.Connection, *, uuid: str, consumer: Consumer) -> bool:
+    """Whether the row named by `uuid` satisfies `consumer`'s own narrowing filter, right now.
+
+    Raises:
+        KeyError: the consumer states no narrowing filter, so there is nothing to check and a caller
+            asking has confused "no filter" with "admits nothing". Only `surface` and `search` are
+            in that position and neither has a row-state condition to re-check.
+    """
+    clause = narrowing(consumer)
+    if clause is None:
+        raise KeyError(f"{consumer} states no narrowing filter, so there is nothing to re-check")
+    sql = (
+        "SELECT EXISTS (SELECT 1 FROM memory "  # noqa: S608 — the clause is a source-level constant from the filter table; the uuid is bound.
+        f"{MEMORY_ALIAS} WHERE {MEMORY_ALIAS}.uuid = ? AND {clause})"
+    )
+    rows = await db.execute_fetchall(sql, (uuid,))
+    return bool(next(iter(rows))[0])
+
+
+async def is_deliverable_member(db: aiosqlite.Connection, uuid: str) -> bool:
+    """Whether this row is still an unconsolidated journal row, i.e. still a deliverable member.
+
+    A row that fails this has left the journal by another path between plan and call — a primary
+    agent retired it, or promoted it. Nothing was lost and it blocks no completion; it is simply not
+    something a consolidator may act on.
+    """
+    return await _satisfies(db, uuid=uuid, consumer=_MEMBER)
+
+
+async def is_targetable(db: aiosqlite.Connection, uuid: str) -> bool:
+    """Whether this row is still an active long-term record, i.e. still a legal `merge` target.
+
+    A merge never rewrites a historical row: `superseded_by` is immutable once set, so a retired row
+    given fresh prose would be neither the current record nor a faithful historical one.
+    """
+    return await _satisfies(db, uuid=uuid, consumer=_TARGET)