iCode-TUI 0.29.1__py3-none-any.whl

chrys/foundation/trajectory/recovery.py

@@ -0,0 +1,225 @@
+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+"""Activation-time recovery for an existing events file.
+
+A crash can leave a torn final line. Appending after it would turn the torn
+bytes into a bad line in the *middle* of the file, where "ignore the last
+line" no longer applies. So before a writer resumes a file it must, under the
+session's exclusive lock: locate the last complete newline, truncate the
+torn tail, and recover ``sequence`` from the last valid event.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from chrys.foundation.trajectory.envelope import (
+    SCHEMA_VERSION,
+    EnvelopeError,
+    TrajectoryEvent,
+    decode_event_line,
+    peek_envelope_header,
+)
+
+_INITIAL_TAIL_BYTES = 64 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class RecoveryScan:
+    """What the tail of an existing events file looks like."""
+
+    file_size: int
+    complete_offset: int
+    """Byte offset just past the last complete newline (0 for an empty file)."""
+    last_sequence: int
+    """Highest slot the file has spoken for (0 when it has spoken for none).
+
+    Usually the last valid event's, but a corrupt line whose header still
+    names its slot counts too: those bytes stay on disk, so handing the same
+    sequence out again would put two lines on one slot.
+    """
+    last_event: TrajectoryEvent | None
+    newer_schema_version: int | None = None
+    """Set when the tail was written under a schema this build must not append after."""
+    unreadable_slots: tuple[int, int] | None = None
+    """Inclusive slot range the file spent on lines no reader can show, if any.
+
+    A damaged line keeps its bytes and its slot but never reaches a reader,
+    so resuming past it leaves a hole in the sequence that nothing explains.
+    A writer that resumes here owes the file a gap over this range first.
+    """
+    unreadable_tail: bool = False
+    """Set when a complete line past the last valid event does not even name its slot.
+
+    A damaged line whose header still parses spends a slot this build can
+    name, and ``unreadable_slots`` accounts for it. One whose header does not
+    parse spends a slot nobody can name — and a slot that cannot be named
+    cannot be skipped over, because the next sequence to hand out is exactly
+    the thing in question. There is nothing to resume from here.
+    """
+
+    @property
+    def had_valid_events(self) -> bool:
+        return self.last_event is not None
+
+    @property
+    def truncated_bytes(self) -> int:
+        return self.file_size - self.complete_offset
+
+
+def scan_for_recovery(path: Path) -> RecoveryScan:
+    """Inspect the tail of *path* without modifying it."""
+    with path.open("rb") as handle:
+        return scan_open_file(handle.fileno())
+
+
+def scan_open_file(fd: int) -> RecoveryScan:
+    """Inspect the tail of the file *fd* is open on, without modifying it.
+
+    This is how a writer scans: the descriptor it will append through is the
+    only thing that names the file it holds. Reopening the pathname to read it
+    can land on a file that replaced it in the meantime, and offsets read from
+    one file then truncate the other.
+    """
+    size = os.fstat(fd).st_size
+    if size == 0:
+        return RecoveryScan(file_size=0, complete_offset=0, last_sequence=0, last_event=None)
+    window = min(size, _INITIAL_TAIL_BYTES)
+    while True:
+        start = size - window
+        data = _read_at(fd, start, window)
+        newline_at = data.rfind(b"\n")
+        if newline_at < 0:
+            if start == 0:
+                # No newline anywhere: the whole file is one torn line.
+                return RecoveryScan(file_size=size, complete_offset=0, last_sequence=0, last_event=None)
+            window = min(size, window * 2)
+            continue
+        complete_offset = start + newline_at + 1
+        tail = _scan_tail(data[: newline_at + 1], whole_file=start == 0)
+        if tail.newer_schema_version is not None:
+            # Nothing past this line can be read here, its slot included,
+            # so there is no sequence to resume from — only a range this
+            # build would write over.
+            return RecoveryScan(
+                file_size=size,
+                complete_offset=complete_offset,
+                last_sequence=0,
+                last_event=None,
+                newer_schema_version=tail.newer_schema_version,
+            )
+        if tail.event is not None:
+            last_sequence = max(tail.event.sequence, tail.claimed_sequence)
+            return RecoveryScan(
+                file_size=size,
+                complete_offset=complete_offset,
+                last_sequence=last_sequence,
+                last_event=tail.event,
+                unreadable_slots=_range_after(tail.event.sequence, last_sequence),
+                unreadable_tail=tail.unreadable_tail,
+            )
+        if start == 0:
+            # The whole file walked and nothing in it decodes. Damaged lines
+            # that still name their slot took those slots — and every slot
+            # below the highest of them is just as unreadable, because no
+            # line anywhere in this file could be read.
+            return RecoveryScan(
+                file_size=size,
+                complete_offset=complete_offset,
+                last_sequence=tail.claimed_sequence,
+                last_event=None,
+                unreadable_slots=_range_after(0, tail.claimed_sequence),
+                unreadable_tail=tail.unreadable_tail,
+            )
+        window = min(size, window * 2)
+
+
+def _range_after(last_readable: int, last_claimed: int) -> tuple[int, int] | None:
+    """The slots between the last readable event and the last claimed one, if any."""
+    if last_claimed <= last_readable:
+        return None
+    return last_readable + 1, last_claimed
+
+
+def _read_at(fd: int, offset: int, length: int) -> bytes:
+    """Read *length* bytes from *offset* without disturbing where writes land.
+
+    ``pread`` where the platform has it; elsewhere a seek, which an append
+    descriptor ignores when it writes. Either way the read is repeated until
+    it has what it asked for or the file ends, because one read call is
+    allowed to return less.
+    """
+    pread = getattr(os, "pread", None)
+    chunks: list[bytes] = []
+    read_so_far = 0
+    while read_so_far < length:
+        want = length - read_so_far
+        at = offset + read_so_far
+        if pread is not None:
+            chunk = pread(fd, want, at)
+        else:
+            os.lseek(fd, at, os.SEEK_SET)
+            chunk = os.read(fd, want)
+        if not chunk:
+            break
+        chunks.append(chunk)
+        read_so_far += len(chunk)
+    return b"".join(chunks)
+
+
+@dataclass(frozen=True, slots=True)
+class _TailScan:
+    """What the last complete lines say about where a writer may resume."""
+
+    event: TrajectoryEvent | None = None
+    newer_schema_version: int | None = None
+    claimed_sequence: int = 0
+    """Highest slot a complete line claims, whether or not this build can read the rest of it."""
+    unreadable_tail: bool = False
+    """Set when a complete line past the returned event claims no slot at all."""
+
+
+def _scan_tail(data: bytes, *, whole_file: bool) -> _TailScan:
+    """The last decodable event among complete lines in *data*, and the slots they claim.
+
+    Walked from the end. A line written under a newer schema stops the walk:
+    a build that cannot read a line cannot know which slot it took, and
+    resuming from an older event would put this runtime's events on slots the
+    newer one already used. A line this build *should* have been able to read
+    but cannot is damage rather than skew — the walk goes on past it — yet its
+    header may still name the slot it took, and a slot named is a slot spent.
+    When *data* is a tail window rather than the whole file, its first line
+    may be a fragment and is skipped.
+    """
+    lines = data.split(b"\n")
+    if lines and lines[-1] == b"":
+        lines.pop()
+    first_index = 0 if whole_file else 1
+    claimed = 0
+    unreadable_tail = False
+    for index in range(len(lines) - 1, first_index - 1, -1):
+        line = lines[index]
+        header = peek_envelope_header(line)
+        if header is not None:
+            if header.schema_version > SCHEMA_VERSION:
+                return _TailScan(newer_schema_version=header.schema_version)
+            claimed = max(claimed, header.sequence)
+        try:
+            return _TailScan(event=decode_event_line(line), claimed_sequence=claimed, unreadable_tail=unreadable_tail)
+        except EnvelopeError:
+            # A line whose header parsed spent a slot this build can name; one
+            # whose header did not spent a slot nobody can name, and the walk
+            # cannot tell how far past the next event the file already went.
+            unreadable_tail = unreadable_tail or header is None
+            continue
+    return _TailScan(claimed_sequence=claimed, unreadable_tail=unreadable_tail)
+
+
+def truncate_torn_tail(fd: int, scan: RecoveryScan) -> int:
+    """Truncate the torn tail described by *scan* through *fd*; return bytes removed."""
+    if scan.truncated_bytes <= 0:
+        return 0
+    os.ftruncate(fd, scan.complete_offset)
+    return scan.truncated_bytes

chrys/foundation/trajectory/revisions.py

@@ -0,0 +1,152 @@
+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+"""Context-revision chains: exact request membership as checkpoints and deltas.
+
+Every wire request's context is a *revision* of the actor's previous one.
+A revision records which items (by ``analytics_item_id``) the request
+carried, in order and with repeats, so the analysis layer can replay the
+exact context of any exchange without re-reading ``session.json``. The
+first revision of a chain and every ``CHECKPOINT_INTERVAL``-th one is a
+full checkpoint; the rest carry only the entries added and removed since
+their parent. A keyed ``membership_hash`` over the full ordered membership
+rides on every revision so a reconstructed chain can be verified.
+
+Entries are ``{item_id, occurrence, position, action}``: ``occurrence`` is
+the 0-based repeat index of that item within the request (so an item sent
+twice is two distinct entries) and ``position`` its 0-based index in the
+request; both are required for exact membership.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any, Final
+
+from chrys.foundation.trajectory.fingerprint import DOMAIN_MEMBERSHIP, canonical_json_bytes, keyed_fingerprint
+from chrys.foundation.trajectory.ids import new_analytics_id
+
+CHECKPOINT_INTERVAL: Final = 16
+"""Every this many revisions of a chain is a full checkpoint (the first always is)."""
+
+ACTION_ADD: Final = "add"
+ACTION_REMOVE: Final = "remove"
+
+MembershipRef = tuple[str, int]
+"""``(item_id, occurrence)`` — one slot of a request's ordered membership."""
+
+
+def membership_of(item_ids: Sequence[str]) -> tuple[MembershipRef, ...]:
+    """Number the repeats of each item id in request order."""
+    seen: dict[str, int] = {}
+    refs: list[MembershipRef] = []
+    for item_id in item_ids:
+        occurrence = seen.get(item_id, 0)
+        seen[item_id] = occurrence + 1
+        refs.append((item_id, occurrence))
+    return tuple(refs)
+
+
+def membership_hash(key: bytes | None, membership: Sequence[MembershipRef]) -> str | None:
+    """Keyed digest of the full ordered membership; ``None`` before a key is loaded."""
+    if key is None:
+        return None
+    return keyed_fingerprint(key, DOMAIN_MEMBERSHIP, canonical_json_bytes([list(ref) for ref in membership]))
+
+
+def _entry(ref: MembershipRef, *, position: int, action: str) -> dict[str, Any]:
+    return {"item_id": ref[0], "occurrence": ref[1], "position": position, "action": action}
+
+
+@dataclass(frozen=True, slots=True)
+class RevisionPlan:
+    """One computed revision, not yet committed to its chain."""
+
+    revision_id: str
+    parent_revision_id: str | None
+    is_checkpoint: bool
+    membership: tuple[MembershipRef, ...]
+    entries: tuple[dict[str, Any], ...]
+    """Checkpoint: every slot as ``add``; delta: removed slots (old positions) then added slots (new positions)."""
+
+    @property
+    def item_count(self) -> int:
+        return len(self.membership)
+
+
+@dataclass(slots=True)
+class RevisionChain:
+    """The revision lineage of one actor within one runtime."""
+
+    last_revision_id: str | None = None
+    membership: tuple[MembershipRef, ...] = ()
+    revisions_since_checkpoint: int = 0
+
+    def plan(self, membership: Sequence[MembershipRef]) -> RevisionPlan:
+        """Compute the next revision for *membership* (a checkpoint when due or cheaper)."""
+        new_membership = tuple(membership)
+        revision_id = new_analytics_id()
+        parent = self.last_revision_id
+        checkpoint_entries = tuple(_entry(ref, position=i, action=ACTION_ADD) for i, ref in enumerate(new_membership))
+        if parent is None or self.revisions_since_checkpoint + 1 >= CHECKPOINT_INTERVAL:
+            return RevisionPlan(revision_id, parent, True, new_membership, checkpoint_entries)
+        old_slots = set(self.membership)
+        new_slots = set(new_membership)
+        if tuple(ref for ref in self.membership if ref in new_slots) != tuple(
+            ref for ref in new_membership if ref in old_slots
+        ):
+            # A delta is replayed as "drop these old positions, insert these
+            # new ones", which reproduces the request only when the slots that
+            # survived kept their relative order. Nothing in the entry shape
+            # can say "these two swapped", so a reorder takes a checkpoint.
+            return RevisionPlan(revision_id, parent, True, new_membership, checkpoint_entries)
+        removed = tuple(
+            _entry(ref, position=i, action=ACTION_REMOVE)
+            for i, ref in enumerate(self.membership)
+            if ref not in new_slots
+        )
+        added = tuple(
+            _entry(ref, position=i, action=ACTION_ADD) for i, ref in enumerate(new_membership) if ref not in old_slots
+        )
+        delta = removed + added
+        if len(delta) >= len(checkpoint_entries):
+            # A delta no smaller than the snapshot buys nothing; checkpoint instead.
+            return RevisionPlan(revision_id, parent, True, new_membership, checkpoint_entries)
+        return RevisionPlan(revision_id, parent, False, new_membership, delta)
+
+    def commit(self, plan: RevisionPlan) -> None:
+        """Advance the chain to *plan* (call only once its events were queued)."""
+        self.last_revision_id = plan.revision_id
+        self.membership = plan.membership
+        self.revisions_since_checkpoint = 0 if plan.is_checkpoint else self.revisions_since_checkpoint + 1
+
+
+@dataclass(slots=True)
+class RevisionRegistry:
+    """Revision chains per actor, shared by every context derived from one root."""
+
+    _chains: dict[str, RevisionChain] = field(default_factory=dict)
+
+    def chain(self, actor_id: str) -> RevisionChain:
+        chain = self._chains.get(actor_id)
+        if chain is None:
+            chain = RevisionChain()
+            self._chains[actor_id] = chain
+        return chain
+
+    def forget(self, actor_id: str) -> None:
+        """Drop an actor's chain (its next revision starts a fresh checkpoint)."""
+        self._chains.pop(actor_id, None)
+
+
+__all__ = [
+    "ACTION_ADD",
+    "ACTION_REMOVE",
+    "CHECKPOINT_INTERVAL",
+    "MembershipRef",
+    "RevisionChain",
+    "RevisionPlan",
+    "RevisionRegistry",
+    "membership_hash",
+    "membership_of",
+]

chrys/foundation/trajectory/segments.py

@@ -0,0 +1,225 @@
+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+"""Segmentation of unbounded event fields into ``event.segment`` lines.
+
+Fields that can grow without bound (context membership refs, large provider
+usage blobs, compaction group lists) never degrade to a hash: the base event
+declares ``segmented_fields[]`` and the field's content continues in one or
+more ``event.segment`` events that each fit the line budget. Two encodings:
+
+* ``array_slice`` — ``entries[]`` is an ordered slice of the original array;
+  concatenating segments by ``segment_index`` restores it.
+* ``object_entries`` — ``entries[]`` holds ``{key, value}`` pairs; a duplicate
+  key within one group is corruption, so writers de-duplicate before packing.
+
+An entry that still exceeds the budget alone is marked ``entry_oversized`` and
+its value is replaced by an ``{omitted, byte_length, value_hash}`` sentinel
+(keyed HMAC) — the only sanctioned "segmented yet degraded" path. Without a
+fingerprint key the sentinel keeps the length and drops the hash: an unkeyed
+digest of a short value is guessable, and identifying the value is not worth
+handing out a way to recover it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Final
+
+from chrys.foundation.trajectory.envelope import LINE_BUDGET_BYTES, SegmentedField, encode_json_value
+from chrys.foundation.trajectory.ids import new_analytics_id
+
+ENCODING_ARRAY_SLICE: Final = "array_slice"
+ENCODING_OBJECT_ENTRIES: Final = "object_entries"
+
+_INDEX_PLACEHOLDER: Final = 10**9
+
+
+class SegmentationError(ValueError):
+    """Raised when a field cannot be segmented under the contract."""
+
+
+@dataclass(frozen=True, slots=True)
+class SegmentPlan:
+    """Result of segmenting one field: its declaration plus segment payloads."""
+
+    declaration: SegmentedField
+    encoding: str
+    segment_payloads: tuple[dict[str, Any], ...]
+    oversized_entry_count: int
+
+    @property
+    def segment_count(self) -> int:
+        return len(self.segment_payloads)
+
+
+LineMeasure = Callable[[Mapping[str, Any]], int]
+"""Return the encoded line length of a full event carrying *payload*."""
+
+ValueHasher = Callable[[bytes], str]
+"""Return the keyed fingerprint of an omitted value's canonical bytes."""
+
+
+def _entry_bytes(entry: Any) -> bytes:
+    return encode_json_value(entry)
+
+
+def _segment_payload(
+    *,
+    segment_group_id: str,
+    parent_event_id: str,
+    field_pointer: str,
+    encoding: str,
+    segment_index: int,
+    segment_count: int,
+    entries: Sequence[Any],
+    entry_oversized: bool,
+) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "segment_group_id": segment_group_id,
+        "parent_event_id": parent_event_id,
+        "field_pointer": field_pointer,
+        "segment_index": segment_index,
+        "segment_count": segment_count,
+        "encoding": encoding,
+        "entries": list(entries),
+    }
+    if entry_oversized:
+        payload["entry_oversized"] = True
+    return payload
+
+
+def _oversized_sentinel(raw: bytes, hasher: ValueHasher | None) -> dict[str, Any]:
+    sentinel: dict[str, Any] = {"omitted": True, "byte_length": len(raw)}
+    if hasher is not None:
+        sentinel["value_hash"] = hasher(raw)
+    return sentinel
+
+
+def plan_segments(
+    *,
+    parent_event_id: str,
+    field_pointer: str,
+    encoding: str,
+    entries: Sequence[Any],
+    measure: LineMeasure,
+    hasher: ValueHasher | None,
+    budget: int = LINE_BUDGET_BYTES,
+) -> SegmentPlan:
+    """Pack *entries* into segment payloads that each encode under *budget* bytes.
+
+    For ``object_entries`` every entry must already be a ``{key, value}`` mapping
+    with string keys; duplicate keys raise :class:`SegmentationError` because the
+    reader treats them as corruption rather than merging.
+    """
+    if encoding not in {ENCODING_ARRAY_SLICE, ENCODING_OBJECT_ENTRIES}:
+        raise SegmentationError(f"Unknown segment encoding: {encoding!r}")
+    if encoding == ENCODING_OBJECT_ENTRIES:
+        seen: set[str] = set()
+        for entry in entries:
+            if not isinstance(entry, Mapping) or set(entry) != {"key", "value"} or not isinstance(entry["key"], str):
+                raise SegmentationError("object_entries segments require {key, value} entries with string keys.")
+            if entry["key"] in seen:
+                raise SegmentationError(f"Duplicate object_entries key before segmentation: {entry['key']!r}")
+            seen.add(entry["key"])
+    segment_group_id = new_analytics_id()
+    overhead = measure(
+        _segment_payload(
+            segment_group_id=segment_group_id,
+            parent_event_id=parent_event_id,
+            field_pointer=field_pointer,
+            encoding=encoding,
+            segment_index=_INDEX_PLACEHOLDER,
+            segment_count=_INDEX_PLACEHOLDER,
+            entries=(),
+            entry_oversized=True,
+        )
+    )
+    if overhead >= budget:
+        raise SegmentationError("Segment envelope alone exceeds the line budget.")
+    capacity = budget - overhead
+
+    groups: list[tuple[list[Any], bool]] = []
+    current: list[Any] = []
+    current_size = 0
+    oversized_count = 0
+    for entry in entries:
+        raw = _entry_bytes(entry)
+        cost = len(raw) + (1 if current else 0)
+        if len(raw) > capacity:
+            if current:
+                groups.append((current, False))
+                current, current_size = [], 0
+            if encoding == ENCODING_OBJECT_ENTRIES:
+                value_raw = _entry_bytes(entry["value"])
+                sentinel: Any = {"key": entry["key"], "value": _oversized_sentinel(value_raw, hasher)}
+            else:
+                sentinel = _oversized_sentinel(raw, hasher)
+            if len(_entry_bytes(sentinel)) > capacity:
+                raise SegmentationError("Oversized-entry sentinel does not fit the line budget.")
+            groups.append(([sentinel], True))
+            oversized_count += 1
+            continue
+        if current_size + cost > capacity:
+            groups.append((current, False))
+            current, current_size = [], 0
+            cost = len(raw)
+        current.append(entry)
+        current_size += cost
+    if current or not groups:
+        groups.append((current, False))
+
+    segment_count = len(groups)
+    payloads = tuple(
+        _segment_payload(
+            segment_group_id=segment_group_id,
+            parent_event_id=parent_event_id,
+            field_pointer=field_pointer,
+            encoding=encoding,
+            segment_index=index,
+            segment_count=segment_count,
+            entries=group,
+            entry_oversized=oversized,
+        )
+        for index, (group, oversized) in enumerate(groups)
+    )
+    for payload in payloads:
+        if measure(payload) > budget:  # pragma: no cover - defensive: sizing above is exact
+            raise SegmentationError("Segment packing exceeded the line budget.")
+    declaration = SegmentedField(
+        field_pointer=field_pointer,
+        segment_group_id=segment_group_id,
+        segment_count=segment_count,
+    )
+    return SegmentPlan(
+        declaration=declaration,
+        encoding=encoding,
+        segment_payloads=payloads,
+        oversized_entry_count=oversized_count,
+    )
+
+
+def object_entries(mapping: Mapping[str, Any]) -> list[dict[str, Any]]:
+    """Convert a mapping into ``object_entries`` form (insertion order kept)."""
+    return [{"key": key, "value": value} for key, value in mapping.items()]
+
+
+def reassemble_array_slice(segment_payloads: Sequence[Mapping[str, Any]]) -> list[Any]:
+    """Concatenate ``array_slice`` segments in ``segment_index`` order."""
+    ordered = sorted(segment_payloads, key=lambda payload: int(payload["segment_index"]))
+    result: list[Any] = []
+    for payload in ordered:
+        result.extend(payload["entries"])
+    return result
+
+
+def reassemble_object_entries(segment_payloads: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+    """Merge ``object_entries`` segments; duplicate keys raise (corruption)."""
+    result: dict[str, Any] = {}
+    for payload in sorted(segment_payloads, key=lambda payload: int(payload["segment_index"])):
+        for entry in payload["entries"]:
+            key = entry["key"]
+            if key in result:
+                raise SegmentationError(f"Duplicate object_entries key across segments: {key!r}")
+            result[key] = entry["value"]
+    return result