bdo-toolkit 1.0.0__py3-none-any.whl

bdo_toolkit/_specs.py

@@ -0,0 +1,285 @@
+"""Turn opcode profile JSON entries into decodable event specs."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Iterable, Optional
+
+from ._protocol import MAX_TARGET_MESSAGE_LENGTH, EventSpec
+from .profiles import OpcodeProfile, ProfileError
+
+
+@dataclass(frozen=True)
+class LoadedSpecProfile:
+    active: bool
+    specs: tuple[EventSpec, ...]
+    source: str
+
+
+def event_specs_from_profile(profile: OpcodeProfile) -> LoadedSpecProfile:
+    """Convert one validated profile object into decoder event specs."""
+    if not profile.active:
+        return LoadedSpecProfile(active=False, specs=(), source=str(profile.path))
+
+    specs: list[EventSpec] = []
+    decodable_events = {"LOOT_PREVIEW", "INVENTORY_TRANSFER", "STORAGE_ITEM_DELTA"}
+    for event, entries in profile.specs.items():
+        for entry in entries:
+            try:
+                spec = _event_spec_from_entry(event, entry)
+            except ValueError as exc:
+                raise ProfileError(
+                    f"Invalid {event} spec in {profile.path}: {exc}"
+                ) from exc
+            if spec is not None:
+                specs.append(spec)
+            elif event in decodable_events:
+                raise ProfileError(
+                    f"Invalid {event} spec in {profile.path}: "
+                    "missing or invalid required fields"
+                )
+
+    normalized = tuple(_dedupe_event_specs(specs))
+    _validate_unambiguous_loot_specs(normalized, source=profile.path)
+    return LoadedSpecProfile(
+        active=True,
+        specs=normalized,
+        source=str(profile.path),
+    )
+
+
+def _validate_loot_profile_entries(
+    entries: Iterable[Mapping[str, object]],
+    *,
+    source: object,
+) -> None:
+    """Validate only the runtime ambiguity introduced by raw LOOT entries."""
+
+    specs: list[EventSpec] = []
+    for entry in entries:
+        try:
+            spec = _event_spec_from_entry("LOOT_PREVIEW", entry)
+        except ValueError as exc:
+            raise ProfileError(
+                f"Invalid LOOT_PREVIEW spec in {source}: {exc}"
+            ) from exc
+        if spec is None:
+            raise ProfileError(
+                f"Invalid LOOT_PREVIEW spec in {source}: "
+                "missing or invalid required fields"
+            )
+        specs.append(spec)
+
+    _validate_unambiguous_loot_specs(
+        _dedupe_event_specs(specs),
+        source=source,
+    )
+
+
+def _validate_unambiguous_loot_specs(
+    specs: Iterable[EventSpec],
+    *,
+    source: object,
+) -> None:
+    """Require one non-overlapping runtime length domain per LOOT opcode."""
+
+    grouped: dict[int, list[EventSpec]] = {}
+    for spec in specs:
+        if spec.label != "LOOT_PREVIEW":
+            continue
+        grouped.setdefault(spec.opcode, []).append(spec)
+
+    for opcode, candidates in grouped.items():
+        for index, first in enumerate(candidates):
+            first_low, first_high = _loot_message_length_domain(first)
+            for second in candidates[index + 1 :]:
+                second_low, second_high = _loot_message_length_domain(second)
+                if max(first_low, second_low) > min(first_high, second_high):
+                    continue
+                raise ProfileError(
+                    f"Ambiguous LOOT_PREVIEW specs in {source}: opcode "
+                    f"0x{opcode:04X} has distinct layouts with overlapping "
+                    "runtime message-length domains: "
+                    f"{_loot_layout_description(first)} and "
+                    f"{_loot_layout_description(second)}. Keep only the layout for "
+                    "the captured game patch, or recalibrate and replace the "
+                    "LOOT_PREVIEW family instead of merging it."
+                )
+
+
+def _loot_message_length_domain(spec: EventSpec) -> tuple[int, int]:
+    exact_length = spec.single_record_message_length
+    if exact_length is not None:
+        return exact_length, exact_length
+    return spec.min_message_length, MAX_TARGET_MESSAGE_LENGTH
+
+
+def _loot_layout_description(spec: EventSpec) -> str:
+    exact_length = spec.single_record_message_length
+    length = (
+        str(exact_length)
+        if exact_length is not None
+        else f"{spec.min_message_length}..{MAX_TARGET_MESSAGE_LENGTH}"
+    )
+    return (
+        f"(length={length}, item_id_offset={spec.item_offset}, "
+        f"quantity_offset={spec.quantity_offset}, "
+        f"item_instance_offset={spec.item_instance_offset!r})"
+    )
+
+
+def _event_spec_from_entry(
+    event: str,
+    entry: Mapping[str, object],
+) -> Optional[EventSpec]:
+    opcode = _parse_opcode(entry.get("opcode"))
+    if opcode is None:
+        return None
+
+    length = _optional_int(entry.get("length"))
+    item_id_offset = _optional_int(entry.get("item_id_offset"))
+    quantity_offset = _optional_int(entry.get("quantity_offset"))
+
+    if event == "LOOT_PREVIEW":
+        if item_id_offset is None or quantity_offset is None:
+            return None
+        item_instance_offset = _optional_int(entry.get("item_instance_offset"))
+        return EventSpec(
+            label="LOOT_PREVIEW",
+            opcode=opcode,
+            item_offset=item_id_offset,
+            quantity_offset=quantity_offset,
+            min_message_length=_minimum_event_length(
+                length,
+                item_id_offset + 4,
+                quantity_offset + 4,
+                (
+                    item_instance_offset + 8
+                    if item_instance_offset is not None
+                    else None
+                ),
+            ),
+            item_instance_offset=item_instance_offset,
+            single_record_message_length=length,
+            default_context="Gathering",
+        )
+
+    if event == "INVENTORY_TRANSFER":
+        if item_id_offset is None or quantity_offset is None:
+            return None
+        inventory_slot_offset = _optional_int(entry.get("inventory_slot_offset"))
+        context_offset = _optional_int(entry.get("context_offset"))
+        item_instance_offset = _optional_int(entry.get("item_instance_offset"))
+        repeat_stride = _optional_int(entry.get("repeat_stride"))
+        return EventSpec(
+            label="INVENTORY_TRANSFER",
+            opcode=opcode,
+            item_offset=item_id_offset,
+            quantity_offset=quantity_offset,
+            min_message_length=_minimum_event_length(
+                length,
+                item_id_offset + 4,
+                quantity_offset + 4,
+                item_instance_offset + 8 if item_instance_offset is not None else None,
+                context_offset + 4 if context_offset is not None else None,
+                inventory_slot_offset + 1 if inventory_slot_offset is not None else None,
+            ),
+            inventory_slot_offset=inventory_slot_offset,
+            source_context_offset=context_offset,
+            item_instance_offset=item_instance_offset,
+            repeat_stride=repeat_stride,
+            # Preserve the calibrated base length even when a single-record
+            # capture could not reveal a repeat stride. Runtime structural
+            # discovery uses it to validate later multi-record messages.
+            single_record_message_length=length,
+        )
+
+    if event == "STORAGE_ITEM_DELTA":
+        quantity_added_offset = _optional_int(entry.get("quantity_added_offset"))
+        destination_instance_offset = _optional_int(
+            entry.get("destination_instance_offset")
+        )
+        if item_id_offset is None or quantity_added_offset is None:
+            return None
+        context_offset = _optional_int(entry.get("context_offset"))
+        record_count_offset = _optional_int(entry.get("record_count_offset"))
+        repeat_stride = _optional_int(entry.get("repeat_stride"))
+        return EventSpec(
+            label="INVENTORY_TO_STORAGE",
+            opcode=opcode,
+            item_offset=item_id_offset,
+            quantity_offset=quantity_added_offset,
+            min_message_length=_minimum_event_length(
+                length,
+                item_id_offset + 4,
+                quantity_added_offset + 4,
+                (
+                    destination_instance_offset + 8
+                    if destination_instance_offset is not None
+                    else None
+                ),
+                context_offset + 4 if context_offset is not None else None,
+                (
+                    record_count_offset + 2
+                    if record_count_offset is not None
+                    else None
+                ),
+            ),
+            source_context_offset=context_offset,
+            record_count_offset=record_count_offset,
+            storage_instance_offset=destination_instance_offset,
+            repeat_stride=repeat_stride,
+            single_record_message_length=length,
+            default_context="Storage",
+        )
+
+    return None
+
+
+def _parse_opcode(value: object) -> Optional[int]:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        opcode = value
+    elif isinstance(value, str):
+        try:
+            opcode = int(value, 16 if value.lower().startswith("0x") else 10)
+        except ValueError:
+            return None
+    else:
+        return None
+
+    if not 0 <= opcode <= 0xFFFF:
+        return None
+    return opcode
+
+
+def _optional_int(value: object) -> Optional[int]:
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+        raise ValueError(f"expected a non-negative integer, got {value!r}")
+    return value
+
+
+def _minimum_event_length(length: Optional[int], *ends: Optional[int]) -> int:
+    minimum = max((end for end in ends if end is not None), default=5)
+    if length is not None and length > 0:
+        minimum = max(minimum, length)
+    return minimum
+
+
+def _dedupe_event_specs(specs: Iterable[EventSpec]) -> list[EventSpec]:
+    output: list[EventSpec] = []
+    # EventSpec is a frozen value object whose fields are all decode-affecting.
+    # Using the complete value as identity preserves same-opcode layouts that
+    # differ in base length, instance offsets, repeat geometry, context width,
+    # or fallback context while still removing literal duplicates.
+    seen: set[EventSpec] = set()
+    for spec in specs:
+        if spec in seen:
+            continue
+        seen.add(spec)
+        output.append(spec)
+    return output

bdo_toolkit/_storage_destination_validation.py

@@ -0,0 +1,167 @@
+"""Conservative runtime revalidation for calibrated storage destinations.
+
+Calibration remains authoritative for event decoding.  This module only looks
+for strong cross-wrapper evidence that the calibrated destination column has
+gone stale; it never selects a replacement column or changes an event.
+"""
+
+from __future__ import annotations
+
+from collections import OrderedDict, defaultdict
+from dataclasses import dataclass, field
+from hashlib import blake2s
+from typing import Optional
+
+from ._protocol import FlowKey, storage_destination_candidates
+
+
+# Four distinct wrappers prevent a short transaction pair from becoming a
+# schema verdict.  Three distinct registered destination values distinguish a
+# real column from the common ``0x05xx -> 0x0005`` one-byte overlap.  A proven
+# column must occur in every candidate-bearing wrapper; there is no majority
+# guess and ambiguous evidence remains silent.
+_MIN_PROOF_WRAPPERS = 4
+_MIN_PROOF_DESTINATIONS = 3
+
+# One character-load hydration currently needs fewer than 80 distinct storage
+# wrappers.  These limits retain a complete observed cohort with headroom while
+# bounding adversarial/reconnect state to small fixed metadata (not packets).
+_MAX_COHORTS = 64
+_MAX_WRAPPERS_PER_COHORT = 128
+
+
+@dataclass(frozen=True)
+class StorageDestinationSchemaMismatch:
+    """Strong evidence that a profile's destination column is stale."""
+
+    opcode: int
+    configured_offset: int
+    observed_offset: int
+    wrapper_count: int
+    distinct_destinations: int
+
+
+@dataclass
+class _DestinationCohort:
+    wrappers: OrderedDict[bytes, tuple[tuple[int, int], ...]] = field(
+        default_factory=OrderedDict
+    )
+    mismatch_reported: bool = False
+
+
+class StorageDestinationValidator:
+    """Revalidate one destination column across bounded connection cohorts.
+
+    Evidence is isolated by TCP four-tuple generation and opcode.  Identical
+    wrappers are counted once, and only registered destination candidates
+    before the first item record participate.  Unknown-town-only wrappers are
+    retained for deduplication but do not vote for any column.
+    """
+
+    def __init__(self) -> None:
+        self._cohorts: OrderedDict[
+            tuple[FlowKey, int, int, int, int], _DestinationCohort
+        ] = OrderedDict()
+
+    def observe(
+        self,
+        *,
+        flow: FlowKey,
+        flow_generation: int,
+        opcode: int,
+        message: bytes,
+        first_item_offset: int,
+        configured_offset: Optional[int],
+    ) -> Optional[StorageDestinationSchemaMismatch]:
+        """Return a mismatch only after one alternate column is proven."""
+
+        if configured_offset is None or first_item_offset <= 5:
+            return None
+
+        # One opcode may legitimately select more than one structural layout.
+        # Destination evidence is comparable only when both the calibrated
+        # destination column and the first-record boundary are the same.
+        # Without this identity, a correct second layout can inherit the
+        # first layout's candidate column and be diagnosed as stale.
+        key = (
+            flow,
+            flow_generation,
+            opcode,
+            first_item_offset,
+            configured_offset,
+        )
+        cohort = self._cohorts.pop(key, None)
+        if cohort is None:
+            cohort = _DestinationCohort()
+        self._cohorts[key] = cohort
+        while len(self._cohorts) > _MAX_COHORTS:
+            self._cohorts.popitem(last=False)
+
+        # A compact cryptographic fingerprint avoids retaining private packet
+        # contents merely to suppress the per-record callbacks of one wrapper.
+        fingerprint = blake2s(message, digest_size=16).digest()
+        if fingerprint in cohort.wrappers:
+            return None
+
+        cohort.wrappers[fingerprint] = storage_destination_candidates(
+            message,
+            before_offset=first_item_offset,
+        )
+        while len(cohort.wrappers) > _MAX_WRAPPERS_PER_COHORT:
+            cohort.wrappers.popitem(last=False)
+
+        if cohort.mismatch_reported:
+            return None
+
+        candidate_wrappers = tuple(
+            candidates for candidates in cohort.wrappers.values() if candidates
+        )
+        if len(candidate_wrappers) < _MIN_PROOF_WRAPPERS:
+            return None
+
+        support: dict[int, int] = defaultdict(int)
+        destination_ids: dict[int, set[int]] = defaultdict(set)
+        for candidates in candidate_wrappers:
+            for offset, storage_id in candidates:
+                support[offset] += 1
+                destination_ids[offset].add(storage_id)
+
+        proven_offsets = tuple(
+            offset
+            for offset, count in support.items()
+            if count == len(candidate_wrappers)
+            and len(destination_ids[offset]) >= _MIN_PROOF_DESTINATIONS
+        )
+        if len(proven_offsets) != 1:
+            return None
+        observed_offset = proven_offsets[0]
+        if observed_offset == configured_offset:
+            return None
+
+        cohort.mismatch_reported = True
+        return StorageDestinationSchemaMismatch(
+            opcode=opcode,
+            configured_offset=configured_offset,
+            observed_offset=observed_offset,
+            wrapper_count=len(candidate_wrappers),
+            distinct_destinations=len(destination_ids[observed_offset]),
+        )
+
+    def close_flow(self, flow: FlowKey) -> None:
+        """Discard every generation retained for a closed TCP four-tuple."""
+
+        for key in tuple(self._cohorts):
+            if key[0] == flow:
+                del self._cohorts[key]
+
+    @property
+    def cohort_count(self) -> int:
+        """Number of retained flow-generation/opcode cohorts (for health tests)."""
+
+        return len(self._cohorts)
+
+    @property
+    def retained_wrapper_count(self) -> int:
+        """Number of compact wrapper fingerprints currently retained."""
+
+        return sum(len(cohort.wrappers) for cohort in self._cohorts.values())

bdo_toolkit/_storage_hydration.py

@@ -0,0 +1,241 @@
+"""Stateful, offset-free storage hydration classification."""
+
+from __future__ import annotations
+
+import dataclasses
+from collections import deque
+from dataclasses import dataclass, field
+from threading import RLock
+from typing import Callable, Optional
+
+from .events import BDOEvent, Flow
+
+
+_HydrationKey = tuple[Flow, int]
+
+
+def _hydration_key(event: BDOEvent) -> _HydrationKey:
+    return event.flow, event._flow_generation
+
+
+@dataclass
+class _HydrationBurst:
+    first_timestamp: float
+    last_timestamp: float
+    anchored: bool
+    opcode: Optional[int]
+    events: list[BDOEvent] = field(default_factory=list)
+    destinations: set[int] = field(default_factory=set)
+    max_record_count: int = 0
+    confirmed: bool = False
+
+
+class StorageHydrationTracker:
+    """Promote neutral storage cohorts to snapshots before app filtering.
+
+    Inventory hydration opens a bounded epoch on the same flow.  Neutral
+    storage wrappers are grouped by wire timestamp after manual/worker origin
+    evidence has had first refusal.  A multi-destination cohort proves state
+    hydration without consulting any patch-specific mode or token byte.
+
+    A broader unanchored threshold retains storage-only snapshot evidence when
+    inventory framing was missed.  Isolated unresolved records remain neutral
+    and therefore never enter ordinary activity filters.
+    """
+
+    BURST_GAP_SECONDS = 0.5
+    MAX_BURST_SECONDS = 1.0
+    HYDRATION_EPOCH_SECONDS = 30.0
+    INVENTORY_GENERATION_GAP_SECONDS = 1.0
+    MIN_ANCHORED_DESTINATIONS = 8
+    MIN_UNANCHORED_DESTINATIONS = 17
+    MIN_UNANCHORED_RECORD_COUNT = 8
+    MAX_PENDING_EVENTS = 10_000
+
+    def __init__(self, emit: Callable[[BDOEvent], object]) -> None:
+        self._emit = emit
+        self._lock = RLock()
+        self._inventory_anchor: dict[_HydrationKey, float] = {}
+        self._inventory_last: dict[_HydrationKey, float] = {}
+        self._bursts: dict[_HydrationKey, _HydrationBurst] = {}
+        self._outbox: deque[BDOEvent] = deque()
+        self._dispatching = False
+
+    def observe_inventory(self, event: BDOEvent) -> None:
+        """Record a proven inventory-hydration boundary and emit the event."""
+        key = _hydration_key(event)
+        with self._lock:
+            last = self._inventory_last.get(key)
+            if (
+                last is None
+                or event.timestamp - last > self.INVENTORY_GENERATION_GAP_SECONDS
+            ):
+                self._flush_key_locked(key)
+                self._inventory_anchor[key] = event.timestamp
+            self._inventory_last[key] = event.timestamp
+            self._queue_locked(event)
+        self._drain_outbox()
+
+    def observe_storage(self, event: BDOEvent) -> None:
+        """Stage one neutral record or pass through an independently live one."""
+        key = _hydration_key(event)
+        if event.event_type != "storage_record":
+            with self._lock:
+                # Positive mutation evidence is an explicit semantic boundary.
+                # It must not inherit a prior confirmed hydration burst, and a
+                # subsequent unknown record must prove a new epoch. Origin
+                # lookahead can deliver an older live event after a newer
+                # hydration anchor, so only retire state at or before this
+                # event's own wire timestamp.
+                burst = self._bursts.get(key)
+                if burst is None or event.timestamp >= burst.first_timestamp:
+                    self._flush_key_locked(key)
+                anchor = self._inventory_anchor.get(key)
+                if anchor is None or event.timestamp >= anchor:
+                    self._inventory_anchor.pop(key, None)
+                    self._inventory_last.pop(key, None)
+                self._queue_locked(event)
+            self._drain_outbox()
+            return
+
+        with self._lock:
+            anchor = self._inventory_anchor.get(key)
+            anchored = (
+                anchor is not None
+                and anchor <= event.timestamp
+                and event.timestamp - anchor <= self.HYDRATION_EPOCH_SECONDS
+            )
+            burst = self._bursts.get(key)
+            if (
+                burst is None
+                or event.timestamp - burst.last_timestamp > self.BURST_GAP_SECONDS
+                or event.timestamp - burst.first_timestamp > self.MAX_BURST_SECONDS
+                or event.timestamp < burst.last_timestamp
+                or burst.anchored != anchored
+                or burst.opcode != event.opcode
+            ):
+                self._flush_key_locked(key)
+                burst = _HydrationBurst(
+                    first_timestamp=event.timestamp,
+                    last_timestamp=event.timestamp,
+                    anchored=anchored,
+                    opcode=event.opcode,
+                )
+                self._bursts[key] = burst
+
+            burst.last_timestamp = event.timestamp
+            if event.storage_id is not None:
+                burst.destinations.add(event.storage_id)
+            if isinstance(event.record_count, int) and not isinstance(
+                event.record_count, bool
+            ):
+                burst.max_record_count = max(
+                    burst.max_record_count,
+                    event.record_count,
+                )
+
+            if burst.confirmed:
+                self._queue_locked(self._as_snapshot(event, burst))
+            else:
+                burst.events.append(event)
+                enough_destinations = len(burst.destinations) >= (
+                    self.MIN_ANCHORED_DESTINATIONS
+                    if anchored
+                    else self.MIN_UNANCHORED_DESTINATIONS
+                )
+                credible_unanchored = (
+                    anchored
+                    or burst.max_record_count >= self.MIN_UNANCHORED_RECORD_COUNT
+                )
+                if enough_destinations and credible_unanchored:
+                    burst.confirmed = True
+                    for pending in burst.events:
+                        self._queue_locked(self._as_snapshot(pending, burst))
+                    burst.events.clear()
+                elif len(burst.events) > self.MAX_PENDING_EVENTS:
+                    # Bounded fail-neutral behavior: an unproven cohort must not
+                    # consume unbounded memory or become activity by default.
+                    self._flush_key_locked(key)
+        self._drain_outbox()
+
+    def flush_stale(self, now: float) -> None:
+        with self._lock:
+            stale = tuple(
+                key
+                for key, burst in self._bursts.items()
+                if now - burst.last_timestamp > self.BURST_GAP_SECONDS
+            )
+            for key in stale:
+                self._flush_key_locked(key)
+            expired_anchors = tuple(
+                key
+                for key, anchor in self._inventory_anchor.items()
+                if now - anchor > self.HYDRATION_EPOCH_SECONDS
+            )
+            for key in expired_anchors:
+                self._inventory_anchor.pop(key, None)
+                self._inventory_last.pop(key, None)
+        self._drain_outbox()
+
+    def close_flow(self, flow: Flow) -> None:
+        with self._lock:
+            keys = {
+                key
+                for key in (
+                    set(self._bursts)
+                    | set(self._inventory_anchor)
+                    | set(self._inventory_last)
+                )
+                if key[0] == flow
+            }
+            for key in keys:
+                self._flush_key_locked(key)
+                self._inventory_anchor.pop(key, None)
+                self._inventory_last.pop(key, None)
+        self._drain_outbox()
+
+    def finalize_all(self) -> None:
+        with self._lock:
+            for key in tuple(self._bursts):
+                self._flush_key_locked(key)
+            self._inventory_anchor.clear()
+            self._inventory_last.clear()
+        self._drain_outbox()
+
+    def _flush_key_locked(self, key: _HydrationKey) -> None:
+        burst = self._bursts.pop(key, None)
+        if burst is None:
+            return
+        # Confirmed events were emitted as they arrived. Only an unconfirmed
+        # burst retains entries, and those remain explicitly neutral.
+        for event in burst.events:
+            self._queue_locked(event)
+
+    @staticmethod
+    def _as_snapshot(event: BDOEvent, burst: _HydrationBurst) -> BDOEvent:
+        return dataclasses.replace(
+            event,
+            event_type="storage_snapshot",
+            source=None,
+        )
+
+    def _queue_locked(self, event: BDOEvent) -> None:
+        self._outbox.append(event)
+
+    def _drain_outbox(self) -> None:
+        with self._lock:
+            if self._dispatching:
+                return
+            self._dispatching = True
+        try:
+            while True:
+                with self._lock:
+                    if not self._outbox:
+                        self._dispatching = False
+                        return
+                    event = self._outbox.popleft()
+                self._emit(event)
+        except BaseException:
+            with self._lock:
+                self._dispatching = False
+            raise

bdo_toolkit/_version.py

@@ -0,0 +1,3 @@
+"""Single source of truth for the BDO Toolkit release version."""
+
+__version__ = "1.0.0"