bdo-toolkit 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bdo_toolkit/__init__.py +87 -0
- bdo_toolkit/_async_sessions.py +651 -0
- bdo_toolkit/_capture_backend.py +194 -0
- bdo_toolkit/_capture_options.py +68 -0
- bdo_toolkit/_capture_runtime.py +626 -0
- bdo_toolkit/_deposit_origin.py +1599 -0
- bdo_toolkit/_engine.py +327 -0
- bdo_toolkit/_framing.py +904 -0
- bdo_toolkit/_profile_runtime.py +157 -0
- bdo_toolkit/_protocol.py +386 -0
- bdo_toolkit/_reassembly.py +654 -0
- bdo_toolkit/_specs.py +285 -0
- bdo_toolkit/_storage_destination_validation.py +167 -0
- bdo_toolkit/_storage_hydration.py +241 -0
- bdo_toolkit/_version.py +3 -0
- bdo_toolkit/calibration.py +3223 -0
- bdo_toolkit/capture.py +1713 -0
- bdo_toolkit/character_state.py +3506 -0
- bdo_toolkit/cli.py +948 -0
- bdo_toolkit/diagnostics.py +51 -0
- bdo_toolkit/events.py +214 -0
- bdo_toolkit/filters.py +105 -0
- bdo_toolkit/item_state.py +48 -0
- bdo_toolkit/origin_learning.py +779 -0
- bdo_toolkit/profiles.py +370 -0
- bdo_toolkit/py.typed +1 -0
- bdo_toolkit/remote_profiles.py +358 -0
- bdo_toolkit/solare/__init__.py +50 -0
- bdo_toolkit/solare/_constants.py +94 -0
- bdo_toolkit/solare/_detail_learning.py +1437 -0
- bdo_toolkit/solare/_details.py +796 -0
- bdo_toolkit/solare/_discovery.py +1212 -0
- bdo_toolkit/solare/_live_tracker.py +472 -0
- bdo_toolkit/solare/_replay_capture.py +182 -0
- bdo_toolkit/solare/_result.py +441 -0
- bdo_toolkit/solare/_scanner.py +203 -0
- bdo_toolkit/solare/_validation.py +11 -0
- bdo_toolkit/solare/async_session.py +444 -0
- bdo_toolkit/solare/models.py +806 -0
- bdo_toolkit/solare/replay.py +62 -0
- bdo_toolkit/solare/session.py +1051 -0
- bdo_toolkit/writers.py +30 -0
- bdo_toolkit-1.0.0.dist-info/METADATA +143 -0
- bdo_toolkit-1.0.0.dist-info/RECORD +48 -0
- bdo_toolkit-1.0.0.dist-info/WHEEL +5 -0
- bdo_toolkit-1.0.0.dist-info/entry_points.txt +2 -0
- bdo_toolkit-1.0.0.dist-info/licenses/LICENSE +21 -0
- bdo_toolkit-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,3506 @@
|
|
|
1
|
+
"""Experimental character-load snapshot diagnostics and state summaries.
|
|
2
|
+
|
|
3
|
+
The framed inventory and storage hydration messages are strong enough to
|
|
4
|
+
enumerate occupied item records. Current inventory frames also expose a
|
|
5
|
+
structurally validated raw container code and slot. Their human-readable
|
|
6
|
+
container interpretations remain provisional, and the packets still do not
|
|
7
|
+
prove storage capacity or whether hydration was triggered by initial login
|
|
8
|
+
versus a character switch. This module keeps those limits explicit while
|
|
9
|
+
providing a queryable model for tools and early adopters.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
from dataclasses import dataclass, replace
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from threading import RLock
|
|
18
|
+
from typing import Any, Iterable, Optional
|
|
19
|
+
|
|
20
|
+
from ._capture_backend import (
|
|
21
|
+
iter_pcap_file,
|
|
22
|
+
make_packet_handler,
|
|
23
|
+
)
|
|
24
|
+
from ._capture_options import PacketCaptureOptions
|
|
25
|
+
from ._capture_runtime import (
|
|
26
|
+
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
|
27
|
+
LivePacketCapture,
|
|
28
|
+
_attach_cleanup_owner,
|
|
29
|
+
)
|
|
30
|
+
from .capture import _EventCollector, _ProfileAuthority, _load_profile_authority
|
|
31
|
+
from ._engine import PacketEngine, toolkit_event_from_record
|
|
32
|
+
from ._protocol import (
|
|
33
|
+
BDOFrame,
|
|
34
|
+
CHARACTER_LOAD_CONTEXT,
|
|
35
|
+
DEFAULT_SERVER_PORTS,
|
|
36
|
+
STORAGE_LOCATIONS,
|
|
37
|
+
EventSpec,
|
|
38
|
+
storage_location,
|
|
39
|
+
)
|
|
40
|
+
from .diagnostics import DecoderHealth
|
|
41
|
+
from .events import BDOEvent
|
|
42
|
+
from .filters import EventFilter
|
|
43
|
+
from .profiles import OpcodeProfile, ProfileError
|
|
44
|
+
|
|
45
|
+
_INVENTORY_GENERATION_GAP_SECONDS = 1.0
|
|
46
|
+
_INVENTORY_TRAILING_DISCOVERY_BYTES = 12
|
|
47
|
+
_STORAGE_DESTINATION_CHUNK_GAP_SECONDS = 1.0
|
|
48
|
+
_STORAGE_EMPTY_WINDOW_MARGIN_SECONDS = 1.0
|
|
49
|
+
_STORAGE_HYDRATION_BURST_GAP_SECONDS = 0.5
|
|
50
|
+
_STORAGE_HYDRATION_MAX_BURST_SECONDS = 1.0
|
|
51
|
+
_STORAGE_HYDRATION_EPOCH_SECONDS = 30.0
|
|
52
|
+
_STORAGE_HYDRATION_MIN_DESTINATIONS = 8
|
|
53
|
+
_ITEM_STATE_SCHEMA_VERSION = 5
|
|
54
|
+
_CHARACTER_LOAD_STARTUP_TIMEOUT_SECONDS = DEFAULT_STARTUP_TIMEOUT_SECONDS
|
|
55
|
+
|
|
56
|
+
# These interpretations agree across the July 17 initial-load and character-
|
|
57
|
+
# switch captures, and the 0x00/0x10/0x0B families agree with legacy research.
|
|
58
|
+
# They deliberately remain local to the experimental character-state API.
|
|
59
|
+
_INVENTORY_CONTAINER_LABELS: dict[int, tuple[str, str]] = {
|
|
60
|
+
0x00: ("Main Inventory", "provisional"),
|
|
61
|
+
0x10: ("Pearl Inventory", "provisional"),
|
|
62
|
+
0x18: ("Global Currencies", "provisional"),
|
|
63
|
+
0x0B: ("Enhancement Inventory", "provisional"),
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
_CURRENCY_NAMES: dict[tuple[int, int], str] = {
|
|
67
|
+
(0x18, 1): "Silver",
|
|
68
|
+
(0x10, 6): "Pearl",
|
|
69
|
+
(0x10, 7): "Loyalties",
|
|
70
|
+
(0x18, 10): "Crow Coin",
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class ItemStateCaptureLimits:
|
|
76
|
+
"""Hard fail-closed bounds for retained item-state observations."""
|
|
77
|
+
|
|
78
|
+
max_relevant_frames: int = 10_000
|
|
79
|
+
max_snapshot_records: int = 50_000
|
|
80
|
+
max_relevant_bytes: int = 64 * 1024 * 1024
|
|
81
|
+
|
|
82
|
+
def __post_init__(self) -> None:
|
|
83
|
+
for name, value in (
|
|
84
|
+
("max_relevant_frames", self.max_relevant_frames),
|
|
85
|
+
("max_snapshot_records", self.max_snapshot_records),
|
|
86
|
+
("max_relevant_bytes", self.max_relevant_bytes),
|
|
87
|
+
):
|
|
88
|
+
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
|
89
|
+
raise ValueError(f"{name} must be a positive integer")
|
|
90
|
+
|
|
91
|
+
def to_dict(self) -> dict[str, int]:
|
|
92
|
+
return {
|
|
93
|
+
"max_relevant_frames": self.max_relevant_frames,
|
|
94
|
+
"max_snapshot_records": self.max_snapshot_records,
|
|
95
|
+
"max_relevant_bytes": self.max_relevant_bytes,
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class ItemStateCaptureLimitError(RuntimeError):
|
|
100
|
+
"""Raised before an item-state accumulator would exceed a hard bound."""
|
|
101
|
+
|
|
102
|
+
def __init__(self, *, limit_name: str, limit: int, attempted: int) -> None:
|
|
103
|
+
self.limit_name = limit_name
|
|
104
|
+
self.limit = limit
|
|
105
|
+
self.attempted = attempted
|
|
106
|
+
super().__init__(
|
|
107
|
+
f"item-state accumulation limit exceeded: {limit_name} "
|
|
108
|
+
f"attempted={attempted} limit={limit}; no partial snapshot was returned"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass(frozen=True, kw_only=True)
|
|
113
|
+
class SnapshotItem:
|
|
114
|
+
"""One occupied item-stack record observed during state hydration."""
|
|
115
|
+
|
|
116
|
+
item_id: int
|
|
117
|
+
quantity: int
|
|
118
|
+
instance: str
|
|
119
|
+
observed_at: float
|
|
120
|
+
base_item_id: Optional[int] = None
|
|
121
|
+
enhancement_level: Optional[int] = None
|
|
122
|
+
enhancement: Optional[str] = None
|
|
123
|
+
inventory_slot: Optional[int] = None
|
|
124
|
+
container_code: Optional[int] = None
|
|
125
|
+
container_name: Optional[str] = None
|
|
126
|
+
container_confidence: Optional[str] = None
|
|
127
|
+
currency_name: Optional[str] = None
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def is_currency_balance(self) -> bool:
|
|
131
|
+
"""Whether this serialized record represents a known wallet balance."""
|
|
132
|
+
return self.currency_name is not None
|
|
133
|
+
|
|
134
|
+
def to_dict(self) -> dict[str, object]:
|
|
135
|
+
output: dict[str, object] = {
|
|
136
|
+
"item_id": self.item_id,
|
|
137
|
+
"quantity": self.quantity,
|
|
138
|
+
"instance": self.instance,
|
|
139
|
+
"observed_at": self.observed_at,
|
|
140
|
+
}
|
|
141
|
+
optional = {
|
|
142
|
+
"base_item_id": self.base_item_id,
|
|
143
|
+
"enhancement_level": self.enhancement_level,
|
|
144
|
+
"enhancement": self.enhancement,
|
|
145
|
+
"inventory_slot": self.inventory_slot,
|
|
146
|
+
"container_code": self.container_code,
|
|
147
|
+
"container_code_hex": (
|
|
148
|
+
f"0x{self.container_code:02X}"
|
|
149
|
+
if self.container_code is not None
|
|
150
|
+
else None
|
|
151
|
+
),
|
|
152
|
+
"container_name": self.container_name,
|
|
153
|
+
"container_confidence": self.container_confidence,
|
|
154
|
+
"currency_name": self.currency_name,
|
|
155
|
+
}
|
|
156
|
+
output.update(
|
|
157
|
+
{key: value for key, value in optional.items() if value is not None}
|
|
158
|
+
)
|
|
159
|
+
return output
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class _ItemQueries:
|
|
163
|
+
items: tuple[SnapshotItem, ...]
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def occupied_stacks(self) -> int:
|
|
167
|
+
return len(self.items)
|
|
168
|
+
|
|
169
|
+
def records_for(self, item_id: int) -> tuple[SnapshotItem, ...]:
|
|
170
|
+
"""Return every distinct occupied stack with the exact encoded item ID."""
|
|
171
|
+
return tuple(item for item in self.items if item.item_id == item_id)
|
|
172
|
+
|
|
173
|
+
def quantity_for(self, item_id: int) -> int:
|
|
174
|
+
"""Sum quantities across distinct stacks with the exact encoded item ID."""
|
|
175
|
+
return sum(item.quantity for item in self.records_for(item_id))
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@dataclass(frozen=True, kw_only=True)
|
|
179
|
+
class InventoryContainerSummary(_ItemQueries):
|
|
180
|
+
"""One structurally classified inventory container (experimental)."""
|
|
181
|
+
|
|
182
|
+
raw_code: int
|
|
183
|
+
name: str
|
|
184
|
+
confidence: str
|
|
185
|
+
items: tuple[SnapshotItem, ...]
|
|
186
|
+
currency_balances: tuple[SnapshotItem, ...]
|
|
187
|
+
|
|
188
|
+
@property
|
|
189
|
+
def serialized_records(self) -> int:
|
|
190
|
+
return len(self.items) + len(self.currency_balances)
|
|
191
|
+
|
|
192
|
+
def currency(self, item_id_or_name: int | str) -> Optional[SnapshotItem]:
|
|
193
|
+
"""Look up a known balance by encoded item ID or display name."""
|
|
194
|
+
if isinstance(item_id_or_name, int):
|
|
195
|
+
return next(
|
|
196
|
+
(
|
|
197
|
+
balance
|
|
198
|
+
for balance in self.currency_balances
|
|
199
|
+
if balance.item_id == item_id_or_name
|
|
200
|
+
),
|
|
201
|
+
None,
|
|
202
|
+
)
|
|
203
|
+
folded = item_id_or_name.casefold()
|
|
204
|
+
return next(
|
|
205
|
+
(
|
|
206
|
+
balance
|
|
207
|
+
for balance in self.currency_balances
|
|
208
|
+
if balance.currency_name is not None
|
|
209
|
+
and balance.currency_name.casefold() == folded
|
|
210
|
+
),
|
|
211
|
+
None,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
def to_dict(self) -> dict[str, object]:
|
|
215
|
+
return {
|
|
216
|
+
"raw_code": self.raw_code,
|
|
217
|
+
"raw_code_hex": f"0x{self.raw_code:02X}",
|
|
218
|
+
"name": self.name,
|
|
219
|
+
"confidence": self.confidence,
|
|
220
|
+
"serialized_records": self.serialized_records,
|
|
221
|
+
"occupied_stacks": self.occupied_stacks,
|
|
222
|
+
"currency_balance_records": len(self.currency_balances),
|
|
223
|
+
"items": [item.to_dict() for item in self.items],
|
|
224
|
+
"currency_balances": [
|
|
225
|
+
balance.to_dict() for balance in self.currency_balances
|
|
226
|
+
],
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
@dataclass(frozen=True, kw_only=True)
|
|
231
|
+
class InventorySnapshotSummary(_ItemQueries):
|
|
232
|
+
"""Canonical inventory state with computed container views."""
|
|
233
|
+
|
|
234
|
+
hydration_observed: bool
|
|
235
|
+
items: tuple[SnapshotItem, ...]
|
|
236
|
+
currency_balances: tuple[SnapshotItem, ...]
|
|
237
|
+
|
|
238
|
+
@property
|
|
239
|
+
def serialized_records(self) -> int:
|
|
240
|
+
"""Distinct current records, including known currency-wallet balances."""
|
|
241
|
+
return self.occupied_stacks + self.currency_balance_records
|
|
242
|
+
|
|
243
|
+
@property
|
|
244
|
+
def currency_balance_records(self) -> int:
|
|
245
|
+
return len(self.currency_balances)
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def unclassified_records(self) -> int:
|
|
249
|
+
return sum(
|
|
250
|
+
item.container_code is None
|
|
251
|
+
for item in (*self.items, *self.currency_balances)
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
@property
|
|
255
|
+
def containers(self) -> tuple[InventoryContainerSummary, ...]:
|
|
256
|
+
"""Compute provisional container views without duplicating stored state."""
|
|
257
|
+
|
|
258
|
+
records = (*self.items, *self.currency_balances)
|
|
259
|
+
containers: list[InventoryContainerSummary] = []
|
|
260
|
+
for raw_code, (name, confidence) in _INVENTORY_CONTAINER_LABELS.items():
|
|
261
|
+
container_records = tuple(
|
|
262
|
+
item for item in records if item.container_code == raw_code
|
|
263
|
+
)
|
|
264
|
+
if not container_records:
|
|
265
|
+
continue
|
|
266
|
+
containers.append(
|
|
267
|
+
InventoryContainerSummary(
|
|
268
|
+
raw_code=raw_code,
|
|
269
|
+
name=name,
|
|
270
|
+
confidence=confidence,
|
|
271
|
+
items=tuple(
|
|
272
|
+
item
|
|
273
|
+
for item in container_records
|
|
274
|
+
if not item.is_currency_balance
|
|
275
|
+
),
|
|
276
|
+
currency_balances=tuple(
|
|
277
|
+
item for item in container_records if item.is_currency_balance
|
|
278
|
+
),
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
return tuple(containers)
|
|
282
|
+
|
|
283
|
+
def container(self, raw_code: int) -> Optional[InventoryContainerSummary]:
|
|
284
|
+
"""Look up a provisionally classified container by its raw byte."""
|
|
285
|
+
return next(
|
|
286
|
+
(
|
|
287
|
+
container
|
|
288
|
+
for container in self.containers
|
|
289
|
+
if container.raw_code == raw_code
|
|
290
|
+
),
|
|
291
|
+
None,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
def container_named(self, name: str) -> Optional[InventoryContainerSummary]:
|
|
295
|
+
"""Convenience lookup by provisional display name."""
|
|
296
|
+
folded = name.casefold()
|
|
297
|
+
return next(
|
|
298
|
+
(
|
|
299
|
+
container
|
|
300
|
+
for container in self.containers
|
|
301
|
+
if container.name.casefold() == folded
|
|
302
|
+
),
|
|
303
|
+
None,
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
def currency(self, item_id_or_name: int | str) -> Optional[SnapshotItem]:
|
|
307
|
+
"""Look up a known currency balance by encoded ID or display name."""
|
|
308
|
+
if isinstance(item_id_or_name, int):
|
|
309
|
+
return next(
|
|
310
|
+
(
|
|
311
|
+
balance
|
|
312
|
+
for balance in self.currency_balances
|
|
313
|
+
if balance.item_id == item_id_or_name
|
|
314
|
+
),
|
|
315
|
+
None,
|
|
316
|
+
)
|
|
317
|
+
folded = item_id_or_name.casefold()
|
|
318
|
+
return next(
|
|
319
|
+
(
|
|
320
|
+
balance
|
|
321
|
+
for balance in self.currency_balances
|
|
322
|
+
if balance.currency_name is not None
|
|
323
|
+
and balance.currency_name.casefold() == folded
|
|
324
|
+
),
|
|
325
|
+
None,
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
def to_dict(self) -> dict[str, object]:
|
|
329
|
+
return {
|
|
330
|
+
"hydration_observed": self.hydration_observed,
|
|
331
|
+
"occupied_stacks": self.occupied_stacks,
|
|
332
|
+
"serialized_records": self.serialized_records,
|
|
333
|
+
"currency_balance_records": self.currency_balance_records,
|
|
334
|
+
"unclassified_records": self.unclassified_records,
|
|
335
|
+
"currency_balances": [
|
|
336
|
+
balance.to_dict() for balance in self.currency_balances
|
|
337
|
+
],
|
|
338
|
+
"items": [item.to_dict() for item in self.items],
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
@dataclass(frozen=True, kw_only=True)
|
|
343
|
+
class StorageSnapshotSummary(_ItemQueries):
|
|
344
|
+
"""Selected current state for one observed storage destination."""
|
|
345
|
+
|
|
346
|
+
storage_id: int
|
|
347
|
+
name: Optional[str]
|
|
348
|
+
name_confidence: Optional[str]
|
|
349
|
+
items: tuple[SnapshotItem, ...]
|
|
350
|
+
current_state_observed: bool
|
|
351
|
+
current_empty: Optional[bool]
|
|
352
|
+
current_identity_complete: Optional[bool]
|
|
353
|
+
|
|
354
|
+
def to_dict(self) -> dict[str, object]:
|
|
355
|
+
return {
|
|
356
|
+
"storage_id": self.storage_id,
|
|
357
|
+
"name": self.name,
|
|
358
|
+
"name_confidence": self.name_confidence,
|
|
359
|
+
"occupied_stacks": self.occupied_stacks,
|
|
360
|
+
"current_state_observed": self.current_state_observed,
|
|
361
|
+
"current_empty": self.current_empty,
|
|
362
|
+
"current_identity_complete": self.current_identity_complete,
|
|
363
|
+
"items": [item.to_dict() for item in self.items],
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
class StorageContents(tuple[StorageSnapshotSummary, ...]):
|
|
368
|
+
"""Tuple-preserving query collection for storage snapshots.
|
|
369
|
+
|
|
370
|
+
Subclassing ``tuple`` retains the complete historical sequence contract,
|
|
371
|
+
including tuple type checks, operators, and generic dataclass traversal,
|
|
372
|
+
while adding cross-destination queries.
|
|
373
|
+
"""
|
|
374
|
+
|
|
375
|
+
def __new__(
|
|
376
|
+
cls,
|
|
377
|
+
values: Iterable[StorageSnapshotSummary] = (),
|
|
378
|
+
) -> "StorageContents":
|
|
379
|
+
return super().__new__(cls, values)
|
|
380
|
+
|
|
381
|
+
def by_id(self, storage_id: int) -> Optional[StorageSnapshotSummary]:
|
|
382
|
+
"""Look up a destination by its numeric protocol key."""
|
|
383
|
+
return next(
|
|
384
|
+
(storage for storage in self if storage.storage_id == storage_id),
|
|
385
|
+
None,
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
def named(self, name: str) -> Optional[StorageSnapshotSummary]:
|
|
389
|
+
"""Look up a destination by its confidence-qualified display name."""
|
|
390
|
+
folded = name.casefold()
|
|
391
|
+
return next(
|
|
392
|
+
(
|
|
393
|
+
storage
|
|
394
|
+
for storage in self
|
|
395
|
+
if storage.name is not None and storage.name.casefold() == folded
|
|
396
|
+
),
|
|
397
|
+
None,
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
def find_item(self, item_id: int) -> tuple[SnapshotItem, ...]:
|
|
401
|
+
"""Return every distinct stack with ``item_id`` across all storages."""
|
|
402
|
+
return tuple(item for storage in self for item in storage.records_for(item_id))
|
|
403
|
+
|
|
404
|
+
def total_quantity(self, item_id: int) -> int:
|
|
405
|
+
"""Sum an exact encoded item ID across every observed storage."""
|
|
406
|
+
return sum(item.quantity for item in self.find_item(item_id))
|
|
407
|
+
|
|
408
|
+
def locations_for(
|
|
409
|
+
self,
|
|
410
|
+
item_id: int,
|
|
411
|
+
) -> tuple[StorageSnapshotSummary, ...]:
|
|
412
|
+
"""Return storage summaries containing at least one matching stack."""
|
|
413
|
+
return tuple(storage for storage in self if storage.records_for(item_id))
|
|
414
|
+
|
|
415
|
+
@property
|
|
416
|
+
def registered_count(self) -> int:
|
|
417
|
+
"""Number of observed destinations present in the installed registry."""
|
|
418
|
+
return sum(storage.storage_id in STORAGE_LOCATIONS for storage in self)
|
|
419
|
+
|
|
420
|
+
@property
|
|
421
|
+
def selected_count(self) -> int:
|
|
422
|
+
"""Number of destinations selected as current state."""
|
|
423
|
+
return sum(storage.current_state_observed for storage in self)
|
|
424
|
+
|
|
425
|
+
@property
|
|
426
|
+
def nonempty_count(self) -> int:
|
|
427
|
+
"""Selected destinations containing at least one occupied stack."""
|
|
428
|
+
return sum(
|
|
429
|
+
storage.current_state_observed and storage.occupied_stacks > 0
|
|
430
|
+
for storage in self
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
@property
|
|
434
|
+
def empty_count(self) -> int:
|
|
435
|
+
"""Selected destinations proven empty by a count-zero wrapper."""
|
|
436
|
+
return sum(storage.current_empty is True for storage in self)
|
|
437
|
+
|
|
438
|
+
@property
|
|
439
|
+
def occupied_stacks(self) -> int:
|
|
440
|
+
"""Occupied stacks across the selected current destination states."""
|
|
441
|
+
return sum(storage.occupied_stacks for storage in self)
|
|
442
|
+
|
|
443
|
+
def to_dict(self) -> dict[str, object]:
|
|
444
|
+
"""Serialize current aggregate counts and each observed destination."""
|
|
445
|
+
return {
|
|
446
|
+
"observed_count": len(self),
|
|
447
|
+
"registered_count": self.registered_count,
|
|
448
|
+
"selected_count": self.selected_count,
|
|
449
|
+
"nonempty_count": self.nonempty_count,
|
|
450
|
+
"empty_count": self.empty_count,
|
|
451
|
+
"occupied_stacks": self.occupied_stacks,
|
|
452
|
+
"destinations": [storage.to_dict() for storage in self],
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
@dataclass(frozen=True)
|
|
457
|
+
class ItemStateCoverage:
|
|
458
|
+
"""Actionable gaps in the observed item-state evidence."""
|
|
459
|
+
|
|
460
|
+
inventory_records_missing_instance: int
|
|
461
|
+
storage_records_missing_instance: int
|
|
462
|
+
selected_storage_records_missing_instance: int
|
|
463
|
+
registered_storage_ids_not_observed: tuple[int, ...]
|
|
464
|
+
unregistered_storage_ids_observed: tuple[int, ...]
|
|
465
|
+
storage_locations_not_selected: int
|
|
466
|
+
storage_locations_with_incomplete_current_identity: int
|
|
467
|
+
|
|
468
|
+
def to_dict(self) -> dict[str, object]:
|
|
469
|
+
return {
|
|
470
|
+
"inventory_records_missing_instance": (
|
|
471
|
+
self.inventory_records_missing_instance
|
|
472
|
+
),
|
|
473
|
+
"storage_records_missing_instance": self.storage_records_missing_instance,
|
|
474
|
+
"selected_storage_records_missing_instance": (
|
|
475
|
+
self.selected_storage_records_missing_instance
|
|
476
|
+
),
|
|
477
|
+
"registered_storage_ids_not_observed": list(
|
|
478
|
+
self.registered_storage_ids_not_observed
|
|
479
|
+
),
|
|
480
|
+
"unregistered_storage_ids_observed": list(
|
|
481
|
+
self.unregistered_storage_ids_observed
|
|
482
|
+
),
|
|
483
|
+
"storage_locations_not_selected": self.storage_locations_not_selected,
|
|
484
|
+
"storage_locations_with_incomplete_current_identity": (
|
|
485
|
+
self.storage_locations_with_incomplete_current_identity
|
|
486
|
+
),
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
@dataclass(frozen=True)
|
|
491
|
+
class ItemStateProvenance:
|
|
492
|
+
"""Machine-readable origin of one assembled item-state snapshot."""
|
|
493
|
+
|
|
494
|
+
capture_mode: str
|
|
495
|
+
profile_source: str
|
|
496
|
+
generation_selection: str = "unknown"
|
|
497
|
+
capture_path: Optional[str] = None
|
|
498
|
+
|
|
499
|
+
def to_dict(self, *, include_capture_path: bool = False) -> dict[str, object]:
|
|
500
|
+
output: dict[str, object] = {
|
|
501
|
+
"capture_mode": self.capture_mode,
|
|
502
|
+
"profile_source": self.profile_source,
|
|
503
|
+
"generation_selection": self.generation_selection,
|
|
504
|
+
}
|
|
505
|
+
if include_capture_path and self.capture_path is not None:
|
|
506
|
+
output["capture_path"] = self.capture_path
|
|
507
|
+
return output
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
@dataclass(frozen=True, kw_only=True)
|
|
511
|
+
class InventoryHydrationDiagnostics:
|
|
512
|
+
"""Inventory wrapper geometry and selection measurements."""
|
|
513
|
+
|
|
514
|
+
raw_records: int
|
|
515
|
+
duplicate_records: int
|
|
516
|
+
group_counts: tuple[int, ...]
|
|
517
|
+
inferred_strides: tuple[int, ...]
|
|
518
|
+
generations_observed: int
|
|
519
|
+
source_opcodes: tuple[int, ...]
|
|
520
|
+
message_lengths: tuple[int, ...]
|
|
521
|
+
|
|
522
|
+
@property
|
|
523
|
+
def groups(self) -> int:
|
|
524
|
+
return len(self.group_counts)
|
|
525
|
+
|
|
526
|
+
@property
|
|
527
|
+
def populated_groups(self) -> int:
|
|
528
|
+
return sum(count > 0 for count in self.group_counts)
|
|
529
|
+
|
|
530
|
+
@property
|
|
531
|
+
def empty_groups(self) -> int:
|
|
532
|
+
return sum(count == 0 for count in self.group_counts)
|
|
533
|
+
|
|
534
|
+
def to_dict(self) -> dict[str, object]:
|
|
535
|
+
return {
|
|
536
|
+
"raw_records": self.raw_records,
|
|
537
|
+
"duplicate_records": self.duplicate_records,
|
|
538
|
+
"groups": self.groups,
|
|
539
|
+
"populated_groups": self.populated_groups,
|
|
540
|
+
"empty_groups": self.empty_groups,
|
|
541
|
+
"group_counts": list(self.group_counts),
|
|
542
|
+
"inferred_strides": list(self.inferred_strides),
|
|
543
|
+
"generations_observed": self.generations_observed,
|
|
544
|
+
"source_opcodes": [
|
|
545
|
+
f"0x{opcode:04X}" for opcode in self.source_opcodes
|
|
546
|
+
],
|
|
547
|
+
"message_lengths": list(self.message_lengths),
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
@dataclass(frozen=True, kw_only=True)
|
|
552
|
+
class StorageDestinationDiagnostics:
|
|
553
|
+
"""All-sweep assembly evidence for one numeric storage destination."""
|
|
554
|
+
|
|
555
|
+
storage_id: int
|
|
556
|
+
raw_records: int
|
|
557
|
+
duplicate_records: int
|
|
558
|
+
groups: int
|
|
559
|
+
empty_envelope_seen: bool
|
|
560
|
+
selected_records: int
|
|
561
|
+
selected_groups: int
|
|
562
|
+
sweeps_observed: int
|
|
563
|
+
selected_sweep: Optional[int]
|
|
564
|
+
missing_instance_records: int
|
|
565
|
+
selected_missing_instance_records: int
|
|
566
|
+
source_opcodes: tuple[int, ...]
|
|
567
|
+
message_lengths: tuple[int, ...]
|
|
568
|
+
|
|
569
|
+
@property
|
|
570
|
+
def superseded_records(self) -> int:
|
|
571
|
+
return max(0, self.raw_records - self.selected_records)
|
|
572
|
+
|
|
573
|
+
@property
|
|
574
|
+
def superseded_groups(self) -> int:
|
|
575
|
+
return max(0, self.groups - self.selected_groups)
|
|
576
|
+
|
|
577
|
+
def to_dict(self) -> dict[str, object]:
|
|
578
|
+
return {
|
|
579
|
+
"storage_id": self.storage_id,
|
|
580
|
+
"raw_records": self.raw_records,
|
|
581
|
+
"duplicate_records": self.duplicate_records,
|
|
582
|
+
"groups": self.groups,
|
|
583
|
+
"empty_envelope_seen": self.empty_envelope_seen,
|
|
584
|
+
"selected_records": self.selected_records,
|
|
585
|
+
"superseded_records": self.superseded_records,
|
|
586
|
+
"selected_groups": self.selected_groups,
|
|
587
|
+
"superseded_groups": self.superseded_groups,
|
|
588
|
+
"sweeps_observed": self.sweeps_observed,
|
|
589
|
+
"selected_sweep": self.selected_sweep,
|
|
590
|
+
"missing_instance_records": self.missing_instance_records,
|
|
591
|
+
"selected_missing_instance_records": (
|
|
592
|
+
self.selected_missing_instance_records
|
|
593
|
+
),
|
|
594
|
+
"source_opcodes": [
|
|
595
|
+
f"0x{opcode:04X}" for opcode in self.source_opcodes
|
|
596
|
+
],
|
|
597
|
+
"message_lengths": list(self.message_lengths),
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
@dataclass(frozen=True, kw_only=True)
|
|
602
|
+
class StorageHydrationDiagnostics:
|
|
603
|
+
"""Aggregate storage assembly evidence with per-destination detail."""
|
|
604
|
+
|
|
605
|
+
records_decoded: int
|
|
606
|
+
records_without_destination: int
|
|
607
|
+
sweeps_observed: int
|
|
608
|
+
selected_sweep: Optional[int]
|
|
609
|
+
destinations: tuple[StorageDestinationDiagnostics, ...]
|
|
610
|
+
|
|
611
|
+
def destination(
|
|
612
|
+
self,
|
|
613
|
+
storage_id: int,
|
|
614
|
+
) -> Optional[StorageDestinationDiagnostics]:
|
|
615
|
+
"""Return diagnostics for one exact numeric destination key."""
|
|
616
|
+
|
|
617
|
+
return next(
|
|
618
|
+
(
|
|
619
|
+
diagnostic
|
|
620
|
+
for diagnostic in self.destinations
|
|
621
|
+
if diagnostic.storage_id == storage_id
|
|
622
|
+
),
|
|
623
|
+
None,
|
|
624
|
+
)
|
|
625
|
+
|
|
626
|
+
def to_dict(self) -> dict[str, object]:
|
|
627
|
+
return {
|
|
628
|
+
"records_decoded": self.records_decoded,
|
|
629
|
+
"records_without_destination": self.records_without_destination,
|
|
630
|
+
"sweeps_observed": self.sweeps_observed,
|
|
631
|
+
"selected_sweep": self.selected_sweep,
|
|
632
|
+
"destinations": [
|
|
633
|
+
diagnostic.to_dict() for diagnostic in self.destinations
|
|
634
|
+
],
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
@dataclass(frozen=True, kw_only=True)
|
|
639
|
+
class ItemStateDiagnostics:
|
|
640
|
+
"""Advanced capture and selection measurements for troubleshooting."""
|
|
641
|
+
|
|
642
|
+
frames_seen: int
|
|
643
|
+
relevant_frames_retained: int
|
|
644
|
+
relevant_bytes_retained: int
|
|
645
|
+
snapshot_records_retained: int
|
|
646
|
+
capture_limits: ItemStateCaptureLimits
|
|
647
|
+
inventory: InventoryHydrationDiagnostics
|
|
648
|
+
storage: StorageHydrationDiagnostics
|
|
649
|
+
|
|
650
|
+
def to_dict(self) -> dict[str, object]:
|
|
651
|
+
return {
|
|
652
|
+
"frames_seen": self.frames_seen,
|
|
653
|
+
"relevant_frames_retained": self.relevant_frames_retained,
|
|
654
|
+
"relevant_bytes_retained": self.relevant_bytes_retained,
|
|
655
|
+
"snapshot_records_retained": self.snapshot_records_retained,
|
|
656
|
+
"capture_limits": self.capture_limits.to_dict(),
|
|
657
|
+
"inventory": self.inventory.to_dict(),
|
|
658
|
+
"storage": self.storage.to_dict(),
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
@dataclass(frozen=True)
|
|
663
|
+
class CharacterStateSnapshot:
|
|
664
|
+
"""Query model assembled from observed character-load hydration records."""
|
|
665
|
+
|
|
666
|
+
inventory: InventorySnapshotSummary
|
|
667
|
+
storages: StorageContents
|
|
668
|
+
provenance: ItemStateProvenance
|
|
669
|
+
coverage: ItemStateCoverage
|
|
670
|
+
decoder_health: DecoderHealth = DecoderHealth()
|
|
671
|
+
warnings: tuple[str, ...] = ()
|
|
672
|
+
diagnostics: Optional[ItemStateDiagnostics] = None
|
|
673
|
+
|
|
674
|
+
def __post_init__(self) -> None:
|
|
675
|
+
if not isinstance(self.inventory, InventorySnapshotSummary):
|
|
676
|
+
raise TypeError("inventory must be an InventorySnapshotSummary")
|
|
677
|
+
if not isinstance(self.provenance, ItemStateProvenance):
|
|
678
|
+
raise TypeError("provenance must be an ItemStateProvenance")
|
|
679
|
+
if not isinstance(self.coverage, ItemStateCoverage):
|
|
680
|
+
raise TypeError("coverage must be an ItemStateCoverage")
|
|
681
|
+
if not isinstance(self.decoder_health, DecoderHealth):
|
|
682
|
+
raise TypeError("decoder_health must be a DecoderHealth")
|
|
683
|
+
if self.diagnostics is not None and not isinstance(
|
|
684
|
+
self.diagnostics, ItemStateDiagnostics
|
|
685
|
+
):
|
|
686
|
+
raise TypeError("diagnostics must be an ItemStateDiagnostics or None")
|
|
687
|
+
object.__setattr__(self, "storages", StorageContents(self.storages))
|
|
688
|
+
|
|
689
|
+
@property
|
|
690
|
+
def schema_version(self) -> int:
|
|
691
|
+
return _ITEM_STATE_SCHEMA_VERSION
|
|
692
|
+
|
|
693
|
+
@property
|
|
694
|
+
def identity_complete(self) -> bool:
|
|
695
|
+
return (
|
|
696
|
+
self.coverage.inventory_records_missing_instance == 0
|
|
697
|
+
and self.coverage.storage_records_missing_instance == 0
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
@property
|
|
701
|
+
def hydration_detected(self) -> bool:
|
|
702
|
+
storage_evidence_observed = (
|
|
703
|
+
self.diagnostics.storage.records_decoded
|
|
704
|
+
if self.diagnostics is not None
|
|
705
|
+
else 0
|
|
706
|
+
)
|
|
707
|
+
return bool(
|
|
708
|
+
self.inventory.hydration_observed
|
|
709
|
+
or storage_evidence_observed
|
|
710
|
+
or self.storages
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
def to_dict(self, *, include_diagnostics: bool = False) -> dict[str, object]:
|
|
714
|
+
output: dict[str, object] = {
|
|
715
|
+
"schema_version": self.schema_version,
|
|
716
|
+
"hydration_detected": self.hydration_detected,
|
|
717
|
+
"identity_complete": self.identity_complete,
|
|
718
|
+
"provenance": self.provenance.to_dict(
|
|
719
|
+
include_capture_path=include_diagnostics
|
|
720
|
+
),
|
|
721
|
+
"coverage": self.coverage.to_dict(),
|
|
722
|
+
"decoder_health": self.decoder_health.to_dict(),
|
|
723
|
+
"inventory": self.inventory.to_dict(),
|
|
724
|
+
"storages": self.storages.to_dict(),
|
|
725
|
+
"warnings": list(self.warnings),
|
|
726
|
+
}
|
|
727
|
+
if include_diagnostics and self.diagnostics is not None:
|
|
728
|
+
output["diagnostics"] = self.diagnostics.to_dict()
|
|
729
|
+
return output
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
@dataclass(frozen=True)
|
|
733
|
+
class _InventoryAssembly:
|
|
734
|
+
summary: InventorySnapshotSummary
|
|
735
|
+
missing_instance_records: int
|
|
736
|
+
diagnostics: InventoryHydrationDiagnostics
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
@dataclass(frozen=True)
|
|
740
|
+
class _StorageAssembly:
|
|
741
|
+
summaries: tuple[StorageSnapshotSummary, ...]
|
|
742
|
+
diagnostics: tuple[StorageDestinationDiagnostics, ...]
|
|
743
|
+
records_without_destination: int
|
|
744
|
+
records_missing_instance: int
|
|
745
|
+
sweeps_observed: int
|
|
746
|
+
selected_sweep: Optional[int]
|
|
747
|
+
unknown_empty_envelopes: int
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
@dataclass(frozen=True)
|
|
751
|
+
class _FrameKey:
|
|
752
|
+
source_ip: str
|
|
753
|
+
source_port: int
|
|
754
|
+
destination_ip: str
|
|
755
|
+
destination_port: int
|
|
756
|
+
flow_generation: int
|
|
757
|
+
stream_sequence: Optional[int]
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
@dataclass(frozen=True)
|
|
761
|
+
class _HydrationAnchor:
|
|
762
|
+
"""One validated inventory-wrapper observation anchoring a load epoch."""
|
|
763
|
+
|
|
764
|
+
timestamp: float
|
|
765
|
+
frame_key: _FrameKey
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
@dataclass(frozen=True)
|
|
769
|
+
class _InventoryGeneration:
|
|
770
|
+
"""Latest inventory hydration burst, including recordless wrappers."""
|
|
771
|
+
|
|
772
|
+
events: tuple[BDOEvent, ...]
|
|
773
|
+
anchors: tuple[_HydrationAnchor, ...]
|
|
774
|
+
start: Optional[float]
|
|
775
|
+
flow_generation_key: Optional[tuple[str, int, str, int, int]]
|
|
776
|
+
generations_observed: int
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
@dataclass(frozen=True)
|
|
780
|
+
class _StorageGroupObservation:
|
|
781
|
+
"""One decoded nonempty frame or validated count-zero storage wrapper."""
|
|
782
|
+
|
|
783
|
+
storage_id: int
|
|
784
|
+
frame_key: _FrameKey
|
|
785
|
+
timestamp: float
|
|
786
|
+
opcode: int
|
|
787
|
+
message_length: Optional[int]
|
|
788
|
+
items: tuple[SnapshotItem, ...]
|
|
789
|
+
raw_records: int
|
|
790
|
+
missing_instance_records: int
|
|
791
|
+
empty: bool
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
@dataclass(frozen=True)
|
|
795
|
+
class _StorageDestinationBlock:
|
|
796
|
+
"""Consecutive chunks belonging to one destination within one sweep."""
|
|
797
|
+
|
|
798
|
+
storage_id: int
|
|
799
|
+
stream_key: tuple[str, int, str, int, int, int]
|
|
800
|
+
groups: tuple[_StorageGroupObservation, ...]
|
|
801
|
+
|
|
802
|
+
@property
|
|
803
|
+
def empty(self) -> bool:
|
|
804
|
+
return all(group.empty for group in self.groups)
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
@dataclass(frozen=True)
|
|
808
|
+
class _StorageEmptySchema:
|
|
809
|
+
spec: EventSpec
|
|
810
|
+
prefix_length: int
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def _event_frame_key(event: BDOEvent) -> _FrameKey:
|
|
814
|
+
sequence = event.extra.get("stream_sequence")
|
|
815
|
+
return _FrameKey(
|
|
816
|
+
event.flow.source_ip,
|
|
817
|
+
event.flow.source_port,
|
|
818
|
+
event.flow.destination_ip,
|
|
819
|
+
event.flow.destination_port,
|
|
820
|
+
event._flow_generation,
|
|
821
|
+
sequence if isinstance(sequence, int) else None,
|
|
822
|
+
)
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def _frame_key(frame: BDOFrame) -> _FrameKey:
|
|
826
|
+
return _FrameKey(
|
|
827
|
+
frame.context.flow.source_ip,
|
|
828
|
+
frame.context.flow.source_port,
|
|
829
|
+
frame.context.flow.destination_ip,
|
|
830
|
+
frame.context.flow.destination_port,
|
|
831
|
+
frame.context.flow_generation,
|
|
832
|
+
frame.stream_sequence,
|
|
833
|
+
)
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def _event_flow_generation_key(event: BDOEvent) -> tuple[str, int, str, int, int]:
|
|
837
|
+
frame_key = _event_frame_key(event)
|
|
838
|
+
return (
|
|
839
|
+
event.flow.source_ip,
|
|
840
|
+
event.flow.source_port,
|
|
841
|
+
event.flow.destination_ip,
|
|
842
|
+
event.flow.destination_port,
|
|
843
|
+
frame_key.flow_generation,
|
|
844
|
+
)
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def _frame_flow_generation_key(frame: BDOFrame) -> tuple[str, int, str, int, int]:
|
|
848
|
+
return (
|
|
849
|
+
frame.context.flow.source_ip,
|
|
850
|
+
frame.context.flow.source_port,
|
|
851
|
+
frame.context.flow.destination_ip,
|
|
852
|
+
frame.context.flow.destination_port,
|
|
853
|
+
frame.context.flow_generation,
|
|
854
|
+
)
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def _anchor_flow_generation_key(
|
|
858
|
+
anchor: _HydrationAnchor,
|
|
859
|
+
) -> tuple[str, int, str, int, int]:
|
|
860
|
+
key = anchor.frame_key
|
|
861
|
+
return (
|
|
862
|
+
key.source_ip,
|
|
863
|
+
key.source_port,
|
|
864
|
+
key.destination_ip,
|
|
865
|
+
key.destination_port,
|
|
866
|
+
key.flow_generation,
|
|
867
|
+
)
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
@dataclass(frozen=True)
|
|
871
|
+
class _InventoryRecordMetadata:
|
|
872
|
+
slot: Optional[int]
|
|
873
|
+
container_code: int
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def _snapshot_item(
|
|
877
|
+
event: BDOEvent,
|
|
878
|
+
instance: str,
|
|
879
|
+
inventory_metadata: Optional[_InventoryRecordMetadata] = None,
|
|
880
|
+
) -> SnapshotItem:
|
|
881
|
+
container_name = None
|
|
882
|
+
container_confidence = None
|
|
883
|
+
currency_name = None
|
|
884
|
+
if inventory_metadata is not None:
|
|
885
|
+
container_name, container_confidence = _INVENTORY_CONTAINER_LABELS[
|
|
886
|
+
inventory_metadata.container_code
|
|
887
|
+
]
|
|
888
|
+
currency_name = _CURRENCY_NAMES.get(
|
|
889
|
+
(inventory_metadata.container_code, event.item_id)
|
|
890
|
+
)
|
|
891
|
+
return SnapshotItem(
|
|
892
|
+
item_id=event.item_id,
|
|
893
|
+
quantity=event.quantity,
|
|
894
|
+
instance=instance,
|
|
895
|
+
observed_at=event.timestamp,
|
|
896
|
+
base_item_id=event.base_item_id,
|
|
897
|
+
enhancement_level=event.enhancement_level,
|
|
898
|
+
enhancement=event.enhancement,
|
|
899
|
+
inventory_slot=(
|
|
900
|
+
inventory_metadata.slot if inventory_metadata is not None else None
|
|
901
|
+
),
|
|
902
|
+
container_code=(
|
|
903
|
+
inventory_metadata.container_code
|
|
904
|
+
if inventory_metadata is not None
|
|
905
|
+
else None
|
|
906
|
+
),
|
|
907
|
+
container_name=container_name,
|
|
908
|
+
container_confidence=container_confidence,
|
|
909
|
+
currency_name=currency_name,
|
|
910
|
+
)
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def _inventory_frame_stride(
|
|
914
|
+
frame: BDOFrame,
|
|
915
|
+
spec: EventSpec,
|
|
916
|
+
group: list[BDOEvent],
|
|
917
|
+
) -> Optional[int]:
|
|
918
|
+
"""Derive and validate one frame's repeated-record geometry."""
|
|
919
|
+
count = len(group)
|
|
920
|
+
base_length = spec.single_record_message_length
|
|
921
|
+
if count < 2 or base_length is None:
|
|
922
|
+
return None
|
|
923
|
+
extra_length = frame.length - base_length
|
|
924
|
+
if extra_length <= 0 or extra_length % (count - 1):
|
|
925
|
+
return None
|
|
926
|
+
stride = extra_length // (count - 1)
|
|
927
|
+
if stride <= _INVENTORY_TRAILING_DISCOVERY_BYTES:
|
|
928
|
+
return None
|
|
929
|
+
if frame.length != base_length + (count - 1) * stride:
|
|
930
|
+
return None
|
|
931
|
+
if len(frame.message) < frame.length:
|
|
932
|
+
return None
|
|
933
|
+
if not _inventory_event_geometry_valid(spec, group, stride):
|
|
934
|
+
return None
|
|
935
|
+
return stride
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
def _spec_candidates_by_opcode(
|
|
939
|
+
specs: Iterable[EventSpec],
|
|
940
|
+
) -> dict[int, tuple[EventSpec, ...]]:
|
|
941
|
+
"""Retain every distinct same-opcode layout in deterministic profile order."""
|
|
942
|
+
grouped: dict[int, list[EventSpec]] = {}
|
|
943
|
+
for spec in specs:
|
|
944
|
+
candidates = grouped.setdefault(spec.opcode, [])
|
|
945
|
+
if spec not in candidates:
|
|
946
|
+
candidates.append(spec)
|
|
947
|
+
return {opcode: tuple(candidates) for opcode, candidates in grouped.items()}
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
def _unique_inventory_multi_layout(
|
|
951
|
+
frame: BDOFrame,
|
|
952
|
+
group: list[BDOEvent],
|
|
953
|
+
candidates: Iterable[EventSpec],
|
|
954
|
+
) -> Optional[tuple[EventSpec, int]]:
|
|
955
|
+
matches: list[tuple[EventSpec, int]] = []
|
|
956
|
+
for spec in candidates:
|
|
957
|
+
if not _frame_has_zero_context(frame, spec):
|
|
958
|
+
continue
|
|
959
|
+
stride = _inventory_frame_stride(frame, spec, group)
|
|
960
|
+
if stride is not None:
|
|
961
|
+
matches.append((spec, stride))
|
|
962
|
+
return matches[0] if len(matches) == 1 else None
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def _unique_inventory_single_layout(
|
|
966
|
+
frame: BDOFrame,
|
|
967
|
+
group: list[BDOEvent],
|
|
968
|
+
candidates: Iterable[EventSpec],
|
|
969
|
+
sibling_strides: dict[EventSpec, set[int]],
|
|
970
|
+
) -> Optional[tuple[EventSpec, int]]:
|
|
971
|
+
matches: list[tuple[EventSpec, int]] = []
|
|
972
|
+
for spec in candidates:
|
|
973
|
+
proven = sibling_strides.get(spec, set())
|
|
974
|
+
if len(proven) != 1:
|
|
975
|
+
continue
|
|
976
|
+
stride = next(iter(proven))
|
|
977
|
+
if (
|
|
978
|
+
not _frame_has_zero_context(frame, spec)
|
|
979
|
+
or spec.single_record_message_length != frame.length
|
|
980
|
+
or not _inventory_event_geometry_valid(spec, group, stride)
|
|
981
|
+
):
|
|
982
|
+
continue
|
|
983
|
+
matches.append((spec, stride))
|
|
984
|
+
return matches[0] if len(matches) == 1 else None
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
def _inventory_event_geometry_valid(
|
|
988
|
+
spec: EventSpec,
|
|
989
|
+
group: list[BDOEvent],
|
|
990
|
+
stride: int,
|
|
991
|
+
) -> bool:
|
|
992
|
+
"""Require a complete, ordered record set for a candidate stride."""
|
|
993
|
+
count = len(group)
|
|
994
|
+
if count == 0:
|
|
995
|
+
return False
|
|
996
|
+
ordered = sorted(
|
|
997
|
+
group,
|
|
998
|
+
key=lambda event: (
|
|
999
|
+
event.record_offset if event.record_offset is not None else -1
|
|
1000
|
+
),
|
|
1001
|
+
)
|
|
1002
|
+
offsets = [event.record_offset for event in ordered]
|
|
1003
|
+
if any(offset is None for offset in offsets):
|
|
1004
|
+
return False
|
|
1005
|
+
if offsets != [spec.item_offset + index * stride for index in range(count)]:
|
|
1006
|
+
return False
|
|
1007
|
+
if any(
|
|
1008
|
+
event.record_count is not None and event.record_count != count
|
|
1009
|
+
for event in ordered
|
|
1010
|
+
):
|
|
1011
|
+
return False
|
|
1012
|
+
indexes = [event.record_index for event in ordered]
|
|
1013
|
+
if any(index is not None for index in indexes):
|
|
1014
|
+
if indexes != list(range(1, count + 1)):
|
|
1015
|
+
return False
|
|
1016
|
+
return True
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def _discover_inventory_tail_layout(
|
|
1020
|
+
frame_groups: list[tuple[BDOFrame, list[BDOEvent], EventSpec, int]],
|
|
1021
|
+
) -> Optional[tuple[int, int]]:
|
|
1022
|
+
"""Jointly discover slot/container columns near the repeated-record tail.
|
|
1023
|
+
|
|
1024
|
+
The layout is accepted only when exactly one pair explains every complete
|
|
1025
|
+
multi-record group. Requiring at least two distinct known container codes
|
|
1026
|
+
rejects padding columns that happen to contain only zeroes.
|
|
1027
|
+
"""
|
|
1028
|
+
if len(frame_groups) < 2:
|
|
1029
|
+
return None
|
|
1030
|
+
common_stride = {stride for _, _, _, stride in frame_groups}
|
|
1031
|
+
if len(common_stride) != 1:
|
|
1032
|
+
return None
|
|
1033
|
+
stride = next(iter(common_stride))
|
|
1034
|
+
window_start = max(0, stride - _INVENTORY_TRAILING_DISCOVERY_BYTES)
|
|
1035
|
+
candidates: list[tuple[int, int]] = []
|
|
1036
|
+
|
|
1037
|
+
for slot_relative in range(window_start, stride):
|
|
1038
|
+
slot_groups: list[list[int]] = []
|
|
1039
|
+
for frame, group, _, _ in frame_groups:
|
|
1040
|
+
ordered = sorted(group, key=lambda event: int(event.record_offset or 0))
|
|
1041
|
+
slots = [
|
|
1042
|
+
frame.message[int(event.record_offset) + slot_relative]
|
|
1043
|
+
for event in ordered
|
|
1044
|
+
if event.record_offset is not None
|
|
1045
|
+
and int(event.record_offset) + slot_relative < len(frame.message)
|
|
1046
|
+
]
|
|
1047
|
+
slot_groups.append(slots)
|
|
1048
|
+
if any(
|
|
1049
|
+
len(slots) != len(group)
|
|
1050
|
+
for slots, (_, group, _, _) in zip(slot_groups, frame_groups)
|
|
1051
|
+
):
|
|
1052
|
+
continue
|
|
1053
|
+
if any(
|
|
1054
|
+
not slots
|
|
1055
|
+
or any(slot == 0xFF for slot in slots)
|
|
1056
|
+
or slots != sorted(slots)
|
|
1057
|
+
or len(set(slots)) != len(slots)
|
|
1058
|
+
for slots in slot_groups
|
|
1059
|
+
):
|
|
1060
|
+
continue
|
|
1061
|
+
|
|
1062
|
+
for container_relative in range(
|
|
1063
|
+
slot_relative + 1, min(stride, slot_relative + 5)
|
|
1064
|
+
):
|
|
1065
|
+
observed_codes: set[int] = set()
|
|
1066
|
+
valid = True
|
|
1067
|
+
for frame, group, _, _ in frame_groups:
|
|
1068
|
+
codes = {
|
|
1069
|
+
frame.message[int(event.record_offset) + container_relative]
|
|
1070
|
+
for event in group
|
|
1071
|
+
if event.record_offset is not None
|
|
1072
|
+
and int(event.record_offset) + container_relative
|
|
1073
|
+
< len(frame.message)
|
|
1074
|
+
}
|
|
1075
|
+
if len(codes) != 1:
|
|
1076
|
+
valid = False
|
|
1077
|
+
break
|
|
1078
|
+
code = next(iter(codes))
|
|
1079
|
+
if code not in _INVENTORY_CONTAINER_LABELS:
|
|
1080
|
+
valid = False
|
|
1081
|
+
break
|
|
1082
|
+
observed_codes.add(code)
|
|
1083
|
+
if valid and len(observed_codes) >= 2:
|
|
1084
|
+
candidates.append((slot_relative, container_relative))
|
|
1085
|
+
|
|
1086
|
+
if len(candidates) != 1:
|
|
1087
|
+
return None
|
|
1088
|
+
return candidates[0]
|
|
1089
|
+
|
|
1090
|
+
|
|
1091
|
+
def _discover_inventory_header_container_offset(
|
|
1092
|
+
frame_groups: list[tuple[BDOFrame, list[BDOEvent], EventSpec, int]],
|
|
1093
|
+
) -> Optional[int]:
|
|
1094
|
+
"""Discover a wrapper-level container byte shared by every sibling record.
|
|
1095
|
+
|
|
1096
|
+
The August wrapper moved the known 00/10/18/0B container identity out of
|
|
1097
|
+
each repeated-record tail and into the prefix immediately before item one.
|
|
1098
|
+
Search the framed prefix instead of pinning that new position, and accept
|
|
1099
|
+
it only when one unique byte column explains at least two container groups.
|
|
1100
|
+
"""
|
|
1101
|
+
if len(frame_groups) < 2:
|
|
1102
|
+
return None
|
|
1103
|
+
search_end = min(spec.item_offset for _, _, spec, _ in frame_groups)
|
|
1104
|
+
candidates: list[int] = []
|
|
1105
|
+
for offset in range(5, search_end):
|
|
1106
|
+
codes = []
|
|
1107
|
+
for frame, _, _, _ in frame_groups:
|
|
1108
|
+
if offset >= len(frame.message):
|
|
1109
|
+
break
|
|
1110
|
+
code = frame.message[offset]
|
|
1111
|
+
if code not in _INVENTORY_CONTAINER_LABELS:
|
|
1112
|
+
break
|
|
1113
|
+
codes.append(code)
|
|
1114
|
+
else:
|
|
1115
|
+
if len(set(codes)) >= 2:
|
|
1116
|
+
candidates.append(offset)
|
|
1117
|
+
return candidates[0] if len(candidates) == 1 else None
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
def _inventory_header_metadata(
|
|
1121
|
+
frame: BDOFrame,
|
|
1122
|
+
group: list[BDOEvent],
|
|
1123
|
+
container_offset: int,
|
|
1124
|
+
) -> Optional[dict[int, _InventoryRecordMetadata]]:
|
|
1125
|
+
if container_offset >= len(frame.message):
|
|
1126
|
+
return None
|
|
1127
|
+
code = frame.message[container_offset]
|
|
1128
|
+
if code not in _INVENTORY_CONTAINER_LABELS:
|
|
1129
|
+
return None
|
|
1130
|
+
return {
|
|
1131
|
+
event.record_offset: _InventoryRecordMetadata(None, code)
|
|
1132
|
+
for event in group
|
|
1133
|
+
if event.record_offset is not None
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
def _inventory_record_metadata(
|
|
1138
|
+
frame: BDOFrame,
|
|
1139
|
+
group: list[BDOEvent],
|
|
1140
|
+
spec: EventSpec,
|
|
1141
|
+
stride: int,
|
|
1142
|
+
layout: tuple[int, int],
|
|
1143
|
+
) -> Optional[dict[int, _InventoryRecordMetadata]]:
|
|
1144
|
+
"""Extract one validated frame's dynamically discovered tail fields."""
|
|
1145
|
+
base_length = spec.single_record_message_length
|
|
1146
|
+
if base_length is None:
|
|
1147
|
+
return None
|
|
1148
|
+
count = len(group)
|
|
1149
|
+
expected_length = base_length if count == 1 else base_length + (count - 1) * stride
|
|
1150
|
+
if frame.length != expected_length or len(frame.message) < frame.length:
|
|
1151
|
+
return None
|
|
1152
|
+
if not _inventory_event_geometry_valid(spec, group, stride):
|
|
1153
|
+
return None
|
|
1154
|
+
|
|
1155
|
+
slot_relative, container_relative = layout
|
|
1156
|
+
extracted: dict[int, _InventoryRecordMetadata] = {}
|
|
1157
|
+
slots: list[int] = []
|
|
1158
|
+
codes: set[int] = set()
|
|
1159
|
+
for event in sorted(group, key=lambda candidate: int(candidate.record_offset or 0)):
|
|
1160
|
+
assert event.record_offset is not None
|
|
1161
|
+
slot_offset = event.record_offset + slot_relative
|
|
1162
|
+
container_offset = event.record_offset + container_relative
|
|
1163
|
+
if max(slot_offset, container_offset) >= len(frame.message):
|
|
1164
|
+
return None
|
|
1165
|
+
slot = frame.message[slot_offset]
|
|
1166
|
+
code = frame.message[container_offset]
|
|
1167
|
+
if slot == 0xFF or code not in _INVENTORY_CONTAINER_LABELS:
|
|
1168
|
+
return None
|
|
1169
|
+
slots.append(slot)
|
|
1170
|
+
codes.add(code)
|
|
1171
|
+
extracted[event.record_offset] = _InventoryRecordMetadata(slot, code)
|
|
1172
|
+
|
|
1173
|
+
if slots != sorted(slots) or len(slots) != len(set(slots)) or len(codes) != 1:
|
|
1174
|
+
return None
|
|
1175
|
+
return extracted
|
|
1176
|
+
|
|
1177
|
+
|
|
1178
|
+
class _CharacterStateAccumulator:
|
|
1179
|
+
def __init__(
|
|
1180
|
+
self,
|
|
1181
|
+
*,
|
|
1182
|
+
profile_source: str,
|
|
1183
|
+
specs: Iterable[EventSpec],
|
|
1184
|
+
capture_mode: str = "unknown",
|
|
1185
|
+
input_path: str | Path | None = None,
|
|
1186
|
+
saved_capture_path: str | Path | None = None,
|
|
1187
|
+
capture_limits: Optional[ItemStateCaptureLimits] = None,
|
|
1188
|
+
) -> None:
|
|
1189
|
+
if capture_limits is not None and not isinstance(
|
|
1190
|
+
capture_limits, ItemStateCaptureLimits
|
|
1191
|
+
):
|
|
1192
|
+
raise TypeError("capture_limits must be an ItemStateCaptureLimits or None")
|
|
1193
|
+
self.profile_source = profile_source
|
|
1194
|
+
self.capture_mode = capture_mode
|
|
1195
|
+
self.input_path = str(input_path) if input_path is not None else None
|
|
1196
|
+
self.saved_capture_path = (
|
|
1197
|
+
str(saved_capture_path) if saved_capture_path is not None else None
|
|
1198
|
+
)
|
|
1199
|
+
self.capture_limits = capture_limits or ItemStateCaptureLimits()
|
|
1200
|
+
self.specs = tuple(specs)
|
|
1201
|
+
self.inventory_specs = tuple(
|
|
1202
|
+
spec for spec in self.specs if spec.label == "INVENTORY_TRANSFER"
|
|
1203
|
+
)
|
|
1204
|
+
self.storage_specs = tuple(
|
|
1205
|
+
spec for spec in self.specs if spec.label == "INVENTORY_TO_STORAGE"
|
|
1206
|
+
)
|
|
1207
|
+
self.relevant_opcodes = {
|
|
1208
|
+
spec.opcode for spec in self.inventory_specs + self.storage_specs
|
|
1209
|
+
}
|
|
1210
|
+
self._lock = RLock()
|
|
1211
|
+
self._frames_seen = 0
|
|
1212
|
+
self._relevant_frames_retained = 0
|
|
1213
|
+
self._relevant_bytes_retained = 0
|
|
1214
|
+
self._snapshot_records_retained = 0
|
|
1215
|
+
self._limit_error: Optional[ItemStateCaptureLimitError] = None
|
|
1216
|
+
self._frames: list[BDOFrame] = []
|
|
1217
|
+
self._seen_frames: set[tuple[_FrameKey, bytes]] = set()
|
|
1218
|
+
self._inventory_events: list[BDOEvent] = []
|
|
1219
|
+
self._storage_events: list[BDOEvent] = []
|
|
1220
|
+
self._neutral_storage_events: list[BDOEvent] = []
|
|
1221
|
+
self._live_storage_boundaries: list[BDOEvent] = []
|
|
1222
|
+
|
|
1223
|
+
@property
|
|
1224
|
+
def frames_seen(self) -> int:
|
|
1225
|
+
with self._lock:
|
|
1226
|
+
return self._frames_seen
|
|
1227
|
+
|
|
1228
|
+
def observe_frame(self, frame: BDOFrame) -> None:
|
|
1229
|
+
with self._lock:
|
|
1230
|
+
self._frames_seen += 1
|
|
1231
|
+
if self._limit_error is not None:
|
|
1232
|
+
raise self._limit_error
|
|
1233
|
+
if frame.opcode not in self.relevant_opcodes:
|
|
1234
|
+
return
|
|
1235
|
+
digest = hashlib.blake2b(frame.message, digest_size=16).digest()
|
|
1236
|
+
dedupe_key = (_frame_key(frame), digest)
|
|
1237
|
+
if dedupe_key in self._seen_frames:
|
|
1238
|
+
return
|
|
1239
|
+
attempted_frames = self._relevant_frames_retained + 1
|
|
1240
|
+
if attempted_frames > self.capture_limits.max_relevant_frames:
|
|
1241
|
+
self._raise_limit(
|
|
1242
|
+
"max_relevant_frames",
|
|
1243
|
+
self.capture_limits.max_relevant_frames,
|
|
1244
|
+
attempted_frames,
|
|
1245
|
+
)
|
|
1246
|
+
attempted_bytes = self._relevant_bytes_retained + len(frame.message)
|
|
1247
|
+
if attempted_bytes > self.capture_limits.max_relevant_bytes:
|
|
1248
|
+
self._raise_limit(
|
|
1249
|
+
"max_relevant_bytes",
|
|
1250
|
+
self.capture_limits.max_relevant_bytes,
|
|
1251
|
+
attempted_bytes,
|
|
1252
|
+
)
|
|
1253
|
+
self._seen_frames.add(dedupe_key)
|
|
1254
|
+
self._frames.append(frame)
|
|
1255
|
+
self._relevant_frames_retained = attempted_frames
|
|
1256
|
+
self._relevant_bytes_retained = attempted_bytes
|
|
1257
|
+
|
|
1258
|
+
def observe_record(self, record: Any, raw_message: bytes) -> None:
|
|
1259
|
+
del raw_message
|
|
1260
|
+
self.observe_event(toolkit_event_from_record(record))
|
|
1261
|
+
|
|
1262
|
+
def observe_event(self, event: BDOEvent) -> None:
|
|
1263
|
+
"""Retain snapshot records and fail-neutral storage candidates."""
|
|
1264
|
+
|
|
1265
|
+
with self._lock:
|
|
1266
|
+
if self._limit_error is not None:
|
|
1267
|
+
raise self._limit_error
|
|
1268
|
+
if event.event_type not in {
|
|
1269
|
+
"inventory_snapshot",
|
|
1270
|
+
"storage_snapshot",
|
|
1271
|
+
"storage_record",
|
|
1272
|
+
"storage_delta",
|
|
1273
|
+
}:
|
|
1274
|
+
return
|
|
1275
|
+
attempted_evidence = (
|
|
1276
|
+
self._snapshot_records_retained
|
|
1277
|
+
+ len(self._live_storage_boundaries)
|
|
1278
|
+
+ 1
|
|
1279
|
+
)
|
|
1280
|
+
if attempted_evidence > self.capture_limits.max_snapshot_records:
|
|
1281
|
+
self._raise_limit(
|
|
1282
|
+
"max_snapshot_records",
|
|
1283
|
+
self.capture_limits.max_snapshot_records,
|
|
1284
|
+
attempted_evidence,
|
|
1285
|
+
)
|
|
1286
|
+
if event.event_type == "inventory_snapshot":
|
|
1287
|
+
self._inventory_events.append(event)
|
|
1288
|
+
elif event.event_type == "storage_snapshot":
|
|
1289
|
+
self._storage_events.append(event)
|
|
1290
|
+
elif event.event_type == "storage_delta":
|
|
1291
|
+
# A proven live mutation is not snapshot content, but it is a
|
|
1292
|
+
# semantic boundary: neutral records on opposite sides must
|
|
1293
|
+
# never be reconciled into one character-load sweep.
|
|
1294
|
+
self._live_storage_boundaries.append(event)
|
|
1295
|
+
return
|
|
1296
|
+
else:
|
|
1297
|
+
self._neutral_storage_events.append(event)
|
|
1298
|
+
self._snapshot_records_retained += 1
|
|
1299
|
+
|
|
1300
|
+
def _raise_limit(self, limit_name: str, limit: int, attempted: int) -> None:
|
|
1301
|
+
error = ItemStateCaptureLimitError(
|
|
1302
|
+
limit_name=limit_name,
|
|
1303
|
+
limit=limit,
|
|
1304
|
+
attempted=attempted,
|
|
1305
|
+
)
|
|
1306
|
+
self._limit_error = error
|
|
1307
|
+
raise error
|
|
1308
|
+
|
|
1309
|
+
def snapshot(
|
|
1310
|
+
self,
|
|
1311
|
+
*,
|
|
1312
|
+
decoder_health: Optional[DecoderHealth] = None,
|
|
1313
|
+
) -> CharacterStateSnapshot:
|
|
1314
|
+
with self._lock:
|
|
1315
|
+
if self._limit_error is not None:
|
|
1316
|
+
raise self._limit_error
|
|
1317
|
+
frames_seen = self._frames_seen
|
|
1318
|
+
relevant_frames_retained = self._relevant_frames_retained
|
|
1319
|
+
relevant_bytes_retained = self._relevant_bytes_retained
|
|
1320
|
+
snapshot_records_retained = self._snapshot_records_retained
|
|
1321
|
+
frames = tuple(self._frames)
|
|
1322
|
+
inventory_events = tuple(self._inventory_events)
|
|
1323
|
+
storage_events = tuple(self._storage_events)
|
|
1324
|
+
neutral_storage_events = tuple(self._neutral_storage_events)
|
|
1325
|
+
live_storage_boundaries = tuple(self._live_storage_boundaries)
|
|
1326
|
+
|
|
1327
|
+
inventory_generation = _latest_inventory_generation(
|
|
1328
|
+
frames,
|
|
1329
|
+
inventory_events,
|
|
1330
|
+
self.inventory_specs,
|
|
1331
|
+
)
|
|
1332
|
+
inventory_events = inventory_generation.events
|
|
1333
|
+
inventory_anchors = inventory_generation.anchors
|
|
1334
|
+
generation_start = inventory_generation.start
|
|
1335
|
+
generations_seen = inventory_generation.generations_observed
|
|
1336
|
+
if generation_start is not None:
|
|
1337
|
+
# Current captures send the compact inventory hydration first and
|
|
1338
|
+
# storage hydration afterward. It is therefore a clean boundary
|
|
1339
|
+
# between separate character loads while retaining repeated
|
|
1340
|
+
# storage sweeps belonging to the same load.
|
|
1341
|
+
selected_flow_generations = {
|
|
1342
|
+
inventory_generation.flow_generation_key
|
|
1343
|
+
}
|
|
1344
|
+
storage_events = tuple(
|
|
1345
|
+
event
|
|
1346
|
+
for event in storage_events
|
|
1347
|
+
if event.timestamp >= generation_start
|
|
1348
|
+
and _event_flow_generation_key(event) in selected_flow_generations
|
|
1349
|
+
)
|
|
1350
|
+
neutral_storage_events = tuple(
|
|
1351
|
+
event
|
|
1352
|
+
for event in neutral_storage_events
|
|
1353
|
+
if event.timestamp >= generation_start
|
|
1354
|
+
and _event_flow_generation_key(event) in selected_flow_generations
|
|
1355
|
+
)
|
|
1356
|
+
live_storage_boundaries = tuple(
|
|
1357
|
+
event
|
|
1358
|
+
for event in live_storage_boundaries
|
|
1359
|
+
if event.timestamp >= generation_start
|
|
1360
|
+
and _event_flow_generation_key(event) in selected_flow_generations
|
|
1361
|
+
)
|
|
1362
|
+
frames = tuple(
|
|
1363
|
+
frame
|
|
1364
|
+
for frame in frames
|
|
1365
|
+
if frame.context.timestamp >= generation_start
|
|
1366
|
+
and _frame_flow_generation_key(frame) in selected_flow_generations
|
|
1367
|
+
)
|
|
1368
|
+
|
|
1369
|
+
fallback_observations: Optional[
|
|
1370
|
+
tuple[_StorageGroupObservation, ...]
|
|
1371
|
+
] = None
|
|
1372
|
+
sparse_fallback_used = False
|
|
1373
|
+
split_reconciliation_used = False
|
|
1374
|
+
if not storage_events:
|
|
1375
|
+
storage_events, selected_observations = (
|
|
1376
|
+
_character_storage_snapshot_fallback(
|
|
1377
|
+
frames,
|
|
1378
|
+
neutral_storage_events,
|
|
1379
|
+
inventory_anchors,
|
|
1380
|
+
self.storage_specs,
|
|
1381
|
+
)
|
|
1382
|
+
)
|
|
1383
|
+
if selected_observations:
|
|
1384
|
+
fallback_observations = selected_observations
|
|
1385
|
+
sparse_fallback_used = True
|
|
1386
|
+
|
|
1387
|
+
if storage_events and neutral_storage_events:
|
|
1388
|
+
(
|
|
1389
|
+
reconciled_events,
|
|
1390
|
+
reconciled_observations,
|
|
1391
|
+
reconciled_count,
|
|
1392
|
+
) = _reconcile_split_storage_hydration(
|
|
1393
|
+
frames,
|
|
1394
|
+
storage_events,
|
|
1395
|
+
neutral_storage_events,
|
|
1396
|
+
live_storage_boundaries,
|
|
1397
|
+
inventory_anchors,
|
|
1398
|
+
self.storage_specs,
|
|
1399
|
+
)
|
|
1400
|
+
if reconciled_count:
|
|
1401
|
+
storage_events = reconciled_events
|
|
1402
|
+
fallback_observations = reconciled_observations
|
|
1403
|
+
split_reconciliation_used = True
|
|
1404
|
+
|
|
1405
|
+
inventory_assembly = self._inventory_summary(
|
|
1406
|
+
frames,
|
|
1407
|
+
inventory_events,
|
|
1408
|
+
generations_observed=generations_seen,
|
|
1409
|
+
)
|
|
1410
|
+
inventory = inventory_assembly.summary
|
|
1411
|
+
storage_assembly = self._storage_summaries(
|
|
1412
|
+
frames,
|
|
1413
|
+
storage_events,
|
|
1414
|
+
observations=fallback_observations,
|
|
1415
|
+
hydration_anchors=inventory_anchors,
|
|
1416
|
+
)
|
|
1417
|
+
storages = storage_assembly.summaries
|
|
1418
|
+
unresolved_storage = storage_assembly.records_without_destination
|
|
1419
|
+
storage_records_missing_instance = (
|
|
1420
|
+
storage_assembly.records_missing_instance
|
|
1421
|
+
)
|
|
1422
|
+
storage_sweeps_observed = storage_assembly.sweeps_observed
|
|
1423
|
+
selected_storage_sweep = storage_assembly.selected_sweep
|
|
1424
|
+
unknown_empty_envelopes = storage_assembly.unknown_empty_envelopes
|
|
1425
|
+
warnings = [
|
|
1426
|
+
"Initial login and character switch use the same observed hydration "
|
|
1427
|
+
"shape; the packet-level trigger is not decoded.",
|
|
1428
|
+
"Inventory container names are provisional interpretations of a "
|
|
1429
|
+
"dynamically discovered raw record field; use the numeric code as "
|
|
1430
|
+
"the experimental identity.",
|
|
1431
|
+
"Count-zero inventory wrappers contain no record-level slot or "
|
|
1432
|
+
"container field and remain unclassified.",
|
|
1433
|
+
"Storage capacity is not decoded; occupied stacks are not maximum capacity.",
|
|
1434
|
+
"Snapshot completion has no proven end marker; stopping capture during "
|
|
1435
|
+
"loading can produce a partial report.",
|
|
1436
|
+
]
|
|
1437
|
+
if generations_seen > 1:
|
|
1438
|
+
warnings.append(
|
|
1439
|
+
f"{generations_seen} inventory hydration generations were observed; "
|
|
1440
|
+
"the report contains only the latest generation."
|
|
1441
|
+
)
|
|
1442
|
+
if not inventory.hydration_observed:
|
|
1443
|
+
warnings.append(
|
|
1444
|
+
"No inventory snapshot records were decoded; verify that the active "
|
|
1445
|
+
"profile has an inventory opcode, context offset, item instance offset, "
|
|
1446
|
+
"and calibrated single-record length."
|
|
1447
|
+
)
|
|
1448
|
+
if storage_events or storages:
|
|
1449
|
+
warnings.append(
|
|
1450
|
+
"No inventory hydration boundary was decoded; storage state "
|
|
1451
|
+
"diagnostics contain all observed records, while current contents "
|
|
1452
|
+
"use the latest inferred sweep and may span multiple loads."
|
|
1453
|
+
)
|
|
1454
|
+
elif not inventory_events:
|
|
1455
|
+
warnings.append(
|
|
1456
|
+
"Inventory hydration was observed only through calibrated count-zero "
|
|
1457
|
+
"wrappers; the empty current state is preserved, but no record-level "
|
|
1458
|
+
"container metadata was available."
|
|
1459
|
+
)
|
|
1460
|
+
if sparse_fallback_used:
|
|
1461
|
+
warnings.append(
|
|
1462
|
+
"Storage hydration was proven by the dedicated character-load "
|
|
1463
|
+
"boundary plus a broad count-zero/nonempty destination cohort; "
|
|
1464
|
+
"the ordinary live stream remained fail-neutral."
|
|
1465
|
+
)
|
|
1466
|
+
if split_reconciliation_used:
|
|
1467
|
+
warnings.append(
|
|
1468
|
+
"Storage hydration records split across timing bursts were "
|
|
1469
|
+
"reconciled only within one inventory-anchored flow generation, "
|
|
1470
|
+
"opcode family, inferred sweep, and live-mutation boundary."
|
|
1471
|
+
)
|
|
1472
|
+
if not storage_events and not storages:
|
|
1473
|
+
warnings.append(
|
|
1474
|
+
"No storage snapshot records were decoded; the capture may be partial "
|
|
1475
|
+
"or the storage wrapper/profile may have changed."
|
|
1476
|
+
)
|
|
1477
|
+
resolved_health = decoder_health or DecoderHealth()
|
|
1478
|
+
if storages and resolved_health.storage_status == "not_observed":
|
|
1479
|
+
validated_messages = len(fallback_observations or ())
|
|
1480
|
+
resolved_health = replace(
|
|
1481
|
+
resolved_health,
|
|
1482
|
+
storage_status="compatible",
|
|
1483
|
+
storage_messages_observed=max(
|
|
1484
|
+
resolved_health.storage_messages_observed,
|
|
1485
|
+
validated_messages,
|
|
1486
|
+
),
|
|
1487
|
+
storage_messages_decoded=max(
|
|
1488
|
+
resolved_health.storage_messages_decoded,
|
|
1489
|
+
validated_messages,
|
|
1490
|
+
),
|
|
1491
|
+
)
|
|
1492
|
+
unregistered_storage_ids = {
|
|
1493
|
+
storage.storage_id
|
|
1494
|
+
for storage in storages
|
|
1495
|
+
if storage.storage_id not in STORAGE_LOCATIONS
|
|
1496
|
+
}
|
|
1497
|
+
if unregistered_storage_ids:
|
|
1498
|
+
selected_unknown_records = sum(
|
|
1499
|
+
event.storage_id is not None
|
|
1500
|
+
and event.storage_id not in STORAGE_LOCATIONS
|
|
1501
|
+
for event in storage_events
|
|
1502
|
+
)
|
|
1503
|
+
resolved_health = replace(
|
|
1504
|
+
resolved_health,
|
|
1505
|
+
storage_status="incompatible",
|
|
1506
|
+
storage_destination_failures=(
|
|
1507
|
+
max(
|
|
1508
|
+
resolved_health.storage_destination_failures,
|
|
1509
|
+
selected_unknown_records,
|
|
1510
|
+
)
|
|
1511
|
+
+ unknown_empty_envelopes
|
|
1512
|
+
),
|
|
1513
|
+
)
|
|
1514
|
+
if resolved_health.storage_status == "incompatible":
|
|
1515
|
+
warnings.append(
|
|
1516
|
+
"The storage decoder reported an incompatible wrapper, geometry, "
|
|
1517
|
+
"or destination field. Recalibrate before treating missing towns "
|
|
1518
|
+
"as empty."
|
|
1519
|
+
)
|
|
1520
|
+
elif (
|
|
1521
|
+
resolved_health.storage_status == "not_observed" and inventory_anchors
|
|
1522
|
+
):
|
|
1523
|
+
warnings.append(
|
|
1524
|
+
"Inventory hydration was observed, but the calibrated storage "
|
|
1525
|
+
"opcode was not observed. The capture may be partial or the storage "
|
|
1526
|
+
"profile may be stale; not_observed is not proof of compatibility."
|
|
1527
|
+
)
|
|
1528
|
+
if unregistered_storage_ids:
|
|
1529
|
+
warnings.append(
|
|
1530
|
+
f"{len(unregistered_storage_ids)} storage destination ID(s) are "
|
|
1531
|
+
"not in the town registry. Their numeric identities were preserved, "
|
|
1532
|
+
"but display names and name-based queries require a registry update."
|
|
1533
|
+
)
|
|
1534
|
+
if unresolved_storage:
|
|
1535
|
+
warnings.append(
|
|
1536
|
+
f"{unresolved_storage} storage snapshot records lacked a numeric "
|
|
1537
|
+
"destination and were excluded from per-storage state."
|
|
1538
|
+
)
|
|
1539
|
+
if inventory_assembly.missing_instance_records:
|
|
1540
|
+
warnings.append(
|
|
1541
|
+
f"{inventory_assembly.missing_instance_records} inventory snapshot records "
|
|
1542
|
+
"lacked observed instance identity and were excluded from "
|
|
1543
|
+
"distinct-stack state."
|
|
1544
|
+
)
|
|
1545
|
+
if storage_records_missing_instance:
|
|
1546
|
+
warnings.append(
|
|
1547
|
+
f"{storage_records_missing_instance} storage snapshot records "
|
|
1548
|
+
"lacked observed instance identity and were excluded from "
|
|
1549
|
+
"distinct-stack state."
|
|
1550
|
+
)
|
|
1551
|
+
not_selected = sum(not storage.current_state_observed for storage in storages)
|
|
1552
|
+
if storage_sweeps_observed > 1:
|
|
1553
|
+
warnings.append(
|
|
1554
|
+
f"{storage_sweeps_observed} storage sweeps were conservatively "
|
|
1555
|
+
f"inferred; current contents use sweep {selected_storage_sweep}, "
|
|
1556
|
+
"while raw record and group counts cover every observed sweep."
|
|
1557
|
+
)
|
|
1558
|
+
if not_selected:
|
|
1559
|
+
warnings.append(
|
|
1560
|
+
f"The latest inferred storage sweep did not revisit {not_selected} "
|
|
1561
|
+
"earlier-observed destinations. Their older items were excluded "
|
|
1562
|
+
"instead of being reported as current; the selected sweep may be "
|
|
1563
|
+
"partial."
|
|
1564
|
+
)
|
|
1565
|
+
observed_registered = {
|
|
1566
|
+
storage.storage_id
|
|
1567
|
+
for storage in storages
|
|
1568
|
+
if storage.storage_id in STORAGE_LOCATIONS
|
|
1569
|
+
}
|
|
1570
|
+
missing_registered = tuple(
|
|
1571
|
+
storage_id
|
|
1572
|
+
for storage_id in STORAGE_LOCATIONS
|
|
1573
|
+
if storage_id not in observed_registered
|
|
1574
|
+
)
|
|
1575
|
+
if generations_seen:
|
|
1576
|
+
generation_selection = "latest_observed_inventory_hydration"
|
|
1577
|
+
elif storage_events or storages:
|
|
1578
|
+
generation_selection = "all_observed_storage_no_inventory_boundary"
|
|
1579
|
+
else:
|
|
1580
|
+
generation_selection = "none_no_hydration_boundary"
|
|
1581
|
+
return CharacterStateSnapshot(
|
|
1582
|
+
inventory=inventory,
|
|
1583
|
+
storages=StorageContents(storages),
|
|
1584
|
+
provenance=ItemStateProvenance(
|
|
1585
|
+
capture_mode=self.capture_mode,
|
|
1586
|
+
profile_source=self.profile_source,
|
|
1587
|
+
generation_selection=generation_selection,
|
|
1588
|
+
capture_path=self.input_path or self.saved_capture_path,
|
|
1589
|
+
),
|
|
1590
|
+
coverage=ItemStateCoverage(
|
|
1591
|
+
inventory_records_missing_instance=(
|
|
1592
|
+
inventory_assembly.missing_instance_records
|
|
1593
|
+
),
|
|
1594
|
+
storage_records_missing_instance=storage_records_missing_instance,
|
|
1595
|
+
selected_storage_records_missing_instance=sum(
|
|
1596
|
+
diagnostic.selected_missing_instance_records
|
|
1597
|
+
for diagnostic in storage_assembly.diagnostics
|
|
1598
|
+
),
|
|
1599
|
+
registered_storage_ids_not_observed=missing_registered,
|
|
1600
|
+
unregistered_storage_ids_observed=tuple(
|
|
1601
|
+
sorted(unregistered_storage_ids)
|
|
1602
|
+
),
|
|
1603
|
+
storage_locations_not_selected=not_selected,
|
|
1604
|
+
storage_locations_with_incomplete_current_identity=sum(
|
|
1605
|
+
storage.current_state_observed
|
|
1606
|
+
and storage.current_identity_complete is False
|
|
1607
|
+
for storage in storages
|
|
1608
|
+
),
|
|
1609
|
+
),
|
|
1610
|
+
decoder_health=resolved_health,
|
|
1611
|
+
warnings=tuple(warnings),
|
|
1612
|
+
diagnostics=ItemStateDiagnostics(
|
|
1613
|
+
frames_seen=frames_seen,
|
|
1614
|
+
relevant_frames_retained=relevant_frames_retained,
|
|
1615
|
+
relevant_bytes_retained=relevant_bytes_retained,
|
|
1616
|
+
snapshot_records_retained=snapshot_records_retained,
|
|
1617
|
+
capture_limits=self.capture_limits,
|
|
1618
|
+
inventory=inventory_assembly.diagnostics,
|
|
1619
|
+
storage=StorageHydrationDiagnostics(
|
|
1620
|
+
records_decoded=len(storage_events),
|
|
1621
|
+
records_without_destination=unresolved_storage,
|
|
1622
|
+
sweeps_observed=storage_sweeps_observed,
|
|
1623
|
+
selected_sweep=selected_storage_sweep,
|
|
1624
|
+
destinations=storage_assembly.diagnostics,
|
|
1625
|
+
),
|
|
1626
|
+
),
|
|
1627
|
+
)
|
|
1628
|
+
|
|
1629
|
+
def _inventory_summary(
|
|
1630
|
+
self,
|
|
1631
|
+
frames: tuple[BDOFrame, ...],
|
|
1632
|
+
events: tuple[BDOEvent, ...],
|
|
1633
|
+
*,
|
|
1634
|
+
generations_observed: int,
|
|
1635
|
+
) -> _InventoryAssembly:
|
|
1636
|
+
groups: dict[_FrameKey, list[BDOEvent]] = {}
|
|
1637
|
+
for event in events:
|
|
1638
|
+
groups.setdefault(_event_frame_key(event), []).append(event)
|
|
1639
|
+
|
|
1640
|
+
frames_by_key = {_frame_key(frame): frame for frame in frames}
|
|
1641
|
+
specs_by_opcode = _spec_candidates_by_opcode(self.inventory_specs)
|
|
1642
|
+
multi_groups_by_spec: dict[
|
|
1643
|
+
EventSpec, list[tuple[BDOFrame, list[BDOEvent], EventSpec, int]]
|
|
1644
|
+
] = {}
|
|
1645
|
+
strides_by_key: dict[_FrameKey, int] = {}
|
|
1646
|
+
selected_specs_by_key: dict[_FrameKey, EventSpec] = {}
|
|
1647
|
+
sibling_strides: dict[EventSpec, set[int]] = {}
|
|
1648
|
+
prefix_candidates: dict[EventSpec, set[int]] = {}
|
|
1649
|
+
|
|
1650
|
+
# A multi-record frame proves its own stride from L, B, and N. Layout
|
|
1651
|
+
# discovery is intentionally separate: stride alone does not prove
|
|
1652
|
+
# where slot/container metadata moved in a new protocol generation.
|
|
1653
|
+
for key, group in groups.items():
|
|
1654
|
+
frame = frames_by_key.get(key)
|
|
1655
|
+
if frame is None:
|
|
1656
|
+
continue
|
|
1657
|
+
selected = _unique_inventory_multi_layout(
|
|
1658
|
+
frame,
|
|
1659
|
+
group,
|
|
1660
|
+
specs_by_opcode.get(frame.opcode, ()),
|
|
1661
|
+
)
|
|
1662
|
+
if selected is None:
|
|
1663
|
+
continue
|
|
1664
|
+
spec, stride = selected
|
|
1665
|
+
strides_by_key[key] = stride
|
|
1666
|
+
selected_specs_by_key[key] = spec
|
|
1667
|
+
sibling_strides.setdefault(spec, set()).add(stride)
|
|
1668
|
+
prefix_candidates.setdefault(spec, set()).add(
|
|
1669
|
+
frame.length - len(group) * stride
|
|
1670
|
+
)
|
|
1671
|
+
multi_groups_by_spec.setdefault(spec, []).append(
|
|
1672
|
+
(frame, group, spec, stride)
|
|
1673
|
+
)
|
|
1674
|
+
|
|
1675
|
+
tail_layouts = {
|
|
1676
|
+
spec: _discover_inventory_tail_layout(frame_groups)
|
|
1677
|
+
for spec, frame_groups in multi_groups_by_spec.items()
|
|
1678
|
+
}
|
|
1679
|
+
header_container_offsets = {
|
|
1680
|
+
spec: _discover_inventory_header_container_offset(frame_groups)
|
|
1681
|
+
for spec, frame_groups in multi_groups_by_spec.items()
|
|
1682
|
+
if tail_layouts.get(spec) is None
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
# A calibrated single-record base and repeat stride also prove the
|
|
1686
|
+
# zero-record prefix. This preserves empty inventory hydration as an
|
|
1687
|
+
# observed state even when the capture contains no occupied records.
|
|
1688
|
+
for candidates in specs_by_opcode.values():
|
|
1689
|
+
for spec in candidates:
|
|
1690
|
+
if (
|
|
1691
|
+
spec.single_record_message_length is not None
|
|
1692
|
+
and spec.repeat_stride is not None
|
|
1693
|
+
and spec.repeat_stride > 0
|
|
1694
|
+
and spec.single_record_message_length > spec.repeat_stride
|
|
1695
|
+
):
|
|
1696
|
+
prefix_candidates.setdefault(spec, set()).add(
|
|
1697
|
+
spec.single_record_message_length - spec.repeat_stride
|
|
1698
|
+
)
|
|
1699
|
+
metadata_by_record: dict[tuple[_FrameKey, int], _InventoryRecordMetadata] = {}
|
|
1700
|
+
for key, group in groups.items():
|
|
1701
|
+
frame = frames_by_key.get(key)
|
|
1702
|
+
if frame is None:
|
|
1703
|
+
continue
|
|
1704
|
+
spec_for_group = selected_specs_by_key.get(key)
|
|
1705
|
+
stride_for_group = strides_by_key.get(key)
|
|
1706
|
+
if stride_for_group is None and len(group) == 1:
|
|
1707
|
+
selected = _unique_inventory_single_layout(
|
|
1708
|
+
frame,
|
|
1709
|
+
group,
|
|
1710
|
+
specs_by_opcode.get(frame.opcode, ()),
|
|
1711
|
+
sibling_strides,
|
|
1712
|
+
)
|
|
1713
|
+
if selected is not None:
|
|
1714
|
+
spec_for_group, stride_for_group = selected
|
|
1715
|
+
selected_specs_by_key[key] = spec_for_group
|
|
1716
|
+
strides_by_key[key] = stride_for_group
|
|
1717
|
+
if spec_for_group is None or stride_for_group is None:
|
|
1718
|
+
continue
|
|
1719
|
+
tail_layout = tail_layouts.get(spec_for_group)
|
|
1720
|
+
if tail_layout is not None:
|
|
1721
|
+
extracted = _inventory_record_metadata(
|
|
1722
|
+
frame,
|
|
1723
|
+
group,
|
|
1724
|
+
spec_for_group,
|
|
1725
|
+
stride_for_group,
|
|
1726
|
+
tail_layout,
|
|
1727
|
+
)
|
|
1728
|
+
else:
|
|
1729
|
+
header_offset = header_container_offsets.get(spec_for_group)
|
|
1730
|
+
if header_offset is None:
|
|
1731
|
+
continue
|
|
1732
|
+
extracted = _inventory_header_metadata(
|
|
1733
|
+
frame,
|
|
1734
|
+
group,
|
|
1735
|
+
header_offset,
|
|
1736
|
+
)
|
|
1737
|
+
if extracted is None:
|
|
1738
|
+
continue
|
|
1739
|
+
metadata_by_record.update(
|
|
1740
|
+
{
|
|
1741
|
+
(key, record_offset): metadata
|
|
1742
|
+
for record_offset, metadata in extracted.items()
|
|
1743
|
+
}
|
|
1744
|
+
)
|
|
1745
|
+
|
|
1746
|
+
latest: dict[str, SnapshotItem] = {}
|
|
1747
|
+
missing_instance = 0
|
|
1748
|
+
for event in events:
|
|
1749
|
+
if event.item_instance is None:
|
|
1750
|
+
missing_instance += 1
|
|
1751
|
+
continue
|
|
1752
|
+
instance = event.item_instance
|
|
1753
|
+
metadata = (
|
|
1754
|
+
metadata_by_record.get((_event_frame_key(event), event.record_offset))
|
|
1755
|
+
if event.record_offset is not None
|
|
1756
|
+
else None
|
|
1757
|
+
)
|
|
1758
|
+
latest[instance] = _snapshot_item(event, instance, metadata)
|
|
1759
|
+
|
|
1760
|
+
prefixes = {
|
|
1761
|
+
spec: next(iter(candidates))
|
|
1762
|
+
for spec, candidates in prefix_candidates.items()
|
|
1763
|
+
if len(candidates) == 1
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
frame_counts: list[int] = []
|
|
1767
|
+
counted_keys: set[_FrameKey] = set()
|
|
1768
|
+
for frame in frames:
|
|
1769
|
+
key = _frame_key(frame)
|
|
1770
|
+
frame_group = groups.get(key)
|
|
1771
|
+
if frame_group is not None and key in selected_specs_by_key:
|
|
1772
|
+
frame_counts.append(len(frame_group))
|
|
1773
|
+
counted_keys.add(key)
|
|
1774
|
+
continue
|
|
1775
|
+
empty_matches = [
|
|
1776
|
+
candidate
|
|
1777
|
+
for candidate in specs_by_opcode.get(frame.opcode, ())
|
|
1778
|
+
if _frame_has_zero_context(frame, candidate)
|
|
1779
|
+
and prefixes.get(candidate) == frame.length
|
|
1780
|
+
]
|
|
1781
|
+
if frame_group is None and len(empty_matches) == 1:
|
|
1782
|
+
frame_counts.append(0)
|
|
1783
|
+
counted_keys.add(key)
|
|
1784
|
+
|
|
1785
|
+
# Events can still be useful when a caller feeds normalized records
|
|
1786
|
+
# without generic frame observations.
|
|
1787
|
+
for key, group in groups.items():
|
|
1788
|
+
if key not in counted_keys:
|
|
1789
|
+
frame_counts.append(len(group))
|
|
1790
|
+
|
|
1791
|
+
identified_records = len(events) - missing_instance
|
|
1792
|
+
duplicate_records = identified_records - len(latest)
|
|
1793
|
+
|
|
1794
|
+
latest_records = tuple(sorted(latest.values(), key=lambda item: item.instance))
|
|
1795
|
+
items = tuple(item for item in latest_records if not item.is_currency_balance)
|
|
1796
|
+
currency_balances = tuple(
|
|
1797
|
+
item for item in latest_records if item.is_currency_balance
|
|
1798
|
+
)
|
|
1799
|
+
source_opcodes = {
|
|
1800
|
+
frame.opcode
|
|
1801
|
+
for frame in frames
|
|
1802
|
+
if _frame_key(frame) in counted_keys
|
|
1803
|
+
}
|
|
1804
|
+
message_lengths = {
|
|
1805
|
+
frame.length
|
|
1806
|
+
for frame in frames
|
|
1807
|
+
if _frame_key(frame) in counted_keys
|
|
1808
|
+
}
|
|
1809
|
+
source_opcodes.update(
|
|
1810
|
+
event.opcode for event in events if event.opcode is not None
|
|
1811
|
+
)
|
|
1812
|
+
message_lengths.update(
|
|
1813
|
+
event.message_length
|
|
1814
|
+
for event in events
|
|
1815
|
+
if isinstance(event.message_length, int)
|
|
1816
|
+
and not isinstance(event.message_length, bool)
|
|
1817
|
+
)
|
|
1818
|
+
return _InventoryAssembly(
|
|
1819
|
+
summary=InventorySnapshotSummary(
|
|
1820
|
+
hydration_observed=bool(frame_counts),
|
|
1821
|
+
items=items,
|
|
1822
|
+
currency_balances=currency_balances,
|
|
1823
|
+
),
|
|
1824
|
+
missing_instance_records=missing_instance,
|
|
1825
|
+
diagnostics=InventoryHydrationDiagnostics(
|
|
1826
|
+
raw_records=len(events),
|
|
1827
|
+
duplicate_records=duplicate_records,
|
|
1828
|
+
group_counts=tuple(frame_counts),
|
|
1829
|
+
inferred_strides=tuple(
|
|
1830
|
+
sorted(
|
|
1831
|
+
{
|
|
1832
|
+
stride
|
|
1833
|
+
for strides in sibling_strides.values()
|
|
1834
|
+
for stride in strides
|
|
1835
|
+
}
|
|
1836
|
+
)
|
|
1837
|
+
),
|
|
1838
|
+
generations_observed=generations_observed,
|
|
1839
|
+
source_opcodes=tuple(sorted(source_opcodes)),
|
|
1840
|
+
message_lengths=tuple(sorted(message_lengths)),
|
|
1841
|
+
),
|
|
1842
|
+
)
|
|
1843
|
+
|
|
1844
|
+
def _storage_summaries(
|
|
1845
|
+
self,
|
|
1846
|
+
frames: tuple[BDOFrame, ...],
|
|
1847
|
+
events: tuple[BDOEvent, ...],
|
|
1848
|
+
*,
|
|
1849
|
+
observations: Optional[tuple[_StorageGroupObservation, ...]] = None,
|
|
1850
|
+
hydration_anchors: Iterable[_HydrationAnchor] = (),
|
|
1851
|
+
) -> _StorageAssembly:
|
|
1852
|
+
events = tuple(events)
|
|
1853
|
+
unresolved = sum(event.storage_id is None for event in events)
|
|
1854
|
+
records_missing_instance = sum(
|
|
1855
|
+
event.storage_instance is None for event in events
|
|
1856
|
+
)
|
|
1857
|
+
resolved_events = tuple(
|
|
1858
|
+
event for event in events if event.storage_id is not None
|
|
1859
|
+
)
|
|
1860
|
+
if observations is None:
|
|
1861
|
+
observations = _storage_group_observations(
|
|
1862
|
+
frames,
|
|
1863
|
+
resolved_events,
|
|
1864
|
+
self.storage_specs,
|
|
1865
|
+
hydration_anchors=hydration_anchors,
|
|
1866
|
+
)
|
|
1867
|
+
unknown_empty_envelopes = sum(
|
|
1868
|
+
observation.empty and observation.storage_id not in STORAGE_LOCATIONS
|
|
1869
|
+
for observation in observations
|
|
1870
|
+
)
|
|
1871
|
+
sweeps = _infer_storage_sweeps(observations)
|
|
1872
|
+
selected_sweep = len(sweeps) if sweeps else None
|
|
1873
|
+
selected_blocks = (
|
|
1874
|
+
{block.storage_id: block for block in sweeps[-1]} if sweeps else {}
|
|
1875
|
+
)
|
|
1876
|
+
|
|
1877
|
+
raw_counts: dict[int, int] = {}
|
|
1878
|
+
missing_instance_counts: dict[int, int] = {}
|
|
1879
|
+
all_records: dict[int, dict[str, SnapshotItem]] = {}
|
|
1880
|
+
group_counts: dict[int, int] = {}
|
|
1881
|
+
empty_ids: set[int] = set()
|
|
1882
|
+
source_opcodes: dict[int, set[int]] = {}
|
|
1883
|
+
message_lengths: dict[int, set[int]] = {}
|
|
1884
|
+
for observation in observations:
|
|
1885
|
+
storage_id = observation.storage_id
|
|
1886
|
+
raw_counts[storage_id] = (
|
|
1887
|
+
raw_counts.get(storage_id, 0) + observation.raw_records
|
|
1888
|
+
)
|
|
1889
|
+
missing_instance_counts[storage_id] = (
|
|
1890
|
+
missing_instance_counts.get(storage_id, 0)
|
|
1891
|
+
+ observation.missing_instance_records
|
|
1892
|
+
)
|
|
1893
|
+
group_counts[storage_id] = group_counts.get(storage_id, 0) + 1
|
|
1894
|
+
source_opcodes.setdefault(storage_id, set()).add(observation.opcode)
|
|
1895
|
+
if observation.message_length is not None:
|
|
1896
|
+
message_lengths.setdefault(storage_id, set()).add(
|
|
1897
|
+
observation.message_length
|
|
1898
|
+
)
|
|
1899
|
+
if observation.empty:
|
|
1900
|
+
empty_ids.add(storage_id)
|
|
1901
|
+
for item in observation.items:
|
|
1902
|
+
all_records.setdefault(storage_id, {})[item.instance] = item
|
|
1903
|
+
|
|
1904
|
+
sweeps_by_storage: dict[int, int] = {}
|
|
1905
|
+
for sweep in sweeps:
|
|
1906
|
+
for block in sweep:
|
|
1907
|
+
sweeps_by_storage[block.storage_id] = (
|
|
1908
|
+
sweeps_by_storage.get(block.storage_id, 0) + 1
|
|
1909
|
+
)
|
|
1910
|
+
|
|
1911
|
+
all_ids = set(group_counts)
|
|
1912
|
+
summaries: list[StorageSnapshotSummary] = []
|
|
1913
|
+
diagnostics_by_id: dict[int, StorageDestinationDiagnostics] = {}
|
|
1914
|
+
for storage_id in all_ids:
|
|
1915
|
+
location = storage_location(storage_id)
|
|
1916
|
+
selected_block = selected_blocks.get(storage_id)
|
|
1917
|
+
current_state_observed = selected_block is not None
|
|
1918
|
+
items_by_instance: dict[str, SnapshotItem] = {}
|
|
1919
|
+
selected_records = 0
|
|
1920
|
+
selected_groups = 0
|
|
1921
|
+
selected_missing_instance_records = 0
|
|
1922
|
+
current_empty: Optional[bool] = None
|
|
1923
|
+
if selected_block is not None:
|
|
1924
|
+
selected_records = sum(
|
|
1925
|
+
group.raw_records for group in selected_block.groups
|
|
1926
|
+
)
|
|
1927
|
+
selected_groups = len(selected_block.groups)
|
|
1928
|
+
selected_missing_instance_records = sum(
|
|
1929
|
+
group.missing_instance_records for group in selected_block.groups
|
|
1930
|
+
)
|
|
1931
|
+
current_empty = selected_block.empty
|
|
1932
|
+
if not current_empty:
|
|
1933
|
+
for group in selected_block.groups:
|
|
1934
|
+
for item in group.items:
|
|
1935
|
+
items_by_instance[item.instance] = item
|
|
1936
|
+
raw_count = raw_counts.get(storage_id, 0)
|
|
1937
|
+
missing_instance_records = missing_instance_counts.get(storage_id, 0)
|
|
1938
|
+
groups = group_counts.get(storage_id, 0)
|
|
1939
|
+
summaries.append(
|
|
1940
|
+
StorageSnapshotSummary(
|
|
1941
|
+
storage_id=storage_id,
|
|
1942
|
+
name=location.name if location is not None else None,
|
|
1943
|
+
name_confidence=(
|
|
1944
|
+
location.confidence if location is not None else None
|
|
1945
|
+
),
|
|
1946
|
+
items=tuple(
|
|
1947
|
+
sorted(
|
|
1948
|
+
items_by_instance.values(), key=lambda item: item.instance
|
|
1949
|
+
)
|
|
1950
|
+
),
|
|
1951
|
+
current_state_observed=current_state_observed,
|
|
1952
|
+
current_empty=current_empty,
|
|
1953
|
+
current_identity_complete=(
|
|
1954
|
+
selected_missing_instance_records == 0
|
|
1955
|
+
if current_state_observed
|
|
1956
|
+
else None
|
|
1957
|
+
),
|
|
1958
|
+
)
|
|
1959
|
+
)
|
|
1960
|
+
diagnostics_by_id[storage_id] = StorageDestinationDiagnostics(
|
|
1961
|
+
storage_id=storage_id,
|
|
1962
|
+
raw_records=raw_count,
|
|
1963
|
+
duplicate_records=max(
|
|
1964
|
+
0,
|
|
1965
|
+
raw_count
|
|
1966
|
+
- missing_instance_records
|
|
1967
|
+
- len(all_records.get(storage_id, {})),
|
|
1968
|
+
),
|
|
1969
|
+
groups=groups,
|
|
1970
|
+
empty_envelope_seen=storage_id in empty_ids,
|
|
1971
|
+
selected_records=selected_records,
|
|
1972
|
+
selected_groups=selected_groups,
|
|
1973
|
+
sweeps_observed=sweeps_by_storage.get(storage_id, 0),
|
|
1974
|
+
selected_sweep=(selected_sweep if current_state_observed else None),
|
|
1975
|
+
missing_instance_records=missing_instance_records,
|
|
1976
|
+
selected_missing_instance_records=selected_missing_instance_records,
|
|
1977
|
+
source_opcodes=tuple(sorted(source_opcodes.get(storage_id, ()))),
|
|
1978
|
+
message_lengths=tuple(sorted(message_lengths.get(storage_id, ()))),
|
|
1979
|
+
)
|
|
1980
|
+
|
|
1981
|
+
summaries.sort(
|
|
1982
|
+
key=lambda summary: (
|
|
1983
|
+
summary.name is None,
|
|
1984
|
+
summary.name.casefold() if summary.name is not None else "",
|
|
1985
|
+
summary.storage_id,
|
|
1986
|
+
)
|
|
1987
|
+
)
|
|
1988
|
+
return _StorageAssembly(
|
|
1989
|
+
summaries=tuple(summaries),
|
|
1990
|
+
diagnostics=tuple(
|
|
1991
|
+
diagnostics_by_id[summary.storage_id] for summary in summaries
|
|
1992
|
+
),
|
|
1993
|
+
records_without_destination=unresolved,
|
|
1994
|
+
records_missing_instance=records_missing_instance,
|
|
1995
|
+
sweeps_observed=len(sweeps),
|
|
1996
|
+
selected_sweep=selected_sweep,
|
|
1997
|
+
unknown_empty_envelopes=unknown_empty_envelopes,
|
|
1998
|
+
)
|
|
1999
|
+
|
|
2000
|
+
|
|
2001
|
+
def _frame_has_zero_context(frame: BDOFrame, spec: EventSpec) -> bool:
|
|
2002
|
+
if spec.source_context_offset is None:
|
|
2003
|
+
return False
|
|
2004
|
+
start = spec.source_context_offset
|
|
2005
|
+
end = start + spec.source_context_length
|
|
2006
|
+
return (
|
|
2007
|
+
end <= len(frame.message) and frame.message[start:end] == CHARACTER_LOAD_CONTEXT
|
|
2008
|
+
)
|
|
2009
|
+
|
|
2010
|
+
|
|
2011
|
+
def _latest_inventory_generation(
|
|
2012
|
+
frames: tuple[BDOFrame, ...],
|
|
2013
|
+
events: tuple[BDOEvent, ...],
|
|
2014
|
+
specs: Iterable[EventSpec],
|
|
2015
|
+
) -> _InventoryGeneration:
|
|
2016
|
+
"""Select the latest inventory burst, including proven count-zero wrappers."""
|
|
2017
|
+
specs_by_opcode = _spec_candidates_by_opcode(specs)
|
|
2018
|
+
observations = {
|
|
2019
|
+
_HydrationAnchor(event.timestamp, _event_frame_key(event)) for event in events
|
|
2020
|
+
}
|
|
2021
|
+
event_frame_keys = {_event_frame_key(event) for event in events}
|
|
2022
|
+
for frame in frames:
|
|
2023
|
+
frame_key = _frame_key(frame)
|
|
2024
|
+
if frame_key in event_frame_keys:
|
|
2025
|
+
continue
|
|
2026
|
+
empty_matches = [
|
|
2027
|
+
spec
|
|
2028
|
+
for spec in specs_by_opcode.get(frame.opcode, ())
|
|
2029
|
+
if spec.single_record_message_length is not None
|
|
2030
|
+
and spec.repeat_stride is not None
|
|
2031
|
+
and spec.repeat_stride > 0
|
|
2032
|
+
and spec.single_record_message_length > spec.repeat_stride
|
|
2033
|
+
and frame.length == spec.single_record_message_length - spec.repeat_stride
|
|
2034
|
+
and _frame_has_zero_context(frame, spec)
|
|
2035
|
+
]
|
|
2036
|
+
if len(empty_matches) == 1:
|
|
2037
|
+
observations.add(_HydrationAnchor(frame.context.timestamp, frame_key))
|
|
2038
|
+
|
|
2039
|
+
if not observations:
|
|
2040
|
+
return _InventoryGeneration(
|
|
2041
|
+
events=events,
|
|
2042
|
+
anchors=(),
|
|
2043
|
+
start=None,
|
|
2044
|
+
flow_generation_key=None,
|
|
2045
|
+
generations_observed=0,
|
|
2046
|
+
)
|
|
2047
|
+
|
|
2048
|
+
ordered = sorted(
|
|
2049
|
+
observations,
|
|
2050
|
+
key=lambda item: (
|
|
2051
|
+
item.timestamp,
|
|
2052
|
+
item.frame_key.source_ip,
|
|
2053
|
+
item.frame_key.source_port,
|
|
2054
|
+
item.frame_key.destination_ip,
|
|
2055
|
+
item.frame_key.destination_port,
|
|
2056
|
+
item.frame_key.flow_generation,
|
|
2057
|
+
item.frame_key.stream_sequence is None,
|
|
2058
|
+
item.frame_key.stream_sequence or 0,
|
|
2059
|
+
),
|
|
2060
|
+
)
|
|
2061
|
+
first = ordered[0]
|
|
2062
|
+
previous_generation_key = _anchor_flow_generation_key(first)
|
|
2063
|
+
generation_starts = [first.timestamp]
|
|
2064
|
+
generation_keys = [previous_generation_key]
|
|
2065
|
+
previous_timestamp = first.timestamp
|
|
2066
|
+
for observation in ordered[1:]:
|
|
2067
|
+
timestamp = observation.timestamp
|
|
2068
|
+
generation_key = _anchor_flow_generation_key(observation)
|
|
2069
|
+
if (
|
|
2070
|
+
timestamp - previous_timestamp > _INVENTORY_GENERATION_GAP_SECONDS
|
|
2071
|
+
or generation_key != previous_generation_key
|
|
2072
|
+
):
|
|
2073
|
+
generation_starts.append(timestamp)
|
|
2074
|
+
generation_keys.append(generation_key)
|
|
2075
|
+
previous_timestamp = timestamp
|
|
2076
|
+
previous_generation_key = generation_key
|
|
2077
|
+
|
|
2078
|
+
latest_start = generation_starts[-1]
|
|
2079
|
+
latest_key = generation_keys[-1]
|
|
2080
|
+
return _InventoryGeneration(
|
|
2081
|
+
events=tuple(
|
|
2082
|
+
event
|
|
2083
|
+
for event in events
|
|
2084
|
+
if event.timestamp >= latest_start
|
|
2085
|
+
and _event_flow_generation_key(event) == latest_key
|
|
2086
|
+
),
|
|
2087
|
+
anchors=tuple(
|
|
2088
|
+
observation
|
|
2089
|
+
for observation in ordered
|
|
2090
|
+
if observation.timestamp >= latest_start
|
|
2091
|
+
and _anchor_flow_generation_key(observation) == latest_key
|
|
2092
|
+
),
|
|
2093
|
+
start=latest_start,
|
|
2094
|
+
flow_generation_key=latest_key,
|
|
2095
|
+
generations_observed=len(generation_starts),
|
|
2096
|
+
)
|
|
2097
|
+
|
|
2098
|
+
|
|
2099
|
+
def _character_storage_snapshot_fallback(
|
|
2100
|
+
frames: Iterable[BDOFrame],
|
|
2101
|
+
neutral_events: Iterable[BDOEvent],
|
|
2102
|
+
hydration_anchors: Iterable[_HydrationAnchor],
|
|
2103
|
+
specs: Iterable[EventSpec],
|
|
2104
|
+
) -> tuple[tuple[BDOEvent, ...], tuple[_StorageGroupObservation, ...]]:
|
|
2105
|
+
"""Prove a sparse storage hydration cohort inside the dedicated API.
|
|
2106
|
+
|
|
2107
|
+
The general live classifier intentionally requires eight *populated*
|
|
2108
|
+
destinations so an uncorrelated worker batch cannot become a snapshot.
|
|
2109
|
+
Character-load capture additionally retains exact count-zero envelopes.
|
|
2110
|
+
Those envelopes can prove the same broad, tightly timed town sweep for an
|
|
2111
|
+
account with only a few populated storages without weakening live event
|
|
2112
|
+
filtering for every application.
|
|
2113
|
+
"""
|
|
2114
|
+
|
|
2115
|
+
frames = tuple(frames)
|
|
2116
|
+
neutral_events = tuple(neutral_events)
|
|
2117
|
+
hydration_anchors = tuple(hydration_anchors)
|
|
2118
|
+
if not hydration_anchors:
|
|
2119
|
+
return (), ()
|
|
2120
|
+
|
|
2121
|
+
anchors: dict[tuple[str, int, str, int, int], float] = {}
|
|
2122
|
+
for hydration_anchor in hydration_anchors:
|
|
2123
|
+
key = _anchor_flow_generation_key(hydration_anchor)
|
|
2124
|
+
anchors[key] = max(
|
|
2125
|
+
anchors.get(key, hydration_anchor.timestamp),
|
|
2126
|
+
hydration_anchor.timestamp,
|
|
2127
|
+
)
|
|
2128
|
+
|
|
2129
|
+
observations = _storage_group_observations(
|
|
2130
|
+
frames,
|
|
2131
|
+
neutral_events,
|
|
2132
|
+
specs,
|
|
2133
|
+
hydration_anchors=hydration_anchors,
|
|
2134
|
+
)
|
|
2135
|
+
by_stream: dict[
|
|
2136
|
+
tuple[str, int, str, int, int, int],
|
|
2137
|
+
list[_StorageGroupObservation],
|
|
2138
|
+
] = {}
|
|
2139
|
+
for observation in observations:
|
|
2140
|
+
frame_key = observation.frame_key
|
|
2141
|
+
stream_key = (
|
|
2142
|
+
frame_key.source_ip,
|
|
2143
|
+
frame_key.source_port,
|
|
2144
|
+
frame_key.destination_ip,
|
|
2145
|
+
frame_key.destination_port,
|
|
2146
|
+
frame_key.flow_generation,
|
|
2147
|
+
observation.opcode,
|
|
2148
|
+
)
|
|
2149
|
+
by_stream.setdefault(stream_key, []).append(observation)
|
|
2150
|
+
|
|
2151
|
+
candidates: list[tuple[_StorageGroupObservation, ...]] = []
|
|
2152
|
+
for stream_key, stream_observations in by_stream.items():
|
|
2153
|
+
anchor = anchors.get(stream_key[:5])
|
|
2154
|
+
if anchor is None:
|
|
2155
|
+
continue
|
|
2156
|
+
ordered = sorted(
|
|
2157
|
+
stream_observations,
|
|
2158
|
+
key=lambda observation: observation.timestamp,
|
|
2159
|
+
)
|
|
2160
|
+
burst: list[_StorageGroupObservation] = []
|
|
2161
|
+
|
|
2162
|
+
def consider() -> None:
|
|
2163
|
+
if not burst:
|
|
2164
|
+
return
|
|
2165
|
+
distinct_destinations = {
|
|
2166
|
+
observation.storage_id
|
|
2167
|
+
for observation in burst
|
|
2168
|
+
if observation.storage_id > 0
|
|
2169
|
+
}
|
|
2170
|
+
if (
|
|
2171
|
+
len(distinct_destinations) >= _STORAGE_HYDRATION_MIN_DESTINATIONS
|
|
2172
|
+
and any(observation.empty for observation in burst)
|
|
2173
|
+
and anchor <= burst[0].timestamp
|
|
2174
|
+
and burst[-1].timestamp - anchor
|
|
2175
|
+
<= _STORAGE_HYDRATION_EPOCH_SECONDS
|
|
2176
|
+
):
|
|
2177
|
+
candidates.append(tuple(burst))
|
|
2178
|
+
|
|
2179
|
+
for observation in ordered:
|
|
2180
|
+
if burst and (
|
|
2181
|
+
observation.timestamp - burst[-1].timestamp
|
|
2182
|
+
> _STORAGE_HYDRATION_BURST_GAP_SECONDS
|
|
2183
|
+
or observation.timestamp - burst[0].timestamp
|
|
2184
|
+
> _STORAGE_HYDRATION_MAX_BURST_SECONDS
|
|
2185
|
+
):
|
|
2186
|
+
consider()
|
|
2187
|
+
burst = []
|
|
2188
|
+
burst.append(observation)
|
|
2189
|
+
consider()
|
|
2190
|
+
|
|
2191
|
+
if not candidates:
|
|
2192
|
+
return (), ()
|
|
2193
|
+
selected = max(candidates, key=lambda cohort: cohort[-1].timestamp)
|
|
2194
|
+
selected_records = {
|
|
2195
|
+
(observation.frame_key, observation.timestamp, observation.storage_id)
|
|
2196
|
+
for observation in selected
|
|
2197
|
+
if not observation.empty
|
|
2198
|
+
}
|
|
2199
|
+
promoted: list[BDOEvent] = []
|
|
2200
|
+
for event in neutral_events:
|
|
2201
|
+
if (
|
|
2202
|
+
_event_frame_key(event),
|
|
2203
|
+
event.timestamp,
|
|
2204
|
+
event.storage_id,
|
|
2205
|
+
) not in selected_records:
|
|
2206
|
+
continue
|
|
2207
|
+
promoted.append(
|
|
2208
|
+
replace(
|
|
2209
|
+
event,
|
|
2210
|
+
event_type="storage_snapshot",
|
|
2211
|
+
source=None,
|
|
2212
|
+
)
|
|
2213
|
+
)
|
|
2214
|
+
return tuple(promoted), selected
|
|
2215
|
+
|
|
2216
|
+
|
|
2217
|
+
def _reconcile_split_storage_hydration(
|
|
2218
|
+
frames: Iterable[BDOFrame],
|
|
2219
|
+
snapshot_events: Iterable[BDOEvent],
|
|
2220
|
+
neutral_events: Iterable[BDOEvent],
|
|
2221
|
+
live_boundaries: Iterable[BDOEvent],
|
|
2222
|
+
hydration_anchors: Iterable[_HydrationAnchor],
|
|
2223
|
+
specs: Iterable[EventSpec],
|
|
2224
|
+
) -> tuple[
|
|
2225
|
+
tuple[BDOEvent, ...],
|
|
2226
|
+
tuple[_StorageGroupObservation, ...],
|
|
2227
|
+
int,
|
|
2228
|
+
]:
|
|
2229
|
+
"""Extend a proven snapshot sweep across harmless timing-burst splits.
|
|
2230
|
+
|
|
2231
|
+
The continuous live classifier deliberately treats a timing gap as a
|
|
2232
|
+
fail-neutral boundary. The dedicated character-state API has stronger
|
|
2233
|
+
evidence: a selected inventory hydration generation plus storage records
|
|
2234
|
+
that were already promoted by the live classifier. Neutral records may
|
|
2235
|
+
join such a proven sweep only on the same flow generation and opcode, in
|
|
2236
|
+
the bounded inventory epoch, and without crossing a positive live
|
|
2237
|
+
mutation. Repeated destinations remain separate sweeps through
|
|
2238
|
+
:func:`_infer_storage_sweeps`.
|
|
2239
|
+
"""
|
|
2240
|
+
|
|
2241
|
+
frames = tuple(frames)
|
|
2242
|
+
snapshot_events = tuple(snapshot_events)
|
|
2243
|
+
neutral_events = tuple(neutral_events)
|
|
2244
|
+
live_boundaries = tuple(live_boundaries)
|
|
2245
|
+
hydration_anchors = tuple(hydration_anchors)
|
|
2246
|
+
specs = tuple(specs)
|
|
2247
|
+
if not snapshot_events or not neutral_events or not hydration_anchors:
|
|
2248
|
+
return snapshot_events, (), 0
|
|
2249
|
+
|
|
2250
|
+
anchors: dict[tuple[str, int, str, int, int], float] = {}
|
|
2251
|
+
for hydration_anchor in hydration_anchors:
|
|
2252
|
+
key = _anchor_flow_generation_key(hydration_anchor)
|
|
2253
|
+
anchors[key] = max(
|
|
2254
|
+
anchors.get(key, hydration_anchor.timestamp),
|
|
2255
|
+
hydration_anchor.timestamp,
|
|
2256
|
+
)
|
|
2257
|
+
|
|
2258
|
+
confirmed_groups = {
|
|
2259
|
+
(_event_frame_key(event), event.timestamp, event.storage_id)
|
|
2260
|
+
for event in snapshot_events
|
|
2261
|
+
if event.storage_id is not None
|
|
2262
|
+
}
|
|
2263
|
+
observations = _storage_group_observations(
|
|
2264
|
+
frames,
|
|
2265
|
+
(*snapshot_events, *neutral_events),
|
|
2266
|
+
specs,
|
|
2267
|
+
hydration_anchors=hydration_anchors,
|
|
2268
|
+
)
|
|
2269
|
+
|
|
2270
|
+
boundaries_by_flow: dict[
|
|
2271
|
+
tuple[str, int, str, int, int],
|
|
2272
|
+
list[tuple[float, bool, int]],
|
|
2273
|
+
] = {}
|
|
2274
|
+
for event in live_boundaries:
|
|
2275
|
+
frame_key = _event_frame_key(event)
|
|
2276
|
+
boundaries_by_flow.setdefault(
|
|
2277
|
+
_event_flow_generation_key(event),
|
|
2278
|
+
[],
|
|
2279
|
+
).append(
|
|
2280
|
+
(
|
|
2281
|
+
event.timestamp,
|
|
2282
|
+
frame_key.stream_sequence is None,
|
|
2283
|
+
frame_key.stream_sequence or 0,
|
|
2284
|
+
)
|
|
2285
|
+
)
|
|
2286
|
+
for boundaries in boundaries_by_flow.values():
|
|
2287
|
+
boundaries.sort()
|
|
2288
|
+
|
|
2289
|
+
cohorts: dict[
|
|
2290
|
+
tuple[str, int, str, int, int, int, int],
|
|
2291
|
+
list[_StorageGroupObservation],
|
|
2292
|
+
] = {}
|
|
2293
|
+
for observation in observations:
|
|
2294
|
+
frame_key = observation.frame_key
|
|
2295
|
+
flow_key = (
|
|
2296
|
+
frame_key.source_ip,
|
|
2297
|
+
frame_key.source_port,
|
|
2298
|
+
frame_key.destination_ip,
|
|
2299
|
+
frame_key.destination_port,
|
|
2300
|
+
frame_key.flow_generation,
|
|
2301
|
+
)
|
|
2302
|
+
anchor = anchors.get(flow_key)
|
|
2303
|
+
if (
|
|
2304
|
+
anchor is None
|
|
2305
|
+
or observation.timestamp < anchor
|
|
2306
|
+
or observation.timestamp - anchor > _STORAGE_HYDRATION_EPOCH_SECONDS
|
|
2307
|
+
):
|
|
2308
|
+
continue
|
|
2309
|
+
order_key = (
|
|
2310
|
+
observation.timestamp,
|
|
2311
|
+
frame_key.stream_sequence is None,
|
|
2312
|
+
frame_key.stream_sequence or 0,
|
|
2313
|
+
)
|
|
2314
|
+
boundary_index = sum(
|
|
2315
|
+
boundary <= order_key
|
|
2316
|
+
for boundary in boundaries_by_flow.get(flow_key, ())
|
|
2317
|
+
)
|
|
2318
|
+
cohorts.setdefault(
|
|
2319
|
+
(*flow_key, observation.opcode, boundary_index),
|
|
2320
|
+
[],
|
|
2321
|
+
).append(observation)
|
|
2322
|
+
|
|
2323
|
+
eligible_groups: set[tuple[_FrameKey, float, int]] = set()
|
|
2324
|
+
for cohort in cohorts.values():
|
|
2325
|
+
for sweep in _infer_storage_sweeps(cohort):
|
|
2326
|
+
sweep_groups = {
|
|
2327
|
+
(group.frame_key, group.timestamp, group.storage_id)
|
|
2328
|
+
for block in sweep
|
|
2329
|
+
for group in block.groups
|
|
2330
|
+
if not group.empty
|
|
2331
|
+
}
|
|
2332
|
+
if sweep_groups & confirmed_groups:
|
|
2333
|
+
eligible_groups.update(sweep_groups)
|
|
2334
|
+
|
|
2335
|
+
promoted: list[BDOEvent] = []
|
|
2336
|
+
for event in neutral_events:
|
|
2337
|
+
group_key = (
|
|
2338
|
+
_event_frame_key(event),
|
|
2339
|
+
event.timestamp,
|
|
2340
|
+
event.storage_id,
|
|
2341
|
+
)
|
|
2342
|
+
if (
|
|
2343
|
+
event.storage_id is None
|
|
2344
|
+
or group_key in confirmed_groups
|
|
2345
|
+
or group_key not in eligible_groups
|
|
2346
|
+
):
|
|
2347
|
+
continue
|
|
2348
|
+
promoted.append(
|
|
2349
|
+
replace(
|
|
2350
|
+
event,
|
|
2351
|
+
event_type="storage_snapshot",
|
|
2352
|
+
source=None,
|
|
2353
|
+
)
|
|
2354
|
+
)
|
|
2355
|
+
|
|
2356
|
+
if not promoted:
|
|
2357
|
+
return snapshot_events, (), 0
|
|
2358
|
+
|
|
2359
|
+
reconciled = tuple(
|
|
2360
|
+
sorted(
|
|
2361
|
+
(*snapshot_events, *promoted),
|
|
2362
|
+
key=lambda event: (
|
|
2363
|
+
event.timestamp,
|
|
2364
|
+
event.flow.source_ip,
|
|
2365
|
+
event.flow.source_port,
|
|
2366
|
+
event.flow.destination_ip,
|
|
2367
|
+
event.flow.destination_port,
|
|
2368
|
+
_event_frame_key(event).flow_generation,
|
|
2369
|
+
_event_frame_key(event).stream_sequence is None,
|
|
2370
|
+
_event_frame_key(event).stream_sequence or 0,
|
|
2371
|
+
event.record_index or 0,
|
|
2372
|
+
),
|
|
2373
|
+
)
|
|
2374
|
+
)
|
|
2375
|
+
reconciled_observations = _storage_group_observations(
|
|
2376
|
+
frames,
|
|
2377
|
+
reconciled,
|
|
2378
|
+
specs,
|
|
2379
|
+
hydration_anchors=hydration_anchors,
|
|
2380
|
+
)
|
|
2381
|
+
return reconciled, reconciled_observations, len(promoted)
|
|
2382
|
+
|
|
2383
|
+
|
|
2384
|
+
def _storage_group_observations(
|
|
2385
|
+
frames: Iterable[BDOFrame],
|
|
2386
|
+
events: Iterable[BDOEvent],
|
|
2387
|
+
specs: Iterable[EventSpec],
|
|
2388
|
+
*,
|
|
2389
|
+
hydration_anchors: Iterable[_HydrationAnchor] = (),
|
|
2390
|
+
) -> tuple[_StorageGroupObservation, ...]:
|
|
2391
|
+
"""Merge record-bearing frames and empty wrappers into capture order."""
|
|
2392
|
+
frames = tuple(frames)
|
|
2393
|
+
events = tuple(events)
|
|
2394
|
+
event_groups: dict[tuple[_FrameKey, float, int], list[BDOEvent]] = {}
|
|
2395
|
+
for event in events:
|
|
2396
|
+
if event.storage_id is None:
|
|
2397
|
+
continue
|
|
2398
|
+
event_groups.setdefault(
|
|
2399
|
+
(_event_frame_key(event), event.timestamp, event.storage_id),
|
|
2400
|
+
[],
|
|
2401
|
+
).append(event)
|
|
2402
|
+
|
|
2403
|
+
observations: list[_StorageGroupObservation] = []
|
|
2404
|
+
for (frame_key, timestamp, storage_id), group in event_groups.items():
|
|
2405
|
+
items_by_instance: dict[str, SnapshotItem] = {}
|
|
2406
|
+
missing_instance_records = 0
|
|
2407
|
+
for event in group:
|
|
2408
|
+
instance = event.storage_instance
|
|
2409
|
+
if instance is None:
|
|
2410
|
+
missing_instance_records += 1
|
|
2411
|
+
continue
|
|
2412
|
+
items_by_instance[instance] = _snapshot_item(event, instance)
|
|
2413
|
+
observations.append(
|
|
2414
|
+
_StorageGroupObservation(
|
|
2415
|
+
storage_id=storage_id,
|
|
2416
|
+
frame_key=frame_key,
|
|
2417
|
+
timestamp=timestamp,
|
|
2418
|
+
opcode=group[0].opcode or 0,
|
|
2419
|
+
message_length=group[0].message_length,
|
|
2420
|
+
items=tuple(
|
|
2421
|
+
sorted(
|
|
2422
|
+
items_by_instance.values(),
|
|
2423
|
+
key=lambda item: item.instance,
|
|
2424
|
+
)
|
|
2425
|
+
),
|
|
2426
|
+
raw_records=len(group),
|
|
2427
|
+
missing_instance_records=missing_instance_records,
|
|
2428
|
+
empty=False,
|
|
2429
|
+
)
|
|
2430
|
+
)
|
|
2431
|
+
|
|
2432
|
+
empty_schemas, hydration_windows = _storage_empty_schemas(
|
|
2433
|
+
events,
|
|
2434
|
+
specs,
|
|
2435
|
+
frames=frames,
|
|
2436
|
+
hydration_anchors=hydration_anchors,
|
|
2437
|
+
)
|
|
2438
|
+
for frame in frames:
|
|
2439
|
+
matches: list[int] = []
|
|
2440
|
+
for schema in empty_schemas.get(frame.opcode, ()):
|
|
2441
|
+
candidate_storage_id = _empty_storage_envelope_id(
|
|
2442
|
+
frame,
|
|
2443
|
+
schema,
|
|
2444
|
+
hydration_windows,
|
|
2445
|
+
)
|
|
2446
|
+
if candidate_storage_id is not None:
|
|
2447
|
+
matches.append(candidate_storage_id)
|
|
2448
|
+
if len(matches) != 1:
|
|
2449
|
+
continue
|
|
2450
|
+
storage_id = matches[0]
|
|
2451
|
+
observations.append(
|
|
2452
|
+
_StorageGroupObservation(
|
|
2453
|
+
storage_id=storage_id,
|
|
2454
|
+
frame_key=_frame_key(frame),
|
|
2455
|
+
timestamp=frame.context.timestamp,
|
|
2456
|
+
opcode=frame.opcode,
|
|
2457
|
+
message_length=frame.length,
|
|
2458
|
+
items=(),
|
|
2459
|
+
raw_records=0,
|
|
2460
|
+
missing_instance_records=0,
|
|
2461
|
+
empty=True,
|
|
2462
|
+
)
|
|
2463
|
+
)
|
|
2464
|
+
|
|
2465
|
+
observations.sort(
|
|
2466
|
+
key=lambda observation: (
|
|
2467
|
+
observation.timestamp,
|
|
2468
|
+
observation.frame_key.source_ip,
|
|
2469
|
+
observation.frame_key.source_port,
|
|
2470
|
+
observation.frame_key.destination_ip,
|
|
2471
|
+
observation.frame_key.destination_port,
|
|
2472
|
+
observation.frame_key.flow_generation,
|
|
2473
|
+
observation.frame_key.stream_sequence is None,
|
|
2474
|
+
observation.frame_key.stream_sequence or 0,
|
|
2475
|
+
observation.opcode,
|
|
2476
|
+
observation.storage_id,
|
|
2477
|
+
observation.empty,
|
|
2478
|
+
)
|
|
2479
|
+
)
|
|
2480
|
+
return tuple(observations)
|
|
2481
|
+
|
|
2482
|
+
|
|
2483
|
+
def _infer_storage_sweeps(
|
|
2484
|
+
observations: Iterable[_StorageGroupObservation],
|
|
2485
|
+
) -> tuple[tuple[_StorageDestinationBlock, ...], ...]:
|
|
2486
|
+
"""Conservatively split ordered destination blocks into repeated sweeps.
|
|
2487
|
+
|
|
2488
|
+
Tightly timed consecutive nonempty groups for one destination are record
|
|
2489
|
+
chunks and stay together. A state change or a gap beyond the observed
|
|
2490
|
+
chunk window closes the block even when the destination is unchanged.
|
|
2491
|
+
Once a closed destination appears again, that repeat begins a new sweep.
|
|
2492
|
+
"""
|
|
2493
|
+
blocks: list[_StorageDestinationBlock] = []
|
|
2494
|
+
current_storage_id: Optional[int] = None
|
|
2495
|
+
current_stream_key: Optional[tuple[str, int, str, int, int, int]] = None
|
|
2496
|
+
current_groups: list[_StorageGroupObservation] = []
|
|
2497
|
+
current_empty: Optional[bool] = None
|
|
2498
|
+
|
|
2499
|
+
def close_current() -> None:
|
|
2500
|
+
nonlocal current_storage_id, current_stream_key, current_groups, current_empty
|
|
2501
|
+
if (
|
|
2502
|
+
current_storage_id is not None
|
|
2503
|
+
and current_stream_key is not None
|
|
2504
|
+
and current_groups
|
|
2505
|
+
):
|
|
2506
|
+
blocks.append(
|
|
2507
|
+
_StorageDestinationBlock(
|
|
2508
|
+
storage_id=current_storage_id,
|
|
2509
|
+
stream_key=current_stream_key,
|
|
2510
|
+
groups=tuple(current_groups),
|
|
2511
|
+
)
|
|
2512
|
+
)
|
|
2513
|
+
current_storage_id = None
|
|
2514
|
+
current_stream_key = None
|
|
2515
|
+
current_groups = []
|
|
2516
|
+
current_empty = None
|
|
2517
|
+
|
|
2518
|
+
for observation in observations:
|
|
2519
|
+
observation_stream_key = (
|
|
2520
|
+
observation.frame_key.source_ip,
|
|
2521
|
+
observation.frame_key.source_port,
|
|
2522
|
+
observation.frame_key.destination_ip,
|
|
2523
|
+
observation.frame_key.destination_port,
|
|
2524
|
+
observation.frame_key.flow_generation,
|
|
2525
|
+
observation.opcode,
|
|
2526
|
+
)
|
|
2527
|
+
previous_timestamp = current_groups[-1].timestamp if current_groups else None
|
|
2528
|
+
if current_storage_id is not None and (
|
|
2529
|
+
observation.storage_id != current_storage_id
|
|
2530
|
+
or observation_stream_key != current_stream_key
|
|
2531
|
+
or observation.empty != current_empty
|
|
2532
|
+
or (
|
|
2533
|
+
previous_timestamp is not None
|
|
2534
|
+
and observation.timestamp - previous_timestamp
|
|
2535
|
+
> _STORAGE_DESTINATION_CHUNK_GAP_SECONDS
|
|
2536
|
+
)
|
|
2537
|
+
):
|
|
2538
|
+
close_current()
|
|
2539
|
+
if current_storage_id is None:
|
|
2540
|
+
current_storage_id = observation.storage_id
|
|
2541
|
+
current_stream_key = observation_stream_key
|
|
2542
|
+
current_empty = observation.empty
|
|
2543
|
+
current_groups.append(observation)
|
|
2544
|
+
close_current()
|
|
2545
|
+
|
|
2546
|
+
sweeps: list[tuple[_StorageDestinationBlock, ...]] = []
|
|
2547
|
+
current_sweep: list[_StorageDestinationBlock] = []
|
|
2548
|
+
destinations_seen: set[int] = set()
|
|
2549
|
+
current_sweep_stream: Optional[tuple[str, int, str, int, int, int]] = None
|
|
2550
|
+
for block in blocks:
|
|
2551
|
+
if current_sweep and (
|
|
2552
|
+
block.storage_id in destinations_seen
|
|
2553
|
+
or block.stream_key != current_sweep_stream
|
|
2554
|
+
):
|
|
2555
|
+
sweeps.append(tuple(current_sweep))
|
|
2556
|
+
current_sweep = []
|
|
2557
|
+
destinations_seen = set()
|
|
2558
|
+
if not current_sweep:
|
|
2559
|
+
current_sweep_stream = block.stream_key
|
|
2560
|
+
current_sweep.append(block)
|
|
2561
|
+
destinations_seen.add(block.storage_id)
|
|
2562
|
+
if current_sweep:
|
|
2563
|
+
sweeps.append(tuple(current_sweep))
|
|
2564
|
+
return tuple(sweeps)
|
|
2565
|
+
|
|
2566
|
+
|
|
2567
|
+
def _anchored_empty_prefix_lengths(
|
|
2568
|
+
frames: Iterable[BDOFrame],
|
|
2569
|
+
events: Iterable[BDOEvent],
|
|
2570
|
+
spec: EventSpec,
|
|
2571
|
+
hydration_windows: dict[
|
|
2572
|
+
tuple[str, int, str, int, int, int],
|
|
2573
|
+
tuple[float, float],
|
|
2574
|
+
],
|
|
2575
|
+
) -> set[int]:
|
|
2576
|
+
"""Infer an empty-wrapper length from a broad anchored town cohort.
|
|
2577
|
+
|
|
2578
|
+
Older complete profiles may predate ``repeat_stride`` persistence. The
|
|
2579
|
+
dedicated character-load boundary still lets us prove a prefix without a
|
|
2580
|
+
patch table: exact count-zero wrappers at one length plus decoded nonempty
|
|
2581
|
+
records must form a tightly timed cohort spanning at least eight numeric
|
|
2582
|
+
destinations. Competing lengths fail closed.
|
|
2583
|
+
"""
|
|
2584
|
+
|
|
2585
|
+
count_offset = spec.record_count_offset
|
|
2586
|
+
destination_offset = spec.source_context_offset
|
|
2587
|
+
if count_offset is None or destination_offset is None:
|
|
2588
|
+
return set()
|
|
2589
|
+
|
|
2590
|
+
empty_by_stream_and_length: dict[
|
|
2591
|
+
tuple[tuple[str, int, str, int, int, int], int],
|
|
2592
|
+
list[tuple[float, int, bool]],
|
|
2593
|
+
] = {}
|
|
2594
|
+
for frame in frames:
|
|
2595
|
+
if frame.opcode != spec.opcode or frame.length > spec.item_offset:
|
|
2596
|
+
continue
|
|
2597
|
+
if max(count_offset + 2, destination_offset + 4) > len(frame.message):
|
|
2598
|
+
continue
|
|
2599
|
+
if int.from_bytes(frame.message[count_offset : count_offset + 2], "little"):
|
|
2600
|
+
continue
|
|
2601
|
+
storage_id = int.from_bytes(
|
|
2602
|
+
frame.message[destination_offset : destination_offset + 4],
|
|
2603
|
+
"little",
|
|
2604
|
+
)
|
|
2605
|
+
if storage_id == 0:
|
|
2606
|
+
continue
|
|
2607
|
+
stream_key = (
|
|
2608
|
+
frame.context.flow.source_ip,
|
|
2609
|
+
frame.context.flow.source_port,
|
|
2610
|
+
frame.context.flow.destination_ip,
|
|
2611
|
+
frame.context.flow.destination_port,
|
|
2612
|
+
frame.context.flow_generation,
|
|
2613
|
+
frame.opcode,
|
|
2614
|
+
)
|
|
2615
|
+
window = hydration_windows.get(stream_key)
|
|
2616
|
+
if window is None or not window[0] <= frame.context.timestamp <= window[1]:
|
|
2617
|
+
continue
|
|
2618
|
+
empty_by_stream_and_length.setdefault(
|
|
2619
|
+
(stream_key, frame.length),
|
|
2620
|
+
[],
|
|
2621
|
+
).append((frame.context.timestamp, storage_id, True))
|
|
2622
|
+
|
|
2623
|
+
nonempty_by_stream: dict[
|
|
2624
|
+
tuple[str, int, str, int, int, int],
|
|
2625
|
+
list[tuple[float, int, bool]],
|
|
2626
|
+
] = {}
|
|
2627
|
+
for event in events:
|
|
2628
|
+
if event.opcode != spec.opcode or not event.storage_id:
|
|
2629
|
+
continue
|
|
2630
|
+
frame_key = _event_frame_key(event)
|
|
2631
|
+
stream_key = (
|
|
2632
|
+
event.flow.source_ip,
|
|
2633
|
+
event.flow.source_port,
|
|
2634
|
+
event.flow.destination_ip,
|
|
2635
|
+
event.flow.destination_port,
|
|
2636
|
+
frame_key.flow_generation,
|
|
2637
|
+
spec.opcode,
|
|
2638
|
+
)
|
|
2639
|
+
nonempty_by_stream.setdefault(stream_key, []).append(
|
|
2640
|
+
(event.timestamp, event.storage_id, False)
|
|
2641
|
+
)
|
|
2642
|
+
|
|
2643
|
+
proven: set[int] = set()
|
|
2644
|
+
for (stream_key, prefix_length), empty_points in empty_by_stream_and_length.items():
|
|
2645
|
+
points = sorted(
|
|
2646
|
+
empty_points + nonempty_by_stream.get(stream_key, []),
|
|
2647
|
+
key=lambda point: point[0],
|
|
2648
|
+
)
|
|
2649
|
+
burst: list[tuple[float, int, bool]] = []
|
|
2650
|
+
|
|
2651
|
+
def consider() -> None:
|
|
2652
|
+
if (
|
|
2653
|
+
burst
|
|
2654
|
+
and any(point[2] for point in burst)
|
|
2655
|
+
and len({point[1] for point in burst})
|
|
2656
|
+
>= _STORAGE_HYDRATION_MIN_DESTINATIONS
|
|
2657
|
+
):
|
|
2658
|
+
proven.add(prefix_length)
|
|
2659
|
+
|
|
2660
|
+
for point in points:
|
|
2661
|
+
if burst and (
|
|
2662
|
+
point[0] - burst[-1][0] > _STORAGE_HYDRATION_BURST_GAP_SECONDS
|
|
2663
|
+
or point[0] - burst[0][0] > _STORAGE_HYDRATION_MAX_BURST_SECONDS
|
|
2664
|
+
):
|
|
2665
|
+
consider()
|
|
2666
|
+
burst = []
|
|
2667
|
+
burst.append(point)
|
|
2668
|
+
consider()
|
|
2669
|
+
return proven
|
|
2670
|
+
|
|
2671
|
+
|
|
2672
|
+
def _storage_empty_schemas(
|
|
2673
|
+
events: Iterable[BDOEvent],
|
|
2674
|
+
specs: Iterable[EventSpec],
|
|
2675
|
+
*,
|
|
2676
|
+
frames: Iterable[BDOFrame] = (),
|
|
2677
|
+
hydration_anchors: Iterable[_HydrationAnchor] = (),
|
|
2678
|
+
) -> tuple[
|
|
2679
|
+
dict[int, tuple[_StorageEmptySchema, ...]],
|
|
2680
|
+
dict[tuple[str, int, str, int, int, int], tuple[float, float]],
|
|
2681
|
+
]:
|
|
2682
|
+
"""Learn empty-wrapper geometry from profile authority and live records."""
|
|
2683
|
+
|
|
2684
|
+
events = tuple(events)
|
|
2685
|
+
frames = tuple(frames)
|
|
2686
|
+
hydration_anchors = tuple(hydration_anchors)
|
|
2687
|
+
storage_specs = tuple(
|
|
2688
|
+
spec for spec in specs if spec.label == "INVENTORY_TO_STORAGE"
|
|
2689
|
+
)
|
|
2690
|
+
by_opcode = _spec_candidates_by_opcode(storage_specs)
|
|
2691
|
+
observed_strides: dict[EventSpec, set[int]] = {
|
|
2692
|
+
spec: set() for spec in storage_specs
|
|
2693
|
+
}
|
|
2694
|
+
windows: dict[tuple[str, int, str, int, int, int], tuple[float, float]] = {}
|
|
2695
|
+
groups: dict[tuple[_FrameKey, float, Optional[int]], list[BDOEvent]] = {}
|
|
2696
|
+
for event in events:
|
|
2697
|
+
groups.setdefault(
|
|
2698
|
+
(_event_frame_key(event), event.timestamp, event.opcode),
|
|
2699
|
+
[],
|
|
2700
|
+
).append(event)
|
|
2701
|
+
if event.opcode is None:
|
|
2702
|
+
continue
|
|
2703
|
+
window_key = (
|
|
2704
|
+
event.flow.source_ip,
|
|
2705
|
+
event.flow.source_port,
|
|
2706
|
+
event.flow.destination_ip,
|
|
2707
|
+
event.flow.destination_port,
|
|
2708
|
+
_event_frame_key(event).flow_generation,
|
|
2709
|
+
event.opcode,
|
|
2710
|
+
)
|
|
2711
|
+
previous = windows.get(window_key)
|
|
2712
|
+
windows[window_key] = (
|
|
2713
|
+
event.timestamp if previous is None else min(previous[0], event.timestamp),
|
|
2714
|
+
event.timestamp if previous is None else max(previous[1], event.timestamp),
|
|
2715
|
+
)
|
|
2716
|
+
|
|
2717
|
+
# The dedicated character-state API has a stronger semantic boundary than
|
|
2718
|
+
# the continuous event stream: a proven inventory hydration generation.
|
|
2719
|
+
# It may therefore retain exact count-zero storage envelopes even when an
|
|
2720
|
+
# account has too few populated towns to satisfy the live classifier's
|
|
2721
|
+
# conservative nonempty-destination threshold. The profile still supplies
|
|
2722
|
+
# the authoritative opcode, count column, destination column, and stride;
|
|
2723
|
+
# this only supplies the bounded time window in which those envelopes may
|
|
2724
|
+
# represent the same character-load cohort.
|
|
2725
|
+
for anchor in hydration_anchors:
|
|
2726
|
+
anchor_key = anchor.frame_key
|
|
2727
|
+
for spec in storage_specs:
|
|
2728
|
+
window_key = (
|
|
2729
|
+
anchor_key.source_ip,
|
|
2730
|
+
anchor_key.source_port,
|
|
2731
|
+
anchor_key.destination_ip,
|
|
2732
|
+
anchor_key.destination_port,
|
|
2733
|
+
anchor_key.flow_generation,
|
|
2734
|
+
spec.opcode,
|
|
2735
|
+
)
|
|
2736
|
+
start = anchor.timestamp
|
|
2737
|
+
end = anchor.timestamp + _STORAGE_HYDRATION_EPOCH_SECONDS
|
|
2738
|
+
previous = windows.get(window_key)
|
|
2739
|
+
windows[window_key] = (
|
|
2740
|
+
start if previous is None else min(previous[0], start),
|
|
2741
|
+
end if previous is None else max(previous[1], end),
|
|
2742
|
+
)
|
|
2743
|
+
|
|
2744
|
+
for (_frame_key_value, _timestamp, opcode), group in groups.items():
|
|
2745
|
+
if opcode is None or len(group) < 2:
|
|
2746
|
+
continue
|
|
2747
|
+
offsets = sorted(
|
|
2748
|
+
event.record_offset
|
|
2749
|
+
for event in group
|
|
2750
|
+
if isinstance(event.record_offset, int)
|
|
2751
|
+
and not isinstance(event.record_offset, bool)
|
|
2752
|
+
)
|
|
2753
|
+
if len(offsets) != len(group):
|
|
2754
|
+
continue
|
|
2755
|
+
strides = {later - earlier for earlier, later in zip(offsets, offsets[1:])}
|
|
2756
|
+
if len(strides) != 1:
|
|
2757
|
+
continue
|
|
2758
|
+
stride = next(iter(strides))
|
|
2759
|
+
if stride <= 0:
|
|
2760
|
+
continue
|
|
2761
|
+
for spec in by_opcode.get(opcode, ()):
|
|
2762
|
+
base_length = spec.single_record_message_length
|
|
2763
|
+
if base_length is None or offsets[0] != spec.item_offset:
|
|
2764
|
+
continue
|
|
2765
|
+
message_lengths = {
|
|
2766
|
+
event.message_length
|
|
2767
|
+
for event in group
|
|
2768
|
+
if isinstance(event.message_length, int)
|
|
2769
|
+
and not isinstance(event.message_length, bool)
|
|
2770
|
+
}
|
|
2771
|
+
if len(message_lengths) != 1:
|
|
2772
|
+
continue
|
|
2773
|
+
message_length = next(iter(message_lengths))
|
|
2774
|
+
if message_length != base_length + (len(group) - 1) * stride:
|
|
2775
|
+
continue
|
|
2776
|
+
observed_strides[spec].add(stride)
|
|
2777
|
+
|
|
2778
|
+
schemas_by_opcode: dict[int, list[_StorageEmptySchema]] = {}
|
|
2779
|
+
for spec in storage_specs:
|
|
2780
|
+
strides = observed_strides[spec]
|
|
2781
|
+
selected_stride: Optional[int] = None
|
|
2782
|
+
prefix_length: Optional[int] = None
|
|
2783
|
+
if len(strides) == 1:
|
|
2784
|
+
selected_stride = next(iter(strides))
|
|
2785
|
+
elif len(strides) > 1:
|
|
2786
|
+
# Conflicting same-capture record geometry is stronger evidence
|
|
2787
|
+
# of ambiguity than a coincidental count-zero cohort. Fail closed.
|
|
2788
|
+
continue
|
|
2789
|
+
elif spec.repeat_stride is not None:
|
|
2790
|
+
selected_stride = spec.repeat_stride
|
|
2791
|
+
else:
|
|
2792
|
+
# Only a profile with no record-stride authority may learn its
|
|
2793
|
+
# empty-envelope length from the dedicated character-load cohort.
|
|
2794
|
+
# A weaker cohort must never override observed or calibrated
|
|
2795
|
+
# repeated-record geometry.
|
|
2796
|
+
inferred_prefixes = _anchored_empty_prefix_lengths(
|
|
2797
|
+
frames,
|
|
2798
|
+
events,
|
|
2799
|
+
spec,
|
|
2800
|
+
windows,
|
|
2801
|
+
)
|
|
2802
|
+
if len(inferred_prefixes) != 1:
|
|
2803
|
+
continue
|
|
2804
|
+
prefix_length = next(iter(inferred_prefixes))
|
|
2805
|
+
base_length = spec.single_record_message_length
|
|
2806
|
+
count_offset = spec.record_count_offset
|
|
2807
|
+
destination_offset = spec.source_context_offset
|
|
2808
|
+
if (
|
|
2809
|
+
count_offset is None
|
|
2810
|
+
or destination_offset is None
|
|
2811
|
+
):
|
|
2812
|
+
continue
|
|
2813
|
+
if prefix_length is None:
|
|
2814
|
+
if base_length is None or selected_stride is None:
|
|
2815
|
+
continue
|
|
2816
|
+
prefix_length = base_length - selected_stride
|
|
2817
|
+
if (
|
|
2818
|
+
prefix_length < 5
|
|
2819
|
+
or count_offset + 2 > prefix_length
|
|
2820
|
+
or destination_offset + 4 > prefix_length
|
|
2821
|
+
or prefix_length > spec.item_offset
|
|
2822
|
+
):
|
|
2823
|
+
continue
|
|
2824
|
+
schemas_by_opcode.setdefault(spec.opcode, []).append(
|
|
2825
|
+
_StorageEmptySchema(spec=spec, prefix_length=prefix_length)
|
|
2826
|
+
)
|
|
2827
|
+
return (
|
|
2828
|
+
{opcode: tuple(schemas) for opcode, schemas in schemas_by_opcode.items()},
|
|
2829
|
+
windows,
|
|
2830
|
+
)
|
|
2831
|
+
|
|
2832
|
+
|
|
2833
|
+
def _empty_storage_envelope_id(
|
|
2834
|
+
frame: BDOFrame,
|
|
2835
|
+
schema: _StorageEmptySchema,
|
|
2836
|
+
hydration_windows: dict[
|
|
2837
|
+
tuple[str, int, str, int, int, int],
|
|
2838
|
+
tuple[float, float],
|
|
2839
|
+
],
|
|
2840
|
+
) -> Optional[int]:
|
|
2841
|
+
"""Return a town only from a learned, in-cohort count-zero envelope."""
|
|
2842
|
+
|
|
2843
|
+
spec = schema.spec
|
|
2844
|
+
count_offset = spec.record_count_offset
|
|
2845
|
+
destination_offset = spec.source_context_offset
|
|
2846
|
+
if count_offset is None or destination_offset is None:
|
|
2847
|
+
return None
|
|
2848
|
+
if frame.length != schema.prefix_length or frame.length > spec.item_offset:
|
|
2849
|
+
return None
|
|
2850
|
+
if max(count_offset + 2, destination_offset + 4) > len(frame.message):
|
|
2851
|
+
return None
|
|
2852
|
+
if int.from_bytes(frame.message[count_offset : count_offset + 2], "little") != 0:
|
|
2853
|
+
return None
|
|
2854
|
+
window_key = (
|
|
2855
|
+
frame.context.flow.source_ip,
|
|
2856
|
+
frame.context.flow.source_port,
|
|
2857
|
+
frame.context.flow.destination_ip,
|
|
2858
|
+
frame.context.flow.destination_port,
|
|
2859
|
+
frame.context.flow_generation,
|
|
2860
|
+
frame.opcode,
|
|
2861
|
+
)
|
|
2862
|
+
window = hydration_windows.get(window_key)
|
|
2863
|
+
if window is None or not (
|
|
2864
|
+
window[0] - _STORAGE_EMPTY_WINDOW_MARGIN_SECONDS
|
|
2865
|
+
<= frame.context.timestamp
|
|
2866
|
+
<= window[1] + _STORAGE_EMPTY_WINDOW_MARGIN_SECONDS
|
|
2867
|
+
):
|
|
2868
|
+
return None
|
|
2869
|
+
storage_id = int.from_bytes(
|
|
2870
|
+
frame.message[destination_offset : destination_offset + 4],
|
|
2871
|
+
"little",
|
|
2872
|
+
)
|
|
2873
|
+
# The calibrated column is authoritative even when a new town has not yet
|
|
2874
|
+
# been added to the display-name registry. Preserve its numeric identity so
|
|
2875
|
+
# coverage and decoder health can report the unresolved mapping instead of
|
|
2876
|
+
# silently dropping an otherwise proven empty destination.
|
|
2877
|
+
return storage_id or None
|
|
2878
|
+
|
|
2879
|
+
|
|
2880
|
+
def _validate_item_state_identity_specs(specs: Iterable[EventSpec]) -> None:
|
|
2881
|
+
"""Reject layouts that cannot prove complete character-state semantics."""
|
|
2882
|
+
missing: list[tuple[EventSpec, tuple[str, ...]]] = []
|
|
2883
|
+
for spec in specs:
|
|
2884
|
+
fields: list[str] = []
|
|
2885
|
+
if spec.label == "INVENTORY_TRANSFER":
|
|
2886
|
+
if spec.item_instance_offset is None:
|
|
2887
|
+
fields.append("item instance")
|
|
2888
|
+
if spec.source_context_offset is None:
|
|
2889
|
+
fields.append("snapshot context")
|
|
2890
|
+
elif spec.label == "INVENTORY_TO_STORAGE":
|
|
2891
|
+
if spec.storage_instance_offset is None:
|
|
2892
|
+
fields.append("storage instance")
|
|
2893
|
+
if spec.source_context_offset is None:
|
|
2894
|
+
fields.append("storage destination")
|
|
2895
|
+
if spec.record_count_offset is None:
|
|
2896
|
+
fields.append("record count")
|
|
2897
|
+
if fields:
|
|
2898
|
+
missing.append((spec, tuple(fields)))
|
|
2899
|
+
if not missing:
|
|
2900
|
+
return
|
|
2901
|
+
descriptions = ", ".join(
|
|
2902
|
+
f"{spec.label}(0x{spec.opcode:04X}: {', '.join(fields)})"
|
|
2903
|
+
for spec, fields in missing
|
|
2904
|
+
)
|
|
2905
|
+
raise ProfileError(
|
|
2906
|
+
"item-state snapshots require calibrated identity and wrapper authority; "
|
|
2907
|
+
f"missing geometry: {descriptions}. Recalibrate the active profile."
|
|
2908
|
+
)
|
|
2909
|
+
|
|
2910
|
+
|
|
2911
|
+
def _active_profile_authority(
|
|
2912
|
+
opcode_profile: str | Path | OpcodeProfile,
|
|
2913
|
+
) -> _ProfileAuthority:
|
|
2914
|
+
authority = _load_profile_authority(opcode_profile)
|
|
2915
|
+
_validate_item_state_identity_specs(authority.loaded_specs.specs)
|
|
2916
|
+
return authority
|
|
2917
|
+
|
|
2918
|
+
|
|
2919
|
+
def analyze_character_load_pcap(
|
|
2920
|
+
path: str | Path,
|
|
2921
|
+
*,
|
|
2922
|
+
opcode_profile: str | Path | OpcodeProfile,
|
|
2923
|
+
ports: tuple[int, ...] = DEFAULT_SERVER_PORTS,
|
|
2924
|
+
capture_limits: Optional[ItemStateCaptureLimits] = None,
|
|
2925
|
+
) -> CharacterStateSnapshot:
|
|
2926
|
+
"""Replay a capture and summarize framed inventory/storage hydration."""
|
|
2927
|
+
options = PacketCaptureOptions(ports=ports)
|
|
2928
|
+
authority = _active_profile_authority(opcode_profile)
|
|
2929
|
+
profile_source = str(authority.profile.path)
|
|
2930
|
+
specs = authority.loaded_specs.specs
|
|
2931
|
+
accumulator = _CharacterStateAccumulator(
|
|
2932
|
+
profile_source=profile_source,
|
|
2933
|
+
specs=specs,
|
|
2934
|
+
capture_mode="pcap_replay",
|
|
2935
|
+
input_path=path,
|
|
2936
|
+
capture_limits=capture_limits,
|
|
2937
|
+
)
|
|
2938
|
+
collector = _EventCollector(
|
|
2939
|
+
server_ports=options.ports,
|
|
2940
|
+
event_filter=EventFilter(
|
|
2941
|
+
event_types={
|
|
2942
|
+
"inventory_snapshot",
|
|
2943
|
+
"storage_snapshot",
|
|
2944
|
+
"storage_record",
|
|
2945
|
+
"storage_delta",
|
|
2946
|
+
}
|
|
2947
|
+
),
|
|
2948
|
+
on_event=accumulator.observe_event,
|
|
2949
|
+
frame_observer=accumulator.observe_frame,
|
|
2950
|
+
_profile_authority=authority,
|
|
2951
|
+
)
|
|
2952
|
+
for _ in iter_pcap_file(Path(path), collector.engine):
|
|
2953
|
+
pass
|
|
2954
|
+
collector.finalize()
|
|
2955
|
+
return accumulator.snapshot(decoder_health=collector.decoder_health)
|
|
2956
|
+
|
|
2957
|
+
|
|
2958
|
+
def _validate_save_pcap_path(path: str | Path) -> Path:
|
|
2959
|
+
if not isinstance(path, (str, Path)):
|
|
2960
|
+
raise TypeError("save_pcap must be a path string, Path, or None")
|
|
2961
|
+
capture_path = Path(path)
|
|
2962
|
+
if capture_path.suffix.casefold() not in {".pcap", ".pcapng"}:
|
|
2963
|
+
raise ValueError("save_pcap must end in .pcap or .pcapng")
|
|
2964
|
+
return capture_path
|
|
2965
|
+
|
|
2966
|
+
|
|
2967
|
+
def _open_packet_writer(path: Path) -> Any:
|
|
2968
|
+
"""Open a Scapy writer matching the requested capture container."""
|
|
2969
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
2970
|
+
if path.exists():
|
|
2971
|
+
raise FileExistsError(f"refusing to overwrite existing capture: {path}")
|
|
2972
|
+
if path.suffix.casefold() == ".pcapng":
|
|
2973
|
+
from scapy.utils import PcapNgWriter # type: ignore
|
|
2974
|
+
|
|
2975
|
+
return PcapNgWriter(str(path))
|
|
2976
|
+
|
|
2977
|
+
from scapy.utils import PcapWriter # type: ignore
|
|
2978
|
+
|
|
2979
|
+
# sync=True makes a live .pcap useful even if the process exits before a
|
|
2980
|
+
# normal stop. PcapNgWriter has no equivalent constructor option and is
|
|
2981
|
+
# still explicitly closed on every session exit path below.
|
|
2982
|
+
return PcapWriter(str(path), append=False, sync=True)
|
|
2983
|
+
|
|
2984
|
+
|
|
2985
|
+
class CharacterLoadSession:
|
|
2986
|
+
"""Experimental live capture session returning a character-state summary."""
|
|
2987
|
+
|
|
2988
|
+
def __init__(
|
|
2989
|
+
self,
|
|
2990
|
+
*,
|
|
2991
|
+
opcode_profile: str | Path | OpcodeProfile,
|
|
2992
|
+
capture_options: Optional[PacketCaptureOptions] = None,
|
|
2993
|
+
save_pcap: str | Path | None = None,
|
|
2994
|
+
capture_limits: Optional[ItemStateCaptureLimits] = None,
|
|
2995
|
+
) -> None:
|
|
2996
|
+
if capture_options is not None and not isinstance(
|
|
2997
|
+
capture_options, PacketCaptureOptions
|
|
2998
|
+
):
|
|
2999
|
+
raise TypeError("capture_options must be a PacketCaptureOptions or None")
|
|
3000
|
+
if capture_limits is not None and not isinstance(
|
|
3001
|
+
capture_limits, ItemStateCaptureLimits
|
|
3002
|
+
):
|
|
3003
|
+
raise TypeError("capture_limits must be an ItemStateCaptureLimits or None")
|
|
3004
|
+
self._capture_options = capture_options or PacketCaptureOptions()
|
|
3005
|
+
self._capture_limits = capture_limits or ItemStateCaptureLimits()
|
|
3006
|
+
self._save_pcap_path = (
|
|
3007
|
+
_validate_save_pcap_path(save_pcap) if save_pcap is not None else None
|
|
3008
|
+
)
|
|
3009
|
+
self._profile_authority = _active_profile_authority(opcode_profile)
|
|
3010
|
+
self._profile_source = str(self._profile_authority.profile.path)
|
|
3011
|
+
self._specs = self._profile_authority.loaded_specs.specs
|
|
3012
|
+
self._start_attempted = False
|
|
3013
|
+
self._accumulator: Optional[_CharacterStateAccumulator] = None
|
|
3014
|
+
self._collector: Optional[_EventCollector] = None
|
|
3015
|
+
self._engine: Optional[PacketEngine] = None
|
|
3016
|
+
self._capture: Optional[LivePacketCapture] = None
|
|
3017
|
+
self._capture_writer: Any = None
|
|
3018
|
+
self._result: Optional[CharacterStateSnapshot] = None
|
|
3019
|
+
self._error: Optional[BaseException] = None
|
|
3020
|
+
|
|
3021
|
+
@property
|
|
3022
|
+
def running(self) -> bool:
|
|
3023
|
+
capture = self._capture
|
|
3024
|
+
return capture is not None and capture.running
|
|
3025
|
+
|
|
3026
|
+
@property
|
|
3027
|
+
def cleanup_incomplete(self) -> bool:
|
|
3028
|
+
"""Whether capture shutdown retained resources for a stop retry."""
|
|
3029
|
+
|
|
3030
|
+
capture = self._capture
|
|
3031
|
+
return capture is not None and capture.cleanup_incomplete
|
|
3032
|
+
|
|
3033
|
+
@property
|
|
3034
|
+
def frames_seen(self) -> int:
|
|
3035
|
+
return self._accumulator.frames_seen if self._accumulator is not None else 0
|
|
3036
|
+
|
|
3037
|
+
@property
|
|
3038
|
+
def decoder_health(self) -> DecoderHealth:
|
|
3039
|
+
"""Current storage-decoder compatibility for this capture."""
|
|
3040
|
+
|
|
3041
|
+
if self._collector is not None:
|
|
3042
|
+
return self._collector.decoder_health
|
|
3043
|
+
if self._result is not None:
|
|
3044
|
+
return self._result.decoder_health
|
|
3045
|
+
return DecoderHealth()
|
|
3046
|
+
|
|
3047
|
+
@property
|
|
3048
|
+
def error(self) -> Optional[BaseException]:
|
|
3049
|
+
"""First background capture or decoder failure, if any."""
|
|
3050
|
+
if self._error is not None:
|
|
3051
|
+
return self._error
|
|
3052
|
+
capture = self._capture
|
|
3053
|
+
return capture.error if capture is not None else None
|
|
3054
|
+
|
|
3055
|
+
@property
|
|
3056
|
+
def save_pcap_path(self) -> Optional[Path]:
|
|
3057
|
+
"""Destination for opt-in raw live packets, if configured."""
|
|
3058
|
+
return self._save_pcap_path
|
|
3059
|
+
|
|
3060
|
+
def start(self) -> None:
|
|
3061
|
+
"""Begin passive capture and return once the adapter is ready.
|
|
3062
|
+
|
|
3063
|
+
A session is single-use, including after a failed startup. Construct a
|
|
3064
|
+
new session to retry with a fresh writer, decoder, and capture handle.
|
|
3065
|
+
If startup reports incomplete cleanup, first call ``stop()`` on this
|
|
3066
|
+
session until the retained capture backend is verified stopped.
|
|
3067
|
+
"""
|
|
3068
|
+
if self._start_attempted:
|
|
3069
|
+
raise RuntimeError(
|
|
3070
|
+
"character-load session is single-use and already started"
|
|
3071
|
+
)
|
|
3072
|
+
self._start_attempted = True
|
|
3073
|
+
self._error = None
|
|
3074
|
+
|
|
3075
|
+
accumulator = _CharacterStateAccumulator(
|
|
3076
|
+
profile_source=self._profile_source,
|
|
3077
|
+
specs=self._specs,
|
|
3078
|
+
capture_mode="live_capture",
|
|
3079
|
+
saved_capture_path=self._save_pcap_path,
|
|
3080
|
+
capture_limits=self._capture_limits,
|
|
3081
|
+
)
|
|
3082
|
+
collector = _EventCollector(
|
|
3083
|
+
server_ports=self._capture_options.ports,
|
|
3084
|
+
event_filter=EventFilter(
|
|
3085
|
+
event_types={
|
|
3086
|
+
"inventory_snapshot",
|
|
3087
|
+
"storage_snapshot",
|
|
3088
|
+
"storage_record",
|
|
3089
|
+
"storage_delta",
|
|
3090
|
+
}
|
|
3091
|
+
),
|
|
3092
|
+
on_event=accumulator.observe_event,
|
|
3093
|
+
frame_observer=accumulator.observe_frame,
|
|
3094
|
+
_profile_authority=self._profile_authority,
|
|
3095
|
+
)
|
|
3096
|
+
engine = collector.engine
|
|
3097
|
+
packet_handler = make_packet_handler(engine)
|
|
3098
|
+
capture_writer = None
|
|
3099
|
+
capture: Optional[LivePacketCapture] = None
|
|
3100
|
+
try:
|
|
3101
|
+
capture_writer = (
|
|
3102
|
+
_open_packet_writer(self._save_pcap_path)
|
|
3103
|
+
if self._save_pcap_path is not None
|
|
3104
|
+
else None
|
|
3105
|
+
)
|
|
3106
|
+
|
|
3107
|
+
def handle_packet(packet: object) -> None:
|
|
3108
|
+
try:
|
|
3109
|
+
# Persist the untouched packet before decoding so parser
|
|
3110
|
+
# failures still retain the packet that exposed them.
|
|
3111
|
+
if capture_writer is not None:
|
|
3112
|
+
capture_writer.write(packet)
|
|
3113
|
+
packet_handler(packet)
|
|
3114
|
+
except BaseException as exc:
|
|
3115
|
+
self._record_error(exc)
|
|
3116
|
+
raise
|
|
3117
|
+
|
|
3118
|
+
capture = LivePacketCapture(
|
|
3119
|
+
capture_options=self._capture_options,
|
|
3120
|
+
on_packet=handle_packet,
|
|
3121
|
+
startup_timeout=_CHARACTER_LOAD_STARTUP_TIMEOUT_SECONDS,
|
|
3122
|
+
)
|
|
3123
|
+
self._accumulator = accumulator
|
|
3124
|
+
self._collector = collector
|
|
3125
|
+
self._engine = engine
|
|
3126
|
+
self._capture_writer = capture_writer
|
|
3127
|
+
self._capture = capture
|
|
3128
|
+
capture.start()
|
|
3129
|
+
except BaseException as exc:
|
|
3130
|
+
self._record_error(exc)
|
|
3131
|
+
if capture is not None and capture.cleanup_incomplete:
|
|
3132
|
+
# The backend may still invoke handle_packet(). Keep its
|
|
3133
|
+
# writer, engine, accumulator, and capture owner reachable so
|
|
3134
|
+
# stop() can safely retry before any dependent resource closes.
|
|
3135
|
+
_attach_cleanup_owner(
|
|
3136
|
+
exc,
|
|
3137
|
+
self,
|
|
3138
|
+
context="character-load capture startup",
|
|
3139
|
+
)
|
|
3140
|
+
raise
|
|
3141
|
+
if capture_writer is not None:
|
|
3142
|
+
try:
|
|
3143
|
+
capture_writer.close()
|
|
3144
|
+
except BaseException:
|
|
3145
|
+
# Preserve the original startup failure.
|
|
3146
|
+
pass
|
|
3147
|
+
self._capture = None
|
|
3148
|
+
self._capture_writer = None
|
|
3149
|
+
self._accumulator = None
|
|
3150
|
+
self._collector = None
|
|
3151
|
+
self._engine = None
|
|
3152
|
+
raise
|
|
3153
|
+
|
|
3154
|
+
def stop(self) -> CharacterStateSnapshot:
|
|
3155
|
+
"""Stop capture, finish reassembly, and return the queryable summary."""
|
|
3156
|
+
if self._result is not None:
|
|
3157
|
+
if self._error is not None:
|
|
3158
|
+
# A cached diagnostic snapshot must never turn a previously
|
|
3159
|
+
# failed run into an apparent success on a repeated stop().
|
|
3160
|
+
raise self._error
|
|
3161
|
+
return self._result
|
|
3162
|
+
if (
|
|
3163
|
+
self._capture is None
|
|
3164
|
+
or self._collector is None
|
|
3165
|
+
or self._engine is None
|
|
3166
|
+
or self._accumulator is None
|
|
3167
|
+
):
|
|
3168
|
+
raise RuntimeError("character-load session was not started")
|
|
3169
|
+
capture = self._capture
|
|
3170
|
+
collector = self._collector
|
|
3171
|
+
engine = self._engine
|
|
3172
|
+
accumulator = self._accumulator
|
|
3173
|
+
capture_writer = self._capture_writer
|
|
3174
|
+
stop_failure: Optional[BaseException] = None
|
|
3175
|
+
try:
|
|
3176
|
+
capture.stop()
|
|
3177
|
+
except BaseException as exc:
|
|
3178
|
+
stop_failure = exc
|
|
3179
|
+
self._record_error(exc)
|
|
3180
|
+
if not capture.stopped:
|
|
3181
|
+
if stop_failure is None:
|
|
3182
|
+
stop_failure = capture.cleanup_error or RuntimeError(
|
|
3183
|
+
"character-load capture cleanup is incomplete"
|
|
3184
|
+
)
|
|
3185
|
+
self._record_error(stop_failure)
|
|
3186
|
+
# The capture callback still owns the writer and decoder. Leave
|
|
3187
|
+
# every dependency intact for a later, verified stop attempt.
|
|
3188
|
+
raise stop_failure
|
|
3189
|
+
capture_error = capture.error
|
|
3190
|
+
if capture_error is not None:
|
|
3191
|
+
self._record_error(capture_error)
|
|
3192
|
+
try:
|
|
3193
|
+
engine.finish()
|
|
3194
|
+
except BaseException as exc:
|
|
3195
|
+
self._record_error(exc)
|
|
3196
|
+
try:
|
|
3197
|
+
collector.finalize()
|
|
3198
|
+
except BaseException as exc:
|
|
3199
|
+
self._record_error(exc)
|
|
3200
|
+
if capture_writer is not None:
|
|
3201
|
+
try:
|
|
3202
|
+
capture_writer.close()
|
|
3203
|
+
except BaseException as exc:
|
|
3204
|
+
self._record_error(exc)
|
|
3205
|
+
result: Optional[CharacterStateSnapshot] = None
|
|
3206
|
+
try:
|
|
3207
|
+
result = accumulator.snapshot(decoder_health=collector.decoder_health)
|
|
3208
|
+
except BaseException as exc:
|
|
3209
|
+
self._record_error(exc)
|
|
3210
|
+
self._capture = None
|
|
3211
|
+
self._capture_writer = None
|
|
3212
|
+
self._collector = None
|
|
3213
|
+
self._engine = None
|
|
3214
|
+
if result is not None:
|
|
3215
|
+
self._result = result
|
|
3216
|
+
if self._error is not None:
|
|
3217
|
+
raise self._error
|
|
3218
|
+
assert result is not None
|
|
3219
|
+
return result
|
|
3220
|
+
|
|
3221
|
+
def _record_error(self, error: BaseException) -> None:
|
|
3222
|
+
if self._error is None:
|
|
3223
|
+
self._error = error
|
|
3224
|
+
|
|
3225
|
+
def __enter__(self) -> "CharacterLoadSession":
|
|
3226
|
+
if not self._start_attempted:
|
|
3227
|
+
self.start()
|
|
3228
|
+
elif self._capture is None and self._result is None:
|
|
3229
|
+
raise RuntimeError(
|
|
3230
|
+
"character-load session is single-use and cannot be restarted"
|
|
3231
|
+
)
|
|
3232
|
+
return self
|
|
3233
|
+
|
|
3234
|
+
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
|
3235
|
+
if self._capture is not None:
|
|
3236
|
+
try:
|
|
3237
|
+
self.stop()
|
|
3238
|
+
except BaseException as cleanup_error:
|
|
3239
|
+
if exc_value is None:
|
|
3240
|
+
raise
|
|
3241
|
+
if self.cleanup_incomplete:
|
|
3242
|
+
_attach_cleanup_owner(
|
|
3243
|
+
exc_value,
|
|
3244
|
+
self,
|
|
3245
|
+
context="character-load capture context",
|
|
3246
|
+
)
|
|
3247
|
+
if hasattr(exc_value, "add_note"):
|
|
3248
|
+
exc_value.add_note(
|
|
3249
|
+
"character-load context cleanup also failed: "
|
|
3250
|
+
f"{cleanup_error!r}"
|
|
3251
|
+
)
|
|
3252
|
+
|
|
3253
|
+
|
|
3254
|
+
def format_character_state(
|
|
3255
|
+
snapshot: CharacterStateSnapshot,
|
|
3256
|
+
*,
|
|
3257
|
+
show_items: bool = False,
|
|
3258
|
+
) -> str:
|
|
3259
|
+
"""Render a stable, honest diagnostic summary for console tools."""
|
|
3260
|
+
diagnostics = snapshot.diagnostics
|
|
3261
|
+
frames_seen = diagnostics.frames_seen if diagnostics is not None else "unavailable"
|
|
3262
|
+
lines = [
|
|
3263
|
+
"CHARACTER LOAD SNAPSHOT DIAGNOSTIC",
|
|
3264
|
+
f"Profile: {snapshot.provenance.profile_source}",
|
|
3265
|
+
f"Generic BDO frames observed: {frames_seen}",
|
|
3266
|
+
(
|
|
3267
|
+
"Storage decoder: "
|
|
3268
|
+
f"{snapshot.decoder_health.storage_status} "
|
|
3269
|
+
f"({snapshot.decoder_health.storage_messages_decoded}/"
|
|
3270
|
+
f"{snapshot.decoder_health.storage_messages_observed} "
|
|
3271
|
+
"observed wrappers decoded)"
|
|
3272
|
+
),
|
|
3273
|
+
(
|
|
3274
|
+
"Hydration packets detected; trigger is unclassified "
|
|
3275
|
+
"(initial login vs character switch)."
|
|
3276
|
+
if snapshot.hydration_detected
|
|
3277
|
+
else "No hydration packets were detected."
|
|
3278
|
+
),
|
|
3279
|
+
"",
|
|
3280
|
+
"INVENTORY SNAPSHOT",
|
|
3281
|
+
]
|
|
3282
|
+
inventory = snapshot.inventory
|
|
3283
|
+
inventory_diagnostics = diagnostics.inventory if diagnostics is not None else None
|
|
3284
|
+
if inventory.hydration_observed:
|
|
3285
|
+
lines.append(
|
|
3286
|
+
f" {inventory.serialized_records} serialized records: "
|
|
3287
|
+
f"{inventory.occupied_stacks} occupied item stacks + "
|
|
3288
|
+
f"{inventory.currency_balance_records} currency balances"
|
|
3289
|
+
)
|
|
3290
|
+
if inventory_diagnostics is not None:
|
|
3291
|
+
lines.extend(
|
|
3292
|
+
[
|
|
3293
|
+
(
|
|
3294
|
+
f" {inventory_diagnostics.groups} groups: "
|
|
3295
|
+
f"{inventory_diagnostics.populated_groups} populated, "
|
|
3296
|
+
f"{inventory_diagnostics.empty_groups} empty"
|
|
3297
|
+
),
|
|
3298
|
+
" group record counts: "
|
|
3299
|
+
+ ", ".join(
|
|
3300
|
+
str(count) for count in inventory_diagnostics.group_counts
|
|
3301
|
+
),
|
|
3302
|
+
" inferred record strides: "
|
|
3303
|
+
+ (
|
|
3304
|
+
", ".join(
|
|
3305
|
+
str(stride)
|
|
3306
|
+
for stride in inventory_diagnostics.inferred_strides
|
|
3307
|
+
)
|
|
3308
|
+
if inventory_diagnostics.inferred_strides
|
|
3309
|
+
else "unavailable"
|
|
3310
|
+
),
|
|
3311
|
+
]
|
|
3312
|
+
)
|
|
3313
|
+
if inventory.containers:
|
|
3314
|
+
lines.append(" provisional containers (raw code is authoritative):")
|
|
3315
|
+
for container in inventory.containers:
|
|
3316
|
+
lines.append(
|
|
3317
|
+
f" {container.name} [0x{container.raw_code:02X}, "
|
|
3318
|
+
f"{container.confidence}]: {container.occupied_stacks} item stacks, "
|
|
3319
|
+
f"{len(container.currency_balances)} currency balances"
|
|
3320
|
+
)
|
|
3321
|
+
else:
|
|
3322
|
+
lines.append(" container/tab labels: unclassified")
|
|
3323
|
+
if inventory_diagnostics is not None and inventory_diagnostics.empty_groups:
|
|
3324
|
+
lines.append(
|
|
3325
|
+
f" {inventory_diagnostics.empty_groups} empty wrappers: unclassified "
|
|
3326
|
+
"(no record-level container field)"
|
|
3327
|
+
)
|
|
3328
|
+
if inventory.unclassified_records:
|
|
3329
|
+
lines.append(
|
|
3330
|
+
f" records without a validated container: "
|
|
3331
|
+
f"{inventory.unclassified_records}"
|
|
3332
|
+
)
|
|
3333
|
+
if inventory.currency_balances:
|
|
3334
|
+
lines.append(" currency balances:")
|
|
3335
|
+
for balance in sorted(
|
|
3336
|
+
inventory.currency_balances,
|
|
3337
|
+
key=lambda item: item.item_id,
|
|
3338
|
+
):
|
|
3339
|
+
lines.append(
|
|
3340
|
+
f" {balance.currency_name}: {balance.quantity:,} "
|
|
3341
|
+
f"(item_id={balance.item_id}, "
|
|
3342
|
+
f"container={balance.container_name}, "
|
|
3343
|
+
f"slot={balance.inventory_slot})"
|
|
3344
|
+
)
|
|
3345
|
+
if (
|
|
3346
|
+
inventory_diagnostics is not None
|
|
3347
|
+
and inventory_diagnostics.duplicate_records
|
|
3348
|
+
):
|
|
3349
|
+
lines.append(
|
|
3350
|
+
" repeated records merged by item instance: "
|
|
3351
|
+
f"{inventory_diagnostics.duplicate_records}"
|
|
3352
|
+
)
|
|
3353
|
+
if snapshot.coverage.inventory_records_missing_instance:
|
|
3354
|
+
lines.append(
|
|
3355
|
+
f" identity-unresolved records excluded: "
|
|
3356
|
+
f"{snapshot.coverage.inventory_records_missing_instance}"
|
|
3357
|
+
)
|
|
3358
|
+
if show_items:
|
|
3359
|
+
for item in inventory.items:
|
|
3360
|
+
lines.append(
|
|
3361
|
+
f" item_id={item.item_id} quantity={item.quantity} "
|
|
3362
|
+
f"instance={item.instance} container={item.container_name or 'unknown'} "
|
|
3363
|
+
f"container_code="
|
|
3364
|
+
f"{f'0x{item.container_code:02X}' if item.container_code is not None else 'unknown'} "
|
|
3365
|
+
f"slot={item.inventory_slot}"
|
|
3366
|
+
)
|
|
3367
|
+
else:
|
|
3368
|
+
lines.append(" NOT DETECTED")
|
|
3369
|
+
|
|
3370
|
+
lines.extend(["", "STORAGE SNAPSHOT"])
|
|
3371
|
+
storage_records_decoded = (
|
|
3372
|
+
diagnostics.storage.records_decoded
|
|
3373
|
+
if diagnostics is not None
|
|
3374
|
+
else None
|
|
3375
|
+
)
|
|
3376
|
+
if storage_records_decoded or snapshot.storages:
|
|
3377
|
+
missing_known_ids = snapshot.coverage.registered_storage_ids_not_observed
|
|
3378
|
+
earlier_only = tuple(
|
|
3379
|
+
storage
|
|
3380
|
+
for storage in snapshot.storages
|
|
3381
|
+
if not storage.current_state_observed
|
|
3382
|
+
)
|
|
3383
|
+
identity_incomplete = tuple(
|
|
3384
|
+
storage
|
|
3385
|
+
for storage in snapshot.storages
|
|
3386
|
+
if storage.current_state_observed
|
|
3387
|
+
and storage.current_identity_complete is False
|
|
3388
|
+
)
|
|
3389
|
+
if earlier_only or identity_incomplete:
|
|
3390
|
+
current_state_parts = [
|
|
3391
|
+
f" {snapshot.storages.nonempty_count} non-empty",
|
|
3392
|
+
f"{snapshot.storages.empty_count} explicitly empty",
|
|
3393
|
+
]
|
|
3394
|
+
if identity_incomplete:
|
|
3395
|
+
current_state_parts.append(
|
|
3396
|
+
f"{len(identity_incomplete)} identity-incomplete"
|
|
3397
|
+
)
|
|
3398
|
+
if earlier_only:
|
|
3399
|
+
current_state_parts.append(
|
|
3400
|
+
f"{len(earlier_only)} earlier-only (current state unavailable)"
|
|
3401
|
+
)
|
|
3402
|
+
current_state_parts.append(f"{len(missing_known_ids)} not observed")
|
|
3403
|
+
current_state_line = ", ".join(current_state_parts)
|
|
3404
|
+
else:
|
|
3405
|
+
current_state_line = (
|
|
3406
|
+
f" {snapshot.storages.nonempty_count} non-empty, "
|
|
3407
|
+
f"{snapshot.storages.empty_count} explicitly empty, "
|
|
3408
|
+
f"{len(missing_known_ids)} not observed"
|
|
3409
|
+
)
|
|
3410
|
+
storage_item_line = (
|
|
3411
|
+
f" {snapshot.storages.occupied_stacks} unique occupied item stacks"
|
|
3412
|
+
)
|
|
3413
|
+
if storage_records_decoded is not None:
|
|
3414
|
+
storage_item_line += (
|
|
3415
|
+
f" from {storage_records_decoded} decoded snapshot records"
|
|
3416
|
+
)
|
|
3417
|
+
lines.extend(
|
|
3418
|
+
[
|
|
3419
|
+
(
|
|
3420
|
+
f" {snapshot.storages.registered_count}/"
|
|
3421
|
+
f"{len(STORAGE_LOCATIONS)} known destinations observed"
|
|
3422
|
+
),
|
|
3423
|
+
current_state_line,
|
|
3424
|
+
storage_item_line,
|
|
3425
|
+
" capacity: unavailable (not present in the decoded item wrappers)",
|
|
3426
|
+
"",
|
|
3427
|
+
]
|
|
3428
|
+
)
|
|
3429
|
+
if diagnostics is not None and diagnostics.storage.sweeps_observed:
|
|
3430
|
+
lines.insert(
|
|
3431
|
+
len(lines) - 1,
|
|
3432
|
+
f" selected inferred storage sweep "
|
|
3433
|
+
f"{diagnostics.storage.selected_sweep}/"
|
|
3434
|
+
f"{diagnostics.storage.sweeps_observed}",
|
|
3435
|
+
)
|
|
3436
|
+
if snapshot.coverage.storage_records_missing_instance:
|
|
3437
|
+
lines.insert(
|
|
3438
|
+
len(lines) - 1,
|
|
3439
|
+
f" identity-unresolved records excluded: "
|
|
3440
|
+
f"{snapshot.coverage.storage_records_missing_instance}",
|
|
3441
|
+
)
|
|
3442
|
+
if missing_known_ids:
|
|
3443
|
+
missing_names = sorted(
|
|
3444
|
+
STORAGE_LOCATIONS[storage_id].name for storage_id in missing_known_ids
|
|
3445
|
+
)
|
|
3446
|
+
lines.append(
|
|
3447
|
+
" known destinations not observed: " + ", ".join(missing_names)
|
|
3448
|
+
)
|
|
3449
|
+
lines.append("")
|
|
3450
|
+
for storage in snapshot.storages:
|
|
3451
|
+
label = storage.name or f"UNKNOWN_STORAGE(0x{storage.storage_id:08x})"
|
|
3452
|
+
if not storage.current_state_observed:
|
|
3453
|
+
lines.append(
|
|
3454
|
+
f" {label}: current state unavailable "
|
|
3455
|
+
f"(observed only in an earlier inferred sweep)"
|
|
3456
|
+
)
|
|
3457
|
+
continue
|
|
3458
|
+
lines.append(
|
|
3459
|
+
f" {label}: {storage.occupied_stacks} occupied item stacks detected"
|
|
3460
|
+
)
|
|
3461
|
+
storage_diagnostics = (
|
|
3462
|
+
diagnostics.storage.destination(storage.storage_id)
|
|
3463
|
+
if diagnostics is not None
|
|
3464
|
+
else None
|
|
3465
|
+
)
|
|
3466
|
+
if (
|
|
3467
|
+
storage_diagnostics is not None
|
|
3468
|
+
and storage_diagnostics.selected_missing_instance_records
|
|
3469
|
+
):
|
|
3470
|
+
lines.append(
|
|
3471
|
+
f" identity-unresolved current records excluded: "
|
|
3472
|
+
f"{storage_diagnostics.selected_missing_instance_records}"
|
|
3473
|
+
)
|
|
3474
|
+
if show_items:
|
|
3475
|
+
for item in storage.items:
|
|
3476
|
+
lines.append(
|
|
3477
|
+
f" item_id={item.item_id} quantity={item.quantity} "
|
|
3478
|
+
f"instance={item.instance}"
|
|
3479
|
+
)
|
|
3480
|
+
else:
|
|
3481
|
+
lines.append(" NOT DETECTED")
|
|
3482
|
+
|
|
3483
|
+
lines.extend(["", "LIMITATIONS"])
|
|
3484
|
+
lines.extend(f" - {warning}" for warning in snapshot.warnings)
|
|
3485
|
+
return "\n".join(lines)
|
|
3486
|
+
|
|
3487
|
+
|
|
3488
|
+
__all__ = [
|
|
3489
|
+
"CharacterLoadSession",
|
|
3490
|
+
"CharacterStateSnapshot",
|
|
3491
|
+
"InventoryContainerSummary",
|
|
3492
|
+
"InventorySnapshotSummary",
|
|
3493
|
+
"ItemStateCaptureLimitError",
|
|
3494
|
+
"ItemStateCaptureLimits",
|
|
3495
|
+
"ItemStateCoverage",
|
|
3496
|
+
"ItemStateDiagnostics",
|
|
3497
|
+
"ItemStateProvenance",
|
|
3498
|
+
"InventoryHydrationDiagnostics",
|
|
3499
|
+
"SnapshotItem",
|
|
3500
|
+
"StorageDestinationDiagnostics",
|
|
3501
|
+
"StorageHydrationDiagnostics",
|
|
3502
|
+
"StorageContents",
|
|
3503
|
+
"StorageSnapshotSummary",
|
|
3504
|
+
"analyze_character_load_pcap",
|
|
3505
|
+
"format_character_state",
|
|
3506
|
+
]
|