bdo-toolkit 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. bdo_toolkit/__init__.py +87 -0
  2. bdo_toolkit/_async_sessions.py +651 -0
  3. bdo_toolkit/_capture_backend.py +194 -0
  4. bdo_toolkit/_capture_options.py +68 -0
  5. bdo_toolkit/_capture_runtime.py +626 -0
  6. bdo_toolkit/_deposit_origin.py +1599 -0
  7. bdo_toolkit/_engine.py +327 -0
  8. bdo_toolkit/_framing.py +904 -0
  9. bdo_toolkit/_profile_runtime.py +157 -0
  10. bdo_toolkit/_protocol.py +386 -0
  11. bdo_toolkit/_reassembly.py +654 -0
  12. bdo_toolkit/_specs.py +285 -0
  13. bdo_toolkit/_storage_destination_validation.py +167 -0
  14. bdo_toolkit/_storage_hydration.py +241 -0
  15. bdo_toolkit/_version.py +3 -0
  16. bdo_toolkit/calibration.py +3223 -0
  17. bdo_toolkit/capture.py +1713 -0
  18. bdo_toolkit/character_state.py +3506 -0
  19. bdo_toolkit/cli.py +948 -0
  20. bdo_toolkit/diagnostics.py +51 -0
  21. bdo_toolkit/events.py +214 -0
  22. bdo_toolkit/filters.py +105 -0
  23. bdo_toolkit/item_state.py +48 -0
  24. bdo_toolkit/origin_learning.py +779 -0
  25. bdo_toolkit/profiles.py +370 -0
  26. bdo_toolkit/py.typed +1 -0
  27. bdo_toolkit/remote_profiles.py +358 -0
  28. bdo_toolkit/solare/__init__.py +50 -0
  29. bdo_toolkit/solare/_constants.py +94 -0
  30. bdo_toolkit/solare/_detail_learning.py +1437 -0
  31. bdo_toolkit/solare/_details.py +796 -0
  32. bdo_toolkit/solare/_discovery.py +1212 -0
  33. bdo_toolkit/solare/_live_tracker.py +472 -0
  34. bdo_toolkit/solare/_replay_capture.py +182 -0
  35. bdo_toolkit/solare/_result.py +441 -0
  36. bdo_toolkit/solare/_scanner.py +203 -0
  37. bdo_toolkit/solare/_validation.py +11 -0
  38. bdo_toolkit/solare/async_session.py +444 -0
  39. bdo_toolkit/solare/models.py +806 -0
  40. bdo_toolkit/solare/replay.py +62 -0
  41. bdo_toolkit/solare/session.py +1051 -0
  42. bdo_toolkit/writers.py +30 -0
  43. bdo_toolkit-1.0.0.dist-info/METADATA +143 -0
  44. bdo_toolkit-1.0.0.dist-info/RECORD +48 -0
  45. bdo_toolkit-1.0.0.dist-info/WHEEL +5 -0
  46. bdo_toolkit-1.0.0.dist-info/entry_points.txt +2 -0
  47. bdo_toolkit-1.0.0.dist-info/licenses/LICENSE +21 -0
  48. bdo_toolkit-1.0.0.dist-info/top_level.txt +1 -0
bdo_toolkit/capture.py ADDED
@@ -0,0 +1,1713 @@
1
+ """Packet capture and pcap replay APIs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import OrderedDict, deque
6
+ from contextvars import ContextVar
7
+ from dataclasses import dataclass
8
+ import math
9
+ import time
10
+ from pathlib import Path
11
+ from queue import Empty, Full, Queue
12
+ from threading import Event, Lock, RLock, Thread, Timer, current_thread
13
+ from typing import Callable, Iterator, Optional
14
+
15
+ from ._capture_backend import (
16
+ iter_pcap_file,
17
+ make_packet_handler,
18
+ validate_server_ports,
19
+ )
20
+ from ._capture_options import LiveCaptureOptions
21
+ from ._capture_runtime import (
22
+ CaptureEndpoint,
23
+ CaptureStats,
24
+ LivePacketCapture,
25
+ _attach_cleanup_owner,
26
+ _capture_is_clean,
27
+ )
28
+ from ._deposit_origin import DecrementSpec, DepositOriginTracker
29
+ from ._engine import PacketEngine, toolkit_event_from_record
30
+ from ._profile_runtime import validate_runtime_profile
31
+ from ._protocol import (
32
+ BDOFrame,
33
+ CHARACTER_LOAD_CONTEXT,
34
+ DEFAULT_SERVER_PORTS,
35
+ FlowKey,
36
+ LootEvent,
37
+ PacketContext,
38
+ )
39
+ from ._storage_destination_validation import StorageDestinationValidator
40
+ from ._storage_hydration import StorageHydrationTracker
41
+ from .diagnostics import DecoderDiagnostic, DecoderHealth
42
+ from ._specs import LoadedSpecProfile
43
+ from .events import BDOEvent, Flow
44
+ from .filters import EventFilter
45
+ from .origin_learning import CompanionObservation
46
+ from .profiles import (
47
+ OpcodeProfile,
48
+ OriginCompanionFamily,
49
+ load_opcode_profile,
50
+ )
51
+
52
+
53
+ _PACKET_WORKER_STOP = object()
54
+ _TARGET_FRAME_HISTORY_LIMIT = 4096
55
+ _TargetFrameKey = tuple[FlowKey, int, Optional[int], float, int, int]
56
+ _INVENTORY_BOUNDARY_COHORT_LIMIT = 64
57
+ _InventoryBoundaryKey = tuple[FlowKey, int, int]
58
+
59
+
60
+ @dataclass
61
+ class _InventoryBoundaryState:
62
+ """One fully decoded inventory wrapper awaiting boundary evidence."""
63
+
64
+ frame_key: _TargetFrameKey
65
+ stream_end: int
66
+ raw_message: bytes
67
+ records: list[LootEvent]
68
+ accepted: bool
69
+
70
+
71
+ _ACTIVE_ORIGIN_SESSIONS: ContextVar[tuple[object, ...]] = ContextVar(
72
+ "bdo_toolkit_active_origin_session",
73
+ default=(),
74
+ )
75
+
76
+
77
+ class CaptureIntegrityError(RuntimeError):
78
+ """Live acquisition lost data before the decoder could inspect it."""
79
+
80
+
81
+ @dataclass(frozen=True)
82
+ class LiveCaptureHealth:
83
+ """Bounded-queue, TCP-reassembly, and native-capture diagnostics.
84
+
85
+ A non-clean result means the event stream may be incomplete. Queue
86
+ overflow is fail-closed: the session records :class:`CaptureIntegrityError`
87
+ and requests shutdown instead of silently continuing with missing packets.
88
+ """
89
+
90
+ packets_accepted: int = 0
91
+ packet_queue_peak: int = 0
92
+ packet_queue_overflows: int = 0
93
+ event_queue_peak: int = 0
94
+ tcp_gap_resets: int = 0
95
+ flow_state_evictions: int = 0
96
+ pcap_received: Optional[int] = None
97
+ pcap_dropped: Optional[int] = None
98
+ pcap_interface_dropped: Optional[int] = None
99
+ capture_buffer_bytes: Optional[int] = None
100
+ capture_buffer_fallback: bool = False
101
+
102
+ @property
103
+ def capture_is_clean(self) -> bool:
104
+ """Whether known acquisition-loss indicators remained clear."""
105
+
106
+ return _capture_is_clean(
107
+ tcp_gap_resets=self.tcp_gap_resets,
108
+ pcap_dropped=self.pcap_dropped,
109
+ pcap_interface_dropped=self.pcap_interface_dropped,
110
+ packet_queue_overflows=self.packet_queue_overflows,
111
+ flow_state_evictions=self.flow_state_evictions,
112
+ )
113
+
114
+ def to_dict(self) -> dict[str, object]:
115
+ """Return a JSON-ready diagnostic mapping."""
116
+
117
+ result: dict[str, object] = {
118
+ "packets_accepted": self.packets_accepted,
119
+ "packet_queue_peak": self.packet_queue_peak,
120
+ "packet_queue_overflows": self.packet_queue_overflows,
121
+ "event_queue_peak": self.event_queue_peak,
122
+ "tcp_gap_resets": self.tcp_gap_resets,
123
+ "flow_state_evictions": self.flow_state_evictions,
124
+ "capture_buffer_fallback": self.capture_buffer_fallback,
125
+ "capture_is_clean": self.capture_is_clean,
126
+ }
127
+ optional = {
128
+ "pcap_received": self.pcap_received,
129
+ "pcap_dropped": self.pcap_dropped,
130
+ "pcap_interface_dropped": self.pcap_interface_dropped,
131
+ "capture_buffer_bytes": self.capture_buffer_bytes,
132
+ }
133
+ result.update(
134
+ {key: value for key, value in optional.items() if value is not None}
135
+ )
136
+ return result
137
+
138
+
139
+ @dataclass(frozen=True)
140
+ class _ProfileAuthority:
141
+ """One profile revision and its derived runtime decoder specifications."""
142
+
143
+ profile: OpcodeProfile
144
+ loaded_specs: LoadedSpecProfile
145
+ decrement_specs: tuple[DecrementSpec, ...]
146
+
147
+
148
+ def _load_profile_authority(
149
+ source: str | Path | OpcodeProfile,
150
+ ) -> _ProfileAuthority:
151
+ if isinstance(source, OpcodeProfile):
152
+ profile = source
153
+ else:
154
+ path = Path(source)
155
+ if not path.is_file():
156
+ raise FileNotFoundError(f"Opcode profile does not exist: {path}")
157
+ profile = load_opcode_profile(path)
158
+ runtime = validate_runtime_profile(profile)
159
+ return _ProfileAuthority(
160
+ profile=profile,
161
+ loaded_specs=runtime.loaded_specs,
162
+ decrement_specs=runtime.decrement_specs,
163
+ )
164
+
165
+
166
+ def _decrement_specs(
167
+ profile: OpcodeProfile,
168
+ ) -> tuple[DecrementSpec, ...]:
169
+ """Compatibility shim for the shared fail-closed runtime validator."""
170
+
171
+ return validate_runtime_profile(profile).decrement_specs
172
+
173
+
174
+ def _origin_companion_families(
175
+ profile: OpcodeProfile,
176
+ ) -> tuple[OriginCompanionFamily, ...]:
177
+ return profile.origin_companion_families
178
+
179
+
180
+ def _needs_origin_tracking(
181
+ event_filter: Optional[EventFilter],
182
+ origin_observer: Optional[Callable[[CompanionObservation], object]],
183
+ ) -> bool:
184
+ if origin_observer is not None or event_filter is None:
185
+ return True
186
+ return event_filter.event_types is None or bool(
187
+ event_filter.event_types
188
+ & {"storage_delta", "storage_record", "storage_snapshot"}
189
+ )
190
+
191
+
192
+ class _EventCollector:
193
+ """Wire a PacketEngine to app-facing events with optional filtering.
194
+
195
+ Live storage deltas and neutral storage records take a short detour through
196
+ the deposit-origin tracker (a few frames of lookahead). Independent manual
197
+ or worker evidence can promote an unfamiliar-mode ``storage_record`` to a
198
+ confirmed-live ``storage_delta``; records without that evidence remain
199
+ neutral. All other events emit immediately, which can place a deferred
200
+ storage event slightly after later events of other types. Timestamps are
201
+ unaffected.
202
+ """
203
+
204
+ def __init__(
205
+ self,
206
+ *,
207
+ server_ports: tuple[int, ...],
208
+ event_filter: Optional[EventFilter] = None,
209
+ on_event: Optional[Callable[[BDOEvent], None]] = None,
210
+ opcode_profile: str | Path | OpcodeProfile | None = None,
211
+ origin_observer: Optional[Callable[[CompanionObservation], object]] = None,
212
+ frame_observer: Optional[Callable[[BDOFrame], object]] = None,
213
+ on_diagnostic: Optional[Callable[[DecoderDiagnostic], object]] = None,
214
+ preflight: bool = True,
215
+ _profile_authority: Optional[_ProfileAuthority] = None,
216
+ ) -> None:
217
+ if _profile_authority is not None:
218
+ if opcode_profile is not None:
219
+ raise TypeError(
220
+ "provide opcode_profile or _profile_authority, not both"
221
+ )
222
+ authority = _profile_authority
223
+ elif opcode_profile is not None:
224
+ authority = _load_profile_authority(opcode_profile)
225
+ else:
226
+ raise TypeError("opcode_profile is required")
227
+ profile = authority.profile
228
+ loaded_specs = authority.loaded_specs
229
+ self._events: deque[BDOEvent] = deque()
230
+ self.event_filter = event_filter
231
+ self.on_event = on_event
232
+ self.on_diagnostic = on_diagnostic
233
+ self.profile_source = f"{loaded_specs.source} active profile"
234
+ self.event_specs = loaded_specs.specs
235
+ self._diagnostic_lock = Lock()
236
+ self._diagnostic_keys: set[tuple[str, Optional[int]]] = set()
237
+ self._storage_messages_observed = 0
238
+ self._storage_messages_decoded = 0
239
+ self._storage_records_decoded = 0
240
+ self._storage_destination_failures = 0
241
+ self._storage_geometry_failures = 0
242
+ self._diagnostics_emitted = 0
243
+ self._storage_incompatible = False
244
+ self._storage_destination_validator = StorageDestinationValidator()
245
+ self._generic_storage_frames: set[_TargetFrameKey] = set()
246
+ self._generic_storage_frame_order: deque[_TargetFrameKey] = deque()
247
+ self._accepted_storage_frames: set[_TargetFrameKey] = set()
248
+ self._accepted_storage_frame_order: deque[_TargetFrameKey] = deque()
249
+ self._generic_inventory_frames: set[_TargetFrameKey] = set()
250
+ self._generic_inventory_frame_order: deque[_TargetFrameKey] = deque()
251
+ self._inventory_boundary_states: OrderedDict[
252
+ _InventoryBoundaryKey, _InventoryBoundaryState
253
+ ] = OrderedDict()
254
+ self._storage_opcodes = frozenset(
255
+ spec.opcode
256
+ for spec in self.event_specs
257
+ if spec.label == "INVENTORY_TO_STORAGE"
258
+ )
259
+ self._inventory_transfer_opcodes = frozenset(
260
+ spec.opcode
261
+ for spec in self.event_specs
262
+ if spec.label == "INVENTORY_TRANSFER"
263
+ )
264
+ self._requested_storage_ids = tuple(
265
+ sorted(event_filter.storage_ids)
266
+ if event_filter is not None and event_filter.storage_ids is not None
267
+ else ()
268
+ )
269
+ self._hydration = StorageHydrationTracker(self._deliver)
270
+ self._tracker: Optional[DepositOriginTracker] = None
271
+ if _needs_origin_tracking(event_filter, origin_observer):
272
+ self._tracker = DepositOriginTracker(
273
+ decrement_specs=authority.decrement_specs,
274
+ emit=self._after_origin,
275
+ origin_observer=origin_observer,
276
+ known_companion_families=_origin_companion_families(profile),
277
+ storage_delta_opcodes=(
278
+ spec.opcode
279
+ for spec in loaded_specs.specs
280
+ if spec.label == "INVENTORY_TO_STORAGE"
281
+ ),
282
+ )
283
+ tracker = self._tracker
284
+ observe_storage_messages = self._storage_is_requested() or tracker is not None
285
+
286
+ def observe_frame(frame: BDOFrame) -> None:
287
+ self._remember_generic_target_frame(frame)
288
+ if tracker is not None:
289
+ tracker.observe_frame(frame)
290
+ if frame_observer is not None:
291
+ frame_observer(frame)
292
+
293
+ self.engine = PacketEngine(
294
+ server_ports=server_ports,
295
+ event_specs=loaded_specs.specs,
296
+ on_event=self._handle_record,
297
+ frame_observer=(
298
+ observe_frame
299
+ if tracker is not None
300
+ or frame_observer is not None
301
+ or self._storage_is_requested()
302
+ or self._inventory_snapshot_is_requested()
303
+ else None
304
+ ),
305
+ stream_observer=(tracker.observe_stream if tracker is not None else None),
306
+ flow_close_observer=self._close_flow,
307
+ message_observer=(
308
+ self._observe_storage_message if observe_storage_messages else None
309
+ ),
310
+ )
311
+ self._preflight_done = False
312
+ if preflight:
313
+ self.preflight()
314
+
315
+ def _handle_record(self, record: LootEvent, raw_message: bytes) -> None:
316
+ is_storage = record.label == "INVENTORY_TO_STORAGE"
317
+ is_inventory_snapshot = (
318
+ record.label == "INVENTORY_TRANSFER"
319
+ and record.source_context_candidate == CHARACTER_LOAD_CONTEXT
320
+ )
321
+ if is_storage:
322
+ frame_key = (
323
+ record.context.flow,
324
+ record.context.flow_generation,
325
+ record.stream_sequence,
326
+ record.context.timestamp,
327
+ record.opcode,
328
+ record.message_length,
329
+ )
330
+ with self._diagnostic_lock:
331
+ if frame_key not in self._accepted_storage_frames:
332
+ # Target-signature recovery may find a valid-looking
333
+ # storage wrapper inside another BDO frame. Only a
334
+ # decoder candidate that also matched the generic
335
+ # top-level boundary may enter origin/hydration/event state.
336
+ return
337
+ if is_inventory_snapshot:
338
+ for accepted_record, accepted_message in self._inventory_records_ready(
339
+ record,
340
+ raw_message,
341
+ ):
342
+ self._handle_accepted_record(accepted_record, accepted_message)
343
+ return
344
+ self._handle_accepted_record(record, raw_message)
345
+
346
+ def _handle_accepted_record(
347
+ self,
348
+ record: LootEvent,
349
+ raw_message: bytes,
350
+ ) -> None:
351
+ """Normalize and classify one record with proven message boundaries."""
352
+
353
+ is_storage = record.label == "INVENTORY_TO_STORAGE"
354
+ if is_storage:
355
+ self._revalidate_storage_destination(record, raw_message)
356
+ event = toolkit_event_from_record(record)
357
+ if is_storage:
358
+ with self._diagnostic_lock:
359
+ self._storage_records_decoded += 1
360
+ missing_destination = event.storage_id is None
361
+ unrecognized_destination = (
362
+ event.storage_id is not None and event.storage_name is None
363
+ )
364
+ if missing_destination or unrecognized_destination:
365
+ self._storage_destination_failures += 1
366
+ self._storage_incompatible = True
367
+ if missing_destination:
368
+ self._emit_diagnostic(
369
+ code="storage_destination_unavailable",
370
+ opcode=event.opcode,
371
+ message=(
372
+ "A storage wrapper decoded, but its destination field "
373
+ "did not resolve to a registered town. Recalibrate the "
374
+ "inventory-to-storage action; storage-ID-filtered events "
375
+ "may otherwise be omitted."
376
+ ),
377
+ )
378
+ elif unrecognized_destination:
379
+ self._emit_diagnostic(
380
+ code="storage_destination_unrecognized",
381
+ opcode=event.opcode,
382
+ message=(
383
+ f"The calibrated storage destination field decoded ID "
384
+ f"0x{event.storage_id:08X}, but that key is not in the "
385
+ "town registry. Its numeric identity was preserved and "
386
+ "no display name is registered for it. Update the "
387
+ "registry before relying on destination display metadata."
388
+ ),
389
+ )
390
+ if event.event_type == "inventory_snapshot":
391
+ self._hydration.observe_inventory(event)
392
+ elif (
393
+ self._tracker is not None
394
+ and event.event_type in {"storage_delta", "storage_record"}
395
+ ):
396
+ # Filtering happens at delivery, AFTER classification, so filters
397
+ # on event_type/source see any evidence-based promotion and the
398
+ # final semantic source verdict.
399
+ self._tracker.register(event, raw_message=raw_message)
400
+ elif event.event_type in {"storage_delta", "storage_record"}:
401
+ self._hydration.observe_storage(event)
402
+ else:
403
+ self._deliver(event)
404
+
405
+ def _inventory_records_ready(
406
+ self,
407
+ record: LootEvent,
408
+ raw_message: bytes,
409
+ ) -> tuple[tuple[LootEvent, bytes], ...]:
410
+ """Require a top-level frame or an exact adjacent target-wrapper pair.
411
+
412
+ The generic scanner can remain synchronized to a false outer-frame
413
+ chain after capture begins midstream. Two fully decoded inventory
414
+ wrappers that are exactly adjacent in one TCP generation recover that
415
+ historical case without restoring the unsafe "any nested signature"
416
+ rule. A lone unframed wrapper is retained only until another wrapper
417
+ proves adjacency; flow close/finalization discard it.
418
+ """
419
+
420
+ stream_sequence = record.stream_sequence
421
+ frame_key: _TargetFrameKey = (
422
+ record.context.flow,
423
+ record.context.flow_generation,
424
+ stream_sequence,
425
+ record.context.timestamp,
426
+ record.opcode,
427
+ record.message_length,
428
+ )
429
+ with self._diagnostic_lock:
430
+ top_level = frame_key in self._generic_inventory_frames
431
+ if stream_sequence is None:
432
+ return ((record, raw_message),) if top_level else ()
433
+
434
+ boundary_key: _InventoryBoundaryKey = (
435
+ record.context.flow,
436
+ record.context.flow_generation,
437
+ record.opcode,
438
+ )
439
+ state = self._inventory_boundary_states.pop(boundary_key, None)
440
+ ready: tuple[tuple[LootEvent, bytes], ...]
441
+
442
+ if state is not None and state.frame_key == frame_key:
443
+ if state.accepted:
444
+ ready = ((record, raw_message),)
445
+ elif top_level:
446
+ ready = tuple(
447
+ (pending, state.raw_message) for pending in state.records
448
+ ) + ((record, raw_message),)
449
+ state.records.clear()
450
+ state.accepted = True
451
+ else:
452
+ state.records.append(record)
453
+ ready = ()
454
+ self._inventory_boundary_states[boundary_key] = state
455
+ return ready
456
+
457
+ if state is not None and state.stream_end == stream_sequence:
458
+ ready = (
459
+ tuple((pending, state.raw_message) for pending in state.records)
460
+ + ((record, raw_message),)
461
+ )
462
+ accepted = True
463
+ elif top_level:
464
+ ready = ((record, raw_message),)
465
+ accepted = True
466
+ else:
467
+ ready = ()
468
+ accepted = False
469
+
470
+ self._inventory_boundary_states[boundary_key] = _InventoryBoundaryState(
471
+ frame_key=frame_key,
472
+ stream_end=(stream_sequence + record.message_length) & 0xFFFFFFFF,
473
+ raw_message=raw_message,
474
+ records=[] if accepted else [record],
475
+ accepted=accepted,
476
+ )
477
+ while (
478
+ len(self._inventory_boundary_states)
479
+ > _INVENTORY_BOUNDARY_COHORT_LIMIT
480
+ ):
481
+ self._inventory_boundary_states.popitem(last=False)
482
+ return ready
483
+
484
+ def _after_origin(self, event: BDOEvent) -> None:
485
+ """Give independent live evidence precedence over hydration inference."""
486
+
487
+ self._hydration.observe_storage(event)
488
+
489
+ def _revalidate_storage_destination(
490
+ self,
491
+ record: LootEvent,
492
+ raw_message: bytes,
493
+ ) -> None:
494
+ """Warn when cross-wrapper evidence disproves the profile's column."""
495
+
496
+ first_item_offset = record.record_offset
497
+ if first_item_offset is None:
498
+ return
499
+ matching_specs = tuple(
500
+ spec
501
+ for spec in self.event_specs
502
+ if spec.label == "INVENTORY_TO_STORAGE"
503
+ and spec.opcode == record.opcode
504
+ and spec.item_offset == first_item_offset
505
+ )
506
+ # Only record one lands at the configured item offset. Multiple
507
+ # matching layouts would not identify which spec the scanner selected,
508
+ # so ambiguous profile state remains fail-neutral here.
509
+ if len(matching_specs) != 1:
510
+ return
511
+ spec = matching_specs[0]
512
+ mismatch = self._storage_destination_validator.observe(
513
+ flow=record.context.flow,
514
+ flow_generation=record.context.flow_generation,
515
+ opcode=record.opcode,
516
+ message=raw_message,
517
+ first_item_offset=first_item_offset,
518
+ configured_offset=spec.source_context_offset,
519
+ )
520
+ if mismatch is None:
521
+ return
522
+ with self._diagnostic_lock:
523
+ self._storage_incompatible = True
524
+ self._emit_diagnostic(
525
+ code="storage_destination_schema_mismatch",
526
+ opcode=record.opcode,
527
+ message=(
528
+ f"Cross-wrapper evidence from {mismatch.wrapper_count} storage "
529
+ f"wrappers and {mismatch.distinct_destinations} registered "
530
+ f"destinations uniquely proves the destination field at byte "
531
+ f"{mismatch.observed_offset}, but the active profile uses byte "
532
+ f"{mismatch.configured_offset}. Events were not relabeled; "
533
+ "recalibrate before relying on storage names or storage-ID filters."
534
+ ),
535
+ )
536
+
537
+ def _close_flow(self, flow: FlowKey) -> None:
538
+ # Origin finalization may emit neutral records into the hydration
539
+ # tracker. Close that tracker only after those records arrive.
540
+ if self._tracker is not None:
541
+ self._tracker.close_flow(flow)
542
+ self._storage_destination_validator.close_flow(flow)
543
+ with self._diagnostic_lock:
544
+ for key in tuple(self._inventory_boundary_states):
545
+ if key[0] == flow:
546
+ del self._inventory_boundary_states[key]
547
+ self._hydration.close_flow(
548
+ Flow(
549
+ source_ip=flow.source_ip,
550
+ source_port=flow.source_port,
551
+ destination_ip=flow.destination_ip,
552
+ destination_port=flow.destination_port,
553
+ )
554
+ )
555
+
556
+ def _deliver(self, event: BDOEvent) -> None:
557
+ if self.event_filter is not None and not self.event_filter.allows(event):
558
+ return
559
+ if self.on_event is not None:
560
+ self.on_event(event)
561
+ else:
562
+ self._events.append(event)
563
+
564
+ @property
565
+ def decoder_health(self) -> DecoderHealth:
566
+ with self._diagnostic_lock:
567
+ status = (
568
+ "incompatible"
569
+ if self._storage_incompatible
570
+ else "compatible"
571
+ if self._storage_messages_decoded
572
+ else "not_observed"
573
+ )
574
+ return DecoderHealth(
575
+ storage_status=status,
576
+ storage_messages_observed=self._storage_messages_observed,
577
+ storage_messages_decoded=self._storage_messages_decoded,
578
+ storage_records_decoded=self._storage_records_decoded,
579
+ storage_destination_failures=self._storage_destination_failures,
580
+ storage_geometry_failures=self._storage_geometry_failures,
581
+ diagnostics_emitted=self._diagnostics_emitted,
582
+ )
583
+
584
+ def _storage_is_requested(self) -> bool:
585
+ if self.event_filter is None or self.event_filter.event_types is None:
586
+ return True
587
+ return bool(
588
+ self.event_filter.event_types
589
+ & {"storage_delta", "storage_record", "storage_snapshot"}
590
+ )
591
+
592
+ def _inventory_snapshot_is_requested(self) -> bool:
593
+ if self.event_filter is None or self.event_filter.event_types is None:
594
+ return True
595
+ return "inventory_snapshot" in self.event_filter.event_types
596
+
597
+ def preflight(self) -> None:
598
+ """Emit profile-level compatibility diagnostics exactly once."""
599
+
600
+ with self._diagnostic_lock:
601
+ if self._preflight_done:
602
+ return
603
+ self._preflight_done = True
604
+ self._preflight_storage_profile()
605
+
606
+ def _preflight_storage_profile(self) -> None:
607
+ if not self._storage_is_requested():
608
+ return
609
+ storage_specs = tuple(
610
+ spec
611
+ for spec in self.event_specs
612
+ if spec.label == "INVENTORY_TO_STORAGE"
613
+ )
614
+ if not storage_specs:
615
+ with self._diagnostic_lock:
616
+ self._storage_incompatible = True
617
+ self._emit_diagnostic(
618
+ code="storage_decoder_incompatible",
619
+ opcode=None,
620
+ message=(
621
+ "The active opcode profile has no storage decoder. "
622
+ "Run inventory-to-storage calibration before relying on "
623
+ "storage events or storage-ID filters."
624
+ ),
625
+ )
626
+ return
627
+ for spec in storage_specs:
628
+ missing = []
629
+ if (
630
+ spec.source_context_offset is None
631
+ or spec.source_context_length != 4
632
+ ):
633
+ missing.append("destination field")
634
+ if spec.record_count_offset is None:
635
+ missing.append("record-count field")
636
+ if spec.storage_instance_offset is None:
637
+ missing.append("storage-instance field")
638
+ if spec.single_record_message_length is None:
639
+ missing.append("single-record length")
640
+ if not missing:
641
+ continue
642
+ with self._diagnostic_lock:
643
+ self._storage_incompatible = True
644
+ self._emit_diagnostic(
645
+ code="storage_decoder_incompatible",
646
+ opcode=spec.opcode,
647
+ message=(
648
+ "The active storage profile is missing its calibrated "
649
+ f"{' and '.join(missing)}. Recalibrate with two validated "
650
+ "record counts (for example a single and an unstackable "
651
+ "multi-record deposit) before relying on storage events "
652
+ "or storage-ID filters."
653
+ ),
654
+ )
655
+
656
+ def _observe_storage_message(
657
+ self,
658
+ opcode: int,
659
+ message_length: int,
660
+ status: str,
661
+ record_count: int,
662
+ context: PacketContext,
663
+ stream_sequence: Optional[int],
664
+ ) -> None:
665
+ del record_count
666
+ frame_key = (
667
+ context.flow,
668
+ context.flow_generation,
669
+ stream_sequence,
670
+ context.timestamp,
671
+ opcode,
672
+ message_length,
673
+ )
674
+ rejected = status != "decoded"
675
+ with self._diagnostic_lock:
676
+ if frame_key not in self._generic_storage_frames:
677
+ return
678
+ self._storage_messages_observed += 1
679
+ if rejected:
680
+ self._storage_geometry_failures += 1
681
+ # The target scanner reports only a complete candidate that
682
+ # exactly matches a generic top-level frame boundary; nested
683
+ # signatures and uniquely valid same-opcode families are
684
+ # excluded before this callback. One rejected storage wrapper
685
+ # is therefore enough to tell a production app that its active
686
+ # profile is incompatible with observed traffic.
687
+ self._storage_incompatible = True
688
+ else:
689
+ self._storage_messages_decoded += 1
690
+ if frame_key not in self._accepted_storage_frames:
691
+ self._accepted_storage_frames.add(frame_key)
692
+ self._accepted_storage_frame_order.append(frame_key)
693
+ while (
694
+ len(self._accepted_storage_frame_order)
695
+ > _TARGET_FRAME_HISTORY_LIMIT
696
+ ):
697
+ expired = self._accepted_storage_frame_order.popleft()
698
+ self._accepted_storage_frames.discard(expired)
699
+ if rejected:
700
+ self._emit_diagnostic(
701
+ code="storage_decoder_incompatible",
702
+ opcode=opcode,
703
+ message=(
704
+ f"Observed storage opcode 0x{opcode:04X} with length "
705
+ f"{message_length}, but its calibrated count and record "
706
+ "geometry did not validate. Recalibrate; if the warning "
707
+ "persists, retain a diagnostic PCAP for decoder review."
708
+ ),
709
+ )
710
+
711
+ def _remember_generic_target_frame(self, frame: BDOFrame) -> None:
712
+ is_storage = frame.opcode in self._storage_opcodes
713
+ is_inventory = frame.opcode in self._inventory_transfer_opcodes
714
+ if not is_storage and not is_inventory:
715
+ return
716
+ key = (
717
+ frame.context.flow,
718
+ frame.context.flow_generation,
719
+ frame.stream_sequence,
720
+ frame.context.timestamp,
721
+ frame.opcode,
722
+ frame.length,
723
+ )
724
+ with self._diagnostic_lock:
725
+ if is_storage and key not in self._generic_storage_frames:
726
+ self._generic_storage_frames.add(key)
727
+ self._generic_storage_frame_order.append(key)
728
+ while (
729
+ len(self._generic_storage_frame_order)
730
+ > _TARGET_FRAME_HISTORY_LIMIT
731
+ ):
732
+ expired = self._generic_storage_frame_order.popleft()
733
+ self._generic_storage_frames.discard(expired)
734
+ if is_inventory and key not in self._generic_inventory_frames:
735
+ self._generic_inventory_frames.add(key)
736
+ self._generic_inventory_frame_order.append(key)
737
+ while (
738
+ len(self._generic_inventory_frame_order)
739
+ > _TARGET_FRAME_HISTORY_LIMIT
740
+ ):
741
+ expired = self._generic_inventory_frame_order.popleft()
742
+ self._generic_inventory_frames.discard(expired)
743
+
744
+ def _emit_diagnostic(
745
+ self,
746
+ *,
747
+ code: str,
748
+ opcode: Optional[int],
749
+ message: str,
750
+ ) -> None:
751
+ key = (code, opcode)
752
+ with self._diagnostic_lock:
753
+ if key in self._diagnostic_keys:
754
+ return
755
+ self._diagnostic_keys.add(key)
756
+ self._diagnostics_emitted += 1
757
+ callback = self.on_diagnostic
758
+ if callback is not None:
759
+ callback(
760
+ DecoderDiagnostic(
761
+ code=code,
762
+ message=message,
763
+ opcode=opcode,
764
+ requested_storage_ids=self._requested_storage_ids,
765
+ )
766
+ )
767
+
768
+ def drain_events(self) -> Iterator[BDOEvent]:
769
+ """Yield and remove all currently delivered events."""
770
+ while self._events:
771
+ yield self._events.popleft()
772
+
773
+ def flush_stale(self, now: float) -> None:
774
+ if self._tracker is not None:
775
+ self._tracker.flush_stale(now)
776
+ self._hydration.flush_stale(now)
777
+
778
+ def finalize(self) -> None:
779
+ with self._diagnostic_lock:
780
+ self._inventory_boundary_states.clear()
781
+ if self._tracker is not None:
782
+ self._tracker.finalize_all()
783
+ self._hydration.finalize_all()
784
+
785
+
786
+ def replay_pcap(
787
+ path: str | Path,
788
+ *,
789
+ opcode_profile: str | Path | OpcodeProfile,
790
+ ports: tuple[int, ...] = DEFAULT_SERVER_PORTS,
791
+ event_filter: Optional[EventFilter] = None,
792
+ origin_observer: Optional[Callable[[CompanionObservation], object]] = None,
793
+ on_diagnostic: Optional[Callable[[DecoderDiagnostic], object]] = None,
794
+ ) -> Iterator[BDOEvent]:
795
+ """Replay a pcap/pcapng file and yield structured events.
796
+
797
+ ``event_filter=None`` preserves the complete decoded replay stream. Live
798
+ capture has a narrower activity-only default because hydration can contain
799
+ thousands of state records.
800
+ """
801
+
802
+ if event_filter is not None and not isinstance(event_filter, EventFilter):
803
+ raise TypeError("event_filter must be an EventFilter or None")
804
+ validated_ports = validate_server_ports(ports)
805
+ collector = _EventCollector(
806
+ server_ports=validated_ports,
807
+ event_filter=event_filter,
808
+ opcode_profile=opcode_profile,
809
+ origin_observer=origin_observer,
810
+ on_diagnostic=on_diagnostic,
811
+ )
812
+ for _ in iter_pcap_file(Path(path), collector.engine):
813
+ yield from collector.drain_events()
814
+ collector.finalize()
815
+ yield from collector.drain_events()
816
+
817
+
818
+ class LiveCaptureSession:
819
+ """Controllable live capture for applications and long-running services.
820
+
821
+ ``start()`` begins capture in Scapy's background thread. Consume events
822
+ with the blocking ``events()`` iterator or the timeout-aware ``poll()``
823
+ method, and call ``stop()`` from the UI/control thread to wake consumers,
824
+ finish TCP reassembly, and finalize pending deposit-origin decisions.
825
+
826
+ A session is single-use and supports one event consumer. Create a new
827
+ session when a stopped feature is started again. With ``event_filter=None``
828
+ the session delivers :meth:`EventFilter.activity` events. Pass an explicit
829
+ filter, including ``EventFilter.all()``, to select different semantics.
830
+ """
831
+
832
+ _POLL_INTERVAL_SECONDS = 0.2
833
+ _DECODER_STOP_TIMEOUT_SECONDS = 5.0
834
+
835
+ def __init__(
836
+ self,
837
+ *,
838
+ opcode_profile: str | Path | OpcodeProfile,
839
+ live_options: Optional[LiveCaptureOptions] = None,
840
+ event_filter: Optional[EventFilter] = None,
841
+ origin_observer: Optional[Callable[[CompanionObservation], object]] = None,
842
+ on_diagnostic: Optional[Callable[[DecoderDiagnostic], object]] = None,
843
+ ) -> None:
844
+ if live_options is not None and not isinstance(live_options, LiveCaptureOptions):
845
+ raise TypeError("live_options must be a LiveCaptureOptions or None")
846
+ if event_filter is not None and not isinstance(event_filter, EventFilter):
847
+ raise TypeError("event_filter must be an EventFilter or None")
848
+ resolved_live_options = live_options or LiveCaptureOptions()
849
+
850
+ self._opcode_profile = opcode_profile
851
+ self._live_options = resolved_live_options
852
+ self._event_filter = (
853
+ event_filter if event_filter is not None else EventFilter.activity()
854
+ )
855
+ self._origin_observer = origin_observer
856
+ self._on_diagnostic = on_diagnostic
857
+
858
+ self._queue: Queue[BDOEvent] = Queue(
859
+ maxsize=resolved_live_options.event_queue_size
860
+ )
861
+ self._packet_queue: Queue[object] = Queue(
862
+ maxsize=resolved_live_options.packet_queue_size
863
+ )
864
+ self._tail_events: deque[BDOEvent] = deque()
865
+ self._tail_lock = Lock()
866
+ self._delivery_lock = RLock()
867
+ self._decoder_lock = Lock()
868
+ self._state_lock = Lock()
869
+ self._cleanup_lock = RLock()
870
+ self._stop_requested = Event()
871
+ self._finalizing = Event()
872
+ self._stopped = Event()
873
+ self._started = False
874
+ self._capture: Optional[LivePacketCapture] = None
875
+ self._collector: Optional[_EventCollector] = None
876
+ self._packet_handler: Optional[Callable[[object], None]] = None
877
+ self._packet_worker: Optional[Thread] = None
878
+ self._packet_worker_stop_signaled = False
879
+ self._stop_monitor: Optional[Thread] = None
880
+ self._error: Optional[BaseException] = None
881
+ self._stop_reason: Optional[str] = None
882
+ self._capture_stats = CaptureStats()
883
+ self._packets_accepted = 0
884
+ self._packet_queue_peak = 0
885
+ self._packet_queue_overflows = 0
886
+ self._event_queue_peak = 0
887
+ self._cleanup_incomplete = False
888
+
889
+ @property
890
+ def running(self) -> bool:
891
+ capture = self._capture
892
+ return (
893
+ self._started
894
+ and not self._stopped.is_set()
895
+ and capture is not None
896
+ and capture.running
897
+ )
898
+
899
+ @property
900
+ def stopped(self) -> bool:
901
+ return self._stopped.is_set()
902
+
903
+ @property
904
+ def cleanup_incomplete(self) -> bool:
905
+ """Whether a failed stop retained pipeline ownership for retry."""
906
+
907
+ capture = self._capture
908
+ with self._state_lock:
909
+ pipeline_incomplete = self._cleanup_incomplete
910
+ return pipeline_incomplete or (
911
+ capture is not None and capture.cleanup_incomplete
912
+ )
913
+
914
+ @property
915
+ def stop_reason(self) -> Optional[str]:
916
+ """Why capture ended: requested, capture-ended, or error."""
917
+ with self._state_lock:
918
+ return self._stop_reason
919
+
920
+ @property
921
+ def error(self) -> Optional[BaseException]:
922
+ """First decoder, observer, or cleanup exception, if one occurred."""
923
+ with self._state_lock:
924
+ return self._error
925
+
926
+ @property
927
+ def endpoint(self) -> Optional[CaptureEndpoint]:
928
+ """Resolved interface, local address, and packet filter after start."""
929
+
930
+ capture = self._capture
931
+ return capture.endpoint if capture is not None else None
932
+
933
+ @property
934
+ def health(self) -> LiveCaptureHealth:
935
+ """Return an immutable, observational integrity snapshot.
936
+
937
+ Active counter families may be sampled at adjacent instants. Sampling
938
+ does not acquire structural TCP-reassembly ownership.
939
+ """
940
+
941
+ capture = self._capture
942
+ stats = self._capture_stats
943
+ if capture is not None and not capture.stopped:
944
+ # Reading native counters is best-effort and must never turn a
945
+ # diagnostic property into a capture failure.
946
+ try:
947
+ stats = capture.snapshot_stats()
948
+ except BaseException:
949
+ stats = capture.stats
950
+ collector = self._collector
951
+ engine = collector.engine if collector is not None else None
952
+ tcp_gap_resets = int(getattr(engine, "tcp_gap_resets", 0))
953
+ flow_state_evictions = int(getattr(engine, "flow_state_evictions", 0))
954
+ pcap_received = stats.received
955
+ pcap_dropped = stats.dropped
956
+ pcap_interface_dropped = stats.interface_dropped
957
+ capture_buffer_bytes = stats.capture_buffer_bytes
958
+ capture_buffer_fallback = (
959
+ capture is not None and capture.buffer_error is not None
960
+ )
961
+ with self._state_lock:
962
+ packets_accepted = self._packets_accepted
963
+ packet_queue_peak = self._packet_queue_peak
964
+ packet_queue_overflows = self._packet_queue_overflows
965
+ event_queue_peak = self._event_queue_peak
966
+ return LiveCaptureHealth(
967
+ packets_accepted=packets_accepted,
968
+ packet_queue_peak=packet_queue_peak,
969
+ packet_queue_overflows=packet_queue_overflows,
970
+ event_queue_peak=event_queue_peak,
971
+ tcp_gap_resets=tcp_gap_resets,
972
+ flow_state_evictions=flow_state_evictions,
973
+ pcap_received=pcap_received,
974
+ pcap_dropped=pcap_dropped,
975
+ pcap_interface_dropped=pcap_interface_dropped,
976
+ capture_buffer_bytes=capture_buffer_bytes,
977
+ capture_buffer_fallback=capture_buffer_fallback,
978
+ )
979
+
980
+ @property
981
+ def decoder_health(self) -> DecoderHealth:
982
+ """Protocol compatibility, independent of packet-capture integrity."""
983
+
984
+ collector = self._collector
985
+ return collector.decoder_health if collector is not None else DecoderHealth()
986
+
987
+ def start(self) -> None:
988
+ """Open capture and hand packets to a bounded decoder worker."""
989
+ with self._cleanup_lock:
990
+ if self._started:
991
+ raise RuntimeError("live capture session was already started")
992
+ collector = _EventCollector(
993
+ server_ports=self._live_options.ports,
994
+ event_filter=self._event_filter,
995
+ on_event=self._enqueue,
996
+ opcode_profile=self._opcode_profile,
997
+ origin_observer=(
998
+ self._notify_origin_observer
999
+ if self._origin_observer is not None
1000
+ else None
1001
+ ),
1002
+ on_diagnostic=(
1003
+ self._notify_diagnostic
1004
+ if self._on_diagnostic is not None
1005
+ else None
1006
+ ),
1007
+ preflight=False,
1008
+ )
1009
+
1010
+ packet_handler = make_packet_handler(collector.engine)
1011
+ live_capture = LivePacketCapture(
1012
+ capture_options=self._live_options,
1013
+ # This callback deliberately performs only one non-blocking
1014
+ # queue handoff. Protocol decode and event backpressure live
1015
+ # on the worker below, not Scapy's native capture thread.
1016
+ on_packet=self._enqueue_packet,
1017
+ )
1018
+ worker = Thread(
1019
+ target=self._run_packet_worker,
1020
+ name="bdo-toolkit-items",
1021
+ daemon=True,
1022
+ )
1023
+ self._collector = collector
1024
+ self._capture = live_capture
1025
+ self._packet_handler = packet_handler
1026
+ self._packet_worker = worker
1027
+ with self._state_lock:
1028
+ self._started = True
1029
+ self._stopped.clear()
1030
+ self._stop_requested.clear()
1031
+ self._finalizing.clear()
1032
+ capture_started = False
1033
+ try:
1034
+ worker.start()
1035
+ live_capture.start()
1036
+ capture_started = True
1037
+ # The session is now fully addressable, so a startup diagnostic
1038
+ # callback may safely call request_stop(). stop()/poll() remain
1039
+ # rejected by the shared decoder-callback guard.
1040
+ preflight = getattr(collector, "preflight", None)
1041
+ if preflight is not None:
1042
+ preflight()
1043
+
1044
+ monitor = Thread(
1045
+ target=self._monitor_stop_request,
1046
+ name="bdo-toolkit-items-stop",
1047
+ daemon=True,
1048
+ )
1049
+ self._stop_monitor = monitor
1050
+ monitor.start()
1051
+ except BaseException as exc:
1052
+ self._rollback_failed_start(
1053
+ exc,
1054
+ capture_started=capture_started,
1055
+ )
1056
+ raise
1057
+
1058
+ def _rollback_failed_start(
1059
+ self,
1060
+ error: BaseException,
1061
+ *,
1062
+ capture_started: bool,
1063
+ ) -> None:
1064
+ """Release every successfully started owner after startup fails.
1065
+
1066
+ This helper runs while ``_cleanup_lock`` is held. A failed auxiliary
1067
+ ``Thread.start()`` is just as transactional as a native startup
1068
+ failure: verified cleanup restores the pre-start state; anything that
1069
+ cannot be verified remains attached to the original exception for a
1070
+ same-session ``stop()`` retry.
1071
+ """
1072
+
1073
+ self._finalizing.set()
1074
+ self._stop_requested.set()
1075
+ capture = self._capture
1076
+ cleanup_failures: list[BaseException] = []
1077
+
1078
+ if capture_started and capture is not None and not capture.stopped:
1079
+ try:
1080
+ self._capture_stats = capture.stop()
1081
+ except BaseException as stop_error:
1082
+ cleanup_failures.append(stop_error)
1083
+
1084
+ capture_incomplete = capture is not None and (
1085
+ capture.cleanup_incomplete
1086
+ or (capture_started and not capture.stopped)
1087
+ )
1088
+ worker = self._packet_worker
1089
+ worker_alive = worker is not None and worker.is_alive()
1090
+
1091
+ # A native callback may still race with this session until capture
1092
+ # termination is verified. Keep the decoder alive in that case; the
1093
+ # stop request makes every later native callback a no-op.
1094
+ if not capture_incomplete:
1095
+ decoder_deadline = (
1096
+ time.monotonic() + self._DECODER_STOP_TIMEOUT_SECONDS
1097
+ )
1098
+ self._signal_packet_worker_stop(decoder_deadline)
1099
+ if (
1100
+ worker_alive
1101
+ and worker is not None
1102
+ and worker is not current_thread()
1103
+ ):
1104
+ worker.join(
1105
+ timeout=max(0.0, decoder_deadline - time.monotonic())
1106
+ )
1107
+ worker_alive = worker is not None and worker.is_alive()
1108
+
1109
+ if capture_incomplete or worker_alive:
1110
+ self._record_error(error)
1111
+ with self._state_lock:
1112
+ self._cleanup_incomplete = True
1113
+ for failure in cleanup_failures:
1114
+ if hasattr(error, "add_note"):
1115
+ error.add_note(
1116
+ "live item capture startup cleanup also failed: "
1117
+ f"{failure!r}"
1118
+ )
1119
+ _attach_cleanup_owner(
1120
+ error,
1121
+ self,
1122
+ context="live item capture startup",
1123
+ )
1124
+ return
1125
+
1126
+ for failure in cleanup_failures:
1127
+ if hasattr(error, "add_note"):
1128
+ error.add_note(
1129
+ "live item capture startup cleanup reported a fully "
1130
+ f"released error: {failure!r}"
1131
+ )
1132
+ self._reset_after_failed_start()
1133
+
1134
+ def _reset_after_failed_start(self) -> None:
1135
+ """Restore reusable pre-start state after verified rollback."""
1136
+
1137
+ self._collector = None
1138
+ self._capture = None
1139
+ self._packet_handler = None
1140
+ self._packet_worker = None
1141
+ self._packet_worker_stop_signaled = False
1142
+ self._stop_monitor = None
1143
+ self._capture_stats = CaptureStats()
1144
+ while True:
1145
+ try:
1146
+ self._packet_queue.get_nowait()
1147
+ except Empty:
1148
+ break
1149
+ while True:
1150
+ try:
1151
+ self._queue.get_nowait()
1152
+ except Empty:
1153
+ break
1154
+ with self._tail_lock:
1155
+ self._tail_events.clear()
1156
+ with self._state_lock:
1157
+ self._started = False
1158
+ self._error = None
1159
+ self._stop_reason = None
1160
+ self._cleanup_incomplete = False
1161
+ self._packets_accepted = 0
1162
+ self._packet_queue_peak = 0
1163
+ self._packet_queue_overflows = 0
1164
+ self._event_queue_peak = 0
1165
+ # Atomic with _started=False so an old request_stop() either wins
1166
+ # before rollback and is cleared here, or observes not-started.
1167
+ self._stopped.clear()
1168
+ self._stop_requested.clear()
1169
+ self._finalizing.clear()
1170
+
1171
+ def stop(self) -> None:
1172
+ """Stop capture and finalize queued events; safe from a control thread."""
1173
+ if (
1174
+ self._packet_worker is current_thread()
1175
+ or self._inside_origin_observer()
1176
+ ):
1177
+ raise RuntimeError(
1178
+ "stop() cannot block inside the live decoder or origin "
1179
+ "observer; use request_stop() instead"
1180
+ )
1181
+ self._require_started()
1182
+ self._finish_stop("requested")
1183
+
1184
+ def request_stop(self) -> None:
1185
+ """Request callback-safe shutdown without waiting for finalization."""
1186
+
1187
+ # Deliberately lock-free: a decoder/origin callback must be able to
1188
+ # request shutdown while a control thread owns _cleanup_lock and waits
1189
+ # for that callback's worker to finish.
1190
+ with self._state_lock:
1191
+ if not self._started:
1192
+ raise RuntimeError("live capture session was not started")
1193
+ if self._stopped.is_set():
1194
+ return
1195
+ # Events produced by packets already accepted before this request
1196
+ # remain deliverable while the monitor joins and drains the worker.
1197
+ self._finalizing.set()
1198
+ self._stop_requested.set()
1199
+
1200
+ def poll(self, timeout: Optional[float] = None) -> Optional[BDOEvent]:
1201
+ """Return one event, or ``None`` on timeout or after the final event.
1202
+
1203
+ ``timeout=None`` waits until an event arrives or the session stops.
1204
+ Even an indefinite poll wakes promptly when another thread calls
1205
+ ``stop()``.
1206
+ """
1207
+ if self._inside_origin_observer():
1208
+ raise RuntimeError(
1209
+ "poll() cannot consume events inside origin_observer; "
1210
+ "consume them after the callback returns"
1211
+ )
1212
+ self._require_started()
1213
+ _validate_poll_timeout(timeout)
1214
+ deadline = None if timeout is None else time.monotonic() + timeout
1215
+
1216
+ while True:
1217
+ self._service_capture_state()
1218
+ try:
1219
+ return self._queue.get_nowait()
1220
+ except Empty:
1221
+ pass
1222
+
1223
+ # Serialize the empty-queue recheck with producers. This lets a
1224
+ # non-blocking UI poll safely flush stale origin decisions without
1225
+ # racing a producer that fills the one available queue slot.
1226
+ with self._delivery_lock:
1227
+ try:
1228
+ return self._queue.get_nowait()
1229
+ except Empty:
1230
+ pass
1231
+ self._flush_stale()
1232
+ try:
1233
+ return self._queue.get_nowait()
1234
+ except Empty:
1235
+ pass
1236
+
1237
+ if self._stopped.is_set():
1238
+ with self._tail_lock:
1239
+ if self._tail_events:
1240
+ return self._tail_events.popleft()
1241
+ self.raise_if_failed()
1242
+ return None
1243
+
1244
+ if deadline is not None:
1245
+ remaining = deadline - time.monotonic()
1246
+ if remaining <= 0:
1247
+ return None
1248
+ wait_seconds = min(self._POLL_INTERVAL_SECONDS, remaining)
1249
+ else:
1250
+ wait_seconds = self._POLL_INTERVAL_SECONDS
1251
+
1252
+ try:
1253
+ return self._queue.get(timeout=wait_seconds)
1254
+ except Empty:
1255
+ continue
1256
+
1257
+ def events(self) -> Iterator[BDOEvent]:
1258
+ """Return a blocking iterator that ends after ``stop()`` and draining."""
1259
+ self._require_started()
1260
+ return self._iterate_events()
1261
+
1262
+ def raise_if_failed(self) -> None:
1263
+ """Re-raise a background capture failure in the calling thread."""
1264
+ error = self.error
1265
+ if error is not None:
1266
+ raise error
1267
+
1268
+ def _iterate_events(self) -> Iterator[BDOEvent]:
1269
+ while True:
1270
+ event = self.poll(timeout=self._POLL_INTERVAL_SECONDS)
1271
+ if event is not None:
1272
+ yield event
1273
+ elif self._stopped.is_set():
1274
+ return
1275
+
1276
+ def _enqueue_packet(self, packet: object) -> None:
1277
+ """Perform the native-callback handoff without blocking Scapy."""
1278
+
1279
+ if self._stop_requested.is_set():
1280
+ return
1281
+ try:
1282
+ self._packet_queue.put_nowait(packet)
1283
+ except Full:
1284
+ with self._state_lock:
1285
+ self._packet_queue_overflows += 1
1286
+ self._record_error(
1287
+ CaptureIntegrityError(
1288
+ "live packet queue overflowed; the event stream may be incomplete"
1289
+ )
1290
+ )
1291
+ self._stop_requested.set()
1292
+ return
1293
+ depth = self._packet_queue.qsize()
1294
+ with self._state_lock:
1295
+ self._packets_accepted += 1
1296
+ self._packet_queue_peak = max(self._packet_queue_peak, depth)
1297
+
1298
+ def _run_packet_worker(self) -> None:
1299
+ """Decode accepted packets in FIFO order away from capture callback."""
1300
+
1301
+ decode_enabled = True
1302
+ while True:
1303
+ packet = self._packet_queue.get()
1304
+ if packet is _PACKET_WORKER_STOP:
1305
+ return
1306
+ if not decode_enabled:
1307
+ # After decoder state fails, retain deterministic shutdown by
1308
+ # draining accepted packets without invoking a corrupt decoder.
1309
+ continue
1310
+ handler = self._packet_handler
1311
+ if handler is None:
1312
+ self._record_error(RuntimeError("live packet decoder was unavailable"))
1313
+ self._stop_requested.set()
1314
+ decode_enabled = False
1315
+ continue
1316
+ try:
1317
+ with self._decoder_lock:
1318
+ handler(packet)
1319
+ except BaseException as exc:
1320
+ self._record_error(exc)
1321
+ self._stop_requested.set()
1322
+ decode_enabled = False
1323
+
1324
+ def _monitor_stop_request(self) -> None:
1325
+ """Service idle state and stop failures without an active consumer."""
1326
+
1327
+ while not self._stop_requested.wait(self._POLL_INTERVAL_SECONDS):
1328
+ capture = self._capture
1329
+ capture_error = capture.error if capture is not None else None
1330
+ if isinstance(capture_error, BaseException):
1331
+ self._record_error(capture_error)
1332
+ self._stop_requested.set()
1333
+ break
1334
+ if capture is not None and not capture.running:
1335
+ self._stop_requested.set()
1336
+ break
1337
+ self._service_engine_clock()
1338
+ if not self._stopped.is_set():
1339
+ capture = self._capture
1340
+ capture_error = capture.error if capture is not None else None
1341
+ if isinstance(capture_error, BaseException):
1342
+ self._record_error(capture_error)
1343
+ reason = (
1344
+ "error"
1345
+ if self.error is not None
1346
+ else "requested"
1347
+ if capture is None or capture.running
1348
+ else "capture-ended"
1349
+ )
1350
+ try:
1351
+ self._finish_stop(reason)
1352
+ except BaseException as exc:
1353
+ # A non-cooperative backend remains owned and retryable. The
1354
+ # public stop()/poll() paths surface the same retained error;
1355
+ # do not leak an unhandled exception from this daemon monitor.
1356
+ self._record_error(exc)
1357
+
1358
+ def _service_engine_clock(self) -> None:
1359
+ collector = self._collector
1360
+ service_gaps = (
1361
+ getattr(collector.engine, "service_gaps", None)
1362
+ if collector is not None
1363
+ else None
1364
+ )
1365
+ if service_gaps is None:
1366
+ return
1367
+ try:
1368
+ with self._decoder_lock:
1369
+ service_gaps(time.time())
1370
+ except BaseException as exc:
1371
+ self._record_error(exc)
1372
+ self._stop_requested.set()
1373
+
1374
+ def _signal_packet_worker_stop(self, deadline: Optional[float] = None) -> bool:
1375
+ worker = self._packet_worker
1376
+ if worker is None:
1377
+ return True
1378
+ if self._packet_worker_stop_signaled:
1379
+ return True
1380
+ if deadline is None:
1381
+ deadline = time.monotonic() + self._DECODER_STOP_TIMEOUT_SECONDS
1382
+ while worker.is_alive():
1383
+ remaining = deadline - time.monotonic()
1384
+ if remaining <= 0:
1385
+ return False
1386
+ try:
1387
+ self._packet_queue.put(
1388
+ _PACKET_WORKER_STOP,
1389
+ timeout=min(self._POLL_INTERVAL_SECONDS, remaining),
1390
+ )
1391
+ self._packet_worker_stop_signaled = True
1392
+ return True
1393
+ except Full:
1394
+ continue
1395
+ return True
1396
+
1397
+ def _enqueue(self, event: BDOEvent) -> None:
1398
+ # During shutdown the sniffer is joined before the consumer necessarily
1399
+ # drains its bounded queue. Route finalized events to a short-lived
1400
+ # tail instead of deadlocking the stop caller.
1401
+ with self._delivery_lock:
1402
+ if self._finalizing.is_set():
1403
+ with self._tail_lock:
1404
+ self._tail_events.append(event)
1405
+ return
1406
+ while not self._stop_requested.is_set():
1407
+ try:
1408
+ self._queue.put(event, timeout=self._POLL_INTERVAL_SECONDS)
1409
+ depth = self._queue.qsize()
1410
+ with self._state_lock:
1411
+ self._event_queue_peak = max(
1412
+ self._event_queue_peak,
1413
+ depth,
1414
+ )
1415
+ return
1416
+ except Full:
1417
+ if self._finalizing.is_set():
1418
+ with self._tail_lock:
1419
+ self._tail_events.append(event)
1420
+ return
1421
+
1422
+ def _service_capture_state(self) -> None:
1423
+ if self._stopped.is_set():
1424
+ return
1425
+ if self._stop_requested.is_set():
1426
+ self._finish_stop("error" if self.error is not None else "requested")
1427
+ return
1428
+ capture = self._capture
1429
+ capture_error = capture.error if capture is not None else None
1430
+ if isinstance(capture_error, BaseException):
1431
+ self._record_error(capture_error)
1432
+ self._finish_stop("error")
1433
+ return
1434
+ if capture is not None and not capture.running:
1435
+ self._finish_stop("error" if self.error is not None else "capture-ended")
1436
+
1437
+ def _flush_stale(self) -> None:
1438
+ collector = self._collector
1439
+ if collector is None or self._stopped.is_set():
1440
+ return
1441
+ try:
1442
+ collector.flush_stale(time.time())
1443
+ except BaseException as exc:
1444
+ self._record_error(exc)
1445
+ self._stop_requested.set()
1446
+
1447
+ def _finish_stop(self, reason: str) -> None:
1448
+ with self._cleanup_lock:
1449
+ if self._stopped.is_set():
1450
+ return
1451
+ self._finalizing.set()
1452
+ self._stop_requested.set()
1453
+ capture = self._capture
1454
+ collector = self._collector
1455
+
1456
+ capture_error = capture.error if capture is not None else None
1457
+ if isinstance(capture_error, BaseException):
1458
+ self._record_error(capture_error)
1459
+
1460
+ if capture is not None and not capture.stopped:
1461
+ stop_failure: Optional[BaseException] = None
1462
+ try:
1463
+ self._capture_stats = capture.stop()
1464
+ except BaseException as exc:
1465
+ stop_failure = exc
1466
+ self._record_error(exc)
1467
+ if not capture.stopped:
1468
+ cleanup_error = (
1469
+ capture.cleanup_error
1470
+ or stop_failure
1471
+ or RuntimeError(
1472
+ "live capture cleanup is incomplete after stop"
1473
+ )
1474
+ )
1475
+ self._record_error(cleanup_error)
1476
+ with self._state_lock:
1477
+ self._cleanup_incomplete = True
1478
+ # Do not queue the worker sentinel or finalize decoder
1479
+ # state while the native callback may still be active.
1480
+ # _stop_requested makes every later callback a no-op.
1481
+ raise cleanup_error
1482
+ capture_error = capture.error
1483
+ if isinstance(capture_error, BaseException):
1484
+ self._record_error(capture_error)
1485
+ elif capture is not None:
1486
+ self._capture_stats = capture.stats
1487
+
1488
+ # Capture is joined before the sentinel is queued, so every packet
1489
+ # accepted by the callback appears ahead of it in FIFO order.
1490
+ decoder_deadline = (
1491
+ time.monotonic() + self._DECODER_STOP_TIMEOUT_SECONDS
1492
+ )
1493
+ self._signal_packet_worker_stop(decoder_deadline)
1494
+ worker = self._packet_worker
1495
+ if worker is not None and worker.is_alive():
1496
+ if worker is not current_thread():
1497
+ worker.join(
1498
+ timeout=max(0.0, decoder_deadline - time.monotonic())
1499
+ )
1500
+ if worker.is_alive():
1501
+ cleanup_error = RuntimeError(
1502
+ "live packet decoder cleanup is incomplete after the "
1503
+ f"{self._DECODER_STOP_TIMEOUT_SECONDS:g}-second deadline"
1504
+ )
1505
+ self._record_error(cleanup_error)
1506
+ with self._state_lock:
1507
+ self._cleanup_incomplete = True
1508
+ # The worker may still hold _decoder_lock or call into the
1509
+ # collector. Retain every dependency and retry only after
1510
+ # its termination can be verified.
1511
+ raise cleanup_error
1512
+ if collector is not None:
1513
+ try:
1514
+ with self._decoder_lock:
1515
+ collector.engine.finish()
1516
+ except BaseException as exc:
1517
+ self._record_error(exc)
1518
+ try:
1519
+ collector.finalize()
1520
+ except BaseException as exc:
1521
+ self._record_error(exc)
1522
+
1523
+ self._packet_handler = None
1524
+
1525
+ with self._state_lock:
1526
+ self._stop_reason = "error" if self._error is not None else reason
1527
+ self._cleanup_incomplete = False
1528
+ self._stopped.set()
1529
+
1530
+ def _record_error(self, error: BaseException) -> None:
1531
+ with self._state_lock:
1532
+ if self._error is None:
1533
+ self._error = error
1534
+
1535
+ def _notify_origin_observer(
1536
+ self,
1537
+ observation: CompanionObservation,
1538
+ ) -> object:
1539
+ callback = self._origin_observer
1540
+ if callback is None:
1541
+ return None
1542
+ active_sessions = _ACTIVE_ORIGIN_SESSIONS.get()
1543
+ token = _ACTIVE_ORIGIN_SESSIONS.set(active_sessions + (self,))
1544
+ try:
1545
+ return callback(observation)
1546
+ finally:
1547
+ _ACTIVE_ORIGIN_SESSIONS.reset(token)
1548
+
1549
+ def _notify_diagnostic(self, diagnostic: DecoderDiagnostic) -> object:
1550
+ callback = self._on_diagnostic
1551
+ if callback is None:
1552
+ return None
1553
+ active_sessions = _ACTIVE_ORIGIN_SESSIONS.get()
1554
+ token = _ACTIVE_ORIGIN_SESSIONS.set(active_sessions + (self,))
1555
+ try:
1556
+ return callback(diagnostic)
1557
+ finally:
1558
+ _ACTIVE_ORIGIN_SESSIONS.reset(token)
1559
+
1560
+ def _inside_origin_observer(self) -> bool:
1561
+ return self in _ACTIVE_ORIGIN_SESSIONS.get()
1562
+
1563
+ def _require_started(self) -> None:
1564
+ # Startup and verified rollback both mutate ``_started`` under the
1565
+ # cleanup lock. Waiting here prevents a concurrent stop/poll from
1566
+ # observing the provisional True and acting on a rolled-back session.
1567
+ with self._cleanup_lock:
1568
+ with self._state_lock:
1569
+ if not self._started:
1570
+ raise RuntimeError("live capture session was not started")
1571
+
1572
+ def __enter__(self) -> "LiveCaptureSession":
1573
+ self.start()
1574
+ return self
1575
+
1576
+ def __exit__(self, exc_type, exc_value, traceback) -> None:
1577
+ if self._started and not self._stopped.is_set():
1578
+ try:
1579
+ self.stop()
1580
+ except BaseException as cleanup_error:
1581
+ if exc_value is None:
1582
+ raise
1583
+ if self.cleanup_incomplete:
1584
+ _attach_cleanup_owner(
1585
+ exc_value,
1586
+ self,
1587
+ context="live item capture context",
1588
+ )
1589
+ if hasattr(exc_value, "add_note"):
1590
+ exc_value.add_note(
1591
+ "live item capture context cleanup also failed: "
1592
+ f"{cleanup_error!r}"
1593
+ )
1594
+
1595
+
1596
+ def capture_live(
1597
+ *,
1598
+ opcode_profile: str | Path | OpcodeProfile,
1599
+ live_options: Optional[LiveCaptureOptions] = None,
1600
+ event_filter: Optional[EventFilter] = None,
1601
+ capture_seconds: Optional[float] = None,
1602
+ origin_observer: Optional[Callable[[CompanionObservation], object]] = None,
1603
+ on_diagnostic: Optional[Callable[[DecoderDiagnostic], object]] = None,
1604
+ ) -> Iterator[BDOEvent]:
1605
+ """Start passive live capture and yield structured events.
1606
+
1607
+ This blocking convenience wrapper is intended for scripts. Applications
1608
+ that need programmatic start/stop control should use
1609
+ :class:`LiveCaptureSession`. When ``event_filter`` is omitted, only
1610
+ :meth:`EventFilter.activity` events are delivered; pass
1611
+ ``EventFilter.all()`` for the complete decoded stream.
1612
+ """
1613
+ _validate_capture_seconds(capture_seconds)
1614
+ session = LiveCaptureSession(
1615
+ opcode_profile=opcode_profile,
1616
+ live_options=live_options,
1617
+ event_filter=event_filter,
1618
+ origin_observer=origin_observer,
1619
+ on_diagnostic=on_diagnostic,
1620
+ )
1621
+ session.start()
1622
+ started_at = time.monotonic()
1623
+ deadline_timer: Optional[Timer] = None
1624
+ if capture_seconds is not None:
1625
+ # The deadline belongs to capture ownership, not generator progress.
1626
+ # It therefore fires even while the consumer is suspended after a
1627
+ # yielded event.
1628
+ def finish_at_deadline() -> None:
1629
+ try:
1630
+ session._finish_stop("timeout")
1631
+ except BaseException as exc:
1632
+ # _finish_stop has retained both the first error and every
1633
+ # owner required for a retry. Avoid an unhandled Timer-thread
1634
+ # traceback while the generator remains the public owner.
1635
+ session._record_error(exc)
1636
+
1637
+ try:
1638
+ deadline_timer = Timer(capture_seconds, finish_at_deadline)
1639
+ deadline_timer.name = "bdo-toolkit-items-deadline"
1640
+ deadline_timer.daemon = True
1641
+ deadline_timer.start()
1642
+ except BaseException as exc:
1643
+ if deadline_timer is not None:
1644
+ deadline_timer.cancel()
1645
+ try:
1646
+ session.stop()
1647
+ except BaseException as cleanup_error:
1648
+ if session.cleanup_incomplete:
1649
+ _attach_cleanup_owner(
1650
+ exc,
1651
+ session,
1652
+ context="timed live item capture startup",
1653
+ )
1654
+ if hasattr(exc, "add_note"):
1655
+ exc.add_note(
1656
+ "timed live item capture startup cleanup also "
1657
+ f"failed: {cleanup_error!r}"
1658
+ )
1659
+ raise
1660
+ try:
1661
+ while True:
1662
+ if capture_seconds is None:
1663
+ poll_timeout = LiveCaptureSession._POLL_INTERVAL_SECONDS
1664
+ else:
1665
+ remaining = capture_seconds - (time.monotonic() - started_at)
1666
+ if remaining <= 0:
1667
+ session._finish_stop("timeout")
1668
+ poll_timeout = 0.0
1669
+ else:
1670
+ poll_timeout = min(
1671
+ LiveCaptureSession._POLL_INTERVAL_SECONDS,
1672
+ remaining,
1673
+ )
1674
+
1675
+ event = session.poll(timeout=poll_timeout)
1676
+ if event is not None:
1677
+ yield event
1678
+ elif session.stopped:
1679
+ break
1680
+ finally:
1681
+ if deadline_timer is not None:
1682
+ deadline_timer.cancel()
1683
+ # No yields during generator close. The session finalizes pending TCP
1684
+ # and origin state; a normal stop path drains it in the loop above.
1685
+ if not session.stopped:
1686
+ try:
1687
+ session.stop()
1688
+ except BaseException as exc:
1689
+ if session.cleanup_incomplete:
1690
+ _attach_cleanup_owner(
1691
+ exc,
1692
+ session,
1693
+ context="live item capture convenience wrapper",
1694
+ )
1695
+ raise
1696
+
1697
+
1698
+ def _validate_poll_timeout(value: Optional[float]) -> None:
1699
+ if value is None:
1700
+ return
1701
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
1702
+ raise ValueError("timeout must be a number or None")
1703
+ if not math.isfinite(value) or value < 0:
1704
+ raise ValueError("timeout must be finite and non-negative")
1705
+
1706
+
1707
+ def _validate_capture_seconds(value: Optional[float]) -> None:
1708
+ if value is None:
1709
+ return
1710
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
1711
+ raise ValueError("capture_seconds must be a number or None")
1712
+ if not math.isfinite(value) or value < 0:
1713
+ raise ValueError("capture_seconds must be finite and non-negative")