oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (497) hide show
  1. oscura/__init__.py +169 -167
  2. oscura/analyzers/__init__.py +3 -0
  3. oscura/analyzers/classification.py +659 -0
  4. oscura/analyzers/digital/edges.py +325 -65
  5. oscura/analyzers/digital/quality.py +293 -166
  6. oscura/analyzers/digital/timing.py +260 -115
  7. oscura/analyzers/digital/timing_numba.py +334 -0
  8. oscura/analyzers/entropy.py +605 -0
  9. oscura/analyzers/eye/diagram.py +176 -109
  10. oscura/analyzers/eye/metrics.py +5 -5
  11. oscura/analyzers/jitter/__init__.py +6 -4
  12. oscura/analyzers/jitter/ber.py +52 -52
  13. oscura/analyzers/jitter/classification.py +156 -0
  14. oscura/analyzers/jitter/decomposition.py +163 -113
  15. oscura/analyzers/jitter/spectrum.py +80 -64
  16. oscura/analyzers/ml/__init__.py +39 -0
  17. oscura/analyzers/ml/features.py +600 -0
  18. oscura/analyzers/ml/signal_classifier.py +604 -0
  19. oscura/analyzers/packet/daq.py +246 -158
  20. oscura/analyzers/packet/parser.py +12 -1
  21. oscura/analyzers/packet/payload.py +50 -2110
  22. oscura/analyzers/packet/payload_analysis.py +361 -181
  23. oscura/analyzers/packet/payload_patterns.py +133 -70
  24. oscura/analyzers/packet/stream.py +84 -23
  25. oscura/analyzers/patterns/__init__.py +26 -5
  26. oscura/analyzers/patterns/anomaly_detection.py +908 -0
  27. oscura/analyzers/patterns/clustering.py +169 -108
  28. oscura/analyzers/patterns/clustering_optimized.py +227 -0
  29. oscura/analyzers/patterns/discovery.py +1 -1
  30. oscura/analyzers/patterns/matching.py +581 -197
  31. oscura/analyzers/patterns/pattern_mining.py +778 -0
  32. oscura/analyzers/patterns/periodic.py +121 -38
  33. oscura/analyzers/patterns/sequences.py +175 -78
  34. oscura/analyzers/power/conduction.py +1 -1
  35. oscura/analyzers/power/soa.py +6 -6
  36. oscura/analyzers/power/switching.py +250 -110
  37. oscura/analyzers/protocol/__init__.py +17 -1
  38. oscura/analyzers/protocols/base.py +6 -6
  39. oscura/analyzers/protocols/ble/__init__.py +38 -0
  40. oscura/analyzers/protocols/ble/analyzer.py +809 -0
  41. oscura/analyzers/protocols/ble/uuids.py +288 -0
  42. oscura/analyzers/protocols/can.py +257 -127
  43. oscura/analyzers/protocols/can_fd.py +107 -80
  44. oscura/analyzers/protocols/flexray.py +139 -80
  45. oscura/analyzers/protocols/hdlc.py +93 -58
  46. oscura/analyzers/protocols/i2c.py +247 -106
  47. oscura/analyzers/protocols/i2s.py +138 -86
  48. oscura/analyzers/protocols/industrial/__init__.py +40 -0
  49. oscura/analyzers/protocols/industrial/bacnet/__init__.py +33 -0
  50. oscura/analyzers/protocols/industrial/bacnet/analyzer.py +708 -0
  51. oscura/analyzers/protocols/industrial/bacnet/encoding.py +412 -0
  52. oscura/analyzers/protocols/industrial/bacnet/services.py +622 -0
  53. oscura/analyzers/protocols/industrial/ethercat/__init__.py +30 -0
  54. oscura/analyzers/protocols/industrial/ethercat/analyzer.py +474 -0
  55. oscura/analyzers/protocols/industrial/ethercat/mailbox.py +339 -0
  56. oscura/analyzers/protocols/industrial/ethercat/topology.py +166 -0
  57. oscura/analyzers/protocols/industrial/modbus/__init__.py +31 -0
  58. oscura/analyzers/protocols/industrial/modbus/analyzer.py +525 -0
  59. oscura/analyzers/protocols/industrial/modbus/crc.py +79 -0
  60. oscura/analyzers/protocols/industrial/modbus/functions.py +436 -0
  61. oscura/analyzers/protocols/industrial/opcua/__init__.py +21 -0
  62. oscura/analyzers/protocols/industrial/opcua/analyzer.py +552 -0
  63. oscura/analyzers/protocols/industrial/opcua/datatypes.py +446 -0
  64. oscura/analyzers/protocols/industrial/opcua/services.py +264 -0
  65. oscura/analyzers/protocols/industrial/profinet/__init__.py +23 -0
  66. oscura/analyzers/protocols/industrial/profinet/analyzer.py +441 -0
  67. oscura/analyzers/protocols/industrial/profinet/dcp.py +263 -0
  68. oscura/analyzers/protocols/industrial/profinet/ptcp.py +200 -0
  69. oscura/analyzers/protocols/jtag.py +180 -98
  70. oscura/analyzers/protocols/lin.py +219 -114
  71. oscura/analyzers/protocols/manchester.py +4 -4
  72. oscura/analyzers/protocols/onewire.py +253 -149
  73. oscura/analyzers/protocols/parallel_bus/__init__.py +20 -0
  74. oscura/analyzers/protocols/parallel_bus/centronics.py +92 -0
  75. oscura/analyzers/protocols/parallel_bus/gpib.py +137 -0
  76. oscura/analyzers/protocols/spi.py +192 -95
  77. oscura/analyzers/protocols/swd.py +321 -167
  78. oscura/analyzers/protocols/uart.py +267 -125
  79. oscura/analyzers/protocols/usb.py +235 -131
  80. oscura/analyzers/side_channel/power.py +17 -12
  81. oscura/analyzers/signal/__init__.py +15 -0
  82. oscura/analyzers/signal/timing_analysis.py +1086 -0
  83. oscura/analyzers/signal_integrity/__init__.py +4 -1
  84. oscura/analyzers/signal_integrity/sparams.py +2 -19
  85. oscura/analyzers/spectral/chunked.py +129 -60
  86. oscura/analyzers/spectral/chunked_fft.py +300 -94
  87. oscura/analyzers/spectral/chunked_wavelet.py +100 -80
  88. oscura/analyzers/statistical/checksum.py +376 -217
  89. oscura/analyzers/statistical/classification.py +229 -107
  90. oscura/analyzers/statistical/entropy.py +78 -53
  91. oscura/analyzers/statistics/correlation.py +407 -211
  92. oscura/analyzers/statistics/outliers.py +2 -2
  93. oscura/analyzers/statistics/streaming.py +30 -5
  94. oscura/analyzers/validation.py +216 -101
  95. oscura/analyzers/waveform/measurements.py +9 -0
  96. oscura/analyzers/waveform/measurements_with_uncertainty.py +31 -15
  97. oscura/analyzers/waveform/spectral.py +500 -228
  98. oscura/api/__init__.py +31 -5
  99. oscura/api/dsl/__init__.py +582 -0
  100. oscura/{dsl → api/dsl}/commands.py +43 -76
  101. oscura/{dsl → api/dsl}/interpreter.py +26 -51
  102. oscura/{dsl → api/dsl}/parser.py +107 -77
  103. oscura/{dsl → api/dsl}/repl.py +2 -2
  104. oscura/api/dsl.py +1 -1
  105. oscura/{integrations → api/integrations}/__init__.py +1 -1
  106. oscura/{integrations → api/integrations}/llm.py +201 -102
  107. oscura/api/operators.py +3 -3
  108. oscura/api/optimization.py +144 -30
  109. oscura/api/rest_server.py +921 -0
  110. oscura/api/server/__init__.py +17 -0
  111. oscura/api/server/dashboard.py +850 -0
  112. oscura/api/server/static/README.md +34 -0
  113. oscura/api/server/templates/base.html +181 -0
  114. oscura/api/server/templates/export.html +120 -0
  115. oscura/api/server/templates/home.html +284 -0
  116. oscura/api/server/templates/protocols.html +58 -0
  117. oscura/api/server/templates/reports.html +43 -0
  118. oscura/api/server/templates/session_detail.html +89 -0
  119. oscura/api/server/templates/sessions.html +83 -0
  120. oscura/api/server/templates/waveforms.html +73 -0
  121. oscura/automotive/__init__.py +8 -1
  122. oscura/automotive/can/__init__.py +10 -0
  123. oscura/automotive/can/checksum.py +3 -1
  124. oscura/automotive/can/dbc_generator.py +590 -0
  125. oscura/automotive/can/message_wrapper.py +121 -74
  126. oscura/automotive/can/patterns.py +98 -21
  127. oscura/automotive/can/session.py +292 -56
  128. oscura/automotive/can/state_machine.py +6 -3
  129. oscura/automotive/can/stimulus_response.py +97 -75
  130. oscura/automotive/dbc/__init__.py +10 -2
  131. oscura/automotive/dbc/generator.py +84 -56
  132. oscura/automotive/dbc/parser.py +6 -6
  133. oscura/automotive/dtc/data.json +17 -102
  134. oscura/automotive/dtc/database.py +2 -2
  135. oscura/automotive/flexray/__init__.py +31 -0
  136. oscura/automotive/flexray/analyzer.py +504 -0
  137. oscura/automotive/flexray/crc.py +185 -0
  138. oscura/automotive/flexray/fibex.py +449 -0
  139. oscura/automotive/j1939/__init__.py +45 -8
  140. oscura/automotive/j1939/analyzer.py +605 -0
  141. oscura/automotive/j1939/spns.py +326 -0
  142. oscura/automotive/j1939/transport.py +306 -0
  143. oscura/automotive/lin/__init__.py +47 -0
  144. oscura/automotive/lin/analyzer.py +612 -0
  145. oscura/automotive/loaders/blf.py +13 -2
  146. oscura/automotive/loaders/csv_can.py +143 -72
  147. oscura/automotive/loaders/dispatcher.py +50 -2
  148. oscura/automotive/loaders/mdf.py +86 -45
  149. oscura/automotive/loaders/pcap.py +111 -61
  150. oscura/automotive/uds/__init__.py +4 -0
  151. oscura/automotive/uds/analyzer.py +725 -0
  152. oscura/automotive/uds/decoder.py +140 -58
  153. oscura/automotive/uds/models.py +7 -1
  154. oscura/automotive/visualization.py +1 -1
  155. oscura/cli/analyze.py +348 -0
  156. oscura/cli/batch.py +142 -122
  157. oscura/cli/benchmark.py +275 -0
  158. oscura/cli/characterize.py +137 -82
  159. oscura/cli/compare.py +224 -131
  160. oscura/cli/completion.py +250 -0
  161. oscura/cli/config_cmd.py +361 -0
  162. oscura/cli/decode.py +164 -87
  163. oscura/cli/export.py +286 -0
  164. oscura/cli/main.py +115 -31
  165. oscura/{onboarding → cli/onboarding}/__init__.py +3 -3
  166. oscura/{onboarding → cli/onboarding}/help.py +80 -58
  167. oscura/{onboarding → cli/onboarding}/tutorials.py +97 -72
  168. oscura/{onboarding → cli/onboarding}/wizard.py +55 -36
  169. oscura/cli/progress.py +147 -0
  170. oscura/cli/shell.py +157 -135
  171. oscura/cli/validate_cmd.py +204 -0
  172. oscura/cli/visualize.py +158 -0
  173. oscura/convenience.py +125 -79
  174. oscura/core/__init__.py +4 -2
  175. oscura/core/backend_selector.py +3 -3
  176. oscura/core/cache.py +126 -15
  177. oscura/core/cancellation.py +1 -1
  178. oscura/{config → core/config}/__init__.py +20 -11
  179. oscura/{config → core/config}/defaults.py +1 -1
  180. oscura/{config → core/config}/loader.py +7 -5
  181. oscura/{config → core/config}/memory.py +5 -5
  182. oscura/{config → core/config}/migration.py +1 -1
  183. oscura/{config → core/config}/pipeline.py +99 -23
  184. oscura/{config → core/config}/preferences.py +1 -1
  185. oscura/{config → core/config}/protocol.py +3 -3
  186. oscura/{config → core/config}/schema.py +426 -272
  187. oscura/{config → core/config}/settings.py +1 -1
  188. oscura/{config → core/config}/thresholds.py +195 -153
  189. oscura/core/correlation.py +5 -6
  190. oscura/core/cross_domain.py +0 -2
  191. oscura/core/debug.py +9 -5
  192. oscura/{extensibility → core/extensibility}/docs.py +158 -70
  193. oscura/{extensibility → core/extensibility}/extensions.py +160 -76
  194. oscura/{extensibility → core/extensibility}/logging.py +1 -1
  195. oscura/{extensibility → core/extensibility}/measurements.py +1 -1
  196. oscura/{extensibility → core/extensibility}/plugins.py +1 -1
  197. oscura/{extensibility → core/extensibility}/templates.py +73 -3
  198. oscura/{extensibility → core/extensibility}/validation.py +1 -1
  199. oscura/core/gpu_backend.py +11 -7
  200. oscura/core/log_query.py +101 -11
  201. oscura/core/logging.py +126 -54
  202. oscura/core/logging_advanced.py +5 -5
  203. oscura/core/memory_limits.py +108 -70
  204. oscura/core/memory_monitor.py +2 -2
  205. oscura/core/memory_progress.py +7 -7
  206. oscura/core/memory_warnings.py +1 -1
  207. oscura/core/numba_backend.py +13 -13
  208. oscura/{plugins → core/plugins}/__init__.py +9 -9
  209. oscura/{plugins → core/plugins}/base.py +7 -7
  210. oscura/{plugins → core/plugins}/cli.py +3 -3
  211. oscura/{plugins → core/plugins}/discovery.py +186 -106
  212. oscura/{plugins → core/plugins}/lifecycle.py +1 -1
  213. oscura/{plugins → core/plugins}/manager.py +7 -7
  214. oscura/{plugins → core/plugins}/registry.py +3 -3
  215. oscura/{plugins → core/plugins}/versioning.py +1 -1
  216. oscura/core/progress.py +16 -1
  217. oscura/core/provenance.py +8 -2
  218. oscura/{schemas → core/schemas}/__init__.py +2 -2
  219. oscura/{schemas → core/schemas}/device_mapping.json +2 -8
  220. oscura/{schemas → core/schemas}/packet_format.json +4 -24
  221. oscura/{schemas → core/schemas}/protocol_definition.json +2 -12
  222. oscura/core/types.py +4 -0
  223. oscura/core/uncertainty.py +3 -3
  224. oscura/correlation/__init__.py +52 -0
  225. oscura/correlation/multi_protocol.py +811 -0
  226. oscura/discovery/auto_decoder.py +117 -35
  227. oscura/discovery/comparison.py +191 -86
  228. oscura/discovery/quality_validator.py +155 -68
  229. oscura/discovery/signal_detector.py +196 -79
  230. oscura/export/__init__.py +18 -8
  231. oscura/export/kaitai_struct.py +513 -0
  232. oscura/export/scapy_layer.py +801 -0
  233. oscura/export/wireshark/generator.py +1 -1
  234. oscura/export/wireshark/templates/dissector.lua.j2 +2 -2
  235. oscura/export/wireshark_dissector.py +746 -0
  236. oscura/guidance/wizard.py +207 -111
  237. oscura/hardware/__init__.py +19 -0
  238. oscura/{acquisition → hardware/acquisition}/__init__.py +4 -4
  239. oscura/{acquisition → hardware/acquisition}/file.py +2 -2
  240. oscura/{acquisition → hardware/acquisition}/hardware.py +7 -7
  241. oscura/{acquisition → hardware/acquisition}/saleae.py +15 -12
  242. oscura/{acquisition → hardware/acquisition}/socketcan.py +1 -1
  243. oscura/{acquisition → hardware/acquisition}/streaming.py +2 -2
  244. oscura/{acquisition → hardware/acquisition}/synthetic.py +3 -3
  245. oscura/{acquisition → hardware/acquisition}/visa.py +33 -11
  246. oscura/hardware/firmware/__init__.py +29 -0
  247. oscura/hardware/firmware/pattern_recognition.py +874 -0
  248. oscura/hardware/hal_detector.py +736 -0
  249. oscura/hardware/security/__init__.py +37 -0
  250. oscura/hardware/security/side_channel_detector.py +1126 -0
  251. oscura/inference/__init__.py +4 -0
  252. oscura/inference/active_learning/observation_table.py +4 -1
  253. oscura/inference/alignment.py +216 -123
  254. oscura/inference/bayesian.py +113 -33
  255. oscura/inference/crc_reverse.py +101 -55
  256. oscura/inference/logic.py +6 -2
  257. oscura/inference/message_format.py +342 -183
  258. oscura/inference/protocol.py +95 -44
  259. oscura/inference/protocol_dsl.py +180 -82
  260. oscura/inference/signal_intelligence.py +1439 -706
  261. oscura/inference/spectral.py +99 -57
  262. oscura/inference/state_machine.py +810 -158
  263. oscura/inference/stream.py +270 -110
  264. oscura/iot/__init__.py +34 -0
  265. oscura/iot/coap/__init__.py +32 -0
  266. oscura/iot/coap/analyzer.py +668 -0
  267. oscura/iot/coap/options.py +212 -0
  268. oscura/iot/lorawan/__init__.py +21 -0
  269. oscura/iot/lorawan/crypto.py +206 -0
  270. oscura/iot/lorawan/decoder.py +801 -0
  271. oscura/iot/lorawan/mac_commands.py +341 -0
  272. oscura/iot/mqtt/__init__.py +27 -0
  273. oscura/iot/mqtt/analyzer.py +999 -0
  274. oscura/iot/mqtt/properties.py +315 -0
  275. oscura/iot/zigbee/__init__.py +31 -0
  276. oscura/iot/zigbee/analyzer.py +615 -0
  277. oscura/iot/zigbee/security.py +153 -0
  278. oscura/iot/zigbee/zcl.py +349 -0
  279. oscura/jupyter/display.py +125 -45
  280. oscura/{exploratory → jupyter/exploratory}/__init__.py +8 -8
  281. oscura/{exploratory → jupyter/exploratory}/error_recovery.py +298 -141
  282. oscura/jupyter/exploratory/fuzzy.py +746 -0
  283. oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py +258 -100
  284. oscura/{exploratory → jupyter/exploratory}/legacy.py +464 -242
  285. oscura/{exploratory → jupyter/exploratory}/parse.py +167 -145
  286. oscura/{exploratory → jupyter/exploratory}/recovery.py +119 -87
  287. oscura/jupyter/exploratory/sync.py +612 -0
  288. oscura/{exploratory → jupyter/exploratory}/unknown.py +299 -176
  289. oscura/jupyter/magic.py +4 -4
  290. oscura/{ui → jupyter/ui}/__init__.py +2 -2
  291. oscura/{ui → jupyter/ui}/formatters.py +3 -3
  292. oscura/{ui → jupyter/ui}/progressive_display.py +153 -82
  293. oscura/loaders/__init__.py +183 -67
  294. oscura/loaders/binary.py +88 -1
  295. oscura/loaders/chipwhisperer.py +153 -137
  296. oscura/loaders/configurable.py +208 -86
  297. oscura/loaders/csv_loader.py +458 -215
  298. oscura/loaders/hdf5_loader.py +278 -119
  299. oscura/loaders/lazy.py +87 -54
  300. oscura/loaders/mmap_loader.py +1 -1
  301. oscura/loaders/numpy_loader.py +253 -116
  302. oscura/loaders/pcap.py +226 -151
  303. oscura/loaders/rigol.py +110 -49
  304. oscura/loaders/sigrok.py +201 -78
  305. oscura/loaders/tdms.py +81 -58
  306. oscura/loaders/tektronix.py +291 -174
  307. oscura/loaders/touchstone.py +182 -87
  308. oscura/loaders/tss.py +456 -0
  309. oscura/loaders/vcd.py +215 -117
  310. oscura/loaders/wav.py +155 -68
  311. oscura/reporting/__init__.py +9 -0
  312. oscura/reporting/analyze.py +352 -146
  313. oscura/reporting/argument_preparer.py +69 -14
  314. oscura/reporting/auto_report.py +97 -61
  315. oscura/reporting/batch.py +131 -58
  316. oscura/reporting/chart_selection.py +57 -45
  317. oscura/reporting/comparison.py +63 -17
  318. oscura/reporting/content/executive.py +76 -24
  319. oscura/reporting/core_formats/multi_format.py +11 -8
  320. oscura/reporting/engine.py +312 -158
  321. oscura/reporting/enhanced_reports.py +949 -0
  322. oscura/reporting/export.py +86 -43
  323. oscura/reporting/formatting/numbers.py +69 -42
  324. oscura/reporting/html.py +139 -58
  325. oscura/reporting/index.py +137 -65
  326. oscura/reporting/output.py +158 -67
  327. oscura/reporting/pdf.py +67 -102
  328. oscura/reporting/plots.py +191 -112
  329. oscura/reporting/sections.py +88 -47
  330. oscura/reporting/standards.py +104 -61
  331. oscura/reporting/summary_generator.py +75 -55
  332. oscura/reporting/tables.py +138 -54
  333. oscura/reporting/templates/enhanced/protocol_re.html +525 -0
  334. oscura/sessions/__init__.py +14 -23
  335. oscura/sessions/base.py +3 -3
  336. oscura/sessions/blackbox.py +106 -10
  337. oscura/sessions/generic.py +2 -2
  338. oscura/sessions/legacy.py +783 -0
  339. oscura/side_channel/__init__.py +63 -0
  340. oscura/side_channel/dpa.py +1025 -0
  341. oscura/utils/__init__.py +15 -1
  342. oscura/utils/bitwise.py +118 -0
  343. oscura/{builders → utils/builders}/__init__.py +1 -1
  344. oscura/{comparison → utils/comparison}/__init__.py +6 -6
  345. oscura/{comparison → utils/comparison}/compare.py +202 -101
  346. oscura/{comparison → utils/comparison}/golden.py +83 -63
  347. oscura/{comparison → utils/comparison}/limits.py +313 -89
  348. oscura/{comparison → utils/comparison}/mask.py +151 -45
  349. oscura/{comparison → utils/comparison}/trace_diff.py +1 -1
  350. oscura/{comparison → utils/comparison}/visualization.py +147 -89
  351. oscura/{component → utils/component}/__init__.py +3 -3
  352. oscura/{component → utils/component}/impedance.py +122 -58
  353. oscura/{component → utils/component}/reactive.py +165 -168
  354. oscura/{component → utils/component}/transmission_line.py +3 -3
  355. oscura/{filtering → utils/filtering}/__init__.py +6 -6
  356. oscura/{filtering → utils/filtering}/base.py +1 -1
  357. oscura/{filtering → utils/filtering}/convenience.py +2 -2
  358. oscura/{filtering → utils/filtering}/design.py +169 -93
  359. oscura/{filtering → utils/filtering}/filters.py +2 -2
  360. oscura/{filtering → utils/filtering}/introspection.py +2 -2
  361. oscura/utils/geometry.py +31 -0
  362. oscura/utils/imports.py +184 -0
  363. oscura/utils/lazy.py +1 -1
  364. oscura/{math → utils/math}/__init__.py +2 -2
  365. oscura/{math → utils/math}/arithmetic.py +114 -48
  366. oscura/{math → utils/math}/interpolation.py +139 -106
  367. oscura/utils/memory.py +129 -66
  368. oscura/utils/memory_advanced.py +92 -9
  369. oscura/utils/memory_extensions.py +10 -8
  370. oscura/{optimization → utils/optimization}/__init__.py +1 -1
  371. oscura/{optimization → utils/optimization}/search.py +2 -2
  372. oscura/utils/performance/__init__.py +58 -0
  373. oscura/utils/performance/caching.py +889 -0
  374. oscura/utils/performance/lsh_clustering.py +333 -0
  375. oscura/utils/performance/memory_optimizer.py +699 -0
  376. oscura/utils/performance/optimizations.py +675 -0
  377. oscura/utils/performance/parallel.py +654 -0
  378. oscura/utils/performance/profiling.py +661 -0
  379. oscura/{pipeline → utils/pipeline}/base.py +1 -1
  380. oscura/{pipeline → utils/pipeline}/composition.py +1 -1
  381. oscura/{pipeline → utils/pipeline}/parallel.py +3 -2
  382. oscura/{pipeline → utils/pipeline}/pipeline.py +1 -1
  383. oscura/{pipeline → utils/pipeline}/reverse_engineering.py +412 -221
  384. oscura/{search → utils/search}/__init__.py +3 -3
  385. oscura/{search → utils/search}/anomaly.py +188 -58
  386. oscura/utils/search/context.py +294 -0
  387. oscura/{search → utils/search}/pattern.py +138 -10
  388. oscura/utils/serial.py +51 -0
  389. oscura/utils/storage/__init__.py +61 -0
  390. oscura/utils/storage/database.py +1166 -0
  391. oscura/{streaming → utils/streaming}/chunked.py +302 -143
  392. oscura/{streaming → utils/streaming}/progressive.py +1 -1
  393. oscura/{streaming → utils/streaming}/realtime.py +3 -2
  394. oscura/{triggering → utils/triggering}/__init__.py +6 -6
  395. oscura/{triggering → utils/triggering}/base.py +6 -6
  396. oscura/{triggering → utils/triggering}/edge.py +2 -2
  397. oscura/{triggering → utils/triggering}/pattern.py +2 -2
  398. oscura/{triggering → utils/triggering}/pulse.py +115 -74
  399. oscura/{triggering → utils/triggering}/window.py +2 -2
  400. oscura/utils/validation.py +32 -0
  401. oscura/validation/__init__.py +121 -0
  402. oscura/{compliance → validation/compliance}/__init__.py +5 -5
  403. oscura/{compliance → validation/compliance}/advanced.py +5 -5
  404. oscura/{compliance → validation/compliance}/masks.py +1 -1
  405. oscura/{compliance → validation/compliance}/reporting.py +127 -53
  406. oscura/{compliance → validation/compliance}/testing.py +114 -52
  407. oscura/validation/compliance_tests.py +915 -0
  408. oscura/validation/fuzzer.py +990 -0
  409. oscura/validation/grammar_tests.py +596 -0
  410. oscura/validation/grammar_validator.py +904 -0
  411. oscura/validation/hil_testing.py +977 -0
  412. oscura/{quality → validation/quality}/__init__.py +4 -4
  413. oscura/{quality → validation/quality}/ensemble.py +251 -171
  414. oscura/{quality → validation/quality}/explainer.py +3 -3
  415. oscura/{quality → validation/quality}/scoring.py +1 -1
  416. oscura/{quality → validation/quality}/warnings.py +4 -4
  417. oscura/validation/regression_suite.py +808 -0
  418. oscura/validation/replay.py +788 -0
  419. oscura/{testing → validation/testing}/__init__.py +2 -2
  420. oscura/{testing → validation/testing}/synthetic.py +5 -5
  421. oscura/visualization/__init__.py +9 -0
  422. oscura/visualization/accessibility.py +1 -1
  423. oscura/visualization/annotations.py +64 -67
  424. oscura/visualization/colors.py +7 -7
  425. oscura/visualization/digital.py +180 -81
  426. oscura/visualization/eye.py +236 -85
  427. oscura/visualization/interactive.py +320 -143
  428. oscura/visualization/jitter.py +587 -247
  429. oscura/visualization/layout.py +169 -134
  430. oscura/visualization/optimization.py +103 -52
  431. oscura/visualization/palettes.py +1 -1
  432. oscura/visualization/power.py +427 -211
  433. oscura/visualization/power_extended.py +626 -297
  434. oscura/visualization/presets.py +2 -0
  435. oscura/visualization/protocols.py +495 -181
  436. oscura/visualization/render.py +79 -63
  437. oscura/visualization/reverse_engineering.py +171 -124
  438. oscura/visualization/signal_integrity.py +460 -279
  439. oscura/visualization/specialized.py +190 -100
  440. oscura/visualization/spectral.py +670 -255
  441. oscura/visualization/thumbnails.py +166 -137
  442. oscura/visualization/waveform.py +150 -63
  443. oscura/workflows/__init__.py +3 -0
  444. oscura/{batch → workflows/batch}/__init__.py +5 -5
  445. oscura/{batch → workflows/batch}/advanced.py +150 -75
  446. oscura/workflows/batch/aggregate.py +531 -0
  447. oscura/workflows/batch/analyze.py +236 -0
  448. oscura/{batch → workflows/batch}/logging.py +2 -2
  449. oscura/{batch → workflows/batch}/metrics.py +1 -1
  450. oscura/workflows/complete_re.py +1144 -0
  451. oscura/workflows/compliance.py +44 -54
  452. oscura/workflows/digital.py +197 -51
  453. oscura/workflows/legacy/__init__.py +12 -0
  454. oscura/{workflow → workflows/legacy}/dag.py +4 -1
  455. oscura/workflows/multi_trace.py +9 -9
  456. oscura/workflows/power.py +42 -62
  457. oscura/workflows/protocol.py +82 -49
  458. oscura/workflows/reverse_engineering.py +351 -150
  459. oscura/workflows/signal_integrity.py +157 -82
  460. oscura-0.7.0.dist-info/METADATA +661 -0
  461. oscura-0.7.0.dist-info/RECORD +591 -0
  462. oscura/batch/aggregate.py +0 -300
  463. oscura/batch/analyze.py +0 -139
  464. oscura/dsl/__init__.py +0 -73
  465. oscura/exceptions.py +0 -59
  466. oscura/exploratory/fuzzy.py +0 -513
  467. oscura/exploratory/sync.py +0 -384
  468. oscura/exporters/__init__.py +0 -94
  469. oscura/exporters/csv.py +0 -303
  470. oscura/exporters/exporters.py +0 -44
  471. oscura/exporters/hdf5.py +0 -217
  472. oscura/exporters/html_export.py +0 -701
  473. oscura/exporters/json_export.py +0 -291
  474. oscura/exporters/markdown_export.py +0 -367
  475. oscura/exporters/matlab_export.py +0 -354
  476. oscura/exporters/npz_export.py +0 -219
  477. oscura/exporters/spice_export.py +0 -210
  478. oscura/search/context.py +0 -149
  479. oscura/session/__init__.py +0 -34
  480. oscura/session/annotations.py +0 -289
  481. oscura/session/history.py +0 -313
  482. oscura/session/session.py +0 -520
  483. oscura/workflow/__init__.py +0 -13
  484. oscura-0.5.1.dist-info/METADATA +0 -583
  485. oscura-0.5.1.dist-info/RECORD +0 -481
  486. /oscura/core/{config.py → config/legacy.py} +0 -0
  487. /oscura/{extensibility → core/extensibility}/__init__.py +0 -0
  488. /oscura/{extensibility → core/extensibility}/registry.py +0 -0
  489. /oscura/{plugins → core/plugins}/isolation.py +0 -0
  490. /oscura/{schemas → core/schemas}/bus_configuration.json +0 -0
  491. /oscura/{builders → utils/builders}/signal_builder.py +0 -0
  492. /oscura/{optimization → utils/optimization}/parallel.py +0 -0
  493. /oscura/{pipeline → utils/pipeline}/__init__.py +0 -0
  494. /oscura/{streaming → utils/streaming}/__init__.py +0 -0
  495. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/WHEEL +0 -0
  496. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/entry_points.txt +0 -0
  497. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,811 @@
1
+ """Multi-protocol session correlation framework.
2
+
3
+ This module provides comprehensive tools for correlating sessions across
4
+ multiple protocols (e.g., CAN + Ethernet + Serial) to discover cross-protocol
5
+ communication patterns, dependencies, and session flows.
6
+
7
+ The framework supports:
8
+ - Message correlation across different protocols
9
+ - Request-response pair detection
10
+ - Dependency graph generation
11
+ - Session flow extraction
12
+ - Cross-protocol visualization
13
+ - Correlation analysis export
14
+
15
+ Example:
16
+ >>> from oscura.correlation import MultiProtocolCorrelator, ProtocolMessage
17
+ >>>
18
+ >>> # Create correlator
19
+ >>> correlator = MultiProtocolCorrelator(
20
+ ... time_window=0.1, # Max 100ms between correlated messages
21
+ ... min_confidence=0.5
22
+ ... )
23
+ >>>
24
+ >>> # Add CAN message
25
+ >>> can_msg = ProtocolMessage(
26
+ ... protocol="can",
27
+ ... timestamp=1.234,
28
+ ... message_id=0x123,
29
+ ... payload=b"\\x01\\x02\\x03\\x04",
30
+ ... source="ECU1",
31
+ ... destination="ECU2"
32
+ ... )
33
+ >>> correlator.add_message(can_msg)
34
+ >>>
35
+ >>> # Add Ethernet message shortly after (likely related)
36
+ >>> eth_msg = ProtocolMessage(
37
+ ... protocol="ethernet",
38
+ ... timestamp=1.238, # 4ms later
39
+ ... payload=b"\\x01\\x02\\x03\\x04\\x05", # Similar payload
40
+ ... source="192.168.1.10",
41
+ ... destination="192.168.1.20"
42
+ ... )
43
+ >>> correlator.add_message(eth_msg)
44
+ >>>
45
+ >>> # Find all correlations
46
+ >>> correlations = correlator.correlate_all()
47
+ >>> for corr in correlations:
48
+ ... print(f"{corr.correlation_type}: {corr.confidence:.2f}")
49
+ ... print(f" Evidence: {', '.join(corr.evidence)}")
50
+ >>>
51
+ >>> # Build dependency graph
52
+ >>> graph = correlator.build_dependency_graph()
53
+ >>> print(f"Nodes: {graph.number_of_nodes()}")
54
+ >>> print(f"Edges: {graph.number_of_edges()}")
55
+ >>>
56
+ >>> # Extract logical sessions
57
+ >>> sessions = correlator.extract_sessions()
58
+ >>> for session in sessions:
59
+ ... print(f"Session: {len(session.messages)} messages")
60
+ ... print(f"Protocols: {', '.join(session.protocols)}")
61
+ ... print(f"Duration: {session.end_time - session.start_time:.3f}s")
62
+
63
+ Correlation Detection Methods:
64
+ The framework uses multiple methods to identify related messages:
65
+
66
+ 1. Timestamp Proximity: Messages within time_window are candidates
67
+ 2. Payload Similarity: Partial or full payload matches using Jaccard
68
+ 3. ID Matching: Matching message IDs across protocols
69
+ 4. Source/Destination: Matching addresses or identifiers
70
+
71
+ Correlation types:
72
+ - broadcast: Near-simultaneous messages across protocols
73
+ - request_response: Request followed by response
74
+ - related_payload: Correlated by payload content
75
+
76
+ References:
77
+ Network protocol analysis
78
+ Session correlation algorithms
79
+ Graph theory for dependency analysis
80
+ """
81
+
82
+ from __future__ import annotations
83
+
84
+ from dataclasses import dataclass, field
85
+ from pathlib import Path
86
+ from typing import Any
87
+
88
+ # Optional dependencies - import at module level for validation
89
+ try:
90
+ import matplotlib
91
+ except ImportError:
92
+ matplotlib = None # type: ignore[assignment]
93
+
94
+ try:
95
+ import networkx
96
+ except ImportError:
97
+ networkx = None # type: ignore[assignment]
98
+
99
+
100
+ @dataclass
101
+ class ProtocolMessage:
102
+ """Generic protocol message for multi-protocol correlation.
103
+
104
+ Represents a message from any protocol with common fields for correlation.
105
+
106
+ Attributes:
107
+ protocol: Protocol name (e.g., "can", "ethernet", "uart", "spi").
108
+ timestamp: Message timestamp in seconds (float for high precision).
109
+ message_id: Message identifier (int for CAN, str for others, None if N/A).
110
+ payload: Message payload as bytes.
111
+ source: Source address/identifier (optional).
112
+ destination: Destination address/identifier (optional).
113
+ metadata: Additional protocol-specific metadata.
114
+
115
+ Example:
116
+ >>> msg = ProtocolMessage(
117
+ ... protocol="can",
118
+ ... timestamp=1.234567,
119
+ ... message_id=0x123,
120
+ ... payload=b"\\x01\\x02\\x03\\x04",
121
+ ... source="ECU1",
122
+ ... metadata={"dlc": 4, "extended": False}
123
+ ... )
124
+ """
125
+
126
+ protocol: str
127
+ timestamp: float
128
+ message_id: str | int | None = None
129
+ payload: bytes = b""
130
+ source: str | None = None
131
+ destination: str | None = None
132
+ metadata: dict[str, Any] = field(default_factory=dict)
133
+
134
+
135
+ @dataclass
136
+ class MessageCorrelation:
137
+ """Correlation between two messages from different protocols.
138
+
139
+ Attributes:
140
+ message1: First message (chronologically earlier).
141
+ message2: Second message (chronologically later).
142
+ correlation_type: Type of correlation detected.
143
+ confidence: Correlation confidence score (0.0-1.0).
144
+ time_delta: Time difference in seconds (message2 - message1).
145
+ evidence: List of evidence strings explaining the correlation.
146
+
147
+ Correlation types:
148
+ - "broadcast": Near-simultaneous messages (<10ms) across protocols
149
+ - "request_response": Request followed by response
150
+ - "related_payload": Correlated by payload similarity
151
+
152
+ Example:
153
+ >>> corr = MessageCorrelation(
154
+ ... message1=can_msg,
155
+ ... message2=eth_msg,
156
+ ... correlation_type="request_response",
157
+ ... confidence=0.85,
158
+ ... time_delta=0.004,
159
+ ... evidence=["Payload similarity: 0.80", "Source-destination match"]
160
+ ... )
161
+ """
162
+
163
+ message1: ProtocolMessage
164
+ message2: ProtocolMessage
165
+ correlation_type: str
166
+ confidence: float
167
+ time_delta: float
168
+ evidence: list[str] = field(default_factory=list)
169
+
170
+
171
+ @dataclass
172
+ class SessionFlow:
173
+ """Cross-protocol session flow.
174
+
175
+ Represents a logical session composed of correlated messages from
176
+ multiple protocols.
177
+
178
+ Attributes:
179
+ start_time: Session start time (first message timestamp).
180
+ end_time: Session end time (last message timestamp).
181
+ messages: All messages in the session (chronologically sorted).
182
+ correlations: All correlations within the session.
183
+ protocols: Set of protocols used in the session.
184
+
185
+ Example:
186
+ >>> session = SessionFlow(
187
+ ... start_time=1.234,
188
+ ... end_time=1.456,
189
+ ... messages=[can_msg1, eth_msg1, can_msg2],
190
+ ... correlations=[corr1, corr2],
191
+ ... protocols={"can", "ethernet"}
192
+ ... )
193
+ >>> duration = session.end_time - session.start_time
194
+ >>> print(f"Session duration: {duration:.3f}s")
195
+ """
196
+
197
+ start_time: float
198
+ end_time: float
199
+ messages: list[ProtocolMessage]
200
+ correlations: list[MessageCorrelation]
201
+ protocols: set[str]
202
+
203
+
204
+ class MultiProtocolCorrelator:
205
+ """Multi-protocol session correlator.
206
+
207
+ Analyzes messages from multiple protocols to discover correlations,
208
+ dependencies, and logical session flows.
209
+
210
+ Attributes:
211
+ time_window: Maximum time between correlated messages (seconds).
212
+ min_confidence: Minimum confidence threshold for correlations.
213
+ messages: List of all added messages.
214
+ correlations: List of discovered correlations.
215
+
216
+ Example:
217
+ >>> correlator = MultiProtocolCorrelator(
218
+ ... time_window=0.1, # 100ms max time difference
219
+ ... min_confidence=0.5 # 50% minimum confidence
220
+ ... )
221
+ >>>
222
+ >>> # Add messages from different protocols
223
+ >>> correlator.add_message(can_msg)
224
+ >>> correlator.add_message(ethernet_msg)
225
+ >>> correlator.add_message(uart_msg)
226
+ >>>
227
+ >>> # Find correlations
228
+ >>> correlations = correlator.correlate_all()
229
+ >>> print(f"Found {len(correlations)} correlations")
230
+ >>>
231
+ >>> # Extract sessions
232
+ >>> sessions = correlator.extract_sessions()
233
+ >>> for session in sessions:
234
+ ... print(f"Session: {len(session.messages)} messages, "
235
+ ... f"{len(session.protocols)} protocols")
236
+ """
237
+
238
+ def __init__(
239
+ self,
240
+ time_window: float = 0.1,
241
+ min_confidence: float = 0.5,
242
+ ) -> None:
243
+ """Initialize multi-protocol correlator.
244
+
245
+ Args:
246
+ time_window: Maximum time between correlated messages (seconds).
247
+ Default 0.1 (100ms). Increase for slower protocols.
248
+ min_confidence: Minimum confidence threshold for correlations (0.0-1.0).
249
+ Default 0.5. Higher values reduce false positives.
250
+
251
+ Raises:
252
+ ValueError: If time_window <= 0 or min_confidence not in [0, 1].
253
+ """
254
+ if time_window <= 0:
255
+ raise ValueError(f"time_window must be positive, got {time_window}")
256
+ if not 0.0 <= min_confidence <= 1.0:
257
+ raise ValueError(f"min_confidence must be in [0, 1], got {min_confidence}")
258
+
259
+ self.time_window = time_window
260
+ self.min_confidence = min_confidence
261
+ self.messages: list[ProtocolMessage] = []
262
+ self.correlations: list[MessageCorrelation] = []
263
+
264
+ def add_message(self, msg: ProtocolMessage) -> None:
265
+ """Add message from any protocol.
266
+
267
+ Messages are stored internally and used for correlation analysis.
268
+ Messages do not need to be added in chronological order.
269
+
270
+ Args:
271
+ msg: Protocol message to add.
272
+
273
+ Example:
274
+ >>> correlator.add_message(ProtocolMessage(
275
+ ... protocol="can",
276
+ ... timestamp=1.234,
277
+ ... message_id=0x123,
278
+ ... payload=b"\\x01\\x02"
279
+ ... ))
280
+ """
281
+ self.messages.append(msg)
282
+
283
+ def correlate_all(self) -> list[MessageCorrelation]:
284
+ """Find all correlations across protocols.
285
+
286
+ Uses multiple correlation methods:
287
+ 1. Timestamp proximity (within time_window)
288
+ 2. Payload similarity (partial or full match)
289
+ 3. ID matching (if available)
290
+ 4. Source/destination matching
291
+
292
+ Returns:
293
+ List of discovered correlations sorted by confidence (descending).
294
+ Only correlations with confidence >= min_confidence are returned.
295
+
296
+ Example:
297
+ >>> correlations = correlator.correlate_all()
298
+ >>> for corr in correlations:
299
+ ... print(f"{corr.message1.protocol} -> {corr.message2.protocol}: "
300
+ ... f"{corr.confidence:.2f}")
301
+ """
302
+ # Sort messages by timestamp for efficient windowing
303
+ sorted_messages = sorted(self.messages, key=lambda m: m.timestamp)
304
+
305
+ correlations = []
306
+
307
+ for i, msg1 in enumerate(sorted_messages):
308
+ # Look for correlations within time window
309
+ for j in range(i + 1, len(sorted_messages)):
310
+ msg2 = sorted_messages[j]
311
+
312
+ # Check time window
313
+ time_delta = msg2.timestamp - msg1.timestamp
314
+ if time_delta > self.time_window:
315
+ break # No point checking further (sorted by time)
316
+
317
+ # Calculate correlation confidence
318
+ confidence, evidence = self._calculate_correlation_confidence(msg1, msg2)
319
+
320
+ if confidence >= self.min_confidence:
321
+ # Determine correlation type
322
+ if time_delta < 0.01 and msg1.protocol != msg2.protocol:
323
+ corr_type = "broadcast"
324
+ elif self._is_request_response(msg1, msg2):
325
+ corr_type = "request_response"
326
+ else:
327
+ corr_type = "related_payload"
328
+
329
+ correlations.append(
330
+ MessageCorrelation(
331
+ message1=msg1,
332
+ message2=msg2,
333
+ correlation_type=corr_type,
334
+ confidence=confidence,
335
+ time_delta=time_delta,
336
+ evidence=evidence,
337
+ )
338
+ )
339
+
340
+ # Sort by confidence (descending)
341
+ correlations.sort(key=lambda c: c.confidence, reverse=True)
342
+
343
+ self.correlations = correlations
344
+ return correlations
345
+
346
+ def find_request_response_pairs(
347
+ self,
348
+ request_protocol: str,
349
+ response_protocol: str,
350
+ ) -> list[MessageCorrelation]:
351
+ """Find request-response pairs across specific protocols.
352
+
353
+ Filters correlations to only those matching the specified protocol pair
354
+ and classified as request-response type.
355
+
356
+ Args:
357
+ request_protocol: Protocol name for requests (e.g., "can").
358
+ response_protocol: Protocol name for responses (e.g., "ethernet").
359
+
360
+ Returns:
361
+ List of request-response correlations matching the protocol pair.
362
+
363
+ Example:
364
+ >>> # Find CAN requests that trigger Ethernet responses
365
+ >>> pairs = correlator.find_request_response_pairs("can", "ethernet")
366
+ >>> for pair in pairs:
367
+ ... print(f"CAN {pair.message1.message_id} -> "
368
+ ... f"Ethernet after {pair.time_delta*1000:.1f}ms")
369
+ """
370
+ if not self.correlations:
371
+ self.correlate_all()
372
+
373
+ return [
374
+ corr
375
+ for corr in self.correlations
376
+ if corr.correlation_type == "request_response"
377
+ and corr.message1.protocol == request_protocol
378
+ and corr.message2.protocol == response_protocol
379
+ ]
380
+
381
+ def build_dependency_graph(self) -> Any:
382
+ """Build NetworkX graph showing message dependencies.
383
+
384
+ Creates a directed graph where:
385
+ - Nodes represent messages (with protocol, timestamp, ID attributes)
386
+ - Edges represent correlations (with confidence, type, time_delta)
387
+
388
+ Returns:
389
+ NetworkX DiGraph with message dependencies.
390
+
391
+ Raises:
392
+ ImportError: If networkx is not installed.
393
+
394
+ Example:
395
+ >>> graph = correlator.build_dependency_graph()
396
+ >>> print(f"Nodes: {graph.number_of_nodes()}")
397
+ >>> print(f"Edges: {graph.number_of_edges()}")
398
+ >>>
399
+ >>> # Find strongly connected components
400
+ >>> import networkx as nx
401
+ >>> components = list(nx.strongly_connected_components(graph))
402
+ """
403
+ try:
404
+ import networkx as nx
405
+ except ImportError as e:
406
+ raise ImportError(
407
+ "networkx is required for graph analysis. "
408
+ "Install with: pip install 'oscura[analysis]'"
409
+ ) from e
410
+
411
+ if not self.correlations:
412
+ self.correlate_all()
413
+
414
+ graph = nx.DiGraph()
415
+
416
+ # Add nodes (messages)
417
+ for i, msg in enumerate(self.messages):
418
+ graph.add_node(
419
+ i,
420
+ protocol=msg.protocol,
421
+ timestamp=msg.timestamp,
422
+ id=msg.message_id,
423
+ label=f"{msg.protocol}:{msg.message_id}",
424
+ )
425
+
426
+ # Add edges (correlations)
427
+ msg_to_idx = {id(msg): i for i, msg in enumerate(self.messages)}
428
+
429
+ for corr in self.correlations:
430
+ idx1 = msg_to_idx[id(corr.message1)]
431
+ idx2 = msg_to_idx[id(corr.message2)]
432
+
433
+ graph.add_edge(
434
+ idx1,
435
+ idx2,
436
+ weight=corr.confidence,
437
+ type=corr.correlation_type,
438
+ time_delta=corr.time_delta,
439
+ )
440
+
441
+ return graph
442
+
443
+ def extract_sessions(self) -> list[SessionFlow]:
444
+ """Extract logical sessions from correlated messages.
445
+
446
+ A session is defined as a connected component in the dependency graph,
447
+ representing a group of messages that are directly or indirectly correlated.
448
+
449
+ Returns:
450
+ List of SessionFlow objects, each representing a logical session.
451
+ Sessions are sorted by start time.
452
+
453
+ Example:
454
+ >>> sessions = correlator.extract_sessions()
455
+ >>> for i, session in enumerate(sessions):
456
+ ... print(f"Session {i+1}:")
457
+ ... print(f" Messages: {len(session.messages)}")
458
+ ... print(f" Protocols: {', '.join(session.protocols)}")
459
+ ... print(f" Duration: {session.end_time - session.start_time:.3f}s")
460
+ """
461
+ if networkx is None:
462
+ raise ImportError("networkx is required for dependency graph building")
463
+ try:
464
+ import networkx as nx
465
+ except ImportError as e:
466
+ raise ImportError(
467
+ "networkx is required for session extraction. "
468
+ "Install with: pip install 'oscura[analysis]'"
469
+ ) from e
470
+
471
+ graph = self.build_dependency_graph()
472
+
473
+ # Find connected components (undirected version)
474
+ undirected = graph.to_undirected()
475
+ components = nx.connected_components(undirected)
476
+
477
+ sessions = []
478
+ for component in components:
479
+ # Get messages in this component
480
+ session_messages = [self.messages[i] for i in component]
481
+ session_messages.sort(key=lambda m: m.timestamp)
482
+
483
+ # Get correlations within this component
484
+ component_set = set(component)
485
+ msg_indices = {id(msg): i for i, msg in enumerate(self.messages)}
486
+
487
+ session_corrs = [
488
+ corr
489
+ for corr in self.correlations
490
+ if msg_indices[id(corr.message1)] in component_set
491
+ and msg_indices[id(corr.message2)] in component_set
492
+ ]
493
+
494
+ # Extract protocols used
495
+ protocols = {msg.protocol for msg in session_messages}
496
+
497
+ sessions.append(
498
+ SessionFlow(
499
+ start_time=session_messages[0].timestamp,
500
+ end_time=session_messages[-1].timestamp,
501
+ messages=session_messages,
502
+ correlations=session_corrs,
503
+ protocols=protocols,
504
+ )
505
+ )
506
+
507
+ # Sort sessions by start time
508
+ sessions.sort(key=lambda s: s.start_time)
509
+
510
+ return sessions
511
+
512
+ def _calculate_correlation_confidence(
513
+ self,
514
+ msg1: ProtocolMessage,
515
+ msg2: ProtocolMessage,
516
+ ) -> tuple[float, list[str]]:
517
+ """Calculate correlation confidence and evidence.
518
+
519
+ Uses multiple signals to estimate correlation likelihood:
520
+ - Payload similarity (Jaccard coefficient)
521
+ - Message ID matching
522
+ - Source/destination address matching
523
+
524
+ Args:
525
+ msg1: First message.
526
+ msg2: Second message.
527
+
528
+ Returns:
529
+ Tuple of (confidence, evidence_list).
530
+ Confidence is clamped to [0.0, 1.0].
531
+ """
532
+ confidence = 0.0
533
+ evidence = []
534
+
535
+ # Same protocol = no cross-protocol correlation
536
+ if msg1.protocol == msg2.protocol:
537
+ return 0.0, []
538
+
539
+ # Check payload similarity (weighted 40%)
540
+ payload_sim = self._payload_similarity(msg1.payload, msg2.payload)
541
+ if payload_sim > 0.5:
542
+ confidence += 0.4 * payload_sim
543
+ evidence.append(f"Payload similarity: {payload_sim:.2f}")
544
+
545
+ # Check ID matching (weighted 30%)
546
+ if msg1.message_id is not None and msg2.message_id is not None:
547
+ if msg1.message_id == msg2.message_id:
548
+ confidence += 0.3
549
+ evidence.append(f"Matching IDs: {msg1.message_id}")
550
+
551
+ # Check source/destination matching (weighted 15% each)
552
+ if msg1.source is not None and msg2.destination is not None:
553
+ if msg1.source == msg2.destination:
554
+ confidence += 0.15
555
+ evidence.append("Source-destination match")
556
+
557
+ if msg1.destination is not None and msg2.source is not None:
558
+ if msg1.destination == msg2.source:
559
+ confidence += 0.15
560
+ evidence.append("Destination-source match")
561
+
562
+ return min(confidence, 1.0), evidence
563
+
564
+ def _payload_similarity(self, payload1: bytes, payload2: bytes) -> float:
565
+ """Calculate payload similarity using Jaccard coefficient.
566
+
567
+ Computes similarity based on:
568
+ 1. Exact containment (one payload contains the other)
569
+ 2. Byte set intersection (Jaccard similarity)
570
+
571
+ Args:
572
+ payload1: First payload.
573
+ payload2: Second payload.
574
+
575
+ Returns:
576
+ Similarity score (0.0-1.0).
577
+ """
578
+ if not payload1 or not payload2:
579
+ return 0.0
580
+
581
+ # Check if one contains the other (exact match or prefix/suffix)
582
+ if payload1 in payload2 or payload2 in payload1:
583
+ return 1.0
584
+
585
+ # Calculate Jaccard similarity of bytes
586
+ set1 = set(payload1)
587
+ set2 = set(payload2)
588
+
589
+ intersection = len(set1 & set2)
590
+ union = len(set1 | set2)
591
+
592
+ return intersection / union if union > 0 else 0.0
593
+
594
+ def _is_request_response(
595
+ self,
596
+ msg1: ProtocolMessage,
597
+ msg2: ProtocolMessage,
598
+ ) -> bool:
599
+ """Determine if messages form a request-response pair.
600
+
601
+ Heuristics:
602
+ - Source/destination swap (request source = response destination)
603
+ - Time delta in typical RPC range (1-100ms)
604
+ - Payload contains request identifier
605
+
606
+ Args:
607
+ msg1: Potential request message.
608
+ msg2: Potential response message.
609
+
610
+ Returns:
611
+ True if messages likely form request-response pair.
612
+ """
613
+ # Check source/destination swap
614
+ if (
615
+ msg1.source is not None
616
+ and msg2.destination is not None
617
+ and msg1.source == msg2.destination
618
+ ):
619
+ # Also check time delta is reasonable for RPC
620
+ time_delta = msg2.timestamp - msg1.timestamp
621
+ if 0.001 <= time_delta <= 0.1: # 1-100ms typical RPC time
622
+ return True
623
+
624
+ return False
625
+
626
+ def visualize_flow(
627
+ self,
628
+ session: SessionFlow,
629
+ output_path: Path,
630
+ ) -> None:
631
+ """Visualize cross-protocol message flow.
632
+
633
+ Generates a timeline diagram showing message flows across protocols.
634
+ Uses matplotlib to create a multi-lane diagram with time on the X-axis
635
+ and different protocols on separate Y-lanes.
636
+
637
+ Args:
638
+ session: Session flow to visualize.
639
+ output_path: Path to save visualization (PNG/PDF/SVG).
640
+
641
+ Raises:
642
+ ImportError: If matplotlib is not installed.
643
+
644
+ Example:
645
+ >>> sessions = correlator.extract_sessions()
646
+ >>> correlator.visualize_flow(sessions[0], Path("session_flow.png"))
647
+ """
648
+ try:
649
+ import matplotlib.pyplot as plt
650
+ except ImportError as e:
651
+ raise ImportError(
652
+ "matplotlib is required for visualization. Install with: pip install matplotlib"
653
+ ) from e
654
+
655
+ fig, ax = plt.subplots(figsize=(12, 6))
656
+
657
+ # Assign Y positions to protocols
658
+ protocols_list = sorted(session.protocols)
659
+ protocol_to_y = {proto: i for i, proto in enumerate(protocols_list)}
660
+
661
+ # Plot messages
662
+ for msg in session.messages:
663
+ y = protocol_to_y[msg.protocol]
664
+ ax.scatter(msg.timestamp, y, s=100, alpha=0.6)
665
+ ax.text(
666
+ msg.timestamp,
667
+ y + 0.1,
668
+ str(msg.message_id) if msg.message_id else "",
669
+ fontsize=8,
670
+ ha="center",
671
+ )
672
+
673
+ # Plot correlations
674
+ msg_to_idx = {id(msg): i for i, msg in enumerate(session.messages)}
675
+ for corr in session.correlations:
676
+ idx1 = msg_to_idx.get(id(corr.message1))
677
+ idx2 = msg_to_idx.get(id(corr.message2))
678
+ if idx1 is not None and idx2 is not None:
679
+ msg1 = session.messages[idx1]
680
+ msg2 = session.messages[idx2]
681
+ y1 = protocol_to_y[msg1.protocol]
682
+ y2 = protocol_to_y[msg2.protocol]
683
+
684
+ # Draw arrow
685
+ ax.annotate(
686
+ "",
687
+ xy=(msg2.timestamp, y2),
688
+ xytext=(msg1.timestamp, y1),
689
+ arrowprops={
690
+ "arrowstyle": "->",
691
+ "alpha": 0.3,
692
+ "lw": 1,
693
+ },
694
+ )
695
+
696
+ # Configure axes
697
+ ax.set_yticks(range(len(protocols_list)))
698
+ ax.set_yticklabels(protocols_list)
699
+ ax.set_xlabel("Time (s)")
700
+ ax.set_ylabel("Protocol")
701
+ ax.set_title(f"Cross-Protocol Message Flow ({len(session.messages)} messages)")
702
+ ax.grid(True, alpha=0.3)
703
+
704
+ plt.tight_layout()
705
+ plt.savefig(output_path, dpi=150)
706
+ plt.close()
707
+
708
+ def export_analysis(
709
+ self,
710
+ output_path: Path,
711
+ format: str = "json",
712
+ ) -> None:
713
+ """Export correlation analysis to file.
714
+
715
+ Exports all messages, correlations, and sessions to specified format.
716
+
717
+ Args:
718
+ output_path: Path to save analysis.
719
+ format: Export format ("json" or "csv").
720
+
721
+ Raises:
722
+ ValueError: If format is not supported.
723
+
724
+ Example:
725
+ >>> correlator.export_analysis(Path("analysis.json"), format="json")
726
+ """
727
+ if matplotlib is None:
728
+ raise ImportError("matplotlib is required for flow visualization")
729
+ if format == "json":
730
+ self._export_json(output_path)
731
+ elif format == "csv":
732
+ self._export_csv(output_path)
733
+ else:
734
+ raise ValueError(f"Unsupported format: {format}")
735
+
736
+ def _export_json(self, output_path: Path) -> None:
737
+ """Export analysis to JSON format."""
738
+ import json
739
+
740
+ # Build export structure
741
+ data = {
742
+ "config": {
743
+ "time_window": self.time_window,
744
+ "min_confidence": self.min_confidence,
745
+ },
746
+ "messages": [
747
+ {
748
+ "protocol": msg.protocol,
749
+ "timestamp": msg.timestamp,
750
+ "message_id": (
751
+ msg.message_id if isinstance(msg.message_id, (int, str)) else None
752
+ ),
753
+ "payload_hex": msg.payload.hex(),
754
+ "source": msg.source,
755
+ "destination": msg.destination,
756
+ "metadata": msg.metadata,
757
+ }
758
+ for msg in self.messages
759
+ ],
760
+ "correlations": [
761
+ {
762
+ "message1_idx": self.messages.index(corr.message1),
763
+ "message2_idx": self.messages.index(corr.message2),
764
+ "correlation_type": corr.correlation_type,
765
+ "confidence": corr.confidence,
766
+ "time_delta": corr.time_delta,
767
+ "evidence": corr.evidence,
768
+ }
769
+ for corr in self.correlations
770
+ ],
771
+ }
772
+
773
+ # Write JSON
774
+ output_path.write_text(json.dumps(data, indent=2))
775
+
776
+ def _export_csv(self, output_path: Path) -> None:
777
+ """Export correlations to CSV format."""
778
+ import csv
779
+
780
+ with output_path.open("w", newline="") as f:
781
+ writer = csv.writer(f)
782
+ writer.writerow(
783
+ [
784
+ "Protocol1",
785
+ "Timestamp1",
786
+ "ID1",
787
+ "Protocol2",
788
+ "Timestamp2",
789
+ "ID2",
790
+ "Type",
791
+ "Confidence",
792
+ "TimeDelta",
793
+ "Evidence",
794
+ ]
795
+ )
796
+
797
+ for corr in self.correlations:
798
+ writer.writerow(
799
+ [
800
+ corr.message1.protocol,
801
+ corr.message1.timestamp,
802
+ corr.message1.message_id,
803
+ corr.message2.protocol,
804
+ corr.message2.timestamp,
805
+ corr.message2.message_id,
806
+ corr.correlation_type,
807
+ corr.confidence,
808
+ corr.time_delta,
809
+ "; ".join(corr.evidence),
810
+ ]
811
+ )