oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (497) hide show
  1. oscura/__init__.py +169 -167
  2. oscura/analyzers/__init__.py +3 -0
  3. oscura/analyzers/classification.py +659 -0
  4. oscura/analyzers/digital/edges.py +325 -65
  5. oscura/analyzers/digital/quality.py +293 -166
  6. oscura/analyzers/digital/timing.py +260 -115
  7. oscura/analyzers/digital/timing_numba.py +334 -0
  8. oscura/analyzers/entropy.py +605 -0
  9. oscura/analyzers/eye/diagram.py +176 -109
  10. oscura/analyzers/eye/metrics.py +5 -5
  11. oscura/analyzers/jitter/__init__.py +6 -4
  12. oscura/analyzers/jitter/ber.py +52 -52
  13. oscura/analyzers/jitter/classification.py +156 -0
  14. oscura/analyzers/jitter/decomposition.py +163 -113
  15. oscura/analyzers/jitter/spectrum.py +80 -64
  16. oscura/analyzers/ml/__init__.py +39 -0
  17. oscura/analyzers/ml/features.py +600 -0
  18. oscura/analyzers/ml/signal_classifier.py +604 -0
  19. oscura/analyzers/packet/daq.py +246 -158
  20. oscura/analyzers/packet/parser.py +12 -1
  21. oscura/analyzers/packet/payload.py +50 -2110
  22. oscura/analyzers/packet/payload_analysis.py +361 -181
  23. oscura/analyzers/packet/payload_patterns.py +133 -70
  24. oscura/analyzers/packet/stream.py +84 -23
  25. oscura/analyzers/patterns/__init__.py +26 -5
  26. oscura/analyzers/patterns/anomaly_detection.py +908 -0
  27. oscura/analyzers/patterns/clustering.py +169 -108
  28. oscura/analyzers/patterns/clustering_optimized.py +227 -0
  29. oscura/analyzers/patterns/discovery.py +1 -1
  30. oscura/analyzers/patterns/matching.py +581 -197
  31. oscura/analyzers/patterns/pattern_mining.py +778 -0
  32. oscura/analyzers/patterns/periodic.py +121 -38
  33. oscura/analyzers/patterns/sequences.py +175 -78
  34. oscura/analyzers/power/conduction.py +1 -1
  35. oscura/analyzers/power/soa.py +6 -6
  36. oscura/analyzers/power/switching.py +250 -110
  37. oscura/analyzers/protocol/__init__.py +17 -1
  38. oscura/analyzers/protocols/base.py +6 -6
  39. oscura/analyzers/protocols/ble/__init__.py +38 -0
  40. oscura/analyzers/protocols/ble/analyzer.py +809 -0
  41. oscura/analyzers/protocols/ble/uuids.py +288 -0
  42. oscura/analyzers/protocols/can.py +257 -127
  43. oscura/analyzers/protocols/can_fd.py +107 -80
  44. oscura/analyzers/protocols/flexray.py +139 -80
  45. oscura/analyzers/protocols/hdlc.py +93 -58
  46. oscura/analyzers/protocols/i2c.py +247 -106
  47. oscura/analyzers/protocols/i2s.py +138 -86
  48. oscura/analyzers/protocols/industrial/__init__.py +40 -0
  49. oscura/analyzers/protocols/industrial/bacnet/__init__.py +33 -0
  50. oscura/analyzers/protocols/industrial/bacnet/analyzer.py +708 -0
  51. oscura/analyzers/protocols/industrial/bacnet/encoding.py +412 -0
  52. oscura/analyzers/protocols/industrial/bacnet/services.py +622 -0
  53. oscura/analyzers/protocols/industrial/ethercat/__init__.py +30 -0
  54. oscura/analyzers/protocols/industrial/ethercat/analyzer.py +474 -0
  55. oscura/analyzers/protocols/industrial/ethercat/mailbox.py +339 -0
  56. oscura/analyzers/protocols/industrial/ethercat/topology.py +166 -0
  57. oscura/analyzers/protocols/industrial/modbus/__init__.py +31 -0
  58. oscura/analyzers/protocols/industrial/modbus/analyzer.py +525 -0
  59. oscura/analyzers/protocols/industrial/modbus/crc.py +79 -0
  60. oscura/analyzers/protocols/industrial/modbus/functions.py +436 -0
  61. oscura/analyzers/protocols/industrial/opcua/__init__.py +21 -0
  62. oscura/analyzers/protocols/industrial/opcua/analyzer.py +552 -0
  63. oscura/analyzers/protocols/industrial/opcua/datatypes.py +446 -0
  64. oscura/analyzers/protocols/industrial/opcua/services.py +264 -0
  65. oscura/analyzers/protocols/industrial/profinet/__init__.py +23 -0
  66. oscura/analyzers/protocols/industrial/profinet/analyzer.py +441 -0
  67. oscura/analyzers/protocols/industrial/profinet/dcp.py +263 -0
  68. oscura/analyzers/protocols/industrial/profinet/ptcp.py +200 -0
  69. oscura/analyzers/protocols/jtag.py +180 -98
  70. oscura/analyzers/protocols/lin.py +219 -114
  71. oscura/analyzers/protocols/manchester.py +4 -4
  72. oscura/analyzers/protocols/onewire.py +253 -149
  73. oscura/analyzers/protocols/parallel_bus/__init__.py +20 -0
  74. oscura/analyzers/protocols/parallel_bus/centronics.py +92 -0
  75. oscura/analyzers/protocols/parallel_bus/gpib.py +137 -0
  76. oscura/analyzers/protocols/spi.py +192 -95
  77. oscura/analyzers/protocols/swd.py +321 -167
  78. oscura/analyzers/protocols/uart.py +267 -125
  79. oscura/analyzers/protocols/usb.py +235 -131
  80. oscura/analyzers/side_channel/power.py +17 -12
  81. oscura/analyzers/signal/__init__.py +15 -0
  82. oscura/analyzers/signal/timing_analysis.py +1086 -0
  83. oscura/analyzers/signal_integrity/__init__.py +4 -1
  84. oscura/analyzers/signal_integrity/sparams.py +2 -19
  85. oscura/analyzers/spectral/chunked.py +129 -60
  86. oscura/analyzers/spectral/chunked_fft.py +300 -94
  87. oscura/analyzers/spectral/chunked_wavelet.py +100 -80
  88. oscura/analyzers/statistical/checksum.py +376 -217
  89. oscura/analyzers/statistical/classification.py +229 -107
  90. oscura/analyzers/statistical/entropy.py +78 -53
  91. oscura/analyzers/statistics/correlation.py +407 -211
  92. oscura/analyzers/statistics/outliers.py +2 -2
  93. oscura/analyzers/statistics/streaming.py +30 -5
  94. oscura/analyzers/validation.py +216 -101
  95. oscura/analyzers/waveform/measurements.py +9 -0
  96. oscura/analyzers/waveform/measurements_with_uncertainty.py +31 -15
  97. oscura/analyzers/waveform/spectral.py +500 -228
  98. oscura/api/__init__.py +31 -5
  99. oscura/api/dsl/__init__.py +582 -0
  100. oscura/{dsl → api/dsl}/commands.py +43 -76
  101. oscura/{dsl → api/dsl}/interpreter.py +26 -51
  102. oscura/{dsl → api/dsl}/parser.py +107 -77
  103. oscura/{dsl → api/dsl}/repl.py +2 -2
  104. oscura/api/dsl.py +1 -1
  105. oscura/{integrations → api/integrations}/__init__.py +1 -1
  106. oscura/{integrations → api/integrations}/llm.py +201 -102
  107. oscura/api/operators.py +3 -3
  108. oscura/api/optimization.py +144 -30
  109. oscura/api/rest_server.py +921 -0
  110. oscura/api/server/__init__.py +17 -0
  111. oscura/api/server/dashboard.py +850 -0
  112. oscura/api/server/static/README.md +34 -0
  113. oscura/api/server/templates/base.html +181 -0
  114. oscura/api/server/templates/export.html +120 -0
  115. oscura/api/server/templates/home.html +284 -0
  116. oscura/api/server/templates/protocols.html +58 -0
  117. oscura/api/server/templates/reports.html +43 -0
  118. oscura/api/server/templates/session_detail.html +89 -0
  119. oscura/api/server/templates/sessions.html +83 -0
  120. oscura/api/server/templates/waveforms.html +73 -0
  121. oscura/automotive/__init__.py +8 -1
  122. oscura/automotive/can/__init__.py +10 -0
  123. oscura/automotive/can/checksum.py +3 -1
  124. oscura/automotive/can/dbc_generator.py +590 -0
  125. oscura/automotive/can/message_wrapper.py +121 -74
  126. oscura/automotive/can/patterns.py +98 -21
  127. oscura/automotive/can/session.py +292 -56
  128. oscura/automotive/can/state_machine.py +6 -3
  129. oscura/automotive/can/stimulus_response.py +97 -75
  130. oscura/automotive/dbc/__init__.py +10 -2
  131. oscura/automotive/dbc/generator.py +84 -56
  132. oscura/automotive/dbc/parser.py +6 -6
  133. oscura/automotive/dtc/data.json +17 -102
  134. oscura/automotive/dtc/database.py +2 -2
  135. oscura/automotive/flexray/__init__.py +31 -0
  136. oscura/automotive/flexray/analyzer.py +504 -0
  137. oscura/automotive/flexray/crc.py +185 -0
  138. oscura/automotive/flexray/fibex.py +449 -0
  139. oscura/automotive/j1939/__init__.py +45 -8
  140. oscura/automotive/j1939/analyzer.py +605 -0
  141. oscura/automotive/j1939/spns.py +326 -0
  142. oscura/automotive/j1939/transport.py +306 -0
  143. oscura/automotive/lin/__init__.py +47 -0
  144. oscura/automotive/lin/analyzer.py +612 -0
  145. oscura/automotive/loaders/blf.py +13 -2
  146. oscura/automotive/loaders/csv_can.py +143 -72
  147. oscura/automotive/loaders/dispatcher.py +50 -2
  148. oscura/automotive/loaders/mdf.py +86 -45
  149. oscura/automotive/loaders/pcap.py +111 -61
  150. oscura/automotive/uds/__init__.py +4 -0
  151. oscura/automotive/uds/analyzer.py +725 -0
  152. oscura/automotive/uds/decoder.py +140 -58
  153. oscura/automotive/uds/models.py +7 -1
  154. oscura/automotive/visualization.py +1 -1
  155. oscura/cli/analyze.py +348 -0
  156. oscura/cli/batch.py +142 -122
  157. oscura/cli/benchmark.py +275 -0
  158. oscura/cli/characterize.py +137 -82
  159. oscura/cli/compare.py +224 -131
  160. oscura/cli/completion.py +250 -0
  161. oscura/cli/config_cmd.py +361 -0
  162. oscura/cli/decode.py +164 -87
  163. oscura/cli/export.py +286 -0
  164. oscura/cli/main.py +115 -31
  165. oscura/{onboarding → cli/onboarding}/__init__.py +3 -3
  166. oscura/{onboarding → cli/onboarding}/help.py +80 -58
  167. oscura/{onboarding → cli/onboarding}/tutorials.py +97 -72
  168. oscura/{onboarding → cli/onboarding}/wizard.py +55 -36
  169. oscura/cli/progress.py +147 -0
  170. oscura/cli/shell.py +157 -135
  171. oscura/cli/validate_cmd.py +204 -0
  172. oscura/cli/visualize.py +158 -0
  173. oscura/convenience.py +125 -79
  174. oscura/core/__init__.py +4 -2
  175. oscura/core/backend_selector.py +3 -3
  176. oscura/core/cache.py +126 -15
  177. oscura/core/cancellation.py +1 -1
  178. oscura/{config → core/config}/__init__.py +20 -11
  179. oscura/{config → core/config}/defaults.py +1 -1
  180. oscura/{config → core/config}/loader.py +7 -5
  181. oscura/{config → core/config}/memory.py +5 -5
  182. oscura/{config → core/config}/migration.py +1 -1
  183. oscura/{config → core/config}/pipeline.py +99 -23
  184. oscura/{config → core/config}/preferences.py +1 -1
  185. oscura/{config → core/config}/protocol.py +3 -3
  186. oscura/{config → core/config}/schema.py +426 -272
  187. oscura/{config → core/config}/settings.py +1 -1
  188. oscura/{config → core/config}/thresholds.py +195 -153
  189. oscura/core/correlation.py +5 -6
  190. oscura/core/cross_domain.py +0 -2
  191. oscura/core/debug.py +9 -5
  192. oscura/{extensibility → core/extensibility}/docs.py +158 -70
  193. oscura/{extensibility → core/extensibility}/extensions.py +160 -76
  194. oscura/{extensibility → core/extensibility}/logging.py +1 -1
  195. oscura/{extensibility → core/extensibility}/measurements.py +1 -1
  196. oscura/{extensibility → core/extensibility}/plugins.py +1 -1
  197. oscura/{extensibility → core/extensibility}/templates.py +73 -3
  198. oscura/{extensibility → core/extensibility}/validation.py +1 -1
  199. oscura/core/gpu_backend.py +11 -7
  200. oscura/core/log_query.py +101 -11
  201. oscura/core/logging.py +126 -54
  202. oscura/core/logging_advanced.py +5 -5
  203. oscura/core/memory_limits.py +108 -70
  204. oscura/core/memory_monitor.py +2 -2
  205. oscura/core/memory_progress.py +7 -7
  206. oscura/core/memory_warnings.py +1 -1
  207. oscura/core/numba_backend.py +13 -13
  208. oscura/{plugins → core/plugins}/__init__.py +9 -9
  209. oscura/{plugins → core/plugins}/base.py +7 -7
  210. oscura/{plugins → core/plugins}/cli.py +3 -3
  211. oscura/{plugins → core/plugins}/discovery.py +186 -106
  212. oscura/{plugins → core/plugins}/lifecycle.py +1 -1
  213. oscura/{plugins → core/plugins}/manager.py +7 -7
  214. oscura/{plugins → core/plugins}/registry.py +3 -3
  215. oscura/{plugins → core/plugins}/versioning.py +1 -1
  216. oscura/core/progress.py +16 -1
  217. oscura/core/provenance.py +8 -2
  218. oscura/{schemas → core/schemas}/__init__.py +2 -2
  219. oscura/{schemas → core/schemas}/device_mapping.json +2 -8
  220. oscura/{schemas → core/schemas}/packet_format.json +4 -24
  221. oscura/{schemas → core/schemas}/protocol_definition.json +2 -12
  222. oscura/core/types.py +4 -0
  223. oscura/core/uncertainty.py +3 -3
  224. oscura/correlation/__init__.py +52 -0
  225. oscura/correlation/multi_protocol.py +811 -0
  226. oscura/discovery/auto_decoder.py +117 -35
  227. oscura/discovery/comparison.py +191 -86
  228. oscura/discovery/quality_validator.py +155 -68
  229. oscura/discovery/signal_detector.py +196 -79
  230. oscura/export/__init__.py +18 -8
  231. oscura/export/kaitai_struct.py +513 -0
  232. oscura/export/scapy_layer.py +801 -0
  233. oscura/export/wireshark/generator.py +1 -1
  234. oscura/export/wireshark/templates/dissector.lua.j2 +2 -2
  235. oscura/export/wireshark_dissector.py +746 -0
  236. oscura/guidance/wizard.py +207 -111
  237. oscura/hardware/__init__.py +19 -0
  238. oscura/{acquisition → hardware/acquisition}/__init__.py +4 -4
  239. oscura/{acquisition → hardware/acquisition}/file.py +2 -2
  240. oscura/{acquisition → hardware/acquisition}/hardware.py +7 -7
  241. oscura/{acquisition → hardware/acquisition}/saleae.py +15 -12
  242. oscura/{acquisition → hardware/acquisition}/socketcan.py +1 -1
  243. oscura/{acquisition → hardware/acquisition}/streaming.py +2 -2
  244. oscura/{acquisition → hardware/acquisition}/synthetic.py +3 -3
  245. oscura/{acquisition → hardware/acquisition}/visa.py +33 -11
  246. oscura/hardware/firmware/__init__.py +29 -0
  247. oscura/hardware/firmware/pattern_recognition.py +874 -0
  248. oscura/hardware/hal_detector.py +736 -0
  249. oscura/hardware/security/__init__.py +37 -0
  250. oscura/hardware/security/side_channel_detector.py +1126 -0
  251. oscura/inference/__init__.py +4 -0
  252. oscura/inference/active_learning/observation_table.py +4 -1
  253. oscura/inference/alignment.py +216 -123
  254. oscura/inference/bayesian.py +113 -33
  255. oscura/inference/crc_reverse.py +101 -55
  256. oscura/inference/logic.py +6 -2
  257. oscura/inference/message_format.py +342 -183
  258. oscura/inference/protocol.py +95 -44
  259. oscura/inference/protocol_dsl.py +180 -82
  260. oscura/inference/signal_intelligence.py +1439 -706
  261. oscura/inference/spectral.py +99 -57
  262. oscura/inference/state_machine.py +810 -158
  263. oscura/inference/stream.py +270 -110
  264. oscura/iot/__init__.py +34 -0
  265. oscura/iot/coap/__init__.py +32 -0
  266. oscura/iot/coap/analyzer.py +668 -0
  267. oscura/iot/coap/options.py +212 -0
  268. oscura/iot/lorawan/__init__.py +21 -0
  269. oscura/iot/lorawan/crypto.py +206 -0
  270. oscura/iot/lorawan/decoder.py +801 -0
  271. oscura/iot/lorawan/mac_commands.py +341 -0
  272. oscura/iot/mqtt/__init__.py +27 -0
  273. oscura/iot/mqtt/analyzer.py +999 -0
  274. oscura/iot/mqtt/properties.py +315 -0
  275. oscura/iot/zigbee/__init__.py +31 -0
  276. oscura/iot/zigbee/analyzer.py +615 -0
  277. oscura/iot/zigbee/security.py +153 -0
  278. oscura/iot/zigbee/zcl.py +349 -0
  279. oscura/jupyter/display.py +125 -45
  280. oscura/{exploratory → jupyter/exploratory}/__init__.py +8 -8
  281. oscura/{exploratory → jupyter/exploratory}/error_recovery.py +298 -141
  282. oscura/jupyter/exploratory/fuzzy.py +746 -0
  283. oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py +258 -100
  284. oscura/{exploratory → jupyter/exploratory}/legacy.py +464 -242
  285. oscura/{exploratory → jupyter/exploratory}/parse.py +167 -145
  286. oscura/{exploratory → jupyter/exploratory}/recovery.py +119 -87
  287. oscura/jupyter/exploratory/sync.py +612 -0
  288. oscura/{exploratory → jupyter/exploratory}/unknown.py +299 -176
  289. oscura/jupyter/magic.py +4 -4
  290. oscura/{ui → jupyter/ui}/__init__.py +2 -2
  291. oscura/{ui → jupyter/ui}/formatters.py +3 -3
  292. oscura/{ui → jupyter/ui}/progressive_display.py +153 -82
  293. oscura/loaders/__init__.py +183 -67
  294. oscura/loaders/binary.py +88 -1
  295. oscura/loaders/chipwhisperer.py +153 -137
  296. oscura/loaders/configurable.py +208 -86
  297. oscura/loaders/csv_loader.py +458 -215
  298. oscura/loaders/hdf5_loader.py +278 -119
  299. oscura/loaders/lazy.py +87 -54
  300. oscura/loaders/mmap_loader.py +1 -1
  301. oscura/loaders/numpy_loader.py +253 -116
  302. oscura/loaders/pcap.py +226 -151
  303. oscura/loaders/rigol.py +110 -49
  304. oscura/loaders/sigrok.py +201 -78
  305. oscura/loaders/tdms.py +81 -58
  306. oscura/loaders/tektronix.py +291 -174
  307. oscura/loaders/touchstone.py +182 -87
  308. oscura/loaders/tss.py +456 -0
  309. oscura/loaders/vcd.py +215 -117
  310. oscura/loaders/wav.py +155 -68
  311. oscura/reporting/__init__.py +9 -0
  312. oscura/reporting/analyze.py +352 -146
  313. oscura/reporting/argument_preparer.py +69 -14
  314. oscura/reporting/auto_report.py +97 -61
  315. oscura/reporting/batch.py +131 -58
  316. oscura/reporting/chart_selection.py +57 -45
  317. oscura/reporting/comparison.py +63 -17
  318. oscura/reporting/content/executive.py +76 -24
  319. oscura/reporting/core_formats/multi_format.py +11 -8
  320. oscura/reporting/engine.py +312 -158
  321. oscura/reporting/enhanced_reports.py +949 -0
  322. oscura/reporting/export.py +86 -43
  323. oscura/reporting/formatting/numbers.py +69 -42
  324. oscura/reporting/html.py +139 -58
  325. oscura/reporting/index.py +137 -65
  326. oscura/reporting/output.py +158 -67
  327. oscura/reporting/pdf.py +67 -102
  328. oscura/reporting/plots.py +191 -112
  329. oscura/reporting/sections.py +88 -47
  330. oscura/reporting/standards.py +104 -61
  331. oscura/reporting/summary_generator.py +75 -55
  332. oscura/reporting/tables.py +138 -54
  333. oscura/reporting/templates/enhanced/protocol_re.html +525 -0
  334. oscura/sessions/__init__.py +14 -23
  335. oscura/sessions/base.py +3 -3
  336. oscura/sessions/blackbox.py +106 -10
  337. oscura/sessions/generic.py +2 -2
  338. oscura/sessions/legacy.py +783 -0
  339. oscura/side_channel/__init__.py +63 -0
  340. oscura/side_channel/dpa.py +1025 -0
  341. oscura/utils/__init__.py +15 -1
  342. oscura/utils/bitwise.py +118 -0
  343. oscura/{builders → utils/builders}/__init__.py +1 -1
  344. oscura/{comparison → utils/comparison}/__init__.py +6 -6
  345. oscura/{comparison → utils/comparison}/compare.py +202 -101
  346. oscura/{comparison → utils/comparison}/golden.py +83 -63
  347. oscura/{comparison → utils/comparison}/limits.py +313 -89
  348. oscura/{comparison → utils/comparison}/mask.py +151 -45
  349. oscura/{comparison → utils/comparison}/trace_diff.py +1 -1
  350. oscura/{comparison → utils/comparison}/visualization.py +147 -89
  351. oscura/{component → utils/component}/__init__.py +3 -3
  352. oscura/{component → utils/component}/impedance.py +122 -58
  353. oscura/{component → utils/component}/reactive.py +165 -168
  354. oscura/{component → utils/component}/transmission_line.py +3 -3
  355. oscura/{filtering → utils/filtering}/__init__.py +6 -6
  356. oscura/{filtering → utils/filtering}/base.py +1 -1
  357. oscura/{filtering → utils/filtering}/convenience.py +2 -2
  358. oscura/{filtering → utils/filtering}/design.py +169 -93
  359. oscura/{filtering → utils/filtering}/filters.py +2 -2
  360. oscura/{filtering → utils/filtering}/introspection.py +2 -2
  361. oscura/utils/geometry.py +31 -0
  362. oscura/utils/imports.py +184 -0
  363. oscura/utils/lazy.py +1 -1
  364. oscura/{math → utils/math}/__init__.py +2 -2
  365. oscura/{math → utils/math}/arithmetic.py +114 -48
  366. oscura/{math → utils/math}/interpolation.py +139 -106
  367. oscura/utils/memory.py +129 -66
  368. oscura/utils/memory_advanced.py +92 -9
  369. oscura/utils/memory_extensions.py +10 -8
  370. oscura/{optimization → utils/optimization}/__init__.py +1 -1
  371. oscura/{optimization → utils/optimization}/search.py +2 -2
  372. oscura/utils/performance/__init__.py +58 -0
  373. oscura/utils/performance/caching.py +889 -0
  374. oscura/utils/performance/lsh_clustering.py +333 -0
  375. oscura/utils/performance/memory_optimizer.py +699 -0
  376. oscura/utils/performance/optimizations.py +675 -0
  377. oscura/utils/performance/parallel.py +654 -0
  378. oscura/utils/performance/profiling.py +661 -0
  379. oscura/{pipeline → utils/pipeline}/base.py +1 -1
  380. oscura/{pipeline → utils/pipeline}/composition.py +1 -1
  381. oscura/{pipeline → utils/pipeline}/parallel.py +3 -2
  382. oscura/{pipeline → utils/pipeline}/pipeline.py +1 -1
  383. oscura/{pipeline → utils/pipeline}/reverse_engineering.py +412 -221
  384. oscura/{search → utils/search}/__init__.py +3 -3
  385. oscura/{search → utils/search}/anomaly.py +188 -58
  386. oscura/utils/search/context.py +294 -0
  387. oscura/{search → utils/search}/pattern.py +138 -10
  388. oscura/utils/serial.py +51 -0
  389. oscura/utils/storage/__init__.py +61 -0
  390. oscura/utils/storage/database.py +1166 -0
  391. oscura/{streaming → utils/streaming}/chunked.py +302 -143
  392. oscura/{streaming → utils/streaming}/progressive.py +1 -1
  393. oscura/{streaming → utils/streaming}/realtime.py +3 -2
  394. oscura/{triggering → utils/triggering}/__init__.py +6 -6
  395. oscura/{triggering → utils/triggering}/base.py +6 -6
  396. oscura/{triggering → utils/triggering}/edge.py +2 -2
  397. oscura/{triggering → utils/triggering}/pattern.py +2 -2
  398. oscura/{triggering → utils/triggering}/pulse.py +115 -74
  399. oscura/{triggering → utils/triggering}/window.py +2 -2
  400. oscura/utils/validation.py +32 -0
  401. oscura/validation/__init__.py +121 -0
  402. oscura/{compliance → validation/compliance}/__init__.py +5 -5
  403. oscura/{compliance → validation/compliance}/advanced.py +5 -5
  404. oscura/{compliance → validation/compliance}/masks.py +1 -1
  405. oscura/{compliance → validation/compliance}/reporting.py +127 -53
  406. oscura/{compliance → validation/compliance}/testing.py +114 -52
  407. oscura/validation/compliance_tests.py +915 -0
  408. oscura/validation/fuzzer.py +990 -0
  409. oscura/validation/grammar_tests.py +596 -0
  410. oscura/validation/grammar_validator.py +904 -0
  411. oscura/validation/hil_testing.py +977 -0
  412. oscura/{quality → validation/quality}/__init__.py +4 -4
  413. oscura/{quality → validation/quality}/ensemble.py +251 -171
  414. oscura/{quality → validation/quality}/explainer.py +3 -3
  415. oscura/{quality → validation/quality}/scoring.py +1 -1
  416. oscura/{quality → validation/quality}/warnings.py +4 -4
  417. oscura/validation/regression_suite.py +808 -0
  418. oscura/validation/replay.py +788 -0
  419. oscura/{testing → validation/testing}/__init__.py +2 -2
  420. oscura/{testing → validation/testing}/synthetic.py +5 -5
  421. oscura/visualization/__init__.py +9 -0
  422. oscura/visualization/accessibility.py +1 -1
  423. oscura/visualization/annotations.py +64 -67
  424. oscura/visualization/colors.py +7 -7
  425. oscura/visualization/digital.py +180 -81
  426. oscura/visualization/eye.py +236 -85
  427. oscura/visualization/interactive.py +320 -143
  428. oscura/visualization/jitter.py +587 -247
  429. oscura/visualization/layout.py +169 -134
  430. oscura/visualization/optimization.py +103 -52
  431. oscura/visualization/palettes.py +1 -1
  432. oscura/visualization/power.py +427 -211
  433. oscura/visualization/power_extended.py +626 -297
  434. oscura/visualization/presets.py +2 -0
  435. oscura/visualization/protocols.py +495 -181
  436. oscura/visualization/render.py +79 -63
  437. oscura/visualization/reverse_engineering.py +171 -124
  438. oscura/visualization/signal_integrity.py +460 -279
  439. oscura/visualization/specialized.py +190 -100
  440. oscura/visualization/spectral.py +670 -255
  441. oscura/visualization/thumbnails.py +166 -137
  442. oscura/visualization/waveform.py +150 -63
  443. oscura/workflows/__init__.py +3 -0
  444. oscura/{batch → workflows/batch}/__init__.py +5 -5
  445. oscura/{batch → workflows/batch}/advanced.py +150 -75
  446. oscura/workflows/batch/aggregate.py +531 -0
  447. oscura/workflows/batch/analyze.py +236 -0
  448. oscura/{batch → workflows/batch}/logging.py +2 -2
  449. oscura/{batch → workflows/batch}/metrics.py +1 -1
  450. oscura/workflows/complete_re.py +1144 -0
  451. oscura/workflows/compliance.py +44 -54
  452. oscura/workflows/digital.py +197 -51
  453. oscura/workflows/legacy/__init__.py +12 -0
  454. oscura/{workflow → workflows/legacy}/dag.py +4 -1
  455. oscura/workflows/multi_trace.py +9 -9
  456. oscura/workflows/power.py +42 -62
  457. oscura/workflows/protocol.py +82 -49
  458. oscura/workflows/reverse_engineering.py +351 -150
  459. oscura/workflows/signal_integrity.py +157 -82
  460. oscura-0.7.0.dist-info/METADATA +661 -0
  461. oscura-0.7.0.dist-info/RECORD +591 -0
  462. oscura/batch/aggregate.py +0 -300
  463. oscura/batch/analyze.py +0 -139
  464. oscura/dsl/__init__.py +0 -73
  465. oscura/exceptions.py +0 -59
  466. oscura/exploratory/fuzzy.py +0 -513
  467. oscura/exploratory/sync.py +0 -384
  468. oscura/exporters/__init__.py +0 -94
  469. oscura/exporters/csv.py +0 -303
  470. oscura/exporters/exporters.py +0 -44
  471. oscura/exporters/hdf5.py +0 -217
  472. oscura/exporters/html_export.py +0 -701
  473. oscura/exporters/json_export.py +0 -291
  474. oscura/exporters/markdown_export.py +0 -367
  475. oscura/exporters/matlab_export.py +0 -354
  476. oscura/exporters/npz_export.py +0 -219
  477. oscura/exporters/spice_export.py +0 -210
  478. oscura/search/context.py +0 -149
  479. oscura/session/__init__.py +0 -34
  480. oscura/session/annotations.py +0 -289
  481. oscura/session/history.py +0 -313
  482. oscura/session/session.py +0 -520
  483. oscura/workflow/__init__.py +0 -13
  484. oscura-0.5.1.dist-info/METADATA +0 -583
  485. oscura-0.5.1.dist-info/RECORD +0 -481
  486. /oscura/core/{config.py → config/legacy.py} +0 -0
  487. /oscura/{extensibility → core/extensibility}/__init__.py +0 -0
  488. /oscura/{extensibility → core/extensibility}/registry.py +0 -0
  489. /oscura/{plugins → core/plugins}/isolation.py +0 -0
  490. /oscura/{schemas → core/schemas}/bus_configuration.json +0 -0
  491. /oscura/{builders → utils/builders}/signal_builder.py +0 -0
  492. /oscura/{optimization → utils/optimization}/parallel.py +0 -0
  493. /oscura/{pipeline → utils/pipeline}/__init__.py +0 -0
  494. /oscura/{streaming → utils/streaming}/__init__.py +0 -0
  495. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/WHEEL +0 -0
  496. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/entry_points.txt +0 -0
  497. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,778 @@
1
+ """Pattern mining and correlation analysis for protocol traffic.
2
+
3
+ This module implements pattern mining algorithms (FP-Growth, Apriori) for
4
+ discovering repeated byte sequences, field correlations, and temporal patterns
5
+ in protocol message traffic.
6
+
7
+ Example:
8
+ >>> from oscura.analyzers.patterns.pattern_mining import PatternMiner
9
+ >>> miner = PatternMiner(min_support=0.1, min_confidence=0.5)
10
+ >>> messages = [b"\\xAA\\xBB\\xCC", b"\\xAA\\xBB\\xDD", b"\\xAA\\xBB\\xCC"]
11
+ >>> patterns = miner.mine_byte_patterns(messages)
12
+ >>> print(f"Found {len(patterns)} frequent patterns")
13
+ >>> rules = miner.find_associations(patterns)
14
+ >>> print(f"Discovered {len(rules)} association rules")
15
+
16
+ References:
17
+ Han et al. (2000): "Mining Frequent Patterns without Candidate Generation"
18
+ Agrawal & Srikant (1994): "Fast Algorithms for Mining Association Rules"
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ from collections import defaultdict
24
+ from dataclasses import dataclass, field
25
+ from pathlib import Path
26
+ from typing import Any, Literal
27
+
28
+ import numpy as np
29
+
30
+
31
+ @dataclass
32
+ class Pattern:
33
+ """Discovered pattern in message traffic.
34
+
35
+ Attributes:
36
+ sequence: Byte sequence or field values (tuple of integers).
37
+ support: Frequency of pattern (0.0-1.0, fraction of messages).
38
+ confidence: Confidence for association rules (optional).
39
+ locations: List of (message_idx, offset) tuples where pattern appears.
40
+ metadata: Additional pattern metadata.
41
+ """
42
+
43
+ sequence: tuple[int, ...]
44
+ support: float
45
+ confidence: float | None = None
46
+ locations: list[tuple[int, int]] = field(default_factory=list)
47
+ metadata: dict[str, Any] = field(default_factory=dict)
48
+
49
+ def __repr__(self) -> str:
50
+ """Generate readable representation."""
51
+ seq_str = " ".join(f"{b:02X}" for b in self.sequence)
52
+ return f"Pattern([{seq_str}], support={self.support:.3f})"
53
+
54
+
55
+ @dataclass
56
+ class AssociationRule:
57
+ """Association rule between patterns (A -> B).
58
+
59
+ Represents the rule: "If A appears, then B follows".
60
+
61
+ Attributes:
62
+ antecedent: Pattern A (if this appears).
63
+ consequent: Pattern B (then this follows).
64
+ support: Frequency of (A, B) appearing together.
65
+ confidence: Probability P(B|A) = support(A, B) / support(A).
66
+ lift: Confidence / P(B), measures rule strength.
67
+ """
68
+
69
+ antecedent: tuple[int, ...]
70
+ consequent: tuple[int, ...]
71
+ support: float
72
+ confidence: float
73
+ lift: float
74
+
75
+ def __repr__(self) -> str:
76
+ """Generate readable representation."""
77
+ ant_str = " ".join(f"{b:02X}" for b in self.antecedent)
78
+ con_str = " ".join(f"{b:02X}" for b in self.consequent)
79
+ return f"Rule([{ant_str}] -> [{con_str}], conf={self.confidence:.3f}, lift={self.lift:.2f})"
80
+
81
+
82
+ @dataclass
83
+ class TemporalPattern:
84
+ """Temporal sequence pattern in events.
85
+
86
+ Represents a sequence of events that occur with regular timing.
87
+
88
+ Attributes:
89
+ events: Sequence of event type names.
90
+ timestamps: Event timestamps (relative to first event).
91
+ avg_interval: Average time between consecutive events.
92
+ variance: Variance in event timing.
93
+ """
94
+
95
+ events: list[str]
96
+ timestamps: list[float]
97
+ avg_interval: float
98
+ variance: float
99
+
100
+ def __repr__(self) -> str:
101
+ """Generate readable representation."""
102
+ events_str = " -> ".join(self.events)
103
+ return (
104
+ f"TemporalPattern([{events_str}], interval={self.avg_interval:.3f}±{self.variance:.3f})"
105
+ )
106
+
107
+
108
+ class PatternMiner:
109
+ """Pattern mining and correlation analysis for protocol traffic.
110
+
111
+ Implements FP-Growth and Apriori algorithms for mining frequent patterns,
112
+ association rule discovery, temporal pattern detection, and field correlation
113
+ analysis.
114
+
115
+ Example:
116
+ >>> miner = PatternMiner(min_support=0.1, min_confidence=0.5)
117
+ >>> patterns = miner.mine_byte_patterns(messages, algorithm="fp_growth")
118
+ >>> rules = miner.find_associations(patterns)
119
+ >>> temporal = miner.mine_temporal_patterns(events, max_gap=1.0)
120
+ """
121
+
122
+ def __init__(
123
+ self,
124
+ min_support: float = 0.1,
125
+ min_confidence: float = 0.5,
126
+ min_pattern_length: int = 2,
127
+ max_pattern_length: int = 10,
128
+ ) -> None:
129
+ """Initialize pattern miner with thresholds.
130
+
131
+ Args:
132
+ min_support: Minimum pattern frequency (0.0-1.0).
133
+ min_confidence: Minimum confidence for association rules (0.0-1.0).
134
+ min_pattern_length: Minimum pattern length in bytes.
135
+ max_pattern_length: Maximum pattern length in bytes.
136
+
137
+ Raises:
138
+ ValueError: If parameters are out of valid range.
139
+ """
140
+ if not 0.0 <= min_support <= 1.0:
141
+ raise ValueError(f"min_support must be in [0.0, 1.0], got {min_support}")
142
+ if not 0.0 <= min_confidence <= 1.0:
143
+ raise ValueError(f"min_confidence must be in [0.0, 1.0], got {min_confidence}")
144
+ if min_pattern_length < 1:
145
+ raise ValueError(f"min_pattern_length must be >= 1, got {min_pattern_length}")
146
+ if max_pattern_length < min_pattern_length:
147
+ raise ValueError(
148
+ f"max_pattern_length ({max_pattern_length}) must be >= "
149
+ f"min_pattern_length ({min_pattern_length})"
150
+ )
151
+
152
+ self.min_support = min_support
153
+ self.min_confidence = min_confidence
154
+ self.min_pattern_length = min_pattern_length
155
+ self.max_pattern_length = max_pattern_length
156
+ self.patterns: list[Pattern] = []
157
+ self.rules: list[AssociationRule] = []
158
+
159
+ def mine_byte_patterns(
160
+ self, messages: list[bytes], algorithm: Literal["fp_growth", "apriori"] = "fp_growth"
161
+ ) -> list[Pattern]:
162
+ """Mine frequent byte patterns from messages.
163
+
164
+ Extracts all subsequences from messages, counts occurrences, and filters
165
+ by minimum support threshold. Returns patterns sorted by frequency.
166
+
167
+ Args:
168
+ messages: List of message byte sequences.
169
+ algorithm: Mining algorithm to use ("fp_growth" or "apriori").
170
+
171
+ Returns:
172
+ List of Pattern objects sorted by support (descending).
173
+
174
+ Raises:
175
+ ValueError: If messages list is empty or algorithm is unknown.
176
+
177
+ Example:
178
+ >>> messages = [b"\\xAA\\xBB\\xCC", b"\\xAA\\xBB\\xDD"]
179
+ >>> patterns = miner.mine_byte_patterns(messages)
180
+ >>> for p in patterns:
181
+ ... print(f"{p.sequence}: {p.support:.2f}")
182
+ """
183
+ if not messages:
184
+ raise ValueError("Messages list cannot be empty")
185
+
186
+ if algorithm not in ("fp_growth", "apriori"):
187
+ raise ValueError(f"Unknown algorithm: {algorithm}")
188
+
189
+ # Extract all subsequences
190
+ pattern_counts: dict[tuple[int, ...], int] = defaultdict(int)
191
+ pattern_locations: dict[tuple[int, ...], list[tuple[int, int]]] = defaultdict(list)
192
+
193
+ for msg_idx, message in enumerate(messages):
194
+ subsequences = self._extract_subsequences(
195
+ message, self.min_pattern_length, self.max_pattern_length
196
+ )
197
+
198
+ for subseq in subsequences:
199
+ # Find all occurrences in this message
200
+ for offset in range(len(message) - len(subseq) + 1):
201
+ if tuple(message[offset : offset + len(subseq)]) == subseq:
202
+ pattern_counts[subseq] += 1
203
+ pattern_locations[subseq].append((msg_idx, offset))
204
+
205
+ # Calculate support
206
+ total_subsequences = sum(pattern_counts.values())
207
+
208
+ # Filter by minimum support
209
+ patterns = []
210
+ for seq, count in pattern_counts.items():
211
+ support = count / total_subsequences if total_subsequences > 0 else 0.0
212
+
213
+ if support >= self.min_support:
214
+ # NECESSARY COPY: Prevents mutations of locations after pattern creation.
215
+ patterns.append(
216
+ Pattern(sequence=seq, support=support, locations=pattern_locations[seq].copy())
217
+ )
218
+
219
+ # Sort by support (most frequent first)
220
+ patterns.sort(key=lambda p: p.support, reverse=True)
221
+
222
+ self.patterns = patterns
223
+ return patterns
224
+
225
+ def mine_field_patterns(
226
+ self, field_sequences: list[list[int]], field_names: list[str]
227
+ ) -> list[Pattern]:
228
+ """Mine patterns in extracted field values.
229
+
230
+ Analyzes sequences of field values across multiple messages to find
231
+ common patterns and value combinations.
232
+
233
+ Args:
234
+ field_sequences: List of field value sequences (one per message).
235
+ field_names: Names of fields in sequences.
236
+
237
+ Returns:
238
+ List of Pattern objects for field value sequences.
239
+
240
+ Raises:
241
+ ValueError: If field_sequences is empty or field_names length mismatch.
242
+
243
+ Example:
244
+ >>> field_sequences = [[0x01, 0x02], [0x01, 0x03], [0x01, 0x02]]
245
+ >>> field_names = ["field_a", "field_b"]
246
+ >>> patterns = miner.mine_field_patterns(field_sequences, field_names)
247
+ """
248
+ if not field_sequences:
249
+ raise ValueError("Field sequences cannot be empty")
250
+
251
+ if field_names and len(field_names) != len(field_sequences[0]):
252
+ raise ValueError(
253
+ f"Field names length ({len(field_names)}) must match "
254
+ f"sequence length ({len(field_sequences[0])})"
255
+ )
256
+
257
+ # Convert field sequences to transactions for pattern mining
258
+ pattern_counts: dict[tuple[int, ...], int] = defaultdict(int)
259
+ pattern_locations: dict[tuple[int, ...], list[tuple[int, int]]] = defaultdict(list)
260
+
261
+ for msg_idx, field_vals in enumerate(field_sequences):
262
+ # Extract subsequences from field values
263
+ for length in range(
264
+ self.min_pattern_length, min(len(field_vals) + 1, self.max_pattern_length + 1)
265
+ ):
266
+ for offset in range(len(field_vals) - length + 1):
267
+ subseq = tuple(field_vals[offset : offset + length])
268
+ pattern_counts[subseq] += 1
269
+ pattern_locations[subseq].append((msg_idx, offset))
270
+
271
+ # Calculate support
272
+ total_patterns = sum(pattern_counts.values())
273
+
274
+ # Filter and create patterns
275
+ patterns = []
276
+ for seq, count in pattern_counts.items():
277
+ support = count / total_patterns if total_patterns > 0 else 0.0
278
+
279
+ if support >= self.min_support:
280
+ # Add field names to metadata
281
+ metadata = {}
282
+ if field_names and len(seq) == 1:
283
+ # Single field pattern
284
+ first_loc = pattern_locations[seq][0] if pattern_locations[seq] else (0, 0)
285
+ offset = first_loc[1]
286
+ if offset < len(field_names):
287
+ metadata["field_name"] = field_names[offset]
288
+
289
+ # NECESSARY COPY: Protects pattern's locations from external mutations.
290
+ patterns.append(
291
+ Pattern(
292
+ sequence=seq,
293
+ support=support,
294
+ locations=pattern_locations[seq].copy(),
295
+ metadata=metadata,
296
+ )
297
+ )
298
+
299
+ # Sort by support
300
+ patterns.sort(key=lambda p: p.support, reverse=True)
301
+
302
+ self.patterns = patterns
303
+ return patterns
304
+
305
+ def find_associations(self, patterns: list[Pattern]) -> list[AssociationRule]:
306
+ """Find association rules between patterns (A -> B).
307
+
308
+ Discovers rules where pattern A appearing implies pattern B follows,
309
+ with confidence and lift metrics.
310
+
311
+ Args:
312
+ patterns: List of patterns to analyze for associations.
313
+
314
+ Returns:
315
+ List of AssociationRule objects sorted by confidence.
316
+
317
+ Example:
318
+ >>> rules = miner.find_associations(patterns)
319
+ >>> for rule in rules:
320
+ ... print(f"{rule.antecedent} -> {rule.consequent}: {rule.confidence:.2f}")
321
+ """
322
+ if not patterns:
323
+ return []
324
+
325
+ rules = []
326
+
327
+ # Generate rules (check if B follows A immediately)
328
+ for i, pattern_a in enumerate(patterns):
329
+ for j, pattern_b in enumerate(patterns):
330
+ if i >= j:
331
+ continue
332
+
333
+ seq_a = pattern_a.sequence
334
+ seq_b = pattern_b.sequence
335
+
336
+ # Find co-occurrences (B appears immediately after A)
337
+ co_occur_count = 0
338
+ total_occurrences = len(pattern_a.locations)
339
+
340
+ for msg_idx_a, offset_a in pattern_a.locations:
341
+ expected_offset_b = offset_a + len(seq_a)
342
+
343
+ # Check if B appears right after A
344
+ for msg_idx_b, offset_b in pattern_b.locations:
345
+ if msg_idx_a == msg_idx_b and offset_b == expected_offset_b:
346
+ co_occur_count += 1
347
+ break
348
+
349
+ if co_occur_count > 0:
350
+ # Calculate metrics
351
+ # Confidence = P(B|A) = count(A->B) / count(A)
352
+ confidence = (
353
+ co_occur_count / total_occurrences if total_occurrences > 0 else 0.0
354
+ )
355
+ # Support is the frequency of the rule in the dataset
356
+ support_ab = pattern_a.support * confidence
357
+ # Lift = confidence / P(B)
358
+ lift = confidence / pattern_b.support if pattern_b.support > 0 else 0.0
359
+
360
+ if confidence >= self.min_confidence:
361
+ rules.append(
362
+ AssociationRule(
363
+ antecedent=seq_a,
364
+ consequent=seq_b,
365
+ support=support_ab,
366
+ confidence=confidence,
367
+ lift=lift,
368
+ )
369
+ )
370
+
371
+ # Sort by confidence
372
+ rules.sort(key=lambda r: r.confidence, reverse=True)
373
+
374
+ self.rules = rules
375
+ return rules
376
+
377
+ def mine_temporal_patterns(
378
+ self, events: list[tuple[float, str]], max_gap: float = 1.0
379
+ ) -> list[TemporalPattern]:
380
+ """Mine temporal event sequences.
381
+
382
+ Finds sequences like [EventA, EventB, EventC] that occur with regular
383
+ timing intervals.
384
+
385
+ Args:
386
+ events: List of (timestamp, event_type) tuples.
387
+ max_gap: Maximum time gap between consecutive events in sequence.
388
+
389
+ Returns:
390
+ List of TemporalPattern objects.
391
+
392
+ Raises:
393
+ ValueError: If events list is empty or max_gap is negative.
394
+
395
+ Example:
396
+ >>> events = [(0.0, "A"), (0.5, "B"), (1.0, "A"), (1.5, "B")]
397
+ >>> patterns = miner.mine_temporal_patterns(events, max_gap=0.6)
398
+ """
399
+ if not events:
400
+ raise ValueError("Events list cannot be empty")
401
+ if max_gap < 0:
402
+ raise ValueError(f"max_gap must be non-negative, got {max_gap}")
403
+
404
+ # Sort events by timestamp
405
+ events_sorted = sorted(events, key=lambda e: e[0])
406
+
407
+ # Find sequences
408
+ sequences: dict[tuple[str, ...], list[list[float]]] = defaultdict(list)
409
+
410
+ for i in range(len(events_sorted)):
411
+ current_seq = [events_sorted[i][1]]
412
+ current_times = [events_sorted[i][0]]
413
+
414
+ for j in range(i + 1, min(i + self.max_pattern_length, len(events_sorted))):
415
+ time_gap = events_sorted[j][0] - current_times[-1]
416
+
417
+ if time_gap <= max_gap:
418
+ current_seq.append(events_sorted[j][1])
419
+ current_times.append(events_sorted[j][0])
420
+
421
+ if len(current_seq) >= self.min_pattern_length:
422
+ seq_tuple = tuple(current_seq)
423
+ # NECESSARY COPY: current_times is mutated in loop.
424
+ # Without .copy(), all sequences would reference final state.
425
+ # Removing would cause: data corruption, identical timestamps.
426
+ sequences[seq_tuple].append(current_times.copy())
427
+ else:
428
+ break
429
+
430
+ # Convert to TemporalPattern objects
431
+ temporal_patterns = []
432
+ for seq, time_lists in sequences.items():
433
+ # Calculate average interval
434
+ all_intervals = []
435
+ for times in time_lists:
436
+ intervals = [times[i + 1] - times[i] for i in range(len(times) - 1)]
437
+ all_intervals.extend(intervals)
438
+
439
+ if all_intervals:
440
+ avg_interval = float(np.mean(all_intervals))
441
+ variance = float(np.var(all_intervals))
442
+
443
+ # Use relative timestamps (first event at 0.0)
444
+ relative_times = [t - time_lists[0][0] for t in time_lists[0]]
445
+
446
+ temporal_patterns.append(
447
+ TemporalPattern(
448
+ events=list(seq),
449
+ timestamps=relative_times,
450
+ avg_interval=avg_interval,
451
+ variance=variance,
452
+ )
453
+ )
454
+
455
+ return temporal_patterns
456
+
457
+ def find_correlations(self, field_data: dict[str, list[float]]) -> dict[tuple[str, str], float]:
458
+ """Calculate Pearson correlation coefficients between fields.
459
+
460
+ Computes pairwise correlations to find relationships between numeric
461
+ field values across messages.
462
+
463
+ Args:
464
+ field_data: Dictionary of field_name -> list of values.
465
+
466
+ Returns:
467
+ Dictionary of (field_a, field_b) -> correlation_coefficient.
468
+
469
+ Raises:
470
+ ValueError: If field_data is empty or fields have different lengths.
471
+
472
+ Example:
473
+ >>> field_data = {"field_a": [1, 2, 3], "field_b": [2, 4, 6]}
474
+ >>> correlations = miner.find_correlations(field_data)
475
+ >>> print(correlations[("field_a", "field_b")])
476
+ 1.0
477
+ """
478
+ if not field_data:
479
+ raise ValueError("Field data cannot be empty")
480
+
481
+ # Validate all fields have same length
482
+ field_names = list(field_data.keys())
483
+ lengths = {name: len(values) for name, values in field_data.items()}
484
+ if len(set(lengths.values())) > 1:
485
+ raise ValueError(f"All fields must have same length, got {lengths}")
486
+
487
+ correlations: dict[tuple[str, str], float] = {}
488
+
489
+ # Compute pairwise correlations
490
+ for i, field_a in enumerate(field_names):
491
+ for j, field_b in enumerate(field_names):
492
+ if i >= j:
493
+ continue
494
+
495
+ values_a = np.array(field_data[field_a], dtype=np.float64)
496
+ values_b = np.array(field_data[field_b], dtype=np.float64)
497
+
498
+ # Calculate Pearson correlation
499
+ if len(values_a) < 2:
500
+ corr = 0.0
501
+ else:
502
+ # Compute correlation coefficient
503
+ mean_a = np.mean(values_a)
504
+ mean_b = np.mean(values_b)
505
+
506
+ centered_a = values_a - mean_a
507
+ centered_b = values_b - mean_b
508
+
509
+ num = np.sum(centered_a * centered_b)
510
+ denom = np.sqrt(np.sum(centered_a**2) * np.sum(centered_b**2))
511
+
512
+ corr = float(num / denom) if denom > 0 else 0.0
513
+
514
+ correlations[(field_a, field_b)] = corr
515
+ # Add symmetric entry
516
+ correlations[(field_b, field_a)] = corr
517
+
518
+ return correlations
519
+
520
+ def _extract_subsequences(
521
+ self, sequence: bytes, min_length: int, max_length: int
522
+ ) -> set[tuple[int, ...]]:
523
+ """Extract all subsequences of given length range.
524
+
525
+ Args:
526
+ sequence: Byte sequence to extract from.
527
+ min_length: Minimum subsequence length.
528
+ max_length: Maximum subsequence length.
529
+
530
+ Returns:
531
+ Set of unique subsequences as tuples.
532
+ """
533
+ subsequences: set[tuple[int, ...]] = set()
534
+
535
+ for length in range(min_length, min(len(sequence) + 1, max_length + 1)):
536
+ for offset in range(len(sequence) - length + 1):
537
+ subseq = tuple(sequence[offset : offset + length])
538
+ subsequences.add(subseq)
539
+
540
+ return subsequences
541
+
542
+ def _fp_growth(
543
+ self, transactions: list[frozenset[int]], min_support: float
544
+ ) -> list[tuple[frozenset[int], int]]:
545
+ """FP-Growth algorithm for frequent itemset mining.
546
+
547
+ Simplified implementation for pattern mining. For production use,
548
+ consider using libraries like mlxtend.
549
+
550
+ Args:
551
+ transactions: List of transaction itemsets.
552
+ min_support: Minimum support threshold (0.0-1.0).
553
+
554
+ Returns:
555
+ List of (itemset, count) tuples.
556
+ """
557
+ # Count item frequencies
558
+ item_counts: dict[int, int] = defaultdict(int)
559
+ for transaction in transactions:
560
+ for item in transaction:
561
+ item_counts[item] += 1
562
+
563
+ # Filter by minimum support
564
+ min_count = int(min_support * len(transactions))
565
+ frequent_items = {item for item, count in item_counts.items() if count >= min_count}
566
+
567
+ # Generate frequent itemsets (simplified - single items only for now)
568
+ result = []
569
+ for item in frequent_items:
570
+ count = item_counts[item]
571
+ result.append((frozenset([item]), count))
572
+
573
+ return result
574
+
575
+ def _apriori(
576
+ self, transactions: list[frozenset[int]], min_support: float
577
+ ) -> list[tuple[frozenset[int], int]]:
578
+ """Apriori algorithm for frequent itemset mining.
579
+
580
+ Simplified implementation. For production use, consider libraries
581
+ like mlxtend.
582
+
583
+ Args:
584
+ transactions: List of transaction itemsets.
585
+ min_support: Minimum support threshold (0.0-1.0).
586
+
587
+ Returns:
588
+ List of (itemset, count) tuples.
589
+ """
590
+ # Use same implementation as FP-Growth for now
591
+ # Full Apriori would generate candidate itemsets iteratively
592
+ return self._fp_growth(transactions, min_support)
593
+
594
+ def visualize_patterns(
595
+ self, output_path: Path, format: Literal["graph", "tree", "heatmap"] = "graph"
596
+ ) -> None:
597
+ """Visualize discovered patterns.
598
+
599
+ Creates visualization of patterns and their relationships. Requires
600
+ matplotlib and networkx for graph visualization.
601
+
602
+ Args:
603
+ output_path: Path to save visualization file.
604
+ format: Visualization format ("graph", "tree", or "heatmap").
605
+
606
+ Raises:
607
+ ValueError: If no patterns have been mined yet.
608
+ ImportError: If required visualization libraries not available.
609
+ """
610
+ if not self.patterns:
611
+ raise ValueError("No patterns to visualize. Run mine_byte_patterns() first.")
612
+
613
+ if format == "heatmap":
614
+ self._visualize_heatmap(output_path)
615
+ elif format in ("graph", "tree"):
616
+ self._visualize_graph(output_path)
617
+
618
+ def _visualize_heatmap(self, output_path: Path) -> None:
619
+ """Create heatmap of pattern support values."""
620
+ try:
621
+ import matplotlib.pyplot as plt
622
+ except ImportError as e:
623
+ raise ImportError("matplotlib required for visualization") from e
624
+
625
+ fig, ax = plt.subplots(figsize=(10, 6))
626
+
627
+ pattern_labels = [" ".join(f"{b:02X}" for b in p.sequence) for p in self.patterns[:20]]
628
+ support_values = [p.support for p in self.patterns[:20]]
629
+
630
+ ax.barh(pattern_labels, support_values)
631
+ ax.set_xlabel("Support")
632
+ ax.set_title("Top 20 Pattern Support Values")
633
+ ax.grid(axis="x", alpha=0.3)
634
+
635
+ plt.tight_layout()
636
+ plt.savefig(output_path, dpi=150, bbox_inches="tight")
637
+ plt.close()
638
+
639
+ def _visualize_graph(self, output_path: Path) -> None:
640
+ """Create graph visualization of pattern relationships."""
641
+ try:
642
+ import matplotlib.pyplot as plt
643
+ import networkx as nx
644
+ except ImportError as e:
645
+ raise ImportError("matplotlib and networkx required for graph visualization") from e
646
+
647
+ # Build graph
648
+ G = self._build_pattern_graph(nx)
649
+
650
+ # Draw graph
651
+ fig, ax = plt.subplots(figsize=(12, 8))
652
+
653
+ pos = nx.spring_layout(G, k=2, iterations=50)
654
+ labels = nx.get_node_attributes(G, "label")
655
+
656
+ nx.draw_networkx_nodes(G, pos, node_size=500, node_color="lightblue", ax=ax)
657
+ nx.draw_networkx_labels(G, pos, labels, font_size=8, ax=ax)
658
+ nx.draw_networkx_edges(G, pos, edge_color="gray", arrows=True, ax=ax)
659
+
660
+ ax.set_title("Pattern Association Graph")
661
+ ax.axis("off")
662
+
663
+ plt.tight_layout()
664
+ plt.savefig(output_path, dpi=150, bbox_inches="tight")
665
+ plt.close()
666
+
667
+ def _build_pattern_graph(self, nx: Any) -> Any:
668
+ """Build networkx graph from patterns and rules."""
669
+ G = nx.DiGraph()
670
+
671
+ # Add patterns as nodes
672
+ for i, pattern in enumerate(self.patterns[:20]):
673
+ label = " ".join(f"{b:02X}" for b in pattern.sequence)
674
+ G.add_node(i, label=label, support=pattern.support)
675
+
676
+ # Add edges for association rules
677
+ for rule in self.rules:
678
+ ant_idx = next(
679
+ (i for i, p in enumerate(self.patterns) if p.sequence == rule.antecedent),
680
+ None,
681
+ )
682
+ con_idx = next(
683
+ (i for i, p in enumerate(self.patterns) if p.sequence == rule.consequent),
684
+ None,
685
+ )
686
+
687
+ if ant_idx is not None and con_idx is not None:
688
+ G.add_edge(ant_idx, con_idx, confidence=rule.confidence, lift=rule.lift)
689
+
690
+ return G
691
+
692
+ def export_rules(
693
+ self, output_path: Path, format: Literal["json", "csv", "yaml"] = "json"
694
+ ) -> None:
695
+ """Export association rules to file.
696
+
697
+ Args:
698
+ output_path: Path to save rules file.
699
+ format: Output format ("json", "csv", or "yaml").
700
+
701
+ Raises:
702
+ ValueError: If no rules have been discovered yet.
703
+ ImportError: If required library not available for format.
704
+ """
705
+ if not self.rules:
706
+ raise ValueError("No rules to export. Run find_associations() first.")
707
+
708
+ if format == "json":
709
+ self._export_json(output_path)
710
+ elif format == "csv":
711
+ self._export_csv(output_path)
712
+ elif format == "yaml":
713
+ self._export_yaml(output_path)
714
+
715
+ def _export_json(self, output_path: Path) -> None:
716
+ """Export rules as JSON.
717
+
718
+ Args:
719
+ output_path: Path to save JSON file.
720
+ """
721
+ import json
722
+
723
+ rules_data = [
724
+ {
725
+ "antecedent": [int(b) for b in rule.antecedent],
726
+ "consequent": [int(b) for b in rule.consequent],
727
+ "support": rule.support,
728
+ "confidence": rule.confidence,
729
+ "lift": rule.lift,
730
+ }
731
+ for rule in self.rules
732
+ ]
733
+
734
+ with output_path.open("w") as f:
735
+ json.dump(rules_data, f, indent=2)
736
+
737
+ def _export_csv(self, output_path: Path) -> None:
738
+ """Export rules as CSV.
739
+
740
+ Args:
741
+ output_path: Path to save CSV file.
742
+ """
743
+ with output_path.open("w") as f:
744
+ f.write("antecedent,consequent,support,confidence,lift\n")
745
+ for rule in self.rules:
746
+ ant_str = " ".join(f"{b:02X}" for b in rule.antecedent)
747
+ con_str = " ".join(f"{b:02X}" for b in rule.consequent)
748
+ f.write(f'"{ant_str}","{con_str}",{rule.support},{rule.confidence},{rule.lift}\n')
749
+
750
+ def _export_yaml(self, output_path: Path) -> None:
751
+ """Export rules as YAML.
752
+
753
+ Args:
754
+ output_path: Path to save YAML file.
755
+ """
756
+ import yaml
757
+
758
+ rules_data = [
759
+ {
760
+ "antecedent": [int(b) for b in rule.antecedent],
761
+ "consequent": [int(b) for b in rule.consequent],
762
+ "support": rule.support,
763
+ "confidence": rule.confidence,
764
+ "lift": rule.lift,
765
+ }
766
+ for rule in self.rules
767
+ ]
768
+
769
+ with output_path.open("w") as f:
770
+ yaml.dump(rules_data, f, default_flow_style=False)
771
+
772
+
773
+ __all__ = [
774
+ "AssociationRule",
775
+ "Pattern",
776
+ "PatternMiner",
777
+ "TemporalPattern",
778
+ ]