oscura 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (513) hide show
  1. oscura/__init__.py +169 -167
  2. oscura/analyzers/__init__.py +3 -0
  3. oscura/analyzers/classification.py +659 -0
  4. oscura/analyzers/digital/__init__.py +0 -48
  5. oscura/analyzers/digital/edges.py +325 -65
  6. oscura/analyzers/digital/extraction.py +0 -195
  7. oscura/analyzers/digital/quality.py +293 -166
  8. oscura/analyzers/digital/timing.py +260 -115
  9. oscura/analyzers/digital/timing_numba.py +334 -0
  10. oscura/analyzers/entropy.py +605 -0
  11. oscura/analyzers/eye/diagram.py +176 -109
  12. oscura/analyzers/eye/metrics.py +5 -5
  13. oscura/analyzers/jitter/__init__.py +6 -4
  14. oscura/analyzers/jitter/ber.py +52 -52
  15. oscura/analyzers/jitter/classification.py +156 -0
  16. oscura/analyzers/jitter/decomposition.py +163 -113
  17. oscura/analyzers/jitter/spectrum.py +80 -64
  18. oscura/analyzers/ml/__init__.py +39 -0
  19. oscura/analyzers/ml/features.py +600 -0
  20. oscura/analyzers/ml/signal_classifier.py +604 -0
  21. oscura/analyzers/packet/daq.py +246 -158
  22. oscura/analyzers/packet/parser.py +12 -1
  23. oscura/analyzers/packet/payload.py +50 -2110
  24. oscura/analyzers/packet/payload_analysis.py +361 -181
  25. oscura/analyzers/packet/payload_patterns.py +133 -70
  26. oscura/analyzers/packet/stream.py +84 -23
  27. oscura/analyzers/patterns/__init__.py +26 -5
  28. oscura/analyzers/patterns/anomaly_detection.py +908 -0
  29. oscura/analyzers/patterns/clustering.py +169 -108
  30. oscura/analyzers/patterns/clustering_optimized.py +227 -0
  31. oscura/analyzers/patterns/discovery.py +1 -1
  32. oscura/analyzers/patterns/matching.py +581 -197
  33. oscura/analyzers/patterns/pattern_mining.py +778 -0
  34. oscura/analyzers/patterns/periodic.py +121 -38
  35. oscura/analyzers/patterns/sequences.py +175 -78
  36. oscura/analyzers/power/conduction.py +1 -1
  37. oscura/analyzers/power/soa.py +6 -6
  38. oscura/analyzers/power/switching.py +250 -110
  39. oscura/analyzers/protocol/__init__.py +17 -1
  40. oscura/analyzers/protocols/__init__.py +1 -22
  41. oscura/analyzers/protocols/base.py +6 -6
  42. oscura/analyzers/protocols/ble/__init__.py +38 -0
  43. oscura/analyzers/protocols/ble/analyzer.py +809 -0
  44. oscura/analyzers/protocols/ble/uuids.py +288 -0
  45. oscura/analyzers/protocols/can.py +257 -127
  46. oscura/analyzers/protocols/can_fd.py +107 -80
  47. oscura/analyzers/protocols/flexray.py +139 -80
  48. oscura/analyzers/protocols/hdlc.py +93 -58
  49. oscura/analyzers/protocols/i2c.py +247 -106
  50. oscura/analyzers/protocols/i2s.py +138 -86
  51. oscura/analyzers/protocols/industrial/__init__.py +40 -0
  52. oscura/analyzers/protocols/industrial/bacnet/__init__.py +33 -0
  53. oscura/analyzers/protocols/industrial/bacnet/analyzer.py +708 -0
  54. oscura/analyzers/protocols/industrial/bacnet/encoding.py +412 -0
  55. oscura/analyzers/protocols/industrial/bacnet/services.py +622 -0
  56. oscura/analyzers/protocols/industrial/ethercat/__init__.py +30 -0
  57. oscura/analyzers/protocols/industrial/ethercat/analyzer.py +474 -0
  58. oscura/analyzers/protocols/industrial/ethercat/mailbox.py +339 -0
  59. oscura/analyzers/protocols/industrial/ethercat/topology.py +166 -0
  60. oscura/analyzers/protocols/industrial/modbus/__init__.py +31 -0
  61. oscura/analyzers/protocols/industrial/modbus/analyzer.py +525 -0
  62. oscura/analyzers/protocols/industrial/modbus/crc.py +79 -0
  63. oscura/analyzers/protocols/industrial/modbus/functions.py +436 -0
  64. oscura/analyzers/protocols/industrial/opcua/__init__.py +21 -0
  65. oscura/analyzers/protocols/industrial/opcua/analyzer.py +552 -0
  66. oscura/analyzers/protocols/industrial/opcua/datatypes.py +446 -0
  67. oscura/analyzers/protocols/industrial/opcua/services.py +264 -0
  68. oscura/analyzers/protocols/industrial/profinet/__init__.py +23 -0
  69. oscura/analyzers/protocols/industrial/profinet/analyzer.py +441 -0
  70. oscura/analyzers/protocols/industrial/profinet/dcp.py +263 -0
  71. oscura/analyzers/protocols/industrial/profinet/ptcp.py +200 -0
  72. oscura/analyzers/protocols/jtag.py +180 -98
  73. oscura/analyzers/protocols/lin.py +219 -114
  74. oscura/analyzers/protocols/manchester.py +4 -4
  75. oscura/analyzers/protocols/onewire.py +253 -149
  76. oscura/analyzers/protocols/parallel_bus/__init__.py +20 -0
  77. oscura/analyzers/protocols/parallel_bus/centronics.py +92 -0
  78. oscura/analyzers/protocols/parallel_bus/gpib.py +137 -0
  79. oscura/analyzers/protocols/spi.py +192 -95
  80. oscura/analyzers/protocols/swd.py +321 -167
  81. oscura/analyzers/protocols/uart.py +267 -125
  82. oscura/analyzers/protocols/usb.py +235 -131
  83. oscura/analyzers/side_channel/power.py +17 -12
  84. oscura/analyzers/signal/__init__.py +15 -0
  85. oscura/analyzers/signal/timing_analysis.py +1086 -0
  86. oscura/analyzers/signal_integrity/__init__.py +4 -1
  87. oscura/analyzers/signal_integrity/sparams.py +2 -19
  88. oscura/analyzers/spectral/chunked.py +129 -60
  89. oscura/analyzers/spectral/chunked_fft.py +300 -94
  90. oscura/analyzers/spectral/chunked_wavelet.py +100 -80
  91. oscura/analyzers/statistical/checksum.py +376 -217
  92. oscura/analyzers/statistical/classification.py +229 -107
  93. oscura/analyzers/statistical/entropy.py +78 -53
  94. oscura/analyzers/statistics/correlation.py +407 -211
  95. oscura/analyzers/statistics/outliers.py +2 -2
  96. oscura/analyzers/statistics/streaming.py +30 -5
  97. oscura/analyzers/validation.py +216 -101
  98. oscura/analyzers/waveform/measurements.py +9 -0
  99. oscura/analyzers/waveform/measurements_with_uncertainty.py +31 -15
  100. oscura/analyzers/waveform/spectral.py +500 -228
  101. oscura/api/__init__.py +31 -5
  102. oscura/api/dsl/__init__.py +582 -0
  103. oscura/{dsl → api/dsl}/commands.py +43 -76
  104. oscura/{dsl → api/dsl}/interpreter.py +26 -51
  105. oscura/{dsl → api/dsl}/parser.py +107 -77
  106. oscura/{dsl → api/dsl}/repl.py +2 -2
  107. oscura/api/dsl.py +1 -1
  108. oscura/{integrations → api/integrations}/__init__.py +1 -1
  109. oscura/{integrations → api/integrations}/llm.py +201 -102
  110. oscura/api/operators.py +3 -3
  111. oscura/api/optimization.py +144 -30
  112. oscura/api/rest_server.py +921 -0
  113. oscura/api/server/__init__.py +17 -0
  114. oscura/api/server/dashboard.py +850 -0
  115. oscura/api/server/static/README.md +34 -0
  116. oscura/api/server/templates/base.html +181 -0
  117. oscura/api/server/templates/export.html +120 -0
  118. oscura/api/server/templates/home.html +284 -0
  119. oscura/api/server/templates/protocols.html +58 -0
  120. oscura/api/server/templates/reports.html +43 -0
  121. oscura/api/server/templates/session_detail.html +89 -0
  122. oscura/api/server/templates/sessions.html +83 -0
  123. oscura/api/server/templates/waveforms.html +73 -0
  124. oscura/automotive/__init__.py +8 -1
  125. oscura/automotive/can/__init__.py +10 -0
  126. oscura/automotive/can/checksum.py +3 -1
  127. oscura/automotive/can/dbc_generator.py +590 -0
  128. oscura/automotive/can/message_wrapper.py +121 -74
  129. oscura/automotive/can/patterns.py +98 -21
  130. oscura/automotive/can/session.py +292 -56
  131. oscura/automotive/can/state_machine.py +6 -3
  132. oscura/automotive/can/stimulus_response.py +97 -75
  133. oscura/automotive/dbc/__init__.py +10 -2
  134. oscura/automotive/dbc/generator.py +84 -56
  135. oscura/automotive/dbc/parser.py +6 -6
  136. oscura/automotive/dtc/data.json +2763 -0
  137. oscura/automotive/dtc/database.py +2 -2
  138. oscura/automotive/flexray/__init__.py +31 -0
  139. oscura/automotive/flexray/analyzer.py +504 -0
  140. oscura/automotive/flexray/crc.py +185 -0
  141. oscura/automotive/flexray/fibex.py +449 -0
  142. oscura/automotive/j1939/__init__.py +45 -8
  143. oscura/automotive/j1939/analyzer.py +605 -0
  144. oscura/automotive/j1939/spns.py +326 -0
  145. oscura/automotive/j1939/transport.py +306 -0
  146. oscura/automotive/lin/__init__.py +47 -0
  147. oscura/automotive/lin/analyzer.py +612 -0
  148. oscura/automotive/loaders/blf.py +13 -2
  149. oscura/automotive/loaders/csv_can.py +143 -72
  150. oscura/automotive/loaders/dispatcher.py +50 -2
  151. oscura/automotive/loaders/mdf.py +86 -45
  152. oscura/automotive/loaders/pcap.py +111 -61
  153. oscura/automotive/uds/__init__.py +4 -0
  154. oscura/automotive/uds/analyzer.py +725 -0
  155. oscura/automotive/uds/decoder.py +140 -58
  156. oscura/automotive/uds/models.py +7 -1
  157. oscura/automotive/visualization.py +1 -1
  158. oscura/cli/analyze.py +348 -0
  159. oscura/cli/batch.py +142 -122
  160. oscura/cli/benchmark.py +275 -0
  161. oscura/cli/characterize.py +137 -82
  162. oscura/cli/compare.py +224 -131
  163. oscura/cli/completion.py +250 -0
  164. oscura/cli/config_cmd.py +361 -0
  165. oscura/cli/decode.py +164 -87
  166. oscura/cli/export.py +286 -0
  167. oscura/cli/main.py +115 -31
  168. oscura/{onboarding → cli/onboarding}/__init__.py +3 -3
  169. oscura/{onboarding → cli/onboarding}/help.py +80 -58
  170. oscura/{onboarding → cli/onboarding}/tutorials.py +97 -72
  171. oscura/{onboarding → cli/onboarding}/wizard.py +55 -36
  172. oscura/cli/progress.py +147 -0
  173. oscura/cli/shell.py +157 -135
  174. oscura/cli/validate_cmd.py +204 -0
  175. oscura/cli/visualize.py +158 -0
  176. oscura/convenience.py +125 -79
  177. oscura/core/__init__.py +4 -2
  178. oscura/core/backend_selector.py +3 -3
  179. oscura/core/cache.py +126 -15
  180. oscura/core/cancellation.py +1 -1
  181. oscura/{config → core/config}/__init__.py +20 -11
  182. oscura/{config → core/config}/defaults.py +1 -1
  183. oscura/{config → core/config}/loader.py +7 -5
  184. oscura/{config → core/config}/memory.py +5 -5
  185. oscura/{config → core/config}/migration.py +1 -1
  186. oscura/{config → core/config}/pipeline.py +99 -23
  187. oscura/{config → core/config}/preferences.py +1 -1
  188. oscura/{config → core/config}/protocol.py +3 -3
  189. oscura/{config → core/config}/schema.py +426 -272
  190. oscura/{config → core/config}/settings.py +1 -1
  191. oscura/{config → core/config}/thresholds.py +195 -153
  192. oscura/core/correlation.py +5 -6
  193. oscura/core/cross_domain.py +0 -2
  194. oscura/core/debug.py +9 -5
  195. oscura/{extensibility → core/extensibility}/docs.py +158 -70
  196. oscura/{extensibility → core/extensibility}/extensions.py +160 -76
  197. oscura/{extensibility → core/extensibility}/logging.py +1 -1
  198. oscura/{extensibility → core/extensibility}/measurements.py +1 -1
  199. oscura/{extensibility → core/extensibility}/plugins.py +1 -1
  200. oscura/{extensibility → core/extensibility}/templates.py +73 -3
  201. oscura/{extensibility → core/extensibility}/validation.py +1 -1
  202. oscura/core/gpu_backend.py +11 -7
  203. oscura/core/log_query.py +101 -11
  204. oscura/core/logging.py +126 -54
  205. oscura/core/logging_advanced.py +5 -5
  206. oscura/core/memory_limits.py +108 -70
  207. oscura/core/memory_monitor.py +2 -2
  208. oscura/core/memory_progress.py +7 -7
  209. oscura/core/memory_warnings.py +1 -1
  210. oscura/core/numba_backend.py +13 -13
  211. oscura/{plugins → core/plugins}/__init__.py +9 -9
  212. oscura/{plugins → core/plugins}/base.py +7 -7
  213. oscura/{plugins → core/plugins}/cli.py +3 -3
  214. oscura/{plugins → core/plugins}/discovery.py +186 -106
  215. oscura/{plugins → core/plugins}/lifecycle.py +1 -1
  216. oscura/{plugins → core/plugins}/manager.py +7 -7
  217. oscura/{plugins → core/plugins}/registry.py +3 -3
  218. oscura/{plugins → core/plugins}/versioning.py +1 -1
  219. oscura/core/progress.py +16 -1
  220. oscura/core/provenance.py +8 -2
  221. oscura/{schemas → core/schemas}/__init__.py +2 -2
  222. oscura/core/schemas/bus_configuration.json +322 -0
  223. oscura/core/schemas/device_mapping.json +182 -0
  224. oscura/core/schemas/packet_format.json +418 -0
  225. oscura/core/schemas/protocol_definition.json +363 -0
  226. oscura/core/types.py +4 -0
  227. oscura/core/uncertainty.py +3 -3
  228. oscura/correlation/__init__.py +52 -0
  229. oscura/correlation/multi_protocol.py +811 -0
  230. oscura/discovery/auto_decoder.py +117 -35
  231. oscura/discovery/comparison.py +191 -86
  232. oscura/discovery/quality_validator.py +155 -68
  233. oscura/discovery/signal_detector.py +196 -79
  234. oscura/export/__init__.py +18 -20
  235. oscura/export/kaitai_struct.py +513 -0
  236. oscura/export/scapy_layer.py +801 -0
  237. oscura/export/wireshark/README.md +15 -15
  238. oscura/export/wireshark/generator.py +1 -1
  239. oscura/export/wireshark/templates/dissector.lua.j2 +2 -2
  240. oscura/export/wireshark_dissector.py +746 -0
  241. oscura/guidance/wizard.py +207 -111
  242. oscura/hardware/__init__.py +19 -0
  243. oscura/{acquisition → hardware/acquisition}/__init__.py +4 -4
  244. oscura/{acquisition → hardware/acquisition}/file.py +2 -2
  245. oscura/{acquisition → hardware/acquisition}/hardware.py +7 -7
  246. oscura/{acquisition → hardware/acquisition}/saleae.py +15 -12
  247. oscura/{acquisition → hardware/acquisition}/socketcan.py +1 -1
  248. oscura/{acquisition → hardware/acquisition}/streaming.py +2 -2
  249. oscura/{acquisition → hardware/acquisition}/synthetic.py +3 -3
  250. oscura/{acquisition → hardware/acquisition}/visa.py +33 -11
  251. oscura/hardware/firmware/__init__.py +29 -0
  252. oscura/hardware/firmware/pattern_recognition.py +874 -0
  253. oscura/hardware/hal_detector.py +736 -0
  254. oscura/hardware/security/__init__.py +37 -0
  255. oscura/hardware/security/side_channel_detector.py +1126 -0
  256. oscura/inference/__init__.py +4 -0
  257. oscura/inference/active_learning/README.md +7 -7
  258. oscura/inference/active_learning/observation_table.py +4 -1
  259. oscura/inference/alignment.py +216 -123
  260. oscura/inference/bayesian.py +113 -33
  261. oscura/inference/crc_reverse.py +101 -55
  262. oscura/inference/logic.py +6 -2
  263. oscura/inference/message_format.py +342 -183
  264. oscura/inference/protocol.py +95 -44
  265. oscura/inference/protocol_dsl.py +180 -82
  266. oscura/inference/signal_intelligence.py +1439 -706
  267. oscura/inference/spectral.py +99 -57
  268. oscura/inference/state_machine.py +810 -158
  269. oscura/inference/stream.py +270 -110
  270. oscura/iot/__init__.py +34 -0
  271. oscura/iot/coap/__init__.py +32 -0
  272. oscura/iot/coap/analyzer.py +668 -0
  273. oscura/iot/coap/options.py +212 -0
  274. oscura/iot/lorawan/__init__.py +21 -0
  275. oscura/iot/lorawan/crypto.py +206 -0
  276. oscura/iot/lorawan/decoder.py +801 -0
  277. oscura/iot/lorawan/mac_commands.py +341 -0
  278. oscura/iot/mqtt/__init__.py +27 -0
  279. oscura/iot/mqtt/analyzer.py +999 -0
  280. oscura/iot/mqtt/properties.py +315 -0
  281. oscura/iot/zigbee/__init__.py +31 -0
  282. oscura/iot/zigbee/analyzer.py +615 -0
  283. oscura/iot/zigbee/security.py +153 -0
  284. oscura/iot/zigbee/zcl.py +349 -0
  285. oscura/jupyter/display.py +125 -45
  286. oscura/{exploratory → jupyter/exploratory}/__init__.py +8 -8
  287. oscura/{exploratory → jupyter/exploratory}/error_recovery.py +298 -141
  288. oscura/jupyter/exploratory/fuzzy.py +746 -0
  289. oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py +258 -100
  290. oscura/{exploratory → jupyter/exploratory}/legacy.py +464 -242
  291. oscura/{exploratory → jupyter/exploratory}/parse.py +167 -145
  292. oscura/{exploratory → jupyter/exploratory}/recovery.py +119 -87
  293. oscura/jupyter/exploratory/sync.py +612 -0
  294. oscura/{exploratory → jupyter/exploratory}/unknown.py +299 -176
  295. oscura/jupyter/magic.py +4 -4
  296. oscura/{ui → jupyter/ui}/__init__.py +2 -2
  297. oscura/{ui → jupyter/ui}/formatters.py +3 -3
  298. oscura/{ui → jupyter/ui}/progressive_display.py +153 -82
  299. oscura/loaders/__init__.py +171 -63
  300. oscura/loaders/binary.py +88 -1
  301. oscura/loaders/chipwhisperer.py +153 -137
  302. oscura/loaders/configurable.py +208 -86
  303. oscura/loaders/csv_loader.py +458 -215
  304. oscura/loaders/hdf5_loader.py +278 -119
  305. oscura/loaders/lazy.py +87 -54
  306. oscura/loaders/mmap_loader.py +1 -1
  307. oscura/loaders/numpy_loader.py +253 -116
  308. oscura/loaders/pcap.py +226 -151
  309. oscura/loaders/rigol.py +110 -49
  310. oscura/loaders/sigrok.py +201 -78
  311. oscura/loaders/tdms.py +81 -58
  312. oscura/loaders/tektronix.py +291 -174
  313. oscura/loaders/touchstone.py +182 -87
  314. oscura/loaders/vcd.py +215 -117
  315. oscura/loaders/wav.py +155 -68
  316. oscura/reporting/__init__.py +9 -7
  317. oscura/reporting/analyze.py +352 -146
  318. oscura/reporting/argument_preparer.py +69 -14
  319. oscura/reporting/auto_report.py +97 -61
  320. oscura/reporting/batch.py +131 -58
  321. oscura/reporting/chart_selection.py +57 -45
  322. oscura/reporting/comparison.py +63 -17
  323. oscura/reporting/content/executive.py +76 -24
  324. oscura/reporting/core_formats/multi_format.py +11 -8
  325. oscura/reporting/engine.py +312 -158
  326. oscura/reporting/enhanced_reports.py +949 -0
  327. oscura/reporting/export.py +86 -43
  328. oscura/reporting/formatting/numbers.py +69 -42
  329. oscura/reporting/html.py +139 -58
  330. oscura/reporting/index.py +137 -65
  331. oscura/reporting/output.py +158 -67
  332. oscura/reporting/pdf.py +67 -102
  333. oscura/reporting/plots.py +191 -112
  334. oscura/reporting/sections.py +88 -47
  335. oscura/reporting/standards.py +104 -61
  336. oscura/reporting/summary_generator.py +75 -55
  337. oscura/reporting/tables.py +138 -54
  338. oscura/reporting/templates/enhanced/protocol_re.html +525 -0
  339. oscura/reporting/templates/index.md +13 -13
  340. oscura/sessions/__init__.py +14 -23
  341. oscura/sessions/base.py +3 -3
  342. oscura/sessions/blackbox.py +106 -10
  343. oscura/sessions/generic.py +2 -2
  344. oscura/sessions/legacy.py +783 -0
  345. oscura/side_channel/__init__.py +63 -0
  346. oscura/side_channel/dpa.py +1025 -0
  347. oscura/utils/__init__.py +15 -1
  348. oscura/utils/autodetect.py +1 -5
  349. oscura/utils/bitwise.py +118 -0
  350. oscura/{builders → utils/builders}/__init__.py +1 -1
  351. oscura/{comparison → utils/comparison}/__init__.py +6 -6
  352. oscura/{comparison → utils/comparison}/compare.py +202 -101
  353. oscura/{comparison → utils/comparison}/golden.py +83 -63
  354. oscura/{comparison → utils/comparison}/limits.py +313 -89
  355. oscura/{comparison → utils/comparison}/mask.py +151 -45
  356. oscura/{comparison → utils/comparison}/trace_diff.py +1 -1
  357. oscura/{comparison → utils/comparison}/visualization.py +147 -89
  358. oscura/{component → utils/component}/__init__.py +3 -3
  359. oscura/{component → utils/component}/impedance.py +122 -58
  360. oscura/{component → utils/component}/reactive.py +165 -168
  361. oscura/{component → utils/component}/transmission_line.py +3 -3
  362. oscura/{filtering → utils/filtering}/__init__.py +6 -6
  363. oscura/{filtering → utils/filtering}/base.py +1 -1
  364. oscura/{filtering → utils/filtering}/convenience.py +2 -2
  365. oscura/{filtering → utils/filtering}/design.py +169 -93
  366. oscura/{filtering → utils/filtering}/filters.py +2 -2
  367. oscura/{filtering → utils/filtering}/introspection.py +2 -2
  368. oscura/utils/geometry.py +31 -0
  369. oscura/utils/imports.py +184 -0
  370. oscura/utils/lazy.py +1 -1
  371. oscura/{math → utils/math}/__init__.py +2 -2
  372. oscura/{math → utils/math}/arithmetic.py +114 -48
  373. oscura/{math → utils/math}/interpolation.py +139 -106
  374. oscura/utils/memory.py +129 -66
  375. oscura/utils/memory_advanced.py +92 -9
  376. oscura/utils/memory_extensions.py +10 -8
  377. oscura/{optimization → utils/optimization}/__init__.py +1 -1
  378. oscura/{optimization → utils/optimization}/search.py +2 -2
  379. oscura/utils/performance/__init__.py +58 -0
  380. oscura/utils/performance/caching.py +889 -0
  381. oscura/utils/performance/lsh_clustering.py +333 -0
  382. oscura/utils/performance/memory_optimizer.py +699 -0
  383. oscura/utils/performance/optimizations.py +675 -0
  384. oscura/utils/performance/parallel.py +654 -0
  385. oscura/utils/performance/profiling.py +661 -0
  386. oscura/{pipeline → utils/pipeline}/base.py +1 -1
  387. oscura/{pipeline → utils/pipeline}/composition.py +11 -3
  388. oscura/{pipeline → utils/pipeline}/parallel.py +3 -2
  389. oscura/{pipeline → utils/pipeline}/pipeline.py +1 -1
  390. oscura/{pipeline → utils/pipeline}/reverse_engineering.py +412 -221
  391. oscura/{search → utils/search}/__init__.py +3 -3
  392. oscura/{search → utils/search}/anomaly.py +188 -58
  393. oscura/utils/search/context.py +294 -0
  394. oscura/{search → utils/search}/pattern.py +138 -10
  395. oscura/utils/serial.py +51 -0
  396. oscura/utils/storage/__init__.py +61 -0
  397. oscura/utils/storage/database.py +1166 -0
  398. oscura/{streaming → utils/streaming}/chunked.py +302 -143
  399. oscura/{streaming → utils/streaming}/progressive.py +1 -1
  400. oscura/{streaming → utils/streaming}/realtime.py +3 -2
  401. oscura/{triggering → utils/triggering}/__init__.py +6 -6
  402. oscura/{triggering → utils/triggering}/base.py +6 -6
  403. oscura/{triggering → utils/triggering}/edge.py +2 -2
  404. oscura/{triggering → utils/triggering}/pattern.py +2 -2
  405. oscura/{triggering → utils/triggering}/pulse.py +115 -74
  406. oscura/{triggering → utils/triggering}/window.py +2 -2
  407. oscura/utils/validation.py +32 -0
  408. oscura/validation/__init__.py +121 -0
  409. oscura/{compliance → validation/compliance}/__init__.py +5 -5
  410. oscura/{compliance → validation/compliance}/advanced.py +5 -5
  411. oscura/{compliance → validation/compliance}/masks.py +1 -1
  412. oscura/{compliance → validation/compliance}/reporting.py +127 -53
  413. oscura/{compliance → validation/compliance}/testing.py +114 -52
  414. oscura/validation/compliance_tests.py +915 -0
  415. oscura/validation/fuzzer.py +990 -0
  416. oscura/validation/grammar_tests.py +596 -0
  417. oscura/validation/grammar_validator.py +904 -0
  418. oscura/validation/hil_testing.py +977 -0
  419. oscura/{quality → validation/quality}/__init__.py +4 -4
  420. oscura/{quality → validation/quality}/ensemble.py +251 -171
  421. oscura/{quality → validation/quality}/explainer.py +3 -3
  422. oscura/{quality → validation/quality}/scoring.py +1 -1
  423. oscura/{quality → validation/quality}/warnings.py +4 -4
  424. oscura/validation/regression_suite.py +808 -0
  425. oscura/validation/replay.py +788 -0
  426. oscura/{testing → validation/testing}/__init__.py +2 -2
  427. oscura/{testing → validation/testing}/synthetic.py +5 -5
  428. oscura/visualization/__init__.py +9 -0
  429. oscura/visualization/accessibility.py +1 -1
  430. oscura/visualization/annotations.py +64 -67
  431. oscura/visualization/colors.py +7 -7
  432. oscura/visualization/digital.py +180 -81
  433. oscura/visualization/eye.py +236 -85
  434. oscura/visualization/interactive.py +320 -143
  435. oscura/visualization/jitter.py +587 -247
  436. oscura/visualization/layout.py +169 -134
  437. oscura/visualization/optimization.py +103 -52
  438. oscura/visualization/palettes.py +1 -1
  439. oscura/visualization/power.py +427 -211
  440. oscura/visualization/power_extended.py +626 -297
  441. oscura/visualization/presets.py +2 -0
  442. oscura/visualization/protocols.py +495 -181
  443. oscura/visualization/render.py +79 -63
  444. oscura/visualization/reverse_engineering.py +171 -124
  445. oscura/visualization/signal_integrity.py +460 -279
  446. oscura/visualization/specialized.py +190 -100
  447. oscura/visualization/spectral.py +670 -255
  448. oscura/visualization/thumbnails.py +166 -137
  449. oscura/visualization/waveform.py +150 -63
  450. oscura/workflows/__init__.py +3 -0
  451. oscura/{batch → workflows/batch}/__init__.py +5 -5
  452. oscura/{batch → workflows/batch}/advanced.py +150 -75
  453. oscura/workflows/batch/aggregate.py +531 -0
  454. oscura/workflows/batch/analyze.py +236 -0
  455. oscura/{batch → workflows/batch}/logging.py +2 -2
  456. oscura/{batch → workflows/batch}/metrics.py +1 -1
  457. oscura/workflows/complete_re.py +1144 -0
  458. oscura/workflows/compliance.py +44 -54
  459. oscura/workflows/digital.py +197 -51
  460. oscura/workflows/legacy/__init__.py +12 -0
  461. oscura/{workflow → workflows/legacy}/dag.py +4 -1
  462. oscura/workflows/multi_trace.py +9 -9
  463. oscura/workflows/power.py +42 -62
  464. oscura/workflows/protocol.py +82 -49
  465. oscura/workflows/reverse_engineering.py +351 -150
  466. oscura/workflows/signal_integrity.py +157 -82
  467. oscura-0.6.0.dist-info/METADATA +643 -0
  468. oscura-0.6.0.dist-info/RECORD +590 -0
  469. oscura/analyzers/digital/ic_database.py +0 -498
  470. oscura/analyzers/digital/timing_paths.py +0 -339
  471. oscura/analyzers/digital/vintage.py +0 -377
  472. oscura/analyzers/digital/vintage_result.py +0 -148
  473. oscura/analyzers/protocols/parallel_bus.py +0 -449
  474. oscura/batch/aggregate.py +0 -300
  475. oscura/batch/analyze.py +0 -139
  476. oscura/dsl/__init__.py +0 -73
  477. oscura/exceptions.py +0 -59
  478. oscura/exploratory/fuzzy.py +0 -513
  479. oscura/exploratory/sync.py +0 -384
  480. oscura/export/wavedrom.py +0 -430
  481. oscura/exporters/__init__.py +0 -94
  482. oscura/exporters/csv.py +0 -303
  483. oscura/exporters/exporters.py +0 -44
  484. oscura/exporters/hdf5.py +0 -217
  485. oscura/exporters/html_export.py +0 -701
  486. oscura/exporters/json_export.py +0 -338
  487. oscura/exporters/markdown_export.py +0 -367
  488. oscura/exporters/matlab_export.py +0 -354
  489. oscura/exporters/npz_export.py +0 -219
  490. oscura/exporters/spice_export.py +0 -210
  491. oscura/exporters/vintage_logic_csv.py +0 -247
  492. oscura/reporting/vintage_logic_report.py +0 -523
  493. oscura/search/context.py +0 -149
  494. oscura/session/__init__.py +0 -34
  495. oscura/session/annotations.py +0 -289
  496. oscura/session/history.py +0 -313
  497. oscura/session/session.py +0 -520
  498. oscura/visualization/digital_advanced.py +0 -718
  499. oscura/visualization/figure_manager.py +0 -156
  500. oscura/workflow/__init__.py +0 -13
  501. oscura-0.5.0.dist-info/METADATA +0 -407
  502. oscura-0.5.0.dist-info/RECORD +0 -486
  503. /oscura/core/{config.py → config/legacy.py} +0 -0
  504. /oscura/{extensibility → core/extensibility}/__init__.py +0 -0
  505. /oscura/{extensibility → core/extensibility}/registry.py +0 -0
  506. /oscura/{plugins → core/plugins}/isolation.py +0 -0
  507. /oscura/{builders → utils/builders}/signal_builder.py +0 -0
  508. /oscura/{optimization → utils/optimization}/parallel.py +0 -0
  509. /oscura/{pipeline → utils/pipeline}/__init__.py +0 -0
  510. /oscura/{streaming → utils/streaming}/__init__.py +0 -0
  511. {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/WHEEL +0 -0
  512. {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/entry_points.txt +0 -0
  513. {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,778 @@
1
+ """Pattern mining and correlation analysis for protocol traffic.
2
+
3
+ This module implements pattern mining algorithms (FP-Growth, Apriori) for
4
+ discovering repeated byte sequences, field correlations, and temporal patterns
5
+ in protocol message traffic.
6
+
7
+ Example:
8
+ >>> from oscura.analyzers.patterns.pattern_mining import PatternMiner
9
+ >>> miner = PatternMiner(min_support=0.1, min_confidence=0.5)
10
+ >>> messages = [b"\\xAA\\xBB\\xCC", b"\\xAA\\xBB\\xDD", b"\\xAA\\xBB\\xCC"]
11
+ >>> patterns = miner.mine_byte_patterns(messages)
12
+ >>> print(f"Found {len(patterns)} frequent patterns")
13
+ >>> rules = miner.find_associations(patterns)
14
+ >>> print(f"Discovered {len(rules)} association rules")
15
+
16
+ References:
17
+ Han et al. (2000): "Mining Frequent Patterns without Candidate Generation"
18
+ Agrawal & Srikant (1994): "Fast Algorithms for Mining Association Rules"
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ from collections import defaultdict
24
+ from dataclasses import dataclass, field
25
+ from pathlib import Path
26
+ from typing import Any, Literal
27
+
28
+ import numpy as np
29
+
30
+
31
+ @dataclass
32
+ class Pattern:
33
+ """Discovered pattern in message traffic.
34
+
35
+ Attributes:
36
+ sequence: Byte sequence or field values (tuple of integers).
37
+ support: Frequency of pattern (0.0-1.0, fraction of messages).
38
+ confidence: Confidence for association rules (optional).
39
+ locations: List of (message_idx, offset) tuples where pattern appears.
40
+ metadata: Additional pattern metadata.
41
+ """
42
+
43
+ sequence: tuple[int, ...]
44
+ support: float
45
+ confidence: float | None = None
46
+ locations: list[tuple[int, int]] = field(default_factory=list)
47
+ metadata: dict[str, Any] = field(default_factory=dict)
48
+
49
+ def __repr__(self) -> str:
50
+ """Generate readable representation."""
51
+ seq_str = " ".join(f"{b:02X}" for b in self.sequence)
52
+ return f"Pattern([{seq_str}], support={self.support:.3f})"
53
+
54
+
55
+ @dataclass
56
+ class AssociationRule:
57
+ """Association rule between patterns (A -> B).
58
+
59
+ Represents the rule: "If A appears, then B follows".
60
+
61
+ Attributes:
62
+ antecedent: Pattern A (if this appears).
63
+ consequent: Pattern B (then this follows).
64
+ support: Frequency of (A, B) appearing together.
65
+ confidence: Probability P(B|A) = support(A, B) / support(A).
66
+ lift: Confidence / P(B), measures rule strength.
67
+ """
68
+
69
+ antecedent: tuple[int, ...]
70
+ consequent: tuple[int, ...]
71
+ support: float
72
+ confidence: float
73
+ lift: float
74
+
75
+ def __repr__(self) -> str:
76
+ """Generate readable representation."""
77
+ ant_str = " ".join(f"{b:02X}" for b in self.antecedent)
78
+ con_str = " ".join(f"{b:02X}" for b in self.consequent)
79
+ return f"Rule([{ant_str}] -> [{con_str}], conf={self.confidence:.3f}, lift={self.lift:.2f})"
80
+
81
+
82
+ @dataclass
83
+ class TemporalPattern:
84
+ """Temporal sequence pattern in events.
85
+
86
+ Represents a sequence of events that occur with regular timing.
87
+
88
+ Attributes:
89
+ events: Sequence of event type names.
90
+ timestamps: Event timestamps (relative to first event).
91
+ avg_interval: Average time between consecutive events.
92
+ variance: Variance in event timing.
93
+ """
94
+
95
+ events: list[str]
96
+ timestamps: list[float]
97
+ avg_interval: float
98
+ variance: float
99
+
100
+ def __repr__(self) -> str:
101
+ """Generate readable representation."""
102
+ events_str = " -> ".join(self.events)
103
+ return (
104
+ f"TemporalPattern([{events_str}], interval={self.avg_interval:.3f}±{self.variance:.3f})"
105
+ )
106
+
107
+
108
+ class PatternMiner:
109
+ """Pattern mining and correlation analysis for protocol traffic.
110
+
111
+ Implements FP-Growth and Apriori algorithms for mining frequent patterns,
112
+ association rule discovery, temporal pattern detection, and field correlation
113
+ analysis.
114
+
115
+ Example:
116
+ >>> miner = PatternMiner(min_support=0.1, min_confidence=0.5)
117
+ >>> patterns = miner.mine_byte_patterns(messages, algorithm="fp_growth")
118
+ >>> rules = miner.find_associations(patterns)
119
+ >>> temporal = miner.mine_temporal_patterns(events, max_gap=1.0)
120
+ """
121
+
122
+ def __init__(
123
+ self,
124
+ min_support: float = 0.1,
125
+ min_confidence: float = 0.5,
126
+ min_pattern_length: int = 2,
127
+ max_pattern_length: int = 10,
128
+ ) -> None:
129
+ """Initialize pattern miner with thresholds.
130
+
131
+ Args:
132
+ min_support: Minimum pattern frequency (0.0-1.0).
133
+ min_confidence: Minimum confidence for association rules (0.0-1.0).
134
+ min_pattern_length: Minimum pattern length in bytes.
135
+ max_pattern_length: Maximum pattern length in bytes.
136
+
137
+ Raises:
138
+ ValueError: If parameters are out of valid range.
139
+ """
140
+ if not 0.0 <= min_support <= 1.0:
141
+ raise ValueError(f"min_support must be in [0.0, 1.0], got {min_support}")
142
+ if not 0.0 <= min_confidence <= 1.0:
143
+ raise ValueError(f"min_confidence must be in [0.0, 1.0], got {min_confidence}")
144
+ if min_pattern_length < 1:
145
+ raise ValueError(f"min_pattern_length must be >= 1, got {min_pattern_length}")
146
+ if max_pattern_length < min_pattern_length:
147
+ raise ValueError(
148
+ f"max_pattern_length ({max_pattern_length}) must be >= "
149
+ f"min_pattern_length ({min_pattern_length})"
150
+ )
151
+
152
+ self.min_support = min_support
153
+ self.min_confidence = min_confidence
154
+ self.min_pattern_length = min_pattern_length
155
+ self.max_pattern_length = max_pattern_length
156
+ self.patterns: list[Pattern] = []
157
+ self.rules: list[AssociationRule] = []
158
+
159
+ def mine_byte_patterns(
160
+ self, messages: list[bytes], algorithm: Literal["fp_growth", "apriori"] = "fp_growth"
161
+ ) -> list[Pattern]:
162
+ """Mine frequent byte patterns from messages.
163
+
164
+ Extracts all subsequences from messages, counts occurrences, and filters
165
+ by minimum support threshold. Returns patterns sorted by frequency.
166
+
167
+ Args:
168
+ messages: List of message byte sequences.
169
+ algorithm: Mining algorithm to use ("fp_growth" or "apriori").
170
+
171
+ Returns:
172
+ List of Pattern objects sorted by support (descending).
173
+
174
+ Raises:
175
+ ValueError: If messages list is empty or algorithm is unknown.
176
+
177
+ Example:
178
+ >>> messages = [b"\\xAA\\xBB\\xCC", b"\\xAA\\xBB\\xDD"]
179
+ >>> patterns = miner.mine_byte_patterns(messages)
180
+ >>> for p in patterns:
181
+ ... print(f"{p.sequence}: {p.support:.2f}")
182
+ """
183
+ if not messages:
184
+ raise ValueError("Messages list cannot be empty")
185
+
186
+ if algorithm not in ("fp_growth", "apriori"):
187
+ raise ValueError(f"Unknown algorithm: {algorithm}")
188
+
189
+ # Extract all subsequences
190
+ pattern_counts: dict[tuple[int, ...], int] = defaultdict(int)
191
+ pattern_locations: dict[tuple[int, ...], list[tuple[int, int]]] = defaultdict(list)
192
+
193
+ for msg_idx, message in enumerate(messages):
194
+ subsequences = self._extract_subsequences(
195
+ message, self.min_pattern_length, self.max_pattern_length
196
+ )
197
+
198
+ for subseq in subsequences:
199
+ # Find all occurrences in this message
200
+ for offset in range(len(message) - len(subseq) + 1):
201
+ if tuple(message[offset : offset + len(subseq)]) == subseq:
202
+ pattern_counts[subseq] += 1
203
+ pattern_locations[subseq].append((msg_idx, offset))
204
+
205
+ # Calculate support
206
+ total_subsequences = sum(pattern_counts.values())
207
+
208
+ # Filter by minimum support
209
+ patterns = []
210
+ for seq, count in pattern_counts.items():
211
+ support = count / total_subsequences if total_subsequences > 0 else 0.0
212
+
213
+ if support >= self.min_support:
214
+ # NECESSARY COPY: Prevents mutations of locations after pattern creation.
215
+ patterns.append(
216
+ Pattern(sequence=seq, support=support, locations=pattern_locations[seq].copy())
217
+ )
218
+
219
+ # Sort by support (most frequent first)
220
+ patterns.sort(key=lambda p: p.support, reverse=True)
221
+
222
+ self.patterns = patterns
223
+ return patterns
224
+
225
+ def mine_field_patterns(
226
+ self, field_sequences: list[list[int]], field_names: list[str]
227
+ ) -> list[Pattern]:
228
+ """Mine patterns in extracted field values.
229
+
230
+ Analyzes sequences of field values across multiple messages to find
231
+ common patterns and value combinations.
232
+
233
+ Args:
234
+ field_sequences: List of field value sequences (one per message).
235
+ field_names: Names of fields in sequences.
236
+
237
+ Returns:
238
+ List of Pattern objects for field value sequences.
239
+
240
+ Raises:
241
+ ValueError: If field_sequences is empty or field_names length mismatch.
242
+
243
+ Example:
244
+ >>> field_sequences = [[0x01, 0x02], [0x01, 0x03], [0x01, 0x02]]
245
+ >>> field_names = ["field_a", "field_b"]
246
+ >>> patterns = miner.mine_field_patterns(field_sequences, field_names)
247
+ """
248
+ if not field_sequences:
249
+ raise ValueError("Field sequences cannot be empty")
250
+
251
+ if field_names and len(field_names) != len(field_sequences[0]):
252
+ raise ValueError(
253
+ f"Field names length ({len(field_names)}) must match "
254
+ f"sequence length ({len(field_sequences[0])})"
255
+ )
256
+
257
+ # Convert field sequences to transactions for pattern mining
258
+ pattern_counts: dict[tuple[int, ...], int] = defaultdict(int)
259
+ pattern_locations: dict[tuple[int, ...], list[tuple[int, int]]] = defaultdict(list)
260
+
261
+ for msg_idx, field_vals in enumerate(field_sequences):
262
+ # Extract subsequences from field values
263
+ for length in range(
264
+ self.min_pattern_length, min(len(field_vals) + 1, self.max_pattern_length + 1)
265
+ ):
266
+ for offset in range(len(field_vals) - length + 1):
267
+ subseq = tuple(field_vals[offset : offset + length])
268
+ pattern_counts[subseq] += 1
269
+ pattern_locations[subseq].append((msg_idx, offset))
270
+
271
+ # Calculate support
272
+ total_patterns = sum(pattern_counts.values())
273
+
274
+ # Filter and create patterns
275
+ patterns = []
276
+ for seq, count in pattern_counts.items():
277
+ support = count / total_patterns if total_patterns > 0 else 0.0
278
+
279
+ if support >= self.min_support:
280
+ # Add field names to metadata
281
+ metadata = {}
282
+ if field_names and len(seq) == 1:
283
+ # Single field pattern
284
+ first_loc = pattern_locations[seq][0] if pattern_locations[seq] else (0, 0)
285
+ offset = first_loc[1]
286
+ if offset < len(field_names):
287
+ metadata["field_name"] = field_names[offset]
288
+
289
+ # NECESSARY COPY: Protects pattern's locations from external mutations.
290
+ patterns.append(
291
+ Pattern(
292
+ sequence=seq,
293
+ support=support,
294
+ locations=pattern_locations[seq].copy(),
295
+ metadata=metadata,
296
+ )
297
+ )
298
+
299
+ # Sort by support
300
+ patterns.sort(key=lambda p: p.support, reverse=True)
301
+
302
+ self.patterns = patterns
303
+ return patterns
304
+
305
+ def find_associations(self, patterns: list[Pattern]) -> list[AssociationRule]:
306
+ """Find association rules between patterns (A -> B).
307
+
308
+ Discovers rules where pattern A appearing implies pattern B follows,
309
+ with confidence and lift metrics.
310
+
311
+ Args:
312
+ patterns: List of patterns to analyze for associations.
313
+
314
+ Returns:
315
+ List of AssociationRule objects sorted by confidence.
316
+
317
+ Example:
318
+ >>> rules = miner.find_associations(patterns)
319
+ >>> for rule in rules:
320
+ ... print(f"{rule.antecedent} -> {rule.consequent}: {rule.confidence:.2f}")
321
+ """
322
+ if not patterns:
323
+ return []
324
+
325
+ rules = []
326
+
327
+ # Generate rules (check if B follows A immediately)
328
+ for i, pattern_a in enumerate(patterns):
329
+ for j, pattern_b in enumerate(patterns):
330
+ if i >= j:
331
+ continue
332
+
333
+ seq_a = pattern_a.sequence
334
+ seq_b = pattern_b.sequence
335
+
336
+ # Find co-occurrences (B appears immediately after A)
337
+ co_occur_count = 0
338
+ total_occurrences = len(pattern_a.locations)
339
+
340
+ for msg_idx_a, offset_a in pattern_a.locations:
341
+ expected_offset_b = offset_a + len(seq_a)
342
+
343
+ # Check if B appears right after A
344
+ for msg_idx_b, offset_b in pattern_b.locations:
345
+ if msg_idx_a == msg_idx_b and offset_b == expected_offset_b:
346
+ co_occur_count += 1
347
+ break
348
+
349
+ if co_occur_count > 0:
350
+ # Calculate metrics
351
+ # Confidence = P(B|A) = count(A->B) / count(A)
352
+ confidence = (
353
+ co_occur_count / total_occurrences if total_occurrences > 0 else 0.0
354
+ )
355
+ # Support is the frequency of the rule in the dataset
356
+ support_ab = pattern_a.support * confidence
357
+ # Lift = confidence / P(B)
358
+ lift = confidence / pattern_b.support if pattern_b.support > 0 else 0.0
359
+
360
+ if confidence >= self.min_confidence:
361
+ rules.append(
362
+ AssociationRule(
363
+ antecedent=seq_a,
364
+ consequent=seq_b,
365
+ support=support_ab,
366
+ confidence=confidence,
367
+ lift=lift,
368
+ )
369
+ )
370
+
371
+ # Sort by confidence
372
+ rules.sort(key=lambda r: r.confidence, reverse=True)
373
+
374
+ self.rules = rules
375
+ return rules
376
+
377
+ def mine_temporal_patterns(
378
+ self, events: list[tuple[float, str]], max_gap: float = 1.0
379
+ ) -> list[TemporalPattern]:
380
+ """Mine temporal event sequences.
381
+
382
+ Finds sequences like [EventA, EventB, EventC] that occur with regular
383
+ timing intervals.
384
+
385
+ Args:
386
+ events: List of (timestamp, event_type) tuples.
387
+ max_gap: Maximum time gap between consecutive events in sequence.
388
+
389
+ Returns:
390
+ List of TemporalPattern objects.
391
+
392
+ Raises:
393
+ ValueError: If events list is empty or max_gap is negative.
394
+
395
+ Example:
396
+ >>> events = [(0.0, "A"), (0.5, "B"), (1.0, "A"), (1.5, "B")]
397
+ >>> patterns = miner.mine_temporal_patterns(events, max_gap=0.6)
398
+ """
399
+ if not events:
400
+ raise ValueError("Events list cannot be empty")
401
+ if max_gap < 0:
402
+ raise ValueError(f"max_gap must be non-negative, got {max_gap}")
403
+
404
+ # Sort events by timestamp
405
+ events_sorted = sorted(events, key=lambda e: e[0])
406
+
407
+ # Find sequences
408
+ sequences: dict[tuple[str, ...], list[list[float]]] = defaultdict(list)
409
+
410
+ for i in range(len(events_sorted)):
411
+ current_seq = [events_sorted[i][1]]
412
+ current_times = [events_sorted[i][0]]
413
+
414
+ for j in range(i + 1, min(i + self.max_pattern_length, len(events_sorted))):
415
+ time_gap = events_sorted[j][0] - current_times[-1]
416
+
417
+ if time_gap <= max_gap:
418
+ current_seq.append(events_sorted[j][1])
419
+ current_times.append(events_sorted[j][0])
420
+
421
+ if len(current_seq) >= self.min_pattern_length:
422
+ seq_tuple = tuple(current_seq)
423
+ # NECESSARY COPY: current_times is mutated in loop.
424
+ # Without .copy(), all sequences would reference final state.
425
+ # Removing would cause: data corruption, identical timestamps.
426
+ sequences[seq_tuple].append(current_times.copy())
427
+ else:
428
+ break
429
+
430
+ # Convert to TemporalPattern objects
431
+ temporal_patterns = []
432
+ for seq, time_lists in sequences.items():
433
+ # Calculate average interval
434
+ all_intervals = []
435
+ for times in time_lists:
436
+ intervals = [times[i + 1] - times[i] for i in range(len(times) - 1)]
437
+ all_intervals.extend(intervals)
438
+
439
+ if all_intervals:
440
+ avg_interval = float(np.mean(all_intervals))
441
+ variance = float(np.var(all_intervals))
442
+
443
+ # Use relative timestamps (first event at 0.0)
444
+ relative_times = [t - time_lists[0][0] for t in time_lists[0]]
445
+
446
+ temporal_patterns.append(
447
+ TemporalPattern(
448
+ events=list(seq),
449
+ timestamps=relative_times,
450
+ avg_interval=avg_interval,
451
+ variance=variance,
452
+ )
453
+ )
454
+
455
+ return temporal_patterns
456
+
457
+ def find_correlations(self, field_data: dict[str, list[float]]) -> dict[tuple[str, str], float]:
458
+ """Calculate Pearson correlation coefficients between fields.
459
+
460
+ Computes pairwise correlations to find relationships between numeric
461
+ field values across messages.
462
+
463
+ Args:
464
+ field_data: Dictionary of field_name -> list of values.
465
+
466
+ Returns:
467
+ Dictionary of (field_a, field_b) -> correlation_coefficient.
468
+
469
+ Raises:
470
+ ValueError: If field_data is empty or fields have different lengths.
471
+
472
+ Example:
473
+ >>> field_data = {"field_a": [1, 2, 3], "field_b": [2, 4, 6]}
474
+ >>> correlations = miner.find_correlations(field_data)
475
+ >>> print(correlations[("field_a", "field_b")])
476
+ 1.0
477
+ """
478
+ if not field_data:
479
+ raise ValueError("Field data cannot be empty")
480
+
481
+ # Validate all fields have same length
482
+ field_names = list(field_data.keys())
483
+ lengths = {name: len(values) for name, values in field_data.items()}
484
+ if len(set(lengths.values())) > 1:
485
+ raise ValueError(f"All fields must have same length, got {lengths}")
486
+
487
+ correlations: dict[tuple[str, str], float] = {}
488
+
489
+ # Compute pairwise correlations
490
+ for i, field_a in enumerate(field_names):
491
+ for j, field_b in enumerate(field_names):
492
+ if i >= j:
493
+ continue
494
+
495
+ values_a = np.array(field_data[field_a], dtype=np.float64)
496
+ values_b = np.array(field_data[field_b], dtype=np.float64)
497
+
498
+ # Calculate Pearson correlation
499
+ if len(values_a) < 2:
500
+ corr = 0.0
501
+ else:
502
+ # Compute correlation coefficient
503
+ mean_a = np.mean(values_a)
504
+ mean_b = np.mean(values_b)
505
+
506
+ centered_a = values_a - mean_a
507
+ centered_b = values_b - mean_b
508
+
509
+ num = np.sum(centered_a * centered_b)
510
+ denom = np.sqrt(np.sum(centered_a**2) * np.sum(centered_b**2))
511
+
512
+ corr = float(num / denom) if denom > 0 else 0.0
513
+
514
+ correlations[(field_a, field_b)] = corr
515
+ # Add symmetric entry
516
+ correlations[(field_b, field_a)] = corr
517
+
518
+ return correlations
519
+
520
+ def _extract_subsequences(
521
+ self, sequence: bytes, min_length: int, max_length: int
522
+ ) -> set[tuple[int, ...]]:
523
+ """Extract all subsequences of given length range.
524
+
525
+ Args:
526
+ sequence: Byte sequence to extract from.
527
+ min_length: Minimum subsequence length.
528
+ max_length: Maximum subsequence length.
529
+
530
+ Returns:
531
+ Set of unique subsequences as tuples.
532
+ """
533
+ subsequences: set[tuple[int, ...]] = set()
534
+
535
+ for length in range(min_length, min(len(sequence) + 1, max_length + 1)):
536
+ for offset in range(len(sequence) - length + 1):
537
+ subseq = tuple(sequence[offset : offset + length])
538
+ subsequences.add(subseq)
539
+
540
+ return subsequences
541
+
542
+ def _fp_growth(
543
+ self, transactions: list[frozenset[int]], min_support: float
544
+ ) -> list[tuple[frozenset[int], int]]:
545
+ """FP-Growth algorithm for frequent itemset mining.
546
+
547
+ Simplified implementation for pattern mining. For production use,
548
+ consider using libraries like mlxtend.
549
+
550
+ Args:
551
+ transactions: List of transaction itemsets.
552
+ min_support: Minimum support threshold (0.0-1.0).
553
+
554
+ Returns:
555
+ List of (itemset, count) tuples.
556
+ """
557
+ # Count item frequencies
558
+ item_counts: dict[int, int] = defaultdict(int)
559
+ for transaction in transactions:
560
+ for item in transaction:
561
+ item_counts[item] += 1
562
+
563
+ # Filter by minimum support
564
+ min_count = int(min_support * len(transactions))
565
+ frequent_items = {item for item, count in item_counts.items() if count >= min_count}
566
+
567
+ # Generate frequent itemsets (simplified - single items only for now)
568
+ result = []
569
+ for item in frequent_items:
570
+ count = item_counts[item]
571
+ result.append((frozenset([item]), count))
572
+
573
+ return result
574
+
575
+ def _apriori(
576
+ self, transactions: list[frozenset[int]], min_support: float
577
+ ) -> list[tuple[frozenset[int], int]]:
578
+ """Apriori algorithm for frequent itemset mining.
579
+
580
+ Simplified implementation. For production use, consider libraries
581
+ like mlxtend.
582
+
583
+ Args:
584
+ transactions: List of transaction itemsets.
585
+ min_support: Minimum support threshold (0.0-1.0).
586
+
587
+ Returns:
588
+ List of (itemset, count) tuples.
589
+ """
590
+ # Use same implementation as FP-Growth for now
591
+ # Full Apriori would generate candidate itemsets iteratively
592
+ return self._fp_growth(transactions, min_support)
593
+
594
+ def visualize_patterns(
595
+ self, output_path: Path, format: Literal["graph", "tree", "heatmap"] = "graph"
596
+ ) -> None:
597
+ """Visualize discovered patterns.
598
+
599
+ Creates visualization of patterns and their relationships. Requires
600
+ matplotlib and networkx for graph visualization.
601
+
602
+ Args:
603
+ output_path: Path to save visualization file.
604
+ format: Visualization format ("graph", "tree", or "heatmap").
605
+
606
+ Raises:
607
+ ValueError: If no patterns have been mined yet.
608
+ ImportError: If required visualization libraries not available.
609
+ """
610
+ if not self.patterns:
611
+ raise ValueError("No patterns to visualize. Run mine_byte_patterns() first.")
612
+
613
+ if format == "heatmap":
614
+ self._visualize_heatmap(output_path)
615
+ elif format in ("graph", "tree"):
616
+ self._visualize_graph(output_path)
617
+
618
+ def _visualize_heatmap(self, output_path: Path) -> None:
619
+ """Create heatmap of pattern support values."""
620
+ try:
621
+ import matplotlib.pyplot as plt
622
+ except ImportError as e:
623
+ raise ImportError("matplotlib required for visualization") from e
624
+
625
+ fig, ax = plt.subplots(figsize=(10, 6))
626
+
627
+ pattern_labels = [" ".join(f"{b:02X}" for b in p.sequence) for p in self.patterns[:20]]
628
+ support_values = [p.support for p in self.patterns[:20]]
629
+
630
+ ax.barh(pattern_labels, support_values)
631
+ ax.set_xlabel("Support")
632
+ ax.set_title("Top 20 Pattern Support Values")
633
+ ax.grid(axis="x", alpha=0.3)
634
+
635
+ plt.tight_layout()
636
+ plt.savefig(output_path, dpi=150, bbox_inches="tight")
637
+ plt.close()
638
+
639
+ def _visualize_graph(self, output_path: Path) -> None:
640
+ """Create graph visualization of pattern relationships."""
641
+ try:
642
+ import matplotlib.pyplot as plt
643
+ import networkx as nx
644
+ except ImportError as e:
645
+ raise ImportError("matplotlib and networkx required for graph visualization") from e
646
+
647
+ # Build graph
648
+ G = self._build_pattern_graph(nx)
649
+
650
+ # Draw graph
651
+ fig, ax = plt.subplots(figsize=(12, 8))
652
+
653
+ pos = nx.spring_layout(G, k=2, iterations=50)
654
+ labels = nx.get_node_attributes(G, "label")
655
+
656
+ nx.draw_networkx_nodes(G, pos, node_size=500, node_color="lightblue", ax=ax)
657
+ nx.draw_networkx_labels(G, pos, labels, font_size=8, ax=ax)
658
+ nx.draw_networkx_edges(G, pos, edge_color="gray", arrows=True, ax=ax)
659
+
660
+ ax.set_title("Pattern Association Graph")
661
+ ax.axis("off")
662
+
663
+ plt.tight_layout()
664
+ plt.savefig(output_path, dpi=150, bbox_inches="tight")
665
+ plt.close()
666
+
667
+ def _build_pattern_graph(self, nx: Any) -> Any:
668
+ """Build networkx graph from patterns and rules."""
669
+ G = nx.DiGraph()
670
+
671
+ # Add patterns as nodes
672
+ for i, pattern in enumerate(self.patterns[:20]):
673
+ label = " ".join(f"{b:02X}" for b in pattern.sequence)
674
+ G.add_node(i, label=label, support=pattern.support)
675
+
676
+ # Add edges for association rules
677
+ for rule in self.rules:
678
+ ant_idx = next(
679
+ (i for i, p in enumerate(self.patterns) if p.sequence == rule.antecedent),
680
+ None,
681
+ )
682
+ con_idx = next(
683
+ (i for i, p in enumerate(self.patterns) if p.sequence == rule.consequent),
684
+ None,
685
+ )
686
+
687
+ if ant_idx is not None and con_idx is not None:
688
+ G.add_edge(ant_idx, con_idx, confidence=rule.confidence, lift=rule.lift)
689
+
690
+ return G
691
+
692
+ def export_rules(
693
+ self, output_path: Path, format: Literal["json", "csv", "yaml"] = "json"
694
+ ) -> None:
695
+ """Export association rules to file.
696
+
697
+ Args:
698
+ output_path: Path to save rules file.
699
+ format: Output format ("json", "csv", or "yaml").
700
+
701
+ Raises:
702
+ ValueError: If no rules have been discovered yet.
703
+ ImportError: If required library not available for format.
704
+ """
705
+ if not self.rules:
706
+ raise ValueError("No rules to export. Run find_associations() first.")
707
+
708
+ if format == "json":
709
+ self._export_json(output_path)
710
+ elif format == "csv":
711
+ self._export_csv(output_path)
712
+ elif format == "yaml":
713
+ self._export_yaml(output_path)
714
+
715
+ def _export_json(self, output_path: Path) -> None:
716
+ """Export rules as JSON.
717
+
718
+ Args:
719
+ output_path: Path to save JSON file.
720
+ """
721
+ import json
722
+
723
+ rules_data = [
724
+ {
725
+ "antecedent": [int(b) for b in rule.antecedent],
726
+ "consequent": [int(b) for b in rule.consequent],
727
+ "support": rule.support,
728
+ "confidence": rule.confidence,
729
+ "lift": rule.lift,
730
+ }
731
+ for rule in self.rules
732
+ ]
733
+
734
+ with output_path.open("w") as f:
735
+ json.dump(rules_data, f, indent=2)
736
+
737
+ def _export_csv(self, output_path: Path) -> None:
738
+ """Export rules as CSV.
739
+
740
+ Args:
741
+ output_path: Path to save CSV file.
742
+ """
743
+ with output_path.open("w") as f:
744
+ f.write("antecedent,consequent,support,confidence,lift\n")
745
+ for rule in self.rules:
746
+ ant_str = " ".join(f"{b:02X}" for b in rule.antecedent)
747
+ con_str = " ".join(f"{b:02X}" for b in rule.consequent)
748
+ f.write(f'"{ant_str}","{con_str}",{rule.support},{rule.confidence},{rule.lift}\n')
749
+
750
+ def _export_yaml(self, output_path: Path) -> None:
751
+ """Export rules as YAML.
752
+
753
+ Args:
754
+ output_path: Path to save YAML file.
755
+ """
756
+ import yaml
757
+
758
+ rules_data = [
759
+ {
760
+ "antecedent": [int(b) for b in rule.antecedent],
761
+ "consequent": [int(b) for b in rule.consequent],
762
+ "support": rule.support,
763
+ "confidence": rule.confidence,
764
+ "lift": rule.lift,
765
+ }
766
+ for rule in self.rules
767
+ ]
768
+
769
+ with output_path.open("w") as f:
770
+ yaml.dump(rules_data, f, default_flow_style=False)
771
+
772
+
773
+ __all__ = [
774
+ "AssociationRule",
775
+ "Pattern",
776
+ "PatternMiner",
777
+ "TemporalPattern",
778
+ ]