oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (497) hide show
  1. oscura/__init__.py +169 -167
  2. oscura/analyzers/__init__.py +3 -0
  3. oscura/analyzers/classification.py +659 -0
  4. oscura/analyzers/digital/edges.py +325 -65
  5. oscura/analyzers/digital/quality.py +293 -166
  6. oscura/analyzers/digital/timing.py +260 -115
  7. oscura/analyzers/digital/timing_numba.py +334 -0
  8. oscura/analyzers/entropy.py +605 -0
  9. oscura/analyzers/eye/diagram.py +176 -109
  10. oscura/analyzers/eye/metrics.py +5 -5
  11. oscura/analyzers/jitter/__init__.py +6 -4
  12. oscura/analyzers/jitter/ber.py +52 -52
  13. oscura/analyzers/jitter/classification.py +156 -0
  14. oscura/analyzers/jitter/decomposition.py +163 -113
  15. oscura/analyzers/jitter/spectrum.py +80 -64
  16. oscura/analyzers/ml/__init__.py +39 -0
  17. oscura/analyzers/ml/features.py +600 -0
  18. oscura/analyzers/ml/signal_classifier.py +604 -0
  19. oscura/analyzers/packet/daq.py +246 -158
  20. oscura/analyzers/packet/parser.py +12 -1
  21. oscura/analyzers/packet/payload.py +50 -2110
  22. oscura/analyzers/packet/payload_analysis.py +361 -181
  23. oscura/analyzers/packet/payload_patterns.py +133 -70
  24. oscura/analyzers/packet/stream.py +84 -23
  25. oscura/analyzers/patterns/__init__.py +26 -5
  26. oscura/analyzers/patterns/anomaly_detection.py +908 -0
  27. oscura/analyzers/patterns/clustering.py +169 -108
  28. oscura/analyzers/patterns/clustering_optimized.py +227 -0
  29. oscura/analyzers/patterns/discovery.py +1 -1
  30. oscura/analyzers/patterns/matching.py +581 -197
  31. oscura/analyzers/patterns/pattern_mining.py +778 -0
  32. oscura/analyzers/patterns/periodic.py +121 -38
  33. oscura/analyzers/patterns/sequences.py +175 -78
  34. oscura/analyzers/power/conduction.py +1 -1
  35. oscura/analyzers/power/soa.py +6 -6
  36. oscura/analyzers/power/switching.py +250 -110
  37. oscura/analyzers/protocol/__init__.py +17 -1
  38. oscura/analyzers/protocols/base.py +6 -6
  39. oscura/analyzers/protocols/ble/__init__.py +38 -0
  40. oscura/analyzers/protocols/ble/analyzer.py +809 -0
  41. oscura/analyzers/protocols/ble/uuids.py +288 -0
  42. oscura/analyzers/protocols/can.py +257 -127
  43. oscura/analyzers/protocols/can_fd.py +107 -80
  44. oscura/analyzers/protocols/flexray.py +139 -80
  45. oscura/analyzers/protocols/hdlc.py +93 -58
  46. oscura/analyzers/protocols/i2c.py +247 -106
  47. oscura/analyzers/protocols/i2s.py +138 -86
  48. oscura/analyzers/protocols/industrial/__init__.py +40 -0
  49. oscura/analyzers/protocols/industrial/bacnet/__init__.py +33 -0
  50. oscura/analyzers/protocols/industrial/bacnet/analyzer.py +708 -0
  51. oscura/analyzers/protocols/industrial/bacnet/encoding.py +412 -0
  52. oscura/analyzers/protocols/industrial/bacnet/services.py +622 -0
  53. oscura/analyzers/protocols/industrial/ethercat/__init__.py +30 -0
  54. oscura/analyzers/protocols/industrial/ethercat/analyzer.py +474 -0
  55. oscura/analyzers/protocols/industrial/ethercat/mailbox.py +339 -0
  56. oscura/analyzers/protocols/industrial/ethercat/topology.py +166 -0
  57. oscura/analyzers/protocols/industrial/modbus/__init__.py +31 -0
  58. oscura/analyzers/protocols/industrial/modbus/analyzer.py +525 -0
  59. oscura/analyzers/protocols/industrial/modbus/crc.py +79 -0
  60. oscura/analyzers/protocols/industrial/modbus/functions.py +436 -0
  61. oscura/analyzers/protocols/industrial/opcua/__init__.py +21 -0
  62. oscura/analyzers/protocols/industrial/opcua/analyzer.py +552 -0
  63. oscura/analyzers/protocols/industrial/opcua/datatypes.py +446 -0
  64. oscura/analyzers/protocols/industrial/opcua/services.py +264 -0
  65. oscura/analyzers/protocols/industrial/profinet/__init__.py +23 -0
  66. oscura/analyzers/protocols/industrial/profinet/analyzer.py +441 -0
  67. oscura/analyzers/protocols/industrial/profinet/dcp.py +263 -0
  68. oscura/analyzers/protocols/industrial/profinet/ptcp.py +200 -0
  69. oscura/analyzers/protocols/jtag.py +180 -98
  70. oscura/analyzers/protocols/lin.py +219 -114
  71. oscura/analyzers/protocols/manchester.py +4 -4
  72. oscura/analyzers/protocols/onewire.py +253 -149
  73. oscura/analyzers/protocols/parallel_bus/__init__.py +20 -0
  74. oscura/analyzers/protocols/parallel_bus/centronics.py +92 -0
  75. oscura/analyzers/protocols/parallel_bus/gpib.py +137 -0
  76. oscura/analyzers/protocols/spi.py +192 -95
  77. oscura/analyzers/protocols/swd.py +321 -167
  78. oscura/analyzers/protocols/uart.py +267 -125
  79. oscura/analyzers/protocols/usb.py +235 -131
  80. oscura/analyzers/side_channel/power.py +17 -12
  81. oscura/analyzers/signal/__init__.py +15 -0
  82. oscura/analyzers/signal/timing_analysis.py +1086 -0
  83. oscura/analyzers/signal_integrity/__init__.py +4 -1
  84. oscura/analyzers/signal_integrity/sparams.py +2 -19
  85. oscura/analyzers/spectral/chunked.py +129 -60
  86. oscura/analyzers/spectral/chunked_fft.py +300 -94
  87. oscura/analyzers/spectral/chunked_wavelet.py +100 -80
  88. oscura/analyzers/statistical/checksum.py +376 -217
  89. oscura/analyzers/statistical/classification.py +229 -107
  90. oscura/analyzers/statistical/entropy.py +78 -53
  91. oscura/analyzers/statistics/correlation.py +407 -211
  92. oscura/analyzers/statistics/outliers.py +2 -2
  93. oscura/analyzers/statistics/streaming.py +30 -5
  94. oscura/analyzers/validation.py +216 -101
  95. oscura/analyzers/waveform/measurements.py +9 -0
  96. oscura/analyzers/waveform/measurements_with_uncertainty.py +31 -15
  97. oscura/analyzers/waveform/spectral.py +500 -228
  98. oscura/api/__init__.py +31 -5
  99. oscura/api/dsl/__init__.py +582 -0
  100. oscura/{dsl → api/dsl}/commands.py +43 -76
  101. oscura/{dsl → api/dsl}/interpreter.py +26 -51
  102. oscura/{dsl → api/dsl}/parser.py +107 -77
  103. oscura/{dsl → api/dsl}/repl.py +2 -2
  104. oscura/api/dsl.py +1 -1
  105. oscura/{integrations → api/integrations}/__init__.py +1 -1
  106. oscura/{integrations → api/integrations}/llm.py +201 -102
  107. oscura/api/operators.py +3 -3
  108. oscura/api/optimization.py +144 -30
  109. oscura/api/rest_server.py +921 -0
  110. oscura/api/server/__init__.py +17 -0
  111. oscura/api/server/dashboard.py +850 -0
  112. oscura/api/server/static/README.md +34 -0
  113. oscura/api/server/templates/base.html +181 -0
  114. oscura/api/server/templates/export.html +120 -0
  115. oscura/api/server/templates/home.html +284 -0
  116. oscura/api/server/templates/protocols.html +58 -0
  117. oscura/api/server/templates/reports.html +43 -0
  118. oscura/api/server/templates/session_detail.html +89 -0
  119. oscura/api/server/templates/sessions.html +83 -0
  120. oscura/api/server/templates/waveforms.html +73 -0
  121. oscura/automotive/__init__.py +8 -1
  122. oscura/automotive/can/__init__.py +10 -0
  123. oscura/automotive/can/checksum.py +3 -1
  124. oscura/automotive/can/dbc_generator.py +590 -0
  125. oscura/automotive/can/message_wrapper.py +121 -74
  126. oscura/automotive/can/patterns.py +98 -21
  127. oscura/automotive/can/session.py +292 -56
  128. oscura/automotive/can/state_machine.py +6 -3
  129. oscura/automotive/can/stimulus_response.py +97 -75
  130. oscura/automotive/dbc/__init__.py +10 -2
  131. oscura/automotive/dbc/generator.py +84 -56
  132. oscura/automotive/dbc/parser.py +6 -6
  133. oscura/automotive/dtc/data.json +17 -102
  134. oscura/automotive/dtc/database.py +2 -2
  135. oscura/automotive/flexray/__init__.py +31 -0
  136. oscura/automotive/flexray/analyzer.py +504 -0
  137. oscura/automotive/flexray/crc.py +185 -0
  138. oscura/automotive/flexray/fibex.py +449 -0
  139. oscura/automotive/j1939/__init__.py +45 -8
  140. oscura/automotive/j1939/analyzer.py +605 -0
  141. oscura/automotive/j1939/spns.py +326 -0
  142. oscura/automotive/j1939/transport.py +306 -0
  143. oscura/automotive/lin/__init__.py +47 -0
  144. oscura/automotive/lin/analyzer.py +612 -0
  145. oscura/automotive/loaders/blf.py +13 -2
  146. oscura/automotive/loaders/csv_can.py +143 -72
  147. oscura/automotive/loaders/dispatcher.py +50 -2
  148. oscura/automotive/loaders/mdf.py +86 -45
  149. oscura/automotive/loaders/pcap.py +111 -61
  150. oscura/automotive/uds/__init__.py +4 -0
  151. oscura/automotive/uds/analyzer.py +725 -0
  152. oscura/automotive/uds/decoder.py +140 -58
  153. oscura/automotive/uds/models.py +7 -1
  154. oscura/automotive/visualization.py +1 -1
  155. oscura/cli/analyze.py +348 -0
  156. oscura/cli/batch.py +142 -122
  157. oscura/cli/benchmark.py +275 -0
  158. oscura/cli/characterize.py +137 -82
  159. oscura/cli/compare.py +224 -131
  160. oscura/cli/completion.py +250 -0
  161. oscura/cli/config_cmd.py +361 -0
  162. oscura/cli/decode.py +164 -87
  163. oscura/cli/export.py +286 -0
  164. oscura/cli/main.py +115 -31
  165. oscura/{onboarding → cli/onboarding}/__init__.py +3 -3
  166. oscura/{onboarding → cli/onboarding}/help.py +80 -58
  167. oscura/{onboarding → cli/onboarding}/tutorials.py +97 -72
  168. oscura/{onboarding → cli/onboarding}/wizard.py +55 -36
  169. oscura/cli/progress.py +147 -0
  170. oscura/cli/shell.py +157 -135
  171. oscura/cli/validate_cmd.py +204 -0
  172. oscura/cli/visualize.py +158 -0
  173. oscura/convenience.py +125 -79
  174. oscura/core/__init__.py +4 -2
  175. oscura/core/backend_selector.py +3 -3
  176. oscura/core/cache.py +126 -15
  177. oscura/core/cancellation.py +1 -1
  178. oscura/{config → core/config}/__init__.py +20 -11
  179. oscura/{config → core/config}/defaults.py +1 -1
  180. oscura/{config → core/config}/loader.py +7 -5
  181. oscura/{config → core/config}/memory.py +5 -5
  182. oscura/{config → core/config}/migration.py +1 -1
  183. oscura/{config → core/config}/pipeline.py +99 -23
  184. oscura/{config → core/config}/preferences.py +1 -1
  185. oscura/{config → core/config}/protocol.py +3 -3
  186. oscura/{config → core/config}/schema.py +426 -272
  187. oscura/{config → core/config}/settings.py +1 -1
  188. oscura/{config → core/config}/thresholds.py +195 -153
  189. oscura/core/correlation.py +5 -6
  190. oscura/core/cross_domain.py +0 -2
  191. oscura/core/debug.py +9 -5
  192. oscura/{extensibility → core/extensibility}/docs.py +158 -70
  193. oscura/{extensibility → core/extensibility}/extensions.py +160 -76
  194. oscura/{extensibility → core/extensibility}/logging.py +1 -1
  195. oscura/{extensibility → core/extensibility}/measurements.py +1 -1
  196. oscura/{extensibility → core/extensibility}/plugins.py +1 -1
  197. oscura/{extensibility → core/extensibility}/templates.py +73 -3
  198. oscura/{extensibility → core/extensibility}/validation.py +1 -1
  199. oscura/core/gpu_backend.py +11 -7
  200. oscura/core/log_query.py +101 -11
  201. oscura/core/logging.py +126 -54
  202. oscura/core/logging_advanced.py +5 -5
  203. oscura/core/memory_limits.py +108 -70
  204. oscura/core/memory_monitor.py +2 -2
  205. oscura/core/memory_progress.py +7 -7
  206. oscura/core/memory_warnings.py +1 -1
  207. oscura/core/numba_backend.py +13 -13
  208. oscura/{plugins → core/plugins}/__init__.py +9 -9
  209. oscura/{plugins → core/plugins}/base.py +7 -7
  210. oscura/{plugins → core/plugins}/cli.py +3 -3
  211. oscura/{plugins → core/plugins}/discovery.py +186 -106
  212. oscura/{plugins → core/plugins}/lifecycle.py +1 -1
  213. oscura/{plugins → core/plugins}/manager.py +7 -7
  214. oscura/{plugins → core/plugins}/registry.py +3 -3
  215. oscura/{plugins → core/plugins}/versioning.py +1 -1
  216. oscura/core/progress.py +16 -1
  217. oscura/core/provenance.py +8 -2
  218. oscura/{schemas → core/schemas}/__init__.py +2 -2
  219. oscura/{schemas → core/schemas}/device_mapping.json +2 -8
  220. oscura/{schemas → core/schemas}/packet_format.json +4 -24
  221. oscura/{schemas → core/schemas}/protocol_definition.json +2 -12
  222. oscura/core/types.py +4 -0
  223. oscura/core/uncertainty.py +3 -3
  224. oscura/correlation/__init__.py +52 -0
  225. oscura/correlation/multi_protocol.py +811 -0
  226. oscura/discovery/auto_decoder.py +117 -35
  227. oscura/discovery/comparison.py +191 -86
  228. oscura/discovery/quality_validator.py +155 -68
  229. oscura/discovery/signal_detector.py +196 -79
  230. oscura/export/__init__.py +18 -8
  231. oscura/export/kaitai_struct.py +513 -0
  232. oscura/export/scapy_layer.py +801 -0
  233. oscura/export/wireshark/generator.py +1 -1
  234. oscura/export/wireshark/templates/dissector.lua.j2 +2 -2
  235. oscura/export/wireshark_dissector.py +746 -0
  236. oscura/guidance/wizard.py +207 -111
  237. oscura/hardware/__init__.py +19 -0
  238. oscura/{acquisition → hardware/acquisition}/__init__.py +4 -4
  239. oscura/{acquisition → hardware/acquisition}/file.py +2 -2
  240. oscura/{acquisition → hardware/acquisition}/hardware.py +7 -7
  241. oscura/{acquisition → hardware/acquisition}/saleae.py +15 -12
  242. oscura/{acquisition → hardware/acquisition}/socketcan.py +1 -1
  243. oscura/{acquisition → hardware/acquisition}/streaming.py +2 -2
  244. oscura/{acquisition → hardware/acquisition}/synthetic.py +3 -3
  245. oscura/{acquisition → hardware/acquisition}/visa.py +33 -11
  246. oscura/hardware/firmware/__init__.py +29 -0
  247. oscura/hardware/firmware/pattern_recognition.py +874 -0
  248. oscura/hardware/hal_detector.py +736 -0
  249. oscura/hardware/security/__init__.py +37 -0
  250. oscura/hardware/security/side_channel_detector.py +1126 -0
  251. oscura/inference/__init__.py +4 -0
  252. oscura/inference/active_learning/observation_table.py +4 -1
  253. oscura/inference/alignment.py +216 -123
  254. oscura/inference/bayesian.py +113 -33
  255. oscura/inference/crc_reverse.py +101 -55
  256. oscura/inference/logic.py +6 -2
  257. oscura/inference/message_format.py +342 -183
  258. oscura/inference/protocol.py +95 -44
  259. oscura/inference/protocol_dsl.py +180 -82
  260. oscura/inference/signal_intelligence.py +1439 -706
  261. oscura/inference/spectral.py +99 -57
  262. oscura/inference/state_machine.py +810 -158
  263. oscura/inference/stream.py +270 -110
  264. oscura/iot/__init__.py +34 -0
  265. oscura/iot/coap/__init__.py +32 -0
  266. oscura/iot/coap/analyzer.py +668 -0
  267. oscura/iot/coap/options.py +212 -0
  268. oscura/iot/lorawan/__init__.py +21 -0
  269. oscura/iot/lorawan/crypto.py +206 -0
  270. oscura/iot/lorawan/decoder.py +801 -0
  271. oscura/iot/lorawan/mac_commands.py +341 -0
  272. oscura/iot/mqtt/__init__.py +27 -0
  273. oscura/iot/mqtt/analyzer.py +999 -0
  274. oscura/iot/mqtt/properties.py +315 -0
  275. oscura/iot/zigbee/__init__.py +31 -0
  276. oscura/iot/zigbee/analyzer.py +615 -0
  277. oscura/iot/zigbee/security.py +153 -0
  278. oscura/iot/zigbee/zcl.py +349 -0
  279. oscura/jupyter/display.py +125 -45
  280. oscura/{exploratory → jupyter/exploratory}/__init__.py +8 -8
  281. oscura/{exploratory → jupyter/exploratory}/error_recovery.py +298 -141
  282. oscura/jupyter/exploratory/fuzzy.py +746 -0
  283. oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py +258 -100
  284. oscura/{exploratory → jupyter/exploratory}/legacy.py +464 -242
  285. oscura/{exploratory → jupyter/exploratory}/parse.py +167 -145
  286. oscura/{exploratory → jupyter/exploratory}/recovery.py +119 -87
  287. oscura/jupyter/exploratory/sync.py +612 -0
  288. oscura/{exploratory → jupyter/exploratory}/unknown.py +299 -176
  289. oscura/jupyter/magic.py +4 -4
  290. oscura/{ui → jupyter/ui}/__init__.py +2 -2
  291. oscura/{ui → jupyter/ui}/formatters.py +3 -3
  292. oscura/{ui → jupyter/ui}/progressive_display.py +153 -82
  293. oscura/loaders/__init__.py +183 -67
  294. oscura/loaders/binary.py +88 -1
  295. oscura/loaders/chipwhisperer.py +153 -137
  296. oscura/loaders/configurable.py +208 -86
  297. oscura/loaders/csv_loader.py +458 -215
  298. oscura/loaders/hdf5_loader.py +278 -119
  299. oscura/loaders/lazy.py +87 -54
  300. oscura/loaders/mmap_loader.py +1 -1
  301. oscura/loaders/numpy_loader.py +253 -116
  302. oscura/loaders/pcap.py +226 -151
  303. oscura/loaders/rigol.py +110 -49
  304. oscura/loaders/sigrok.py +201 -78
  305. oscura/loaders/tdms.py +81 -58
  306. oscura/loaders/tektronix.py +291 -174
  307. oscura/loaders/touchstone.py +182 -87
  308. oscura/loaders/tss.py +456 -0
  309. oscura/loaders/vcd.py +215 -117
  310. oscura/loaders/wav.py +155 -68
  311. oscura/reporting/__init__.py +9 -0
  312. oscura/reporting/analyze.py +352 -146
  313. oscura/reporting/argument_preparer.py +69 -14
  314. oscura/reporting/auto_report.py +97 -61
  315. oscura/reporting/batch.py +131 -58
  316. oscura/reporting/chart_selection.py +57 -45
  317. oscura/reporting/comparison.py +63 -17
  318. oscura/reporting/content/executive.py +76 -24
  319. oscura/reporting/core_formats/multi_format.py +11 -8
  320. oscura/reporting/engine.py +312 -158
  321. oscura/reporting/enhanced_reports.py +949 -0
  322. oscura/reporting/export.py +86 -43
  323. oscura/reporting/formatting/numbers.py +69 -42
  324. oscura/reporting/html.py +139 -58
  325. oscura/reporting/index.py +137 -65
  326. oscura/reporting/output.py +158 -67
  327. oscura/reporting/pdf.py +67 -102
  328. oscura/reporting/plots.py +191 -112
  329. oscura/reporting/sections.py +88 -47
  330. oscura/reporting/standards.py +104 -61
  331. oscura/reporting/summary_generator.py +75 -55
  332. oscura/reporting/tables.py +138 -54
  333. oscura/reporting/templates/enhanced/protocol_re.html +525 -0
  334. oscura/sessions/__init__.py +14 -23
  335. oscura/sessions/base.py +3 -3
  336. oscura/sessions/blackbox.py +106 -10
  337. oscura/sessions/generic.py +2 -2
  338. oscura/sessions/legacy.py +783 -0
  339. oscura/side_channel/__init__.py +63 -0
  340. oscura/side_channel/dpa.py +1025 -0
  341. oscura/utils/__init__.py +15 -1
  342. oscura/utils/bitwise.py +118 -0
  343. oscura/{builders → utils/builders}/__init__.py +1 -1
  344. oscura/{comparison → utils/comparison}/__init__.py +6 -6
  345. oscura/{comparison → utils/comparison}/compare.py +202 -101
  346. oscura/{comparison → utils/comparison}/golden.py +83 -63
  347. oscura/{comparison → utils/comparison}/limits.py +313 -89
  348. oscura/{comparison → utils/comparison}/mask.py +151 -45
  349. oscura/{comparison → utils/comparison}/trace_diff.py +1 -1
  350. oscura/{comparison → utils/comparison}/visualization.py +147 -89
  351. oscura/{component → utils/component}/__init__.py +3 -3
  352. oscura/{component → utils/component}/impedance.py +122 -58
  353. oscura/{component → utils/component}/reactive.py +165 -168
  354. oscura/{component → utils/component}/transmission_line.py +3 -3
  355. oscura/{filtering → utils/filtering}/__init__.py +6 -6
  356. oscura/{filtering → utils/filtering}/base.py +1 -1
  357. oscura/{filtering → utils/filtering}/convenience.py +2 -2
  358. oscura/{filtering → utils/filtering}/design.py +169 -93
  359. oscura/{filtering → utils/filtering}/filters.py +2 -2
  360. oscura/{filtering → utils/filtering}/introspection.py +2 -2
  361. oscura/utils/geometry.py +31 -0
  362. oscura/utils/imports.py +184 -0
  363. oscura/utils/lazy.py +1 -1
  364. oscura/{math → utils/math}/__init__.py +2 -2
  365. oscura/{math → utils/math}/arithmetic.py +114 -48
  366. oscura/{math → utils/math}/interpolation.py +139 -106
  367. oscura/utils/memory.py +129 -66
  368. oscura/utils/memory_advanced.py +92 -9
  369. oscura/utils/memory_extensions.py +10 -8
  370. oscura/{optimization → utils/optimization}/__init__.py +1 -1
  371. oscura/{optimization → utils/optimization}/search.py +2 -2
  372. oscura/utils/performance/__init__.py +58 -0
  373. oscura/utils/performance/caching.py +889 -0
  374. oscura/utils/performance/lsh_clustering.py +333 -0
  375. oscura/utils/performance/memory_optimizer.py +699 -0
  376. oscura/utils/performance/optimizations.py +675 -0
  377. oscura/utils/performance/parallel.py +654 -0
  378. oscura/utils/performance/profiling.py +661 -0
  379. oscura/{pipeline → utils/pipeline}/base.py +1 -1
  380. oscura/{pipeline → utils/pipeline}/composition.py +1 -1
  381. oscura/{pipeline → utils/pipeline}/parallel.py +3 -2
  382. oscura/{pipeline → utils/pipeline}/pipeline.py +1 -1
  383. oscura/{pipeline → utils/pipeline}/reverse_engineering.py +412 -221
  384. oscura/{search → utils/search}/__init__.py +3 -3
  385. oscura/{search → utils/search}/anomaly.py +188 -58
  386. oscura/utils/search/context.py +294 -0
  387. oscura/{search → utils/search}/pattern.py +138 -10
  388. oscura/utils/serial.py +51 -0
  389. oscura/utils/storage/__init__.py +61 -0
  390. oscura/utils/storage/database.py +1166 -0
  391. oscura/{streaming → utils/streaming}/chunked.py +302 -143
  392. oscura/{streaming → utils/streaming}/progressive.py +1 -1
  393. oscura/{streaming → utils/streaming}/realtime.py +3 -2
  394. oscura/{triggering → utils/triggering}/__init__.py +6 -6
  395. oscura/{triggering → utils/triggering}/base.py +6 -6
  396. oscura/{triggering → utils/triggering}/edge.py +2 -2
  397. oscura/{triggering → utils/triggering}/pattern.py +2 -2
  398. oscura/{triggering → utils/triggering}/pulse.py +115 -74
  399. oscura/{triggering → utils/triggering}/window.py +2 -2
  400. oscura/utils/validation.py +32 -0
  401. oscura/validation/__init__.py +121 -0
  402. oscura/{compliance → validation/compliance}/__init__.py +5 -5
  403. oscura/{compliance → validation/compliance}/advanced.py +5 -5
  404. oscura/{compliance → validation/compliance}/masks.py +1 -1
  405. oscura/{compliance → validation/compliance}/reporting.py +127 -53
  406. oscura/{compliance → validation/compliance}/testing.py +114 -52
  407. oscura/validation/compliance_tests.py +915 -0
  408. oscura/validation/fuzzer.py +990 -0
  409. oscura/validation/grammar_tests.py +596 -0
  410. oscura/validation/grammar_validator.py +904 -0
  411. oscura/validation/hil_testing.py +977 -0
  412. oscura/{quality → validation/quality}/__init__.py +4 -4
  413. oscura/{quality → validation/quality}/ensemble.py +251 -171
  414. oscura/{quality → validation/quality}/explainer.py +3 -3
  415. oscura/{quality → validation/quality}/scoring.py +1 -1
  416. oscura/{quality → validation/quality}/warnings.py +4 -4
  417. oscura/validation/regression_suite.py +808 -0
  418. oscura/validation/replay.py +788 -0
  419. oscura/{testing → validation/testing}/__init__.py +2 -2
  420. oscura/{testing → validation/testing}/synthetic.py +5 -5
  421. oscura/visualization/__init__.py +9 -0
  422. oscura/visualization/accessibility.py +1 -1
  423. oscura/visualization/annotations.py +64 -67
  424. oscura/visualization/colors.py +7 -7
  425. oscura/visualization/digital.py +180 -81
  426. oscura/visualization/eye.py +236 -85
  427. oscura/visualization/interactive.py +320 -143
  428. oscura/visualization/jitter.py +587 -247
  429. oscura/visualization/layout.py +169 -134
  430. oscura/visualization/optimization.py +103 -52
  431. oscura/visualization/palettes.py +1 -1
  432. oscura/visualization/power.py +427 -211
  433. oscura/visualization/power_extended.py +626 -297
  434. oscura/visualization/presets.py +2 -0
  435. oscura/visualization/protocols.py +495 -181
  436. oscura/visualization/render.py +79 -63
  437. oscura/visualization/reverse_engineering.py +171 -124
  438. oscura/visualization/signal_integrity.py +460 -279
  439. oscura/visualization/specialized.py +190 -100
  440. oscura/visualization/spectral.py +670 -255
  441. oscura/visualization/thumbnails.py +166 -137
  442. oscura/visualization/waveform.py +150 -63
  443. oscura/workflows/__init__.py +3 -0
  444. oscura/{batch → workflows/batch}/__init__.py +5 -5
  445. oscura/{batch → workflows/batch}/advanced.py +150 -75
  446. oscura/workflows/batch/aggregate.py +531 -0
  447. oscura/workflows/batch/analyze.py +236 -0
  448. oscura/{batch → workflows/batch}/logging.py +2 -2
  449. oscura/{batch → workflows/batch}/metrics.py +1 -1
  450. oscura/workflows/complete_re.py +1144 -0
  451. oscura/workflows/compliance.py +44 -54
  452. oscura/workflows/digital.py +197 -51
  453. oscura/workflows/legacy/__init__.py +12 -0
  454. oscura/{workflow → workflows/legacy}/dag.py +4 -1
  455. oscura/workflows/multi_trace.py +9 -9
  456. oscura/workflows/power.py +42 -62
  457. oscura/workflows/protocol.py +82 -49
  458. oscura/workflows/reverse_engineering.py +351 -150
  459. oscura/workflows/signal_integrity.py +157 -82
  460. oscura-0.7.0.dist-info/METADATA +661 -0
  461. oscura-0.7.0.dist-info/RECORD +591 -0
  462. oscura/batch/aggregate.py +0 -300
  463. oscura/batch/analyze.py +0 -139
  464. oscura/dsl/__init__.py +0 -73
  465. oscura/exceptions.py +0 -59
  466. oscura/exploratory/fuzzy.py +0 -513
  467. oscura/exploratory/sync.py +0 -384
  468. oscura/exporters/__init__.py +0 -94
  469. oscura/exporters/csv.py +0 -303
  470. oscura/exporters/exporters.py +0 -44
  471. oscura/exporters/hdf5.py +0 -217
  472. oscura/exporters/html_export.py +0 -701
  473. oscura/exporters/json_export.py +0 -291
  474. oscura/exporters/markdown_export.py +0 -367
  475. oscura/exporters/matlab_export.py +0 -354
  476. oscura/exporters/npz_export.py +0 -219
  477. oscura/exporters/spice_export.py +0 -210
  478. oscura/search/context.py +0 -149
  479. oscura/session/__init__.py +0 -34
  480. oscura/session/annotations.py +0 -289
  481. oscura/session/history.py +0 -313
  482. oscura/session/session.py +0 -520
  483. oscura/workflow/__init__.py +0 -13
  484. oscura-0.5.1.dist-info/METADATA +0 -583
  485. oscura-0.5.1.dist-info/RECORD +0 -481
  486. /oscura/core/{config.py → config/legacy.py} +0 -0
  487. /oscura/{extensibility → core/extensibility}/__init__.py +0 -0
  488. /oscura/{extensibility → core/extensibility}/registry.py +0 -0
  489. /oscura/{plugins → core/plugins}/isolation.py +0 -0
  490. /oscura/{schemas → core/schemas}/bus_configuration.json +0 -0
  491. /oscura/{builders → utils/builders}/signal_builder.py +0 -0
  492. /oscura/{optimization → utils/optimization}/parallel.py +0 -0
  493. /oscura/{pipeline → utils/pipeline}/__init__.py +0 -0
  494. /oscura/{streaming → utils/streaming}/__init__.py +0 -0
  495. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/WHEEL +0 -0
  496. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/entry_points.txt +0 -0
  497. {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,908 @@
1
+ """Anomaly detection system for identifying unusual patterns in protocol traffic.
2
+
3
+ This module provides multi-method anomaly detection to identify unusual patterns
4
+ in protocol traffic using statistical and machine learning methods.
5
+
6
+ Key capabilities:
7
+ - Statistical anomaly detection (Z-score, IQR, modified Z-score)
8
+ - Time-series anomaly detection (rate analysis, timing analysis)
9
+ - ML-based detection (Isolation Forest, One-Class SVM) - optional with scikit-learn
10
+ - Detect: unexpected message rates, unusual field values, timing anomalies, sequence violations
11
+ - Return anomaly scores and explanations
12
+ - Support online detection (streaming data)
13
+ - Export anomaly reports with context
14
+
15
+ Detection methods:
16
+ - Z-score: Standard deviation-based outlier detection
17
+ - IQR: Interquartile range-based outlier detection
18
+ - Modified Z-score: Median absolute deviation-based (robust to outliers)
19
+ - Isolation Forest: Tree-based anomaly detection (requires sklearn)
20
+ - One-Class SVM: Support vector-based anomaly detection (requires sklearn)
21
+
22
+ Typical workflow:
23
+ 1. Train detector on normal baseline data (optional for statistical methods)
24
+ 2. Detect anomalies in new data points (batch or streaming)
25
+ 3. Export anomaly reports with explanations and context
26
+
27
+ Example:
28
+ >>> from oscura.analyzers.patterns.anomaly_detection import AnomalyDetector, AnomalyDetectionConfig
29
+ >>> config = AnomalyDetectionConfig(methods=["zscore", "iqr"])
30
+ >>> detector = AnomalyDetector(config)
31
+ >>> # Detect field value anomalies
32
+ >>> anomalies = detector.detect_field_value_anomaly(
33
+ ... field_values=[1.0, 1.1, 0.9, 1.2, 10.0, 1.0],
34
+ ... field_name="voltage"
35
+ ... )
36
+ >>> for anomaly in anomalies:
37
+ ... print(f"{anomaly.anomaly_type}: {anomaly.explanation}")
38
+ value: voltage value 10.00 deviates significantly from expected 2.53
39
+
40
+ References:
41
+ Isolation Forest: Liu et al. (2008) "Isolation Forest"
42
+ One-Class SVM: Schölkopf et al. (2001) "Estimating the Support of a High-Dimensional Distribution"
43
+ """
44
+
45
+ from __future__ import annotations
46
+
47
+ import json
48
+ import logging
49
+ from dataclasses import dataclass, field
50
+ from pathlib import Path
51
+ from typing import TYPE_CHECKING, Any, ClassVar
52
+
53
+ import numpy as np
54
+
55
+ if TYPE_CHECKING:
56
+ from numpy.typing import NDArray
57
+
58
+ logger = logging.getLogger(__name__)
59
+
60
+
61
+ @dataclass
62
+ class Anomaly:
63
+ """Detected anomaly.
64
+
65
+ Attributes:
66
+ timestamp: Time of anomaly occurrence
67
+ anomaly_type: Type of anomaly detected (rate, value, timing, sequence, protocol)
68
+ score: Anomaly score (0.0-1.0, higher = more anomalous)
69
+ message_index: Index of message containing anomaly (None if not message-specific)
70
+ field_name: Name of field with anomaly (None if not field-specific)
71
+ expected_value: Expected value based on baseline/model
72
+ actual_value: Actual value observed
73
+ explanation: Human-readable explanation of anomaly
74
+ context: Additional context information (rates, thresholds, etc.)
75
+
76
+ Example:
77
+ >>> anomaly = Anomaly(
78
+ ... timestamp=1234.5,
79
+ ... anomaly_type="value",
80
+ ... score=0.95,
81
+ ... message_index=42,
82
+ ... field_name="voltage",
83
+ ... expected_value=3.3,
84
+ ... actual_value=15.0,
85
+ ... explanation="Voltage spike detected"
86
+ ... )
87
+ """
88
+
89
+ timestamp: float
90
+ anomaly_type: str # "rate", "value", "timing", "sequence", "protocol"
91
+ score: float # 0.0-1.0, higher = more anomalous
92
+ message_index: int | None = None
93
+ field_name: str | None = None
94
+ expected_value: Any = None
95
+ actual_value: Any = None
96
+ explanation: str = ""
97
+ context: dict[str, Any] = field(default_factory=dict)
98
+
99
+
100
+ @dataclass
101
+ class AnomalyDetectionConfig:
102
+ """Anomaly detection configuration.
103
+
104
+ Attributes:
105
+ methods: Detection methods to use (zscore, iqr, modified_zscore, isolation_forest, one_class_svm)
106
+ zscore_threshold: Standard deviations for Z-score method (default: 3.0)
107
+ iqr_multiplier: Multiplier for IQR method (default: 1.5)
108
+ contamination: Expected outlier fraction for ML methods (default: 0.1)
109
+ window_size: Window size for streaming detection (default: 100)
110
+
111
+ Example:
112
+ >>> config = AnomalyDetectionConfig(
113
+ ... methods=["zscore", "iqr"],
114
+ ... zscore_threshold=2.5,
115
+ ... window_size=200
116
+ ... )
117
+ """
118
+
119
+ methods: list[str] = field(default_factory=lambda: ["zscore", "isolation_forest"])
120
+ zscore_threshold: float = 3.0 # Standard deviations
121
+ iqr_multiplier: float = 1.5
122
+ contamination: float = 0.1 # Expected outlier fraction
123
+ window_size: int = 100 # For streaming detection
124
+
125
+
126
+ class AnomalyDetector:
127
+ """Multi-method anomaly detection system.
128
+
129
+ This class provides various anomaly detection methods for protocol traffic analysis.
130
+ It supports both statistical methods (Z-score, IQR) and ML-based methods
131
+ (Isolation Forest, One-Class SVM - if scikit-learn is available).
132
+
133
+ Attributes:
134
+ config: Detection configuration
135
+ models: Trained ML models (if applicable)
136
+ baselines: Baseline statistics for comparison
137
+ anomalies: Detected anomalies
138
+
139
+ Detection methods:
140
+ Statistical: zscore, iqr, modified_zscore
141
+ ML-based: isolation_forest, one_class_svm (requires scikit-learn)
142
+ Time-series: rate analysis, timing analysis
143
+
144
+ Example:
145
+ >>> detector = AnomalyDetector()
146
+ >>> # Detect message rate anomalies
147
+ >>> timestamps = [0.0, 0.1, 0.2, 0.3, 5.0, 5.1] # Gap at 0.3-5.0
148
+ >>> anomalies = detector.detect_message_rate_anomaly(timestamps)
149
+ >>> # Detect field value anomalies
150
+ >>> values = [1.0, 1.1, 0.9, 1.2, 10.0, 1.0] # Spike at 10.0
151
+ >>> anomalies = detector.detect_field_value_anomaly(values, "voltage")
152
+ """
153
+
154
+ # Detection methods
155
+ STATISTICAL_METHODS: ClassVar[list[str]] = ["zscore", "iqr", "modified_zscore"]
156
+ ML_METHODS: ClassVar[list[str]] = ["isolation_forest", "one_class_svm", "autoencoder"]
157
+ TIMESERIES_METHODS: ClassVar[list[str]] = ["arima", "seasonal_decomposition"]
158
+
159
+ def __init__(self, config: AnomalyDetectionConfig | None = None) -> None:
160
+ """Initialize anomaly detector.
161
+
162
+ Args:
163
+ config: Anomaly detection configuration. If None, uses default config.
164
+
165
+ Example:
166
+ >>> detector = AnomalyDetector()
167
+ >>> config = AnomalyDetectionConfig(methods=["zscore"], zscore_threshold=2.5)
168
+ >>> detector_custom = AnomalyDetector(config)
169
+ """
170
+ self.config = config or AnomalyDetectionConfig()
171
+ self.models: dict[str, Any] = {}
172
+ self.baselines: dict[str, Any] = {}
173
+ self.anomalies: list[Anomaly] = []
174
+
175
+ def train(self, normal_data: list[dict[str, Any]], features: list[str]) -> None:
176
+ """Train anomaly detection models on normal baseline data.
177
+
178
+ Args:
179
+ normal_data: List of normal data points (dicts with feature values)
180
+ features: List of feature names to use for training
181
+
182
+ Raises:
183
+ ImportError: If ML methods requested but scikit-learn not available
184
+ ValueError: If insufficient data for training
185
+
186
+ Example:
187
+ >>> normal_data = [
188
+ ... {"voltage": 3.3, "current": 0.5},
189
+ ... {"voltage": 3.2, "current": 0.6},
190
+ ... ]
191
+ >>> detector.train(normal_data, features=["voltage", "current"])
192
+ """
193
+ if len(normal_data) < 10:
194
+ raise ValueError("Need at least 10 samples for training")
195
+
196
+ # Extract features
197
+ X = np.array([[d[f] for f in features] for d in normal_data])
198
+
199
+ # Calculate baseline statistics
200
+ for i, feature in enumerate(features):
201
+ self.baselines[feature] = {
202
+ "mean": float(np.mean(X[:, i])),
203
+ "std": float(np.std(X[:, i])),
204
+ "median": float(np.median(X[:, i])),
205
+ "q1": float(np.percentile(X[:, i], 25)),
206
+ "q3": float(np.percentile(X[:, i], 75)),
207
+ }
208
+
209
+ # Train ML models if requested
210
+ for method in self.config.methods:
211
+ if method == "isolation_forest":
212
+ self._train_isolation_forest(X)
213
+ elif method == "one_class_svm":
214
+ self._train_one_class_svm(X)
215
+
216
+ def detect(self, data: dict[str, Any], timestamp: float = 0.0) -> list[Anomaly]:
217
+ """Detect anomalies in new data point.
218
+
219
+ Args:
220
+ data: Data point to analyze (dict with feature values)
221
+ timestamp: Timestamp of data point
222
+
223
+ Returns:
224
+ List of detected anomalies
225
+
226
+ Example:
227
+ >>> anomalies = detector.detect(
228
+ ... {"voltage": 10.0, "current": 0.5},
229
+ ... timestamp=123.45
230
+ ... )
231
+ """
232
+ anomalies = []
233
+
234
+ # Check each feature against baselines
235
+ for feature, value in data.items():
236
+ if feature in self.baselines:
237
+ baseline = self.baselines[feature]
238
+
239
+ # Z-score check
240
+ if "zscore" in self.config.methods:
241
+ z_score = abs(value - baseline["mean"]) / (baseline["std"] + 1e-10)
242
+ if z_score > self.config.zscore_threshold:
243
+ anomalies.append(
244
+ Anomaly(
245
+ timestamp=timestamp,
246
+ anomaly_type="value",
247
+ score=min(z_score / 5.0, 1.0),
248
+ field_name=feature,
249
+ expected_value=baseline["mean"],
250
+ actual_value=value,
251
+ explanation=f"{feature} value {value:.2f} deviates {z_score:.1f} std devs from expected {baseline['mean']:.2f}",
252
+ context={"z_score": z_score, "method": "zscore"},
253
+ )
254
+ )
255
+
256
+ # IQR check
257
+ if "iqr" in self.config.methods:
258
+ iqr = baseline["q3"] - baseline["q1"]
259
+ lower_bound = baseline["q1"] - self.config.iqr_multiplier * iqr
260
+ upper_bound = baseline["q3"] + self.config.iqr_multiplier * iqr
261
+
262
+ if value < lower_bound or value > upper_bound:
263
+ anomalies.append(
264
+ Anomaly(
265
+ timestamp=timestamp,
266
+ anomaly_type="value",
267
+ score=0.8,
268
+ field_name=feature,
269
+ expected_value=baseline["median"],
270
+ actual_value=value,
271
+ explanation=f"{feature} value {value:.2f} outside IQR bounds [{lower_bound:.2f}, {upper_bound:.2f}]",
272
+ context={
273
+ "iqr": iqr,
274
+ "bounds": (lower_bound, upper_bound),
275
+ "method": "iqr",
276
+ },
277
+ )
278
+ )
279
+
280
+ self.anomalies.extend(anomalies)
281
+ return anomalies
282
+
283
+ def detect_batch(
284
+ self, data_points: list[dict[str, Any]], timestamps: list[float]
285
+ ) -> list[Anomaly]:
286
+ """Detect anomalies in batch of data.
287
+
288
+ Args:
289
+ data_points: List of data points to analyze
290
+ timestamps: Corresponding timestamps for each data point
291
+
292
+ Returns:
293
+ List of all detected anomalies
294
+
295
+ Example:
296
+ >>> data = [
297
+ ... {"voltage": 3.3},
298
+ ... {"voltage": 10.0}, # Anomaly
299
+ ... ]
300
+ >>> timestamps = [0.0, 1.0]
301
+ >>> anomalies = detector.detect_batch(data, timestamps)
302
+ """
303
+ all_anomalies = []
304
+ for data, timestamp in zip(data_points, timestamps, strict=True):
305
+ anomalies = self.detect(data, timestamp)
306
+ all_anomalies.extend(anomalies)
307
+ return all_anomalies
308
+
309
+ def detect_message_rate_anomaly(
310
+ self, timestamps: list[float], window_size: int = 100
311
+ ) -> list[Anomaly]:
312
+ """Detect anomalous message rates (bursts, gaps).
313
+
314
+ Uses sliding window to calculate message rate, then detects outliers
315
+ in rate distribution using Z-score method.
316
+
317
+ Args:
318
+ timestamps: List of message timestamps
319
+ window_size: Size of sliding window for rate calculation
320
+
321
+ Returns:
322
+ List of detected rate anomalies
323
+
324
+ Example:
325
+ >>> timestamps = [0.0, 0.1, 0.2, 0.3, 5.0, 5.1] # Gap at 0.3-5.0
326
+ >>> anomalies = detector.detect_message_rate_anomaly(timestamps)
327
+ >>> for a in anomalies:
328
+ ... print(a.explanation)
329
+ Message gap detected: 2.1 msg/s vs expected 10.0 msg/s
330
+ """
331
+ if len(timestamps) < window_size:
332
+ return []
333
+
334
+ # Calculate message rates in windows
335
+ rates = []
336
+ window_timestamps = []
337
+
338
+ for i in range(len(timestamps) - window_size):
339
+ window = timestamps[i : i + window_size]
340
+ time_span = window[-1] - window[0]
341
+ rate = window_size / time_span if time_span > 0 else 0
342
+ rates.append(rate)
343
+ window_timestamps.append(window[window_size // 2])
344
+
345
+ # Detect anomalies in rates using dual-threshold approach
346
+ rate_array = np.array(rates)
347
+ mean_rate = np.mean(rate_array)
348
+ std_rate = np.std(rate_array)
349
+ median_rate = np.median(rate_array)
350
+
351
+ # For extremely constant rates, avoid false positives from floating point errors
352
+ if std_rate < mean_rate * 0.0001: # Coefficient of variation < 0.01%
353
+ return []
354
+
355
+ # Use dual-threshold approach for rate anomaly detection:
356
+ # 1. Statistical outlier detection (Z-score)
357
+ # 2. Rate deviation threshold (>30% change from median)
358
+ statistical_outliers = self._zscore_detection(rate_array, self.config.zscore_threshold)
359
+
360
+ # Rate-based threshold: detect significant rate changes (>30% deviation from median)
361
+ rate_deviations = np.abs(rate_array - median_rate) / median_rate
362
+ significant_deviations = rate_deviations > 0.3 # 30% threshold
363
+
364
+ # Combine: anomaly if either statistical outlier OR significant deviation
365
+ outliers = statistical_outliers | significant_deviations
366
+
367
+ # Create Anomaly objects
368
+ anomalies = []
369
+
370
+ for idx, is_outlier in enumerate(outliers):
371
+ if is_outlier:
372
+ if rates[idx] > mean_rate:
373
+ explanation = f"Message burst detected: {rates[idx]:.1f} msg/s vs expected {mean_rate:.1f} msg/s"
374
+ else:
375
+ explanation = f"Message gap detected: {rates[idx]:.1f} msg/s vs expected {mean_rate:.1f} msg/s"
376
+
377
+ anomalies.append(
378
+ Anomaly(
379
+ timestamp=window_timestamps[idx],
380
+ anomaly_type="rate",
381
+ score=min(abs(rates[idx] - mean_rate) / mean_rate, 1.0),
382
+ explanation=explanation,
383
+ context={"rate": rates[idx], "expected_rate": mean_rate},
384
+ )
385
+ )
386
+
387
+ self.anomalies.extend(anomalies)
388
+ return anomalies
389
+
390
+ def detect_field_value_anomaly(
391
+ self, field_values: list[float], field_name: str, method: str = "modified_zscore"
392
+ ) -> list[Anomaly]:
393
+ """Detect unusual field values using statistical methods.
394
+
395
+ Args:
396
+ field_values: List of field values to analyze
397
+ field_name: Name of the field being analyzed
398
+ method: Detection method to use (zscore, iqr, modified_zscore)
399
+
400
+ Returns:
401
+ List of detected value anomalies
402
+
403
+ Raises:
404
+ ValueError: If method is unknown
405
+
406
+ Example:
407
+ >>> values = [1.0, 1.1, 0.9, 1.2, 10.0, 1.0]
408
+ >>> anomalies = detector.detect_field_value_anomaly(values, "voltage")
409
+ >>> print(anomalies[0].explanation)
410
+ voltage value 10.00 deviates significantly from expected 2.53
411
+ """
412
+ values = np.array(field_values)
413
+
414
+ # For very small datasets (n=2), use simple ratio-based detection
415
+ if len(values) == 2:
416
+ # Detect if one value is >5x or <0.2x the other
417
+ ratio = values.max() / (values.min() + 1e-10)
418
+ if ratio > 5.0:
419
+ # The larger value is anomalous
420
+ outliers = values == values.max()
421
+ else:
422
+ outliers = np.zeros(len(values), dtype=bool)
423
+ elif method == "zscore":
424
+ outliers = self._zscore_detection(values, self.config.zscore_threshold)
425
+ elif method == "iqr":
426
+ outliers = self._iqr_detection(values, self.config.iqr_multiplier)
427
+ elif method == "modified_zscore":
428
+ outliers = self._modified_zscore_detection(values, self.config.zscore_threshold)
429
+ else:
430
+ raise ValueError(f"Unknown method: {method}")
431
+
432
+ # Create Anomaly objects
433
+ anomalies = []
434
+ mean_val = np.mean(values)
435
+
436
+ for idx, is_outlier in enumerate(outliers):
437
+ if is_outlier:
438
+ score = abs(values[idx] - mean_val) / (np.std(values) + 1e-10)
439
+ score = min(score / 5.0, 1.0) # Normalize to 0-1
440
+
441
+ anomalies.append(
442
+ Anomaly(
443
+ timestamp=float(idx), # Index as timestamp
444
+ anomaly_type="value",
445
+ score=score,
446
+ message_index=idx,
447
+ field_name=field_name,
448
+ expected_value=mean_val,
449
+ actual_value=values[idx],
450
+ explanation=f"{field_name} value {values[idx]:.2f} deviates significantly from expected {mean_val:.2f}",
451
+ context={"method": method},
452
+ )
453
+ )
454
+
455
+ self.anomalies.extend(anomalies)
456
+ return anomalies
457
+
458
+ def detect_timing_anomaly(
459
+ self,
460
+ inter_arrival_times: list[float],
461
+ expected_period: float | None = None,
462
+ ) -> list[Anomaly]:
463
+ """Detect timing anomalies (jitter, drift, unexpected delays).
464
+
465
+ Args:
466
+ inter_arrival_times: List of inter-arrival times between messages
467
+ expected_period: Expected period (None = calculate from data)
468
+
469
+ Returns:
470
+ List of detected timing anomalies
471
+
472
+ Example:
473
+ >>> inter_arrival = [0.1, 0.1, 0.1, 1.0, 0.1] # Delay at index 3
474
+ >>> anomalies = detector.detect_timing_anomaly(inter_arrival)
475
+ >>> print(anomalies[0].explanation)
476
+ Timing anomaly: 1.00s vs expected 0.28s
477
+ """
478
+ if not inter_arrival_times:
479
+ return []
480
+
481
+ times = np.array(inter_arrival_times)
482
+
483
+ # Use expected period or calculate from data
484
+ if expected_period is None:
485
+ expected_period = float(np.median(times))
486
+
487
+ # For very small datasets (n=2), use ratio-based detection
488
+ if len(times) == 2:
489
+ # Detect if one value is >5x or <0.2x the other
490
+ ratio = times.max() / (times.min() + 1e-10)
491
+ if ratio > 5.0:
492
+ # The larger value is anomalous (unexpected delay)
493
+ outliers = times == times.max()
494
+ else:
495
+ outliers = np.zeros(len(times), dtype=bool)
496
+ else:
497
+ # Use statistical detection when no expected period provided
498
+ outliers = self._zscore_detection(times, self.config.zscore_threshold)
499
+ else:
500
+ # When expected period is provided, detect deviations from it directly
501
+ # Use threshold-based detection (e.g., >50% deviation from expected)
502
+ deviations = np.abs(times - expected_period) / expected_period
503
+ outliers = deviations > 0.5 # 50% deviation threshold
504
+
505
+ # Create Anomaly objects
506
+ anomalies = []
507
+ for idx, is_outlier in enumerate(outliers):
508
+ if is_outlier:
509
+ anomalies.append(
510
+ Anomaly(
511
+ timestamp=float(idx),
512
+ anomaly_type="timing",
513
+ score=min(abs(times[idx] - expected_period) / expected_period, 1.0),
514
+ message_index=idx,
515
+ expected_value=expected_period,
516
+ actual_value=times[idx],
517
+ explanation=f"Timing anomaly: {times[idx]:.2f}s vs expected {expected_period:.2f}s",
518
+ context={"expected_period": expected_period},
519
+ )
520
+ )
521
+
522
+ self.anomalies.extend(anomalies)
523
+ return anomalies
524
+
525
+ def detect_sequence_anomaly(
526
+ self, sequences: list[list[int]], trained_model: Any = None
527
+ ) -> list[Anomaly]:
528
+ """Detect unusual byte/message sequences.
529
+
530
+ Analyzes sequences for unusual patterns. If trained_model is provided,
531
+ uses it for prediction. Otherwise uses statistical analysis.
532
+
533
+ Args:
534
+ sequences: List of byte/message sequences
535
+ trained_model: Trained sequence model (optional)
536
+
537
+ Returns:
538
+ List of detected sequence anomalies
539
+
540
+ Example:
541
+ >>> sequences = [[0x01, 0x02, 0x03], [0x01, 0x02, 0xFF]] # Last byte unusual
542
+ >>> anomalies = detector.detect_sequence_anomaly(sequences)
543
+ """
544
+ anomalies = []
545
+
546
+ # Dual-threshold approach: detect unusual sequence lengths
547
+ lengths = np.array([len(seq) for seq in sequences])
548
+ median_length = np.median(lengths)
549
+ mean_length = np.mean(lengths)
550
+
551
+ # Statistical outlier detection
552
+ statistical_outliers = self._zscore_detection(lengths, self.config.zscore_threshold)
553
+
554
+ # Length-based threshold: detect sequences >2x median length or <0.5x median length
555
+ length_ratios = lengths / median_length
556
+ significant_length_deviations = (length_ratios > 2.0) | (length_ratios < 0.5)
557
+
558
+ # Combine: anomaly if either statistical outlier OR significant length deviation
559
+ outliers = statistical_outliers | significant_length_deviations
560
+
561
+ for idx, is_outlier in enumerate(outliers):
562
+ if is_outlier:
563
+ anomalies.append(
564
+ Anomaly(
565
+ timestamp=float(idx),
566
+ anomaly_type="sequence",
567
+ score=0.7,
568
+ message_index=idx,
569
+ expected_value=mean_length,
570
+ actual_value=lengths[idx],
571
+ explanation=f"Unusual sequence length: {lengths[idx]} bytes vs expected {mean_length:.0f} bytes",
572
+ context={"sequence_length": lengths[idx]},
573
+ )
574
+ )
575
+
576
+ self.anomalies.extend(anomalies)
577
+ return anomalies
578
+
579
+ def _zscore_detection(
580
+ self, values: NDArray[np.floating[Any]], threshold: float = 3.0
581
+ ) -> NDArray[np.bool_]:
582
+ """Z-score based outlier detection.
583
+
584
+ Z-score measures how many standard deviations a value is from the mean.
585
+ Outliers are typically defined as |Z-score| > threshold (commonly 3.0).
586
+
587
+ Args:
588
+ values: Array of values to analyze
589
+ threshold: Z-score threshold for outliers (default: 3.0)
590
+
591
+ Returns:
592
+ Boolean array indicating outliers (True = outlier)
593
+
594
+ Example:
595
+ >>> values = np.array([1.0, 1.1, 0.9, 10.0])
596
+ >>> outliers = detector._zscore_detection(values, threshold=2.0)
597
+ >>> print(outliers)
598
+ [False False False True]
599
+ """
600
+ if len(values) < 2:
601
+ return np.zeros(len(values), dtype=bool)
602
+
603
+ mean = np.mean(values)
604
+ std = np.std(values)
605
+
606
+ if std == 0:
607
+ return np.zeros(len(values), dtype=bool)
608
+
609
+ z_scores = np.abs((values - mean) / std)
610
+ outliers: NDArray[np.bool_] = z_scores > threshold
611
+
612
+ return outliers
613
+
614
+ def _iqr_detection(
615
+ self, values: NDArray[np.floating[Any]], multiplier: float = 1.5
616
+ ) -> NDArray[np.bool_]:
617
+ """Interquartile range (IQR) based outlier detection.
618
+
619
+ IQR is the range between 25th and 75th percentiles. Outliers are
620
+ values outside [Q1 - multiplier*IQR, Q3 + multiplier*IQR].
621
+
622
+ Args:
623
+ values: Array of values to analyze
624
+ multiplier: IQR multiplier for bounds (default: 1.5)
625
+
626
+ Returns:
627
+ Boolean array indicating outliers (True = outlier)
628
+
629
+ Example:
630
+ >>> values = np.array([1.0, 1.1, 0.9, 10.0])
631
+ >>> outliers = detector._iqr_detection(values)
632
+ >>> print(outliers)
633
+ [False False False True]
634
+ """
635
+ if len(values) < 4:
636
+ return np.zeros(len(values), dtype=bool)
637
+
638
+ q1 = np.percentile(values, 25)
639
+ q3 = np.percentile(values, 75)
640
+ iqr = q3 - q1
641
+
642
+ lower_bound = q1 - multiplier * iqr
643
+ upper_bound = q3 + multiplier * iqr
644
+
645
+ outliers: NDArray[np.bool_] = (values < lower_bound) | (values > upper_bound)
646
+
647
+ return outliers
648
+
649
+ def _modified_zscore_detection(
650
+ self, values: NDArray[np.floating[Any]], threshold: float = 2.5
651
+ ) -> NDArray[np.bool_]:
652
+ """Modified Z-score based outlier detection using median absolute deviation.
653
+
654
+ More robust to outliers than standard Z-score. Uses median instead of mean
655
+ and MAD instead of standard deviation.
656
+
657
+ Args:
658
+ values: Array of values to analyze
659
+ threshold: Modified Z-score threshold (default: 2.5, lower than standard 3.5)
660
+
661
+ Returns:
662
+ Boolean array indicating outliers (True = outlier)
663
+
664
+ Example:
665
+ >>> values = np.array([1.0, 1.1, 0.9, 10.0])
666
+ >>> outliers = detector._modified_zscore_detection(values)
667
+ >>> print(outliers)
668
+ [False False False True]
669
+ """
670
+ if len(values) < 2:
671
+ return np.zeros(len(values), dtype=bool)
672
+
673
+ median = np.median(values)
674
+ mad = np.median(np.abs(values - median))
675
+
676
+ if mad == 0:
677
+ # When MAD is 0, use a different approach
678
+ # Check if any values differ from the median
679
+ result_outliers: NDArray[np.bool_] = values != median
680
+ return result_outliers
681
+
682
+ # Modified Z-score = 0.6745 * (x - median) / MAD
683
+ modified_z_scores = 0.6745 * np.abs(values - median) / mad
684
+ outliers: NDArray[np.bool_] = modified_z_scores > threshold
685
+
686
+ return outliers
687
+
688
+ def _isolation_forest_detection(
689
+ self, X: NDArray[np.floating[Any]], contamination: float = 0.1
690
+ ) -> NDArray[np.bool_]:
691
+ """Isolation Forest based anomaly detection.
692
+
693
+ Isolation Forest isolates anomalies by randomly selecting features
694
+ and split values. Anomalies require fewer splits to isolate.
695
+
696
+ Args:
697
+ X: Feature matrix (n_samples, n_features)
698
+ contamination: Expected outlier fraction (0.0-0.5)
699
+
700
+ Returns:
701
+ Boolean array indicating outliers (True = outlier)
702
+
703
+ Raises:
704
+ ImportError: If scikit-learn is not installed
705
+
706
+ Example:
707
+ >>> X = np.array([[1.0, 2.0], [1.1, 2.1], [10.0, 20.0]])
708
+ >>> outliers = detector._isolation_forest_detection(X)
709
+ """
710
+ try:
711
+ from sklearn.ensemble import IsolationForest
712
+ except ImportError as e:
713
+ raise ImportError(
714
+ "scikit-learn is required for Isolation Forest. "
715
+ "Install with: pip install scikit-learn"
716
+ ) from e
717
+
718
+ if len(X.shape) == 1:
719
+ X = X.reshape(-1, 1)
720
+
721
+ model = IsolationForest(contamination=contamination, random_state=42, n_estimators=100)
722
+
723
+ predictions = model.fit_predict(X)
724
+ outliers: NDArray[np.bool_] = predictions == -1
725
+
726
+ # Store model for later use
727
+ self.models["isolation_forest"] = model
728
+
729
+ return outliers
730
+
731
+ def _one_class_svm_detection(
732
+ self, X: NDArray[np.floating[Any]], nu: float = 0.1
733
+ ) -> NDArray[np.bool_]:
734
+ """One-Class SVM based anomaly detection.
735
+
736
+ One-Class SVM learns a decision boundary around normal data points.
737
+ Points outside this boundary are considered anomalies.
738
+
739
+ Args:
740
+ X: Feature matrix (n_samples, n_features)
741
+ nu: Upper bound on fraction of outliers (0.0-1.0)
742
+
743
+ Returns:
744
+ Boolean array indicating outliers (True = outlier)
745
+
746
+ Raises:
747
+ ImportError: If scikit-learn is not installed
748
+
749
+ Example:
750
+ >>> X = np.array([[1.0, 2.0], [1.1, 2.1], [10.0, 20.0]])
751
+ >>> outliers = detector._one_class_svm_detection(X)
752
+ """
753
+ try:
754
+ from sklearn.svm import OneClassSVM
755
+ except ImportError as e:
756
+ raise ImportError(
757
+ "scikit-learn is required for One-Class SVM. Install with: pip install scikit-learn"
758
+ ) from e
759
+
760
+ if len(X.shape) == 1:
761
+ X = X.reshape(-1, 1)
762
+
763
+ model = OneClassSVM(nu=nu, kernel="rbf", gamma="auto")
764
+
765
+ predictions = model.fit_predict(X)
766
+ outliers: NDArray[np.bool_] = predictions == -1
767
+
768
+ # Store model for later use
769
+ self.models["one_class_svm"] = model
770
+
771
+ return outliers
772
+
773
+ def _train_isolation_forest(self, X: NDArray[np.floating[Any]]) -> None:
774
+ """Train Isolation Forest model.
775
+
776
+ Args:
777
+ X: Training data (n_samples, n_features)
778
+ """
779
+ try:
780
+ from sklearn.ensemble import IsolationForest
781
+ except ImportError:
782
+ logger.warning("scikit-learn not available, skipping Isolation Forest training")
783
+ return
784
+
785
+ model = IsolationForest(
786
+ contamination=self.config.contamination, random_state=42, n_estimators=100
787
+ )
788
+ model.fit(X)
789
+ self.models["isolation_forest"] = model
790
+
791
+ def _train_one_class_svm(self, X: NDArray[np.floating[Any]]) -> None:
792
+ """Train One-Class SVM model.
793
+
794
+ Args:
795
+ X: Training data (n_samples, n_features)
796
+ """
797
+ try:
798
+ from sklearn.svm import OneClassSVM
799
+ except ImportError:
800
+ logger.warning("scikit-learn not available, skipping One-Class SVM training")
801
+ return
802
+
803
+ model = OneClassSVM(nu=self.config.contamination, kernel="rbf", gamma="auto")
804
+ model.fit(X)
805
+ self.models["one_class_svm"] = model
806
+
807
+ def export_report(self, output_path: Path, format: str = "json") -> None:
808
+ """Export anomaly report with context and explanations.
809
+
810
+ Args:
811
+ output_path: Path to output file
812
+ format: Export format (json, txt)
813
+
814
+ Raises:
815
+ ValueError: If format is unsupported
816
+
817
+ Example:
818
+ >>> detector.export_report(Path("anomalies.json"), format="json")
819
+ >>> detector.export_report(Path("anomalies.txt"), format="txt")
820
+ """
821
+ if format == "json":
822
+ self._export_json(output_path)
823
+ elif format == "txt":
824
+ self._export_txt(output_path)
825
+ else:
826
+ raise ValueError(f"Unsupported format: {format}")
827
+
828
+ def _export_json(self, output_path: Path) -> None:
829
+ """Export anomalies to JSON format.
830
+
831
+ Args:
832
+ output_path: Path to output JSON file
833
+ """
834
+ report = {
835
+ "config": {
836
+ "methods": self.config.methods,
837
+ "zscore_threshold": self.config.zscore_threshold,
838
+ "iqr_multiplier": self.config.iqr_multiplier,
839
+ "contamination": self.config.contamination,
840
+ "window_size": self.config.window_size,
841
+ },
842
+ "summary": {
843
+ "total_anomalies": len(self.anomalies),
844
+ "by_type": self._count_by_type(),
845
+ },
846
+ "anomalies": [
847
+ {
848
+ "timestamp": a.timestamp,
849
+ "anomaly_type": a.anomaly_type,
850
+ "score": a.score,
851
+ "message_index": a.message_index,
852
+ "field_name": a.field_name,
853
+ "expected_value": a.expected_value,
854
+ "actual_value": a.actual_value,
855
+ "explanation": a.explanation,
856
+ "context": a.context,
857
+ }
858
+ for a in self.anomalies
859
+ ],
860
+ }
861
+
862
+ with output_path.open("w") as f:
863
+ json.dump(report, f, indent=2)
864
+
865
+ def _export_txt(self, output_path: Path) -> None:
866
+ """Export anomalies to text format.
867
+
868
+ Args:
869
+ output_path: Path to output text file
870
+ """
871
+ with output_path.open("w") as f:
872
+ f.write("Anomaly Detection Report\n")
873
+ f.write("=" * 80 + "\n\n")
874
+
875
+ f.write("Configuration:\n")
876
+ f.write(f" Methods: {', '.join(self.config.methods)}\n")
877
+ f.write(f" Z-score threshold: {self.config.zscore_threshold}\n")
878
+ f.write(f" IQR multiplier: {self.config.iqr_multiplier}\n\n")
879
+
880
+ f.write("Summary:\n")
881
+ f.write(f" Total anomalies: {len(self.anomalies)}\n")
882
+ f.write(" By type:\n")
883
+ for anomaly_type, count in self._count_by_type().items():
884
+ f.write(f" {anomaly_type}: {count}\n")
885
+ f.write("\n")
886
+
887
+ f.write("Anomalies:\n")
888
+ f.write("-" * 80 + "\n")
889
+ for i, a in enumerate(self.anomalies, 1):
890
+ f.write(f"{i}. [{a.anomaly_type}] {a.explanation}\n")
891
+ f.write(f" Timestamp: {a.timestamp:.3f}\n")
892
+ f.write(f" Score: {a.score:.3f}\n")
893
+ if a.field_name:
894
+ f.write(f" Field: {a.field_name}\n")
895
+ if a.message_index is not None:
896
+ f.write(f" Message index: {a.message_index}\n")
897
+ f.write("\n")
898
+
899
+ def _count_by_type(self) -> dict[str, int]:
900
+ """Count anomalies by type.
901
+
902
+ Returns:
903
+ Dictionary mapping anomaly type to count
904
+ """
905
+ counts: dict[str, int] = {}
906
+ for anomaly in self.anomalies:
907
+ counts[anomaly.anomaly_type] = counts.get(anomaly.anomaly_type, 0) + 1
908
+ return counts