oscura 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (513) hide show
  1. oscura/__init__.py +169 -167
  2. oscura/analyzers/__init__.py +3 -0
  3. oscura/analyzers/classification.py +659 -0
  4. oscura/analyzers/digital/__init__.py +0 -48
  5. oscura/analyzers/digital/edges.py +325 -65
  6. oscura/analyzers/digital/extraction.py +0 -195
  7. oscura/analyzers/digital/quality.py +293 -166
  8. oscura/analyzers/digital/timing.py +260 -115
  9. oscura/analyzers/digital/timing_numba.py +334 -0
  10. oscura/analyzers/entropy.py +605 -0
  11. oscura/analyzers/eye/diagram.py +176 -109
  12. oscura/analyzers/eye/metrics.py +5 -5
  13. oscura/analyzers/jitter/__init__.py +6 -4
  14. oscura/analyzers/jitter/ber.py +52 -52
  15. oscura/analyzers/jitter/classification.py +156 -0
  16. oscura/analyzers/jitter/decomposition.py +163 -113
  17. oscura/analyzers/jitter/spectrum.py +80 -64
  18. oscura/analyzers/ml/__init__.py +39 -0
  19. oscura/analyzers/ml/features.py +600 -0
  20. oscura/analyzers/ml/signal_classifier.py +604 -0
  21. oscura/analyzers/packet/daq.py +246 -158
  22. oscura/analyzers/packet/parser.py +12 -1
  23. oscura/analyzers/packet/payload.py +50 -2110
  24. oscura/analyzers/packet/payload_analysis.py +361 -181
  25. oscura/analyzers/packet/payload_patterns.py +133 -70
  26. oscura/analyzers/packet/stream.py +84 -23
  27. oscura/analyzers/patterns/__init__.py +26 -5
  28. oscura/analyzers/patterns/anomaly_detection.py +908 -0
  29. oscura/analyzers/patterns/clustering.py +169 -108
  30. oscura/analyzers/patterns/clustering_optimized.py +227 -0
  31. oscura/analyzers/patterns/discovery.py +1 -1
  32. oscura/analyzers/patterns/matching.py +581 -197
  33. oscura/analyzers/patterns/pattern_mining.py +778 -0
  34. oscura/analyzers/patterns/periodic.py +121 -38
  35. oscura/analyzers/patterns/sequences.py +175 -78
  36. oscura/analyzers/power/conduction.py +1 -1
  37. oscura/analyzers/power/soa.py +6 -6
  38. oscura/analyzers/power/switching.py +250 -110
  39. oscura/analyzers/protocol/__init__.py +17 -1
  40. oscura/analyzers/protocols/__init__.py +1 -22
  41. oscura/analyzers/protocols/base.py +6 -6
  42. oscura/analyzers/protocols/ble/__init__.py +38 -0
  43. oscura/analyzers/protocols/ble/analyzer.py +809 -0
  44. oscura/analyzers/protocols/ble/uuids.py +288 -0
  45. oscura/analyzers/protocols/can.py +257 -127
  46. oscura/analyzers/protocols/can_fd.py +107 -80
  47. oscura/analyzers/protocols/flexray.py +139 -80
  48. oscura/analyzers/protocols/hdlc.py +93 -58
  49. oscura/analyzers/protocols/i2c.py +247 -106
  50. oscura/analyzers/protocols/i2s.py +138 -86
  51. oscura/analyzers/protocols/industrial/__init__.py +40 -0
  52. oscura/analyzers/protocols/industrial/bacnet/__init__.py +33 -0
  53. oscura/analyzers/protocols/industrial/bacnet/analyzer.py +708 -0
  54. oscura/analyzers/protocols/industrial/bacnet/encoding.py +412 -0
  55. oscura/analyzers/protocols/industrial/bacnet/services.py +622 -0
  56. oscura/analyzers/protocols/industrial/ethercat/__init__.py +30 -0
  57. oscura/analyzers/protocols/industrial/ethercat/analyzer.py +474 -0
  58. oscura/analyzers/protocols/industrial/ethercat/mailbox.py +339 -0
  59. oscura/analyzers/protocols/industrial/ethercat/topology.py +166 -0
  60. oscura/analyzers/protocols/industrial/modbus/__init__.py +31 -0
  61. oscura/analyzers/protocols/industrial/modbus/analyzer.py +525 -0
  62. oscura/analyzers/protocols/industrial/modbus/crc.py +79 -0
  63. oscura/analyzers/protocols/industrial/modbus/functions.py +436 -0
  64. oscura/analyzers/protocols/industrial/opcua/__init__.py +21 -0
  65. oscura/analyzers/protocols/industrial/opcua/analyzer.py +552 -0
  66. oscura/analyzers/protocols/industrial/opcua/datatypes.py +446 -0
  67. oscura/analyzers/protocols/industrial/opcua/services.py +264 -0
  68. oscura/analyzers/protocols/industrial/profinet/__init__.py +23 -0
  69. oscura/analyzers/protocols/industrial/profinet/analyzer.py +441 -0
  70. oscura/analyzers/protocols/industrial/profinet/dcp.py +263 -0
  71. oscura/analyzers/protocols/industrial/profinet/ptcp.py +200 -0
  72. oscura/analyzers/protocols/jtag.py +180 -98
  73. oscura/analyzers/protocols/lin.py +219 -114
  74. oscura/analyzers/protocols/manchester.py +4 -4
  75. oscura/analyzers/protocols/onewire.py +253 -149
  76. oscura/analyzers/protocols/parallel_bus/__init__.py +20 -0
  77. oscura/analyzers/protocols/parallel_bus/centronics.py +92 -0
  78. oscura/analyzers/protocols/parallel_bus/gpib.py +137 -0
  79. oscura/analyzers/protocols/spi.py +192 -95
  80. oscura/analyzers/protocols/swd.py +321 -167
  81. oscura/analyzers/protocols/uart.py +267 -125
  82. oscura/analyzers/protocols/usb.py +235 -131
  83. oscura/analyzers/side_channel/power.py +17 -12
  84. oscura/analyzers/signal/__init__.py +15 -0
  85. oscura/analyzers/signal/timing_analysis.py +1086 -0
  86. oscura/analyzers/signal_integrity/__init__.py +4 -1
  87. oscura/analyzers/signal_integrity/sparams.py +2 -19
  88. oscura/analyzers/spectral/chunked.py +129 -60
  89. oscura/analyzers/spectral/chunked_fft.py +300 -94
  90. oscura/analyzers/spectral/chunked_wavelet.py +100 -80
  91. oscura/analyzers/statistical/checksum.py +376 -217
  92. oscura/analyzers/statistical/classification.py +229 -107
  93. oscura/analyzers/statistical/entropy.py +78 -53
  94. oscura/analyzers/statistics/correlation.py +407 -211
  95. oscura/analyzers/statistics/outliers.py +2 -2
  96. oscura/analyzers/statistics/streaming.py +30 -5
  97. oscura/analyzers/validation.py +216 -101
  98. oscura/analyzers/waveform/measurements.py +9 -0
  99. oscura/analyzers/waveform/measurements_with_uncertainty.py +31 -15
  100. oscura/analyzers/waveform/spectral.py +500 -228
  101. oscura/api/__init__.py +31 -5
  102. oscura/api/dsl/__init__.py +582 -0
  103. oscura/{dsl → api/dsl}/commands.py +43 -76
  104. oscura/{dsl → api/dsl}/interpreter.py +26 -51
  105. oscura/{dsl → api/dsl}/parser.py +107 -77
  106. oscura/{dsl → api/dsl}/repl.py +2 -2
  107. oscura/api/dsl.py +1 -1
  108. oscura/{integrations → api/integrations}/__init__.py +1 -1
  109. oscura/{integrations → api/integrations}/llm.py +201 -102
  110. oscura/api/operators.py +3 -3
  111. oscura/api/optimization.py +144 -30
  112. oscura/api/rest_server.py +921 -0
  113. oscura/api/server/__init__.py +17 -0
  114. oscura/api/server/dashboard.py +850 -0
  115. oscura/api/server/static/README.md +34 -0
  116. oscura/api/server/templates/base.html +181 -0
  117. oscura/api/server/templates/export.html +120 -0
  118. oscura/api/server/templates/home.html +284 -0
  119. oscura/api/server/templates/protocols.html +58 -0
  120. oscura/api/server/templates/reports.html +43 -0
  121. oscura/api/server/templates/session_detail.html +89 -0
  122. oscura/api/server/templates/sessions.html +83 -0
  123. oscura/api/server/templates/waveforms.html +73 -0
  124. oscura/automotive/__init__.py +8 -1
  125. oscura/automotive/can/__init__.py +10 -0
  126. oscura/automotive/can/checksum.py +3 -1
  127. oscura/automotive/can/dbc_generator.py +590 -0
  128. oscura/automotive/can/message_wrapper.py +121 -74
  129. oscura/automotive/can/patterns.py +98 -21
  130. oscura/automotive/can/session.py +292 -56
  131. oscura/automotive/can/state_machine.py +6 -3
  132. oscura/automotive/can/stimulus_response.py +97 -75
  133. oscura/automotive/dbc/__init__.py +10 -2
  134. oscura/automotive/dbc/generator.py +84 -56
  135. oscura/automotive/dbc/parser.py +6 -6
  136. oscura/automotive/dtc/data.json +2763 -0
  137. oscura/automotive/dtc/database.py +2 -2
  138. oscura/automotive/flexray/__init__.py +31 -0
  139. oscura/automotive/flexray/analyzer.py +504 -0
  140. oscura/automotive/flexray/crc.py +185 -0
  141. oscura/automotive/flexray/fibex.py +449 -0
  142. oscura/automotive/j1939/__init__.py +45 -8
  143. oscura/automotive/j1939/analyzer.py +605 -0
  144. oscura/automotive/j1939/spns.py +326 -0
  145. oscura/automotive/j1939/transport.py +306 -0
  146. oscura/automotive/lin/__init__.py +47 -0
  147. oscura/automotive/lin/analyzer.py +612 -0
  148. oscura/automotive/loaders/blf.py +13 -2
  149. oscura/automotive/loaders/csv_can.py +143 -72
  150. oscura/automotive/loaders/dispatcher.py +50 -2
  151. oscura/automotive/loaders/mdf.py +86 -45
  152. oscura/automotive/loaders/pcap.py +111 -61
  153. oscura/automotive/uds/__init__.py +4 -0
  154. oscura/automotive/uds/analyzer.py +725 -0
  155. oscura/automotive/uds/decoder.py +140 -58
  156. oscura/automotive/uds/models.py +7 -1
  157. oscura/automotive/visualization.py +1 -1
  158. oscura/cli/analyze.py +348 -0
  159. oscura/cli/batch.py +142 -122
  160. oscura/cli/benchmark.py +275 -0
  161. oscura/cli/characterize.py +137 -82
  162. oscura/cli/compare.py +224 -131
  163. oscura/cli/completion.py +250 -0
  164. oscura/cli/config_cmd.py +361 -0
  165. oscura/cli/decode.py +164 -87
  166. oscura/cli/export.py +286 -0
  167. oscura/cli/main.py +115 -31
  168. oscura/{onboarding → cli/onboarding}/__init__.py +3 -3
  169. oscura/{onboarding → cli/onboarding}/help.py +80 -58
  170. oscura/{onboarding → cli/onboarding}/tutorials.py +97 -72
  171. oscura/{onboarding → cli/onboarding}/wizard.py +55 -36
  172. oscura/cli/progress.py +147 -0
  173. oscura/cli/shell.py +157 -135
  174. oscura/cli/validate_cmd.py +204 -0
  175. oscura/cli/visualize.py +158 -0
  176. oscura/convenience.py +125 -79
  177. oscura/core/__init__.py +4 -2
  178. oscura/core/backend_selector.py +3 -3
  179. oscura/core/cache.py +126 -15
  180. oscura/core/cancellation.py +1 -1
  181. oscura/{config → core/config}/__init__.py +20 -11
  182. oscura/{config → core/config}/defaults.py +1 -1
  183. oscura/{config → core/config}/loader.py +7 -5
  184. oscura/{config → core/config}/memory.py +5 -5
  185. oscura/{config → core/config}/migration.py +1 -1
  186. oscura/{config → core/config}/pipeline.py +99 -23
  187. oscura/{config → core/config}/preferences.py +1 -1
  188. oscura/{config → core/config}/protocol.py +3 -3
  189. oscura/{config → core/config}/schema.py +426 -272
  190. oscura/{config → core/config}/settings.py +1 -1
  191. oscura/{config → core/config}/thresholds.py +195 -153
  192. oscura/core/correlation.py +5 -6
  193. oscura/core/cross_domain.py +0 -2
  194. oscura/core/debug.py +9 -5
  195. oscura/{extensibility → core/extensibility}/docs.py +158 -70
  196. oscura/{extensibility → core/extensibility}/extensions.py +160 -76
  197. oscura/{extensibility → core/extensibility}/logging.py +1 -1
  198. oscura/{extensibility → core/extensibility}/measurements.py +1 -1
  199. oscura/{extensibility → core/extensibility}/plugins.py +1 -1
  200. oscura/{extensibility → core/extensibility}/templates.py +73 -3
  201. oscura/{extensibility → core/extensibility}/validation.py +1 -1
  202. oscura/core/gpu_backend.py +11 -7
  203. oscura/core/log_query.py +101 -11
  204. oscura/core/logging.py +126 -54
  205. oscura/core/logging_advanced.py +5 -5
  206. oscura/core/memory_limits.py +108 -70
  207. oscura/core/memory_monitor.py +2 -2
  208. oscura/core/memory_progress.py +7 -7
  209. oscura/core/memory_warnings.py +1 -1
  210. oscura/core/numba_backend.py +13 -13
  211. oscura/{plugins → core/plugins}/__init__.py +9 -9
  212. oscura/{plugins → core/plugins}/base.py +7 -7
  213. oscura/{plugins → core/plugins}/cli.py +3 -3
  214. oscura/{plugins → core/plugins}/discovery.py +186 -106
  215. oscura/{plugins → core/plugins}/lifecycle.py +1 -1
  216. oscura/{plugins → core/plugins}/manager.py +7 -7
  217. oscura/{plugins → core/plugins}/registry.py +3 -3
  218. oscura/{plugins → core/plugins}/versioning.py +1 -1
  219. oscura/core/progress.py +16 -1
  220. oscura/core/provenance.py +8 -2
  221. oscura/{schemas → core/schemas}/__init__.py +2 -2
  222. oscura/core/schemas/bus_configuration.json +322 -0
  223. oscura/core/schemas/device_mapping.json +182 -0
  224. oscura/core/schemas/packet_format.json +418 -0
  225. oscura/core/schemas/protocol_definition.json +363 -0
  226. oscura/core/types.py +4 -0
  227. oscura/core/uncertainty.py +3 -3
  228. oscura/correlation/__init__.py +52 -0
  229. oscura/correlation/multi_protocol.py +811 -0
  230. oscura/discovery/auto_decoder.py +117 -35
  231. oscura/discovery/comparison.py +191 -86
  232. oscura/discovery/quality_validator.py +155 -68
  233. oscura/discovery/signal_detector.py +196 -79
  234. oscura/export/__init__.py +18 -20
  235. oscura/export/kaitai_struct.py +513 -0
  236. oscura/export/scapy_layer.py +801 -0
  237. oscura/export/wireshark/README.md +15 -15
  238. oscura/export/wireshark/generator.py +1 -1
  239. oscura/export/wireshark/templates/dissector.lua.j2 +2 -2
  240. oscura/export/wireshark_dissector.py +746 -0
  241. oscura/guidance/wizard.py +207 -111
  242. oscura/hardware/__init__.py +19 -0
  243. oscura/{acquisition → hardware/acquisition}/__init__.py +4 -4
  244. oscura/{acquisition → hardware/acquisition}/file.py +2 -2
  245. oscura/{acquisition → hardware/acquisition}/hardware.py +7 -7
  246. oscura/{acquisition → hardware/acquisition}/saleae.py +15 -12
  247. oscura/{acquisition → hardware/acquisition}/socketcan.py +1 -1
  248. oscura/{acquisition → hardware/acquisition}/streaming.py +2 -2
  249. oscura/{acquisition → hardware/acquisition}/synthetic.py +3 -3
  250. oscura/{acquisition → hardware/acquisition}/visa.py +33 -11
  251. oscura/hardware/firmware/__init__.py +29 -0
  252. oscura/hardware/firmware/pattern_recognition.py +874 -0
  253. oscura/hardware/hal_detector.py +736 -0
  254. oscura/hardware/security/__init__.py +37 -0
  255. oscura/hardware/security/side_channel_detector.py +1126 -0
  256. oscura/inference/__init__.py +4 -0
  257. oscura/inference/active_learning/README.md +7 -7
  258. oscura/inference/active_learning/observation_table.py +4 -1
  259. oscura/inference/alignment.py +216 -123
  260. oscura/inference/bayesian.py +113 -33
  261. oscura/inference/crc_reverse.py +101 -55
  262. oscura/inference/logic.py +6 -2
  263. oscura/inference/message_format.py +342 -183
  264. oscura/inference/protocol.py +95 -44
  265. oscura/inference/protocol_dsl.py +180 -82
  266. oscura/inference/signal_intelligence.py +1439 -706
  267. oscura/inference/spectral.py +99 -57
  268. oscura/inference/state_machine.py +810 -158
  269. oscura/inference/stream.py +270 -110
  270. oscura/iot/__init__.py +34 -0
  271. oscura/iot/coap/__init__.py +32 -0
  272. oscura/iot/coap/analyzer.py +668 -0
  273. oscura/iot/coap/options.py +212 -0
  274. oscura/iot/lorawan/__init__.py +21 -0
  275. oscura/iot/lorawan/crypto.py +206 -0
  276. oscura/iot/lorawan/decoder.py +801 -0
  277. oscura/iot/lorawan/mac_commands.py +341 -0
  278. oscura/iot/mqtt/__init__.py +27 -0
  279. oscura/iot/mqtt/analyzer.py +999 -0
  280. oscura/iot/mqtt/properties.py +315 -0
  281. oscura/iot/zigbee/__init__.py +31 -0
  282. oscura/iot/zigbee/analyzer.py +615 -0
  283. oscura/iot/zigbee/security.py +153 -0
  284. oscura/iot/zigbee/zcl.py +349 -0
  285. oscura/jupyter/display.py +125 -45
  286. oscura/{exploratory → jupyter/exploratory}/__init__.py +8 -8
  287. oscura/{exploratory → jupyter/exploratory}/error_recovery.py +298 -141
  288. oscura/jupyter/exploratory/fuzzy.py +746 -0
  289. oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py +258 -100
  290. oscura/{exploratory → jupyter/exploratory}/legacy.py +464 -242
  291. oscura/{exploratory → jupyter/exploratory}/parse.py +167 -145
  292. oscura/{exploratory → jupyter/exploratory}/recovery.py +119 -87
  293. oscura/jupyter/exploratory/sync.py +612 -0
  294. oscura/{exploratory → jupyter/exploratory}/unknown.py +299 -176
  295. oscura/jupyter/magic.py +4 -4
  296. oscura/{ui → jupyter/ui}/__init__.py +2 -2
  297. oscura/{ui → jupyter/ui}/formatters.py +3 -3
  298. oscura/{ui → jupyter/ui}/progressive_display.py +153 -82
  299. oscura/loaders/__init__.py +171 -63
  300. oscura/loaders/binary.py +88 -1
  301. oscura/loaders/chipwhisperer.py +153 -137
  302. oscura/loaders/configurable.py +208 -86
  303. oscura/loaders/csv_loader.py +458 -215
  304. oscura/loaders/hdf5_loader.py +278 -119
  305. oscura/loaders/lazy.py +87 -54
  306. oscura/loaders/mmap_loader.py +1 -1
  307. oscura/loaders/numpy_loader.py +253 -116
  308. oscura/loaders/pcap.py +226 -151
  309. oscura/loaders/rigol.py +110 -49
  310. oscura/loaders/sigrok.py +201 -78
  311. oscura/loaders/tdms.py +81 -58
  312. oscura/loaders/tektronix.py +291 -174
  313. oscura/loaders/touchstone.py +182 -87
  314. oscura/loaders/vcd.py +215 -117
  315. oscura/loaders/wav.py +155 -68
  316. oscura/reporting/__init__.py +9 -7
  317. oscura/reporting/analyze.py +352 -146
  318. oscura/reporting/argument_preparer.py +69 -14
  319. oscura/reporting/auto_report.py +97 -61
  320. oscura/reporting/batch.py +131 -58
  321. oscura/reporting/chart_selection.py +57 -45
  322. oscura/reporting/comparison.py +63 -17
  323. oscura/reporting/content/executive.py +76 -24
  324. oscura/reporting/core_formats/multi_format.py +11 -8
  325. oscura/reporting/engine.py +312 -158
  326. oscura/reporting/enhanced_reports.py +949 -0
  327. oscura/reporting/export.py +86 -43
  328. oscura/reporting/formatting/numbers.py +69 -42
  329. oscura/reporting/html.py +139 -58
  330. oscura/reporting/index.py +137 -65
  331. oscura/reporting/output.py +158 -67
  332. oscura/reporting/pdf.py +67 -102
  333. oscura/reporting/plots.py +191 -112
  334. oscura/reporting/sections.py +88 -47
  335. oscura/reporting/standards.py +104 -61
  336. oscura/reporting/summary_generator.py +75 -55
  337. oscura/reporting/tables.py +138 -54
  338. oscura/reporting/templates/enhanced/protocol_re.html +525 -0
  339. oscura/reporting/templates/index.md +13 -13
  340. oscura/sessions/__init__.py +14 -23
  341. oscura/sessions/base.py +3 -3
  342. oscura/sessions/blackbox.py +106 -10
  343. oscura/sessions/generic.py +2 -2
  344. oscura/sessions/legacy.py +783 -0
  345. oscura/side_channel/__init__.py +63 -0
  346. oscura/side_channel/dpa.py +1025 -0
  347. oscura/utils/__init__.py +15 -1
  348. oscura/utils/autodetect.py +1 -5
  349. oscura/utils/bitwise.py +118 -0
  350. oscura/{builders → utils/builders}/__init__.py +1 -1
  351. oscura/{comparison → utils/comparison}/__init__.py +6 -6
  352. oscura/{comparison → utils/comparison}/compare.py +202 -101
  353. oscura/{comparison → utils/comparison}/golden.py +83 -63
  354. oscura/{comparison → utils/comparison}/limits.py +313 -89
  355. oscura/{comparison → utils/comparison}/mask.py +151 -45
  356. oscura/{comparison → utils/comparison}/trace_diff.py +1 -1
  357. oscura/{comparison → utils/comparison}/visualization.py +147 -89
  358. oscura/{component → utils/component}/__init__.py +3 -3
  359. oscura/{component → utils/component}/impedance.py +122 -58
  360. oscura/{component → utils/component}/reactive.py +165 -168
  361. oscura/{component → utils/component}/transmission_line.py +3 -3
  362. oscura/{filtering → utils/filtering}/__init__.py +6 -6
  363. oscura/{filtering → utils/filtering}/base.py +1 -1
  364. oscura/{filtering → utils/filtering}/convenience.py +2 -2
  365. oscura/{filtering → utils/filtering}/design.py +169 -93
  366. oscura/{filtering → utils/filtering}/filters.py +2 -2
  367. oscura/{filtering → utils/filtering}/introspection.py +2 -2
  368. oscura/utils/geometry.py +31 -0
  369. oscura/utils/imports.py +184 -0
  370. oscura/utils/lazy.py +1 -1
  371. oscura/{math → utils/math}/__init__.py +2 -2
  372. oscura/{math → utils/math}/arithmetic.py +114 -48
  373. oscura/{math → utils/math}/interpolation.py +139 -106
  374. oscura/utils/memory.py +129 -66
  375. oscura/utils/memory_advanced.py +92 -9
  376. oscura/utils/memory_extensions.py +10 -8
  377. oscura/{optimization → utils/optimization}/__init__.py +1 -1
  378. oscura/{optimization → utils/optimization}/search.py +2 -2
  379. oscura/utils/performance/__init__.py +58 -0
  380. oscura/utils/performance/caching.py +889 -0
  381. oscura/utils/performance/lsh_clustering.py +333 -0
  382. oscura/utils/performance/memory_optimizer.py +699 -0
  383. oscura/utils/performance/optimizations.py +675 -0
  384. oscura/utils/performance/parallel.py +654 -0
  385. oscura/utils/performance/profiling.py +661 -0
  386. oscura/{pipeline → utils/pipeline}/base.py +1 -1
  387. oscura/{pipeline → utils/pipeline}/composition.py +11 -3
  388. oscura/{pipeline → utils/pipeline}/parallel.py +3 -2
  389. oscura/{pipeline → utils/pipeline}/pipeline.py +1 -1
  390. oscura/{pipeline → utils/pipeline}/reverse_engineering.py +412 -221
  391. oscura/{search → utils/search}/__init__.py +3 -3
  392. oscura/{search → utils/search}/anomaly.py +188 -58
  393. oscura/utils/search/context.py +294 -0
  394. oscura/{search → utils/search}/pattern.py +138 -10
  395. oscura/utils/serial.py +51 -0
  396. oscura/utils/storage/__init__.py +61 -0
  397. oscura/utils/storage/database.py +1166 -0
  398. oscura/{streaming → utils/streaming}/chunked.py +302 -143
  399. oscura/{streaming → utils/streaming}/progressive.py +1 -1
  400. oscura/{streaming → utils/streaming}/realtime.py +3 -2
  401. oscura/{triggering → utils/triggering}/__init__.py +6 -6
  402. oscura/{triggering → utils/triggering}/base.py +6 -6
  403. oscura/{triggering → utils/triggering}/edge.py +2 -2
  404. oscura/{triggering → utils/triggering}/pattern.py +2 -2
  405. oscura/{triggering → utils/triggering}/pulse.py +115 -74
  406. oscura/{triggering → utils/triggering}/window.py +2 -2
  407. oscura/utils/validation.py +32 -0
  408. oscura/validation/__init__.py +121 -0
  409. oscura/{compliance → validation/compliance}/__init__.py +5 -5
  410. oscura/{compliance → validation/compliance}/advanced.py +5 -5
  411. oscura/{compliance → validation/compliance}/masks.py +1 -1
  412. oscura/{compliance → validation/compliance}/reporting.py +127 -53
  413. oscura/{compliance → validation/compliance}/testing.py +114 -52
  414. oscura/validation/compliance_tests.py +915 -0
  415. oscura/validation/fuzzer.py +990 -0
  416. oscura/validation/grammar_tests.py +596 -0
  417. oscura/validation/grammar_validator.py +904 -0
  418. oscura/validation/hil_testing.py +977 -0
  419. oscura/{quality → validation/quality}/__init__.py +4 -4
  420. oscura/{quality → validation/quality}/ensemble.py +251 -171
  421. oscura/{quality → validation/quality}/explainer.py +3 -3
  422. oscura/{quality → validation/quality}/scoring.py +1 -1
  423. oscura/{quality → validation/quality}/warnings.py +4 -4
  424. oscura/validation/regression_suite.py +808 -0
  425. oscura/validation/replay.py +788 -0
  426. oscura/{testing → validation/testing}/__init__.py +2 -2
  427. oscura/{testing → validation/testing}/synthetic.py +5 -5
  428. oscura/visualization/__init__.py +9 -0
  429. oscura/visualization/accessibility.py +1 -1
  430. oscura/visualization/annotations.py +64 -67
  431. oscura/visualization/colors.py +7 -7
  432. oscura/visualization/digital.py +180 -81
  433. oscura/visualization/eye.py +236 -85
  434. oscura/visualization/interactive.py +320 -143
  435. oscura/visualization/jitter.py +587 -247
  436. oscura/visualization/layout.py +169 -134
  437. oscura/visualization/optimization.py +103 -52
  438. oscura/visualization/palettes.py +1 -1
  439. oscura/visualization/power.py +427 -211
  440. oscura/visualization/power_extended.py +626 -297
  441. oscura/visualization/presets.py +2 -0
  442. oscura/visualization/protocols.py +495 -181
  443. oscura/visualization/render.py +79 -63
  444. oscura/visualization/reverse_engineering.py +171 -124
  445. oscura/visualization/signal_integrity.py +460 -279
  446. oscura/visualization/specialized.py +190 -100
  447. oscura/visualization/spectral.py +670 -255
  448. oscura/visualization/thumbnails.py +166 -137
  449. oscura/visualization/waveform.py +150 -63
  450. oscura/workflows/__init__.py +3 -0
  451. oscura/{batch → workflows/batch}/__init__.py +5 -5
  452. oscura/{batch → workflows/batch}/advanced.py +150 -75
  453. oscura/workflows/batch/aggregate.py +531 -0
  454. oscura/workflows/batch/analyze.py +236 -0
  455. oscura/{batch → workflows/batch}/logging.py +2 -2
  456. oscura/{batch → workflows/batch}/metrics.py +1 -1
  457. oscura/workflows/complete_re.py +1144 -0
  458. oscura/workflows/compliance.py +44 -54
  459. oscura/workflows/digital.py +197 -51
  460. oscura/workflows/legacy/__init__.py +12 -0
  461. oscura/{workflow → workflows/legacy}/dag.py +4 -1
  462. oscura/workflows/multi_trace.py +9 -9
  463. oscura/workflows/power.py +42 -62
  464. oscura/workflows/protocol.py +82 -49
  465. oscura/workflows/reverse_engineering.py +351 -150
  466. oscura/workflows/signal_integrity.py +157 -82
  467. oscura-0.6.0.dist-info/METADATA +643 -0
  468. oscura-0.6.0.dist-info/RECORD +590 -0
  469. oscura/analyzers/digital/ic_database.py +0 -498
  470. oscura/analyzers/digital/timing_paths.py +0 -339
  471. oscura/analyzers/digital/vintage.py +0 -377
  472. oscura/analyzers/digital/vintage_result.py +0 -148
  473. oscura/analyzers/protocols/parallel_bus.py +0 -449
  474. oscura/batch/aggregate.py +0 -300
  475. oscura/batch/analyze.py +0 -139
  476. oscura/dsl/__init__.py +0 -73
  477. oscura/exceptions.py +0 -59
  478. oscura/exploratory/fuzzy.py +0 -513
  479. oscura/exploratory/sync.py +0 -384
  480. oscura/export/wavedrom.py +0 -430
  481. oscura/exporters/__init__.py +0 -94
  482. oscura/exporters/csv.py +0 -303
  483. oscura/exporters/exporters.py +0 -44
  484. oscura/exporters/hdf5.py +0 -217
  485. oscura/exporters/html_export.py +0 -701
  486. oscura/exporters/json_export.py +0 -338
  487. oscura/exporters/markdown_export.py +0 -367
  488. oscura/exporters/matlab_export.py +0 -354
  489. oscura/exporters/npz_export.py +0 -219
  490. oscura/exporters/spice_export.py +0 -210
  491. oscura/exporters/vintage_logic_csv.py +0 -247
  492. oscura/reporting/vintage_logic_report.py +0 -523
  493. oscura/search/context.py +0 -149
  494. oscura/session/__init__.py +0 -34
  495. oscura/session/annotations.py +0 -289
  496. oscura/session/history.py +0 -313
  497. oscura/session/session.py +0 -520
  498. oscura/visualization/digital_advanced.py +0 -718
  499. oscura/visualization/figure_manager.py +0 -156
  500. oscura/workflow/__init__.py +0 -13
  501. oscura-0.5.0.dist-info/METADATA +0 -407
  502. oscura-0.5.0.dist-info/RECORD +0 -486
  503. /oscura/core/{config.py → config/legacy.py} +0 -0
  504. /oscura/{extensibility → core/extensibility}/__init__.py +0 -0
  505. /oscura/{extensibility → core/extensibility}/registry.py +0 -0
  506. /oscura/{plugins → core/plugins}/isolation.py +0 -0
  507. /oscura/{builders → utils/builders}/signal_builder.py +0 -0
  508. /oscura/{optimization → utils/optimization}/parallel.py +0 -0
  509. /oscura/{pipeline → utils/pipeline}/__init__.py +0 -0
  510. /oscura/{streaming → utils/streaming}/__init__.py +0 -0
  511. {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/WHEEL +0 -0
  512. {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/entry_points.txt +0 -0
  513. {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,908 @@
1
+ """Anomaly detection system for identifying unusual patterns in protocol traffic.
2
+
3
+ This module provides multi-method anomaly detection to identify unusual patterns
4
+ in protocol traffic using statistical and machine learning methods.
5
+
6
+ Key capabilities:
7
+ - Statistical anomaly detection (Z-score, IQR, modified Z-score)
8
+ - Time-series anomaly detection (rate analysis, timing analysis)
9
+ - ML-based detection (Isolation Forest, One-Class SVM) - optional with scikit-learn
10
+ - Detect: unexpected message rates, unusual field values, timing anomalies, sequence violations
11
+ - Return anomaly scores and explanations
12
+ - Support online detection (streaming data)
13
+ - Export anomaly reports with context
14
+
15
+ Detection methods:
16
+ - Z-score: Standard deviation-based outlier detection
17
+ - IQR: Interquartile range-based outlier detection
18
+ - Modified Z-score: Median absolute deviation-based (robust to outliers)
19
+ - Isolation Forest: Tree-based anomaly detection (requires sklearn)
20
+ - One-Class SVM: Support vector-based anomaly detection (requires sklearn)
21
+
22
+ Typical workflow:
23
+ 1. Train detector on normal baseline data (optional for statistical methods)
24
+ 2. Detect anomalies in new data points (batch or streaming)
25
+ 3. Export anomaly reports with explanations and context
26
+
27
+ Example:
28
+ >>> from oscura.analyzers.patterns.anomaly_detection import AnomalyDetector, AnomalyDetectionConfig
29
+ >>> config = AnomalyDetectionConfig(methods=["zscore", "iqr"])
30
+ >>> detector = AnomalyDetector(config)
31
+ >>> # Detect field value anomalies
32
+ >>> anomalies = detector.detect_field_value_anomaly(
33
+ ... field_values=[1.0, 1.1, 0.9, 1.2, 10.0, 1.0],
34
+ ... field_name="voltage"
35
+ ... )
36
+ >>> for anomaly in anomalies:
37
+ ... print(f"{anomaly.anomaly_type}: {anomaly.explanation}")
38
+ value: voltage value 10.00 deviates significantly from expected 2.53
39
+
40
+ References:
41
+ Isolation Forest: Liu et al. (2008) "Isolation Forest"
42
+ One-Class SVM: Schölkopf et al. (2001) "Estimating the Support of a High-Dimensional Distribution"
43
+ """
44
+
45
+ from __future__ import annotations
46
+
47
+ import json
48
+ import logging
49
+ from dataclasses import dataclass, field
50
+ from pathlib import Path
51
+ from typing import TYPE_CHECKING, Any, ClassVar
52
+
53
+ import numpy as np
54
+
55
+ if TYPE_CHECKING:
56
+ from numpy.typing import NDArray
57
+
58
+ logger = logging.getLogger(__name__)
59
+
60
+
61
+ @dataclass
62
+ class Anomaly:
63
+ """Detected anomaly.
64
+
65
+ Attributes:
66
+ timestamp: Time of anomaly occurrence
67
+ anomaly_type: Type of anomaly detected (rate, value, timing, sequence, protocol)
68
+ score: Anomaly score (0.0-1.0, higher = more anomalous)
69
+ message_index: Index of message containing anomaly (None if not message-specific)
70
+ field_name: Name of field with anomaly (None if not field-specific)
71
+ expected_value: Expected value based on baseline/model
72
+ actual_value: Actual value observed
73
+ explanation: Human-readable explanation of anomaly
74
+ context: Additional context information (rates, thresholds, etc.)
75
+
76
+ Example:
77
+ >>> anomaly = Anomaly(
78
+ ... timestamp=1234.5,
79
+ ... anomaly_type="value",
80
+ ... score=0.95,
81
+ ... message_index=42,
82
+ ... field_name="voltage",
83
+ ... expected_value=3.3,
84
+ ... actual_value=15.0,
85
+ ... explanation="Voltage spike detected"
86
+ ... )
87
+ """
88
+
89
+ timestamp: float
90
+ anomaly_type: str # "rate", "value", "timing", "sequence", "protocol"
91
+ score: float # 0.0-1.0, higher = more anomalous
92
+ message_index: int | None = None
93
+ field_name: str | None = None
94
+ expected_value: Any = None
95
+ actual_value: Any = None
96
+ explanation: str = ""
97
+ context: dict[str, Any] = field(default_factory=dict)
98
+
99
+
100
+ @dataclass
101
+ class AnomalyDetectionConfig:
102
+ """Anomaly detection configuration.
103
+
104
+ Attributes:
105
+ methods: Detection methods to use (zscore, iqr, modified_zscore, isolation_forest, one_class_svm)
106
+ zscore_threshold: Standard deviations for Z-score method (default: 3.0)
107
+ iqr_multiplier: Multiplier for IQR method (default: 1.5)
108
+ contamination: Expected outlier fraction for ML methods (default: 0.1)
109
+ window_size: Window size for streaming detection (default: 100)
110
+
111
+ Example:
112
+ >>> config = AnomalyDetectionConfig(
113
+ ... methods=["zscore", "iqr"],
114
+ ... zscore_threshold=2.5,
115
+ ... window_size=200
116
+ ... )
117
+ """
118
+
119
+ methods: list[str] = field(default_factory=lambda: ["zscore", "isolation_forest"])
120
+ zscore_threshold: float = 3.0 # Standard deviations
121
+ iqr_multiplier: float = 1.5
122
+ contamination: float = 0.1 # Expected outlier fraction
123
+ window_size: int = 100 # For streaming detection
124
+
125
+
126
+ class AnomalyDetector:
127
+ """Multi-method anomaly detection system.
128
+
129
+ This class provides various anomaly detection methods for protocol traffic analysis.
130
+ It supports both statistical methods (Z-score, IQR) and ML-based methods
131
+ (Isolation Forest, One-Class SVM - if scikit-learn is available).
132
+
133
+ Attributes:
134
+ config: Detection configuration
135
+ models: Trained ML models (if applicable)
136
+ baselines: Baseline statistics for comparison
137
+ anomalies: Detected anomalies
138
+
139
+ Detection methods:
140
+ Statistical: zscore, iqr, modified_zscore
141
+ ML-based: isolation_forest, one_class_svm (requires scikit-learn)
142
+ Time-series: rate analysis, timing analysis
143
+
144
+ Example:
145
+ >>> detector = AnomalyDetector()
146
+ >>> # Detect message rate anomalies
147
+ >>> timestamps = [0.0, 0.1, 0.2, 0.3, 5.0, 5.1] # Gap at 0.3-5.0
148
+ >>> anomalies = detector.detect_message_rate_anomaly(timestamps)
149
+ >>> # Detect field value anomalies
150
+ >>> values = [1.0, 1.1, 0.9, 1.2, 10.0, 1.0] # Spike at 10.0
151
+ >>> anomalies = detector.detect_field_value_anomaly(values, "voltage")
152
+ """
153
+
154
+ # Detection methods
155
+ STATISTICAL_METHODS: ClassVar[list[str]] = ["zscore", "iqr", "modified_zscore"]
156
+ ML_METHODS: ClassVar[list[str]] = ["isolation_forest", "one_class_svm", "autoencoder"]
157
+ TIMESERIES_METHODS: ClassVar[list[str]] = ["arima", "seasonal_decomposition"]
158
+
159
+ def __init__(self, config: AnomalyDetectionConfig | None = None) -> None:
160
+ """Initialize anomaly detector.
161
+
162
+ Args:
163
+ config: Anomaly detection configuration. If None, uses default config.
164
+
165
+ Example:
166
+ >>> detector = AnomalyDetector()
167
+ >>> config = AnomalyDetectionConfig(methods=["zscore"], zscore_threshold=2.5)
168
+ >>> detector_custom = AnomalyDetector(config)
169
+ """
170
+ self.config = config or AnomalyDetectionConfig()
171
+ self.models: dict[str, Any] = {}
172
+ self.baselines: dict[str, Any] = {}
173
+ self.anomalies: list[Anomaly] = []
174
+
175
+ def train(self, normal_data: list[dict[str, Any]], features: list[str]) -> None:
176
+ """Train anomaly detection models on normal baseline data.
177
+
178
+ Args:
179
+ normal_data: List of normal data points (dicts with feature values)
180
+ features: List of feature names to use for training
181
+
182
+ Raises:
183
+ ImportError: If ML methods requested but scikit-learn not available
184
+ ValueError: If insufficient data for training
185
+
186
+ Example:
187
+ >>> normal_data = [
188
+ ... {"voltage": 3.3, "current": 0.5},
189
+ ... {"voltage": 3.2, "current": 0.6},
190
+ ... ]
191
+ >>> detector.train(normal_data, features=["voltage", "current"])
192
+ """
193
+ if len(normal_data) < 10:
194
+ raise ValueError("Need at least 10 samples for training")
195
+
196
+ # Extract features
197
+ X = np.array([[d[f] for f in features] for d in normal_data])
198
+
199
+ # Calculate baseline statistics
200
+ for i, feature in enumerate(features):
201
+ self.baselines[feature] = {
202
+ "mean": float(np.mean(X[:, i])),
203
+ "std": float(np.std(X[:, i])),
204
+ "median": float(np.median(X[:, i])),
205
+ "q1": float(np.percentile(X[:, i], 25)),
206
+ "q3": float(np.percentile(X[:, i], 75)),
207
+ }
208
+
209
+ # Train ML models if requested
210
+ for method in self.config.methods:
211
+ if method == "isolation_forest":
212
+ self._train_isolation_forest(X)
213
+ elif method == "one_class_svm":
214
+ self._train_one_class_svm(X)
215
+
216
+ def detect(self, data: dict[str, Any], timestamp: float = 0.0) -> list[Anomaly]:
217
+ """Detect anomalies in new data point.
218
+
219
+ Args:
220
+ data: Data point to analyze (dict with feature values)
221
+ timestamp: Timestamp of data point
222
+
223
+ Returns:
224
+ List of detected anomalies
225
+
226
+ Example:
227
+ >>> anomalies = detector.detect(
228
+ ... {"voltage": 10.0, "current": 0.5},
229
+ ... timestamp=123.45
230
+ ... )
231
+ """
232
+ anomalies = []
233
+
234
+ # Check each feature against baselines
235
+ for feature, value in data.items():
236
+ if feature in self.baselines:
237
+ baseline = self.baselines[feature]
238
+
239
+ # Z-score check
240
+ if "zscore" in self.config.methods:
241
+ z_score = abs(value - baseline["mean"]) / (baseline["std"] + 1e-10)
242
+ if z_score > self.config.zscore_threshold:
243
+ anomalies.append(
244
+ Anomaly(
245
+ timestamp=timestamp,
246
+ anomaly_type="value",
247
+ score=min(z_score / 5.0, 1.0),
248
+ field_name=feature,
249
+ expected_value=baseline["mean"],
250
+ actual_value=value,
251
+ explanation=f"{feature} value {value:.2f} deviates {z_score:.1f} std devs from expected {baseline['mean']:.2f}",
252
+ context={"z_score": z_score, "method": "zscore"},
253
+ )
254
+ )
255
+
256
+ # IQR check
257
+ if "iqr" in self.config.methods:
258
+ iqr = baseline["q3"] - baseline["q1"]
259
+ lower_bound = baseline["q1"] - self.config.iqr_multiplier * iqr
260
+ upper_bound = baseline["q3"] + self.config.iqr_multiplier * iqr
261
+
262
+ if value < lower_bound or value > upper_bound:
263
+ anomalies.append(
264
+ Anomaly(
265
+ timestamp=timestamp,
266
+ anomaly_type="value",
267
+ score=0.8,
268
+ field_name=feature,
269
+ expected_value=baseline["median"],
270
+ actual_value=value,
271
+ explanation=f"{feature} value {value:.2f} outside IQR bounds [{lower_bound:.2f}, {upper_bound:.2f}]",
272
+ context={
273
+ "iqr": iqr,
274
+ "bounds": (lower_bound, upper_bound),
275
+ "method": "iqr",
276
+ },
277
+ )
278
+ )
279
+
280
+ self.anomalies.extend(anomalies)
281
+ return anomalies
282
+
283
+ def detect_batch(
284
+ self, data_points: list[dict[str, Any]], timestamps: list[float]
285
+ ) -> list[Anomaly]:
286
+ """Detect anomalies in batch of data.
287
+
288
+ Args:
289
+ data_points: List of data points to analyze
290
+ timestamps: Corresponding timestamps for each data point
291
+
292
+ Returns:
293
+ List of all detected anomalies
294
+
295
+ Example:
296
+ >>> data = [
297
+ ... {"voltage": 3.3},
298
+ ... {"voltage": 10.0}, # Anomaly
299
+ ... ]
300
+ >>> timestamps = [0.0, 1.0]
301
+ >>> anomalies = detector.detect_batch(data, timestamps)
302
+ """
303
+ all_anomalies = []
304
+ for data, timestamp in zip(data_points, timestamps, strict=True):
305
+ anomalies = self.detect(data, timestamp)
306
+ all_anomalies.extend(anomalies)
307
+ return all_anomalies
308
+
309
+ def detect_message_rate_anomaly(
310
+ self, timestamps: list[float], window_size: int = 100
311
+ ) -> list[Anomaly]:
312
+ """Detect anomalous message rates (bursts, gaps).
313
+
314
+ Uses sliding window to calculate message rate, then detects outliers
315
+ in rate distribution using Z-score method.
316
+
317
+ Args:
318
+ timestamps: List of message timestamps
319
+ window_size: Size of sliding window for rate calculation
320
+
321
+ Returns:
322
+ List of detected rate anomalies
323
+
324
+ Example:
325
+ >>> timestamps = [0.0, 0.1, 0.2, 0.3, 5.0, 5.1] # Gap at 0.3-5.0
326
+ >>> anomalies = detector.detect_message_rate_anomaly(timestamps)
327
+ >>> for a in anomalies:
328
+ ... print(a.explanation)
329
+ Message gap detected: 2.1 msg/s vs expected 10.0 msg/s
330
+ """
331
+ if len(timestamps) < window_size:
332
+ return []
333
+
334
+ # Calculate message rates in windows
335
+ rates = []
336
+ window_timestamps = []
337
+
338
+ for i in range(len(timestamps) - window_size):
339
+ window = timestamps[i : i + window_size]
340
+ time_span = window[-1] - window[0]
341
+ rate = window_size / time_span if time_span > 0 else 0
342
+ rates.append(rate)
343
+ window_timestamps.append(window[window_size // 2])
344
+
345
+ # Detect anomalies in rates using dual-threshold approach
346
+ rate_array = np.array(rates)
347
+ mean_rate = np.mean(rate_array)
348
+ std_rate = np.std(rate_array)
349
+ median_rate = np.median(rate_array)
350
+
351
+ # For extremely constant rates, avoid false positives from floating point errors
352
+ if std_rate < mean_rate * 0.0001: # Coefficient of variation < 0.01%
353
+ return []
354
+
355
+ # Use dual-threshold approach for rate anomaly detection:
356
+ # 1. Statistical outlier detection (Z-score)
357
+ # 2. Rate deviation threshold (>30% change from median)
358
+ statistical_outliers = self._zscore_detection(rate_array, self.config.zscore_threshold)
359
+
360
+ # Rate-based threshold: detect significant rate changes (>30% deviation from median)
361
+ rate_deviations = np.abs(rate_array - median_rate) / median_rate
362
+ significant_deviations = rate_deviations > 0.3 # 30% threshold
363
+
364
+ # Combine: anomaly if either statistical outlier OR significant deviation
365
+ outliers = statistical_outliers | significant_deviations
366
+
367
+ # Create Anomaly objects
368
+ anomalies = []
369
+
370
+ for idx, is_outlier in enumerate(outliers):
371
+ if is_outlier:
372
+ if rates[idx] > mean_rate:
373
+ explanation = f"Message burst detected: {rates[idx]:.1f} msg/s vs expected {mean_rate:.1f} msg/s"
374
+ else:
375
+ explanation = f"Message gap detected: {rates[idx]:.1f} msg/s vs expected {mean_rate:.1f} msg/s"
376
+
377
+ anomalies.append(
378
+ Anomaly(
379
+ timestamp=window_timestamps[idx],
380
+ anomaly_type="rate",
381
+ score=min(abs(rates[idx] - mean_rate) / mean_rate, 1.0),
382
+ explanation=explanation,
383
+ context={"rate": rates[idx], "expected_rate": mean_rate},
384
+ )
385
+ )
386
+
387
+ self.anomalies.extend(anomalies)
388
+ return anomalies
389
+
390
+ def detect_field_value_anomaly(
391
+ self, field_values: list[float], field_name: str, method: str = "modified_zscore"
392
+ ) -> list[Anomaly]:
393
+ """Detect unusual field values using statistical methods.
394
+
395
+ Args:
396
+ field_values: List of field values to analyze
397
+ field_name: Name of the field being analyzed
398
+ method: Detection method to use (zscore, iqr, modified_zscore)
399
+
400
+ Returns:
401
+ List of detected value anomalies
402
+
403
+ Raises:
404
+ ValueError: If method is unknown
405
+
406
+ Example:
407
+ >>> values = [1.0, 1.1, 0.9, 1.2, 10.0, 1.0]
408
+ >>> anomalies = detector.detect_field_value_anomaly(values, "voltage")
409
+ >>> print(anomalies[0].explanation)
410
+ voltage value 10.00 deviates significantly from expected 2.53
411
+ """
412
+ values = np.array(field_values)
413
+
414
+ # For very small datasets (n=2), use simple ratio-based detection
415
+ if len(values) == 2:
416
+ # Detect if one value is >5x or <0.2x the other
417
+ ratio = values.max() / (values.min() + 1e-10)
418
+ if ratio > 5.0:
419
+ # The larger value is anomalous
420
+ outliers = values == values.max()
421
+ else:
422
+ outliers = np.zeros(len(values), dtype=bool)
423
+ elif method == "zscore":
424
+ outliers = self._zscore_detection(values, self.config.zscore_threshold)
425
+ elif method == "iqr":
426
+ outliers = self._iqr_detection(values, self.config.iqr_multiplier)
427
+ elif method == "modified_zscore":
428
+ outliers = self._modified_zscore_detection(values, self.config.zscore_threshold)
429
+ else:
430
+ raise ValueError(f"Unknown method: {method}")
431
+
432
+ # Create Anomaly objects
433
+ anomalies = []
434
+ mean_val = np.mean(values)
435
+
436
+ for idx, is_outlier in enumerate(outliers):
437
+ if is_outlier:
438
+ score = abs(values[idx] - mean_val) / (np.std(values) + 1e-10)
439
+ score = min(score / 5.0, 1.0) # Normalize to 0-1
440
+
441
+ anomalies.append(
442
+ Anomaly(
443
+ timestamp=float(idx), # Index as timestamp
444
+ anomaly_type="value",
445
+ score=score,
446
+ message_index=idx,
447
+ field_name=field_name,
448
+ expected_value=mean_val,
449
+ actual_value=values[idx],
450
+ explanation=f"{field_name} value {values[idx]:.2f} deviates significantly from expected {mean_val:.2f}",
451
+ context={"method": method},
452
+ )
453
+ )
454
+
455
+ self.anomalies.extend(anomalies)
456
+ return anomalies
457
+
458
+ def detect_timing_anomaly(
459
+ self,
460
+ inter_arrival_times: list[float],
461
+ expected_period: float | None = None,
462
+ ) -> list[Anomaly]:
463
+ """Detect timing anomalies (jitter, drift, unexpected delays).
464
+
465
+ Args:
466
+ inter_arrival_times: List of inter-arrival times between messages
467
+ expected_period: Expected period (None = calculate from data)
468
+
469
+ Returns:
470
+ List of detected timing anomalies
471
+
472
+ Example:
473
+ >>> inter_arrival = [0.1, 0.1, 0.1, 1.0, 0.1] # Delay at index 3
474
+ >>> anomalies = detector.detect_timing_anomaly(inter_arrival)
475
+ >>> print(anomalies[0].explanation)
476
+ Timing anomaly: 1.00s vs expected 0.28s
477
+ """
478
+ if not inter_arrival_times:
479
+ return []
480
+
481
+ times = np.array(inter_arrival_times)
482
+
483
+ # Use expected period or calculate from data
484
+ if expected_period is None:
485
+ expected_period = float(np.median(times))
486
+
487
+ # For very small datasets (n=2), use ratio-based detection
488
+ if len(times) == 2:
489
+ # Detect if one value is >5x or <0.2x the other
490
+ ratio = times.max() / (times.min() + 1e-10)
491
+ if ratio > 5.0:
492
+ # The larger value is anomalous (unexpected delay)
493
+ outliers = times == times.max()
494
+ else:
495
+ outliers = np.zeros(len(times), dtype=bool)
496
+ else:
497
+ # Use statistical detection when no expected period provided
498
+ outliers = self._zscore_detection(times, self.config.zscore_threshold)
499
+ else:
500
+ # When expected period is provided, detect deviations from it directly
501
+ # Use threshold-based detection (e.g., >50% deviation from expected)
502
+ deviations = np.abs(times - expected_period) / expected_period
503
+ outliers = deviations > 0.5 # 50% deviation threshold
504
+
505
+ # Create Anomaly objects
506
+ anomalies = []
507
+ for idx, is_outlier in enumerate(outliers):
508
+ if is_outlier:
509
+ anomalies.append(
510
+ Anomaly(
511
+ timestamp=float(idx),
512
+ anomaly_type="timing",
513
+ score=min(abs(times[idx] - expected_period) / expected_period, 1.0),
514
+ message_index=idx,
515
+ expected_value=expected_period,
516
+ actual_value=times[idx],
517
+ explanation=f"Timing anomaly: {times[idx]:.2f}s vs expected {expected_period:.2f}s",
518
+ context={"expected_period": expected_period},
519
+ )
520
+ )
521
+
522
+ self.anomalies.extend(anomalies)
523
+ return anomalies
524
+
525
+ def detect_sequence_anomaly(
526
+ self, sequences: list[list[int]], trained_model: Any = None
527
+ ) -> list[Anomaly]:
528
+ """Detect unusual byte/message sequences.
529
+
530
+ Analyzes sequences for unusual patterns. If trained_model is provided,
531
+ uses it for prediction. Otherwise uses statistical analysis.
532
+
533
+ Args:
534
+ sequences: List of byte/message sequences
535
+ trained_model: Trained sequence model (optional)
536
+
537
+ Returns:
538
+ List of detected sequence anomalies
539
+
540
+ Example:
541
+ >>> sequences = [[0x01, 0x02, 0x03], [0x01, 0x02, 0xFF]] # Last byte unusual
542
+ >>> anomalies = detector.detect_sequence_anomaly(sequences)
543
+ """
544
+ anomalies = []
545
+
546
+ # Dual-threshold approach: detect unusual sequence lengths
547
+ lengths = np.array([len(seq) for seq in sequences])
548
+ median_length = np.median(lengths)
549
+ mean_length = np.mean(lengths)
550
+
551
+ # Statistical outlier detection
552
+ statistical_outliers = self._zscore_detection(lengths, self.config.zscore_threshold)
553
+
554
+ # Length-based threshold: detect sequences >2x median length or <0.5x median length
555
+ length_ratios = lengths / median_length
556
+ significant_length_deviations = (length_ratios > 2.0) | (length_ratios < 0.5)
557
+
558
+ # Combine: anomaly if either statistical outlier OR significant length deviation
559
+ outliers = statistical_outliers | significant_length_deviations
560
+
561
+ for idx, is_outlier in enumerate(outliers):
562
+ if is_outlier:
563
+ anomalies.append(
564
+ Anomaly(
565
+ timestamp=float(idx),
566
+ anomaly_type="sequence",
567
+ score=0.7,
568
+ message_index=idx,
569
+ expected_value=mean_length,
570
+ actual_value=lengths[idx],
571
+ explanation=f"Unusual sequence length: {lengths[idx]} bytes vs expected {mean_length:.0f} bytes",
572
+ context={"sequence_length": lengths[idx]},
573
+ )
574
+ )
575
+
576
+ self.anomalies.extend(anomalies)
577
+ return anomalies
578
+
579
+ def _zscore_detection(
580
+ self, values: NDArray[np.floating[Any]], threshold: float = 3.0
581
+ ) -> NDArray[np.bool_]:
582
+ """Z-score based outlier detection.
583
+
584
+ Z-score measures how many standard deviations a value is from the mean.
585
+ Outliers are typically defined as |Z-score| > threshold (commonly 3.0).
586
+
587
+ Args:
588
+ values: Array of values to analyze
589
+ threshold: Z-score threshold for outliers (default: 3.0)
590
+
591
+ Returns:
592
+ Boolean array indicating outliers (True = outlier)
593
+
594
+ Example:
595
+ >>> values = np.array([1.0, 1.1, 0.9, 10.0])
596
+ >>> outliers = detector._zscore_detection(values, threshold=2.0)
597
+ >>> print(outliers)
598
+ [False False False True]
599
+ """
600
+ if len(values) < 2:
601
+ return np.zeros(len(values), dtype=bool)
602
+
603
+ mean = np.mean(values)
604
+ std = np.std(values)
605
+
606
+ if std == 0:
607
+ return np.zeros(len(values), dtype=bool)
608
+
609
+ z_scores = np.abs((values - mean) / std)
610
+ outliers: NDArray[np.bool_] = z_scores > threshold
611
+
612
+ return outliers
613
+
614
+ def _iqr_detection(
615
+ self, values: NDArray[np.floating[Any]], multiplier: float = 1.5
616
+ ) -> NDArray[np.bool_]:
617
+ """Interquartile range (IQR) based outlier detection.
618
+
619
+ IQR is the range between 25th and 75th percentiles. Outliers are
620
+ values outside [Q1 - multiplier*IQR, Q3 + multiplier*IQR].
621
+
622
+ Args:
623
+ values: Array of values to analyze
624
+ multiplier: IQR multiplier for bounds (default: 1.5)
625
+
626
+ Returns:
627
+ Boolean array indicating outliers (True = outlier)
628
+
629
+ Example:
630
+ >>> values = np.array([1.0, 1.1, 0.9, 10.0])
631
+ >>> outliers = detector._iqr_detection(values)
632
+ >>> print(outliers)
633
+ [False False False True]
634
+ """
635
+ if len(values) < 4:
636
+ return np.zeros(len(values), dtype=bool)
637
+
638
+ q1 = np.percentile(values, 25)
639
+ q3 = np.percentile(values, 75)
640
+ iqr = q3 - q1
641
+
642
+ lower_bound = q1 - multiplier * iqr
643
+ upper_bound = q3 + multiplier * iqr
644
+
645
+ outliers: NDArray[np.bool_] = (values < lower_bound) | (values > upper_bound)
646
+
647
+ return outliers
648
+
649
+ def _modified_zscore_detection(
650
+ self, values: NDArray[np.floating[Any]], threshold: float = 2.5
651
+ ) -> NDArray[np.bool_]:
652
+ """Modified Z-score based outlier detection using median absolute deviation.
653
+
654
+ More robust to outliers than standard Z-score. Uses median instead of mean
655
+ and MAD instead of standard deviation.
656
+
657
+ Args:
658
+ values: Array of values to analyze
659
+ threshold: Modified Z-score threshold (default: 2.5, lower than standard 3.5)
660
+
661
+ Returns:
662
+ Boolean array indicating outliers (True = outlier)
663
+
664
+ Example:
665
+ >>> values = np.array([1.0, 1.1, 0.9, 10.0])
666
+ >>> outliers = detector._modified_zscore_detection(values)
667
+ >>> print(outliers)
668
+ [False False False True]
669
+ """
670
+ if len(values) < 2:
671
+ return np.zeros(len(values), dtype=bool)
672
+
673
+ median = np.median(values)
674
+ mad = np.median(np.abs(values - median))
675
+
676
+ if mad == 0:
677
+ # When MAD is 0, use a different approach
678
+ # Check if any values differ from the median
679
+ result_outliers: NDArray[np.bool_] = values != median
680
+ return result_outliers
681
+
682
+ # Modified Z-score = 0.6745 * (x - median) / MAD
683
+ modified_z_scores = 0.6745 * np.abs(values - median) / mad
684
+ outliers: NDArray[np.bool_] = modified_z_scores > threshold
685
+
686
+ return outliers
687
+
688
+ def _isolation_forest_detection(
689
+ self, X: NDArray[np.floating[Any]], contamination: float = 0.1
690
+ ) -> NDArray[np.bool_]:
691
+ """Isolation Forest based anomaly detection.
692
+
693
+ Isolation Forest isolates anomalies by randomly selecting features
694
+ and split values. Anomalies require fewer splits to isolate.
695
+
696
+ Args:
697
+ X: Feature matrix (n_samples, n_features)
698
+ contamination: Expected outlier fraction (0.0-0.5)
699
+
700
+ Returns:
701
+ Boolean array indicating outliers (True = outlier)
702
+
703
+ Raises:
704
+ ImportError: If scikit-learn is not installed
705
+
706
+ Example:
707
+ >>> X = np.array([[1.0, 2.0], [1.1, 2.1], [10.0, 20.0]])
708
+ >>> outliers = detector._isolation_forest_detection(X)
709
+ """
710
+ try:
711
+ from sklearn.ensemble import IsolationForest
712
+ except ImportError as e:
713
+ raise ImportError(
714
+ "scikit-learn is required for Isolation Forest. "
715
+ "Install with: pip install scikit-learn"
716
+ ) from e
717
+
718
+ if len(X.shape) == 1:
719
+ X = X.reshape(-1, 1)
720
+
721
+ model = IsolationForest(contamination=contamination, random_state=42, n_estimators=100)
722
+
723
+ predictions = model.fit_predict(X)
724
+ outliers: NDArray[np.bool_] = predictions == -1
725
+
726
+ # Store model for later use
727
+ self.models["isolation_forest"] = model
728
+
729
+ return outliers
730
+
731
+ def _one_class_svm_detection(
732
+ self, X: NDArray[np.floating[Any]], nu: float = 0.1
733
+ ) -> NDArray[np.bool_]:
734
+ """One-Class SVM based anomaly detection.
735
+
736
+ One-Class SVM learns a decision boundary around normal data points.
737
+ Points outside this boundary are considered anomalies.
738
+
739
+ Args:
740
+ X: Feature matrix (n_samples, n_features)
741
+ nu: Upper bound on fraction of outliers (0.0-1.0)
742
+
743
+ Returns:
744
+ Boolean array indicating outliers (True = outlier)
745
+
746
+ Raises:
747
+ ImportError: If scikit-learn is not installed
748
+
749
+ Example:
750
+ >>> X = np.array([[1.0, 2.0], [1.1, 2.1], [10.0, 20.0]])
751
+ >>> outliers = detector._one_class_svm_detection(X)
752
+ """
753
+ try:
754
+ from sklearn.svm import OneClassSVM
755
+ except ImportError as e:
756
+ raise ImportError(
757
+ "scikit-learn is required for One-Class SVM. Install with: pip install scikit-learn"
758
+ ) from e
759
+
760
+ if len(X.shape) == 1:
761
+ X = X.reshape(-1, 1)
762
+
763
+ model = OneClassSVM(nu=nu, kernel="rbf", gamma="auto")
764
+
765
+ predictions = model.fit_predict(X)
766
+ outliers: NDArray[np.bool_] = predictions == -1
767
+
768
+ # Store model for later use
769
+ self.models["one_class_svm"] = model
770
+
771
+ return outliers
772
+
773
+ def _train_isolation_forest(self, X: NDArray[np.floating[Any]]) -> None:
774
+ """Train Isolation Forest model.
775
+
776
+ Args:
777
+ X: Training data (n_samples, n_features)
778
+ """
779
+ try:
780
+ from sklearn.ensemble import IsolationForest
781
+ except ImportError:
782
+ logger.warning("scikit-learn not available, skipping Isolation Forest training")
783
+ return
784
+
785
+ model = IsolationForest(
786
+ contamination=self.config.contamination, random_state=42, n_estimators=100
787
+ )
788
+ model.fit(X)
789
+ self.models["isolation_forest"] = model
790
+
791
+ def _train_one_class_svm(self, X: NDArray[np.floating[Any]]) -> None:
792
+ """Train One-Class SVM model.
793
+
794
+ Args:
795
+ X: Training data (n_samples, n_features)
796
+ """
797
+ try:
798
+ from sklearn.svm import OneClassSVM
799
+ except ImportError:
800
+ logger.warning("scikit-learn not available, skipping One-Class SVM training")
801
+ return
802
+
803
+ model = OneClassSVM(nu=self.config.contamination, kernel="rbf", gamma="auto")
804
+ model.fit(X)
805
+ self.models["one_class_svm"] = model
806
+
807
+ def export_report(self, output_path: Path, format: str = "json") -> None:
808
+ """Export anomaly report with context and explanations.
809
+
810
+ Args:
811
+ output_path: Path to output file
812
+ format: Export format (json, txt)
813
+
814
+ Raises:
815
+ ValueError: If format is unsupported
816
+
817
+ Example:
818
+ >>> detector.export_report(Path("anomalies.json"), format="json")
819
+ >>> detector.export_report(Path("anomalies.txt"), format="txt")
820
+ """
821
+ if format == "json":
822
+ self._export_json(output_path)
823
+ elif format == "txt":
824
+ self._export_txt(output_path)
825
+ else:
826
+ raise ValueError(f"Unsupported format: {format}")
827
+
828
+ def _export_json(self, output_path: Path) -> None:
829
+ """Export anomalies to JSON format.
830
+
831
+ Args:
832
+ output_path: Path to output JSON file
833
+ """
834
+ report = {
835
+ "config": {
836
+ "methods": self.config.methods,
837
+ "zscore_threshold": self.config.zscore_threshold,
838
+ "iqr_multiplier": self.config.iqr_multiplier,
839
+ "contamination": self.config.contamination,
840
+ "window_size": self.config.window_size,
841
+ },
842
+ "summary": {
843
+ "total_anomalies": len(self.anomalies),
844
+ "by_type": self._count_by_type(),
845
+ },
846
+ "anomalies": [
847
+ {
848
+ "timestamp": a.timestamp,
849
+ "anomaly_type": a.anomaly_type,
850
+ "score": a.score,
851
+ "message_index": a.message_index,
852
+ "field_name": a.field_name,
853
+ "expected_value": a.expected_value,
854
+ "actual_value": a.actual_value,
855
+ "explanation": a.explanation,
856
+ "context": a.context,
857
+ }
858
+ for a in self.anomalies
859
+ ],
860
+ }
861
+
862
+ with output_path.open("w") as f:
863
+ json.dump(report, f, indent=2)
864
+
865
+ def _export_txt(self, output_path: Path) -> None:
866
+ """Export anomalies to text format.
867
+
868
+ Args:
869
+ output_path: Path to output text file
870
+ """
871
+ with output_path.open("w") as f:
872
+ f.write("Anomaly Detection Report\n")
873
+ f.write("=" * 80 + "\n\n")
874
+
875
+ f.write("Configuration:\n")
876
+ f.write(f" Methods: {', '.join(self.config.methods)}\n")
877
+ f.write(f" Z-score threshold: {self.config.zscore_threshold}\n")
878
+ f.write(f" IQR multiplier: {self.config.iqr_multiplier}\n\n")
879
+
880
+ f.write("Summary:\n")
881
+ f.write(f" Total anomalies: {len(self.anomalies)}\n")
882
+ f.write(" By type:\n")
883
+ for anomaly_type, count in self._count_by_type().items():
884
+ f.write(f" {anomaly_type}: {count}\n")
885
+ f.write("\n")
886
+
887
+ f.write("Anomalies:\n")
888
+ f.write("-" * 80 + "\n")
889
+ for i, a in enumerate(self.anomalies, 1):
890
+ f.write(f"{i}. [{a.anomaly_type}] {a.explanation}\n")
891
+ f.write(f" Timestamp: {a.timestamp:.3f}\n")
892
+ f.write(f" Score: {a.score:.3f}\n")
893
+ if a.field_name:
894
+ f.write(f" Field: {a.field_name}\n")
895
+ if a.message_index is not None:
896
+ f.write(f" Message index: {a.message_index}\n")
897
+ f.write("\n")
898
+
899
+ def _count_by_type(self) -> dict[str, int]:
900
+ """Count anomalies by type.
901
+
902
+ Returns:
903
+ Dictionary mapping anomaly type to count
904
+ """
905
+ counts: dict[str, int] = {}
906
+ for anomaly in self.anomalies:
907
+ counts[anomaly.anomaly_type] = counts.get(anomaly.anomaly_type, 0) + 1
908
+ return counts