oscura 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oscura/__init__.py +169 -167
- oscura/analyzers/__init__.py +3 -0
- oscura/analyzers/classification.py +659 -0
- oscura/analyzers/digital/__init__.py +0 -48
- oscura/analyzers/digital/edges.py +325 -65
- oscura/analyzers/digital/extraction.py +0 -195
- oscura/analyzers/digital/quality.py +293 -166
- oscura/analyzers/digital/timing.py +260 -115
- oscura/analyzers/digital/timing_numba.py +334 -0
- oscura/analyzers/entropy.py +605 -0
- oscura/analyzers/eye/diagram.py +176 -109
- oscura/analyzers/eye/metrics.py +5 -5
- oscura/analyzers/jitter/__init__.py +6 -4
- oscura/analyzers/jitter/ber.py +52 -52
- oscura/analyzers/jitter/classification.py +156 -0
- oscura/analyzers/jitter/decomposition.py +163 -113
- oscura/analyzers/jitter/spectrum.py +80 -64
- oscura/analyzers/ml/__init__.py +39 -0
- oscura/analyzers/ml/features.py +600 -0
- oscura/analyzers/ml/signal_classifier.py +604 -0
- oscura/analyzers/packet/daq.py +246 -158
- oscura/analyzers/packet/parser.py +12 -1
- oscura/analyzers/packet/payload.py +50 -2110
- oscura/analyzers/packet/payload_analysis.py +361 -181
- oscura/analyzers/packet/payload_patterns.py +133 -70
- oscura/analyzers/packet/stream.py +84 -23
- oscura/analyzers/patterns/__init__.py +26 -5
- oscura/analyzers/patterns/anomaly_detection.py +908 -0
- oscura/analyzers/patterns/clustering.py +169 -108
- oscura/analyzers/patterns/clustering_optimized.py +227 -0
- oscura/analyzers/patterns/discovery.py +1 -1
- oscura/analyzers/patterns/matching.py +581 -197
- oscura/analyzers/patterns/pattern_mining.py +778 -0
- oscura/analyzers/patterns/periodic.py +121 -38
- oscura/analyzers/patterns/sequences.py +175 -78
- oscura/analyzers/power/conduction.py +1 -1
- oscura/analyzers/power/soa.py +6 -6
- oscura/analyzers/power/switching.py +250 -110
- oscura/analyzers/protocol/__init__.py +17 -1
- oscura/analyzers/protocols/__init__.py +1 -22
- oscura/analyzers/protocols/base.py +6 -6
- oscura/analyzers/protocols/ble/__init__.py +38 -0
- oscura/analyzers/protocols/ble/analyzer.py +809 -0
- oscura/analyzers/protocols/ble/uuids.py +288 -0
- oscura/analyzers/protocols/can.py +257 -127
- oscura/analyzers/protocols/can_fd.py +107 -80
- oscura/analyzers/protocols/flexray.py +139 -80
- oscura/analyzers/protocols/hdlc.py +93 -58
- oscura/analyzers/protocols/i2c.py +247 -106
- oscura/analyzers/protocols/i2s.py +138 -86
- oscura/analyzers/protocols/industrial/__init__.py +40 -0
- oscura/analyzers/protocols/industrial/bacnet/__init__.py +33 -0
- oscura/analyzers/protocols/industrial/bacnet/analyzer.py +708 -0
- oscura/analyzers/protocols/industrial/bacnet/encoding.py +412 -0
- oscura/analyzers/protocols/industrial/bacnet/services.py +622 -0
- oscura/analyzers/protocols/industrial/ethercat/__init__.py +30 -0
- oscura/analyzers/protocols/industrial/ethercat/analyzer.py +474 -0
- oscura/analyzers/protocols/industrial/ethercat/mailbox.py +339 -0
- oscura/analyzers/protocols/industrial/ethercat/topology.py +166 -0
- oscura/analyzers/protocols/industrial/modbus/__init__.py +31 -0
- oscura/analyzers/protocols/industrial/modbus/analyzer.py +525 -0
- oscura/analyzers/protocols/industrial/modbus/crc.py +79 -0
- oscura/analyzers/protocols/industrial/modbus/functions.py +436 -0
- oscura/analyzers/protocols/industrial/opcua/__init__.py +21 -0
- oscura/analyzers/protocols/industrial/opcua/analyzer.py +552 -0
- oscura/analyzers/protocols/industrial/opcua/datatypes.py +446 -0
- oscura/analyzers/protocols/industrial/opcua/services.py +264 -0
- oscura/analyzers/protocols/industrial/profinet/__init__.py +23 -0
- oscura/analyzers/protocols/industrial/profinet/analyzer.py +441 -0
- oscura/analyzers/protocols/industrial/profinet/dcp.py +263 -0
- oscura/analyzers/protocols/industrial/profinet/ptcp.py +200 -0
- oscura/analyzers/protocols/jtag.py +180 -98
- oscura/analyzers/protocols/lin.py +219 -114
- oscura/analyzers/protocols/manchester.py +4 -4
- oscura/analyzers/protocols/onewire.py +253 -149
- oscura/analyzers/protocols/parallel_bus/__init__.py +20 -0
- oscura/analyzers/protocols/parallel_bus/centronics.py +92 -0
- oscura/analyzers/protocols/parallel_bus/gpib.py +137 -0
- oscura/analyzers/protocols/spi.py +192 -95
- oscura/analyzers/protocols/swd.py +321 -167
- oscura/analyzers/protocols/uart.py +267 -125
- oscura/analyzers/protocols/usb.py +235 -131
- oscura/analyzers/side_channel/power.py +17 -12
- oscura/analyzers/signal/__init__.py +15 -0
- oscura/analyzers/signal/timing_analysis.py +1086 -0
- oscura/analyzers/signal_integrity/__init__.py +4 -1
- oscura/analyzers/signal_integrity/sparams.py +2 -19
- oscura/analyzers/spectral/chunked.py +129 -60
- oscura/analyzers/spectral/chunked_fft.py +300 -94
- oscura/analyzers/spectral/chunked_wavelet.py +100 -80
- oscura/analyzers/statistical/checksum.py +376 -217
- oscura/analyzers/statistical/classification.py +229 -107
- oscura/analyzers/statistical/entropy.py +78 -53
- oscura/analyzers/statistics/correlation.py +407 -211
- oscura/analyzers/statistics/outliers.py +2 -2
- oscura/analyzers/statistics/streaming.py +30 -5
- oscura/analyzers/validation.py +216 -101
- oscura/analyzers/waveform/measurements.py +9 -0
- oscura/analyzers/waveform/measurements_with_uncertainty.py +31 -15
- oscura/analyzers/waveform/spectral.py +500 -228
- oscura/api/__init__.py +31 -5
- oscura/api/dsl/__init__.py +582 -0
- oscura/{dsl → api/dsl}/commands.py +43 -76
- oscura/{dsl → api/dsl}/interpreter.py +26 -51
- oscura/{dsl → api/dsl}/parser.py +107 -77
- oscura/{dsl → api/dsl}/repl.py +2 -2
- oscura/api/dsl.py +1 -1
- oscura/{integrations → api/integrations}/__init__.py +1 -1
- oscura/{integrations → api/integrations}/llm.py +201 -102
- oscura/api/operators.py +3 -3
- oscura/api/optimization.py +144 -30
- oscura/api/rest_server.py +921 -0
- oscura/api/server/__init__.py +17 -0
- oscura/api/server/dashboard.py +850 -0
- oscura/api/server/static/README.md +34 -0
- oscura/api/server/templates/base.html +181 -0
- oscura/api/server/templates/export.html +120 -0
- oscura/api/server/templates/home.html +284 -0
- oscura/api/server/templates/protocols.html +58 -0
- oscura/api/server/templates/reports.html +43 -0
- oscura/api/server/templates/session_detail.html +89 -0
- oscura/api/server/templates/sessions.html +83 -0
- oscura/api/server/templates/waveforms.html +73 -0
- oscura/automotive/__init__.py +8 -1
- oscura/automotive/can/__init__.py +10 -0
- oscura/automotive/can/checksum.py +3 -1
- oscura/automotive/can/dbc_generator.py +590 -0
- oscura/automotive/can/message_wrapper.py +121 -74
- oscura/automotive/can/patterns.py +98 -21
- oscura/automotive/can/session.py +292 -56
- oscura/automotive/can/state_machine.py +6 -3
- oscura/automotive/can/stimulus_response.py +97 -75
- oscura/automotive/dbc/__init__.py +10 -2
- oscura/automotive/dbc/generator.py +84 -56
- oscura/automotive/dbc/parser.py +6 -6
- oscura/automotive/dtc/data.json +2763 -0
- oscura/automotive/dtc/database.py +2 -2
- oscura/automotive/flexray/__init__.py +31 -0
- oscura/automotive/flexray/analyzer.py +504 -0
- oscura/automotive/flexray/crc.py +185 -0
- oscura/automotive/flexray/fibex.py +449 -0
- oscura/automotive/j1939/__init__.py +45 -8
- oscura/automotive/j1939/analyzer.py +605 -0
- oscura/automotive/j1939/spns.py +326 -0
- oscura/automotive/j1939/transport.py +306 -0
- oscura/automotive/lin/__init__.py +47 -0
- oscura/automotive/lin/analyzer.py +612 -0
- oscura/automotive/loaders/blf.py +13 -2
- oscura/automotive/loaders/csv_can.py +143 -72
- oscura/automotive/loaders/dispatcher.py +50 -2
- oscura/automotive/loaders/mdf.py +86 -45
- oscura/automotive/loaders/pcap.py +111 -61
- oscura/automotive/uds/__init__.py +4 -0
- oscura/automotive/uds/analyzer.py +725 -0
- oscura/automotive/uds/decoder.py +140 -58
- oscura/automotive/uds/models.py +7 -1
- oscura/automotive/visualization.py +1 -1
- oscura/cli/analyze.py +348 -0
- oscura/cli/batch.py +142 -122
- oscura/cli/benchmark.py +275 -0
- oscura/cli/characterize.py +137 -82
- oscura/cli/compare.py +224 -131
- oscura/cli/completion.py +250 -0
- oscura/cli/config_cmd.py +361 -0
- oscura/cli/decode.py +164 -87
- oscura/cli/export.py +286 -0
- oscura/cli/main.py +115 -31
- oscura/{onboarding → cli/onboarding}/__init__.py +3 -3
- oscura/{onboarding → cli/onboarding}/help.py +80 -58
- oscura/{onboarding → cli/onboarding}/tutorials.py +97 -72
- oscura/{onboarding → cli/onboarding}/wizard.py +55 -36
- oscura/cli/progress.py +147 -0
- oscura/cli/shell.py +157 -135
- oscura/cli/validate_cmd.py +204 -0
- oscura/cli/visualize.py +158 -0
- oscura/convenience.py +125 -79
- oscura/core/__init__.py +4 -2
- oscura/core/backend_selector.py +3 -3
- oscura/core/cache.py +126 -15
- oscura/core/cancellation.py +1 -1
- oscura/{config → core/config}/__init__.py +20 -11
- oscura/{config → core/config}/defaults.py +1 -1
- oscura/{config → core/config}/loader.py +7 -5
- oscura/{config → core/config}/memory.py +5 -5
- oscura/{config → core/config}/migration.py +1 -1
- oscura/{config → core/config}/pipeline.py +99 -23
- oscura/{config → core/config}/preferences.py +1 -1
- oscura/{config → core/config}/protocol.py +3 -3
- oscura/{config → core/config}/schema.py +426 -272
- oscura/{config → core/config}/settings.py +1 -1
- oscura/{config → core/config}/thresholds.py +195 -153
- oscura/core/correlation.py +5 -6
- oscura/core/cross_domain.py +0 -2
- oscura/core/debug.py +9 -5
- oscura/{extensibility → core/extensibility}/docs.py +158 -70
- oscura/{extensibility → core/extensibility}/extensions.py +160 -76
- oscura/{extensibility → core/extensibility}/logging.py +1 -1
- oscura/{extensibility → core/extensibility}/measurements.py +1 -1
- oscura/{extensibility → core/extensibility}/plugins.py +1 -1
- oscura/{extensibility → core/extensibility}/templates.py +73 -3
- oscura/{extensibility → core/extensibility}/validation.py +1 -1
- oscura/core/gpu_backend.py +11 -7
- oscura/core/log_query.py +101 -11
- oscura/core/logging.py +126 -54
- oscura/core/logging_advanced.py +5 -5
- oscura/core/memory_limits.py +108 -70
- oscura/core/memory_monitor.py +2 -2
- oscura/core/memory_progress.py +7 -7
- oscura/core/memory_warnings.py +1 -1
- oscura/core/numba_backend.py +13 -13
- oscura/{plugins → core/plugins}/__init__.py +9 -9
- oscura/{plugins → core/plugins}/base.py +7 -7
- oscura/{plugins → core/plugins}/cli.py +3 -3
- oscura/{plugins → core/plugins}/discovery.py +186 -106
- oscura/{plugins → core/plugins}/lifecycle.py +1 -1
- oscura/{plugins → core/plugins}/manager.py +7 -7
- oscura/{plugins → core/plugins}/registry.py +3 -3
- oscura/{plugins → core/plugins}/versioning.py +1 -1
- oscura/core/progress.py +16 -1
- oscura/core/provenance.py +8 -2
- oscura/{schemas → core/schemas}/__init__.py +2 -2
- oscura/core/schemas/bus_configuration.json +322 -0
- oscura/core/schemas/device_mapping.json +182 -0
- oscura/core/schemas/packet_format.json +418 -0
- oscura/core/schemas/protocol_definition.json +363 -0
- oscura/core/types.py +4 -0
- oscura/core/uncertainty.py +3 -3
- oscura/correlation/__init__.py +52 -0
- oscura/correlation/multi_protocol.py +811 -0
- oscura/discovery/auto_decoder.py +117 -35
- oscura/discovery/comparison.py +191 -86
- oscura/discovery/quality_validator.py +155 -68
- oscura/discovery/signal_detector.py +196 -79
- oscura/export/__init__.py +18 -20
- oscura/export/kaitai_struct.py +513 -0
- oscura/export/scapy_layer.py +801 -0
- oscura/export/wireshark/README.md +15 -15
- oscura/export/wireshark/generator.py +1 -1
- oscura/export/wireshark/templates/dissector.lua.j2 +2 -2
- oscura/export/wireshark_dissector.py +746 -0
- oscura/guidance/wizard.py +207 -111
- oscura/hardware/__init__.py +19 -0
- oscura/{acquisition → hardware/acquisition}/__init__.py +4 -4
- oscura/{acquisition → hardware/acquisition}/file.py +2 -2
- oscura/{acquisition → hardware/acquisition}/hardware.py +7 -7
- oscura/{acquisition → hardware/acquisition}/saleae.py +15 -12
- oscura/{acquisition → hardware/acquisition}/socketcan.py +1 -1
- oscura/{acquisition → hardware/acquisition}/streaming.py +2 -2
- oscura/{acquisition → hardware/acquisition}/synthetic.py +3 -3
- oscura/{acquisition → hardware/acquisition}/visa.py +33 -11
- oscura/hardware/firmware/__init__.py +29 -0
- oscura/hardware/firmware/pattern_recognition.py +874 -0
- oscura/hardware/hal_detector.py +736 -0
- oscura/hardware/security/__init__.py +37 -0
- oscura/hardware/security/side_channel_detector.py +1126 -0
- oscura/inference/__init__.py +4 -0
- oscura/inference/active_learning/README.md +7 -7
- oscura/inference/active_learning/observation_table.py +4 -1
- oscura/inference/alignment.py +216 -123
- oscura/inference/bayesian.py +113 -33
- oscura/inference/crc_reverse.py +101 -55
- oscura/inference/logic.py +6 -2
- oscura/inference/message_format.py +342 -183
- oscura/inference/protocol.py +95 -44
- oscura/inference/protocol_dsl.py +180 -82
- oscura/inference/signal_intelligence.py +1439 -706
- oscura/inference/spectral.py +99 -57
- oscura/inference/state_machine.py +810 -158
- oscura/inference/stream.py +270 -110
- oscura/iot/__init__.py +34 -0
- oscura/iot/coap/__init__.py +32 -0
- oscura/iot/coap/analyzer.py +668 -0
- oscura/iot/coap/options.py +212 -0
- oscura/iot/lorawan/__init__.py +21 -0
- oscura/iot/lorawan/crypto.py +206 -0
- oscura/iot/lorawan/decoder.py +801 -0
- oscura/iot/lorawan/mac_commands.py +341 -0
- oscura/iot/mqtt/__init__.py +27 -0
- oscura/iot/mqtt/analyzer.py +999 -0
- oscura/iot/mqtt/properties.py +315 -0
- oscura/iot/zigbee/__init__.py +31 -0
- oscura/iot/zigbee/analyzer.py +615 -0
- oscura/iot/zigbee/security.py +153 -0
- oscura/iot/zigbee/zcl.py +349 -0
- oscura/jupyter/display.py +125 -45
- oscura/{exploratory → jupyter/exploratory}/__init__.py +8 -8
- oscura/{exploratory → jupyter/exploratory}/error_recovery.py +298 -141
- oscura/jupyter/exploratory/fuzzy.py +746 -0
- oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py +258 -100
- oscura/{exploratory → jupyter/exploratory}/legacy.py +464 -242
- oscura/{exploratory → jupyter/exploratory}/parse.py +167 -145
- oscura/{exploratory → jupyter/exploratory}/recovery.py +119 -87
- oscura/jupyter/exploratory/sync.py +612 -0
- oscura/{exploratory → jupyter/exploratory}/unknown.py +299 -176
- oscura/jupyter/magic.py +4 -4
- oscura/{ui → jupyter/ui}/__init__.py +2 -2
- oscura/{ui → jupyter/ui}/formatters.py +3 -3
- oscura/{ui → jupyter/ui}/progressive_display.py +153 -82
- oscura/loaders/__init__.py +171 -63
- oscura/loaders/binary.py +88 -1
- oscura/loaders/chipwhisperer.py +153 -137
- oscura/loaders/configurable.py +208 -86
- oscura/loaders/csv_loader.py +458 -215
- oscura/loaders/hdf5_loader.py +278 -119
- oscura/loaders/lazy.py +87 -54
- oscura/loaders/mmap_loader.py +1 -1
- oscura/loaders/numpy_loader.py +253 -116
- oscura/loaders/pcap.py +226 -151
- oscura/loaders/rigol.py +110 -49
- oscura/loaders/sigrok.py +201 -78
- oscura/loaders/tdms.py +81 -58
- oscura/loaders/tektronix.py +291 -174
- oscura/loaders/touchstone.py +182 -87
- oscura/loaders/vcd.py +215 -117
- oscura/loaders/wav.py +155 -68
- oscura/reporting/__init__.py +9 -7
- oscura/reporting/analyze.py +352 -146
- oscura/reporting/argument_preparer.py +69 -14
- oscura/reporting/auto_report.py +97 -61
- oscura/reporting/batch.py +131 -58
- oscura/reporting/chart_selection.py +57 -45
- oscura/reporting/comparison.py +63 -17
- oscura/reporting/content/executive.py +76 -24
- oscura/reporting/core_formats/multi_format.py +11 -8
- oscura/reporting/engine.py +312 -158
- oscura/reporting/enhanced_reports.py +949 -0
- oscura/reporting/export.py +86 -43
- oscura/reporting/formatting/numbers.py +69 -42
- oscura/reporting/html.py +139 -58
- oscura/reporting/index.py +137 -65
- oscura/reporting/output.py +158 -67
- oscura/reporting/pdf.py +67 -102
- oscura/reporting/plots.py +191 -112
- oscura/reporting/sections.py +88 -47
- oscura/reporting/standards.py +104 -61
- oscura/reporting/summary_generator.py +75 -55
- oscura/reporting/tables.py +138 -54
- oscura/reporting/templates/enhanced/protocol_re.html +525 -0
- oscura/reporting/templates/index.md +13 -13
- oscura/sessions/__init__.py +14 -23
- oscura/sessions/base.py +3 -3
- oscura/sessions/blackbox.py +106 -10
- oscura/sessions/generic.py +2 -2
- oscura/sessions/legacy.py +783 -0
- oscura/side_channel/__init__.py +63 -0
- oscura/side_channel/dpa.py +1025 -0
- oscura/utils/__init__.py +15 -1
- oscura/utils/autodetect.py +1 -5
- oscura/utils/bitwise.py +118 -0
- oscura/{builders → utils/builders}/__init__.py +1 -1
- oscura/{comparison → utils/comparison}/__init__.py +6 -6
- oscura/{comparison → utils/comparison}/compare.py +202 -101
- oscura/{comparison → utils/comparison}/golden.py +83 -63
- oscura/{comparison → utils/comparison}/limits.py +313 -89
- oscura/{comparison → utils/comparison}/mask.py +151 -45
- oscura/{comparison → utils/comparison}/trace_diff.py +1 -1
- oscura/{comparison → utils/comparison}/visualization.py +147 -89
- oscura/{component → utils/component}/__init__.py +3 -3
- oscura/{component → utils/component}/impedance.py +122 -58
- oscura/{component → utils/component}/reactive.py +165 -168
- oscura/{component → utils/component}/transmission_line.py +3 -3
- oscura/{filtering → utils/filtering}/__init__.py +6 -6
- oscura/{filtering → utils/filtering}/base.py +1 -1
- oscura/{filtering → utils/filtering}/convenience.py +2 -2
- oscura/{filtering → utils/filtering}/design.py +169 -93
- oscura/{filtering → utils/filtering}/filters.py +2 -2
- oscura/{filtering → utils/filtering}/introspection.py +2 -2
- oscura/utils/geometry.py +31 -0
- oscura/utils/imports.py +184 -0
- oscura/utils/lazy.py +1 -1
- oscura/{math → utils/math}/__init__.py +2 -2
- oscura/{math → utils/math}/arithmetic.py +114 -48
- oscura/{math → utils/math}/interpolation.py +139 -106
- oscura/utils/memory.py +129 -66
- oscura/utils/memory_advanced.py +92 -9
- oscura/utils/memory_extensions.py +10 -8
- oscura/{optimization → utils/optimization}/__init__.py +1 -1
- oscura/{optimization → utils/optimization}/search.py +2 -2
- oscura/utils/performance/__init__.py +58 -0
- oscura/utils/performance/caching.py +889 -0
- oscura/utils/performance/lsh_clustering.py +333 -0
- oscura/utils/performance/memory_optimizer.py +699 -0
- oscura/utils/performance/optimizations.py +675 -0
- oscura/utils/performance/parallel.py +654 -0
- oscura/utils/performance/profiling.py +661 -0
- oscura/{pipeline → utils/pipeline}/base.py +1 -1
- oscura/{pipeline → utils/pipeline}/composition.py +11 -3
- oscura/{pipeline → utils/pipeline}/parallel.py +3 -2
- oscura/{pipeline → utils/pipeline}/pipeline.py +1 -1
- oscura/{pipeline → utils/pipeline}/reverse_engineering.py +412 -221
- oscura/{search → utils/search}/__init__.py +3 -3
- oscura/{search → utils/search}/anomaly.py +188 -58
- oscura/utils/search/context.py +294 -0
- oscura/{search → utils/search}/pattern.py +138 -10
- oscura/utils/serial.py +51 -0
- oscura/utils/storage/__init__.py +61 -0
- oscura/utils/storage/database.py +1166 -0
- oscura/{streaming → utils/streaming}/chunked.py +302 -143
- oscura/{streaming → utils/streaming}/progressive.py +1 -1
- oscura/{streaming → utils/streaming}/realtime.py +3 -2
- oscura/{triggering → utils/triggering}/__init__.py +6 -6
- oscura/{triggering → utils/triggering}/base.py +6 -6
- oscura/{triggering → utils/triggering}/edge.py +2 -2
- oscura/{triggering → utils/triggering}/pattern.py +2 -2
- oscura/{triggering → utils/triggering}/pulse.py +115 -74
- oscura/{triggering → utils/triggering}/window.py +2 -2
- oscura/utils/validation.py +32 -0
- oscura/validation/__init__.py +121 -0
- oscura/{compliance → validation/compliance}/__init__.py +5 -5
- oscura/{compliance → validation/compliance}/advanced.py +5 -5
- oscura/{compliance → validation/compliance}/masks.py +1 -1
- oscura/{compliance → validation/compliance}/reporting.py +127 -53
- oscura/{compliance → validation/compliance}/testing.py +114 -52
- oscura/validation/compliance_tests.py +915 -0
- oscura/validation/fuzzer.py +990 -0
- oscura/validation/grammar_tests.py +596 -0
- oscura/validation/grammar_validator.py +904 -0
- oscura/validation/hil_testing.py +977 -0
- oscura/{quality → validation/quality}/__init__.py +4 -4
- oscura/{quality → validation/quality}/ensemble.py +251 -171
- oscura/{quality → validation/quality}/explainer.py +3 -3
- oscura/{quality → validation/quality}/scoring.py +1 -1
- oscura/{quality → validation/quality}/warnings.py +4 -4
- oscura/validation/regression_suite.py +808 -0
- oscura/validation/replay.py +788 -0
- oscura/{testing → validation/testing}/__init__.py +2 -2
- oscura/{testing → validation/testing}/synthetic.py +5 -5
- oscura/visualization/__init__.py +9 -0
- oscura/visualization/accessibility.py +1 -1
- oscura/visualization/annotations.py +64 -67
- oscura/visualization/colors.py +7 -7
- oscura/visualization/digital.py +180 -81
- oscura/visualization/eye.py +236 -85
- oscura/visualization/interactive.py +320 -143
- oscura/visualization/jitter.py +587 -247
- oscura/visualization/layout.py +169 -134
- oscura/visualization/optimization.py +103 -52
- oscura/visualization/palettes.py +1 -1
- oscura/visualization/power.py +427 -211
- oscura/visualization/power_extended.py +626 -297
- oscura/visualization/presets.py +2 -0
- oscura/visualization/protocols.py +495 -181
- oscura/visualization/render.py +79 -63
- oscura/visualization/reverse_engineering.py +171 -124
- oscura/visualization/signal_integrity.py +460 -279
- oscura/visualization/specialized.py +190 -100
- oscura/visualization/spectral.py +670 -255
- oscura/visualization/thumbnails.py +166 -137
- oscura/visualization/waveform.py +150 -63
- oscura/workflows/__init__.py +3 -0
- oscura/{batch → workflows/batch}/__init__.py +5 -5
- oscura/{batch → workflows/batch}/advanced.py +150 -75
- oscura/workflows/batch/aggregate.py +531 -0
- oscura/workflows/batch/analyze.py +236 -0
- oscura/{batch → workflows/batch}/logging.py +2 -2
- oscura/{batch → workflows/batch}/metrics.py +1 -1
- oscura/workflows/complete_re.py +1144 -0
- oscura/workflows/compliance.py +44 -54
- oscura/workflows/digital.py +197 -51
- oscura/workflows/legacy/__init__.py +12 -0
- oscura/{workflow → workflows/legacy}/dag.py +4 -1
- oscura/workflows/multi_trace.py +9 -9
- oscura/workflows/power.py +42 -62
- oscura/workflows/protocol.py +82 -49
- oscura/workflows/reverse_engineering.py +351 -150
- oscura/workflows/signal_integrity.py +157 -82
- oscura-0.6.0.dist-info/METADATA +643 -0
- oscura-0.6.0.dist-info/RECORD +590 -0
- oscura/analyzers/digital/ic_database.py +0 -498
- oscura/analyzers/digital/timing_paths.py +0 -339
- oscura/analyzers/digital/vintage.py +0 -377
- oscura/analyzers/digital/vintage_result.py +0 -148
- oscura/analyzers/protocols/parallel_bus.py +0 -449
- oscura/batch/aggregate.py +0 -300
- oscura/batch/analyze.py +0 -139
- oscura/dsl/__init__.py +0 -73
- oscura/exceptions.py +0 -59
- oscura/exploratory/fuzzy.py +0 -513
- oscura/exploratory/sync.py +0 -384
- oscura/export/wavedrom.py +0 -430
- oscura/exporters/__init__.py +0 -94
- oscura/exporters/csv.py +0 -303
- oscura/exporters/exporters.py +0 -44
- oscura/exporters/hdf5.py +0 -217
- oscura/exporters/html_export.py +0 -701
- oscura/exporters/json_export.py +0 -338
- oscura/exporters/markdown_export.py +0 -367
- oscura/exporters/matlab_export.py +0 -354
- oscura/exporters/npz_export.py +0 -219
- oscura/exporters/spice_export.py +0 -210
- oscura/exporters/vintage_logic_csv.py +0 -247
- oscura/reporting/vintage_logic_report.py +0 -523
- oscura/search/context.py +0 -149
- oscura/session/__init__.py +0 -34
- oscura/session/annotations.py +0 -289
- oscura/session/history.py +0 -313
- oscura/session/session.py +0 -520
- oscura/visualization/digital_advanced.py +0 -718
- oscura/visualization/figure_manager.py +0 -156
- oscura/workflow/__init__.py +0 -13
- oscura-0.5.0.dist-info/METADATA +0 -407
- oscura-0.5.0.dist-info/RECORD +0 -486
- /oscura/core/{config.py → config/legacy.py} +0 -0
- /oscura/{extensibility → core/extensibility}/__init__.py +0 -0
- /oscura/{extensibility → core/extensibility}/registry.py +0 -0
- /oscura/{plugins → core/plugins}/isolation.py +0 -0
- /oscura/{builders → utils/builders}/signal_builder.py +0 -0
- /oscura/{optimization → utils/optimization}/parallel.py +0 -0
- /oscura/{pipeline → utils/pipeline}/__init__.py +0 -0
- /oscura/{streaming → utils/streaming}/__init__.py +0 -0
- {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/WHEEL +0 -0
- {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/entry_points.txt +0 -0
- {oscura-0.5.0.dist-info → oscura-0.6.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,908 @@
|
|
|
1
|
+
"""Anomaly detection system for identifying unusual patterns in protocol traffic.
|
|
2
|
+
|
|
3
|
+
This module provides multi-method anomaly detection to identify unusual patterns
|
|
4
|
+
in protocol traffic using statistical and machine learning methods.
|
|
5
|
+
|
|
6
|
+
Key capabilities:
|
|
7
|
+
- Statistical anomaly detection (Z-score, IQR, modified Z-score)
|
|
8
|
+
- Time-series anomaly detection (rate analysis, timing analysis)
|
|
9
|
+
- ML-based detection (Isolation Forest, One-Class SVM) - optional with scikit-learn
|
|
10
|
+
- Detect: unexpected message rates, unusual field values, timing anomalies, sequence violations
|
|
11
|
+
- Return anomaly scores and explanations
|
|
12
|
+
- Support online detection (streaming data)
|
|
13
|
+
- Export anomaly reports with context
|
|
14
|
+
|
|
15
|
+
Detection methods:
|
|
16
|
+
- Z-score: Standard deviation-based outlier detection
|
|
17
|
+
- IQR: Interquartile range-based outlier detection
|
|
18
|
+
- Modified Z-score: Median absolute deviation-based (robust to outliers)
|
|
19
|
+
- Isolation Forest: Tree-based anomaly detection (requires sklearn)
|
|
20
|
+
- One-Class SVM: Support vector-based anomaly detection (requires sklearn)
|
|
21
|
+
|
|
22
|
+
Typical workflow:
|
|
23
|
+
1. Train detector on normal baseline data (optional for statistical methods)
|
|
24
|
+
2. Detect anomalies in new data points (batch or streaming)
|
|
25
|
+
3. Export anomaly reports with explanations and context
|
|
26
|
+
|
|
27
|
+
Example:
|
|
28
|
+
>>> from oscura.analyzers.patterns.anomaly_detection import AnomalyDetector, AnomalyDetectionConfig
|
|
29
|
+
>>> config = AnomalyDetectionConfig(methods=["zscore", "iqr"])
|
|
30
|
+
>>> detector = AnomalyDetector(config)
|
|
31
|
+
>>> # Detect field value anomalies
|
|
32
|
+
>>> anomalies = detector.detect_field_value_anomaly(
|
|
33
|
+
... field_values=[1.0, 1.1, 0.9, 1.2, 10.0, 1.0],
|
|
34
|
+
... field_name="voltage"
|
|
35
|
+
... )
|
|
36
|
+
>>> for anomaly in anomalies:
|
|
37
|
+
... print(f"{anomaly.anomaly_type}: {anomaly.explanation}")
|
|
38
|
+
value: voltage value 10.00 deviates significantly from expected 2.53
|
|
39
|
+
|
|
40
|
+
References:
|
|
41
|
+
Isolation Forest: Liu et al. (2008) "Isolation Forest"
|
|
42
|
+
One-Class SVM: Schölkopf et al. (2001) "Estimating the Support of a High-Dimensional Distribution"
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import json
|
|
48
|
+
import logging
|
|
49
|
+
from dataclasses import dataclass, field
|
|
50
|
+
from pathlib import Path
|
|
51
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
|
52
|
+
|
|
53
|
+
import numpy as np
|
|
54
|
+
|
|
55
|
+
if TYPE_CHECKING:
|
|
56
|
+
from numpy.typing import NDArray
|
|
57
|
+
|
|
58
|
+
logger = logging.getLogger(__name__)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class Anomaly:
|
|
63
|
+
"""Detected anomaly.
|
|
64
|
+
|
|
65
|
+
Attributes:
|
|
66
|
+
timestamp: Time of anomaly occurrence
|
|
67
|
+
anomaly_type: Type of anomaly detected (rate, value, timing, sequence, protocol)
|
|
68
|
+
score: Anomaly score (0.0-1.0, higher = more anomalous)
|
|
69
|
+
message_index: Index of message containing anomaly (None if not message-specific)
|
|
70
|
+
field_name: Name of field with anomaly (None if not field-specific)
|
|
71
|
+
expected_value: Expected value based on baseline/model
|
|
72
|
+
actual_value: Actual value observed
|
|
73
|
+
explanation: Human-readable explanation of anomaly
|
|
74
|
+
context: Additional context information (rates, thresholds, etc.)
|
|
75
|
+
|
|
76
|
+
Example:
|
|
77
|
+
>>> anomaly = Anomaly(
|
|
78
|
+
... timestamp=1234.5,
|
|
79
|
+
... anomaly_type="value",
|
|
80
|
+
... score=0.95,
|
|
81
|
+
... message_index=42,
|
|
82
|
+
... field_name="voltage",
|
|
83
|
+
... expected_value=3.3,
|
|
84
|
+
... actual_value=15.0,
|
|
85
|
+
... explanation="Voltage spike detected"
|
|
86
|
+
... )
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
timestamp: float
|
|
90
|
+
anomaly_type: str # "rate", "value", "timing", "sequence", "protocol"
|
|
91
|
+
score: float # 0.0-1.0, higher = more anomalous
|
|
92
|
+
message_index: int | None = None
|
|
93
|
+
field_name: str | None = None
|
|
94
|
+
expected_value: Any = None
|
|
95
|
+
actual_value: Any = None
|
|
96
|
+
explanation: str = ""
|
|
97
|
+
context: dict[str, Any] = field(default_factory=dict)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class AnomalyDetectionConfig:
|
|
102
|
+
"""Anomaly detection configuration.
|
|
103
|
+
|
|
104
|
+
Attributes:
|
|
105
|
+
methods: Detection methods to use (zscore, iqr, modified_zscore, isolation_forest, one_class_svm)
|
|
106
|
+
zscore_threshold: Standard deviations for Z-score method (default: 3.0)
|
|
107
|
+
iqr_multiplier: Multiplier for IQR method (default: 1.5)
|
|
108
|
+
contamination: Expected outlier fraction for ML methods (default: 0.1)
|
|
109
|
+
window_size: Window size for streaming detection (default: 100)
|
|
110
|
+
|
|
111
|
+
Example:
|
|
112
|
+
>>> config = AnomalyDetectionConfig(
|
|
113
|
+
... methods=["zscore", "iqr"],
|
|
114
|
+
... zscore_threshold=2.5,
|
|
115
|
+
... window_size=200
|
|
116
|
+
... )
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
methods: list[str] = field(default_factory=lambda: ["zscore", "isolation_forest"])
|
|
120
|
+
zscore_threshold: float = 3.0 # Standard deviations
|
|
121
|
+
iqr_multiplier: float = 1.5
|
|
122
|
+
contamination: float = 0.1 # Expected outlier fraction
|
|
123
|
+
window_size: int = 100 # For streaming detection
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class AnomalyDetector:
|
|
127
|
+
"""Multi-method anomaly detection system.
|
|
128
|
+
|
|
129
|
+
This class provides various anomaly detection methods for protocol traffic analysis.
|
|
130
|
+
It supports both statistical methods (Z-score, IQR) and ML-based methods
|
|
131
|
+
(Isolation Forest, One-Class SVM - if scikit-learn is available).
|
|
132
|
+
|
|
133
|
+
Attributes:
|
|
134
|
+
config: Detection configuration
|
|
135
|
+
models: Trained ML models (if applicable)
|
|
136
|
+
baselines: Baseline statistics for comparison
|
|
137
|
+
anomalies: Detected anomalies
|
|
138
|
+
|
|
139
|
+
Detection methods:
|
|
140
|
+
Statistical: zscore, iqr, modified_zscore
|
|
141
|
+
ML-based: isolation_forest, one_class_svm (requires scikit-learn)
|
|
142
|
+
Time-series: rate analysis, timing analysis
|
|
143
|
+
|
|
144
|
+
Example:
|
|
145
|
+
>>> detector = AnomalyDetector()
|
|
146
|
+
>>> # Detect message rate anomalies
|
|
147
|
+
>>> timestamps = [0.0, 0.1, 0.2, 0.3, 5.0, 5.1] # Gap at 0.3-5.0
|
|
148
|
+
>>> anomalies = detector.detect_message_rate_anomaly(timestamps)
|
|
149
|
+
>>> # Detect field value anomalies
|
|
150
|
+
>>> values = [1.0, 1.1, 0.9, 1.2, 10.0, 1.0] # Spike at 10.0
|
|
151
|
+
>>> anomalies = detector.detect_field_value_anomaly(values, "voltage")
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
# Detection methods
|
|
155
|
+
STATISTICAL_METHODS: ClassVar[list[str]] = ["zscore", "iqr", "modified_zscore"]
|
|
156
|
+
ML_METHODS: ClassVar[list[str]] = ["isolation_forest", "one_class_svm", "autoencoder"]
|
|
157
|
+
TIMESERIES_METHODS: ClassVar[list[str]] = ["arima", "seasonal_decomposition"]
|
|
158
|
+
|
|
159
|
+
def __init__(self, config: AnomalyDetectionConfig | None = None) -> None:
|
|
160
|
+
"""Initialize anomaly detector.
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
config: Anomaly detection configuration. If None, uses default config.
|
|
164
|
+
|
|
165
|
+
Example:
|
|
166
|
+
>>> detector = AnomalyDetector()
|
|
167
|
+
>>> config = AnomalyDetectionConfig(methods=["zscore"], zscore_threshold=2.5)
|
|
168
|
+
>>> detector_custom = AnomalyDetector(config)
|
|
169
|
+
"""
|
|
170
|
+
self.config = config or AnomalyDetectionConfig()
|
|
171
|
+
self.models: dict[str, Any] = {}
|
|
172
|
+
self.baselines: dict[str, Any] = {}
|
|
173
|
+
self.anomalies: list[Anomaly] = []
|
|
174
|
+
|
|
175
|
+
def train(self, normal_data: list[dict[str, Any]], features: list[str]) -> None:
|
|
176
|
+
"""Train anomaly detection models on normal baseline data.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
normal_data: List of normal data points (dicts with feature values)
|
|
180
|
+
features: List of feature names to use for training
|
|
181
|
+
|
|
182
|
+
Raises:
|
|
183
|
+
ImportError: If ML methods requested but scikit-learn not available
|
|
184
|
+
ValueError: If insufficient data for training
|
|
185
|
+
|
|
186
|
+
Example:
|
|
187
|
+
>>> normal_data = [
|
|
188
|
+
... {"voltage": 3.3, "current": 0.5},
|
|
189
|
+
... {"voltage": 3.2, "current": 0.6},
|
|
190
|
+
... ]
|
|
191
|
+
>>> detector.train(normal_data, features=["voltage", "current"])
|
|
192
|
+
"""
|
|
193
|
+
if len(normal_data) < 10:
|
|
194
|
+
raise ValueError("Need at least 10 samples for training")
|
|
195
|
+
|
|
196
|
+
# Extract features
|
|
197
|
+
X = np.array([[d[f] for f in features] for d in normal_data])
|
|
198
|
+
|
|
199
|
+
# Calculate baseline statistics
|
|
200
|
+
for i, feature in enumerate(features):
|
|
201
|
+
self.baselines[feature] = {
|
|
202
|
+
"mean": float(np.mean(X[:, i])),
|
|
203
|
+
"std": float(np.std(X[:, i])),
|
|
204
|
+
"median": float(np.median(X[:, i])),
|
|
205
|
+
"q1": float(np.percentile(X[:, i], 25)),
|
|
206
|
+
"q3": float(np.percentile(X[:, i], 75)),
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
# Train ML models if requested
|
|
210
|
+
for method in self.config.methods:
|
|
211
|
+
if method == "isolation_forest":
|
|
212
|
+
self._train_isolation_forest(X)
|
|
213
|
+
elif method == "one_class_svm":
|
|
214
|
+
self._train_one_class_svm(X)
|
|
215
|
+
|
|
216
|
+
def detect(self, data: dict[str, Any], timestamp: float = 0.0) -> list[Anomaly]:
|
|
217
|
+
"""Detect anomalies in new data point.
|
|
218
|
+
|
|
219
|
+
Args:
|
|
220
|
+
data: Data point to analyze (dict with feature values)
|
|
221
|
+
timestamp: Timestamp of data point
|
|
222
|
+
|
|
223
|
+
Returns:
|
|
224
|
+
List of detected anomalies
|
|
225
|
+
|
|
226
|
+
Example:
|
|
227
|
+
>>> anomalies = detector.detect(
|
|
228
|
+
... {"voltage": 10.0, "current": 0.5},
|
|
229
|
+
... timestamp=123.45
|
|
230
|
+
... )
|
|
231
|
+
"""
|
|
232
|
+
anomalies = []
|
|
233
|
+
|
|
234
|
+
# Check each feature against baselines
|
|
235
|
+
for feature, value in data.items():
|
|
236
|
+
if feature in self.baselines:
|
|
237
|
+
baseline = self.baselines[feature]
|
|
238
|
+
|
|
239
|
+
# Z-score check
|
|
240
|
+
if "zscore" in self.config.methods:
|
|
241
|
+
z_score = abs(value - baseline["mean"]) / (baseline["std"] + 1e-10)
|
|
242
|
+
if z_score > self.config.zscore_threshold:
|
|
243
|
+
anomalies.append(
|
|
244
|
+
Anomaly(
|
|
245
|
+
timestamp=timestamp,
|
|
246
|
+
anomaly_type="value",
|
|
247
|
+
score=min(z_score / 5.0, 1.0),
|
|
248
|
+
field_name=feature,
|
|
249
|
+
expected_value=baseline["mean"],
|
|
250
|
+
actual_value=value,
|
|
251
|
+
explanation=f"{feature} value {value:.2f} deviates {z_score:.1f} std devs from expected {baseline['mean']:.2f}",
|
|
252
|
+
context={"z_score": z_score, "method": "zscore"},
|
|
253
|
+
)
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
# IQR check
|
|
257
|
+
if "iqr" in self.config.methods:
|
|
258
|
+
iqr = baseline["q3"] - baseline["q1"]
|
|
259
|
+
lower_bound = baseline["q1"] - self.config.iqr_multiplier * iqr
|
|
260
|
+
upper_bound = baseline["q3"] + self.config.iqr_multiplier * iqr
|
|
261
|
+
|
|
262
|
+
if value < lower_bound or value > upper_bound:
|
|
263
|
+
anomalies.append(
|
|
264
|
+
Anomaly(
|
|
265
|
+
timestamp=timestamp,
|
|
266
|
+
anomaly_type="value",
|
|
267
|
+
score=0.8,
|
|
268
|
+
field_name=feature,
|
|
269
|
+
expected_value=baseline["median"],
|
|
270
|
+
actual_value=value,
|
|
271
|
+
explanation=f"{feature} value {value:.2f} outside IQR bounds [{lower_bound:.2f}, {upper_bound:.2f}]",
|
|
272
|
+
context={
|
|
273
|
+
"iqr": iqr,
|
|
274
|
+
"bounds": (lower_bound, upper_bound),
|
|
275
|
+
"method": "iqr",
|
|
276
|
+
},
|
|
277
|
+
)
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
self.anomalies.extend(anomalies)
|
|
281
|
+
return anomalies
|
|
282
|
+
|
|
283
|
+
def detect_batch(
|
|
284
|
+
self, data_points: list[dict[str, Any]], timestamps: list[float]
|
|
285
|
+
) -> list[Anomaly]:
|
|
286
|
+
"""Detect anomalies in batch of data.
|
|
287
|
+
|
|
288
|
+
Args:
|
|
289
|
+
data_points: List of data points to analyze
|
|
290
|
+
timestamps: Corresponding timestamps for each data point
|
|
291
|
+
|
|
292
|
+
Returns:
|
|
293
|
+
List of all detected anomalies
|
|
294
|
+
|
|
295
|
+
Example:
|
|
296
|
+
>>> data = [
|
|
297
|
+
... {"voltage": 3.3},
|
|
298
|
+
... {"voltage": 10.0}, # Anomaly
|
|
299
|
+
... ]
|
|
300
|
+
>>> timestamps = [0.0, 1.0]
|
|
301
|
+
>>> anomalies = detector.detect_batch(data, timestamps)
|
|
302
|
+
"""
|
|
303
|
+
all_anomalies = []
|
|
304
|
+
for data, timestamp in zip(data_points, timestamps, strict=True):
|
|
305
|
+
anomalies = self.detect(data, timestamp)
|
|
306
|
+
all_anomalies.extend(anomalies)
|
|
307
|
+
return all_anomalies
|
|
308
|
+
|
|
309
|
+
def detect_message_rate_anomaly(
|
|
310
|
+
self, timestamps: list[float], window_size: int = 100
|
|
311
|
+
) -> list[Anomaly]:
|
|
312
|
+
"""Detect anomalous message rates (bursts, gaps).
|
|
313
|
+
|
|
314
|
+
Uses sliding window to calculate message rate, then detects outliers
|
|
315
|
+
in rate distribution using Z-score method.
|
|
316
|
+
|
|
317
|
+
Args:
|
|
318
|
+
timestamps: List of message timestamps
|
|
319
|
+
window_size: Size of sliding window for rate calculation
|
|
320
|
+
|
|
321
|
+
Returns:
|
|
322
|
+
List of detected rate anomalies
|
|
323
|
+
|
|
324
|
+
Example:
|
|
325
|
+
>>> timestamps = [0.0, 0.1, 0.2, 0.3, 5.0, 5.1] # Gap at 0.3-5.0
|
|
326
|
+
>>> anomalies = detector.detect_message_rate_anomaly(timestamps)
|
|
327
|
+
>>> for a in anomalies:
|
|
328
|
+
... print(a.explanation)
|
|
329
|
+
Message gap detected: 2.1 msg/s vs expected 10.0 msg/s
|
|
330
|
+
"""
|
|
331
|
+
if len(timestamps) < window_size:
|
|
332
|
+
return []
|
|
333
|
+
|
|
334
|
+
# Calculate message rates in windows
|
|
335
|
+
rates = []
|
|
336
|
+
window_timestamps = []
|
|
337
|
+
|
|
338
|
+
for i in range(len(timestamps) - window_size):
|
|
339
|
+
window = timestamps[i : i + window_size]
|
|
340
|
+
time_span = window[-1] - window[0]
|
|
341
|
+
rate = window_size / time_span if time_span > 0 else 0
|
|
342
|
+
rates.append(rate)
|
|
343
|
+
window_timestamps.append(window[window_size // 2])
|
|
344
|
+
|
|
345
|
+
# Detect anomalies in rates using dual-threshold approach
|
|
346
|
+
rate_array = np.array(rates)
|
|
347
|
+
mean_rate = np.mean(rate_array)
|
|
348
|
+
std_rate = np.std(rate_array)
|
|
349
|
+
median_rate = np.median(rate_array)
|
|
350
|
+
|
|
351
|
+
# For extremely constant rates, avoid false positives from floating point errors
|
|
352
|
+
if std_rate < mean_rate * 0.0001: # Coefficient of variation < 0.01%
|
|
353
|
+
return []
|
|
354
|
+
|
|
355
|
+
# Use dual-threshold approach for rate anomaly detection:
|
|
356
|
+
# 1. Statistical outlier detection (Z-score)
|
|
357
|
+
# 2. Rate deviation threshold (>30% change from median)
|
|
358
|
+
statistical_outliers = self._zscore_detection(rate_array, self.config.zscore_threshold)
|
|
359
|
+
|
|
360
|
+
# Rate-based threshold: detect significant rate changes (>30% deviation from median)
|
|
361
|
+
rate_deviations = np.abs(rate_array - median_rate) / median_rate
|
|
362
|
+
significant_deviations = rate_deviations > 0.3 # 30% threshold
|
|
363
|
+
|
|
364
|
+
# Combine: anomaly if either statistical outlier OR significant deviation
|
|
365
|
+
outliers = statistical_outliers | significant_deviations
|
|
366
|
+
|
|
367
|
+
# Create Anomaly objects
|
|
368
|
+
anomalies = []
|
|
369
|
+
|
|
370
|
+
for idx, is_outlier in enumerate(outliers):
|
|
371
|
+
if is_outlier:
|
|
372
|
+
if rates[idx] > mean_rate:
|
|
373
|
+
explanation = f"Message burst detected: {rates[idx]:.1f} msg/s vs expected {mean_rate:.1f} msg/s"
|
|
374
|
+
else:
|
|
375
|
+
explanation = f"Message gap detected: {rates[idx]:.1f} msg/s vs expected {mean_rate:.1f} msg/s"
|
|
376
|
+
|
|
377
|
+
anomalies.append(
|
|
378
|
+
Anomaly(
|
|
379
|
+
timestamp=window_timestamps[idx],
|
|
380
|
+
anomaly_type="rate",
|
|
381
|
+
score=min(abs(rates[idx] - mean_rate) / mean_rate, 1.0),
|
|
382
|
+
explanation=explanation,
|
|
383
|
+
context={"rate": rates[idx], "expected_rate": mean_rate},
|
|
384
|
+
)
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
self.anomalies.extend(anomalies)
|
|
388
|
+
return anomalies
|
|
389
|
+
|
|
390
|
+
def detect_field_value_anomaly(
|
|
391
|
+
self, field_values: list[float], field_name: str, method: str = "modified_zscore"
|
|
392
|
+
) -> list[Anomaly]:
|
|
393
|
+
"""Detect unusual field values using statistical methods.
|
|
394
|
+
|
|
395
|
+
Args:
|
|
396
|
+
field_values: List of field values to analyze
|
|
397
|
+
field_name: Name of the field being analyzed
|
|
398
|
+
method: Detection method to use (zscore, iqr, modified_zscore)
|
|
399
|
+
|
|
400
|
+
Returns:
|
|
401
|
+
List of detected value anomalies
|
|
402
|
+
|
|
403
|
+
Raises:
|
|
404
|
+
ValueError: If method is unknown
|
|
405
|
+
|
|
406
|
+
Example:
|
|
407
|
+
>>> values = [1.0, 1.1, 0.9, 1.2, 10.0, 1.0]
|
|
408
|
+
>>> anomalies = detector.detect_field_value_anomaly(values, "voltage")
|
|
409
|
+
>>> print(anomalies[0].explanation)
|
|
410
|
+
voltage value 10.00 deviates significantly from expected 2.53
|
|
411
|
+
"""
|
|
412
|
+
values = np.array(field_values)
|
|
413
|
+
|
|
414
|
+
# For very small datasets (n=2), use simple ratio-based detection
|
|
415
|
+
if len(values) == 2:
|
|
416
|
+
# Detect if one value is >5x or <0.2x the other
|
|
417
|
+
ratio = values.max() / (values.min() + 1e-10)
|
|
418
|
+
if ratio > 5.0:
|
|
419
|
+
# The larger value is anomalous
|
|
420
|
+
outliers = values == values.max()
|
|
421
|
+
else:
|
|
422
|
+
outliers = np.zeros(len(values), dtype=bool)
|
|
423
|
+
elif method == "zscore":
|
|
424
|
+
outliers = self._zscore_detection(values, self.config.zscore_threshold)
|
|
425
|
+
elif method == "iqr":
|
|
426
|
+
outliers = self._iqr_detection(values, self.config.iqr_multiplier)
|
|
427
|
+
elif method == "modified_zscore":
|
|
428
|
+
outliers = self._modified_zscore_detection(values, self.config.zscore_threshold)
|
|
429
|
+
else:
|
|
430
|
+
raise ValueError(f"Unknown method: {method}")
|
|
431
|
+
|
|
432
|
+
# Create Anomaly objects
|
|
433
|
+
anomalies = []
|
|
434
|
+
mean_val = np.mean(values)
|
|
435
|
+
|
|
436
|
+
for idx, is_outlier in enumerate(outliers):
|
|
437
|
+
if is_outlier:
|
|
438
|
+
score = abs(values[idx] - mean_val) / (np.std(values) + 1e-10)
|
|
439
|
+
score = min(score / 5.0, 1.0) # Normalize to 0-1
|
|
440
|
+
|
|
441
|
+
anomalies.append(
|
|
442
|
+
Anomaly(
|
|
443
|
+
timestamp=float(idx), # Index as timestamp
|
|
444
|
+
anomaly_type="value",
|
|
445
|
+
score=score,
|
|
446
|
+
message_index=idx,
|
|
447
|
+
field_name=field_name,
|
|
448
|
+
expected_value=mean_val,
|
|
449
|
+
actual_value=values[idx],
|
|
450
|
+
explanation=f"{field_name} value {values[idx]:.2f} deviates significantly from expected {mean_val:.2f}",
|
|
451
|
+
context={"method": method},
|
|
452
|
+
)
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
self.anomalies.extend(anomalies)
|
|
456
|
+
return anomalies
|
|
457
|
+
|
|
458
|
+
def detect_timing_anomaly(
|
|
459
|
+
self,
|
|
460
|
+
inter_arrival_times: list[float],
|
|
461
|
+
expected_period: float | None = None,
|
|
462
|
+
) -> list[Anomaly]:
|
|
463
|
+
"""Detect timing anomalies (jitter, drift, unexpected delays).
|
|
464
|
+
|
|
465
|
+
Args:
|
|
466
|
+
inter_arrival_times: List of inter-arrival times between messages
|
|
467
|
+
expected_period: Expected period (None = calculate from data)
|
|
468
|
+
|
|
469
|
+
Returns:
|
|
470
|
+
List of detected timing anomalies
|
|
471
|
+
|
|
472
|
+
Example:
|
|
473
|
+
>>> inter_arrival = [0.1, 0.1, 0.1, 1.0, 0.1] # Delay at index 3
|
|
474
|
+
>>> anomalies = detector.detect_timing_anomaly(inter_arrival)
|
|
475
|
+
>>> print(anomalies[0].explanation)
|
|
476
|
+
Timing anomaly: 1.00s vs expected 0.28s
|
|
477
|
+
"""
|
|
478
|
+
if not inter_arrival_times:
|
|
479
|
+
return []
|
|
480
|
+
|
|
481
|
+
times = np.array(inter_arrival_times)
|
|
482
|
+
|
|
483
|
+
# Use expected period or calculate from data
|
|
484
|
+
if expected_period is None:
|
|
485
|
+
expected_period = float(np.median(times))
|
|
486
|
+
|
|
487
|
+
# For very small datasets (n=2), use ratio-based detection
|
|
488
|
+
if len(times) == 2:
|
|
489
|
+
# Detect if one value is >5x or <0.2x the other
|
|
490
|
+
ratio = times.max() / (times.min() + 1e-10)
|
|
491
|
+
if ratio > 5.0:
|
|
492
|
+
# The larger value is anomalous (unexpected delay)
|
|
493
|
+
outliers = times == times.max()
|
|
494
|
+
else:
|
|
495
|
+
outliers = np.zeros(len(times), dtype=bool)
|
|
496
|
+
else:
|
|
497
|
+
# Use statistical detection when no expected period provided
|
|
498
|
+
outliers = self._zscore_detection(times, self.config.zscore_threshold)
|
|
499
|
+
else:
|
|
500
|
+
# When expected period is provided, detect deviations from it directly
|
|
501
|
+
# Use threshold-based detection (e.g., >50% deviation from expected)
|
|
502
|
+
deviations = np.abs(times - expected_period) / expected_period
|
|
503
|
+
outliers = deviations > 0.5 # 50% deviation threshold
|
|
504
|
+
|
|
505
|
+
# Create Anomaly objects
|
|
506
|
+
anomalies = []
|
|
507
|
+
for idx, is_outlier in enumerate(outliers):
|
|
508
|
+
if is_outlier:
|
|
509
|
+
anomalies.append(
|
|
510
|
+
Anomaly(
|
|
511
|
+
timestamp=float(idx),
|
|
512
|
+
anomaly_type="timing",
|
|
513
|
+
score=min(abs(times[idx] - expected_period) / expected_period, 1.0),
|
|
514
|
+
message_index=idx,
|
|
515
|
+
expected_value=expected_period,
|
|
516
|
+
actual_value=times[idx],
|
|
517
|
+
explanation=f"Timing anomaly: {times[idx]:.2f}s vs expected {expected_period:.2f}s",
|
|
518
|
+
context={"expected_period": expected_period},
|
|
519
|
+
)
|
|
520
|
+
)
|
|
521
|
+
|
|
522
|
+
self.anomalies.extend(anomalies)
|
|
523
|
+
return anomalies
|
|
524
|
+
|
|
525
|
+
def detect_sequence_anomaly(
|
|
526
|
+
self, sequences: list[list[int]], trained_model: Any = None
|
|
527
|
+
) -> list[Anomaly]:
|
|
528
|
+
"""Detect unusual byte/message sequences.
|
|
529
|
+
|
|
530
|
+
Analyzes sequences for unusual patterns. If trained_model is provided,
|
|
531
|
+
uses it for prediction. Otherwise uses statistical analysis.
|
|
532
|
+
|
|
533
|
+
Args:
|
|
534
|
+
sequences: List of byte/message sequences
|
|
535
|
+
trained_model: Trained sequence model (optional)
|
|
536
|
+
|
|
537
|
+
Returns:
|
|
538
|
+
List of detected sequence anomalies
|
|
539
|
+
|
|
540
|
+
Example:
|
|
541
|
+
>>> sequences = [[0x01, 0x02, 0x03], [0x01, 0x02, 0xFF]] # Last byte unusual
|
|
542
|
+
>>> anomalies = detector.detect_sequence_anomaly(sequences)
|
|
543
|
+
"""
|
|
544
|
+
anomalies = []
|
|
545
|
+
|
|
546
|
+
# Dual-threshold approach: detect unusual sequence lengths
|
|
547
|
+
lengths = np.array([len(seq) for seq in sequences])
|
|
548
|
+
median_length = np.median(lengths)
|
|
549
|
+
mean_length = np.mean(lengths)
|
|
550
|
+
|
|
551
|
+
# Statistical outlier detection
|
|
552
|
+
statistical_outliers = self._zscore_detection(lengths, self.config.zscore_threshold)
|
|
553
|
+
|
|
554
|
+
# Length-based threshold: detect sequences >2x median length or <0.5x median length
|
|
555
|
+
length_ratios = lengths / median_length
|
|
556
|
+
significant_length_deviations = (length_ratios > 2.0) | (length_ratios < 0.5)
|
|
557
|
+
|
|
558
|
+
# Combine: anomaly if either statistical outlier OR significant length deviation
|
|
559
|
+
outliers = statistical_outliers | significant_length_deviations
|
|
560
|
+
|
|
561
|
+
for idx, is_outlier in enumerate(outliers):
|
|
562
|
+
if is_outlier:
|
|
563
|
+
anomalies.append(
|
|
564
|
+
Anomaly(
|
|
565
|
+
timestamp=float(idx),
|
|
566
|
+
anomaly_type="sequence",
|
|
567
|
+
score=0.7,
|
|
568
|
+
message_index=idx,
|
|
569
|
+
expected_value=mean_length,
|
|
570
|
+
actual_value=lengths[idx],
|
|
571
|
+
explanation=f"Unusual sequence length: {lengths[idx]} bytes vs expected {mean_length:.0f} bytes",
|
|
572
|
+
context={"sequence_length": lengths[idx]},
|
|
573
|
+
)
|
|
574
|
+
)
|
|
575
|
+
|
|
576
|
+
self.anomalies.extend(anomalies)
|
|
577
|
+
return anomalies
|
|
578
|
+
|
|
579
|
+
def _zscore_detection(
|
|
580
|
+
self, values: NDArray[np.floating[Any]], threshold: float = 3.0
|
|
581
|
+
) -> NDArray[np.bool_]:
|
|
582
|
+
"""Z-score based outlier detection.
|
|
583
|
+
|
|
584
|
+
Z-score measures how many standard deviations a value is from the mean.
|
|
585
|
+
Outliers are typically defined as |Z-score| > threshold (commonly 3.0).
|
|
586
|
+
|
|
587
|
+
Args:
|
|
588
|
+
values: Array of values to analyze
|
|
589
|
+
threshold: Z-score threshold for outliers (default: 3.0)
|
|
590
|
+
|
|
591
|
+
Returns:
|
|
592
|
+
Boolean array indicating outliers (True = outlier)
|
|
593
|
+
|
|
594
|
+
Example:
|
|
595
|
+
>>> values = np.array([1.0, 1.1, 0.9, 10.0])
|
|
596
|
+
>>> outliers = detector._zscore_detection(values, threshold=2.0)
|
|
597
|
+
>>> print(outliers)
|
|
598
|
+
[False False False True]
|
|
599
|
+
"""
|
|
600
|
+
if len(values) < 2:
|
|
601
|
+
return np.zeros(len(values), dtype=bool)
|
|
602
|
+
|
|
603
|
+
mean = np.mean(values)
|
|
604
|
+
std = np.std(values)
|
|
605
|
+
|
|
606
|
+
if std == 0:
|
|
607
|
+
return np.zeros(len(values), dtype=bool)
|
|
608
|
+
|
|
609
|
+
z_scores = np.abs((values - mean) / std)
|
|
610
|
+
outliers: NDArray[np.bool_] = z_scores > threshold
|
|
611
|
+
|
|
612
|
+
return outliers
|
|
613
|
+
|
|
614
|
+
def _iqr_detection(
|
|
615
|
+
self, values: NDArray[np.floating[Any]], multiplier: float = 1.5
|
|
616
|
+
) -> NDArray[np.bool_]:
|
|
617
|
+
"""Interquartile range (IQR) based outlier detection.
|
|
618
|
+
|
|
619
|
+
IQR is the range between 25th and 75th percentiles. Outliers are
|
|
620
|
+
values outside [Q1 - multiplier*IQR, Q3 + multiplier*IQR].
|
|
621
|
+
|
|
622
|
+
Args:
|
|
623
|
+
values: Array of values to analyze
|
|
624
|
+
multiplier: IQR multiplier for bounds (default: 1.5)
|
|
625
|
+
|
|
626
|
+
Returns:
|
|
627
|
+
Boolean array indicating outliers (True = outlier)
|
|
628
|
+
|
|
629
|
+
Example:
|
|
630
|
+
>>> values = np.array([1.0, 1.1, 0.9, 10.0])
|
|
631
|
+
>>> outliers = detector._iqr_detection(values)
|
|
632
|
+
>>> print(outliers)
|
|
633
|
+
[False False False True]
|
|
634
|
+
"""
|
|
635
|
+
if len(values) < 4:
|
|
636
|
+
return np.zeros(len(values), dtype=bool)
|
|
637
|
+
|
|
638
|
+
q1 = np.percentile(values, 25)
|
|
639
|
+
q3 = np.percentile(values, 75)
|
|
640
|
+
iqr = q3 - q1
|
|
641
|
+
|
|
642
|
+
lower_bound = q1 - multiplier * iqr
|
|
643
|
+
upper_bound = q3 + multiplier * iqr
|
|
644
|
+
|
|
645
|
+
outliers: NDArray[np.bool_] = (values < lower_bound) | (values > upper_bound)
|
|
646
|
+
|
|
647
|
+
return outliers
|
|
648
|
+
|
|
649
|
+
def _modified_zscore_detection(
|
|
650
|
+
self, values: NDArray[np.floating[Any]], threshold: float = 2.5
|
|
651
|
+
) -> NDArray[np.bool_]:
|
|
652
|
+
"""Modified Z-score based outlier detection using median absolute deviation.
|
|
653
|
+
|
|
654
|
+
More robust to outliers than standard Z-score. Uses median instead of mean
|
|
655
|
+
and MAD instead of standard deviation.
|
|
656
|
+
|
|
657
|
+
Args:
|
|
658
|
+
values: Array of values to analyze
|
|
659
|
+
threshold: Modified Z-score threshold (default: 2.5, lower than standard 3.5)
|
|
660
|
+
|
|
661
|
+
Returns:
|
|
662
|
+
Boolean array indicating outliers (True = outlier)
|
|
663
|
+
|
|
664
|
+
Example:
|
|
665
|
+
>>> values = np.array([1.0, 1.1, 0.9, 10.0])
|
|
666
|
+
>>> outliers = detector._modified_zscore_detection(values)
|
|
667
|
+
>>> print(outliers)
|
|
668
|
+
[False False False True]
|
|
669
|
+
"""
|
|
670
|
+
if len(values) < 2:
|
|
671
|
+
return np.zeros(len(values), dtype=bool)
|
|
672
|
+
|
|
673
|
+
median = np.median(values)
|
|
674
|
+
mad = np.median(np.abs(values - median))
|
|
675
|
+
|
|
676
|
+
if mad == 0:
|
|
677
|
+
# When MAD is 0, use a different approach
|
|
678
|
+
# Check if any values differ from the median
|
|
679
|
+
result_outliers: NDArray[np.bool_] = values != median
|
|
680
|
+
return result_outliers
|
|
681
|
+
|
|
682
|
+
# Modified Z-score = 0.6745 * (x - median) / MAD
|
|
683
|
+
modified_z_scores = 0.6745 * np.abs(values - median) / mad
|
|
684
|
+
outliers: NDArray[np.bool_] = modified_z_scores > threshold
|
|
685
|
+
|
|
686
|
+
return outliers
|
|
687
|
+
|
|
688
|
+
def _isolation_forest_detection(
|
|
689
|
+
self, X: NDArray[np.floating[Any]], contamination: float = 0.1
|
|
690
|
+
) -> NDArray[np.bool_]:
|
|
691
|
+
"""Isolation Forest based anomaly detection.
|
|
692
|
+
|
|
693
|
+
Isolation Forest isolates anomalies by randomly selecting features
|
|
694
|
+
and split values. Anomalies require fewer splits to isolate.
|
|
695
|
+
|
|
696
|
+
Args:
|
|
697
|
+
X: Feature matrix (n_samples, n_features)
|
|
698
|
+
contamination: Expected outlier fraction (0.0-0.5)
|
|
699
|
+
|
|
700
|
+
Returns:
|
|
701
|
+
Boolean array indicating outliers (True = outlier)
|
|
702
|
+
|
|
703
|
+
Raises:
|
|
704
|
+
ImportError: If scikit-learn is not installed
|
|
705
|
+
|
|
706
|
+
Example:
|
|
707
|
+
>>> X = np.array([[1.0, 2.0], [1.1, 2.1], [10.0, 20.0]])
|
|
708
|
+
>>> outliers = detector._isolation_forest_detection(X)
|
|
709
|
+
"""
|
|
710
|
+
try:
|
|
711
|
+
from sklearn.ensemble import IsolationForest
|
|
712
|
+
except ImportError as e:
|
|
713
|
+
raise ImportError(
|
|
714
|
+
"scikit-learn is required for Isolation Forest. "
|
|
715
|
+
"Install with: pip install scikit-learn"
|
|
716
|
+
) from e
|
|
717
|
+
|
|
718
|
+
if len(X.shape) == 1:
|
|
719
|
+
X = X.reshape(-1, 1)
|
|
720
|
+
|
|
721
|
+
model = IsolationForest(contamination=contamination, random_state=42, n_estimators=100)
|
|
722
|
+
|
|
723
|
+
predictions = model.fit_predict(X)
|
|
724
|
+
outliers: NDArray[np.bool_] = predictions == -1
|
|
725
|
+
|
|
726
|
+
# Store model for later use
|
|
727
|
+
self.models["isolation_forest"] = model
|
|
728
|
+
|
|
729
|
+
return outliers
|
|
730
|
+
|
|
731
|
+
def _one_class_svm_detection(
|
|
732
|
+
self, X: NDArray[np.floating[Any]], nu: float = 0.1
|
|
733
|
+
) -> NDArray[np.bool_]:
|
|
734
|
+
"""One-Class SVM based anomaly detection.
|
|
735
|
+
|
|
736
|
+
One-Class SVM learns a decision boundary around normal data points.
|
|
737
|
+
Points outside this boundary are considered anomalies.
|
|
738
|
+
|
|
739
|
+
Args:
|
|
740
|
+
X: Feature matrix (n_samples, n_features)
|
|
741
|
+
nu: Upper bound on fraction of outliers (0.0-1.0)
|
|
742
|
+
|
|
743
|
+
Returns:
|
|
744
|
+
Boolean array indicating outliers (True = outlier)
|
|
745
|
+
|
|
746
|
+
Raises:
|
|
747
|
+
ImportError: If scikit-learn is not installed
|
|
748
|
+
|
|
749
|
+
Example:
|
|
750
|
+
>>> X = np.array([[1.0, 2.0], [1.1, 2.1], [10.0, 20.0]])
|
|
751
|
+
>>> outliers = detector._one_class_svm_detection(X)
|
|
752
|
+
"""
|
|
753
|
+
try:
|
|
754
|
+
from sklearn.svm import OneClassSVM
|
|
755
|
+
except ImportError as e:
|
|
756
|
+
raise ImportError(
|
|
757
|
+
"scikit-learn is required for One-Class SVM. Install with: pip install scikit-learn"
|
|
758
|
+
) from e
|
|
759
|
+
|
|
760
|
+
if len(X.shape) == 1:
|
|
761
|
+
X = X.reshape(-1, 1)
|
|
762
|
+
|
|
763
|
+
model = OneClassSVM(nu=nu, kernel="rbf", gamma="auto")
|
|
764
|
+
|
|
765
|
+
predictions = model.fit_predict(X)
|
|
766
|
+
outliers: NDArray[np.bool_] = predictions == -1
|
|
767
|
+
|
|
768
|
+
# Store model for later use
|
|
769
|
+
self.models["one_class_svm"] = model
|
|
770
|
+
|
|
771
|
+
return outliers
|
|
772
|
+
|
|
773
|
+
def _train_isolation_forest(self, X: NDArray[np.floating[Any]]) -> None:
|
|
774
|
+
"""Train Isolation Forest model.
|
|
775
|
+
|
|
776
|
+
Args:
|
|
777
|
+
X: Training data (n_samples, n_features)
|
|
778
|
+
"""
|
|
779
|
+
try:
|
|
780
|
+
from sklearn.ensemble import IsolationForest
|
|
781
|
+
except ImportError:
|
|
782
|
+
logger.warning("scikit-learn not available, skipping Isolation Forest training")
|
|
783
|
+
return
|
|
784
|
+
|
|
785
|
+
model = IsolationForest(
|
|
786
|
+
contamination=self.config.contamination, random_state=42, n_estimators=100
|
|
787
|
+
)
|
|
788
|
+
model.fit(X)
|
|
789
|
+
self.models["isolation_forest"] = model
|
|
790
|
+
|
|
791
|
+
def _train_one_class_svm(self, X: NDArray[np.floating[Any]]) -> None:
|
|
792
|
+
"""Train One-Class SVM model.
|
|
793
|
+
|
|
794
|
+
Args:
|
|
795
|
+
X: Training data (n_samples, n_features)
|
|
796
|
+
"""
|
|
797
|
+
try:
|
|
798
|
+
from sklearn.svm import OneClassSVM
|
|
799
|
+
except ImportError:
|
|
800
|
+
logger.warning("scikit-learn not available, skipping One-Class SVM training")
|
|
801
|
+
return
|
|
802
|
+
|
|
803
|
+
model = OneClassSVM(nu=self.config.contamination, kernel="rbf", gamma="auto")
|
|
804
|
+
model.fit(X)
|
|
805
|
+
self.models["one_class_svm"] = model
|
|
806
|
+
|
|
807
|
+
def export_report(self, output_path: Path, format: str = "json") -> None:
|
|
808
|
+
"""Export anomaly report with context and explanations.
|
|
809
|
+
|
|
810
|
+
Args:
|
|
811
|
+
output_path: Path to output file
|
|
812
|
+
format: Export format (json, txt)
|
|
813
|
+
|
|
814
|
+
Raises:
|
|
815
|
+
ValueError: If format is unsupported
|
|
816
|
+
|
|
817
|
+
Example:
|
|
818
|
+
>>> detector.export_report(Path("anomalies.json"), format="json")
|
|
819
|
+
>>> detector.export_report(Path("anomalies.txt"), format="txt")
|
|
820
|
+
"""
|
|
821
|
+
if format == "json":
|
|
822
|
+
self._export_json(output_path)
|
|
823
|
+
elif format == "txt":
|
|
824
|
+
self._export_txt(output_path)
|
|
825
|
+
else:
|
|
826
|
+
raise ValueError(f"Unsupported format: {format}")
|
|
827
|
+
|
|
828
|
+
def _export_json(self, output_path: Path) -> None:
|
|
829
|
+
"""Export anomalies to JSON format.
|
|
830
|
+
|
|
831
|
+
Args:
|
|
832
|
+
output_path: Path to output JSON file
|
|
833
|
+
"""
|
|
834
|
+
report = {
|
|
835
|
+
"config": {
|
|
836
|
+
"methods": self.config.methods,
|
|
837
|
+
"zscore_threshold": self.config.zscore_threshold,
|
|
838
|
+
"iqr_multiplier": self.config.iqr_multiplier,
|
|
839
|
+
"contamination": self.config.contamination,
|
|
840
|
+
"window_size": self.config.window_size,
|
|
841
|
+
},
|
|
842
|
+
"summary": {
|
|
843
|
+
"total_anomalies": len(self.anomalies),
|
|
844
|
+
"by_type": self._count_by_type(),
|
|
845
|
+
},
|
|
846
|
+
"anomalies": [
|
|
847
|
+
{
|
|
848
|
+
"timestamp": a.timestamp,
|
|
849
|
+
"anomaly_type": a.anomaly_type,
|
|
850
|
+
"score": a.score,
|
|
851
|
+
"message_index": a.message_index,
|
|
852
|
+
"field_name": a.field_name,
|
|
853
|
+
"expected_value": a.expected_value,
|
|
854
|
+
"actual_value": a.actual_value,
|
|
855
|
+
"explanation": a.explanation,
|
|
856
|
+
"context": a.context,
|
|
857
|
+
}
|
|
858
|
+
for a in self.anomalies
|
|
859
|
+
],
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
with output_path.open("w") as f:
|
|
863
|
+
json.dump(report, f, indent=2)
|
|
864
|
+
|
|
865
|
+
def _export_txt(self, output_path: Path) -> None:
|
|
866
|
+
"""Export anomalies to text format.
|
|
867
|
+
|
|
868
|
+
Args:
|
|
869
|
+
output_path: Path to output text file
|
|
870
|
+
"""
|
|
871
|
+
with output_path.open("w") as f:
|
|
872
|
+
f.write("Anomaly Detection Report\n")
|
|
873
|
+
f.write("=" * 80 + "\n\n")
|
|
874
|
+
|
|
875
|
+
f.write("Configuration:\n")
|
|
876
|
+
f.write(f" Methods: {', '.join(self.config.methods)}\n")
|
|
877
|
+
f.write(f" Z-score threshold: {self.config.zscore_threshold}\n")
|
|
878
|
+
f.write(f" IQR multiplier: {self.config.iqr_multiplier}\n\n")
|
|
879
|
+
|
|
880
|
+
f.write("Summary:\n")
|
|
881
|
+
f.write(f" Total anomalies: {len(self.anomalies)}\n")
|
|
882
|
+
f.write(" By type:\n")
|
|
883
|
+
for anomaly_type, count in self._count_by_type().items():
|
|
884
|
+
f.write(f" {anomaly_type}: {count}\n")
|
|
885
|
+
f.write("\n")
|
|
886
|
+
|
|
887
|
+
f.write("Anomalies:\n")
|
|
888
|
+
f.write("-" * 80 + "\n")
|
|
889
|
+
for i, a in enumerate(self.anomalies, 1):
|
|
890
|
+
f.write(f"{i}. [{a.anomaly_type}] {a.explanation}\n")
|
|
891
|
+
f.write(f" Timestamp: {a.timestamp:.3f}\n")
|
|
892
|
+
f.write(f" Score: {a.score:.3f}\n")
|
|
893
|
+
if a.field_name:
|
|
894
|
+
f.write(f" Field: {a.field_name}\n")
|
|
895
|
+
if a.message_index is not None:
|
|
896
|
+
f.write(f" Message index: {a.message_index}\n")
|
|
897
|
+
f.write("\n")
|
|
898
|
+
|
|
899
|
+
def _count_by_type(self) -> dict[str, int]:
|
|
900
|
+
"""Count anomalies by type.
|
|
901
|
+
|
|
902
|
+
Returns:
|
|
903
|
+
Dictionary mapping anomaly type to count
|
|
904
|
+
"""
|
|
905
|
+
counts: dict[str, int] = {}
|
|
906
|
+
for anomaly in self.anomalies:
|
|
907
|
+
counts[anomaly.anomaly_type] = counts.get(anomaly.anomaly_type, 0) + 1
|
|
908
|
+
return counts
|