oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oscura/__init__.py +169 -167
- oscura/analyzers/__init__.py +3 -0
- oscura/analyzers/classification.py +659 -0
- oscura/analyzers/digital/edges.py +325 -65
- oscura/analyzers/digital/quality.py +293 -166
- oscura/analyzers/digital/timing.py +260 -115
- oscura/analyzers/digital/timing_numba.py +334 -0
- oscura/analyzers/entropy.py +605 -0
- oscura/analyzers/eye/diagram.py +176 -109
- oscura/analyzers/eye/metrics.py +5 -5
- oscura/analyzers/jitter/__init__.py +6 -4
- oscura/analyzers/jitter/ber.py +52 -52
- oscura/analyzers/jitter/classification.py +156 -0
- oscura/analyzers/jitter/decomposition.py +163 -113
- oscura/analyzers/jitter/spectrum.py +80 -64
- oscura/analyzers/ml/__init__.py +39 -0
- oscura/analyzers/ml/features.py +600 -0
- oscura/analyzers/ml/signal_classifier.py +604 -0
- oscura/analyzers/packet/daq.py +246 -158
- oscura/analyzers/packet/parser.py +12 -1
- oscura/analyzers/packet/payload.py +50 -2110
- oscura/analyzers/packet/payload_analysis.py +361 -181
- oscura/analyzers/packet/payload_patterns.py +133 -70
- oscura/analyzers/packet/stream.py +84 -23
- oscura/analyzers/patterns/__init__.py +26 -5
- oscura/analyzers/patterns/anomaly_detection.py +908 -0
- oscura/analyzers/patterns/clustering.py +169 -108
- oscura/analyzers/patterns/clustering_optimized.py +227 -0
- oscura/analyzers/patterns/discovery.py +1 -1
- oscura/analyzers/patterns/matching.py +581 -197
- oscura/analyzers/patterns/pattern_mining.py +778 -0
- oscura/analyzers/patterns/periodic.py +121 -38
- oscura/analyzers/patterns/sequences.py +175 -78
- oscura/analyzers/power/conduction.py +1 -1
- oscura/analyzers/power/soa.py +6 -6
- oscura/analyzers/power/switching.py +250 -110
- oscura/analyzers/protocol/__init__.py +17 -1
- oscura/analyzers/protocols/base.py +6 -6
- oscura/analyzers/protocols/ble/__init__.py +38 -0
- oscura/analyzers/protocols/ble/analyzer.py +809 -0
- oscura/analyzers/protocols/ble/uuids.py +288 -0
- oscura/analyzers/protocols/can.py +257 -127
- oscura/analyzers/protocols/can_fd.py +107 -80
- oscura/analyzers/protocols/flexray.py +139 -80
- oscura/analyzers/protocols/hdlc.py +93 -58
- oscura/analyzers/protocols/i2c.py +247 -106
- oscura/analyzers/protocols/i2s.py +138 -86
- oscura/analyzers/protocols/industrial/__init__.py +40 -0
- oscura/analyzers/protocols/industrial/bacnet/__init__.py +33 -0
- oscura/analyzers/protocols/industrial/bacnet/analyzer.py +708 -0
- oscura/analyzers/protocols/industrial/bacnet/encoding.py +412 -0
- oscura/analyzers/protocols/industrial/bacnet/services.py +622 -0
- oscura/analyzers/protocols/industrial/ethercat/__init__.py +30 -0
- oscura/analyzers/protocols/industrial/ethercat/analyzer.py +474 -0
- oscura/analyzers/protocols/industrial/ethercat/mailbox.py +339 -0
- oscura/analyzers/protocols/industrial/ethercat/topology.py +166 -0
- oscura/analyzers/protocols/industrial/modbus/__init__.py +31 -0
- oscura/analyzers/protocols/industrial/modbus/analyzer.py +525 -0
- oscura/analyzers/protocols/industrial/modbus/crc.py +79 -0
- oscura/analyzers/protocols/industrial/modbus/functions.py +436 -0
- oscura/analyzers/protocols/industrial/opcua/__init__.py +21 -0
- oscura/analyzers/protocols/industrial/opcua/analyzer.py +552 -0
- oscura/analyzers/protocols/industrial/opcua/datatypes.py +446 -0
- oscura/analyzers/protocols/industrial/opcua/services.py +264 -0
- oscura/analyzers/protocols/industrial/profinet/__init__.py +23 -0
- oscura/analyzers/protocols/industrial/profinet/analyzer.py +441 -0
- oscura/analyzers/protocols/industrial/profinet/dcp.py +263 -0
- oscura/analyzers/protocols/industrial/profinet/ptcp.py +200 -0
- oscura/analyzers/protocols/jtag.py +180 -98
- oscura/analyzers/protocols/lin.py +219 -114
- oscura/analyzers/protocols/manchester.py +4 -4
- oscura/analyzers/protocols/onewire.py +253 -149
- oscura/analyzers/protocols/parallel_bus/__init__.py +20 -0
- oscura/analyzers/protocols/parallel_bus/centronics.py +92 -0
- oscura/analyzers/protocols/parallel_bus/gpib.py +137 -0
- oscura/analyzers/protocols/spi.py +192 -95
- oscura/analyzers/protocols/swd.py +321 -167
- oscura/analyzers/protocols/uart.py +267 -125
- oscura/analyzers/protocols/usb.py +235 -131
- oscura/analyzers/side_channel/power.py +17 -12
- oscura/analyzers/signal/__init__.py +15 -0
- oscura/analyzers/signal/timing_analysis.py +1086 -0
- oscura/analyzers/signal_integrity/__init__.py +4 -1
- oscura/analyzers/signal_integrity/sparams.py +2 -19
- oscura/analyzers/spectral/chunked.py +129 -60
- oscura/analyzers/spectral/chunked_fft.py +300 -94
- oscura/analyzers/spectral/chunked_wavelet.py +100 -80
- oscura/analyzers/statistical/checksum.py +376 -217
- oscura/analyzers/statistical/classification.py +229 -107
- oscura/analyzers/statistical/entropy.py +78 -53
- oscura/analyzers/statistics/correlation.py +407 -211
- oscura/analyzers/statistics/outliers.py +2 -2
- oscura/analyzers/statistics/streaming.py +30 -5
- oscura/analyzers/validation.py +216 -101
- oscura/analyzers/waveform/measurements.py +9 -0
- oscura/analyzers/waveform/measurements_with_uncertainty.py +31 -15
- oscura/analyzers/waveform/spectral.py +500 -228
- oscura/api/__init__.py +31 -5
- oscura/api/dsl/__init__.py +582 -0
- oscura/{dsl → api/dsl}/commands.py +43 -76
- oscura/{dsl → api/dsl}/interpreter.py +26 -51
- oscura/{dsl → api/dsl}/parser.py +107 -77
- oscura/{dsl → api/dsl}/repl.py +2 -2
- oscura/api/dsl.py +1 -1
- oscura/{integrations → api/integrations}/__init__.py +1 -1
- oscura/{integrations → api/integrations}/llm.py +201 -102
- oscura/api/operators.py +3 -3
- oscura/api/optimization.py +144 -30
- oscura/api/rest_server.py +921 -0
- oscura/api/server/__init__.py +17 -0
- oscura/api/server/dashboard.py +850 -0
- oscura/api/server/static/README.md +34 -0
- oscura/api/server/templates/base.html +181 -0
- oscura/api/server/templates/export.html +120 -0
- oscura/api/server/templates/home.html +284 -0
- oscura/api/server/templates/protocols.html +58 -0
- oscura/api/server/templates/reports.html +43 -0
- oscura/api/server/templates/session_detail.html +89 -0
- oscura/api/server/templates/sessions.html +83 -0
- oscura/api/server/templates/waveforms.html +73 -0
- oscura/automotive/__init__.py +8 -1
- oscura/automotive/can/__init__.py +10 -0
- oscura/automotive/can/checksum.py +3 -1
- oscura/automotive/can/dbc_generator.py +590 -0
- oscura/automotive/can/message_wrapper.py +121 -74
- oscura/automotive/can/patterns.py +98 -21
- oscura/automotive/can/session.py +292 -56
- oscura/automotive/can/state_machine.py +6 -3
- oscura/automotive/can/stimulus_response.py +97 -75
- oscura/automotive/dbc/__init__.py +10 -2
- oscura/automotive/dbc/generator.py +84 -56
- oscura/automotive/dbc/parser.py +6 -6
- oscura/automotive/dtc/data.json +17 -102
- oscura/automotive/dtc/database.py +2 -2
- oscura/automotive/flexray/__init__.py +31 -0
- oscura/automotive/flexray/analyzer.py +504 -0
- oscura/automotive/flexray/crc.py +185 -0
- oscura/automotive/flexray/fibex.py +449 -0
- oscura/automotive/j1939/__init__.py +45 -8
- oscura/automotive/j1939/analyzer.py +605 -0
- oscura/automotive/j1939/spns.py +326 -0
- oscura/automotive/j1939/transport.py +306 -0
- oscura/automotive/lin/__init__.py +47 -0
- oscura/automotive/lin/analyzer.py +612 -0
- oscura/automotive/loaders/blf.py +13 -2
- oscura/automotive/loaders/csv_can.py +143 -72
- oscura/automotive/loaders/dispatcher.py +50 -2
- oscura/automotive/loaders/mdf.py +86 -45
- oscura/automotive/loaders/pcap.py +111 -61
- oscura/automotive/uds/__init__.py +4 -0
- oscura/automotive/uds/analyzer.py +725 -0
- oscura/automotive/uds/decoder.py +140 -58
- oscura/automotive/uds/models.py +7 -1
- oscura/automotive/visualization.py +1 -1
- oscura/cli/analyze.py +348 -0
- oscura/cli/batch.py +142 -122
- oscura/cli/benchmark.py +275 -0
- oscura/cli/characterize.py +137 -82
- oscura/cli/compare.py +224 -131
- oscura/cli/completion.py +250 -0
- oscura/cli/config_cmd.py +361 -0
- oscura/cli/decode.py +164 -87
- oscura/cli/export.py +286 -0
- oscura/cli/main.py +115 -31
- oscura/{onboarding → cli/onboarding}/__init__.py +3 -3
- oscura/{onboarding → cli/onboarding}/help.py +80 -58
- oscura/{onboarding → cli/onboarding}/tutorials.py +97 -72
- oscura/{onboarding → cli/onboarding}/wizard.py +55 -36
- oscura/cli/progress.py +147 -0
- oscura/cli/shell.py +157 -135
- oscura/cli/validate_cmd.py +204 -0
- oscura/cli/visualize.py +158 -0
- oscura/convenience.py +125 -79
- oscura/core/__init__.py +4 -2
- oscura/core/backend_selector.py +3 -3
- oscura/core/cache.py +126 -15
- oscura/core/cancellation.py +1 -1
- oscura/{config → core/config}/__init__.py +20 -11
- oscura/{config → core/config}/defaults.py +1 -1
- oscura/{config → core/config}/loader.py +7 -5
- oscura/{config → core/config}/memory.py +5 -5
- oscura/{config → core/config}/migration.py +1 -1
- oscura/{config → core/config}/pipeline.py +99 -23
- oscura/{config → core/config}/preferences.py +1 -1
- oscura/{config → core/config}/protocol.py +3 -3
- oscura/{config → core/config}/schema.py +426 -272
- oscura/{config → core/config}/settings.py +1 -1
- oscura/{config → core/config}/thresholds.py +195 -153
- oscura/core/correlation.py +5 -6
- oscura/core/cross_domain.py +0 -2
- oscura/core/debug.py +9 -5
- oscura/{extensibility → core/extensibility}/docs.py +158 -70
- oscura/{extensibility → core/extensibility}/extensions.py +160 -76
- oscura/{extensibility → core/extensibility}/logging.py +1 -1
- oscura/{extensibility → core/extensibility}/measurements.py +1 -1
- oscura/{extensibility → core/extensibility}/plugins.py +1 -1
- oscura/{extensibility → core/extensibility}/templates.py +73 -3
- oscura/{extensibility → core/extensibility}/validation.py +1 -1
- oscura/core/gpu_backend.py +11 -7
- oscura/core/log_query.py +101 -11
- oscura/core/logging.py +126 -54
- oscura/core/logging_advanced.py +5 -5
- oscura/core/memory_limits.py +108 -70
- oscura/core/memory_monitor.py +2 -2
- oscura/core/memory_progress.py +7 -7
- oscura/core/memory_warnings.py +1 -1
- oscura/core/numba_backend.py +13 -13
- oscura/{plugins → core/plugins}/__init__.py +9 -9
- oscura/{plugins → core/plugins}/base.py +7 -7
- oscura/{plugins → core/plugins}/cli.py +3 -3
- oscura/{plugins → core/plugins}/discovery.py +186 -106
- oscura/{plugins → core/plugins}/lifecycle.py +1 -1
- oscura/{plugins → core/plugins}/manager.py +7 -7
- oscura/{plugins → core/plugins}/registry.py +3 -3
- oscura/{plugins → core/plugins}/versioning.py +1 -1
- oscura/core/progress.py +16 -1
- oscura/core/provenance.py +8 -2
- oscura/{schemas → core/schemas}/__init__.py +2 -2
- oscura/{schemas → core/schemas}/device_mapping.json +2 -8
- oscura/{schemas → core/schemas}/packet_format.json +4 -24
- oscura/{schemas → core/schemas}/protocol_definition.json +2 -12
- oscura/core/types.py +4 -0
- oscura/core/uncertainty.py +3 -3
- oscura/correlation/__init__.py +52 -0
- oscura/correlation/multi_protocol.py +811 -0
- oscura/discovery/auto_decoder.py +117 -35
- oscura/discovery/comparison.py +191 -86
- oscura/discovery/quality_validator.py +155 -68
- oscura/discovery/signal_detector.py +196 -79
- oscura/export/__init__.py +18 -8
- oscura/export/kaitai_struct.py +513 -0
- oscura/export/scapy_layer.py +801 -0
- oscura/export/wireshark/generator.py +1 -1
- oscura/export/wireshark/templates/dissector.lua.j2 +2 -2
- oscura/export/wireshark_dissector.py +746 -0
- oscura/guidance/wizard.py +207 -111
- oscura/hardware/__init__.py +19 -0
- oscura/{acquisition → hardware/acquisition}/__init__.py +4 -4
- oscura/{acquisition → hardware/acquisition}/file.py +2 -2
- oscura/{acquisition → hardware/acquisition}/hardware.py +7 -7
- oscura/{acquisition → hardware/acquisition}/saleae.py +15 -12
- oscura/{acquisition → hardware/acquisition}/socketcan.py +1 -1
- oscura/{acquisition → hardware/acquisition}/streaming.py +2 -2
- oscura/{acquisition → hardware/acquisition}/synthetic.py +3 -3
- oscura/{acquisition → hardware/acquisition}/visa.py +33 -11
- oscura/hardware/firmware/__init__.py +29 -0
- oscura/hardware/firmware/pattern_recognition.py +874 -0
- oscura/hardware/hal_detector.py +736 -0
- oscura/hardware/security/__init__.py +37 -0
- oscura/hardware/security/side_channel_detector.py +1126 -0
- oscura/inference/__init__.py +4 -0
- oscura/inference/active_learning/observation_table.py +4 -1
- oscura/inference/alignment.py +216 -123
- oscura/inference/bayesian.py +113 -33
- oscura/inference/crc_reverse.py +101 -55
- oscura/inference/logic.py +6 -2
- oscura/inference/message_format.py +342 -183
- oscura/inference/protocol.py +95 -44
- oscura/inference/protocol_dsl.py +180 -82
- oscura/inference/signal_intelligence.py +1439 -706
- oscura/inference/spectral.py +99 -57
- oscura/inference/state_machine.py +810 -158
- oscura/inference/stream.py +270 -110
- oscura/iot/__init__.py +34 -0
- oscura/iot/coap/__init__.py +32 -0
- oscura/iot/coap/analyzer.py +668 -0
- oscura/iot/coap/options.py +212 -0
- oscura/iot/lorawan/__init__.py +21 -0
- oscura/iot/lorawan/crypto.py +206 -0
- oscura/iot/lorawan/decoder.py +801 -0
- oscura/iot/lorawan/mac_commands.py +341 -0
- oscura/iot/mqtt/__init__.py +27 -0
- oscura/iot/mqtt/analyzer.py +999 -0
- oscura/iot/mqtt/properties.py +315 -0
- oscura/iot/zigbee/__init__.py +31 -0
- oscura/iot/zigbee/analyzer.py +615 -0
- oscura/iot/zigbee/security.py +153 -0
- oscura/iot/zigbee/zcl.py +349 -0
- oscura/jupyter/display.py +125 -45
- oscura/{exploratory → jupyter/exploratory}/__init__.py +8 -8
- oscura/{exploratory → jupyter/exploratory}/error_recovery.py +298 -141
- oscura/jupyter/exploratory/fuzzy.py +746 -0
- oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py +258 -100
- oscura/{exploratory → jupyter/exploratory}/legacy.py +464 -242
- oscura/{exploratory → jupyter/exploratory}/parse.py +167 -145
- oscura/{exploratory → jupyter/exploratory}/recovery.py +119 -87
- oscura/jupyter/exploratory/sync.py +612 -0
- oscura/{exploratory → jupyter/exploratory}/unknown.py +299 -176
- oscura/jupyter/magic.py +4 -4
- oscura/{ui → jupyter/ui}/__init__.py +2 -2
- oscura/{ui → jupyter/ui}/formatters.py +3 -3
- oscura/{ui → jupyter/ui}/progressive_display.py +153 -82
- oscura/loaders/__init__.py +183 -67
- oscura/loaders/binary.py +88 -1
- oscura/loaders/chipwhisperer.py +153 -137
- oscura/loaders/configurable.py +208 -86
- oscura/loaders/csv_loader.py +458 -215
- oscura/loaders/hdf5_loader.py +278 -119
- oscura/loaders/lazy.py +87 -54
- oscura/loaders/mmap_loader.py +1 -1
- oscura/loaders/numpy_loader.py +253 -116
- oscura/loaders/pcap.py +226 -151
- oscura/loaders/rigol.py +110 -49
- oscura/loaders/sigrok.py +201 -78
- oscura/loaders/tdms.py +81 -58
- oscura/loaders/tektronix.py +291 -174
- oscura/loaders/touchstone.py +182 -87
- oscura/loaders/tss.py +456 -0
- oscura/loaders/vcd.py +215 -117
- oscura/loaders/wav.py +155 -68
- oscura/reporting/__init__.py +9 -0
- oscura/reporting/analyze.py +352 -146
- oscura/reporting/argument_preparer.py +69 -14
- oscura/reporting/auto_report.py +97 -61
- oscura/reporting/batch.py +131 -58
- oscura/reporting/chart_selection.py +57 -45
- oscura/reporting/comparison.py +63 -17
- oscura/reporting/content/executive.py +76 -24
- oscura/reporting/core_formats/multi_format.py +11 -8
- oscura/reporting/engine.py +312 -158
- oscura/reporting/enhanced_reports.py +949 -0
- oscura/reporting/export.py +86 -43
- oscura/reporting/formatting/numbers.py +69 -42
- oscura/reporting/html.py +139 -58
- oscura/reporting/index.py +137 -65
- oscura/reporting/output.py +158 -67
- oscura/reporting/pdf.py +67 -102
- oscura/reporting/plots.py +191 -112
- oscura/reporting/sections.py +88 -47
- oscura/reporting/standards.py +104 -61
- oscura/reporting/summary_generator.py +75 -55
- oscura/reporting/tables.py +138 -54
- oscura/reporting/templates/enhanced/protocol_re.html +525 -0
- oscura/sessions/__init__.py +14 -23
- oscura/sessions/base.py +3 -3
- oscura/sessions/blackbox.py +106 -10
- oscura/sessions/generic.py +2 -2
- oscura/sessions/legacy.py +783 -0
- oscura/side_channel/__init__.py +63 -0
- oscura/side_channel/dpa.py +1025 -0
- oscura/utils/__init__.py +15 -1
- oscura/utils/bitwise.py +118 -0
- oscura/{builders → utils/builders}/__init__.py +1 -1
- oscura/{comparison → utils/comparison}/__init__.py +6 -6
- oscura/{comparison → utils/comparison}/compare.py +202 -101
- oscura/{comparison → utils/comparison}/golden.py +83 -63
- oscura/{comparison → utils/comparison}/limits.py +313 -89
- oscura/{comparison → utils/comparison}/mask.py +151 -45
- oscura/{comparison → utils/comparison}/trace_diff.py +1 -1
- oscura/{comparison → utils/comparison}/visualization.py +147 -89
- oscura/{component → utils/component}/__init__.py +3 -3
- oscura/{component → utils/component}/impedance.py +122 -58
- oscura/{component → utils/component}/reactive.py +165 -168
- oscura/{component → utils/component}/transmission_line.py +3 -3
- oscura/{filtering → utils/filtering}/__init__.py +6 -6
- oscura/{filtering → utils/filtering}/base.py +1 -1
- oscura/{filtering → utils/filtering}/convenience.py +2 -2
- oscura/{filtering → utils/filtering}/design.py +169 -93
- oscura/{filtering → utils/filtering}/filters.py +2 -2
- oscura/{filtering → utils/filtering}/introspection.py +2 -2
- oscura/utils/geometry.py +31 -0
- oscura/utils/imports.py +184 -0
- oscura/utils/lazy.py +1 -1
- oscura/{math → utils/math}/__init__.py +2 -2
- oscura/{math → utils/math}/arithmetic.py +114 -48
- oscura/{math → utils/math}/interpolation.py +139 -106
- oscura/utils/memory.py +129 -66
- oscura/utils/memory_advanced.py +92 -9
- oscura/utils/memory_extensions.py +10 -8
- oscura/{optimization → utils/optimization}/__init__.py +1 -1
- oscura/{optimization → utils/optimization}/search.py +2 -2
- oscura/utils/performance/__init__.py +58 -0
- oscura/utils/performance/caching.py +889 -0
- oscura/utils/performance/lsh_clustering.py +333 -0
- oscura/utils/performance/memory_optimizer.py +699 -0
- oscura/utils/performance/optimizations.py +675 -0
- oscura/utils/performance/parallel.py +654 -0
- oscura/utils/performance/profiling.py +661 -0
- oscura/{pipeline → utils/pipeline}/base.py +1 -1
- oscura/{pipeline → utils/pipeline}/composition.py +1 -1
- oscura/{pipeline → utils/pipeline}/parallel.py +3 -2
- oscura/{pipeline → utils/pipeline}/pipeline.py +1 -1
- oscura/{pipeline → utils/pipeline}/reverse_engineering.py +412 -221
- oscura/{search → utils/search}/__init__.py +3 -3
- oscura/{search → utils/search}/anomaly.py +188 -58
- oscura/utils/search/context.py +294 -0
- oscura/{search → utils/search}/pattern.py +138 -10
- oscura/utils/serial.py +51 -0
- oscura/utils/storage/__init__.py +61 -0
- oscura/utils/storage/database.py +1166 -0
- oscura/{streaming → utils/streaming}/chunked.py +302 -143
- oscura/{streaming → utils/streaming}/progressive.py +1 -1
- oscura/{streaming → utils/streaming}/realtime.py +3 -2
- oscura/{triggering → utils/triggering}/__init__.py +6 -6
- oscura/{triggering → utils/triggering}/base.py +6 -6
- oscura/{triggering → utils/triggering}/edge.py +2 -2
- oscura/{triggering → utils/triggering}/pattern.py +2 -2
- oscura/{triggering → utils/triggering}/pulse.py +115 -74
- oscura/{triggering → utils/triggering}/window.py +2 -2
- oscura/utils/validation.py +32 -0
- oscura/validation/__init__.py +121 -0
- oscura/{compliance → validation/compliance}/__init__.py +5 -5
- oscura/{compliance → validation/compliance}/advanced.py +5 -5
- oscura/{compliance → validation/compliance}/masks.py +1 -1
- oscura/{compliance → validation/compliance}/reporting.py +127 -53
- oscura/{compliance → validation/compliance}/testing.py +114 -52
- oscura/validation/compliance_tests.py +915 -0
- oscura/validation/fuzzer.py +990 -0
- oscura/validation/grammar_tests.py +596 -0
- oscura/validation/grammar_validator.py +904 -0
- oscura/validation/hil_testing.py +977 -0
- oscura/{quality → validation/quality}/__init__.py +4 -4
- oscura/{quality → validation/quality}/ensemble.py +251 -171
- oscura/{quality → validation/quality}/explainer.py +3 -3
- oscura/{quality → validation/quality}/scoring.py +1 -1
- oscura/{quality → validation/quality}/warnings.py +4 -4
- oscura/validation/regression_suite.py +808 -0
- oscura/validation/replay.py +788 -0
- oscura/{testing → validation/testing}/__init__.py +2 -2
- oscura/{testing → validation/testing}/synthetic.py +5 -5
- oscura/visualization/__init__.py +9 -0
- oscura/visualization/accessibility.py +1 -1
- oscura/visualization/annotations.py +64 -67
- oscura/visualization/colors.py +7 -7
- oscura/visualization/digital.py +180 -81
- oscura/visualization/eye.py +236 -85
- oscura/visualization/interactive.py +320 -143
- oscura/visualization/jitter.py +587 -247
- oscura/visualization/layout.py +169 -134
- oscura/visualization/optimization.py +103 -52
- oscura/visualization/palettes.py +1 -1
- oscura/visualization/power.py +427 -211
- oscura/visualization/power_extended.py +626 -297
- oscura/visualization/presets.py +2 -0
- oscura/visualization/protocols.py +495 -181
- oscura/visualization/render.py +79 -63
- oscura/visualization/reverse_engineering.py +171 -124
- oscura/visualization/signal_integrity.py +460 -279
- oscura/visualization/specialized.py +190 -100
- oscura/visualization/spectral.py +670 -255
- oscura/visualization/thumbnails.py +166 -137
- oscura/visualization/waveform.py +150 -63
- oscura/workflows/__init__.py +3 -0
- oscura/{batch → workflows/batch}/__init__.py +5 -5
- oscura/{batch → workflows/batch}/advanced.py +150 -75
- oscura/workflows/batch/aggregate.py +531 -0
- oscura/workflows/batch/analyze.py +236 -0
- oscura/{batch → workflows/batch}/logging.py +2 -2
- oscura/{batch → workflows/batch}/metrics.py +1 -1
- oscura/workflows/complete_re.py +1144 -0
- oscura/workflows/compliance.py +44 -54
- oscura/workflows/digital.py +197 -51
- oscura/workflows/legacy/__init__.py +12 -0
- oscura/{workflow → workflows/legacy}/dag.py +4 -1
- oscura/workflows/multi_trace.py +9 -9
- oscura/workflows/power.py +42 -62
- oscura/workflows/protocol.py +82 -49
- oscura/workflows/reverse_engineering.py +351 -150
- oscura/workflows/signal_integrity.py +157 -82
- oscura-0.7.0.dist-info/METADATA +661 -0
- oscura-0.7.0.dist-info/RECORD +591 -0
- oscura/batch/aggregate.py +0 -300
- oscura/batch/analyze.py +0 -139
- oscura/dsl/__init__.py +0 -73
- oscura/exceptions.py +0 -59
- oscura/exploratory/fuzzy.py +0 -513
- oscura/exploratory/sync.py +0 -384
- oscura/exporters/__init__.py +0 -94
- oscura/exporters/csv.py +0 -303
- oscura/exporters/exporters.py +0 -44
- oscura/exporters/hdf5.py +0 -217
- oscura/exporters/html_export.py +0 -701
- oscura/exporters/json_export.py +0 -291
- oscura/exporters/markdown_export.py +0 -367
- oscura/exporters/matlab_export.py +0 -354
- oscura/exporters/npz_export.py +0 -219
- oscura/exporters/spice_export.py +0 -210
- oscura/search/context.py +0 -149
- oscura/session/__init__.py +0 -34
- oscura/session/annotations.py +0 -289
- oscura/session/history.py +0 -313
- oscura/session/session.py +0 -520
- oscura/workflow/__init__.py +0 -13
- oscura-0.5.1.dist-info/METADATA +0 -583
- oscura-0.5.1.dist-info/RECORD +0 -481
- /oscura/core/{config.py → config/legacy.py} +0 -0
- /oscura/{extensibility → core/extensibility}/__init__.py +0 -0
- /oscura/{extensibility → core/extensibility}/registry.py +0 -0
- /oscura/{plugins → core/plugins}/isolation.py +0 -0
- /oscura/{schemas → core/schemas}/bus_configuration.json +0 -0
- /oscura/{builders → utils/builders}/signal_builder.py +0 -0
- /oscura/{optimization → utils/optimization}/parallel.py +0 -0
- /oscura/{pipeline → utils/pipeline}/__init__.py +0 -0
- /oscura/{streaming → utils/streaming}/__init__.py +0 -0
- {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/WHEEL +0 -0
- {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/entry_points.txt +0 -0
- {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,778 @@
|
|
|
1
|
+
"""Pattern mining and correlation analysis for protocol traffic.
|
|
2
|
+
|
|
3
|
+
This module implements pattern mining algorithms (FP-Growth, Apriori) for
|
|
4
|
+
discovering repeated byte sequences, field correlations, and temporal patterns
|
|
5
|
+
in protocol message traffic.
|
|
6
|
+
|
|
7
|
+
Example:
|
|
8
|
+
>>> from oscura.analyzers.patterns.pattern_mining import PatternMiner
|
|
9
|
+
>>> miner = PatternMiner(min_support=0.1, min_confidence=0.5)
|
|
10
|
+
>>> messages = [b"\\xAA\\xBB\\xCC", b"\\xAA\\xBB\\xDD", b"\\xAA\\xBB\\xCC"]
|
|
11
|
+
>>> patterns = miner.mine_byte_patterns(messages)
|
|
12
|
+
>>> print(f"Found {len(patterns)} frequent patterns")
|
|
13
|
+
>>> rules = miner.find_associations(patterns)
|
|
14
|
+
>>> print(f"Discovered {len(rules)} association rules")
|
|
15
|
+
|
|
16
|
+
References:
|
|
17
|
+
Han et al. (2000): "Mining Frequent Patterns without Candidate Generation"
|
|
18
|
+
Agrawal & Srikant (1994): "Fast Algorithms for Mining Association Rules"
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from collections import defaultdict
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Any, Literal
|
|
27
|
+
|
|
28
|
+
import numpy as np
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class Pattern:
|
|
33
|
+
"""Discovered pattern in message traffic.
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
sequence: Byte sequence or field values (tuple of integers).
|
|
37
|
+
support: Frequency of pattern (0.0-1.0, fraction of messages).
|
|
38
|
+
confidence: Confidence for association rules (optional).
|
|
39
|
+
locations: List of (message_idx, offset) tuples where pattern appears.
|
|
40
|
+
metadata: Additional pattern metadata.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
sequence: tuple[int, ...]
|
|
44
|
+
support: float
|
|
45
|
+
confidence: float | None = None
|
|
46
|
+
locations: list[tuple[int, int]] = field(default_factory=list)
|
|
47
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
48
|
+
|
|
49
|
+
def __repr__(self) -> str:
|
|
50
|
+
"""Generate readable representation."""
|
|
51
|
+
seq_str = " ".join(f"{b:02X}" for b in self.sequence)
|
|
52
|
+
return f"Pattern([{seq_str}], support={self.support:.3f})"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class AssociationRule:
|
|
57
|
+
"""Association rule between patterns (A -> B).
|
|
58
|
+
|
|
59
|
+
Represents the rule: "If A appears, then B follows".
|
|
60
|
+
|
|
61
|
+
Attributes:
|
|
62
|
+
antecedent: Pattern A (if this appears).
|
|
63
|
+
consequent: Pattern B (then this follows).
|
|
64
|
+
support: Frequency of (A, B) appearing together.
|
|
65
|
+
confidence: Probability P(B|A) = support(A, B) / support(A).
|
|
66
|
+
lift: Confidence / P(B), measures rule strength.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
antecedent: tuple[int, ...]
|
|
70
|
+
consequent: tuple[int, ...]
|
|
71
|
+
support: float
|
|
72
|
+
confidence: float
|
|
73
|
+
lift: float
|
|
74
|
+
|
|
75
|
+
def __repr__(self) -> str:
|
|
76
|
+
"""Generate readable representation."""
|
|
77
|
+
ant_str = " ".join(f"{b:02X}" for b in self.antecedent)
|
|
78
|
+
con_str = " ".join(f"{b:02X}" for b in self.consequent)
|
|
79
|
+
return f"Rule([{ant_str}] -> [{con_str}], conf={self.confidence:.3f}, lift={self.lift:.2f})"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass
|
|
83
|
+
class TemporalPattern:
|
|
84
|
+
"""Temporal sequence pattern in events.
|
|
85
|
+
|
|
86
|
+
Represents a sequence of events that occur with regular timing.
|
|
87
|
+
|
|
88
|
+
Attributes:
|
|
89
|
+
events: Sequence of event type names.
|
|
90
|
+
timestamps: Event timestamps (relative to first event).
|
|
91
|
+
avg_interval: Average time between consecutive events.
|
|
92
|
+
variance: Variance in event timing.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
events: list[str]
|
|
96
|
+
timestamps: list[float]
|
|
97
|
+
avg_interval: float
|
|
98
|
+
variance: float
|
|
99
|
+
|
|
100
|
+
def __repr__(self) -> str:
|
|
101
|
+
"""Generate readable representation."""
|
|
102
|
+
events_str = " -> ".join(self.events)
|
|
103
|
+
return (
|
|
104
|
+
f"TemporalPattern([{events_str}], interval={self.avg_interval:.3f}±{self.variance:.3f})"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class PatternMiner:
|
|
109
|
+
"""Pattern mining and correlation analysis for protocol traffic.
|
|
110
|
+
|
|
111
|
+
Implements FP-Growth and Apriori algorithms for mining frequent patterns,
|
|
112
|
+
association rule discovery, temporal pattern detection, and field correlation
|
|
113
|
+
analysis.
|
|
114
|
+
|
|
115
|
+
Example:
|
|
116
|
+
>>> miner = PatternMiner(min_support=0.1, min_confidence=0.5)
|
|
117
|
+
>>> patterns = miner.mine_byte_patterns(messages, algorithm="fp_growth")
|
|
118
|
+
>>> rules = miner.find_associations(patterns)
|
|
119
|
+
>>> temporal = miner.mine_temporal_patterns(events, max_gap=1.0)
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
def __init__(
|
|
123
|
+
self,
|
|
124
|
+
min_support: float = 0.1,
|
|
125
|
+
min_confidence: float = 0.5,
|
|
126
|
+
min_pattern_length: int = 2,
|
|
127
|
+
max_pattern_length: int = 10,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""Initialize pattern miner with thresholds.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
min_support: Minimum pattern frequency (0.0-1.0).
|
|
133
|
+
min_confidence: Minimum confidence for association rules (0.0-1.0).
|
|
134
|
+
min_pattern_length: Minimum pattern length in bytes.
|
|
135
|
+
max_pattern_length: Maximum pattern length in bytes.
|
|
136
|
+
|
|
137
|
+
Raises:
|
|
138
|
+
ValueError: If parameters are out of valid range.
|
|
139
|
+
"""
|
|
140
|
+
if not 0.0 <= min_support <= 1.0:
|
|
141
|
+
raise ValueError(f"min_support must be in [0.0, 1.0], got {min_support}")
|
|
142
|
+
if not 0.0 <= min_confidence <= 1.0:
|
|
143
|
+
raise ValueError(f"min_confidence must be in [0.0, 1.0], got {min_confidence}")
|
|
144
|
+
if min_pattern_length < 1:
|
|
145
|
+
raise ValueError(f"min_pattern_length must be >= 1, got {min_pattern_length}")
|
|
146
|
+
if max_pattern_length < min_pattern_length:
|
|
147
|
+
raise ValueError(
|
|
148
|
+
f"max_pattern_length ({max_pattern_length}) must be >= "
|
|
149
|
+
f"min_pattern_length ({min_pattern_length})"
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
self.min_support = min_support
|
|
153
|
+
self.min_confidence = min_confidence
|
|
154
|
+
self.min_pattern_length = min_pattern_length
|
|
155
|
+
self.max_pattern_length = max_pattern_length
|
|
156
|
+
self.patterns: list[Pattern] = []
|
|
157
|
+
self.rules: list[AssociationRule] = []
|
|
158
|
+
|
|
159
|
+
def mine_byte_patterns(
|
|
160
|
+
self, messages: list[bytes], algorithm: Literal["fp_growth", "apriori"] = "fp_growth"
|
|
161
|
+
) -> list[Pattern]:
|
|
162
|
+
"""Mine frequent byte patterns from messages.
|
|
163
|
+
|
|
164
|
+
Extracts all subsequences from messages, counts occurrences, and filters
|
|
165
|
+
by minimum support threshold. Returns patterns sorted by frequency.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
messages: List of message byte sequences.
|
|
169
|
+
algorithm: Mining algorithm to use ("fp_growth" or "apriori").
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
List of Pattern objects sorted by support (descending).
|
|
173
|
+
|
|
174
|
+
Raises:
|
|
175
|
+
ValueError: If messages list is empty or algorithm is unknown.
|
|
176
|
+
|
|
177
|
+
Example:
|
|
178
|
+
>>> messages = [b"\\xAA\\xBB\\xCC", b"\\xAA\\xBB\\xDD"]
|
|
179
|
+
>>> patterns = miner.mine_byte_patterns(messages)
|
|
180
|
+
>>> for p in patterns:
|
|
181
|
+
... print(f"{p.sequence}: {p.support:.2f}")
|
|
182
|
+
"""
|
|
183
|
+
if not messages:
|
|
184
|
+
raise ValueError("Messages list cannot be empty")
|
|
185
|
+
|
|
186
|
+
if algorithm not in ("fp_growth", "apriori"):
|
|
187
|
+
raise ValueError(f"Unknown algorithm: {algorithm}")
|
|
188
|
+
|
|
189
|
+
# Extract all subsequences
|
|
190
|
+
pattern_counts: dict[tuple[int, ...], int] = defaultdict(int)
|
|
191
|
+
pattern_locations: dict[tuple[int, ...], list[tuple[int, int]]] = defaultdict(list)
|
|
192
|
+
|
|
193
|
+
for msg_idx, message in enumerate(messages):
|
|
194
|
+
subsequences = self._extract_subsequences(
|
|
195
|
+
message, self.min_pattern_length, self.max_pattern_length
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
for subseq in subsequences:
|
|
199
|
+
# Find all occurrences in this message
|
|
200
|
+
for offset in range(len(message) - len(subseq) + 1):
|
|
201
|
+
if tuple(message[offset : offset + len(subseq)]) == subseq:
|
|
202
|
+
pattern_counts[subseq] += 1
|
|
203
|
+
pattern_locations[subseq].append((msg_idx, offset))
|
|
204
|
+
|
|
205
|
+
# Calculate support
|
|
206
|
+
total_subsequences = sum(pattern_counts.values())
|
|
207
|
+
|
|
208
|
+
# Filter by minimum support
|
|
209
|
+
patterns = []
|
|
210
|
+
for seq, count in pattern_counts.items():
|
|
211
|
+
support = count / total_subsequences if total_subsequences > 0 else 0.0
|
|
212
|
+
|
|
213
|
+
if support >= self.min_support:
|
|
214
|
+
# NECESSARY COPY: Prevents mutations of locations after pattern creation.
|
|
215
|
+
patterns.append(
|
|
216
|
+
Pattern(sequence=seq, support=support, locations=pattern_locations[seq].copy())
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
# Sort by support (most frequent first)
|
|
220
|
+
patterns.sort(key=lambda p: p.support, reverse=True)
|
|
221
|
+
|
|
222
|
+
self.patterns = patterns
|
|
223
|
+
return patterns
|
|
224
|
+
|
|
225
|
+
def mine_field_patterns(
|
|
226
|
+
self, field_sequences: list[list[int]], field_names: list[str]
|
|
227
|
+
) -> list[Pattern]:
|
|
228
|
+
"""Mine patterns in extracted field values.
|
|
229
|
+
|
|
230
|
+
Analyzes sequences of field values across multiple messages to find
|
|
231
|
+
common patterns and value combinations.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
field_sequences: List of field value sequences (one per message).
|
|
235
|
+
field_names: Names of fields in sequences.
|
|
236
|
+
|
|
237
|
+
Returns:
|
|
238
|
+
List of Pattern objects for field value sequences.
|
|
239
|
+
|
|
240
|
+
Raises:
|
|
241
|
+
ValueError: If field_sequences is empty or field_names length mismatch.
|
|
242
|
+
|
|
243
|
+
Example:
|
|
244
|
+
>>> field_sequences = [[0x01, 0x02], [0x01, 0x03], [0x01, 0x02]]
|
|
245
|
+
>>> field_names = ["field_a", "field_b"]
|
|
246
|
+
>>> patterns = miner.mine_field_patterns(field_sequences, field_names)
|
|
247
|
+
"""
|
|
248
|
+
if not field_sequences:
|
|
249
|
+
raise ValueError("Field sequences cannot be empty")
|
|
250
|
+
|
|
251
|
+
if field_names and len(field_names) != len(field_sequences[0]):
|
|
252
|
+
raise ValueError(
|
|
253
|
+
f"Field names length ({len(field_names)}) must match "
|
|
254
|
+
f"sequence length ({len(field_sequences[0])})"
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
# Convert field sequences to transactions for pattern mining
|
|
258
|
+
pattern_counts: dict[tuple[int, ...], int] = defaultdict(int)
|
|
259
|
+
pattern_locations: dict[tuple[int, ...], list[tuple[int, int]]] = defaultdict(list)
|
|
260
|
+
|
|
261
|
+
for msg_idx, field_vals in enumerate(field_sequences):
|
|
262
|
+
# Extract subsequences from field values
|
|
263
|
+
for length in range(
|
|
264
|
+
self.min_pattern_length, min(len(field_vals) + 1, self.max_pattern_length + 1)
|
|
265
|
+
):
|
|
266
|
+
for offset in range(len(field_vals) - length + 1):
|
|
267
|
+
subseq = tuple(field_vals[offset : offset + length])
|
|
268
|
+
pattern_counts[subseq] += 1
|
|
269
|
+
pattern_locations[subseq].append((msg_idx, offset))
|
|
270
|
+
|
|
271
|
+
# Calculate support
|
|
272
|
+
total_patterns = sum(pattern_counts.values())
|
|
273
|
+
|
|
274
|
+
# Filter and create patterns
|
|
275
|
+
patterns = []
|
|
276
|
+
for seq, count in pattern_counts.items():
|
|
277
|
+
support = count / total_patterns if total_patterns > 0 else 0.0
|
|
278
|
+
|
|
279
|
+
if support >= self.min_support:
|
|
280
|
+
# Add field names to metadata
|
|
281
|
+
metadata = {}
|
|
282
|
+
if field_names and len(seq) == 1:
|
|
283
|
+
# Single field pattern
|
|
284
|
+
first_loc = pattern_locations[seq][0] if pattern_locations[seq] else (0, 0)
|
|
285
|
+
offset = first_loc[1]
|
|
286
|
+
if offset < len(field_names):
|
|
287
|
+
metadata["field_name"] = field_names[offset]
|
|
288
|
+
|
|
289
|
+
# NECESSARY COPY: Protects pattern's locations from external mutations.
|
|
290
|
+
patterns.append(
|
|
291
|
+
Pattern(
|
|
292
|
+
sequence=seq,
|
|
293
|
+
support=support,
|
|
294
|
+
locations=pattern_locations[seq].copy(),
|
|
295
|
+
metadata=metadata,
|
|
296
|
+
)
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
# Sort by support
|
|
300
|
+
patterns.sort(key=lambda p: p.support, reverse=True)
|
|
301
|
+
|
|
302
|
+
self.patterns = patterns
|
|
303
|
+
return patterns
|
|
304
|
+
|
|
305
|
+
def find_associations(self, patterns: list[Pattern]) -> list[AssociationRule]:
|
|
306
|
+
"""Find association rules between patterns (A -> B).
|
|
307
|
+
|
|
308
|
+
Discovers rules where pattern A appearing implies pattern B follows,
|
|
309
|
+
with confidence and lift metrics.
|
|
310
|
+
|
|
311
|
+
Args:
|
|
312
|
+
patterns: List of patterns to analyze for associations.
|
|
313
|
+
|
|
314
|
+
Returns:
|
|
315
|
+
List of AssociationRule objects sorted by confidence.
|
|
316
|
+
|
|
317
|
+
Example:
|
|
318
|
+
>>> rules = miner.find_associations(patterns)
|
|
319
|
+
>>> for rule in rules:
|
|
320
|
+
... print(f"{rule.antecedent} -> {rule.consequent}: {rule.confidence:.2f}")
|
|
321
|
+
"""
|
|
322
|
+
if not patterns:
|
|
323
|
+
return []
|
|
324
|
+
|
|
325
|
+
rules = []
|
|
326
|
+
|
|
327
|
+
# Generate rules (check if B follows A immediately)
|
|
328
|
+
for i, pattern_a in enumerate(patterns):
|
|
329
|
+
for j, pattern_b in enumerate(patterns):
|
|
330
|
+
if i >= j:
|
|
331
|
+
continue
|
|
332
|
+
|
|
333
|
+
seq_a = pattern_a.sequence
|
|
334
|
+
seq_b = pattern_b.sequence
|
|
335
|
+
|
|
336
|
+
# Find co-occurrences (B appears immediately after A)
|
|
337
|
+
co_occur_count = 0
|
|
338
|
+
total_occurrences = len(pattern_a.locations)
|
|
339
|
+
|
|
340
|
+
for msg_idx_a, offset_a in pattern_a.locations:
|
|
341
|
+
expected_offset_b = offset_a + len(seq_a)
|
|
342
|
+
|
|
343
|
+
# Check if B appears right after A
|
|
344
|
+
for msg_idx_b, offset_b in pattern_b.locations:
|
|
345
|
+
if msg_idx_a == msg_idx_b and offset_b == expected_offset_b:
|
|
346
|
+
co_occur_count += 1
|
|
347
|
+
break
|
|
348
|
+
|
|
349
|
+
if co_occur_count > 0:
|
|
350
|
+
# Calculate metrics
|
|
351
|
+
# Confidence = P(B|A) = count(A->B) / count(A)
|
|
352
|
+
confidence = (
|
|
353
|
+
co_occur_count / total_occurrences if total_occurrences > 0 else 0.0
|
|
354
|
+
)
|
|
355
|
+
# Support is the frequency of the rule in the dataset
|
|
356
|
+
support_ab = pattern_a.support * confidence
|
|
357
|
+
# Lift = confidence / P(B)
|
|
358
|
+
lift = confidence / pattern_b.support if pattern_b.support > 0 else 0.0
|
|
359
|
+
|
|
360
|
+
if confidence >= self.min_confidence:
|
|
361
|
+
rules.append(
|
|
362
|
+
AssociationRule(
|
|
363
|
+
antecedent=seq_a,
|
|
364
|
+
consequent=seq_b,
|
|
365
|
+
support=support_ab,
|
|
366
|
+
confidence=confidence,
|
|
367
|
+
lift=lift,
|
|
368
|
+
)
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
# Sort by confidence
|
|
372
|
+
rules.sort(key=lambda r: r.confidence, reverse=True)
|
|
373
|
+
|
|
374
|
+
self.rules = rules
|
|
375
|
+
return rules
|
|
376
|
+
|
|
377
|
+
def mine_temporal_patterns(
|
|
378
|
+
self, events: list[tuple[float, str]], max_gap: float = 1.0
|
|
379
|
+
) -> list[TemporalPattern]:
|
|
380
|
+
"""Mine temporal event sequences.
|
|
381
|
+
|
|
382
|
+
Finds sequences like [EventA, EventB, EventC] that occur with regular
|
|
383
|
+
timing intervals.
|
|
384
|
+
|
|
385
|
+
Args:
|
|
386
|
+
events: List of (timestamp, event_type) tuples.
|
|
387
|
+
max_gap: Maximum time gap between consecutive events in sequence.
|
|
388
|
+
|
|
389
|
+
Returns:
|
|
390
|
+
List of TemporalPattern objects.
|
|
391
|
+
|
|
392
|
+
Raises:
|
|
393
|
+
ValueError: If events list is empty or max_gap is negative.
|
|
394
|
+
|
|
395
|
+
Example:
|
|
396
|
+
>>> events = [(0.0, "A"), (0.5, "B"), (1.0, "A"), (1.5, "B")]
|
|
397
|
+
>>> patterns = miner.mine_temporal_patterns(events, max_gap=0.6)
|
|
398
|
+
"""
|
|
399
|
+
if not events:
|
|
400
|
+
raise ValueError("Events list cannot be empty")
|
|
401
|
+
if max_gap < 0:
|
|
402
|
+
raise ValueError(f"max_gap must be non-negative, got {max_gap}")
|
|
403
|
+
|
|
404
|
+
# Sort events by timestamp
|
|
405
|
+
events_sorted = sorted(events, key=lambda e: e[0])
|
|
406
|
+
|
|
407
|
+
# Find sequences
|
|
408
|
+
sequences: dict[tuple[str, ...], list[list[float]]] = defaultdict(list)
|
|
409
|
+
|
|
410
|
+
for i in range(len(events_sorted)):
|
|
411
|
+
current_seq = [events_sorted[i][1]]
|
|
412
|
+
current_times = [events_sorted[i][0]]
|
|
413
|
+
|
|
414
|
+
for j in range(i + 1, min(i + self.max_pattern_length, len(events_sorted))):
|
|
415
|
+
time_gap = events_sorted[j][0] - current_times[-1]
|
|
416
|
+
|
|
417
|
+
if time_gap <= max_gap:
|
|
418
|
+
current_seq.append(events_sorted[j][1])
|
|
419
|
+
current_times.append(events_sorted[j][0])
|
|
420
|
+
|
|
421
|
+
if len(current_seq) >= self.min_pattern_length:
|
|
422
|
+
seq_tuple = tuple(current_seq)
|
|
423
|
+
# NECESSARY COPY: current_times is mutated in loop.
|
|
424
|
+
# Without .copy(), all sequences would reference final state.
|
|
425
|
+
# Removing would cause: data corruption, identical timestamps.
|
|
426
|
+
sequences[seq_tuple].append(current_times.copy())
|
|
427
|
+
else:
|
|
428
|
+
break
|
|
429
|
+
|
|
430
|
+
# Convert to TemporalPattern objects
|
|
431
|
+
temporal_patterns = []
|
|
432
|
+
for seq, time_lists in sequences.items():
|
|
433
|
+
# Calculate average interval
|
|
434
|
+
all_intervals = []
|
|
435
|
+
for times in time_lists:
|
|
436
|
+
intervals = [times[i + 1] - times[i] for i in range(len(times) - 1)]
|
|
437
|
+
all_intervals.extend(intervals)
|
|
438
|
+
|
|
439
|
+
if all_intervals:
|
|
440
|
+
avg_interval = float(np.mean(all_intervals))
|
|
441
|
+
variance = float(np.var(all_intervals))
|
|
442
|
+
|
|
443
|
+
# Use relative timestamps (first event at 0.0)
|
|
444
|
+
relative_times = [t - time_lists[0][0] for t in time_lists[0]]
|
|
445
|
+
|
|
446
|
+
temporal_patterns.append(
|
|
447
|
+
TemporalPattern(
|
|
448
|
+
events=list(seq),
|
|
449
|
+
timestamps=relative_times,
|
|
450
|
+
avg_interval=avg_interval,
|
|
451
|
+
variance=variance,
|
|
452
|
+
)
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
return temporal_patterns
|
|
456
|
+
|
|
457
|
+
def find_correlations(self, field_data: dict[str, list[float]]) -> dict[tuple[str, str], float]:
|
|
458
|
+
"""Calculate Pearson correlation coefficients between fields.
|
|
459
|
+
|
|
460
|
+
Computes pairwise correlations to find relationships between numeric
|
|
461
|
+
field values across messages.
|
|
462
|
+
|
|
463
|
+
Args:
|
|
464
|
+
field_data: Dictionary of field_name -> list of values.
|
|
465
|
+
|
|
466
|
+
Returns:
|
|
467
|
+
Dictionary of (field_a, field_b) -> correlation_coefficient.
|
|
468
|
+
|
|
469
|
+
Raises:
|
|
470
|
+
ValueError: If field_data is empty or fields have different lengths.
|
|
471
|
+
|
|
472
|
+
Example:
|
|
473
|
+
>>> field_data = {"field_a": [1, 2, 3], "field_b": [2, 4, 6]}
|
|
474
|
+
>>> correlations = miner.find_correlations(field_data)
|
|
475
|
+
>>> print(correlations[("field_a", "field_b")])
|
|
476
|
+
1.0
|
|
477
|
+
"""
|
|
478
|
+
if not field_data:
|
|
479
|
+
raise ValueError("Field data cannot be empty")
|
|
480
|
+
|
|
481
|
+
# Validate all fields have same length
|
|
482
|
+
field_names = list(field_data.keys())
|
|
483
|
+
lengths = {name: len(values) for name, values in field_data.items()}
|
|
484
|
+
if len(set(lengths.values())) > 1:
|
|
485
|
+
raise ValueError(f"All fields must have same length, got {lengths}")
|
|
486
|
+
|
|
487
|
+
correlations: dict[tuple[str, str], float] = {}
|
|
488
|
+
|
|
489
|
+
# Compute pairwise correlations
|
|
490
|
+
for i, field_a in enumerate(field_names):
|
|
491
|
+
for j, field_b in enumerate(field_names):
|
|
492
|
+
if i >= j:
|
|
493
|
+
continue
|
|
494
|
+
|
|
495
|
+
values_a = np.array(field_data[field_a], dtype=np.float64)
|
|
496
|
+
values_b = np.array(field_data[field_b], dtype=np.float64)
|
|
497
|
+
|
|
498
|
+
# Calculate Pearson correlation
|
|
499
|
+
if len(values_a) < 2:
|
|
500
|
+
corr = 0.0
|
|
501
|
+
else:
|
|
502
|
+
# Compute correlation coefficient
|
|
503
|
+
mean_a = np.mean(values_a)
|
|
504
|
+
mean_b = np.mean(values_b)
|
|
505
|
+
|
|
506
|
+
centered_a = values_a - mean_a
|
|
507
|
+
centered_b = values_b - mean_b
|
|
508
|
+
|
|
509
|
+
num = np.sum(centered_a * centered_b)
|
|
510
|
+
denom = np.sqrt(np.sum(centered_a**2) * np.sum(centered_b**2))
|
|
511
|
+
|
|
512
|
+
corr = float(num / denom) if denom > 0 else 0.0
|
|
513
|
+
|
|
514
|
+
correlations[(field_a, field_b)] = corr
|
|
515
|
+
# Add symmetric entry
|
|
516
|
+
correlations[(field_b, field_a)] = corr
|
|
517
|
+
|
|
518
|
+
return correlations
|
|
519
|
+
|
|
520
|
+
def _extract_subsequences(
|
|
521
|
+
self, sequence: bytes, min_length: int, max_length: int
|
|
522
|
+
) -> set[tuple[int, ...]]:
|
|
523
|
+
"""Extract all subsequences of given length range.
|
|
524
|
+
|
|
525
|
+
Args:
|
|
526
|
+
sequence: Byte sequence to extract from.
|
|
527
|
+
min_length: Minimum subsequence length.
|
|
528
|
+
max_length: Maximum subsequence length.
|
|
529
|
+
|
|
530
|
+
Returns:
|
|
531
|
+
Set of unique subsequences as tuples.
|
|
532
|
+
"""
|
|
533
|
+
subsequences: set[tuple[int, ...]] = set()
|
|
534
|
+
|
|
535
|
+
for length in range(min_length, min(len(sequence) + 1, max_length + 1)):
|
|
536
|
+
for offset in range(len(sequence) - length + 1):
|
|
537
|
+
subseq = tuple(sequence[offset : offset + length])
|
|
538
|
+
subsequences.add(subseq)
|
|
539
|
+
|
|
540
|
+
return subsequences
|
|
541
|
+
|
|
542
|
+
def _fp_growth(
|
|
543
|
+
self, transactions: list[frozenset[int]], min_support: float
|
|
544
|
+
) -> list[tuple[frozenset[int], int]]:
|
|
545
|
+
"""FP-Growth algorithm for frequent itemset mining.
|
|
546
|
+
|
|
547
|
+
Simplified implementation for pattern mining. For production use,
|
|
548
|
+
consider using libraries like mlxtend.
|
|
549
|
+
|
|
550
|
+
Args:
|
|
551
|
+
transactions: List of transaction itemsets.
|
|
552
|
+
min_support: Minimum support threshold (0.0-1.0).
|
|
553
|
+
|
|
554
|
+
Returns:
|
|
555
|
+
List of (itemset, count) tuples.
|
|
556
|
+
"""
|
|
557
|
+
# Count item frequencies
|
|
558
|
+
item_counts: dict[int, int] = defaultdict(int)
|
|
559
|
+
for transaction in transactions:
|
|
560
|
+
for item in transaction:
|
|
561
|
+
item_counts[item] += 1
|
|
562
|
+
|
|
563
|
+
# Filter by minimum support
|
|
564
|
+
min_count = int(min_support * len(transactions))
|
|
565
|
+
frequent_items = {item for item, count in item_counts.items() if count >= min_count}
|
|
566
|
+
|
|
567
|
+
# Generate frequent itemsets (simplified - single items only for now)
|
|
568
|
+
result = []
|
|
569
|
+
for item in frequent_items:
|
|
570
|
+
count = item_counts[item]
|
|
571
|
+
result.append((frozenset([item]), count))
|
|
572
|
+
|
|
573
|
+
return result
|
|
574
|
+
|
|
575
|
+
def _apriori(
|
|
576
|
+
self, transactions: list[frozenset[int]], min_support: float
|
|
577
|
+
) -> list[tuple[frozenset[int], int]]:
|
|
578
|
+
"""Apriori algorithm for frequent itemset mining.
|
|
579
|
+
|
|
580
|
+
Simplified implementation. For production use, consider libraries
|
|
581
|
+
like mlxtend.
|
|
582
|
+
|
|
583
|
+
Args:
|
|
584
|
+
transactions: List of transaction itemsets.
|
|
585
|
+
min_support: Minimum support threshold (0.0-1.0).
|
|
586
|
+
|
|
587
|
+
Returns:
|
|
588
|
+
List of (itemset, count) tuples.
|
|
589
|
+
"""
|
|
590
|
+
# Use same implementation as FP-Growth for now
|
|
591
|
+
# Full Apriori would generate candidate itemsets iteratively
|
|
592
|
+
return self._fp_growth(transactions, min_support)
|
|
593
|
+
|
|
594
|
+
def visualize_patterns(
|
|
595
|
+
self, output_path: Path, format: Literal["graph", "tree", "heatmap"] = "graph"
|
|
596
|
+
) -> None:
|
|
597
|
+
"""Visualize discovered patterns.
|
|
598
|
+
|
|
599
|
+
Creates visualization of patterns and their relationships. Requires
|
|
600
|
+
matplotlib and networkx for graph visualization.
|
|
601
|
+
|
|
602
|
+
Args:
|
|
603
|
+
output_path: Path to save visualization file.
|
|
604
|
+
format: Visualization format ("graph", "tree", or "heatmap").
|
|
605
|
+
|
|
606
|
+
Raises:
|
|
607
|
+
ValueError: If no patterns have been mined yet.
|
|
608
|
+
ImportError: If required visualization libraries not available.
|
|
609
|
+
"""
|
|
610
|
+
if not self.patterns:
|
|
611
|
+
raise ValueError("No patterns to visualize. Run mine_byte_patterns() first.")
|
|
612
|
+
|
|
613
|
+
if format == "heatmap":
|
|
614
|
+
self._visualize_heatmap(output_path)
|
|
615
|
+
elif format in ("graph", "tree"):
|
|
616
|
+
self._visualize_graph(output_path)
|
|
617
|
+
|
|
618
|
+
def _visualize_heatmap(self, output_path: Path) -> None:
|
|
619
|
+
"""Create heatmap of pattern support values."""
|
|
620
|
+
try:
|
|
621
|
+
import matplotlib.pyplot as plt
|
|
622
|
+
except ImportError as e:
|
|
623
|
+
raise ImportError("matplotlib required for visualization") from e
|
|
624
|
+
|
|
625
|
+
fig, ax = plt.subplots(figsize=(10, 6))
|
|
626
|
+
|
|
627
|
+
pattern_labels = [" ".join(f"{b:02X}" for b in p.sequence) for p in self.patterns[:20]]
|
|
628
|
+
support_values = [p.support for p in self.patterns[:20]]
|
|
629
|
+
|
|
630
|
+
ax.barh(pattern_labels, support_values)
|
|
631
|
+
ax.set_xlabel("Support")
|
|
632
|
+
ax.set_title("Top 20 Pattern Support Values")
|
|
633
|
+
ax.grid(axis="x", alpha=0.3)
|
|
634
|
+
|
|
635
|
+
plt.tight_layout()
|
|
636
|
+
plt.savefig(output_path, dpi=150, bbox_inches="tight")
|
|
637
|
+
plt.close()
|
|
638
|
+
|
|
639
|
+
def _visualize_graph(self, output_path: Path) -> None:
|
|
640
|
+
"""Create graph visualization of pattern relationships."""
|
|
641
|
+
try:
|
|
642
|
+
import matplotlib.pyplot as plt
|
|
643
|
+
import networkx as nx
|
|
644
|
+
except ImportError as e:
|
|
645
|
+
raise ImportError("matplotlib and networkx required for graph visualization") from e
|
|
646
|
+
|
|
647
|
+
# Build graph
|
|
648
|
+
G = self._build_pattern_graph(nx)
|
|
649
|
+
|
|
650
|
+
# Draw graph
|
|
651
|
+
fig, ax = plt.subplots(figsize=(12, 8))
|
|
652
|
+
|
|
653
|
+
pos = nx.spring_layout(G, k=2, iterations=50)
|
|
654
|
+
labels = nx.get_node_attributes(G, "label")
|
|
655
|
+
|
|
656
|
+
nx.draw_networkx_nodes(G, pos, node_size=500, node_color="lightblue", ax=ax)
|
|
657
|
+
nx.draw_networkx_labels(G, pos, labels, font_size=8, ax=ax)
|
|
658
|
+
nx.draw_networkx_edges(G, pos, edge_color="gray", arrows=True, ax=ax)
|
|
659
|
+
|
|
660
|
+
ax.set_title("Pattern Association Graph")
|
|
661
|
+
ax.axis("off")
|
|
662
|
+
|
|
663
|
+
plt.tight_layout()
|
|
664
|
+
plt.savefig(output_path, dpi=150, bbox_inches="tight")
|
|
665
|
+
plt.close()
|
|
666
|
+
|
|
667
|
+
def _build_pattern_graph(self, nx: Any) -> Any:
|
|
668
|
+
"""Build networkx graph from patterns and rules."""
|
|
669
|
+
G = nx.DiGraph()
|
|
670
|
+
|
|
671
|
+
# Add patterns as nodes
|
|
672
|
+
for i, pattern in enumerate(self.patterns[:20]):
|
|
673
|
+
label = " ".join(f"{b:02X}" for b in pattern.sequence)
|
|
674
|
+
G.add_node(i, label=label, support=pattern.support)
|
|
675
|
+
|
|
676
|
+
# Add edges for association rules
|
|
677
|
+
for rule in self.rules:
|
|
678
|
+
ant_idx = next(
|
|
679
|
+
(i for i, p in enumerate(self.patterns) if p.sequence == rule.antecedent),
|
|
680
|
+
None,
|
|
681
|
+
)
|
|
682
|
+
con_idx = next(
|
|
683
|
+
(i for i, p in enumerate(self.patterns) if p.sequence == rule.consequent),
|
|
684
|
+
None,
|
|
685
|
+
)
|
|
686
|
+
|
|
687
|
+
if ant_idx is not None and con_idx is not None:
|
|
688
|
+
G.add_edge(ant_idx, con_idx, confidence=rule.confidence, lift=rule.lift)
|
|
689
|
+
|
|
690
|
+
return G
|
|
691
|
+
|
|
692
|
+
def export_rules(
|
|
693
|
+
self, output_path: Path, format: Literal["json", "csv", "yaml"] = "json"
|
|
694
|
+
) -> None:
|
|
695
|
+
"""Export association rules to file.
|
|
696
|
+
|
|
697
|
+
Args:
|
|
698
|
+
output_path: Path to save rules file.
|
|
699
|
+
format: Output format ("json", "csv", or "yaml").
|
|
700
|
+
|
|
701
|
+
Raises:
|
|
702
|
+
ValueError: If no rules have been discovered yet.
|
|
703
|
+
ImportError: If required library not available for format.
|
|
704
|
+
"""
|
|
705
|
+
if not self.rules:
|
|
706
|
+
raise ValueError("No rules to export. Run find_associations() first.")
|
|
707
|
+
|
|
708
|
+
if format == "json":
|
|
709
|
+
self._export_json(output_path)
|
|
710
|
+
elif format == "csv":
|
|
711
|
+
self._export_csv(output_path)
|
|
712
|
+
elif format == "yaml":
|
|
713
|
+
self._export_yaml(output_path)
|
|
714
|
+
|
|
715
|
+
def _export_json(self, output_path: Path) -> None:
|
|
716
|
+
"""Export rules as JSON.
|
|
717
|
+
|
|
718
|
+
Args:
|
|
719
|
+
output_path: Path to save JSON file.
|
|
720
|
+
"""
|
|
721
|
+
import json
|
|
722
|
+
|
|
723
|
+
rules_data = [
|
|
724
|
+
{
|
|
725
|
+
"antecedent": [int(b) for b in rule.antecedent],
|
|
726
|
+
"consequent": [int(b) for b in rule.consequent],
|
|
727
|
+
"support": rule.support,
|
|
728
|
+
"confidence": rule.confidence,
|
|
729
|
+
"lift": rule.lift,
|
|
730
|
+
}
|
|
731
|
+
for rule in self.rules
|
|
732
|
+
]
|
|
733
|
+
|
|
734
|
+
with output_path.open("w") as f:
|
|
735
|
+
json.dump(rules_data, f, indent=2)
|
|
736
|
+
|
|
737
|
+
def _export_csv(self, output_path: Path) -> None:
|
|
738
|
+
"""Export rules as CSV.
|
|
739
|
+
|
|
740
|
+
Args:
|
|
741
|
+
output_path: Path to save CSV file.
|
|
742
|
+
"""
|
|
743
|
+
with output_path.open("w") as f:
|
|
744
|
+
f.write("antecedent,consequent,support,confidence,lift\n")
|
|
745
|
+
for rule in self.rules:
|
|
746
|
+
ant_str = " ".join(f"{b:02X}" for b in rule.antecedent)
|
|
747
|
+
con_str = " ".join(f"{b:02X}" for b in rule.consequent)
|
|
748
|
+
f.write(f'"{ant_str}","{con_str}",{rule.support},{rule.confidence},{rule.lift}\n')
|
|
749
|
+
|
|
750
|
+
def _export_yaml(self, output_path: Path) -> None:
|
|
751
|
+
"""Export rules as YAML.
|
|
752
|
+
|
|
753
|
+
Args:
|
|
754
|
+
output_path: Path to save YAML file.
|
|
755
|
+
"""
|
|
756
|
+
import yaml
|
|
757
|
+
|
|
758
|
+
rules_data = [
|
|
759
|
+
{
|
|
760
|
+
"antecedent": [int(b) for b in rule.antecedent],
|
|
761
|
+
"consequent": [int(b) for b in rule.consequent],
|
|
762
|
+
"support": rule.support,
|
|
763
|
+
"confidence": rule.confidence,
|
|
764
|
+
"lift": rule.lift,
|
|
765
|
+
}
|
|
766
|
+
for rule in self.rules
|
|
767
|
+
]
|
|
768
|
+
|
|
769
|
+
with output_path.open("w") as f:
|
|
770
|
+
yaml.dump(rules_data, f, default_flow_style=False)
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
__all__ = [
|
|
774
|
+
"AssociationRule",
|
|
775
|
+
"Pattern",
|
|
776
|
+
"PatternMiner",
|
|
777
|
+
"TemporalPattern",
|
|
778
|
+
]
|