oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oscura/__init__.py +169 -167
- oscura/analyzers/__init__.py +3 -0
- oscura/analyzers/classification.py +659 -0
- oscura/analyzers/digital/edges.py +325 -65
- oscura/analyzers/digital/quality.py +293 -166
- oscura/analyzers/digital/timing.py +260 -115
- oscura/analyzers/digital/timing_numba.py +334 -0
- oscura/analyzers/entropy.py +605 -0
- oscura/analyzers/eye/diagram.py +176 -109
- oscura/analyzers/eye/metrics.py +5 -5
- oscura/analyzers/jitter/__init__.py +6 -4
- oscura/analyzers/jitter/ber.py +52 -52
- oscura/analyzers/jitter/classification.py +156 -0
- oscura/analyzers/jitter/decomposition.py +163 -113
- oscura/analyzers/jitter/spectrum.py +80 -64
- oscura/analyzers/ml/__init__.py +39 -0
- oscura/analyzers/ml/features.py +600 -0
- oscura/analyzers/ml/signal_classifier.py +604 -0
- oscura/analyzers/packet/daq.py +246 -158
- oscura/analyzers/packet/parser.py +12 -1
- oscura/analyzers/packet/payload.py +50 -2110
- oscura/analyzers/packet/payload_analysis.py +361 -181
- oscura/analyzers/packet/payload_patterns.py +133 -70
- oscura/analyzers/packet/stream.py +84 -23
- oscura/analyzers/patterns/__init__.py +26 -5
- oscura/analyzers/patterns/anomaly_detection.py +908 -0
- oscura/analyzers/patterns/clustering.py +169 -108
- oscura/analyzers/patterns/clustering_optimized.py +227 -0
- oscura/analyzers/patterns/discovery.py +1 -1
- oscura/analyzers/patterns/matching.py +581 -197
- oscura/analyzers/patterns/pattern_mining.py +778 -0
- oscura/analyzers/patterns/periodic.py +121 -38
- oscura/analyzers/patterns/sequences.py +175 -78
- oscura/analyzers/power/conduction.py +1 -1
- oscura/analyzers/power/soa.py +6 -6
- oscura/analyzers/power/switching.py +250 -110
- oscura/analyzers/protocol/__init__.py +17 -1
- oscura/analyzers/protocols/base.py +6 -6
- oscura/analyzers/protocols/ble/__init__.py +38 -0
- oscura/analyzers/protocols/ble/analyzer.py +809 -0
- oscura/analyzers/protocols/ble/uuids.py +288 -0
- oscura/analyzers/protocols/can.py +257 -127
- oscura/analyzers/protocols/can_fd.py +107 -80
- oscura/analyzers/protocols/flexray.py +139 -80
- oscura/analyzers/protocols/hdlc.py +93 -58
- oscura/analyzers/protocols/i2c.py +247 -106
- oscura/analyzers/protocols/i2s.py +138 -86
- oscura/analyzers/protocols/industrial/__init__.py +40 -0
- oscura/analyzers/protocols/industrial/bacnet/__init__.py +33 -0
- oscura/analyzers/protocols/industrial/bacnet/analyzer.py +708 -0
- oscura/analyzers/protocols/industrial/bacnet/encoding.py +412 -0
- oscura/analyzers/protocols/industrial/bacnet/services.py +622 -0
- oscura/analyzers/protocols/industrial/ethercat/__init__.py +30 -0
- oscura/analyzers/protocols/industrial/ethercat/analyzer.py +474 -0
- oscura/analyzers/protocols/industrial/ethercat/mailbox.py +339 -0
- oscura/analyzers/protocols/industrial/ethercat/topology.py +166 -0
- oscura/analyzers/protocols/industrial/modbus/__init__.py +31 -0
- oscura/analyzers/protocols/industrial/modbus/analyzer.py +525 -0
- oscura/analyzers/protocols/industrial/modbus/crc.py +79 -0
- oscura/analyzers/protocols/industrial/modbus/functions.py +436 -0
- oscura/analyzers/protocols/industrial/opcua/__init__.py +21 -0
- oscura/analyzers/protocols/industrial/opcua/analyzer.py +552 -0
- oscura/analyzers/protocols/industrial/opcua/datatypes.py +446 -0
- oscura/analyzers/protocols/industrial/opcua/services.py +264 -0
- oscura/analyzers/protocols/industrial/profinet/__init__.py +23 -0
- oscura/analyzers/protocols/industrial/profinet/analyzer.py +441 -0
- oscura/analyzers/protocols/industrial/profinet/dcp.py +263 -0
- oscura/analyzers/protocols/industrial/profinet/ptcp.py +200 -0
- oscura/analyzers/protocols/jtag.py +180 -98
- oscura/analyzers/protocols/lin.py +219 -114
- oscura/analyzers/protocols/manchester.py +4 -4
- oscura/analyzers/protocols/onewire.py +253 -149
- oscura/analyzers/protocols/parallel_bus/__init__.py +20 -0
- oscura/analyzers/protocols/parallel_bus/centronics.py +92 -0
- oscura/analyzers/protocols/parallel_bus/gpib.py +137 -0
- oscura/analyzers/protocols/spi.py +192 -95
- oscura/analyzers/protocols/swd.py +321 -167
- oscura/analyzers/protocols/uart.py +267 -125
- oscura/analyzers/protocols/usb.py +235 -131
- oscura/analyzers/side_channel/power.py +17 -12
- oscura/analyzers/signal/__init__.py +15 -0
- oscura/analyzers/signal/timing_analysis.py +1086 -0
- oscura/analyzers/signal_integrity/__init__.py +4 -1
- oscura/analyzers/signal_integrity/sparams.py +2 -19
- oscura/analyzers/spectral/chunked.py +129 -60
- oscura/analyzers/spectral/chunked_fft.py +300 -94
- oscura/analyzers/spectral/chunked_wavelet.py +100 -80
- oscura/analyzers/statistical/checksum.py +376 -217
- oscura/analyzers/statistical/classification.py +229 -107
- oscura/analyzers/statistical/entropy.py +78 -53
- oscura/analyzers/statistics/correlation.py +407 -211
- oscura/analyzers/statistics/outliers.py +2 -2
- oscura/analyzers/statistics/streaming.py +30 -5
- oscura/analyzers/validation.py +216 -101
- oscura/analyzers/waveform/measurements.py +9 -0
- oscura/analyzers/waveform/measurements_with_uncertainty.py +31 -15
- oscura/analyzers/waveform/spectral.py +500 -228
- oscura/api/__init__.py +31 -5
- oscura/api/dsl/__init__.py +582 -0
- oscura/{dsl → api/dsl}/commands.py +43 -76
- oscura/{dsl → api/dsl}/interpreter.py +26 -51
- oscura/{dsl → api/dsl}/parser.py +107 -77
- oscura/{dsl → api/dsl}/repl.py +2 -2
- oscura/api/dsl.py +1 -1
- oscura/{integrations → api/integrations}/__init__.py +1 -1
- oscura/{integrations → api/integrations}/llm.py +201 -102
- oscura/api/operators.py +3 -3
- oscura/api/optimization.py +144 -30
- oscura/api/rest_server.py +921 -0
- oscura/api/server/__init__.py +17 -0
- oscura/api/server/dashboard.py +850 -0
- oscura/api/server/static/README.md +34 -0
- oscura/api/server/templates/base.html +181 -0
- oscura/api/server/templates/export.html +120 -0
- oscura/api/server/templates/home.html +284 -0
- oscura/api/server/templates/protocols.html +58 -0
- oscura/api/server/templates/reports.html +43 -0
- oscura/api/server/templates/session_detail.html +89 -0
- oscura/api/server/templates/sessions.html +83 -0
- oscura/api/server/templates/waveforms.html +73 -0
- oscura/automotive/__init__.py +8 -1
- oscura/automotive/can/__init__.py +10 -0
- oscura/automotive/can/checksum.py +3 -1
- oscura/automotive/can/dbc_generator.py +590 -0
- oscura/automotive/can/message_wrapper.py +121 -74
- oscura/automotive/can/patterns.py +98 -21
- oscura/automotive/can/session.py +292 -56
- oscura/automotive/can/state_machine.py +6 -3
- oscura/automotive/can/stimulus_response.py +97 -75
- oscura/automotive/dbc/__init__.py +10 -2
- oscura/automotive/dbc/generator.py +84 -56
- oscura/automotive/dbc/parser.py +6 -6
- oscura/automotive/dtc/data.json +17 -102
- oscura/automotive/dtc/database.py +2 -2
- oscura/automotive/flexray/__init__.py +31 -0
- oscura/automotive/flexray/analyzer.py +504 -0
- oscura/automotive/flexray/crc.py +185 -0
- oscura/automotive/flexray/fibex.py +449 -0
- oscura/automotive/j1939/__init__.py +45 -8
- oscura/automotive/j1939/analyzer.py +605 -0
- oscura/automotive/j1939/spns.py +326 -0
- oscura/automotive/j1939/transport.py +306 -0
- oscura/automotive/lin/__init__.py +47 -0
- oscura/automotive/lin/analyzer.py +612 -0
- oscura/automotive/loaders/blf.py +13 -2
- oscura/automotive/loaders/csv_can.py +143 -72
- oscura/automotive/loaders/dispatcher.py +50 -2
- oscura/automotive/loaders/mdf.py +86 -45
- oscura/automotive/loaders/pcap.py +111 -61
- oscura/automotive/uds/__init__.py +4 -0
- oscura/automotive/uds/analyzer.py +725 -0
- oscura/automotive/uds/decoder.py +140 -58
- oscura/automotive/uds/models.py +7 -1
- oscura/automotive/visualization.py +1 -1
- oscura/cli/analyze.py +348 -0
- oscura/cli/batch.py +142 -122
- oscura/cli/benchmark.py +275 -0
- oscura/cli/characterize.py +137 -82
- oscura/cli/compare.py +224 -131
- oscura/cli/completion.py +250 -0
- oscura/cli/config_cmd.py +361 -0
- oscura/cli/decode.py +164 -87
- oscura/cli/export.py +286 -0
- oscura/cli/main.py +115 -31
- oscura/{onboarding → cli/onboarding}/__init__.py +3 -3
- oscura/{onboarding → cli/onboarding}/help.py +80 -58
- oscura/{onboarding → cli/onboarding}/tutorials.py +97 -72
- oscura/{onboarding → cli/onboarding}/wizard.py +55 -36
- oscura/cli/progress.py +147 -0
- oscura/cli/shell.py +157 -135
- oscura/cli/validate_cmd.py +204 -0
- oscura/cli/visualize.py +158 -0
- oscura/convenience.py +125 -79
- oscura/core/__init__.py +4 -2
- oscura/core/backend_selector.py +3 -3
- oscura/core/cache.py +126 -15
- oscura/core/cancellation.py +1 -1
- oscura/{config → core/config}/__init__.py +20 -11
- oscura/{config → core/config}/defaults.py +1 -1
- oscura/{config → core/config}/loader.py +7 -5
- oscura/{config → core/config}/memory.py +5 -5
- oscura/{config → core/config}/migration.py +1 -1
- oscura/{config → core/config}/pipeline.py +99 -23
- oscura/{config → core/config}/preferences.py +1 -1
- oscura/{config → core/config}/protocol.py +3 -3
- oscura/{config → core/config}/schema.py +426 -272
- oscura/{config → core/config}/settings.py +1 -1
- oscura/{config → core/config}/thresholds.py +195 -153
- oscura/core/correlation.py +5 -6
- oscura/core/cross_domain.py +0 -2
- oscura/core/debug.py +9 -5
- oscura/{extensibility → core/extensibility}/docs.py +158 -70
- oscura/{extensibility → core/extensibility}/extensions.py +160 -76
- oscura/{extensibility → core/extensibility}/logging.py +1 -1
- oscura/{extensibility → core/extensibility}/measurements.py +1 -1
- oscura/{extensibility → core/extensibility}/plugins.py +1 -1
- oscura/{extensibility → core/extensibility}/templates.py +73 -3
- oscura/{extensibility → core/extensibility}/validation.py +1 -1
- oscura/core/gpu_backend.py +11 -7
- oscura/core/log_query.py +101 -11
- oscura/core/logging.py +126 -54
- oscura/core/logging_advanced.py +5 -5
- oscura/core/memory_limits.py +108 -70
- oscura/core/memory_monitor.py +2 -2
- oscura/core/memory_progress.py +7 -7
- oscura/core/memory_warnings.py +1 -1
- oscura/core/numba_backend.py +13 -13
- oscura/{plugins → core/plugins}/__init__.py +9 -9
- oscura/{plugins → core/plugins}/base.py +7 -7
- oscura/{plugins → core/plugins}/cli.py +3 -3
- oscura/{plugins → core/plugins}/discovery.py +186 -106
- oscura/{plugins → core/plugins}/lifecycle.py +1 -1
- oscura/{plugins → core/plugins}/manager.py +7 -7
- oscura/{plugins → core/plugins}/registry.py +3 -3
- oscura/{plugins → core/plugins}/versioning.py +1 -1
- oscura/core/progress.py +16 -1
- oscura/core/provenance.py +8 -2
- oscura/{schemas → core/schemas}/__init__.py +2 -2
- oscura/{schemas → core/schemas}/device_mapping.json +2 -8
- oscura/{schemas → core/schemas}/packet_format.json +4 -24
- oscura/{schemas → core/schemas}/protocol_definition.json +2 -12
- oscura/core/types.py +4 -0
- oscura/core/uncertainty.py +3 -3
- oscura/correlation/__init__.py +52 -0
- oscura/correlation/multi_protocol.py +811 -0
- oscura/discovery/auto_decoder.py +117 -35
- oscura/discovery/comparison.py +191 -86
- oscura/discovery/quality_validator.py +155 -68
- oscura/discovery/signal_detector.py +196 -79
- oscura/export/__init__.py +18 -8
- oscura/export/kaitai_struct.py +513 -0
- oscura/export/scapy_layer.py +801 -0
- oscura/export/wireshark/generator.py +1 -1
- oscura/export/wireshark/templates/dissector.lua.j2 +2 -2
- oscura/export/wireshark_dissector.py +746 -0
- oscura/guidance/wizard.py +207 -111
- oscura/hardware/__init__.py +19 -0
- oscura/{acquisition → hardware/acquisition}/__init__.py +4 -4
- oscura/{acquisition → hardware/acquisition}/file.py +2 -2
- oscura/{acquisition → hardware/acquisition}/hardware.py +7 -7
- oscura/{acquisition → hardware/acquisition}/saleae.py +15 -12
- oscura/{acquisition → hardware/acquisition}/socketcan.py +1 -1
- oscura/{acquisition → hardware/acquisition}/streaming.py +2 -2
- oscura/{acquisition → hardware/acquisition}/synthetic.py +3 -3
- oscura/{acquisition → hardware/acquisition}/visa.py +33 -11
- oscura/hardware/firmware/__init__.py +29 -0
- oscura/hardware/firmware/pattern_recognition.py +874 -0
- oscura/hardware/hal_detector.py +736 -0
- oscura/hardware/security/__init__.py +37 -0
- oscura/hardware/security/side_channel_detector.py +1126 -0
- oscura/inference/__init__.py +4 -0
- oscura/inference/active_learning/observation_table.py +4 -1
- oscura/inference/alignment.py +216 -123
- oscura/inference/bayesian.py +113 -33
- oscura/inference/crc_reverse.py +101 -55
- oscura/inference/logic.py +6 -2
- oscura/inference/message_format.py +342 -183
- oscura/inference/protocol.py +95 -44
- oscura/inference/protocol_dsl.py +180 -82
- oscura/inference/signal_intelligence.py +1439 -706
- oscura/inference/spectral.py +99 -57
- oscura/inference/state_machine.py +810 -158
- oscura/inference/stream.py +270 -110
- oscura/iot/__init__.py +34 -0
- oscura/iot/coap/__init__.py +32 -0
- oscura/iot/coap/analyzer.py +668 -0
- oscura/iot/coap/options.py +212 -0
- oscura/iot/lorawan/__init__.py +21 -0
- oscura/iot/lorawan/crypto.py +206 -0
- oscura/iot/lorawan/decoder.py +801 -0
- oscura/iot/lorawan/mac_commands.py +341 -0
- oscura/iot/mqtt/__init__.py +27 -0
- oscura/iot/mqtt/analyzer.py +999 -0
- oscura/iot/mqtt/properties.py +315 -0
- oscura/iot/zigbee/__init__.py +31 -0
- oscura/iot/zigbee/analyzer.py +615 -0
- oscura/iot/zigbee/security.py +153 -0
- oscura/iot/zigbee/zcl.py +349 -0
- oscura/jupyter/display.py +125 -45
- oscura/{exploratory → jupyter/exploratory}/__init__.py +8 -8
- oscura/{exploratory → jupyter/exploratory}/error_recovery.py +298 -141
- oscura/jupyter/exploratory/fuzzy.py +746 -0
- oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py +258 -100
- oscura/{exploratory → jupyter/exploratory}/legacy.py +464 -242
- oscura/{exploratory → jupyter/exploratory}/parse.py +167 -145
- oscura/{exploratory → jupyter/exploratory}/recovery.py +119 -87
- oscura/jupyter/exploratory/sync.py +612 -0
- oscura/{exploratory → jupyter/exploratory}/unknown.py +299 -176
- oscura/jupyter/magic.py +4 -4
- oscura/{ui → jupyter/ui}/__init__.py +2 -2
- oscura/{ui → jupyter/ui}/formatters.py +3 -3
- oscura/{ui → jupyter/ui}/progressive_display.py +153 -82
- oscura/loaders/__init__.py +183 -67
- oscura/loaders/binary.py +88 -1
- oscura/loaders/chipwhisperer.py +153 -137
- oscura/loaders/configurable.py +208 -86
- oscura/loaders/csv_loader.py +458 -215
- oscura/loaders/hdf5_loader.py +278 -119
- oscura/loaders/lazy.py +87 -54
- oscura/loaders/mmap_loader.py +1 -1
- oscura/loaders/numpy_loader.py +253 -116
- oscura/loaders/pcap.py +226 -151
- oscura/loaders/rigol.py +110 -49
- oscura/loaders/sigrok.py +201 -78
- oscura/loaders/tdms.py +81 -58
- oscura/loaders/tektronix.py +291 -174
- oscura/loaders/touchstone.py +182 -87
- oscura/loaders/tss.py +456 -0
- oscura/loaders/vcd.py +215 -117
- oscura/loaders/wav.py +155 -68
- oscura/reporting/__init__.py +9 -0
- oscura/reporting/analyze.py +352 -146
- oscura/reporting/argument_preparer.py +69 -14
- oscura/reporting/auto_report.py +97 -61
- oscura/reporting/batch.py +131 -58
- oscura/reporting/chart_selection.py +57 -45
- oscura/reporting/comparison.py +63 -17
- oscura/reporting/content/executive.py +76 -24
- oscura/reporting/core_formats/multi_format.py +11 -8
- oscura/reporting/engine.py +312 -158
- oscura/reporting/enhanced_reports.py +949 -0
- oscura/reporting/export.py +86 -43
- oscura/reporting/formatting/numbers.py +69 -42
- oscura/reporting/html.py +139 -58
- oscura/reporting/index.py +137 -65
- oscura/reporting/output.py +158 -67
- oscura/reporting/pdf.py +67 -102
- oscura/reporting/plots.py +191 -112
- oscura/reporting/sections.py +88 -47
- oscura/reporting/standards.py +104 -61
- oscura/reporting/summary_generator.py +75 -55
- oscura/reporting/tables.py +138 -54
- oscura/reporting/templates/enhanced/protocol_re.html +525 -0
- oscura/sessions/__init__.py +14 -23
- oscura/sessions/base.py +3 -3
- oscura/sessions/blackbox.py +106 -10
- oscura/sessions/generic.py +2 -2
- oscura/sessions/legacy.py +783 -0
- oscura/side_channel/__init__.py +63 -0
- oscura/side_channel/dpa.py +1025 -0
- oscura/utils/__init__.py +15 -1
- oscura/utils/bitwise.py +118 -0
- oscura/{builders → utils/builders}/__init__.py +1 -1
- oscura/{comparison → utils/comparison}/__init__.py +6 -6
- oscura/{comparison → utils/comparison}/compare.py +202 -101
- oscura/{comparison → utils/comparison}/golden.py +83 -63
- oscura/{comparison → utils/comparison}/limits.py +313 -89
- oscura/{comparison → utils/comparison}/mask.py +151 -45
- oscura/{comparison → utils/comparison}/trace_diff.py +1 -1
- oscura/{comparison → utils/comparison}/visualization.py +147 -89
- oscura/{component → utils/component}/__init__.py +3 -3
- oscura/{component → utils/component}/impedance.py +122 -58
- oscura/{component → utils/component}/reactive.py +165 -168
- oscura/{component → utils/component}/transmission_line.py +3 -3
- oscura/{filtering → utils/filtering}/__init__.py +6 -6
- oscura/{filtering → utils/filtering}/base.py +1 -1
- oscura/{filtering → utils/filtering}/convenience.py +2 -2
- oscura/{filtering → utils/filtering}/design.py +169 -93
- oscura/{filtering → utils/filtering}/filters.py +2 -2
- oscura/{filtering → utils/filtering}/introspection.py +2 -2
- oscura/utils/geometry.py +31 -0
- oscura/utils/imports.py +184 -0
- oscura/utils/lazy.py +1 -1
- oscura/{math → utils/math}/__init__.py +2 -2
- oscura/{math → utils/math}/arithmetic.py +114 -48
- oscura/{math → utils/math}/interpolation.py +139 -106
- oscura/utils/memory.py +129 -66
- oscura/utils/memory_advanced.py +92 -9
- oscura/utils/memory_extensions.py +10 -8
- oscura/{optimization → utils/optimization}/__init__.py +1 -1
- oscura/{optimization → utils/optimization}/search.py +2 -2
- oscura/utils/performance/__init__.py +58 -0
- oscura/utils/performance/caching.py +889 -0
- oscura/utils/performance/lsh_clustering.py +333 -0
- oscura/utils/performance/memory_optimizer.py +699 -0
- oscura/utils/performance/optimizations.py +675 -0
- oscura/utils/performance/parallel.py +654 -0
- oscura/utils/performance/profiling.py +661 -0
- oscura/{pipeline → utils/pipeline}/base.py +1 -1
- oscura/{pipeline → utils/pipeline}/composition.py +1 -1
- oscura/{pipeline → utils/pipeline}/parallel.py +3 -2
- oscura/{pipeline → utils/pipeline}/pipeline.py +1 -1
- oscura/{pipeline → utils/pipeline}/reverse_engineering.py +412 -221
- oscura/{search → utils/search}/__init__.py +3 -3
- oscura/{search → utils/search}/anomaly.py +188 -58
- oscura/utils/search/context.py +294 -0
- oscura/{search → utils/search}/pattern.py +138 -10
- oscura/utils/serial.py +51 -0
- oscura/utils/storage/__init__.py +61 -0
- oscura/utils/storage/database.py +1166 -0
- oscura/{streaming → utils/streaming}/chunked.py +302 -143
- oscura/{streaming → utils/streaming}/progressive.py +1 -1
- oscura/{streaming → utils/streaming}/realtime.py +3 -2
- oscura/{triggering → utils/triggering}/__init__.py +6 -6
- oscura/{triggering → utils/triggering}/base.py +6 -6
- oscura/{triggering → utils/triggering}/edge.py +2 -2
- oscura/{triggering → utils/triggering}/pattern.py +2 -2
- oscura/{triggering → utils/triggering}/pulse.py +115 -74
- oscura/{triggering → utils/triggering}/window.py +2 -2
- oscura/utils/validation.py +32 -0
- oscura/validation/__init__.py +121 -0
- oscura/{compliance → validation/compliance}/__init__.py +5 -5
- oscura/{compliance → validation/compliance}/advanced.py +5 -5
- oscura/{compliance → validation/compliance}/masks.py +1 -1
- oscura/{compliance → validation/compliance}/reporting.py +127 -53
- oscura/{compliance → validation/compliance}/testing.py +114 -52
- oscura/validation/compliance_tests.py +915 -0
- oscura/validation/fuzzer.py +990 -0
- oscura/validation/grammar_tests.py +596 -0
- oscura/validation/grammar_validator.py +904 -0
- oscura/validation/hil_testing.py +977 -0
- oscura/{quality → validation/quality}/__init__.py +4 -4
- oscura/{quality → validation/quality}/ensemble.py +251 -171
- oscura/{quality → validation/quality}/explainer.py +3 -3
- oscura/{quality → validation/quality}/scoring.py +1 -1
- oscura/{quality → validation/quality}/warnings.py +4 -4
- oscura/validation/regression_suite.py +808 -0
- oscura/validation/replay.py +788 -0
- oscura/{testing → validation/testing}/__init__.py +2 -2
- oscura/{testing → validation/testing}/synthetic.py +5 -5
- oscura/visualization/__init__.py +9 -0
- oscura/visualization/accessibility.py +1 -1
- oscura/visualization/annotations.py +64 -67
- oscura/visualization/colors.py +7 -7
- oscura/visualization/digital.py +180 -81
- oscura/visualization/eye.py +236 -85
- oscura/visualization/interactive.py +320 -143
- oscura/visualization/jitter.py +587 -247
- oscura/visualization/layout.py +169 -134
- oscura/visualization/optimization.py +103 -52
- oscura/visualization/palettes.py +1 -1
- oscura/visualization/power.py +427 -211
- oscura/visualization/power_extended.py +626 -297
- oscura/visualization/presets.py +2 -0
- oscura/visualization/protocols.py +495 -181
- oscura/visualization/render.py +79 -63
- oscura/visualization/reverse_engineering.py +171 -124
- oscura/visualization/signal_integrity.py +460 -279
- oscura/visualization/specialized.py +190 -100
- oscura/visualization/spectral.py +670 -255
- oscura/visualization/thumbnails.py +166 -137
- oscura/visualization/waveform.py +150 -63
- oscura/workflows/__init__.py +3 -0
- oscura/{batch → workflows/batch}/__init__.py +5 -5
- oscura/{batch → workflows/batch}/advanced.py +150 -75
- oscura/workflows/batch/aggregate.py +531 -0
- oscura/workflows/batch/analyze.py +236 -0
- oscura/{batch → workflows/batch}/logging.py +2 -2
- oscura/{batch → workflows/batch}/metrics.py +1 -1
- oscura/workflows/complete_re.py +1144 -0
- oscura/workflows/compliance.py +44 -54
- oscura/workflows/digital.py +197 -51
- oscura/workflows/legacy/__init__.py +12 -0
- oscura/{workflow → workflows/legacy}/dag.py +4 -1
- oscura/workflows/multi_trace.py +9 -9
- oscura/workflows/power.py +42 -62
- oscura/workflows/protocol.py +82 -49
- oscura/workflows/reverse_engineering.py +351 -150
- oscura/workflows/signal_integrity.py +157 -82
- oscura-0.7.0.dist-info/METADATA +661 -0
- oscura-0.7.0.dist-info/RECORD +591 -0
- oscura/batch/aggregate.py +0 -300
- oscura/batch/analyze.py +0 -139
- oscura/dsl/__init__.py +0 -73
- oscura/exceptions.py +0 -59
- oscura/exploratory/fuzzy.py +0 -513
- oscura/exploratory/sync.py +0 -384
- oscura/exporters/__init__.py +0 -94
- oscura/exporters/csv.py +0 -303
- oscura/exporters/exporters.py +0 -44
- oscura/exporters/hdf5.py +0 -217
- oscura/exporters/html_export.py +0 -701
- oscura/exporters/json_export.py +0 -291
- oscura/exporters/markdown_export.py +0 -367
- oscura/exporters/matlab_export.py +0 -354
- oscura/exporters/npz_export.py +0 -219
- oscura/exporters/spice_export.py +0 -210
- oscura/search/context.py +0 -149
- oscura/session/__init__.py +0 -34
- oscura/session/annotations.py +0 -289
- oscura/session/history.py +0 -313
- oscura/session/session.py +0 -520
- oscura/workflow/__init__.py +0 -13
- oscura-0.5.1.dist-info/METADATA +0 -583
- oscura-0.5.1.dist-info/RECORD +0 -481
- /oscura/core/{config.py → config/legacy.py} +0 -0
- /oscura/{extensibility → core/extensibility}/__init__.py +0 -0
- /oscura/{extensibility → core/extensibility}/registry.py +0 -0
- /oscura/{plugins → core/plugins}/isolation.py +0 -0
- /oscura/{schemas → core/schemas}/bus_configuration.json +0 -0
- /oscura/{builders → utils/builders}/signal_builder.py +0 -0
- /oscura/{optimization → utils/optimization}/parallel.py +0 -0
- /oscura/{pipeline → utils/pipeline}/__init__.py +0 -0
- /oscura/{streaming → utils/streaming}/__init__.py +0 -0
- {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/WHEEL +0 -0
- {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/entry_points.txt +0 -0
- {oscura-0.5.1.dist-info → oscura-0.7.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,811 @@
|
|
|
1
|
+
"""Multi-protocol session correlation framework.
|
|
2
|
+
|
|
3
|
+
This module provides comprehensive tools for correlating sessions across
|
|
4
|
+
multiple protocols (e.g., CAN + Ethernet + Serial) to discover cross-protocol
|
|
5
|
+
communication patterns, dependencies, and session flows.
|
|
6
|
+
|
|
7
|
+
The framework supports:
|
|
8
|
+
- Message correlation across different protocols
|
|
9
|
+
- Request-response pair detection
|
|
10
|
+
- Dependency graph generation
|
|
11
|
+
- Session flow extraction
|
|
12
|
+
- Cross-protocol visualization
|
|
13
|
+
- Correlation analysis export
|
|
14
|
+
|
|
15
|
+
Example:
|
|
16
|
+
>>> from oscura.correlation import MultiProtocolCorrelator, ProtocolMessage
|
|
17
|
+
>>>
|
|
18
|
+
>>> # Create correlator
|
|
19
|
+
>>> correlator = MultiProtocolCorrelator(
|
|
20
|
+
... time_window=0.1, # Max 100ms between correlated messages
|
|
21
|
+
... min_confidence=0.5
|
|
22
|
+
... )
|
|
23
|
+
>>>
|
|
24
|
+
>>> # Add CAN message
|
|
25
|
+
>>> can_msg = ProtocolMessage(
|
|
26
|
+
... protocol="can",
|
|
27
|
+
... timestamp=1.234,
|
|
28
|
+
... message_id=0x123,
|
|
29
|
+
... payload=b"\\x01\\x02\\x03\\x04",
|
|
30
|
+
... source="ECU1",
|
|
31
|
+
... destination="ECU2"
|
|
32
|
+
... )
|
|
33
|
+
>>> correlator.add_message(can_msg)
|
|
34
|
+
>>>
|
|
35
|
+
>>> # Add Ethernet message shortly after (likely related)
|
|
36
|
+
>>> eth_msg = ProtocolMessage(
|
|
37
|
+
... protocol="ethernet",
|
|
38
|
+
... timestamp=1.238, # 4ms later
|
|
39
|
+
... payload=b"\\x01\\x02\\x03\\x04\\x05", # Similar payload
|
|
40
|
+
... source="192.168.1.10",
|
|
41
|
+
... destination="192.168.1.20"
|
|
42
|
+
... )
|
|
43
|
+
>>> correlator.add_message(eth_msg)
|
|
44
|
+
>>>
|
|
45
|
+
>>> # Find all correlations
|
|
46
|
+
>>> correlations = correlator.correlate_all()
|
|
47
|
+
>>> for corr in correlations:
|
|
48
|
+
... print(f"{corr.correlation_type}: {corr.confidence:.2f}")
|
|
49
|
+
... print(f" Evidence: {', '.join(corr.evidence)}")
|
|
50
|
+
>>>
|
|
51
|
+
>>> # Build dependency graph
|
|
52
|
+
>>> graph = correlator.build_dependency_graph()
|
|
53
|
+
>>> print(f"Nodes: {graph.number_of_nodes()}")
|
|
54
|
+
>>> print(f"Edges: {graph.number_of_edges()}")
|
|
55
|
+
>>>
|
|
56
|
+
>>> # Extract logical sessions
|
|
57
|
+
>>> sessions = correlator.extract_sessions()
|
|
58
|
+
>>> for session in sessions:
|
|
59
|
+
... print(f"Session: {len(session.messages)} messages")
|
|
60
|
+
... print(f"Protocols: {', '.join(session.protocols)}")
|
|
61
|
+
... print(f"Duration: {session.end_time - session.start_time:.3f}s")
|
|
62
|
+
|
|
63
|
+
Correlation Detection Methods:
|
|
64
|
+
The framework uses multiple methods to identify related messages:
|
|
65
|
+
|
|
66
|
+
1. Timestamp Proximity: Messages within time_window are candidates
|
|
67
|
+
2. Payload Similarity: Partial or full payload matches using Jaccard
|
|
68
|
+
3. ID Matching: Matching message IDs across protocols
|
|
69
|
+
4. Source/Destination: Matching addresses or identifiers
|
|
70
|
+
|
|
71
|
+
Correlation types:
|
|
72
|
+
- broadcast: Near-simultaneous messages across protocols
|
|
73
|
+
- request_response: Request followed by response
|
|
74
|
+
- related_payload: Correlated by payload content
|
|
75
|
+
|
|
76
|
+
References:
|
|
77
|
+
Network protocol analysis
|
|
78
|
+
Session correlation algorithms
|
|
79
|
+
Graph theory for dependency analysis
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
from __future__ import annotations
|
|
83
|
+
|
|
84
|
+
from dataclasses import dataclass, field
|
|
85
|
+
from pathlib import Path
|
|
86
|
+
from typing import Any
|
|
87
|
+
|
|
88
|
+
# Optional dependencies - import at module level for validation
|
|
89
|
+
try:
|
|
90
|
+
import matplotlib
|
|
91
|
+
except ImportError:
|
|
92
|
+
matplotlib = None # type: ignore[assignment]
|
|
93
|
+
|
|
94
|
+
try:
|
|
95
|
+
import networkx
|
|
96
|
+
except ImportError:
|
|
97
|
+
networkx = None # type: ignore[assignment]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class ProtocolMessage:
|
|
102
|
+
"""Generic protocol message for multi-protocol correlation.
|
|
103
|
+
|
|
104
|
+
Represents a message from any protocol with common fields for correlation.
|
|
105
|
+
|
|
106
|
+
Attributes:
|
|
107
|
+
protocol: Protocol name (e.g., "can", "ethernet", "uart", "spi").
|
|
108
|
+
timestamp: Message timestamp in seconds (float for high precision).
|
|
109
|
+
message_id: Message identifier (int for CAN, str for others, None if N/A).
|
|
110
|
+
payload: Message payload as bytes.
|
|
111
|
+
source: Source address/identifier (optional).
|
|
112
|
+
destination: Destination address/identifier (optional).
|
|
113
|
+
metadata: Additional protocol-specific metadata.
|
|
114
|
+
|
|
115
|
+
Example:
|
|
116
|
+
>>> msg = ProtocolMessage(
|
|
117
|
+
... protocol="can",
|
|
118
|
+
... timestamp=1.234567,
|
|
119
|
+
... message_id=0x123,
|
|
120
|
+
... payload=b"\\x01\\x02\\x03\\x04",
|
|
121
|
+
... source="ECU1",
|
|
122
|
+
... metadata={"dlc": 4, "extended": False}
|
|
123
|
+
... )
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
protocol: str
|
|
127
|
+
timestamp: float
|
|
128
|
+
message_id: str | int | None = None
|
|
129
|
+
payload: bytes = b""
|
|
130
|
+
source: str | None = None
|
|
131
|
+
destination: str | None = None
|
|
132
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass
|
|
136
|
+
class MessageCorrelation:
|
|
137
|
+
"""Correlation between two messages from different protocols.
|
|
138
|
+
|
|
139
|
+
Attributes:
|
|
140
|
+
message1: First message (chronologically earlier).
|
|
141
|
+
message2: Second message (chronologically later).
|
|
142
|
+
correlation_type: Type of correlation detected.
|
|
143
|
+
confidence: Correlation confidence score (0.0-1.0).
|
|
144
|
+
time_delta: Time difference in seconds (message2 - message1).
|
|
145
|
+
evidence: List of evidence strings explaining the correlation.
|
|
146
|
+
|
|
147
|
+
Correlation types:
|
|
148
|
+
- "broadcast": Near-simultaneous messages (<10ms) across protocols
|
|
149
|
+
- "request_response": Request followed by response
|
|
150
|
+
- "related_payload": Correlated by payload similarity
|
|
151
|
+
|
|
152
|
+
Example:
|
|
153
|
+
>>> corr = MessageCorrelation(
|
|
154
|
+
... message1=can_msg,
|
|
155
|
+
... message2=eth_msg,
|
|
156
|
+
... correlation_type="request_response",
|
|
157
|
+
... confidence=0.85,
|
|
158
|
+
... time_delta=0.004,
|
|
159
|
+
... evidence=["Payload similarity: 0.80", "Source-destination match"]
|
|
160
|
+
... )
|
|
161
|
+
"""
|
|
162
|
+
|
|
163
|
+
message1: ProtocolMessage
|
|
164
|
+
message2: ProtocolMessage
|
|
165
|
+
correlation_type: str
|
|
166
|
+
confidence: float
|
|
167
|
+
time_delta: float
|
|
168
|
+
evidence: list[str] = field(default_factory=list)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@dataclass
|
|
172
|
+
class SessionFlow:
|
|
173
|
+
"""Cross-protocol session flow.
|
|
174
|
+
|
|
175
|
+
Represents a logical session composed of correlated messages from
|
|
176
|
+
multiple protocols.
|
|
177
|
+
|
|
178
|
+
Attributes:
|
|
179
|
+
start_time: Session start time (first message timestamp).
|
|
180
|
+
end_time: Session end time (last message timestamp).
|
|
181
|
+
messages: All messages in the session (chronologically sorted).
|
|
182
|
+
correlations: All correlations within the session.
|
|
183
|
+
protocols: Set of protocols used in the session.
|
|
184
|
+
|
|
185
|
+
Example:
|
|
186
|
+
>>> session = SessionFlow(
|
|
187
|
+
... start_time=1.234,
|
|
188
|
+
... end_time=1.456,
|
|
189
|
+
... messages=[can_msg1, eth_msg1, can_msg2],
|
|
190
|
+
... correlations=[corr1, corr2],
|
|
191
|
+
... protocols={"can", "ethernet"}
|
|
192
|
+
... )
|
|
193
|
+
>>> duration = session.end_time - session.start_time
|
|
194
|
+
>>> print(f"Session duration: {duration:.3f}s")
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
start_time: float
|
|
198
|
+
end_time: float
|
|
199
|
+
messages: list[ProtocolMessage]
|
|
200
|
+
correlations: list[MessageCorrelation]
|
|
201
|
+
protocols: set[str]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
class MultiProtocolCorrelator:
|
|
205
|
+
"""Multi-protocol session correlator.
|
|
206
|
+
|
|
207
|
+
Analyzes messages from multiple protocols to discover correlations,
|
|
208
|
+
dependencies, and logical session flows.
|
|
209
|
+
|
|
210
|
+
Attributes:
|
|
211
|
+
time_window: Maximum time between correlated messages (seconds).
|
|
212
|
+
min_confidence: Minimum confidence threshold for correlations.
|
|
213
|
+
messages: List of all added messages.
|
|
214
|
+
correlations: List of discovered correlations.
|
|
215
|
+
|
|
216
|
+
Example:
|
|
217
|
+
>>> correlator = MultiProtocolCorrelator(
|
|
218
|
+
... time_window=0.1, # 100ms max time difference
|
|
219
|
+
... min_confidence=0.5 # 50% minimum confidence
|
|
220
|
+
... )
|
|
221
|
+
>>>
|
|
222
|
+
>>> # Add messages from different protocols
|
|
223
|
+
>>> correlator.add_message(can_msg)
|
|
224
|
+
>>> correlator.add_message(ethernet_msg)
|
|
225
|
+
>>> correlator.add_message(uart_msg)
|
|
226
|
+
>>>
|
|
227
|
+
>>> # Find correlations
|
|
228
|
+
>>> correlations = correlator.correlate_all()
|
|
229
|
+
>>> print(f"Found {len(correlations)} correlations")
|
|
230
|
+
>>>
|
|
231
|
+
>>> # Extract sessions
|
|
232
|
+
>>> sessions = correlator.extract_sessions()
|
|
233
|
+
>>> for session in sessions:
|
|
234
|
+
... print(f"Session: {len(session.messages)} messages, "
|
|
235
|
+
... f"{len(session.protocols)} protocols")
|
|
236
|
+
"""
|
|
237
|
+
|
|
238
|
+
def __init__(
|
|
239
|
+
self,
|
|
240
|
+
time_window: float = 0.1,
|
|
241
|
+
min_confidence: float = 0.5,
|
|
242
|
+
) -> None:
|
|
243
|
+
"""Initialize multi-protocol correlator.
|
|
244
|
+
|
|
245
|
+
Args:
|
|
246
|
+
time_window: Maximum time between correlated messages (seconds).
|
|
247
|
+
Default 0.1 (100ms). Increase for slower protocols.
|
|
248
|
+
min_confidence: Minimum confidence threshold for correlations (0.0-1.0).
|
|
249
|
+
Default 0.5. Higher values reduce false positives.
|
|
250
|
+
|
|
251
|
+
Raises:
|
|
252
|
+
ValueError: If time_window <= 0 or min_confidence not in [0, 1].
|
|
253
|
+
"""
|
|
254
|
+
if time_window <= 0:
|
|
255
|
+
raise ValueError(f"time_window must be positive, got {time_window}")
|
|
256
|
+
if not 0.0 <= min_confidence <= 1.0:
|
|
257
|
+
raise ValueError(f"min_confidence must be in [0, 1], got {min_confidence}")
|
|
258
|
+
|
|
259
|
+
self.time_window = time_window
|
|
260
|
+
self.min_confidence = min_confidence
|
|
261
|
+
self.messages: list[ProtocolMessage] = []
|
|
262
|
+
self.correlations: list[MessageCorrelation] = []
|
|
263
|
+
|
|
264
|
+
def add_message(self, msg: ProtocolMessage) -> None:
|
|
265
|
+
"""Add message from any protocol.
|
|
266
|
+
|
|
267
|
+
Messages are stored internally and used for correlation analysis.
|
|
268
|
+
Messages do not need to be added in chronological order.
|
|
269
|
+
|
|
270
|
+
Args:
|
|
271
|
+
msg: Protocol message to add.
|
|
272
|
+
|
|
273
|
+
Example:
|
|
274
|
+
>>> correlator.add_message(ProtocolMessage(
|
|
275
|
+
... protocol="can",
|
|
276
|
+
... timestamp=1.234,
|
|
277
|
+
... message_id=0x123,
|
|
278
|
+
... payload=b"\\x01\\x02"
|
|
279
|
+
... ))
|
|
280
|
+
"""
|
|
281
|
+
self.messages.append(msg)
|
|
282
|
+
|
|
283
|
+
def correlate_all(self) -> list[MessageCorrelation]:
|
|
284
|
+
"""Find all correlations across protocols.
|
|
285
|
+
|
|
286
|
+
Uses multiple correlation methods:
|
|
287
|
+
1. Timestamp proximity (within time_window)
|
|
288
|
+
2. Payload similarity (partial or full match)
|
|
289
|
+
3. ID matching (if available)
|
|
290
|
+
4. Source/destination matching
|
|
291
|
+
|
|
292
|
+
Returns:
|
|
293
|
+
List of discovered correlations sorted by confidence (descending).
|
|
294
|
+
Only correlations with confidence >= min_confidence are returned.
|
|
295
|
+
|
|
296
|
+
Example:
|
|
297
|
+
>>> correlations = correlator.correlate_all()
|
|
298
|
+
>>> for corr in correlations:
|
|
299
|
+
... print(f"{corr.message1.protocol} -> {corr.message2.protocol}: "
|
|
300
|
+
... f"{corr.confidence:.2f}")
|
|
301
|
+
"""
|
|
302
|
+
# Sort messages by timestamp for efficient windowing
|
|
303
|
+
sorted_messages = sorted(self.messages, key=lambda m: m.timestamp)
|
|
304
|
+
|
|
305
|
+
correlations = []
|
|
306
|
+
|
|
307
|
+
for i, msg1 in enumerate(sorted_messages):
|
|
308
|
+
# Look for correlations within time window
|
|
309
|
+
for j in range(i + 1, len(sorted_messages)):
|
|
310
|
+
msg2 = sorted_messages[j]
|
|
311
|
+
|
|
312
|
+
# Check time window
|
|
313
|
+
time_delta = msg2.timestamp - msg1.timestamp
|
|
314
|
+
if time_delta > self.time_window:
|
|
315
|
+
break # No point checking further (sorted by time)
|
|
316
|
+
|
|
317
|
+
# Calculate correlation confidence
|
|
318
|
+
confidence, evidence = self._calculate_correlation_confidence(msg1, msg2)
|
|
319
|
+
|
|
320
|
+
if confidence >= self.min_confidence:
|
|
321
|
+
# Determine correlation type
|
|
322
|
+
if time_delta < 0.01 and msg1.protocol != msg2.protocol:
|
|
323
|
+
corr_type = "broadcast"
|
|
324
|
+
elif self._is_request_response(msg1, msg2):
|
|
325
|
+
corr_type = "request_response"
|
|
326
|
+
else:
|
|
327
|
+
corr_type = "related_payload"
|
|
328
|
+
|
|
329
|
+
correlations.append(
|
|
330
|
+
MessageCorrelation(
|
|
331
|
+
message1=msg1,
|
|
332
|
+
message2=msg2,
|
|
333
|
+
correlation_type=corr_type,
|
|
334
|
+
confidence=confidence,
|
|
335
|
+
time_delta=time_delta,
|
|
336
|
+
evidence=evidence,
|
|
337
|
+
)
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
# Sort by confidence (descending)
|
|
341
|
+
correlations.sort(key=lambda c: c.confidence, reverse=True)
|
|
342
|
+
|
|
343
|
+
self.correlations = correlations
|
|
344
|
+
return correlations
|
|
345
|
+
|
|
346
|
+
def find_request_response_pairs(
|
|
347
|
+
self,
|
|
348
|
+
request_protocol: str,
|
|
349
|
+
response_protocol: str,
|
|
350
|
+
) -> list[MessageCorrelation]:
|
|
351
|
+
"""Find request-response pairs across specific protocols.
|
|
352
|
+
|
|
353
|
+
Filters correlations to only those matching the specified protocol pair
|
|
354
|
+
and classified as request-response type.
|
|
355
|
+
|
|
356
|
+
Args:
|
|
357
|
+
request_protocol: Protocol name for requests (e.g., "can").
|
|
358
|
+
response_protocol: Protocol name for responses (e.g., "ethernet").
|
|
359
|
+
|
|
360
|
+
Returns:
|
|
361
|
+
List of request-response correlations matching the protocol pair.
|
|
362
|
+
|
|
363
|
+
Example:
|
|
364
|
+
>>> # Find CAN requests that trigger Ethernet responses
|
|
365
|
+
>>> pairs = correlator.find_request_response_pairs("can", "ethernet")
|
|
366
|
+
>>> for pair in pairs:
|
|
367
|
+
... print(f"CAN {pair.message1.message_id} -> "
|
|
368
|
+
... f"Ethernet after {pair.time_delta*1000:.1f}ms")
|
|
369
|
+
"""
|
|
370
|
+
if not self.correlations:
|
|
371
|
+
self.correlate_all()
|
|
372
|
+
|
|
373
|
+
return [
|
|
374
|
+
corr
|
|
375
|
+
for corr in self.correlations
|
|
376
|
+
if corr.correlation_type == "request_response"
|
|
377
|
+
and corr.message1.protocol == request_protocol
|
|
378
|
+
and corr.message2.protocol == response_protocol
|
|
379
|
+
]
|
|
380
|
+
|
|
381
|
+
def build_dependency_graph(self) -> Any:
|
|
382
|
+
"""Build NetworkX graph showing message dependencies.
|
|
383
|
+
|
|
384
|
+
Creates a directed graph where:
|
|
385
|
+
- Nodes represent messages (with protocol, timestamp, ID attributes)
|
|
386
|
+
- Edges represent correlations (with confidence, type, time_delta)
|
|
387
|
+
|
|
388
|
+
Returns:
|
|
389
|
+
NetworkX DiGraph with message dependencies.
|
|
390
|
+
|
|
391
|
+
Raises:
|
|
392
|
+
ImportError: If networkx is not installed.
|
|
393
|
+
|
|
394
|
+
Example:
|
|
395
|
+
>>> graph = correlator.build_dependency_graph()
|
|
396
|
+
>>> print(f"Nodes: {graph.number_of_nodes()}")
|
|
397
|
+
>>> print(f"Edges: {graph.number_of_edges()}")
|
|
398
|
+
>>>
|
|
399
|
+
>>> # Find strongly connected components
|
|
400
|
+
>>> import networkx as nx
|
|
401
|
+
>>> components = list(nx.strongly_connected_components(graph))
|
|
402
|
+
"""
|
|
403
|
+
try:
|
|
404
|
+
import networkx as nx
|
|
405
|
+
except ImportError as e:
|
|
406
|
+
raise ImportError(
|
|
407
|
+
"networkx is required for graph analysis. "
|
|
408
|
+
"Install with: pip install 'oscura[analysis]'"
|
|
409
|
+
) from e
|
|
410
|
+
|
|
411
|
+
if not self.correlations:
|
|
412
|
+
self.correlate_all()
|
|
413
|
+
|
|
414
|
+
graph = nx.DiGraph()
|
|
415
|
+
|
|
416
|
+
# Add nodes (messages)
|
|
417
|
+
for i, msg in enumerate(self.messages):
|
|
418
|
+
graph.add_node(
|
|
419
|
+
i,
|
|
420
|
+
protocol=msg.protocol,
|
|
421
|
+
timestamp=msg.timestamp,
|
|
422
|
+
id=msg.message_id,
|
|
423
|
+
label=f"{msg.protocol}:{msg.message_id}",
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
# Add edges (correlations)
|
|
427
|
+
msg_to_idx = {id(msg): i for i, msg in enumerate(self.messages)}
|
|
428
|
+
|
|
429
|
+
for corr in self.correlations:
|
|
430
|
+
idx1 = msg_to_idx[id(corr.message1)]
|
|
431
|
+
idx2 = msg_to_idx[id(corr.message2)]
|
|
432
|
+
|
|
433
|
+
graph.add_edge(
|
|
434
|
+
idx1,
|
|
435
|
+
idx2,
|
|
436
|
+
weight=corr.confidence,
|
|
437
|
+
type=corr.correlation_type,
|
|
438
|
+
time_delta=corr.time_delta,
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
return graph
|
|
442
|
+
|
|
443
|
+
def extract_sessions(self) -> list[SessionFlow]:
|
|
444
|
+
"""Extract logical sessions from correlated messages.
|
|
445
|
+
|
|
446
|
+
A session is defined as a connected component in the dependency graph,
|
|
447
|
+
representing a group of messages that are directly or indirectly correlated.
|
|
448
|
+
|
|
449
|
+
Returns:
|
|
450
|
+
List of SessionFlow objects, each representing a logical session.
|
|
451
|
+
Sessions are sorted by start time.
|
|
452
|
+
|
|
453
|
+
Example:
|
|
454
|
+
>>> sessions = correlator.extract_sessions()
|
|
455
|
+
>>> for i, session in enumerate(sessions):
|
|
456
|
+
... print(f"Session {i+1}:")
|
|
457
|
+
... print(f" Messages: {len(session.messages)}")
|
|
458
|
+
... print(f" Protocols: {', '.join(session.protocols)}")
|
|
459
|
+
... print(f" Duration: {session.end_time - session.start_time:.3f}s")
|
|
460
|
+
"""
|
|
461
|
+
if networkx is None:
|
|
462
|
+
raise ImportError("networkx is required for dependency graph building")
|
|
463
|
+
try:
|
|
464
|
+
import networkx as nx
|
|
465
|
+
except ImportError as e:
|
|
466
|
+
raise ImportError(
|
|
467
|
+
"networkx is required for session extraction. "
|
|
468
|
+
"Install with: pip install 'oscura[analysis]'"
|
|
469
|
+
) from e
|
|
470
|
+
|
|
471
|
+
graph = self.build_dependency_graph()
|
|
472
|
+
|
|
473
|
+
# Find connected components (undirected version)
|
|
474
|
+
undirected = graph.to_undirected()
|
|
475
|
+
components = nx.connected_components(undirected)
|
|
476
|
+
|
|
477
|
+
sessions = []
|
|
478
|
+
for component in components:
|
|
479
|
+
# Get messages in this component
|
|
480
|
+
session_messages = [self.messages[i] for i in component]
|
|
481
|
+
session_messages.sort(key=lambda m: m.timestamp)
|
|
482
|
+
|
|
483
|
+
# Get correlations within this component
|
|
484
|
+
component_set = set(component)
|
|
485
|
+
msg_indices = {id(msg): i for i, msg in enumerate(self.messages)}
|
|
486
|
+
|
|
487
|
+
session_corrs = [
|
|
488
|
+
corr
|
|
489
|
+
for corr in self.correlations
|
|
490
|
+
if msg_indices[id(corr.message1)] in component_set
|
|
491
|
+
and msg_indices[id(corr.message2)] in component_set
|
|
492
|
+
]
|
|
493
|
+
|
|
494
|
+
# Extract protocols used
|
|
495
|
+
protocols = {msg.protocol for msg in session_messages}
|
|
496
|
+
|
|
497
|
+
sessions.append(
|
|
498
|
+
SessionFlow(
|
|
499
|
+
start_time=session_messages[0].timestamp,
|
|
500
|
+
end_time=session_messages[-1].timestamp,
|
|
501
|
+
messages=session_messages,
|
|
502
|
+
correlations=session_corrs,
|
|
503
|
+
protocols=protocols,
|
|
504
|
+
)
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
# Sort sessions by start time
|
|
508
|
+
sessions.sort(key=lambda s: s.start_time)
|
|
509
|
+
|
|
510
|
+
return sessions
|
|
511
|
+
|
|
512
|
+
def _calculate_correlation_confidence(
|
|
513
|
+
self,
|
|
514
|
+
msg1: ProtocolMessage,
|
|
515
|
+
msg2: ProtocolMessage,
|
|
516
|
+
) -> tuple[float, list[str]]:
|
|
517
|
+
"""Calculate correlation confidence and evidence.
|
|
518
|
+
|
|
519
|
+
Uses multiple signals to estimate correlation likelihood:
|
|
520
|
+
- Payload similarity (Jaccard coefficient)
|
|
521
|
+
- Message ID matching
|
|
522
|
+
- Source/destination address matching
|
|
523
|
+
|
|
524
|
+
Args:
|
|
525
|
+
msg1: First message.
|
|
526
|
+
msg2: Second message.
|
|
527
|
+
|
|
528
|
+
Returns:
|
|
529
|
+
Tuple of (confidence, evidence_list).
|
|
530
|
+
Confidence is clamped to [0.0, 1.0].
|
|
531
|
+
"""
|
|
532
|
+
confidence = 0.0
|
|
533
|
+
evidence = []
|
|
534
|
+
|
|
535
|
+
# Same protocol = no cross-protocol correlation
|
|
536
|
+
if msg1.protocol == msg2.protocol:
|
|
537
|
+
return 0.0, []
|
|
538
|
+
|
|
539
|
+
# Check payload similarity (weighted 40%)
|
|
540
|
+
payload_sim = self._payload_similarity(msg1.payload, msg2.payload)
|
|
541
|
+
if payload_sim > 0.5:
|
|
542
|
+
confidence += 0.4 * payload_sim
|
|
543
|
+
evidence.append(f"Payload similarity: {payload_sim:.2f}")
|
|
544
|
+
|
|
545
|
+
# Check ID matching (weighted 30%)
|
|
546
|
+
if msg1.message_id is not None and msg2.message_id is not None:
|
|
547
|
+
if msg1.message_id == msg2.message_id:
|
|
548
|
+
confidence += 0.3
|
|
549
|
+
evidence.append(f"Matching IDs: {msg1.message_id}")
|
|
550
|
+
|
|
551
|
+
# Check source/destination matching (weighted 15% each)
|
|
552
|
+
if msg1.source is not None and msg2.destination is not None:
|
|
553
|
+
if msg1.source == msg2.destination:
|
|
554
|
+
confidence += 0.15
|
|
555
|
+
evidence.append("Source-destination match")
|
|
556
|
+
|
|
557
|
+
if msg1.destination is not None and msg2.source is not None:
|
|
558
|
+
if msg1.destination == msg2.source:
|
|
559
|
+
confidence += 0.15
|
|
560
|
+
evidence.append("Destination-source match")
|
|
561
|
+
|
|
562
|
+
return min(confidence, 1.0), evidence
|
|
563
|
+
|
|
564
|
+
def _payload_similarity(self, payload1: bytes, payload2: bytes) -> float:
|
|
565
|
+
"""Calculate payload similarity using Jaccard coefficient.
|
|
566
|
+
|
|
567
|
+
Computes similarity based on:
|
|
568
|
+
1. Exact containment (one payload contains the other)
|
|
569
|
+
2. Byte set intersection (Jaccard similarity)
|
|
570
|
+
|
|
571
|
+
Args:
|
|
572
|
+
payload1: First payload.
|
|
573
|
+
payload2: Second payload.
|
|
574
|
+
|
|
575
|
+
Returns:
|
|
576
|
+
Similarity score (0.0-1.0).
|
|
577
|
+
"""
|
|
578
|
+
if not payload1 or not payload2:
|
|
579
|
+
return 0.0
|
|
580
|
+
|
|
581
|
+
# Check if one contains the other (exact match or prefix/suffix)
|
|
582
|
+
if payload1 in payload2 or payload2 in payload1:
|
|
583
|
+
return 1.0
|
|
584
|
+
|
|
585
|
+
# Calculate Jaccard similarity of bytes
|
|
586
|
+
set1 = set(payload1)
|
|
587
|
+
set2 = set(payload2)
|
|
588
|
+
|
|
589
|
+
intersection = len(set1 & set2)
|
|
590
|
+
union = len(set1 | set2)
|
|
591
|
+
|
|
592
|
+
return intersection / union if union > 0 else 0.0
|
|
593
|
+
|
|
594
|
+
def _is_request_response(
|
|
595
|
+
self,
|
|
596
|
+
msg1: ProtocolMessage,
|
|
597
|
+
msg2: ProtocolMessage,
|
|
598
|
+
) -> bool:
|
|
599
|
+
"""Determine if messages form a request-response pair.
|
|
600
|
+
|
|
601
|
+
Heuristics:
|
|
602
|
+
- Source/destination swap (request source = response destination)
|
|
603
|
+
- Time delta in typical RPC range (1-100ms)
|
|
604
|
+
- Payload contains request identifier
|
|
605
|
+
|
|
606
|
+
Args:
|
|
607
|
+
msg1: Potential request message.
|
|
608
|
+
msg2: Potential response message.
|
|
609
|
+
|
|
610
|
+
Returns:
|
|
611
|
+
True if messages likely form request-response pair.
|
|
612
|
+
"""
|
|
613
|
+
# Check source/destination swap
|
|
614
|
+
if (
|
|
615
|
+
msg1.source is not None
|
|
616
|
+
and msg2.destination is not None
|
|
617
|
+
and msg1.source == msg2.destination
|
|
618
|
+
):
|
|
619
|
+
# Also check time delta is reasonable for RPC
|
|
620
|
+
time_delta = msg2.timestamp - msg1.timestamp
|
|
621
|
+
if 0.001 <= time_delta <= 0.1: # 1-100ms typical RPC time
|
|
622
|
+
return True
|
|
623
|
+
|
|
624
|
+
return False
|
|
625
|
+
|
|
626
|
+
def visualize_flow(
|
|
627
|
+
self,
|
|
628
|
+
session: SessionFlow,
|
|
629
|
+
output_path: Path,
|
|
630
|
+
) -> None:
|
|
631
|
+
"""Visualize cross-protocol message flow.
|
|
632
|
+
|
|
633
|
+
Generates a timeline diagram showing message flows across protocols.
|
|
634
|
+
Uses matplotlib to create a multi-lane diagram with time on the X-axis
|
|
635
|
+
and different protocols on separate Y-lanes.
|
|
636
|
+
|
|
637
|
+
Args:
|
|
638
|
+
session: Session flow to visualize.
|
|
639
|
+
output_path: Path to save visualization (PNG/PDF/SVG).
|
|
640
|
+
|
|
641
|
+
Raises:
|
|
642
|
+
ImportError: If matplotlib is not installed.
|
|
643
|
+
|
|
644
|
+
Example:
|
|
645
|
+
>>> sessions = correlator.extract_sessions()
|
|
646
|
+
>>> correlator.visualize_flow(sessions[0], Path("session_flow.png"))
|
|
647
|
+
"""
|
|
648
|
+
try:
|
|
649
|
+
import matplotlib.pyplot as plt
|
|
650
|
+
except ImportError as e:
|
|
651
|
+
raise ImportError(
|
|
652
|
+
"matplotlib is required for visualization. Install with: pip install matplotlib"
|
|
653
|
+
) from e
|
|
654
|
+
|
|
655
|
+
fig, ax = plt.subplots(figsize=(12, 6))
|
|
656
|
+
|
|
657
|
+
# Assign Y positions to protocols
|
|
658
|
+
protocols_list = sorted(session.protocols)
|
|
659
|
+
protocol_to_y = {proto: i for i, proto in enumerate(protocols_list)}
|
|
660
|
+
|
|
661
|
+
# Plot messages
|
|
662
|
+
for msg in session.messages:
|
|
663
|
+
y = protocol_to_y[msg.protocol]
|
|
664
|
+
ax.scatter(msg.timestamp, y, s=100, alpha=0.6)
|
|
665
|
+
ax.text(
|
|
666
|
+
msg.timestamp,
|
|
667
|
+
y + 0.1,
|
|
668
|
+
str(msg.message_id) if msg.message_id else "",
|
|
669
|
+
fontsize=8,
|
|
670
|
+
ha="center",
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
# Plot correlations
|
|
674
|
+
msg_to_idx = {id(msg): i for i, msg in enumerate(session.messages)}
|
|
675
|
+
for corr in session.correlations:
|
|
676
|
+
idx1 = msg_to_idx.get(id(corr.message1))
|
|
677
|
+
idx2 = msg_to_idx.get(id(corr.message2))
|
|
678
|
+
if idx1 is not None and idx2 is not None:
|
|
679
|
+
msg1 = session.messages[idx1]
|
|
680
|
+
msg2 = session.messages[idx2]
|
|
681
|
+
y1 = protocol_to_y[msg1.protocol]
|
|
682
|
+
y2 = protocol_to_y[msg2.protocol]
|
|
683
|
+
|
|
684
|
+
# Draw arrow
|
|
685
|
+
ax.annotate(
|
|
686
|
+
"",
|
|
687
|
+
xy=(msg2.timestamp, y2),
|
|
688
|
+
xytext=(msg1.timestamp, y1),
|
|
689
|
+
arrowprops={
|
|
690
|
+
"arrowstyle": "->",
|
|
691
|
+
"alpha": 0.3,
|
|
692
|
+
"lw": 1,
|
|
693
|
+
},
|
|
694
|
+
)
|
|
695
|
+
|
|
696
|
+
# Configure axes
|
|
697
|
+
ax.set_yticks(range(len(protocols_list)))
|
|
698
|
+
ax.set_yticklabels(protocols_list)
|
|
699
|
+
ax.set_xlabel("Time (s)")
|
|
700
|
+
ax.set_ylabel("Protocol")
|
|
701
|
+
ax.set_title(f"Cross-Protocol Message Flow ({len(session.messages)} messages)")
|
|
702
|
+
ax.grid(True, alpha=0.3)
|
|
703
|
+
|
|
704
|
+
plt.tight_layout()
|
|
705
|
+
plt.savefig(output_path, dpi=150)
|
|
706
|
+
plt.close()
|
|
707
|
+
|
|
708
|
+
def export_analysis(
|
|
709
|
+
self,
|
|
710
|
+
output_path: Path,
|
|
711
|
+
format: str = "json",
|
|
712
|
+
) -> None:
|
|
713
|
+
"""Export correlation analysis to file.
|
|
714
|
+
|
|
715
|
+
Exports all messages, correlations, and sessions to specified format.
|
|
716
|
+
|
|
717
|
+
Args:
|
|
718
|
+
output_path: Path to save analysis.
|
|
719
|
+
format: Export format ("json" or "csv").
|
|
720
|
+
|
|
721
|
+
Raises:
|
|
722
|
+
ValueError: If format is not supported.
|
|
723
|
+
|
|
724
|
+
Example:
|
|
725
|
+
>>> correlator.export_analysis(Path("analysis.json"), format="json")
|
|
726
|
+
"""
|
|
727
|
+
if matplotlib is None:
|
|
728
|
+
raise ImportError("matplotlib is required for flow visualization")
|
|
729
|
+
if format == "json":
|
|
730
|
+
self._export_json(output_path)
|
|
731
|
+
elif format == "csv":
|
|
732
|
+
self._export_csv(output_path)
|
|
733
|
+
else:
|
|
734
|
+
raise ValueError(f"Unsupported format: {format}")
|
|
735
|
+
|
|
736
|
+
def _export_json(self, output_path: Path) -> None:
|
|
737
|
+
"""Export analysis to JSON format."""
|
|
738
|
+
import json
|
|
739
|
+
|
|
740
|
+
# Build export structure
|
|
741
|
+
data = {
|
|
742
|
+
"config": {
|
|
743
|
+
"time_window": self.time_window,
|
|
744
|
+
"min_confidence": self.min_confidence,
|
|
745
|
+
},
|
|
746
|
+
"messages": [
|
|
747
|
+
{
|
|
748
|
+
"protocol": msg.protocol,
|
|
749
|
+
"timestamp": msg.timestamp,
|
|
750
|
+
"message_id": (
|
|
751
|
+
msg.message_id if isinstance(msg.message_id, (int, str)) else None
|
|
752
|
+
),
|
|
753
|
+
"payload_hex": msg.payload.hex(),
|
|
754
|
+
"source": msg.source,
|
|
755
|
+
"destination": msg.destination,
|
|
756
|
+
"metadata": msg.metadata,
|
|
757
|
+
}
|
|
758
|
+
for msg in self.messages
|
|
759
|
+
],
|
|
760
|
+
"correlations": [
|
|
761
|
+
{
|
|
762
|
+
"message1_idx": self.messages.index(corr.message1),
|
|
763
|
+
"message2_idx": self.messages.index(corr.message2),
|
|
764
|
+
"correlation_type": corr.correlation_type,
|
|
765
|
+
"confidence": corr.confidence,
|
|
766
|
+
"time_delta": corr.time_delta,
|
|
767
|
+
"evidence": corr.evidence,
|
|
768
|
+
}
|
|
769
|
+
for corr in self.correlations
|
|
770
|
+
],
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
# Write JSON
|
|
774
|
+
output_path.write_text(json.dumps(data, indent=2))
|
|
775
|
+
|
|
776
|
+
def _export_csv(self, output_path: Path) -> None:
|
|
777
|
+
"""Export correlations to CSV format."""
|
|
778
|
+
import csv
|
|
779
|
+
|
|
780
|
+
with output_path.open("w", newline="") as f:
|
|
781
|
+
writer = csv.writer(f)
|
|
782
|
+
writer.writerow(
|
|
783
|
+
[
|
|
784
|
+
"Protocol1",
|
|
785
|
+
"Timestamp1",
|
|
786
|
+
"ID1",
|
|
787
|
+
"Protocol2",
|
|
788
|
+
"Timestamp2",
|
|
789
|
+
"ID2",
|
|
790
|
+
"Type",
|
|
791
|
+
"Confidence",
|
|
792
|
+
"TimeDelta",
|
|
793
|
+
"Evidence",
|
|
794
|
+
]
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
for corr in self.correlations:
|
|
798
|
+
writer.writerow(
|
|
799
|
+
[
|
|
800
|
+
corr.message1.protocol,
|
|
801
|
+
corr.message1.timestamp,
|
|
802
|
+
corr.message1.message_id,
|
|
803
|
+
corr.message2.protocol,
|
|
804
|
+
corr.message2.timestamp,
|
|
805
|
+
corr.message2.message_id,
|
|
806
|
+
corr.correlation_type,
|
|
807
|
+
corr.confidence,
|
|
808
|
+
corr.time_delta,
|
|
809
|
+
"; ".join(corr.evidence),
|
|
810
|
+
]
|
|
811
|
+
)
|