oscura 0.7.0__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oscura/__init__.py +19 -19
- oscura/analyzers/__init__.py +2 -0
- oscura/analyzers/digital/extraction.py +2 -3
- oscura/analyzers/digital/quality.py +1 -1
- oscura/analyzers/digital/timing.py +1 -1
- oscura/analyzers/eye/__init__.py +5 -1
- oscura/analyzers/eye/generation.py +501 -0
- oscura/analyzers/jitter/__init__.py +6 -6
- oscura/analyzers/jitter/timing.py +419 -0
- oscura/analyzers/patterns/__init__.py +94 -0
- oscura/analyzers/patterns/reverse_engineering.py +991 -0
- oscura/analyzers/power/__init__.py +35 -12
- oscura/analyzers/power/basic.py +3 -3
- oscura/analyzers/power/soa.py +1 -1
- oscura/analyzers/power/switching.py +3 -3
- oscura/analyzers/signal_classification.py +529 -0
- oscura/analyzers/signal_integrity/sparams.py +3 -3
- oscura/analyzers/statistics/__init__.py +4 -0
- oscura/analyzers/statistics/basic.py +152 -0
- oscura/analyzers/statistics/correlation.py +47 -6
- oscura/analyzers/validation.py +1 -1
- oscura/analyzers/waveform/__init__.py +2 -0
- oscura/analyzers/waveform/measurements.py +329 -163
- oscura/analyzers/waveform/measurements_with_uncertainty.py +91 -35
- oscura/analyzers/waveform/spectral.py +498 -54
- oscura/api/dsl/commands.py +15 -6
- oscura/api/server/templates/base.html +137 -146
- oscura/api/server/templates/export.html +84 -110
- oscura/api/server/templates/home.html +248 -267
- oscura/api/server/templates/protocols.html +44 -48
- oscura/api/server/templates/reports.html +27 -35
- oscura/api/server/templates/session_detail.html +68 -78
- oscura/api/server/templates/sessions.html +62 -72
- oscura/api/server/templates/waveforms.html +54 -64
- oscura/automotive/__init__.py +1 -1
- oscura/automotive/can/session.py +1 -1
- oscura/automotive/dbc/generator.py +638 -23
- oscura/automotive/dtc/data.json +102 -17
- oscura/automotive/uds/decoder.py +99 -6
- oscura/cli/analyze.py +8 -2
- oscura/cli/batch.py +36 -5
- oscura/cli/characterize.py +18 -4
- oscura/cli/export.py +47 -5
- oscura/cli/main.py +2 -0
- oscura/cli/onboarding/wizard.py +10 -6
- oscura/cli/pipeline.py +585 -0
- oscura/cli/visualize.py +6 -4
- oscura/convenience.py +400 -32
- oscura/core/config/loader.py +0 -1
- oscura/core/measurement_result.py +286 -0
- oscura/core/progress.py +1 -1
- oscura/core/schemas/device_mapping.json +8 -2
- oscura/core/schemas/packet_format.json +24 -4
- oscura/core/schemas/protocol_definition.json +12 -2
- oscura/core/types.py +300 -199
- oscura/correlation/multi_protocol.py +1 -1
- oscura/export/legacy/__init__.py +11 -0
- oscura/export/legacy/wav.py +75 -0
- oscura/exporters/__init__.py +19 -0
- oscura/exporters/wireshark.py +809 -0
- oscura/hardware/acquisition/file.py +5 -19
- oscura/hardware/acquisition/saleae.py +10 -10
- oscura/hardware/acquisition/socketcan.py +4 -6
- oscura/hardware/acquisition/synthetic.py +1 -5
- oscura/hardware/acquisition/visa.py +6 -6
- oscura/hardware/security/side_channel_detector.py +5 -508
- oscura/inference/message_format.py +686 -1
- oscura/jupyter/display.py +2 -2
- oscura/jupyter/magic.py +3 -3
- oscura/loaders/__init__.py +17 -12
- oscura/loaders/binary.py +1 -1
- oscura/loaders/chipwhisperer.py +1 -2
- oscura/loaders/configurable.py +1 -1
- oscura/loaders/csv_loader.py +2 -2
- oscura/loaders/hdf5_loader.py +1 -1
- oscura/loaders/lazy.py +6 -1
- oscura/loaders/mmap_loader.py +0 -1
- oscura/loaders/numpy_loader.py +8 -7
- oscura/loaders/preprocessing.py +3 -5
- oscura/loaders/rigol.py +21 -7
- oscura/loaders/sigrok.py +2 -5
- oscura/loaders/tdms.py +3 -2
- oscura/loaders/tektronix.py +38 -32
- oscura/loaders/tss.py +20 -27
- oscura/loaders/vcd.py +13 -8
- oscura/loaders/wav.py +1 -6
- oscura/pipeline/__init__.py +76 -0
- oscura/pipeline/handlers/__init__.py +165 -0
- oscura/pipeline/handlers/analyzers.py +1045 -0
- oscura/pipeline/handlers/decoders.py +899 -0
- oscura/pipeline/handlers/exporters.py +1103 -0
- oscura/pipeline/handlers/filters.py +891 -0
- oscura/pipeline/handlers/loaders.py +640 -0
- oscura/pipeline/handlers/transforms.py +768 -0
- oscura/reporting/__init__.py +88 -1
- oscura/reporting/automation.py +348 -0
- oscura/reporting/citations.py +374 -0
- oscura/reporting/core.py +54 -0
- oscura/reporting/formatting/__init__.py +11 -0
- oscura/reporting/formatting/measurements.py +320 -0
- oscura/reporting/html.py +57 -0
- oscura/reporting/interpretation.py +431 -0
- oscura/reporting/summary.py +329 -0
- oscura/reporting/templates/enhanced/protocol_re.html +504 -503
- oscura/reporting/visualization.py +542 -0
- oscura/side_channel/__init__.py +38 -57
- oscura/utils/builders/signal_builder.py +5 -5
- oscura/utils/comparison/compare.py +7 -9
- oscura/utils/comparison/golden.py +1 -1
- oscura/utils/filtering/convenience.py +2 -2
- oscura/utils/math/arithmetic.py +38 -62
- oscura/utils/math/interpolation.py +20 -20
- oscura/utils/pipeline/__init__.py +4 -17
- oscura/utils/progressive.py +1 -4
- oscura/utils/triggering/edge.py +1 -1
- oscura/utils/triggering/pattern.py +2 -2
- oscura/utils/triggering/pulse.py +2 -2
- oscura/utils/triggering/window.py +3 -3
- oscura/validation/hil_testing.py +11 -11
- oscura/visualization/__init__.py +47 -284
- oscura/visualization/batch.py +160 -0
- oscura/visualization/plot.py +542 -53
- oscura/visualization/styles.py +184 -318
- oscura/workflows/__init__.py +2 -0
- oscura/workflows/batch/advanced.py +1 -1
- oscura/workflows/batch/aggregate.py +7 -8
- oscura/workflows/complete_re.py +251 -23
- oscura/workflows/digital.py +27 -4
- oscura/workflows/multi_trace.py +136 -17
- oscura/workflows/waveform.py +788 -0
- {oscura-0.7.0.dist-info → oscura-0.10.0.dist-info}/METADATA +59 -79
- {oscura-0.7.0.dist-info → oscura-0.10.0.dist-info}/RECORD +135 -149
- oscura/side_channel/dpa.py +0 -1025
- oscura/utils/optimization/__init__.py +0 -19
- oscura/utils/optimization/parallel.py +0 -443
- oscura/utils/optimization/search.py +0 -532
- oscura/utils/pipeline/base.py +0 -338
- oscura/utils/pipeline/composition.py +0 -248
- oscura/utils/pipeline/parallel.py +0 -449
- oscura/utils/pipeline/pipeline.py +0 -375
- oscura/utils/search/__init__.py +0 -16
- oscura/utils/search/anomaly.py +0 -424
- oscura/utils/search/context.py +0 -294
- oscura/utils/search/pattern.py +0 -288
- oscura/utils/storage/__init__.py +0 -61
- oscura/utils/storage/database.py +0 -1166
- oscura/visualization/accessibility.py +0 -526
- oscura/visualization/annotations.py +0 -371
- oscura/visualization/axis_scaling.py +0 -305
- oscura/visualization/colors.py +0 -451
- oscura/visualization/digital.py +0 -436
- oscura/visualization/eye.py +0 -571
- oscura/visualization/histogram.py +0 -281
- oscura/visualization/interactive.py +0 -1035
- oscura/visualization/jitter.py +0 -1042
- oscura/visualization/keyboard.py +0 -394
- oscura/visualization/layout.py +0 -400
- oscura/visualization/optimization.py +0 -1079
- oscura/visualization/palettes.py +0 -446
- oscura/visualization/power.py +0 -508
- oscura/visualization/power_extended.py +0 -955
- oscura/visualization/presets.py +0 -469
- oscura/visualization/protocols.py +0 -1246
- oscura/visualization/render.py +0 -223
- oscura/visualization/rendering.py +0 -444
- oscura/visualization/reverse_engineering.py +0 -838
- oscura/visualization/signal_integrity.py +0 -989
- oscura/visualization/specialized.py +0 -643
- oscura/visualization/spectral.py +0 -1226
- oscura/visualization/thumbnails.py +0 -340
- oscura/visualization/time_axis.py +0 -351
- oscura/visualization/waveform.py +0 -454
- {oscura-0.7.0.dist-info → oscura-0.10.0.dist-info}/WHEEL +0 -0
- {oscura-0.7.0.dist-info → oscura-0.10.0.dist-info}/entry_points.txt +0 -0
- {oscura-0.7.0.dist-info → oscura-0.10.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,16 +1,26 @@
|
|
|
1
|
-
"""DBC file generator
|
|
1
|
+
"""DBC file generator with intelligent signal detection and naming.
|
|
2
2
|
|
|
3
|
-
This module provides
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
This module provides enhanced DBC generation from CAN reverse engineering,
|
|
4
|
+
featuring:
|
|
5
|
+
- Correlation-based multi-bit signal detection
|
|
6
|
+
- Intelligent signal naming from stimulus labels and patterns
|
|
7
|
+
- Automatic endianness detection for multi-byte signals
|
|
6
8
|
|
|
7
|
-
|
|
9
|
+
The generator analyzes bit-level correlations to identify signal boundaries,
|
|
10
|
+
applies pattern recognition for intelligent naming, and automatically detects
|
|
11
|
+
the correct byte order for multi-byte signals.
|
|
8
12
|
"""
|
|
9
13
|
|
|
10
14
|
from __future__ import annotations
|
|
11
15
|
|
|
16
|
+
import re
|
|
17
|
+
from collections import Counter
|
|
12
18
|
from pathlib import Path
|
|
13
|
-
from typing import TYPE_CHECKING
|
|
19
|
+
from typing import TYPE_CHECKING, ClassVar
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
from numpy.typing import NDArray
|
|
23
|
+
from scipy.stats import pearsonr
|
|
14
24
|
|
|
15
25
|
if TYPE_CHECKING:
|
|
16
26
|
from oscura.automotive.can.discovery import DiscoveryDocument
|
|
@@ -24,17 +34,587 @@ from oscura.automotive.can.dbc_generator import (
|
|
|
24
34
|
DBCSignal,
|
|
25
35
|
)
|
|
26
36
|
|
|
27
|
-
__all__ = ["DBCGenerator"]
|
|
37
|
+
__all__ = ["DBCGenerator", "SignalDetector", "SignalNamer"]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class SignalDetector:
|
|
41
|
+
"""Detect signal boundaries using bit-level correlation analysis.
|
|
42
|
+
|
|
43
|
+
This class analyzes bit-level patterns across multiple CAN messages to identify
|
|
44
|
+
multi-bit signals that change together. Correlation drops indicate signal boundaries.
|
|
45
|
+
|
|
46
|
+
Example:
|
|
47
|
+
>>> detector = SignalDetector()
|
|
48
|
+
>>> data = [b'\\x00\\x10', b'\\x00\\x20', b'\\x00\\x30']
|
|
49
|
+
>>> signals = detector.detect_correlated_bits(data)
|
|
50
|
+
>>> # Returns [(8, 8, 'big_endian')] for changing second byte
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
@staticmethod
|
|
54
|
+
def detect_correlated_bits(
|
|
55
|
+
data_bytes: list[bytes], min_correlation: float = 0.85
|
|
56
|
+
) -> list[tuple[int, int, str]]:
|
|
57
|
+
"""Detect multi-bit signals using correlation analysis.
|
|
58
|
+
|
|
59
|
+
Analyzes bit-level correlation across messages to identify bits that change
|
|
60
|
+
together as a group. When correlation drops between adjacent bits, a signal
|
|
61
|
+
boundary is detected.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
data_bytes: List of CAN message data bytes (multiple messages).
|
|
65
|
+
min_correlation: Minimum correlation threshold (0.0-1.0). Higher values
|
|
66
|
+
require stronger correlation to group bits together.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
List of (start_bit, bit_length, byte_order) tuples representing detected
|
|
70
|
+
signals. byte_order is 'little_endian' or 'big_endian'.
|
|
71
|
+
|
|
72
|
+
Example:
|
|
73
|
+
>>> detector = SignalDetector()
|
|
74
|
+
>>> # Messages with 16-bit counter at byte 1-2
|
|
75
|
+
>>> data = [b'\\x00\\x00\\x10', b'\\x00\\x00\\x20', b'\\x00\\x00\\x30']
|
|
76
|
+
>>> signals = detector.detect_correlated_bits(data)
|
|
77
|
+
>>> # Returns [(8, 16, 'big_endian')] for the counter signal
|
|
78
|
+
"""
|
|
79
|
+
if len(data_bytes) < 2:
|
|
80
|
+
return []
|
|
81
|
+
|
|
82
|
+
# Determine max byte length across all messages
|
|
83
|
+
max_len = max(len(d) for d in data_bytes)
|
|
84
|
+
if max_len == 0:
|
|
85
|
+
return []
|
|
86
|
+
|
|
87
|
+
# Convert to bit arrays (pad shorter messages to max length)
|
|
88
|
+
bit_arrays = []
|
|
89
|
+
for data in data_bytes:
|
|
90
|
+
padded = data + b"\x00" * (max_len - len(data))
|
|
91
|
+
bits = np.unpackbits(np.frombuffer(padded, dtype=np.uint8))
|
|
92
|
+
bit_arrays.append(bits)
|
|
93
|
+
|
|
94
|
+
bit_matrix = np.array(bit_arrays)
|
|
95
|
+
|
|
96
|
+
# Need at least 2 messages for correlation
|
|
97
|
+
if bit_matrix.shape[0] < 2:
|
|
98
|
+
return []
|
|
99
|
+
|
|
100
|
+
signals = []
|
|
101
|
+
visited = set()
|
|
102
|
+
|
|
103
|
+
# Scan each bit position for potential signal starts
|
|
104
|
+
for start_bit in range(max_len * 8):
|
|
105
|
+
if start_bit in visited:
|
|
106
|
+
continue
|
|
107
|
+
|
|
108
|
+
# Skip constant bits (no variance = no information)
|
|
109
|
+
bit_column = bit_matrix[:, start_bit]
|
|
110
|
+
if np.std(bit_column) == 0:
|
|
111
|
+
visited.add(start_bit)
|
|
112
|
+
continue
|
|
113
|
+
|
|
114
|
+
# Find correlated adjacent bits by comparing each successive bit
|
|
115
|
+
correlated_bits = [start_bit]
|
|
116
|
+
current_bit = start_bit
|
|
117
|
+
|
|
118
|
+
while current_bit + 1 < max_len * 8:
|
|
119
|
+
next_bit = current_bit + 1
|
|
120
|
+
next_column = bit_matrix[:, next_bit]
|
|
121
|
+
|
|
122
|
+
# Skip constant bits (signal boundary detected)
|
|
123
|
+
if np.std(next_column) == 0:
|
|
124
|
+
break
|
|
125
|
+
|
|
126
|
+
# Calculate Pearson correlation between current and next bit
|
|
127
|
+
try:
|
|
128
|
+
correlation, _ = pearsonr(bit_column, next_column)
|
|
129
|
+
except Exception:
|
|
130
|
+
correlation = 0.0
|
|
131
|
+
|
|
132
|
+
# High correlation means bits change together (same signal)
|
|
133
|
+
if abs(correlation) >= min_correlation:
|
|
134
|
+
correlated_bits.append(next_bit)
|
|
135
|
+
current_bit = next_bit
|
|
136
|
+
else:
|
|
137
|
+
# Correlation dropped - signal boundary detected
|
|
138
|
+
break
|
|
139
|
+
|
|
140
|
+
# Create signal from correlated bits
|
|
141
|
+
if len(correlated_bits) >= 1:
|
|
142
|
+
bit_length = len(correlated_bits)
|
|
143
|
+
visited.update(correlated_bits)
|
|
144
|
+
|
|
145
|
+
# Detect endianness for multi-byte signals
|
|
146
|
+
byte_order = SignalDetector._detect_byte_order(
|
|
147
|
+
bit_matrix, start_bit, bit_length, data_bytes
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
signals.append((start_bit, bit_length, byte_order))
|
|
151
|
+
|
|
152
|
+
return signals
|
|
153
|
+
|
|
154
|
+
@staticmethod
|
|
155
|
+
def _detect_byte_order(
|
|
156
|
+
bit_matrix: NDArray[np.uint8], start_bit: int, bit_length: int, data_bytes: list[bytes]
|
|
157
|
+
) -> str:
|
|
158
|
+
"""Detect byte order (endianness) for multi-byte signal.
|
|
159
|
+
|
|
160
|
+
Compares big-endian vs little-endian interpretations of the signal data
|
|
161
|
+
to determine which byte order produces more realistic value progressions.
|
|
162
|
+
Automotive CAN typically uses big-endian (Motorola) byte order.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
bit_matrix: Matrix of bit values across messages (unused but kept for API).
|
|
166
|
+
start_bit: Signal start bit position.
|
|
167
|
+
bit_length: Signal length in bits.
|
|
168
|
+
data_bytes: Original message data bytes.
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
'little_endian' or 'big_endian'.
|
|
172
|
+
"""
|
|
173
|
+
# Single-byte signals default to little endian (no byte order issues)
|
|
174
|
+
if bit_length <= 8:
|
|
175
|
+
return "little_endian"
|
|
176
|
+
|
|
177
|
+
# Multi-byte signals: extract and compare both byte orders
|
|
178
|
+
start_byte = start_bit // 8
|
|
179
|
+
num_bytes = (bit_length + 7) // 8
|
|
180
|
+
|
|
181
|
+
# Extract values in both byte orders
|
|
182
|
+
big_endian_values = []
|
|
183
|
+
little_endian_values = []
|
|
184
|
+
|
|
185
|
+
for data in data_bytes:
|
|
186
|
+
if len(data) < start_byte + num_bytes:
|
|
187
|
+
continue
|
|
188
|
+
|
|
189
|
+
byte_slice = data[start_byte : start_byte + num_bytes]
|
|
190
|
+
|
|
191
|
+
# Big-endian (most significant byte first - typical for automotive)
|
|
192
|
+
big_val = int.from_bytes(byte_slice, byteorder="big")
|
|
193
|
+
big_endian_values.append(big_val)
|
|
194
|
+
|
|
195
|
+
# Little-endian (least significant byte first)
|
|
196
|
+
little_val = int.from_bytes(byte_slice, byteorder="little")
|
|
197
|
+
little_endian_values.append(little_val)
|
|
198
|
+
|
|
199
|
+
if not big_endian_values:
|
|
200
|
+
return "big_endian" # Default for automotive CAN
|
|
201
|
+
|
|
202
|
+
# Heuristic 1: Variance of differences (smoother = correct)
|
|
203
|
+
# Automotive signals typically change gradually, not with huge jumps
|
|
204
|
+
big_diffs = np.diff(big_endian_values) if len(big_endian_values) > 1 else [0]
|
|
205
|
+
little_diffs = np.diff(little_endian_values) if len(little_endian_values) > 1 else [0]
|
|
206
|
+
|
|
207
|
+
big_variance = np.var(big_diffs)
|
|
208
|
+
little_variance = np.var(little_diffs)
|
|
209
|
+
|
|
210
|
+
# Heuristic 2: Realistic value ranges
|
|
211
|
+
# Byte-swapped values often have unrealistic magnitudes
|
|
212
|
+
little_max = max(little_endian_values)
|
|
213
|
+
|
|
214
|
+
# Prefer big-endian (automotive industry standard)
|
|
215
|
+
# Only use little-endian if significantly smoother AND reasonable range
|
|
216
|
+
if little_variance < big_variance * 0.5 and little_max < 100000:
|
|
217
|
+
return "little_endian"
|
|
218
|
+
|
|
219
|
+
return "big_endian"
|
|
220
|
+
|
|
221
|
+
@staticmethod
|
|
222
|
+
def analyze_stimulus_correlation(
|
|
223
|
+
baseline_data: list[bytes],
|
|
224
|
+
stimulus_data: list[bytes],
|
|
225
|
+
stimulus_label: str,
|
|
226
|
+
) -> list[tuple[int, int, str, str]]:
|
|
227
|
+
"""Detect signals that correlate with stimulus events.
|
|
228
|
+
|
|
229
|
+
Compares baseline (no action) vs stimulus (action performed) captures to identify
|
|
230
|
+
which bits change in response to the stimulus. This enables rapid signal mapping
|
|
231
|
+
during reverse engineering.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
baseline_data: Baseline message data (no stimulus).
|
|
235
|
+
stimulus_data: Stimulus message data (during/after stimulus).
|
|
236
|
+
stimulus_label: Stimulus description (e.g., "rpm_increase", "brake_pressed").
|
|
237
|
+
|
|
238
|
+
Returns:
|
|
239
|
+
List of (start_bit, bit_length, byte_order, stimulus_hint) tuples for signals
|
|
240
|
+
that changed significantly during stimulus.
|
|
241
|
+
|
|
242
|
+
Example:
|
|
243
|
+
>>> detector = SignalDetector()
|
|
244
|
+
>>> baseline = [b'\\x00\\x00\\x10', b'\\x00\\x00\\x10']
|
|
245
|
+
>>> stimulus = [b'\\x00\\x00\\x20', b'\\x00\\x00\\x30']
|
|
246
|
+
>>> signals = detector.analyze_stimulus_correlation(
|
|
247
|
+
... baseline, stimulus, "rpm_increase"
|
|
248
|
+
... )
|
|
249
|
+
>>> # Returns [(8, 16, 'big_endian', 'rpm_increase')]
|
|
250
|
+
"""
|
|
251
|
+
if len(baseline_data) < 2 or len(stimulus_data) < 2:
|
|
252
|
+
return []
|
|
253
|
+
|
|
254
|
+
# Detect signals in stimulus dataset
|
|
255
|
+
stimulus_signals = SignalDetector.detect_correlated_bits(stimulus_data)
|
|
256
|
+
|
|
257
|
+
# Find signals that changed between baseline and stimulus
|
|
258
|
+
changed_signals = []
|
|
259
|
+
|
|
260
|
+
for start_bit, bit_length, byte_order in stimulus_signals:
|
|
261
|
+
# Extract values for this signal in both datasets
|
|
262
|
+
baseline_values = SignalDetector._extract_signal_values(
|
|
263
|
+
baseline_data, start_bit, bit_length
|
|
264
|
+
)
|
|
265
|
+
stimulus_values = SignalDetector._extract_signal_values(
|
|
266
|
+
stimulus_data, start_bit, bit_length
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
# Check if signal changed significantly
|
|
270
|
+
if SignalDetector._signal_changed_significantly(baseline_values, stimulus_values):
|
|
271
|
+
changed_signals.append((start_bit, bit_length, byte_order, stimulus_label))
|
|
272
|
+
|
|
273
|
+
return changed_signals
|
|
274
|
+
|
|
275
|
+
@staticmethod
|
|
276
|
+
def _extract_signal_values(
|
|
277
|
+
data_bytes: list[bytes], start_bit: int, bit_length: int
|
|
278
|
+
) -> list[int]:
|
|
279
|
+
"""Extract signal values from message data.
|
|
280
|
+
|
|
281
|
+
Args:
|
|
282
|
+
data_bytes: Message data bytes.
|
|
283
|
+
start_bit: Signal start bit.
|
|
284
|
+
bit_length: Signal bit length.
|
|
285
|
+
|
|
286
|
+
Returns:
|
|
287
|
+
List of extracted signal values.
|
|
288
|
+
"""
|
|
289
|
+
values = []
|
|
290
|
+
start_byte = start_bit // 8
|
|
291
|
+
num_bytes = (bit_length + 7) // 8
|
|
292
|
+
|
|
293
|
+
for data in data_bytes:
|
|
294
|
+
if len(data) < start_byte + num_bytes:
|
|
295
|
+
continue
|
|
296
|
+
|
|
297
|
+
byte_slice = data[start_byte : start_byte + num_bytes]
|
|
298
|
+
value = int.from_bytes(byte_slice, byteorder="big")
|
|
299
|
+
values.append(value)
|
|
300
|
+
|
|
301
|
+
return values
|
|
302
|
+
|
|
303
|
+
@staticmethod
|
|
304
|
+
def _signal_changed_significantly(baseline: list[int], stimulus: list[int]) -> bool:
|
|
305
|
+
"""Check if signal changed significantly between baseline and stimulus.
|
|
306
|
+
|
|
307
|
+
Args:
|
|
308
|
+
baseline: Baseline signal values.
|
|
309
|
+
stimulus: Stimulus signal values.
|
|
310
|
+
|
|
311
|
+
Returns:
|
|
312
|
+
True if signal changed significantly.
|
|
313
|
+
"""
|
|
314
|
+
if not baseline or not stimulus:
|
|
315
|
+
return False
|
|
316
|
+
|
|
317
|
+
# Check mean difference
|
|
318
|
+
baseline_mean = np.mean(baseline)
|
|
319
|
+
stimulus_mean = np.mean(stimulus)
|
|
320
|
+
|
|
321
|
+
# Check variance difference
|
|
322
|
+
baseline_std = np.std(baseline)
|
|
323
|
+
stimulus_std = np.std(stimulus)
|
|
324
|
+
|
|
325
|
+
# Signal changed if mean or variance changed significantly
|
|
326
|
+
mean_change = abs(stimulus_mean - baseline_mean)
|
|
327
|
+
variance_change = abs(stimulus_std - baseline_std)
|
|
328
|
+
|
|
329
|
+
# Thresholds: >10% mean change or >50% variance increase
|
|
330
|
+
if baseline_mean > 0:
|
|
331
|
+
mean_change_pct = mean_change / baseline_mean
|
|
332
|
+
if mean_change_pct > 0.1:
|
|
333
|
+
return True
|
|
334
|
+
|
|
335
|
+
return bool(variance_change > baseline_std * 0.5)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
class SignalNamer:
|
|
339
|
+
"""Generate intelligent signal names from stimulus labels and patterns.
|
|
340
|
+
|
|
341
|
+
This class applies pattern recognition and heuristics to generate meaningful
|
|
342
|
+
signal names from stimulus labels (e.g., "rpm_increase" → "EngineRPM") and
|
|
343
|
+
value patterns (e.g., sequential counter → "MsgCounter").
|
|
344
|
+
|
|
345
|
+
Example:
|
|
346
|
+
>>> namer = SignalNamer()
|
|
347
|
+
>>> namer.name_from_stimulus("rpm_increase", (0, 8000))
|
|
348
|
+
'EngineRPM'
|
|
349
|
+
>>> namer.name_from_pattern([0, 1, 2, 3, 4], 8)
|
|
350
|
+
'MsgCounter'
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
# Pattern-based name mappings (automotive domain knowledge)
|
|
354
|
+
STIMULUS_PATTERNS: ClassVar[dict[str, str]] = {
|
|
355
|
+
r"rpm|engine.*speed": "EngineRPM",
|
|
356
|
+
r"throttle|accel|pedal": "ThrottlePosition",
|
|
357
|
+
r"brake": "BrakeStatus",
|
|
358
|
+
r"speed|velocity": "VehicleSpeed",
|
|
359
|
+
r"steer|wheel": "SteeringAngle",
|
|
360
|
+
r"temp|temperature": "Temperature",
|
|
361
|
+
r"voltage|volt": "Voltage",
|
|
362
|
+
r"current|amp": "Current",
|
|
363
|
+
r"pressure": "Pressure",
|
|
364
|
+
r"gear": "GearPosition",
|
|
365
|
+
r"torque": "Torque",
|
|
366
|
+
r"fuel": "FuelLevel",
|
|
367
|
+
r"battery": "BatteryVoltage",
|
|
368
|
+
r"coolant": "CoolantTemp",
|
|
369
|
+
r"oil": "OilPressure",
|
|
370
|
+
r"wheel.*speed": "WheelSpeed",
|
|
371
|
+
r"door": "DoorStatus",
|
|
372
|
+
r"window": "WindowPosition",
|
|
373
|
+
r"seat": "SeatPosition",
|
|
374
|
+
r"mirror": "MirrorPosition",
|
|
375
|
+
r"wiper": "WiperStatus",
|
|
376
|
+
r"light|lamp": "LightStatus",
|
|
377
|
+
r"turn.*signal": "TurnSignal",
|
|
378
|
+
r"horn": "HornStatus",
|
|
379
|
+
r"airbag": "AirbagStatus",
|
|
380
|
+
r"abs": "ABSStatus",
|
|
381
|
+
r"traction": "TractionControl",
|
|
382
|
+
r"cruise": "CruiseControl",
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
# Value range-based suffixes
|
|
386
|
+
VALUE_RANGE_SUFFIXES: ClassVar[dict[tuple[int, int], str]] = {
|
|
387
|
+
(0, 100): "_pct",
|
|
388
|
+
(0, 360): "_deg",
|
|
389
|
+
(-180, 180): "_deg",
|
|
390
|
+
(0, 1): "_flag",
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
@staticmethod
|
|
394
|
+
def name_from_stimulus(
|
|
395
|
+
stimulus_label: str,
|
|
396
|
+
value_range: tuple[float, float] | None = None,
|
|
397
|
+
is_signed: bool = False,
|
|
398
|
+
) -> str:
|
|
399
|
+
"""Generate signal name from stimulus label.
|
|
400
|
+
|
|
401
|
+
Extracts meaningful signal names from user actions during reverse engineering.
|
|
402
|
+
For example, "rpm_increase" becomes "EngineRPM" using pattern matching.
|
|
403
|
+
|
|
404
|
+
Args:
|
|
405
|
+
stimulus_label: Stimulus label (e.g., "rpm_increase", "brake_pressed").
|
|
406
|
+
value_range: Observed (min, max) value range for suffix hints.
|
|
407
|
+
is_signed: Whether signal is signed (adds "_signed" suffix if needed).
|
|
408
|
+
|
|
409
|
+
Returns:
|
|
410
|
+
Signal name in PascalCase with appropriate suffix.
|
|
411
|
+
|
|
412
|
+
Example:
|
|
413
|
+
>>> namer = SignalNamer()
|
|
414
|
+
>>> namer.name_from_stimulus("rpm_increase", (0, 8000))
|
|
415
|
+
'EngineRPM'
|
|
416
|
+
>>> namer.name_from_stimulus("throttle_50pct", (0, 100))
|
|
417
|
+
'ThrottlePosition_pct'
|
|
418
|
+
>>> namer.name_from_stimulus("brake_pressed")
|
|
419
|
+
'BrakeStatus'
|
|
420
|
+
"""
|
|
421
|
+
# Clean stimulus label (remove punctuation, convert to lowercase)
|
|
422
|
+
label_lower = stimulus_label.lower()
|
|
423
|
+
label_lower = re.sub(r"[_\-\.]", " ", label_lower)
|
|
424
|
+
|
|
425
|
+
# Try pattern matching against known automotive terms
|
|
426
|
+
for pattern, name in SignalNamer.STIMULUS_PATTERNS.items():
|
|
427
|
+
if re.search(pattern, label_lower):
|
|
428
|
+
base_name = name
|
|
429
|
+
break
|
|
430
|
+
else:
|
|
431
|
+
# No pattern match - extract meaningful words
|
|
432
|
+
words = label_lower.split()
|
|
433
|
+
# Remove common action words that don't indicate signal meaning
|
|
434
|
+
filtered = [
|
|
435
|
+
w
|
|
436
|
+
for w in words
|
|
437
|
+
if w
|
|
438
|
+
not in {
|
|
439
|
+
"increase",
|
|
440
|
+
"decrease",
|
|
441
|
+
"pressed",
|
|
442
|
+
"released",
|
|
443
|
+
"on",
|
|
444
|
+
"off",
|
|
445
|
+
"activate",
|
|
446
|
+
"deactivate",
|
|
447
|
+
"set",
|
|
448
|
+
"get",
|
|
449
|
+
}
|
|
450
|
+
]
|
|
451
|
+
if filtered:
|
|
452
|
+
base_name = "".join(w.capitalize() for w in filtered)
|
|
453
|
+
else:
|
|
454
|
+
base_name = "Signal"
|
|
455
|
+
|
|
456
|
+
# Add range-based suffix for additional context
|
|
457
|
+
suffix = ""
|
|
458
|
+
if value_range:
|
|
459
|
+
min_val, max_val = value_range
|
|
460
|
+
for (range_min, range_max), range_suffix in SignalNamer.VALUE_RANGE_SUFFIXES.items():
|
|
461
|
+
if range_min <= min_val <= max_val <= range_max:
|
|
462
|
+
suffix = range_suffix
|
|
463
|
+
break
|
|
464
|
+
|
|
465
|
+
# Add signed suffix if needed (and no other suffix)
|
|
466
|
+
if is_signed and not suffix:
|
|
467
|
+
suffix = "_signed"
|
|
468
|
+
|
|
469
|
+
return base_name + suffix
|
|
470
|
+
|
|
471
|
+
@staticmethod
|
|
472
|
+
def name_from_pattern(
|
|
473
|
+
byte_values: list[int],
|
|
474
|
+
bit_length: int,
|
|
475
|
+
is_signed: bool = False,
|
|
476
|
+
) -> str:
|
|
477
|
+
"""Generate signal name from value pattern analysis.
|
|
478
|
+
|
|
479
|
+
Analyzes the observed values to detect common patterns like counters,
|
|
480
|
+
checksums, flags, and enumerations.
|
|
481
|
+
|
|
482
|
+
Args:
|
|
483
|
+
byte_values: Observed signal values across multiple messages.
|
|
484
|
+
bit_length: Signal length in bits.
|
|
485
|
+
is_signed: Whether signal is signed.
|
|
486
|
+
|
|
487
|
+
Returns:
|
|
488
|
+
Signal name based on detected pattern (e.g., "MsgCounter", "Checksum").
|
|
489
|
+
|
|
490
|
+
Example:
|
|
491
|
+
>>> namer = SignalNamer()
|
|
492
|
+
>>> namer.name_from_pattern([0, 1, 2, 3, 4, 5], 8)
|
|
493
|
+
'MsgCounter'
|
|
494
|
+
>>> namer.name_from_pattern([0, 0, 0, 1, 1, 1], 1)
|
|
495
|
+
'StatusFlag'
|
|
496
|
+
>>> namer.name_from_pattern([0x3A, 0x7F, 0x92, 0x1B], 8)
|
|
497
|
+
'Checksum'
|
|
498
|
+
"""
|
|
499
|
+
if not byte_values:
|
|
500
|
+
return "UnknownSignal"
|
|
501
|
+
|
|
502
|
+
unique_values = set(byte_values)
|
|
503
|
+
value_count = len(unique_values)
|
|
504
|
+
|
|
505
|
+
# Pattern 1: Counter (sequential incrementing values)
|
|
506
|
+
if value_count > 4 and len(byte_values) > 1:
|
|
507
|
+
diffs = [byte_values[i + 1] - byte_values[i] for i in range(len(byte_values) - 1)]
|
|
508
|
+
if diffs:
|
|
509
|
+
common_diff = Counter(diffs).most_common(1)[0][0]
|
|
510
|
+
# Rolling counter with increment of 1
|
|
511
|
+
if common_diff == 1 and value_count > len(byte_values) * 0.5:
|
|
512
|
+
return "MsgCounter"
|
|
513
|
+
# Frame counter with other increment
|
|
514
|
+
elif common_diff != 0:
|
|
515
|
+
return "FrameCounter"
|
|
516
|
+
|
|
517
|
+
# Pattern 2: Boolean/flag (only 0 and 1)
|
|
518
|
+
if unique_values <= {0, 1} and bit_length == 1:
|
|
519
|
+
return "StatusFlag"
|
|
520
|
+
|
|
521
|
+
# Pattern 3: Checksum (high entropy, covers most of value range)
|
|
522
|
+
max_possible = (1 << bit_length) - 1
|
|
523
|
+
if value_count > max_possible * 0.75:
|
|
524
|
+
return "Checksum"
|
|
525
|
+
|
|
526
|
+
# Pattern 4: Enum (few discrete values)
|
|
527
|
+
if value_count <= 8:
|
|
528
|
+
if bit_length <= 4:
|
|
529
|
+
return "StatusCode"
|
|
530
|
+
else:
|
|
531
|
+
return "ModeSelect"
|
|
532
|
+
|
|
533
|
+
# Pattern 5: Value signal (continuous range analysis)
|
|
534
|
+
min_val, max_val = min(byte_values), max(byte_values)
|
|
535
|
+
|
|
536
|
+
# Percentage-like (0-100 range)
|
|
537
|
+
if min_val >= 0 and max_val <= 100:
|
|
538
|
+
return "Value_pct"
|
|
539
|
+
|
|
540
|
+
# Angle-like (0-360 range)
|
|
541
|
+
if min_val >= 0 and max_val <= 360:
|
|
542
|
+
return "Angle_deg"
|
|
543
|
+
|
|
544
|
+
# Temperature-like (signed, reasonable range for °C)
|
|
545
|
+
if is_signed and -40 <= min_val <= 150:
|
|
546
|
+
return "Temperature"
|
|
547
|
+
|
|
548
|
+
# Generic data signal (no pattern detected)
|
|
549
|
+
return "DataSignal"
|
|
550
|
+
|
|
551
|
+
@staticmethod
|
|
552
|
+
def generate_name(
|
|
553
|
+
start_bit: int,
|
|
554
|
+
bit_length: int,
|
|
555
|
+
byte_values: list[int] | None = None,
|
|
556
|
+
stimulus_label: str | None = None,
|
|
557
|
+
value_range: tuple[float, float] | None = None,
|
|
558
|
+
is_signed: bool = False,
|
|
559
|
+
) -> str:
|
|
560
|
+
"""Generate signal name using all available heuristics.
|
|
561
|
+
|
|
562
|
+
Combines stimulus-based, pattern-based, and position-based naming strategies
|
|
563
|
+
to generate the most descriptive possible signal name.
|
|
564
|
+
|
|
565
|
+
Args:
|
|
566
|
+
start_bit: Signal start bit position.
|
|
567
|
+
bit_length: Signal length in bits.
|
|
568
|
+
byte_values: Observed values (optional).
|
|
569
|
+
stimulus_label: Stimulus label if available (optional).
|
|
570
|
+
value_range: Observed value range (optional).
|
|
571
|
+
is_signed: Whether signal is signed.
|
|
572
|
+
|
|
573
|
+
Returns:
|
|
574
|
+
Best possible signal name based on available information.
|
|
575
|
+
|
|
576
|
+
Example:
|
|
577
|
+
>>> namer = SignalNamer()
|
|
578
|
+
>>> # Stimulus-based naming (most preferred)
|
|
579
|
+
>>> namer.generate_name(0, 16, stimulus_label="rpm_increase")
|
|
580
|
+
'EngineRPM'
|
|
581
|
+
>>> # Pattern-based naming (fallback)
|
|
582
|
+
>>> namer.generate_name(0, 8, byte_values=[0, 1, 2, 3])
|
|
583
|
+
'MsgCounter'
|
|
584
|
+
>>> # Position-based naming (last resort)
|
|
585
|
+
>>> namer.generate_name(16, 8)
|
|
586
|
+
'Signal_B2_b0_8bit'
|
|
587
|
+
"""
|
|
588
|
+
# Strategy 1: Stimulus-based naming (highest confidence)
|
|
589
|
+
if stimulus_label:
|
|
590
|
+
return SignalNamer.name_from_stimulus(stimulus_label, value_range, is_signed)
|
|
591
|
+
|
|
592
|
+
# Strategy 2: Pattern-based naming (medium confidence)
|
|
593
|
+
if byte_values:
|
|
594
|
+
return SignalNamer.name_from_pattern(byte_values, bit_length, is_signed)
|
|
595
|
+
|
|
596
|
+
# Strategy 3: Position-based naming (last resort)
|
|
597
|
+
byte_pos = start_bit // 8
|
|
598
|
+
bit_offset = start_bit % 8
|
|
599
|
+
return f"Signal_B{byte_pos}_b{bit_offset}_{bit_length}bit"
|
|
28
600
|
|
|
29
601
|
|
|
30
602
|
class DBCGenerator:
|
|
31
|
-
"""
|
|
603
|
+
"""Enhanced DBC generator with intelligent signal detection and naming.
|
|
32
604
|
|
|
33
|
-
This class provides
|
|
34
|
-
|
|
35
|
-
|
|
605
|
+
This class provides advanced DBC generation featuring:
|
|
606
|
+
- Correlation-based signal boundary detection
|
|
607
|
+
- Intelligent signal naming from stimulus labels
|
|
608
|
+
- Automatic endianness detection
|
|
36
609
|
|
|
37
|
-
|
|
610
|
+
The generator can work with discovery documents or raw CAN sessions to produce
|
|
611
|
+
high-quality DBC files with meaningful signal names and correct byte orders.
|
|
612
|
+
|
|
613
|
+
Example:
|
|
614
|
+
>>> from oscura.automotive.can.discovery import DiscoveryDocument
|
|
615
|
+
>>> doc = DiscoveryDocument()
|
|
616
|
+
>>> # ... add messages and signals ...
|
|
617
|
+
>>> DBCGenerator.generate(doc, "output.dbc")
|
|
38
618
|
"""
|
|
39
619
|
|
|
40
620
|
@staticmethod
|
|
@@ -43,30 +623,38 @@ class DBCGenerator:
|
|
|
43
623
|
output_path: Path | str,
|
|
44
624
|
min_confidence: float = 0.0,
|
|
45
625
|
include_comments: bool = True,
|
|
626
|
+
enable_correlation_detection: bool = True,
|
|
627
|
+
enable_intelligent_naming: bool = True,
|
|
46
628
|
) -> None:
|
|
47
|
-
"""Generate DBC file from discovery document.
|
|
629
|
+
"""Generate DBC file from discovery document with enhancements.
|
|
48
630
|
|
|
49
631
|
Args:
|
|
50
632
|
discovery: DiscoveryDocument with discovered signals.
|
|
51
633
|
output_path: Output DBC file path.
|
|
52
|
-
min_confidence: Minimum confidence threshold for including signals.
|
|
53
|
-
include_comments: Include evidence as comments in DBC.
|
|
634
|
+
min_confidence: Minimum confidence threshold for including signals (0.0-1.0).
|
|
635
|
+
include_comments: Include evidence as comments in DBC file.
|
|
636
|
+
enable_correlation_detection: Use correlation-based signal detection (reserved).
|
|
637
|
+
enable_intelligent_naming: Use intelligent signal naming.
|
|
54
638
|
|
|
55
639
|
Example:
|
|
56
640
|
>>> from oscura.automotive.can.discovery import DiscoveryDocument
|
|
57
641
|
>>> doc = DiscoveryDocument()
|
|
58
642
|
>>> # ... add messages and signals ...
|
|
59
|
-
>>> DBCGenerator.generate(
|
|
643
|
+
>>> DBCGenerator.generate(
|
|
644
|
+
... doc,
|
|
645
|
+
... "output.dbc",
|
|
646
|
+
... min_confidence=0.8,
|
|
647
|
+
... enable_intelligent_naming=True
|
|
648
|
+
... )
|
|
60
649
|
"""
|
|
61
|
-
|
|
62
650
|
path = Path(output_path)
|
|
63
651
|
|
|
64
|
-
# Create instance of
|
|
652
|
+
# Create instance of underlying DBC generator implementation
|
|
65
653
|
gen = _DBCGeneratorImpl()
|
|
66
654
|
|
|
67
655
|
# Convert discovery document to DBC format
|
|
68
656
|
for msg_id, msg_discovery in sorted(discovery.messages.items()):
|
|
69
|
-
# Filter signals by confidence
|
|
657
|
+
# Filter signals by confidence threshold
|
|
70
658
|
signals = [s for s in msg_discovery.signals if s.confidence >= min_confidence]
|
|
71
659
|
|
|
72
660
|
if not signals:
|
|
@@ -75,14 +663,41 @@ class DBCGenerator:
|
|
|
75
663
|
# Convert signals to DBC format
|
|
76
664
|
dbc_signals = []
|
|
77
665
|
for sig in signals:
|
|
78
|
-
#
|
|
79
|
-
# Floats are typically represented as unsigned in DBC
|
|
666
|
+
# Determine value type (unsigned/signed)
|
|
80
667
|
value_type: str = sig.value_type
|
|
81
668
|
if value_type not in ("unsigned", "signed"):
|
|
82
669
|
value_type = "unsigned"
|
|
83
670
|
|
|
671
|
+
# Apply intelligent naming if enabled
|
|
672
|
+
signal_name = sig.name
|
|
673
|
+
if enable_intelligent_naming:
|
|
674
|
+
# Extract stimulus hint from evidence
|
|
675
|
+
stimulus_hint = None
|
|
676
|
+
for evidence in sig.evidence:
|
|
677
|
+
if "stimulus:" in evidence.lower():
|
|
678
|
+
stimulus_hint = evidence.split(":", 1)[1].strip()
|
|
679
|
+
break
|
|
680
|
+
|
|
681
|
+
# Generate better name using all available information
|
|
682
|
+
value_range = None
|
|
683
|
+
if sig.min_value is not None and sig.max_value is not None:
|
|
684
|
+
value_range = (sig.min_value, sig.max_value)
|
|
685
|
+
|
|
686
|
+
improved_name = SignalNamer.generate_name(
|
|
687
|
+
start_bit=sig.start_bit,
|
|
688
|
+
bit_length=sig.length,
|
|
689
|
+
stimulus_label=stimulus_hint,
|
|
690
|
+
value_range=value_range,
|
|
691
|
+
is_signed=(value_type == "signed"),
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
# Use improved name if original is generic
|
|
695
|
+
if signal_name.startswith(("Signal_", "Byte_", "Unknown")):
|
|
696
|
+
signal_name = improved_name
|
|
697
|
+
|
|
698
|
+
# Create DBC signal
|
|
84
699
|
dbc_sig = DBCSignal(
|
|
85
|
-
name=
|
|
700
|
+
name=signal_name,
|
|
86
701
|
start_bit=sig.start_bit,
|
|
87
702
|
bit_length=sig.length,
|
|
88
703
|
byte_order=sig.byte_order,
|
|
@@ -120,7 +735,7 @@ class DBCGenerator:
|
|
|
120
735
|
def generate_from_session(
|
|
121
736
|
session: CANSession,
|
|
122
737
|
output_path: Path | str,
|
|
123
|
-
min_confidence: float = 0.8,
|
|
738
|
+
min_confidence: float = 0.8,
|
|
124
739
|
) -> None:
|
|
125
740
|
"""Generate DBC file from CANSession with documented signals.
|
|
126
741
|
|