oscura 0.1.2__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oscura/__init__.py +1 -1
- oscura/analyzers/packet/payload_extraction.py +2 -4
- oscura/analyzers/packet/stream.py +5 -5
- oscura/analyzers/patterns/__init__.py +4 -3
- oscura/analyzers/patterns/clustering.py +3 -1
- oscura/analyzers/power/ac_power.py +0 -2
- oscura/analyzers/power/basic.py +0 -2
- oscura/analyzers/power/ripple.py +0 -2
- oscura/analyzers/signal_integrity/embedding.py +0 -2
- oscura/analyzers/signal_integrity/sparams.py +28 -206
- oscura/analyzers/spectral/fft.py +0 -2
- oscura/analyzers/statistical/__init__.py +3 -3
- oscura/analyzers/statistical/checksum.py +2 -0
- oscura/analyzers/statistical/classification.py +2 -0
- oscura/analyzers/statistical/entropy.py +11 -9
- oscura/analyzers/statistical/ngrams.py +4 -2
- oscura/api/fluent.py +2 -2
- oscura/automotive/__init__.py +1 -3
- oscura/automotive/can/__init__.py +0 -2
- oscura/automotive/dbc/__init__.py +0 -2
- oscura/automotive/dtc/__init__.py +0 -2
- oscura/automotive/dtc/data.json +2763 -0
- oscura/automotive/dtc/database.py +37 -2769
- oscura/automotive/j1939/__init__.py +0 -2
- oscura/automotive/loaders/__init__.py +0 -2
- oscura/automotive/loaders/asc.py +0 -2
- oscura/automotive/loaders/blf.py +0 -2
- oscura/automotive/loaders/csv_can.py +0 -2
- oscura/automotive/obd/__init__.py +0 -2
- oscura/automotive/uds/__init__.py +0 -2
- oscura/automotive/uds/models.py +0 -2
- oscura/cli/main.py +0 -2
- oscura/cli/shell.py +0 -2
- oscura/config/loader.py +0 -2
- oscura/core/backend_selector.py +1 -1
- oscura/core/correlation.py +0 -2
- oscura/core/exceptions.py +56 -2
- oscura/core/lazy.py +5 -3
- oscura/core/memory_limits.py +0 -2
- oscura/core/numba_backend.py +5 -7
- oscura/core/uncertainty.py +3 -3
- oscura/dsl/interpreter.py +2 -0
- oscura/dsl/parser.py +8 -6
- oscura/exploratory/error_recovery.py +3 -3
- oscura/exploratory/parse.py +2 -0
- oscura/exploratory/recovery.py +2 -0
- oscura/exploratory/sync.py +2 -0
- oscura/export/wireshark/generator.py +1 -1
- oscura/export/wireshark/type_mapping.py +2 -0
- oscura/exporters/hdf5.py +1 -3
- oscura/extensibility/templates.py +0 -8
- oscura/inference/active_learning/lstar.py +2 -4
- oscura/inference/active_learning/observation_table.py +0 -2
- oscura/inference/active_learning/oracle.py +3 -1
- oscura/inference/active_learning/teachers/simulator.py +1 -3
- oscura/inference/alignment.py +2 -0
- oscura/inference/message_format.py +2 -0
- oscura/inference/protocol_dsl.py +7 -5
- oscura/inference/sequences.py +12 -14
- oscura/inference/state_machine.py +2 -0
- oscura/integrations/llm.py +3 -1
- oscura/jupyter/display.py +0 -2
- oscura/loaders/__init__.py +67 -51
- oscura/loaders/pcap.py +1 -1
- oscura/loaders/touchstone.py +221 -0
- oscura/math/arithmetic.py +0 -2
- oscura/optimization/parallel.py +9 -6
- oscura/pipeline/composition.py +0 -2
- oscura/plugins/cli.py +0 -2
- oscura/reporting/comparison.py +0 -2
- oscura/reporting/config.py +1 -1
- oscura/reporting/formatting/emphasis.py +2 -0
- oscura/reporting/formatting/numbers.py +0 -2
- oscura/reporting/output.py +1 -3
- oscura/reporting/sections.py +0 -2
- oscura/search/anomaly.py +2 -0
- oscura/session/session.py +80 -13
- oscura/testing/synthetic.py +2 -0
- oscura/ui/formatters.py +4 -2
- oscura/utils/buffer.py +2 -2
- oscura/utils/lazy.py +5 -5
- oscura/utils/memory_advanced.py +2 -2
- oscura/utils/memory_extensions.py +2 -2
- oscura/visualization/colors.py +0 -2
- oscura/visualization/power.py +2 -0
- oscura/workflows/multi_trace.py +2 -0
- {oscura-0.1.2.dist-info → oscura-0.3.0.dist-info}/METADATA +37 -16
- {oscura-0.1.2.dist-info → oscura-0.3.0.dist-info}/RECORD +91 -89
- {oscura-0.1.2.dist-info → oscura-0.3.0.dist-info}/WHEEL +0 -0
- {oscura-0.1.2.dist-info → oscura-0.3.0.dist-info}/entry_points.txt +0 -0
- {oscura-0.1.2.dist-info → oscura-0.3.0.dist-info}/licenses/LICENSE +0 -0
oscura/__init__.py
CHANGED
|
@@ -6,8 +6,6 @@ This module provides payload extraction from PCAP packets with metadata
|
|
|
6
6
|
preservation, filtering, and multiple output formats.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
|
-
from __future__ import annotations
|
|
10
|
-
|
|
11
9
|
from collections.abc import Iterator, Sequence
|
|
12
10
|
from dataclasses import dataclass
|
|
13
11
|
from typing import Any, Literal
|
|
@@ -122,7 +120,7 @@ class PayloadExtractor:
|
|
|
122
120
|
packets: Sequence[dict[str, Any] | bytes],
|
|
123
121
|
protocol: str | None = None,
|
|
124
122
|
port_filter: tuple[int | None, int | None] | None = None,
|
|
125
|
-
) -> list[PayloadInfo]:
|
|
123
|
+
) -> list["PayloadInfo"]:
|
|
126
124
|
"""Extract payloads from all packets with metadata.
|
|
127
125
|
|
|
128
126
|
Implements RE-PAY-001: Batch payload extraction with metadata.
|
|
@@ -186,7 +184,7 @@ class PayloadExtractor:
|
|
|
186
184
|
def iter_payloads(
|
|
187
185
|
self,
|
|
188
186
|
packets: Sequence[dict[str, Any] | bytes],
|
|
189
|
-
) -> Iterator[PayloadInfo]:
|
|
187
|
+
) -> Iterator["PayloadInfo"]:
|
|
190
188
|
"""Iterate over payloads for memory-efficient processing.
|
|
191
189
|
|
|
192
190
|
Implements RE-PAY-001: Streaming payload iteration.
|
|
@@ -22,11 +22,11 @@ from typing import TYPE_CHECKING, Any, BinaryIO, TypeVar
|
|
|
22
22
|
|
|
23
23
|
from oscura.analyzers.packet.parser import BinaryParser
|
|
24
24
|
|
|
25
|
+
T = TypeVar("T")
|
|
26
|
+
|
|
25
27
|
if TYPE_CHECKING:
|
|
26
28
|
from collections.abc import Callable, Iterator
|
|
27
29
|
|
|
28
|
-
T = TypeVar("T")
|
|
29
|
-
|
|
30
30
|
|
|
31
31
|
@dataclass
|
|
32
32
|
class StreamPacket:
|
|
@@ -282,7 +282,7 @@ def pipeline(
|
|
|
282
282
|
yield from result
|
|
283
283
|
|
|
284
284
|
|
|
285
|
-
def batch
|
|
285
|
+
def batch(
|
|
286
286
|
source: Iterator[T],
|
|
287
287
|
size: int,
|
|
288
288
|
) -> Iterator[list[T]]:
|
|
@@ -311,7 +311,7 @@ def batch[T](
|
|
|
311
311
|
yield current_batch
|
|
312
312
|
|
|
313
313
|
|
|
314
|
-
def take
|
|
314
|
+
def take(source: Iterator[T], n: int) -> Iterator[T]:
|
|
315
315
|
"""Take first n items.
|
|
316
316
|
|
|
317
317
|
Args:
|
|
@@ -329,7 +329,7 @@ def take[T](source: Iterator[T], n: int) -> Iterator[T]:
|
|
|
329
329
|
count += 1 # noqa: SIM113
|
|
330
330
|
|
|
331
331
|
|
|
332
|
-
def skip
|
|
332
|
+
def skip(source: Iterator[T], n: int) -> Iterator[T]:
|
|
333
333
|
"""Skip first n items.
|
|
334
334
|
|
|
335
335
|
Args:
|
|
@@ -22,6 +22,8 @@ Author: Oscura Development Team
|
|
|
22
22
|
|
|
23
23
|
# Periodic pattern detection (PAT-001)
|
|
24
24
|
# Pattern clustering (PAT-004)
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
25
27
|
from .clustering import (
|
|
26
28
|
ClusteringResult,
|
|
27
29
|
ClusterResult,
|
|
@@ -125,7 +127,7 @@ def find_motifs(
|
|
|
125
127
|
return results
|
|
126
128
|
|
|
127
129
|
|
|
128
|
-
def extract_motif(data: Any, start: int, length: int) ->
|
|
130
|
+
def extract_motif(data: Any, start: int, length: int) -> NDArray[np.generic]:
|
|
129
131
|
"""Extract a motif from data.
|
|
130
132
|
|
|
131
133
|
Args:
|
|
@@ -137,10 +139,9 @@ def extract_motif(data: Any, start: int, length: int) -> "NDArray[np.generic]":
|
|
|
137
139
|
Extracted motif as numpy array.
|
|
138
140
|
"""
|
|
139
141
|
import numpy as np
|
|
140
|
-
from numpy.typing import NDArray
|
|
141
142
|
|
|
142
143
|
data_arr = np.asarray(data)
|
|
143
|
-
result
|
|
144
|
+
result = data_arr[start : start + length]
|
|
144
145
|
return result
|
|
145
146
|
|
|
146
147
|
|
|
@@ -7,6 +7,8 @@ using various distance metrics and clustering approaches.
|
|
|
7
7
|
Author: Oscura Development Team
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
10
12
|
from dataclasses import dataclass
|
|
11
13
|
from typing import Literal
|
|
12
14
|
|
|
@@ -984,7 +986,7 @@ class PatternClusterer:
|
|
|
984
986
|
|
|
985
987
|
def fit(
|
|
986
988
|
self, patterns: list[bytes | np.ndarray[tuple[int], np.dtype[np.uint8]]]
|
|
987
|
-
) ->
|
|
989
|
+
) -> PatternClusterer:
|
|
988
990
|
"""Fit the clusterer to patterns (sklearn-style interface).
|
|
989
991
|
|
|
990
992
|
Args:
|
oscura/analyzers/power/basic.py
CHANGED
oscura/analyzers/power/ripple.py
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
|
-
"""S-Parameter handling and
|
|
1
|
+
"""S-Parameter handling and analysis.
|
|
2
2
|
|
|
3
|
-
This module provides
|
|
4
|
-
|
|
3
|
+
This module provides S-parameter calculations including return loss and
|
|
4
|
+
insertion loss for signal integrity analysis.
|
|
5
5
|
|
|
6
|
+
For loading Touchstone files, use:
|
|
7
|
+
>>> from oscura.loaders import load_touchstone
|
|
6
8
|
|
|
7
9
|
Example:
|
|
8
|
-
>>> from oscura.
|
|
10
|
+
>>> from oscura.loaders import load_touchstone
|
|
11
|
+
>>> from oscura.analyzers.signal_integrity.sparams import return_loss
|
|
9
12
|
>>> s_params = load_touchstone("cable.s2p")
|
|
10
13
|
>>> rl = return_loss(s_params, frequency=1e9)
|
|
11
14
|
|
|
@@ -16,16 +19,11 @@ References:
|
|
|
16
19
|
|
|
17
20
|
from __future__ import annotations
|
|
18
21
|
|
|
19
|
-
import contextlib
|
|
20
|
-
import re
|
|
21
22
|
from dataclasses import dataclass, field
|
|
22
|
-
from
|
|
23
|
-
from typing import TYPE_CHECKING
|
|
23
|
+
from typing import TYPE_CHECKING, Any
|
|
24
24
|
|
|
25
25
|
import numpy as np
|
|
26
26
|
|
|
27
|
-
from oscura.core.exceptions import FormatError, LoaderError
|
|
28
|
-
|
|
29
27
|
if TYPE_CHECKING:
|
|
30
28
|
from numpy.typing import NDArray
|
|
31
29
|
|
|
@@ -94,201 +92,6 @@ class SParameterData:
|
|
|
94
92
|
)
|
|
95
93
|
|
|
96
94
|
|
|
97
|
-
def load_touchstone(path: str | Path) -> SParameterData:
|
|
98
|
-
"""Load S-parameter data from Touchstone file.
|
|
99
|
-
|
|
100
|
-
Supports .s1p through .s8p formats and both Touchstone 1.0
|
|
101
|
-
and 2.0 file formats.
|
|
102
|
-
|
|
103
|
-
Args:
|
|
104
|
-
path: Path to Touchstone file.
|
|
105
|
-
|
|
106
|
-
Returns:
|
|
107
|
-
SParameterData with loaded S-parameters.
|
|
108
|
-
|
|
109
|
-
Raises:
|
|
110
|
-
LoaderError: If file cannot be read.
|
|
111
|
-
FormatError: If file format is invalid.
|
|
112
|
-
|
|
113
|
-
Example:
|
|
114
|
-
>>> s_params = load_touchstone("cable.s2p")
|
|
115
|
-
>>> print(f"Loaded {s_params.n_ports}-port, {len(s_params.frequencies)} points")
|
|
116
|
-
|
|
117
|
-
References:
|
|
118
|
-
Touchstone 2.0 File Format Specification
|
|
119
|
-
"""
|
|
120
|
-
path = Path(path)
|
|
121
|
-
|
|
122
|
-
if not path.exists():
|
|
123
|
-
raise LoaderError(f"File not found: {path}")
|
|
124
|
-
|
|
125
|
-
# Determine number of ports from extension
|
|
126
|
-
suffix = path.suffix.lower()
|
|
127
|
-
match = re.match(r"\.s(\d+)p", suffix)
|
|
128
|
-
if not match:
|
|
129
|
-
raise FormatError(f"Unsupported file extension: {suffix}")
|
|
130
|
-
|
|
131
|
-
n_ports = int(match.group(1))
|
|
132
|
-
|
|
133
|
-
try:
|
|
134
|
-
with open(path) as f:
|
|
135
|
-
lines = f.readlines()
|
|
136
|
-
except Exception as e:
|
|
137
|
-
raise LoaderError(f"Failed to read file: {e}") # noqa: B904
|
|
138
|
-
|
|
139
|
-
return _parse_touchstone(lines, n_ports, str(path))
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
def _parse_touchstone(
|
|
143
|
-
lines: list[str],
|
|
144
|
-
n_ports: int,
|
|
145
|
-
source_file: str,
|
|
146
|
-
) -> SParameterData:
|
|
147
|
-
"""Parse Touchstone file content.
|
|
148
|
-
|
|
149
|
-
Args:
|
|
150
|
-
lines: File lines.
|
|
151
|
-
n_ports: Number of ports.
|
|
152
|
-
source_file: Source file path.
|
|
153
|
-
|
|
154
|
-
Returns:
|
|
155
|
-
Parsed SParameterData.
|
|
156
|
-
|
|
157
|
-
Raises:
|
|
158
|
-
FormatError: If file format is invalid.
|
|
159
|
-
"""
|
|
160
|
-
comments = []
|
|
161
|
-
option_line = None
|
|
162
|
-
data_lines = []
|
|
163
|
-
|
|
164
|
-
for line in lines:
|
|
165
|
-
line = line.strip()
|
|
166
|
-
|
|
167
|
-
if not line:
|
|
168
|
-
continue
|
|
169
|
-
|
|
170
|
-
if line.startswith("!"):
|
|
171
|
-
comments.append(line[1:].strip())
|
|
172
|
-
elif line.startswith("#"):
|
|
173
|
-
option_line = line
|
|
174
|
-
else:
|
|
175
|
-
data_lines.append(line)
|
|
176
|
-
|
|
177
|
-
# Parse option line
|
|
178
|
-
freq_unit = 1e9 # Default GHz
|
|
179
|
-
format_type = "ma" # Default MA (magnitude/angle)
|
|
180
|
-
z0 = 50.0
|
|
181
|
-
|
|
182
|
-
if option_line:
|
|
183
|
-
option_line = option_line.lower()
|
|
184
|
-
parts = option_line.split()
|
|
185
|
-
|
|
186
|
-
for i, part in enumerate(parts):
|
|
187
|
-
if part in ("hz", "khz", "mhz", "ghz"):
|
|
188
|
-
freq_unit = {
|
|
189
|
-
"hz": 1.0,
|
|
190
|
-
"khz": 1e3,
|
|
191
|
-
"mhz": 1e6,
|
|
192
|
-
"ghz": 1e9,
|
|
193
|
-
}[part]
|
|
194
|
-
elif part in ("db", "ma", "ri"):
|
|
195
|
-
format_type = part
|
|
196
|
-
elif part == "r":
|
|
197
|
-
# Reference impedance follows
|
|
198
|
-
if i + 1 < len(parts):
|
|
199
|
-
with contextlib.suppress(ValueError):
|
|
200
|
-
z0 = float(parts[i + 1])
|
|
201
|
-
|
|
202
|
-
# Parse data
|
|
203
|
-
frequencies = []
|
|
204
|
-
s_data = []
|
|
205
|
-
|
|
206
|
-
# Number of S-parameters per frequency
|
|
207
|
-
n_s_params = n_ports * n_ports
|
|
208
|
-
|
|
209
|
-
i = 0
|
|
210
|
-
while i < len(data_lines):
|
|
211
|
-
# First line has frequency and first S-parameters
|
|
212
|
-
parts = data_lines[i].split()
|
|
213
|
-
|
|
214
|
-
if len(parts) < 1:
|
|
215
|
-
i += 1
|
|
216
|
-
continue
|
|
217
|
-
|
|
218
|
-
freq = float(parts[0]) * freq_unit
|
|
219
|
-
frequencies.append(freq)
|
|
220
|
-
|
|
221
|
-
# Collect all S-parameter values for this frequency
|
|
222
|
-
s_values = []
|
|
223
|
-
|
|
224
|
-
# Add values from first line
|
|
225
|
-
for j in range(1, len(parts), 2):
|
|
226
|
-
if j + 1 < len(parts):
|
|
227
|
-
val1 = float(parts[j])
|
|
228
|
-
val2 = float(parts[j + 1])
|
|
229
|
-
s_values.append((val1, val2))
|
|
230
|
-
|
|
231
|
-
i += 1
|
|
232
|
-
|
|
233
|
-
# Continue collecting from subsequent lines if needed
|
|
234
|
-
while len(s_values) < n_s_params and i < len(data_lines):
|
|
235
|
-
parts = data_lines[i].split()
|
|
236
|
-
|
|
237
|
-
# Check if this is a new frequency (has odd number of values)
|
|
238
|
-
try:
|
|
239
|
-
float(parts[0])
|
|
240
|
-
if len(parts) % 2 == 1:
|
|
241
|
-
break # New frequency line
|
|
242
|
-
except (ValueError, IndexError):
|
|
243
|
-
pass
|
|
244
|
-
|
|
245
|
-
for j in range(0, len(parts), 2):
|
|
246
|
-
if j + 1 < len(parts):
|
|
247
|
-
val1 = float(parts[j])
|
|
248
|
-
val2 = float(parts[j + 1])
|
|
249
|
-
s_values.append((val1, val2))
|
|
250
|
-
|
|
251
|
-
i += 1
|
|
252
|
-
|
|
253
|
-
# Convert to complex based on format
|
|
254
|
-
s_complex = []
|
|
255
|
-
for val1, val2 in s_values:
|
|
256
|
-
if format_type == "ri":
|
|
257
|
-
# Real/Imaginary
|
|
258
|
-
s_complex.append(complex(val1, val2))
|
|
259
|
-
elif format_type == "ma":
|
|
260
|
-
# Magnitude/Angle (degrees)
|
|
261
|
-
mag = val1
|
|
262
|
-
angle_rad = np.radians(val2)
|
|
263
|
-
s_complex.append(mag * np.exp(1j * angle_rad))
|
|
264
|
-
elif format_type == "db":
|
|
265
|
-
# dB/Angle (degrees)
|
|
266
|
-
mag = 10 ** (val1 / 20)
|
|
267
|
-
angle_rad = np.radians(val2)
|
|
268
|
-
s_complex.append(mag * np.exp(1j * angle_rad))
|
|
269
|
-
|
|
270
|
-
# Reshape into matrix
|
|
271
|
-
if len(s_complex) == n_s_params:
|
|
272
|
-
s_matrix = np.array(s_complex).reshape(n_ports, n_ports)
|
|
273
|
-
s_data.append(s_matrix)
|
|
274
|
-
|
|
275
|
-
if len(frequencies) == 0:
|
|
276
|
-
raise FormatError("No valid frequency points found")
|
|
277
|
-
|
|
278
|
-
frequencies_arr = np.array(frequencies, dtype=np.float64)
|
|
279
|
-
s_matrix_arr = np.array(s_data, dtype=np.complex128)
|
|
280
|
-
|
|
281
|
-
return SParameterData(
|
|
282
|
-
frequencies=frequencies_arr,
|
|
283
|
-
s_matrix=s_matrix_arr,
|
|
284
|
-
n_ports=n_ports,
|
|
285
|
-
z0=z0,
|
|
286
|
-
format=format_type,
|
|
287
|
-
source_file=source_file,
|
|
288
|
-
comments=comments,
|
|
289
|
-
)
|
|
290
|
-
|
|
291
|
-
|
|
292
95
|
def return_loss(
|
|
293
96
|
s_params: SParameterData,
|
|
294
97
|
frequency: float | None = None,
|
|
@@ -474,11 +277,30 @@ def _abcd_to_s_single(abcd: NDArray[np.complex128], z0: float) -> NDArray[np.com
|
|
|
474
277
|
return np.array([[S11, S12], [S21, S22]], dtype=np.complex128)
|
|
475
278
|
|
|
476
279
|
|
|
280
|
+
# Backward compatibility: load_touchstone moved to loaders module
|
|
281
|
+
# Import with deprecation warning
|
|
282
|
+
def __getattr__(name: str) -> Any:
|
|
283
|
+
"""Provide backward compatibility for load_touchstone."""
|
|
284
|
+
if name == "load_touchstone":
|
|
285
|
+
import warnings
|
|
286
|
+
|
|
287
|
+
from oscura.loaders.touchstone import load_touchstone
|
|
288
|
+
|
|
289
|
+
warnings.warn(
|
|
290
|
+
"Importing load_touchstone from oscura.analyzers.signal_integrity.sparams "
|
|
291
|
+
"is deprecated. Use 'from oscura.loaders import load_touchstone' instead.",
|
|
292
|
+
DeprecationWarning,
|
|
293
|
+
stacklevel=2,
|
|
294
|
+
)
|
|
295
|
+
return load_touchstone
|
|
296
|
+
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
|
297
|
+
|
|
298
|
+
|
|
477
299
|
__all__ = [
|
|
478
300
|
"SParameterData",
|
|
479
301
|
"abcd_to_s",
|
|
480
302
|
"insertion_loss",
|
|
481
|
-
"load_touchstone",
|
|
303
|
+
"load_touchstone", # Backward compatibility via __getattr__ # noqa: F822
|
|
482
304
|
"return_loss",
|
|
483
305
|
"s_to_abcd",
|
|
484
306
|
]
|
oscura/analyzers/spectral/fft.py
CHANGED
|
@@ -17,6 +17,8 @@ Requirements:
|
|
|
17
17
|
- RE-ENT-002: Byte Frequency Distribution
|
|
18
18
|
"""
|
|
19
19
|
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
20
22
|
from typing import TYPE_CHECKING, Union
|
|
21
23
|
|
|
22
24
|
import numpy as np
|
|
@@ -134,9 +136,7 @@ entropy = shannon_entropy
|
|
|
134
136
|
DataType = Union[bytes, bytearray, "NDArray[np.uint8]"]
|
|
135
137
|
|
|
136
138
|
|
|
137
|
-
def entropy_windowed(
|
|
138
|
-
data: DataType, window_size: int = 256, step: int = 1
|
|
139
|
-
) -> "NDArray[np.float64]":
|
|
139
|
+
def entropy_windowed(data: DataType, window_size: int = 256, step: int = 1) -> NDArray[np.float64]:
|
|
140
140
|
"""Windowed entropy calculation (alias for sliding_entropy)."""
|
|
141
141
|
return sliding_entropy(data, window_size=window_size, step=step)
|
|
142
142
|
|
|
@@ -5,6 +5,8 @@ This module provides tools for detecting checksum and CRC fields in binary
|
|
|
5
5
|
messages by analyzing field correlations and testing common algorithms.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
8
10
|
from dataclasses import dataclass, field
|
|
9
11
|
from typing import TYPE_CHECKING, Any, Literal, Union
|
|
10
12
|
|
|
@@ -8,6 +8,8 @@ transitions for field boundary identification, and classifying data types
|
|
|
8
8
|
based on entropy characteristics.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
11
13
|
from collections import Counter
|
|
12
14
|
from dataclasses import dataclass, field
|
|
13
15
|
from typing import TYPE_CHECKING, Literal, Union
|
|
@@ -78,8 +80,8 @@ class ByteFrequencyResult:
|
|
|
78
80
|
printable_ratio: Proportion of printable ASCII.
|
|
79
81
|
"""
|
|
80
82
|
|
|
81
|
-
counts:
|
|
82
|
-
frequencies:
|
|
83
|
+
counts: NDArray[np.int64]
|
|
84
|
+
frequencies: NDArray[np.float64]
|
|
83
85
|
entropy: float
|
|
84
86
|
unique_bytes: int
|
|
85
87
|
most_common: list[tuple[int, int]]
|
|
@@ -103,8 +105,8 @@ class FrequencyAnomalyResult:
|
|
|
103
105
|
"""
|
|
104
106
|
|
|
105
107
|
anomalous_bytes: list[int]
|
|
106
|
-
z_scores:
|
|
107
|
-
is_anomalous:
|
|
108
|
+
z_scores: NDArray[np.float64]
|
|
109
|
+
is_anomalous: NDArray[np.bool_]
|
|
108
110
|
expected_frequency: float
|
|
109
111
|
|
|
110
112
|
|
|
@@ -221,7 +223,7 @@ def bit_entropy(data: DataType) -> float:
|
|
|
221
223
|
|
|
222
224
|
def sliding_entropy(
|
|
223
225
|
data: DataType, window: int = 256, step: int = 64, window_size: int | None = None
|
|
224
|
-
) ->
|
|
226
|
+
) -> NDArray[np.float64]:
|
|
225
227
|
"""Calculate sliding window entropy profile.
|
|
226
228
|
|
|
227
229
|
: Shannon Entropy Analysis
|
|
@@ -563,7 +565,7 @@ def classify_by_entropy(data: DataType) -> EntropyResult:
|
|
|
563
565
|
)
|
|
564
566
|
|
|
565
567
|
|
|
566
|
-
def entropy_profile(data: DataType, window: int = 256) ->
|
|
568
|
+
def entropy_profile(data: DataType, window: int = 256) -> NDArray[np.float64]:
|
|
567
569
|
"""Generate entropy profile for visualization.
|
|
568
570
|
|
|
569
571
|
: Shannon Entropy Analysis
|
|
@@ -588,7 +590,7 @@ def entropy_profile(data: DataType, window: int = 256) -> "NDArray[np.float64]":
|
|
|
588
590
|
return sliding_entropy(data, window=window, step=step)
|
|
589
591
|
|
|
590
592
|
|
|
591
|
-
def entropy_histogram(data: DataType) -> tuple[
|
|
593
|
+
def entropy_histogram(data: DataType) -> tuple[NDArray[np.intp], NDArray[np.float64]]:
|
|
592
594
|
"""Generate byte frequency histogram.
|
|
593
595
|
|
|
594
596
|
: Shannon Entropy Analysis
|
|
@@ -799,7 +801,7 @@ def detect_frequency_anomalies(data: DataType, z_threshold: float = 3.0) -> Freq
|
|
|
799
801
|
|
|
800
802
|
def compare_byte_distributions(
|
|
801
803
|
data_a: DataType, data_b: DataType
|
|
802
|
-
) -> tuple[float, float,
|
|
804
|
+
) -> tuple[float, float, NDArray[np.float64]]:
|
|
803
805
|
"""Compare byte frequency distributions between two data samples.
|
|
804
806
|
|
|
805
807
|
Implements RE-ENT-002: Byte Frequency Distribution.
|
|
@@ -849,7 +851,7 @@ def compare_byte_distributions(
|
|
|
849
851
|
|
|
850
852
|
def sliding_byte_frequency(
|
|
851
853
|
data: DataType, window: int = 256, step: int = 64, byte_value: int | None = None
|
|
852
|
-
) ->
|
|
854
|
+
) -> NDArray[np.float64]:
|
|
853
855
|
"""Compute sliding window byte frequency profile.
|
|
854
856
|
|
|
855
857
|
Implements RE-ENT-002: Byte Frequency Distribution.
|
|
@@ -6,6 +6,8 @@ in binary data, useful for pattern identification, data characterization,
|
|
|
6
6
|
and protocol fingerprinting.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
9
11
|
from collections import Counter
|
|
10
12
|
from dataclasses import dataclass
|
|
11
13
|
from typing import TYPE_CHECKING, Any, Union
|
|
@@ -310,7 +312,7 @@ def find_unusual_ngrams(
|
|
|
310
312
|
return unusual
|
|
311
313
|
|
|
312
314
|
|
|
313
|
-
def ngram_heatmap(data: DataType, n: int = 2) ->
|
|
315
|
+
def ngram_heatmap(data: DataType, n: int = 2) -> NDArray[np.float64]:
|
|
314
316
|
"""Generate n-gram co-occurrence heatmap.
|
|
315
317
|
|
|
316
318
|
: N-gram Frequency Analysis
|
|
@@ -586,7 +588,7 @@ class NGramAnalyzer:
|
|
|
586
588
|
"""
|
|
587
589
|
return find_unusual_ngrams(data, baseline=baseline, n=self.n, z_threshold=z_threshold)
|
|
588
590
|
|
|
589
|
-
def heatmap(self, data: DataType) ->
|
|
591
|
+
def heatmap(self, data: DataType) -> NDArray[np.float64]:
|
|
590
592
|
"""Generate bigram heatmap.
|
|
591
593
|
|
|
592
594
|
Args:
|
oscura/api/fluent.py
CHANGED
|
@@ -7,7 +7,7 @@ expressing signal analysis operations in a readable, intuitive way.
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
from dataclasses import dataclass, field
|
|
10
|
-
from typing import TYPE_CHECKING, Any, TypeVar
|
|
10
|
+
from typing import TYPE_CHECKING, Any, Generic, TypeVar
|
|
11
11
|
|
|
12
12
|
import numpy as np
|
|
13
13
|
|
|
@@ -26,7 +26,7 @@ __all__ = [
|
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
@dataclass
|
|
29
|
-
class FluentResult[T]:
|
|
29
|
+
class FluentResult(Generic[T]):
|
|
30
30
|
"""Result container with fluent interface.
|
|
31
31
|
|
|
32
32
|
Provides method chaining for result processing.
|
oscura/automotive/__init__.py
CHANGED