lupaxa-magic-bytes 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lupaxa/__init__.py +1 -0
- lupaxa/magic_bytes/__init__.py +5 -0
- lupaxa/magic_bytes/__main__.py +6 -0
- lupaxa/magic_bytes/analyse.py +276 -0
- lupaxa/magic_bytes/cli.py +928 -0
- lupaxa/magic_bytes/compare.py +117 -0
- lupaxa/magic_bytes/config.py +153 -0
- lupaxa/magic_bytes/errors.py +75 -0
- lupaxa/magic_bytes/formats.py +1302 -0
- lupaxa/magic_bytes/hexview.py +96 -0
- lupaxa/magic_bytes/htmlreport.py +79 -0
- lupaxa/magic_bytes/io.py +173 -0
- lupaxa/magic_bytes/models.py +191 -0
- lupaxa/magic_bytes/py.typed +0 -0
- lupaxa/magic_bytes/scan.py +160 -0
- lupaxa/magic_bytes/signatures.py +727 -0
- lupaxa/magic_bytes/version.py +8 -0
- lupaxa_magic_bytes-0.1.0.dist-info/METADATA +261 -0
- lupaxa_magic_bytes-0.1.0.dist-info/RECORD +22 -0
- lupaxa_magic_bytes-0.1.0.dist-info/WHEEL +4 -0
- lupaxa_magic_bytes-0.1.0.dist-info/entry_points.txt +2 -0
- lupaxa_magic_bytes-0.1.0.dist-info/licenses/LICENCE +21 -0
lupaxa/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Lupaxa namespace package."""
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
"""Identification, inspection, and carving decisions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from lupaxa.magic_bytes.config import Settings
|
|
9
|
+
from lupaxa.magic_bytes.errors import ArgumentError, ExtractionError, LimitError, NotFoundError
|
|
10
|
+
from lupaxa.magic_bytes.formats import VALIDATORS, validate_format
|
|
11
|
+
from lupaxa.magic_bytes.io import Blob
|
|
12
|
+
from lupaxa.magic_bytes.models import Budget, Candidate, Region, Validation
|
|
13
|
+
from lupaxa.magic_bytes.signatures import Signature, catalogue, evidence_lines, matches
|
|
14
|
+
|
|
15
|
+
RANK = {"confirmed": 5, "high": 4, "medium": 3, "low": 2, "unknown": 1}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def make_budget(settings: Settings, timeout: float | None = None) -> Budget:
|
|
19
|
+
"""Build a budget from settings and an optional CLI timeout."""
|
|
20
|
+
return Budget(
|
|
21
|
+
timeout_seconds=settings.timeout_seconds if timeout is None else timeout,
|
|
22
|
+
max_candidates=settings.max_candidates,
|
|
23
|
+
max_bytes=settings.max_scan_bytes,
|
|
24
|
+
max_depth=settings.max_depth,
|
|
25
|
+
max_decompressed=settings.max_decompressed_bytes,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def identify(
|
|
30
|
+
blob: Blob,
|
|
31
|
+
settings: Settings,
|
|
32
|
+
budget: Budget,
|
|
33
|
+
*,
|
|
34
|
+
level: str,
|
|
35
|
+
deep: bool = False,
|
|
36
|
+
signatures_only: bool = False,
|
|
37
|
+
base_offset: int | None = None,
|
|
38
|
+
strict: bool = False,
|
|
39
|
+
) -> list[Candidate]:
|
|
40
|
+
"""Return every relevant format candidate, strongest first."""
|
|
41
|
+
found: list[Candidate] = []
|
|
42
|
+
seen: set[tuple[str, int]] = set()
|
|
43
|
+
for signature in catalogue(settings):
|
|
44
|
+
for base in _bases(blob, signature, settings, budget, deep, base_offset):
|
|
45
|
+
key = (signature.id, base)
|
|
46
|
+
if key in seen:
|
|
47
|
+
continue
|
|
48
|
+
seen.add(key)
|
|
49
|
+
if not matches(blob, signature, base) and not _container_hit(blob, signature, base):
|
|
50
|
+
continue
|
|
51
|
+
try:
|
|
52
|
+
budget.take_candidate()
|
|
53
|
+
except LimitError:
|
|
54
|
+
break
|
|
55
|
+
found.append(_candidate(blob, signature, base, settings, budget, level, signatures_only, strict))
|
|
56
|
+
if budget.incomplete:
|
|
57
|
+
break
|
|
58
|
+
if not found:
|
|
59
|
+
return []
|
|
60
|
+
found.sort(key=lambda item: (-RANK[item.confidence], -_priority(item, settings), item.offset != 0, item.offset))
|
|
61
|
+
found[0].primary = True
|
|
62
|
+
return found
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _bases(
|
|
66
|
+
blob: Blob,
|
|
67
|
+
signature: Signature,
|
|
68
|
+
settings: Settings,
|
|
69
|
+
budget: Budget,
|
|
70
|
+
deep: bool,
|
|
71
|
+
forced: int | None,
|
|
72
|
+
) -> list[int]:
|
|
73
|
+
if forced is not None:
|
|
74
|
+
return [forced]
|
|
75
|
+
if not signature.needle:
|
|
76
|
+
return [0] if matches(blob, signature, 0) else []
|
|
77
|
+
if signature.id == "zip":
|
|
78
|
+
tail = blob.find(b"PK\x05\x06", max(0, blob.size - 65557), blob.size, limit=1)
|
|
79
|
+
if tail:
|
|
80
|
+
return [0]
|
|
81
|
+
if not deep and not signature.scan:
|
|
82
|
+
return [0] if matches(blob, signature, 0) else []
|
|
83
|
+
end = (
|
|
84
|
+
blob.size
|
|
85
|
+
if deep
|
|
86
|
+
else min(blob.size, max(settings.header_window, signature.needle_bias + len(signature.needle) + 64))
|
|
87
|
+
)
|
|
88
|
+
if deep:
|
|
89
|
+
end = min(end, settings.max_scan_bytes)
|
|
90
|
+
limit = 16 if not signature.scan else min(64, budget.max_candidates)
|
|
91
|
+
hits = blob.find(signature.needle, 0, end, limit=limit, alignment=max(1, signature.alignment))
|
|
92
|
+
bases = [hit - signature.needle_bias for hit in hits if hit >= signature.needle_bias]
|
|
93
|
+
if matches(blob, signature, 0) and 0 not in bases:
|
|
94
|
+
bases.insert(0, 0)
|
|
95
|
+
return list(dict.fromkeys(bases))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _container_hit(blob: Blob, signature: Signature, base: int) -> bool:
|
|
99
|
+
if signature.id == "zip" and base == 0:
|
|
100
|
+
return bool(blob.find(b"PK\x05\x06", max(0, blob.size - 65557), blob.size, limit=1))
|
|
101
|
+
if signature.id == "pdf":
|
|
102
|
+
return b"%PDF-" in blob.read_at(base, 1024)
|
|
103
|
+
return False
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _priority(candidate: Candidate, settings: Settings) -> int:
|
|
107
|
+
for signature in catalogue(settings):
|
|
108
|
+
if signature.id == candidate.format_id:
|
|
109
|
+
return signature.priority
|
|
110
|
+
return 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _candidate(
|
|
114
|
+
blob: Blob,
|
|
115
|
+
signature: Signature,
|
|
116
|
+
base: int,
|
|
117
|
+
settings: Settings,
|
|
118
|
+
budget: Budget,
|
|
119
|
+
level: str,
|
|
120
|
+
signatures_only: bool,
|
|
121
|
+
strict: bool,
|
|
122
|
+
) -> Candidate:
|
|
123
|
+
validation: Validation | None = None
|
|
124
|
+
if not signatures_only and signature.id in VALIDATORS:
|
|
125
|
+
try:
|
|
126
|
+
validation = validate_format(signature.id, blob, base, level, budget, strict=strict)
|
|
127
|
+
except LimitError:
|
|
128
|
+
budget.incomplete = True
|
|
129
|
+
confidence = _confidence(signature, validation, signatures_only)
|
|
130
|
+
attributes = list(validation.attributes) if validation else ["signature-only"]
|
|
131
|
+
if validation is None:
|
|
132
|
+
attributes = ["signature-only"]
|
|
133
|
+
contradictions = []
|
|
134
|
+
if validation and validation.status == "failed":
|
|
135
|
+
contradictions = [item.detail for item in validation.checks if not item.passed and item.severity == "error"]
|
|
136
|
+
return Candidate(
|
|
137
|
+
format_id=signature.id,
|
|
138
|
+
name=signature.name,
|
|
139
|
+
mime=signature.mime,
|
|
140
|
+
extensions=list(signature.extensions),
|
|
141
|
+
offset=base,
|
|
142
|
+
confidence=confidence,
|
|
143
|
+
evidence=evidence_lines(blob, signature, base) or [f"{signature.name} signature"],
|
|
144
|
+
contradictions=contradictions,
|
|
145
|
+
validation=validation,
|
|
146
|
+
variant=validation.variant if validation else signature.variant,
|
|
147
|
+
warnings=list(validation.warnings) if validation else [],
|
|
148
|
+
attributes=attributes,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _confidence(signature: Signature, validation: Validation | None, signatures_only: bool) -> str:
|
|
153
|
+
if validation and validation.status == "passed":
|
|
154
|
+
return "confirmed"
|
|
155
|
+
if validation and validation.status == "partial":
|
|
156
|
+
return "high"
|
|
157
|
+
if signatures_only or validation is None:
|
|
158
|
+
return "high" if signature.strength == "strong" else signature.strength
|
|
159
|
+
if signature.strength == "strong":
|
|
160
|
+
return "high"
|
|
161
|
+
return "medium" if signature.strength == "medium" else "low"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def extension_note(path: str, candidate: Candidate | None) -> str | None:
|
|
165
|
+
"""Describe an extension mismatch. A mismatch is an observation, not malice."""
|
|
166
|
+
if candidate is None:
|
|
167
|
+
return None
|
|
168
|
+
suffix = Path(path).suffix.lower()
|
|
169
|
+
if not suffix:
|
|
170
|
+
return "The filename has no extension."
|
|
171
|
+
if suffix not in candidate.extensions:
|
|
172
|
+
expected = ", ".join(candidate.extensions) or "none"
|
|
173
|
+
return f"Extension {suffix} is not a usual extension for {candidate.name} ({expected})."
|
|
174
|
+
return None
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def trailing_region(blob: Blob, candidate: Candidate | None) -> Region | None:
|
|
178
|
+
"""Return bytes after a validator's logical end, when one was established."""
|
|
179
|
+
if candidate is None or candidate.validation is None or candidate.validation.logical_end is None:
|
|
180
|
+
return None
|
|
181
|
+
end = candidate.validation.logical_end
|
|
182
|
+
if end >= blob.size:
|
|
183
|
+
return None
|
|
184
|
+
kind = "overlay" if candidate.format_id == "pe" else "trailing"
|
|
185
|
+
description = (
|
|
186
|
+
"PE overlay is a normal place for appended data." if kind == "overlay" else "Bytes outside the logical file."
|
|
187
|
+
)
|
|
188
|
+
return Region(end, blob.size, kind, kind, description, candidate.format_id, "passed", [])
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def shannon(blob: Blob, start: int, length: int) -> float:
|
|
192
|
+
"""Shannon entropy in bits per byte, from 0 to 8."""
|
|
193
|
+
if length < 0 or start < 0:
|
|
194
|
+
raise ArgumentError("Entropy ranges must be zero or positive.")
|
|
195
|
+
counts = [0] * 256
|
|
196
|
+
remain = min(length, max(0, blob.size - start))
|
|
197
|
+
offset = start
|
|
198
|
+
total = 0
|
|
199
|
+
while remain:
|
|
200
|
+
chunk = blob.read_at(offset, min(1024 * 1024, remain))
|
|
201
|
+
if not chunk:
|
|
202
|
+
break
|
|
203
|
+
for byte in chunk:
|
|
204
|
+
counts[byte] += 1
|
|
205
|
+
total += len(chunk)
|
|
206
|
+
offset += len(chunk)
|
|
207
|
+
remain -= len(chunk)
|
|
208
|
+
if total == 0:
|
|
209
|
+
return 0.0
|
|
210
|
+
entropy = 0.0
|
|
211
|
+
for count in counts:
|
|
212
|
+
if count:
|
|
213
|
+
probability = count / total
|
|
214
|
+
entropy -= probability * math.log2(probability)
|
|
215
|
+
return entropy
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def block_entropy(blob: Blob, start: int, length: int, block_size: int) -> list[tuple[int, float]]:
|
|
219
|
+
"""Entropy of successive fixed blocks."""
|
|
220
|
+
if block_size < 1:
|
|
221
|
+
raise ArgumentError("Block size must be positive.")
|
|
222
|
+
rows: list[tuple[int, float]] = []
|
|
223
|
+
offset = start
|
|
224
|
+
end = min(blob.size, start + length)
|
|
225
|
+
while offset < end:
|
|
226
|
+
size = min(block_size, end - offset)
|
|
227
|
+
rows.append((offset, shannon(blob, offset, size)))
|
|
228
|
+
offset += size
|
|
229
|
+
return rows
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def select_candidate(candidates: list[Candidate], format_id: str | None) -> Candidate:
|
|
233
|
+
"""Choose the requested format, or the primary candidate."""
|
|
234
|
+
if format_id:
|
|
235
|
+
for candidate in candidates:
|
|
236
|
+
if candidate.format_id == format_id:
|
|
237
|
+
return candidate
|
|
238
|
+
raise NotFoundError(f"No {format_id} candidate was found.")
|
|
239
|
+
if not candidates:
|
|
240
|
+
raise NotFoundError("No recognised format.")
|
|
241
|
+
return next(item for item in candidates if item.primary)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def carving_objects(candidates: list[Candidate], blob: Blob) -> list[dict[str, object]]:
|
|
245
|
+
"""List objects that can be copied out without running them."""
|
|
246
|
+
objects: list[dict[str, object]] = []
|
|
247
|
+
index = 1
|
|
248
|
+
for candidate in candidates:
|
|
249
|
+
validation = candidate.validation
|
|
250
|
+
if validation is None or validation.logical_end is None:
|
|
251
|
+
continue
|
|
252
|
+
length = validation.logical_end - candidate.offset
|
|
253
|
+
if length <= 0:
|
|
254
|
+
continue
|
|
255
|
+
boundary = "exact" if validation.boundary == "exact" and validation.status == "passed" else validation.boundary
|
|
256
|
+
objects.append(
|
|
257
|
+
{
|
|
258
|
+
"id": index,
|
|
259
|
+
"format": candidate.format_id,
|
|
260
|
+
"offset": candidate.offset,
|
|
261
|
+
"length": length,
|
|
262
|
+
"boundary": boundary,
|
|
263
|
+
"validated": validation.status == "passed" and validation.boundary == "exact",
|
|
264
|
+
"attributes": list(candidate.attributes),
|
|
265
|
+
}
|
|
266
|
+
)
|
|
267
|
+
index += 1
|
|
268
|
+
if blob.size and not objects:
|
|
269
|
+
return objects
|
|
270
|
+
return objects
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def require_extract_length(length: int, settings: Settings) -> None:
|
|
274
|
+
"""Reject an extraction that exceeds the configured limit."""
|
|
275
|
+
if length > settings.max_extract_size:
|
|
276
|
+
raise ExtractionError(f"Extraction of {length} bytes exceeds the limit.")
|