lupaxa-magic-bytes 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
lupaxa/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """Lupaxa namespace package."""
@@ -0,0 +1,5 @@
1
+ """Offline binary identification, validation, and carving."""
2
+
3
+ from lupaxa.magic_bytes.version import __version__, get_version
4
+
5
+ __all__ = ["__version__", "get_version"]
@@ -0,0 +1,6 @@
1
+ """python -m lupaxa.magic_bytes."""
2
+
3
+ from lupaxa.magic_bytes.cli import main
4
+
5
+ if __name__ == "__main__":
6
+ raise SystemExit(main())
@@ -0,0 +1,276 @@
1
+ """Identification, inspection, and carving decisions."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from pathlib import Path
7
+
8
+ from lupaxa.magic_bytes.config import Settings
9
+ from lupaxa.magic_bytes.errors import ArgumentError, ExtractionError, LimitError, NotFoundError
10
+ from lupaxa.magic_bytes.formats import VALIDATORS, validate_format
11
+ from lupaxa.magic_bytes.io import Blob
12
+ from lupaxa.magic_bytes.models import Budget, Candidate, Region, Validation
13
+ from lupaxa.magic_bytes.signatures import Signature, catalogue, evidence_lines, matches
14
+
15
+ RANK = {"confirmed": 5, "high": 4, "medium": 3, "low": 2, "unknown": 1}
16
+
17
+
18
+ def make_budget(settings: Settings, timeout: float | None = None) -> Budget:
19
+ """Build a budget from settings and an optional CLI timeout."""
20
+ return Budget(
21
+ timeout_seconds=settings.timeout_seconds if timeout is None else timeout,
22
+ max_candidates=settings.max_candidates,
23
+ max_bytes=settings.max_scan_bytes,
24
+ max_depth=settings.max_depth,
25
+ max_decompressed=settings.max_decompressed_bytes,
26
+ )
27
+
28
+
29
+ def identify(
30
+ blob: Blob,
31
+ settings: Settings,
32
+ budget: Budget,
33
+ *,
34
+ level: str,
35
+ deep: bool = False,
36
+ signatures_only: bool = False,
37
+ base_offset: int | None = None,
38
+ strict: bool = False,
39
+ ) -> list[Candidate]:
40
+ """Return every relevant format candidate, strongest first."""
41
+ found: list[Candidate] = []
42
+ seen: set[tuple[str, int]] = set()
43
+ for signature in catalogue(settings):
44
+ for base in _bases(blob, signature, settings, budget, deep, base_offset):
45
+ key = (signature.id, base)
46
+ if key in seen:
47
+ continue
48
+ seen.add(key)
49
+ if not matches(blob, signature, base) and not _container_hit(blob, signature, base):
50
+ continue
51
+ try:
52
+ budget.take_candidate()
53
+ except LimitError:
54
+ break
55
+ found.append(_candidate(blob, signature, base, settings, budget, level, signatures_only, strict))
56
+ if budget.incomplete:
57
+ break
58
+ if not found:
59
+ return []
60
+ found.sort(key=lambda item: (-RANK[item.confidence], -_priority(item, settings), item.offset != 0, item.offset))
61
+ found[0].primary = True
62
+ return found
63
+
64
+
65
+ def _bases(
66
+ blob: Blob,
67
+ signature: Signature,
68
+ settings: Settings,
69
+ budget: Budget,
70
+ deep: bool,
71
+ forced: int | None,
72
+ ) -> list[int]:
73
+ if forced is not None:
74
+ return [forced]
75
+ if not signature.needle:
76
+ return [0] if matches(blob, signature, 0) else []
77
+ if signature.id == "zip":
78
+ tail = blob.find(b"PK\x05\x06", max(0, blob.size - 65557), blob.size, limit=1)
79
+ if tail:
80
+ return [0]
81
+ if not deep and not signature.scan:
82
+ return [0] if matches(blob, signature, 0) else []
83
+ end = (
84
+ blob.size
85
+ if deep
86
+ else min(blob.size, max(settings.header_window, signature.needle_bias + len(signature.needle) + 64))
87
+ )
88
+ if deep:
89
+ end = min(end, settings.max_scan_bytes)
90
+ limit = 16 if not signature.scan else min(64, budget.max_candidates)
91
+ hits = blob.find(signature.needle, 0, end, limit=limit, alignment=max(1, signature.alignment))
92
+ bases = [hit - signature.needle_bias for hit in hits if hit >= signature.needle_bias]
93
+ if matches(blob, signature, 0) and 0 not in bases:
94
+ bases.insert(0, 0)
95
+ return list(dict.fromkeys(bases))
96
+
97
+
98
+ def _container_hit(blob: Blob, signature: Signature, base: int) -> bool:
99
+ if signature.id == "zip" and base == 0:
100
+ return bool(blob.find(b"PK\x05\x06", max(0, blob.size - 65557), blob.size, limit=1))
101
+ if signature.id == "pdf":
102
+ return b"%PDF-" in blob.read_at(base, 1024)
103
+ return False
104
+
105
+
106
+ def _priority(candidate: Candidate, settings: Settings) -> int:
107
+ for signature in catalogue(settings):
108
+ if signature.id == candidate.format_id:
109
+ return signature.priority
110
+ return 0
111
+
112
+
113
+ def _candidate(
114
+ blob: Blob,
115
+ signature: Signature,
116
+ base: int,
117
+ settings: Settings,
118
+ budget: Budget,
119
+ level: str,
120
+ signatures_only: bool,
121
+ strict: bool,
122
+ ) -> Candidate:
123
+ validation: Validation | None = None
124
+ if not signatures_only and signature.id in VALIDATORS:
125
+ try:
126
+ validation = validate_format(signature.id, blob, base, level, budget, strict=strict)
127
+ except LimitError:
128
+ budget.incomplete = True
129
+ confidence = _confidence(signature, validation, signatures_only)
130
+ attributes = list(validation.attributes) if validation else ["signature-only"]
131
+ if validation is None:
132
+ attributes = ["signature-only"]
133
+ contradictions = []
134
+ if validation and validation.status == "failed":
135
+ contradictions = [item.detail for item in validation.checks if not item.passed and item.severity == "error"]
136
+ return Candidate(
137
+ format_id=signature.id,
138
+ name=signature.name,
139
+ mime=signature.mime,
140
+ extensions=list(signature.extensions),
141
+ offset=base,
142
+ confidence=confidence,
143
+ evidence=evidence_lines(blob, signature, base) or [f"{signature.name} signature"],
144
+ contradictions=contradictions,
145
+ validation=validation,
146
+ variant=validation.variant if validation else signature.variant,
147
+ warnings=list(validation.warnings) if validation else [],
148
+ attributes=attributes,
149
+ )
150
+
151
+
152
+ def _confidence(signature: Signature, validation: Validation | None, signatures_only: bool) -> str:
153
+ if validation and validation.status == "passed":
154
+ return "confirmed"
155
+ if validation and validation.status == "partial":
156
+ return "high"
157
+ if signatures_only or validation is None:
158
+ return "high" if signature.strength == "strong" else signature.strength
159
+ if signature.strength == "strong":
160
+ return "high"
161
+ return "medium" if signature.strength == "medium" else "low"
162
+
163
+
164
+ def extension_note(path: str, candidate: Candidate | None) -> str | None:
165
+ """Describe an extension mismatch. A mismatch is an observation, not malice."""
166
+ if candidate is None:
167
+ return None
168
+ suffix = Path(path).suffix.lower()
169
+ if not suffix:
170
+ return "The filename has no extension."
171
+ if suffix not in candidate.extensions:
172
+ expected = ", ".join(candidate.extensions) or "none"
173
+ return f"Extension {suffix} is not a usual extension for {candidate.name} ({expected})."
174
+ return None
175
+
176
+
177
+ def trailing_region(blob: Blob, candidate: Candidate | None) -> Region | None:
178
+ """Return bytes after a validator's logical end, when one was established."""
179
+ if candidate is None or candidate.validation is None or candidate.validation.logical_end is None:
180
+ return None
181
+ end = candidate.validation.logical_end
182
+ if end >= blob.size:
183
+ return None
184
+ kind = "overlay" if candidate.format_id == "pe" else "trailing"
185
+ description = (
186
+ "PE overlay is a normal place for appended data." if kind == "overlay" else "Bytes outside the logical file."
187
+ )
188
+ return Region(end, blob.size, kind, kind, description, candidate.format_id, "passed", [])
189
+
190
+
191
+ def shannon(blob: Blob, start: int, length: int) -> float:
192
+ """Shannon entropy in bits per byte, from 0 to 8."""
193
+ if length < 0 or start < 0:
194
+ raise ArgumentError("Entropy ranges must be zero or positive.")
195
+ counts = [0] * 256
196
+ remain = min(length, max(0, blob.size - start))
197
+ offset = start
198
+ total = 0
199
+ while remain:
200
+ chunk = blob.read_at(offset, min(1024 * 1024, remain))
201
+ if not chunk:
202
+ break
203
+ for byte in chunk:
204
+ counts[byte] += 1
205
+ total += len(chunk)
206
+ offset += len(chunk)
207
+ remain -= len(chunk)
208
+ if total == 0:
209
+ return 0.0
210
+ entropy = 0.0
211
+ for count in counts:
212
+ if count:
213
+ probability = count / total
214
+ entropy -= probability * math.log2(probability)
215
+ return entropy
216
+
217
+
218
+ def block_entropy(blob: Blob, start: int, length: int, block_size: int) -> list[tuple[int, float]]:
219
+ """Entropy of successive fixed blocks."""
220
+ if block_size < 1:
221
+ raise ArgumentError("Block size must be positive.")
222
+ rows: list[tuple[int, float]] = []
223
+ offset = start
224
+ end = min(blob.size, start + length)
225
+ while offset < end:
226
+ size = min(block_size, end - offset)
227
+ rows.append((offset, shannon(blob, offset, size)))
228
+ offset += size
229
+ return rows
230
+
231
+
232
+ def select_candidate(candidates: list[Candidate], format_id: str | None) -> Candidate:
233
+ """Choose the requested format, or the primary candidate."""
234
+ if format_id:
235
+ for candidate in candidates:
236
+ if candidate.format_id == format_id:
237
+ return candidate
238
+ raise NotFoundError(f"No {format_id} candidate was found.")
239
+ if not candidates:
240
+ raise NotFoundError("No recognised format.")
241
+ return next(item for item in candidates if item.primary)
242
+
243
+
244
+ def carving_objects(candidates: list[Candidate], blob: Blob) -> list[dict[str, object]]:
245
+ """List objects that can be copied out without running them."""
246
+ objects: list[dict[str, object]] = []
247
+ index = 1
248
+ for candidate in candidates:
249
+ validation = candidate.validation
250
+ if validation is None or validation.logical_end is None:
251
+ continue
252
+ length = validation.logical_end - candidate.offset
253
+ if length <= 0:
254
+ continue
255
+ boundary = "exact" if validation.boundary == "exact" and validation.status == "passed" else validation.boundary
256
+ objects.append(
257
+ {
258
+ "id": index,
259
+ "format": candidate.format_id,
260
+ "offset": candidate.offset,
261
+ "length": length,
262
+ "boundary": boundary,
263
+ "validated": validation.status == "passed" and validation.boundary == "exact",
264
+ "attributes": list(candidate.attributes),
265
+ }
266
+ )
267
+ index += 1
268
+ if blob.size and not objects:
269
+ return objects
270
+ return objects
271
+
272
+
273
+ def require_extract_length(length: int, settings: Settings) -> None:
274
+ """Reject an extraction that exceeds the configured limit."""
275
+ if length > settings.max_extract_size:
276
+ raise ExtractionError(f"Extraction of {length} bytes exceeds the limit.")