codevanta 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codevanta/__init__.py +1 -0
- codevanta/analysis/__init__.py +1 -0
- codevanta/analysis/binary_diff.py +1168 -0
- codevanta/analysis/callgraph.py +542 -0
- codevanta/analysis/cfg.py +1201 -0
- codevanta/analysis/dataflow.py +762 -0
- codevanta/analysis/debugger.py +289 -0
- codevanta/analysis/debuginfo.py +304 -0
- codevanta/analysis/decompile.py +1094 -0
- codevanta/analysis/decompilers/__init__.py +250 -0
- codevanta/analysis/decompilers/angr_backend.py +251 -0
- codevanta/analysis/decompilers/base.py +278 -0
- codevanta/analysis/decompilers/ghidra_backend.py +316 -0
- codevanta/analysis/dynamic.py +756 -0
- codevanta/analysis/emulate.py +1349 -0
- codevanta/analysis/evidence_bundle.py +736 -0
- codevanta/analysis/function_hash.py +562 -0
- codevanta/analysis/graph_export.py +441 -0
- codevanta/analysis/html_view.py +695 -0
- codevanta/analysis/jump_tables.py +546 -0
- codevanta/analysis/solver.py +715 -0
- codevanta/analysis/symbolic.py +1809 -0
- codevanta/analysis/trace.py +646 -0
- codevanta/analysis/type_propagation.py +1055 -0
- codevanta/analysis/type_recovery.py +1214 -0
- codevanta/analysis/vtables.py +572 -0
- codevanta/analysis/xrefs.py +434 -0
- codevanta/api/__init__.py +1 -0
- codevanta/cli/__init__.py +1 -0
- codevanta/cli/main.py +3038 -0
- codevanta/core/__init__.py +31 -0
- codevanta/core/disassembler.py +121 -0
- codevanta/core/entropy.py +237 -0
- codevanta/core/functions.py +176 -0
- codevanta/core/loader.py +1351 -0
- codevanta/core/report.py +306 -0
- codevanta/core/strings.py +109 -0
- codevanta/evidence/__init__.py +7 -0
- codevanta/evidence/confidence.py +36 -0
- codevanta/evidence/models.py +43 -0
- codevanta/evidence/provenance.py +37 -0
- codevanta/gui/__init__.py +46 -0
- codevanta/gui/layout.py +1384 -0
- codevanta/gui/server.py +1305 -0
- codevanta/ir/__init__.py +6 -0
- codevanta/ir/adapters/__init__.py +31 -0
- codevanta/ir/base.py +31 -0
- codevanta/ir/instruction.py +42 -0
- codevanta/mcp_server/__init__.py +5 -0
- codevanta/mcp_server/server.py +710 -0
- codevanta/mcp_server/tools.py +1364 -0
- codevanta/plugins/__init__.py +748 -0
- codevanta/setup/__init__.py +28 -0
- codevanta/setup/apply.py +22 -0
- codevanta/setup/clients.py +285 -0
- codevanta/setup/plan.py +79 -0
- codevanta/setup/verify.py +43 -0
- codevanta/storage/__init__.py +1 -0
- codevanta/storage/session.py +514 -0
- codevanta-0.1.0.dist-info/METADATA +688 -0
- codevanta-0.1.0.dist-info/RECORD +65 -0
- codevanta-0.1.0.dist-info/WHEEL +5 -0
- codevanta-0.1.0.dist-info/entry_points.txt +2 -0
- codevanta-0.1.0.dist-info/licenses/LICENSE +21 -0
- codevanta-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,736 @@
|
|
|
1
|
+
"""Self-contained, tamper-evident evidence bundles for one analysis subject.
|
|
2
|
+
|
|
3
|
+
An analysis result is only useful to somebody else if that somebody can check
|
|
4
|
+
it. A bundle therefore pins the *input* and every *output* in one JSON
|
|
5
|
+
document:
|
|
6
|
+
|
|
7
|
+
* ``manifest.subject`` / ``manifest.subject_sha256`` identify the binary that
|
|
8
|
+
was analysed and its SHA-256 digest, so a verifier can re-hash the file and
|
|
9
|
+
confirm they are looking at the same bytes.
|
|
10
|
+
* ``entries`` holds one record per analysis payload (``cfg``, ``callgraph``,
|
|
11
|
+
``strings``, ...) with the payload inlined next to its own digest and size.
|
|
12
|
+
* ``manifest.entries_sha256`` is a single digest over the whole entry table, so
|
|
13
|
+
adding, removing or renaming an entry -- or editing one entry's recorded
|
|
14
|
+
digest, size or media type -- is detectable as one mismatch.
|
|
15
|
+
|
|
16
|
+
Canonical form
|
|
17
|
+
--------------
|
|
18
|
+
|
|
19
|
+
Every payload is reduced to *canonical JSON* before it is hashed:
|
|
20
|
+
|
|
21
|
+
.. code-block:: python
|
|
22
|
+
|
|
23
|
+
json.dumps(payload, sort_keys=True, separators=(",", ":"),
|
|
24
|
+
ensure_ascii=True, allow_nan=False)
|
|
25
|
+
|
|
26
|
+
The result is a byte string that does not depend on the producer's key
|
|
27
|
+
insertion order, indentation or locale, which is what makes
|
|
28
|
+
|
|
29
|
+
.. code-block:: python
|
|
30
|
+
|
|
31
|
+
hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest()
|
|
32
|
+
|
|
33
|
+
reproducible by a third party who only has the bundle. ``allow_nan=False``
|
|
34
|
+
means ``NaN``/``Infinity`` are rejected rather than written as Python-only
|
|
35
|
+
tokens, so the document stays parseable by strict JSON readers.
|
|
36
|
+
|
|
37
|
+
The entry-table digest is the SHA-256 of canonical JSON of a list of
|
|
38
|
+
``[name, sha256, bytes, media_type]`` rows sorted by ``name``::
|
|
39
|
+
|
|
40
|
+
[[name, sha256_hex, payload_bytes, media_type], ...]
|
|
41
|
+
|
|
42
|
+
so it can be recomputed in any language, not just Python
|
|
43
|
+
(:func:`entries_digest` exposes the same recipe).
|
|
44
|
+
|
|
45
|
+
What a bundle does and does not prove
|
|
46
|
+
-------------------------------------
|
|
47
|
+
|
|
48
|
+
It proves **integrity**: :func:`verify_bundle` recomputes every payload digest
|
|
49
|
+
and the entry-table digest, so any edit -- one flipped character inside a
|
|
50
|
+
payload, an edited ``sha256``, an edited ``entries_sha256``, a dropped entry --
|
|
51
|
+
is reported with the entry name and the expected/actual values. Two builds
|
|
52
|
+
from the same inputs serialise to identical bytes apart from ``created_at``.
|
|
53
|
+
|
|
54
|
+
It does **not** prove **authenticity**: there is no signature and no secret, so
|
|
55
|
+
anybody who can rewrite the whole document can also recompute every digest.
|
|
56
|
+
``subject`` is a caller-supplied label and is *not* trusted input; only
|
|
57
|
+
``subject_sha256`` binds the bundle to real bytes. Treat a bundle as evidence
|
|
58
|
+
about a file, not as a statement about who produced it.
|
|
59
|
+
|
|
60
|
+
Bounds and failure modes
|
|
61
|
+
------------------------
|
|
62
|
+
|
|
63
|
+
* at most :data:`MAX_ENTRIES` (256) entries;
|
|
64
|
+
* at most :data:`MAX_PAYLOAD_BYTES` (16 MiB) of canonical JSON per payload;
|
|
65
|
+
* the subject must exist and be a readable regular file;
|
|
66
|
+
* payloads that are not JSON-serialisable are rejected.
|
|
67
|
+
|
|
68
|
+
Every rejection raises :class:`~codevanta.core.report.AnalysisError`; a bundle
|
|
69
|
+
is never written partially. Only the standard library is used.
|
|
70
|
+
|
|
71
|
+
Reading a bundle back
|
|
72
|
+
---------------------
|
|
73
|
+
|
|
74
|
+
The file written by :func:`write_bundle` is exactly :meth:`Bundle.to_dict`
|
|
75
|
+
serialised with ``sort_keys=True``, ``indent=2`` and ``ensure_ascii=True``, so
|
|
76
|
+
it diffs and reviews well. :func:`read_bundle` rejects a structurally broken
|
|
77
|
+
document (missing payload, malformed digest) with
|
|
78
|
+
:class:`AnalysisError`; for forensic checks on a possibly damaged file, parse
|
|
79
|
+
the JSON and hand the mapping to :func:`verify_bundle`, which reports damage
|
|
80
|
+
instead of raising.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
from __future__ import annotations
|
|
84
|
+
|
|
85
|
+
import hashlib
|
|
86
|
+
import json
|
|
87
|
+
import re
|
|
88
|
+
from collections.abc import Mapping
|
|
89
|
+
from dataclasses import dataclass, field
|
|
90
|
+
from datetime import datetime, timezone
|
|
91
|
+
from pathlib import Path
|
|
92
|
+
from types import MappingProxyType
|
|
93
|
+
from typing import Any
|
|
94
|
+
|
|
95
|
+
from codevanta.core.report import AnalysisError
|
|
96
|
+
|
|
97
|
+
__all__ = [
|
|
98
|
+
"DEFAULT_ALGORITHM",
|
|
99
|
+
"MAX_ENTRIES",
|
|
100
|
+
"MAX_PAYLOAD_BYTES",
|
|
101
|
+
"MEDIA_TYPE_JSON",
|
|
102
|
+
"SUBJECT_CHUNK_BYTES",
|
|
103
|
+
"SUPPORTED_ALGORITHMS",
|
|
104
|
+
"TOOL_VERSION",
|
|
105
|
+
"Bundle",
|
|
106
|
+
"BundleManifest",
|
|
107
|
+
"BundleVerification",
|
|
108
|
+
"build_bundle",
|
|
109
|
+
"canonical_json",
|
|
110
|
+
"dumps_bundle",
|
|
111
|
+
"entries_digest",
|
|
112
|
+
"hash_subject",
|
|
113
|
+
"read_bundle",
|
|
114
|
+
"verify_bundle",
|
|
115
|
+
"write_bundle",
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
#: Digest recorded in :attr:`BundleManifest.algorithm` and used everywhere here.
|
|
119
|
+
DEFAULT_ALGORITHM = "sha256"
|
|
120
|
+
|
|
121
|
+
#: Algorithms :func:`verify_bundle` can recompute. Anything else is a problem,
|
|
122
|
+
#: never a silent pass.
|
|
123
|
+
SUPPORTED_ALGORITHMS = frozenset({DEFAULT_ALGORITHM})
|
|
124
|
+
|
|
125
|
+
#: Version string stamped into every manifest. Mirrors
|
|
126
|
+
#: :data:`codevanta.mcp_server.server.SERVER_VERSION`.
|
|
127
|
+
TOOL_VERSION = "0.1.0"
|
|
128
|
+
|
|
129
|
+
#: Media type recorded for a canonically serialised JSON payload.
|
|
130
|
+
MEDIA_TYPE_JSON = "application/json"
|
|
131
|
+
|
|
132
|
+
#: Hard ceiling on entries in one bundle (bounded work for the verifier).
|
|
133
|
+
MAX_ENTRIES = 256
|
|
134
|
+
|
|
135
|
+
#: Hard ceiling on one payload's canonical JSON, in bytes.
|
|
136
|
+
MAX_PAYLOAD_BYTES = 16 * 1024 * 1024
|
|
137
|
+
|
|
138
|
+
#: Read size used when hashing the subject binary: the file is streamed, never
|
|
139
|
+
#: loaded whole.
|
|
140
|
+
SUBJECT_CHUNK_BYTES = 1024 * 1024
|
|
141
|
+
|
|
142
|
+
#: Shape of a SHA-256 hex digest as stored in a bundle.
|
|
143
|
+
_SHA256_HEX = re.compile(r"[0-9a-f]{64}")
|
|
144
|
+
|
|
145
|
+
#: Manifest keys that must be non-blank strings.
|
|
146
|
+
_MANIFEST_TEXT_KEYS = (
|
|
147
|
+
"created_at",
|
|
148
|
+
"tool_version",
|
|
149
|
+
"subject",
|
|
150
|
+
"subject_sha256",
|
|
151
|
+
"entries_sha256",
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _utc_now_iso() -> str:
|
|
156
|
+
"""Return the current UTC time as an ISO-8601 string."""
|
|
157
|
+
return datetime.now(timezone.utc).isoformat()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _sha256_hex(data: bytes) -> str:
|
|
161
|
+
"""Return the lowercase SHA-256 hex digest of ``data``."""
|
|
162
|
+
return hashlib.new(DEFAULT_ALGORITHM, data).hexdigest()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def canonical_json(payload: Any) -> str:
|
|
166
|
+
"""Serialise ``payload`` to the canonical JSON text that gets hashed.
|
|
167
|
+
|
|
168
|
+
Key order, indentation, non-ASCII escaping and float formatting are all
|
|
169
|
+
fixed, so equal payloads always produce equal bytes and therefore equal
|
|
170
|
+
digests.
|
|
171
|
+
|
|
172
|
+
Args:
|
|
173
|
+
payload: Any JSON-serialisable value.
|
|
174
|
+
|
|
175
|
+
Returns:
|
|
176
|
+
The canonical JSON text (always ASCII, no newlines).
|
|
177
|
+
|
|
178
|
+
Raises:
|
|
179
|
+
AnalysisError: When ``payload`` is not JSON-serialisable or contains a
|
|
180
|
+
``NaN``/``Infinity`` value.
|
|
181
|
+
"""
|
|
182
|
+
try:
|
|
183
|
+
return json.dumps(
|
|
184
|
+
payload,
|
|
185
|
+
sort_keys=True,
|
|
186
|
+
separators=(",", ":"),
|
|
187
|
+
ensure_ascii=True,
|
|
188
|
+
allow_nan=False,
|
|
189
|
+
)
|
|
190
|
+
except (TypeError, ValueError) as exc:
|
|
191
|
+
raise AnalysisError(f"payload is not canonical JSON: {exc}") from exc
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def entries_digest(entries: Mapping[str, Mapping[str, Any]]) -> str:
|
|
195
|
+
"""Return the combined digest over an entry table.
|
|
196
|
+
|
|
197
|
+
The recipe is the SHA-256 of canonical JSON of
|
|
198
|
+
``[[name, sha256, bytes, media_type], ...]`` sorted by ``name``; it is
|
|
199
|
+
documented in the module docstring so non-Python verifiers can reproduce
|
|
200
|
+
it.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
entries: Mapping of entry name to a record carrying ``sha256``,
|
|
204
|
+
``bytes`` and ``media_type``.
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
The lowercase hex digest of the whole table.
|
|
208
|
+
|
|
209
|
+
Raises:
|
|
210
|
+
AnalysisError: When a record is missing or carries malformed metadata.
|
|
211
|
+
"""
|
|
212
|
+
table: list[list[Any]] = []
|
|
213
|
+
for name in sorted(entries):
|
|
214
|
+
record = entries[name]
|
|
215
|
+
digest = record.get("sha256")
|
|
216
|
+
size = record.get("bytes")
|
|
217
|
+
media_type = record.get("media_type")
|
|
218
|
+
if not isinstance(digest, str) or not isinstance(size, int) or isinstance(size, bool):
|
|
219
|
+
raise AnalysisError(f"entry {name!r} has malformed digest metadata")
|
|
220
|
+
if not isinstance(media_type, str):
|
|
221
|
+
raise AnalysisError(f"entry {name!r} has a malformed media_type")
|
|
222
|
+
table.append([name, digest, size, media_type])
|
|
223
|
+
return _sha256_hex(canonical_json(table).encode("utf-8"))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def hash_subject(subject_path: str | Path) -> str:
|
|
227
|
+
"""Stream ``subject_path`` through SHA-256 and return the hex digest.
|
|
228
|
+
|
|
229
|
+
The file is read in :data:`SUBJECT_CHUNK_BYTES` chunks, so hashing a large
|
|
230
|
+
binary never loads it into memory.
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
subject_path: Path to the analysed binary.
|
|
234
|
+
|
|
235
|
+
Returns:
|
|
236
|
+
The lowercase hex digest of the file's bytes.
|
|
237
|
+
|
|
238
|
+
Raises:
|
|
239
|
+
AnalysisError: When the path does not exist, is not a regular file or
|
|
240
|
+
cannot be read.
|
|
241
|
+
"""
|
|
242
|
+
target = Path(subject_path)
|
|
243
|
+
try:
|
|
244
|
+
exists = target.exists()
|
|
245
|
+
is_file = target.is_file()
|
|
246
|
+
except OSError as exc: # pragma: no cover - exotic filesystem errors only
|
|
247
|
+
raise AnalysisError(f"cannot inspect subject binary {target}: {exc}") from exc
|
|
248
|
+
if not exists:
|
|
249
|
+
raise AnalysisError(f"subject binary not found: {target}")
|
|
250
|
+
if not is_file:
|
|
251
|
+
raise AnalysisError(f"subject binary is not a regular file: {target}")
|
|
252
|
+
|
|
253
|
+
digest = hashlib.new(DEFAULT_ALGORITHM)
|
|
254
|
+
try:
|
|
255
|
+
with target.open("rb") as handle:
|
|
256
|
+
for chunk in iter(lambda: handle.read(SUBJECT_CHUNK_BYTES), b""):
|
|
257
|
+
digest.update(chunk)
|
|
258
|
+
except OSError as exc:
|
|
259
|
+
raise AnalysisError(f"cannot read subject binary {target}: {exc}") from exc
|
|
260
|
+
return digest.hexdigest()
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
@dataclass(frozen=True)
|
|
265
|
+
class BundleManifest:
|
|
266
|
+
"""What a bundle covers: the subject, its digest and the entry-table digest.
|
|
267
|
+
|
|
268
|
+
Attributes:
|
|
269
|
+
created_at: ISO-8601 UTC timestamp of the build.
|
|
270
|
+
tool_version: CODEVANTA version that produced the bundle.
|
|
271
|
+
subject: Human-readable label for the analysed binary -- the file name by
|
|
272
|
+
default, or the caller's ``label``. Never trusted input.
|
|
273
|
+
subject_sha256: Digest of the analysed bytes; this is what binds the
|
|
274
|
+
bundle to a real file.
|
|
275
|
+
entry_count: Number of entries in the bundle.
|
|
276
|
+
entries_sha256: Combined digest over the whole entry table.
|
|
277
|
+
algorithm: Digest algorithm name. Only ``"sha256"`` is supported.
|
|
278
|
+
"""
|
|
279
|
+
|
|
280
|
+
created_at: str
|
|
281
|
+
tool_version: str
|
|
282
|
+
subject: str
|
|
283
|
+
subject_sha256: str
|
|
284
|
+
entry_count: int
|
|
285
|
+
entries_sha256: str
|
|
286
|
+
algorithm: str = DEFAULT_ALGORITHM
|
|
287
|
+
|
|
288
|
+
def to_dict(self) -> dict[str, Any]:
|
|
289
|
+
"""Serialise the manifest as a JSON-safe dict."""
|
|
290
|
+
return {
|
|
291
|
+
"algorithm": self.algorithm,
|
|
292
|
+
"created_at": self.created_at,
|
|
293
|
+
"entries_sha256": self.entries_sha256,
|
|
294
|
+
"entry_count": self.entry_count,
|
|
295
|
+
"subject": self.subject,
|
|
296
|
+
"subject_sha256": self.subject_sha256,
|
|
297
|
+
"tool_version": self.tool_version,
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
@dataclass(frozen=True)
|
|
302
|
+
class Bundle:
|
|
303
|
+
"""A subject digest plus the hashed payloads that describe it.
|
|
304
|
+
|
|
305
|
+
Attributes:
|
|
306
|
+
manifest: The bundle's :class:`BundleManifest`.
|
|
307
|
+
entries: Read-only mapping of entry name to ``{"sha256", "bytes",
|
|
308
|
+
"media_type"}``, sorted by name. The payload itself is not part of
|
|
309
|
+
this mapping; use :meth:`payload`.
|
|
310
|
+
|
|
311
|
+
Instances are frozen and the mappings are :class:`types.MappingProxyType`,
|
|
312
|
+
so a built bundle cannot be mutated behind the verifier's back.
|
|
313
|
+
"""
|
|
314
|
+
|
|
315
|
+
manifest: BundleManifest
|
|
316
|
+
entries: Mapping[str, Mapping[str, Any]]
|
|
317
|
+
_payload_texts: Mapping[str, str] = field(repr=False, compare=False)
|
|
318
|
+
|
|
319
|
+
def __post_init__(self) -> None:
|
|
320
|
+
records = {
|
|
321
|
+
name: MappingProxyType(dict(self.entries[name])) for name in sorted(self.entries)
|
|
322
|
+
}
|
|
323
|
+
object.__setattr__(self, "entries", MappingProxyType(records))
|
|
324
|
+
object.__setattr__(self, "_payload_texts", MappingProxyType(dict(self._payload_texts)))
|
|
325
|
+
|
|
326
|
+
@property
|
|
327
|
+
def names(self) -> tuple[str, ...]:
|
|
328
|
+
"""Entry names in the deterministic (sorted) order."""
|
|
329
|
+
return tuple(sorted(self.entries))
|
|
330
|
+
|
|
331
|
+
def payload(self, name: str) -> Any:
|
|
332
|
+
"""Return the payload stored under ``name``.
|
|
333
|
+
|
|
334
|
+
The value is decoded from the exact canonical text that was hashed, so
|
|
335
|
+
the returned object always reproduces the recorded digest.
|
|
336
|
+
|
|
337
|
+
Raises:
|
|
338
|
+
AnalysisError: When the bundle has no such entry.
|
|
339
|
+
"""
|
|
340
|
+
text = self._payload_texts.get(name)
|
|
341
|
+
if text is None:
|
|
342
|
+
raise AnalysisError(f"bundle has no entry {name!r}")
|
|
343
|
+
return json.loads(text)
|
|
344
|
+
|
|
345
|
+
def to_dict(self) -> dict[str, Any]:
|
|
346
|
+
"""Serialise the bundle, inlining every payload next to its digest.
|
|
347
|
+
|
|
348
|
+
The returned dict is JSON-safe and is exactly what
|
|
349
|
+
:func:`write_bundle` persists.
|
|
350
|
+
"""
|
|
351
|
+
entries: dict[str, dict[str, Any]] = {}
|
|
352
|
+
for name in self.names:
|
|
353
|
+
record = dict(self.entries[name])
|
|
354
|
+
record["payload"] = self.payload(name)
|
|
355
|
+
entries[name] = record
|
|
356
|
+
return {"manifest": self.manifest.to_dict(), "entries": entries}
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
@dataclass(frozen=True)
|
|
360
|
+
class BundleVerification:
|
|
361
|
+
"""Outcome of :func:`verify_bundle`.
|
|
362
|
+
|
|
363
|
+
Attributes:
|
|
364
|
+
passed: ``True`` only when no problem was found.
|
|
365
|
+
problems: Human-readable findings, each naming the entry or manifest
|
|
366
|
+
field that failed and the expected/actual values.
|
|
367
|
+
manifest: The manifest that was checked; fields that could not be parsed
|
|
368
|
+
are reported as empty/zero rather than raised.
|
|
369
|
+
"""
|
|
370
|
+
|
|
371
|
+
passed: bool
|
|
372
|
+
problems: tuple[str, ...]
|
|
373
|
+
manifest: BundleManifest
|
|
374
|
+
|
|
375
|
+
def to_dict(self) -> dict[str, Any]:
|
|
376
|
+
"""Serialise the verification outcome as a JSON-safe dict."""
|
|
377
|
+
return {
|
|
378
|
+
"passed": self.passed,
|
|
379
|
+
"problems": list(self.problems),
|
|
380
|
+
"manifest": self.manifest.to_dict(),
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
@dataclass(frozen=True)
|
|
385
|
+
class _ParsedDocument:
|
|
386
|
+
"""Structural view of a bundle document, with damage recorded in problems."""
|
|
387
|
+
|
|
388
|
+
manifest: BundleManifest
|
|
389
|
+
entries: dict[str, dict[str, Any]]
|
|
390
|
+
payloads: dict[str, str]
|
|
391
|
+
names: tuple[str, ...]
|
|
392
|
+
problems: tuple[str, ...]
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _entry_metadata(
|
|
397
|
+
name: str, record: Mapping[str, Any], problems: list[str]
|
|
398
|
+
) -> dict[str, Any] | None:
|
|
399
|
+
"""Extract ``{"sha256", "bytes", "media_type"}``, or report why not."""
|
|
400
|
+
digest = record.get("sha256")
|
|
401
|
+
if not isinstance(digest, str) or _SHA256_HEX.fullmatch(digest) is None:
|
|
402
|
+
problems.append(f"entry {name!r}: sha256 is missing or malformed ({digest!r})")
|
|
403
|
+
return None
|
|
404
|
+
size = record.get("bytes")
|
|
405
|
+
if not isinstance(size, int) or isinstance(size, bool) or size < 0:
|
|
406
|
+
problems.append(f"entry {name!r}: byte length is missing or malformed ({size!r})")
|
|
407
|
+
return None
|
|
408
|
+
media_type = record.get("media_type")
|
|
409
|
+
if not isinstance(media_type, str) or not media_type:
|
|
410
|
+
problems.append(f"entry {name!r}: media_type is missing or malformed ({media_type!r})")
|
|
411
|
+
return None
|
|
412
|
+
return {"sha256": digest, "bytes": size, "media_type": media_type}
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _parse_manifest(raw: Any, problems: list[str]) -> BundleManifest:
|
|
416
|
+
"""Parse a manifest, recording damage instead of raising."""
|
|
417
|
+
if not isinstance(raw, Mapping):
|
|
418
|
+
problems.append("manifest: missing or not a JSON object")
|
|
419
|
+
return BundleManifest(
|
|
420
|
+
created_at="",
|
|
421
|
+
tool_version="",
|
|
422
|
+
subject="",
|
|
423
|
+
subject_sha256="",
|
|
424
|
+
entry_count=0,
|
|
425
|
+
entries_sha256="",
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
fields: dict[str, str] = {}
|
|
429
|
+
for key in _MANIFEST_TEXT_KEYS:
|
|
430
|
+
value = raw.get(key)
|
|
431
|
+
if not isinstance(value, str) or not value:
|
|
432
|
+
problems.append(f"manifest: {key} is missing or not a non-empty string")
|
|
433
|
+
value = ""
|
|
434
|
+
fields[key] = value
|
|
435
|
+
for key in ("subject_sha256", "entries_sha256"):
|
|
436
|
+
if fields[key] and _SHA256_HEX.fullmatch(fields[key]) is None:
|
|
437
|
+
problems.append(f"manifest: {key} is not a 64-character lowercase hex digest")
|
|
438
|
+
|
|
439
|
+
count = raw.get("entry_count")
|
|
440
|
+
if not isinstance(count, int) or isinstance(count, bool) or count < 0:
|
|
441
|
+
problems.append(f"manifest: entry_count is missing or malformed ({count!r})")
|
|
442
|
+
count = 0
|
|
443
|
+
|
|
444
|
+
algorithm = raw.get("algorithm")
|
|
445
|
+
if not isinstance(algorithm, str) or not algorithm:
|
|
446
|
+
problems.append(f"manifest: algorithm is missing or malformed ({algorithm!r})")
|
|
447
|
+
algorithm = DEFAULT_ALGORITHM
|
|
448
|
+
elif algorithm not in SUPPORTED_ALGORITHMS:
|
|
449
|
+
problems.append(f"manifest: unsupported algorithm {algorithm!r}")
|
|
450
|
+
|
|
451
|
+
return BundleManifest(
|
|
452
|
+
created_at=fields["created_at"],
|
|
453
|
+
tool_version=fields["tool_version"],
|
|
454
|
+
subject=fields["subject"],
|
|
455
|
+
subject_sha256=fields["subject_sha256"],
|
|
456
|
+
entry_count=count,
|
|
457
|
+
entries_sha256=fields["entries_sha256"],
|
|
458
|
+
algorithm=algorithm,
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _parse_document(document: Mapping[str, Any]) -> _ParsedDocument:
|
|
463
|
+
"""Parse a bundle document into metadata, canonical payloads and problems."""
|
|
464
|
+
problems: list[str] = []
|
|
465
|
+
manifest = _parse_manifest(document.get("manifest"), problems)
|
|
466
|
+
|
|
467
|
+
raw_entries = document.get("entries")
|
|
468
|
+
if not isinstance(raw_entries, Mapping):
|
|
469
|
+
problems.append("bundle: entries is missing or not a JSON object")
|
|
470
|
+
raw_entries = {}
|
|
471
|
+
|
|
472
|
+
names: list[str] = []
|
|
473
|
+
entries: dict[str, dict[str, Any]] = {}
|
|
474
|
+
payloads: dict[str, str] = {}
|
|
475
|
+
for name in sorted(raw_entries, key=str):
|
|
476
|
+
if not isinstance(name, str) or not name.strip():
|
|
477
|
+
problems.append(f"entry {name!r}: name must be a non-empty string")
|
|
478
|
+
continue
|
|
479
|
+
names.append(name)
|
|
480
|
+
record = raw_entries[name]
|
|
481
|
+
if not isinstance(record, Mapping):
|
|
482
|
+
problems.append(f"entry {name!r}: record is not a JSON object")
|
|
483
|
+
continue
|
|
484
|
+
metadata = _entry_metadata(name, record, problems)
|
|
485
|
+
if metadata is None:
|
|
486
|
+
continue
|
|
487
|
+
entries[name] = metadata
|
|
488
|
+
if "payload" not in record:
|
|
489
|
+
problems.append(f"entry {name!r}: payload is missing")
|
|
490
|
+
continue
|
|
491
|
+
try:
|
|
492
|
+
payloads[name] = canonical_json(record["payload"])
|
|
493
|
+
except AnalysisError as exc:
|
|
494
|
+
problems.append(f"entry {name!r}: {exc}")
|
|
495
|
+
|
|
496
|
+
return _ParsedDocument(
|
|
497
|
+
manifest=manifest,
|
|
498
|
+
entries=entries,
|
|
499
|
+
payloads=payloads,
|
|
500
|
+
names=tuple(names),
|
|
501
|
+
problems=tuple(problems),
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def build_bundle(
|
|
508
|
+
subject_path: str | Path,
|
|
509
|
+
payloads: Mapping[str, Any],
|
|
510
|
+
*,
|
|
511
|
+
label: str | None = None,
|
|
512
|
+
) -> Bundle:
|
|
513
|
+
"""Hash a binary and a set of analysis payloads into one bundle.
|
|
514
|
+
|
|
515
|
+
Entries are sorted by name, so two builds from the same inputs serialise to
|
|
516
|
+
identical bytes regardless of the ``payloads`` insertion order.
|
|
517
|
+
|
|
518
|
+
Args:
|
|
519
|
+
subject_path: The analysed binary; it must exist and be readable.
|
|
520
|
+
payloads: Mapping of entry name to any JSON-serialisable payload.
|
|
521
|
+
label: Optional subject label. Defaults to the binary's file name.
|
|
522
|
+
|
|
523
|
+
Returns:
|
|
524
|
+
The populated :class:`Bundle`.
|
|
525
|
+
|
|
526
|
+
Raises:
|
|
527
|
+
AnalysisError: When the subject is missing/unreadable, when ``payloads``
|
|
528
|
+
is not a mapping, when a name is blank, when there are more than
|
|
529
|
+
:data:`MAX_ENTRIES` entries, or when a payload is not
|
|
530
|
+
JSON-serialisable or exceeds :data:`MAX_PAYLOAD_BYTES`.
|
|
531
|
+
"""
|
|
532
|
+
subject_sha256 = hash_subject(subject_path)
|
|
533
|
+
|
|
534
|
+
if not isinstance(payloads, Mapping):
|
|
535
|
+
raise AnalysisError(f"payloads must be a mapping, got {type(payloads).__name__}")
|
|
536
|
+
if len(payloads) > MAX_ENTRIES:
|
|
537
|
+
raise AnalysisError(f"too many entries ({len(payloads)} > limit {MAX_ENTRIES})")
|
|
538
|
+
|
|
539
|
+
subject = label if label is not None else Path(subject_path).name
|
|
540
|
+
if not isinstance(subject, str) or not subject.strip():
|
|
541
|
+
raise AnalysisError(f"subject label must be a non-empty string, got {subject!r}")
|
|
542
|
+
|
|
543
|
+
entries: dict[str, dict[str, Any]] = {}
|
|
544
|
+
payload_texts: dict[str, str] = {}
|
|
545
|
+
for name in sorted(payloads, key=str):
|
|
546
|
+
if not isinstance(name, str) or not name.strip():
|
|
547
|
+
raise AnalysisError(f"entry name must be a non-empty string, got {name!r}")
|
|
548
|
+
text = canonical_json(payloads[name])
|
|
549
|
+
data = text.encode("utf-8")
|
|
550
|
+
if len(data) > MAX_PAYLOAD_BYTES:
|
|
551
|
+
raise AnalysisError(
|
|
552
|
+
f"payload {name!r} is too large ({len(data)} bytes > limit {MAX_PAYLOAD_BYTES})"
|
|
553
|
+
)
|
|
554
|
+
payload_texts[name] = text
|
|
555
|
+
entries[name] = {
|
|
556
|
+
"sha256": _sha256_hex(data),
|
|
557
|
+
"bytes": len(data),
|
|
558
|
+
"media_type": MEDIA_TYPE_JSON,
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
manifest = BundleManifest(
|
|
562
|
+
created_at=_utc_now_iso(),
|
|
563
|
+
tool_version=TOOL_VERSION,
|
|
564
|
+
subject=subject,
|
|
565
|
+
subject_sha256=subject_sha256,
|
|
566
|
+
entry_count=len(entries),
|
|
567
|
+
entries_sha256=entries_digest(entries),
|
|
568
|
+
)
|
|
569
|
+
return Bundle(manifest=manifest, entries=entries, _payload_texts=payload_texts)
|
|
570
|
+
|
|
571
|
+
def verify_bundle(
|
|
572
|
+
bundle_or_dict: Bundle | Mapping[str, Any],
|
|
573
|
+
*,
|
|
574
|
+
subject_path: str | Path | None = None,
|
|
575
|
+
) -> BundleVerification:
|
|
576
|
+
"""Recompute every digest in a bundle and report each mismatch.
|
|
577
|
+
|
|
578
|
+
Two independent checks run:
|
|
579
|
+
|
|
580
|
+
1. per entry -- the inlined payload is re-canonicalised, re-hashed and
|
|
581
|
+
re-measured, then compared with the recorded ``sha256`` and ``bytes``;
|
|
582
|
+
2. per table -- :func:`entries_digest` over the recorded entry table is
|
|
583
|
+
compared with ``manifest.entries_sha256``, which catches added, removed,
|
|
584
|
+
renamed or re-metadata'ed entries, including an edited ``sha256`` whose
|
|
585
|
+
payload was left alone.
|
|
586
|
+
|
|
587
|
+
``manifest.entry_count`` and ``manifest.algorithm`` are checked too, and
|
|
588
|
+
structurally damaged input is reported as a problem rather than raised.
|
|
589
|
+
|
|
590
|
+
Args:
|
|
591
|
+
bundle_or_dict: A :class:`Bundle` or the mapping returned by
|
|
592
|
+
:meth:`Bundle.to_dict` (or parsed from a written file).
|
|
593
|
+
subject_path: Optional path to the analysed binary. When given, its
|
|
594
|
+
digest is recomputed and compared with ``manifest.subject_sha256``,
|
|
595
|
+
which is the only way to confirm the bundle describes *this* file.
|
|
596
|
+
|
|
597
|
+
Returns:
|
|
598
|
+
A :class:`BundleVerification` whose ``problems`` name the failing entry
|
|
599
|
+
and the expected/actual values.
|
|
600
|
+
|
|
601
|
+
Raises:
|
|
602
|
+
AnalysisError: When ``bundle_or_dict`` is neither a bundle nor a
|
|
603
|
+
mapping, or when ``subject_path`` cannot be read.
|
|
604
|
+
"""
|
|
605
|
+
if isinstance(bundle_or_dict, Bundle):
|
|
606
|
+
document: Mapping[str, Any] = bundle_or_dict.to_dict()
|
|
607
|
+
elif isinstance(bundle_or_dict, Mapping):
|
|
608
|
+
document = bundle_or_dict
|
|
609
|
+
else:
|
|
610
|
+
raise AnalysisError(
|
|
611
|
+
f"bundle must be a Bundle or a mapping, got {type(bundle_or_dict).__name__}"
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
parsed = _parse_document(document)
|
|
615
|
+
problems: list[str] = list(parsed.problems)
|
|
616
|
+
|
|
617
|
+
for name in sorted(parsed.payloads):
|
|
618
|
+
data = parsed.payloads[name].encode("utf-8")
|
|
619
|
+
recorded = parsed.entries[name]
|
|
620
|
+
actual_digest = _sha256_hex(data)
|
|
621
|
+
if actual_digest != recorded["sha256"]:
|
|
622
|
+
problems.append(
|
|
623
|
+
f"entry {name!r}: payload sha256 mismatch "
|
|
624
|
+
f"(expected {recorded['sha256']}, actual {actual_digest})"
|
|
625
|
+
)
|
|
626
|
+
if len(data) != recorded["bytes"]:
|
|
627
|
+
problems.append(
|
|
628
|
+
f"entry {name!r}: payload size mismatch "
|
|
629
|
+
f"(expected {recorded['bytes']}, actual {len(data)})"
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
if parsed.manifest.entry_count != len(parsed.names):
|
|
633
|
+
problems.append(
|
|
634
|
+
f"manifest: entry_count mismatch "
|
|
635
|
+
f"(expected {parsed.manifest.entry_count}, actual {len(parsed.names)})"
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
actual_entries = entries_digest(parsed.entries)
|
|
639
|
+
if actual_entries != parsed.manifest.entries_sha256:
|
|
640
|
+
problems.append(
|
|
641
|
+
f"manifest: entries_sha256 mismatch "
|
|
642
|
+
f"(expected {parsed.manifest.entries_sha256}, actual {actual_entries})"
|
|
643
|
+
)
|
|
644
|
+
|
|
645
|
+
if subject_path is not None:
|
|
646
|
+
actual_subject = hash_subject(subject_path)
|
|
647
|
+
if actual_subject != parsed.manifest.subject_sha256:
|
|
648
|
+
problems.append(
|
|
649
|
+
f"subject {str(subject_path)!r}: sha256 mismatch "
|
|
650
|
+
f"(expected {parsed.manifest.subject_sha256}, actual {actual_subject})"
|
|
651
|
+
)
|
|
652
|
+
|
|
653
|
+
return BundleVerification(
|
|
654
|
+
passed=not problems,
|
|
655
|
+
problems=tuple(problems),
|
|
656
|
+
manifest=parsed.manifest,
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def dumps_bundle(bundle: Bundle) -> str:
|
|
661
|
+
"""Return the canonical JSON text of ``bundle``, as written to disk.
|
|
662
|
+
|
|
663
|
+
Keys are sorted, indentation is two spaces, non-ASCII is escaped and the
|
|
664
|
+
text ends with a single newline, so two bundles with the same content
|
|
665
|
+
serialise to identical bytes.
|
|
666
|
+
"""
|
|
667
|
+
return json.dumps(bundle.to_dict(), sort_keys=True, indent=2, ensure_ascii=True) + "\n"
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def write_bundle(bundle: Bundle, path: str | Path) -> Path:
|
|
671
|
+
"""Write ``bundle`` to ``path`` as canonical JSON.
|
|
672
|
+
|
|
673
|
+
Args:
|
|
674
|
+
bundle: The bundle to persist.
|
|
675
|
+
path: Destination file; parent directories must already exist.
|
|
676
|
+
|
|
677
|
+
Returns:
|
|
678
|
+
The destination path.
|
|
679
|
+
|
|
680
|
+
Raises:
|
|
681
|
+
AnalysisError: When the file cannot be written.
|
|
682
|
+
"""
|
|
683
|
+
target = Path(path)
|
|
684
|
+
try:
|
|
685
|
+
target.write_text(dumps_bundle(bundle), encoding="utf-8", newline="\n")
|
|
686
|
+
except OSError as exc:
|
|
687
|
+
raise AnalysisError(f"cannot write bundle {target}: {exc}") from exc
|
|
688
|
+
return target
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def _load_document(path: str | Path) -> Mapping[str, Any]:
|
|
692
|
+
"""Read and parse a bundle file, raising ``AnalysisError`` on damage."""
|
|
693
|
+
target = Path(path)
|
|
694
|
+
try:
|
|
695
|
+
with target.open(encoding="utf-8") as handle:
|
|
696
|
+
text = handle.read()
|
|
697
|
+
except UnicodeDecodeError as exc:
|
|
698
|
+
raise AnalysisError(f"bundle {target} is not UTF-8 text: {exc}") from exc
|
|
699
|
+
except OSError as exc:
|
|
700
|
+
raise AnalysisError(f"cannot read bundle {target}: {exc}") from exc
|
|
701
|
+
try:
|
|
702
|
+
document = json.loads(text)
|
|
703
|
+
except json.JSONDecodeError as exc:
|
|
704
|
+
raise AnalysisError(f"bundle {target} is not valid JSON: {exc}") from exc
|
|
705
|
+
if not isinstance(document, Mapping):
|
|
706
|
+
raise AnalysisError(f"bundle {target} is not a JSON object")
|
|
707
|
+
return document
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def read_bundle(path: str | Path) -> Bundle:
|
|
711
|
+
"""Load a bundle previously written by :func:`write_bundle`.
|
|
712
|
+
|
|
713
|
+
Structure is checked so the returned bundle is usable; digests are *not*
|
|
714
|
+
checked here -- that is :func:`verify_bundle`'s job, and keeping the two
|
|
715
|
+
separate lets a caller round-trip and then re-verify.
|
|
716
|
+
|
|
717
|
+
Args:
|
|
718
|
+
path: Bundle file to read.
|
|
719
|
+
|
|
720
|
+
Returns:
|
|
721
|
+
The :class:`Bundle` described by the file.
|
|
722
|
+
|
|
723
|
+
Raises:
|
|
724
|
+
AnalysisError: When the file cannot be read, is not JSON, or is
|
|
725
|
+
structurally damaged (missing manifest fields, missing payload,
|
|
726
|
+
malformed digest).
|
|
727
|
+
"""
|
|
728
|
+
document = _load_document(path)
|
|
729
|
+
parsed = _parse_document(document)
|
|
730
|
+
if parsed.problems:
|
|
731
|
+
raise AnalysisError(f"invalid bundle {path}: {'; '.join(parsed.problems)}")
|
|
732
|
+
return Bundle(
|
|
733
|
+
manifest=parsed.manifest,
|
|
734
|
+
entries=parsed.entries,
|
|
735
|
+
_payload_texts=parsed.payloads,
|
|
736
|
+
)
|