scientific-method-engine 19.2.0__tar.gz → 19.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/PKG-INFO +1 -1
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/pyproject.toml +1 -1
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/cli.py +4 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/ranges.py +16 -10
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/reach.py +146 -29
- scientific_method_engine-19.3.0/src/scientific_method_engine/x86/table_bounds.py +810 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/trace.py +7 -3
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_ranges.py +11 -1
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_reach.py +162 -0
- scientific_method_engine-19.3.0/tests/test_table_bounds.py +389 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/.gitignore +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/LICENSE +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/README.md +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/inventory.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/machine.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/reports.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/oracle.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_config_encoding.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_enter_frames.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_indirect_far_transfers.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_interrupt_models.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_inventory_check.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_join_terms.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_return_checks.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_term_limit.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_upstream_gaps.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_volatile_memory.py +0 -0
- {scientific_method_engine-19.2.0 → scientific_method_engine-19.3.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 19.
|
|
3
|
+
Version: 19.3.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "19.
|
|
8
|
+
version = "19.3.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -27,6 +27,10 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
27
27
|
"a call to a noReturn routine, a noReturn call and a noReturn interrupt do not continue at the next\n"
|
|
28
28
|
"instruction, a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
|
|
29
29
|
"is reported contradicted, and a noReturn call lists its targets' return sites and is never contradicted;\n"
|
|
30
|
+
"stops cut this query's walk and claim nothing about the build: a stops routine is reached but not read\n"
|
|
31
|
+
"and a call to it does not continue at the next instruction, a stops site is decoded and nothing after it\n"
|
|
32
|
+
"is followed, and each is repeated with its reason and where it cut the walk, not checked or assumed;\n"
|
|
33
|
+
"the noReturn check reads past the stops, and returnCheckGaps lists what it could not read there;\n"
|
|
30
34
|
"indirectCalls declares the targets of a computed call as a table read from the build or a list,\n"
|
|
31
35
|
"and the walk calls each of them; one not declared exhaustive stays unresolved;\n"
|
|
32
36
|
"controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
|
|
@@ -9,9 +9,9 @@ operation without a transfer function has no range, and its output is unknown.
|
|
|
9
9
|
|
|
10
10
|
``Evaluation`` runs one instruction's p-code over ``RegisterRanges``. Memory is not modelled:
|
|
11
11
|
a ``LOAD`` or a direct ``ram`` operand yields an unknown value and a ``STORE`` changes no
|
|
12
|
-
register. Control flow, joins across paths, widening and branch refinement belong to the
|
|
13
|
-
analysis that walks the code (
|
|
14
|
-
|
|
12
|
+
register. Control flow, memory, joins across paths, widening and branch refinement belong to the
|
|
13
|
+
analysis that walks the code (``table_bounds.py``); ``Evaluation.execute`` stops at the first
|
|
14
|
+
control-flow operation and hands it back.
|
|
15
15
|
"""
|
|
16
16
|
from math import gcd
|
|
17
17
|
|
|
@@ -455,7 +455,8 @@ class RegisterRanges:
|
|
|
455
455
|
"""The ranges register varnodes hold, keyed by ``(offset, size)`` in the register space.
|
|
456
456
|
|
|
457
457
|
Held varnodes never overlap. A read of a varnode inside a held one takes its bytes; a read
|
|
458
|
-
that held varnodes
|
|
458
|
+
that covers held varnodes is pieced from them, with every byte no held varnode covers unknown;
|
|
459
|
+
a read that a held varnode only partly overlaps is unknown. A write keeps the
|
|
459
460
|
bytes of overlapped varnodes that it does not cover. x86 SLEIGH's register space is
|
|
460
461
|
little-endian, so byte ``k`` of a varnode holds its bits ``8k`` to ``8k + 7``.
|
|
461
462
|
"""
|
|
@@ -489,13 +490,18 @@ class RegisterRanges:
|
|
|
489
490
|
for (o, n), r in self.held.items():
|
|
490
491
|
if o <= offset and offset + size <= o + n:
|
|
491
492
|
return resize(shift_right(r, (offset - o) * 8), bits)
|
|
493
|
+
end = offset + size
|
|
494
|
+
inside = sorted((o, n) for (o, n) in self.held if o < end and offset < o + n)
|
|
495
|
+
if not inside or any(o < offset or o + n > end for o, n in inside):
|
|
496
|
+
return Range.top(bits)
|
|
497
|
+
# Held varnodes inside the read, with every byte between them unknown.
|
|
492
498
|
parts, at = [], offset
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
at
|
|
499
|
+
for o, n in inside + [(end, 0)]:
|
|
500
|
+
if at < o:
|
|
501
|
+
parts.append(Range.top((o - at) * 8))
|
|
502
|
+
if n:
|
|
503
|
+
parts.append(self.held[(o, n)])
|
|
504
|
+
at = o + n
|
|
499
505
|
result = parts[0]
|
|
500
506
|
for part in parts[1:]:
|
|
501
507
|
result = t_piece(result.bits + part.bits, part, result)
|
|
@@ -11,6 +11,9 @@ ROOT = -1
|
|
|
11
11
|
CALLS = ("call", "lcall")
|
|
12
12
|
UNSUPPORTED = "unsupported control-transfer frame encoding"
|
|
13
13
|
LEAF_OVERLAP = "leaf start inside a reached instruction; the leaf is not decoded, so its boundary is unchecked"
|
|
14
|
+
STOP_OVERLAP = ("stop routine start inside a reached instruction; the routine is not decoded, so its boundary is "
|
|
15
|
+
"unchecked")
|
|
16
|
+
CHECK_CONTESTED = "contested instruction in the noReturn return check's walk"
|
|
14
17
|
|
|
15
18
|
|
|
16
19
|
def _text(ins):
|
|
@@ -47,6 +50,42 @@ def _leaves(value, image, starts):
|
|
|
47
50
|
return leaves
|
|
48
51
|
|
|
49
52
|
|
|
53
|
+
def _stops(value, image, starts, leaves, no_return_routines):
|
|
54
|
+
"""``(routines, sites)``: the query's stop routines and stop sites, each mapped to its reason (ADR 0039)."""
|
|
55
|
+
if not isinstance(value, list) or len(value) > 256:
|
|
56
|
+
raise ValueError("stops must be a list of at most 256 objects")
|
|
57
|
+
routines, sites = {}, {}
|
|
58
|
+
for stop in value:
|
|
59
|
+
if not isinstance(stop, dict) or set(stop) not in ({"routine", "reason"}, {"site", "reason"}):
|
|
60
|
+
raise ValueError("Each stop needs reason and exactly one of routine and site")
|
|
61
|
+
kind = "routine" if "routine" in stop else "site"
|
|
62
|
+
at = integer(stop[kind], 0, len(image.data) - 1, f"stop {kind}")
|
|
63
|
+
if image.region(at) is None:
|
|
64
|
+
raise ValueError(f"stop {kind} {at} is outside declared code")
|
|
65
|
+
if not isinstance(stop["reason"], str) or not stop["reason"].strip():
|
|
66
|
+
raise ValueError("A stop needs a nonempty reason")
|
|
67
|
+
found = routines if kind == "routine" else sites
|
|
68
|
+
if at in found:
|
|
69
|
+
raise ValueError(f"Duplicate stop {kind}")
|
|
70
|
+
if at in starts:
|
|
71
|
+
raise ValueError(f"stop {kind} {at} is a start; the walk would read nothing past it")
|
|
72
|
+
if kind == "routine" and at in leaves:
|
|
73
|
+
raise ValueError(f"stop routine {at} is a leaf; a stop routine is not read either, so give it once, as a stop")
|
|
74
|
+
if kind == "routine" and at in no_return_routines:
|
|
75
|
+
raise ValueError(f"stop routine {at} is declared noReturn; the declaration is a claim about the build and "
|
|
76
|
+
"the stop a cut of this query, so give one of them")
|
|
77
|
+
if kind == "site" and at in leaves:
|
|
78
|
+
raise ValueError(f"stop site {at} is a leaf; the walk does not decode a leaf, so the stop would never be read")
|
|
79
|
+
if kind == "site" and image.decode(at) is None:
|
|
80
|
+
raise ValueError(f"stop site {at} does not decode")
|
|
81
|
+
found[at] = stop["reason"]
|
|
82
|
+
both = sorted(set(routines) & set(sites))
|
|
83
|
+
if both:
|
|
84
|
+
raise ValueError(f"stop {both[0]} is given as a routine and as a site; a stop routine is not decoded and a "
|
|
85
|
+
"stop site is, so give one of them")
|
|
86
|
+
return routines, sites
|
|
87
|
+
|
|
88
|
+
|
|
50
89
|
def no_return_declarations(value, image):
|
|
51
90
|
"""``(routines, interrupts, calls)``: the declared routines, interrupt sites and call sites that never return.
|
|
52
91
|
|
|
@@ -160,12 +199,20 @@ def successor_kinds(at, ins, step):
|
|
|
160
199
|
return rows
|
|
161
200
|
|
|
162
201
|
|
|
163
|
-
def successor_graph(image, seen, **step):
|
|
164
|
-
"""Each site of ``seen`` mapped to its ``successor_kinds`` rows that stay in ``seen
|
|
165
|
-
|
|
202
|
+
def successor_graph(image, seen, kept=frozenset(), **step):
|
|
203
|
+
"""Each site of ``seen`` mapped to its ``successor_kinds`` rows that stay in ``seen`` or lead to ``kept``,
|
|
204
|
+
under ``cfg_step``'s ``step`` options."""
|
|
205
|
+
return {at: [(s, kind) for s, kind in successor_kinds(at, ins, cfg_step(image, at, ins, **step))
|
|
206
|
+
if s in seen or s in kept]
|
|
166
207
|
for at, ins in seen.items()}
|
|
167
208
|
|
|
168
209
|
|
|
210
|
+
def _call_site_rows(sites, seen):
|
|
211
|
+
"""Each call site in ``sites`` as ``{ "site", "following", "followingRead" }``, from its instruction in ``seen``."""
|
|
212
|
+
return [{"site": site, "following": site + seen[site].size, "followingRead": site + seen[site].size in seen}
|
|
213
|
+
for site in sites]
|
|
214
|
+
|
|
215
|
+
|
|
169
216
|
def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, read, calls, call_targets):
|
|
170
217
|
"""The ``noReturn`` report rows of the declared ``routines``, ``calls`` and ``interrupts``.
|
|
171
218
|
|
|
@@ -179,9 +226,7 @@ def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, rea
|
|
|
179
226
|
"""
|
|
180
227
|
rows = [{"routine": at, "reason": reason, "reached": at in reached, "read": at in read,
|
|
181
228
|
"returnSites": returns[at], "contradicted": bool(returns[at]),
|
|
182
|
-
"callSites":
|
|
183
|
-
"followingRead": site + seen[site].size in seen}
|
|
184
|
-
for site in call_sites.get(at, [])]}
|
|
229
|
+
"callSites": _call_site_rows(call_sites.get(at, []), seen)}
|
|
185
230
|
for at, reason in routines.items()]
|
|
186
231
|
rows += [{"call": at, "reason": row["reason"], "reached": at in seen,
|
|
187
232
|
"targets": [{"routine": target, "read": target in read, "returnSites": returns.get(target, [])}
|
|
@@ -276,7 +321,10 @@ def reach(image, config):
|
|
|
276
321
|
interrupt to the next instruction. A ``leaves`` routine is reached but never decoded. A call to a
|
|
277
322
|
``noReturn`` routine, a ``noReturn`` call site and a ``noReturn`` interrupt site do not continue
|
|
278
323
|
at the next instruction, nor does a declared indirect call that is exhaustive and whose targets
|
|
279
|
-
are all ``noReturn``.
|
|
324
|
+
are all ``noReturn``. A ``stops`` routine is reached but never decoded, and a call to it does not
|
|
325
|
+
continue at the next instruction, under the same rule as a call to a ``noReturn`` routine; a
|
|
326
|
+
``stops`` site is decoded and nothing after it is followed. A stop narrows the query and claims
|
|
327
|
+
nothing about the build, so it is neither checked nor listed among the assumptions.
|
|
280
328
|
Each reached target gets one chain with the fewest calls, the assumptions its route rests on,
|
|
281
329
|
and the routines on every route to it within the read graph. With ``decodedRanges`` the report
|
|
282
330
|
also gives every instruction the walk decoded, as ``decoded_ranges`` groups them.
|
|
@@ -316,14 +364,27 @@ def reach(image, config):
|
|
|
316
364
|
instruction_limit = config.get("instructionLimit", 10000)
|
|
317
365
|
no_return_routines, no_return_interrupts, no_return_sites, declared = no_return_inputs(
|
|
318
366
|
config.get("noReturn", []), image, indirect_calls)
|
|
367
|
+
stop_routines, stop_sites = _stops(config.get("stops", []), image, set(starts), leaves, no_return_routines)
|
|
368
|
+
# A call-site control must be resolved, and the walk follows nothing at a stop site.
|
|
369
|
+
for at in controls:
|
|
370
|
+
if at in stop_sites:
|
|
371
|
+
raise ValueError(f"control {at} is a stop site, whose call the walk does not follow; give it as an "
|
|
372
|
+
"instructionControls entry")
|
|
373
|
+
# On the walk's terms a stop routine is a leaf that never returns; the report keeps the two apart.
|
|
374
|
+
unread = frozenset(leaves) | frozenset(stop_routines)
|
|
375
|
+
options = declared | {"no_return_calls": declared["no_return_calls"] | frozenset(stop_routines)}
|
|
319
376
|
seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
|
|
320
|
-
stops=frozenset(
|
|
377
|
+
stops=unread, ends=frozenset(stop_sites), **options)
|
|
321
378
|
|
|
322
379
|
# Each reached call site with its resolved or declared targets.
|
|
323
|
-
graph, unresolved, interrupts, call_targets = {}, [], [], {}
|
|
380
|
+
graph, unresolved, interrupts, call_targets, stop_successors = {}, [], [], {}, {}
|
|
324
381
|
for at, ins in sorted(seen.items()):
|
|
325
|
-
step = cfg_step(image, at, ins, follow_interrupts=True, **
|
|
326
|
-
|
|
382
|
+
step = cfg_step(image, at, ins, follow_interrupts=True, **options)
|
|
383
|
+
if at in stop_sites:
|
|
384
|
+
# Nothing at a stop site is followed, so its transfer is neither resolved nor unresolved.
|
|
385
|
+
graph[at], stop_successors[at] = [], step.successors
|
|
386
|
+
continue
|
|
387
|
+
graph[at] = [(s, kind) for s, kind in successor_kinds(at, ins, step) if s in seen or s in unread]
|
|
327
388
|
m = base_mnemonic(ins)
|
|
328
389
|
text = _text(ins)
|
|
329
390
|
kind = "call" if m in CALLS else "return" if m in RETURNS else "jump"
|
|
@@ -338,7 +399,7 @@ def reach(image, config):
|
|
|
338
399
|
if m in INTERRUPTS and at not in no_return_interrupts:
|
|
339
400
|
vector, conditional = interrupt_vector(image.flat, ins, at)
|
|
340
401
|
interrupts.append({"site": at, "instruction": text, "vector": vector, "conditional": conditional})
|
|
341
|
-
for at in
|
|
402
|
+
for at in unread:
|
|
342
403
|
graph.setdefault(at, [])
|
|
343
404
|
|
|
344
405
|
distance, parent, routine = fewest_calls(graph, starts)
|
|
@@ -406,7 +467,8 @@ def reach(image, config):
|
|
|
406
467
|
if at in routine_starts:
|
|
407
468
|
cut.append(at)
|
|
408
469
|
at = idom[at]
|
|
409
|
-
row |= {"reached": True, "routine": routine[target], "leaf": target in leaves,
|
|
470
|
+
row |= {"reached": True, "routine": routine[target], "leaf": target in leaves,
|
|
471
|
+
"stop": target in stop_routines or target in stop_sites, **chain(target),
|
|
410
472
|
"throughEveryRoute": cut[::-1]}
|
|
411
473
|
else:
|
|
412
474
|
row |= {"reached": False, **unreached(target)}
|
|
@@ -419,14 +481,22 @@ def reach(image, config):
|
|
|
419
481
|
# The walk never decodes a leaf, so its overlap check cannot see a leaf start inside a decoded instruction.
|
|
420
482
|
gaps += [{"site": at, "reason": LEAF_OVERLAP, "insideInstruction": holding_instruction(seen, at)}
|
|
421
483
|
for at in sorted(leaves) if at in distance and holding_instruction(seen, at) is not None]
|
|
484
|
+
gaps += [{"site": at, "reason": STOP_OVERLAP, "insideInstruction": holding_instruction(seen, at)}
|
|
485
|
+
for at in sorted(stop_routines) if at in distance and holding_instruction(seen, at) is not None]
|
|
422
486
|
# Kept apart from the gap rows, which the result limit can cut.
|
|
423
487
|
stopped = any(g["reason"] == LIMIT_REASON for g in walk_gaps)
|
|
424
488
|
# A control at a site a noReturn declaration kept the walk from names that declaration, which may be the cause.
|
|
425
489
|
def ends(at, found):
|
|
426
|
-
return call_ends(at, found[0], indirect_calls,
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
490
|
+
return call_ends(at, found[0], indirect_calls, options["no_return_calls"])
|
|
491
|
+
|
|
492
|
+
def ended_by(at, found):
|
|
493
|
+
parts = []
|
|
494
|
+
for label, named_routines in (("noReturn", no_return_routines), ("stop", stop_routines)):
|
|
495
|
+
named = [str(target) for target in found if target in named_routines]
|
|
496
|
+
if named:
|
|
497
|
+
parts.append(f"{label} routine{'s' if len(named) > 1 else ''} " + ", ".join(named))
|
|
498
|
+
return f"the call at {at} to " + " and ".join(parts)
|
|
499
|
+
cut_by = {at + seen[at].size: ended_by(at, found) for at, found in sorted(call_targets.items()) if ends(at, found)}
|
|
430
500
|
# A call to a noReturn routine can be declared as a call too; the message then names both declarations.
|
|
431
501
|
for at, row in no_return_sites.items():
|
|
432
502
|
if at in seen:
|
|
@@ -434,6 +504,10 @@ def reach(image, config):
|
|
|
434
504
|
cut_by[row["following"]] = f"{cut_by[row['following']]} and {cause}" if row["following"] in cut_by else cause
|
|
435
505
|
cut_by |= {row["following"]: f"the noReturn interrupt at {at}"
|
|
436
506
|
for at, row in no_return_interrupts.items() if at in seen}
|
|
507
|
+
# A site past a stop site may be reached by no other route; the first stop names the cut.
|
|
508
|
+
for at, found in sorted(stop_successors.items()):
|
|
509
|
+
for successor in found:
|
|
510
|
+
cut_by.setdefault(successor, f"the stop at {at}")
|
|
437
511
|
# Not reached and reached-but-unresolved need different fixes, so each failure says which it is.
|
|
438
512
|
failures = []
|
|
439
513
|
for label, sites, passed in (("control", controls, call_targets), ("instruction control", instruction_controls, seen)):
|
|
@@ -446,6 +520,9 @@ def reach(image, config):
|
|
|
446
520
|
failures.append(f"{label} {at} is reached but its call target is unresolved ({'; '.join(reasons)})")
|
|
447
521
|
elif at in leaves and at in distance:
|
|
448
522
|
failures.append(f"{label} {at} is the start of a leaf, which the walk reaches but does not decode")
|
|
523
|
+
elif at in stop_routines and at in distance:
|
|
524
|
+
failures.append(f"{label} {at} is the start of a stop routine, which the walk reaches but does not "
|
|
525
|
+
"decode")
|
|
449
526
|
else:
|
|
450
527
|
state = unreached(at)
|
|
451
528
|
detail = (f"inside the reached instruction at {state['insideInstruction']}" if "insideInstruction" in state
|
|
@@ -466,19 +543,49 @@ def reach(image, config):
|
|
|
466
543
|
ended_sites.setdefault(target, []).append(site)
|
|
467
544
|
leaf_rows = [{"routine": at, "reason": reason, "reached": at in distance, "callSites": call_sites.get(at, [])}
|
|
468
545
|
for at, reason in leaves.items()]
|
|
546
|
+
stop_rows = [{"routine": at, "reason": reason, "reached": at in distance,
|
|
547
|
+
"callSites": _call_site_rows(call_sites.get(at, []), seen)}
|
|
548
|
+
for at, reason in stop_routines.items()]
|
|
549
|
+
for at, reason in stop_sites.items():
|
|
550
|
+
ins = image.decode(at)
|
|
551
|
+
found = stop_successors[at] if at in seen else cfg_step(image, at, ins, follow_interrupts=True,
|
|
552
|
+
**options).successors
|
|
553
|
+
stop_rows.append({"site": at, "reason": reason, "reached": at in seen, "instruction": _text(ins),
|
|
554
|
+
"routine": routine.get(at), "successors": [{"site": s, "read": s in seen} for s in found]})
|
|
469
555
|
# Each call the declaration kept from its return site, so a reviewer can check what follows it.
|
|
470
556
|
returning_leaves = set(leaves) - set(no_return_routines)
|
|
471
557
|
site_targets = {at: call_targets.get(at, []) for at in no_return_sites}
|
|
472
558
|
checked = set(no_return_routines) | {target for found in site_targets.values() for target in found}
|
|
473
|
-
|
|
474
|
-
|
|
559
|
+
check_graph, read, check_stopped, check_gap_rows = graph, seen, False, []
|
|
560
|
+
# A stop the walk never met cut nothing, so the walk read what a walk without stops reads.
|
|
561
|
+
stop_met = (any(at in distance for at in stop_routines)
|
|
562
|
+
or any(at in seen or at in contested or at in overlapping for at in stop_sites))
|
|
563
|
+
if stop_met and checked:
|
|
564
|
+
# A stop cuts the query and says nothing about the build, so the return check reads the reached
|
|
565
|
+
# declared routines without the stops: code a stop hid can still contradict a declaration.
|
|
566
|
+
read, check_gaps, _, _, check_contested = walk(image, sorted(at for at in checked if at in distance),
|
|
567
|
+
instruction_limit, follow_interrupts=True,
|
|
568
|
+
stops=frozenset(leaves), **declared)
|
|
569
|
+
check_stopped = any(g["reason"] == LIMIT_REASON for g in check_gaps)
|
|
570
|
+
# What the check could not read past a stop leaves its declarations unchecked there, as a gap of the
|
|
571
|
+
# query's own walk would; the rows the query's walk already holds are not repeated.
|
|
572
|
+
known = {(g["site"], g["reason"]) for g in walk_gaps}
|
|
573
|
+
check_gap_rows = sorted(
|
|
574
|
+
[g for g in check_gaps if g["reason"] != LIMIT_REASON and (g["site"], g["reason"]) not in known
|
|
575
|
+
and g["site"] not in check_contested]
|
|
576
|
+
+ [{"site": at, "reason": CHECK_CONTESTED} for at in check_contested if at not in contested],
|
|
577
|
+
key=lambda g: (g["site"], g["reason"]))
|
|
578
|
+
check_graph = successor_graph(image, read, kept=frozenset(leaves), follow_interrupts=True, **declared)
|
|
579
|
+
returns = {at: returns_from(check_graph, read, returning_leaves, at) for at in checked}
|
|
580
|
+
no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, ended_sites, seen, distance, read,
|
|
475
581
|
no_return_sites, site_targets)
|
|
476
582
|
contradicted = any(row.get("contradicted") for row in no_return)
|
|
477
583
|
# A declared target the walk entered but established no instruction at shows in gaps or contested too,
|
|
478
584
|
# unless the walk stopped at its instruction limit before reading it.
|
|
479
585
|
indirect_rows = [declaration | {"reached": site in seen, "routine": routine.get(site),
|
|
480
586
|
"unreadTargets": [t for t in declaration["targets"]
|
|
481
|
-
if site in seen and
|
|
587
|
+
if site in seen and site not in stop_sites
|
|
588
|
+
and t not in seen and t not in unread]}
|
|
482
589
|
for site, declaration in indirect_calls.items()]
|
|
483
590
|
counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
|
|
484
591
|
"interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
|
|
@@ -492,20 +599,23 @@ def reach(image, config):
|
|
|
492
599
|
assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
|
|
493
600
|
"it gives, and only those when it is declared exhaustive")
|
|
494
601
|
assumptions += no_return_assumptions(no_return_routines, no_return_interrupts, no_return_sites)
|
|
602
|
+
stopped = stopped or check_stopped
|
|
495
603
|
# Every range is kept whatever the result limit, so a caller can place any site in the walk; the
|
|
496
604
|
# instruction limit bounds their number.
|
|
497
605
|
ranges = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
|
|
498
|
-
return {"starts": starts, "targets": rows, "leaves": leaf_rows, "
|
|
606
|
+
return {"starts": starts, "targets": rows, "leaves": leaf_rows, "stops": stop_rows, "noReturn": no_return,
|
|
607
|
+
"indirectCalls": indirect_rows,
|
|
499
608
|
"reachedRoutines": sorted(routine_starts), **ranges, "counts": counts,
|
|
500
609
|
"unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
|
|
501
|
-
"contested": sorted(contested)[:limit],
|
|
502
|
-
"truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested)),
|
|
610
|
+
"contested": sorted(contested)[:limit], "returnCheckGaps": check_gap_rows[:limit],
|
|
611
|
+
"truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested, check_gap_rows)),
|
|
503
612
|
"instructionLimitReached": stopped,
|
|
504
613
|
"controls": [{"site": at, "target": call_targets[at][0]} for at in controls],
|
|
505
614
|
"instructionControls": [{"site": at, "instruction": _text(seen[at]), "routine": routine.get(at)}
|
|
506
615
|
for at in instruction_controls],
|
|
507
616
|
"negativeUsable": bool(controls or instruction_controls)
|
|
508
|
-
and not (stopped or unresolved or gaps or contested or contradicted
|
|
617
|
+
and not (stopped or unresolved or gaps or contested or contradicted
|
|
618
|
+
or check_gap_rows),
|
|
509
619
|
"assumptions": assumptions,
|
|
510
620
|
"exclusions": ["computed call and jump targets no declaration gives (listed in unresolved)", "unrelocated far calls",
|
|
511
621
|
"code reached only from outside the starts", "runtime reachability"],
|
|
@@ -513,7 +623,7 @@ def reach(image, config):
|
|
|
513
623
|
"throughEveryRoute lists the routine starts every read route to the target passes, "
|
|
514
624
|
"and an unresolved transfer may add a route that passes none of them. "
|
|
515
625
|
"negativeUsable needs a control of either kind, a walk that did not stop at its instruction limit, "
|
|
516
|
-
"and no unresolved transfer, gap, contested instruction or contradicted noReturn "
|
|
626
|
+
"and no unresolved transfer, gap, returnCheckGaps row, contested instruction or contradicted noReturn "
|
|
517
627
|
"routine, and still rests on the listed assumptions, leaves and noReturn declarations. "
|
|
518
628
|
"A declared indirect call counts as resolved when it is declared exhaustive: its table "
|
|
519
629
|
"rows are read from the build's bytes and its targets list rests on its evidence alone, "
|
|
@@ -524,11 +634,18 @@ def reach(image, config):
|
|
|
524
634
|
"A noReturn call is never contradicted, since whether its target returns to it can "
|
|
525
635
|
"depend on what the caller passes, which the walk does not follow; the return sites "
|
|
526
636
|
"of its targets are listed, and the declaration rests on its reason alone. "
|
|
637
|
+
"A stop narrows the question and claims nothing about the build: the walk reads no "
|
|
638
|
+
"further at a stop site, nor into a stop routine or past a call to it, so a target "
|
|
639
|
+
"reached only past a stop is not reached, and negativeUsable answers the narrowed "
|
|
640
|
+
"question. The noReturn return check reads past the stops, with a walk of its own, and "
|
|
641
|
+
"returnCheckGaps lists what that walk could not read that the query's walk did not "
|
|
642
|
+
"record; any row keeps negativeUsable false, as a gap would. "
|
|
527
643
|
"decodedRanges, when asked for, covers the instructions the walk decoded; no range "
|
|
528
|
-
"starts at a leaf or
|
|
644
|
+
"starts at a leaf, a stop routine or a contested, unresolved overlapping or undecodable start, though "
|
|
529
645
|
"one can lie inside a decoded instruction. A site inside a range lies in a decoded "
|
|
530
646
|
"instruction, which may start before it; a target at the site shows whether one starts "
|
|
531
647
|
"there. "
|
|
532
|
-
"When instructionLimitReached holds, the walk
|
|
533
|
-
"every list and count covers only the part read, which
|
|
534
|
-
"
|
|
648
|
+
"When instructionLimitReached holds, the walk, or the return check's walk, stopped "
|
|
649
|
+
"before reading all it reaches: every list and count covers only the part read, which "
|
|
650
|
+
"part depends on the walk order, an unreached target may lie past the stop, and an "
|
|
651
|
+
"uncontradicted noReturn routine may return in code the check did not read."}
|