scientific-method-engine 18.0.0__tar.gz → 18.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/PKG-INFO +1 -1
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/pyproject.toml +1 -1
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/cli.py +6 -1
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/reach.py +164 -32
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/trace.py +16 -7
- scientific_method_engine-18.2.0/tests/test_reach.py +387 -0
- scientific_method_engine-18.0.0/tests/test_reach.py +0 -190
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/.gitignore +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/LICENSE +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/README.md +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/inventory.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/machine.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/reports.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/oracle.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_config_encoding.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_enter_frames.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_indirect_far_transfers.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_interrupt_models.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_inventory_check.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_join_terms.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_return_checks.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_term_limit.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_upstream_gaps.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_volatile_memory.py +0 -0
- {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 18.
|
|
3
|
+
Version: 18.2.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "18.
|
|
8
|
+
version = "18.2.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -23,7 +23,12 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
23
23
|
"or continues at another address than the next instruction (ghidraFallsThroughElsewhere).\n"
|
|
24
24
|
"reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
|
|
25
25
|
"and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
|
|
26
|
-
"a leaves routine is reached but not read, and its reason is repeated in the report
|
|
26
|
+
"a leaves routine is reached but not read, and its reason is repeated in the report;\n"
|
|
27
|
+
"a call to a noReturn routine and a noReturn interrupt do not continue at the next instruction,\n"
|
|
28
|
+
"and a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
|
|
29
|
+
"is reported contradicted;\n"
|
|
30
|
+
"controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
|
|
31
|
+
"and a failed control fails the report saying whether it was reached.\n"
|
|
27
32
|
"inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
|
|
28
33
|
"that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
|
|
29
34
|
"with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""Transitive reachability from established starts to target sites over the entry-path CFG."""
|
|
2
2
|
from collections import deque
|
|
3
3
|
from .image import integer
|
|
4
|
-
from .trace import walk, cfg_step, base_mnemonic, unsupported_transfer, holding_instruction, INTERRUPTS, RETURNS,
|
|
4
|
+
from .trace import (walk, cfg_step, base_mnemonic, unsupported_transfer, holding_instruction, INTERRUPTS, RETURNS,
|
|
5
|
+
LIMIT_REASON, OVERLAP_REASON, UNDECODED_REASON)
|
|
5
6
|
from .pcode_backend import interrupt_vector
|
|
6
7
|
|
|
7
8
|
# The virtual root of the dominator computation; no file offset is negative.
|
|
@@ -11,9 +12,13 @@ UNSUPPORTED = "unsupported control-transfer frame encoding"
|
|
|
11
12
|
LEAF_OVERLAP = "leaf start inside a reached instruction; the leaf is not decoded, so its boundary is unchecked"
|
|
12
13
|
|
|
13
14
|
|
|
14
|
-
def
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
def _text(ins):
|
|
16
|
+
return (ins.mnemonic + " " + ins.op_str).strip()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _sites(value, label, image, limit=256, least=1):
|
|
20
|
+
if not isinstance(value, list) or not least <= len(value) <= limit or len(set(map(repr, value))) != len(value):
|
|
21
|
+
raise ValueError(f"{label} must be a list of {least}..{limit} distinct file offsets")
|
|
17
22
|
for at in value:
|
|
18
23
|
integer(at, 0, len(image.data) - 1, label)
|
|
19
24
|
if image.region(at) is None:
|
|
@@ -41,6 +46,59 @@ def _leaves(value, image, starts):
|
|
|
41
46
|
return leaves
|
|
42
47
|
|
|
43
48
|
|
|
49
|
+
def _no_return(value, image):
|
|
50
|
+
"""The declared routines and interrupt sites that never return, each with its reason."""
|
|
51
|
+
if not isinstance(value, list) or len(value) > 256:
|
|
52
|
+
raise ValueError("noReturn must be a list of at most 256 objects")
|
|
53
|
+
routines, interrupts = {}, {}
|
|
54
|
+
for entry in value:
|
|
55
|
+
if not isinstance(entry, dict) or set(entry) not in ({"routine", "reason"}, {"interrupt", "reason"}):
|
|
56
|
+
raise ValueError("Each noReturn entry needs reason and exactly one of routine and interrupt")
|
|
57
|
+
kind = "routine" if "routine" in entry else "interrupt"
|
|
58
|
+
at = integer(entry[kind], 0, len(image.data) - 1, f"noReturn {kind}")
|
|
59
|
+
if image.region(at) is None:
|
|
60
|
+
raise ValueError(f"noReturn {kind} {at} is outside declared code")
|
|
61
|
+
if not isinstance(entry["reason"], str) or not entry["reason"].strip():
|
|
62
|
+
raise ValueError("A noReturn entry needs a nonempty reason")
|
|
63
|
+
found = routines if kind == "routine" else interrupts
|
|
64
|
+
if at in found:
|
|
65
|
+
raise ValueError(f"Duplicate noReturn {kind}")
|
|
66
|
+
if kind == "interrupt":
|
|
67
|
+
ins = image.decode(at)
|
|
68
|
+
if ins is None or base_mnemonic(ins) not in INTERRUPTS:
|
|
69
|
+
raise ValueError(f"noReturn interrupt {at} is not an interrupt instruction")
|
|
70
|
+
vector, conditional = interrupt_vector(image.flat, ins, at)
|
|
71
|
+
if conditional:
|
|
72
|
+
raise ValueError(f"noReturn interrupt {at} is conditional and continues when it does not interrupt")
|
|
73
|
+
found[at] = {"reason": entry["reason"], "vector": vector, "following": at + ins.size}
|
|
74
|
+
continue
|
|
75
|
+
found[at] = entry["reason"]
|
|
76
|
+
return routines, interrupts
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _returns_from(graph, seen, returning_leaves, routine):
|
|
80
|
+
"""Where ``routine``'s own paths return: each return instruction, and each leaf they enter other than by a call.
|
|
81
|
+
|
|
82
|
+
Calls are stepped over at the return sites the graph keeps. The walk assumes that a leaf returns,
|
|
83
|
+
so a jump, branch, table row or fall-through into one in ``returning_leaves`` returns too.
|
|
84
|
+
"""
|
|
85
|
+
found, stack, visited = [], [routine], set()
|
|
86
|
+
while stack:
|
|
87
|
+
at = stack.pop()
|
|
88
|
+
if at in visited:
|
|
89
|
+
continue
|
|
90
|
+
visited.add(at)
|
|
91
|
+
if at in returning_leaves:
|
|
92
|
+
found.append(at)
|
|
93
|
+
continue
|
|
94
|
+
if at not in seen:
|
|
95
|
+
continue
|
|
96
|
+
if base_mnemonic(seen[at]) in RETURNS:
|
|
97
|
+
found.append(at)
|
|
98
|
+
stack.extend(s for s, kind in graph.get(at, []) if kind != "call")
|
|
99
|
+
return sorted(found)
|
|
100
|
+
|
|
101
|
+
|
|
44
102
|
def _kinds(at, ins, step):
|
|
45
103
|
"""Each successor of ``step`` with how the CFG reaches it: call, return, table, interrupt, jump or fall."""
|
|
46
104
|
m, following = base_mnemonic(ins), at + ins.size
|
|
@@ -95,8 +153,10 @@ def reach(image, config):
|
|
|
95
153
|
The walk decodes from ``starts`` (established region entries) and follows every resolved call
|
|
96
154
|
into its callee and on at its return site, every resolved jump and branch, the rows of a
|
|
97
155
|
declared indirect jump table, and every interrupt to the next instruction. A ``leaves`` routine
|
|
98
|
-
is reached but never decoded.
|
|
99
|
-
|
|
156
|
+
is reached but never decoded. A call to a ``noReturn`` routine and a ``noReturn`` interrupt site
|
|
157
|
+
do not continue at the next instruction. Each reached target gets one chain with the fewest
|
|
158
|
+
calls, the assumptions its route rests on, and the routines on every route to it within the
|
|
159
|
+
read graph.
|
|
100
160
|
"""
|
|
101
161
|
from .reports import entries
|
|
102
162
|
starts = _sites(config.get("starts"), "starts", image)
|
|
@@ -106,20 +166,33 @@ def reach(image, config):
|
|
|
106
166
|
raise ValueError(f"Reach start {at} must be an established region entry")
|
|
107
167
|
targets = _sites(config.get("targets"), "targets", image)
|
|
108
168
|
leaves = _leaves(config.get("leaves", []), image, set(starts))
|
|
109
|
-
controls = config.get("controls", [])
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
169
|
+
controls = _sites(config.get("controls", []), "controls", image, least=0)
|
|
170
|
+
instruction_controls = _sites(config.get("instructionControls", []), "instructionControls", image, least=0)
|
|
171
|
+
# The walk decodes a site as image.decode does, so a control whose bytes are no call can never pass.
|
|
172
|
+
for at in controls:
|
|
173
|
+
ins = image.decode(at)
|
|
174
|
+
if ins is None or base_mnemonic(ins) not in CALLS:
|
|
175
|
+
decoded = f"decodes as {_text(ins)}" if ins is not None else "does not decode"
|
|
176
|
+
raise ValueError(f"control {at} is not a call site ({decoded}); controls are call sites the walk must "
|
|
177
|
+
"reach and resolve, and instructionControls are sites the walk must decode")
|
|
178
|
+
# The walk decodes every start it is given, so a start as an instruction control would pass whatever the walk misses.
|
|
179
|
+
for at in instruction_controls:
|
|
180
|
+
if at in starts:
|
|
181
|
+
raise ValueError(f"instruction control {at} is a start; the walk decodes every start, so it shows nothing "
|
|
182
|
+
"the walk found")
|
|
113
183
|
limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
|
|
114
184
|
instruction_limit = config.get("instructionLimit", 10000)
|
|
115
|
-
|
|
185
|
+
no_return_routines, no_return_interrupts = _no_return(config.get("noReturn", []), image)
|
|
186
|
+
declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
|
|
187
|
+
seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
|
|
188
|
+
stops=frozenset(leaves), **declared)
|
|
116
189
|
|
|
117
190
|
graph, unresolved, interrupts, resolved_calls = {}, [], [], {}
|
|
118
191
|
for at, ins in sorted(seen.items()):
|
|
119
|
-
step = cfg_step(image, at, ins, follow_interrupts=True)
|
|
192
|
+
step = cfg_step(image, at, ins, follow_interrupts=True, **declared)
|
|
120
193
|
graph[at] = [(s, kind) for s, kind in _kinds(at, ins, step) if s in seen or s in leaves]
|
|
121
194
|
m = base_mnemonic(ins)
|
|
122
|
-
text = (ins
|
|
195
|
+
text = _text(ins)
|
|
123
196
|
kind = "call" if m in CALLS else "return" if m in RETURNS else "jump"
|
|
124
197
|
if unsupported_transfer(image, ins):
|
|
125
198
|
unresolved.append({"site": at, "instruction": text, "kind": kind, "reason": UNSUPPORTED})
|
|
@@ -129,7 +202,7 @@ def reach(image, config):
|
|
|
129
202
|
"reason": edge["provenance"].get("reason", "target outside declared regions")})
|
|
130
203
|
elif m in CALLS:
|
|
131
204
|
resolved_calls[at] = edge["target"]
|
|
132
|
-
if m in INTERRUPTS:
|
|
205
|
+
if m in INTERRUPTS and at not in no_return_interrupts:
|
|
133
206
|
vector, conditional = interrupt_vector(image.flat, ins, at)
|
|
134
207
|
interrupts.append({"site": at, "instruction": text, "vector": vector, "conditional": conditional})
|
|
135
208
|
for at in leaves:
|
|
@@ -191,6 +264,19 @@ def reach(image, config):
|
|
|
191
264
|
"route": {"assumedReturns": assumed_returns[::-1], "declaredTableJumps": tables[::-1],
|
|
192
265
|
"interruptsContinued": continued[::-1]}}
|
|
193
266
|
|
|
267
|
+
# The walk decodes both instructions of an unresolved overlap and keeps neither, so their starts are not in seen.
|
|
268
|
+
overlapping = {g["site"] for g in walk_gaps if g["reason"] == OVERLAP_REASON}
|
|
269
|
+
# A site an edge leads to whose bytes do not decode is reached, though no instruction starts there.
|
|
270
|
+
undecodable = {g["site"] for g in walk_gaps if g["reason"] == UNDECODED_REASON}
|
|
271
|
+
|
|
272
|
+
def unreached(site):
|
|
273
|
+
holder = holding_instruction(seen, site)
|
|
274
|
+
if holder is not None:
|
|
275
|
+
return {"status": "inside a reached instruction", "insideInstruction": holder}
|
|
276
|
+
return {"status": "start of a contested instruction" if site in contested
|
|
277
|
+
else "start of an unresolved overlapping instruction" if site in overlapping
|
|
278
|
+
else "reached but not decodable" if site in undecodable else "not reached"}
|
|
279
|
+
|
|
194
280
|
rows = []
|
|
195
281
|
for target in targets:
|
|
196
282
|
row = {"target": target}
|
|
@@ -203,10 +289,7 @@ def reach(image, config):
|
|
|
203
289
|
row |= {"reached": True, "routine": routine[target], "leaf": target in leaves, **chain(target),
|
|
204
290
|
"throughEveryRoute": cut[::-1]}
|
|
205
291
|
else:
|
|
206
|
-
|
|
207
|
-
row |= {"reached": False, "status": "inside a reached instruction" if holder is not None
|
|
208
|
-
else "start of a contested instruction" if target in contested else "not reached",
|
|
209
|
-
**({"insideInstruction": holder} if holder is not None else {})}
|
|
292
|
+
row |= {"reached": False, **unreached(target)}
|
|
210
293
|
rows.append(row)
|
|
211
294
|
|
|
212
295
|
for row in unresolved + interrupts:
|
|
@@ -216,38 +299,87 @@ def reach(image, config):
|
|
|
216
299
|
# The walk never decodes a leaf, so its overlap check cannot see a leaf start inside a decoded instruction.
|
|
217
300
|
gaps += [{"site": at, "reason": LEAF_OVERLAP, "insideInstruction": holding_instruction(seen, at)}
|
|
218
301
|
for at in sorted(leaves) if at in distance and holding_instruction(seen, at) is not None]
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
302
|
+
# Kept apart from the gap rows, which the result limit can cut.
|
|
303
|
+
stopped = any(g["reason"] == LIMIT_REASON for g in walk_gaps)
|
|
304
|
+
# A control at a site a noReturn declaration kept the walk from names that declaration, which may be the cause.
|
|
305
|
+
cut_by = {at + seen[at].size: f"the call at {at} to noReturn routine {target}"
|
|
306
|
+
for at, target in sorted(resolved_calls.items()) if target in no_return_routines}
|
|
307
|
+
cut_by |= {row["following"]: f"the noReturn interrupt at {at}"
|
|
308
|
+
for at, row in no_return_interrupts.items() if at in seen}
|
|
309
|
+
# Not reached and reached-but-unresolved need different fixes, so each failure says which it is.
|
|
310
|
+
failures = []
|
|
311
|
+
for label, sites, passed in (("control", controls, resolved_calls), ("instruction control", instruction_controls, seen)):
|
|
312
|
+
for at in sites:
|
|
313
|
+
if at in passed:
|
|
314
|
+
continue
|
|
315
|
+
# Only a call-site control can be decoded and still fail.
|
|
316
|
+
if at in seen:
|
|
317
|
+
reasons = sorted({row["reason"] for row in unresolved if row["site"] == at})
|
|
318
|
+
failures.append(f"{label} {at} is reached but its call target is unresolved ({'; '.join(reasons)})")
|
|
319
|
+
elif at in leaves and at in distance:
|
|
320
|
+
failures.append(f"{label} {at} is the start of a leaf, which the walk reaches but does not decode")
|
|
321
|
+
else:
|
|
322
|
+
state = unreached(at)
|
|
323
|
+
detail = (f"inside the reached instruction at {state['insideInstruction']}" if "insideInstruction" in state
|
|
324
|
+
else "the " + state["status"] if state["status"].startswith("start") else state["status"])
|
|
325
|
+
cut = cut_by.get(at)
|
|
326
|
+
failures.append(f"{label} {at} is {detail}"
|
|
327
|
+
+ (f" (it follows {cut})" if cut else "")
|
|
328
|
+
+ (" (the walk stopped at its instruction limit)" if stopped else ""))
|
|
329
|
+
if failures:
|
|
330
|
+
raise ValueError("Positive controls failed: " + "; ".join(failures))
|
|
222
331
|
call_sites = {}
|
|
223
332
|
for site, target in sorted(resolved_calls.items()):
|
|
224
333
|
call_sites.setdefault(target, []).append(site)
|
|
225
334
|
leaf_rows = [{"routine": at, "reason": reason, "reached": at in distance, "callSites": call_sites.get(at, [])}
|
|
226
335
|
for at, reason in leaves.items()]
|
|
227
|
-
#
|
|
228
|
-
|
|
336
|
+
# Each call the declaration kept from its return site, so a reviewer can check what follows it.
|
|
337
|
+
no_return_rows, returning_leaves = [], set(leaves) - set(no_return_routines)
|
|
338
|
+
for at, reason in no_return_routines.items():
|
|
339
|
+
returns = _returns_from(graph, seen, returning_leaves, at)
|
|
340
|
+
no_return_rows.append({"routine": at, "reason": reason, "reached": at in distance, "read": at in seen,
|
|
341
|
+
"returnSites": returns, "contradicted": bool(returns),
|
|
342
|
+
"callSites": [{"site": site, "following": site + seen[site].size,
|
|
343
|
+
"followingRead": site + seen[site].size in seen}
|
|
344
|
+
for site in call_sites.get(at, [])]})
|
|
345
|
+
no_return_rows += [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
|
|
346
|
+
"following": row["following"], "followingRead": row["following"] in seen}
|
|
347
|
+
for at, row in no_return_interrupts.items()]
|
|
348
|
+
contradicted = any(row.get("contradicted") for row in no_return_rows)
|
|
229
349
|
counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
|
|
230
350
|
"interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
|
|
231
|
-
|
|
351
|
+
assumptions = ["each reached call returns to its next instruction"
|
|
352
|
+
+ (", except a call to a noReturn routine" if no_return_routines else ""),
|
|
353
|
+
"each reached interrupt returns to its next instruction"
|
|
354
|
+
+ (", except at a noReturn interrupt site" if no_return_interrupts else "")
|
|
355
|
+
+ "; interrupt handlers are not read",
|
|
356
|
+
"each leaf calls nothing, for the reason it gives",
|
|
357
|
+
"each declared indirect jump table holds the routes its declaration gives"]
|
|
358
|
+
if no_return_rows:
|
|
359
|
+
assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
|
|
360
|
+
return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return_rows,
|
|
232
361
|
"reachedRoutines": sorted(routine_starts), "counts": counts,
|
|
233
362
|
"unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
|
|
234
363
|
"contested": sorted(contested)[:limit],
|
|
235
364
|
"truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested)),
|
|
236
365
|
"instructionLimitReached": stopped,
|
|
237
366
|
"controls": [{"site": at, "target": resolved_calls[at]} for at in controls],
|
|
238
|
-
"
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
367
|
+
"instructionControls": [{"site": at, "instruction": _text(seen[at]), "routine": routine.get(at)}
|
|
368
|
+
for at in instruction_controls],
|
|
369
|
+
"negativeUsable": bool(controls or instruction_controls)
|
|
370
|
+
and not (stopped or unresolved or gaps or contested or contradicted),
|
|
371
|
+
"assumptions": assumptions,
|
|
243
372
|
"exclusions": ["computed call and jump targets (listed in unresolved)", "unrelocated far calls",
|
|
244
373
|
"code reached only from outside the starts", "runtime reachability"],
|
|
245
374
|
"interpretation": "Routes over the decoded entry-path CFG from the starts. A chain has the fewest calls; "
|
|
246
375
|
"throughEveryRoute lists the routine starts every read route to the target passes, "
|
|
247
376
|
"and an unresolved transfer may add a route that passes none of them. "
|
|
248
|
-
"negativeUsable needs
|
|
249
|
-
"and no unresolved transfer, gap
|
|
250
|
-
"and still rests on the listed assumptions and
|
|
377
|
+
"negativeUsable needs a control of either kind, a walk that did not stop at its instruction limit, "
|
|
378
|
+
"and no unresolved transfer, gap, contested instruction or contradicted noReturn "
|
|
379
|
+
"routine, and still rests on the listed assumptions, leaves and noReturn declarations. "
|
|
380
|
+
"A noReturn routine is contradicted when a return instruction, or a leaf entered "
|
|
381
|
+
"other than by a call, lies on its own read paths; an empty returnSites means only "
|
|
382
|
+
"that the walk read none. "
|
|
251
383
|
"When instructionLimitReached holds, the walk stopped before reading all it reaches: "
|
|
252
384
|
"every list and count covers only the part read, which part depends on the walk order, "
|
|
253
385
|
"and an unreached target may lie past the stop."}
|
|
@@ -88,6 +88,8 @@ def far_pointer_transfer(state, image):
|
|
|
88
88
|
# The gap reason walk() records when it stops at its instruction limit with code left to read.
|
|
89
89
|
LIMIT_REASON = "instruction limit"
|
|
90
90
|
OVERLAP_REASON = "overlapping entry-path instructions; boundary unresolved"
|
|
91
|
+
# The gap reason walk() records at a site it reaches but cannot decode.
|
|
92
|
+
UNDECODED_REASON = "undecoded or unmapped edge"
|
|
91
93
|
RETURNS = {"ret": "near return", "retf": "far return", "iret": "interrupt return", "iretd": "interrupt return"}
|
|
92
94
|
INTERRUPTS = ("int", "int1", "int3", "into")
|
|
93
95
|
PORTS = ("in", "out", "insb", "insw", "insd", "outsb", "outsw", "outsd")
|
|
@@ -178,7 +180,7 @@ class Step(NamedTuple):
|
|
|
178
180
|
|
|
179
181
|
|
|
180
182
|
def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False,
|
|
181
|
-
modeled_interrupts=frozenset()):
|
|
183
|
+
modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset()):
|
|
182
184
|
"""The successors of ``ins`` decoded at ``at``, under the rule ``walk`` and the ``uses`` caller continuation share.
|
|
183
185
|
|
|
184
186
|
An unsupported transfer encoding, a return, an unconditional jump without a resolved target,
|
|
@@ -187,7 +189,9 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
|
|
|
187
189
|
conditional jump or loop continues at its resolved target and at the next instruction. A declared
|
|
188
190
|
indirect jump continues at its table rows only. A call continues at its resolved target and at
|
|
189
191
|
its return site; with ``step_over_calls`` it continues at its return site only and its target
|
|
190
|
-
is neither resolved nor reported.
|
|
192
|
+
is neither resolved nor reported. A call whose resolved target is in ``no_return_calls``, and an
|
|
193
|
+
interrupt whose site is in ``no_return_interrupts``, have no return site: the query declared that
|
|
194
|
+
they never come back.
|
|
191
195
|
"""
|
|
192
196
|
m, following = base_mnemonic(ins), at + ins.size
|
|
193
197
|
if unsupported_transfer(image, ins):
|
|
@@ -214,8 +218,10 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
|
|
|
214
218
|
gaps.append({"site": at, "reason": provenance.get("reason", "target outside declared regions")})
|
|
215
219
|
else:
|
|
216
220
|
successors.append(target)
|
|
217
|
-
if m in ("jmp", "ljmp"):
|
|
221
|
+
if m in ("jmp", "ljmp") or (calls and target in no_return_calls):
|
|
218
222
|
return Step(successors, False, None, gaps, edges)
|
|
223
|
+
if m in INTERRUPTS and at in no_return_interrupts:
|
|
224
|
+
return Step(successors, False, None, gaps, edges)
|
|
219
225
|
if m == "hlt" or (m in INTERRUPTS and not follow_interrupts and at not in modeled_interrupts):
|
|
220
226
|
gaps.append({"site": at, "reason": "hardware or interrupt boundary"})
|
|
221
227
|
return Step(successors, False, None, gaps, edges)
|
|
@@ -231,7 +237,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
|
|
|
231
237
|
|
|
232
238
|
|
|
233
239
|
def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset(),
|
|
234
|
-
modeled_interrupts=frozenset()):
|
|
240
|
+
modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset()):
|
|
235
241
|
"""Decode the CFG reached from ``entries`` and check its instruction boundaries.
|
|
236
242
|
|
|
237
243
|
A port access in the flat model records a gap and ends that branch, as ``trace`` stops there.
|
|
@@ -240,7 +246,9 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
|
|
|
240
246
|
for callers that list each interrupt as an assumption. ``modeled_interrupts`` continues past the
|
|
241
247
|
interrupts at those sites only, the ones ``modeled_interrupt_sites`` names, which ``trace``
|
|
242
248
|
continues past under a call model (ADR 0017). A site in ``stops`` is never decoded:
|
|
243
|
-
the walk reaches it and goes no further, so it is in no returned set.
|
|
249
|
+
the walk reaches it and goes no further, so it is in no returned set. ``no_return_calls`` and
|
|
250
|
+
``no_return_interrupts`` end the branch at a call to one of those routines and at one of those
|
|
251
|
+
interrupt sites, as ``cfg_step`` describes.
|
|
244
252
|
"""
|
|
245
253
|
instruction_limit(image, limit)
|
|
246
254
|
pending, seen, gaps, edges = list(entries), {}, [], []
|
|
@@ -259,11 +267,12 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
|
|
|
259
267
|
break
|
|
260
268
|
ins = image.decode(at)
|
|
261
269
|
if ins is None:
|
|
262
|
-
gaps.append({"site": at, "reason":
|
|
270
|
+
gaps.append({"site": at, "reason": UNDECODED_REASON})
|
|
263
271
|
continue
|
|
264
272
|
seen[at] = ins
|
|
265
273
|
step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts,
|
|
266
|
-
modeled_interrupts=modeled_interrupts
|
|
274
|
+
modeled_interrupts=modeled_interrupts, no_return_calls=no_return_calls,
|
|
275
|
+
no_return_interrupts=no_return_interrupts)
|
|
267
276
|
successors[at] = step.successors
|
|
268
277
|
gaps.extend(step.gaps)
|
|
269
278
|
edges.extend(step.edges)
|