scientific-method-engine 19.1.0__tar.gz → 19.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/PKG-INFO +1 -1
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/pyproject.toml +1 -1
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/cli.py +6 -1
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/ranges.py +16 -10
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/reach.py +181 -30
- scientific_method_engine-19.3.0/src/scientific_method_engine/x86/table_bounds.py +810 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/trace.py +7 -3
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_ranges.py +11 -1
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_reach.py +202 -0
- scientific_method_engine-19.3.0/tests/test_table_bounds.py +389 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/.gitignore +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/LICENSE +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/README.md +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/inventory.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/machine.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/reports.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/oracle.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_config_encoding.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_enter_frames.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_indirect_far_transfers.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_interrupt_models.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_inventory_check.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_join_terms.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_return_checks.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_term_limit.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_upstream_gaps.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_volatile_memory.py +0 -0
- {scientific_method_engine-19.1.0 → scientific_method_engine-19.3.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 19.
|
|
3
|
+
Version: 19.3.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "19.
|
|
8
|
+
version = "19.3.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -27,10 +27,15 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
27
27
|
"a call to a noReturn routine, a noReturn call and a noReturn interrupt do not continue at the next\n"
|
|
28
28
|
"instruction, a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
|
|
29
29
|
"is reported contradicted, and a noReturn call lists its targets' return sites and is never contradicted;\n"
|
|
30
|
+
"stops cut this query's walk and claim nothing about the build: a stops routine is reached but not read\n"
|
|
31
|
+
"and a call to it does not continue at the next instruction, a stops site is decoded and nothing after it\n"
|
|
32
|
+
"is followed, and each is repeated with its reason and where it cut the walk, not checked or assumed;\n"
|
|
33
|
+
"the noReturn check reads past the stops, and returnCheckGaps lists what it could not read there;\n"
|
|
30
34
|
"indirectCalls declares the targets of a computed call as a table read from the build or a list,\n"
|
|
31
35
|
"and the walk calls each of them; one not declared exhaustive stays unresolved;\n"
|
|
32
36
|
"controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
|
|
33
|
-
"and a failed control fails the report saying whether it was reached
|
|
37
|
+
"and a failed control fails the report saying whether it was reached;\n"
|
|
38
|
+
"decodedRanges true lists every instruction the walk decoded as half-open ranges with their routine.\n"
|
|
34
39
|
"inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
|
|
35
40
|
"that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
|
|
36
41
|
"with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
|
|
@@ -9,9 +9,9 @@ operation without a transfer function has no range, and its output is unknown.
|
|
|
9
9
|
|
|
10
10
|
``Evaluation`` runs one instruction's p-code over ``RegisterRanges``. Memory is not modelled:
|
|
11
11
|
a ``LOAD`` or a direct ``ram`` operand yields an unknown value and a ``STORE`` changes no
|
|
12
|
-
register. Control flow, joins across paths, widening and branch refinement belong to the
|
|
13
|
-
analysis that walks the code (
|
|
14
|
-
|
|
12
|
+
register. Control flow, memory, joins across paths, widening and branch refinement belong to the
|
|
13
|
+
analysis that walks the code (``table_bounds.py``); ``Evaluation.execute`` stops at the first
|
|
14
|
+
control-flow operation and hands it back.
|
|
15
15
|
"""
|
|
16
16
|
from math import gcd
|
|
17
17
|
|
|
@@ -455,7 +455,8 @@ class RegisterRanges:
|
|
|
455
455
|
"""The ranges register varnodes hold, keyed by ``(offset, size)`` in the register space.
|
|
456
456
|
|
|
457
457
|
Held varnodes never overlap. A read of a varnode inside a held one takes its bytes; a read
|
|
458
|
-
that held varnodes
|
|
458
|
+
that covers held varnodes is pieced from them, with every byte no held varnode covers unknown;
|
|
459
|
+
a read that a held varnode only partly overlaps is unknown. A write keeps the
|
|
459
460
|
bytes of overlapped varnodes that it does not cover. x86 SLEIGH's register space is
|
|
460
461
|
little-endian, so byte ``k`` of a varnode holds its bits ``8k`` to ``8k + 7``.
|
|
461
462
|
"""
|
|
@@ -489,13 +490,18 @@ class RegisterRanges:
|
|
|
489
490
|
for (o, n), r in self.held.items():
|
|
490
491
|
if o <= offset and offset + size <= o + n:
|
|
491
492
|
return resize(shift_right(r, (offset - o) * 8), bits)
|
|
493
|
+
end = offset + size
|
|
494
|
+
inside = sorted((o, n) for (o, n) in self.held if o < end and offset < o + n)
|
|
495
|
+
if not inside or any(o < offset or o + n > end for o, n in inside):
|
|
496
|
+
return Range.top(bits)
|
|
497
|
+
# Held varnodes inside the read, with every byte between them unknown.
|
|
492
498
|
parts, at = [], offset
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
at
|
|
499
|
+
for o, n in inside + [(end, 0)]:
|
|
500
|
+
if at < o:
|
|
501
|
+
parts.append(Range.top((o - at) * 8))
|
|
502
|
+
if n:
|
|
503
|
+
parts.append(self.held[(o, n)])
|
|
504
|
+
at = o + n
|
|
499
505
|
result = parts[0]
|
|
500
506
|
for part in parts[1:]:
|
|
501
507
|
result = t_piece(result.bits + part.bits, part, result)
|
|
@@ -11,6 +11,9 @@ ROOT = -1
|
|
|
11
11
|
CALLS = ("call", "lcall")
|
|
12
12
|
UNSUPPORTED = "unsupported control-transfer frame encoding"
|
|
13
13
|
LEAF_OVERLAP = "leaf start inside a reached instruction; the leaf is not decoded, so its boundary is unchecked"
|
|
14
|
+
STOP_OVERLAP = ("stop routine start inside a reached instruction; the routine is not decoded, so its boundary is "
|
|
15
|
+
"unchecked")
|
|
16
|
+
CHECK_CONTESTED = "contested instruction in the noReturn return check's walk"
|
|
14
17
|
|
|
15
18
|
|
|
16
19
|
def _text(ins):
|
|
@@ -47,6 +50,42 @@ def _leaves(value, image, starts):
|
|
|
47
50
|
return leaves
|
|
48
51
|
|
|
49
52
|
|
|
53
|
+
def _stops(value, image, starts, leaves, no_return_routines):
|
|
54
|
+
"""``(routines, sites)``: the query's stop routines and stop sites, each mapped to its reason (ADR 0039)."""
|
|
55
|
+
if not isinstance(value, list) or len(value) > 256:
|
|
56
|
+
raise ValueError("stops must be a list of at most 256 objects")
|
|
57
|
+
routines, sites = {}, {}
|
|
58
|
+
for stop in value:
|
|
59
|
+
if not isinstance(stop, dict) or set(stop) not in ({"routine", "reason"}, {"site", "reason"}):
|
|
60
|
+
raise ValueError("Each stop needs reason and exactly one of routine and site")
|
|
61
|
+
kind = "routine" if "routine" in stop else "site"
|
|
62
|
+
at = integer(stop[kind], 0, len(image.data) - 1, f"stop {kind}")
|
|
63
|
+
if image.region(at) is None:
|
|
64
|
+
raise ValueError(f"stop {kind} {at} is outside declared code")
|
|
65
|
+
if not isinstance(stop["reason"], str) or not stop["reason"].strip():
|
|
66
|
+
raise ValueError("A stop needs a nonempty reason")
|
|
67
|
+
found = routines if kind == "routine" else sites
|
|
68
|
+
if at in found:
|
|
69
|
+
raise ValueError(f"Duplicate stop {kind}")
|
|
70
|
+
if at in starts:
|
|
71
|
+
raise ValueError(f"stop {kind} {at} is a start; the walk would read nothing past it")
|
|
72
|
+
if kind == "routine" and at in leaves:
|
|
73
|
+
raise ValueError(f"stop routine {at} is a leaf; a stop routine is not read either, so give it once, as a stop")
|
|
74
|
+
if kind == "routine" and at in no_return_routines:
|
|
75
|
+
raise ValueError(f"stop routine {at} is declared noReturn; the declaration is a claim about the build and "
|
|
76
|
+
"the stop a cut of this query, so give one of them")
|
|
77
|
+
if kind == "site" and at in leaves:
|
|
78
|
+
raise ValueError(f"stop site {at} is a leaf; the walk does not decode a leaf, so the stop would never be read")
|
|
79
|
+
if kind == "site" and image.decode(at) is None:
|
|
80
|
+
raise ValueError(f"stop site {at} does not decode")
|
|
81
|
+
found[at] = stop["reason"]
|
|
82
|
+
both = sorted(set(routines) & set(sites))
|
|
83
|
+
if both:
|
|
84
|
+
raise ValueError(f"stop {both[0]} is given as a routine and as a site; a stop routine is not decoded and a "
|
|
85
|
+
"stop site is, so give one of them")
|
|
86
|
+
return routines, sites
|
|
87
|
+
|
|
88
|
+
|
|
50
89
|
def no_return_declarations(value, image):
|
|
51
90
|
"""``(routines, interrupts, calls)``: the declared routines, interrupt sites and call sites that never return.
|
|
52
91
|
|
|
@@ -160,12 +199,20 @@ def successor_kinds(at, ins, step):
|
|
|
160
199
|
return rows
|
|
161
200
|
|
|
162
201
|
|
|
163
|
-
def successor_graph(image, seen, **step):
|
|
164
|
-
"""Each site of ``seen`` mapped to its ``successor_kinds`` rows that stay in ``seen
|
|
165
|
-
|
|
202
|
+
def successor_graph(image, seen, kept=frozenset(), **step):
|
|
203
|
+
"""Each site of ``seen`` mapped to its ``successor_kinds`` rows that stay in ``seen`` or lead to ``kept``,
|
|
204
|
+
under ``cfg_step``'s ``step`` options."""
|
|
205
|
+
return {at: [(s, kind) for s, kind in successor_kinds(at, ins, cfg_step(image, at, ins, **step))
|
|
206
|
+
if s in seen or s in kept]
|
|
166
207
|
for at, ins in seen.items()}
|
|
167
208
|
|
|
168
209
|
|
|
210
|
+
def _call_site_rows(sites, seen):
|
|
211
|
+
"""Each call site in ``sites`` as ``{ "site", "following", "followingRead" }``, from its instruction in ``seen``."""
|
|
212
|
+
return [{"site": site, "following": site + seen[site].size, "followingRead": site + seen[site].size in seen}
|
|
213
|
+
for site in sites]
|
|
214
|
+
|
|
215
|
+
|
|
169
216
|
def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, read, calls, call_targets):
|
|
170
217
|
"""The ``noReturn`` report rows of the declared ``routines``, ``calls`` and ``interrupts``.
|
|
171
218
|
|
|
@@ -179,9 +226,7 @@ def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, rea
|
|
|
179
226
|
"""
|
|
180
227
|
rows = [{"routine": at, "reason": reason, "reached": at in reached, "read": at in read,
|
|
181
228
|
"returnSites": returns[at], "contradicted": bool(returns[at]),
|
|
182
|
-
"callSites":
|
|
183
|
-
"followingRead": site + seen[site].size in seen}
|
|
184
|
-
for site in call_sites.get(at, [])]}
|
|
229
|
+
"callSites": _call_site_rows(call_sites.get(at, []), seen)}
|
|
185
230
|
for at, reason in routines.items()]
|
|
186
231
|
rows += [{"call": at, "reason": row["reason"], "reached": at in seen,
|
|
187
232
|
"targets": [{"routine": target, "read": target in read, "returnSites": returns.get(target, [])}
|
|
@@ -219,6 +264,28 @@ def fewest_calls(graph, starts):
|
|
|
219
264
|
return distance, parent, routine
|
|
220
265
|
|
|
221
266
|
|
|
267
|
+
def decoded_ranges(seen, routine):
|
|
268
|
+
"""The instructions in ``seen`` as half-open ``[start, end)`` runs, each with the ``routine`` they were read in.
|
|
269
|
+
|
|
270
|
+
A run holds instructions that abut, each starting where the one before it ends, and that
|
|
271
|
+
``routine`` gives the same routine. A site on no route from a start has the routine None. Runs
|
|
272
|
+
overlap only where ``seen`` holds instructions that overlap, which the walk keeps when it proves
|
|
273
|
+
both boundaries.
|
|
274
|
+
"""
|
|
275
|
+
# Past an overlap, the run an instruction continues need not be the last one opened, so each open
|
|
276
|
+
# run is found by where it ends and in which routine.
|
|
277
|
+
ranges, ending = [], {}
|
|
278
|
+
for at, ins in sorted(seen.items()):
|
|
279
|
+
owner = routine.get(at)
|
|
280
|
+
run = ending.pop((at, owner), None)
|
|
281
|
+
if run is None:
|
|
282
|
+
run = {"start": at, "end": at, "routine": owner}
|
|
283
|
+
ranges.append(run)
|
|
284
|
+
run["end"] = at + ins.size
|
|
285
|
+
ending[(run["end"], owner)] = run
|
|
286
|
+
return ranges
|
|
287
|
+
|
|
288
|
+
|
|
222
289
|
def _dominators(order, predecessors):
|
|
223
290
|
"""Immediate dominators (Cooper, Harvey and Kennedy) of every node in ``order``, a reverse postorder from ROOT."""
|
|
224
291
|
index = {node: i for i, node in enumerate(order)}
|
|
@@ -254,9 +321,13 @@ def reach(image, config):
|
|
|
254
321
|
interrupt to the next instruction. A ``leaves`` routine is reached but never decoded. A call to a
|
|
255
322
|
``noReturn`` routine, a ``noReturn`` call site and a ``noReturn`` interrupt site do not continue
|
|
256
323
|
at the next instruction, nor does a declared indirect call that is exhaustive and whose targets
|
|
257
|
-
are all ``noReturn``.
|
|
324
|
+
are all ``noReturn``. A ``stops`` routine is reached but never decoded, and a call to it does not
|
|
325
|
+
continue at the next instruction, under the same rule as a call to a ``noReturn`` routine; a
|
|
326
|
+
``stops`` site is decoded and nothing after it is followed. A stop narrows the query and claims
|
|
327
|
+
nothing about the build, so it is neither checked nor listed among the assumptions.
|
|
258
328
|
Each reached target gets one chain with the fewest calls, the assumptions its route rests on,
|
|
259
|
-
and the routines on every route to it within the read graph.
|
|
329
|
+
and the routines on every route to it within the read graph. With ``decodedRanges`` the report
|
|
330
|
+
also gives every instruction the walk decoded, as ``decoded_ranges`` groups them.
|
|
260
331
|
"""
|
|
261
332
|
from .reports import entries
|
|
262
333
|
starts = _sites(config.get("starts"), "starts", image)
|
|
@@ -287,17 +358,33 @@ def reach(image, config):
|
|
|
287
358
|
raise ValueError(f"instruction control {at} is a start; the walk decodes every start, so it shows nothing "
|
|
288
359
|
"the walk found")
|
|
289
360
|
limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
|
|
361
|
+
with_ranges = config.get("decodedRanges", False)
|
|
362
|
+
if not isinstance(with_ranges, bool):
|
|
363
|
+
raise ValueError("decodedRanges must be a boolean")
|
|
290
364
|
instruction_limit = config.get("instructionLimit", 10000)
|
|
291
365
|
no_return_routines, no_return_interrupts, no_return_sites, declared = no_return_inputs(
|
|
292
366
|
config.get("noReturn", []), image, indirect_calls)
|
|
367
|
+
stop_routines, stop_sites = _stops(config.get("stops", []), image, set(starts), leaves, no_return_routines)
|
|
368
|
+
# A call-site control must be resolved, and the walk follows nothing at a stop site.
|
|
369
|
+
for at in controls:
|
|
370
|
+
if at in stop_sites:
|
|
371
|
+
raise ValueError(f"control {at} is a stop site, whose call the walk does not follow; give it as an "
|
|
372
|
+
"instructionControls entry")
|
|
373
|
+
# On the walk's terms a stop routine is a leaf that never returns; the report keeps the two apart.
|
|
374
|
+
unread = frozenset(leaves) | frozenset(stop_routines)
|
|
375
|
+
options = declared | {"no_return_calls": declared["no_return_calls"] | frozenset(stop_routines)}
|
|
293
376
|
seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
|
|
294
|
-
stops=frozenset(
|
|
377
|
+
stops=unread, ends=frozenset(stop_sites), **options)
|
|
295
378
|
|
|
296
379
|
# Each reached call site with its resolved or declared targets.
|
|
297
|
-
graph, unresolved, interrupts, call_targets = {}, [], [], {}
|
|
380
|
+
graph, unresolved, interrupts, call_targets, stop_successors = {}, [], [], {}, {}
|
|
298
381
|
for at, ins in sorted(seen.items()):
|
|
299
|
-
step = cfg_step(image, at, ins, follow_interrupts=True, **
|
|
300
|
-
|
|
382
|
+
step = cfg_step(image, at, ins, follow_interrupts=True, **options)
|
|
383
|
+
if at in stop_sites:
|
|
384
|
+
# Nothing at a stop site is followed, so its transfer is neither resolved nor unresolved.
|
|
385
|
+
graph[at], stop_successors[at] = [], step.successors
|
|
386
|
+
continue
|
|
387
|
+
graph[at] = [(s, kind) for s, kind in successor_kinds(at, ins, step) if s in seen or s in unread]
|
|
301
388
|
m = base_mnemonic(ins)
|
|
302
389
|
text = _text(ins)
|
|
303
390
|
kind = "call" if m in CALLS else "return" if m in RETURNS else "jump"
|
|
@@ -312,7 +399,7 @@ def reach(image, config):
|
|
|
312
399
|
if m in INTERRUPTS and at not in no_return_interrupts:
|
|
313
400
|
vector, conditional = interrupt_vector(image.flat, ins, at)
|
|
314
401
|
interrupts.append({"site": at, "instruction": text, "vector": vector, "conditional": conditional})
|
|
315
|
-
for at in
|
|
402
|
+
for at in unread:
|
|
316
403
|
graph.setdefault(at, [])
|
|
317
404
|
|
|
318
405
|
distance, parent, routine = fewest_calls(graph, starts)
|
|
@@ -380,7 +467,8 @@ def reach(image, config):
|
|
|
380
467
|
if at in routine_starts:
|
|
381
468
|
cut.append(at)
|
|
382
469
|
at = idom[at]
|
|
383
|
-
row |= {"reached": True, "routine": routine[target], "leaf": target in leaves,
|
|
470
|
+
row |= {"reached": True, "routine": routine[target], "leaf": target in leaves,
|
|
471
|
+
"stop": target in stop_routines or target in stop_sites, **chain(target),
|
|
384
472
|
"throughEveryRoute": cut[::-1]}
|
|
385
473
|
else:
|
|
386
474
|
row |= {"reached": False, **unreached(target)}
|
|
@@ -393,14 +481,22 @@ def reach(image, config):
|
|
|
393
481
|
# The walk never decodes a leaf, so its overlap check cannot see a leaf start inside a decoded instruction.
|
|
394
482
|
gaps += [{"site": at, "reason": LEAF_OVERLAP, "insideInstruction": holding_instruction(seen, at)}
|
|
395
483
|
for at in sorted(leaves) if at in distance and holding_instruction(seen, at) is not None]
|
|
484
|
+
gaps += [{"site": at, "reason": STOP_OVERLAP, "insideInstruction": holding_instruction(seen, at)}
|
|
485
|
+
for at in sorted(stop_routines) if at in distance and holding_instruction(seen, at) is not None]
|
|
396
486
|
# Kept apart from the gap rows, which the result limit can cut.
|
|
397
487
|
stopped = any(g["reason"] == LIMIT_REASON for g in walk_gaps)
|
|
398
488
|
# A control at a site a noReturn declaration kept the walk from names that declaration, which may be the cause.
|
|
399
489
|
def ends(at, found):
|
|
400
|
-
return call_ends(at, found[0], indirect_calls,
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
490
|
+
return call_ends(at, found[0], indirect_calls, options["no_return_calls"])
|
|
491
|
+
|
|
492
|
+
def ended_by(at, found):
|
|
493
|
+
parts = []
|
|
494
|
+
for label, named_routines in (("noReturn", no_return_routines), ("stop", stop_routines)):
|
|
495
|
+
named = [str(target) for target in found if target in named_routines]
|
|
496
|
+
if named:
|
|
497
|
+
parts.append(f"{label} routine{'s' if len(named) > 1 else ''} " + ", ".join(named))
|
|
498
|
+
return f"the call at {at} to " + " and ".join(parts)
|
|
499
|
+
cut_by = {at + seen[at].size: ended_by(at, found) for at, found in sorted(call_targets.items()) if ends(at, found)}
|
|
404
500
|
# A call to a noReturn routine can be declared as a call too; the message then names both declarations.
|
|
405
501
|
for at, row in no_return_sites.items():
|
|
406
502
|
if at in seen:
|
|
@@ -408,6 +504,10 @@ def reach(image, config):
|
|
|
408
504
|
cut_by[row["following"]] = f"{cut_by[row['following']]} and {cause}" if row["following"] in cut_by else cause
|
|
409
505
|
cut_by |= {row["following"]: f"the noReturn interrupt at {at}"
|
|
410
506
|
for at, row in no_return_interrupts.items() if at in seen}
|
|
507
|
+
# A site past a stop site may be reached by no other route; the first stop names the cut.
|
|
508
|
+
for at, found in sorted(stop_successors.items()):
|
|
509
|
+
for successor in found:
|
|
510
|
+
cut_by.setdefault(successor, f"the stop at {at}")
|
|
411
511
|
# Not reached and reached-but-unresolved need different fixes, so each failure says which it is.
|
|
412
512
|
failures = []
|
|
413
513
|
for label, sites, passed in (("control", controls, call_targets), ("instruction control", instruction_controls, seen)):
|
|
@@ -420,6 +520,9 @@ def reach(image, config):
|
|
|
420
520
|
failures.append(f"{label} {at} is reached but its call target is unresolved ({'; '.join(reasons)})")
|
|
421
521
|
elif at in leaves and at in distance:
|
|
422
522
|
failures.append(f"{label} {at} is the start of a leaf, which the walk reaches but does not decode")
|
|
523
|
+
elif at in stop_routines and at in distance:
|
|
524
|
+
failures.append(f"{label} {at} is the start of a stop routine, which the walk reaches but does not "
|
|
525
|
+
"decode")
|
|
423
526
|
else:
|
|
424
527
|
state = unreached(at)
|
|
425
528
|
detail = (f"inside the reached instruction at {state['insideInstruction']}" if "insideInstruction" in state
|
|
@@ -440,19 +543,49 @@ def reach(image, config):
|
|
|
440
543
|
ended_sites.setdefault(target, []).append(site)
|
|
441
544
|
leaf_rows = [{"routine": at, "reason": reason, "reached": at in distance, "callSites": call_sites.get(at, [])}
|
|
442
545
|
for at, reason in leaves.items()]
|
|
546
|
+
stop_rows = [{"routine": at, "reason": reason, "reached": at in distance,
|
|
547
|
+
"callSites": _call_site_rows(call_sites.get(at, []), seen)}
|
|
548
|
+
for at, reason in stop_routines.items()]
|
|
549
|
+
for at, reason in stop_sites.items():
|
|
550
|
+
ins = image.decode(at)
|
|
551
|
+
found = stop_successors[at] if at in seen else cfg_step(image, at, ins, follow_interrupts=True,
|
|
552
|
+
**options).successors
|
|
553
|
+
stop_rows.append({"site": at, "reason": reason, "reached": at in seen, "instruction": _text(ins),
|
|
554
|
+
"routine": routine.get(at), "successors": [{"site": s, "read": s in seen} for s in found]})
|
|
443
555
|
# Each call the declaration kept from its return site, so a reviewer can check what follows it.
|
|
444
556
|
returning_leaves = set(leaves) - set(no_return_routines)
|
|
445
557
|
site_targets = {at: call_targets.get(at, []) for at in no_return_sites}
|
|
446
558
|
checked = set(no_return_routines) | {target for found in site_targets.values() for target in found}
|
|
447
|
-
|
|
448
|
-
|
|
559
|
+
check_graph, read, check_stopped, check_gap_rows = graph, seen, False, []
|
|
560
|
+
# A stop the walk never met cut nothing, so the walk read what a walk without stops reads.
|
|
561
|
+
stop_met = (any(at in distance for at in stop_routines)
|
|
562
|
+
or any(at in seen or at in contested or at in overlapping for at in stop_sites))
|
|
563
|
+
if stop_met and checked:
|
|
564
|
+
# A stop cuts the query and says nothing about the build, so the return check reads the reached
|
|
565
|
+
# declared routines without the stops: code a stop hid can still contradict a declaration.
|
|
566
|
+
read, check_gaps, _, _, check_contested = walk(image, sorted(at for at in checked if at in distance),
|
|
567
|
+
instruction_limit, follow_interrupts=True,
|
|
568
|
+
stops=frozenset(leaves), **declared)
|
|
569
|
+
check_stopped = any(g["reason"] == LIMIT_REASON for g in check_gaps)
|
|
570
|
+
# What the check could not read past a stop leaves its declarations unchecked there, as a gap of the
|
|
571
|
+
# query's own walk would; the rows the query's walk already holds are not repeated.
|
|
572
|
+
known = {(g["site"], g["reason"]) for g in walk_gaps}
|
|
573
|
+
check_gap_rows = sorted(
|
|
574
|
+
[g for g in check_gaps if g["reason"] != LIMIT_REASON and (g["site"], g["reason"]) not in known
|
|
575
|
+
and g["site"] not in check_contested]
|
|
576
|
+
+ [{"site": at, "reason": CHECK_CONTESTED} for at in check_contested if at not in contested],
|
|
577
|
+
key=lambda g: (g["site"], g["reason"]))
|
|
578
|
+
check_graph = successor_graph(image, read, kept=frozenset(leaves), follow_interrupts=True, **declared)
|
|
579
|
+
returns = {at: returns_from(check_graph, read, returning_leaves, at) for at in checked}
|
|
580
|
+
no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, ended_sites, seen, distance, read,
|
|
449
581
|
no_return_sites, site_targets)
|
|
450
582
|
contradicted = any(row.get("contradicted") for row in no_return)
|
|
451
583
|
# A declared target the walk entered but established no instruction at shows in gaps or contested too,
|
|
452
584
|
# unless the walk stopped at its instruction limit before reading it.
|
|
453
585
|
indirect_rows = [declaration | {"reached": site in seen, "routine": routine.get(site),
|
|
454
586
|
"unreadTargets": [t for t in declaration["targets"]
|
|
455
|
-
if site in seen and
|
|
587
|
+
if site in seen and site not in stop_sites
|
|
588
|
+
and t not in seen and t not in unread]}
|
|
456
589
|
for site, declaration in indirect_calls.items()]
|
|
457
590
|
counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
|
|
458
591
|
"interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
|
|
@@ -466,17 +599,23 @@ def reach(image, config):
|
|
|
466
599
|
assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
|
|
467
600
|
"it gives, and only those when it is declared exhaustive")
|
|
468
601
|
assumptions += no_return_assumptions(no_return_routines, no_return_interrupts, no_return_sites)
|
|
469
|
-
|
|
470
|
-
|
|
602
|
+
stopped = stopped or check_stopped
|
|
603
|
+
# Every range is kept whatever the result limit, so a caller can place any site in the walk; the
|
|
604
|
+
# instruction limit bounds their number.
|
|
605
|
+
ranges = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
|
|
606
|
+
return {"starts": starts, "targets": rows, "leaves": leaf_rows, "stops": stop_rows, "noReturn": no_return,
|
|
607
|
+
"indirectCalls": indirect_rows,
|
|
608
|
+
"reachedRoutines": sorted(routine_starts), **ranges, "counts": counts,
|
|
471
609
|
"unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
|
|
472
|
-
"contested": sorted(contested)[:limit],
|
|
473
|
-
"truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested)),
|
|
610
|
+
"contested": sorted(contested)[:limit], "returnCheckGaps": check_gap_rows[:limit],
|
|
611
|
+
"truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested, check_gap_rows)),
|
|
474
612
|
"instructionLimitReached": stopped,
|
|
475
613
|
"controls": [{"site": at, "target": call_targets[at][0]} for at in controls],
|
|
476
614
|
"instructionControls": [{"site": at, "instruction": _text(seen[at]), "routine": routine.get(at)}
|
|
477
615
|
for at in instruction_controls],
|
|
478
616
|
"negativeUsable": bool(controls or instruction_controls)
|
|
479
|
-
and not (stopped or unresolved or gaps or contested or contradicted
|
|
617
|
+
and not (stopped or unresolved or gaps or contested or contradicted
|
|
618
|
+
or check_gap_rows),
|
|
480
619
|
"assumptions": assumptions,
|
|
481
620
|
"exclusions": ["computed call and jump targets no declaration gives (listed in unresolved)", "unrelocated far calls",
|
|
482
621
|
"code reached only from outside the starts", "runtime reachability"],
|
|
@@ -484,7 +623,7 @@ def reach(image, config):
|
|
|
484
623
|
"throughEveryRoute lists the routine starts every read route to the target passes, "
|
|
485
624
|
"and an unresolved transfer may add a route that passes none of them. "
|
|
486
625
|
"negativeUsable needs a control of either kind, a walk that did not stop at its instruction limit, "
|
|
487
|
-
"and no unresolved transfer, gap, contested instruction or contradicted noReturn "
|
|
626
|
+
"and no unresolved transfer, gap, returnCheckGaps row, contested instruction or contradicted noReturn "
|
|
488
627
|
"routine, and still rests on the listed assumptions, leaves and noReturn declarations. "
|
|
489
628
|
"A declared indirect call counts as resolved when it is declared exhaustive: its table "
|
|
490
629
|
"rows are read from the build's bytes and its targets list rests on its evidence alone, "
|
|
@@ -495,6 +634,18 @@ def reach(image, config):
|
|
|
495
634
|
"A noReturn call is never contradicted, since whether its target returns to it can "
|
|
496
635
|
"depend on what the caller passes, which the walk does not follow; the return sites "
|
|
497
636
|
"of its targets are listed, and the declaration rests on its reason alone. "
|
|
498
|
-
"
|
|
499
|
-
"
|
|
500
|
-
"
|
|
637
|
+
"A stop narrows the question and claims nothing about the build: the walk reads no "
|
|
638
|
+
"further at a stop site, nor into a stop routine or past a call to it, so a target "
|
|
639
|
+
"reached only past a stop is not reached, and negativeUsable answers the narrowed "
|
|
640
|
+
"question. The noReturn return check reads past the stops, with a walk of its own, and "
|
|
641
|
+
"returnCheckGaps lists what that walk could not read that the query's walk did not "
|
|
642
|
+
"record; any row keeps negativeUsable false, as a gap would. "
|
|
643
|
+
"decodedRanges, when asked for, covers the instructions the walk decoded; no range "
|
|
644
|
+
"starts at a leaf, a stop routine or a contested, unresolved overlapping or undecodable start, though "
|
|
645
|
+
"one can lie inside a decoded instruction. A site inside a range lies in a decoded "
|
|
646
|
+
"instruction, which may start before it; a target at the site shows whether one starts "
|
|
647
|
+
"there. "
|
|
648
|
+
"When instructionLimitReached holds, the walk, or the return check's walk, stopped "
|
|
649
|
+
"before reading all it reaches: every list and count covers only the part read, which "
|
|
650
|
+
"part depends on the walk order, an unreached target may lie past the stop, and an "
|
|
651
|
+
"uncontradicted noReturn routine may return in code the check did not read."}
|