scientific-method-engine 18.4.0__tar.gz → 19.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/PKG-INFO +1 -1
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/pyproject.toml +1 -1
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/cli.py +7 -6
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/inventory.py +135 -41
- scientific_method_engine-19.1.0/src/scientific_method_engine/x86/ranges.py +592 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/reach.py +78 -23
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/reports.py +5 -3
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/trace.py +14 -9
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_inventory_check.py +116 -4
- scientific_method_engine-19.1.0/tests/test_ranges.py +419 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_reach.py +95 -2
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/.gitignore +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/LICENSE +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/README.md +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/machine.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/oracle.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_config_encoding.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_enter_frames.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_indirect_far_transfers.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_interrupt_models.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_join_terms.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_return_checks.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_term_limit.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_upstream_gaps.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_volatile_memory.py +0 -0
- {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version:
|
|
3
|
+
Version: 19.1.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "
|
|
8
|
+
version = "19.1.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -24,9 +24,9 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
24
24
|
"reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
|
|
25
25
|
"and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
|
|
26
26
|
"a leaves routine is reached but not read, and its reason is repeated in the report;\n"
|
|
27
|
-
"a call to a noReturn routine and a noReturn interrupt do not continue at the next
|
|
28
|
-
"
|
|
29
|
-
"is reported contradicted;\n"
|
|
27
|
+
"a call to a noReturn routine, a noReturn call and a noReturn interrupt do not continue at the next\n"
|
|
28
|
+
"instruction, a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
|
|
29
|
+
"is reported contradicted, and a noReturn call lists its targets' return sites and is never contradicted;\n"
|
|
30
30
|
"indirectCalls declares the targets of a computed call as a table read from the build or a list,\n"
|
|
31
31
|
"and the walk calls each of them; one not declared exhaustive stays unresolved;\n"
|
|
32
32
|
"controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
|
|
@@ -36,9 +36,10 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
36
36
|
"with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
|
|
37
37
|
"a target inside an instruction the entry-path walk established from another start names it as insideInstruction.\n"
|
|
38
38
|
"It lists each row whose start lies inside an instruction the walk established (rowStarts), with the\n"
|
|
39
|
-
"routine the walk read it in
|
|
40
|
-
"
|
|
41
|
-
"
|
|
39
|
+
"routine the walk read it in, and each row start at or inside an instruction of an overlap the walk left\n"
|
|
40
|
+
"unresolved, with the instructions it overlaps; a row start in bytes the walk did not decode is counted, not placed.\n"
|
|
41
|
+
"noReturn takes reach's declarations: the walk stops at a call to a declared routine, at a declared call\n"
|
|
42
|
+
"and at a declared interrupt, a declared routine with a return on its own read paths is reported contradicted, and\n"
|
|
42
43
|
"rowsPastNoReturn lists each row whose body continues past such a call or interrupt.\n"
|
|
43
44
|
"indirectCalls takes reach's declarations: each declared target of a reached computed call is a call target.\n"
|
|
44
45
|
"uses matches accesses whose footprint intersects query offset..offset+width; its rawCandidates give each unreached\n"
|
|
@@ -3,7 +3,7 @@ import re
|
|
|
3
3
|
from bisect import bisect_right
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from .image import integer
|
|
6
|
-
from .trace import LIMIT_REASON, CONTESTED_REASON, walk, holding_instruction
|
|
6
|
+
from .trace import LIMIT_REASON, CONTESTED_REASON, OVERLAP_REASON, walk, holding_instruction
|
|
7
7
|
from .dispatch import indirect_call_declarations, call_ends
|
|
8
8
|
|
|
9
9
|
MAX_INVENTORY = 16 * 1024 * 1024
|
|
@@ -139,15 +139,53 @@ def _row_fields(row):
|
|
|
139
139
|
return {k: row[k] for k in ("start", "size", "name") if k in row}
|
|
140
140
|
|
|
141
141
|
|
|
142
|
-
def
|
|
143
|
-
"""
|
|
142
|
+
def _routine_fields(image, starts, routine, prefix=""):
|
|
143
|
+
"""The ``routine``, ``routineAddress`` and ``routineIsRow`` fields for a routine start, or none when it is None.
|
|
144
|
+
|
|
145
|
+
A ``prefix`` names the fields after it, as ``rowStartRoutine`` for ``prefix="rowStart"``.
|
|
146
|
+
"""
|
|
147
|
+
if routine is None:
|
|
148
|
+
return {}
|
|
149
|
+
place = _target_place(image, routine)
|
|
150
|
+
name = prefix + "Routine" if prefix else "routine"
|
|
151
|
+
return {name: routine, name + "Address": place[2], name + "IsRow": place[:2] in starts}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _overlaps(image, starts, at, decoded, routine_of):
|
|
155
|
+
"""Every instruction in ``decoded`` other than the one at ``at`` whose bytes meet the bytes of the
|
|
156
|
+
instruction at ``at``, or the byte ``at`` when no instruction starts there, with its evidence.
|
|
157
|
+
|
|
158
|
+
``decoded`` maps a site to ``(instruction, evidence)``.
|
|
159
|
+
"""
|
|
160
|
+
end = at + decoded[at][0].size if at in decoded else at + 1
|
|
161
|
+
found = []
|
|
162
|
+
# An x86 instruction is at most 15 bytes, so only the starts just before the site can reach into it.
|
|
163
|
+
for site in range(max(at - 14, 0), end):
|
|
164
|
+
if site == at or site not in decoded or site + decoded[site][0].size <= at:
|
|
165
|
+
continue
|
|
166
|
+
ins, evidence = decoded[site]
|
|
167
|
+
found.append({"site": site, "address": _target_place(image, site)[2], "size": ins.size, "text": _text(ins),
|
|
168
|
+
"evidence": evidence, **_routine_fields(image, starts, routine_of(site))})
|
|
169
|
+
return found
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _row_starts(image, rows, starts, domains, seen, contested, unresolved, routine_of):
|
|
173
|
+
"""Each row whose start lies inside an instruction the entry-path walk established, or at or inside
|
|
174
|
+
an instruction of an overlap the walk left unresolved, and counts of the rest.
|
|
144
175
|
|
|
145
176
|
A start is checked against the walk's instructions only; bytes the walk did not decode are not
|
|
146
|
-
decoded here, so a row start in them is counted as not read rather than placed.
|
|
147
|
-
|
|
177
|
+
decoded here, so a row start in them is counted as not read rather than placed. ``unresolved``
|
|
178
|
+
holds the instructions at the walk's overlap gaps, which it decoded on the entry path and
|
|
179
|
+
rejected because no boundary between them is proven. A row start at or inside one is listed with
|
|
180
|
+
every instruction it overlaps, since the row's own entry can be one side of the overlap. A start
|
|
181
|
+
at or inside an instruction the walk decoded but rejected as contested is counted as contested.
|
|
148
182
|
"""
|
|
149
183
|
found, counts = [], {"instructionStarts": 0, "insideAnInstruction": 0, "overlappingInstructionStarts": 0,
|
|
150
|
-
"contested": 0, "notRead": 0}
|
|
184
|
+
"unresolvedOverlaps": 0, "contested": 0, "notRead": 0}
|
|
185
|
+
# Every instruction the walk decoded, with its evidence. Only a row start at an unresolved overlap reads it.
|
|
186
|
+
decoded = ({at: (ins, CONTESTED_REASON) for at, ins in contested.items()}
|
|
187
|
+
| {at: (ins, OVERLAP_REASON) for at, ins in unresolved.items()}
|
|
188
|
+
| {at: (ins, "entry-path instruction") for at, ins in seen.items()}) if unresolved else {}
|
|
151
189
|
for row in rows:
|
|
152
190
|
offsets = _offsets(image, domains, row["space"], row["at"])
|
|
153
191
|
if not offsets:
|
|
@@ -156,6 +194,25 @@ def _row_starts(image, rows, starts, domains, seen, contested, routine_of):
|
|
|
156
194
|
if held is None:
|
|
157
195
|
if any(at in seen for at in offsets):
|
|
158
196
|
counts["instructionStarts"] += 1
|
|
197
|
+
continue
|
|
198
|
+
hit = next(((at, holder) for at in offsets for holder in [holding_instruction(unresolved, at)]
|
|
199
|
+
if at in unresolved or holder is not None), None)
|
|
200
|
+
if hit is not None:
|
|
201
|
+
at, holder = hit
|
|
202
|
+
counts["unresolvedOverlaps"] += 1
|
|
203
|
+
entry = {**_row_fields(row), "site": at}
|
|
204
|
+
if at in unresolved:
|
|
205
|
+
entry |= {"status": "start of an unresolved overlapping instruction",
|
|
206
|
+
"rowStartInstructionSize": unresolved[at].size, "rowStartInstructionText": _text(unresolved[at]),
|
|
207
|
+
**_routine_fields(image, starts, routine_of(at), "rowStart")}
|
|
208
|
+
else:
|
|
209
|
+
entry["status"] = "inside an unresolved overlapping instruction"
|
|
210
|
+
if holder is not None:
|
|
211
|
+
entry |= {"insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
|
|
212
|
+
"insideInstructionSize": unresolved[holder].size,
|
|
213
|
+
"insideInstructionText": _text(unresolved[holder]),
|
|
214
|
+
**_routine_fields(image, starts, routine_of(holder))}
|
|
215
|
+
found.append(entry | {"overlaps": _overlaps(image, starts, at, decoded, routine_of)})
|
|
159
216
|
elif any(at in contested or holding_instruction(contested, at) is not None for at in offsets):
|
|
160
217
|
counts["contested"] += 1
|
|
161
218
|
else:
|
|
@@ -164,27 +221,25 @@ def _row_starts(image, rows, starts, domains, seen, contested, routine_of):
|
|
|
164
221
|
at, holder = held
|
|
165
222
|
overlapping = at in seen
|
|
166
223
|
counts["overlappingInstructionStarts" if overlapping else "insideAnInstruction"] += 1
|
|
167
|
-
routine = routine_of(holder)
|
|
168
224
|
entry = {**_row_fields(row), "site": at,
|
|
169
225
|
"status": "start of an overlapping instruction" if overlapping else "inside an instruction",
|
|
170
226
|
"insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
|
|
171
227
|
"insideInstructionSize": seen[holder].size, "insideInstructionText": _text(seen[holder])}
|
|
172
228
|
if overlapping:
|
|
173
|
-
entry |= {"rowStartInstructionSize": seen[at].size, "rowStartInstructionText": _text(seen[at])
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
entry |= {"routine": routine, "routineAddress": place[2], "routineIsRow": place[:2] in starts}
|
|
177
|
-
found.append(entry)
|
|
229
|
+
entry |= {"rowStartInstructionSize": seen[at].size, "rowStartInstructionText": _text(seen[at]),
|
|
230
|
+
**_routine_fields(image, starts, routine_of(at), "rowStart")}
|
|
231
|
+
found.append(entry | _routine_fields(image, starts, routine_of(holder)))
|
|
178
232
|
return found, counts
|
|
179
233
|
|
|
180
234
|
|
|
181
235
|
def _no_return_check(image, routines, declared, limit):
|
|
182
|
-
"""``(returns, read, stopped)``: the return sites on each
|
|
236
|
+
"""``(returns, read, stopped)``: the return sites on each of ``routines``' own read paths, the
|
|
183
237
|
instructions the walk established, and whether the walk stopped.
|
|
184
238
|
|
|
185
|
-
|
|
186
|
-
one, as ``reach`` reads them, so a
|
|
187
|
-
|
|
239
|
+
``routines`` are the declared routines, or the targets of the declared calls. The walk starts at
|
|
240
|
+
them and continues past every interrupt except a declared one, as ``reach`` reads them, so a
|
|
241
|
+
routine that ends in an interrupt that returns is contradicted by what follows it. Calls are
|
|
242
|
+
stepped over at their return sites, except calls to declared routines and declared calls.
|
|
188
243
|
A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
|
|
189
244
|
has no return sites and is not in ``read``.
|
|
190
245
|
"""
|
|
@@ -212,27 +267,28 @@ def inventory_check(image, config):
|
|
|
212
267
|
the first of an instruction the entry-path walk established names that instruction.
|
|
213
268
|
|
|
214
269
|
The rows are checked against the same walk: each row whose start lies inside an instruction the
|
|
215
|
-
walk established
|
|
216
|
-
|
|
270
|
+
walk established, or at or inside an instruction of an overlap the walk left unresolved, is listed
|
|
271
|
+
in ``rowStarts``. ``noReturn`` declarations, in ``reach``'s shape, end the walk at a call to a
|
|
272
|
+
declared routine, at a declared call site and at a declared interrupt site, are checked for a
|
|
217
273
|
return on the routine's own paths, and list in ``rowsPastNoReturn`` each row whose body
|
|
218
274
|
continues past such a call or interrupt. ``indirectCalls`` declarations, in ``reach``'s shape,
|
|
219
275
|
make each declared target of a reached computed call a call target, and the walk reads on in them.
|
|
220
276
|
"""
|
|
221
277
|
from .reports import call_controls, direct_calls, entries, search_coverage
|
|
222
|
-
from .reach import
|
|
278
|
+
from .reach import (no_return_inputs, no_return_rows, successor_graph, fewest_calls, call_assumption,
|
|
279
|
+
no_return_assumptions)
|
|
223
280
|
if "inventory" not in config:
|
|
224
281
|
raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
|
|
225
282
|
rows = read_inventory(config["inventory"], image)
|
|
226
283
|
limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
|
|
227
|
-
no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
|
|
228
284
|
indirect_calls = indirect_call_declarations(config.get("indirectCalls", []), image)
|
|
285
|
+
no_return_routines, no_return_interrupts, no_return_sites, declared = no_return_inputs(
|
|
286
|
+
config.get("noReturn", []), image, indirect_calls)
|
|
229
287
|
for at in config.get("controls", []) if isinstance(config.get("controls"), list) else []:
|
|
230
288
|
# call_controls refuses a control that is no integer.
|
|
231
289
|
if type(at) is int and at in indirect_calls:
|
|
232
290
|
raise ValueError(f"Positive control {at} is a declared indirect call, whose targets rest on its "
|
|
233
291
|
"declaration; a control must resolve from its own encoding")
|
|
234
|
-
declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts),
|
|
235
|
-
"indirect_calls": indirect_calls}
|
|
236
292
|
calls = direct_calls(image, config, **declared)
|
|
237
293
|
|
|
238
294
|
def ends_branch(row):
|
|
@@ -310,27 +366,49 @@ def inventory_check(image, config):
|
|
|
310
366
|
entry["rows"] = [_row_fields(rows[i]) for i in held]
|
|
311
367
|
missing.append(entry)
|
|
312
368
|
|
|
369
|
+
# The instructions of the overlaps the walk left unresolved. The walk decoded each at its gap site.
|
|
370
|
+
overlapping = {g["site"]: image.decode(g["site"]) for g in calls["gaps"] if g.get("reason") == OVERLAP_REASON}
|
|
371
|
+
|
|
313
372
|
# The routine the walk read an instruction in, by the route with the fewest calls, as reach names it.
|
|
373
|
+
# An unresolved instruction is a node that ends its route, so it gets the routine of the established
|
|
374
|
+
# instruction that steps into it, or its own when it is an entry, and no route passes through it.
|
|
314
375
|
# The graph is built only when a row start needs it.
|
|
315
376
|
routines = None
|
|
316
377
|
|
|
317
378
|
def routine_of(site):
|
|
318
379
|
nonlocal routines
|
|
319
380
|
if routines is None:
|
|
320
|
-
graph = successor_graph(image, seen, **declared)
|
|
321
|
-
|
|
381
|
+
graph = successor_graph(image, seen | overlapping, **declared)
|
|
382
|
+
graph = {at: [] if at in overlapping else successors for at, successors in graph.items()}
|
|
383
|
+
routines = fewest_calls(graph, [at for at in entries(image) if at in seen or at in overlapping])[2]
|
|
322
384
|
return routines.get(site)
|
|
323
|
-
row_starts, start_counts = _row_starts(image, rows, starts, domains, seen, contested, routine_of)
|
|
385
|
+
row_starts, start_counts = _row_starts(image, rows, starts, domains, seen, contested, overlapping, routine_of)
|
|
324
386
|
|
|
325
|
-
# Each declaration with the calls it kept from their return sites, as reach reports them
|
|
326
|
-
|
|
327
|
-
|
|
387
|
+
# Each declaration with the calls it kept from their return sites, as reach reports them: only a call the
|
|
388
|
+
# entry-path walk established has targets, so a raw byte candidate at an unreached declared site gives none.
|
|
389
|
+
site_targets = {at: [] for at in no_return_sites}
|
|
390
|
+
for row in calls["rows"]:
|
|
391
|
+
if (row["site"] in site_targets and row["site"] in seen and row["target"] is not None
|
|
392
|
+
and row["target"] not in site_targets[row["site"]]):
|
|
393
|
+
site_targets[row["site"]].append(row["target"])
|
|
394
|
+
# The targets of the declared calls get a walk of their own, so that their code, overlapping a declared
|
|
395
|
+
# routine's or spending the limit first, cannot change what the routine check reads.
|
|
396
|
+
call_targets = {target for found in site_targets.values() for target in found} - set(no_return_routines)
|
|
397
|
+
checked = set(no_return_routines) | call_targets
|
|
398
|
+
check_limit = config.get("instructionLimit", 10000)
|
|
399
|
+
returns, routine_read, routine_stopped = _no_return_check(image, no_return_routines, declared, check_limit)
|
|
400
|
+
target_returns, target_read, target_stopped = _no_return_check(image, call_targets, declared, check_limit)
|
|
401
|
+
returns |= target_returns
|
|
402
|
+
check_read = ({at for at in no_return_routines if at in routine_read}
|
|
403
|
+
| {at for at in call_targets if at in target_read})
|
|
404
|
+
check_stopped = routine_stopped or target_stopped
|
|
328
405
|
entry_calls = {}
|
|
329
406
|
for row in calls["rows"]:
|
|
330
407
|
if row["target"] in no_return_routines and row["site"] in seen and ends_branch(row):
|
|
331
408
|
entry_calls.setdefault(row["target"], []).append(row["site"])
|
|
332
409
|
no_return = no_return_rows(no_return_routines, no_return_interrupts, returns,
|
|
333
|
-
{at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read
|
|
410
|
+
{at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read,
|
|
411
|
+
no_return_sites, site_targets)
|
|
334
412
|
|
|
335
413
|
# A row whose body holds a declared call or interrupt and the byte after it still covers what follows.
|
|
336
414
|
# A declared computed call has one row for each target, and is listed once with all of them.
|
|
@@ -343,8 +421,14 @@ def inventory_check(image, config):
|
|
|
343
421
|
declared_ends.add(row["site"])
|
|
344
422
|
routine = list(indirect_calls[row["site"]]["targets"])
|
|
345
423
|
ends.append((row["site"], image.decode(row["site"]).size, "call", routine, row["classification"]))
|
|
346
|
-
|
|
347
|
-
|
|
424
|
+
# A declared call site is listed as one, whatever its targets, unless a routine declaration already listed it.
|
|
425
|
+
def site_classification(at):
|
|
426
|
+
return "entry-path instruction" if at in seen else CONTESTED_REASON if at in contested else "declared site"
|
|
427
|
+
listed = {site for site, *_ in ends}
|
|
428
|
+
ends += [(at, row["following"] - at, "call site",
|
|
429
|
+
site_targets[at][0] if len(site_targets[at]) == 1 else site_targets[at] or None, site_classification(at))
|
|
430
|
+
for at, row in no_return_sites.items() if at not in listed]
|
|
431
|
+
ends += [(at, row["following"] - at, "interrupt", None, site_classification(at))
|
|
348
432
|
for at, row in no_return_interrupts.items()]
|
|
349
433
|
# Only a site the entry-path walk established counts a row; the rest are listed with their evidence.
|
|
350
434
|
past, past_rows, past_unread, past_unverified = [], set(), set(), set()
|
|
@@ -412,6 +496,12 @@ def inventory_check(image, config):
|
|
|
412
496
|
summary += (f", {start_counts['overlappingInstructionStarts']} of them at the start of an overlapping "
|
|
413
497
|
"instruction it also established")
|
|
414
498
|
summary += "."
|
|
499
|
+
if start_counts["unresolvedOverlaps"]:
|
|
500
|
+
summary += " " + _count(start_counts["unresolvedOverlaps"],
|
|
501
|
+
"row start lies at or inside an instruction of an overlap the walk left unresolved, "
|
|
502
|
+
"listed with the instructions it overlaps.",
|
|
503
|
+
"row starts lie at or inside an instruction of an overlap the walk left unresolved, "
|
|
504
|
+
"each listed with the instructions it overlaps.")
|
|
415
505
|
if start_counts["notRead"]:
|
|
416
506
|
summary += " " + _count(start_counts["notRead"], "row start in declared code lies", "row starts in declared code lie") \
|
|
417
507
|
+ " in bytes where the walk established no instruction, so its boundary is not checked."
|
|
@@ -433,16 +523,17 @@ def inventory_check(image, config):
|
|
|
433
523
|
if unread:
|
|
434
524
|
summary += " " + _count(unread, "noReturn routine is", "noReturn routines are") \
|
|
435
525
|
+ " not an instruction the check walk established, so its return check reads nothing."
|
|
436
|
-
if
|
|
526
|
+
if routine_stopped:
|
|
437
527
|
summary += " The walk that checks the noReturn routines stopped at its instruction limit, so it can miss returns."
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
528
|
+
if target_stopped:
|
|
529
|
+
summary += (" The walk that checks the targets of the noReturn calls stopped at its instruction limit, so it "
|
|
530
|
+
"can miss returns.")
|
|
531
|
+
assumptions = [call_assumption(no_return_routines, no_return_sites)]
|
|
532
|
+
if checked:
|
|
441
533
|
assumptions.append("each interrupt the noReturn check reads returns to its next instruction"
|
|
442
534
|
+ (", except at a noReturn interrupt site" if no_return_interrupts else "")
|
|
443
535
|
+ "; interrupt handlers are not read")
|
|
444
|
-
|
|
445
|
-
assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
|
|
536
|
+
assumptions += no_return_assumptions(no_return_routines, no_return_interrupts, no_return_sites)
|
|
446
537
|
if indirect_calls:
|
|
447
538
|
assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
|
|
448
539
|
"it gives, and only those when it is declared exhaustive")
|
|
@@ -469,10 +560,13 @@ def inventory_check(image, config):
|
|
|
469
560
|
"notation and compared with its rows. A target only raw byte candidates or contested "
|
|
470
561
|
"instructions call is not shown to be code. The counts are lower bounds on the targets "
|
|
471
562
|
"the inventory lacks: calls the search does not resolve can add more. Row starts are "
|
|
472
|
-
"compared only with the instructions the entry-path walk
|
|
473
|
-
"
|
|
474
|
-
"
|
|
563
|
+
"compared only with the instructions the entry-path walk decoded; a start at or inside an instruction "
|
|
564
|
+
"of an overlap the walk left unresolved is listed with every instruction it overlaps, and a "
|
|
565
|
+
"start where it established no instruction is counted as notRead or contested and not placed, "
|
|
566
|
+
"so linear decoding past data never reports one. A start inside an instruction can be deliberately overlapping code, "
|
|
475
567
|
"which the row names with both instructions for a reader to judge. rowsPastNoReturn rests "
|
|
476
568
|
"on the noReturn declarations: followingRead says whether the walk reached the bytes after "
|
|
477
569
|
"the call by another route, only a site the walk established counts a row, and an empty "
|
|
478
|
-
"returnSites means only that the walk read no return."
|
|
570
|
+
"returnSites means only that the walk read no return. A noReturn call is never "
|
|
571
|
+
"contradicted: the return sites of its targets are listed, and the declaration rests "
|
|
572
|
+
"on its reason alone."}
|