scientific-method-engine 18.2.0__tar.gz → 18.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/PKG-INFO +1 -1
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/pyproject.toml +1 -1
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/cli.py +5 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/inventory.py +195 -8
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/reach.py +62 -35
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/reports.py +7 -5
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_inventory_check.py +170 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/.gitignore +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/LICENSE +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/README.md +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/machine.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/trace.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/oracle.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_config_encoding.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_enter_frames.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_indirect_far_transfers.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_interrupt_models.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_join_terms.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_reach.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_return_checks.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_term_limit.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_upstream_gaps.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_volatile_memory.py +0 -0
- {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 18.
|
|
3
|
+
Version: 18.3.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "18.
|
|
8
|
+
version = "18.3.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -33,6 +33,11 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
33
33
|
"that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
|
|
34
34
|
"with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
|
|
35
35
|
"a target inside an instruction the entry-path walk established from another start names it as insideInstruction.\n"
|
|
36
|
+
"It lists each row whose start lies inside an instruction the walk established (rowStarts), with the\n"
|
|
37
|
+
"routine the walk read it in; a row start in bytes the walk did not decode is counted, not placed.\n"
|
|
38
|
+
"noReturn takes reach's declarations: the walk stops at a call to a declared routine and at a declared\n"
|
|
39
|
+
"interrupt, a declared routine with a return on its own read paths is reported contradicted, and\n"
|
|
40
|
+
"rowsPastNoReturn lists each row whose body continues past such a call or interrupt.\n"
|
|
36
41
|
"uses matches accesses whose footprint intersects query offset..offset+width; its rawCandidates give each unreached\n"
|
|
37
42
|
"operand's encoded displacement, width, access, segment register and intersection, and never count as uses.\n"
|
|
38
43
|
"trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
|
|
@@ -3,7 +3,7 @@ import re
|
|
|
3
3
|
from bisect import bisect_right
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from .image import integer
|
|
6
|
-
from .trace import LIMIT_REASON
|
|
6
|
+
from .trace import LIMIT_REASON, CONTESTED_REASON, walk, holding_instruction
|
|
7
7
|
|
|
8
8
|
MAX_INVENTORY = 16 * 1024 * 1024
|
|
9
9
|
MAX_ROWS = 200000
|
|
@@ -121,6 +121,81 @@ def _domains(image):
|
|
|
121
121
|
yield kind, low, low + r["end"] - r["start"]
|
|
122
122
|
|
|
123
123
|
|
|
124
|
+
def _offsets(image, domains, space, number):
|
|
125
|
+
"""The file offsets a place in an inventory's notation names, one for each declared region that maps it.
|
|
126
|
+
|
|
127
|
+
``domains`` is ``_domains(image)`` as a list, in the order of ``image.regions``.
|
|
128
|
+
"""
|
|
129
|
+
return [r["start"] + number - low for r, (kind, low, high) in zip(image.regions, domains)
|
|
130
|
+
if kind == space and low <= number < high]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _text(ins):
|
|
134
|
+
return (ins.mnemonic + " " + ins.op_str).strip()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _row_fields(row):
|
|
138
|
+
return {k: row[k] for k in ("start", "size", "name") if k in row}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _row_starts(image, rows, starts, domains, seen, contested, routine_of):
|
|
142
|
+
"""Each row whose start lies inside an instruction the entry-path walk established, and counts of the rest.
|
|
143
|
+
|
|
144
|
+
A start is checked against the walk's instructions only; bytes the walk did not decode are not
|
|
145
|
+
decoded here, so a row start in them is counted as not read rather than placed. A start at or
|
|
146
|
+
inside an instruction the walk decoded but rejected as contested is counted as contested.
|
|
147
|
+
"""
|
|
148
|
+
found, counts = [], {"instructionStarts": 0, "insideAnInstruction": 0, "overlappingInstructionStarts": 0,
|
|
149
|
+
"contested": 0, "notRead": 0}
|
|
150
|
+
for row in rows:
|
|
151
|
+
offsets = _offsets(image, domains, row["space"], row["at"])
|
|
152
|
+
if not offsets:
|
|
153
|
+
continue
|
|
154
|
+
held = next(((at, holder) for at in offsets for holder in [holding_instruction(seen, at)] if holder is not None), None)
|
|
155
|
+
if held is None:
|
|
156
|
+
if any(at in seen for at in offsets):
|
|
157
|
+
counts["instructionStarts"] += 1
|
|
158
|
+
elif any(at in contested or holding_instruction(contested, at) is not None for at in offsets):
|
|
159
|
+
counts["contested"] += 1
|
|
160
|
+
else:
|
|
161
|
+
counts["notRead"] += 1
|
|
162
|
+
continue
|
|
163
|
+
at, holder = held
|
|
164
|
+
overlapping = at in seen
|
|
165
|
+
counts["overlappingInstructionStarts" if overlapping else "insideAnInstruction"] += 1
|
|
166
|
+
routine = routine_of(holder)
|
|
167
|
+
entry = {**_row_fields(row), "site": at,
|
|
168
|
+
"status": "start of an overlapping instruction" if overlapping else "inside an instruction",
|
|
169
|
+
"insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
|
|
170
|
+
"insideInstructionSize": seen[holder].size, "insideInstructionText": _text(seen[holder])}
|
|
171
|
+
if overlapping:
|
|
172
|
+
entry |= {"rowStartInstructionSize": seen[at].size, "rowStartInstructionText": _text(seen[at])}
|
|
173
|
+
if routine is not None:
|
|
174
|
+
place = _target_place(image, routine)
|
|
175
|
+
entry |= {"routine": routine, "routineAddress": place[2], "routineIsRow": place[:2] in starts}
|
|
176
|
+
found.append(entry)
|
|
177
|
+
return found, counts
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _no_return_check(image, routines, declared, limit):
|
|
181
|
+
"""``(returns, read, stopped)``: the return sites on each declared routine's own read paths, the
|
|
182
|
+
instructions the walk established, and whether the walk stopped.
|
|
183
|
+
|
|
184
|
+
The walk starts at the declared routines and continues past every interrupt except a declared
|
|
185
|
+
one, as ``reach`` reads them, so a routine that ends in an interrupt that returns is contradicted
|
|
186
|
+
by what follows it. Calls are stepped over at their return sites, except calls to declared routines.
|
|
187
|
+
A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
|
|
188
|
+
has no return sites and is not in ``read``.
|
|
189
|
+
"""
|
|
190
|
+
from .reach import returns_from, successor_graph
|
|
191
|
+
if not routines:
|
|
192
|
+
return {}, {}, False
|
|
193
|
+
seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True, **declared)
|
|
194
|
+
graph = successor_graph(image, seen, follow_interrupts=True, **declared)
|
|
195
|
+
return ({at: returns_from(graph, seen, set(), at) for at in routines}, seen,
|
|
196
|
+
any(g["reason"] == LIMIT_REASON for g in gaps))
|
|
197
|
+
|
|
198
|
+
|
|
124
199
|
def _count(n, one, many):
|
|
125
200
|
"""``n`` followed by the singular or the plural wording."""
|
|
126
201
|
return f"{n} {one if n == 1 else many}"
|
|
@@ -134,14 +209,22 @@ def inventory_check(image, config):
|
|
|
134
209
|
reported with one calling site, near or far, and the evidence of its best site: a call on the
|
|
135
210
|
entry path, a contested instruction, or only a raw byte candidate. A target that is a byte past
|
|
136
211
|
the first of an instruction the entry-path walk established names that instruction.
|
|
212
|
+
|
|
213
|
+
The rows are checked against the same walk: each row whose start lies inside an instruction the
|
|
214
|
+
walk established is listed in ``rowStarts``. ``noReturn`` declarations, in ``reach``'s shape, end
|
|
215
|
+
the walk at a call to a declared routine and at a declared interrupt site, are checked for a
|
|
216
|
+
return on the routine's own paths, and list in ``rowsPastNoReturn`` each row whose body
|
|
217
|
+
continues past such a call or interrupt.
|
|
137
218
|
"""
|
|
138
|
-
from .reports import call_controls, direct_calls, search_coverage
|
|
139
|
-
from .
|
|
219
|
+
from .reports import call_controls, direct_calls, entries, search_coverage
|
|
220
|
+
from .reach import no_return_declarations, no_return_rows, successor_graph, fewest_calls
|
|
140
221
|
if "inventory" not in config:
|
|
141
222
|
raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
|
|
142
223
|
rows = read_inventory(config["inventory"], image)
|
|
143
224
|
limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
|
|
144
|
-
|
|
225
|
+
no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
|
|
226
|
+
declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
|
|
227
|
+
calls = direct_calls(image, config, **declared)
|
|
145
228
|
seen, contested = calls["seen"], calls["contested"]
|
|
146
229
|
|
|
147
230
|
starts = {(row["space"], row["at"]) for row in rows}
|
|
@@ -211,9 +294,63 @@ def inventory_check(image, config):
|
|
|
211
294
|
entry |= {"insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
|
|
212
295
|
"insideInstructionSize": seen[holder].size}
|
|
213
296
|
if held:
|
|
214
|
-
entry["rows"] = [
|
|
297
|
+
entry["rows"] = [_row_fields(rows[i]) for i in held]
|
|
215
298
|
missing.append(entry)
|
|
216
299
|
|
|
300
|
+
# The routine the walk read an instruction in, by the route with the fewest calls, as reach names it.
|
|
301
|
+
# The graph is built only when a row start needs it.
|
|
302
|
+
routines = None
|
|
303
|
+
|
|
304
|
+
def routine_of(site):
|
|
305
|
+
nonlocal routines
|
|
306
|
+
if routines is None:
|
|
307
|
+
graph = successor_graph(image, seen, **declared)
|
|
308
|
+
routines = fewest_calls(graph, [at for at in entries(image) if at in seen])[2]
|
|
309
|
+
return routines.get(site)
|
|
310
|
+
row_starts, start_counts = _row_starts(image, rows, starts, domains, seen, contested, routine_of)
|
|
311
|
+
|
|
312
|
+
# Each declaration with the calls it kept from their return sites, as reach reports them.
|
|
313
|
+
returns, check_read, check_stopped = _no_return_check(image, no_return_routines, declared,
|
|
314
|
+
config.get("instructionLimit", 10000))
|
|
315
|
+
entry_calls = {}
|
|
316
|
+
for row in calls["rows"]:
|
|
317
|
+
if row["target"] in no_return_routines and row["site"] in seen:
|
|
318
|
+
entry_calls.setdefault(row["target"], []).append(row["site"])
|
|
319
|
+
no_return = no_return_rows(no_return_routines, no_return_interrupts, returns,
|
|
320
|
+
{at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read)
|
|
321
|
+
|
|
322
|
+
# A row whose body holds a declared call or interrupt and the byte after it still covers what follows.
|
|
323
|
+
ends = [(row["site"], image.decode(row["site"]).size, "call", row["target"], row["classification"])
|
|
324
|
+
for row in calls["rows"] if row["target"] in no_return_routines]
|
|
325
|
+
ends += [(at, row["following"] - at, "interrupt", None,
|
|
326
|
+
"entry-path instruction" if at in seen else CONTESTED_REASON if at in contested else "declared site")
|
|
327
|
+
for at, row in no_return_interrupts.items()]
|
|
328
|
+
# Only a site the entry-path walk established counts a row; the rest are listed with their evidence.
|
|
329
|
+
past, past_rows, past_unread, past_unverified = [], set(), set(), set()
|
|
330
|
+
for site, size, kind, routine, classification in sorted(ends):
|
|
331
|
+
here, after = _target_place(image, site), _target_place(image, site + size)
|
|
332
|
+
# The byte after the site must be the next one in the inventory's notation too, or no body range can hold both.
|
|
333
|
+
if after is None or after[:2] != (here[0], here[1] + size):
|
|
334
|
+
continue
|
|
335
|
+
for i in owners(*here[:2]):
|
|
336
|
+
piece = next(((begin, end) for begin, end in rows[i]["body"] if begin <= after[1] < end), None)
|
|
337
|
+
if piece is None:
|
|
338
|
+
continue
|
|
339
|
+
entry = {**_row_fields(rows[i]), "kind": kind, "site": site, "siteAddress": here[2],
|
|
340
|
+
"siteClassification": classification}
|
|
341
|
+
if routine is not None:
|
|
342
|
+
entry["routine"] = routine
|
|
343
|
+
past.append(entry | {"following": site + size, "followingAddress": after[2],
|
|
344
|
+
"followingRead": site + size in seen, "bytesAfter": piece[1] - after[1]})
|
|
345
|
+
# A row with several such calls or interrupts is listed once for each, and counted once.
|
|
346
|
+
if site not in seen:
|
|
347
|
+
past_unverified.add(i)
|
|
348
|
+
continue
|
|
349
|
+
past_rows.add(i)
|
|
350
|
+
if site + size not in seen:
|
|
351
|
+
past_unread.add(i)
|
|
352
|
+
past_unverified -= past_rows
|
|
353
|
+
|
|
217
354
|
controls = call_controls(calls, config)
|
|
218
355
|
coverage = search_coverage(image, calls["read"])
|
|
219
356
|
partial = any(c["partial"] for c in coverage)
|
|
@@ -245,12 +382,55 @@ def inventory_check(image, config):
|
|
|
245
382
|
if LIMIT_REASON in stopped:
|
|
246
383
|
summary += " The entry-path walk stopped at its instruction limit, so it can reach more calls."
|
|
247
384
|
summary += " Calls the search does not resolve can add targets."
|
|
385
|
+
inside = start_counts["insideAnInstruction"] + start_counts["overlappingInstructionStarts"]
|
|
386
|
+
if inside:
|
|
387
|
+
summary += " " + _count(inside, "row start lies", "row starts lie") + " inside an instruction the entry-path walk established"
|
|
388
|
+
if start_counts["overlappingInstructionStarts"]:
|
|
389
|
+
summary += (f", {start_counts['overlappingInstructionStarts']} of them at the start of an overlapping "
|
|
390
|
+
"instruction it also established")
|
|
391
|
+
summary += "."
|
|
392
|
+
if start_counts["notRead"]:
|
|
393
|
+
summary += " " + _count(start_counts["notRead"], "row start in declared code lies", "row starts in declared code lie") \
|
|
394
|
+
+ " in bytes where the walk established no instruction, so its boundary is not checked."
|
|
395
|
+
if past_rows:
|
|
396
|
+
summary += (" " + _count(len(past_rows), "row continues", "rows continue") + " past a call or interrupt declared "
|
|
397
|
+
f"noReturn, {len(past_unread)} of them where the walk read no instruction after it by another route.")
|
|
398
|
+
if past_unverified:
|
|
399
|
+
summary += (" " + _count(len(past_unverified), "more row holds", "more rows hold") + " such a call or interrupt "
|
|
400
|
+
"that only raw bytes, a contested instruction or an unreached declaration show, so it is not "
|
|
401
|
+
"shown to run.")
|
|
402
|
+
if start_counts["contested"]:
|
|
403
|
+
summary += " " + _count(start_counts["contested"], "row start lies", "row starts lie") \
|
|
404
|
+
+ " at or inside an instruction the walk rejected as contested, so its boundary is not checked."
|
|
405
|
+
contradicted = sum(row.get("contradicted", False) for row in no_return)
|
|
406
|
+
if contradicted:
|
|
407
|
+
summary += " " + _count(contradicted, "noReturn routine has", "noReturn routines have") \
|
|
408
|
+
+ " a return on its own read paths, so the declaration is contradicted."
|
|
409
|
+
unread = sum(not row["read"] for row in no_return if "routine" in row)
|
|
410
|
+
if unread:
|
|
411
|
+
summary += " " + _count(unread, "noReturn routine is", "noReturn routines are") \
|
|
412
|
+
+ " not an instruction the check walk established, so its return check reads nothing."
|
|
413
|
+
if check_stopped:
|
|
414
|
+
summary += " The walk that checks the noReturn routines stopped at its instruction limit, so it can miss returns."
|
|
415
|
+
assumptions = ["each reached call returns to its next instruction"
|
|
416
|
+
+ (", except a call to a noReturn routine" if no_return_routines else "")]
|
|
417
|
+
if no_return_routines:
|
|
418
|
+
assumptions.append("each interrupt the noReturn check reads returns to its next instruction"
|
|
419
|
+
+ (", except at a noReturn interrupt site" if no_return_interrupts else "")
|
|
420
|
+
+ "; interrupt handlers are not read")
|
|
421
|
+
if no_return:
|
|
422
|
+
assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
|
|
248
423
|
return {"inventory": {"path": config["inventory"], "rows": len(rows)},
|
|
249
424
|
"targets": missing[:limit], "unresolved": unresolved[:limit],
|
|
250
425
|
"rowsOutsideDeclaredCode": outside_code[:limit],
|
|
426
|
+
"rowStarts": row_starts[:limit], "noReturn": no_return, "rowsPastNoReturn": past[:limit],
|
|
251
427
|
"counts": {"callTargets": len(by_target), **tally, "unresolvedCalls": len(unresolved),
|
|
252
|
-
"inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code)
|
|
253
|
-
|
|
428
|
+
"inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code),
|
|
429
|
+
"rowStarts": start_counts, "rowsPastNoReturn": len(past_rows),
|
|
430
|
+
"rowsPastNoReturnUnreadAfter": len(past_unread),
|
|
431
|
+
"rowsPastNoReturnUnverified": len(past_unverified)},
|
|
432
|
+
"truncated": any(len(x) > limit for x in (missing, unresolved, outside_code, row_starts, past)),
|
|
433
|
+
"noReturnCheckLimitReached": check_stopped, "assumptions": assumptions,
|
|
254
434
|
"summary": summary, "controls": controls,
|
|
255
435
|
"searched": [r for r in image.regions if r["name"] in calls["scans"]], "coverage": coverage,
|
|
256
436
|
"partialSearch": partial, "gaps": calls["gaps"],
|
|
@@ -259,4 +439,11 @@ def inventory_check(image, config):
|
|
|
259
439
|
"interpretation": "Each distinct resolved direct call target in the searched regions, placed in the inventory's "
|
|
260
440
|
"notation and compared with its rows. A target only raw byte candidates or contested "
|
|
261
441
|
"instructions call is not shown to be code. The counts are lower bounds on the targets "
|
|
262
|
-
"the inventory lacks: calls the search does not resolve can add more."
|
|
442
|
+
"the inventory lacks: calls the search does not resolve can add more. Row starts are "
|
|
443
|
+
"compared only with the instructions the entry-path walk established; a start where it "
|
|
444
|
+
"established no instruction is counted as notRead or contested and not placed, so linear decoding past data "
|
|
445
|
+
"never reports one. A start inside an instruction can be deliberately overlapping code, "
|
|
446
|
+
"which the row names with both instructions for a reader to judge. rowsPastNoReturn rests "
|
|
447
|
+
"on the noReturn declarations: followingRead says whether the walk reached the bytes after "
|
|
448
|
+
"the call by another route, only a site the walk established counts a row, and an empty "
|
|
449
|
+
"returnSites means only that the walk read no return."}
|
|
@@ -46,7 +46,7 @@ def _leaves(value, image, starts):
|
|
|
46
46
|
return leaves
|
|
47
47
|
|
|
48
48
|
|
|
49
|
-
def
|
|
49
|
+
def no_return_declarations(value, image):
|
|
50
50
|
"""The declared routines and interrupt sites that never return, each with its reason."""
|
|
51
51
|
if not isinstance(value, list) or len(value) > 256:
|
|
52
52
|
raise ValueError("noReturn must be a list of at most 256 objects")
|
|
@@ -76,7 +76,7 @@ def _no_return(value, image):
|
|
|
76
76
|
return routines, interrupts
|
|
77
77
|
|
|
78
78
|
|
|
79
|
-
def
|
|
79
|
+
def returns_from(graph, seen, returning_leaves, routine):
|
|
80
80
|
"""Where ``routine``'s own paths return: each return instruction, and each leaf they enter other than by a call.
|
|
81
81
|
|
|
82
82
|
Calls are stepped over at the return sites the graph keeps. The walk assumes that a leaf returns,
|
|
@@ -99,7 +99,7 @@ def _returns_from(graph, seen, returning_leaves, routine):
|
|
|
99
99
|
return sorted(found)
|
|
100
100
|
|
|
101
101
|
|
|
102
|
-
def
|
|
102
|
+
def successor_kinds(at, ins, step):
|
|
103
103
|
"""Each successor of ``step`` with how the CFG reaches it: call, return, table, interrupt, jump or fall."""
|
|
104
104
|
m, following = base_mnemonic(ins), at + ins.size
|
|
105
105
|
targets = {e["target"] for e in step.edges if e["target"] is not None}
|
|
@@ -121,6 +121,56 @@ def _kinds(at, ins, step):
|
|
|
121
121
|
return rows
|
|
122
122
|
|
|
123
123
|
|
|
124
|
+
def successor_graph(image, seen, **step):
|
|
125
|
+
"""Each site of ``seen`` mapped to its ``successor_kinds`` rows that stay in ``seen``, under ``cfg_step``'s ``step`` options."""
|
|
126
|
+
return {at: [(s, kind) for s, kind in successor_kinds(at, ins, cfg_step(image, at, ins, **step)) if s in seen]
|
|
127
|
+
for at, ins in seen.items()}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, read):
|
|
131
|
+
"""The ``noReturn`` report rows of the declared ``routines`` and ``interrupts``.
|
|
132
|
+
|
|
133
|
+
``returns`` gives each routine's return sites, ``call_sites`` the reached calls to each routine,
|
|
134
|
+
``seen`` the instructions the calls and interrupts were read in, ``reached`` the sites a walk from
|
|
135
|
+
the starts reached and ``read`` the sites the return check decoded.
|
|
136
|
+
"""
|
|
137
|
+
rows = [{"routine": at, "reason": reason, "reached": at in reached, "read": at in read,
|
|
138
|
+
"returnSites": returns[at], "contradicted": bool(returns[at]),
|
|
139
|
+
"callSites": [{"site": site, "following": site + seen[site].size,
|
|
140
|
+
"followingRead": site + seen[site].size in seen}
|
|
141
|
+
for site in call_sites.get(at, [])]}
|
|
142
|
+
for at, reason in routines.items()]
|
|
143
|
+
return rows + [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
|
|
144
|
+
"following": row["following"], "followingRead": row["following"] in seen}
|
|
145
|
+
for at, row in interrupts.items()]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def fewest_calls(graph, starts):
|
|
149
|
+
"""``(distance, parent, routine)`` for every site ``graph`` reaches from ``starts`` by the route with the fewest calls.
|
|
150
|
+
|
|
151
|
+
``graph`` maps a site to its ``successor_kinds`` rows. ``distance`` counts the calls on the route,
|
|
152
|
+
``parent`` gives each site's ``(predecessor, kind)`` on it (None at a start), and ``routine`` the
|
|
153
|
+
target of the route's last call, or the start.
|
|
154
|
+
"""
|
|
155
|
+
# Entering a callee costs one, every other edge nothing (0-1 breadth-first search).
|
|
156
|
+
distance, parent, routine = {}, {}, {}
|
|
157
|
+
queue = deque()
|
|
158
|
+
for at in starts:
|
|
159
|
+
distance[at], parent[at], routine[at] = 0, None, at
|
|
160
|
+
queue.append((0, at))
|
|
161
|
+
while queue:
|
|
162
|
+
cost, at = queue.popleft()
|
|
163
|
+
if cost > distance[at]:
|
|
164
|
+
continue
|
|
165
|
+
for successor, kind in graph.get(at, []):
|
|
166
|
+
step_cost = cost + (kind == "call")
|
|
167
|
+
if successor not in distance or step_cost < distance[successor]:
|
|
168
|
+
distance[successor], parent[successor] = step_cost, (at, kind)
|
|
169
|
+
routine[successor] = successor if kind == "call" else routine[at]
|
|
170
|
+
(queue.append if kind == "call" else queue.appendleft)((step_cost, successor))
|
|
171
|
+
return distance, parent, routine
|
|
172
|
+
|
|
173
|
+
|
|
124
174
|
def _dominators(order, predecessors):
|
|
125
175
|
"""Immediate dominators (Cooper, Harvey and Kennedy) of every node in ``order``, a reverse postorder from ROOT."""
|
|
126
176
|
index = {node: i for i, node in enumerate(order)}
|
|
@@ -182,7 +232,7 @@ def reach(image, config):
|
|
|
182
232
|
"the walk found")
|
|
183
233
|
limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
|
|
184
234
|
instruction_limit = config.get("instructionLimit", 10000)
|
|
185
|
-
no_return_routines, no_return_interrupts =
|
|
235
|
+
no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
|
|
186
236
|
declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
|
|
187
237
|
seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
|
|
188
238
|
stops=frozenset(leaves), **declared)
|
|
@@ -190,7 +240,7 @@ def reach(image, config):
|
|
|
190
240
|
graph, unresolved, interrupts, resolved_calls = {}, [], [], {}
|
|
191
241
|
for at, ins in sorted(seen.items()):
|
|
192
242
|
step = cfg_step(image, at, ins, follow_interrupts=True, **declared)
|
|
193
|
-
graph[at] = [(s, kind) for s, kind in
|
|
243
|
+
graph[at] = [(s, kind) for s, kind in successor_kinds(at, ins, step) if s in seen or s in leaves]
|
|
194
244
|
m = base_mnemonic(ins)
|
|
195
245
|
text = _text(ins)
|
|
196
246
|
kind = "call" if m in CALLS else "return" if m in RETURNS else "jump"
|
|
@@ -208,22 +258,7 @@ def reach(image, config):
|
|
|
208
258
|
for at in leaves:
|
|
209
259
|
graph.setdefault(at, [])
|
|
210
260
|
|
|
211
|
-
|
|
212
|
-
distance, parent, routine = {}, {}, {}
|
|
213
|
-
queue = deque()
|
|
214
|
-
for at in starts:
|
|
215
|
-
distance[at], parent[at], routine[at] = 0, None, at
|
|
216
|
-
queue.append((0, at))
|
|
217
|
-
while queue:
|
|
218
|
-
cost, at = queue.popleft()
|
|
219
|
-
if cost > distance[at]:
|
|
220
|
-
continue
|
|
221
|
-
for successor, kind in graph.get(at, []):
|
|
222
|
-
step_cost = cost + (kind == "call")
|
|
223
|
-
if successor not in distance or step_cost < distance[successor]:
|
|
224
|
-
distance[successor], parent[successor] = step_cost, (at, kind)
|
|
225
|
-
routine[successor] = successor if kind == "call" else routine[at]
|
|
226
|
-
(queue.append if kind == "call" else queue.appendleft)((step_cost, successor))
|
|
261
|
+
distance, parent, routine = fewest_calls(graph, starts)
|
|
227
262
|
|
|
228
263
|
# Dominators over everything reached, from a virtual root that leads to every start.
|
|
229
264
|
successors = {ROOT: list(starts)} | {at: [s for s, _ in graph.get(at, [])] for at in distance}
|
|
@@ -334,18 +369,10 @@ def reach(image, config):
|
|
|
334
369
|
leaf_rows = [{"routine": at, "reason": reason, "reached": at in distance, "callSites": call_sites.get(at, [])}
|
|
335
370
|
for at, reason in leaves.items()]
|
|
336
371
|
# Each call the declaration kept from its return site, so a reviewer can check what follows it.
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
"returnSites": returns, "contradicted": bool(returns),
|
|
342
|
-
"callSites": [{"site": site, "following": site + seen[site].size,
|
|
343
|
-
"followingRead": site + seen[site].size in seen}
|
|
344
|
-
for site in call_sites.get(at, [])]})
|
|
345
|
-
no_return_rows += [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
|
|
346
|
-
"following": row["following"], "followingRead": row["following"] in seen}
|
|
347
|
-
for at, row in no_return_interrupts.items()]
|
|
348
|
-
contradicted = any(row.get("contradicted") for row in no_return_rows)
|
|
372
|
+
returning_leaves = set(leaves) - set(no_return_routines)
|
|
373
|
+
returns = {at: returns_from(graph, seen, returning_leaves, at) for at in no_return_routines}
|
|
374
|
+
no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, call_sites, seen, distance, seen)
|
|
375
|
+
contradicted = any(row.get("contradicted") for row in no_return)
|
|
349
376
|
counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
|
|
350
377
|
"interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
|
|
351
378
|
assumptions = ["each reached call returns to its next instruction"
|
|
@@ -355,9 +382,9 @@ def reach(image, config):
|
|
|
355
382
|
+ "; interrupt handlers are not read",
|
|
356
383
|
"each leaf calls nothing, for the reason it gives",
|
|
357
384
|
"each declared indirect jump table holds the routes its declaration gives"]
|
|
358
|
-
if
|
|
385
|
+
if no_return:
|
|
359
386
|
assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
|
|
360
|
-
return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn":
|
|
387
|
+
return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return,
|
|
361
388
|
"reachedRoutines": sorted(routine_starts), "counts": counts,
|
|
362
389
|
"unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
|
|
363
390
|
"contested": sorted(contested)[:limit],
|
|
@@ -85,19 +85,21 @@ def search_coverage(image, spans):
|
|
|
85
85
|
return rows
|
|
86
86
|
|
|
87
87
|
|
|
88
|
-
def direct_calls(image, config):
|
|
88
|
+
def direct_calls(image, config, no_return_calls=frozenset(), no_return_interrupts=frozenset()):
|
|
89
89
|
"""Every direct call site in the searched regions, as ``incoming`` and ``inventory-check`` read them.
|
|
90
90
|
|
|
91
|
-
Walks the entry-path CFG from every established entry,
|
|
92
|
-
``
|
|
93
|
-
|
|
91
|
+
Walks the entry-path CFG from every established entry, ending a branch at a call to a routine in
|
|
92
|
+
``no_return_calls`` and at an interrupt site in ``no_return_interrupts`` as ``walk`` does, then
|
|
93
|
+
scans every byte of the regions ``searchRegions`` names (all regions by default) for E8 and 9A
|
|
94
|
+
call starts, and adds each reached call the scan cannot see (one that starts with a prefix). Returns a dict with the walk's
|
|
94
95
|
``seen``, ``gaps`` (with a ``raw scan limit`` gap where ``scanLimit`` stopped a region),
|
|
95
96
|
``edges``, ``undecoded`` and ``contested``; ``rows``, every call row in the order it was read,
|
|
96
97
|
each with its ``target`` (None when unresolved); ``scanned``, those rows by site, leaving out
|
|
97
98
|
the reached calls whose frame encoding the walk does not model; ``scans``, the region names
|
|
98
99
|
searched; and ``read``, the byte span the scan read in each.
|
|
99
100
|
"""
|
|
100
|
-
seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000)
|
|
101
|
+
seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000),
|
|
102
|
+
no_return_calls=no_return_calls, no_return_interrupts=no_return_interrupts)
|
|
101
103
|
rows, scanned = [], {}
|
|
102
104
|
|
|
103
105
|
def classify(at):
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_inventory_check.py
RENAMED
|
@@ -171,6 +171,157 @@ class InventoryCheckTests(unittest.TestCase):
|
|
|
171
171
|
self.assertEqual([t["insideInstruction"] for t in twice["targets"]], [1, 9])
|
|
172
172
|
self.assertIn(" 2 of them start inside an instruction the entry-path walk established from another start.", twice["summary"])
|
|
173
173
|
|
|
174
|
+
def test_a_row_start_inside_an_established_far_call_names_the_call_and_the_routine_it_was_read_in(self):
|
|
175
|
+
# 0000 calls 0004, a routine no row starts at: push bp; mov bp,sp; call far 1000:000E; pop bp; ret.
|
|
176
|
+
# The row 1000:000A starts three bytes into that five-byte far call.
|
|
177
|
+
data = bytes.fromhex("e80100" "c3" "55" "8bec" "9a0e000010" "5d" "c3" "cb")
|
|
178
|
+
relocation = [{"site": 10, "segment": 0x1000, "evidence": "synthetic relocation"}]
|
|
179
|
+
r = self.check("start\tsize\tname\n1000:0000\t4\tmain\n1000:000A\t4\tshifted\n1000:000E\t1\tfar\n", data=data,
|
|
180
|
+
relocations=relocation)
|
|
181
|
+
self.assertEqual(r["rowStarts"], [{
|
|
182
|
+
"start": "1000:000A", "size": 4, "name": "shifted", "site": 10, "status": "inside an instruction",
|
|
183
|
+
"insideInstruction": 7, "insideInstructionAddress": "1000:0007", "insideInstructionSize": 5,
|
|
184
|
+
"insideInstructionText": "lcall 0x1000, 0xe", "routine": 4, "routineAddress": "1000:0004", "routineIsRow": False}])
|
|
185
|
+
self.assertEqual(r["counts"]["rowStarts"],
|
|
186
|
+
{"instructionStarts": 2, "insideAnInstruction": 1, "overlappingInstructionStarts": 0,
|
|
187
|
+
"contested": 0, "notRead": 0})
|
|
188
|
+
self.assertIn(" 1 row start lies inside an instruction the entry-path walk established.", r["summary"])
|
|
189
|
+
# The routine's real entry is still a call target no row starts at.
|
|
190
|
+
self.assertEqual([(t["address"], t["status"]) for t in r["targets"]], [("1000:0004", "outside every row")])
|
|
191
|
+
# Rows at instruction starts pass.
|
|
192
|
+
fixed = self.check("start\tsize\n1000:0000\t4\n1000:0004\t10\n1000:000E\t1\n", data=data, relocations=relocation)
|
|
193
|
+
self.assertEqual(fixed["rowStarts"], [])
|
|
194
|
+
self.assertEqual(fixed["counts"]["rowStarts"]["instructionStarts"], 3)
|
|
195
|
+
self.assertNotIn("row start", fixed["summary"])
|
|
196
|
+
# The result limit cuts the list, not the count.
|
|
197
|
+
cut = self.check("start\tsize\n1000:0000\t4\n1000:0009\t1\n1000:000A\t1\n", data=data, relocations=relocation, limit=1)
|
|
198
|
+
self.assertEqual([row["start"] for row in cut["rowStarts"]], ["1000:0009"])
|
|
199
|
+
self.assertEqual(cut["counts"]["rowStarts"]["insideAnInstruction"], 2)
|
|
200
|
+
self.assertTrue(cut["truncated"])
|
|
201
|
+
|
|
202
|
+
def test_a_row_start_only_a_misaligned_decode_would_cut_is_not_placed_inside_an_instruction(self):
|
|
203
|
+
# 0000 calls 0007. 0004..0006 is data the walk never decodes; decoded linearly, its 9A byte starts a
|
|
204
|
+
# five-byte far call over 0007. The row at 0007 is an established start, and the row at 0005, in the
|
|
205
|
+
# data, is counted as not read.
|
|
206
|
+
data = bytes.fromhex("e80400" "c3" "9a9090" "c3" "90")
|
|
207
|
+
r = self.check("start\tsize\n1000:0000\t4\n1000:0005\t1\n1000:0007\t1\n", data=data)
|
|
208
|
+
self.assertEqual(r["rowStarts"], [])
|
|
209
|
+
self.assertEqual(r["counts"]["rowStarts"],
|
|
210
|
+
{"instructionStarts": 2, "insideAnInstruction": 0, "overlappingInstructionStarts": 0,
|
|
211
|
+
"contested": 0, "notRead": 1})
|
|
212
|
+
self.assertIn(" 1 row start in declared code lies in bytes where the walk established no instruction, so its boundary is not checked.",
|
|
213
|
+
r["summary"])
|
|
214
|
+
self.assertNotIn("inside an instruction the entry-path walk established", r["summary"])
|
|
215
|
+
|
|
216
|
+
def test_a_row_at_a_deliberately_overlapping_instruction_is_listed_once_with_both_instructions(self):
|
|
217
|
+
# 0001 or bh,0 holds an IRET byte at 0002, which the push-CS/call frame at 0004 calls, so the walk
|
|
218
|
+
# establishes both instructions and the row at 0002 starts one inside the other.
|
|
219
|
+
data = bytes.fromhex("9c" "80cf00" "0e" "e8faff" "c3")
|
|
220
|
+
r = self.check("start\tsize\n1000:0000\t9\n1000:0002\t1\n", data=data)
|
|
221
|
+
self.assertEqual(r["rowStarts"], [{
|
|
222
|
+
"start": "1000:0002", "size": 1, "site": 2, "status": "start of an overlapping instruction",
|
|
223
|
+
"insideInstruction": 1, "insideInstructionAddress": "1000:0001", "insideInstructionSize": 3,
|
|
224
|
+
"insideInstructionText": "or bh, 0", "rowStartInstructionSize": 1, "rowStartInstructionText": "iret",
|
|
225
|
+
"routine": 0, "routineAddress": "1000:0000", "routineIsRow": True}])
|
|
226
|
+
self.assertEqual(r["counts"]["rowStarts"]["overlappingInstructionStarts"], 1)
|
|
227
|
+
self.assertIn(" 1 row start lies inside an instruction the entry-path walk established, 1 of them at the start "
|
|
228
|
+
"of an overlapping instruction it also established.", r["summary"])
|
|
229
|
+
|
|
230
|
+
def test_a_row_past_a_call_to_a_no_return_routine_is_listed_and_the_bytes_after_the_call_are_not_walked(self):
|
|
231
|
+
# 0000 calls the exit routine at 0008 (mov ah,4Ch; int 21h). The row 1000:0000 runs on over 0003..0007,
|
|
232
|
+
# where E8 FD FF decodes as a call to 0003 and the walk reads it unless the routine is declared.
|
|
233
|
+
data = bytes.fromhex("e80500" "e8fdff" "0000" "b44c" "cd21")
|
|
234
|
+
inventory = "start\tsize\n1000:0000\t8\n1000:0008\t4\n"
|
|
235
|
+
exit_routine = [{"routine": 8, "reason": "synthetic exit"}]
|
|
236
|
+
r = self.check(inventory, data=data, noReturn=exit_routine)
|
|
237
|
+
self.assertEqual(r["rowsPastNoReturn"], [{
|
|
238
|
+
"start": "1000:0000", "size": 8, "kind": "call", "site": 0, "siteAddress": "1000:0000",
|
|
239
|
+
"siteClassification": "entry-path instruction", "routine": 8, "following": 3, "followingAddress": "1000:0003",
|
|
240
|
+
"followingRead": False, "bytesAfter": 5}])
|
|
241
|
+
self.assertEqual(r["noReturn"], [{"routine": 8, "reason": "synthetic exit", "reached": True, "read": True,
|
|
242
|
+
"returnSites": [], "contradicted": False,
|
|
243
|
+
"callSites": [{"site": 0, "following": 3, "followingRead": False}]}])
|
|
244
|
+
self.assertEqual((r["counts"]["rowsPastNoReturn"], r["counts"]["rowsPastNoReturnUnreadAfter"]), (1, 1))
|
|
245
|
+
self.assertIn(" 1 row continues past a call or interrupt declared noReturn, 1 of them where the walk read no "
|
|
246
|
+
"instruction after it by another route.", r["summary"])
|
|
247
|
+
self.assertEqual(r["assumptions"], ["each reached call returns to its next instruction, except a call to a noReturn routine",
|
|
248
|
+
"each interrupt the noReturn check reads returns to its next instruction; "
|
|
249
|
+
"interrupt handlers are not read",
|
|
250
|
+
"each noReturn routine and interrupt never returns, for the reason it gives"])
|
|
251
|
+
# The call in the bytes after the exit call is only a raw byte candidate once the routine is declared.
|
|
252
|
+
self.assertEqual([(t["target"], t["evidence"]) for t in r["targets"]], [(3, "raw byte candidate only")])
|
|
253
|
+
undeclared = self.check(inventory, data=data)
|
|
254
|
+
self.assertEqual([(t["target"], t["evidence"]) for t in undeclared["targets"]], [(3, "entry-path call")])
|
|
255
|
+
self.assertEqual((undeclared["rowsPastNoReturn"], undeclared["noReturn"]), ([], []))
|
|
256
|
+
self.assertEqual(undeclared["assumptions"], ["each reached call returns to its next instruction"])
|
|
257
|
+
# A row that ends at the call's last byte does not continue past it.
|
|
258
|
+
ended = self.check("start\tsize\n1000:0000\t3\n1000:0008\t4\n", data=data, noReturn=exit_routine)
|
|
259
|
+
self.assertEqual(ended["rowsPastNoReturn"], [])
|
|
260
|
+
|
|
261
|
+
def test_a_row_with_two_calls_to_a_no_return_routine_lists_both_and_counts_one_row(self):
|
|
262
|
+
# 0000 jz 0005; 0002 call 000C; 0005 jnz 000A; 0007 call 000C; 000A ret; 000B nop; 000C mov ah,4Ch; int 21h.
|
|
263
|
+
data = bytes.fromhex("7403" "e80700" "7503" "e80200" "c3" "90" "b44c" "cd21")
|
|
264
|
+
exit_routine = [{"routine": 12, "reason": "synthetic exit"}]
|
|
265
|
+
r = self.check("start\tsize\n1000:0000\t12\n1000:000C\t4\n", data=data, noReturn=exit_routine)
|
|
266
|
+
self.assertEqual([(row["start"], row["site"], row["followingRead"]) for row in r["rowsPastNoReturn"]],
|
|
267
|
+
[("1000:0000", 2, True), ("1000:0000", 7, True)])
|
|
268
|
+
self.assertEqual((r["counts"]["rowsPastNoReturn"], r["counts"]["rowsPastNoReturnUnreadAfter"]), (1, 0))
|
|
269
|
+
self.assertIn(" 1 row continues past a call or interrupt declared noReturn, 0 of them where", r["summary"])
|
|
270
|
+
# The result limit cuts the list, not the count.
|
|
271
|
+
cut = self.check("start\tsize\n1000:0000\t12\n1000:000C\t4\n", data=data, noReturn=exit_routine, limit=1)
|
|
272
|
+
self.assertEqual([row["site"] for row in cut["rowsPastNoReturn"]], [2])
|
|
273
|
+
self.assertEqual(cut["counts"]["rowsPastNoReturn"], 1)
|
|
274
|
+
self.assertTrue(cut["truncated"])
|
|
275
|
+
|
|
276
|
+
def test_a_no_return_routine_the_check_walk_does_not_establish_is_not_read(self):
|
|
277
|
+
# 0000 ret; 0001 is a 0F byte that does not decode, declared as a routine that never returns.
|
|
278
|
+
r = self.check("start\tsize\n1000:0000\t1\n", data=bytes.fromhex("c3" "0f"),
|
|
279
|
+
noReturn=[{"routine": 1, "reason": "synthetic exit"}])
|
|
280
|
+
self.assertEqual([(row["read"], row["returnSites"], row["contradicted"]) for row in r["noReturn"]], [(False, [], False)])
|
|
281
|
+
self.assertIn(" 1 noReturn routine is not an instruction the check walk established, so its return check reads nothing.",
|
|
282
|
+
r["summary"])
|
|
283
|
+
|
|
284
|
+
def test_a_row_past_a_no_return_interrupt_is_listed_and_the_interrupt_is_no_boundary_gap(self):
|
|
285
|
+
# mov ah,4Ch; int 21h; three data bytes; ret. The row covers all eight bytes.
|
|
286
|
+
data = bytes.fromhex("b44c" "cd21" "4f4b00" "c3")
|
|
287
|
+
inventory = "start\tsize\n1000:0000\t8\n"
|
|
288
|
+
r = self.check(inventory, data=data, noReturn=[{"interrupt": 2, "reason": "synthetic terminate"}])
|
|
289
|
+
self.assertEqual(r["rowsPastNoReturn"], [{
|
|
290
|
+
"start": "1000:0000", "size": 8, "kind": "interrupt", "site": 2, "siteAddress": "1000:0002",
|
|
291
|
+
"siteClassification": "entry-path instruction", "following": 4, "followingAddress": "1000:0004",
|
|
292
|
+
"followingRead": False, "bytesAfter": 4}])
|
|
293
|
+
self.assertEqual(r["noReturn"], [{"interrupt": 2, "reason": "synthetic terminate", "vector": 0x21, "reached": True,
|
|
294
|
+
"following": 4, "followingRead": False}])
|
|
295
|
+
self.assertEqual([g for g in r["gaps"] if g.get("site") == 2], [])
|
|
296
|
+
undeclared = self.check(inventory, data=data)
|
|
297
|
+
self.assertEqual([g["reason"] for g in undeclared["gaps"] if g.get("site") == 2], ["hardware or interrupt boundary"])
|
|
298
|
+
with self.assertRaisesRegex(ValueError, "noReturn interrupt 0 is not an interrupt instruction"):
|
|
299
|
+
self.check(inventory, data=data, noReturn=[{"interrupt": 0, "reason": "synthetic"}])
|
|
300
|
+
|
|
301
|
+
def test_a_no_return_routine_is_checked_past_its_interrupts_and_a_call_reached_another_way_says_so(self):
|
|
302
|
+
# 0000 jnz 0005; 0002 call 0006; 0005 ret. 0006 mov ah,4Ch; 0008 int 21h; 000A ret.
|
|
303
|
+
data = bytes.fromhex("7503" "e80100" "c3" "b44c" "cd21" "c3")
|
|
304
|
+
inventory = "start\tsize\n1000:0000\t6\n1000:0006\t5\n"
|
|
305
|
+
exit_routine = {"routine": 6, "reason": "synthetic exit"}
|
|
306
|
+
# The entry-path walk ends at the interrupt, but the check reads past it to the RET, as reach does.
|
|
307
|
+
r = self.check(inventory, data=data, noReturn=[exit_routine])
|
|
308
|
+
self.assertEqual([(row["returnSites"], row["contradicted"]) for row in r["noReturn"]], [([10], True)])
|
|
309
|
+
self.assertIn(" 1 noReturn routine has a return on its own read paths, so the declaration is contradicted.", r["summary"])
|
|
310
|
+
self.assertEqual([(row["site"], row["following"], row["followingRead"]) for row in r["rowsPastNoReturn"]],
|
|
311
|
+
[(2, 5, True)])
|
|
312
|
+
self.assertEqual(r["counts"]["rowsPastNoReturnUnreadAfter"], 0)
|
|
313
|
+
self.assertIn(", 0 of them where the walk read no instruction after it by another route.", r["summary"])
|
|
314
|
+
# Declaring the interrupt too ends the routine's paths there.
|
|
315
|
+
both = self.check(inventory, data=data, noReturn=[exit_routine, {"interrupt": 8, "reason": "synthetic terminate"}])
|
|
316
|
+
self.assertEqual([(row.get("returnSites"), row.get("contradicted")) for row in both["noReturn"]],
|
|
317
|
+
[([], False), (None, None)])
|
|
318
|
+
self.assertNotIn("contradicted", both["summary"])
|
|
319
|
+
# The check walk shares instructionLimit and says when it stopped.
|
|
320
|
+
stopped = self.check(inventory, data=data, noReturn=[exit_routine], instructionLimit=2)
|
|
321
|
+
self.assertTrue(stopped["noReturnCheckLimitReached"])
|
|
322
|
+
self.assertIn("The walk that checks the noReturn routines stopped at its instruction limit", stopped["summary"])
|
|
323
|
+
self.assertFalse(r["noReturnCheckLimitReached"])
|
|
324
|
+
|
|
174
325
|
def test_a_contested_instruction_holds_no_target(self):
|
|
175
326
|
# Entries 0 and 1 overlap, so the call at 0003 is contested. The raw call at 000D targets 0004, a byte of
|
|
176
327
|
# that contested call, and names no holder, since only established instructions count.
|
|
@@ -182,6 +333,25 @@ class InventoryCheckTests(unittest.TestCase):
|
|
|
182
333
|
[(4, "raw byte candidate only"), (10, "contested call only")])
|
|
183
334
|
self.assertTrue(all("insideInstruction" not in t for t in r["targets"]))
|
|
184
335
|
self.assertEqual(sum(t["insideAnInstruction"] for t in r["counts"].values() if isinstance(t, dict)), 0)
|
|
336
|
+
# A row starting at the contested call is counted as contested. The row at 0000, an entry of the
|
|
337
|
+
# unresolved overlapping pair, is not read: the walk established no instruction there.
|
|
338
|
+
rows = self.inventory("start\tsize\n1000:0000\t3\n1000:0003\t14\n", name="contested.tsv")
|
|
339
|
+
r = run_report(data, config(data, regions, inventory=rows), "inventory-check")
|
|
340
|
+
self.assertEqual((r["counts"]["rowStarts"]["contested"], r["counts"]["rowStarts"]["notRead"]), (1, 1))
|
|
341
|
+
self.assertIn(" 1 row start lies at or inside an instruction the walk rejected as contested, so its boundary "
|
|
342
|
+
"is not checked.", r["summary"])
|
|
343
|
+
|
|
344
|
+
def test_a_raw_byte_call_to_a_no_return_routine_is_listed_but_not_counted_as_a_row_past_it(self):
|
|
345
|
+
# 0000 ret; 0001 E8 01 00 are data bytes that decode as a call to the exit routine at 0005; 0004 data.
|
|
346
|
+
data = bytes.fromhex("c3" "e80100" "00" "b44c" "cd21")
|
|
347
|
+
r = self.check("start\tsize\n1000:0000\t5\n1000:0005\t4\n", data=data,
|
|
348
|
+
noReturn=[{"routine": 5, "reason": "synthetic exit"}])
|
|
349
|
+
self.assertEqual([(row["site"], row["siteClassification"]) for row in r["rowsPastNoReturn"]],
|
|
350
|
+
[(1, "raw byte candidate")])
|
|
351
|
+
self.assertEqual((r["counts"]["rowsPastNoReturn"], r["counts"]["rowsPastNoReturnUnverified"]), (0, 1))
|
|
352
|
+
self.assertNotIn("row continues past", r["summary"])
|
|
353
|
+
self.assertIn(" 1 more row holds such a call or interrupt that only raw bytes, a contested instruction or an "
|
|
354
|
+
"unreached declaration show, so it is not shown to run.", r["summary"])
|
|
185
355
|
|
|
186
356
|
def test_a_scan_limit_short_of_the_code_makes_the_search_partial(self):
|
|
187
357
|
# The region lies in no container or declared segment, so only the scan gap shows the stop.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_config_encoding.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_effect_order.py
RENAMED
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_enter_frames.py
RENAMED
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_entry_frame.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_interrupt_models.py
RENAMED
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_join_terms.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_memory_scopes.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_return_checks.py
RENAMED
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_successor_rule.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_term_limit.py
RENAMED
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_upstream_gaps.py
RENAMED
|
File without changes
|
{scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_volatile_memory.py
RENAMED
|
File without changes
|
|
File without changes
|