scientific-method-engine 9.0.0__tar.gz → 10.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/PKG-INFO +11 -2
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/README.md +10 -1
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/pyproject.toml +1 -1
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/cli.py +1 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/effect_order.py +1 -1
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/image.py +55 -3
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/relational.py +84 -7
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/reports.py +5 -2
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/trace.py +79 -5
- scientific_method_engine-10.0.0/tests/test_indirect_far_transfers.py +351 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_interrupt_models.py +51 -3
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_oracle.py +19 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_relational_controls.py +105 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/.gitignore +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/LICENSE +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/machine.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/oracle.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-9.0.0 → scientific_method_engine-10.0.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version:
|
|
3
|
+
Version: 10.0.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -145,6 +145,14 @@ boundary and continuation limits remain explicit gaps or stops. Partial-table sp
|
|
|
145
145
|
partial evidence, never complete dispatch or native-reachability claims. See the
|
|
146
146
|
[table contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#evidenced-indirect-jump-tables).
|
|
147
147
|
|
|
148
|
+
A far call or jump through an `m16:16` pointer is followed when the path stored both words and
|
|
149
|
+
the address names declared code through one region's exact mapping, or names a source FBOV
|
|
150
|
+
trampoline whose overlay entry is declared. The event keeps the pointer read, each word's
|
|
151
|
+
producers and writers, and how the target was admitted. An unknown or partly unknown word, an
|
|
152
|
+
address outside declared code, a segment alias of resident code, an overlay's analysis segment and
|
|
153
|
+
an `m16:32` pointer stop the path with a named reason. See
|
|
154
|
+
[indirect far transfers](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#indirect-far-transfers-through-a-traced-pointer).
|
|
155
|
+
|
|
148
156
|
A call model may also sit at an `INT n` instruction in the real-mode model. The interrupt is
|
|
149
157
|
still reported as a hardware boundary with its vector, the handler is not executed, and each case
|
|
150
158
|
returns to the next instruction with SP and CS as before the interrupt. Registers, flags and memory
|
|
@@ -169,7 +177,8 @@ The trace-family commands (`trace`, `arguments`, `effects`, `returns`, `guards`,
|
|
|
169
177
|
writer, containment, value relation and value origin) evaluated on every bounded path. A violated
|
|
170
178
|
control fails the report. A control left undecided by a stop, a limit or an unread call is
|
|
171
179
|
reported as `undecided` and never counts as held. Access reports also give each byte's
|
|
172
|
-
`writeOrder`, or why it has no modeled value.
|
|
180
|
+
`writeOrder`, or why it has no modeled value. A last-writer control with an `address` asks the
|
|
181
|
+
same of memory the program never reads back, at a checkpoint. See the
|
|
173
182
|
[controls contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#relational-controls).
|
|
174
183
|
|
|
175
184
|
A trace-family query whose entry lies inside a function body names the function's entry in
|
|
@@ -129,6 +129,14 @@ boundary and continuation limits remain explicit gaps or stops. Partial-table sp
|
|
|
129
129
|
partial evidence, never complete dispatch or native-reachability claims. See the
|
|
130
130
|
[table contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#evidenced-indirect-jump-tables).
|
|
131
131
|
|
|
132
|
+
A far call or jump through an `m16:16` pointer is followed when the path stored both words and
|
|
133
|
+
the address names declared code through one region's exact mapping, or names a source FBOV
|
|
134
|
+
trampoline whose overlay entry is declared. The event keeps the pointer read, each word's
|
|
135
|
+
producers and writers, and how the target was admitted. An unknown or partly unknown word, an
|
|
136
|
+
address outside declared code, a segment alias of resident code, an overlay's analysis segment and
|
|
137
|
+
an `m16:32` pointer stop the path with a named reason. See
|
|
138
|
+
[indirect far transfers](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#indirect-far-transfers-through-a-traced-pointer).
|
|
139
|
+
|
|
132
140
|
A call model may also sit at an `INT n` instruction in the real-mode model. The interrupt is
|
|
133
141
|
still reported as a hardware boundary with its vector, the handler is not executed, and each case
|
|
134
142
|
returns to the next instruction with SP and CS as before the interrupt. Registers, flags and memory
|
|
@@ -153,7 +161,8 @@ The trace-family commands (`trace`, `arguments`, `effects`, `returns`, `guards`,
|
|
|
153
161
|
writer, containment, value relation and value origin) evaluated on every bounded path. A violated
|
|
154
162
|
control fails the report. A control left undecided by a stop, a limit or an unread call is
|
|
155
163
|
reported as `undecided` and never counts as held. Access reports also give each byte's
|
|
156
|
-
`writeOrder`, or why it has no modeled value.
|
|
164
|
+
`writeOrder`, or why it has no modeled value. A last-writer control with an `address` asks the
|
|
165
|
+
same of memory the program never reads back, at a checkpoint. See the
|
|
157
166
|
[controls contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#relational-controls).
|
|
158
167
|
|
|
159
168
|
A trace-family query whose entry lies inside a function body names the function's entry in
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "
|
|
8
|
+
version = "10.0.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -23,6 +23,7 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
23
23
|
"or continues at another address than the next instruction (ghidraFallsThroughElsewhere).\n"
|
|
24
24
|
"trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
|
|
25
25
|
"a violated control fails the report; an undecided one is reported and never counts as held.\n"
|
|
26
|
+
"A lastWriter control with an address inspects that memory at its checkpoint anchors without a read.\n"
|
|
26
27
|
"entryFrame {from} starts an entry inside the function at from, in the frame a trace from there observed.\n"
|
|
27
28
|
"Declared table continuations are separate conditional paths that spend continuationBudget\n"
|
|
28
29
|
"(paths, totalSteps, maxSteps, visitLimit, stringIterations); ordinary computed transfers remain stopped.\n"
|
|
@@ -4,7 +4,7 @@ from bisect import bisect_right
|
|
|
4
4
|
from .memory_scopes import model_scopes, without_scopes
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
KINDS = {"read", "write", "call", "call-return", "return", "branch", "compare", "flag-assumption",
|
|
7
|
+
KINDS = {"read", "write", "call", "far-jump", "call-return", "return", "branch", "compare", "flag-assumption",
|
|
8
8
|
"arithmetic", "value-transfer", "conversion", "flag-write", "flags-save", "flags-restore", "local-iret", "string-operation", "declared-jump-continuation",
|
|
9
9
|
"hardware-boundary"}
|
|
10
10
|
|
|
@@ -3,6 +3,7 @@ import re
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
import xxhash
|
|
5
5
|
from capstone import Cs, CS_ARCH_X86, CS_MODE_16, CS_MODE_32
|
|
6
|
+
from capstone.x86 import X86_OP_MEM
|
|
6
7
|
from .pe import prepare_pe
|
|
7
8
|
import capstone
|
|
8
9
|
|
|
@@ -109,6 +110,8 @@ class Image:
|
|
|
109
110
|
self.decoder = Cs(CS_ARCH_X86, CS_MODE_32 if self.flat else CS_MODE_16)
|
|
110
111
|
self.decoder.detail = True
|
|
111
112
|
self.cache = {}
|
|
113
|
+
# overlayExports rows by trampoline site, built on the first far_pointer_target call.
|
|
114
|
+
self.trampoline_rows = None
|
|
112
115
|
self.relocations = config.get("relocations", [])
|
|
113
116
|
if not isinstance(self.relocations, list) or len(self.relocations) > 100000:
|
|
114
117
|
raise ValueError("Invalid relocation metadata")
|
|
@@ -125,10 +128,13 @@ class Image:
|
|
|
125
128
|
def region(self, site):
|
|
126
129
|
return next((r for r in self.regions if r["start"] <= site < r["end"]), None)
|
|
127
130
|
|
|
131
|
+
def mapped_region(self, segment, ip):
|
|
132
|
+
"""The region whose declared mapping covers ``segment:ip`` exactly, or None. At most one does."""
|
|
133
|
+
return next((r for r in self.regions if r["segment"] == segment and r["ip"] <= ip < r["ip"] + r["end"] - r["start"]), None)
|
|
134
|
+
|
|
128
135
|
def offset(self, segment, ip):
|
|
129
|
-
|
|
130
|
-
if
|
|
131
|
-
r = exact[0]
|
|
136
|
+
r = self.mapped_region(segment, ip)
|
|
137
|
+
if r is not None:
|
|
132
138
|
return r["start"] + ip - r["ip"]
|
|
133
139
|
if self.flat:
|
|
134
140
|
return None
|
|
@@ -157,6 +163,10 @@ class Image:
|
|
|
157
163
|
def far_target(self, site, ins):
|
|
158
164
|
if self.flat:
|
|
159
165
|
return None, {"reason": "far transfer is outside the PE32 flat model"}
|
|
166
|
+
if ins.operands and ins.operands[0].type == X86_OP_MEM:
|
|
167
|
+
# The pointer is read from memory when the instruction runs; only a traced path knows it
|
|
168
|
+
# (far_pointer_target).
|
|
169
|
+
return None, {"reason": "computed transfer remains unresolved"}
|
|
160
170
|
# ptr16:16 immediate only. Operand-size-prefixed far calls are unsupported.
|
|
161
171
|
if ins.size != 5 or self.data[site] not in (0x9a, 0xea):
|
|
162
172
|
return None, {"reason": "unsupported far transfer encoding"}
|
|
@@ -170,6 +180,48 @@ class Image:
|
|
|
170
180
|
target = integer(fixup["target"], 0, len(self.data) - 1, "canonical target")
|
|
171
181
|
return target, {"rawSegment": raw, "offset": ip, "resolvedSegment": fixup["segment"], "relocation": fixup}
|
|
172
182
|
|
|
183
|
+
def far_pointer_target(self, segment, offset):
|
|
184
|
+
"""The code a known ``segment:offset`` far pointer names, as ``(target, admission)``.
|
|
185
|
+
|
|
186
|
+
The pointer is admitted only through the exact declared mapping of one region: ``segment``
|
|
187
|
+
equals the region's segment and ``offset`` lies in its IP range. A region with a
|
|
188
|
+
``container`` is an overlay's analysis view, whose segment is not a load address, so a
|
|
189
|
+
pointer that names it is refused. So is a pointer that reaches declared resident code only
|
|
190
|
+
through another segment (a canonical alias), because the code would then run under a CS
|
|
191
|
+
and IP the region's mapping does not describe. A pointer at a source FBOV trampoline
|
|
192
|
+
(``overlayExports``) continues at that trampoline's overlay entry, as an immediate far
|
|
193
|
+
transfer through a relocated trampoline does. Returns ``(None, {"reason": ...})`` when the
|
|
194
|
+
pointer is not admitted.
|
|
195
|
+
"""
|
|
196
|
+
region = self.mapped_region(segment, offset)
|
|
197
|
+
if region is None:
|
|
198
|
+
if self.offset(segment, offset) is not None:
|
|
199
|
+
return None, {"reason": "far pointer names declared resident code only through a segment alias of its mapping"}
|
|
200
|
+
return None, {"reason": "far pointer names no declared code region"}
|
|
201
|
+
if region.get("container") is not None:
|
|
202
|
+
return None, {"reason": "far pointer names overlay code by its analysis segment, which is not a load address",
|
|
203
|
+
"region": region["name"]}
|
|
204
|
+
site = region["start"] + offset - region["ip"]
|
|
205
|
+
admission = {"rule": "exact declared region mapping", "region": region["name"], "regionSegment": region["segment"],
|
|
206
|
+
"regionIp": region["ip"], "regionStart": region["start"], "regionEvidence": region["evidence"]}
|
|
207
|
+
if self.trampoline_rows is None:
|
|
208
|
+
# Indexed once: a traced path can take many indirect far transfers, and the source FBOV
|
|
209
|
+
# tables can list thousands of trampolines.
|
|
210
|
+
self.trampoline_rows = {}
|
|
211
|
+
for row in self.config.get("overlayExports", []):
|
|
212
|
+
if isinstance(row, dict):
|
|
213
|
+
self.trampoline_rows.setdefault(row.get("trampoline"), []).append(row)
|
|
214
|
+
exports = self.trampoline_rows.get(site, [])
|
|
215
|
+
if not exports:
|
|
216
|
+
return site, admission
|
|
217
|
+
if len(exports) > 1 or type(exports[0].get("entry")) is not int:
|
|
218
|
+
return None, {**admission, "reason": "far pointer names an FBOV trampoline without one overlay entry"}
|
|
219
|
+
export = exports[0]
|
|
220
|
+
admission["trampoline"] = {k: export.get(k) for k in ("trampoline", "descriptor", "entry", "evidence")}
|
|
221
|
+
if self.region(export["entry"]) is None:
|
|
222
|
+
return None, {**admission, "reason": "the overlay entry the FBOV trampoline names lies outside declared code regions"}
|
|
223
|
+
return export["entry"], admission
|
|
224
|
+
|
|
173
225
|
def file_offset(self, va, width=1):
|
|
174
226
|
"""Only loaded raw PE bytes; zero-fill and alignment padding are not source extents."""
|
|
175
227
|
metadata = self.config.get("peMetadata")
|
|
@@ -7,7 +7,9 @@ whose paths were not all read (a stop, a limit, a dropped path) is undecided, ne
|
|
|
7
7
|
from bisect import bisect_right
|
|
8
8
|
from math import inf
|
|
9
9
|
from .image import integer
|
|
10
|
-
from .machine import ALIASES, NO_WRITE, State
|
|
10
|
+
from .machine import ALIASES, NO_WRITE, State, StopPath
|
|
11
|
+
from .memory_scopes import SEGMENTS
|
|
12
|
+
from .values import const, op
|
|
11
13
|
|
|
12
14
|
KINDS = ("reach", "order", "lastWriter", "containment", "relation", "origin")
|
|
13
15
|
OPERATORS = ("eq", "ne", "lt", "le", "gt", "ge")
|
|
@@ -91,6 +93,69 @@ def _operand(operand, image, name, budget):
|
|
|
91
93
|
_term(operand, image, name)
|
|
92
94
|
|
|
93
95
|
|
|
96
|
+
def _address(control, image):
|
|
97
|
+
"""Check a lastWriter address: a segment register, an optional base register, a displacement and a width."""
|
|
98
|
+
address, name = control["address"], control["name"]
|
|
99
|
+
if not isinstance(address, dict) or set(address) - {"segment", "base", "displacement", "width"} or "segment" not in address:
|
|
100
|
+
raise ValueError(f"Relational control {name} address names segment, and optionally base, displacement and width")
|
|
101
|
+
if address["segment"] not in SEGMENTS:
|
|
102
|
+
raise ValueError(f"Relational control {name} address segment must be a segment register")
|
|
103
|
+
if "base" in address:
|
|
104
|
+
base = address["base"]
|
|
105
|
+
if (not isinstance(base, str) or base not in ALIASES or ALIASES[base][0] in SEGMENTS or ALIASES[base][1] != 0
|
|
106
|
+
or ALIASES[base][2] != image.bits):
|
|
107
|
+
raise ValueError(f"Relational control {name} address base must be a {image.bits}-bit general register")
|
|
108
|
+
# Signed or unsigned in the address width; a wider value would silently name another offset.
|
|
109
|
+
displacement = address.get("displacement", 0)
|
|
110
|
+
if type(displacement) is not int or not -(1 << (image.bits - 1)) <= displacement < 1 << image.bits:
|
|
111
|
+
raise ValueError(f"Relational control {name} address displacement must be a {image.bits}-bit integer")
|
|
112
|
+
integer(address.get("width"), 1, 32, "lastWriter address width")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def checkpoint_anchors(config):
|
|
116
|
+
"""The sites of every relational control's checkpoint anchors, which the trace adds to checkpoints.
|
|
117
|
+
|
|
118
|
+
Reads controls that validate_controls accepted.
|
|
119
|
+
"""
|
|
120
|
+
sites = set()
|
|
121
|
+
for control in config.get("relationalControls", []):
|
|
122
|
+
anchors = control["at"] if isinstance(control["at"], list) else [control["at"]]
|
|
123
|
+
sites.update(a["site"] for a in anchors if a.get("event") == "checkpoint")
|
|
124
|
+
return sites
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def memory_probes(config):
|
|
128
|
+
"""The lastWriter addresses to inspect at each checkpoint site: {site: [(control name, address)]}.
|
|
129
|
+
|
|
130
|
+
Reads controls that validate_controls accepted.
|
|
131
|
+
"""
|
|
132
|
+
probes = {}
|
|
133
|
+
for control in config.get("relationalControls", []):
|
|
134
|
+
if control.get("kind") == "lastWriter" and "address" in control:
|
|
135
|
+
anchors = control["at"] if isinstance(control["at"], list) else [control["at"]]
|
|
136
|
+
for site in {a["site"] for a in anchors}:
|
|
137
|
+
probes.setdefault(site, []).append((control["name"], control["address"]))
|
|
138
|
+
return probes
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def probe_memory(state, name, address):
|
|
142
|
+
"""What the model holds at a lastWriter address before the instruction runs, without a read event.
|
|
143
|
+
|
|
144
|
+
The row carries ``byteProducers`` as a read of those bytes would report them, or ``unresolved``
|
|
145
|
+
when the address cannot be inspected.
|
|
146
|
+
"""
|
|
147
|
+
offset = const(address.get("displacement", 0), state.bits)
|
|
148
|
+
if "base" in address:
|
|
149
|
+
offset = op("add", state.reg(address["base"]), offset)
|
|
150
|
+
segment = state.segment(address["segment"])
|
|
151
|
+
row = {"control": name, "segment": segment.report(), "offset": offset.report(), "width": address["width"]}
|
|
152
|
+
try:
|
|
153
|
+
keys = state.keys(segment, offset, address["width"])[3]
|
|
154
|
+
except StopPath as error:
|
|
155
|
+
return {**row, "unresolved": str(error)}
|
|
156
|
+
return {**row, "byteProducers": [state.byte_writer(i, key) for i, key in enumerate(keys)]}
|
|
157
|
+
|
|
158
|
+
|
|
94
159
|
def validate_controls(config, image):
|
|
95
160
|
"""Check every relational control before tracing; return them unchanged."""
|
|
96
161
|
controls = config.get("relationalControls", [])
|
|
@@ -145,11 +210,15 @@ def validate_controls(config, image):
|
|
|
145
210
|
if not isinstance(same, dict) or set(same) != {"before", "at"} or not all(isinstance(v, str) and v for v in same.values()):
|
|
146
211
|
raise ValueError(f"Relational control {name} sameValue names a before field and an at field")
|
|
147
212
|
elif kind == "lastWriter":
|
|
148
|
-
allowed |= {"writers", "byteWriters"}
|
|
149
|
-
_anchors(control, image, events=("read",))
|
|
213
|
+
allowed |= {"writers", "byteWriters", "address"}
|
|
214
|
+
_anchors(control, image, events=("checkpoint",) if "address" in control else ("read",))
|
|
215
|
+
if "address" in control:
|
|
216
|
+
_address(control, image)
|
|
150
217
|
lists = [control["writers"]] if "writers" in control else control.get("byteWriters")
|
|
151
218
|
if ("writers" in control) == ("byteWriters" in control) or not isinstance(lists, list) or not 1 <= len(lists) <= 32:
|
|
152
219
|
raise ValueError(f"Relational control {name} needs writers or byteWriters")
|
|
220
|
+
if "address" in control and "byteWriters" in control and len(lists) != control["address"]["width"]:
|
|
221
|
+
raise ValueError(f"Relational control {name} byteWriters lists {len(lists)} bytes; its address has {control['address']['width']}")
|
|
153
222
|
for writers in lists:
|
|
154
223
|
if not isinstance(writers, list) or not 1 <= len(writers) <= 64:
|
|
155
224
|
raise ValueError(f"Relational control {name} writer lists hold 1..64 entries")
|
|
@@ -542,11 +611,17 @@ def _occurrence(control, path, anchor, image):
|
|
|
542
611
|
verdicts.append(verdict)
|
|
543
612
|
return _worst(verdicts), detail
|
|
544
613
|
if kind == "lastWriter":
|
|
545
|
-
|
|
546
|
-
if
|
|
547
|
-
|
|
614
|
+
accessed = anchor
|
|
615
|
+
if "address" in control:
|
|
616
|
+
accessed = next(p for p in anchor["memoryProbes"] if p["control"] == control["name"])
|
|
617
|
+
if "unresolved" in accessed:
|
|
618
|
+
return "undecided", {"reason": f"the address cannot be inspected here: {accessed['unresolved']}",
|
|
619
|
+
"address": {k: accessed[k] for k in ("segment", "offset", "width")}}
|
|
620
|
+
lists = control.get("byteWriters") or [control["writers"]] * accessed["width"]
|
|
621
|
+
if len(lists) != accessed["width"]:
|
|
622
|
+
raise ValueError(f"Relational control {control['name']} byteWriters lists {len(lists)} bytes; the access at {anchor['site']} has {accessed['width']}")
|
|
548
623
|
rows, verdicts = [], []
|
|
549
|
-
for row, allowed in zip(
|
|
624
|
+
for row, allowed in zip(accessed["byteProducers"], lists):
|
|
550
625
|
if row.get("writeOrder") is not None:
|
|
551
626
|
writer = events[row["writeOrder"]]
|
|
552
627
|
verdict = "held" if writer["site"] in allowed else "violated"
|
|
@@ -560,6 +635,8 @@ def _occurrence(control, path, anchor, image):
|
|
|
560
635
|
verdicts.append(verdict)
|
|
561
636
|
via = path.frame_branch[order]
|
|
562
637
|
detail = {"bytes": rows, "via": via and {"site": via["site"], "taken": via["taken"], "order": via["order"]}}
|
|
638
|
+
if accessed is not anchor:
|
|
639
|
+
detail["address"] = {k: accessed[k] for k in ("segment", "offset", "width")}
|
|
563
640
|
wrong = [r for r in rows if r["verdict"] == "violated"]
|
|
564
641
|
if wrong:
|
|
565
642
|
detail["reason"] = "; ".join(f"byte {r['index']} was written at site {r['writer']['site']}, not a listed writer" if r["writer"]
|
|
@@ -1770,16 +1770,19 @@ def _run_report(image, config, command):
|
|
|
1770
1770
|
if command == "returns":
|
|
1771
1771
|
report = return_flows(report, config)
|
|
1772
1772
|
if command != "trace":
|
|
1773
|
-
kinds = {"arguments": ("address-formation", "read", "call", "call-return"), "effects": ("address-formation", "write", "call", "call-return", "return", "branch", "string-operation", "hardware-boundary",
|
|
1773
|
+
kinds = {"arguments": ("address-formation", "read", "call", "call-return"), "effects": ("address-formation", "write", "call", "far-jump", "call-return", "return", "branch", "string-operation", "hardware-boundary",
|
|
1774
1774
|
"flag-assumption", "flag-write", "flags-save", "flags-restore", "local-iret"),
|
|
1775
1775
|
"returns": ("return", "call-return", "compare", "branch", "write"),
|
|
1776
|
-
"guards": ("compare", "branch", "read", "write", "call", "call-return"),
|
|
1776
|
+
"guards": ("compare", "branch", "read", "write", "call", "far-jump", "call-return"),
|
|
1777
1777
|
"memory": ("read", "write", "address-formation")}[command]
|
|
1778
1778
|
for path in report["paths"]:
|
|
1779
1779
|
# Returns keep the transfers, conversions and reads that depend on a declared result.
|
|
1780
1780
|
consumed = {c["order"] for f in path.get("returnFlows", {}).get("results", ()) for c in f["consumers"]}
|
|
1781
1781
|
# Arguments keeps the writes its argument-frame slots cite as writers.
|
|
1782
1782
|
consumed |= {s["writerOrder"] for f in path.get("argumentFrames", ()) for s in f["slots"] if s["writerSite"] is not None}
|
|
1783
|
+
# A kept indirect far CALL or JMP keeps the pointer read its provenance cites.
|
|
1784
|
+
consumed |= {e["provenance"]["pointerRead"]["order"] for e in path["events"] if e["kind"] in kinds
|
|
1785
|
+
and isinstance(e.get("provenance"), dict) and "pointerRead" in e["provenance"]}
|
|
1783
1786
|
path["events"] = [e for e in path["events"] if e["kind"] in kinds or e["order"] in consumed or
|
|
1784
1787
|
(command == "effects" and e["kind"] == "read" and (e.get("nearPointerAccessCandidates") or e.get("nearPointerArgumentCandidates")))]
|
|
1785
1788
|
return report
|
|
@@ -6,11 +6,12 @@ from .image import integer
|
|
|
6
6
|
from .machine import (State, StopPath, REGISTERS, ALIASES, BRANCH_CONDITIONS, string_instruction,
|
|
7
7
|
string_count, string_effect, check_string_form, compare_string, repeated, string_width,
|
|
8
8
|
FLAT_PORT_REASON)
|
|
9
|
-
from .values import const, unknown, sources, op,
|
|
9
|
+
from .values import const, unknown, sources, op, join
|
|
10
10
|
from .result_flow import validate_contracts, result_contracts
|
|
11
11
|
from .loops import LoopTracker
|
|
12
12
|
from .memory_scopes import validate_scopes, capture_scopes, retain_scopes, scope_history
|
|
13
13
|
from .pcode_backend import interrupt_vector
|
|
14
|
+
from .relational import checkpoint_anchors, memory_probes, probe_memory
|
|
14
15
|
|
|
15
16
|
def call_target(image, site, ins):
|
|
16
17
|
if ins.mnemonic in ("lcall", "ljmp"):
|
|
@@ -21,6 +22,63 @@ def call_target(image, site, ins):
|
|
|
21
22
|
return None, {"reason": "computed transfer remains unresolved"}
|
|
22
23
|
|
|
23
24
|
|
|
25
|
+
def far_pointer_word(state, read, low):
|
|
26
|
+
"""One word of the far pointer ``read`` has just fetched, as ``(row, word)``.
|
|
27
|
+
|
|
28
|
+
``row`` reports the word's value and the producers and writers of its two bytes. ``word`` is the
|
|
29
|
+
modeled value of those bytes with all their sources, including the markers of declared return
|
|
30
|
+
results that ``producers`` leaves out, or None when a byte has no modeled value.
|
|
31
|
+
"""
|
|
32
|
+
rows = read["byteProducers"][low // 8:low // 8 + 2]
|
|
33
|
+
interval = read["interval"]
|
|
34
|
+
seg, base, start = interval["segment"], interval["base"], interval["start"]
|
|
35
|
+
keys = [(seg, base, start + i if seg == ("linear",) else (start + i) % (1 << state.bits))
|
|
36
|
+
for i in (low // 8, low // 8 + 1)]
|
|
37
|
+
word = join([state.memory[key] for key in keys]) if all(key in state.memory for key in keys) else None
|
|
38
|
+
return ({"value": None if word is None else word.number,
|
|
39
|
+
"producers": sorted({p for row in rows for p in row["producers"]}), "bytes": rows}, word)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def far_pointer_transfer(state, image):
|
|
43
|
+
"""Resolve an indirect far CALL or JMP from the m16:16 pointer the trace has just read.
|
|
44
|
+
|
|
45
|
+
Called right after ``state.get`` read the operand, so the newest event is that read; any other
|
|
46
|
+
newest event is an engine error, so the provenance never cites the wrong event. Returns
|
|
47
|
+
``(target, provenance, code_segment)``. ``provenance`` keeps the read's order and address, its
|
|
48
|
+
effective segment register, and both words with the producers and writers of their bytes. The
|
|
49
|
+
target is admitted only when both words are known and ``Image.far_pointer_target`` admits the
|
|
50
|
+
address they form; otherwise ``target`` and ``code_segment`` are None and ``provenance`` names
|
|
51
|
+
the reason. ``code_segment`` is the value CS takes: the segment word with every source of its
|
|
52
|
+
bytes (declared return results included), or the overlay
|
|
53
|
+
entry region's segment when the pointer names an FBOV trampoline.
|
|
54
|
+
"""
|
|
55
|
+
read = state.events[-1]
|
|
56
|
+
if read["kind"] != "read" or read["site"] != state.at or read["width"] != 4:
|
|
57
|
+
raise RuntimeError("an indirect far transfer needs its own m16:16 read as the newest event")
|
|
58
|
+
(offset_word, _), (segment_word, segment_value) = far_pointer_word(state, read, 0), far_pointer_word(state, read, 16)
|
|
59
|
+
provenance = {"encoding": "m16:16",
|
|
60
|
+
"pointerRead": {"order": read["order"], "segment": read["segment"], "offset": read["offset"],
|
|
61
|
+
"segmentRegister": read["effectiveSegmentRegister"]},
|
|
62
|
+
"offsetWord": offset_word, "segmentWord": segment_word}
|
|
63
|
+
unknown_words = [name for name, word in (("offset", offset_word), ("segment", segment_word)) if word["value"] is None]
|
|
64
|
+
if unknown_words:
|
|
65
|
+
provenance["reason"] = "far pointer " + " and ".join(unknown_words) + (" words" if len(unknown_words) > 1 else " word") + " unknown"
|
|
66
|
+
return None, provenance, None
|
|
67
|
+
provenance["loadedAddress"] = f"{segment_word['value']:04X}:{offset_word['value']:04X}"
|
|
68
|
+
target, admission = image.far_pointer_target(segment_word["value"], offset_word["value"])
|
|
69
|
+
admission = dict(admission)
|
|
70
|
+
reason = admission.pop("reason", None)
|
|
71
|
+
provenance["admission"] = admission
|
|
72
|
+
if target is None:
|
|
73
|
+
provenance["reason"] = reason
|
|
74
|
+
return None, provenance, None
|
|
75
|
+
if "trampoline" in admission:
|
|
76
|
+
return target, provenance, const(image.region(target)["segment"], 16, state.at)
|
|
77
|
+
# CS keeps every source of the segment word's bytes, so a declared result stored there stays a
|
|
78
|
+
# result origin of CS.
|
|
79
|
+
return target, provenance, segment_value
|
|
80
|
+
|
|
81
|
+
|
|
24
82
|
OVERLAP_REASON = "overlapping entry-path instructions; boundary unresolved"
|
|
25
83
|
RETURNS = {"ret": "near return", "retf": "far return", "iret": "interrupt return", "iretd": "interrupt return"}
|
|
26
84
|
INTERRUPTS = ("int", "int1", "int3", "into")
|
|
@@ -499,7 +557,10 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
|
|
|
499
557
|
total_string_steps = 0
|
|
500
558
|
string_limit = budget_input(config, "stringIterations")
|
|
501
559
|
total_limit = budget_input(config, "totalSteps")
|
|
502
|
-
|
|
560
|
+
# A control anchored at a checkpoint gets that checkpoint; a lastWriter control with an address
|
|
561
|
+
# inspects it there.
|
|
562
|
+
probes = memory_probes(config)
|
|
563
|
+
checkpoints = set(config.get("checkpoints", [])) | checkpoint_anchors(config)
|
|
503
564
|
# How often one path may pass the same instruction; a loop with a known bound needs it raised.
|
|
504
565
|
visit_limit = budget_input(config, "visitLimit")
|
|
505
566
|
# An unset field takes the value of the matching ordinary input.
|
|
@@ -732,7 +793,8 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
|
|
|
732
793
|
elif state.visits[at] > visit_limit:
|
|
733
794
|
raise StopPath(f"instruction repeated more than {visit_limit} times; raise visitLimit or read the loop's bound")
|
|
734
795
|
if at in checkpoints:
|
|
735
|
-
state.event("checkpoint", registers=snapshot(state)
|
|
796
|
+
state.event("checkpoint", registers=snapshot(state),
|
|
797
|
+
**({"memoryProbes": [probe_memory(state, n, a) for n, a in probes[at]]} if at in probes else {}))
|
|
736
798
|
m, following = ins.mnemonic, at + ins.size
|
|
737
799
|
is_string = string_instruction(ins)
|
|
738
800
|
if (0xf2 in ins.prefix or 0xf3 in ins.prefix) and not is_string:
|
|
@@ -801,9 +863,11 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
|
|
|
801
863
|
break
|
|
802
864
|
if m in ("call", "lcall"):
|
|
803
865
|
target, provenance = call_target(image, at, ins)
|
|
804
|
-
indirect_value = None
|
|
866
|
+
indirect_value = code_segment = None
|
|
805
867
|
if target is None and ins.operands and ins.operands[0].type in (X86_OP_REG, X86_OP_MEM):
|
|
806
868
|
indirect_value = state.get(ins, ins.operands[0], image)
|
|
869
|
+
if m == "lcall":
|
|
870
|
+
target, provenance, code_segment = far_pointer_transfer(state, image)
|
|
807
871
|
guard_checks = []
|
|
808
872
|
if indirect_value is not None:
|
|
809
873
|
for g in state.guards:
|
|
@@ -869,7 +933,8 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
|
|
|
869
933
|
"continuation": following, "returnIP": return_ip, "callSite": at,
|
|
870
934
|
"callerCS": state.reg("cs"), "localFlagsFrame": flags_frame})
|
|
871
935
|
if m == "lcall":
|
|
872
|
-
|
|
936
|
+
# An indirect far call loads CS from the pointer's segment word (far_pointer_transfer).
|
|
937
|
+
state.setreg("cs", const(target_region["segment"], 16, at) if code_segment is None else code_segment, at)
|
|
873
938
|
state.at = target
|
|
874
939
|
continue
|
|
875
940
|
if m in ("iret", "iretd"):
|
|
@@ -922,6 +987,15 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
|
|
|
922
987
|
continue
|
|
923
988
|
if m in ("jmp", "ljmp"):
|
|
924
989
|
target, provenance = call_target(image, at, ins)
|
|
990
|
+
if target is None and m == "ljmp" and ins.operands[0].type == X86_OP_MEM:
|
|
991
|
+
pointer = state.get(ins, ins.operands[0], image)
|
|
992
|
+
target, provenance, code_segment = far_pointer_transfer(state, image)
|
|
993
|
+
state.event("far-jump", target=target, provenance=provenance, indirectValue=pointer.report())
|
|
994
|
+
if target is None:
|
|
995
|
+
raise StopPath("unresolved jump: " + provenance["reason"])
|
|
996
|
+
state.setreg("cs", code_segment, at)
|
|
997
|
+
state.at = target
|
|
998
|
+
continue
|
|
925
999
|
if target is None:
|
|
926
1000
|
if continue_declared_jumps and at in image.indirect_jumps:
|
|
927
1001
|
deferred.append(deepcopy(state))
|