scientific-method-engine 6.2.0__tar.gz → 7.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/PKG-INFO +9 -1
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/README.md +8 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/pyproject.toml +1 -1
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/cli.py +3 -1
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/machine.py +23 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/relational.py +8 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/reports.py +70 -26
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/trace.py +69 -9
- scientific_method_engine-7.0.0/tests/test_entry_frame.py +156 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_x86.py +93 -13
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/.gitignore +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/LICENSE +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/oracle.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_table_continuations.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version:
|
|
3
|
+
Version: 7.0.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -161,3 +161,11 @@ control fails the report. A control left undecided by a stop, a limit or an unre
|
|
|
161
161
|
reported as `undecided` and never counts as held. Access reports also give each byte's
|
|
162
162
|
`writeOrder`, or why it has no modeled value. See the
|
|
163
163
|
[controls contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#relational-controls).
|
|
164
|
+
|
|
165
|
+
A trace-family query whose entry lies inside a function body names the function's entry in
|
|
166
|
+
`entryFrame: { "from": <site> }`. The engine traces from there to the query's entry and, when SP
|
|
167
|
+
sits at one offset from the function's entry SP at every arrival, starts the query in that frame,
|
|
168
|
+
so the function's return balances and its paths can be read to the end. The report's `entryFrame`
|
|
169
|
+
gives the offsets, or the reasons the frame is not established, in which case the query runs as
|
|
170
|
+
it would without the input. See the
|
|
171
|
+
[narrower entry contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#a-narrower-entry-inside-its-functions-frame).
|
|
@@ -145,3 +145,11 @@ control fails the report. A control left undecided by a stop, a limit or an unre
|
|
|
145
145
|
reported as `undecided` and never counts as held. Access reports also give each byte's
|
|
146
146
|
`writeOrder`, or why it has no modeled value. See the
|
|
147
147
|
[controls contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#relational-controls).
|
|
148
|
+
|
|
149
|
+
A trace-family query whose entry lies inside a function body names the function's entry in
|
|
150
|
+
`entryFrame: { "from": <site> }`. The engine traces from there to the query's entry and, when SP
|
|
151
|
+
sits at one offset from the function's entry SP at every arrival, starts the query in that frame,
|
|
152
|
+
so the function's return balances and its paths can be read to the end. The report's `entryFrame`
|
|
153
|
+
gives the offsets, or the reasons the frame is not established, in which case the query runs as
|
|
154
|
+
it would without the input. See the
|
|
155
|
+
[narrower entry contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#a-narrower-entry-inside-its-functions-frame).
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "
|
|
8
|
+
version = "7.0.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -17,9 +17,11 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
17
17
|
"effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
|
|
18
18
|
"effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
|
|
19
19
|
"callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges;\n"
|
|
20
|
-
"agreed is false where Ghidra ends the function at a call, conditional jump or interrupt the engine reads past (ghidraEndsFunction)
|
|
20
|
+
"agreed is false where Ghidra ends the function at a call, conditional jump or interrupt the engine reads past (ghidraEndsFunction),\n"
|
|
21
|
+
"or continues past a jmp, ljmp, return or hlt the engine stops at (ghidraContinues).\n"
|
|
21
22
|
"trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
|
|
22
23
|
"a violated control fails the report; an undecided one is reported and never counts as held.\n"
|
|
24
|
+
"entryFrame {from} starts an entry inside the function at from, in the frame a trace from there observed.\n"
|
|
23
25
|
"Declared table continuations are separate conditional paths that spend continuationBudget\n"
|
|
24
26
|
"(paths, totalSteps, maxSteps, visitLimit, stringIterations); ordinary computed transfers remain stopped.\n"
|
|
25
27
|
"Port accesses and interrupts are hardware-boundary events; portInputs supplies port reads as assumptions.\n"
|
|
@@ -171,6 +171,29 @@ class State:
|
|
|
171
171
|
self.event("flag-assumption", flag="DF", value=self.direction_flag.report(),
|
|
172
172
|
evidence="explicit query starting hypothesis, not native state")
|
|
173
173
|
|
|
174
|
+
def enter_frame(self, frame):
|
|
175
|
+
"""Place the entry inside the enclosing function's frame that an entryFrame trace established.
|
|
176
|
+
|
|
177
|
+
The root frame keeps the function's entry SP, so the function's own return balances against
|
|
178
|
+
it. SP, and BP when the trace found it at one offset, become offsets from that SP. A frame
|
|
179
|
+
that is absent or not established leaves the state as it was.
|
|
180
|
+
"""
|
|
181
|
+
if not frame or not frame["established"]:
|
|
182
|
+
return
|
|
183
|
+
base = self.frames[0]["sp"]
|
|
184
|
+
self.setreg(self.sp, op("add", base, const(frame["sp"], self.bits)), None)
|
|
185
|
+
if frame["bp"] is not None:
|
|
186
|
+
self.setreg(self.bp, op("add", base, const(frame["bp"], self.bits)), None)
|
|
187
|
+
|
|
188
|
+
def frame_offset(self, value):
|
|
189
|
+
"""A register value as a signed offset from the current frame's entry SP, or None when it is not one."""
|
|
190
|
+
base, delta = address_parts(self.frames[-1]["sp"])
|
|
191
|
+
other, other_delta = address_parts(value)
|
|
192
|
+
if other != base or base == ("absolute",):
|
|
193
|
+
return None
|
|
194
|
+
offset = (other_delta - delta) % (1 << self.bits)
|
|
195
|
+
return offset - (1 << self.bits) if offset >> (self.bits - 1) else offset
|
|
196
|
+
|
|
174
197
|
def forget_flags(self, keep_carry=False):
|
|
175
198
|
carry = self.carry_value() if keep_carry else None
|
|
176
199
|
self.flags = None
|
|
@@ -661,7 +661,13 @@ def evaluate_controls(report, config, image):
|
|
|
661
661
|
limit = integer(config.get("controlOccurrenceLimit", 4096), 1, 100000, "control occurrence limit")
|
|
662
662
|
spent = 0
|
|
663
663
|
entry_state = State(config["entry"], image, config)
|
|
664
|
+
frame = report.get("entryFrame")
|
|
665
|
+
# Entry register references read the entry as the trace started it, inside the observed frame.
|
|
666
|
+
entry_state.enter_frame(frame)
|
|
664
667
|
supplied = {ALIASES[r][0] for r in config.get("registers", {})}
|
|
668
|
+
if frame and frame["established"] and frame["bp"] is not None:
|
|
669
|
+
# The frame states BP as an offset from the entry SP, so no unknown input names BP.
|
|
670
|
+
supplied.add("ebp")
|
|
665
671
|
case_supplied = {(m["site"], ALIASES[r][0]) for m in config.get("callModels", [])
|
|
666
672
|
for case in m["cases"] for r in case.get("registers", {})}
|
|
667
673
|
paths = [_Path(p, entry_state, supplied, case_supplied) for p in report["paths"]]
|
|
@@ -716,6 +722,8 @@ def evaluate_controls(report, config, image):
|
|
|
716
722
|
if capped:
|
|
717
723
|
break
|
|
718
724
|
reasons = []
|
|
725
|
+
if frame and not frame["established"] and any(not r["returned"] for r in rows):
|
|
726
|
+
reasons.append("entryFrame was not established: " + "; ".join(frame["reasons"]))
|
|
719
727
|
if unread:
|
|
720
728
|
reasons.append("paths the trace did not read: " + "; ".join(g["reason"] for g in unread))
|
|
721
729
|
if capped:
|
|
@@ -1041,7 +1041,7 @@ def callees(image, config):
|
|
|
1041
1041
|
and not s["omittedRoutes"]
|
|
1042
1042
|
and not any(d.get("reason") in capped for at in s["entries"] for d in nodes[at]["dependencies"])
|
|
1043
1043
|
and not any(d.get("reason") in capped for i in s["dependencyEdges"] for d in edges[i]["dependencies"]))
|
|
1044
|
-
cross_check = _ghidra_cross_check(export, nodes, outgoing, omitted) if export is not None else None
|
|
1044
|
+
cross_check = _ghidra_cross_check(image, export, nodes, outgoing, omitted) if export is not None else None
|
|
1045
1045
|
known = {"sharedSites": {e["site"] for e in edges if shared_control(e)},
|
|
1046
1046
|
"recursiveSites": {e["site"] for e in edges if e["classification"] == "recursivePath"},
|
|
1047
1047
|
"writeSites": {o["site"] for n in nodes.values() for o in n["memoryObservations"] if o["boundaryUsable"] and "write" in o["access"]},
|
|
@@ -1121,22 +1121,60 @@ def _ghidra_key(g):
|
|
|
1121
1121
|
return g["site"], g["target"]
|
|
1122
1122
|
|
|
1123
1123
|
|
|
1124
|
+
# The flow types Ghidra's RefType builds with a fall-through (FlowType.hasFallthrough). Every other flow type ends the
|
|
1125
|
+
# function at its instruction, CONDITIONAL_CALL_TERMINATOR included.
|
|
1126
|
+
GHIDRA_FALL_THROUGH_FLOWS = frozenset((
|
|
1127
|
+
"FALL_THROUGH", "CONDITIONAL_JUMP", "UNCONDITIONAL_CALL", "CONDITIONAL_CALL", "CONDITIONAL_TERMINATOR",
|
|
1128
|
+
"COMPUTED_CALL", "CONDITIONAL_COMPUTED_CALL", "CONDITIONAL_COMPUTED_JUMP", "CALL_OVERRIDE_UNCONDITIONAL",
|
|
1129
|
+
"CALLOTHER_OVERRIDE_CALL"))
|
|
1130
|
+
|
|
1131
|
+
|
|
1124
1132
|
def _ghidra_falls_through(g):
|
|
1125
1133
|
"""Whether Ghidra continues to the next instruction at an exported edge's site, and what that was read from.
|
|
1126
1134
|
|
|
1127
1135
|
The export's fallsThrough also reflects a user's fall-through override. Copies of the script that leave it out
|
|
1128
|
-
are read by the flow type's name, which misses such an override
|
|
1136
|
+
are read by the flow type's name, which misses such an override: a flow type Ghidra gives a fall-through
|
|
1137
|
+
(GHIDRA_FALL_THROUGH_FLOWS) continues, and every other one ends the function.
|
|
1129
1138
|
"""
|
|
1130
1139
|
exported = g["fallsThrough"] is not None
|
|
1131
|
-
|
|
1140
|
+
named = g["flow"] in GHIDRA_FALL_THROUGH_FLOWS
|
|
1141
|
+
return {"ghidraFallsThrough": g["fallsThrough"] if exported else named,
|
|
1132
1142
|
"ghidraFallsThroughBasis": "fallsThrough" if exported else "flowName"}
|
|
1133
1143
|
|
|
1134
1144
|
|
|
1135
|
-
def
|
|
1145
|
+
def _engine_reads_on(image, ins):
|
|
1146
|
+
"""Whether the engine's body reading continues to the instruction after ins.
|
|
1147
|
+
|
|
1148
|
+
It stops at a jmp, ljmp, return or hlt, and reads on past every other instruction, including a call, a conditional
|
|
1149
|
+
jump and an interrupt. At a transfer outside the frame model it stops with a gap, so the result is None: the engine
|
|
1150
|
+
decided nothing there. It is None too when the engine did not read ins.
|
|
1151
|
+
"""
|
|
1152
|
+
if ins is None or unsupported_transfer(image, ins):
|
|
1153
|
+
return None
|
|
1154
|
+
m = base_mnemonic(ins)
|
|
1155
|
+
return not (m in RETURNS or m in ("hlt", "jmp", "ljmp"))
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def _ghidra_cross_check(image, export, nodes, outgoing, omitted):
|
|
1136
1159
|
"""Compare the engine's edges with Ghidra's for each caller both read; a Ghidra-only edge stays unchecked."""
|
|
1137
1160
|
callers = export["callers"]
|
|
1138
1161
|
compared = sorted(nodes.keys() & callers.keys())
|
|
1139
1162
|
rows = []
|
|
1163
|
+
# Rows where Ghidra ends the function at an instruction the engine reads past, and rows where Ghidra continues past
|
|
1164
|
+
# an instruction the engine stops at. Either way the two analyses disagree on the function's extent.
|
|
1165
|
+
ends, continues = [], []
|
|
1166
|
+
|
|
1167
|
+
def compare_extent(row, g, ins):
|
|
1168
|
+
reads_on = _engine_reads_on(image, ins)
|
|
1169
|
+
if reads_on is None:
|
|
1170
|
+
return row
|
|
1171
|
+
row |= _ghidra_falls_through(g)
|
|
1172
|
+
if reads_on and row["ghidraFallsThrough"] is False:
|
|
1173
|
+
ends.append(row)
|
|
1174
|
+
elif not reads_on and row["ghidraFallsThrough"] is True:
|
|
1175
|
+
continues.append(row)
|
|
1176
|
+
return row
|
|
1177
|
+
|
|
1140
1178
|
for caller in compared:
|
|
1141
1179
|
ours = outgoing.get(caller, [])
|
|
1142
1180
|
theirs = callers[caller]
|
|
@@ -1148,12 +1186,9 @@ def _ghidra_cross_check(export, nodes, outgoing, omitted):
|
|
|
1148
1186
|
g = matches.get((e["site"], e["target"]))
|
|
1149
1187
|
row = {"caller": caller, "site": e["site"], "target": e["target"], "engineEdge": e["id"],
|
|
1150
1188
|
"result": "engineOnly" if g is None else "agreement", "ghidraFlow": g["flow"] if g else None}
|
|
1151
|
-
# The engine reads on past every call and every conditional tail transfer, and stops at a jmp or ljmp.
|
|
1152
1189
|
# Ghidra ends the function at a call to a callee it treats as non-returning (CALL_TERMINATOR) or at an
|
|
1153
|
-
# instruction whose fall-through a user cleared.
|
|
1154
|
-
if g is
|
|
1155
|
-
row |= _ghidra_falls_through(g)
|
|
1156
|
-
rows.append(row)
|
|
1190
|
+
# instruction whose fall-through a user cleared, and continues past a jmp a user gave a fall-through.
|
|
1191
|
+
rows.append(row if g is None else compare_extent(row, g, instructions[e["site"]]))
|
|
1157
1192
|
for g in theirs:
|
|
1158
1193
|
if g["site"] is not None and _ghidra_key(g) in read:
|
|
1159
1194
|
continue
|
|
@@ -1162,35 +1197,40 @@ def _ghidra_cross_check(export, nodes, outgoing, omitted):
|
|
|
1162
1197
|
# SLEIGH lifts INT, INT1, INT3 and INTO to a computed call with no target, while the engine assumes the
|
|
1163
1198
|
# interrupt returns to the next instruction and records no edge. At INT1 and INT3 Ghidra's flow is a
|
|
1164
1199
|
# terminator that ends the function there.
|
|
1165
|
-
rows.append({"caller": caller, "site": g["site"], "target": None, "siteAddress": g["siteAddress"],
|
|
1166
|
-
|
|
1167
|
-
|
|
1200
|
+
rows.append(compare_extent({"caller": caller, "site": g["site"], "target": None, "siteAddress": g["siteAddress"],
|
|
1201
|
+
"targetAddress": None, "ghidraFlow": g["flow"], "result": "interrupt",
|
|
1202
|
+
"engineEdge": None}, g, ins))
|
|
1168
1203
|
continue
|
|
1169
|
-
# Ghidra's edge is evidence the engine did not check; it never becomes an engine edge.
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1204
|
+
# Ghidra's edge is evidence the engine did not check; it never becomes an engine edge. Its fall-through is
|
|
1205
|
+
# still compared where the engine read the instruction at its site.
|
|
1206
|
+
engine_edge = next((e["id"] for e in ours if g["site"] is not None and e["site"] == g["site"]), None)
|
|
1207
|
+
rows.append(compare_extent({"caller": caller, "site": g["site"], "target": g["target"], "siteAddress": g["siteAddress"],
|
|
1208
|
+
"targetAddress": g["targetAddress"], "ghidraFlow": g["flow"], "result": "ghidraOnly",
|
|
1209
|
+
"checked": False, "engineEdge": engine_edge}, g, ins))
|
|
1173
1210
|
counts = {kind: sum(r["result"] == kind for r in rows) for kind in ("agreement", "engineOnly", "ghidraOnly", "interrupt")}
|
|
1174
|
-
counts["ghidraEndsFunction"] =
|
|
1211
|
+
counts["ghidraEndsFunction"] = len(ends)
|
|
1212
|
+
counts["ghidraContinues"] = len(continues)
|
|
1175
1213
|
not_compared = {"engineCallers": sorted(nodes.keys() - callers.keys()), "ghidraCallers": sorted(callers.keys() - nodes.keys()),
|
|
1176
1214
|
"unmappedGhidraFunctions": export["unmappedFunctions"], "missingGhidraEntries": export["missingEntries"],
|
|
1177
1215
|
"unreadGhidraFunctions": export["unreadFunctions"],
|
|
1178
1216
|
"omittedEngineRoutes": [o["id"] for o in omitted if o["entry"] in callers]}
|
|
1179
|
-
# A site agrees only when every edge either analysis read there agrees and Ghidra continues past it
|
|
1180
|
-
|
|
1217
|
+
# A site agrees only when every edge either analysis read there agrees and Ghidra continues past it exactly when the
|
|
1218
|
+
# engine does.
|
|
1219
|
+
disputed = {r["site"] for r in rows if r["result"] != "agreement"} | {r["site"] for r in ends + continues}
|
|
1181
1220
|
return {"comparedCallers": compared, "edges": rows, "counts": counts, "notCompared": not_compared,
|
|
1182
1221
|
"agreed": (not counts["engineOnly"] and not counts["ghidraOnly"] and not any(not_compared.values())
|
|
1183
|
-
and not counts["ghidraEndsFunction"]),
|
|
1222
|
+
and not counts["ghidraEndsFunction"] and not counts["ghidraContinues"]),
|
|
1184
1223
|
"agreementSites": {r["site"] for r in rows if r["result"] == "agreement"} - disputed,
|
|
1185
1224
|
"interpretation": "Edges of each caller that both the engine and the Ghidra export read, matched by site and target "
|
|
1186
1225
|
"file offset; an unresolved call matches an unresolved call at its site, and a Ghidra target without a file offset "
|
|
1187
1226
|
"matches no engine edge. An interrupt row is Ghidra's targetless call at an instruction the engine read as an "
|
|
1188
|
-
"interrupt and assumed to return.
|
|
1189
|
-
"
|
|
1190
|
-
"
|
|
1191
|
-
"
|
|
1192
|
-
"
|
|
1193
|
-
"
|
|
1227
|
+
"interrupt and assumed to return. A row at an instruction the engine read carries whether Ghidra continues "
|
|
1228
|
+
"to the next instruction there (ghidraFallsThrough), read from the export's fallsThrough or, in an export "
|
|
1229
|
+
"without it, from the flow name (ghidraFallsThroughBasis). The engine reads on past every call, conditional jump "
|
|
1230
|
+
"and interrupt and stops at a jmp, ljmp, return or hlt. A row where Ghidra ends the function at an instruction "
|
|
1231
|
+
"the engine reads past (ghidraEndsFunction) or continues past one the engine stops at (ghidraContinues) "
|
|
1232
|
+
"counts against agreed. A ghidraOnly edge is Ghidra's claim: the engine did not check it and never adds it "
|
|
1233
|
+
"to its graph. Agreement means both analyses read the edge, not that it executes."}
|
|
1194
1234
|
|
|
1195
1235
|
|
|
1196
1236
|
# These branches test CX/ECX (LOOPE/LOOPNE also ZF), so an adjacent CMP/TEST never describes their predicate.
|
|
@@ -1538,6 +1578,8 @@ def _run_report(image, config, command):
|
|
|
1538
1578
|
raise ValueError("relationalControls apply only to " + ", ".join(TRACE_COMMANDS))
|
|
1539
1579
|
if "controlOccurrenceLimit" in config and command not in TRACE_COMMANDS:
|
|
1540
1580
|
raise ValueError("controlOccurrenceLimit applies only to " + ", ".join(TRACE_COMMANDS))
|
|
1581
|
+
if "entryFrame" in config and command not in TRACE_COMMANDS:
|
|
1582
|
+
raise ValueError("entryFrame applies only to " + ", ".join(TRACE_COMMANDS))
|
|
1541
1583
|
if command == "operand":
|
|
1542
1584
|
return operand_provenance(image, config)
|
|
1543
1585
|
if command == "target":
|
|
@@ -1577,6 +1619,8 @@ def _run_report(image, config, command):
|
|
|
1577
1619
|
report = argument_frames(report, image)
|
|
1578
1620
|
if command == "allocation":
|
|
1579
1621
|
result = allocations(report, config)
|
|
1622
|
+
if "entryFrame" in report:
|
|
1623
|
+
result["entryFrame"] = report["entryFrame"]
|
|
1580
1624
|
if controls is not None:
|
|
1581
1625
|
result["relationalControls"] = controls
|
|
1582
1626
|
return result
|
|
@@ -323,7 +323,55 @@ def validate_continuation_budget(config):
|
|
|
323
323
|
for key, value in budget.items()}
|
|
324
324
|
|
|
325
325
|
|
|
326
|
-
|
|
326
|
+
# The stop of an entryFrame trace's path at its first arrival at the query's entry.
|
|
327
|
+
ENTRY_ARRIVAL = "reached the query's entry"
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def entry_frame(image, config, entry):
|
|
331
|
+
"""The query entry's place in the frame of the function that holds it, observed on a trace from that function's entry.
|
|
332
|
+
|
|
333
|
+
Returns None without an ``entryFrame`` input. Each path from ``entryFrame.from`` stops at its first
|
|
334
|
+
arrival at the entry, since the query itself traces everything after it. The frame is established
|
|
335
|
+
when every path was read until it arrived or returned, at least one arrived, every arrival was in
|
|
336
|
+
a frame of that function, and SP sat at one offset from that frame's entry SP at all of them. BP is
|
|
337
|
+
stated only when it also sat at one offset at all of them.
|
|
338
|
+
"""
|
|
339
|
+
spec = config.get("entryFrame")
|
|
340
|
+
if spec is None:
|
|
341
|
+
return None
|
|
342
|
+
if not isinstance(spec, dict) or set(spec) != {"from"}:
|
|
343
|
+
raise ValueError("entryFrame takes only from, the entry of the function that holds the query's entry")
|
|
344
|
+
start = integer(spec["from"], 0, len(image.data) - 1, "entryFrame.from")
|
|
345
|
+
registers = config.get("registers", {})
|
|
346
|
+
if isinstance(registers, dict) and any(ALIASES.get(r, ("",))[0] in ("esp", "ebp") for r in registers):
|
|
347
|
+
raise ValueError("entryFrame observes SP and BP at the entry; registers cannot also supply them")
|
|
348
|
+
# Checkpoints and return contracts only add report detail, which this trace never reports.
|
|
349
|
+
prefix = {k: v for k, v in config.items()
|
|
350
|
+
if k not in ("entryFrame", "relationalControls", "controlOccurrenceLimit", "checkpoints", "returnContracts")}
|
|
351
|
+
report = trace(image, {**prefix, "entry": start}, continue_declared_jumps=False, track_loops=False, arrive=entry)
|
|
352
|
+
arrivals = [p["arrival"] for p in report["paths"] if p["stop"] == ENTRY_ARRIVAL]
|
|
353
|
+
reasons = [f"a path from {start} stopped at {p['stopSite']} before reaching the entry: {p['stop']}"
|
|
354
|
+
for p in report["paths"] if not p["returned"] and p["stop"] != ENTRY_ARRIVAL]
|
|
355
|
+
reasons += [f"the trace from {start} left paths unread: {g['reason']}" for g in report["gaps"]]
|
|
356
|
+
reasons += [f"the entry was reached inside a call to {a['frameEntry']}" for a in arrivals if a["frameEntry"] != start]
|
|
357
|
+
if not arrivals:
|
|
358
|
+
reasons.append(f"no path from {start} reached the entry")
|
|
359
|
+
offsets = sorted({a["sp"] for a in arrivals if a["sp"] is not None})
|
|
360
|
+
if any(a["sp"] is None for a in arrivals):
|
|
361
|
+
reasons.append("SP at an arrival is not an offset from the function's entry SP")
|
|
362
|
+
elif len(offsets) > 1:
|
|
363
|
+
reasons.append("arrivals reach the entry with SP at different offsets: " + ", ".join(map(str, offsets)))
|
|
364
|
+
established = not reasons
|
|
365
|
+
bp = {a["bp"] for a in arrivals}
|
|
366
|
+
return {"from": start, "established": established, "sp": offsets[0] if established else None,
|
|
367
|
+
"bp": next(iter(bp)) if established and len(bp) == 1 else None,
|
|
368
|
+
"arrivals": len(arrivals), "pathsRead": len(report["paths"]), "stepsUsed": report["stepsUsed"],
|
|
369
|
+
"reasons": reasons,
|
|
370
|
+
"meaning": "offsets from the entry SP of the function at from, observed at each first arrival on the "
|
|
371
|
+
"traced paths under the query's own inputs; bp null leaves BP unknown"}
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None):
|
|
327
375
|
"""Trace bounded paths, preserving declared-table continuations as separate conditional evidence.
|
|
328
376
|
|
|
329
377
|
Ordinary paths run first. A path stopped at a declared indirect jump is then
|
|
@@ -332,7 +380,8 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True):
|
|
|
332
380
|
what the ordinary paths spend never leaves the continuations without budget.
|
|
333
381
|
Callers that read only ordinary paths pass continue_declared_jumps=False.
|
|
334
382
|
Callers that never report the paths pass track_loops=False, and their paths
|
|
335
|
-
carry no ``loops`` record.
|
|
383
|
+
carry no ``loops`` record. entry_frame passes arrive, the query entry:
|
|
384
|
+
each path stops at its first arrival at the site with an ``arrival`` record.
|
|
336
385
|
"""
|
|
337
386
|
entry = integer(config.get("entry"), 0, len(image.data) - 1, "entry")
|
|
338
387
|
if not any(entry in r["entries"] for r in image.regions):
|
|
@@ -369,7 +418,9 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True):
|
|
|
369
418
|
if r not in ALIASES or type(n) is not int or not 0 <= n < 1 << ALIASES[r][2]:
|
|
370
419
|
raise ValueError("Invalid model register")
|
|
371
420
|
explicit_continuation_budget = validate_continuation_budget(config)
|
|
421
|
+
observed_frame = entry_frame(image, config, entry) if arrive is None else None
|
|
372
422
|
root = State(entry, image, config)
|
|
423
|
+
root.enter_frame(observed_frame)
|
|
373
424
|
# Each path carries its own loop record; forks copy it with the rest of the state.
|
|
374
425
|
root.loops = (LoopTracker(integer(config.get("loopIterationLimit", 64), 1, 1024, "loopIterationLimit"))
|
|
375
426
|
if track_loops else None)
|
|
@@ -417,6 +468,9 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True):
|
|
|
417
468
|
path = {"returned": returned, "stop": reason, "stopSite": None if returned else s.at, "steps": s.steps,
|
|
418
469
|
"instructionPath": s.path, "guards": s.guards, "events": s.events,
|
|
419
470
|
"registers": snapshot(s), "conditionalModels": s.conditional}
|
|
471
|
+
if reason == ENTRY_ARRIVAL:
|
|
472
|
+
path["arrival"] = {"frameEntry": s.frames[-1]["entry"], "sp": s.frame_offset(s.reg(s.sp)),
|
|
473
|
+
"bp": s.frame_offset(s.reg(s.bp))}
|
|
420
474
|
if s.loops is not None:
|
|
421
475
|
path["loops"] = s.loops.report()
|
|
422
476
|
assumptions = getattr(s, "declared_jump_assumptions", [])
|
|
@@ -524,6 +578,9 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True):
|
|
|
524
578
|
state = pending.pop()
|
|
525
579
|
try:
|
|
526
580
|
while True:
|
|
581
|
+
if arrive is not None and state.at == arrive:
|
|
582
|
+
finish(state, ENTRY_ARRIVAL)
|
|
583
|
+
break
|
|
527
584
|
if continuing:
|
|
528
585
|
if state.continuation_steps >= continuation_budget["maxSteps"]:
|
|
529
586
|
raise StopPath("continuation step limit; loop progress unresolved")
|
|
@@ -808,10 +865,13 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True):
|
|
|
808
865
|
else:
|
|
809
866
|
ordinary_steps, ordinary_string_steps = total_steps, total_string_steps
|
|
810
867
|
continuation_steps = continuation_string_steps = 0
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
868
|
+
result = {"paths": outputs, "declaredContinuationPaths": conditional_outputs, "gaps": global_gaps,
|
|
869
|
+
"hardwareBoundaries": hardware_placement(outputs, conditional_outputs, global_gaps[:first_continuation_gap]),
|
|
870
|
+
"completeWithinModel": not global_gaps[:first_continuation_gap] and bool(outputs) and all(p["returned"] for p in outputs),
|
|
871
|
+
"nativeReachability": "unconfirmed", "stepsUsed": ordinary_steps, "stringIterationsUsed": ordinary_string_steps,
|
|
872
|
+
"continuationStepsUsed": continuation_steps, "continuationStringIterationsUsed": continuation_string_steps,
|
|
873
|
+
"limits": {"steps": max_steps, "paths": ordinary_max_paths, "depth": max_depth,
|
|
874
|
+
"continuation": continuation_budget}}
|
|
875
|
+
if observed_frame is not None:
|
|
876
|
+
result["entryFrame"] = observed_frame
|
|
877
|
+
return result
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""A narrower entry inside a function's frame, observed from the function's entry (entryFrame). Synthetic code only."""
|
|
2
|
+
import unittest
|
|
3
|
+
from test_x86 import Code, report
|
|
4
|
+
|
|
5
|
+
STACK = {"ss": 0x3000, "ds": 0x2000}
|
|
6
|
+
# push bp; mov bp, sp; sub sp, 4; push si
|
|
7
|
+
PROLOGUE = "55 8b ec 83 ec 04 56"
|
|
8
|
+
# pop si; mov sp, bp; pop bp; ret
|
|
9
|
+
EPILOGUE = "5e 8b e5 5d c3"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def cleanup():
|
|
13
|
+
"""A guard skips the assignment to [bp-2]; the cleanup read takes it on both edges."""
|
|
14
|
+
return (Code().emit(PROLOGUE).label("narrow").emit("85 c0").branch("74", "skip").label("assign")
|
|
15
|
+
.emit("c7 46 fe 01 00").label("skip").label("read").emit("8b 46 fe " + EPILOGUE))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def run(c, command="trace", entries=("narrow",), **extra):
|
|
19
|
+
data = c.bytes()
|
|
20
|
+
regions = [{"name": "synthetic", "start": 0, "end": len(data), "ip": 0, "segment": 0x1000, "resident": True,
|
|
21
|
+
"entries": [0, *(c.labels[e] for e in entries)], "evidence": "synthetic declared code extent"}]
|
|
22
|
+
return report(c, command, regions=regions, entry=c.labels[entries[0]], **{"registers": STACK, **extra})
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def slot(c, *writers):
|
|
26
|
+
return [{"name": "slot", "kind": "lastWriter", "at": {"site": c.labels["read"], "event": "read"},
|
|
27
|
+
"writers": [c.labels["assign"], *writers]}]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def verdict(result):
|
|
31
|
+
return result["relationalControls"]["controls"][0]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class EntryFrameTests(unittest.TestCase):
|
|
35
|
+
def test_without_an_entry_frame_the_function_return_stops_every_path(self):
|
|
36
|
+
c = cleanup()
|
|
37
|
+
r = run(c, relationalControls=slot(c, "entryState"))
|
|
38
|
+
self.assertNotIn("entryFrame", r)
|
|
39
|
+
self.assertEqual({p["stop"] for p in r["paths"]}, {"return frame or stack balance differs from the call"})
|
|
40
|
+
self.assertEqual(verdict(r)["verdict"], "undecided")
|
|
41
|
+
|
|
42
|
+
def test_the_observed_frame_lets_the_function_return_and_the_control_hold(self):
|
|
43
|
+
c = cleanup()
|
|
44
|
+
r = run(c, relationalControls=slot(c, "entryState"), entryFrame={"from": 0})
|
|
45
|
+
frame = r["entryFrame"]
|
|
46
|
+
self.assertTrue(frame["established"])
|
|
47
|
+
self.assertEqual((frame["sp"], frame["bp"], frame["arrivals"], frame["reasons"]), (-8, -2, 1, []))
|
|
48
|
+
self.assertTrue(r["completeWithinModel"])
|
|
49
|
+
self.assertTrue(all(p["returned"] for p in r["paths"]))
|
|
50
|
+
result = verdict(r)
|
|
51
|
+
self.assertEqual(result["verdict"], "held")
|
|
52
|
+
edges = {o["via"]["taken"]: o["bytes"][0] for p in result["paths"] for o in p["occurrences"]}
|
|
53
|
+
self.assertEqual(edges[False]["writer"]["site"], c.labels["assign"])
|
|
54
|
+
self.assertEqual(edges[True]["unwritten"]["cause"], "no write on this path")
|
|
55
|
+
|
|
56
|
+
def test_a_skipped_assignment_violates_the_control_inside_the_observed_frame(self):
|
|
57
|
+
c = cleanup()
|
|
58
|
+
with self.assertRaisesRegex(ValueError, "slot violated on path"):
|
|
59
|
+
run(c, relationalControls=slot(c), entryFrame={"from": 0})
|
|
60
|
+
|
|
61
|
+
def test_arguments_count_from_the_function_entry_sp(self):
|
|
62
|
+
# The narrower entry reads the function's first argument through BP.
|
|
63
|
+
c = Code().emit(PROLOGUE).label("narrow").emit("8b 46 04 " + EPILOGUE)
|
|
64
|
+
r = run(c, "arguments", entryFrame={"from": 0})
|
|
65
|
+
read = next(e for e in r["paths"][0]["events"] if e["kind"] == "read" and e["site"] == c.labels["narrow"])
|
|
66
|
+
self.assertEqual(read["argument"]["offsetFromEntrySP"], 2)
|
|
67
|
+
|
|
68
|
+
def test_arrivals_with_different_sp_leave_the_frame_unestablished(self):
|
|
69
|
+
# One edge pushes an extra word before the narrower entry.
|
|
70
|
+
c = (Code().emit("55 8b ec 85 c0").branch("74", "join").emit("53").label("join").label("narrow")
|
|
71
|
+
.emit("8b e5 5d c3").label("dead").emit("90"))
|
|
72
|
+
r = run(c, entryFrame={"from": 0}, relationalControls=[
|
|
73
|
+
{"name": "dead", "kind": "reach", "at": {"site": c.labels["dead"]}, "expect": "never"}])
|
|
74
|
+
frame = r["entryFrame"]
|
|
75
|
+
self.assertFalse(frame["established"])
|
|
76
|
+
self.assertEqual((frame["sp"], frame["arrivals"]), (None, 2))
|
|
77
|
+
self.assertIn("different offsets: -4, -2", " ".join(frame["reasons"]))
|
|
78
|
+
self.assertEqual({p["stop"] for p in r["paths"]}, {"return frame or stack balance differs from the call"})
|
|
79
|
+
result = verdict(r)
|
|
80
|
+
self.assertEqual(result["verdict"], "undecided")
|
|
81
|
+
self.assertIn("entryFrame was not established", " ".join(result["reasons"]))
|
|
82
|
+
|
|
83
|
+
def test_a_stop_before_the_entry_leaves_the_frame_unestablished(self):
|
|
84
|
+
# An unresolved call on one edge stops that path before it reaches the narrower entry.
|
|
85
|
+
c = Code().emit("55 8b ec 85 c0").branch("74", "narrow").emit("ff d3").label("narrow").emit("5d c3")
|
|
86
|
+
frame = run(c, entryFrame={"from": 0})["entryFrame"]
|
|
87
|
+
self.assertFalse(frame["established"])
|
|
88
|
+
self.assertIn("before reaching the entry", " ".join(frame["reasons"]))
|
|
89
|
+
|
|
90
|
+
def test_a_path_limit_before_the_entry_leaves_the_frame_unestablished(self):
|
|
91
|
+
c = Code().emit("55 8b ec 85 c0").branch("74", "narrow").emit("90").label("narrow").emit("5d c3")
|
|
92
|
+
frame = run(c, entryFrame={"from": 0}, maxPaths=1)["entryFrame"]
|
|
93
|
+
self.assertFalse(frame["established"])
|
|
94
|
+
self.assertIn("left paths unread: path limit", " ".join(frame["reasons"]))
|
|
95
|
+
|
|
96
|
+
def test_only_the_first_arrival_counts_and_a_stop_after_it_is_the_query_s_own(self):
|
|
97
|
+
# After the narrower entry an unresolved call stops one edge; the query reports that stop itself.
|
|
98
|
+
c = (Code().emit(PROLOGUE).label("narrow").emit("85 c0").branch("74", "out").emit("ff d3").label("out")
|
|
99
|
+
.emit(EPILOGUE))
|
|
100
|
+
r = run(c, entryFrame={"from": 0})
|
|
101
|
+
self.assertTrue(r["entryFrame"]["established"])
|
|
102
|
+
self.assertEqual(r["entryFrame"]["pathsRead"], 1)
|
|
103
|
+
self.assertEqual(sorted(p["returned"] for p in r["paths"]), [False, True])
|
|
104
|
+
|
|
105
|
+
def test_an_entry_no_path_reaches_leaves_the_frame_unestablished(self):
|
|
106
|
+
c = Code().emit("55 8b ec 5d c3").label("narrow").emit("c3")
|
|
107
|
+
frame = run(c, entryFrame={"from": 0})["entryFrame"]
|
|
108
|
+
self.assertEqual((frame["established"], frame["arrivals"]), (False, 0))
|
|
109
|
+
self.assertIn("no path from 0 reached the entry", frame["reasons"])
|
|
110
|
+
|
|
111
|
+
def test_an_entry_reached_inside_another_function_leaves_the_frame_unestablished(self):
|
|
112
|
+
c = Code().emit("55 8b ec").branch("e8", "helper").emit("5d c3").label("helper").label("narrow").emit("c3")
|
|
113
|
+
frame = run(c, entryFrame={"from": 0})["entryFrame"]
|
|
114
|
+
self.assertFalse(frame["established"])
|
|
115
|
+
self.assertIn(f"inside a call to {c.labels['helper']}", " ".join(frame["reasons"]))
|
|
116
|
+
|
|
117
|
+
def test_bp_that_is_not_a_frame_offset_stays_unknown(self):
|
|
118
|
+
# BP holds a loaded word at the narrower entry, so only SP is stated.
|
|
119
|
+
c = Code().emit("55 8b 2e 20 00").label("narrow").emit("5d c3")
|
|
120
|
+
r = run(c, entryFrame={"from": 0})
|
|
121
|
+
self.assertEqual((r["entryFrame"]["established"], r["entryFrame"]["sp"], r["entryFrame"]["bp"]), (True, -2, None))
|
|
122
|
+
self.assertTrue(r["paths"][0]["returned"])
|
|
123
|
+
|
|
124
|
+
def test_an_entry_bp_input_the_frame_states_is_undecided(self):
|
|
125
|
+
# BP enters as an offset from the entry SP, so no unknown input names it; that never violates.
|
|
126
|
+
c = (Code().emit(PROLOGUE).label("narrow").emit("89 e8 85 c0").label("branch").branch("74", "out").label("out")
|
|
127
|
+
.emit(EPILOGUE))
|
|
128
|
+
rule = {"name": "frame pointer", "kind": "origin", "at": {"site": c.labels["branch"], "event": "branch"},
|
|
129
|
+
"value": {"field": "left"}, "expect": {"inputs": {"include": [{"entryRegister": "bp"}]}}}
|
|
130
|
+
r = run(c, relationalControls=[rule], entryFrame={"from": 0})
|
|
131
|
+
self.assertEqual(r["entryFrame"]["bp"], -2)
|
|
132
|
+
result = verdict(r)
|
|
133
|
+
self.assertEqual(result["verdict"], "undecided")
|
|
134
|
+
self.assertIn("supplies its entry value", result["paths"][0]["occurrences"][0]["reason"])
|
|
135
|
+
# SP stays an unknown input, so a value derived from it holds.
|
|
136
|
+
held = run(c, relationalControls=[{**rule, "expect": {"inputs": {"include": [{"entryRegister": "sp"}]}}}],
|
|
137
|
+
entryFrame={"from": 0})
|
|
138
|
+
self.assertEqual(verdict(held)["verdict"], "held")
|
|
139
|
+
|
|
140
|
+
def test_rejected_inputs(self):
|
|
141
|
+
c = cleanup()
|
|
142
|
+
for value, message in (({"from": 0, "sp": -8}, "takes only from"), (0, "takes only from"),
|
|
143
|
+
({"from": -1}, "entryFrame.from")):
|
|
144
|
+
with self.assertRaisesRegex(ValueError, message):
|
|
145
|
+
run(c, entryFrame=value)
|
|
146
|
+
with self.assertRaisesRegex(ValueError, "registers cannot also supply them"):
|
|
147
|
+
run(c, entryFrame={"from": 0}, registers={**STACK, "bp": 0x100})
|
|
148
|
+
with self.assertRaisesRegex(ValueError, "entryFrame applies only to"):
|
|
149
|
+
run(c, "bounds", entryFrame={"from": 0}, query={"entry": 0})
|
|
150
|
+
# The function entry must itself be an established entry.
|
|
151
|
+
with self.assertRaisesRegex(ValueError, "established region entry"):
|
|
152
|
+
run(c, entryFrame={"from": c.labels["assign"]})
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
if __name__ == "__main__":
|
|
156
|
+
unittest.main()
|
|
@@ -681,7 +681,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
|
|
|
681
681
|
# Ghidra's computed target never becomes an engine edge.
|
|
682
682
|
self.assertIsNone(r["edges"][2]["target"])
|
|
683
683
|
self.assertEqual(r["edges"][2]["classification"], "unresolved")
|
|
684
|
-
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 2, "ghidraOnly": 1, "interrupt": 0, "ghidraEndsFunction": 0})
|
|
684
|
+
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 2, "ghidraOnly": 1, "interrupt": 0, "ghidraEndsFunction": 0, "ghidraContinues": 0})
|
|
685
685
|
self.assertEqual(check["comparedCallers"], [0, b])
|
|
686
686
|
self.assertEqual(check["notCompared"]["engineCallers"], [a])
|
|
687
687
|
self.assertFalse(check["agreed"])
|
|
@@ -695,7 +695,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
|
|
|
695
695
|
r = run_report(data, cfg, "callees")
|
|
696
696
|
check = r["ghidraCrossCheck"]
|
|
697
697
|
self.assertTrue(check["agreed"])
|
|
698
|
-
self.assertEqual(check["counts"], {"agreement": 2, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0, "ghidraEndsFunction": 0})
|
|
698
|
+
self.assertEqual(check["counts"], {"agreement": 2, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0, "ghidraEndsFunction": 0, "ghidraContinues": 0})
|
|
699
699
|
# Agreement on an unresolved call leaves the engine's edge unresolved.
|
|
700
700
|
self.assertEqual(r["edges"][1]["classification"], "unresolved")
|
|
701
701
|
|
|
@@ -707,7 +707,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
|
|
|
707
707
|
cfg = configuration(data, ghidraCallEdges=export)
|
|
708
708
|
cfg["regions"][0]["entries"] = [0, 6]
|
|
709
709
|
check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
|
|
710
|
-
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 1, "ghidraOnly": 1, "interrupt": 0, "ghidraEndsFunction": 0})
|
|
710
|
+
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 1, "ghidraOnly": 1, "interrupt": 0, "ghidraEndsFunction": 0, "ghidraContinues": 0})
|
|
711
711
|
self.assertFalse(check["agreed"])
|
|
712
712
|
cfg["controls"] = {"ghidraAgreementSites": [3]}
|
|
713
713
|
with self.assertRaisesRegex(ValueError, "positive control missed: ghidraAgreementSites"):
|
|
@@ -723,7 +723,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
|
|
|
723
723
|
r = run_report(data, cfg, "callees")
|
|
724
724
|
check = r["ghidraCrossCheck"]
|
|
725
725
|
self.assertTrue(check["agreed"])
|
|
726
|
-
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 2, "ghidraEndsFunction": 0})
|
|
726
|
+
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 2, "ghidraEndsFunction": 0, "ghidraContinues": 0})
|
|
727
727
|
rows = [e for e in check["edges"] if e["result"] == "interrupt"]
|
|
728
728
|
self.assertEqual([(e["site"], e["ghidraFallsThrough"]) for e in rows], [(0, True), (2, True)])
|
|
729
729
|
self.assertEqual([e["site"] for e in r["edges"]], [3])
|
|
@@ -748,7 +748,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
|
|
|
748
748
|
data = bytes.fromhex("cc c3")
|
|
749
749
|
cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, None, "COMPUTED_CALL_TERMINATOR")]}))
|
|
750
750
|
check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
|
|
751
|
-
self.assertEqual(check["counts"], {"agreement": 0, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 1, "ghidraEndsFunction": 1})
|
|
751
|
+
self.assertEqual(check["counts"], {"agreement": 0, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 1, "ghidraEndsFunction": 1, "ghidraContinues": 0})
|
|
752
752
|
self.assertFalse(check["agreed"])
|
|
753
753
|
|
|
754
754
|
def test_the_exported_falls_through_decides_an_interrupt_over_the_flow_name(self):
|
|
@@ -790,7 +790,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
|
|
|
790
790
|
self.assertEqual(row["result"], "agreement")
|
|
791
791
|
self.assertEqual((row["ghidraFallsThrough"], row["ghidraFallsThroughBasis"]), (falls_through, "fallsThrough"))
|
|
792
792
|
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0,
|
|
793
|
-
"ghidraEndsFunction": 0 if falls_through else 1})
|
|
793
|
+
"ghidraEndsFunction": 0 if falls_through else 1, "ghidraContinues": 0})
|
|
794
794
|
self.assertEqual(check["agreed"], agreed)
|
|
795
795
|
# A call Ghidra ends the function at is no agreement site.
|
|
796
796
|
cfg["controls"] = {"ghidraAgreementSites": [0]}
|
|
@@ -814,19 +814,99 @@ class GhidraCrossCheckTests(unittest.TestCase):
|
|
|
814
814
|
self.assertEqual(check["counts"]["ghidraEndsFunction"], 0 if falls_through else 1)
|
|
815
815
|
self.assertEqual(check["agreed"], falls_through)
|
|
816
816
|
|
|
817
|
-
def
|
|
818
|
-
# jmp 3; ret; ret.
|
|
817
|
+
def test_a_jmp_tail_transfer_ghidra_continues_past_counts_against_agreed(self):
|
|
818
|
+
# jmp 3; ret; ret. The engine stops at the unconditional jump into the entry at 3. Ghidra stops there too,
|
|
819
|
+
# unless a user gave the jump a fall-through.
|
|
819
820
|
data = bytes.fromhex("eb 01 c3 c3")
|
|
820
|
-
|
|
821
|
+
for falls_through in (False, True):
|
|
822
|
+
with self.subTest(fallsThrough=falls_through):
|
|
823
|
+
cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 3, "UNCONDITIONAL_JUMP", falls_through)], 3: []}))
|
|
824
|
+
cfg["regions"][0]["entries"] = [0, 3]
|
|
825
|
+
check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
|
|
826
|
+
[row] = check["edges"]
|
|
827
|
+
self.assertEqual(row["result"], "agreement")
|
|
828
|
+
self.assertEqual((row["ghidraFallsThrough"], row["ghidraFallsThroughBasis"]), (falls_through, "fallsThrough"))
|
|
829
|
+
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0,
|
|
830
|
+
"ghidraEndsFunction": 0, "ghidraContinues": 1 if falls_through else 0})
|
|
831
|
+
self.assertEqual(check["agreed"], not falls_through)
|
|
832
|
+
# A jump Ghidra continues past is no agreement site.
|
|
833
|
+
cfg["controls"] = {"ghidraAgreementSites": [0]}
|
|
834
|
+
if falls_through:
|
|
835
|
+
with self.assertRaisesRegex(ValueError, "positive control missed: ghidraAgreementSites"):
|
|
836
|
+
run_report(data, cfg, "callees")
|
|
837
|
+
else:
|
|
838
|
+
run_report(data, cfg, "callees")
|
|
839
|
+
|
|
840
|
+
def test_a_jmp_in_an_export_without_falls_through_reads_the_flow_name(self):
|
|
841
|
+
# An unconditional jump's flow never falls through, so an export from an older copy of the script agrees at the
|
|
842
|
+
# jump. A user's fall-through override on it goes unseen.
|
|
843
|
+
data = bytes.fromhex("eb 01 c3 c3")
|
|
844
|
+
cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 3, "UNCONDITIONAL_JUMP")], 3: []}),
|
|
821
845
|
controls={"ghidraAgreementSites": [0]})
|
|
822
846
|
cfg["regions"][0]["entries"] = [0, 3]
|
|
823
847
|
check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
|
|
824
848
|
[row] = check["edges"]
|
|
825
|
-
self.assertEqual(row["
|
|
826
|
-
self.
|
|
827
|
-
self.assertEqual(check["counts"]["ghidraEndsFunction"], 0)
|
|
849
|
+
self.assertEqual((row["ghidraFallsThrough"], row["ghidraFallsThroughBasis"]), (False, "flowName"))
|
|
850
|
+
self.assertEqual(check["counts"]["ghidraContinues"], 0)
|
|
828
851
|
self.assertTrue(check["agreed"])
|
|
829
852
|
|
|
853
|
+
def test_a_conditional_flow_name_falls_through_as_ghidra_defines_it(self):
|
|
854
|
+
# je 3; ret; ret. The engine reads on past the conditional jump. Ghidra's CONDITIONAL_TERMINATOR has a
|
|
855
|
+
# fall-through, while its CONDITIONAL_CALL_TERMINATOR does not, although both names contain TERMINATOR.
|
|
856
|
+
data = bytes.fromhex("74 01 c3 c3")
|
|
857
|
+
for flow, falls_through in (("CONDITIONAL_TERMINATOR", True), ("CONDITIONAL_CALL_TERMINATOR", False)):
|
|
858
|
+
with self.subTest(flow=flow):
|
|
859
|
+
cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 3, flow)], 3: []}))
|
|
860
|
+
cfg["regions"][0]["entries"] = [0, 3]
|
|
861
|
+
check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
|
|
862
|
+
[row] = check["edges"]
|
|
863
|
+
self.assertEqual((row["ghidraFallsThrough"], row["ghidraFallsThroughBasis"]), (falls_through, "flowName"))
|
|
864
|
+
self.assertEqual(check["counts"]["ghidraEndsFunction"], 0 if falls_through else 1)
|
|
865
|
+
self.assertEqual(check["agreed"], falls_through)
|
|
866
|
+
|
|
867
|
+
def test_a_ghidra_only_call_carries_where_ghidra_ends_the_function(self):
|
|
868
|
+
# call 4; ret; ret; ret. The engine resolves the call to 4 and reads on to the ret after it. Ghidra resolves it
|
|
869
|
+
# to 5, and ends the function there when it treats that callee as non-returning.
|
|
870
|
+
data = bytes.fromhex("e8 01 00 c3 c3 c3")
|
|
871
|
+
for flow, falls_through in (("UNCONDITIONAL_CALL", True), ("CALL_TERMINATOR", False)):
|
|
872
|
+
with self.subTest(flow=flow):
|
|
873
|
+
cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 5, flow, falls_through)], 4: [], 5: []}))
|
|
874
|
+
cfg["regions"][0]["entries"] = [0, 4, 5]
|
|
875
|
+
check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
|
|
876
|
+
[ours, theirs] = check["edges"]
|
|
877
|
+
self.assertEqual((ours["result"], ours["target"]), ("engineOnly", 4))
|
|
878
|
+
self.assertNotIn("ghidraFallsThrough", ours)
|
|
879
|
+
self.assertEqual((theirs["result"], theirs["target"], theirs["engineEdge"]), ("ghidraOnly", 5, ours["engineEdge"]))
|
|
880
|
+
self.assertEqual((theirs["ghidraFallsThrough"], theirs["ghidraFallsThroughBasis"]), (falls_through, "fallsThrough"))
|
|
881
|
+
self.assertEqual(check["counts"], {"agreement": 0, "engineOnly": 1, "ghidraOnly": 1, "interrupt": 0,
|
|
882
|
+
"ghidraEndsFunction": 0 if falls_through else 1, "ghidraContinues": 0})
|
|
883
|
+
self.assertFalse(check["agreed"])
|
|
884
|
+
|
|
885
|
+
def test_a_ghidra_only_edge_at_an_instruction_the_engine_did_not_read_has_no_falls_through(self):
|
|
886
|
+
# ret; ret. Ghidra claims a call at 1, past the end of the engine's body for the entry at 0, so the engine has no
|
|
887
|
+
# reading there to compare Ghidra's fall-through with.
|
|
888
|
+
data = bytes.fromhex("c3 c3")
|
|
889
|
+
cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(1, None, "COMPUTED_CALL_TERMINATOR", False)]}))
|
|
890
|
+
check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
|
|
891
|
+
[row] = check["edges"]
|
|
892
|
+
self.assertEqual(row["result"], "ghidraOnly")
|
|
893
|
+
self.assertNotIn("ghidraFallsThrough", row)
|
|
894
|
+
self.assertEqual((check["counts"]["ghidraEndsFunction"], check["counts"]["ghidraContinues"]), (0, 0))
|
|
895
|
+
self.assertFalse(check["agreed"])
|
|
896
|
+
|
|
897
|
+
def test_a_ghidra_only_edge_at_a_transfer_outside_the_frame_model_has_no_falls_through(self):
|
|
898
|
+
# o32 call 7; ret; ret. The engine stops at the operand-size call with a gap and decides nothing about
|
|
899
|
+
# fall-through there, so Ghidra's terminator is not compared.
|
|
900
|
+
data = bytes.fromhex("66 e8 01 00 00 00 c3 c3")
|
|
901
|
+
cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 7, "CALL_TERMINATOR", False)], 7: []}))
|
|
902
|
+
cfg["regions"][0]["entries"] = [0, 7]
|
|
903
|
+
check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
|
|
904
|
+
[row] = check["edges"]
|
|
905
|
+
self.assertEqual(row["result"], "ghidraOnly")
|
|
906
|
+
self.assertNotIn("ghidraFallsThrough", row)
|
|
907
|
+
self.assertEqual((check["counts"]["ghidraEndsFunction"], check["counts"]["ghidraContinues"]), (0, 0))
|
|
908
|
+
self.assertFalse(check["agreed"])
|
|
909
|
+
|
|
830
910
|
def test_a_conditional_tail_transfer_ghidra_ends_the_function_at_counts_against_agreed(self):
|
|
831
911
|
# je 3; ret; ret. The engine reads on past the conditional jump into the entry at 3; Ghidra ends the function
|
|
832
912
|
# there when a user cleared the jump's fall-through.
|
|
@@ -869,7 +949,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
|
|
|
869
949
|
cfg["regions"][0]["entries"] = [0, 6]
|
|
870
950
|
r = run_report(data, cfg, "callees")
|
|
871
951
|
check = r["ghidraCrossCheck"]
|
|
872
|
-
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0, "ghidraEndsFunction": 0})
|
|
952
|
+
self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0, "ghidraEndsFunction": 0, "ghidraContinues": 0})
|
|
873
953
|
self.assertEqual(check["notCompared"]["omittedEngineRoutes"], [r["omittedRoutes"][0]["id"]])
|
|
874
954
|
self.assertFalse(check["agreed"])
|
|
875
955
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_continuation_budget.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_effect_order.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_memory_scopes.py
RENAMED
|
File without changes
|
{scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_nested_frame_request.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_relational_controls.py
RENAMED
|
File without changes
|
{scientific_method_engine-6.2.0 → scientific_method_engine-7.0.0}/tests/test_table_continuations.py
RENAMED
|
File without changes
|