scientific-method-engine 3.1.0__tar.gz → 4.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/PKG-INFO +11 -3
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/README.md +10 -2
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/pyproject.toml +1 -1
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/cli.py +3 -1
- scientific_method_engine-4.0.0/src/scientific_method_engine/x86/loops.py +317 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/machine.py +56 -1
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/reports.py +3 -2
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/result_flow.py +18 -3
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/trace.py +112 -32
- scientific_method_engine-4.0.0/tests/test_continuation_budget.py +179 -0
- scientific_method_engine-4.0.0/tests/test_loops.py +267 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_table_continuations.py +4 -2
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/.gitignore +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/LICENSE +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/oracle.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-4.0.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version:
|
|
3
|
+
Version: 4.0.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -125,12 +125,20 @@ pre-call write prefixes and bounded local restoration witnesses. Unknown modeled
|
|
|
125
125
|
or nested service effects stay separate; no return code establishes rollback or
|
|
126
126
|
transactionality. See [the full contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#ordered-effect-path-summaries).
|
|
127
127
|
|
|
128
|
+
Each traced path carries a `loops` record: the restart edges the path took and, between
|
|
129
|
+
consecutive arrivals at a loop head, which registers, flags and written bytes changed, the
|
|
130
|
+
signedness and operands of each gate, and whether the state repeats an earlier arrival. It never
|
|
131
|
+
reports that a loop terminates or that a retry succeeded. `loopIterationLimit` caps the records.
|
|
132
|
+
See [loop progress](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#loop-restart-edges-and-iteration-changes).
|
|
133
|
+
|
|
128
134
|
Evidenced segmented16 `indirectJumps` declarations also expose separate
|
|
129
135
|
`declaredContinuationPaths`. Ordinary paths retain their unresolved transfer;
|
|
130
136
|
conditional routes preserve prefix/child effects with target-choice, live-table
|
|
131
137
|
and selector assumptions. Effects summaries never mark those routes complete.
|
|
132
|
-
Concrete values/field addresses reject inconsistent rows
|
|
133
|
-
path
|
|
138
|
+
Concrete values/field addresses reject inconsistent rows. Continuations run after
|
|
139
|
+
every ordinary path on their own `continuationBudget` (paths, total and per-path steps,
|
|
140
|
+
visits, string iterations), so ordinary paths are the same with or without them. Overlap,
|
|
141
|
+
boundary and continuation limits remain explicit gaps or stops. Partial-table splits are
|
|
134
142
|
partial evidence, never complete dispatch or native-reachability claims. See the
|
|
135
143
|
[table contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#evidenced-indirect-jump-tables).
|
|
136
144
|
|
|
@@ -109,12 +109,20 @@ pre-call write prefixes and bounded local restoration witnesses. Unknown modeled
|
|
|
109
109
|
or nested service effects stay separate; no return code establishes rollback or
|
|
110
110
|
transactionality. See [the full contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#ordered-effect-path-summaries).
|
|
111
111
|
|
|
112
|
+
Each traced path carries a `loops` record: the restart edges the path took and, between
|
|
113
|
+
consecutive arrivals at a loop head, which registers, flags and written bytes changed, the
|
|
114
|
+
signedness and operands of each gate, and whether the state repeats an earlier arrival. It never
|
|
115
|
+
reports that a loop terminates or that a retry succeeded. `loopIterationLimit` caps the records.
|
|
116
|
+
See [loop progress](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#loop-restart-edges-and-iteration-changes).
|
|
117
|
+
|
|
112
118
|
Evidenced segmented16 `indirectJumps` declarations also expose separate
|
|
113
119
|
`declaredContinuationPaths`. Ordinary paths retain their unresolved transfer;
|
|
114
120
|
conditional routes preserve prefix/child effects with target-choice, live-table
|
|
115
121
|
and selector assumptions. Effects summaries never mark those routes complete.
|
|
116
|
-
Concrete values/field addresses reject inconsistent rows
|
|
117
|
-
path
|
|
122
|
+
Concrete values/field addresses reject inconsistent rows. Continuations run after
|
|
123
|
+
every ordinary path on their own `continuationBudget` (paths, total and per-path steps,
|
|
124
|
+
visits, string iterations), so ordinary paths are the same with or without them. Overlap,
|
|
125
|
+
boundary and continuation limits remain explicit gaps or stops. Partial-table splits are
|
|
118
126
|
partial evidence, never complete dispatch or native-reachability claims. See the
|
|
119
127
|
[table contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#evidenced-indirect-jump-tables).
|
|
120
128
|
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "
|
|
8
|
+
version = "4.0.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -19,9 +19,11 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
19
19
|
"callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges.\n"
|
|
20
20
|
"trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
|
|
21
21
|
"a violated control fails the report; an undecided one is reported and never counts as held.\n"
|
|
22
|
-
"Declared table continuations are separate conditional paths
|
|
22
|
+
"Declared table continuations are separate conditional paths that spend continuationBudget\n"
|
|
23
|
+
"(paths, totalSteps, maxSteps, visitLimit, stringIterations); ordinary computed transfers remain stopped.\n"
|
|
23
24
|
"Port accesses and interrupts are hardware-boundary events; portInputs supplies port reads as assumptions.\n"
|
|
24
25
|
"callModels[].preservesMemory keeps explicit, bounded pre-call byte scopes across a modeled call; other memory stays unknown.\n"
|
|
26
|
+
"Each traced path's loops record lists restart edges and what changed between iterations (loopIterationLimit).\n"
|
|
25
27
|
" scientific-method-engine ghidra-scripts")
|
|
26
28
|
|
|
27
29
|
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""Restart edges and what changed between consecutive traced iterations of a loop on one path.
|
|
2
|
+
|
|
3
|
+
A restart edge is a transfer to its own site's address or below that returns to an instruction the
|
|
4
|
+
same call activation already ran. Its target is the loop head. Edges are read off the path with no
|
|
5
|
+
control-flow graph; the address order keeps a forward branch to a join an earlier iteration ran,
|
|
6
|
+
such as an if/else join in the body, from heading a loop. Each arrival at a head is compared with
|
|
7
|
+
the previous arrival at that head: registers, flags, the bytes of modeled memory written in between, and the operands of every
|
|
8
|
+
branch the iteration evaluated in the loop's frame. These are facts about the traced iterations of
|
|
9
|
+
one path. They never establish that a loop terminates, stays bounded or that an iteration made the
|
|
10
|
+
progress its author intended.
|
|
11
|
+
"""
|
|
12
|
+
from collections import namedtuple
|
|
13
|
+
from .machine import MEMORY_CLEARED, REGISTERS
|
|
14
|
+
from .result_flow import predicate_domain
|
|
15
|
+
from .values import Value, extract
|
|
16
|
+
|
|
17
|
+
INTERPRETATION = ("restart edges and changes between consecutive traced iterations of this path only; "
|
|
18
|
+
"no termination, bound, progress or native reachability claim")
|
|
19
|
+
GATE_OPERANDS = ("left", "right", "count", "carry")
|
|
20
|
+
|
|
21
|
+
# One arrival at an instruction: its number among the arrivals of its activation, the event count and
|
|
22
|
+
# write-log length then, and the flag state, which the write log does not record.
|
|
23
|
+
Arrival = namedtuple("Arrival", "arrival event log flags")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def relation(before, after):
|
|
27
|
+
"""Compare two expressions: identical terms, two different known numbers, or undecided."""
|
|
28
|
+
if before == after:
|
|
29
|
+
return "unchanged"
|
|
30
|
+
if before is not None and after is not None and before[0] == "constant" and after[0] == "constant":
|
|
31
|
+
return "changed"
|
|
32
|
+
return "differentExpression"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _value(term):
|
|
36
|
+
if term is None:
|
|
37
|
+
return None
|
|
38
|
+
return {"expression": term, "value": term[1] if term[0] == "constant" else None}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _views(flat, root, term):
|
|
42
|
+
"""The register names compared for one root: the full register in the flat model; in the
|
|
43
|
+
segmented model its 16-bit view, plus the full register keyed by its upper bytes.
|
|
44
|
+
|
|
45
|
+
A 16-bit write leaves the upper half as a join of byte extracts, so the upper half is compared
|
|
46
|
+
byte by byte; comparing it as one 16-bit extract would report an untouched half as changed.
|
|
47
|
+
"""
|
|
48
|
+
if flat or len(root) == 2:
|
|
49
|
+
return [(root, term)]
|
|
50
|
+
value = Value(32, term)
|
|
51
|
+
return [(root[1:], extract(value, 0, 16).term), (root, tuple(extract(value, n, 8).term for n in (16, 24)))]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def compare_registers(flat, before, after):
|
|
55
|
+
"""Sort registers into unchanged and changed between two {root: term} maps."""
|
|
56
|
+
rows = {"unchanged": [], "changed": []}
|
|
57
|
+
for root in REGISTERS:
|
|
58
|
+
old, new = before[root], after[root]
|
|
59
|
+
views = list(zip(_views(flat, root, old), _views(flat, root, new)))
|
|
60
|
+
for index, ((name, a), (_, b)) in enumerate(views):
|
|
61
|
+
if index == 1:
|
|
62
|
+
# The upper half is listed only when it differs; the root then reports its full value.
|
|
63
|
+
if a == b:
|
|
64
|
+
continue
|
|
65
|
+
a, b = old, new
|
|
66
|
+
outcome = relation(a, b)
|
|
67
|
+
if outcome == "unchanged":
|
|
68
|
+
rows["unchanged"].append(name)
|
|
69
|
+
else:
|
|
70
|
+
rows["changed"].append({"register": name, "relation": outcome, "before": _value(a), "after": _value(b)})
|
|
71
|
+
return rows
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def flag_state(state):
|
|
75
|
+
"""The modeled flags as comparable terms.
|
|
76
|
+
|
|
77
|
+
A comparable producer names the flags by its operation and operands. Otherwise the arithmetic
|
|
78
|
+
flags p-code computed are compared by value. Only flags the model forgot are named by their
|
|
79
|
+
epoch, because they are distinct unknowns.
|
|
80
|
+
"""
|
|
81
|
+
if state.flags is not None:
|
|
82
|
+
arithmetic = ("producer", state.flags[2], state.flags[0].term, state.flags[1].term)
|
|
83
|
+
elif state.flag_values is not None:
|
|
84
|
+
arithmetic = ("values",) + tuple(sorted((name, value.term) for name, value in state.flag_values.items()))
|
|
85
|
+
else:
|
|
86
|
+
arithmetic = ("unknown", state.flag_epoch)
|
|
87
|
+
return (arithmetic, None if state.carry is None else state.carry.term,
|
|
88
|
+
state.direction_flag.term, state.interrupt_flag.term)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _current(state, key):
|
|
92
|
+
"""A logged key's term now: a register's, a memory byte's or None. A memory-cleared marker has
|
|
93
|
+
no current term, so a state from before the clear never matches."""
|
|
94
|
+
if key[0] == "register":
|
|
95
|
+
return state.regs[key[1]].term
|
|
96
|
+
value = state.memory.get(key) if len(key) == 3 else None
|
|
97
|
+
return None if value is None else value.term
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _since(state, start):
|
|
101
|
+
"""For each key the write log changed since ``start``: its term at ``start``, first entry first."""
|
|
102
|
+
before = {}
|
|
103
|
+
for key, old in state.write_log.since(start):
|
|
104
|
+
if key not in before:
|
|
105
|
+
before[key] = None if old is None else old.term
|
|
106
|
+
return before
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def memory_changes(before, state):
|
|
110
|
+
"""The memory bytes among ``before``: (term at the earlier arrival, term now). None is a byte the
|
|
111
|
+
model holds no value for."""
|
|
112
|
+
return {key: (term, _current(state, key)) for key, term in before.items() if len(key) == 3}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def registers_at(before, state):
|
|
116
|
+
"""Each register root's term at the earlier arrival, rebuilt from its first later log entry."""
|
|
117
|
+
return {root: before.get(("register", root), state.regs[root].term) for root in REGISTERS}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _intervals(changes):
|
|
121
|
+
rows = []
|
|
122
|
+
for key in sorted(changes, key=lambda k: (repr(k[0]), repr(k[1]), k[2])):
|
|
123
|
+
before, after = changes[key]
|
|
124
|
+
status = ("invalidated" if after is None else "writtenOverUnmodeled" if before is None
|
|
125
|
+
else relation(before, after))
|
|
126
|
+
segment, base, offset = key
|
|
127
|
+
last = rows[-1] if rows else None
|
|
128
|
+
if (last and last["segment"] == segment and last["base"] == base and last["end"] == offset
|
|
129
|
+
and last["status"] == status):
|
|
130
|
+
last["end"] = offset + 1
|
|
131
|
+
else:
|
|
132
|
+
rows.append({"segment": segment, "base": base, "start": offset, "end": offset + 1, "status": status})
|
|
133
|
+
return rows
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def repeated_arrival(state, history, token):
|
|
137
|
+
"""The earliest arrival in ``history`` whose modeled state equals the current one, or None.
|
|
138
|
+
|
|
139
|
+
One backward pass over the write log serves every candidate: walking from the newest entry, a
|
|
140
|
+
key's value at an earlier point is the previous value of its oldest entry after that point, so
|
|
141
|
+
the set of keys that differ from now is kept up to date as the walk passes each candidate. A
|
|
142
|
+
byte the model holds no value for at either point may hold different contents, so it never
|
|
143
|
+
matches.
|
|
144
|
+
"""
|
|
145
|
+
candidates = sorted((t for t in history if t.flags == token.flags),
|
|
146
|
+
key=lambda t: (t.log, t.arrival), reverse=True)
|
|
147
|
+
if not candidates:
|
|
148
|
+
return None
|
|
149
|
+
oldest = candidates[-1].log
|
|
150
|
+
entries = list(state.write_log.since(oldest))
|
|
151
|
+
position = len(entries)
|
|
152
|
+
differing = set()
|
|
153
|
+
earliest = None
|
|
154
|
+
for candidate in candidates:
|
|
155
|
+
while oldest + position > candidate.log:
|
|
156
|
+
position -= 1
|
|
157
|
+
key, old = entries[position]
|
|
158
|
+
now = _current(state, key)
|
|
159
|
+
if old is None or now is None or old.term != now:
|
|
160
|
+
differing.add(key)
|
|
161
|
+
else:
|
|
162
|
+
differing.discard(key)
|
|
163
|
+
if not differing:
|
|
164
|
+
earliest = candidate.arrival if earliest is None else min(earliest, candidate.arrival)
|
|
165
|
+
return earliest
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _gate(event):
|
|
169
|
+
gate = {"site": event["site"], "order": event["order"], "predicate": event["predicate"],
|
|
170
|
+
"predicateDomain": predicate_domain(event["predicate"]), "taken": event["taken"],
|
|
171
|
+
"operands": {k: _value(event[k]["expression"]) for k in GATE_OPERANDS if isinstance(event.get(k), dict)}}
|
|
172
|
+
# LOOPE and LOOPNE also test ZF; their flag test's record is kept beside the counter.
|
|
173
|
+
for k in ("operation", "flagProducer", "decidedBy", "reason", "zeroFlag"):
|
|
174
|
+
if k in event:
|
|
175
|
+
gate[k] = event[k]
|
|
176
|
+
return gate
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _compare_gates(previous, gates):
|
|
180
|
+
"""Mark each gate against the gate at the same position of the previous iteration."""
|
|
181
|
+
if previous is None or not (previous or gates):
|
|
182
|
+
# A first iteration, or two iterations with no gate in the loop's frame: nothing to compare.
|
|
183
|
+
return None
|
|
184
|
+
same_shape = [g["site"] for g in previous] == [g["site"] for g in gates]
|
|
185
|
+
outcomes = []
|
|
186
|
+
for index, gate in enumerate(gates):
|
|
187
|
+
if not same_shape:
|
|
188
|
+
gate["operandsSincePreviousIteration"] = "gate sequence differs"
|
|
189
|
+
continue
|
|
190
|
+
before, after = previous[index]["operands"], gate["operands"]
|
|
191
|
+
if not after or set(before) != set(after):
|
|
192
|
+
gate["operandsSincePreviousIteration"] = "unresolved"
|
|
193
|
+
outcomes.append(None)
|
|
194
|
+
continue
|
|
195
|
+
relations = {k: relation(before[k]["expression"], after[k]["expression"]) for k in after}
|
|
196
|
+
result = ("changed" if "changed" in relations.values() else
|
|
197
|
+
"differentExpression" if "differentExpression" in relations.values() else "unchanged")
|
|
198
|
+
gate["operandsSincePreviousIteration"] = result
|
|
199
|
+
outcomes.append(result)
|
|
200
|
+
if not same_shape or [g["taken"] for g in previous] != [g["taken"] for g in gates]:
|
|
201
|
+
return False
|
|
202
|
+
if "changed" in outcomes:
|
|
203
|
+
return False
|
|
204
|
+
if None in outcomes or "differentExpression" in outcomes:
|
|
205
|
+
return None
|
|
206
|
+
return True
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class LoopTracker:
|
|
210
|
+
"""Per-path loop bookkeeping. ``arrive`` runs once before each traced instruction.
|
|
211
|
+
|
|
212
|
+
Each call activation is identified by a number stored in its ``state.frames`` entry the first
|
|
213
|
+
time the tracker sees it, so a call inside a loop body never mistakes the callee's frame for the
|
|
214
|
+
caller's.
|
|
215
|
+
"""
|
|
216
|
+
|
|
217
|
+
def __init__(self, limit):
|
|
218
|
+
self.limit = limit
|
|
219
|
+
self.frames = {}
|
|
220
|
+
self.activations = 0
|
|
221
|
+
self.previous = None
|
|
222
|
+
self.edges = {}
|
|
223
|
+
self.iterations = []
|
|
224
|
+
self.omitted = 0
|
|
225
|
+
|
|
226
|
+
def __deepcopy__(self, memo):
|
|
227
|
+
# Arrivals, gate lists and recorded iterations are never changed once stored, so a fork
|
|
228
|
+
# copies only the containers that keep growing.
|
|
229
|
+
copy = LoopTracker.__new__(LoopTracker)
|
|
230
|
+
copy.limit, copy.activations, copy.previous, copy.omitted = self.limit, self.activations, self.previous, self.omitted
|
|
231
|
+
copy.frames = {activation: {"entry": frame["entry"], "tokens": dict(frame["tokens"]),
|
|
232
|
+
"heads": {head: {"history": None if loop["history"] is None else list(loop["history"]),
|
|
233
|
+
"gates": loop["gates"]}
|
|
234
|
+
for head, loop in frame["heads"].items()}}
|
|
235
|
+
for activation, frame in self.frames.items()}
|
|
236
|
+
copy.edges = {key: dict(edge) for key, edge in self.edges.items()}
|
|
237
|
+
copy.iterations = list(self.iterations)
|
|
238
|
+
return copy
|
|
239
|
+
|
|
240
|
+
def _frame(self, state):
|
|
241
|
+
top = state.frames[-1]
|
|
242
|
+
activation = top.get("loopActivation")
|
|
243
|
+
if activation is None:
|
|
244
|
+
self.activations += 1
|
|
245
|
+
activation = top["loopActivation"] = self.activations
|
|
246
|
+
# A new activation: drop the records of activations that returned, since ids never recur.
|
|
247
|
+
live = {f.get("loopActivation") for f in state.frames}
|
|
248
|
+
for gone in [a for a in self.frames if a not in live]:
|
|
249
|
+
del self.frames[gone]
|
|
250
|
+
self.frames[activation] = {"entry": top["entry"], "tokens": {}, "heads": {}}
|
|
251
|
+
return activation, self.frames[activation]
|
|
252
|
+
|
|
253
|
+
def arrive(self, state, at, ins):
|
|
254
|
+
activation, frame = self._frame(state)
|
|
255
|
+
previous, self.previous = self.previous, (at, at + ins.size, activation, ins.mnemonic)
|
|
256
|
+
last = frame["tokens"].get(at)
|
|
257
|
+
token = Arrival(1 if last is None else last.arrival + 1, len(state.events), len(state.write_log),
|
|
258
|
+
flag_state(state))
|
|
259
|
+
# Only a transfer to the site's own address or below closes a loop. Every cycle on a path has
|
|
260
|
+
# one, since fall-throughs run forward, and a forward branch to a join an earlier iteration
|
|
261
|
+
# ran stays an ordinary arrival.
|
|
262
|
+
if (last is not None and previous is not None and previous[2] == activation and at != previous[1]
|
|
263
|
+
and at <= previous[0]):
|
|
264
|
+
self._restart(state, frame, previous, at, last, token)
|
|
265
|
+
else:
|
|
266
|
+
loop = frame["heads"].get(at)
|
|
267
|
+
if loop is not None and loop["history"] is not None:
|
|
268
|
+
# An arrival at a known head that is not a restart, such as an inner loop's entry on
|
|
269
|
+
# the next outer iteration, is an earlier state later iterations may repeat. It starts a
|
|
270
|
+
# new entry into the loop, so the next iteration's gates are not compared with those
|
|
271
|
+
# of the last iteration before the loop was left.
|
|
272
|
+
loop["history"].append(token)
|
|
273
|
+
loop["gates"] = None
|
|
274
|
+
frame["tokens"][at] = token
|
|
275
|
+
|
|
276
|
+
def _restart(self, state, frame, previous, head, start, token):
|
|
277
|
+
site, _, activation, kind = previous
|
|
278
|
+
depth = len(state.frames) - 1
|
|
279
|
+
key = (activation, site, head)
|
|
280
|
+
edge = self.edges.get(key)
|
|
281
|
+
if edge is None:
|
|
282
|
+
edge = self.edges[key] = {"entry": frame["entry"], "depth": depth, "activation": activation,
|
|
283
|
+
"site": site, "target": head, "kind": kind, "traversals": 0,
|
|
284
|
+
"firstOrder": len(state.events)}
|
|
285
|
+
edge["traversals"] += 1
|
|
286
|
+
loop = frame["heads"].setdefault(head, {"history": [start], "gates": None})
|
|
287
|
+
if loop["history"] is None:
|
|
288
|
+
self.omitted += 1
|
|
289
|
+
return
|
|
290
|
+
if len(self.iterations) >= self.limit:
|
|
291
|
+
# Past the limit the history is dropped, so no later iteration is compared with a stale one.
|
|
292
|
+
loop["history"] = None
|
|
293
|
+
self.omitted += 1
|
|
294
|
+
return
|
|
295
|
+
# The iteration runs from the previous arrival at the head, however the path got there.
|
|
296
|
+
gates = [_gate(e) for e in state.events[start.event:] if e["kind"] == "branch" and e["depth"] == depth]
|
|
297
|
+
repeated = _compare_gates(loop["gates"], gates)
|
|
298
|
+
loop["gates"] = gates
|
|
299
|
+
before = _since(state, start.log)
|
|
300
|
+
now = {root: state.regs[root].term for root in REGISTERS}
|
|
301
|
+
self.iterations.append({
|
|
302
|
+
"head": head, "entry": frame["entry"], "depth": depth, "activation": activation,
|
|
303
|
+
"fromArrival": start.arrival, "toArrival": token.arrival,
|
|
304
|
+
"restartEdge": {"site": site, "kind": kind},
|
|
305
|
+
"fromOrder": start.event, "toOrder": token.event,
|
|
306
|
+
"registers": compare_registers(state.flat, registers_at(before, state), now),
|
|
307
|
+
"flags": "unchanged" if start.flags == token.flags else "differ",
|
|
308
|
+
"memory": _intervals(memory_changes(before, state)), "memoryForgotten": MEMORY_CLEARED in before,
|
|
309
|
+
"gates": gates,
|
|
310
|
+
"gateOperandsRepeated": repeated, "stateRepeatsArrival": repeated_arrival(state, loop["history"], token)})
|
|
311
|
+
loop["history"].append(token)
|
|
312
|
+
|
|
313
|
+
def report(self):
|
|
314
|
+
"""The ``loops`` field of a traced path."""
|
|
315
|
+
return {"restartEdges": list(self.edges.values()), "iterations": self.iterations,
|
|
316
|
+
"iterationLimit": self.limit, "iterationsOmitted": self.omitted,
|
|
317
|
+
"allIterationsRecorded": not self.omitted, "interpretation": INTERPRETATION}
|
|
@@ -18,6 +18,53 @@ class StopPath(Exception):
|
|
|
18
18
|
pass
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
# The write-log key that marks a point where the model forgot all memory.
|
|
22
|
+
MEMORY_CLEARED = ("memory-cleared",)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class WriteLog:
|
|
26
|
+
"""Append-only list of (key, previous value) pairs, one per register or memory byte a write changes.
|
|
27
|
+
|
|
28
|
+
A memory key is (segment, base, offset); a register key is ("register", root). A previous memory
|
|
29
|
+
value is None when the model held no value for the byte. ``(MEMORY_CLEARED, None)`` marks a
|
|
30
|
+
point where the model forgot all memory. Copies made at a fork share the entries
|
|
31
|
+
recorded before the fork, which are immutable, so forking a path never copies its log.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self):
|
|
35
|
+
self.chunks = [] # (absolute index of the first entry, tuple of entries), oldest first
|
|
36
|
+
self.tail = []
|
|
37
|
+
self.length = 0
|
|
38
|
+
|
|
39
|
+
def __len__(self):
|
|
40
|
+
return self.length
|
|
41
|
+
|
|
42
|
+
def append(self, entry):
|
|
43
|
+
self.tail.append(entry)
|
|
44
|
+
self.length += 1
|
|
45
|
+
|
|
46
|
+
def extend(self, entries):
|
|
47
|
+
for entry in entries:
|
|
48
|
+
self.append(entry)
|
|
49
|
+
|
|
50
|
+
def since(self, start):
|
|
51
|
+
"""The entries at absolute index ``start`` and later, oldest first."""
|
|
52
|
+
for first, chunk in self.chunks:
|
|
53
|
+
if first + len(chunk) > start:
|
|
54
|
+
yield from chunk[max(0, start - first):]
|
|
55
|
+
first = self.length - len(self.tail)
|
|
56
|
+
yield from self.tail[max(0, start - first):]
|
|
57
|
+
|
|
58
|
+
def __deepcopy__(self, memo):
|
|
59
|
+
if self.tail:
|
|
60
|
+
self.chunks.append((self.length - len(self.tail), tuple(self.tail)))
|
|
61
|
+
self.tail = []
|
|
62
|
+
copy = WriteLog()
|
|
63
|
+
copy.chunks = list(self.chunks)
|
|
64
|
+
copy.length = self.length
|
|
65
|
+
return copy
|
|
66
|
+
|
|
67
|
+
|
|
21
68
|
class Shared(dict):
|
|
22
69
|
"""Read-only query configuration that every copy of a path shares instead of copying."""
|
|
23
70
|
|
|
@@ -51,6 +98,9 @@ class State:
|
|
|
51
98
|
self.semantics = BACKEND
|
|
52
99
|
self.sp, self.bp = ("esp", "ebp") if self.flat else ("sp", "bp")
|
|
53
100
|
self.at = entry
|
|
101
|
+
# Every register and memory byte a write changes, with its previous value, in order, so a
|
|
102
|
+
# reader can recover the state as it stood at an earlier point of the path (x86/loops.py).
|
|
103
|
+
self.write_log = WriteLog()
|
|
54
104
|
self.regs = {r: unknown("initial:" + r, ALIASES[r][2]) for r in REGISTERS}
|
|
55
105
|
# Producers per register byte, so a partial write replaces only the bytes it stores.
|
|
56
106
|
self.reg_sources = {r: [()] * (ALIASES[r][2] // 8) for r in REGISTERS}
|
|
@@ -169,6 +219,9 @@ class State:
|
|
|
169
219
|
self.flag_values = None
|
|
170
220
|
|
|
171
221
|
def clear_memory(self):
|
|
222
|
+
# Bytes the model never held may change too, so the log marks the clear itself.
|
|
223
|
+
self.write_log.extend(self.memory.items())
|
|
224
|
+
self.write_log.append((MEMORY_CLEARED, None))
|
|
172
225
|
self.memory.clear()
|
|
173
226
|
self.memory_groups.clear()
|
|
174
227
|
self.unread_memory.clear()
|
|
@@ -227,6 +280,7 @@ class State:
|
|
|
227
280
|
root, low, bits = alias(name)
|
|
228
281
|
value = resize(value, bits)
|
|
229
282
|
value = Value(bits, value.term, sources(value, site=site))
|
|
283
|
+
self.write_log.append((("register", root), self.regs[root]))
|
|
230
284
|
if bits == ALIASES[root][2]:
|
|
231
285
|
self.regs[root] = value
|
|
232
286
|
else:
|
|
@@ -279,7 +333,7 @@ class State:
|
|
|
279
333
|
for key in list(members):
|
|
280
334
|
if may_alias(key, written, self.bits, self.flat):
|
|
281
335
|
uncertain.append(key)
|
|
282
|
-
self.memory.pop(key, None)
|
|
336
|
+
self.write_log.append((key, self.memory.pop(key, None)))
|
|
283
337
|
self.unread_memory.pop(key, None)
|
|
284
338
|
self.memory_writers.pop(key, None)
|
|
285
339
|
self.lost_memory[key] = {"cause": "dropped by a possibly aliasing write", "order": len(self.events)}
|
|
@@ -287,6 +341,7 @@ class State:
|
|
|
287
341
|
if not members:
|
|
288
342
|
del self.memory_groups[group]
|
|
289
343
|
for i, key in enumerate(keys):
|
|
344
|
+
self.write_log.append((key, self.memory.get(key)))
|
|
290
345
|
self.memory[key] = extract(write, i * 8, 8)
|
|
291
346
|
self.unread_memory.pop(key, None)
|
|
292
347
|
self.memory_writers[key] = len(self.events)
|
|
@@ -231,7 +231,7 @@ def uses(image, config):
|
|
|
231
231
|
stops.setdefault(root, "entry not traced: entry or total instruction budget exhausted")
|
|
232
232
|
break
|
|
233
233
|
report = trace(image, {**config, "entry": at, "totalSteps": remaining, "stringIterations": string_remaining},
|
|
234
|
-
continue_declared_jumps=False)
|
|
234
|
+
continue_declared_jumps=False, track_loops=False)
|
|
235
235
|
remaining -= report["stepsUsed"]
|
|
236
236
|
string_remaining -= report["stringIterationsUsed"]
|
|
237
237
|
if not report["completeWithinModel"]:
|
|
@@ -501,7 +501,8 @@ def dispatch(image, config):
|
|
|
501
501
|
raise ValueError("Dispatch table mapping differs from PE source sections")
|
|
502
502
|
for value in values:
|
|
503
503
|
integer(value, 0, (1 << ALIASES[input_reg][2]) - 1, "input value")
|
|
504
|
-
report = trace(image, {**config, "registers": {**config.get("registers", {}), input_reg: value}}, continue_declared_jumps=False
|
|
504
|
+
report = trace(image, {**config, "registers": {**config.get("registers", {}), input_reg: value}}, continue_declared_jumps=False,
|
|
505
|
+
track_loops=False)
|
|
505
506
|
outcomes = []
|
|
506
507
|
for path in report["paths"]:
|
|
507
508
|
reached = bool(path["instructionPath"]) and path["instructionPath"][-1] == site
|
|
@@ -58,6 +58,17 @@ def result_contracts(state, contracts, entry):
|
|
|
58
58
|
DOMAINS = {"l": "signed", "le": "signed", "s": "signed", "o": "signed", "c": "unsigned", "be": "unsigned"}
|
|
59
59
|
|
|
60
60
|
|
|
61
|
+
def predicate_domain(predicate):
|
|
62
|
+
"""How a branch reads what it tests: ``signed``, ``unsigned``, ``counter`` or ``flags/equality``.
|
|
63
|
+
|
|
64
|
+
LOOP, LOOPE, LOOPNE, JCXZ and JECXZ test CX or ECX against zero (``counter``); the conditional
|
|
65
|
+
jumps read the flags of their producer.
|
|
66
|
+
"""
|
|
67
|
+
if predicate.startswith("loop") or predicate in ("jcxz", "jecxz"):
|
|
68
|
+
return "counter"
|
|
69
|
+
return DOMAINS.get(BRANCH_CONDITIONS.get(predicate, (None,))[0], "flags/equality")
|
|
70
|
+
|
|
71
|
+
|
|
61
72
|
VALUE_FIELDS = ("sourceValue", "resultValue", "destinationContainerValue", "result", "value", "left", "right", "carry", "count")
|
|
62
73
|
CONSUMER_KINDS = ("value-transfer", "conversion", "read", "write", "compare", "branch", "return")
|
|
63
74
|
ROW_FIELDS = ("kind", "site", "entry", "depth", "order", "operation", "source", "destination", "sourceBits", "destinationBits",
|
|
@@ -77,6 +88,8 @@ def return_flows(report, config):
|
|
|
77
88
|
consumer_limit = integer(config.get("returnConsumerLimit", 256), 1, 10000, "return consumer limit")
|
|
78
89
|
analysis_limit = integer(config.get("returnFlowAnalysisLimit", 1000000), 1, 10000000, "return flow analysis limit")
|
|
79
90
|
steps, capped = 0, False
|
|
91
|
+
# Gaps raised while declared-table continuations ran say nothing about an ordinary path.
|
|
92
|
+
ordinary_gaps = [g for g in report["gaps"] if g.get("route") != "declaredContinuation"]
|
|
80
93
|
for path in report["paths"]:
|
|
81
94
|
events = path["events"]
|
|
82
95
|
event_values = {}
|
|
@@ -114,8 +127,10 @@ def return_flows(report, config):
|
|
|
114
127
|
row.update(values=values, dependentValueFields=list(dependent), returnWidthRelationships=widths,
|
|
115
128
|
relationship="producer dependency only; not value or storage identity")
|
|
116
129
|
if later["kind"] == "branch":
|
|
117
|
-
|
|
118
|
-
|
|
130
|
+
domain = predicate_domain(later.get("predicate", ""))
|
|
131
|
+
# returnFlows has reported LOOP and JCXZ as flags/equality since it shipped, and
|
|
132
|
+
# consumers read that value, so it stays until a major release changes it.
|
|
133
|
+
row["predicateDomain"] = "flags/equality" if domain == "counter" else domain
|
|
119
134
|
consumers.append(row)
|
|
120
135
|
flows.append({"originOrder": event["order"], "originSite": event["site"], "calleeEntry": contract["entry"],
|
|
121
136
|
"callSite": event.get("callSite"), "callerEntry": event.get("callerEntry"),
|
|
@@ -125,7 +140,7 @@ def return_flows(report, config):
|
|
|
125
140
|
path["returnFlows"] = {"results": flows, "resultsOmitted": omitted, "resultLimit": limit,
|
|
126
141
|
"consumerLimit": consumer_limit,
|
|
127
142
|
"complete": not omitted and all(not f["consumersOmitted"] and f["consumerScanComplete"] for f in flows)
|
|
128
|
-
and path["returned"] and not
|
|
143
|
+
and path["returned"] and not ordinary_gaps,
|
|
129
144
|
"interpretation": "conditional static dependency paths; encodings are declared evidence, not live occurrence; a branch never establishes initialization, accepted contents or extent"}
|
|
130
145
|
report["returnFlowAnalysis"] = {"limit": analysis_limit, "steps": steps, "capped": capped}
|
|
131
146
|
return report
|