scientific-method-engine 3.1.0__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/PKG-INFO +7 -1
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/README.md +6 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/pyproject.toml +1 -1
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/cli.py +1 -0
- scientific_method_engine-3.2.0/src/scientific_method_engine/x86/loops.py +317 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/machine.py +56 -1
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/reports.py +3 -2
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/result_flow.py +15 -2
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/trace.py +13 -3
- scientific_method_engine-3.2.0/tests/test_loops.py +267 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/.gitignore +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/LICENSE +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/oracle.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -125,6 +125,12 @@ pre-call write prefixes and bounded local restoration witnesses. Unknown modeled
|
|
|
125
125
|
or nested service effects stay separate; no return code establishes rollback or
|
|
126
126
|
transactionality. See [the full contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#ordered-effect-path-summaries).
|
|
127
127
|
|
|
128
|
+
Each traced path carries a `loops` record: the restart edges the path took and, between
|
|
129
|
+
consecutive arrivals at a loop head, which registers, flags and written bytes changed, the
|
|
130
|
+
signedness and operands of each gate, and whether the state repeats an earlier arrival. It never
|
|
131
|
+
reports that a loop terminates or that a retry succeeded. `loopIterationLimit` caps the records.
|
|
132
|
+
See [loop progress](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#loop-restart-edges-and-iteration-changes).
|
|
133
|
+
|
|
128
134
|
Evidenced segmented16 `indirectJumps` declarations also expose separate
|
|
129
135
|
`declaredContinuationPaths`. Ordinary paths retain their unresolved transfer;
|
|
130
136
|
conditional routes preserve prefix/child effects with target-choice, live-table
|
|
@@ -109,6 +109,12 @@ pre-call write prefixes and bounded local restoration witnesses. Unknown modeled
|
|
|
109
109
|
or nested service effects stay separate; no return code establishes rollback or
|
|
110
110
|
transactionality. See [the full contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#ordered-effect-path-summaries).
|
|
111
111
|
|
|
112
|
+
Each traced path carries a `loops` record: the restart edges the path took and, between
|
|
113
|
+
consecutive arrivals at a loop head, which registers, flags and written bytes changed, the
|
|
114
|
+
signedness and operands of each gate, and whether the state repeats an earlier arrival. It never
|
|
115
|
+
reports that a loop terminates or that a retry succeeded. `loopIterationLimit` caps the records.
|
|
116
|
+
See [loop progress](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#loop-restart-edges-and-iteration-changes).
|
|
117
|
+
|
|
112
118
|
Evidenced segmented16 `indirectJumps` declarations also expose separate
|
|
113
119
|
`declaredContinuationPaths`. Ordinary paths retain their unresolved transfer;
|
|
114
120
|
conditional routes preserve prefix/child effects with target-choice, live-table
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "3.
|
|
8
|
+
version = "3.2.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -22,6 +22,7 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
22
22
|
"Declared table continuations are separate conditional paths; ordinary computed transfers remain stopped.\n"
|
|
23
23
|
"Port accesses and interrupts are hardware-boundary events; portInputs supplies port reads as assumptions.\n"
|
|
24
24
|
"callModels[].preservesMemory keeps explicit, bounded pre-call byte scopes across a modeled call; other memory stays unknown.\n"
|
|
25
|
+
"Each traced path's loops record lists restart edges and what changed between iterations (loopIterationLimit).\n"
|
|
25
26
|
" scientific-method-engine ghidra-scripts")
|
|
26
27
|
|
|
27
28
|
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""Restart edges and what changed between consecutive traced iterations of a loop on one path.
|
|
2
|
+
|
|
3
|
+
A restart edge is a transfer to its own site's address or below that returns to an instruction the
|
|
4
|
+
same call activation already ran. Its target is the loop head. Edges are read off the path with no
|
|
5
|
+
control-flow graph; the address order keeps a forward branch to a join an earlier iteration ran,
|
|
6
|
+
such as an if/else join in the body, from heading a loop. Each arrival at a head is compared with
|
|
7
|
+
the previous arrival at that head: registers, flags, the bytes of modeled memory written in between, and the operands of every
|
|
8
|
+
branch the iteration evaluated in the loop's frame. These are facts about the traced iterations of
|
|
9
|
+
one path. They never establish that a loop terminates, stays bounded or that an iteration made the
|
|
10
|
+
progress its author intended.
|
|
11
|
+
"""
|
|
12
|
+
from collections import namedtuple
|
|
13
|
+
from .machine import MEMORY_CLEARED, REGISTERS
|
|
14
|
+
from .result_flow import predicate_domain
|
|
15
|
+
from .values import Value, extract
|
|
16
|
+
|
|
17
|
+
INTERPRETATION = ("restart edges and changes between consecutive traced iterations of this path only; "
|
|
18
|
+
"no termination, bound, progress or native reachability claim")
|
|
19
|
+
GATE_OPERANDS = ("left", "right", "count", "carry")
|
|
20
|
+
|
|
21
|
+
# One arrival at an instruction: its number among the arrivals of its activation, the event count and
|
|
22
|
+
# write-log length then, and the flag state, which the write log does not record.
|
|
23
|
+
Arrival = namedtuple("Arrival", "arrival event log flags")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def relation(before, after):
|
|
27
|
+
"""Compare two expressions: identical terms, two different known numbers, or undecided."""
|
|
28
|
+
if before == after:
|
|
29
|
+
return "unchanged"
|
|
30
|
+
if before is not None and after is not None and before[0] == "constant" and after[0] == "constant":
|
|
31
|
+
return "changed"
|
|
32
|
+
return "differentExpression"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _value(term):
|
|
36
|
+
if term is None:
|
|
37
|
+
return None
|
|
38
|
+
return {"expression": term, "value": term[1] if term[0] == "constant" else None}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _views(flat, root, term):
|
|
42
|
+
"""The register names compared for one root: the full register in the flat model; in the
|
|
43
|
+
segmented model its 16-bit view, plus the full register keyed by its upper bytes.
|
|
44
|
+
|
|
45
|
+
A 16-bit write leaves the upper half as a join of byte extracts, so the upper half is compared
|
|
46
|
+
byte by byte; comparing it as one 16-bit extract would report an untouched half as changed.
|
|
47
|
+
"""
|
|
48
|
+
if flat or len(root) == 2:
|
|
49
|
+
return [(root, term)]
|
|
50
|
+
value = Value(32, term)
|
|
51
|
+
return [(root[1:], extract(value, 0, 16).term), (root, tuple(extract(value, n, 8).term for n in (16, 24)))]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def compare_registers(flat, before, after):
|
|
55
|
+
"""Sort registers into unchanged and changed between two {root: term} maps."""
|
|
56
|
+
rows = {"unchanged": [], "changed": []}
|
|
57
|
+
for root in REGISTERS:
|
|
58
|
+
old, new = before[root], after[root]
|
|
59
|
+
views = list(zip(_views(flat, root, old), _views(flat, root, new)))
|
|
60
|
+
for index, ((name, a), (_, b)) in enumerate(views):
|
|
61
|
+
if index == 1:
|
|
62
|
+
# The upper half is listed only when it differs; the root then reports its full value.
|
|
63
|
+
if a == b:
|
|
64
|
+
continue
|
|
65
|
+
a, b = old, new
|
|
66
|
+
outcome = relation(a, b)
|
|
67
|
+
if outcome == "unchanged":
|
|
68
|
+
rows["unchanged"].append(name)
|
|
69
|
+
else:
|
|
70
|
+
rows["changed"].append({"register": name, "relation": outcome, "before": _value(a), "after": _value(b)})
|
|
71
|
+
return rows
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def flag_state(state):
|
|
75
|
+
"""The modeled flags as comparable terms.
|
|
76
|
+
|
|
77
|
+
A comparable producer names the flags by its operation and operands. Otherwise the arithmetic
|
|
78
|
+
flags p-code computed are compared by value. Only flags the model forgot are named by their
|
|
79
|
+
epoch, because they are distinct unknowns.
|
|
80
|
+
"""
|
|
81
|
+
if state.flags is not None:
|
|
82
|
+
arithmetic = ("producer", state.flags[2], state.flags[0].term, state.flags[1].term)
|
|
83
|
+
elif state.flag_values is not None:
|
|
84
|
+
arithmetic = ("values",) + tuple(sorted((name, value.term) for name, value in state.flag_values.items()))
|
|
85
|
+
else:
|
|
86
|
+
arithmetic = ("unknown", state.flag_epoch)
|
|
87
|
+
return (arithmetic, None if state.carry is None else state.carry.term,
|
|
88
|
+
state.direction_flag.term, state.interrupt_flag.term)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _current(state, key):
|
|
92
|
+
"""A logged key's term now: a register's, a memory byte's or None. A memory-cleared marker has
|
|
93
|
+
no current term, so a state from before the clear never matches."""
|
|
94
|
+
if key[0] == "register":
|
|
95
|
+
return state.regs[key[1]].term
|
|
96
|
+
value = state.memory.get(key) if len(key) == 3 else None
|
|
97
|
+
return None if value is None else value.term
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _since(state, start):
|
|
101
|
+
"""For each key the write log changed since ``start``: its term at ``start``, first entry first."""
|
|
102
|
+
before = {}
|
|
103
|
+
for key, old in state.write_log.since(start):
|
|
104
|
+
if key not in before:
|
|
105
|
+
before[key] = None if old is None else old.term
|
|
106
|
+
return before
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def memory_changes(before, state):
|
|
110
|
+
"""The memory bytes among ``before``: (term at the earlier arrival, term now). None is a byte the
|
|
111
|
+
model holds no value for."""
|
|
112
|
+
return {key: (term, _current(state, key)) for key, term in before.items() if len(key) == 3}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def registers_at(before, state):
|
|
116
|
+
"""Each register root's term at the earlier arrival, rebuilt from its first later log entry."""
|
|
117
|
+
return {root: before.get(("register", root), state.regs[root].term) for root in REGISTERS}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _intervals(changes):
|
|
121
|
+
rows = []
|
|
122
|
+
for key in sorted(changes, key=lambda k: (repr(k[0]), repr(k[1]), k[2])):
|
|
123
|
+
before, after = changes[key]
|
|
124
|
+
status = ("invalidated" if after is None else "writtenOverUnmodeled" if before is None
|
|
125
|
+
else relation(before, after))
|
|
126
|
+
segment, base, offset = key
|
|
127
|
+
last = rows[-1] if rows else None
|
|
128
|
+
if (last and last["segment"] == segment and last["base"] == base and last["end"] == offset
|
|
129
|
+
and last["status"] == status):
|
|
130
|
+
last["end"] = offset + 1
|
|
131
|
+
else:
|
|
132
|
+
rows.append({"segment": segment, "base": base, "start": offset, "end": offset + 1, "status": status})
|
|
133
|
+
return rows
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def repeated_arrival(state, history, token):
|
|
137
|
+
"""The earliest arrival in ``history`` whose modeled state equals the current one, or None.
|
|
138
|
+
|
|
139
|
+
One backward pass over the write log serves every candidate: walking from the newest entry, a
|
|
140
|
+
key's value at an earlier point is the previous value of its oldest entry after that point, so
|
|
141
|
+
the set of keys that differ from now is kept up to date as the walk passes each candidate. A
|
|
142
|
+
byte the model holds no value for at either point may hold different contents, so it never
|
|
143
|
+
matches.
|
|
144
|
+
"""
|
|
145
|
+
candidates = sorted((t for t in history if t.flags == token.flags),
|
|
146
|
+
key=lambda t: (t.log, t.arrival), reverse=True)
|
|
147
|
+
if not candidates:
|
|
148
|
+
return None
|
|
149
|
+
oldest = candidates[-1].log
|
|
150
|
+
entries = list(state.write_log.since(oldest))
|
|
151
|
+
position = len(entries)
|
|
152
|
+
differing = set()
|
|
153
|
+
earliest = None
|
|
154
|
+
for candidate in candidates:
|
|
155
|
+
while oldest + position > candidate.log:
|
|
156
|
+
position -= 1
|
|
157
|
+
key, old = entries[position]
|
|
158
|
+
now = _current(state, key)
|
|
159
|
+
if old is None or now is None or old.term != now:
|
|
160
|
+
differing.add(key)
|
|
161
|
+
else:
|
|
162
|
+
differing.discard(key)
|
|
163
|
+
if not differing:
|
|
164
|
+
earliest = candidate.arrival if earliest is None else min(earliest, candidate.arrival)
|
|
165
|
+
return earliest
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _gate(event):
|
|
169
|
+
gate = {"site": event["site"], "order": event["order"], "predicate": event["predicate"],
|
|
170
|
+
"predicateDomain": predicate_domain(event["predicate"]), "taken": event["taken"],
|
|
171
|
+
"operands": {k: _value(event[k]["expression"]) for k in GATE_OPERANDS if isinstance(event.get(k), dict)}}
|
|
172
|
+
# LOOPE and LOOPNE also test ZF; their flag test's record is kept beside the counter.
|
|
173
|
+
for k in ("operation", "flagProducer", "decidedBy", "reason", "zeroFlag"):
|
|
174
|
+
if k in event:
|
|
175
|
+
gate[k] = event[k]
|
|
176
|
+
return gate
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _compare_gates(previous, gates):
|
|
180
|
+
"""Mark each gate against the gate at the same position of the previous iteration."""
|
|
181
|
+
if previous is None or not (previous or gates):
|
|
182
|
+
# A first iteration, or two iterations with no gate in the loop's frame: nothing to compare.
|
|
183
|
+
return None
|
|
184
|
+
same_shape = [g["site"] for g in previous] == [g["site"] for g in gates]
|
|
185
|
+
outcomes = []
|
|
186
|
+
for index, gate in enumerate(gates):
|
|
187
|
+
if not same_shape:
|
|
188
|
+
gate["operandsSincePreviousIteration"] = "gate sequence differs"
|
|
189
|
+
continue
|
|
190
|
+
before, after = previous[index]["operands"], gate["operands"]
|
|
191
|
+
if not after or set(before) != set(after):
|
|
192
|
+
gate["operandsSincePreviousIteration"] = "unresolved"
|
|
193
|
+
outcomes.append(None)
|
|
194
|
+
continue
|
|
195
|
+
relations = {k: relation(before[k]["expression"], after[k]["expression"]) for k in after}
|
|
196
|
+
result = ("changed" if "changed" in relations.values() else
|
|
197
|
+
"differentExpression" if "differentExpression" in relations.values() else "unchanged")
|
|
198
|
+
gate["operandsSincePreviousIteration"] = result
|
|
199
|
+
outcomes.append(result)
|
|
200
|
+
if not same_shape or [g["taken"] for g in previous] != [g["taken"] for g in gates]:
|
|
201
|
+
return False
|
|
202
|
+
if "changed" in outcomes:
|
|
203
|
+
return False
|
|
204
|
+
if None in outcomes or "differentExpression" in outcomes:
|
|
205
|
+
return None
|
|
206
|
+
return True
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class LoopTracker:
|
|
210
|
+
"""Per-path loop bookkeeping. ``arrive`` runs once before each traced instruction.
|
|
211
|
+
|
|
212
|
+
Each call activation is identified by a number stored in its ``state.frames`` entry the first
|
|
213
|
+
time the tracker sees it, so a call inside a loop body never mistakes the callee's frame for the
|
|
214
|
+
caller's.
|
|
215
|
+
"""
|
|
216
|
+
|
|
217
|
+
def __init__(self, limit):
|
|
218
|
+
self.limit = limit
|
|
219
|
+
self.frames = {}
|
|
220
|
+
self.activations = 0
|
|
221
|
+
self.previous = None
|
|
222
|
+
self.edges = {}
|
|
223
|
+
self.iterations = []
|
|
224
|
+
self.omitted = 0
|
|
225
|
+
|
|
226
|
+
def __deepcopy__(self, memo):
|
|
227
|
+
# Arrivals, gate lists and recorded iterations are never changed once stored, so a fork
|
|
228
|
+
# copies only the containers that keep growing.
|
|
229
|
+
copy = LoopTracker.__new__(LoopTracker)
|
|
230
|
+
copy.limit, copy.activations, copy.previous, copy.omitted = self.limit, self.activations, self.previous, self.omitted
|
|
231
|
+
copy.frames = {activation: {"entry": frame["entry"], "tokens": dict(frame["tokens"]),
|
|
232
|
+
"heads": {head: {"history": None if loop["history"] is None else list(loop["history"]),
|
|
233
|
+
"gates": loop["gates"]}
|
|
234
|
+
for head, loop in frame["heads"].items()}}
|
|
235
|
+
for activation, frame in self.frames.items()}
|
|
236
|
+
copy.edges = {key: dict(edge) for key, edge in self.edges.items()}
|
|
237
|
+
copy.iterations = list(self.iterations)
|
|
238
|
+
return copy
|
|
239
|
+
|
|
240
|
+
def _frame(self, state):
|
|
241
|
+
top = state.frames[-1]
|
|
242
|
+
activation = top.get("loopActivation")
|
|
243
|
+
if activation is None:
|
|
244
|
+
self.activations += 1
|
|
245
|
+
activation = top["loopActivation"] = self.activations
|
|
246
|
+
# A new activation: drop the records of activations that returned, since ids never recur.
|
|
247
|
+
live = {f.get("loopActivation") for f in state.frames}
|
|
248
|
+
for gone in [a for a in self.frames if a not in live]:
|
|
249
|
+
del self.frames[gone]
|
|
250
|
+
self.frames[activation] = {"entry": top["entry"], "tokens": {}, "heads": {}}
|
|
251
|
+
return activation, self.frames[activation]
|
|
252
|
+
|
|
253
|
+
def arrive(self, state, at, ins):
|
|
254
|
+
activation, frame = self._frame(state)
|
|
255
|
+
previous, self.previous = self.previous, (at, at + ins.size, activation, ins.mnemonic)
|
|
256
|
+
last = frame["tokens"].get(at)
|
|
257
|
+
token = Arrival(1 if last is None else last.arrival + 1, len(state.events), len(state.write_log),
|
|
258
|
+
flag_state(state))
|
|
259
|
+
# Only a transfer to the site's own address or below closes a loop. Every cycle on a path has
|
|
260
|
+
# one, since fall-throughs run forward, and a forward branch to a join an earlier iteration
|
|
261
|
+
# ran stays an ordinary arrival.
|
|
262
|
+
if (last is not None and previous is not None and previous[2] == activation and at != previous[1]
|
|
263
|
+
and at <= previous[0]):
|
|
264
|
+
self._restart(state, frame, previous, at, last, token)
|
|
265
|
+
else:
|
|
266
|
+
loop = frame["heads"].get(at)
|
|
267
|
+
if loop is not None and loop["history"] is not None:
|
|
268
|
+
# An arrival at a known head that is not a restart, such as an inner loop's entry on
|
|
269
|
+
# the next outer iteration, is an earlier state later iterations may repeat. It starts a
|
|
270
|
+
# new entry into the loop, so the next iteration's gates are not compared with those
|
|
271
|
+
# of the last iteration before the loop was left.
|
|
272
|
+
loop["history"].append(token)
|
|
273
|
+
loop["gates"] = None
|
|
274
|
+
frame["tokens"][at] = token
|
|
275
|
+
|
|
276
|
+
def _restart(self, state, frame, previous, head, start, token):
|
|
277
|
+
site, _, activation, kind = previous
|
|
278
|
+
depth = len(state.frames) - 1
|
|
279
|
+
key = (activation, site, head)
|
|
280
|
+
edge = self.edges.get(key)
|
|
281
|
+
if edge is None:
|
|
282
|
+
edge = self.edges[key] = {"entry": frame["entry"], "depth": depth, "activation": activation,
|
|
283
|
+
"site": site, "target": head, "kind": kind, "traversals": 0,
|
|
284
|
+
"firstOrder": len(state.events)}
|
|
285
|
+
edge["traversals"] += 1
|
|
286
|
+
loop = frame["heads"].setdefault(head, {"history": [start], "gates": None})
|
|
287
|
+
if loop["history"] is None:
|
|
288
|
+
self.omitted += 1
|
|
289
|
+
return
|
|
290
|
+
if len(self.iterations) >= self.limit:
|
|
291
|
+
# Past the limit the history is dropped, so no later iteration is compared with a stale one.
|
|
292
|
+
loop["history"] = None
|
|
293
|
+
self.omitted += 1
|
|
294
|
+
return
|
|
295
|
+
# The iteration runs from the previous arrival at the head, however the path got there.
|
|
296
|
+
gates = [_gate(e) for e in state.events[start.event:] if e["kind"] == "branch" and e["depth"] == depth]
|
|
297
|
+
repeated = _compare_gates(loop["gates"], gates)
|
|
298
|
+
loop["gates"] = gates
|
|
299
|
+
before = _since(state, start.log)
|
|
300
|
+
now = {root: state.regs[root].term for root in REGISTERS}
|
|
301
|
+
self.iterations.append({
|
|
302
|
+
"head": head, "entry": frame["entry"], "depth": depth, "activation": activation,
|
|
303
|
+
"fromArrival": start.arrival, "toArrival": token.arrival,
|
|
304
|
+
"restartEdge": {"site": site, "kind": kind},
|
|
305
|
+
"fromOrder": start.event, "toOrder": token.event,
|
|
306
|
+
"registers": compare_registers(state.flat, registers_at(before, state), now),
|
|
307
|
+
"flags": "unchanged" if start.flags == token.flags else "differ",
|
|
308
|
+
"memory": _intervals(memory_changes(before, state)), "memoryForgotten": MEMORY_CLEARED in before,
|
|
309
|
+
"gates": gates,
|
|
310
|
+
"gateOperandsRepeated": repeated, "stateRepeatsArrival": repeated_arrival(state, loop["history"], token)})
|
|
311
|
+
loop["history"].append(token)
|
|
312
|
+
|
|
313
|
+
def report(self):
|
|
314
|
+
"""The ``loops`` field of a traced path."""
|
|
315
|
+
return {"restartEdges": list(self.edges.values()), "iterations": self.iterations,
|
|
316
|
+
"iterationLimit": self.limit, "iterationsOmitted": self.omitted,
|
|
317
|
+
"allIterationsRecorded": not self.omitted, "interpretation": INTERPRETATION}
|
|
@@ -18,6 +18,53 @@ class StopPath(Exception):
|
|
|
18
18
|
pass
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
# The write-log key that marks a point where the model forgot all memory.
|
|
22
|
+
MEMORY_CLEARED = ("memory-cleared",)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class WriteLog:
|
|
26
|
+
"""Append-only list of (key, previous value) pairs, one per register or memory byte a write changes.
|
|
27
|
+
|
|
28
|
+
A memory key is (segment, base, offset); a register key is ("register", root). A previous memory
|
|
29
|
+
value is None when the model held no value for the byte. ``(MEMORY_CLEARED, None)`` marks a
|
|
30
|
+
point where the model forgot all memory. Copies made at a fork share the entries
|
|
31
|
+
recorded before the fork, which are immutable, so forking a path never copies its log.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self):
|
|
35
|
+
self.chunks = [] # (absolute index of the first entry, tuple of entries), oldest first
|
|
36
|
+
self.tail = []
|
|
37
|
+
self.length = 0
|
|
38
|
+
|
|
39
|
+
def __len__(self):
|
|
40
|
+
return self.length
|
|
41
|
+
|
|
42
|
+
def append(self, entry):
|
|
43
|
+
self.tail.append(entry)
|
|
44
|
+
self.length += 1
|
|
45
|
+
|
|
46
|
+
def extend(self, entries):
|
|
47
|
+
for entry in entries:
|
|
48
|
+
self.append(entry)
|
|
49
|
+
|
|
50
|
+
def since(self, start):
|
|
51
|
+
"""The entries at absolute index ``start`` and later, oldest first."""
|
|
52
|
+
for first, chunk in self.chunks:
|
|
53
|
+
if first + len(chunk) > start:
|
|
54
|
+
yield from chunk[max(0, start - first):]
|
|
55
|
+
first = self.length - len(self.tail)
|
|
56
|
+
yield from self.tail[max(0, start - first):]
|
|
57
|
+
|
|
58
|
+
def __deepcopy__(self, memo):
|
|
59
|
+
if self.tail:
|
|
60
|
+
self.chunks.append((self.length - len(self.tail), tuple(self.tail)))
|
|
61
|
+
self.tail = []
|
|
62
|
+
copy = WriteLog()
|
|
63
|
+
copy.chunks = list(self.chunks)
|
|
64
|
+
copy.length = self.length
|
|
65
|
+
return copy
|
|
66
|
+
|
|
67
|
+
|
|
21
68
|
class Shared(dict):
|
|
22
69
|
"""Read-only query configuration that every copy of a path shares instead of copying."""
|
|
23
70
|
|
|
@@ -51,6 +98,9 @@ class State:
|
|
|
51
98
|
self.semantics = BACKEND
|
|
52
99
|
self.sp, self.bp = ("esp", "ebp") if self.flat else ("sp", "bp")
|
|
53
100
|
self.at = entry
|
|
101
|
+
# Every register and memory byte a write changes, with its previous value, in order, so a
|
|
102
|
+
# reader can recover the state as it stood at an earlier point of the path (x86/loops.py).
|
|
103
|
+
self.write_log = WriteLog()
|
|
54
104
|
self.regs = {r: unknown("initial:" + r, ALIASES[r][2]) for r in REGISTERS}
|
|
55
105
|
# Producers per register byte, so a partial write replaces only the bytes it stores.
|
|
56
106
|
self.reg_sources = {r: [()] * (ALIASES[r][2] // 8) for r in REGISTERS}
|
|
@@ -169,6 +219,9 @@ class State:
|
|
|
169
219
|
self.flag_values = None
|
|
170
220
|
|
|
171
221
|
def clear_memory(self):
|
|
222
|
+
# Bytes the model never held may change too, so the log marks the clear itself.
|
|
223
|
+
self.write_log.extend(self.memory.items())
|
|
224
|
+
self.write_log.append((MEMORY_CLEARED, None))
|
|
172
225
|
self.memory.clear()
|
|
173
226
|
self.memory_groups.clear()
|
|
174
227
|
self.unread_memory.clear()
|
|
@@ -227,6 +280,7 @@ class State:
|
|
|
227
280
|
root, low, bits = alias(name)
|
|
228
281
|
value = resize(value, bits)
|
|
229
282
|
value = Value(bits, value.term, sources(value, site=site))
|
|
283
|
+
self.write_log.append((("register", root), self.regs[root]))
|
|
230
284
|
if bits == ALIASES[root][2]:
|
|
231
285
|
self.regs[root] = value
|
|
232
286
|
else:
|
|
@@ -279,7 +333,7 @@ class State:
|
|
|
279
333
|
for key in list(members):
|
|
280
334
|
if may_alias(key, written, self.bits, self.flat):
|
|
281
335
|
uncertain.append(key)
|
|
282
|
-
self.memory.pop(key, None)
|
|
336
|
+
self.write_log.append((key, self.memory.pop(key, None)))
|
|
283
337
|
self.unread_memory.pop(key, None)
|
|
284
338
|
self.memory_writers.pop(key, None)
|
|
285
339
|
self.lost_memory[key] = {"cause": "dropped by a possibly aliasing write", "order": len(self.events)}
|
|
@@ -287,6 +341,7 @@ class State:
|
|
|
287
341
|
if not members:
|
|
288
342
|
del self.memory_groups[group]
|
|
289
343
|
for i, key in enumerate(keys):
|
|
344
|
+
self.write_log.append((key, self.memory.get(key)))
|
|
290
345
|
self.memory[key] = extract(write, i * 8, 8)
|
|
291
346
|
self.unread_memory.pop(key, None)
|
|
292
347
|
self.memory_writers[key] = len(self.events)
|
|
@@ -231,7 +231,7 @@ def uses(image, config):
|
|
|
231
231
|
stops.setdefault(root, "entry not traced: entry or total instruction budget exhausted")
|
|
232
232
|
break
|
|
233
233
|
report = trace(image, {**config, "entry": at, "totalSteps": remaining, "stringIterations": string_remaining},
|
|
234
|
-
continue_declared_jumps=False)
|
|
234
|
+
continue_declared_jumps=False, track_loops=False)
|
|
235
235
|
remaining -= report["stepsUsed"]
|
|
236
236
|
string_remaining -= report["stringIterationsUsed"]
|
|
237
237
|
if not report["completeWithinModel"]:
|
|
@@ -501,7 +501,8 @@ def dispatch(image, config):
|
|
|
501
501
|
raise ValueError("Dispatch table mapping differs from PE source sections")
|
|
502
502
|
for value in values:
|
|
503
503
|
integer(value, 0, (1 << ALIASES[input_reg][2]) - 1, "input value")
|
|
504
|
-
report = trace(image, {**config, "registers": {**config.get("registers", {}), input_reg: value}}, continue_declared_jumps=False
|
|
504
|
+
report = trace(image, {**config, "registers": {**config.get("registers", {}), input_reg: value}}, continue_declared_jumps=False,
|
|
505
|
+
track_loops=False)
|
|
505
506
|
outcomes = []
|
|
506
507
|
for path in report["paths"]:
|
|
507
508
|
reached = bool(path["instructionPath"]) and path["instructionPath"][-1] == site
|
|
@@ -58,6 +58,17 @@ def result_contracts(state, contracts, entry):
|
|
|
58
58
|
DOMAINS = {"l": "signed", "le": "signed", "s": "signed", "o": "signed", "c": "unsigned", "be": "unsigned"}
|
|
59
59
|
|
|
60
60
|
|
|
61
|
+
def predicate_domain(predicate):
|
|
62
|
+
"""How a branch reads what it tests: ``signed``, ``unsigned``, ``counter`` or ``flags/equality``.
|
|
63
|
+
|
|
64
|
+
LOOP, LOOPE, LOOPNE, JCXZ and JECXZ test CX or ECX against zero (``counter``); the conditional
|
|
65
|
+
jumps read the flags of their producer.
|
|
66
|
+
"""
|
|
67
|
+
if predicate.startswith("loop") or predicate in ("jcxz", "jecxz"):
|
|
68
|
+
return "counter"
|
|
69
|
+
return DOMAINS.get(BRANCH_CONDITIONS.get(predicate, (None,))[0], "flags/equality")
|
|
70
|
+
|
|
71
|
+
|
|
61
72
|
VALUE_FIELDS = ("sourceValue", "resultValue", "destinationContainerValue", "result", "value", "left", "right", "carry", "count")
|
|
62
73
|
CONSUMER_KINDS = ("value-transfer", "conversion", "read", "write", "compare", "branch", "return")
|
|
63
74
|
ROW_FIELDS = ("kind", "site", "entry", "depth", "order", "operation", "source", "destination", "sourceBits", "destinationBits",
|
|
@@ -114,8 +125,10 @@ def return_flows(report, config):
|
|
|
114
125
|
row.update(values=values, dependentValueFields=list(dependent), returnWidthRelationships=widths,
|
|
115
126
|
relationship="producer dependency only; not value or storage identity")
|
|
116
127
|
if later["kind"] == "branch":
|
|
117
|
-
|
|
118
|
-
|
|
128
|
+
domain = predicate_domain(later.get("predicate", ""))
|
|
129
|
+
# returnFlows has reported LOOP and JCXZ as flags/equality since it shipped, and
|
|
130
|
+
# consumers read that value, so it stays until a major release changes it.
|
|
131
|
+
row["predicateDomain"] = "flags/equality" if domain == "counter" else domain
|
|
119
132
|
consumers.append(row)
|
|
120
133
|
flows.append({"originOrder": event["order"], "originSite": event["site"], "calleeEntry": contract["entry"],
|
|
121
134
|
"callSite": event.get("callSite"), "callerEntry": event.get("callerEntry"),
|
|
@@ -6,6 +6,7 @@ from .machine import (State, StopPath, REGISTERS, ALIASES, BRANCH_CONDITIONS, st
|
|
|
6
6
|
string_count, string_effect, check_string_form, compare_string, repeated, string_width)
|
|
7
7
|
from .values import const, unknown, sources, op, Value
|
|
8
8
|
from .result_flow import validate_contracts, result_contracts
|
|
9
|
+
from .loops import LoopTracker
|
|
9
10
|
from .memory_scopes import validate_scopes, capture_scopes, retain_scopes, scope_history
|
|
10
11
|
|
|
11
12
|
def call_target(image, site, ins):
|
|
@@ -278,13 +279,14 @@ def snapshot(state):
|
|
|
278
279
|
return {name: state.reg(name).report() for name in ALIASES}
|
|
279
280
|
|
|
280
281
|
|
|
281
|
-
def trace(image, config, continue_declared_jumps=True):
|
|
282
|
+
def trace(image, config, continue_declared_jumps=True, track_loops=True):
|
|
282
283
|
"""Trace bounded paths, preserving declared-table continuations as separate conditional evidence.
|
|
283
284
|
|
|
284
285
|
Ordinary paths run first. A path stopped at a declared indirect jump is then
|
|
285
286
|
continued once per surviving table target, so the continuations never take
|
|
286
287
|
budget from an ordinary path. Callers that read only ordinary paths pass
|
|
287
|
-
continue_declared_jumps=False.
|
|
288
|
+
continue_declared_jumps=False. Callers that never report the paths pass
|
|
289
|
+
track_loops=False, and their paths carry no ``loops`` record.
|
|
288
290
|
"""
|
|
289
291
|
entry = integer(config.get("entry"), 0, len(image.data) - 1, "entry")
|
|
290
292
|
if not any(entry in r["entries"] for r in image.regions):
|
|
@@ -320,7 +322,11 @@ def trace(image, config, continue_declared_jumps=True):
|
|
|
320
322
|
for r, n in case.get("registers", {}).items():
|
|
321
323
|
if r not in ALIASES or type(n) is not int or not 0 <= n < 1 << ALIASES[r][2]:
|
|
322
324
|
raise ValueError("Invalid model register")
|
|
323
|
-
|
|
325
|
+
root = State(entry, image, config)
|
|
326
|
+
# Each path carries its own loop record; forks copy it with the rest of the state.
|
|
327
|
+
root.loops = (LoopTracker(integer(config.get("loopIterationLimit", 64), 1, 1024, "loopIterationLimit"))
|
|
328
|
+
if track_loops else None)
|
|
329
|
+
pending, outputs, global_gaps = [root], [], []
|
|
324
330
|
conditional_outputs = []
|
|
325
331
|
# States stopped at a declared jump site, continued after the ordinary paths.
|
|
326
332
|
deferred = []
|
|
@@ -354,6 +360,8 @@ def trace(image, config, continue_declared_jumps=True):
|
|
|
354
360
|
path = {"returned": returned, "stop": reason, "stopSite": None if returned else s.at, "steps": s.steps,
|
|
355
361
|
"instructionPath": s.path, "guards": s.guards, "events": s.events,
|
|
356
362
|
"registers": snapshot(s), "conditionalModels": s.conditional}
|
|
363
|
+
if s.loops is not None:
|
|
364
|
+
path["loops"] = s.loops.report()
|
|
357
365
|
assumptions = getattr(s, "declared_jump_assumptions", [])
|
|
358
366
|
if assumptions:
|
|
359
367
|
path["declaredJumpAssumptions"] = assumptions
|
|
@@ -454,6 +462,8 @@ def trace(image, config, continue_declared_jumps=True):
|
|
|
454
462
|
raise StopPath("undecoded or unmapped instruction")
|
|
455
463
|
state.steps += 1
|
|
456
464
|
state.path.append(at)
|
|
465
|
+
if state.loops is not None:
|
|
466
|
+
state.loops.arrive(state, at, ins)
|
|
457
467
|
state.visits[at] = state.visits.get(at, 0) + 1
|
|
458
468
|
if state.visits[at] > visit_limit:
|
|
459
469
|
raise StopPath(f"instruction repeated more than {visit_limit} times; raise visitLimit or read the loop's bound")
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Loop restart edges and iteration changes on synthetic code only."""
|
|
2
|
+
import unittest
|
|
3
|
+
from copy import deepcopy
|
|
4
|
+
from test_x86 import Code, report
|
|
5
|
+
from scientific_method_engine.x86.loops import LoopTracker, _compare_gates
|
|
6
|
+
from scientific_method_engine.x86.result_flow import predicate_domain
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def stopped(result):
|
|
10
|
+
return next(p for p in result["paths"] if not p["returned"])
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def statuses(iteration):
|
|
14
|
+
return [row["status"] for row in iteration["memory"]]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class LoopProgressTests(unittest.TestCase):
|
|
18
|
+
def counted(self):
|
|
19
|
+
# mov cx, 3; xor ax, ax; head: inc ax; loop head; ret
|
|
20
|
+
return Code().emit("b9 03 00 31 c0").label("head").emit("40").label("back").branch("e2", "head").emit("c3")
|
|
21
|
+
|
|
22
|
+
def test_counted_loop_reports_its_restart_edge_and_what_each_iteration_changed(self):
|
|
23
|
+
c = self.counted()
|
|
24
|
+
path = report(c)["paths"][0]
|
|
25
|
+
self.assertTrue(path["returned"])
|
|
26
|
+
loops = path["loops"]
|
|
27
|
+
self.assertEqual([(e["site"], e["target"], e["kind"], e["traversals"], e["depth"]) for e in loops["restartEdges"]],
|
|
28
|
+
[(c.labels["back"], c.labels["head"], "loop", 2, 0)])
|
|
29
|
+
self.assertEqual(len(loops["iterations"]), 2)
|
|
30
|
+
first, second = loops["iterations"]
|
|
31
|
+
self.assertEqual((first["fromArrival"], first["toArrival"]), (1, 2))
|
|
32
|
+
changed = {row["register"]: (row["relation"], row["before"]["value"], row["after"]["value"])
|
|
33
|
+
for row in first["registers"]["changed"]}
|
|
34
|
+
self.assertEqual(changed, {"ax": ("changed", 0, 1), "cx": ("changed", 3, 2)})
|
|
35
|
+
self.assertIn("si", first["registers"]["unchanged"])
|
|
36
|
+
gate = first["gates"][0]
|
|
37
|
+
self.assertEqual((gate["predicate"], gate["predicateDomain"], gate["taken"]), ("loop", "counter", True))
|
|
38
|
+
self.assertEqual(gate["operands"]["count"]["value"], 2)
|
|
39
|
+
# The first iteration has no earlier one to compare its gates with.
|
|
40
|
+
self.assertIsNone(first["gateOperandsRepeated"])
|
|
41
|
+
self.assertEqual(second["gates"][0]["operandsSincePreviousIteration"], "changed")
|
|
42
|
+
self.assertFalse(second["gateOperandsRepeated"])
|
|
43
|
+
self.assertIsNone(second["stateRepeatsArrival"])
|
|
44
|
+
self.assertTrue(loops["allIterationsRecorded"])
|
|
45
|
+
self.assertIn("no termination", loops["interpretation"])
|
|
46
|
+
|
|
47
|
+
def test_iteration_that_changes_nothing_is_flagged_and_the_visit_limit_still_stops(self):
|
|
48
|
+
# head: cmp si, 16; jae out; jmp head; out: ret
|
|
49
|
+
c = Code().label("head").emit("83 fe 10").branch("73", "out").label("back").branch("eb", "head").label("out").emit("c3")
|
|
50
|
+
r = report(c)
|
|
51
|
+
exit_path = next(p for p in r["paths"] if p["returned"])
|
|
52
|
+
self.assertEqual(exit_path["loops"]["restartEdges"], [])
|
|
53
|
+
path = stopped(r)
|
|
54
|
+
self.assertIn("visitLimit", path["stop"])
|
|
55
|
+
loops = path["loops"]
|
|
56
|
+
self.assertEqual(loops["restartEdges"][0]["site"], c.labels["back"])
|
|
57
|
+
self.assertEqual(loops["restartEdges"][0]["kind"], "jmp")
|
|
58
|
+
last = loops["iterations"][-1]
|
|
59
|
+
self.assertEqual(last["registers"]["changed"], [])
|
|
60
|
+
self.assertEqual(last["memory"], [])
|
|
61
|
+
self.assertTrue(last["gateOperandsRepeated"])
|
|
62
|
+
self.assertEqual(last["gates"][0]["predicateDomain"], "unsigned")
|
|
63
|
+
self.assertEqual(last["gates"][0]["operandsSincePreviousIteration"], "unchanged")
|
|
64
|
+
# The first arrival came before any flag producer, so the second is the first repeated state.
|
|
65
|
+
self.assertIsNone(loops["iterations"][0]["stateRepeatsArrival"])
|
|
66
|
+
self.assertEqual(last["stateRepeatsArrival"], 2)
|
|
67
|
+
|
|
68
|
+
def test_wrapped_index_restores_an_earlier_state_without_any_consecutive_repeat(self):
|
|
69
|
+
# mov si, 14; head: inc si; and si, 15; cmp byte [si+0x100], 0; jne head; ret
|
|
70
|
+
c = Code().emit("be 0e 00").label("head").emit("46 83 e6 0f 80 bc 00 01 00").branch("75", "head").emit("c3")
|
|
71
|
+
path = stopped(report(c, visitLimit=20))
|
|
72
|
+
iterations = path["loops"]["iterations"]
|
|
73
|
+
repeats = [(i["toArrival"], i["stateRepeatsArrival"]) for i in iterations if i["stateRepeatsArrival"]]
|
|
74
|
+
self.assertEqual(repeats[0], (18, 2))
|
|
75
|
+
self.assertTrue(all(i["gateOperandsRepeated"] is not True for i in iterations))
|
|
76
|
+
self.assertEqual(iterations[0]["gates"][0]["predicateDomain"], "flags/equality")
|
|
77
|
+
# Each read names another slot, so the model cannot tell whether the compared bytes differ.
|
|
78
|
+
self.assertEqual(iterations[1]["gates"][0]["operandsSincePreviousIteration"], "differentExpression")
|
|
79
|
+
|
|
80
|
+
def test_rewriting_a_free_slot_with_the_same_value_leaves_the_state_unchanged(self):
|
|
81
|
+
# head: mov byte [0x200], 0; cmp si, 16; jae out; jmp head; out: ret
|
|
82
|
+
c = Code().label("head").emit("c6 06 00 02 00 83 fe 10").branch("73", "out").branch("eb", "head").label("out").emit("c3")
|
|
83
|
+
iterations = stopped(report(c))["loops"]["iterations"]
|
|
84
|
+
self.assertEqual(statuses(iterations[0]), ["writtenOverUnmodeled"])
|
|
85
|
+
self.assertEqual(statuses(iterations[1]), ["unchanged"])
|
|
86
|
+
self.assertEqual((iterations[1]["memory"][0]["start"], iterations[1]["memory"][0]["end"]), (0x200, 0x201))
|
|
87
|
+
self.assertEqual(iterations[1]["stateRepeatsArrival"], 2)
|
|
88
|
+
|
|
89
|
+
def test_a_possibly_aliasing_write_reports_the_invalidated_bytes(self):
|
|
90
|
+
# head: mov byte [bx], 1; mov byte [di], 2; cmp si, 16; jae out; jmp head; out: ret
|
|
91
|
+
c = Code().label("head").emit("c6 07 01 c6 05 02 83 fe 10").branch("73", "out").branch("eb", "head").label("out").emit("c3")
|
|
92
|
+
iterations = stopped(report(c))["loops"]["iterations"]
|
|
93
|
+
self.assertEqual(sorted(statuses(iterations[1])), ["invalidated", "unchanged"])
|
|
94
|
+
# The model holds no byte for [bx] at either arrival; its contents may differ, so no repeated state is claimed.
|
|
95
|
+
self.assertIsNone(iterations[1]["stateRepeatsArrival"])
|
|
96
|
+
|
|
97
|
+
def test_a_call_model_that_forgets_memory_is_reported_on_the_iteration(self):
|
|
98
|
+
# mov cx, 3; head: call f (modeled); loop head; ret; f: ret
|
|
99
|
+
c = Code().emit("b9 03 00").label("head").label("call").branch("e8", "f").branch("e2", "head").emit("c3")
|
|
100
|
+
c.label("f").emit("c3")
|
|
101
|
+
registers = ["eax", "ebx", "ecx", "edx", "esi", "edi", "ebp", "esp", "cs", "ds", "es", "ss", "fs", "gs"]
|
|
102
|
+
model = {"site": c.labels["call"], "evidence": "synthetic service", "preserves": registers, "cases": [{}]}
|
|
103
|
+
iterations = report(c, callModels=[model])["paths"][0]["loops"]["iterations"]
|
|
104
|
+
# The model held no byte, so nothing is listed, yet the callee may have written any byte.
|
|
105
|
+
self.assertEqual([(i["memory"], i["memoryForgotten"]) for i in iterations], [([], True), ([], True)])
|
|
106
|
+
self.assertIsNone(iterations[1]["stateRepeatsArrival"])
|
|
107
|
+
plain = report(self.counted())["paths"][0]["loops"]["iterations"]
|
|
108
|
+
self.assertEqual({i["memoryForgotten"] for i in plain}, {False})
|
|
109
|
+
|
|
110
|
+
def test_restart_after_a_collision_is_a_second_restart_edge_to_the_same_head(self):
|
|
111
|
+
# mov si, 0; head: cmp di, 0; jne collide; inc si; cmp si, 3; jb head; ret; collide: xor di, di; jmp head
|
|
112
|
+
c = Code().emit("be 00 00").label("head").emit("83 ff 00").branch("75", "collide").emit("46 83 fe 03")
|
|
113
|
+
c.label("next").branch("72", "head").emit("c3").label("collide").emit("31 ff").label("restart").branch("eb", "head")
|
|
114
|
+
r = report(c)
|
|
115
|
+
path = next(p for p in r["paths"] if p["loops"]["restartEdges"] and len(p["loops"]["restartEdges"]) == 2)
|
|
116
|
+
self.assertTrue(path["returned"])
|
|
117
|
+
edges = {(e["site"], e["kind"]): e["traversals"] for e in path["loops"]["restartEdges"]}
|
|
118
|
+
self.assertEqual(edges, {(c.labels["restart"], "jmp"): 1, (c.labels["next"], "jb"): 2})
|
|
119
|
+
sites = [i["restartEdge"]["site"] for i in path["loops"]["iterations"]]
|
|
120
|
+
self.assertEqual(sites, [c.labels["restart"], c.labels["next"], c.labels["next"]])
|
|
121
|
+
self.assertEqual({g["predicateDomain"] for i in path["loops"]["iterations"] for g in i["gates"]},
|
|
122
|
+
{"flags/equality", "unsigned"})
|
|
123
|
+
|
|
124
|
+
def test_signed_gate_keeps_its_domain(self):
|
|
125
|
+
# mov si, 0; head: inc si; cmp si, 3; jl head; ret
|
|
126
|
+
c = Code().emit("be 00 00").label("head").emit("46 83 fe 03").branch("7c", "head").emit("c3")
|
|
127
|
+
gates = [g for i in report(c)["paths"][0]["loops"]["iterations"] for g in i["gates"]]
|
|
128
|
+
self.assertEqual({(g["predicate"], g["predicateDomain"]) for g in gates}, {("jl", "signed")})
|
|
129
|
+
|
|
130
|
+
def test_a_loope_gate_keeps_its_zero_flag_test(self):
|
|
131
|
+
# mov cx, 3; cmp si, 0; head: loope head; ret
|
|
132
|
+
c = Code().emit("b9 03 00 83 fe 00").label("head").branch("e1", "head").emit("c3")
|
|
133
|
+
gates = [g for p in report(c)["paths"] for i in p["loops"]["iterations"] for g in i["gates"]]
|
|
134
|
+
self.assertTrue(gates)
|
|
135
|
+
for gate in gates:
|
|
136
|
+
self.assertEqual((gate["predicate"], gate["predicateDomain"]), ("loope", "counter"))
|
|
137
|
+
self.assertEqual(gate["zeroFlag"]["predicate"], "je")
|
|
138
|
+
self.assertEqual(gate["zeroFlag"]["left"]["expression"], ("extract", ("unknown", "initial:esi"), 0, 16, 32))
|
|
139
|
+
|
|
140
|
+
def test_a_forward_branch_to_a_join_an_earlier_iteration_ran_is_not_a_restart_edge(self):
|
|
141
|
+
# mov cx, 3; head: test cx, 1; je skip; inc ax; skip: loop head; ret
|
|
142
|
+
c = Code().emit("b9 03 00").label("head").emit("f7 c1 01 00").label("je").branch("74", "skip").emit("40")
|
|
143
|
+
c.label("skip").branch("e2", "head").emit("c3")
|
|
144
|
+
paths = report(c)["paths"]
|
|
145
|
+
self.assertTrue(paths)
|
|
146
|
+
for path in paths:
|
|
147
|
+
loops = path["loops"]
|
|
148
|
+
self.assertEqual({(e["site"], e["target"], e["kind"]) for e in loops["restartEdges"]},
|
|
149
|
+
{(c.labels["skip"], c.labels["head"], "loop")})
|
|
150
|
+
self.assertEqual({i["head"] for i in loops["iterations"]}, {c.labels["head"]})
|
|
151
|
+
|
|
152
|
+
def test_a_rotated_loop_entered_by_a_forward_jump_to_its_test_is_headed_at_its_body(self):
|
|
153
|
+
# mov cx, 3; jmp test; body: inc ax; test: dec cx; jnz body; ret
|
|
154
|
+
c = Code().emit("b9 03 00").branch("eb", "test").label("body").emit("40").label("test").emit("49")
|
|
155
|
+
c.label("jnz").branch("75", "body").emit("c3")
|
|
156
|
+
loops = next(p for p in report(c)["paths"] if p["returned"])["loops"]
|
|
157
|
+
self.assertEqual([(e["site"], e["target"], e["kind"], e["traversals"]) for e in loops["restartEdges"]],
|
|
158
|
+
[(c.labels["jnz"], c.labels["body"], "jne", 1)])
|
|
159
|
+
self.assertEqual([i["head"] for i in loops["iterations"]], [c.labels["body"]])
|
|
160
|
+
|
|
161
|
+
def test_calling_one_function_twice_is_not_a_restart_and_each_activation_keeps_its_loop(self):
|
|
162
|
+
# call f; call f; ret; f: mov cx, 2; L: loop L; ret
|
|
163
|
+
c = Code().branch("e8", "f").branch("e8", "f").emit("c3").label("f").emit("b9 02 00").label("L").branch("e2", "L").emit("c3")
|
|
164
|
+
loops = report(c)["paths"][0]["loops"]
|
|
165
|
+
edges = loops["restartEdges"]
|
|
166
|
+
self.assertEqual([(e["entry"], e["depth"], e["target"], e["traversals"]) for e in edges],
|
|
167
|
+
[(c.labels["f"], 1, c.labels["L"], 1)] * 2)
|
|
168
|
+
self.assertEqual(len({e["activation"] for e in edges}), 2)
|
|
169
|
+
self.assertEqual(len(loops["iterations"]), 2)
|
|
170
|
+
|
|
171
|
+
def test_iteration_limit_is_reached_and_reported(self):
|
|
172
|
+
loops = report(self.counted(), loopIterationLimit=1)["paths"][0]["loops"]
|
|
173
|
+
self.assertEqual(len(loops["iterations"]), 1)
|
|
174
|
+
self.assertEqual(loops["iterationsOmitted"], 1)
|
|
175
|
+
self.assertFalse(loops["allIterationsRecorded"])
|
|
176
|
+
self.assertEqual(loops["restartEdges"][0]["traversals"], 2)
|
|
177
|
+
for bad in (0, 1025, "4"):
|
|
178
|
+
with self.assertRaises(ValueError):
|
|
179
|
+
report(self.counted(), loopIterationLimit=bad)
|
|
180
|
+
|
|
181
|
+
def test_effects_paths_carry_the_loop_record(self):
|
|
182
|
+
loops = report(self.counted(), "effects")["paths"][0]["loops"]
|
|
183
|
+
self.assertEqual(len(loops["iterations"]), 2)
|
|
184
|
+
|
|
185
|
+
def test_a_call_inside_the_loop_body_keeps_the_callers_restart_edge(self):
|
|
186
|
+
# mov cx, 3; head: call f; loop head; ret; f: ret
|
|
187
|
+
c = Code().emit("b9 03 00").label("head").branch("e8", "f").label("back").branch("e2", "head").emit("c3")
|
|
188
|
+
c.label("f").emit("c3")
|
|
189
|
+
loops = report(c)["paths"][0]["loops"]
|
|
190
|
+
self.assertEqual([(e["site"], e["target"], e["kind"], e["traversals"], e["depth"]) for e in loops["restartEdges"]],
|
|
191
|
+
[(c.labels["back"], c.labels["head"], "loop", 2, 0)])
|
|
192
|
+
self.assertEqual([(i["fromArrival"], i["toArrival"]) for i in loops["iterations"]], [(1, 2), (2, 3)])
|
|
193
|
+
self.assertEqual([[g["predicate"] for g in i["gates"]] for i in loops["iterations"]], [["loop"], ["loop"]])
|
|
194
|
+
|
|
195
|
+
def test_an_inner_iteration_starts_at_the_previous_arrival_at_its_head(self):
|
|
196
|
+
# mov bx, 2; outer: mov cx, 2; inner: inc ax; loop inner; dec bx; jnz outer; ret
|
|
197
|
+
c = Code().emit("bb 02 00").label("outer").emit("b9 02 00").label("inner").emit("40")
|
|
198
|
+
c.branch("e2", "inner").emit("4b").branch("75", "outer").emit("c3")
|
|
199
|
+
loops = report(c)["paths"][0]["loops"]
|
|
200
|
+
inner = [i for i in loops["iterations"] if i["head"] == c.labels["inner"]]
|
|
201
|
+
self.assertEqual([(i["fromArrival"], i["toArrival"]) for i in inner], [(1, 2), (3, 4)])
|
|
202
|
+
# The outer gate ran between the inner head's second and third arrivals, outside both inner iterations.
|
|
203
|
+
self.assertEqual([[g["predicate"] for g in i["gates"]] for i in inner], [["loop"], ["loop"]])
|
|
204
|
+
# The second outer iteration enters the inner loop again with CX reset, so its first inner
|
|
205
|
+
# iteration is not compared with the last one before the inner loop was left.
|
|
206
|
+
self.assertEqual([i["gateOperandsRepeated"] for i in inner], [None, None])
|
|
207
|
+
self.assertNotIn("operandsSincePreviousIteration", inner[1]["gates"][0])
|
|
208
|
+
outer = [i for i in loops["iterations"] if i["head"] == c.labels["outer"]]
|
|
209
|
+
self.assertEqual([g["predicate"] for g in outer[0]["gates"]], ["loop", "loop", "jne"])
|
|
210
|
+
|
|
211
|
+
def test_flags_recomputed_to_the_same_values_repeat(self):
|
|
212
|
+
# mov cx, 1; head: dec cx; inc cx; jnz head; ret
|
|
213
|
+
c = Code().emit("b9 01 00").label("head").emit("49 41").branch("75", "head").emit("c3")
|
|
214
|
+
iterations = stopped(report(c))["loops"]["iterations"]
|
|
215
|
+
self.assertEqual(iterations[0]["flags"], "differ")
|
|
216
|
+
self.assertEqual(iterations[1]["flags"], "unchanged")
|
|
217
|
+
self.assertEqual(iterations[1]["registers"]["changed"], [])
|
|
218
|
+
self.assertEqual(iterations[1]["stateRepeatsArrival"], 2)
|
|
219
|
+
|
|
220
|
+
def test_a_16_bit_write_leaves_the_32_bit_register_unchanged_in_the_segmented_model(self):
|
|
221
|
+
# mov cx, 3; head: inc ax; loop head; ret
|
|
222
|
+
c = Code().emit("b9 03 00").label("head").emit("40").branch("e2", "head").emit("c3")
|
|
223
|
+
first = report(c)["paths"][0]["loops"]["iterations"][0]
|
|
224
|
+
self.assertEqual({row["register"] for row in first["registers"]["changed"]}, {"ax", "cx"})
|
|
225
|
+
# mov cx, 3; head: add eax, 0x10000; loop head; ret
|
|
226
|
+
c = Code().emit("b9 03 00").label("head").emit("66 05 00 00 01 00").branch("e2", "head").emit("c3")
|
|
227
|
+
first = report(c)["paths"][0]["loops"]["iterations"][0]
|
|
228
|
+
self.assertIn("eax", {row["register"] for row in first["registers"]["changed"]})
|
|
229
|
+
|
|
230
|
+
def test_an_iteration_without_the_previous_iterations_gates_is_not_a_repeat(self):
|
|
231
|
+
gate = {"site": 4, "taken": True, "operands": {"count": {"expression": ("constant", 1)}}}
|
|
232
|
+
self.assertIs(_compare_gates([gate], []), False)
|
|
233
|
+
self.assertIsNone(_compare_gates([], []))
|
|
234
|
+
self.assertIsNone(_compare_gates(None, []))
|
|
235
|
+
|
|
236
|
+
def test_forking_a_path_shares_its_write_log_and_recorded_arrivals(self):
|
|
237
|
+
from scientific_method_engine.x86.machine import WriteLog
|
|
238
|
+
log = WriteLog()
|
|
239
|
+
log.extend([(("register", "eax"), None), (("register", "ebx"), None)])
|
|
240
|
+
child = deepcopy(log)
|
|
241
|
+
child.append((("register", "ecx"), None))
|
|
242
|
+
log.append((("register", "edx"), None))
|
|
243
|
+
self.assertIs(child.chunks[0][1], log.chunks[0][1])
|
|
244
|
+
self.assertEqual([k[1] for k, _ in child.since(1)], ["ebx", "ecx"])
|
|
245
|
+
self.assertEqual([k[1] for k, _ in log.since(2)], ["edx"])
|
|
246
|
+
tracker = LoopTracker(4)
|
|
247
|
+
tracker.frames[1] = {"entry": 0, "tokens": {0: ("arrival",)}, "heads": {}}
|
|
248
|
+
copy = deepcopy(tracker)
|
|
249
|
+
self.assertIs(copy.frames[1]["tokens"][0], tracker.frames[1]["tokens"][0])
|
|
250
|
+
self.assertIsNot(copy.frames[1]["tokens"], tracker.frames[1]["tokens"])
|
|
251
|
+
|
|
252
|
+
def test_one_predicate_domain_serves_loop_gates_and_return_flows(self):
|
|
253
|
+
self.assertEqual([predicate_domain(p) for p in ("loop", "loopne", "jcxz", "jecxz", "jl", "jae", "jne")],
|
|
254
|
+
["counter", "counter", "counter", "counter", "signed", "unsigned", "flags/equality"])
|
|
255
|
+
# call f; mov cx, ax; L: loop L; ret; f: mov ax, 2; ret
|
|
256
|
+
c = Code().branch("e8", "f").emit("89 c1").label("L").branch("e2", "L").emit("c3").label("f").emit("b8 02 00 c3")
|
|
257
|
+
contract = {"entry": c.labels["f"], "register": "ax", "evidence": "synthetic result contract"}
|
|
258
|
+
path = report(c, "returns", returnContracts=[contract])["paths"][0]
|
|
259
|
+
consumers = [row for flow in path["returnFlows"]["results"] for row in flow["consumers"] if row["kind"] == "branch"]
|
|
260
|
+
self.assertTrue(consumers)
|
|
261
|
+
# returnFlows keeps the value it shipped with for LOOP; the loop record names the counter.
|
|
262
|
+
self.assertEqual({row["predicateDomain"] for row in consumers}, {"flags/equality"})
|
|
263
|
+
self.assertEqual({g["predicateDomain"] for i in path["loops"]["iterations"] for g in i["gates"]}, {"counter"})
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
if __name__ == "__main__":
|
|
267
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_effect_order.py
RENAMED
|
File without changes
|
{scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_memory_scopes.py
RENAMED
|
File without changes
|
{scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_nested_frame_request.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_relational_controls.py
RENAMED
|
File without changes
|
{scientific_method_engine-3.1.0 → scientific_method_engine-3.2.0}/tests/test_table_continuations.py
RENAMED
|
File without changes
|
|
File without changes
|