scientific-method-engine 3.0.0__tar.gz → 3.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/PKG-INFO +15 -1
  2. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/README.md +14 -0
  3. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/pyproject.toml +1 -1
  4. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/cli.py +3 -0
  5. scientific_method_engine-3.2.0/src/scientific_method_engine/x86/loops.py +317 -0
  6. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/machine.py +109 -2
  7. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/memory_scopes.py +11 -2
  8. scientific_method_engine-3.2.0/src/scientific_method_engine/x86/relational.py +745 -0
  9. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/reports.py +21 -4
  10. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/result_flow.py +15 -2
  11. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/trace.py +16 -5
  12. scientific_method_engine-3.2.0/tests/test_loops.py +267 -0
  13. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/test_memory_scopes.py +5 -3
  14. scientific_method_engine-3.2.0/tests/test_relational_controls.py +561 -0
  15. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/.gitignore +0 -0
  16. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/LICENSE +0 -0
  17. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/__init__.py +0 -0
  18. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/__main__.py +0 -0
  19. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  20. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  21. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  22. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  23. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  24. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  25. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  26. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  27. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  28. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  29. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  30. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  31. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  32. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  33. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  34. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  35. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  36. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  37. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  38. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  39. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  40. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  41. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  42. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  43. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  44. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  45. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  46. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  47. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  48. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  49. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  50. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  51. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  52. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  53. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/image.py +0 -0
  54. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  55. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  56. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/pe.py +0 -0
  57. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/src/scientific_method_engine/x86/values.py +0 -0
  58. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/oracle.py +0 -0
  59. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/test_dispatch.py +0 -0
  60. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/test_effect_order.py +0 -0
  61. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/test_nested_frame_request.py +0 -0
  62. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/test_oracle.py +0 -0
  63. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/test_pe.py +0 -0
  64. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/test_table_continuations.py +0 -0
  65. {scientific_method_engine-3.0.0 → scientific_method_engine-3.2.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 3.0.0
3
+ Version: 3.2.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -125,6 +125,12 @@ pre-call write prefixes and bounded local restoration witnesses. Unknown modeled
125
125
  or nested service effects stay separate; no return code establishes rollback or
126
126
  transactionality. See [the full contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#ordered-effect-path-summaries).
127
127
 
128
+ Each traced path carries a `loops` record: the restart edges the path took and, between
129
+ consecutive arrivals at a loop head, which registers, flags and written bytes changed, the
130
+ signedness and operands of each gate, and whether the state repeats an earlier arrival. It never
131
+ reports that a loop terminates or that a retry succeeded. `loopIterationLimit` caps the records.
132
+ See [loop progress](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#loop-restart-edges-and-iteration-changes).
133
+
128
134
  Evidenced segmented16 `indirectJumps` declarations also expose separate
129
135
  `declaredContinuationPaths`. Ordinary paths retain their unresolved transfer;
130
136
  conditional routes preserve prefix/child effects with target-choice, live-table
@@ -142,3 +148,11 @@ survive the model's memory invalidation, and each scope is reported with the mod
142
148
  return and the effect summary. Everything else the service may do stays unknown. This input needs
143
149
  prepared protocol 3, so upgrade the reader and the engine together. See the
144
150
  [call model contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#limits-and-assumptions).
151
+
152
+ The trace-family commands (`trace`, `arguments`, `effects`, `returns`, `guards`, `memory`,
153
+ `allocation`) check `relationalControls`: assertions over reported values (reach, order, last
154
+ writer, containment, value relation and value origin) evaluated on every bounded path. A violated
155
+ control fails the report. A control left undecided by a stop, a limit or an unread call is
156
+ reported as `undecided` and never counts as held. Access reports also give each byte's
157
+ `writeOrder`, or why it has no modeled value. See the
158
+ [controls contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#relational-controls).
@@ -109,6 +109,12 @@ pre-call write prefixes and bounded local restoration witnesses. Unknown modeled
109
109
  or nested service effects stay separate; no return code establishes rollback or
110
110
  transactionality. See [the full contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#ordered-effect-path-summaries).
111
111
 
112
+ Each traced path carries a `loops` record: the restart edges the path took and, between
113
+ consecutive arrivals at a loop head, which registers, flags and written bytes changed, the
114
+ signedness and operands of each gate, and whether the state repeats an earlier arrival. It never
115
+ reports that a loop terminates or that a retry succeeded. `loopIterationLimit` caps the records.
116
+ See [loop progress](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#loop-restart-edges-and-iteration-changes).
117
+
112
118
  Evidenced segmented16 `indirectJumps` declarations also expose separate
113
119
  `declaredContinuationPaths`. Ordinary paths retain their unresolved transfer;
114
120
  conditional routes preserve prefix/child effects with target-choice, live-table
@@ -126,3 +132,11 @@ survive the model's memory invalidation, and each scope is reported with the mod
126
132
  return and the effect summary. Everything else the service may do stays unknown. This input needs
127
133
  prepared protocol 3, so upgrade the reader and the engine together. See the
128
134
  [call model contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#limits-and-assumptions).
135
+
136
+ The trace-family commands (`trace`, `arguments`, `effects`, `returns`, `guards`, `memory`,
137
+ `allocation`) check `relationalControls`: assertions over reported values (reach, order, last
138
+ writer, containment, value relation and value origin) evaluated on every bounded path. A violated
139
+ control fails the report. A control left undecided by a stop, a limit or an unread call is
140
+ reported as `undecided` and never counts as held. Access reports also give each byte's
141
+ `writeOrder`, or why it has no modeled value. See the
142
+ [controls contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#relational-controls).
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "3.0.0"
8
+ version = "3.2.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -17,9 +17,12 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
17
17
  "effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
18
18
  "effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
19
19
  "callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges.\n"
20
+ "trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
21
+ "a violated control fails the report; an undecided one is reported and never counts as held.\n"
20
22
  "Declared table continuations are separate conditional paths; ordinary computed transfers remain stopped.\n"
21
23
  "Port accesses and interrupts are hardware-boundary events; portInputs supplies port reads as assumptions.\n"
22
24
  "callModels[].preservesMemory keeps explicit, bounded pre-call byte scopes across a modeled call; other memory stays unknown.\n"
25
+ "Each traced path's loops record lists restart edges and what changed between iterations (loopIterationLimit).\n"
23
26
  " scientific-method-engine ghidra-scripts")
24
27
 
25
28
 
@@ -0,0 +1,317 @@
1
+ """Restart edges and what changed between consecutive traced iterations of a loop on one path.
2
+
3
+ A restart edge is a transfer to its own site's address or below that returns to an instruction the
4
+ same call activation already ran. Its target is the loop head. Edges are read off the path with no
5
+ control-flow graph; the address order keeps a forward branch to a join an earlier iteration ran,
6
+ such as an if/else join in the body, from heading a loop. Each arrival at a head is compared with
7
+ the previous arrival at that head: registers, flags, the bytes of modeled memory written in between, and the operands of every
8
+ branch the iteration evaluated in the loop's frame. These are facts about the traced iterations of
9
+ one path. They never establish that a loop terminates, stays bounded or that an iteration made the
10
+ progress its author intended.
11
+ """
12
+ from collections import namedtuple
13
+ from .machine import MEMORY_CLEARED, REGISTERS
14
+ from .result_flow import predicate_domain
15
+ from .values import Value, extract
16
+
17
+ INTERPRETATION = ("restart edges and changes between consecutive traced iterations of this path only; "
18
+ "no termination, bound, progress or native reachability claim")
19
+ GATE_OPERANDS = ("left", "right", "count", "carry")
20
+
21
+ # One arrival at an instruction: its number among the arrivals of its activation, the event count and
22
+ # write-log length then, and the flag state, which the write log does not record.
23
+ Arrival = namedtuple("Arrival", "arrival event log flags")
24
+
25
+
26
+ def relation(before, after):
27
+ """Compare two expressions: identical terms, two different known numbers, or undecided."""
28
+ if before == after:
29
+ return "unchanged"
30
+ if before is not None and after is not None and before[0] == "constant" and after[0] == "constant":
31
+ return "changed"
32
+ return "differentExpression"
33
+
34
+
35
+ def _value(term):
36
+ if term is None:
37
+ return None
38
+ return {"expression": term, "value": term[1] if term[0] == "constant" else None}
39
+
40
+
41
+ def _views(flat, root, term):
42
+ """The register names compared for one root: the full register in the flat model; in the
43
+ segmented model its 16-bit view, plus the full register keyed by its upper bytes.
44
+
45
+ A 16-bit write leaves the upper half as a join of byte extracts, so the upper half is compared
46
+ byte by byte; comparing it as one 16-bit extract would report an untouched half as changed.
47
+ """
48
+ if flat or len(root) == 2:
49
+ return [(root, term)]
50
+ value = Value(32, term)
51
+ return [(root[1:], extract(value, 0, 16).term), (root, tuple(extract(value, n, 8).term for n in (16, 24)))]
52
+
53
+
54
+ def compare_registers(flat, before, after):
55
+ """Sort registers into unchanged and changed between two {root: term} maps."""
56
+ rows = {"unchanged": [], "changed": []}
57
+ for root in REGISTERS:
58
+ old, new = before[root], after[root]
59
+ views = list(zip(_views(flat, root, old), _views(flat, root, new)))
60
+ for index, ((name, a), (_, b)) in enumerate(views):
61
+ if index == 1:
62
+ # The upper half is listed only when it differs; the root then reports its full value.
63
+ if a == b:
64
+ continue
65
+ a, b = old, new
66
+ outcome = relation(a, b)
67
+ if outcome == "unchanged":
68
+ rows["unchanged"].append(name)
69
+ else:
70
+ rows["changed"].append({"register": name, "relation": outcome, "before": _value(a), "after": _value(b)})
71
+ return rows
72
+
73
+
74
+ def flag_state(state):
75
+ """The modeled flags as comparable terms.
76
+
77
+ A comparable producer names the flags by its operation and operands. Otherwise the arithmetic
78
+ flags p-code computed are compared by value. Only flags the model forgot are named by their
79
+ epoch, because they are distinct unknowns.
80
+ """
81
+ if state.flags is not None:
82
+ arithmetic = ("producer", state.flags[2], state.flags[0].term, state.flags[1].term)
83
+ elif state.flag_values is not None:
84
+ arithmetic = ("values",) + tuple(sorted((name, value.term) for name, value in state.flag_values.items()))
85
+ else:
86
+ arithmetic = ("unknown", state.flag_epoch)
87
+ return (arithmetic, None if state.carry is None else state.carry.term,
88
+ state.direction_flag.term, state.interrupt_flag.term)
89
+
90
+
91
+ def _current(state, key):
92
+ """A logged key's term now: a register's, a memory byte's or None. A memory-cleared marker has
93
+ no current term, so a state from before the clear never matches."""
94
+ if key[0] == "register":
95
+ return state.regs[key[1]].term
96
+ value = state.memory.get(key) if len(key) == 3 else None
97
+ return None if value is None else value.term
98
+
99
+
100
+ def _since(state, start):
101
+ """For each key the write log changed since ``start``: its term at ``start``, first entry first."""
102
+ before = {}
103
+ for key, old in state.write_log.since(start):
104
+ if key not in before:
105
+ before[key] = None if old is None else old.term
106
+ return before
107
+
108
+
109
+ def memory_changes(before, state):
110
+ """The memory bytes among ``before``: (term at the earlier arrival, term now). None is a byte the
111
+ model holds no value for."""
112
+ return {key: (term, _current(state, key)) for key, term in before.items() if len(key) == 3}
113
+
114
+
115
+ def registers_at(before, state):
116
+ """Each register root's term at the earlier arrival, rebuilt from its first later log entry."""
117
+ return {root: before.get(("register", root), state.regs[root].term) for root in REGISTERS}
118
+
119
+
120
+ def _intervals(changes):
121
+ rows = []
122
+ for key in sorted(changes, key=lambda k: (repr(k[0]), repr(k[1]), k[2])):
123
+ before, after = changes[key]
124
+ status = ("invalidated" if after is None else "writtenOverUnmodeled" if before is None
125
+ else relation(before, after))
126
+ segment, base, offset = key
127
+ last = rows[-1] if rows else None
128
+ if (last and last["segment"] == segment and last["base"] == base and last["end"] == offset
129
+ and last["status"] == status):
130
+ last["end"] = offset + 1
131
+ else:
132
+ rows.append({"segment": segment, "base": base, "start": offset, "end": offset + 1, "status": status})
133
+ return rows
134
+
135
+
136
+ def repeated_arrival(state, history, token):
137
+ """The earliest arrival in ``history`` whose modeled state equals the current one, or None.
138
+
139
+ One backward pass over the write log serves every candidate: walking from the newest entry, a
140
+ key's value at an earlier point is the previous value of its oldest entry after that point, so
141
+ the set of keys that differ from now is kept up to date as the walk passes each candidate. A
142
+ byte the model holds no value for at either point may hold different contents, so it never
143
+ matches.
144
+ """
145
+ candidates = sorted((t for t in history if t.flags == token.flags),
146
+ key=lambda t: (t.log, t.arrival), reverse=True)
147
+ if not candidates:
148
+ return None
149
+ oldest = candidates[-1].log
150
+ entries = list(state.write_log.since(oldest))
151
+ position = len(entries)
152
+ differing = set()
153
+ earliest = None
154
+ for candidate in candidates:
155
+ while oldest + position > candidate.log:
156
+ position -= 1
157
+ key, old = entries[position]
158
+ now = _current(state, key)
159
+ if old is None or now is None or old.term != now:
160
+ differing.add(key)
161
+ else:
162
+ differing.discard(key)
163
+ if not differing:
164
+ earliest = candidate.arrival if earliest is None else min(earliest, candidate.arrival)
165
+ return earliest
166
+
167
+
168
+ def _gate(event):
169
+ gate = {"site": event["site"], "order": event["order"], "predicate": event["predicate"],
170
+ "predicateDomain": predicate_domain(event["predicate"]), "taken": event["taken"],
171
+ "operands": {k: _value(event[k]["expression"]) for k in GATE_OPERANDS if isinstance(event.get(k), dict)}}
172
+ # LOOPE and LOOPNE also test ZF; their flag test's record is kept beside the counter.
173
+ for k in ("operation", "flagProducer", "decidedBy", "reason", "zeroFlag"):
174
+ if k in event:
175
+ gate[k] = event[k]
176
+ return gate
177
+
178
+
179
+ def _compare_gates(previous, gates):
180
+ """Mark each gate against the gate at the same position of the previous iteration."""
181
+ if previous is None or not (previous or gates):
182
+ # A first iteration, or two iterations with no gate in the loop's frame: nothing to compare.
183
+ return None
184
+ same_shape = [g["site"] for g in previous] == [g["site"] for g in gates]
185
+ outcomes = []
186
+ for index, gate in enumerate(gates):
187
+ if not same_shape:
188
+ gate["operandsSincePreviousIteration"] = "gate sequence differs"
189
+ continue
190
+ before, after = previous[index]["operands"], gate["operands"]
191
+ if not after or set(before) != set(after):
192
+ gate["operandsSincePreviousIteration"] = "unresolved"
193
+ outcomes.append(None)
194
+ continue
195
+ relations = {k: relation(before[k]["expression"], after[k]["expression"]) for k in after}
196
+ result = ("changed" if "changed" in relations.values() else
197
+ "differentExpression" if "differentExpression" in relations.values() else "unchanged")
198
+ gate["operandsSincePreviousIteration"] = result
199
+ outcomes.append(result)
200
+ if not same_shape or [g["taken"] for g in previous] != [g["taken"] for g in gates]:
201
+ return False
202
+ if "changed" in outcomes:
203
+ return False
204
+ if None in outcomes or "differentExpression" in outcomes:
205
+ return None
206
+ return True
207
+
208
+
209
+ class LoopTracker:
210
+ """Per-path loop bookkeeping. ``arrive`` runs once before each traced instruction.
211
+
212
+ Each call activation is identified by a number stored in its ``state.frames`` entry the first
213
+ time the tracker sees it, so a call inside a loop body never mistakes the callee's frame for the
214
+ caller's.
215
+ """
216
+
217
+ def __init__(self, limit):
218
+ self.limit = limit
219
+ self.frames = {}
220
+ self.activations = 0
221
+ self.previous = None
222
+ self.edges = {}
223
+ self.iterations = []
224
+ self.omitted = 0
225
+
226
+ def __deepcopy__(self, memo):
227
+ # Arrivals, gate lists and recorded iterations are never changed once stored, so a fork
228
+ # copies only the containers that keep growing.
229
+ copy = LoopTracker.__new__(LoopTracker)
230
+ copy.limit, copy.activations, copy.previous, copy.omitted = self.limit, self.activations, self.previous, self.omitted
231
+ copy.frames = {activation: {"entry": frame["entry"], "tokens": dict(frame["tokens"]),
232
+ "heads": {head: {"history": None if loop["history"] is None else list(loop["history"]),
233
+ "gates": loop["gates"]}
234
+ for head, loop in frame["heads"].items()}}
235
+ for activation, frame in self.frames.items()}
236
+ copy.edges = {key: dict(edge) for key, edge in self.edges.items()}
237
+ copy.iterations = list(self.iterations)
238
+ return copy
239
+
240
+ def _frame(self, state):
241
+ top = state.frames[-1]
242
+ activation = top.get("loopActivation")
243
+ if activation is None:
244
+ self.activations += 1
245
+ activation = top["loopActivation"] = self.activations
246
+ # A new activation: drop the records of activations that returned, since ids never recur.
247
+ live = {f.get("loopActivation") for f in state.frames}
248
+ for gone in [a for a in self.frames if a not in live]:
249
+ del self.frames[gone]
250
+ self.frames[activation] = {"entry": top["entry"], "tokens": {}, "heads": {}}
251
+ return activation, self.frames[activation]
252
+
253
+ def arrive(self, state, at, ins):
254
+ activation, frame = self._frame(state)
255
+ previous, self.previous = self.previous, (at, at + ins.size, activation, ins.mnemonic)
256
+ last = frame["tokens"].get(at)
257
+ token = Arrival(1 if last is None else last.arrival + 1, len(state.events), len(state.write_log),
258
+ flag_state(state))
259
+ # Only a transfer to the site's own address or below closes a loop. Every cycle on a path has
260
+ # one, since fall-throughs run forward, and a forward branch to a join an earlier iteration
261
+ # ran stays an ordinary arrival.
262
+ if (last is not None and previous is not None and previous[2] == activation and at != previous[1]
263
+ and at <= previous[0]):
264
+ self._restart(state, frame, previous, at, last, token)
265
+ else:
266
+ loop = frame["heads"].get(at)
267
+ if loop is not None and loop["history"] is not None:
268
+ # An arrival at a known head that is not a restart, such as an inner loop's entry on
269
+ # the next outer iteration, is an earlier state later iterations may repeat. It starts a
270
+ # new entry into the loop, so the next iteration's gates are not compared with those
271
+ # of the last iteration before the loop was left.
272
+ loop["history"].append(token)
273
+ loop["gates"] = None
274
+ frame["tokens"][at] = token
275
+
276
+ def _restart(self, state, frame, previous, head, start, token):
277
+ site, _, activation, kind = previous
278
+ depth = len(state.frames) - 1
279
+ key = (activation, site, head)
280
+ edge = self.edges.get(key)
281
+ if edge is None:
282
+ edge = self.edges[key] = {"entry": frame["entry"], "depth": depth, "activation": activation,
283
+ "site": site, "target": head, "kind": kind, "traversals": 0,
284
+ "firstOrder": len(state.events)}
285
+ edge["traversals"] += 1
286
+ loop = frame["heads"].setdefault(head, {"history": [start], "gates": None})
287
+ if loop["history"] is None:
288
+ self.omitted += 1
289
+ return
290
+ if len(self.iterations) >= self.limit:
291
+ # Past the limit the history is dropped, so no later iteration is compared with a stale one.
292
+ loop["history"] = None
293
+ self.omitted += 1
294
+ return
295
+ # The iteration runs from the previous arrival at the head, however the path got there.
296
+ gates = [_gate(e) for e in state.events[start.event:] if e["kind"] == "branch" and e["depth"] == depth]
297
+ repeated = _compare_gates(loop["gates"], gates)
298
+ loop["gates"] = gates
299
+ before = _since(state, start.log)
300
+ now = {root: state.regs[root].term for root in REGISTERS}
301
+ self.iterations.append({
302
+ "head": head, "entry": frame["entry"], "depth": depth, "activation": activation,
303
+ "fromArrival": start.arrival, "toArrival": token.arrival,
304
+ "restartEdge": {"site": site, "kind": kind},
305
+ "fromOrder": start.event, "toOrder": token.event,
306
+ "registers": compare_registers(state.flat, registers_at(before, state), now),
307
+ "flags": "unchanged" if start.flags == token.flags else "differ",
308
+ "memory": _intervals(memory_changes(before, state)), "memoryForgotten": MEMORY_CLEARED in before,
309
+ "gates": gates,
310
+ "gateOperandsRepeated": repeated, "stateRepeatsArrival": repeated_arrival(state, loop["history"], token)})
311
+ loop["history"].append(token)
312
+
313
+ def report(self):
314
+ """The ``loops`` field of a traced path."""
315
+ return {"restartEdges": list(self.edges.values()), "iterations": self.iterations,
316
+ "iterationLimit": self.limit, "iterationsOmitted": self.omitted,
317
+ "allIterationsRecorded": not self.omitted, "interpretation": INTERPRETATION}
@@ -18,6 +18,53 @@ class StopPath(Exception):
18
18
  pass
19
19
 
20
20
 
21
+ # The write-log key that marks a point where the model forgot all memory.
22
+ MEMORY_CLEARED = ("memory-cleared",)
23
+
24
+
25
+ class WriteLog:
26
+ """Append-only list of (key, previous value) pairs, one per register or memory byte a write changes.
27
+
28
+ A memory key is (segment, base, offset); a register key is ("register", root). A previous memory
29
+ value is None when the model held no value for the byte. ``(MEMORY_CLEARED, None)`` marks a
30
+ point where the model forgot all memory. Copies made at a fork share the entries
31
+ recorded before the fork, which are immutable, so forking a path never copies its log.
32
+ """
33
+
34
+ def __init__(self):
35
+ self.chunks = [] # (absolute index of the first entry, tuple of entries), oldest first
36
+ self.tail = []
37
+ self.length = 0
38
+
39
+ def __len__(self):
40
+ return self.length
41
+
42
+ def append(self, entry):
43
+ self.tail.append(entry)
44
+ self.length += 1
45
+
46
+ def extend(self, entries):
47
+ for entry in entries:
48
+ self.append(entry)
49
+
50
+ def since(self, start):
51
+ """The entries at absolute index ``start`` and later, oldest first."""
52
+ for first, chunk in self.chunks:
53
+ if first + len(chunk) > start:
54
+ yield from chunk[max(0, start - first):]
55
+ first = self.length - len(self.tail)
56
+ yield from self.tail[max(0, start - first):]
57
+
58
+ def __deepcopy__(self, memo):
59
+ if self.tail:
60
+ self.chunks.append((self.length - len(self.tail), tuple(self.tail)))
61
+ self.tail = []
62
+ copy = WriteLog()
63
+ copy.chunks = list(self.chunks)
64
+ copy.length = self.length
65
+ return copy
66
+
67
+
21
68
  class Shared(dict):
22
69
  """Read-only query configuration that every copy of a path shares instead of copying."""
23
70
 
@@ -51,6 +98,9 @@ class State:
51
98
  self.semantics = BACKEND
52
99
  self.sp, self.bp = ("esp", "ebp") if self.flat else ("sp", "bp")
53
100
  self.at = entry
101
+ # Every register and memory byte a write changes, with its previous value, in order, so a
102
+ # reader can recover the state as it stood at an earlier point of the path (x86/loops.py).
103
+ self.write_log = WriteLog()
54
104
  self.regs = {r: unknown("initial:" + r, ALIASES[r][2]) for r in REGISTERS}
55
105
  # Producers per register byte, so a partial write replaces only the bytes it stores.
56
106
  self.reg_sources = {r: [()] * (ALIASES[r][2] // 8) for r in REGISTERS}
@@ -67,6 +117,17 @@ class State:
67
117
  # Keys grouped by (segment, base) so a write scans only groups that can alias it.
68
118
  self.memory_groups = {}
69
119
  self.memory_epoch = 0
120
+ # The order of the write event that stored each modeled byte. lost_memory holds the
121
+ # byteProducers "unwritten" row of bytes with no value that an ordinary write explains (a
122
+ # possibly aliasing write dropped them) or that a preservesMemory scope kept without a
123
+ # value. memory_cleared is the order of the event that dropped every byte (a modeled call),
124
+ # or None. writes maps each alias group to {domain: order of the last write with it} since
125
+ # that event, so a read of a byte this path never stored can name a write that may have
126
+ # stored it without scanning the writes of its own group.
127
+ self.memory_writers = {}
128
+ self.lost_memory = {}
129
+ self.memory_cleared = None
130
+ self.writes = {}
70
131
  # Bytes a preservesMemory scope kept without a value (ADR 0009): key -> the unknown term
71
132
  # name they had before the modeled call. They stay unread: no value, no producer.
72
133
  self.unread_memory = {}
@@ -158,9 +219,17 @@ class State:
158
219
  self.flag_values = None
159
220
 
160
221
  def clear_memory(self):
222
+ # Bytes the model never held may change too, so the log marks the clear itself.
223
+ self.write_log.extend(self.memory.items())
224
+ self.write_log.append((MEMORY_CLEARED, None))
161
225
  self.memory.clear()
162
226
  self.memory_groups.clear()
163
227
  self.unread_memory.clear()
228
+ self.memory_writers.clear()
229
+ self.lost_memory.clear()
230
+ self.writes.clear()
231
+ # The caller records the event that explains the loss right after clearing.
232
+ self.memory_cleared = len(self.events)
164
233
  self.memory_epoch += 1
165
234
 
166
235
  def unread_term(self, key):
@@ -171,6 +240,37 @@ class State:
171
240
  """The modeled value of one memory byte, or an unknown term produced by the current site."""
172
241
  return self.memory[key] if key in self.memory else unknown(self.unread_term(key), 8, self.at)
173
242
 
243
+ def unwritten(self, key):
244
+ """Why a byte has no modeled value: the cause and the order of the event behind it."""
245
+ # Writes to the key's own (segment, base) group store other offsets; any other write may alias.
246
+ # A later aliasing write is newer than the one that dropped the byte, so it is the one named.
247
+ lost = self.lost_memory.get(key)
248
+ group, a = key[:2], key_domain(key, self.bits, self.flat)
249
+ latest = None
250
+ for other, domains in self.writes.items():
251
+ if other == group:
252
+ continue
253
+ if a is None:
254
+ # Every write may alias a byte with no concrete domain; the group's newest decides.
255
+ order = max(domains.values())
256
+ else:
257
+ order = max((o for b, o in domains.items() if b is None or not (a[1] <= b[0] or b[1] <= a[0])), default=None)
258
+ if order is not None and (latest is None or order > latest):
259
+ latest = order
260
+ if latest is not None and (lost is None or lost["order"] is None or latest > lost["order"]):
261
+ return {"cause": "possibly written by an aliasing write", "order": latest}
262
+ if lost is not None:
263
+ return dict(lost)
264
+ if self.memory_cleared is not None:
265
+ return {"cause": "dropped by a modeled call", "order": self.memory_cleared}
266
+ return {"cause": "no write on this path", "order": None}
267
+
268
+ def byte_writer(self, index, key):
269
+ """The byteProducers row of one accessed byte: its producers and the write that stored it."""
270
+ if key in self.memory:
271
+ return {"index": index, "producers": producers(self.memory[key]), "writeOrder": self.memory_writers.get(key)}
272
+ return {"index": index, "producers": [], "writeOrder": None, "unwritten": self.unwritten(key)}
273
+
174
274
  def reg(self, name):
175
275
  root, low, bits = alias(name)
176
276
  value = extract(self.regs[root], low, bits)
@@ -180,6 +280,7 @@ class State:
180
280
  root, low, bits = alias(name)
181
281
  value = resize(value, bits)
182
282
  value = Value(bits, value.term, sources(value, site=site))
283
+ self.write_log.append((("register", root), self.regs[root]))
183
284
  if bits == ALIASES[root][2]:
184
285
  self.regs[root] = value
185
286
  else:
@@ -225,20 +326,26 @@ class State:
225
326
  self.memory_epoch += 1
226
327
 
227
328
  written = written_domain(seg, base, delta, width, self.bits, self.flat)
329
+ self.writes.setdefault((seg, base), {})[written] = len(self.events)
228
330
  for group, members in list(self.memory_groups.items()):
229
331
  if group == (seg, base):
230
332
  continue
231
333
  for key in list(members):
232
334
  if may_alias(key, written, self.bits, self.flat):
233
335
  uncertain.append(key)
234
- self.memory.pop(key, None)
336
+ self.write_log.append((key, self.memory.pop(key, None)))
235
337
  self.unread_memory.pop(key, None)
338
+ self.memory_writers.pop(key, None)
339
+ self.lost_memory[key] = {"cause": "dropped by a possibly aliasing write", "order": len(self.events)}
236
340
  members.discard(key)
237
341
  if not members:
238
342
  del self.memory_groups[group]
239
343
  for i, key in enumerate(keys):
344
+ self.write_log.append((key, self.memory.get(key)))
240
345
  self.memory[key] = extract(write, i * 8, 8)
241
346
  self.unread_memory.pop(key, None)
347
+ self.memory_writers[key] = len(self.events)
348
+ self.lost_memory.pop(key, None)
242
349
  self.memory_groups.setdefault((seg, base), set()).update(keys)
243
350
  value = write
244
351
  missing = []
@@ -257,7 +364,7 @@ class State:
257
364
  event = self.event("write" if write is not None else "read", segment=segment.report(), offset=offset.report(),
258
365
  width=width, effectiveSegmentRegister=addressing_register, segmentInterpretation="base" if self.flat else "selector-paragraph", interval={"segment": seg, "base": base, "start": delta, "end": delta + width},
259
366
  value=value.report(), missingByteProducers=missing,
260
- byteProducers=[{"index": i, "producers": producers(self.memory[key]) if key in self.memory else []} for i, key in enumerate(keys)],
367
+ byteProducers=[self.byte_writer(i, key) for i, key in enumerate(keys)],
261
368
  guards=deepcopy(relevant), role=role,
262
369
  uncertainAliasesInvalidated=len(uncertain))
263
370
  f = self.frames[-1]
@@ -95,12 +95,21 @@ def capture_scopes(state, model):
95
95
  return values, unread, descriptions
96
96
 
97
97
 
98
- def retain_scopes(state, values, unread):
98
+ def scope_history(state, values, unread):
99
+ """Snapshot, before the call, the write that stored each captured byte and why each uncached one has no value."""
100
+ return ({key: state.memory_writers[key] for key in values if key in state.memory_writers},
101
+ {key: state.unwritten(key) for key in unread})
102
+
103
+
104
+ def retain_scopes(state, values, unread, history):
99
105
  """Put the captured bytes back after the model invalidated memory. Later writes still apply.
100
106
 
101
- All scoped keys are concrete linear bytes, so they share one alias group.
107
+ All scoped keys are concrete linear bytes, so they share one alias group. ``history`` from
108
+ ``scope_history`` keeps each byte's reported writer, or its reason for having no value.
102
109
  """
103
110
  state.memory.update(values)
104
111
  state.unread_memory.update(unread)
112
+ state.memory_writers.update(history[0])
113
+ state.lost_memory.update(history[1])
105
114
  if values or unread:
106
115
  state.memory_groups.setdefault((("linear",), ("absolute",)), set()).update(values, unread)