scientific-method-engine 7.1.0__tar.gz → 7.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/PKG-INFO +8 -6
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/README.md +7 -5
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/pyproject.toml +1 -1
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/cli.py +3 -1
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +9 -4
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/argument_frames.py +57 -57
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/machine.py +50 -9
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/memory_scopes.py +38 -21
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/reports.py +87 -75
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/trace.py +86 -44
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_memory_scopes.py +149 -3
- scientific_method_engine-7.3.0/tests/test_successor_rule.py +103 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_x86.py +345 -16
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/.gitignore +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/LICENSE +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/oracle.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-7.1.0 → scientific_method_engine-7.3.0}/tests/test_table_continuations.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 7.
|
|
3
|
+
Version: 7.3.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -92,7 +92,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
|
|
|
92
92
|
|---|---|---|
|
|
93
93
|
| `ExportBoundedFlow` | entry, instruction limit (1..10000), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON |
|
|
94
94
|
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
|
|
95
|
-
| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override)
|
|
95
|
+
| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override), `fallsThrough` (whether Ghidra continues to the next instruction at the site, after any fall-through override) and `fallsThroughTo`/`fallsThroughToAddress` (where a fall-through override sends Ghidra instead, or null), as the `ghidraCallEdges` input of `callees` |
|
|
96
96
|
| `ExportFunctionFingerprints` | output TSV path (replaced only when the export completes) | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
|
|
97
97
|
|
|
98
98
|
Repairing the analysis (these change the Ghidra program, so run them before reports and keep the
|
|
@@ -148,10 +148,12 @@ partial evidence, never complete dispatch or native-reachability claims. See the
|
|
|
148
148
|
A call model may declare `preservesMemory`: up to 32 scopes (4,096 bytes in total), each naming a
|
|
149
149
|
segment register, an address-width base register, an optional displacement, a byte count and the
|
|
150
150
|
evidence for assuming the service leaves those bytes alone. Scopes resolve against the pre-call
|
|
151
|
-
registers.
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
registers. The base may be symbolic, such as BP in an `entryFrame` query, and then keeps the bytes
|
|
152
|
+
at those offsets from its value. An unknown segment, a concrete interval that crosses the end of
|
|
153
|
+
the address space, or two scopes that may share a byte stop the path. Only the scoped bytes survive
|
|
154
|
+
the model's memory invalidation, and each scope is reported with the model, the modeled return and
|
|
155
|
+
the effect summary. Everything else the service may do stays unknown. This input needs prepared
|
|
156
|
+
protocol 3, so upgrade the reader and the engine together. See the
|
|
155
157
|
[call model contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#limits-and-assumptions).
|
|
156
158
|
|
|
157
159
|
The trace-family commands (`trace`, `arguments`, `effects`, `returns`, `guards`, `memory`,
|
|
@@ -76,7 +76,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
|
|
|
76
76
|
|---|---|---|
|
|
77
77
|
| `ExportBoundedFlow` | entry, instruction limit (1..10000), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON |
|
|
78
78
|
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
|
|
79
|
-
| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override)
|
|
79
|
+
| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override), `fallsThrough` (whether Ghidra continues to the next instruction at the site, after any fall-through override) and `fallsThroughTo`/`fallsThroughToAddress` (where a fall-through override sends Ghidra instead, or null), as the `ghidraCallEdges` input of `callees` |
|
|
80
80
|
| `ExportFunctionFingerprints` | output TSV path (replaced only when the export completes) | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
|
|
81
81
|
|
|
82
82
|
Repairing the analysis (these change the Ghidra program, so run them before reports and keep the
|
|
@@ -132,10 +132,12 @@ partial evidence, never complete dispatch or native-reachability claims. See the
|
|
|
132
132
|
A call model may declare `preservesMemory`: up to 32 scopes (4,096 bytes in total), each naming a
|
|
133
133
|
segment register, an address-width base register, an optional displacement, a byte count and the
|
|
134
134
|
evidence for assuming the service leaves those bytes alone. Scopes resolve against the pre-call
|
|
135
|
-
registers.
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
135
|
+
registers. The base may be symbolic, such as BP in an `entryFrame` query, and then keeps the bytes
|
|
136
|
+
at those offsets from its value. An unknown segment, a concrete interval that crosses the end of
|
|
137
|
+
the address space, or two scopes that may share a byte stop the path. Only the scoped bytes survive
|
|
138
|
+
the model's memory invalidation, and each scope is reported with the model, the modeled return and
|
|
139
|
+
the effect summary. Everything else the service may do stays unknown. This input needs prepared
|
|
140
|
+
protocol 3, so upgrade the reader and the engine together. See the
|
|
139
141
|
[call model contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#limits-and-assumptions).
|
|
140
142
|
|
|
141
143
|
The trace-family commands (`trace`, `arguments`, `effects`, `returns`, `guards`, `memory`,
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "7.
|
|
8
|
+
version = "7.3.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -18,7 +18,8 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
18
18
|
"effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
|
|
19
19
|
"callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges;\n"
|
|
20
20
|
"agreed is false where Ghidra ends the function at a call, conditional jump or interrupt the engine reads past (ghidraEndsFunction),\n"
|
|
21
|
-
"
|
|
21
|
+
"continues past a jmp, ljmp, return or hlt the engine stops at (ghidraContinues),\n"
|
|
22
|
+
"or continues at another address than the next instruction (ghidraFallsThroughElsewhere).\n"
|
|
22
23
|
"trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
|
|
23
24
|
"a violated control fails the report; an undecided one is reported and never counts as held.\n"
|
|
24
25
|
"entryFrame {from} starts an entry inside the function at from, in the frame a trace from there observed.\n"
|
|
@@ -26,6 +27,7 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
26
27
|
"(paths, totalSteps, maxSteps, visitLimit, stringIterations); ordinary computed transfers remain stopped.\n"
|
|
27
28
|
"Port accesses and interrupts are hardware-boundary events; portInputs supplies port reads as assumptions.\n"
|
|
28
29
|
"callModels[].preservesMemory keeps explicit, bounded pre-call byte scopes across a modeled call; other memory stays unknown.\n"
|
|
30
|
+
"A scope needs a concrete segment; its base may be symbolic, such as BP in an entryFrame query.\n"
|
|
29
31
|
"Each traced path's loops record lists restart edges and what changed between iterations (loopIterationLimit).\n"
|
|
30
32
|
" scientific-method-engine ghidra-scripts")
|
|
31
33
|
|
|
@@ -106,17 +106,22 @@ public class ExportCallEdges extends GhidraScript {
|
|
|
106
106
|
|
|
107
107
|
// The flow is Ghidra's flow type after any flow override. fallsThrough is true when Ghidra continues to
|
|
108
108
|
// the next instruction at the site. getFallThrough() reflects a fall-through override, which can clear it
|
|
109
|
-
// or send it to another address; hasFallthrough() stays true for the second, so it is not used.
|
|
109
|
+
// or send it to another address; hasFallthrough() stays true for the second, so it is not used. When the
|
|
110
|
+
// override sends it to another address, fallsThroughTo and fallsThroughToAddress name that address and
|
|
111
|
+
// fallsThrough is false; otherwise both are null.
|
|
110
112
|
private String edge(Instruction instruction, Address target, FlowType flow) {
|
|
111
113
|
Address fallThrough = instruction.getFallThrough();
|
|
112
|
-
|
|
113
|
-
|
|
114
|
+
Address next = instruction.getAddress().addWrap(instruction.getDefaultFallThroughOffset());
|
|
115
|
+
boolean fallsThrough = fallThrough != null && fallThrough.equals(next);
|
|
116
|
+
Address elsewhere = fallThrough == null || fallsThrough ? null : fallThrough;
|
|
114
117
|
return "{\"site\": " + offset(instruction.getAddress())
|
|
115
118
|
+ ", \"siteAddress\": " + quote(instruction.getAddress().toString())
|
|
116
119
|
+ ", \"target\": " + (target == null ? "null" : offset(target))
|
|
117
120
|
+ ", \"targetAddress\": " + (target == null ? "null" : quote(target.toString()))
|
|
118
121
|
+ ", \"flow\": " + quote(flow.toString())
|
|
119
|
-
+ ", \"fallsThrough\": " + fallsThrough
|
|
122
|
+
+ ", \"fallsThrough\": " + fallsThrough
|
|
123
|
+
+ ", \"fallsThroughTo\": " + (elsewhere == null ? "null" : offset(elsewhere))
|
|
124
|
+
+ ", \"fallsThroughToAddress\": " + (elsewhere == null ? "null" : quote(elsewhere.toString())) + "}";
|
|
120
125
|
}
|
|
121
126
|
|
|
122
127
|
// The file offset the engine uses for an address, or null when the address has no file bytes.
|
|
@@ -7,9 +7,6 @@ amount never join or split slots. Whatever a read did not settle on the path sta
|
|
|
7
7
|
"""
|
|
8
8
|
from capstone.x86 import X86_OP_IMM, X86_OP_REG
|
|
9
9
|
|
|
10
|
-
from .machine import may_alias, written_domain
|
|
11
|
-
from .memory_scopes import model_scopes
|
|
12
|
-
|
|
13
10
|
# The argument bytes one frame maps; a wider frame is mapped up to here and stays open.
|
|
14
11
|
WINDOW_BYTES = 256
|
|
15
12
|
|
|
@@ -39,60 +36,43 @@ def _covered(event, segment, base, start, width, modulus):
|
|
|
39
36
|
return [at for at in offsets if 0 <= at < width]
|
|
40
37
|
|
|
41
38
|
|
|
42
|
-
def
|
|
43
|
-
"""The
|
|
39
|
+
def _slot_states(events, recorded, width):
|
|
40
|
+
"""The write that last stored each frame byte when the call ran, and why each byte without one has none.
|
|
44
41
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
or not it dropped a cached byte. The rule is the one the machine's ``unwritten`` applies, so a
|
|
49
|
-
slot and a callee read of it that runs before any callee write through another segment or base
|
|
50
|
-
name the same write.
|
|
42
|
+
`recorded` is the call event's ``argumentSlots``, from ``State.argument_slots``: the machine's own
|
|
43
|
+
record of each byte, so a slot and a callee read of it that runs before any callee write through
|
|
44
|
+
another segment or base name the same write.
|
|
51
45
|
"""
|
|
52
|
-
writers,
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
event = events[j]
|
|
56
|
-
interval = event.get("interval")
|
|
57
|
-
if event["kind"] == "write" and interval and (interval["segment"], interval["base"]) != (segment, base):
|
|
58
|
-
written = written_domain(interval["segment"], interval["base"], interval["start"],
|
|
59
|
-
interval["end"] - interval["start"], image.bits, image.flat)
|
|
60
|
-
for at, key in enumerate(keys):
|
|
61
|
-
if at not in writers and at not in invalidated and may_alias(key, written, image.bits, image.flat):
|
|
62
|
-
invalidated[at] = f"memory possibly overwritten through another address by the write at {event['site']}"
|
|
63
|
-
if event["kind"] == "call-return" and event.get("unknownMemoryEffects"):
|
|
64
|
-
kept = set()
|
|
65
|
-
if segment == ("linear",):
|
|
66
|
-
kept = {at for scope in model_scopes(models, event)
|
|
67
|
-
for at in range(scope["linearStart"], scope["linearEnd"])}
|
|
68
|
-
for at in range(width):
|
|
69
|
-
if at not in writers and at not in invalidated and start + at not in kept:
|
|
70
|
-
invalidated[at] = "memory invalidated by the modeled call at " + str(event["callSite"])
|
|
71
|
-
for at in _covered(event, segment, base, start, width, modulus):
|
|
72
|
-
if at not in invalidated:
|
|
73
|
-
writers.setdefault(at, event)
|
|
74
|
-
if len(writers) + len(invalidated) == width:
|
|
46
|
+
writers, reasons = {}, {}
|
|
47
|
+
for offset, run, order, unwritten in recorded["runs"]:
|
|
48
|
+
if offset >= width:
|
|
75
49
|
break
|
|
76
|
-
|
|
50
|
+
target, value = (writers, events[order]) if unwritten is None else (reasons, _reason(events, unwritten))
|
|
51
|
+
for at in range(offset, min(offset + run, width)):
|
|
52
|
+
target[at] = value
|
|
53
|
+
return writers, reasons
|
|
54
|
+
|
|
77
55
|
|
|
56
|
+
def _reason(events, unwritten):
|
|
57
|
+
"""A slot's reason for a byte's ``unwritten`` cause, naming the event's site."""
|
|
58
|
+
cause, order = unwritten["cause"], unwritten["order"]
|
|
59
|
+
if cause in ("possibly written by an aliasing write", "dropped by a possibly aliasing write"):
|
|
60
|
+
return f"memory possibly overwritten through another address by the write at {events[order]['site']}"
|
|
61
|
+
if cause == "dropped by a modeled call":
|
|
62
|
+
return "memory invalidated by the modeled call at " + str(events[order]["callSite"])
|
|
63
|
+
return cause
|
|
78
64
|
|
|
79
|
-
|
|
65
|
+
|
|
66
|
+
def _frame(image, events, index):
|
|
80
67
|
call = events[index]
|
|
81
68
|
depth = call["depth"]
|
|
82
|
-
|
|
69
|
+
recorded = call.pop("argumentSlots")
|
|
83
70
|
end = next((j for j in range(index + 1, len(events))
|
|
84
71
|
if events[j]["kind"] == "call-return" and events[j].get("callSite") == call["site"] and events[j]["depth"] == depth), None)
|
|
85
72
|
inside = events[index + 1:end if end is not None else len(events)]
|
|
86
73
|
open_reasons = []
|
|
87
|
-
|
|
88
|
-
return None
|
|
89
|
-
# The return-address push comes last and sits at the callee's entry SP; the frame's return bytes lie above it.
|
|
90
|
-
entry = pushes[-1]["interval"]
|
|
91
|
-
segment, base = entry["segment"], entry["base"]
|
|
74
|
+
segment, base, start = recorded["segment"], recorded["base"], recorded["start"]
|
|
92
75
|
modulus = None if segment == ("linear",) else 1 << image.bits
|
|
93
|
-
start = entry["start"] + call["returnFrameBytes"]
|
|
94
|
-
if modulus:
|
|
95
|
-
start %= modulus
|
|
96
76
|
reads = [e for e in inside if e["kind"] == "read" and e.get("argument") and e["depth"] == depth + 1]
|
|
97
77
|
returned = next((e for e in inside if e["kind"] == "return" and e["depth"] == depth + 1 and e.get("callSite") == call["site"]), None)
|
|
98
78
|
callee_cleanup = returned["cleanupBytes"] if returned else None
|
|
@@ -112,7 +92,7 @@ def _frame(image, events, index, models):
|
|
|
112
92
|
|
|
113
93
|
# Slots: runs of argument bytes that one write event last covered.
|
|
114
94
|
slots, by_byte = [], {}
|
|
115
|
-
writers, reasons =
|
|
95
|
+
writers, reasons = _slot_states(events, recorded, width)
|
|
116
96
|
for at in range(width):
|
|
117
97
|
writer = writers.get(at)
|
|
118
98
|
key = writer["order"] if writer else reasons[at]
|
|
@@ -202,36 +182,56 @@ def argument_frames(report, image):
|
|
|
202
182
|
|
|
203
183
|
A path gets one map per traced call, from caller-written slots to callee reads. A site's
|
|
204
184
|
groupings agree only when every path that traced a call there settled on the same read widths
|
|
205
|
-
and groupings. Separately, a site's widths are consistent when its paths
|
|
206
|
-
and no two reads
|
|
207
|
-
a path that skipped a read leaves the site consistent but not agreed.
|
|
185
|
+
and groupings. Separately, a site's widths are consistent when its paths read at least one byte
|
|
186
|
+
the caller wrote and no two reads with a different offset, width or grouping both saw one shared
|
|
187
|
+
byte from the caller; a path that skipped a read leaves the site consistent but not agreed.
|
|
188
|
+
A read that saw bytes the callee stored itself, or bytes with no known caller writer, stays
|
|
189
|
+
listed and marked with the paths it was made on that way, and its conflicts stay listed, but
|
|
190
|
+
those bytes do not decide whether the widths are consistent. The report must come from
|
|
191
|
+
``trace`` with ``argument_window=WINDOW_BYTES``: each traced call's slots come from its
|
|
192
|
+
``argumentSlots`` record, which this removes from every path, declared continuations included.
|
|
208
193
|
"""
|
|
209
194
|
sites = {}
|
|
210
195
|
for path_index, path in enumerate(report["paths"]):
|
|
211
196
|
events = path["events"]
|
|
212
|
-
|
|
213
|
-
path["argumentFrames"] = [f for f in frames if f is not None]
|
|
197
|
+
path["argumentFrames"] = [_frame(image, events, i) for i, e in enumerate(events) if e["kind"] == "call" and "returnFrameBytes" in e]
|
|
214
198
|
for frame in path["argumentFrames"]:
|
|
215
199
|
sites.setdefault(frame["callSite"], []).append((path_index, frame))
|
|
200
|
+
# Continuations get no frames; their calls' slot records are dropped unread.
|
|
201
|
+
for path in report.get("declaredContinuationPaths", ()):
|
|
202
|
+
for event in path["events"]:
|
|
203
|
+
event.pop("argumentSlots", None)
|
|
216
204
|
report["argumentFrameSites"] = []
|
|
217
205
|
for site, rows in sorted(sites.items()):
|
|
218
206
|
# A set holds each read's grouping too: a far-pointer load and a plain dword over the same bytes disagree.
|
|
219
207
|
widths = sorted({tuple(sorted({(g["offset"], g["width"], g["grouping"]) for g in f["groupings"]})) for _, f in rows})
|
|
220
|
-
|
|
208
|
+
# Each read's paths, split by whether every byte it saw came from the caller's slot writer, and the
|
|
209
|
+
# frame bytes it saw from the slot writer on any path.
|
|
210
|
+
read_on, from_caller, not_from_caller, caller_bytes = {}, {}, {}, {}
|
|
221
211
|
for i, f in rows:
|
|
222
212
|
for g in f["groupings"]:
|
|
223
|
-
|
|
213
|
+
key = (g["offset"], g["width"], g["grouping"])
|
|
214
|
+
read_on.setdefault(key, set()).add(i)
|
|
215
|
+
(not_from_caller if g["bytesNotFromSlotWriter"] else from_caller).setdefault(key, set()).add(i)
|
|
216
|
+
mapped = range(min(g["width"], f["mappedBytes"] - g["offset"]))
|
|
217
|
+
caller_bytes.setdefault(key, set()).update(g["offset"] + at for at in mapped if at not in g["bytesNotFromSlotWriter"])
|
|
224
218
|
reads = sorted(read_on)
|
|
225
219
|
# Distinct reads that share a byte conflict: different intervals, or one interval grouped two ways.
|
|
226
|
-
|
|
227
|
-
|
|
220
|
+
pairs = [(a, b) for n, a in enumerate(reads) for b in reads[n + 1:] if _overlap(a, b)]
|
|
221
|
+
conflicting = [[{"offset": o, "width": w, "grouping": k} for o, w, k in pair] for pair in pairs]
|
|
222
|
+
# Only the caller's bytes decide consistency: a pair counts when both reads saw one shared byte from
|
|
223
|
+
# the slot writer. A callee reusing its argument slot as a local neither breaks nor supplies the
|
|
224
|
+
# width the caller's argument is read at, but a read past the caller's bytes still reads them.
|
|
225
|
+
caller_conflict = any(caller_bytes[a] & caller_bytes[b] for a, b in pairs)
|
|
228
226
|
report["argumentFrameSites"].append({
|
|
229
227
|
"callSite": site, "paths": sorted({i for i, _ in rows}), "frames": len(rows),
|
|
230
228
|
"unsettledPaths": sorted({i for i, f in rows if not f["settledOnThisPath"]}),
|
|
231
229
|
"readWidthSets": [[{"offset": o, "width": w, "grouping": k} for o, w, k in group] for group in widths],
|
|
232
|
-
"readWidths": [{"offset": o, "width": w, "grouping": k, "paths": sorted(read_on[(o, w, k)])
|
|
230
|
+
"readWidths": [{"offset": o, "width": w, "grouping": k, "paths": sorted(read_on[(o, w, k)]),
|
|
231
|
+
"fromCallerOnPaths": sorted(from_caller.get((o, w, k), ())),
|
|
232
|
+
"notFromCallerOnPaths": sorted(not_from_caller.get((o, w, k), ()))} for o, w, k in reads],
|
|
233
233
|
"conflictingWidths": conflicting,
|
|
234
|
-
"widthsConsistent":
|
|
234
|
+
"widthsConsistent": any(caller_bytes.values()) and not caller_conflict,
|
|
235
235
|
"agreed": len(widths) == 1 and all(f["settledOnThisPath"] for _, f in rows),
|
|
236
236
|
"interpretation": "read widths per traced path; paths that never reached this call are not represented"})
|
|
237
237
|
return report
|
|
@@ -136,6 +136,8 @@ class State:
|
|
|
136
136
|
# name they had before the modeled call. They stay unread: no value, no producer.
|
|
137
137
|
self.unread_memory = {}
|
|
138
138
|
self.events = []
|
|
139
|
+
# Bytes above each traced call's return frame that the call event records (argument_slots); 0 records none.
|
|
140
|
+
self.argument_window = 0
|
|
139
141
|
# Value transfers are recorded only for queries that trace declared return results.
|
|
140
142
|
self.value_transfers = bool(config.get("returnContracts"))
|
|
141
143
|
self.guards = []
|
|
@@ -267,23 +269,30 @@ class State:
|
|
|
267
269
|
"""The modeled value of one memory byte, or an unknown term produced by the current site."""
|
|
268
270
|
return self.memory[key] if key in self.memory else unknown(self.unread_term(key), 8, self.at)
|
|
269
271
|
|
|
270
|
-
def
|
|
271
|
-
"""
|
|
272
|
+
def latest_aliasing_write(self, group, domain):
|
|
273
|
+
"""The order of the newest write outside `group` that may have stored a byte of `domain`, or None."""
|
|
272
274
|
# Writes to the key's own (segment, base) group store other offsets; any other write may alias.
|
|
273
|
-
# A later aliasing write is newer than the one that dropped the byte, so it is the one named.
|
|
274
|
-
lost = self.lost_memory.get(key)
|
|
275
|
-
group, a = key[:2], key_domain(key, self.bits, self.flat)
|
|
276
275
|
latest = None
|
|
277
276
|
for other, domains in self.writes.items():
|
|
278
277
|
if other == group:
|
|
279
278
|
continue
|
|
280
|
-
if
|
|
279
|
+
if domain is None:
|
|
281
280
|
# Every write may alias a byte with no concrete domain; the group's newest decides.
|
|
282
281
|
order = max(domains.values())
|
|
283
282
|
else:
|
|
284
|
-
order = max((o for b, o in domains.items() if b is None or not (
|
|
283
|
+
order = max((o for b, o in domains.items() if b is None or not (domain[1] <= b[0] or b[1] <= domain[0])), default=None)
|
|
285
284
|
if order is not None and (latest is None or order > latest):
|
|
286
285
|
latest = order
|
|
286
|
+
return latest
|
|
287
|
+
|
|
288
|
+
def unwritten(self, key):
|
|
289
|
+
"""Why a byte has no modeled value: the cause and the order of the event behind it."""
|
|
290
|
+
return self._unwritten(key, self.latest_aliasing_write(key[:2], key_domain(key, self.bits, self.flat)))
|
|
291
|
+
|
|
292
|
+
def _unwritten(self, key, latest):
|
|
293
|
+
# `latest` is latest_aliasing_write for the key's group and domain. A later aliasing write is
|
|
294
|
+
# newer than the one that dropped the byte, so it is the one named.
|
|
295
|
+
lost = self.lost_memory.get(key)
|
|
287
296
|
if latest is not None and (lost is None or lost["order"] is None or latest > lost["order"]):
|
|
288
297
|
return {"cause": "possibly written by an aliasing write", "order": latest}
|
|
289
298
|
if lost is not None:
|
|
@@ -298,6 +307,34 @@ class State:
|
|
|
298
307
|
return {"index": index, "producers": producers(self.memory[key]), "writeOrder": self.memory_writers.get(key)}
|
|
299
308
|
return {"index": index, "producers": [], "writeOrder": None, "unwritten": self.unwritten(key)}
|
|
300
309
|
|
|
310
|
+
def argument_slots(self, return_bytes, window):
|
|
311
|
+
"""The stack bytes above a call's return frame as a read of them would see them now.
|
|
312
|
+
|
|
313
|
+
Called right after a traced call pushed its return frame. Covers `window` bytes from the first
|
|
314
|
+
byte above the return frame and returns ``{"segment", "base", "start", "runs"}``: `start` is
|
|
315
|
+
that byte's offset in the stack's (segment, base) group, and each run is
|
|
316
|
+
``(offset, width, writeOrder, unwritten)`` for adjacent bytes that ``byte_writer`` would give
|
|
317
|
+
the same write order, or the same ``unwritten`` cause and order.
|
|
318
|
+
"""
|
|
319
|
+
segment, base, delta = self.location(self.segment("ss"), self.reg(self.sp))
|
|
320
|
+
linear = segment == ("linear",)
|
|
321
|
+
start = delta + return_bytes if linear else (delta + return_bytes) % (1 << self.bits)
|
|
322
|
+
runs, aliasing = [], {}
|
|
323
|
+
for at in range(window):
|
|
324
|
+
key = (segment, base, start + at if linear else (start + at) % (1 << self.bits))
|
|
325
|
+
if key in self.memory:
|
|
326
|
+
seen = (self.memory_writers.get(key), None)
|
|
327
|
+
else:
|
|
328
|
+
domain = key_domain(key, self.bits, self.flat)
|
|
329
|
+
if domain not in aliasing:
|
|
330
|
+
aliasing[domain] = self.latest_aliasing_write(key[:2], domain)
|
|
331
|
+
seen = (None, self._unwritten(key, aliasing[domain]))
|
|
332
|
+
if runs and runs[-1][2:] == seen:
|
|
333
|
+
runs[-1] = (runs[-1][0], runs[-1][1] + 1, *seen)
|
|
334
|
+
else:
|
|
335
|
+
runs.append((at, 1, *seen))
|
|
336
|
+
return {"segment": segment, "base": base, "start": start, "runs": runs}
|
|
337
|
+
|
|
301
338
|
def reg(self, name):
|
|
302
339
|
root, low, bits = alias(name)
|
|
303
340
|
value = extract(self.regs[root], low, bits)
|
|
@@ -516,8 +553,12 @@ def may_alias(key, written, bits, flat):
|
|
|
516
553
|
|
|
517
554
|
Different symbolic segments or bases may alias. Concrete linear locations do not.
|
|
518
555
|
"""
|
|
519
|
-
|
|
520
|
-
|
|
556
|
+
return domains_may_overlap(key_domain(key, bits, flat), written)
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def domains_may_overlap(a, b):
|
|
560
|
+
"""Whether two linear byte domains may share a byte; an unknown (None) domain may share any."""
|
|
561
|
+
return a is None or b is None or not (a[1] <= b[0] or b[1] <= a[0])
|
|
521
562
|
|
|
522
563
|
|
|
523
564
|
def string_width(ins, flat):
|
|
@@ -4,8 +4,8 @@ A scope is a query hypothesis that a modeled service leaves a named byte range a
|
|
|
4
4
|
call. It is evidence-layer bookkeeping: it reads and writes no memory on the path, adds no read or
|
|
5
5
|
write event and computes no instruction value or flag (ADR 0003).
|
|
6
6
|
"""
|
|
7
|
-
from .machine import ALIASES, StopPath
|
|
8
|
-
from .values import const
|
|
7
|
+
from .machine import ALIASES, StopPath, domains_may_overlap, written_domain
|
|
8
|
+
from .values import const, op
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
SEGMENTS = ("cs", "ds", "es", "ss", "fs", "gs")
|
|
@@ -60,25 +60,39 @@ def capture_scopes(state, model):
|
|
|
60
60
|
"""Resolve each scope against the pre-call state and snapshot its bytes.
|
|
61
61
|
|
|
62
62
|
Returns ``(values, unread, descriptions)``: the cached bytes, the unknown term names of bytes
|
|
63
|
-
the model had no value for, and one report entry per scope.
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
the model had no value for, and one report entry per scope. The segment must be concrete. The
|
|
64
|
+
base may be concrete, or a symbolic value such as SP or BP at an offset from an unknown entry SP
|
|
65
|
+
(ADR 0013): the scope's bytes are then keyed by that value, as reads and writes through it are.
|
|
66
|
+
Raises ``StopPath`` when the segment is not concrete, a concrete interval leaves the address
|
|
67
|
+
space, two scopes on one base value share a byte, or two scopes on different base values may
|
|
68
|
+
share a linear byte (segment aliases included). A symbolic base may address any byte of its
|
|
69
|
+
segment, so it may alias every scope whose segment range overlaps that segment. An uncached
|
|
70
|
+
byte stays uncached after the call.
|
|
66
71
|
"""
|
|
67
|
-
values, unread, descriptions,
|
|
72
|
+
values, unread, descriptions, resolved = {}, {}, [], []
|
|
68
73
|
for scope in model.get("preservesMemory", []):
|
|
69
74
|
segment, base = state.segment(scope["segment"]), state.reg(scope["base"])
|
|
70
|
-
if segment.number is None
|
|
71
|
-
raise StopPath("preservesMemory address unresolved: segment
|
|
75
|
+
if segment.number is None:
|
|
76
|
+
raise StopPath("preservesMemory address unresolved: the segment must be concrete before the call")
|
|
72
77
|
displacement = scope.get("displacement", 0)
|
|
73
|
-
offset = base.number + displacement
|
|
74
78
|
size = scope["bytes"]
|
|
75
|
-
if
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
79
|
+
if base.number is not None:
|
|
80
|
+
if not 0 <= base.number + displacement <= (1 << state.bits) - size:
|
|
81
|
+
raise StopPath("preservesMemory interval crosses the address boundary")
|
|
82
|
+
offset = const(base.number + displacement, state.bits)
|
|
83
|
+
else:
|
|
84
|
+
offset = op("add", base, const(displacement, state.bits))
|
|
85
|
+
seg, group_base, start, keys = state.keys(segment, offset, size)
|
|
86
|
+
domain = written_domain(seg, group_base, start, size, state.bits, state.flat)
|
|
87
|
+
for group, prior_keys, prior_domain in resolved:
|
|
88
|
+
if group == (seg, group_base):
|
|
89
|
+
shared = not prior_keys.isdisjoint(keys)
|
|
90
|
+
else:
|
|
91
|
+
shared = domains_may_overlap(prior_domain, domain)
|
|
92
|
+
if shared:
|
|
93
|
+
raise StopPath("preservesMemory intervals overlap or alias")
|
|
94
|
+
resolved.append(((seg, group_base), set(keys), domain))
|
|
95
|
+
linear = seg == ("linear",)
|
|
82
96
|
cached = 0
|
|
83
97
|
for key in keys:
|
|
84
98
|
if key in state.memory:
|
|
@@ -89,7 +103,9 @@ def capture_scopes(state, model):
|
|
|
89
103
|
descriptions.append({
|
|
90
104
|
"segmentRegister": scope["segment"], "segment": segment.report(),
|
|
91
105
|
"baseRegister": scope["base"], "base": base.report(), "displacement": displacement,
|
|
92
|
-
"offset": offset,
|
|
106
|
+
"offset": offset.number,
|
|
107
|
+
"interval": {"segment": seg, "base": group_base, "start": start, "end": start + size},
|
|
108
|
+
"linearStart": start if linear else None, "linearEnd": start + size if linear else None, "bytes": size,
|
|
93
109
|
"cachedBytes": cached, "uncachedBytes": size - cached, "evidence": scope["evidence"],
|
|
94
110
|
"meaning": "explicit pre-call memory-preservation hypothesis; memory outside every scope is unknown"})
|
|
95
111
|
return values, unread, descriptions
|
|
@@ -104,15 +120,16 @@ def scope_history(state, values, unread):
|
|
|
104
120
|
def retain_scopes(state, values, unread, history):
|
|
105
121
|
"""Put the captured bytes back after the model invalidated memory. Later writes still apply.
|
|
106
122
|
|
|
107
|
-
|
|
108
|
-
|
|
123
|
+
Each byte rejoins the alias group of its own segment and base, so a later write that may alias
|
|
124
|
+
it drops it as it drops any other byte. ``history`` from ``scope_history`` keeps each byte's
|
|
125
|
+
reported writer, or its reason for having no value.
|
|
109
126
|
"""
|
|
110
127
|
state.memory.update(values)
|
|
111
128
|
state.unread_memory.update(unread)
|
|
112
129
|
state.memory_writers.update(history[0])
|
|
113
130
|
state.lost_memory.update(history[1])
|
|
114
|
-
|
|
115
|
-
state.memory_groups.setdefault(
|
|
131
|
+
for key in [*values, *unread]:
|
|
132
|
+
state.memory_groups.setdefault(key[:2], set()).add(key)
|
|
116
133
|
|
|
117
134
|
|
|
118
135
|
def model_scopes(models, event):
|