scientific-method-engine 7.1.0__tar.gz → 7.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/argument_frames.py +57 -57
  4. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/machine.py +44 -7
  5. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/reports.py +2 -2
  6. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/trace.py +7 -1
  7. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_x86.py +149 -2
  8. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/.gitignore +0 -0
  9. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/LICENSE +0 -0
  10. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/README.md +0 -0
  11. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/__init__.py +0 -0
  12. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/__main__.py +0 -0
  13. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/cli.py +0 -0
  14. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  15. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  16. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  17. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  18. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  19. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  20. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  21. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  22. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  23. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  24. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  25. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  26. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  27. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  28. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  29. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  30. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  31. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  32. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  33. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  34. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  35. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  36. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  37. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  38. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  39. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  40. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  41. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  42. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  43. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  44. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  45. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  46. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  47. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  48. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/image.py +0 -0
  49. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/loops.py +0 -0
  50. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  51. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  52. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  53. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/pe.py +0 -0
  54. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/relational.py +0 -0
  55. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  56. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/src/scientific_method_engine/x86/values.py +0 -0
  57. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/oracle.py +0 -0
  58. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_continuation_budget.py +0 -0
  59. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_dispatch.py +0 -0
  60. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_effect_order.py +0 -0
  61. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_entry_frame.py +0 -0
  62. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_loops.py +0 -0
  63. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_memory_scopes.py +0 -0
  64. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_nested_frame_request.py +0 -0
  65. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_oracle.py +0 -0
  66. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_pe.py +0 -0
  67. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_relational_controls.py +0 -0
  68. {scientific_method_engine-7.1.0 → scientific_method_engine-7.2.0}/tests/test_table_continuations.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 7.1.0
3
+ Version: 7.2.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "7.1.0"
8
+ version = "7.2.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -7,9 +7,6 @@ amount never join or split slots. Whatever a read did not settle on the path sta
7
7
  """
8
8
  from capstone.x86 import X86_OP_IMM, X86_OP_REG
9
9
 
10
- from .machine import may_alias, written_domain
11
- from .memory_scopes import model_scopes
12
-
13
10
  # The argument bytes one frame maps; a wider frame is mapped up to here and stays open.
14
11
  WINDOW_BYTES = 256
15
12
 
@@ -39,60 +36,43 @@ def _covered(event, segment, base, start, width, modulus):
39
36
  return [at for at in offsets if 0 <= at < width]
40
37
 
41
38
 
42
- def _writers(events, before, segment, base, start, width, modulus, image, models):
43
- """The last write before index `before` that covered each frame byte, and why each byte without one has none.
39
+ def _slot_states(events, recorded, width):
40
+ """The write that last stored each frame byte when the call ran, and why each byte without one has none.
44
41
 
45
- A modeled call drops every byte outside its `preservesMemory` scopes. Scopes name linear bytes,
46
- so they keep only frame bytes of a linear stack; the search goes on past the call for those.
47
- A write through another segment or base may have stored every frame byte it may alias, whether
48
- or not it dropped a cached byte. The rule is the one the machine's ``unwritten`` applies, so a
49
- slot and a callee read of it that runs before any callee write through another segment or base
50
- name the same write.
42
+ `recorded` is the call event's ``argumentSlots``, from ``State.argument_slots``: the machine's own
43
+ record of each byte, so a slot and a callee read of it that runs before any callee write through
44
+ another segment or base name the same write.
51
45
  """
52
- writers, invalidated = {}, {}
53
- keys = [(segment, base, (start + at) % modulus if modulus else start + at) for at in range(width)]
54
- for j in range(before - 1, -1, -1):
55
- event = events[j]
56
- interval = event.get("interval")
57
- if event["kind"] == "write" and interval and (interval["segment"], interval["base"]) != (segment, base):
58
- written = written_domain(interval["segment"], interval["base"], interval["start"],
59
- interval["end"] - interval["start"], image.bits, image.flat)
60
- for at, key in enumerate(keys):
61
- if at not in writers and at not in invalidated and may_alias(key, written, image.bits, image.flat):
62
- invalidated[at] = f"memory possibly overwritten through another address by the write at {event['site']}"
63
- if event["kind"] == "call-return" and event.get("unknownMemoryEffects"):
64
- kept = set()
65
- if segment == ("linear",):
66
- kept = {at for scope in model_scopes(models, event)
67
- for at in range(scope["linearStart"], scope["linearEnd"])}
68
- for at in range(width):
69
- if at not in writers and at not in invalidated and start + at not in kept:
70
- invalidated[at] = "memory invalidated by the modeled call at " + str(event["callSite"])
71
- for at in _covered(event, segment, base, start, width, modulus):
72
- if at not in invalidated:
73
- writers.setdefault(at, event)
74
- if len(writers) + len(invalidated) == width:
46
+ writers, reasons = {}, {}
47
+ for offset, run, order, unwritten in recorded["runs"]:
48
+ if offset >= width:
75
49
  break
76
- return writers, {at: invalidated.get(at, "no write on this path") for at in range(width) if at not in writers}
50
+ target, value = (writers, events[order]) if unwritten is None else (reasons, _reason(events, unwritten))
51
+ for at in range(offset, min(offset + run, width)):
52
+ target[at] = value
53
+ return writers, reasons
54
+
77
55
 
56
+ def _reason(events, unwritten):
57
+ """A slot's reason for a byte's ``unwritten`` cause, naming the event's site."""
58
+ cause, order = unwritten["cause"], unwritten["order"]
59
+ if cause in ("possibly written by an aliasing write", "dropped by a possibly aliasing write"):
60
+ return f"memory possibly overwritten through another address by the write at {events[order]['site']}"
61
+ if cause == "dropped by a modeled call":
62
+ return "memory invalidated by the modeled call at " + str(events[order]["callSite"])
63
+ return cause
78
64
 
79
- def _frame(image, events, index, models):
65
+
66
+ def _frame(image, events, index):
80
67
  call = events[index]
81
68
  depth = call["depth"]
82
- pushes = [e for e in events[index + 1:index + 3] if e["kind"] == "write" and e.get("role") == "push" and e["site"] == call["site"]]
69
+ recorded = call.pop("argumentSlots")
83
70
  end = next((j for j in range(index + 1, len(events))
84
71
  if events[j]["kind"] == "call-return" and events[j].get("callSite") == call["site"] and events[j]["depth"] == depth), None)
85
72
  inside = events[index + 1:end if end is not None else len(events)]
86
73
  open_reasons = []
87
- if not pushes:
88
- return None
89
- # The return-address push comes last and sits at the callee's entry SP; the frame's return bytes lie above it.
90
- entry = pushes[-1]["interval"]
91
- segment, base = entry["segment"], entry["base"]
74
+ segment, base, start = recorded["segment"], recorded["base"], recorded["start"]
92
75
  modulus = None if segment == ("linear",) else 1 << image.bits
93
- start = entry["start"] + call["returnFrameBytes"]
94
- if modulus:
95
- start %= modulus
96
76
  reads = [e for e in inside if e["kind"] == "read" and e.get("argument") and e["depth"] == depth + 1]
97
77
  returned = next((e for e in inside if e["kind"] == "return" and e["depth"] == depth + 1 and e.get("callSite") == call["site"]), None)
98
78
  callee_cleanup = returned["cleanupBytes"] if returned else None
@@ -112,7 +92,7 @@ def _frame(image, events, index, models):
112
92
 
113
93
  # Slots: runs of argument bytes that one write event last covered.
114
94
  slots, by_byte = [], {}
115
- writers, reasons = _writers(events, index, segment, base, start, width, modulus, image, models)
95
+ writers, reasons = _slot_states(events, recorded, width)
116
96
  for at in range(width):
117
97
  writer = writers.get(at)
118
98
  key = writer["order"] if writer else reasons[at]
@@ -202,36 +182,56 @@ def argument_frames(report, image):
202
182
 
203
183
  A path gets one map per traced call, from caller-written slots to callee reads. A site's
204
184
  groupings agree only when every path that traced a call there settled on the same read widths
205
- and groupings. Separately, a site's widths are consistent when its paths made at least one read
206
- and no two reads on any of them cover the same byte with a different offset, width or grouping;
207
- a path that skipped a read leaves the site consistent but not agreed.
185
+ and groupings. Separately, a site's widths are consistent when its paths read at least one byte
186
+ the caller wrote and no two reads with a different offset, width or grouping both saw one shared
187
+ byte from the caller; a path that skipped a read leaves the site consistent but not agreed.
188
+ A read that saw bytes the callee stored itself, or bytes with no known caller writer, stays
189
+ listed and marked with the paths it was made on that way, and its conflicts stay listed, but
190
+ those bytes do not decide whether the widths are consistent. The report must come from
191
+ ``trace`` with ``argument_window=WINDOW_BYTES``: each traced call's slots come from its
192
+ ``argumentSlots`` record, which this removes from every path, declared continuations included.
208
193
  """
209
194
  sites = {}
210
195
  for path_index, path in enumerate(report["paths"]):
211
196
  events = path["events"]
212
- frames = [_frame(image, events, i, path["conditionalModels"]) for i, e in enumerate(events) if e["kind"] == "call" and "returnFrameBytes" in e]
213
- path["argumentFrames"] = [f for f in frames if f is not None]
197
+ path["argumentFrames"] = [_frame(image, events, i) for i, e in enumerate(events) if e["kind"] == "call" and "returnFrameBytes" in e]
214
198
  for frame in path["argumentFrames"]:
215
199
  sites.setdefault(frame["callSite"], []).append((path_index, frame))
200
+ # Continuations get no frames; their calls' slot records are dropped unread.
201
+ for path in report.get("declaredContinuationPaths", ()):
202
+ for event in path["events"]:
203
+ event.pop("argumentSlots", None)
216
204
  report["argumentFrameSites"] = []
217
205
  for site, rows in sorted(sites.items()):
218
206
  # A set holds each read's grouping too: a far-pointer load and a plain dword over the same bytes disagree.
219
207
  widths = sorted({tuple(sorted({(g["offset"], g["width"], g["grouping"]) for g in f["groupings"]})) for _, f in rows})
220
- read_on = {}
208
+ # Each read's paths, split by whether every byte it saw came from the caller's slot writer, and the
209
+ # frame bytes it saw from the slot writer on any path.
210
+ read_on, from_caller, not_from_caller, caller_bytes = {}, {}, {}, {}
221
211
  for i, f in rows:
222
212
  for g in f["groupings"]:
223
- read_on.setdefault((g["offset"], g["width"], g["grouping"]), set()).add(i)
213
+ key = (g["offset"], g["width"], g["grouping"])
214
+ read_on.setdefault(key, set()).add(i)
215
+ (not_from_caller if g["bytesNotFromSlotWriter"] else from_caller).setdefault(key, set()).add(i)
216
+ mapped = range(min(g["width"], f["mappedBytes"] - g["offset"]))
217
+ caller_bytes.setdefault(key, set()).update(g["offset"] + at for at in mapped if at not in g["bytesNotFromSlotWriter"])
224
218
  reads = sorted(read_on)
225
219
  # Distinct reads that share a byte conflict: different intervals, or one interval grouped two ways.
226
- conflicting = [[{"offset": o, "width": w, "grouping": k} for o, w, k in (a, b)]
227
- for n, a in enumerate(reads) for b in reads[n + 1:] if _overlap(a, b)]
220
+ pairs = [(a, b) for n, a in enumerate(reads) for b in reads[n + 1:] if _overlap(a, b)]
221
+ conflicting = [[{"offset": o, "width": w, "grouping": k} for o, w, k in pair] for pair in pairs]
222
+ # Only the caller's bytes decide consistency: a pair counts when both reads saw one shared byte from
223
+ # the slot writer. A callee reusing its argument slot as a local neither breaks nor supplies the
224
+ # width the caller's argument is read at, but a read past the caller's bytes still reads them.
225
+ caller_conflict = any(caller_bytes[a] & caller_bytes[b] for a, b in pairs)
228
226
  report["argumentFrameSites"].append({
229
227
  "callSite": site, "paths": sorted({i for i, _ in rows}), "frames": len(rows),
230
228
  "unsettledPaths": sorted({i for i, f in rows if not f["settledOnThisPath"]}),
231
229
  "readWidthSets": [[{"offset": o, "width": w, "grouping": k} for o, w, k in group] for group in widths],
232
- "readWidths": [{"offset": o, "width": w, "grouping": k, "paths": sorted(read_on[(o, w, k)])} for o, w, k in reads],
230
+ "readWidths": [{"offset": o, "width": w, "grouping": k, "paths": sorted(read_on[(o, w, k)]),
231
+ "fromCallerOnPaths": sorted(from_caller.get((o, w, k), ())),
232
+ "notFromCallerOnPaths": sorted(not_from_caller.get((o, w, k), ()))} for o, w, k in reads],
233
233
  "conflictingWidths": conflicting,
234
- "widthsConsistent": bool(reads) and not conflicting,
234
+ "widthsConsistent": any(caller_bytes.values()) and not caller_conflict,
235
235
  "agreed": len(widths) == 1 and all(f["settledOnThisPath"] for _, f in rows),
236
236
  "interpretation": "read widths per traced path; paths that never reached this call are not represented"})
237
237
  return report
@@ -136,6 +136,8 @@ class State:
136
136
  # name they had before the modeled call. They stay unread: no value, no producer.
137
137
  self.unread_memory = {}
138
138
  self.events = []
139
+ # Bytes above each traced call's return frame that the call event records (argument_slots); 0 records none.
140
+ self.argument_window = 0
139
141
  # Value transfers are recorded only for queries that trace declared return results.
140
142
  self.value_transfers = bool(config.get("returnContracts"))
141
143
  self.guards = []
@@ -267,23 +269,30 @@ class State:
267
269
  """The modeled value of one memory byte, or an unknown term produced by the current site."""
268
270
  return self.memory[key] if key in self.memory else unknown(self.unread_term(key), 8, self.at)
269
271
 
270
- def unwritten(self, key):
271
- """Why a byte has no modeled value: the cause and the order of the event behind it."""
272
+ def latest_aliasing_write(self, group, domain):
273
+ """The order of the newest write outside `group` that may have stored a byte of `domain`, or None."""
272
274
  # Writes to the key's own (segment, base) group store other offsets; any other write may alias.
273
- # A later aliasing write is newer than the one that dropped the byte, so it is the one named.
274
- lost = self.lost_memory.get(key)
275
- group, a = key[:2], key_domain(key, self.bits, self.flat)
276
275
  latest = None
277
276
  for other, domains in self.writes.items():
278
277
  if other == group:
279
278
  continue
280
- if a is None:
279
+ if domain is None:
281
280
  # Every write may alias a byte with no concrete domain; the group's newest decides.
282
281
  order = max(domains.values())
283
282
  else:
284
- order = max((o for b, o in domains.items() if b is None or not (a[1] <= b[0] or b[1] <= a[0])), default=None)
283
+ order = max((o for b, o in domains.items() if b is None or not (domain[1] <= b[0] or b[1] <= domain[0])), default=None)
285
284
  if order is not None and (latest is None or order > latest):
286
285
  latest = order
286
+ return latest
287
+
288
+ def unwritten(self, key):
289
+ """Why a byte has no modeled value: the cause and the order of the event behind it."""
290
+ return self._unwritten(key, self.latest_aliasing_write(key[:2], key_domain(key, self.bits, self.flat)))
291
+
292
+ def _unwritten(self, key, latest):
293
+ # `latest` is latest_aliasing_write for the key's group and domain. A later aliasing write is
294
+ # newer than the one that dropped the byte, so it is the one named.
295
+ lost = self.lost_memory.get(key)
287
296
  if latest is not None and (lost is None or lost["order"] is None or latest > lost["order"]):
288
297
  return {"cause": "possibly written by an aliasing write", "order": latest}
289
298
  if lost is not None:
@@ -298,6 +307,34 @@ class State:
298
307
  return {"index": index, "producers": producers(self.memory[key]), "writeOrder": self.memory_writers.get(key)}
299
308
  return {"index": index, "producers": [], "writeOrder": None, "unwritten": self.unwritten(key)}
300
309
 
310
+ def argument_slots(self, return_bytes, window):
311
+ """The stack bytes above a call's return frame as a read of them would see them now.
312
+
313
+ Called right after a traced call pushed its return frame. Covers `window` bytes from the first
314
+ byte above the return frame and returns ``{"segment", "base", "start", "runs"}``: `start` is
315
+ that byte's offset in the stack's (segment, base) group, and each run is
316
+ ``(offset, width, writeOrder, unwritten)`` for adjacent bytes that ``byte_writer`` would give
317
+ the same write order, or the same ``unwritten`` cause and order.
318
+ """
319
+ segment, base, delta = self.location(self.segment("ss"), self.reg(self.sp))
320
+ linear = segment == ("linear",)
321
+ start = delta + return_bytes if linear else (delta + return_bytes) % (1 << self.bits)
322
+ runs, aliasing = [], {}
323
+ for at in range(window):
324
+ key = (segment, base, start + at if linear else (start + at) % (1 << self.bits))
325
+ if key in self.memory:
326
+ seen = (self.memory_writers.get(key), None)
327
+ else:
328
+ domain = key_domain(key, self.bits, self.flat)
329
+ if domain not in aliasing:
330
+ aliasing[domain] = self.latest_aliasing_write(key[:2], domain)
331
+ seen = (None, self._unwritten(key, aliasing[domain]))
332
+ if runs and runs[-1][2:] == seen:
333
+ runs[-1] = (runs[-1][0], runs[-1][1] + 1, *seen)
334
+ else:
335
+ runs.append((at, 1, *seen))
336
+ return {"segment": segment, "base": base, "start": start, "runs": runs}
337
+
301
338
  def reg(self, name):
302
339
  root, low, bits = alias(name)
303
340
  value = extract(self.regs[root], low, bits)
@@ -8,7 +8,7 @@ from .machine import State, StopPath, REGISTERS, ALIASES, segment_register, stri
8
8
  from .values import unknown
9
9
  from .effect_order import effect_ordering
10
10
  from .relational import validate_controls, evaluate_controls
11
- from .argument_frames import argument_frames, stack_cleanup
11
+ from .argument_frames import WINDOW_BYTES, argument_frames, stack_cleanup
12
12
  from .memory_scopes import model_scopes
13
13
  from .result_flow import return_flows
14
14
  from .image import Image, integer
@@ -1736,7 +1736,7 @@ def _run_report(image, config, command):
1736
1736
  if a.get(field):
1737
1737
  checkpoints.add(a[field]["site"])
1738
1738
  config = {**config, "checkpoints": sorted(checkpoints)}
1739
- report = trace(image, config)
1739
+ report = trace(image, config, argument_window=WINDOW_BYTES if command == "arguments" else 0)
1740
1740
  # Controls read the complete event stream, before any command narrows it.
1741
1741
  controls = evaluate_controls(report, config, image)
1742
1742
  if command in ("arguments", "effects"):
@@ -371,7 +371,7 @@ def entry_frame(image, config, entry):
371
371
  "traced paths under the query's own inputs; bp null leaves BP unknown"}
372
372
 
373
373
 
374
- def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False):
374
+ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False, argument_window=0):
375
375
  """Trace bounded paths, preserving declared-table continuations as separate conditional evidence.
376
376
 
377
377
  Ordinary paths run first. A path stopped at a declared indirect jump is then
@@ -386,6 +386,9 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
386
386
  call_stacks=True, and each stopped path carries ``callStack``: the traced
387
387
  calls still open at the stop, outermost first, with each one's return site.
388
388
  A path limit gap inside a called function carries the same ``callStack``.
389
+ A positive ``argument_window`` gives each traced call event an ``argumentSlots`` entry: the
390
+ first ``argument_window`` bytes above its return frame as ``State.argument_slots`` saw them
391
+ when the call ran. ``argument_frames`` reads and removes it.
389
392
  """
390
393
  entry = integer(config.get("entry"), 0, len(image.data) - 1, "entry")
391
394
  if not any(entry in r["entries"] for r in image.regions):
@@ -425,6 +428,7 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
425
428
  observed_frame = entry_frame(image, config, entry) if arrive is None else None
426
429
  root = State(entry, image, config)
427
430
  root.enter_frame(observed_frame)
431
+ root.argument_window = argument_window
428
432
  # Each path carries its own loop record; forks copy it with the rest of the state.
429
433
  root.loops = (LoopTracker(integer(config.get("loopIterationLimit", 64), 1, 1024, "loopIterationLimit"))
430
434
  if track_loops else None)
@@ -760,6 +764,8 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
760
764
  flags_frame = False
761
765
  # A traced call records its return-frame width; argumentFrames maps the slots above it.
762
766
  call_event["returnFrameBytes"] = 4 if m == "lcall" or push_cs else image.bits // 8
767
+ if state.argument_window:
768
+ call_event["argumentSlots"] = state.argument_slots(call_event["returnFrameBytes"], state.argument_window)
763
769
  state.frames.append({"entry": target, "sp": state.reg(state.sp), "returnBytes": call_event["returnFrameBytes"],
764
770
  "frameSource": "push-CS/near-call; matching far return required" if push_cs else m,
765
771
  "continuation": following, "returnIP": return_ip, "callSite": at,
@@ -1,5 +1,6 @@
1
1
  """Synthetic machine code only. No original binaries or analysis artifacts."""
2
2
  import hashlib
3
+ import itertools
3
4
  import json
4
5
  import os
5
6
  from pathlib import Path
@@ -603,8 +604,76 @@ class ArgumentFrameTests(unittest.TestCase):
603
604
  self.assertIn("the slot at 2 was not read by the callee on this path", r["paths"][bypass]["argumentFrames"][0]["openReasons"])
604
605
  self.assertTrue(site["widthsConsistent"])
605
606
  self.assertEqual(site["conflictingWidths"], [])
606
- self.assertEqual(site["readWidths"], [{"offset": 0, "width": 2, "grouping": "consumed width only", "paths": [0, 1]},
607
- {"offset": 2, "width": 2, "grouping": "consumed width only", "paths": [1 - bypass]}])
607
+ self.assertEqual(site["readWidths"], [{"offset": 0, "width": 2, "grouping": "consumed width only", "paths": [0, 1],
608
+ "fromCallerOnPaths": [0, 1], "notFromCallerOnPaths": []},
609
+ {"offset": 2, "width": 2, "grouping": "consumed width only", "paths": [1 - bypass],
610
+ "fromCallerOnPaths": [1 - bypass], "notFromCallerOnPaths": []}])
611
+
612
+ def test_a_callee_reusing_its_argument_slot_does_not_decide_consistency(self):
613
+ # One path stores a word over the first argument and reads four bytes there as a dword; the
614
+ # other reads the caller's word. The pair is listed, but only the word read saw the caller's bytes.
615
+ c = Code().emit("6a 01 6a 02").branch("e8", "callee").emit("83 c4 04 c3")
616
+ c.label("callee").emit("55 89 e5 85 f6").branch("74", "reuse").emit("8b 46 04").branch("eb", "done")
617
+ c.label("reuse").emit("c7 46 04 00 00 66 8b 46 04").label("done").emit("5d c3")
618
+ r, frames = self.frames(c)
619
+ self.assertEqual(len(frames), 2)
620
+ reuse = next(i for i, path in enumerate(r["paths"]) if path["argumentFrames"][0]["groupings"][0]["width"] == 4)
621
+ self.assertEqual(r["paths"][reuse]["argumentFrames"][0]["groupings"][0]["bytesNotFromSlotWriter"], [0, 1])
622
+ site = r["argumentFrameSites"][0]
623
+ self.assertEqual(site["readWidths"], [{"offset": 0, "width": 2, "grouping": "consumed width only", "paths": [1 - reuse],
624
+ "fromCallerOnPaths": [1 - reuse], "notFromCallerOnPaths": []},
625
+ {"offset": 0, "width": 4, "grouping": "consumed width only", "paths": [reuse],
626
+ "fromCallerOnPaths": [], "notFromCallerOnPaths": [reuse]}])
627
+ self.assertEqual(site["conflictingWidths"], [[{"offset": 0, "width": 2, "grouping": "consumed width only"},
628
+ {"offset": 0, "width": 4, "grouping": "consumed width only"}]])
629
+ self.assertTrue(site["widthsConsistent"])
630
+ self.assertFalse(site["agreed"])
631
+ # Control: when the dword read sees the caller's bytes, the same pair makes the site inconsistent.
632
+ c = Code().emit("6a 01 6a 02").branch("e8", "callee").emit("83 c4 04 c3")
633
+ c.label("callee").emit("55 89 e5 85 f6").branch("74", "wide").emit("8b 46 04").branch("eb", "done")
634
+ c.label("wide").emit("66 8b 46 04").label("done").emit("5d c3")
635
+ site = self.frames(c)[0]["argumentFrameSites"][0]
636
+ self.assertEqual(len(site["conflictingWidths"]), 1)
637
+ self.assertEqual([w["notFromCallerOnPaths"] for w in site["readWidths"]], [[], []])
638
+ self.assertFalse(site["widthsConsistent"])
639
+
640
+ def test_a_read_past_the_callers_bytes_still_conflicts_on_the_bytes_it_saw(self):
641
+ # One pushed word, read as a dword on one path and as a word on the other. The dword's upper
642
+ # bytes have no writer, but its lower bytes are the caller's word, so the widths disagree.
643
+ c = Code().emit("6a 01").branch("e8", "callee").emit("83 c4 02 c3")
644
+ c.label("callee").emit("55 89 e5 85 f6").branch("74", "wide").emit("8b 46 04").branch("eb", "done")
645
+ c.label("wide").emit("66 8b 46 04").label("done").emit("5d c3")
646
+ r, _ = self.frames(c)
647
+ wide = next(i for i, path in enumerate(r["paths"]) if path["argumentFrames"][0]["groupings"][0]["width"] == 4)
648
+ self.assertEqual(r["paths"][wide]["argumentFrames"][0]["groupings"][0]["bytesNotFromSlotWriter"], [2, 3])
649
+ site = r["argumentFrameSites"][0]
650
+ self.assertEqual([(w["width"], w["fromCallerOnPaths"], w["notFromCallerOnPaths"]) for w in site["readWidths"]],
651
+ [(2, [1 - wide], []), (4, [], [wide])])
652
+ self.assertEqual(len(site["conflictingWidths"]), 1)
653
+ self.assertFalse(site["widthsConsistent"])
654
+ # A dword over a stored first word and the caller's second word conflicts with a word read of the second.
655
+ c = Code().emit("6a 01 6a 02").branch("e8", "callee").emit("83 c4 04 c3")
656
+ c.label("callee").emit("55 89 e5 85 f6").branch("74", "reuse").emit("8b 46 04 8b 5e 06").branch("eb", "done")
657
+ c.label("reuse").emit("c7 46 04 00 00 66 8b 46 04").label("done").emit("5d c3")
658
+ site = self.frames(c)[0]["argumentFrameSites"][0]
659
+ self.assertEqual([(w["offset"], w["width"]) for w in site["readWidths"]], [(0, 2), (0, 4), (2, 2)])
660
+ self.assertEqual(len(site["conflictingWidths"]), 2)
661
+ self.assertFalse(site["widthsConsistent"])
662
+
663
+ def test_a_site_read_only_after_the_callee_stored_its_slot_is_not_consistent(self):
664
+ # The callee stores over its argument and then reads it: the read is listed, but none saw the caller's word.
665
+ c = Code().emit("6a 01").branch("e8", "callee").emit("83 c4 02 c3").label("callee").emit("55 89 e5 c7 46 04 09 00 8b 46 04 5d c3")
666
+ site = self.frames(c)[0]["argumentFrameSites"][0]
667
+ self.assertEqual(site["readWidths"], [{"offset": 0, "width": 2, "grouping": "consumed width only", "paths": [0],
668
+ "fromCallerOnPaths": [], "notFromCallerOnPaths": [0]}])
669
+ self.assertEqual(site["conflictingWidths"], [])
670
+ self.assertFalse(site["widthsConsistent"])
671
+ # A path that reads the caller's word and then the stored word at the same width lists the path both ways.
672
+ c = Code().emit("6a 01").branch("e8", "callee").emit("83 c4 02 c3")
673
+ c.label("callee").emit("55 89 e5 8b 46 04 c7 46 04 09 00 8b 5e 04 5d c3")
674
+ site = self.frames(c)[0]["argumentFrameSites"][0]
675
+ self.assertEqual([(w["paths"], w["fromCallerOnPaths"], w["notFromCallerOnPaths"]) for w in site["readWidths"]], [([0], [0], [0])])
676
+ self.assertTrue(site["widthsConsistent"])
608
677
 
609
678
  def test_paths_that_group_the_same_bytes_differently_keep_the_site_open(self):
610
679
  # Both paths read four bytes at offset 0: one as a far pointer with LES, one as a 32-bit dword.
@@ -658,6 +727,84 @@ class ArgumentFrameTests(unittest.TestCase):
658
727
  self.assertEqual([(s["offset"], s["width"], s["writerSite"]) for s in frame["slots"]], [(0, 2, 0)])
659
728
  self.assertTrue(frame["settledOnThisPath"], frame["openReasons"])
660
729
 
730
+ def test_every_slot_agrees_with_the_callee_read_of_the_same_byte(self):
731
+ # Each caller fills a frame through pushes, a reserved word, stores through DS or ES and a
732
+ # modeled service, on a symbolic or concrete stack. The callee reads every pushed word, on
733
+ # one variant after an ES store of its own. A read that runs before any callee write
734
+ # through another segment or base sees each byte as the slot does: the same write order,
735
+ # or an unwritten cause naming the event the slot's reason names.
736
+ callers = [("6a 01 6a 02", 2), ("6a 01 c7 06 00 01 05 00 6a 02", 2), ("83 ec 02 88 07", 1),
737
+ ("6a 01 26 c6 07 01", 1), ("6a 01 SERVICE 6a 02", 2), ("83 ec 02 SERVICE 26 c6 07 01", 1),
738
+ ("6a 01 89 e3 c7 07 09 00", 1), ("6a 01 SERVICE", 1)]
739
+ registers = [{}, {"ss": 0x2000, "sp": 0x100}, {"ss": 0x2000, "sp": 0x100, "ds": 0x2000},
740
+ {"ss": 0x2000, "sp": 0x100, "ds": 0x3000}]
741
+ scopes = [None, {"segment": "ss", "base": "sp", "displacement": 0, "bytes": 1, "evidence": "synthetic kept byte"}]
742
+ compared, causes = 0, set()
743
+ for (caller, words), regs, scope, callee_store in itertools.product(callers, registers, scopes, (False, True)):
744
+ c, sites = Code(), []
745
+ for n, part in enumerate(caller.split("SERVICE")):
746
+ if n:
747
+ sites.append(len(c.data))
748
+ c.branch("e8", "service")
749
+ c.emit(part)
750
+ c.branch("e8", "callee").emit("83 c4 %02x c3" % (2 * words)).label("service").emit("c3")
751
+ c.label("callee").emit("55 89 e5" + (" 26 c6 07 01" if callee_store else "")
752
+ + "".join(" 8b 46 %02x" % (4 + 2 * n) for n in range(words)) + " 5d c3")
753
+ models = [{"site": site, "evidence": "synthetic service", "preserves": ["ss", "esp", "ds", "es", "ebx"], "cases": [{}],
754
+ **({"preservesMemory": [scope]} if scope else {})} for site in sites]
755
+ extra = {"registers": regs, "callModels": models}
756
+ traced, mapped = report(c, "trace", **extra), report(c, "arguments", **extra)
757
+ self.assertEqual(len(traced["paths"]), len(mapped["paths"]))
758
+ for path, mapped_path in zip(traced["paths"], mapped["paths"]):
759
+ every = path["events"]
760
+ for frame in mapped_path["argumentFrames"]:
761
+ push = every[frame["callOrder"] + 1]["interval"]
762
+ group = (push["segment"], push["base"])
763
+ for grouping in frame["groupings"]:
764
+ between = every[frame["callOrder"] + 1:grouping["readOrder"]]
765
+ if any(e["kind"] == "write" and (e["interval"]["segment"], e["interval"]["base"]) != group for e in between):
766
+ continue
767
+ read = every[grouping["readOrder"]]
768
+ for i, row in enumerate(read["byteProducers"]):
769
+ at = grouping["offset"] + i
770
+ slot = next(s for s in frame["slots"] if s["offset"] <= at < s["offset"] + s["width"])
771
+ if slot["writerSite"] is not None:
772
+ self.assertEqual(row["writeOrder"], slot["writerOrder"], (c.data.hex(), regs, scope))
773
+ causes.add("written")
774
+ else:
775
+ cause, order = row["unwritten"]["cause"], row["unwritten"]["order"]
776
+ expected = ("no write on this path" if cause == "no write on this path" else
777
+ "memory invalidated by the modeled call at " + str(every[order]["callSite"])
778
+ if cause == "dropped by a modeled call" else
779
+ f"memory possibly overwritten through another address by the write at {every[order]['site']}")
780
+ self.assertEqual(slot["reason"], expected, (c.data.hex(), regs, scope))
781
+ causes.add(cause)
782
+ compared += 1
783
+ self.assertGreater(compared, 100)
784
+ self.assertEqual(causes, {"written", "no write on this path", "dropped by a modeled call",
785
+ "possibly written by an aliasing write", "dropped by a possibly aliasing write"})
786
+
787
+ def test_the_slot_record_stays_out_of_every_reported_event(self):
788
+ # The ordinary path stops at a declared table jump; its continuation calls the callee.
789
+ c = Code().emit("ff e3").label("target").emit("6a 01").branch("e8", "callee").emit("83 c4 02 c3")
790
+ c.label("callee").emit("55 89 e5 8b 46 04 5d c3").label("table")
791
+ data = c.bytes() + c.labels["target"].to_bytes(2, "little")
792
+ cfg = configuration(data, indirectJumps=[{"site": 0, "evidence": "synthetic table consumer", "exhaustive": True,
793
+ "table": {"start": c.labels["table"], "count": 1, "stride": 2,
794
+ "evidence": "synthetic table words"}}])
795
+ cfg["regions"][0]["end"] = c.labels["table"]
796
+ for command in ("arguments", "trace"):
797
+ r = run_report(data, cfg, command)
798
+ self.assertEqual(len(r["declaredContinuationPaths"]), 1)
799
+ calls = [e for group in ("paths", "declaredContinuationPaths") for path in r[group] for e in path["events"] if e["kind"] == "call"]
800
+ self.assertTrue(calls)
801
+ self.assertFalse([e for e in calls if "argumentSlots" in e])
802
+ # A call on an ordinary path keeps no record either.
803
+ c = Code().emit("6a 01").branch("e8", "callee").emit("83 c4 02 c3").label("callee").emit("55 89 e5 8b 46 04 5d c3")
804
+ r, frames = self.frames(c)
805
+ self.assertEqual(len(frames), 1)
806
+ self.assertFalse([e for e in events(r, "call") if "argumentSlots" in e])
807
+
661
808
 
662
809
  class GhidraCrossCheckTests(unittest.TestCase):
663
810
  # root: call a (site 0); call b (site 3); call bx (site 6); ret. a and b return.