scientific-method-engine 1.0.1__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/PKG-INFO +10 -1
  2. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/README.md +9 -0
  3. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/pyproject.toml +1 -1
  4. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/__init__.py +1 -1
  5. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/cli.py +2 -0
  6. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/effect_order.py +18 -8
  7. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/machine.py +44 -10
  8. scientific_method_engine-2.1.0/src/scientific_method_engine/x86/memory_scopes.py +106 -0
  9. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/pcode.py +23 -1
  10. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/pcode_backend.py +65 -6
  11. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/reports.py +28 -6
  12. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/trace.py +83 -4
  13. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/values.py +3 -0
  14. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/tests/oracle.py +40 -6
  15. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/tests/test_effect_order.py +14 -3
  16. scientific_method_engine-2.1.0/tests/test_memory_scopes.py +293 -0
  17. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/tests/test_oracle.py +54 -1
  18. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/tests/test_pe.py +12 -0
  19. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/tests/test_x86.py +247 -7
  20. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/.gitignore +0 -0
  21. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/LICENSE +0 -0
  22. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/__main__.py +0 -0
  23. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  24. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  25. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  26. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  27. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  28. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  29. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  30. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  31. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  32. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  33. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  34. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  35. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  36. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  37. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  38. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  39. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  40. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  41. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  42. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  43. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  44. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  45. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  46. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  47. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  48. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  49. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  50. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  51. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  52. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  53. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  54. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  55. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/image.py +0 -0
  56. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/pe.py +0 -0
  57. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  58. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/tests/test_dispatch.py +0 -0
  59. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/tests/test_nested_frame_request.py +0 -0
  60. {scientific_method_engine-1.0.1 → scientific_method_engine-2.1.0}/tests/test_table_continuations.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 1.0.1
3
+ Version: 2.1.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -128,3 +128,12 @@ Concrete values/field addresses reject inconsistent rows; overlap and shared
128
128
  path/step/visit/total/boundary limits remain explicit. Partial-table splits are
129
129
  partial evidence, never complete dispatch or native-reachability claims. See the
130
130
  [table contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#evidenced-indirect-jump-tables).
131
+
132
+ A call model may declare `preservesMemory`: up to 32 scopes (4,096 bytes in total), each naming a
133
+ segment register, an address-width base register, an optional displacement, a byte count and the
134
+ evidence for assuming the service leaves those bytes alone. Scopes resolve against the pre-call
135
+ registers. An unknown, wrapping or overlapping interval stops the path. Only the scoped bytes
136
+ survive the model's memory invalidation, and each scope is reported with the model, the modeled
137
+ return and the effect summary. Everything else the service may do stays unknown. This input needs
138
+ prepared protocol 3, so upgrade the reader and the engine together. See the
139
+ [call model contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#limits-and-assumptions).
@@ -112,3 +112,12 @@ Concrete values/field addresses reject inconsistent rows; overlap and shared
112
112
  path/step/visit/total/boundary limits remain explicit. Partial-table splits are
113
113
  partial evidence, never complete dispatch or native-reachability claims. See the
114
114
  [table contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#evidenced-indirect-jump-tables).
115
+
116
+ A call model may declare `preservesMemory`: up to 32 scopes (4,096 bytes in total), each naming a
117
+ segment register, an address-width base register, an optional displacement, a byte count and the
118
+ evidence for assuming the service leaves those bytes alone. Scopes resolve against the pre-call
119
+ registers. An unknown, wrapping or overlapping interval stops the path. Only the scoped bytes
120
+ survive the model's memory invalidation, and each scope is reported with the model, the modeled
121
+ return and the effect summary. Everything else the service may do stays unknown. This input needs
122
+ prepared protocol 3, so upgrade the reader and the engine together. See the
123
+ [call model contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#limits-and-assumptions).
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "1.0.1"
8
+ version = "2.1.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -2,4 +2,4 @@
2
2
 
3
3
  # The reader passes this number with every prepared config it pipes to the engine. A reader and
4
4
  # engine that disagree on it refuse to run instead of exchanging relocation data in another shape.
5
- PREPARED_PROTOCOL = 2
5
+ PREPARED_PROTOCOL = 3
@@ -18,6 +18,8 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
18
18
  "effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
19
19
  "callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges.\n"
20
20
  "Declared table continuations are separate conditional paths; ordinary computed transfers remain stopped.\n"
21
+ "Port accesses and interrupts are hardware-boundary events; portInputs supplies port reads as assumptions.\n"
22
+ "callModels[].preservesMemory keeps explicit, bounded pre-call byte scopes across a modeled call; other memory stays unknown.\n"
21
23
  " scientific-method-engine ghidra-scripts")
22
24
 
23
25
 
@@ -3,7 +3,8 @@ from bisect import bisect_right
3
3
 
4
4
 
5
5
  KINDS = {"read", "write", "call", "call-return", "return", "branch", "compare", "flag-assumption",
6
- "arithmetic", "value-transfer", "conversion", "flag-write", "flags-save", "flags-restore", "local-iret", "string-operation", "declared-jump-continuation"}
6
+ "arithmetic", "value-transfer", "conversion", "flag-write", "flags-save", "flags-restore", "local-iret", "string-operation", "declared-jump-continuation",
7
+ "hardware-boundary"}
7
8
 
8
9
 
9
10
  def _storage(event):
@@ -29,7 +30,7 @@ def effect_ordering(report):
29
30
  combined = [(False, i, p) for i, p in enumerate(report["paths"])]
30
31
  combined += [(True, i, p) for i, p in enumerate(report.get("declaredContinuationPaths", []))]
31
32
  for conditional, index, path in combined:
32
- timeline, writes, calls, witnesses = [], [], [], []
33
+ timeline, writes, calls, witnesses, hardware = [], [], [], [], []
33
34
  snapshots = {}
34
35
  pending = {}
35
36
  unknown_orders = []
@@ -38,7 +39,9 @@ def effect_ordering(report):
38
39
  kind = event["kind"]
39
40
  if kind in KINDS:
40
41
  timeline.append(event)
41
- if kind == "read":
42
+ if kind == "hardware-boundary":
43
+ hardware.append(event["order"])
44
+ elif kind == "read":
42
45
  key = _storage(event)
43
46
  if key is not None:
44
47
  snapshots.setdefault(key, {})[event["value"].get("expression")] = event
@@ -60,7 +63,8 @@ def effect_ordering(report):
60
63
  call = {"order": event["order"], "site": event["site"], "entry": event["entry"],
61
64
  "depth": event["depth"], "target": event.get("target"),
62
65
  "writesBeforeCount": len(writes), "status": "unresolved-or-stopped",
63
- "unknownEffects": True, "continuation": None, "_unknownStart": len(unknown_orders)}
66
+ "unknownEffects": True, "continuation": None, "preservedMemoryScopes": [],
67
+ "_unknownStart": len(unknown_orders)}
64
68
  calls.append(call)
65
69
  pending.setdefault((event["site"], event["depth"]), []).append(call)
66
70
  elif kind == "call-return":
@@ -68,11 +72,14 @@ def effect_ordering(report):
68
72
  if stack:
69
73
  call = stack.pop()
70
74
  modeled = event.get("modeled", False)
75
+ scopes = event.get("preservedMemoryScopes", []) if modeled else []
71
76
  call.update(status="modeled-return" if modeled else "traced-return",
72
77
  unknownEffects=bool(modeled or event.get("unknownMemoryEffects") or len(unknown_orders) > call.pop("_unknownStart")),
73
- returnOrder=event["order"], writesAfterCount=len(writes),
74
- continuation="assumes balanced returning service; its memory/flag effects are unknown" if modeled
75
- else "local callee return reached within the instruction model")
78
+ returnOrder=event["order"], writesAfterCount=len(writes), preservedMemoryScopes=scopes,
79
+ continuation="local callee return reached within the instruction model" if not modeled
80
+ else "assumes balanced returning service; memory outside its preservedMemoryScopes "
81
+ "hypotheses and its flag effects are unknown" if scopes
82
+ else "assumes balanced returning service; its memory/flag effects are unknown")
76
83
  if event.get("modeled") or event.get("unknownMemoryEffects"):
77
84
  unknown_orders.append(event["order"])
78
85
  for call in calls:
@@ -84,9 +91,12 @@ def effect_ordering(report):
84
91
  destination = conditional_summaries if conditional else summaries
85
92
  destination.append({"path": index, "declaredJumpAssumptions": path.get("declaredJumpAssumptions", []), "returned": path["returned"], "stop": boundary,
86
93
  "guards": path["guards"], "timeline": timeline,
94
+ "conditionalModels": path.get("conditionalModels", []),
87
95
  "writeOrders": [w["order"] for w in writes], "calls": calls,
88
96
  "localRestorationWitnesses": witnesses,
89
- "effectCompleteWithinModel": bool(path["returned"] and not unknown_orders and not conditional),
97
+ # Port and interrupt effects stay out of writeOrders: RAM writes never stand for device state.
98
+ "hardwareBoundaryOrders": hardware,
99
+ "effectCompleteWithinModel": bool(path["returned"] and not unknown_orders and not conditional and not hardware),
90
100
  "transactionality": "not established; local writes and result codes cannot prove external rollback"})
91
101
  report["effectOrdering"] = {"paths": summaries, "declaredContinuationPaths": conditional_summaries,
92
102
  "allPathsRead": report["completeWithinModel"],
@@ -18,6 +18,13 @@ class StopPath(Exception):
18
18
  pass
19
19
 
20
20
 
21
+ class Shared(dict):
22
+ """Read-only query configuration that every copy of a path shares instead of copying."""
23
+
24
+ def __deepcopy__(self, memo):
25
+ return self
26
+
27
+
21
28
  def segment_register(ins, mem):
22
29
  """Name the segment register an explicit memory operand uses (override or stack/data default)."""
23
30
  if mem.segment:
@@ -60,6 +67,9 @@ class State:
60
67
  # Keys grouped by (segment, base) so a write scans only groups that can alias it.
61
68
  self.memory_groups = {}
62
69
  self.memory_epoch = 0
70
+ # Bytes a preservesMemory scope kept without a value (ADR 0009): key -> the unknown term
71
+ # name they had before the modeled call. They stay unread: no value, no producer.
72
+ self.unread_memory = {}
63
73
  self.events = []
64
74
  # Value transfers are recorded only for queries that trace declared return results.
65
75
  self.value_transfers = bool(config.get("returnContracts"))
@@ -84,6 +94,8 @@ class State:
84
94
  self.visits = {}
85
95
  self.path = []
86
96
  self.conditional = []
97
+ # Values the query supplies for port reads, by site; trace validates them.
98
+ self.port_inputs = Shared({row["site"]: row for row in config.get("portInputs", [])})
87
99
  flags = config.get("flags", {})
88
100
  if not isinstance(flags, dict) or set(flags) - {"direction"}:
89
101
  raise ValueError("Only an explicit starting direction flag is supported")
@@ -148,8 +160,17 @@ class State:
148
160
  def clear_memory(self):
149
161
  self.memory.clear()
150
162
  self.memory_groups.clear()
163
+ self.unread_memory.clear()
151
164
  self.memory_epoch += 1
152
165
 
166
+ def unread_term(self, key):
167
+ """Name the unknown term of a byte with no modeled value: a kept scope term or the epoch's."""
168
+ return self.unread_memory.get(key, f"memory:{self.memory_epoch}:{key}")
169
+
170
+ def byte(self, key):
171
+ """The modeled value of one memory byte, or an unknown term produced by the current site."""
172
+ return self.memory[key] if key in self.memory else unknown(self.unread_term(key), 8, self.at)
173
+
153
174
  def reg(self, name):
154
175
  root, low, bits = alias(name)
155
176
  value = extract(self.regs[root], low, bits)
@@ -194,7 +215,7 @@ class State:
194
215
  def peek(self, segment, offset, width):
195
216
  """Inspect modeled memory without reporting an access the program never performed."""
196
217
  _, _, _, keys = self.keys(segment, offset, width)
197
- return join([self.memory.get(key, unknown(f"memory:{self.memory_epoch}:{key}", 8, self.at)) for key in keys])
218
+ return join([self.byte(key) for key in keys])
198
219
 
199
220
  def access(self, segment, offset, width, write=None, role=None, addressing_register=None):
200
221
  seg, base, delta, keys = self.keys(segment, offset, width)
@@ -221,18 +242,20 @@ class State:
221
242
  disjoint = a is not None and b is not None and (a[1] <= b[0] or b[1] <= a[0])
222
243
  if not disjoint:
223
244
  uncertain.append(key)
224
- del self.memory[key]
245
+ self.memory.pop(key, None)
246
+ self.unread_memory.pop(key, None)
225
247
  members.discard(key)
226
248
  if not members:
227
249
  del self.memory_groups[group]
228
250
  for i, key in enumerate(keys):
229
251
  self.memory[key] = extract(write, i * 8, 8)
252
+ self.unread_memory.pop(key, None)
230
253
  self.memory_groups.setdefault((seg, base), set()).update(keys)
231
254
  value = write
232
255
  missing = []
233
256
  else:
234
257
  missing = [i for i, key in enumerate(keys) if key not in self.memory]
235
- value = join([self.memory.get(key, unknown(f"memory:{self.memory_epoch}:{key}", 8, self.at)) for key in keys])
258
+ value = join([self.byte(key) for key in keys])
236
259
  relevant = []
237
260
  for g in self.guards:
238
261
  left = g.get("left", {}).get("expression")
@@ -320,8 +343,9 @@ for names, condition in ((("jne", "jnz"), "z"), (("jae", "jnb", "jnc"), "c"), ((
320
343
  for name in names:
321
344
  BRANCH_CONDITIONS[name] = (condition, True)
322
345
 
323
- # One-byte opcodes of the string forms; CMPS and SCAS repeat while a comparison holds.
324
- STRING_OPCODES = (0xA4, 0xA5, 0xAA, 0xAB, 0xAC, 0xAD)
346
+ # One-byte opcodes of the string forms (MOVS, STOS, LODS, INS, OUTS); CMPS and SCAS repeat while a
347
+ # comparison holds.
348
+ STRING_OPCODES = (0xA4, 0xA5, 0xAA, 0xAB, 0xAC, 0xAD, 0x6C, 0x6D, 0x6E, 0x6F)
325
349
  COMPARE_STRING_OPCODES = (0xA6, 0xA7, 0xAE, 0xAF)
326
350
 
327
351
 
@@ -338,6 +362,16 @@ def repeated(ins):
338
362
  return 0xF3 in ins.prefix or (0xF2 in ins.prefix and compare_string(ins))
339
363
 
340
364
 
365
+ def string_width(ins, flat):
366
+ """The element width of a string form: a byte for even opcodes, else the operand size."""
367
+ return 1 if ins.opcode[0] % 2 == 0 else (4 if (0x66 in ins.prefix) != flat else 2)
368
+
369
+
370
+ def string_operation(ins):
371
+ """The string operation's name without its width suffix: movs, stos, lods, cmps, scas, ins or outs."""
372
+ return ins.mnemonic.split()[-1][:-1]
373
+
374
+
341
375
  def string_count(state, ins):
342
376
  return state.reg("ecx" if state.flat else "cx") if repeated(ins) else const(1, state.bits)
343
377
 
@@ -356,8 +390,8 @@ def string_effect(state, ins, count, remaining, charge=None):
356
390
  the count runs out, within ``remaining`` iterations, and calls ``charge(1)`` for each one,
357
391
  because it cannot reserve its iterations before they run.
358
392
  """
359
- width = 1 if ins.opcode[0] % 2 == 0 else (4 if (0x66 in ins.prefix) != state.flat else 2)
360
- operation = ins.mnemonic.split()[-1][:4]
393
+ width = string_width(ins, state.flat)
394
+ operation = string_operation(ins)
361
395
  compare = compare_string(ins)
362
396
  state.event("string-operation", operation=operation, width=width, repetitions=count.report(),
363
397
  direction=state.direction_flag.report(), repeat=repeated(ins),
@@ -370,9 +404,9 @@ def string_effect(state, ins, count, remaining, charge=None):
370
404
  return 0
371
405
  if state.direction_flag.number is None:
372
406
  raise StopPath("Direction flag unresolved; conditional string paths required")
373
- # MOVS/LODS decode their source as the second memory operand and CMPS as the first, carrying
374
- # any segment override.
375
- source_name = (segment_register(ins, ins.operands[1].mem) if operation in ("movs", "lods") else
407
+ # MOVS/LODS/OUTS decode their source as the second operand and CMPS as the first, carrying any
408
+ # segment override.
409
+ source_name = (segment_register(ins, ins.operands[1].mem) if operation in ("movs", "lods", "outs") else
376
410
  segment_register(ins, ins.operands[0].mem) if operation == "cmps" else None)
377
411
  counter = "ecx" if state.flat else "cx"
378
412
  if not compare or not repeated(ins):
@@ -0,0 +1,106 @@
1
+ """Explicit, bounded memory hypotheses on modeled calls (ADR 0009).
2
+
3
+ A scope is a query hypothesis that a modeled service leaves a named byte range as it was before the
4
+ call. It is evidence-layer bookkeeping: it reads and writes no memory on the path, adds no read or
5
+ write event and computes no instruction value or flag (ADR 0003).
6
+ """
7
+ from .machine import ALIASES, StopPath
8
+ from .values import const
9
+
10
+
11
+ SEGMENTS = ("cs", "ds", "es", "ss", "fs", "gs")
12
+ FIELDS = {"segment", "base", "displacement", "bytes", "evidence"}
13
+ MAX_SCOPES = 32
14
+ MAX_BYTES = 4096
15
+
16
+
17
+ def validate_scopes(model, bits, flat):
18
+ """Check the shape and budgets of a call model's ``preservesMemory`` before tracing.
19
+
20
+ Every model is checked, including one whose call site no path reaches. Raises ``ValueError``
21
+ on a malformed scope, more than 32 scopes or more than 4,096 bytes in total, two scopes on the
22
+ same segment and base whose ranges overlap, or (segmented images) a scope whose segment register
23
+ the model does not preserve: reads after the call could not address it.
24
+ """
25
+ scopes = model.get("preservesMemory", [])
26
+ if not isinstance(scopes, list):
27
+ raise ValueError("preservesMemory must be a list of scopes")
28
+ if len(scopes) > MAX_SCOPES:
29
+ raise ValueError(f"preservesMemory scope limit is {MAX_SCOPES}")
30
+ total, declared = 0, {}
31
+ for scope in scopes:
32
+ if not isinstance(scope, dict) or set(scope) - FIELDS:
33
+ raise ValueError("preservesMemory scope fields are segment, base, displacement, bytes and evidence")
34
+ if scope.get("segment") not in SEGMENTS:
35
+ raise ValueError("preservesMemory segment must name a segment register")
36
+ base = scope.get("base")
37
+ if not isinstance(base, str) or base not in ALIASES or base in SEGMENTS or ALIASES[base][2] != bits:
38
+ raise ValueError("preservesMemory base must be an address-width general register")
39
+ displacement = scope.get("displacement", 0)
40
+ if type(displacement) is not int or not -(1 << (bits - 1)) <= displacement < 1 << (bits - 1):
41
+ raise ValueError("preservesMemory displacement must be a signed address-width integer")
42
+ size = scope.get("bytes")
43
+ if type(size) is not int or not 1 <= size <= MAX_BYTES:
44
+ raise ValueError(f"preservesMemory bytes must be 1..{MAX_BYTES}")
45
+ total += size
46
+ if total > MAX_BYTES:
47
+ raise ValueError(f"preservesMemory total byte limit is {MAX_BYTES}")
48
+ if not isinstance(scope.get("evidence"), str) or not scope["evidence"].strip():
49
+ raise ValueError("preservesMemory scope requires nonempty evidence")
50
+ # The model replaces every unpreserved segment register except CS with an unknown value.
51
+ if not flat and scope["segment"] != "cs" and scope["segment"] not in model.get("preserves", []):
52
+ raise ValueError(f"preservesMemory segment {scope['segment']} must also be listed in the model's preserves")
53
+ ranges = declared.setdefault((scope["segment"], ALIASES[base][0]), [])
54
+ if any(displacement < end and start < displacement + size for start, end in ranges):
55
+ raise ValueError("preservesMemory scopes on one segment and base overlap")
56
+ ranges.append((displacement, displacement + size))
57
+
58
+
59
+ def capture_scopes(state, model):
60
+ """Resolve each scope against the pre-call state and snapshot its bytes.
61
+
62
+ Returns ``(values, unread, descriptions)``: the cached bytes, the unknown term names of bytes
63
+ the model had no value for, and one report entry per scope. Raises ``StopPath`` when a segment
64
+ or base is not concrete, an interval leaves the address space, or two intervals share a linear
65
+ byte (segment aliases included). An uncached byte stays uncached after the call.
66
+ """
67
+ values, unread, descriptions, intervals = {}, {}, [], []
68
+ for scope in model.get("preservesMemory", []):
69
+ segment, base = state.segment(scope["segment"]), state.reg(scope["base"])
70
+ if segment.number is None or base.number is None:
71
+ raise StopPath("preservesMemory address unresolved: segment and base must be concrete before the call")
72
+ displacement = scope.get("displacement", 0)
73
+ offset = base.number + displacement
74
+ size = scope["bytes"]
75
+ if offset < 0 or offset + size > 1 << state.bits:
76
+ raise StopPath("preservesMemory interval crosses the address boundary")
77
+ _, _, _, keys = state.keys(segment, const(offset, state.bits), size)
78
+ start, end = keys[0][2], keys[-1][2] + 1
79
+ if any(start < prior_end and prior_start < end for prior_start, prior_end in intervals):
80
+ raise StopPath("preservesMemory intervals overlap or alias")
81
+ intervals.append((start, end))
82
+ cached = 0
83
+ for key in keys:
84
+ if key in state.memory:
85
+ cached += 1
86
+ values[key] = state.memory[key]
87
+ else:
88
+ unread[key] = state.unread_term(key)
89
+ descriptions.append({
90
+ "segmentRegister": scope["segment"], "segment": segment.report(),
91
+ "baseRegister": scope["base"], "base": base.report(), "displacement": displacement,
92
+ "offset": offset, "linearStart": start, "linearEnd": end, "bytes": size,
93
+ "cachedBytes": cached, "uncachedBytes": size - cached, "evidence": scope["evidence"],
94
+ "meaning": "explicit pre-call memory-preservation hypothesis; memory outside every scope is unknown"})
95
+ return values, unread, descriptions
96
+
97
+
98
+ def retain_scopes(state, values, unread):
99
+ """Put the captured bytes back after the model invalidated memory. Later writes still apply.
100
+
101
+ All scoped keys are concrete linear bytes, so they share one alias group.
102
+ """
103
+ state.memory.update(values)
104
+ state.unread_memory.update(unread)
105
+ if values or unread:
106
+ state.memory_groups.setdefault((("linear",), ("absolute",)), set()).update(values, unread)
@@ -95,13 +95,17 @@ class Run:
95
95
  ``registers`` holds the last value written to each register before ``setreg`` added its site,
96
96
  and ``flags`` the arithmetic flags the instruction wrote. ``present(value)`` may rewrite a
97
97
  value written to a register or memory into an equal term of the form reports use.
98
+ ``port(direction, port, value, size)`` handles SLEIGH's ``in`` (direction ``"input"``, value
99
+ None, returns the value read) and ``out`` (``"output"``) user operations; without it they
100
+ stop the path.
98
101
  """
99
102
 
100
- def __init__(self, state, ops, memory, constant=None, flags=None, present=None):
103
+ def __init__(self, state, ops, memory, constant=None, flags=None, present=None, port=None):
101
104
  self.state = state
102
105
  self.ops = ops
103
106
  self.memory = memory
104
107
  self.constant = constant
108
+ self.port = port
105
109
  self.present = present or (lambda value: value)
106
110
  self.temps = {}
107
111
  self.registers = {}
@@ -124,6 +128,13 @@ class Run:
124
128
  if code == "CALLOTHER":
125
129
  if o.userop in ("LOCK", "UNLOCK") and o.output is None:
126
130
  continue # Atomicity markers; a single path has no other bus master.
131
+ if o.userop in ("in", "out") and self.port is not None:
132
+ if o.userop == "in":
133
+ self.write(o.output, self.port("input", self.read(o.inputs[1]), None, o.output[2]))
134
+ else:
135
+ port, data = self.port_operands(o.inputs[1:])
136
+ self.port("output", self.read(port), self.read(data), data[2])
137
+ continue
127
138
  if o.userop != "segment":
128
139
  raise StopPath("Unsupported p-code user operation: " + str(o.userop))
129
140
  name = self.register_name(o.inputs[1])
@@ -139,6 +150,17 @@ class Run:
139
150
  self.write(o.output, evaluate(code, arguments, o.output[2] * 8, self.state.at))
140
151
  return None
141
152
 
153
+ def port_operands(self, inputs):
154
+ """The port and data varnodes of an ``out`` operation.
155
+
156
+ SLEIGH writes OUT as ``out(port, value)`` and OUTS as ``out(value, DX)``. The port is the
157
+ operand that is a constant or DX; the data is never either.
158
+ """
159
+ ports = [v for v in inputs if v[0] == "const" or (v[0] == "register" and self.register_name(v) == "DX")]
160
+ if len(inputs) != 2 or len(ports) != 1:
161
+ raise StopPath("p-code port output has no single port operand")
162
+ return ports[0], inputs[1] if inputs[0] == ports[0] else inputs[0]
163
+
142
164
  def register_name(self, v):
143
165
  name = LIFTER.register(self.state.flat, v[1], v[2])
144
166
  if name is None:
@@ -10,10 +10,10 @@ What this module and ``pcode.Run`` use of a ``State``:
10
10
  - Methods: ``get``, ``put``, ``reg``, ``setreg``, ``segment``, ``access``, ``address``, ``event``,
11
11
  ``set_flags``, ``forget_flags``, ``carry_value``, ``save_flags`` and ``restore_flags``.
12
12
  - Read only: ``at``, ``bits``, ``flat``, ``sp``, ``flags``, ``flag_serial``, ``flag_epoch``,
13
- ``unknown_flag_site``, ``segment_bases`` and ``value_transfers``.
13
+ ``unknown_flag_site``, ``segment_bases``, ``value_transfers``, ``events`` and ``port_inputs``.
14
14
  - Read and assigned: ``flag_values`` (the arithmetic flags p-code computed, or None), ``carry``,
15
15
  ``direction_flag`` and ``interrupt_flag``. ``conditional`` is appended to (the divide-error
16
- assumption).
16
+ and port-input assumptions).
17
17
 
18
18
  Every member above belongs to one path, and ``trace`` copies a path with ``deepcopy`` when a branch
19
19
  splits it. The backend keeps no path state of its own: ``Pypcode.__deepcopy__`` returns the same
@@ -356,7 +356,8 @@ class Frame:
356
356
  WIDTHS[value.term] = value.bits
357
357
  LEAVES.add(value.term)
358
358
  present = (lambda value: self.present(value, self)) if self.present else None
359
- self.run = Run(self.state, self.ops, self.memory, self.constant, present=present)
359
+ self.run = Run(self.state, self.ops, self.memory, self.constant, present=present,
360
+ port=lambda *access: port_access(self.state, self.ins, *access))
360
361
  self.run.execute()
361
362
  for value in [*self.loaded.values(), *self.run.temps.values(), *self.run.registers.values()]:
362
363
  if isinstance(value, Value): # Real-mode temporaries may hold a segmented Address.
@@ -470,7 +471,8 @@ class Pypcode:
470
471
  def string_iteration(self, state, ins, operation, width, source_segment):
471
472
  """Apply one iteration of an accepted string form; see ``machine.string_effect``.
472
473
 
473
- ``source_segment`` names the source operand's segment register (MOVS, LODS and CMPS only).
474
+ ``source_segment`` names the source operand's segment register (MOVS, LODS, OUTS and CMPS
475
+ only).
474
476
  p-code steps SI and DI by the direction flag. For a repeated CMPS or SCAS, returns whether
475
477
  the repeat condition holds afterwards (1, 0 or unknown); otherwise None.
476
478
  """
@@ -506,7 +508,7 @@ class Pypcode:
506
508
 
507
509
  ops, tail = string_ops(state.flat, bytes(ins.bytes), state.at, operation)
508
510
  accumulator = state.reg({1: "al", 2: "ax", 4: "eax"}[width]) if operation == "scas" else None
509
- run = Run(state, ops, memory)
511
+ run = Run(state, ops, memory, port=lambda *access: port_access(state, ins, *access))
510
512
  run.execute()
511
513
  if operation not in ("cmps", "scas"):
512
514
  return None
@@ -909,6 +911,62 @@ def flag_write(state, ins, image):
909
911
  interpretation="local flag effect only; interrupts and timing are not simulated")
910
912
 
911
913
 
914
+ OUTPUT_MEANING = ("the value leaves the instruction model at this port; device state, timing and "
915
+ "rendered output are not modeled")
916
+ INPUT_MEANING = ("the value a device returns is not modeled; a supplied value is a query assumption, "
917
+ "never native device state")
918
+
919
+
920
+ def port_access(state, ins, direction, port, value, size):
921
+ """Report one port input or output as a hardware-boundary event; return the value read.
922
+
923
+ The port number and the value written come from the p-code. A value read is unknown unless the
924
+ query's ``portInputs`` supplies one for this site, which the path then lists as an assumption.
925
+ The flat model stops after the event, because I/O privilege decides whether the access faults.
926
+ """
927
+ port = resize(port, 16)
928
+ fields = {"boundary": "port-" + direction, "mnemonic": ins.mnemonic.split()[-1], "port": port.report(),
929
+ "portKnown": port.number is not None, "width": size}
930
+ if direction == "output":
931
+ state.event("hardware-boundary", **fields, value=value.report(), interpretation=OUTPUT_MEANING)
932
+ result = None
933
+ else:
934
+ supplied = state.port_inputs.get(state.at)
935
+ if supplied is None:
936
+ result = unknown(f"port-input:{state.at}:{len(state.events)}", size * 8, state.at)
937
+ fields["valueSource"] = "unknown"
938
+ else:
939
+ result = const(supplied["value"], size * 8, state.at)
940
+ fields.update(valueSource="query assumption", evidence=supplied["evidence"])
941
+ assumption = {"site": state.at, "evidence": supplied["evidence"],
942
+ "assumption": "port input value supplied by the query; device state unconfirmed"}
943
+ # Every read at the site uses the same supplied value, so the path lists it once.
944
+ if assumption not in state.conditional:
945
+ state.conditional.append(assumption)
946
+ state.event("hardware-boundary", **fields, value=result.report(), interpretation=INPUT_MEANING)
947
+ if state.flat:
948
+ raise StopPath("port access in the flat model depends on I/O privilege, which is not modeled")
949
+ return result
950
+
951
+
952
+ def interrupt_vector(flat, ins, site):
953
+ """The vector p-code's ``swi`` operation names, and whether the interrupt is conditional (INTO)."""
954
+ ops, _ = LIFTER.ops(flat, bytes(ins.bytes), site)
955
+ call = next((o for o in ops if o.code == "CALLOTHER" and o.userop == "swi"), None)
956
+ vector = call.inputs[1][1] if call is not None and call.inputs[1][0] == "const" else None
957
+ return vector, any(o.code == "CBRANCH" for o in ops)
958
+
959
+
960
+ def interrupt(state, ins, image):
961
+ vector, conditional = interrupt_vector(state.flat, ins, state.at)
962
+ if conditional:
963
+ # INTO interrupts only when OF is set; that branch is not modeled.
964
+ raise StopPath("Unsupported instruction semantics: " + ins.mnemonic)
965
+ state.event("hardware-boundary", boundary="interrupt", mnemonic=ins.mnemonic, vector=vector,
966
+ interpretation="the interrupt handler and its effects are not modeled")
967
+ raise StopPath("interrupt handler is not modeled; later effects are not read")
968
+
969
+
912
970
  HANDLERS = {}
913
971
  for names, handler in ((("mov", "movzx", "movsx", "xchg"), move), (("nop",), nop), (("lea",), lea),
914
972
  (("lds", "les"), far_pointer), (("push",), push), (("pop",), pop), (("leave",), leave),
@@ -917,7 +975,8 @@ for names, handler in ((("mov", "movzx", "movsx", "xchg"), move), (("nop",), nop
917
975
  (("neg",), negate), (("adc", "sbb"), carry_arithmetic), (("clc", "stc", "cmc"), carry_flag),
918
976
  (("shl", "sal", "shr", "sar"), shift), (("rol", "ror", "rcl", "rcr"), rotate),
919
977
  (("mul", "imul"), multiply), (("div", "idiv"), divide),
920
- (("cbw", "cwde", "cwd", "cdq"), conversion), (("cld", "std", "cli", "sti"), flag_write)):
978
+ (("cbw", "cwde", "cwd", "cdq"), conversion), (("cld", "std", "cli", "sti"), flag_write),
979
+ (("in", "out"), run_plain), (("int", "int1", "int3", "into"), interrupt)):
921
980
  for name in names:
922
981
  HANDLERS[name] = handler
923
982
 
@@ -4,13 +4,14 @@ from bisect import bisect_right
4
4
  from collections import deque
5
5
  from capstone import CS_AC_READ, CS_AC_WRITE
6
6
  from capstone.x86 import X86_OP_IMM, X86_OP_MEM, X86_OP_REG
7
- from .machine import State, StopPath, REGISTERS, ALIASES, segment_register
7
+ from .machine import State, StopPath, REGISTERS, ALIASES, segment_register, string_instruction
8
8
  from .values import unknown
9
9
  from .effect_order import effect_ordering
10
10
  from .result_flow import return_flows
11
11
  from .image import Image, integer
12
12
  from .trace import (trace, walk, call_target, unsupported_transfer, uncovered, base_mnemonic, OVERLAP_REASON, CONTESTED_REASON,
13
- RETURNS, INTERRUPTS, PORTS)
13
+ RETURNS, INTERRUPTS, PORTS, PORT_INPUTS, port_width)
14
+ from .pcode_backend import interrupt_vector
14
15
 
15
16
 
16
17
  def entries(image):
@@ -591,7 +592,10 @@ def allocations(report, config):
591
592
  "orderedWrites": [e for e in path["events"][event["order"]+1:] if e["kind"] == "write"],
592
593
  "arithmetic": [e for e in path["events"] if e["kind"] == "arithmetic"],
593
594
  "guards": path["guards"], "pathStop": path["stop"],
594
- "allocatorEffects": "conditional model; memory unresolved" if returns and returns.get("modeled") else "see path writes and unresolved exits",
595
+ "allocatorEffects": ("see path writes and unresolved exits" if not (returns and returns.get("modeled")) else
596
+ "conditional model; memory unresolved outside its preservedMemoryScopes" if returns.get("preservedMemoryScopes") else
597
+ "conditional model; memory unresolved"),
598
+ "preservedMemoryScopes": returns.get("preservedMemoryScopes", []) if returns else [],
595
599
  "extentObservation": extent, "pointerObservation": pointer, "writeComparisons": comparisons,
596
600
  "observedExtentBytes": capacity,
597
601
  "capacity": "conditional on evidenced extent units and pointer identity" if extent else "unresolved: request units and bounded writes do not establish allocated extent",
@@ -745,16 +749,32 @@ def call_target_report(image, config):
745
749
  return result
746
750
 
747
751
 
752
+ def hardware_boundary(image, at, ins):
753
+ """The static description of one interrupt or port instruction, from its decoding and p-code."""
754
+ m = base_mnemonic(ins)
755
+ if m in INTERRUPTS:
756
+ vector, conditional = interrupt_vector(image.flat, ins, at)
757
+ return {"site": at, "boundary": "interrupt", "mnemonic": m, "vector": vector, "conditional": conditional}
758
+ port = next(o for o in ins.operands if o.type == X86_OP_IMM or (o.type == X86_OP_REG and ins.reg_name(o.reg) == "dx"))
759
+ return {"site": at, "boundary": "port-input" if m in PORT_INPUTS else "port-output", "mnemonic": m,
760
+ "port": {"source": "immediate", "value": port.imm} if port.type == X86_OP_IMM else {"source": "register", "register": "dx"},
761
+ "width": port_width(ins, image.flat), "stringForm": string_instruction(ins),
762
+ # F2 on INS/OUTS repeats on hardware too; trace stops that form as unsupported.
763
+ "repeated": 0xF2 in ins.prefix or 0xF3 in ins.prefix}
764
+
765
+
748
766
  def body(image, entry, limit=10000):
749
767
  """Every instruction one entry reaches without entering a callee, and every way out of it.
750
768
 
751
769
  Calls, interrupts and port accesses are followed to the next instruction, and each such
752
- continuation is listed as an assumption. A direct jump or conditional branch to another
753
- established entry or another region, and every far jump, is a tail transfer.
770
+ continuation is listed as an assumption. Interrupts and port accesses are also listed as
771
+ hardware boundaries. A direct jump or conditional branch to another established entry or
772
+ another region, and every far jump, is a tail transfer.
754
773
  """
755
774
  integer(limit, 1, 100000, "instruction limit")
756
775
  established = set(entries(image))
757
776
  pending, seen, exits, calls, gaps, assumed, shared = [entry], {}, [], [], [], [], set()
777
+ hardware = []
758
778
 
759
779
  def leaves(at, target):
760
780
  return (target in established and target != entry) or image.region(target) is not image.region(at)
@@ -783,6 +803,7 @@ def body(image, entry, limit=10000):
783
803
  exits.append({"site": at, "kind": "halt"})
784
804
  continue
785
805
  if m in INTERRUPTS or m in PORTS:
806
+ hardware.append(hardware_boundary(image, at, ins))
786
807
  assumed.append({"site": at, "assumption": ("the interrupt returns to the next instruction" if m in INTERRUPTS
787
808
  else "the port access continues to the next instruction")})
788
809
  pending.append(following)
@@ -847,6 +868,7 @@ def body(image, entry, limit=10000):
847
868
  "span": {"start": runs[0][0], "end": runs[-1][1]} if runs else None, "coveredBytes": covered,
848
869
  "exits": sorted(exits, key=lambda e: e["site"]), "calls": sorted(calls, key=lambda c: c["site"]),
849
870
  "assumedContinuations": sorted(assumed, key=lambda a: a["site"]), "sharedEntries": sorted(shared),
871
+ "hardwareBoundaries": sorted(hardware, key=lambda h: h["site"]),
850
872
  "gaps": gaps, "complete": bool(exits) and not gaps}
851
873
 
852
874
 
@@ -1489,7 +1511,7 @@ def _run_report(image, config, command):
1489
1511
  if command == "returns":
1490
1512
  report = return_flows(report, config)
1491
1513
  if command != "trace":
1492
- kinds = {"arguments": ("address-formation", "read", "call", "call-return"), "effects": ("address-formation", "write", "call", "call-return", "return", "branch", "string-operation",
1514
+ kinds = {"arguments": ("address-formation", "read", "call", "call-return"), "effects": ("address-formation", "write", "call", "call-return", "return", "branch", "string-operation", "hardware-boundary",
1493
1515
  "flag-assumption", "flag-write", "flags-save", "flags-restore", "local-iret"),
1494
1516
  "returns": ("return", "call-return", "compare", "branch", "write"),
1495
1517
  "guards": ("compare", "branch", "read", "write", "call", "call-return"),