scientific-method-engine 16.0.0__tar.gz → 16.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/PKG-INFO +11 -1
  2. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/README.md +10 -0
  3. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/pyproject.toml +1 -1
  4. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/cli.py +2 -0
  5. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/machine.py +96 -10
  6. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/memory_scopes.py +11 -5
  7. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/trace.py +4 -2
  8. scientific_method_engine-16.1.0/tests/test_volatile_memory.py +194 -0
  9. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/.gitignore +0 -0
  10. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/LICENSE +0 -0
  11. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/__init__.py +0 -0
  12. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/__main__.py +0 -0
  13. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  14. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  15. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  16. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  17. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  18. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  19. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  20. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  21. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  22. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  23. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  24. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  25. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  26. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  27. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  28. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  29. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  30. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  31. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  32. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  33. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  34. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  35. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  36. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  37. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  38. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  39. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  40. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  41. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  42. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  43. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  44. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  45. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  46. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  47. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  48. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  49. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  50. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  51. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  52. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/image.py +0 -0
  53. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/inventory.py +0 -0
  54. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/loops.py +0 -0
  55. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  56. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  57. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/pe.py +0 -0
  58. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/reach.py +0 -0
  59. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/relational.py +0 -0
  60. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/reports.py +0 -0
  61. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  62. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/src/scientific_method_engine/x86/values.py +0 -0
  63. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/oracle.py +0 -0
  64. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_continuation_budget.py +0 -0
  65. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_dispatch.py +0 -0
  66. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_effect_order.py +0 -0
  67. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_enter_frames.py +0 -0
  68. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_entry_frame.py +0 -0
  69. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_indirect_far_transfers.py +0 -0
  70. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_interrupt_models.py +0 -0
  71. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_inventory_check.py +0 -0
  72. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_join_terms.py +0 -0
  73. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_loops.py +0 -0
  74. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_memory_scopes.py +0 -0
  75. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_nested_frame_request.py +0 -0
  76. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_oracle.py +0 -0
  77. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_pe.py +0 -0
  78. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_reach.py +0 -0
  79. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_relational_controls.py +0 -0
  80. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_return_checks.py +0 -0
  81. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_successor_rule.py +0 -0
  82. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_table_continuations.py +0 -0
  83. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_term_limit.py +0 -0
  84. {scientific_method_engine-16.0.0 → scientific_method_engine-16.1.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 16.0.0
3
+ Version: 16.1.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -183,6 +183,16 @@ reported as `undecided` and never counts as held. Access reports also give each
183
183
  same of memory the program never reads back, at a checkpoint. See the
184
184
  [controls contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#relational-controls).
185
185
 
186
+ Two reads of a byte the model holds no value for read one term until a write that may store it or a
187
+ modeled call, which assumes nothing else changes memory. A query whose program polls memory that
188
+ hardware, DMA or an interrupt handler updates lists those addresses in `volatileMemory`: up to 64
189
+ rows of `segment` (a real-mode paragraph, or 0 in the PE32 flat model), `offset`, `bytes` and
190
+ `evidence`. Each read of a byte inside a declared range reads its own term, ignoring any value the
191
+ path stored, and its `byteProducers` row gives cause `declared volatile` with the read's order and
192
+ the declared row's index in `volatileMemory`. A read whose address is not concrete but may name a
193
+ declared byte keeps its term and carries `mayBeVolatile: true`. See the
194
+ [memory model](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#bounded-instruction-reports).
195
+
186
196
  A trace-family query whose entry lies inside a function body names the function's entry in
187
197
  `entryFrame: { "from": <site> }`. The engine traces from there to the query's entry and, when SP
188
198
  sits at one offset from the function's entry SP at every arrival, starts the query in that frame,
@@ -167,6 +167,16 @@ reported as `undecided` and never counts as held. Access reports also give each
167
167
  same of memory the program never reads back, at a checkpoint. See the
168
168
  [controls contract](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#relational-controls).
169
169
 
170
+ Two reads of a byte the model holds no value for read one term until a write that may store it or a
171
+ modeled call, which assumes nothing else changes memory. A query whose program polls memory that
172
+ hardware, DMA or an interrupt handler updates lists those addresses in `volatileMemory`: up to 64
173
+ rows of `segment` (a real-mode paragraph, or 0 in the PE32 flat model), `offset`, `bytes` and
174
+ `evidence`. Each read of a byte inside a declared range reads its own term, ignoring any value the
175
+ path stored, and its `byteProducers` row gives cause `declared volatile` with the read's order and
176
+ the declared row's index in `volatileMemory`. A read whose address is not concrete but may name a
177
+ declared byte keeps its term and carries `mayBeVolatile: true`. See the
178
+ [memory model](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md#bounded-instruction-reports).
179
+
170
180
  A trace-family query whose entry lies inside a function body names the function's entry in
171
181
  `entryFrame: { "from": <site> }`. The engine traces from there to the query's entry and, when SP
172
182
  sits at one offset from the function's entry SP at every arrival, starts the query in that frame,
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "16.0.0"
8
+ version = "16.1.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -35,6 +35,8 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
35
35
  "Declared table continuations are separate conditional paths that spend continuationBudget\n"
36
36
  "(paths, totalSteps, maxSteps, visitLimit, stringIterations); ordinary computed transfers remain stopped.\n"
37
37
  "Port accesses and interrupts are hardware-boundary events; portInputs supplies port reads as assumptions.\n"
38
+ "volatileMemory lists memory hardware or an interrupt may change (segment, offset, bytes, evidence);\n"
39
+ "each read of a declared byte reads its own term, cause declared volatile.\n"
38
40
  "A callModels entry at an INT n site (real mode) returns past the interrupt under its cases (leavesFlags: true\n"
39
41
  "keeps the interrupt's FLAGS word on the stack); other interrupts stop.\n"
40
42
  "callModels[].preservesMemory keeps explicit, bounded pre-call byte scopes across a modeled call; other memory stays unknown.\n"
@@ -1,4 +1,5 @@
1
1
  """Path state for the evidence layer. Instruction semantics come from pypcode (pcode_backend.py)."""
2
+ from bisect import bisect_right
2
3
  from copy import deepcopy
3
4
  from capstone.x86 import X86_OP_REG, X86_OP_IMM, X86_OP_MEM
4
5
  from .values import Value, const, unknown, op, extract, join, resize, sources, address_parts, producers
@@ -28,6 +29,42 @@ MEMORY_CLEARED = ("memory-cleared",)
28
29
  # The ``unwritten`` cause of a byte that no write on the path stored.
29
30
  NO_WRITE = "no write on this path"
30
31
 
32
+ # The ``unwritten`` cause of a read byte that lies inside a range the query's ``volatileMemory`` declares.
33
+ VOLATILE = "declared volatile"
34
+ VOLATILE_FIELDS = {"segment", "offset", "bytes", "evidence"}
35
+ MAX_VOLATILE_RANGES = 64
36
+
37
+
38
+ def volatile_ranges(config, bits, flat):
39
+ """Check the query's ``volatileMemory`` and return its ranges as linear ``(start, end, row)`` triples, sorted.
40
+
41
+ Each row names a concrete ``segment`` (a real-mode paragraph; 0 in the PE32 flat model, whose
42
+ data segment bases are 0), an ``offset``, a byte count and nonempty ``evidence``. Raises
43
+ ``ValueError`` on a malformed row, more than 64 rows, a range past the end of its segment, or
44
+ two ranges that share a linear byte.
45
+ """
46
+ rows = config.get("volatileMemory", [])
47
+ if not isinstance(rows, list) or len(rows) > MAX_VOLATILE_RANGES:
48
+ raise ValueError(f"volatileMemory must be a list of at most {MAX_VOLATILE_RANGES} ranges")
49
+ ranges = []
50
+ for index, row in enumerate(rows):
51
+ if not isinstance(row, dict) or set(row) != VOLATILE_FIELDS:
52
+ raise ValueError("volatileMemory ranges name segment, offset, bytes and evidence")
53
+ segment, offset, size = row["segment"], row["offset"], row["bytes"]
54
+ if type(segment) is not int or not 0 <= segment <= 0xFFFF or flat and segment != 0:
55
+ raise ValueError("volatileMemory segment must be a 16-bit paragraph, or 0 in the PE32 flat model")
56
+ if type(offset) is not int or type(size) is not int or offset < 0 or size < 1 or offset + size > 1 << bits:
57
+ raise ValueError("volatileMemory offset and bytes must give a nonempty range inside the segment")
58
+ if not isinstance(row["evidence"], str) or not row["evidence"].strip():
59
+ raise ValueError("volatileMemory range requires nonempty evidence")
60
+ start = segment * (1 if flat else 16) + offset
61
+ ranges.append((start, start + size, index))
62
+ ranges.sort()
63
+ for before, after in zip(ranges, ranges[1:]):
64
+ if after[0] < before[1]:
65
+ raise ValueError("volatileMemory ranges overlap")
66
+ return tuple(ranges)
67
+
31
68
 
32
69
  class WriteLog:
33
70
  """Append-only list of (key, previous value) pairs, one per register or memory byte a write changes.
@@ -166,6 +203,9 @@ class State:
166
203
  self.conditional = []
167
204
  # Values the query supplies for port reads, by site; trace validates them.
168
205
  self.port_inputs = Shared({row["site"]: row for row in config.get("portInputs", [])})
206
+ # The query's volatileMemory ranges, which every copy of the path shares.
207
+ ranges = volatile_ranges(config, self.bits, self.flat)
208
+ self.volatile = Shared(ranges=ranges, starts=[r[0] for r in ranges])
169
209
  flags = config.get("flags", {})
170
210
  if not isinstance(flags, dict) or set(flags) - {"direction"}:
171
211
  raise ValueError("Only an explicit starting direction flag is supported")
@@ -274,16 +314,46 @@ class State:
274
314
  built): the newest write that may alias it, the write that dropped its value, the modeled
275
315
  call that cleared memory, or none. Reads of the byte name the same term until such an event
276
316
  happens, so a reload across writes that cannot store the byte reads the earlier term, and the
277
- term changes exactly when the reported cause does.
317
+ term changes exactly when the reported cause does. A declared volatile byte's row names the
318
+ read itself, so each read of it has its own term.
278
319
  """
279
- if key in self.unread_memory:
320
+ volatile = unwritten is not None and unwritten["cause"] == VOLATILE
321
+ if key in self.unread_memory and not volatile:
280
322
  return self.unread_memory[key]
281
323
  order = (unwritten or self.unwritten(key))["order"]
282
324
  return f"memory:{'entry' if order is None else order}:{key}"
283
325
 
284
326
  def byte(self, key, unwritten=None):
285
- """The modeled value of one memory byte, or an unknown term produced by the current site."""
286
- return self.memory[key] if key in self.memory else unknown(self.unread_term(key, unwritten), 8, self.at)
327
+ """The modeled value of one memory byte, or an unknown term produced by the current site.
328
+
329
+ A read passes the byte's ``unwritten`` row, which a byte has exactly when the read sees no
330
+ modeled value, so a declared volatile byte reads the term that row names even when the path
331
+ stored a value there.
332
+ """
333
+ if unwritten is None and key in self.memory:
334
+ return self.memory[key]
335
+ return unknown(self.unread_term(key, unwritten), 8, self.at)
336
+
337
+ def volatility(self, key):
338
+ """How a byte relates to the query's volatileMemory ranges.
339
+
340
+ Returns ``("inside", row)`` with the index of the declared row whose range holds the byte when
341
+ the byte has a concrete linear address, ``("possible", None)`` when its address is not
342
+ concrete and the linear bytes it may name overlap a declared range, and ``None`` otherwise.
343
+ """
344
+ ranges = self.volatile["ranges"]
345
+ if not ranges:
346
+ return None
347
+ domain = key_domain(key, self.bits, self.flat)
348
+ if domain is None:
349
+ return "possible", None
350
+ at = bisect_right(self.volatile["starts"], domain[0]) - 1
351
+ if key[0] == ("linear",):
352
+ return ("inside", ranges[at][2]) if at >= 0 and domain[0] < ranges[at][1] else None
353
+ # The range starting at or before the domain overlaps it when it ends past the domain's start;
354
+ # the next range overlaps it when it starts before the domain's end.
355
+ overlaps = at >= 0 and domain[0] < ranges[at][1] or at + 1 < len(ranges) and ranges[at + 1][0] < domain[1]
356
+ return ("possible", None) if overlaps else None
287
357
 
288
358
  def latest_aliasing_write(self, group, domain):
289
359
  """The order of the newest write outside `group` that may have stored a byte of `domain`, or None."""
@@ -317,11 +387,27 @@ class State:
317
387
  return {"cause": "dropped by a modeled call", "order": self.memory_cleared}
318
388
  return {"cause": NO_WRITE, "order": None}
319
389
 
320
- def byte_writer(self, index, key):
321
- """The byteProducers row of one accessed byte: its producers and the write that stored it."""
390
+ def byte_writer(self, index, key, stored=False):
391
+ """The byteProducers row of one accessed byte: its producers and the write that stored it.
392
+
393
+ A row for a byte a read sees (``stored`` false) follows the query's volatileMemory: a byte
394
+ inside a declared range has no producers or writer and an ``unwritten`` row with cause
395
+ ``declared volatile`` and the order of the event this row belongs to, which names its term,
396
+ plus ``volatileMemory``, the index of the declared row. A byte whose address may name a
397
+ declared range carries ``mayBeVolatile: true`` and keeps its ordinary row. ``stored`` rows
398
+ report what a write has just stored.
399
+ """
400
+ volatility = None if stored else self.volatility(key)
401
+ if volatility is not None and volatility[0] == "inside":
402
+ return {"index": index, "producers": [], "writeOrder": None,
403
+ "unwritten": {"cause": VOLATILE, "order": len(self.events)}, "volatileMemory": volatility[1]}
322
404
  if key in self.memory:
323
- return {"index": index, "producers": producers(self.memory[key]), "writeOrder": self.memory_writers.get(key)}
324
- return {"index": index, "producers": [], "writeOrder": None, "unwritten": self.unwritten(key)}
405
+ row = {"index": index, "producers": producers(self.memory[key]), "writeOrder": self.memory_writers.get(key)}
406
+ else:
407
+ row = {"index": index, "producers": [], "writeOrder": None, "unwritten": self.unwritten(key)}
408
+ if volatility is not None:
409
+ row["mayBeVolatile"] = True
410
+ return row
325
411
 
326
412
  def argument_slots(self, return_bytes, window):
327
413
  """The stack bytes above a call's return frame as a read of them would see them now.
@@ -442,11 +528,11 @@ class State:
442
528
  self.memory_groups.setdefault((seg, base), set()).update(keys)
443
529
  value = write
444
530
  missing = []
445
- rows = [self.byte_writer(i, key) for i, key in enumerate(keys)]
531
+ rows = [self.byte_writer(i, key, stored=True) for i, key in enumerate(keys)]
446
532
  else:
447
- missing = [i for i, key in enumerate(keys) if key not in self.memory]
448
533
  # Each row's "unwritten" cause names the term of a byte with no value, so it is built once.
449
534
  rows = [self.byte_writer(i, key) for i, key in enumerate(keys)]
535
+ missing = [row["index"] for row in rows if "unwritten" in row]
450
536
  value = join([self.byte(key, row.get("unwritten")) for key, row in zip(keys, rows)])
451
537
  relevant = []
452
538
  for g in self.guards:
@@ -60,8 +60,10 @@ def capture_scopes(state, model, frame_bytes=0):
60
60
  """Resolve each scope against the pre-call state and snapshot its bytes.
61
61
 
62
62
  Returns ``(values, unread, descriptions)``: the cached bytes, the unknown term names of bytes
63
- the model had no value for, and one report entry per scope. The segment must be concrete. The
64
- base may be concrete, or a symbolic value such as SP or BP at an offset from an unknown entry SP
63
+ the model had no value for, and one report entry per scope. The entry counts the scope's bytes
64
+ in ``cachedBytes``, ``uncachedBytes`` and ``volatileBytes``: a byte inside a range the query's
65
+ ``volatileMemory`` declares counts only as volatile, since a read of it ignores any kept value.
66
+ The segment must be concrete. The base may be concrete, or a symbolic value such as SP or BP at an offset from an unknown entry SP
65
67
  (ADR 0013): the scope's bytes are then keyed by that value, as reads and writes through it are.
66
68
  Raises ``StopPath`` when the segment is not concrete, a concrete interval leaves the address
67
69
  space, two scopes on one base value share a byte, or two scopes on different base values may
@@ -110,10 +112,13 @@ def capture_scopes(state, model, frame_bytes=0):
110
112
  raise StopPath("preservesMemory scope covers the return frame the processor writes below SP")
111
113
  resolved.append(((seg, group_base), set(keys), domain))
112
114
  linear = seg == ("linear",)
113
- cached = 0
115
+ cached = volatile = 0
114
116
  for key in keys:
117
+ # A read of a declared volatile byte ignores the kept value, so it counts apart.
118
+ inside = (state.volatility(key) or ("",))[0] == "inside"
119
+ volatile += inside
115
120
  if key in state.memory:
116
- cached += 1
121
+ cached += not inside
117
122
  values[key] = state.memory[key]
118
123
  else:
119
124
  unread[key] = state.unread_term(key)
@@ -123,7 +128,8 @@ def capture_scopes(state, model, frame_bytes=0):
123
128
  "offset": offset.number,
124
129
  "interval": {"segment": seg, "base": group_base, "start": start, "end": start + size},
125
130
  "linearStart": start if linear else None, "linearEnd": start + size if linear else None, "bytes": size,
126
- "cachedBytes": cached, "uncachedBytes": size - cached, "evidence": scope["evidence"],
131
+ "cachedBytes": cached, "uncachedBytes": size - cached - volatile, "volatileBytes": volatile,
132
+ "evidence": scope["evidence"],
127
133
  "meaning": "explicit pre-call memory-preservation hypothesis; memory outside every scope is unknown"})
128
134
  return values, unread, descriptions
129
135
 
@@ -31,14 +31,16 @@ def far_pointer_word(state, read, low):
31
31
 
32
32
  ``row`` reports the word's value and the producers and writers of its two bytes. ``word`` is the
33
33
  modeled value of those bytes with all their sources, including the markers of declared return
34
- results that ``producers`` leaves out, or None when a byte has no modeled value.
34
+ results that ``producers`` leaves out, or None when a byte has no modeled value or lies in
35
+ declared volatile memory, whose read has no producer.
35
36
  """
36
37
  rows = read["byteProducers"][low // 8:low // 8 + 2]
37
38
  interval = read["interval"]
38
39
  seg, base, start = interval["segment"], interval["base"], interval["start"]
39
40
  keys = [(seg, base, start + i if seg == ("linear",) else (start + i) % (1 << state.bits))
40
41
  for i in (low // 8, low // 8 + 1)]
41
- word = join([state.memory[key] for key in keys]) if all(key in state.memory for key in keys) else None
42
+ word = (join([state.memory[key] for key in keys])
43
+ if all(key in state.memory and "unwritten" not in row for key, row in zip(keys, rows)) else None)
42
44
  return ({"value": None if word is None else word.number,
43
45
  "producers": sorted({p for row in rows for p in row["producers"]}), "bytes": rows}, word)
44
46
 
@@ -0,0 +1,194 @@
1
+ """Declared volatile memory: reads of bytes hardware or an interrupt may change. Synthetic bytes only."""
2
+ import unittest
3
+
4
+ from test_x86 import Code, report, events
5
+ from test_relational_controls import control, verdict
6
+ from test_pe import report as pe_report, events as pe_events
7
+ import test_indirect_far_transfers as far
8
+
9
+ FRAME = {"ds": 0x2000, "ss": 0x3000, "sp": 0xff00}
10
+ # The word at DS:0100 in FRAME.
11
+ WORD = {"segment": 0x2000, "offset": 0x100, "bytes": 2, "evidence": "synthetic polled status word"}
12
+
13
+
14
+ def volatile(**fields):
15
+ return [{**WORD, **fields}]
16
+
17
+
18
+ class ReloadTests(unittest.TestCase):
19
+ def reload(self, store="", registers=FRAME, ranges=(), kind="sameValue"):
20
+ # Load the word at DS:0100, test it, run `store` on the nonzero branch, then reload the word.
21
+ self.c = (Code().emit("a1 00 01 85 c0").label("test").branch("74", "out").label("store").emit(store)
22
+ .label("use").emit("8b 1e 00 01").label("out").emit("c3"))
23
+ if kind == "sameValue":
24
+ rule = control("reload", "order", before={"site": self.c.labels["test"], "event": "branch"},
25
+ branch={"taken": False}, sameValue={"before": "left", "at": "value"},
26
+ at={"site": self.c.labels["use"], "event": "read"})
27
+ else:
28
+ rule = control("reload", "lastWriter", writers=[self.c.labels["store"]],
29
+ at={"site": self.c.labels["use"], "event": "read"})
30
+ self.result = report(self.c, relationalControls=[rule], registers=registers, volatileMemory=list(ranges))
31
+ return verdict(self.result, "reload")
32
+
33
+ def reads(self):
34
+ """The load and the reload on the path that reaches the reload."""
35
+ path = next(p for p in self.result["paths"]
36
+ if any(e["kind"] == "read" and e["site"] == self.c.labels["use"] for e in p["events"]))
37
+ return [e for e in path["events"] if e["kind"] == "read"]
38
+
39
+ def occurrence(self, result):
40
+ return next(o for p in result["paths"] for o in p["occurrences"])
41
+
42
+ def test_a_reload_outside_every_declared_range_keeps_the_tested_word(self):
43
+ # Positive control: the same query holds with no range, and with a range that misses the word.
44
+ for name, ranges in (("no range", ()), ("range elsewhere", volatile(offset=0x200)),
45
+ ("range in another segment", volatile(segment=0x40, offset=0x6C))):
46
+ with self.subTest(name):
47
+ self.assertEqual(self.reload(ranges=ranges)["verdict"], "held")
48
+ for read in self.reads():
49
+ for row in read["byteProducers"]:
50
+ self.assertNotIn("volatileMemory", row)
51
+ self.assertNotIn("mayBeVolatile", row)
52
+
53
+ def test_each_read_of_a_declared_volatile_word_has_its_own_term(self):
54
+ result = self.reload(ranges=volatile())
55
+ self.assertEqual(result["verdict"], "undecided")
56
+ self.assertIn("not shown equal", self.occurrence(result)["reason"])
57
+ first, second = self.reads()
58
+ self.assertNotEqual(first["value"]["expression"], second["value"]["expression"])
59
+ for read in (first, second):
60
+ self.assertEqual(read["missingByteProducers"], [0, 1])
61
+ for row in read["byteProducers"]:
62
+ self.assertEqual(row["unwritten"], {"cause": "declared volatile", "order": read["order"]})
63
+ self.assertEqual((row["producers"], row["writeOrder"], row["volatileMemory"]), ([], None, 0))
64
+
65
+ def test_a_range_over_one_byte_of_the_word_changes_only_that_byte(self):
66
+ self.assertEqual(self.reload(ranges=volatile(offset=0x101, bytes=1))["verdict"], "undecided")
67
+ low, high = self.reads()[1]["byteProducers"]
68
+ self.assertEqual(low["unwritten"]["cause"], "no write on this path")
69
+ self.assertNotIn("volatileMemory", low)
70
+ self.assertEqual((high["unwritten"]["cause"], high["volatileMemory"]), ("declared volatile", 0))
71
+
72
+ def test_a_path_store_to_a_volatile_word_is_not_read_back(self):
73
+ store = "c7 06 00 01 05 00" # mov word [0100], 5
74
+ self.assertEqual(self.reload(store, kind="lastWriter")["verdict"], "held")
75
+ self.assertEqual(self.reads()[1]["value"]["value"], 5)
76
+ result = self.reload(store, ranges=volatile(), kind="lastWriter")
77
+ self.assertEqual(result["verdict"], "undecided")
78
+ reload = self.reads()[1]
79
+ self.assertIsNone(reload["value"]["value"])
80
+ self.assertEqual(self.occurrence(result)["bytes"][0]["unwritten"],
81
+ {"cause": "declared volatile", "order": reload["order"]})
82
+ # The write itself still reports what it stored.
83
+ write = events(self.result, "write")[0]
84
+ self.assertEqual([row["writeOrder"] for row in write["byteProducers"]], [write["order"]] * 2)
85
+ self.assertNotIn("unwritten", write["byteProducers"][0])
86
+
87
+ def test_a_read_whose_address_may_name_a_declared_range_is_flagged_and_keeps_its_term(self):
88
+ # Unknown DS: the word may lie anywhere, so it may lie in the declared range.
89
+ self.assertEqual(self.reload(registers={}, ranges=volatile())["verdict"], "held")
90
+ for read in self.reads():
91
+ self.assertTrue(all(row["mayBeVolatile"] for row in read["byteProducers"]))
92
+ self.assertTrue(all("volatileMemory" not in row for row in read["byteProducers"]))
93
+
94
+ def test_a_concrete_segment_flags_symbolic_offsets_only_when_it_spans_a_range(self):
95
+ # mov bx, [0300]; mov ax, [bx]; ret: the second read has a concrete segment and a symbolic offset.
96
+ code = Code().emit("8b 1e 00 03").label("deref").emit("8b 07 c3")
97
+ for name, ranges, flagged in (("range in DS", volatile(), True),
98
+ ("range outside DS", volatile(segment=0x40, offset=0x6C), False)):
99
+ with self.subTest(name):
100
+ result = report(code, registers=FRAME, volatileMemory=ranges)
101
+ pointer, deref = events(result, "read")
102
+ self.assertNotIn("mayBeVolatile", pointer["byteProducers"][0])
103
+ self.assertEqual("mayBeVolatile" in deref["byteProducers"][0], flagged)
104
+
105
+
106
+ class ProbeTests(unittest.TestCase):
107
+ def test_an_address_probe_over_a_volatile_word_names_its_checkpoint(self):
108
+ # mov [0100], ax; ret, with a lastWriter address at the return's checkpoint and no read of the word.
109
+ code = Code().label("store").emit("a3 00 01").label("end").emit("c3")
110
+ rule = control("slot", "lastWriter", writers=[code.labels["store"]],
111
+ at={"site": code.labels["end"], "event": "checkpoint"},
112
+ address={"segment": "ds", "displacement": 0x100, "width": 2})
113
+ self.assertEqual(verdict(report(code, relationalControls=[rule], registers=FRAME), "slot")["verdict"], "held")
114
+ result = report(code, relationalControls=[rule], registers=FRAME, volatileMemory=volatile())
115
+ self.assertEqual(verdict(result, "slot")["verdict"], "undecided")
116
+ checkpoint = events(result, "checkpoint")[0]
117
+ for row in checkpoint["memoryProbes"][0]["byteProducers"]:
118
+ self.assertEqual(row["unwritten"], {"cause": "declared volatile", "order": checkpoint["order"]})
119
+ self.assertEqual(row["volatileMemory"], 0)
120
+
121
+
122
+ class ScopeTests(unittest.TestCase):
123
+ def test_a_preserved_scope_counts_declared_volatile_bytes_apart(self):
124
+ # mov word [00FE], 7; mov word [0100], 5; mov bx, 00FE; call service (modeled, keeps DS:BX..+4); reload.
125
+ c = (Code().emit("c7 06 fe 00 07 00 c7 06 00 01 05 00 bb fe 00").label("service").branch("e8", "external")
126
+ .label("reload").emit("a1 00 01 c3").label("external").emit("c3"))
127
+ model = {"site": c.labels["service"], "preserves": ["ds", "ss"],
128
+ "preservesMemory": [{"segment": "ds", "base": "bx", "bytes": 4, "evidence": "synthetic kept buffer"}],
129
+ "evidence": "synthetic returning service; other memory unknown", "cases": [{}]}
130
+ for name, ranges, counts, value in (("no range", [], (4, 0, 0), 5), ("range", volatile(), (2, 0, 2), None)):
131
+ with self.subTest(name):
132
+ result = report(c, registers=FRAME, callModels=[model], volatileMemory=ranges)
133
+ kept = result["paths"][0]["conditionalModels"][0]["preservedMemoryScopes"][0]
134
+ self.assertEqual((kept["cachedBytes"], kept["uncachedBytes"], kept["volatileBytes"]), counts)
135
+ reload = next(e for e in events(result, "read") if e["site"] == c.labels["reload"])
136
+ self.assertEqual(reload["value"]["value"], value)
137
+
138
+
139
+ class PollLoopTests(unittest.TestCase):
140
+ def test_a_polled_timer_word_can_change_between_iterations(self):
141
+ # l: mov ax, es:[006C]; cmp ax, dx; je l; ret
142
+ code = Code().label("l").emit("26 a1 6c 00 39 d0").branch("74", "l").emit("c3")
143
+ timer = {"segment": 0x40, "offset": 0x6C, "bytes": 2, "evidence": "synthetic timer counter"}
144
+
145
+ def exits(ranges):
146
+ result = report(code, registers={"es": 0x40}, volatileMemory=ranges, visitLimit=3)
147
+ return sorted(sum(1 for e in p["events"] if e["kind"] == "read") for p in result["paths"] if p["returned"])
148
+
149
+ # Without the range every poll reads one term, so the path leaves only after the first poll.
150
+ self.assertEqual(exits([]), [1])
151
+ self.assertEqual(exits([timer]), [1, 2, 3])
152
+
153
+
154
+ class FarPointerTests(unittest.TestCase):
155
+ def test_a_pointer_in_volatile_memory_is_not_followed(self):
156
+ parts = (far.STORE_OFFSET, far.STORE_SEGMENT, far.CALL, "c3")
157
+ self.assertTrue(far.run(*parts)["paths"][0]["returned"])
158
+ pointer = {"segment": far.FRAME["ds"], "offset": 0x200, "bytes": 4, "evidence": "synthetic vector a handler rewrites"}
159
+ path = far.run(*parts, volatileMemory=[pointer])["paths"][0]
160
+ self.assertEqual(path["stop"], "unresolved call: far pointer offset and segment words unknown")
161
+
162
+
163
+ class ValidationTests(unittest.TestCase):
164
+ def test_malformed_ranges_are_rejected(self):
165
+ for name, ranges, message in (
166
+ ("not a list", WORD, "at most 64 ranges"),
167
+ ("too many", [dict(WORD, offset=n * 2) for n in range(65)], "at most 64 ranges"),
168
+ ("missing evidence", [{k: v for k, v in WORD.items() if k != "evidence"}], "name segment, offset"),
169
+ ("extra field", volatile(width=2), "name segment, offset"),
170
+ ("blank evidence", volatile(evidence=" "), "nonempty evidence"),
171
+ ("segment register name", volatile(segment="es"), "16-bit paragraph"),
172
+ ("empty", volatile(bytes=0), "nonempty range"),
173
+ ("past the segment", volatile(offset=0xFFFF), "nonempty range"),
174
+ ("overlap through another segment", volatile() + volatile(segment=0x2010, offset=0, evidence="x"),
175
+ "overlap")):
176
+ with self.subTest(name):
177
+ with self.assertRaisesRegex(ValueError, message):
178
+ report("c3", volatileMemory=ranges)
179
+
180
+ def test_the_flat_model_takes_linear_offsets_in_segment_zero(self):
181
+ code = bytes.fromhex("a1 00 20 40 00 c3") # mov eax, [00402000]; ret
182
+ with self.assertRaisesRegex(ValueError, "or 0 in the PE32 flat model"):
183
+ pe_report(code, volatileMemory=volatile(segment=1, offset=0x402000))
184
+ read = pe_events(pe_report(code, volatileMemory=volatile(segment=0, offset=0x402002)), "read")[0]
185
+ self.assertEqual([row["unwritten"]["cause"] for row in read["byteProducers"]],
186
+ ["no write on this path"] * 2 + ["declared volatile"] * 2)
187
+
188
+ def test_adjacent_ranges_are_accepted(self):
189
+ result = report("c3", volatileMemory=volatile() + volatile(offset=0x102, evidence="next word"))
190
+ self.assertTrue(result["paths"][0]["returned"])
191
+
192
+
193
+ if __name__ == "__main__":
194
+ unittest.main()