scientific-method-engine 13.7.0__tar.gz → 13.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/reports.py +33 -16
  4. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/trace.py +37 -13
  5. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_successor_rule.py +27 -6
  6. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_x86.py +100 -0
  7. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/.gitignore +0 -0
  8. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/LICENSE +0 -0
  9. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/README.md +0 -0
  10. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/__init__.py +0 -0
  11. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/__main__.py +0 -0
  12. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/cli.py +0 -0
  13. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  14. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  15. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  16. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  17. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  18. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  19. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  20. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  21. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  22. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  23. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  24. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  25. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  26. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  27. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  28. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  29. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  30. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  31. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  32. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  33. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  34. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  35. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  36. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  37. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  38. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  39. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  40. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  41. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  42. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  43. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  44. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  45. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  46. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  47. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  48. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  49. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  50. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  51. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/image.py +0 -0
  52. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/inventory.py +0 -0
  53. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/loops.py +0 -0
  54. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/machine.py +0 -0
  55. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  56. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  57. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  58. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/pe.py +0 -0
  59. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/reach.py +0 -0
  60. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/relational.py +0 -0
  61. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  62. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/src/scientific_method_engine/x86/values.py +0 -0
  63. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/oracle.py +0 -0
  64. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_continuation_budget.py +0 -0
  65. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_dispatch.py +0 -0
  66. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_effect_order.py +0 -0
  67. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_enter_frames.py +0 -0
  68. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_entry_frame.py +0 -0
  69. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_indirect_far_transfers.py +0 -0
  70. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_interrupt_models.py +0 -0
  71. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_inventory_check.py +0 -0
  72. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_loops.py +0 -0
  73. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_memory_scopes.py +0 -0
  74. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_nested_frame_request.py +0 -0
  75. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_oracle.py +0 -0
  76. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_pe.py +0 -0
  77. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_reach.py +0 -0
  78. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_relational_controls.py +0 -0
  79. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_return_checks.py +0 -0
  80. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_table_continuations.py +0 -0
  81. {scientific_method_engine-13.7.0 → scientific_method_engine-13.8.0}/tests/test_term_limit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 13.7.0
3
+ Version: 13.8.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "13.7.0"
8
+ version = "13.8.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -13,7 +13,7 @@ from .memory_scopes import model_scopes
13
13
  from .result_flow import return_flows
14
14
  from .image import Image, integer
15
15
  from .trace import (trace, walk, cfg_step, call_target, unsupported_transfer, uncovered, holding_instruction, base_mnemonic, OVERLAP_REASON, CONTESTED_REASON,
16
- RETURNS, INTERRUPTS, PORTS, PORT_INPUTS, port_width, budget_input)
16
+ RETURNS, INTERRUPTS, PORTS, PORT_INPUTS, port_width, budget_input, modeled_interrupt_sites)
17
17
  from .pcode_backend import interrupt_vector
18
18
 
19
19
 
@@ -236,14 +236,21 @@ CFG_OPERAND = "entry-CFG operand past a stop; values and callee effects unresolv
236
236
  PORT_OPERAND = "operand past a PE32 port access; values and continuation unresolved"
237
237
 
238
238
 
239
- # dependsOn reasons in ``uses`` for the calls and port accesses a conditionalAccesses row is reached past.
239
+ # dependsOn reasons in ``uses`` for the calls, interrupts and port accesses a conditionalAccesses row is reached past.
240
240
  STEPPED_CALL = "call past a stop; assumed to return"
241
241
  STEPPED_PORT = "port access past a stop; assumed to continue"
242
+ STEPPED_INTERRUPT = "interrupt past a stop; assumed to return to the next instruction"
243
+ STEPPED_MODELED_INTERRUPT = "modeled interrupt past a stop; returns to the next instruction as its call model declares"
244
+
245
+
246
+ def _stepped_interrupt(at, modeled):
247
+ """The dependsOn row for the interrupt at ``at``, stepped over past a stop; ``modeled`` holds the sites the query models."""
248
+ return {"site": at, "reason": STEPPED_MODELED_INTERRUPT if at in modeled else STEPPED_INTERRUPT}
242
249
  OPEN_CALL = "call open at a stop inside its callee; continued at its return site, assumed to return"
243
250
 
244
251
 
245
252
  def _function_exit(image, start, limit, follow_flat_ports, cache):
246
- """Walk one function's CFG from ``start``, stepping over calls, and say whether it reaches a return.
253
+ """Walk one function's CFG from ``start``, stepping over calls and interrupts, and say whether it reaches a return.
247
254
 
248
255
  Returns the decoded sites on a route from ``start`` to a return instruction, whether one was
249
256
  reached, and whether the walk stopped at ``limit`` first. A PE32 port access ends its branch
@@ -265,8 +272,9 @@ def _function_exit(image, start, limit, follow_flat_ports, cache):
265
272
  if ins is None:
266
273
  continue
267
274
  seen[at] = ins
268
- # The same successor rule as walk(), except that a call continues only at its return site.
269
- step = cfg_step(image, at, ins, follow_flat_ports, step_over_calls=True)
275
+ # The same successor rule as walk(), except that a call continues only at its return site
276
+ # and an interrupt at the next instruction, as the inventory past a stop follows both.
277
+ step = cfg_step(image, at, ins, follow_flat_ports, step_over_calls=True, follow_interrupts=True)
270
278
  successors[at] = step.successors
271
279
  if step.returns and not (image.flat and base_mnemonic(ins) in ("iret", "iretd")):
272
280
  exits.append(at)
@@ -286,14 +294,15 @@ def _function_exit(image, start, limit, follow_flat_ports, cache):
286
294
  return cache[key]
287
295
 
288
296
 
289
- def _caller_continuations(image, stop, reason, stack, limit, cache):
297
+ def _caller_continuations(image, stop, reason, stack, limit, cache, modeled=frozenset()):
290
298
  """Return sites at which ``uses`` continues its inventory past a stop inside a called function.
291
299
 
292
300
  ``stack`` holds the traced calls still open at ``stop`` as (call site, return site) pairs,
293
301
  outermost first. A return site is continued only when the CFG from the stop, or from the
294
302
  return site inside it, reaches a return of the called function. Each continuation depends on
295
- the stop, on every open call from the stop out to that return site, and on every call or PE32
296
- port access stepped over on the way to those returns. Returns (return site, dependsOn,
303
+ the stop, on every open call from the stop out to that return site, and on every call,
304
+ interrupt or PE32 port access stepped over on the way to those returns, an interrupt at a site in
305
+ ``modeled`` (the query's modeled interrupts) named as modeled. Returns (return site, dependsOn,
297
306
  reached without crossing a PE32 port access) rows and the gaps of walks that reached ``limit``.
298
307
  """
299
308
  rows = []
@@ -317,6 +326,8 @@ def _caller_continuations(image, stop, reason, stack, limit, cache):
317
326
  continue
318
327
  if ins.mnemonic in ("call", "lcall"):
319
328
  depends.append({"site": at, "reason": STEPPED_CALL})
329
+ elif base_mnemonic(ins) in INTERRUPTS:
330
+ depends.append(_stepped_interrupt(at, modeled))
320
331
  elif image.flat and base_mnemonic(ins) in PORTS:
321
332
  depends.append({"site": at, "reason": STEPPED_PORT})
322
333
  depends.append({"site": call_site, "reason": OPEN_CALL})
@@ -343,7 +354,11 @@ def uses(image, config):
343
354
  if not isinstance(controls, list) or len(controls) > 256:
344
355
  raise ValueError("Invalid positive controls")
345
356
  result_limit = integer(config.get("limit", 100), 1, 10000, "result limit")
346
- seen, gaps, _, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000))
357
+ # The entry walk continues past the interrupts the trace continues past under a call model, so
358
+ # the accesses traced after one are on the entry path.
359
+ modeled = modeled_interrupt_sites(image, config.get("callModels", []))
360
+ seen, gaps, _, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000),
361
+ modeled_interrupts=modeled)
347
362
  # A site reached only through a rejected start is as unverified as the start itself.
348
363
  unverified = {g["site"] for g in gaps if g.get("reason") == OVERLAP_REASON} | set(contested)
349
364
  matches, unresolved, unique = [], [], set()
@@ -411,19 +426,19 @@ def uses(image, config):
411
426
  # reading one callee later shows exactly which accesses depended on it.
412
427
  reported = {(e["site"], e["kind"]) for e in matches + unresolved}
413
428
  instruction_limit = config.get("instructionLimit", 10000)
414
- # This inventory assumes execution continues past each stop, so it also follows PE32 port accesses
415
- # and names each one below.
429
+ # This inventory assumes execution continues past each stop, so it also follows interrupts, which
430
+ # it assumes return to the next instruction, and PE32 port accesses, and names each one below.
416
431
  # A stop inside a called function would end the inventory at that function's return. The code
417
432
  # after each call still open at the stop is inventoried too, from the call's return site, and
418
433
  # depends on the stop and on every call between them returning.
419
434
  returning, exit_walks = [], {}
420
435
  for root, stacks in sorted(open_calls.items()):
421
436
  for stack in sorted(stacks):
422
- rows, frame_gaps = _caller_continuations(image, root, stops[root], stack, instruction_limit, exit_walks)
437
+ rows, frame_gaps = _caller_continuations(image, root, stops[root], stack, instruction_limit, exit_walks, modeled)
423
438
  returning.extend(rows)
424
439
  gaps.extend(g for g in frame_gaps if g not in gaps)
425
440
  seeds = list(stops) + [start for start, _, _ in returning]
426
- after_stop, stop_gaps, _, _, _ = (walk(image, seeds, instruction_limit, follow_flat_ports=True)
441
+ after_stop, stop_gaps, _, _, _ = (walk(image, seeds, instruction_limit, follow_flat_ports=True, follow_interrupts=True)
427
442
  if seeds else ({}, [], None, None, None))
428
443
  gaps.extend(g for g in stop_gaps if g["reason"] == "instruction limit")
429
444
  # In PE32 a site the stops reach only by continuing past a port access is named as such, even when
@@ -434,13 +449,15 @@ def uses(image, config):
434
449
  if image.flat and (any(base_mnemonic(ins) in PORTS for ins in after_stop.values())
435
450
  or any(not port_free for _, _, port_free in returning)):
436
451
  port_free_seeds = list(stops) + [start for start, _, port_free in returning if port_free]
437
- before_ports, port_gaps, _, _, _ = walk(image, port_free_seeds, instruction_limit)
452
+ before_ports, port_gaps, _, _, _ = walk(image, port_free_seeds, instruction_limit, follow_interrupts=True)
438
453
  if not any(g["reason"] == "instruction limit" for g in port_gaps):
439
454
  port_only = set(after_stop) - set(before_ports)
440
- # A call past a stop was never traced either, so code after it also depends on it returning.
455
+ # A call or interrupt past a stop was never traced either, so code after it also depends on it returning.
441
456
  starts = [(root, [{"site": root, "reason": reason}]) for root, reason in stops.items()]
442
457
  starts += [(at + ins.size, [{"site": at, "reason": STEPPED_CALL}])
443
458
  for at, ins in after_stop.items() if ins.mnemonic in ("call", "lcall") and at not in stops]
459
+ starts += [(at + ins.size, [_stepped_interrupt(at, modeled)])
460
+ for at, ins in after_stop.items() if base_mnemonic(ins) in INTERRUPTS and at not in stops]
444
461
  if image.flat:
445
462
  starts += [(at + ins.size, [{"site": at, "reason": STEPPED_PORT}])
446
463
  for at, ins in after_stop.items() if base_mnemonic(ins) in PORTS and at not in stops]
@@ -452,7 +469,7 @@ def uses(image, config):
452
469
  named.extend(d for d in row_depends if d not in named)
453
470
  depends = {}
454
471
  for start, row_depends in by_start.items():
455
- reached, _, _, _, _ = walk(image, [start], instruction_limit, follow_flat_ports=True)
472
+ reached, _, _, _, _ = walk(image, [start], instruction_limit, follow_flat_ports=True, follow_interrupts=True)
456
473
  for at in reached:
457
474
  named = depends.setdefault(at, [])
458
475
  named.extend(d for d in row_depends if d not in named)
@@ -103,6 +103,19 @@ def modeled_interrupt(image, ins, site):
103
103
  return None if vector in (1, 3) else vector
104
104
 
105
105
 
106
+ def modeled_interrupt_sites(image, models):
107
+ """The sites in ``models`` (a query's ``callModels``) of the interrupts ``trace`` continues past.
108
+
109
+ Each is an ``INT n`` that ``modeled_interrupt`` accepts. A malformed row is skipped here, since
110
+ ``trace`` rejects it with its own message.
111
+ """
112
+ if not isinstance(models, list):
113
+ return frozenset()
114
+ sites = (m.get("site") for m in models if isinstance(m, dict))
115
+ return frozenset(at for at in sites if type(at) is int and 0 <= at < len(image.data)
116
+ and modeled_interrupt(image, image.decode(at), at) is not None)
117
+
118
+
106
119
  def base_mnemonic(ins):
107
120
  # Capstone names REP/REPNE/BND prefixes in the mnemonic ("repz ret", "rep insb", "bnd jmp").
108
121
  return ins.mnemonic.split()[-1]
@@ -146,9 +159,9 @@ class Step(NamedTuple):
146
159
 
147
160
  ``successors`` are the sites the branch continues at, in the order a walk pushes them; the
148
161
  branch ends at the instruction when there are none. ``returns`` is true for a return
149
- instruction. ``return_site`` is the site a call continues at once its callee returns, or None
150
- for anything else and, when the call's target is followed, for a call whose target is that same
151
- site. ``gaps`` and ``edges`` are the gap and transfer-edge rows ``walk`` reports for the
162
+ instruction. ``return_site`` is the site a call continues at once its callee returns, or an
163
+ interrupt the walk continues past once its handler returns, or None for anything else and, when
164
+ the call's target is followed, for a call whose target is that same site. ``gaps`` and ``edges`` are the gap and transfer-edge rows ``walk`` reports for the
152
165
  instruction. ``supplied`` is true when the successors are the rows of a declared indirect jump
153
166
  table, which never prove an instruction boundary.
154
167
  """
@@ -160,12 +173,13 @@ class Step(NamedTuple):
160
173
  supplied: bool = False
161
174
 
162
175
 
163
- def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False):
176
+ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False,
177
+ modeled_interrupts=frozenset()):
164
178
  """The successors of ``ins`` decoded at ``at``, under the rule ``walk`` and the ``uses`` caller continuation share.
165
179
 
166
180
  An unsupported transfer encoding, a return, an unconditional jump without a resolved target,
167
- ``hlt``, an interrupt unless ``follow_interrupts`` (which continues at the next instruction, as
168
- ``body()`` does) and, unless ``follow_flat_ports``, a PE32 port access end the branch. A
181
+ ``hlt``, an interrupt unless ``follow_interrupts`` or its site is in ``modeled_interrupts`` (either
182
+ continues at the next instruction, as ``body()`` does) and, unless ``follow_flat_ports``, a PE32 port access end the branch. A
169
183
  conditional jump or loop continues at its resolved target and at the next instruction. A declared
170
184
  indirect jump continues at its table rows only. A call continues at its resolved target and at
171
185
  its return site; with ``step_over_calls`` it continues at its return site only and its target
@@ -198,7 +212,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
198
212
  successors.append(target)
199
213
  if m in ("jmp", "ljmp"):
200
214
  return Step(successors, False, None, gaps, edges)
201
- if m == "hlt" or (m in INTERRUPTS and not follow_interrupts):
215
+ if m == "hlt" or (m in INTERRUPTS and not follow_interrupts and at not in modeled_interrupts):
202
216
  gaps.append({"site": at, "reason": "hardware or interrupt boundary"})
203
217
  return Step(successors, False, None, gaps, edges)
204
218
  # In the flat model I/O privilege decides whether a port access faults, so the walk claims nothing after it.
@@ -206,24 +220,29 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
206
220
  if m in PORTS and image.flat and not follow_flat_ports:
207
221
  gaps.append({"site": at, "reason": FLAT_PORT_REASON})
208
222
  return Step(successors, False, None, gaps, edges)
209
- return_site = following if calls and following not in successors else None
223
+ # The site after an interrupt, like a call's return site, is reached only if the handler returns.
224
+ return_site = following if (calls or m in INTERRUPTS) and following not in successors else None
210
225
  successors.append(following)
211
226
  return Step(successors, False, return_site, gaps, edges)
212
227
 
213
228
 
214
- def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset()):
229
+ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset(),
230
+ modeled_interrupts=frozenset()):
215
231
  """Decode the CFG reached from ``entries`` and check its instruction boundaries.
216
232
 
217
233
  A port access in the flat model records a gap and ends that branch, as ``trace`` stops there.
218
234
  ``follow_flat_ports`` continues past it instead, for callers that name each port access their
219
235
  results depend on. ``follow_interrupts`` continues past an interrupt at the next instruction,
220
- for callers that list each interrupt as an assumption. A site in ``stops`` is never decoded:
236
+ for callers that list each interrupt as an assumption. ``modeled_interrupts`` continues past the
237
+ interrupts at those sites only, the ones ``modeled_interrupt_sites`` names, which ``trace``
238
+ continues past under a call model (ADR 0017). A site in ``stops`` is never decoded:
221
239
  the walk reaches it and goes no further, so it is in no returned set.
222
240
  """
223
241
  integer(limit, 1, 100000, "instruction limit")
224
242
  pending, seen, gaps, edges = list(entries), {}, [], []
225
243
  # Decoded successors of each instruction, so a proof can be checked for independence below.
226
- # A call's return site is reached only if the callee returns, so it never proves an overlapping start.
244
+ # A call's return site is reached only if the callee returns, and the site after an interrupt only
245
+ # if its handler returns, so neither proves an overlapping start.
227
246
  successors, returns, supplied_edges = {}, set(), set()
228
247
  while pending:
229
248
  at = pending.pop()
@@ -237,7 +256,8 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
237
256
  gaps.append({"site": at, "reason": "undecoded or unmapped edge"})
238
257
  continue
239
258
  seen[at] = ins
240
- step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts)
259
+ step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts,
260
+ modeled_interrupts=modeled_interrupts)
241
261
  successors[at] = step.successors
242
262
  gaps.extend(step.gaps)
243
263
  edges.extend(step.edges)
@@ -635,6 +655,9 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
635
655
  if r not in ALIASES or type(n) is not int or not 0 <= n < 1 << ALIASES[r][2]:
636
656
  raise ValueError("Invalid model register")
637
657
  model_at = {model["site"]: model for model in models}
658
+ # The boundary walk for declared table targets continues past the interrupts the trace continues past.
659
+ # Only that walk reads the set, so callers that skip declared tables do not decode the model sites.
660
+ modeled_interrupts = modeled_interrupt_sites(image, models) if continue_declared_jumps else frozenset()
638
661
  explicit_continuation_budget = validate_continuation_budget(config)
639
662
  observed_frame = entry_frame(image, config, entry) if arrive is None else None
640
663
  root = State(entry, image, config)
@@ -811,7 +834,8 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
811
834
  if boundary_budget < 1:
812
835
  global_gaps.append({"site": s.at, "reason": "conditional table boundary instruction limit"})
813
836
  return
814
- seen, walk_gaps, _, _, contested = walk(image, [root_entry], boundary_budget)
837
+ seen, walk_gaps, _, _, contested = walk(image, [root_entry], boundary_budget,
838
+ modeled_interrupts=modeled_interrupts)
815
839
  # walk drops rejected overlapping starts and contested instructions from seen,
816
840
  # but it decoded them, so they are charged with the established ones.
817
841
  overlapping = sum(g["reason"] == OVERLAP_REASON for g in walk_gaps)
@@ -4,7 +4,7 @@ from test_x86 import configuration
4
4
  from test_pe import CODE_RAW, fixture as pe_fixture
5
5
  from scientific_method_engine.x86.image import Image
6
6
  from scientific_method_engine.x86.reports import _function_exit
7
- from scientific_method_engine.x86.trace import walk
7
+ from scientific_method_engine.x86.trace import walk, modeled_interrupt_sites
8
8
 
9
9
 
10
10
  def table(code, targets, exhaustive):
@@ -37,9 +37,9 @@ REAL_MODE = [
37
37
  ("call to its return site", "e8 00 00 c3", {}, {3}, {3}, (), False),
38
38
  ("far call", "9a 06 00 00 10 c3 c3", FAR_FIXUP, {5}, {5}, {6}, False),
39
39
  ("far jump", "ea 06 00 00 10 c3 c3", FAR_FIXUP, {6}, {6}, (), False),
40
- ("interrupt", "cd 21 c3", {}, set(), set(), (), False),
41
- ("breakpoint", "cc c3", {}, set(), set(), (), False),
42
- ("overflow interrupt", "ce c3", {}, set(), set(), (), False),
40
+ ("interrupt", "cd 21 c3", {}, {2}, {2}, (), False),
41
+ ("breakpoint", "cc c3", {}, {1}, {1}, (), False),
42
+ ("overflow interrupt", "ce c3", {}, {1}, {1}, (), False),
43
43
  ("hlt", "f4 c3", {}, set(), set(), (), False),
44
44
  ("port input", "ec c3", {}, {1}, {1}, (), False),
45
45
  ("port output", "e6 60 c3", {}, {2}, {2}, (), False),
@@ -59,14 +59,15 @@ FLAT = [
59
59
  ("port input", "ec c3", set(), {1}, (), False),
60
60
  ("port output", "e6 60 c3", set(), {2}, (), False),
61
61
  ("string port output", "f3 6e c3", set(), {2}, (), False),
62
- ("interrupt", "cd 2e c3", set(), set(), (), False),
62
+ ("interrupt", "cd 2e c3", {2}, {2}, (), False),
63
63
  ("hlt", "f4 c3", set(), set(), (), False),
64
64
  ]
65
65
 
66
66
 
67
67
  class SuccessorRuleTests(unittest.TestCase):
68
68
  def check(self, image, start, expected, stepped, returns, follow):
69
- seen, _, _, _, _ = walk(image, [start], follow_flat_ports=follow)
69
+ # The uses inventory past a stop follows interrupts in both walks.
70
+ seen, _, _, _, _ = walk(image, [start], follow_flat_ports=follow, follow_interrupts=True)
70
71
  route, exits, truncated = _function_exit(image, start, 10000, follow, {})
71
72
  walked = {at - start for at in seen} - {0}
72
73
  routed = {at - start for at in route}
@@ -91,6 +92,26 @@ class SuccessorRuleTests(unittest.TestCase):
91
92
  with self.subTest(exhaustive=exhaustive, follow_flat_ports=follow):
92
93
  self.check(Image(data, configuration(data, **extra)), 0, {3, 4}, (), False, follow)
93
94
 
95
+ def test_entry_walk_continues_past_modeled_interrupts_only(self):
96
+ # (code, whether a call model at 0 is used, sites walked without and with the model at 0)
97
+ cases = [("cd 21 c3", True, {0}, {0, 2}), ("cc c3", False, {0}, {0}), ("ce c3", False, {0}, {0})]
98
+ for code, used, without, with_model in cases:
99
+ data = bytes.fromhex(code)
100
+ image = Image(data, configuration(data))
101
+ modeled = modeled_interrupt_sites(image, [{"site": 0}])
102
+ with self.subTest(code):
103
+ self.assertEqual(modeled, frozenset({0}) if used else frozenset())
104
+ seen, gaps, _, _, _ = walk(image, [0])
105
+ self.assertEqual(set(seen), without)
106
+ self.assertIn({"site": 0, "reason": "hardware or interrupt boundary"}, gaps)
107
+ seen, gaps, _, _, _ = walk(image, [0], modeled_interrupts=modeled)
108
+ self.assertEqual(set(seen), with_model)
109
+ self.assertEqual(any(g["reason"] == "hardware or interrupt boundary" for g in gaps), not used)
110
+ # In the PE32 model a call model at an interrupt is not used.
111
+ data, config = pe_fixture("cd 2e c3")
112
+ image = Image(data, config)
113
+ self.assertEqual(modeled_interrupt_sites(image, [{"site": CODE_RAW}]), frozenset())
114
+
94
115
  def test_flat_model_walks_continue_at_the_same_sites(self):
95
116
  for name, code, without, with_ports, stepped, returns in FLAT:
96
117
  data, config = pe_fixture(code)
@@ -2636,6 +2636,97 @@ class ReporterTests(unittest.TestCase):
2636
2636
  self.assertIsNone(later["value"]["value"])
2637
2637
  self.assertFalse(result["negativeUsable"])
2638
2638
 
2639
+ def test_access_past_an_unmodeled_interrupt_is_conditional_and_satisfies_a_control(self):
2640
+ # mov ah,35h; int 21h; mov dx,[0002h]; ret
2641
+ data = bytes.fromhex("b4 35 cd 21 8b 16 02 00 c3")
2642
+ config = configuration(data, query={"offset": 2, "width": 2, "access": "read"}, controls=[4])
2643
+ result = run_report(data, config, "uses")
2644
+ self.assertEqual(result["matches"], [])
2645
+ self.assertEqual(result["unresolvedAccesses"], [])
2646
+ [later] = result["conditionalAccesses"]
2647
+ self.assertEqual((later["site"], later["address"]), (4, "overlaps query"))
2648
+ self.assertEqual(later["dependsOn"], [{"site": 2, "reason": "interrupt handler is not modeled; later effects are not read"}])
2649
+ self.assertIsNone(later["segment"]["value"])
2650
+ self.assertIsNone(later["value"]["value"])
2651
+ self.assertIn({"site": 2, "reason": "hardware or interrupt boundary"}, result["gaps"])
2652
+ self.assertFalse(result["negativeUsable"])
2653
+ # The inventory claims nothing about an instruction that is no use of the query.
2654
+ config["controls"] = [0]
2655
+ with self.assertRaisesRegex(ValueError, "control 0 missed"):
2656
+ run_report(data, config, "uses")
2657
+
2658
+ def test_interrupt_past_a_stop_is_named_and_stepped(self):
2659
+ # An unread call stops the trace; the interrupt after it is stepped over and named.
2660
+ code = Code().branch("e8", "external").emit("cd 21 8b 16 02 00 c3").label("external").emit("c3")
2661
+ data = code.bytes()
2662
+ config = configuration(data, query={"offset": 2, "width": 2}, controls=[5])
2663
+ config["regions"][0]["end"] = code.labels["external"]
2664
+ [later] = run_report(data, config, "uses")["conditionalAccesses"]
2665
+ self.assertEqual(later["site"], 5)
2666
+ self.assertEqual([(d["site"], d["reason"]) for d in later["dependsOn"]],
2667
+ [(0, "unresolved call: outside mapped code"),
2668
+ (3, "interrupt past a stop; assumed to return to the next instruction")])
2669
+ # Under a call model the interrupt is named as modeled, since the query declares its return.
2670
+ config["callModels"] = [{"site": 3, "evidence": "synthetic service returns", "cases": [{}]}]
2671
+ [later] = run_report(data, config, "uses")["conditionalAccesses"]
2672
+ self.assertEqual([(d["site"], d["reason"]) for d in later["dependsOn"]],
2673
+ [(0, "unresolved call: outside mapped code"),
2674
+ (3, "modeled interrupt past a stop; returns to the next instruction as its call model declares")])
2675
+
2676
+ def test_interrupt_stop_inside_a_callee_continues_at_the_callers_return_site(self):
2677
+ # The wrapper stops at its interrupt; the caller's read after the call is still inventoried.
2678
+ code = Code().branch("e8", "wrapper").emit("8b 16 02 00 c3").label("wrapper").emit("b4 35 cd 21 c3")
2679
+ data = code.bytes()
2680
+ wrapper = code.labels["wrapper"]
2681
+ result = run_report(data, configuration(data, query={"offset": 2, "width": 2}, controls=[3]), "uses")
2682
+ later = next(e for e in result["conditionalAccesses"] if e["site"] == 3)
2683
+ self.assertEqual([(d["site"], d["reason"]) for d in later["dependsOn"]],
2684
+ [(0, "call open at a stop inside its callee; continued at its return site, assumed to return"),
2685
+ (wrapper + 2, "interrupt handler is not modeled; later effects are not read")])
2686
+ # A stop before the interrupt names the interrupt it steps over on the way to the return.
2687
+ code = Code().branch("e8", "wrapper").emit("8b 16 02 00 c3").label("wrapper").emit("0f 20 c0 cd 21 c3")
2688
+ data = code.bytes()
2689
+ wrapper = code.labels["wrapper"]
2690
+ result = run_report(data, configuration(data, query={"offset": 2, "width": 2}, controls=[3]), "uses")
2691
+ later = next(e for e in result["conditionalAccesses"] if e["site"] == 3)
2692
+ self.assertEqual([(d["site"], d["reason"]) for d in later["dependsOn"]],
2693
+ [(0, "call open at a stop inside its callee; continued at its return site, assumed to return"),
2694
+ (wrapper, "Unsupported register: cr0"),
2695
+ (wrapper + 3, "interrupt past a stop; assumed to return to the next instruction")])
2696
+
2697
+ def test_access_traced_past_a_modeled_interrupt_is_on_the_entry_path(self):
2698
+ data = bytes.fromhex("b4 35 cd 21 8b 16 02 00 c3")
2699
+ config = configuration(data, query={"offset": 2, "width": 2, "access": "read"}, controls=[4],
2700
+ callModels=[{"site": 2, "evidence": "synthetic service returns", "cases": [{}]}])
2701
+ result = run_report(data, config, "uses")
2702
+ [match] = result["matches"]
2703
+ self.assertEqual(match["site"], 4)
2704
+ self.assertEqual(tuple(match["segment"]["expression"]), ("unknown", "modeled-call:2:ds"))
2705
+ self.assertEqual(result["unresolvedAccesses"], [])
2706
+ self.assertEqual(result["conditionalAccesses"], [])
2707
+ self.assertNotIn({"site": 2, "reason": "hardware or interrupt boundary"}, result["gaps"])
2708
+ self.assertFalse(result["negativeUsable"])
2709
+ # A model at INT 3 is not used, so the walk stops there and the access stays conditional.
2710
+ data = bytes.fromhex("cd 03 8b 16 02 00 c3")
2711
+ config = configuration(data, query={"offset": 2, "width": 2}, controls=[2],
2712
+ callModels=[{"site": 0, "evidence": "synthetic", "cases": [{}]}])
2713
+ result = run_report(data, config, "uses")
2714
+ self.assertEqual(result["matches"], [])
2715
+ self.assertEqual([e["site"] for e in result["conditionalAccesses"]], [2])
2716
+ self.assertIn({"site": 0, "reason": "hardware or interrupt boundary"}, result["gaps"])
2717
+
2718
+ def test_declared_table_target_past_a_modeled_interrupt_is_a_verified_boundary(self):
2719
+ c = Code().emit("cd 21 ff e3").label("target").emit("c3").label("table")
2720
+ data = c.bytes() + c.labels["target"].to_bytes(2, "little")
2721
+ cfg = configuration(data, callModels=[{"site": 0, "evidence": "synthetic service returns", "cases": [{}]}],
2722
+ indirectJumps=[{"site": 2, "evidence": "synthetic table consumer", "exhaustive": True,
2723
+ "table": {"start": c.labels["table"], "count": 1, "stride": 2,
2724
+ "evidence": "synthetic table words"}}])
2725
+ cfg["regions"][0]["end"] = c.labels["table"]
2726
+ r = run_report(data, cfg, "trace")
2727
+ self.assertEqual(len(r["declaredContinuationPaths"]), 1)
2728
+ self.assertFalse([g for g in r["gaps"] if "boundary is unresolved" in g["reason"]])
2729
+
2639
2730
  def test_access_past_two_unread_calls_names_both(self):
2640
2731
  code = Code().branch("e8", "external").branch("e8", "external").emit("a0 20 02 c3").label("external").emit("c3")
2641
2732
  data = code.bytes()
@@ -3042,6 +3133,15 @@ class ReporterTests(unittest.TestCase):
3042
3133
  result=report(code,"incoming",target=code.labels["helper"])
3043
3134
  self.assertEqual({g["site"] for g in result["gaps"] if "overlapping" in g["reason"]},{1,3,4})
3044
3135
 
3136
+ def test_site_after_a_modeled_interrupt_does_not_prove_an_overlapping_start(self):
3137
+ # The helper jumps into the interrupt's vector byte, so the RET after the interrupt is never a start.
3138
+ data = bytes.fromhex("90 cd 21 c3 90 90 90 90 eb f8")
3139
+ config = configuration(data, query={"offset": 0x200, "width": 2},
3140
+ callModels=[{"site": 1, "evidence": "synthetic service returns", "cases": [{}]}])
3141
+ config["regions"][0]["entries"] = [0, 8]
3142
+ result = run_report(data, config, "uses")
3143
+ self.assertEqual({g["site"] for g in result["gaps"] if g.get("reason") == OVERLAP_REASON}, {1, 2, 3})
3144
+
3045
3145
  def test_call_reached_only_through_rejected_start_is_contested_not_confirmed(self):
3046
3146
  # Entries 0 (mov ax) and 1 (nop) conflict; the call at 3 is reached from both but from no accepted start.
3047
3147
  data=bytes.fromhex("b8 90 90 e8 04 00 c7 06 00 02 90 c3 c3")