scientific-method-engine 18.0.0__tar.gz → 18.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/cli.py +6 -1
  4. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/reach.py +164 -32
  5. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/trace.py +16 -7
  6. scientific_method_engine-18.2.0/tests/test_reach.py +387 -0
  7. scientific_method_engine-18.0.0/tests/test_reach.py +0 -190
  8. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/.gitignore +0 -0
  9. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/LICENSE +0 -0
  10. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/README.md +0 -0
  11. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/__init__.py +0 -0
  12. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/__main__.py +0 -0
  13. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  14. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  15. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  16. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  17. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  18. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  19. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  20. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  21. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  22. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  23. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  24. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  25. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  26. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  27. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  28. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  29. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  30. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  31. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  32. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  33. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  34. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  35. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  36. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  37. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  38. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  39. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  40. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  41. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  42. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  43. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  44. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  45. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  46. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  47. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  48. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  49. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  50. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  51. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  52. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/image.py +0 -0
  53. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/inventory.py +0 -0
  54. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/loops.py +0 -0
  55. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/machine.py +0 -0
  56. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  57. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  58. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  59. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pe.py +0 -0
  60. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/relational.py +0 -0
  61. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/reports.py +0 -0
  62. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  63. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/values.py +0 -0
  64. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/oracle.py +0 -0
  65. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_config_encoding.py +0 -0
  66. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_continuation_budget.py +0 -0
  67. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_dispatch.py +0 -0
  68. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_effect_order.py +0 -0
  69. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_enter_frames.py +0 -0
  70. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_entry_frame.py +0 -0
  71. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_indirect_far_transfers.py +0 -0
  72. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_interrupt_models.py +0 -0
  73. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_inventory_check.py +0 -0
  74. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_join_terms.py +0 -0
  75. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_loops.py +0 -0
  76. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_memory_scopes.py +0 -0
  77. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_nested_frame_request.py +0 -0
  78. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_oracle.py +0 -0
  79. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_pe.py +0 -0
  80. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_relational_controls.py +0 -0
  81. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_return_checks.py +0 -0
  82. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_successor_rule.py +0 -0
  83. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_table_continuations.py +0 -0
  84. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_term_limit.py +0 -0
  85. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_upstream_gaps.py +0 -0
  86. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_volatile_memory.py +0 -0
  87. {scientific_method_engine-18.0.0 → scientific_method_engine-18.2.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 18.0.0
3
+ Version: 18.2.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "18.0.0"
8
+ version = "18.2.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -23,7 +23,12 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
23
23
  "or continues at another address than the next instruction (ghidraFallsThroughElsewhere).\n"
24
24
  "reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
25
25
  "and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
26
- "a leaves routine is reached but not read, and its reason is repeated in the report.\n"
26
+ "a leaves routine is reached but not read, and its reason is repeated in the report;\n"
27
+ "a call to a noReturn routine and a noReturn interrupt do not continue at the next instruction,\n"
28
+ "and a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
29
+ "is reported contradicted;\n"
30
+ "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
31
+ "and a failed control fails the report saying whether it was reached.\n"
27
32
  "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
28
33
  "that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
29
34
  "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
@@ -1,7 +1,8 @@
1
1
  """Transitive reachability from established starts to target sites over the entry-path CFG."""
2
2
  from collections import deque
3
3
  from .image import integer
4
- from .trace import walk, cfg_step, base_mnemonic, unsupported_transfer, holding_instruction, INTERRUPTS, RETURNS, LIMIT_REASON
4
+ from .trace import (walk, cfg_step, base_mnemonic, unsupported_transfer, holding_instruction, INTERRUPTS, RETURNS,
5
+ LIMIT_REASON, OVERLAP_REASON, UNDECODED_REASON)
5
6
  from .pcode_backend import interrupt_vector
6
7
 
7
8
  # The virtual root of the dominator computation; no file offset is negative.
@@ -11,9 +12,13 @@ UNSUPPORTED = "unsupported control-transfer frame encoding"
11
12
  LEAF_OVERLAP = "leaf start inside a reached instruction; the leaf is not decoded, so its boundary is unchecked"
12
13
 
13
14
 
14
- def _sites(value, label, image, limit=256):
15
- if not isinstance(value, list) or not 1 <= len(value) <= limit or len(set(map(repr, value))) != len(value):
16
- raise ValueError(f"{label} must be a list of 1..{limit} distinct file offsets")
15
+ def _text(ins):
16
+ return (ins.mnemonic + " " + ins.op_str).strip()
17
+
18
+
19
+ def _sites(value, label, image, limit=256, least=1):
20
+ if not isinstance(value, list) or not least <= len(value) <= limit or len(set(map(repr, value))) != len(value):
21
+ raise ValueError(f"{label} must be a list of {least}..{limit} distinct file offsets")
17
22
  for at in value:
18
23
  integer(at, 0, len(image.data) - 1, label)
19
24
  if image.region(at) is None:
@@ -41,6 +46,59 @@ def _leaves(value, image, starts):
41
46
  return leaves
42
47
 
43
48
 
49
+ def _no_return(value, image):
50
+ """The declared routines and interrupt sites that never return, each with its reason."""
51
+ if not isinstance(value, list) or len(value) > 256:
52
+ raise ValueError("noReturn must be a list of at most 256 objects")
53
+ routines, interrupts = {}, {}
54
+ for entry in value:
55
+ if not isinstance(entry, dict) or set(entry) not in ({"routine", "reason"}, {"interrupt", "reason"}):
56
+ raise ValueError("Each noReturn entry needs reason and exactly one of routine and interrupt")
57
+ kind = "routine" if "routine" in entry else "interrupt"
58
+ at = integer(entry[kind], 0, len(image.data) - 1, f"noReturn {kind}")
59
+ if image.region(at) is None:
60
+ raise ValueError(f"noReturn {kind} {at} is outside declared code")
61
+ if not isinstance(entry["reason"], str) or not entry["reason"].strip():
62
+ raise ValueError("A noReturn entry needs a nonempty reason")
63
+ found = routines if kind == "routine" else interrupts
64
+ if at in found:
65
+ raise ValueError(f"Duplicate noReturn {kind}")
66
+ if kind == "interrupt":
67
+ ins = image.decode(at)
68
+ if ins is None or base_mnemonic(ins) not in INTERRUPTS:
69
+ raise ValueError(f"noReturn interrupt {at} is not an interrupt instruction")
70
+ vector, conditional = interrupt_vector(image.flat, ins, at)
71
+ if conditional:
72
+ raise ValueError(f"noReturn interrupt {at} is conditional and continues when it does not interrupt")
73
+ found[at] = {"reason": entry["reason"], "vector": vector, "following": at + ins.size}
74
+ continue
75
+ found[at] = entry["reason"]
76
+ return routines, interrupts
77
+
78
+
79
+ def _returns_from(graph, seen, returning_leaves, routine):
80
+ """Where ``routine``'s own paths return: each return instruction, and each leaf they enter other than by a call.
81
+
82
+ Calls are stepped over at the return sites the graph keeps. The walk assumes that a leaf returns,
83
+ so a jump, branch, table row or fall-through into one in ``returning_leaves`` returns too.
84
+ """
85
+ found, stack, visited = [], [routine], set()
86
+ while stack:
87
+ at = stack.pop()
88
+ if at in visited:
89
+ continue
90
+ visited.add(at)
91
+ if at in returning_leaves:
92
+ found.append(at)
93
+ continue
94
+ if at not in seen:
95
+ continue
96
+ if base_mnemonic(seen[at]) in RETURNS:
97
+ found.append(at)
98
+ stack.extend(s for s, kind in graph.get(at, []) if kind != "call")
99
+ return sorted(found)
100
+
101
+
44
102
  def _kinds(at, ins, step):
45
103
  """Each successor of ``step`` with how the CFG reaches it: call, return, table, interrupt, jump or fall."""
46
104
  m, following = base_mnemonic(ins), at + ins.size
@@ -95,8 +153,10 @@ def reach(image, config):
95
153
  The walk decodes from ``starts`` (established region entries) and follows every resolved call
96
154
  into its callee and on at its return site, every resolved jump and branch, the rows of a
97
155
  declared indirect jump table, and every interrupt to the next instruction. A ``leaves`` routine
98
- is reached but never decoded. Each reached target gets one chain with the fewest calls, the
99
- assumptions its route rests on, and the routines on every route to it within the read graph.
156
+ is reached but never decoded. A call to a ``noReturn`` routine and a ``noReturn`` interrupt site
157
+ do not continue at the next instruction. Each reached target gets one chain with the fewest
158
+ calls, the assumptions its route rests on, and the routines on every route to it within the
159
+ read graph.
100
160
  """
101
161
  from .reports import entries
102
162
  starts = _sites(config.get("starts"), "starts", image)
@@ -106,20 +166,33 @@ def reach(image, config):
106
166
  raise ValueError(f"Reach start {at} must be an established region entry")
107
167
  targets = _sites(config.get("targets"), "targets", image)
108
168
  leaves = _leaves(config.get("leaves", []), image, set(starts))
109
- controls = config.get("controls", [])
110
- if (not isinstance(controls, list) or len(controls) > 256 or any(type(at) is not int for at in controls)
111
- or len(set(controls)) != len(controls)):
112
- raise ValueError("Positive controls must be at most 256 distinct file offsets")
169
+ controls = _sites(config.get("controls", []), "controls", image, least=0)
170
+ instruction_controls = _sites(config.get("instructionControls", []), "instructionControls", image, least=0)
171
+ # The walk decodes a site as image.decode does, so a control whose bytes are no call can never pass.
172
+ for at in controls:
173
+ ins = image.decode(at)
174
+ if ins is None or base_mnemonic(ins) not in CALLS:
175
+ decoded = f"decodes as {_text(ins)}" if ins is not None else "does not decode"
176
+ raise ValueError(f"control {at} is not a call site ({decoded}); controls are call sites the walk must "
177
+ "reach and resolve, and instructionControls are sites the walk must decode")
178
+ # The walk decodes every start it is given, so a start as an instruction control would pass whatever the walk misses.
179
+ for at in instruction_controls:
180
+ if at in starts:
181
+ raise ValueError(f"instruction control {at} is a start; the walk decodes every start, so it shows nothing "
182
+ "the walk found")
113
183
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
114
184
  instruction_limit = config.get("instructionLimit", 10000)
115
- seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True, stops=frozenset(leaves))
185
+ no_return_routines, no_return_interrupts = _no_return(config.get("noReturn", []), image)
186
+ declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
187
+ seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
188
+ stops=frozenset(leaves), **declared)
116
189
 
117
190
  graph, unresolved, interrupts, resolved_calls = {}, [], [], {}
118
191
  for at, ins in sorted(seen.items()):
119
- step = cfg_step(image, at, ins, follow_interrupts=True)
192
+ step = cfg_step(image, at, ins, follow_interrupts=True, **declared)
120
193
  graph[at] = [(s, kind) for s, kind in _kinds(at, ins, step) if s in seen or s in leaves]
121
194
  m = base_mnemonic(ins)
122
- text = (ins.mnemonic + " " + ins.op_str).strip()
195
+ text = _text(ins)
123
196
  kind = "call" if m in CALLS else "return" if m in RETURNS else "jump"
124
197
  if unsupported_transfer(image, ins):
125
198
  unresolved.append({"site": at, "instruction": text, "kind": kind, "reason": UNSUPPORTED})
@@ -129,7 +202,7 @@ def reach(image, config):
129
202
  "reason": edge["provenance"].get("reason", "target outside declared regions")})
130
203
  elif m in CALLS:
131
204
  resolved_calls[at] = edge["target"]
132
- if m in INTERRUPTS:
205
+ if m in INTERRUPTS and at not in no_return_interrupts:
133
206
  vector, conditional = interrupt_vector(image.flat, ins, at)
134
207
  interrupts.append({"site": at, "instruction": text, "vector": vector, "conditional": conditional})
135
208
  for at in leaves:
@@ -191,6 +264,19 @@ def reach(image, config):
191
264
  "route": {"assumedReturns": assumed_returns[::-1], "declaredTableJumps": tables[::-1],
192
265
  "interruptsContinued": continued[::-1]}}
193
266
 
267
+ # The walk decodes both instructions of an unresolved overlap and keeps neither, so their starts are not in seen.
268
+ overlapping = {g["site"] for g in walk_gaps if g["reason"] == OVERLAP_REASON}
269
+ # A site an edge leads to whose bytes do not decode is reached, though no instruction starts there.
270
+ undecodable = {g["site"] for g in walk_gaps if g["reason"] == UNDECODED_REASON}
271
+
272
+ def unreached(site):
273
+ holder = holding_instruction(seen, site)
274
+ if holder is not None:
275
+ return {"status": "inside a reached instruction", "insideInstruction": holder}
276
+ return {"status": "start of a contested instruction" if site in contested
277
+ else "start of an unresolved overlapping instruction" if site in overlapping
278
+ else "reached but not decodable" if site in undecodable else "not reached"}
279
+
194
280
  rows = []
195
281
  for target in targets:
196
282
  row = {"target": target}
@@ -203,10 +289,7 @@ def reach(image, config):
203
289
  row |= {"reached": True, "routine": routine[target], "leaf": target in leaves, **chain(target),
204
290
  "throughEveryRoute": cut[::-1]}
205
291
  else:
206
- holder = holding_instruction(seen, target)
207
- row |= {"reached": False, "status": "inside a reached instruction" if holder is not None
208
- else "start of a contested instruction" if target in contested else "not reached",
209
- **({"insideInstruction": holder} if holder is not None else {})}
292
+ row |= {"reached": False, **unreached(target)}
210
293
  rows.append(row)
211
294
 
212
295
  for row in unresolved + interrupts:
@@ -216,38 +299,87 @@ def reach(image, config):
216
299
  # The walk never decodes a leaf, so its overlap check cannot see a leaf start inside a decoded instruction.
217
300
  gaps += [{"site": at, "reason": LEAF_OVERLAP, "insideInstruction": holding_instruction(seen, at)}
218
301
  for at in sorted(leaves) if at in distance and holding_instruction(seen, at) is not None]
219
- for at in controls:
220
- if at not in resolved_calls:
221
- raise ValueError(f"Positive control {at} missed or not resolved")
302
+ # Kept apart from the gap rows, which the result limit can cut.
303
+ stopped = any(g["reason"] == LIMIT_REASON for g in walk_gaps)
304
+ # A control at a site a noReturn declaration kept the walk from names that declaration, which may be the cause.
305
+ cut_by = {at + seen[at].size: f"the call at {at} to noReturn routine {target}"
306
+ for at, target in sorted(resolved_calls.items()) if target in no_return_routines}
307
+ cut_by |= {row["following"]: f"the noReturn interrupt at {at}"
308
+ for at, row in no_return_interrupts.items() if at in seen}
309
+ # Not reached and reached-but-unresolved need different fixes, so each failure says which it is.
310
+ failures = []
311
+ for label, sites, passed in (("control", controls, resolved_calls), ("instruction control", instruction_controls, seen)):
312
+ for at in sites:
313
+ if at in passed:
314
+ continue
315
+ # Only a call-site control can be decoded and still fail.
316
+ if at in seen:
317
+ reasons = sorted({row["reason"] for row in unresolved if row["site"] == at})
318
+ failures.append(f"{label} {at} is reached but its call target is unresolved ({'; '.join(reasons)})")
319
+ elif at in leaves and at in distance:
320
+ failures.append(f"{label} {at} is the start of a leaf, which the walk reaches but does not decode")
321
+ else:
322
+ state = unreached(at)
323
+ detail = (f"inside the reached instruction at {state['insideInstruction']}" if "insideInstruction" in state
324
+ else "the " + state["status"] if state["status"].startswith("start") else state["status"])
325
+ cut = cut_by.get(at)
326
+ failures.append(f"{label} {at} is {detail}"
327
+ + (f" (it follows {cut})" if cut else "")
328
+ + (" (the walk stopped at its instruction limit)" if stopped else ""))
329
+ if failures:
330
+ raise ValueError("Positive controls failed: " + "; ".join(failures))
222
331
  call_sites = {}
223
332
  for site, target in sorted(resolved_calls.items()):
224
333
  call_sites.setdefault(target, []).append(site)
225
334
  leaf_rows = [{"routine": at, "reason": reason, "reached": at in distance, "callSites": call_sites.get(at, [])}
226
335
  for at, reason in leaves.items()]
227
- # Kept apart from the gap rows, which the result limit can cut.
228
- stopped = any(g["reason"] == LIMIT_REASON for g in walk_gaps)
336
+ # Each call the declaration kept from its return site, so a reviewer can check what follows it.
337
+ no_return_rows, returning_leaves = [], set(leaves) - set(no_return_routines)
338
+ for at, reason in no_return_routines.items():
339
+ returns = _returns_from(graph, seen, returning_leaves, at)
340
+ no_return_rows.append({"routine": at, "reason": reason, "reached": at in distance, "read": at in seen,
341
+ "returnSites": returns, "contradicted": bool(returns),
342
+ "callSites": [{"site": site, "following": site + seen[site].size,
343
+ "followingRead": site + seen[site].size in seen}
344
+ for site in call_sites.get(at, [])]})
345
+ no_return_rows += [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
346
+ "following": row["following"], "followingRead": row["following"] in seen}
347
+ for at, row in no_return_interrupts.items()]
348
+ contradicted = any(row.get("contradicted") for row in no_return_rows)
229
349
  counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
230
350
  "interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
231
- return {"starts": starts, "targets": rows, "leaves": leaf_rows,
351
+ assumptions = ["each reached call returns to its next instruction"
352
+ + (", except a call to a noReturn routine" if no_return_routines else ""),
353
+ "each reached interrupt returns to its next instruction"
354
+ + (", except at a noReturn interrupt site" if no_return_interrupts else "")
355
+ + "; interrupt handlers are not read",
356
+ "each leaf calls nothing, for the reason it gives",
357
+ "each declared indirect jump table holds the routes its declaration gives"]
358
+ if no_return_rows:
359
+ assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
360
+ return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return_rows,
232
361
  "reachedRoutines": sorted(routine_starts), "counts": counts,
233
362
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
234
363
  "contested": sorted(contested)[:limit],
235
364
  "truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested)),
236
365
  "instructionLimitReached": stopped,
237
366
  "controls": [{"site": at, "target": resolved_calls[at]} for at in controls],
238
- "negativeUsable": bool(controls) and not (stopped or unresolved or gaps or contested),
239
- "assumptions": ["each reached call returns to its next instruction",
240
- "each reached interrupt returns to its next instruction; interrupt handlers are not read",
241
- "each leaf calls nothing, for the reason it gives",
242
- "each declared indirect jump table holds the routes its declaration gives"],
367
+ "instructionControls": [{"site": at, "instruction": _text(seen[at]), "routine": routine.get(at)}
368
+ for at in instruction_controls],
369
+ "negativeUsable": bool(controls or instruction_controls)
370
+ and not (stopped or unresolved or gaps or contested or contradicted),
371
+ "assumptions": assumptions,
243
372
  "exclusions": ["computed call and jump targets (listed in unresolved)", "unrelocated far calls",
244
373
  "code reached only from outside the starts", "runtime reachability"],
245
374
  "interpretation": "Routes over the decoded entry-path CFG from the starts. A chain has the fewest calls; "
246
375
  "throughEveryRoute lists the routine starts every read route to the target passes, "
247
376
  "and an unresolved transfer may add a route that passes none of them. "
248
- "negativeUsable needs controls, a walk that did not stop at its instruction limit, "
249
- "and no unresolved transfer, gap or contested instruction, "
250
- "and still rests on the listed assumptions and leaves. "
377
+ "negativeUsable needs a control of either kind, a walk that did not stop at its instruction limit, "
378
+ "and no unresolved transfer, gap, contested instruction or contradicted noReturn "
379
+ "routine, and still rests on the listed assumptions, leaves and noReturn declarations. "
380
+ "A noReturn routine is contradicted when a return instruction, or a leaf entered "
381
+ "other than by a call, lies on its own read paths; an empty returnSites means only "
382
+ "that the walk read none. "
251
383
  "When instructionLimitReached holds, the walk stopped before reading all it reaches: "
252
384
  "every list and count covers only the part read, which part depends on the walk order, "
253
385
  "and an unreached target may lie past the stop."}
@@ -88,6 +88,8 @@ def far_pointer_transfer(state, image):
88
88
  # The gap reason walk() records when it stops at its instruction limit with code left to read.
89
89
  LIMIT_REASON = "instruction limit"
90
90
  OVERLAP_REASON = "overlapping entry-path instructions; boundary unresolved"
91
+ # The gap reason walk() records at a site it reaches but cannot decode.
92
+ UNDECODED_REASON = "undecoded or unmapped edge"
91
93
  RETURNS = {"ret": "near return", "retf": "far return", "iret": "interrupt return", "iretd": "interrupt return"}
92
94
  INTERRUPTS = ("int", "int1", "int3", "into")
93
95
  PORTS = ("in", "out", "insb", "insw", "insd", "outsb", "outsw", "outsd")
@@ -178,7 +180,7 @@ class Step(NamedTuple):
178
180
 
179
181
 
180
182
  def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False,
181
- modeled_interrupts=frozenset()):
183
+ modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset()):
182
184
  """The successors of ``ins`` decoded at ``at``, under the rule ``walk`` and the ``uses`` caller continuation share.
183
185
 
184
186
  An unsupported transfer encoding, a return, an unconditional jump without a resolved target,
@@ -187,7 +189,9 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
187
189
  conditional jump or loop continues at its resolved target and at the next instruction. A declared
188
190
  indirect jump continues at its table rows only. A call continues at its resolved target and at
189
191
  its return site; with ``step_over_calls`` it continues at its return site only and its target
190
- is neither resolved nor reported.
192
+ is neither resolved nor reported. A call whose resolved target is in ``no_return_calls``, and an
193
+ interrupt whose site is in ``no_return_interrupts``, have no return site: the query declared that
194
+ they never come back.
191
195
  """
192
196
  m, following = base_mnemonic(ins), at + ins.size
193
197
  if unsupported_transfer(image, ins):
@@ -214,8 +218,10 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
214
218
  gaps.append({"site": at, "reason": provenance.get("reason", "target outside declared regions")})
215
219
  else:
216
220
  successors.append(target)
217
- if m in ("jmp", "ljmp"):
221
+ if m in ("jmp", "ljmp") or (calls and target in no_return_calls):
218
222
  return Step(successors, False, None, gaps, edges)
223
+ if m in INTERRUPTS and at in no_return_interrupts:
224
+ return Step(successors, False, None, gaps, edges)
219
225
  if m == "hlt" or (m in INTERRUPTS and not follow_interrupts and at not in modeled_interrupts):
220
226
  gaps.append({"site": at, "reason": "hardware or interrupt boundary"})
221
227
  return Step(successors, False, None, gaps, edges)
@@ -231,7 +237,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
231
237
 
232
238
 
233
239
  def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset(),
234
- modeled_interrupts=frozenset()):
240
+ modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset()):
235
241
  """Decode the CFG reached from ``entries`` and check its instruction boundaries.
236
242
 
237
243
  A port access in the flat model records a gap and ends that branch, as ``trace`` stops there.
@@ -240,7 +246,9 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
240
246
  for callers that list each interrupt as an assumption. ``modeled_interrupts`` continues past the
241
247
  interrupts at those sites only, the ones ``modeled_interrupt_sites`` names, which ``trace``
242
248
  continues past under a call model (ADR 0017). A site in ``stops`` is never decoded:
243
- the walk reaches it and goes no further, so it is in no returned set.
249
+ the walk reaches it and goes no further, so it is in no returned set. ``no_return_calls`` and
250
+ ``no_return_interrupts`` end the branch at a call to one of those routines and at one of those
251
+ interrupt sites, as ``cfg_step`` describes.
244
252
  """
245
253
  instruction_limit(image, limit)
246
254
  pending, seen, gaps, edges = list(entries), {}, [], []
@@ -259,11 +267,12 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
259
267
  break
260
268
  ins = image.decode(at)
261
269
  if ins is None:
262
- gaps.append({"site": at, "reason": "undecoded or unmapped edge"})
270
+ gaps.append({"site": at, "reason": UNDECODED_REASON})
263
271
  continue
264
272
  seen[at] = ins
265
273
  step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts,
266
- modeled_interrupts=modeled_interrupts)
274
+ modeled_interrupts=modeled_interrupts, no_return_calls=no_return_calls,
275
+ no_return_interrupts=no_return_interrupts)
267
276
  successors[at] = step.successors
268
277
  gaps.extend(step.gaps)
269
278
  edges.extend(step.edges)