scientific-method-engine 18.1.0__tar.gz → 18.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/cli.py +3 -0
  4. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/reach.py +101 -13
  5. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/trace.py +13 -6
  6. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_reach.py +125 -0
  7. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/.gitignore +0 -0
  8. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/LICENSE +0 -0
  9. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/README.md +0 -0
  10. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/__init__.py +0 -0
  11. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/__main__.py +0 -0
  12. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  13. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  14. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  15. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  16. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  17. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  18. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  19. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  20. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  21. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  22. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  23. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  24. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  25. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  26. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  27. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  28. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  29. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  30. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  31. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  32. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  33. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  34. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  35. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  36. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  37. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  38. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  39. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  40. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  41. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  42. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  43. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  44. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  45. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  46. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  47. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  48. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  49. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  50. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  51. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/image.py +0 -0
  52. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/inventory.py +0 -0
  53. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/loops.py +0 -0
  54. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/machine.py +0 -0
  55. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  56. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  57. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  58. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/pe.py +0 -0
  59. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/relational.py +0 -0
  60. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/reports.py +0 -0
  61. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  62. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/src/scientific_method_engine/x86/values.py +0 -0
  63. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/oracle.py +0 -0
  64. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_config_encoding.py +0 -0
  65. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_continuation_budget.py +0 -0
  66. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_dispatch.py +0 -0
  67. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_effect_order.py +0 -0
  68. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_enter_frames.py +0 -0
  69. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_entry_frame.py +0 -0
  70. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_indirect_far_transfers.py +0 -0
  71. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_interrupt_models.py +0 -0
  72. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_inventory_check.py +0 -0
  73. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_join_terms.py +0 -0
  74. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_loops.py +0 -0
  75. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_memory_scopes.py +0 -0
  76. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_nested_frame_request.py +0 -0
  77. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_oracle.py +0 -0
  78. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_pe.py +0 -0
  79. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_relational_controls.py +0 -0
  80. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_return_checks.py +0 -0
  81. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_successor_rule.py +0 -0
  82. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_table_continuations.py +0 -0
  83. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_term_limit.py +0 -0
  84. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_upstream_gaps.py +0 -0
  85. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_volatile_memory.py +0 -0
  86. {scientific_method_engine-18.1.0 → scientific_method_engine-18.2.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 18.1.0
3
+ Version: 18.2.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "18.1.0"
8
+ version = "18.2.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -24,6 +24,9 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
24
24
  "reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
25
25
  "and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
26
26
  "a leaves routine is reached but not read, and its reason is repeated in the report;\n"
27
+ "a call to a noReturn routine and a noReturn interrupt do not continue at the next instruction,\n"
28
+ "and a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
29
+ "is reported contradicted;\n"
27
30
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
28
31
  "and a failed control fails the report saying whether it was reached.\n"
29
32
  "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
@@ -46,6 +46,59 @@ def _leaves(value, image, starts):
46
46
  return leaves
47
47
 
48
48
 
49
+ def _no_return(value, image):
50
+ """The declared routines and interrupt sites that never return, each with its reason."""
51
+ if not isinstance(value, list) or len(value) > 256:
52
+ raise ValueError("noReturn must be a list of at most 256 objects")
53
+ routines, interrupts = {}, {}
54
+ for entry in value:
55
+ if not isinstance(entry, dict) or set(entry) not in ({"routine", "reason"}, {"interrupt", "reason"}):
56
+ raise ValueError("Each noReturn entry needs reason and exactly one of routine and interrupt")
57
+ kind = "routine" if "routine" in entry else "interrupt"
58
+ at = integer(entry[kind], 0, len(image.data) - 1, f"noReturn {kind}")
59
+ if image.region(at) is None:
60
+ raise ValueError(f"noReturn {kind} {at} is outside declared code")
61
+ if not isinstance(entry["reason"], str) or not entry["reason"].strip():
62
+ raise ValueError("A noReturn entry needs a nonempty reason")
63
+ found = routines if kind == "routine" else interrupts
64
+ if at in found:
65
+ raise ValueError(f"Duplicate noReturn {kind}")
66
+ if kind == "interrupt":
67
+ ins = image.decode(at)
68
+ if ins is None or base_mnemonic(ins) not in INTERRUPTS:
69
+ raise ValueError(f"noReturn interrupt {at} is not an interrupt instruction")
70
+ vector, conditional = interrupt_vector(image.flat, ins, at)
71
+ if conditional:
72
+ raise ValueError(f"noReturn interrupt {at} is conditional and continues when it does not interrupt")
73
+ found[at] = {"reason": entry["reason"], "vector": vector, "following": at + ins.size}
74
+ continue
75
+ found[at] = entry["reason"]
76
+ return routines, interrupts
77
+
78
+
79
+ def _returns_from(graph, seen, returning_leaves, routine):
80
+ """Where ``routine``'s own paths return: each return instruction, and each leaf they enter other than by a call.
81
+
82
+ Calls are stepped over at the return sites the graph keeps. The walk assumes that a leaf returns,
83
+ so a jump, branch, table row or fall-through into one in ``returning_leaves`` returns too.
84
+ """
85
+ found, stack, visited = [], [routine], set()
86
+ while stack:
87
+ at = stack.pop()
88
+ if at in visited:
89
+ continue
90
+ visited.add(at)
91
+ if at in returning_leaves:
92
+ found.append(at)
93
+ continue
94
+ if at not in seen:
95
+ continue
96
+ if base_mnemonic(seen[at]) in RETURNS:
97
+ found.append(at)
98
+ stack.extend(s for s, kind in graph.get(at, []) if kind != "call")
99
+ return sorted(found)
100
+
101
+
49
102
  def _kinds(at, ins, step):
50
103
  """Each successor of ``step`` with how the CFG reaches it: call, return, table, interrupt, jump or fall."""
51
104
  m, following = base_mnemonic(ins), at + ins.size
@@ -100,8 +153,10 @@ def reach(image, config):
100
153
  The walk decodes from ``starts`` (established region entries) and follows every resolved call
101
154
  into its callee and on at its return site, every resolved jump and branch, the rows of a
102
155
  declared indirect jump table, and every interrupt to the next instruction. A ``leaves`` routine
103
- is reached but never decoded. Each reached target gets one chain with the fewest calls, the
104
- assumptions its route rests on, and the routines on every route to it within the read graph.
156
+ is reached but never decoded. A call to a ``noReturn`` routine and a ``noReturn`` interrupt site
157
+ do not continue at the next instruction. Each reached target gets one chain with the fewest
158
+ calls, the assumptions its route rests on, and the routines on every route to it within the
159
+ read graph.
105
160
  """
106
161
  from .reports import entries
107
162
  starts = _sites(config.get("starts"), "starts", image)
@@ -127,11 +182,14 @@ def reach(image, config):
127
182
  "the walk found")
128
183
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
129
184
  instruction_limit = config.get("instructionLimit", 10000)
130
- seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True, stops=frozenset(leaves))
185
+ no_return_routines, no_return_interrupts = _no_return(config.get("noReturn", []), image)
186
+ declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
187
+ seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
188
+ stops=frozenset(leaves), **declared)
131
189
 
132
190
  graph, unresolved, interrupts, resolved_calls = {}, [], [], {}
133
191
  for at, ins in sorted(seen.items()):
134
- step = cfg_step(image, at, ins, follow_interrupts=True)
192
+ step = cfg_step(image, at, ins, follow_interrupts=True, **declared)
135
193
  graph[at] = [(s, kind) for s, kind in _kinds(at, ins, step) if s in seen or s in leaves]
136
194
  m = base_mnemonic(ins)
137
195
  text = _text(ins)
@@ -144,7 +202,7 @@ def reach(image, config):
144
202
  "reason": edge["provenance"].get("reason", "target outside declared regions")})
145
203
  elif m in CALLS:
146
204
  resolved_calls[at] = edge["target"]
147
- if m in INTERRUPTS:
205
+ if m in INTERRUPTS and at not in no_return_interrupts:
148
206
  vector, conditional = interrupt_vector(image.flat, ins, at)
149
207
  interrupts.append({"site": at, "instruction": text, "vector": vector, "conditional": conditional})
150
208
  for at in leaves:
@@ -243,6 +301,11 @@ def reach(image, config):
243
301
  for at in sorted(leaves) if at in distance and holding_instruction(seen, at) is not None]
244
302
  # Kept apart from the gap rows, which the result limit can cut.
245
303
  stopped = any(g["reason"] == LIMIT_REASON for g in walk_gaps)
304
+ # A control at a site a noReturn declaration kept the walk from names that declaration, which may be the cause.
305
+ cut_by = {at + seen[at].size: f"the call at {at} to noReturn routine {target}"
306
+ for at, target in sorted(resolved_calls.items()) if target in no_return_routines}
307
+ cut_by |= {row["following"]: f"the noReturn interrupt at {at}"
308
+ for at, row in no_return_interrupts.items() if at in seen}
246
309
  # Not reached and reached-but-unresolved need different fixes, so each failure says which it is.
247
310
  failures = []
248
311
  for label, sites, passed in (("control", controls, resolved_calls), ("instruction control", instruction_controls, seen)):
@@ -259,7 +322,9 @@ def reach(image, config):
259
322
  state = unreached(at)
260
323
  detail = (f"inside the reached instruction at {state['insideInstruction']}" if "insideInstruction" in state
261
324
  else "the " + state["status"] if state["status"].startswith("start") else state["status"])
325
+ cut = cut_by.get(at)
262
326
  failures.append(f"{label} {at} is {detail}"
327
+ + (f" (it follows {cut})" if cut else "")
263
328
  + (" (the walk stopped at its instruction limit)" if stopped else ""))
264
329
  if failures:
265
330
  raise ValueError("Positive controls failed: " + "; ".join(failures))
@@ -268,9 +333,31 @@ def reach(image, config):
268
333
  call_sites.setdefault(target, []).append(site)
269
334
  leaf_rows = [{"routine": at, "reason": reason, "reached": at in distance, "callSites": call_sites.get(at, [])}
270
335
  for at, reason in leaves.items()]
336
+ # Each call the declaration kept from its return site, so a reviewer can check what follows it.
337
+ no_return_rows, returning_leaves = [], set(leaves) - set(no_return_routines)
338
+ for at, reason in no_return_routines.items():
339
+ returns = _returns_from(graph, seen, returning_leaves, at)
340
+ no_return_rows.append({"routine": at, "reason": reason, "reached": at in distance, "read": at in seen,
341
+ "returnSites": returns, "contradicted": bool(returns),
342
+ "callSites": [{"site": site, "following": site + seen[site].size,
343
+ "followingRead": site + seen[site].size in seen}
344
+ for site in call_sites.get(at, [])]})
345
+ no_return_rows += [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
346
+ "following": row["following"], "followingRead": row["following"] in seen}
347
+ for at, row in no_return_interrupts.items()]
348
+ contradicted = any(row.get("contradicted") for row in no_return_rows)
271
349
  counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
272
350
  "interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
273
- return {"starts": starts, "targets": rows, "leaves": leaf_rows,
351
+ assumptions = ["each reached call returns to its next instruction"
352
+ + (", except a call to a noReturn routine" if no_return_routines else ""),
353
+ "each reached interrupt returns to its next instruction"
354
+ + (", except at a noReturn interrupt site" if no_return_interrupts else "")
355
+ + "; interrupt handlers are not read",
356
+ "each leaf calls nothing, for the reason it gives",
357
+ "each declared indirect jump table holds the routes its declaration gives"]
358
+ if no_return_rows:
359
+ assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
360
+ return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return_rows,
274
361
  "reachedRoutines": sorted(routine_starts), "counts": counts,
275
362
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
276
363
  "contested": sorted(contested)[:limit],
@@ -279,19 +366,20 @@ def reach(image, config):
279
366
  "controls": [{"site": at, "target": resolved_calls[at]} for at in controls],
280
367
  "instructionControls": [{"site": at, "instruction": _text(seen[at]), "routine": routine.get(at)}
281
368
  for at in instruction_controls],
282
- "negativeUsable": bool(controls or instruction_controls) and not (stopped or unresolved or gaps or contested),
283
- "assumptions": ["each reached call returns to its next instruction",
284
- "each reached interrupt returns to its next instruction; interrupt handlers are not read",
285
- "each leaf calls nothing, for the reason it gives",
286
- "each declared indirect jump table holds the routes its declaration gives"],
369
+ "negativeUsable": bool(controls or instruction_controls)
370
+ and not (stopped or unresolved or gaps or contested or contradicted),
371
+ "assumptions": assumptions,
287
372
  "exclusions": ["computed call and jump targets (listed in unresolved)", "unrelocated far calls",
288
373
  "code reached only from outside the starts", "runtime reachability"],
289
374
  "interpretation": "Routes over the decoded entry-path CFG from the starts. A chain has the fewest calls; "
290
375
  "throughEveryRoute lists the routine starts every read route to the target passes, "
291
376
  "and an unresolved transfer may add a route that passes none of them. "
292
377
  "negativeUsable needs a control of either kind, a walk that did not stop at its instruction limit, "
293
- "and no unresolved transfer, gap or contested instruction, "
294
- "and still rests on the listed assumptions and leaves. "
378
+ "and no unresolved transfer, gap, contested instruction or contradicted noReturn "
379
+ "routine, and still rests on the listed assumptions, leaves and noReturn declarations. "
380
+ "A noReturn routine is contradicted when a return instruction, or a leaf entered "
381
+ "other than by a call, lies on its own read paths; an empty returnSites means only "
382
+ "that the walk read none. "
295
383
  "When instructionLimitReached holds, the walk stopped before reading all it reaches: "
296
384
  "every list and count covers only the part read, which part depends on the walk order, "
297
385
  "and an unreached target may lie past the stop."}
@@ -180,7 +180,7 @@ class Step(NamedTuple):
180
180
 
181
181
 
182
182
  def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False,
183
- modeled_interrupts=frozenset()):
183
+ modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset()):
184
184
  """The successors of ``ins`` decoded at ``at``, under the rule ``walk`` and the ``uses`` caller continuation share.
185
185
 
186
186
  An unsupported transfer encoding, a return, an unconditional jump without a resolved target,
@@ -189,7 +189,9 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
189
189
  conditional jump or loop continues at its resolved target and at the next instruction. A declared
190
190
  indirect jump continues at its table rows only. A call continues at its resolved target and at
191
191
  its return site; with ``step_over_calls`` it continues at its return site only and its target
192
- is neither resolved nor reported.
192
+ is neither resolved nor reported. A call whose resolved target is in ``no_return_calls``, and an
193
+ interrupt whose site is in ``no_return_interrupts``, have no return site: the query declared that
194
+ they never come back.
193
195
  """
194
196
  m, following = base_mnemonic(ins), at + ins.size
195
197
  if unsupported_transfer(image, ins):
@@ -216,8 +218,10 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
216
218
  gaps.append({"site": at, "reason": provenance.get("reason", "target outside declared regions")})
217
219
  else:
218
220
  successors.append(target)
219
- if m in ("jmp", "ljmp"):
221
+ if m in ("jmp", "ljmp") or (calls and target in no_return_calls):
220
222
  return Step(successors, False, None, gaps, edges)
223
+ if m in INTERRUPTS and at in no_return_interrupts:
224
+ return Step(successors, False, None, gaps, edges)
221
225
  if m == "hlt" or (m in INTERRUPTS and not follow_interrupts and at not in modeled_interrupts):
222
226
  gaps.append({"site": at, "reason": "hardware or interrupt boundary"})
223
227
  return Step(successors, False, None, gaps, edges)
@@ -233,7 +237,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
233
237
 
234
238
 
235
239
  def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset(),
236
- modeled_interrupts=frozenset()):
240
+ modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset()):
237
241
  """Decode the CFG reached from ``entries`` and check its instruction boundaries.
238
242
 
239
243
  A port access in the flat model records a gap and ends that branch, as ``trace`` stops there.
@@ -242,7 +246,9 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
242
246
  for callers that list each interrupt as an assumption. ``modeled_interrupts`` continues past the
243
247
  interrupts at those sites only, the ones ``modeled_interrupt_sites`` names, which ``trace``
244
248
  continues past under a call model (ADR 0017). A site in ``stops`` is never decoded:
245
- the walk reaches it and goes no further, so it is in no returned set.
249
+ the walk reaches it and goes no further, so it is in no returned set. ``no_return_calls`` and
250
+ ``no_return_interrupts`` end the branch at a call to one of those routines and at one of those
251
+ interrupt sites, as ``cfg_step`` describes.
246
252
  """
247
253
  instruction_limit(image, limit)
248
254
  pending, seen, gaps, edges = list(entries), {}, [], []
@@ -265,7 +271,8 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
265
271
  continue
266
272
  seen[at] = ins
267
273
  step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts,
268
- modeled_interrupts=modeled_interrupts)
274
+ modeled_interrupts=modeled_interrupts, no_return_calls=no_return_calls,
275
+ no_return_interrupts=no_return_interrupts)
269
276
  successors[at] = step.successors
270
277
  gaps.extend(step.gaps)
271
278
  edges.extend(step.edges)
@@ -36,6 +36,21 @@ TABLE = {"site": 0x12, "exhaustive": True, "evidence": "synthetic: bx is gated t
36
36
  "table": {"start": 0x30, "count": 2, "stride": 2, "evidence": "synthetic: two rows under cmp bx, 1"}}
37
37
 
38
38
 
39
+ # 0000 calls B at 0008, then X at 0010, which ends the program through its interrupt. The bytes after
40
+ # the second call are data that decode as a mov whose immediate runs over B's entry.
41
+ EXITS = bytes.fromhex(
42
+ "e80500" # 0000 call 0008
43
+ "e80a00" # 0003 call 0010
44
+ "b841" # 0006 data: decodes as mov ax, 0B841h through 0008
45
+ "b890c3" # 0008 B: mov ax, 0C390h
46
+ "c3" # 000B ret
47
+ "90909090" # 000C padding
48
+ "b44c" # 0010 X: mov ah, 4Ch
49
+ "cd21" # 0012 int 21h
50
+ "c3" # 0014 what a walk past the interrupt reads
51
+ )
52
+
53
+
39
54
  def config(data=DISPATCH, entries=(0,), **extra):
40
55
  return {"regions": [{"name": "synthetic", "start": 0, "end": len(data), "ip": 0, "segment": 0x1000, "resident": True,
41
56
  "entries": list(entries), "evidence": "synthetic declared code extent"}],
@@ -172,6 +187,105 @@ class ReachTests(unittest.TestCase):
172
187
  self.assertEqual(r["counts"]["unresolved"], 2)
173
188
  self.assertTrue(r["truncated"])
174
189
 
190
+ def test_a_no_return_declaration_keeps_the_bytes_after_its_calls_unread(self):
191
+ exits = "synthetic: ends the program through its interrupt"
192
+ # Without a declaration the walk reads the data after 0003 as an instruction over B's entry.
193
+ r = run_report(EXITS, config(EXITS, targets=[9], controls=[0, 3]), "reach")
194
+ self.assertIn("overlapping entry-path instructions; boundary unresolved", [g["reason"] for g in r["gaps"]])
195
+ self.assertTrue(r["contested"])
196
+ self.assertFalse(r["negativeUsable"])
197
+ self.assertEqual(r["noReturn"], [])
198
+ # Declaring X a leaf leaves it unread, but the call to it still falls through into the data.
199
+ r = run_report(EXITS, config(EXITS, targets=[9], controls=[0, 3],
200
+ leaves=[{"routine": 0x10, "reason": "synthetic leaf"}]), "reach")
201
+ self.assertIn("overlapping entry-path instructions; boundary unresolved", [g["reason"] for g in r["gaps"]])
202
+ r = run_report(EXITS, config(EXITS, targets=[9], controls=[0, 3], noReturn=[
203
+ {"routine": 0x10, "reason": exits}, {"interrupt": 0x12, "reason": exits}]), "reach")
204
+ self.assertEqual((r["gaps"], r["contested"], r["unresolved"], r["interrupts"]), ([], [], [], []))
205
+ # B is read from its own entry only, so its second byte is inside its first instruction.
206
+ self.assertEqual((r["targets"][0]["status"], r["targets"][0]["insideInstruction"]),
207
+ ("inside a reached instruction", 8))
208
+ self.assertTrue(r["negativeUsable"])
209
+ self.assertEqual(r["noReturn"], [
210
+ {"routine": 0x10, "reason": exits, "reached": True, "read": True, "returnSites": [],
211
+ "contradicted": False, "callSites": [{"site": 3, "following": 6, "followingRead": False}]},
212
+ {"interrupt": 0x12, "reason": exits, "vector": 0x21, "reached": True, "following": 0x14,
213
+ "followingRead": False}])
214
+ self.assertIn("each reached call returns to its next instruction, except a call to a noReturn routine",
215
+ r["assumptions"])
216
+ self.assertIn("each reached interrupt returns to its next instruction, except at a noReturn interrupt "
217
+ "site; interrupt handlers are not read", r["assumptions"])
218
+ self.assertIn("each noReturn routine and interrupt never returns, for the reason it gives", r["assumptions"])
219
+
220
+ def test_a_no_return_routine_with_a_read_return_is_contradicted(self):
221
+ # Declared alone, X's interrupt is assumed to return, so the walk reads the ret after it.
222
+ r = run_report(EXITS, config(EXITS, targets=[9], controls=[0, 3],
223
+ noReturn=[{"routine": 0x10, "reason": "synthetic exit"}]), "reach")
224
+ row = r["noReturn"][0]
225
+ self.assertEqual((row["returnSites"], row["contradicted"]), ([0x14], True))
226
+ self.assertEqual(r["gaps"], [])
227
+ self.assertFalse(r["negativeUsable"])
228
+
229
+ def test_a_no_return_leaf_is_reached_unread_and_unchecked(self):
230
+ r = run_report(EXITS, config(EXITS, targets=[9], controls=[0, 3],
231
+ leaves=[{"routine": 0x10, "reason": "synthetic leaf"}],
232
+ noReturn=[{"routine": 0x10, "reason": "synthetic exit"}]), "reach")
233
+ row = r["noReturn"][0]
234
+ self.assertEqual((row["reached"], row["read"], row["returnSites"]), (True, False, []))
235
+ self.assertEqual(r["gaps"], [])
236
+ self.assertTrue(r["negativeUsable"])
237
+
238
+ def test_the_site_after_a_no_return_call_is_still_read_through_a_jump(self):
239
+ # 0000 call 0004; ret. 0004 je 0009; call 000A; 0009 ret. 000A mov ah, 4Ch; int 21h.
240
+ data = bytes.fromhex("e80100c3" "7403" "e80100" "c3" "b44c" "cd21")
241
+ exits = [{"routine": 0xA, "reason": "synthetic exit"}, {"interrupt": 0xC, "reason": "synthetic exit"}]
242
+ r = run_report(data, config(data, targets=[9], controls=[6], noReturn=exits), "reach")
243
+ self.assertTrue(r["targets"][0]["reached"])
244
+ self.assertEqual(r["targets"][0]["chain"], [{"routine": 0}, {"callSite": 0, "routine": 4}])
245
+ self.assertEqual(r["noReturn"][0]["callSites"], [{"site": 6, "following": 9, "followingRead": True}])
246
+ self.assertTrue(r["negativeUsable"])
247
+
248
+ def test_a_no_return_routine_that_jumps_into_a_leaf_is_contradicted(self):
249
+ # 0000 call 0004; 0003 ret. 0004 X: jmp 0007; 0006 nop. 0007 L, a leaf the walk assumes returns.
250
+ data = bytes.fromhex("e80100" "c3" "eb01" "90" "c3")
251
+ leaf = [{"routine": 7, "reason": "synthetic leaf"}]
252
+ r = run_report(data, config(data, targets=[3], controls=[0], leaves=leaf,
253
+ noReturn=[{"routine": 4, "reason": "synthetic exit"}]), "reach")
254
+ row = r["noReturn"][0]
255
+ self.assertEqual((row["returnSites"], row["contradicted"]), ([7], True))
256
+ self.assertFalse(r["negativeUsable"])
257
+ # A leaf that is declared noReturn as well is no way back.
258
+ r = run_report(data, config(data, targets=[3], controls=[0], leaves=leaf,
259
+ noReturn=[{"routine": 4, "reason": "synthetic exit"},
260
+ {"routine": 7, "reason": "synthetic exit"}]), "reach")
261
+ self.assertEqual([row["contradicted"] for row in r["noReturn"]], [False, False])
262
+ self.assertTrue(r["negativeUsable"])
263
+
264
+ def test_a_control_after_a_no_return_call_or_interrupt_names_the_declaration(self):
265
+ exits = [{"routine": 0x10, "reason": "synthetic exit"}, {"interrupt": 0x12, "reason": "synthetic exit"}]
266
+ for site, message in ((6, "instruction control 6 is not reached "
267
+ r"\(it follows the call at 3 to noReturn routine 16\)$"),
268
+ (0x14, "instruction control 20 is not reached "
269
+ r"\(it follows the noReturn interrupt at 18\)$")):
270
+ with self.subTest(site=site), self.assertRaisesRegex(ValueError, "Positive controls failed: " + message):
271
+ run_report(EXITS, config(EXITS, targets=[9], instructionControls=[site], noReturn=exits), "reach")
272
+
273
+ def test_an_unreached_no_return_interrupt_still_gives_its_following_site(self):
274
+ # 0000 ret; 0001 int 21h, which nothing reaches.
275
+ data = bytes.fromhex("c3" "cd21")
276
+ r = run_report(data, config(data, targets=[1], noReturn=[{"interrupt": 1, "reason": "synthetic exit"}]),
277
+ "reach")
278
+ self.assertEqual(r["noReturn"], [{"interrupt": 1, "reason": "synthetic exit", "vector": 0x21,
279
+ "reached": False, "following": 3, "followingRead": False}])
280
+
281
+ def test_a_conditional_interrupt_cannot_be_declared_no_return(self):
282
+ data = bytes.fromhex("cec3") # into; ret
283
+ with self.assertRaisesRegex(ValueError, "noReturn interrupt 0 is conditional"):
284
+ run_report(data, config(data, targets=[1], noReturn=[{"interrupt": 0, "reason": "synthetic"}]), "reach")
285
+ exit_interrupt = {"interrupt": 0x12, "reason": "synthetic"}
286
+ with self.assertRaisesRegex(ValueError, "Duplicate noReturn interrupt"):
287
+ run_report(EXITS, config(EXITS, targets=[9], noReturn=[exit_interrupt, exit_interrupt]), "reach")
288
+
175
289
  def test_a_control_that_is_no_call_is_refused_before_the_walk_and_an_instruction_control_takes_it(self):
176
290
  # 0000 calls 0004, which stores AX at [0100] and returns.
177
291
  data = bytes.fromhex("e80100c3" "a30001" "c3")
@@ -247,6 +361,17 @@ class ReachTests(unittest.TestCase):
247
361
  ({"leaves": [{"routine": 0x18}]}, "routine and reason"),
248
362
  ({"leaves": [{"routine": 0x18, "reason": " "}]}, "nonempty reason"),
249
363
  ({"leaves": [{"routine": 0, "reason": "x"}]}, "start cannot be a leaf"),
364
+ ({"noReturn": {}}, "noReturn must be a list"),
365
+ ({"noReturn": [{"routine": 0x18}]}, "exactly one of routine and interrupt"),
366
+ ({"noReturn": [{"routine": 0x18, "interrupt": 0x18, "reason": "x"}]},
367
+ "exactly one of routine and interrupt"),
368
+ ({"noReturn": [{"routine": 0x1000, "reason": "x"}]}, "noReturn routine"),
369
+ ({"noReturn": [{"routine": 0x18, "reason": ""}]}, "nonempty reason"),
370
+ ({"noReturn": [{"routine": 0x18, "reason": "x"}, {"routine": 0x18, "reason": "y"}]},
371
+ "Duplicate noReturn routine"),
372
+ ({"noReturn": [{"interrupt": 0x18, "reason": "x"}]}, "not an interrupt instruction"),
373
+ ({"noReturn": [{"routine": 0x18, "reason": "x"}] * 257},
374
+ "noReturn must be a list of at most 256"),
250
375
  ({"controls": [0x3, 0x3]}, "distinct file offsets"),
251
376
  ({"controls": [0x1000]}, "controls must be an integer"),
252
377
  ({"instructionControls": [0x3, 0x3]}, "distinct file offsets"),