scientific-method-engine 19.0.0__tar.gz → 19.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/cli.py +7 -6
  4. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/inventory.py +49 -23
  5. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/reach.py +114 -25
  6. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/reports.py +5 -3
  7. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/trace.py +14 -9
  8. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_inventory_check.py +51 -0
  9. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_reach.py +135 -2
  10. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/.gitignore +0 -0
  11. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/LICENSE +0 -0
  12. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/README.md +0 -0
  13. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/__init__.py +0 -0
  14. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/__main__.py +0 -0
  15. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  16. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  17. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  18. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  19. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  20. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  21. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  22. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  23. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  24. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  25. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  26. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  27. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  28. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  29. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  30. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  31. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  32. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  33. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  34. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  35. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  36. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  37. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  38. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  39. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  40. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  41. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  42. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  43. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  44. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  45. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  46. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  47. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  48. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  49. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  50. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  51. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  52. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  53. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  54. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/image.py +0 -0
  55. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/loops.py +0 -0
  56. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/machine.py +0 -0
  57. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  58. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  59. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  60. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/pe.py +0 -0
  61. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/ranges.py +0 -0
  62. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/relational.py +0 -0
  63. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  64. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/values.py +0 -0
  65. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/oracle.py +0 -0
  66. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_config_encoding.py +0 -0
  67. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_continuation_budget.py +0 -0
  68. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_dispatch.py +0 -0
  69. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_effect_order.py +0 -0
  70. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_enter_frames.py +0 -0
  71. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_entry_frame.py +0 -0
  72. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_indirect_far_transfers.py +0 -0
  73. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_interrupt_models.py +0 -0
  74. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_join_terms.py +0 -0
  75. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_loops.py +0 -0
  76. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_memory_scopes.py +0 -0
  77. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_nested_frame_request.py +0 -0
  78. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_oracle.py +0 -0
  79. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_pe.py +0 -0
  80. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_ranges.py +0 -0
  81. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_relational_controls.py +0 -0
  82. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_return_checks.py +0 -0
  83. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_successor_rule.py +0 -0
  84. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_table_continuations.py +0 -0
  85. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_term_limit.py +0 -0
  86. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_upstream_gaps.py +0 -0
  87. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_volatile_memory.py +0 -0
  88. {scientific_method_engine-19.0.0 → scientific_method_engine-19.2.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 19.0.0
3
+ Version: 19.2.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "19.0.0"
8
+ version = "19.2.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -24,13 +24,14 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
24
24
  "reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
25
25
  "and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
26
26
  "a leaves routine is reached but not read, and its reason is repeated in the report;\n"
27
- "a call to a noReturn routine and a noReturn interrupt do not continue at the next instruction,\n"
28
- "and a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
29
- "is reported contradicted;\n"
27
+ "a call to a noReturn routine, a noReturn call and a noReturn interrupt do not continue at the next\n"
28
+ "instruction, a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
29
+ "is reported contradicted, and a noReturn call lists its targets' return sites and is never contradicted;\n"
30
30
  "indirectCalls declares the targets of a computed call as a table read from the build or a list,\n"
31
31
  "and the walk calls each of them; one not declared exhaustive stays unresolved;\n"
32
32
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
33
- "and a failed control fails the report saying whether it was reached.\n"
33
+ "and a failed control fails the report saying whether it was reached;\n"
34
+ "decodedRanges true lists every instruction the walk decoded as half-open ranges with their routine.\n"
34
35
  "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
35
36
  "that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
36
37
  "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
@@ -38,8 +39,8 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
38
39
  "It lists each row whose start lies inside an instruction the walk established (rowStarts), with the\n"
39
40
  "routine the walk read it in, and each row start at or inside an instruction of an overlap the walk left\n"
40
41
  "unresolved, with the instructions it overlaps; a row start in bytes the walk did not decode is counted, not placed.\n"
41
- "noReturn takes reach's declarations: the walk stops at a call to a declared routine and at a declared\n"
42
- "interrupt, a declared routine with a return on its own read paths is reported contradicted, and\n"
42
+ "noReturn takes reach's declarations: the walk stops at a call to a declared routine, at a declared call\n"
43
+ "and at a declared interrupt, a declared routine with a return on its own read paths is reported contradicted, and\n"
43
44
  "rowsPastNoReturn lists each row whose body continues past such a call or interrupt.\n"
44
45
  "indirectCalls takes reach's declarations: each declared target of a reached computed call is a call target.\n"
45
46
  "uses matches accesses whose footprint intersects query offset..offset+width; its rawCandidates give each unreached\n"
@@ -233,12 +233,13 @@ def _row_starts(image, rows, starts, domains, seen, contested, unresolved, routi
233
233
 
234
234
 
235
235
  def _no_return_check(image, routines, declared, limit):
236
- """``(returns, read, stopped)``: the return sites on each declared routine's own read paths, the
236
+ """``(returns, read, stopped)``: the return sites on each of ``routines``' own read paths, the
237
237
  instructions the walk established, and whether the walk stopped.
238
238
 
239
- The walk starts at the declared routines and continues past every interrupt except a declared
240
- one, as ``reach`` reads them, so a routine that ends in an interrupt that returns is contradicted
241
- by what follows it. Calls are stepped over at their return sites, except calls to declared routines.
239
+ ``routines`` are the declared routines, or the targets of the declared calls. The walk starts at
240
+ them and continues past every interrupt except a declared one, as ``reach`` reads them, so a
241
+ routine that ends in an interrupt that returns is contradicted by what follows it. Calls are
242
+ stepped over at their return sites, except calls to declared routines and declared calls.
242
243
  A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
243
244
  has no return sites and is not in ``read``.
244
245
  """
@@ -267,27 +268,27 @@ def inventory_check(image, config):
267
268
 
268
269
  The rows are checked against the same walk: each row whose start lies inside an instruction the
269
270
  walk established, or at or inside an instruction of an overlap the walk left unresolved, is listed
270
- in ``rowStarts``. ``noReturn`` declarations, in ``reach``'s shape, end
271
- the walk at a call to a declared routine and at a declared interrupt site, are checked for a
271
+ in ``rowStarts``. ``noReturn`` declarations, in ``reach``'s shape, end the walk at a call to a
272
+ declared routine, at a declared call site and at a declared interrupt site, are checked for a
272
273
  return on the routine's own paths, and list in ``rowsPastNoReturn`` each row whose body
273
274
  continues past such a call or interrupt. ``indirectCalls`` declarations, in ``reach``'s shape,
274
275
  make each declared target of a reached computed call a call target, and the walk reads on in them.
275
276
  """
276
277
  from .reports import call_controls, direct_calls, entries, search_coverage
277
- from .reach import no_return_declarations, no_return_rows, successor_graph, fewest_calls
278
+ from .reach import (no_return_inputs, no_return_rows, successor_graph, fewest_calls, call_assumption,
279
+ no_return_assumptions)
278
280
  if "inventory" not in config:
279
281
  raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
280
282
  rows = read_inventory(config["inventory"], image)
281
283
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
282
- no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
283
284
  indirect_calls = indirect_call_declarations(config.get("indirectCalls", []), image)
285
+ no_return_routines, no_return_interrupts, no_return_sites, declared = no_return_inputs(
286
+ config.get("noReturn", []), image, indirect_calls)
284
287
  for at in config.get("controls", []) if isinstance(config.get("controls"), list) else []:
285
288
  # call_controls refuses a control that is no integer.
286
289
  if type(at) is int and at in indirect_calls:
287
290
  raise ValueError(f"Positive control {at} is a declared indirect call, whose targets rest on its "
288
291
  "declaration; a control must resolve from its own encoding")
289
- declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts),
290
- "indirect_calls": indirect_calls}
291
292
  calls = direct_calls(image, config, **declared)
292
293
 
293
294
  def ends_branch(row):
@@ -383,15 +384,31 @@ def inventory_check(image, config):
383
384
  return routines.get(site)
384
385
  row_starts, start_counts = _row_starts(image, rows, starts, domains, seen, contested, overlapping, routine_of)
385
386
 
386
- # Each declaration with the calls it kept from their return sites, as reach reports them.
387
- returns, check_read, check_stopped = _no_return_check(image, no_return_routines, declared,
388
- config.get("instructionLimit", 10000))
387
+ # Each declaration with the calls it kept from their return sites, as reach reports them: only a call the
388
+ # entry-path walk established has targets, so a raw byte candidate at an unreached declared site gives none.
389
+ site_targets = {at: [] for at in no_return_sites}
390
+ for row in calls["rows"]:
391
+ if (row["site"] in site_targets and row["site"] in seen and row["target"] is not None
392
+ and row["target"] not in site_targets[row["site"]]):
393
+ site_targets[row["site"]].append(row["target"])
394
+ # The targets of the declared calls get a walk of their own, so that their code, overlapping a declared
395
+ # routine's or spending the limit first, cannot change what the routine check reads.
396
+ call_targets = {target for found in site_targets.values() for target in found} - set(no_return_routines)
397
+ checked = set(no_return_routines) | call_targets
398
+ check_limit = config.get("instructionLimit", 10000)
399
+ returns, routine_read, routine_stopped = _no_return_check(image, no_return_routines, declared, check_limit)
400
+ target_returns, target_read, target_stopped = _no_return_check(image, call_targets, declared, check_limit)
401
+ returns |= target_returns
402
+ check_read = ({at for at in no_return_routines if at in routine_read}
403
+ | {at for at in call_targets if at in target_read})
404
+ check_stopped = routine_stopped or target_stopped
389
405
  entry_calls = {}
390
406
  for row in calls["rows"]:
391
407
  if row["target"] in no_return_routines and row["site"] in seen and ends_branch(row):
392
408
  entry_calls.setdefault(row["target"], []).append(row["site"])
393
409
  no_return = no_return_rows(no_return_routines, no_return_interrupts, returns,
394
- {at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read)
410
+ {at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read,
411
+ no_return_sites, site_targets)
395
412
 
396
413
  # A row whose body holds a declared call or interrupt and the byte after it still covers what follows.
397
414
  # A declared computed call has one row for each target, and is listed once with all of them.
@@ -404,8 +421,14 @@ def inventory_check(image, config):
404
421
  declared_ends.add(row["site"])
405
422
  routine = list(indirect_calls[row["site"]]["targets"])
406
423
  ends.append((row["site"], image.decode(row["site"]).size, "call", routine, row["classification"]))
407
- ends += [(at, row["following"] - at, "interrupt", None,
408
- "entry-path instruction" if at in seen else CONTESTED_REASON if at in contested else "declared site")
424
+ # A declared call site is listed as one, whatever its targets, unless a routine declaration already listed it.
425
+ def site_classification(at):
426
+ return "entry-path instruction" if at in seen else CONTESTED_REASON if at in contested else "declared site"
427
+ listed = {site for site, *_ in ends}
428
+ ends += [(at, row["following"] - at, "call site",
429
+ site_targets[at][0] if len(site_targets[at]) == 1 else site_targets[at] or None, site_classification(at))
430
+ for at, row in no_return_sites.items() if at not in listed]
431
+ ends += [(at, row["following"] - at, "interrupt", None, site_classification(at))
409
432
  for at, row in no_return_interrupts.items()]
410
433
  # Only a site the entry-path walk established counts a row; the rest are listed with their evidence.
411
434
  past, past_rows, past_unread, past_unverified = [], set(), set(), set()
@@ -500,16 +523,17 @@ def inventory_check(image, config):
500
523
  if unread:
501
524
  summary += " " + _count(unread, "noReturn routine is", "noReturn routines are") \
502
525
  + " not an instruction the check walk established, so its return check reads nothing."
503
- if check_stopped:
526
+ if routine_stopped:
504
527
  summary += " The walk that checks the noReturn routines stopped at its instruction limit, so it can miss returns."
505
- assumptions = ["each reached call returns to its next instruction"
506
- + (", except a call to a noReturn routine" if no_return_routines else "")]
507
- if no_return_routines:
528
+ if target_stopped:
529
+ summary += (" The walk that checks the targets of the noReturn calls stopped at its instruction limit, so it "
530
+ "can miss returns.")
531
+ assumptions = [call_assumption(no_return_routines, no_return_sites)]
532
+ if checked:
508
533
  assumptions.append("each interrupt the noReturn check reads returns to its next instruction"
509
534
  + (", except at a noReturn interrupt site" if no_return_interrupts else "")
510
535
  + "; interrupt handlers are not read")
511
- if no_return:
512
- assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
536
+ assumptions += no_return_assumptions(no_return_routines, no_return_interrupts, no_return_sites)
513
537
  if indirect_calls:
514
538
  assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
515
539
  "it gives, and only those when it is declared exhaustive")
@@ -543,4 +567,6 @@ def inventory_check(image, config):
543
567
  "which the row names with both instructions for a reader to judge. rowsPastNoReturn rests "
544
568
  "on the noReturn declarations: followingRead says whether the walk reached the bytes after "
545
569
  "the call by another route, only a site the walk established counts a row, and an empty "
546
- "returnSites means only that the walk read no return."}
570
+ "returnSites means only that the walk read no return. A noReturn call is never "
571
+ "contradicted: the return sites of its targets are listed, and the declaration rests "
572
+ "on its reason alone."}
@@ -48,22 +48,33 @@ def _leaves(value, image, starts):
48
48
 
49
49
 
50
50
  def no_return_declarations(value, image):
51
- """The declared routines and interrupt sites that never return, each with its reason."""
51
+ """``(routines, interrupts, calls)``: the declared routines, interrupt sites and call sites that never return.
52
+
53
+ A routine maps to its reason; an interrupt or call site to its reason and ``following``, the
54
+ site of the next instruction, and an interrupt to its ``vector`` too.
55
+ """
52
56
  if not isinstance(value, list) or len(value) > 256:
53
57
  raise ValueError("noReturn must be a list of at most 256 objects")
54
- routines, interrupts = {}, {}
58
+ routines, interrupts, calls = {}, {}, {}
55
59
  for entry in value:
56
- if not isinstance(entry, dict) or set(entry) not in ({"routine", "reason"}, {"interrupt", "reason"}):
57
- raise ValueError("Each noReturn entry needs reason and exactly one of routine and interrupt")
58
- kind = "routine" if "routine" in entry else "interrupt"
60
+ if not isinstance(entry, dict) or set(entry) not in ({"routine", "reason"}, {"interrupt", "reason"},
61
+ {"call", "reason"}):
62
+ raise ValueError("Each noReturn entry needs reason and exactly one of routine, call and interrupt")
63
+ kind = next(k for k in ("routine", "interrupt", "call") if k in entry)
59
64
  at = integer(entry[kind], 0, len(image.data) - 1, f"noReturn {kind}")
60
65
  if image.region(at) is None:
61
66
  raise ValueError(f"noReturn {kind} {at} is outside declared code")
62
67
  if not isinstance(entry["reason"], str) or not entry["reason"].strip():
63
68
  raise ValueError("A noReturn entry needs a nonempty reason")
64
- found = routines if kind == "routine" else interrupts
69
+ found = {"routine": routines, "interrupt": interrupts, "call": calls}[kind]
65
70
  if at in found:
66
71
  raise ValueError(f"Duplicate noReturn {kind}")
72
+ if kind == "call":
73
+ ins = image.decode(at)
74
+ if ins is None or base_mnemonic(ins) not in CALLS:
75
+ raise ValueError(f"noReturn call {at} is not a call instruction")
76
+ found[at] = {"reason": entry["reason"], "following": at + ins.size}
77
+ continue
67
78
  if kind == "interrupt":
68
79
  ins = image.decode(at)
69
80
  if ins is None or base_mnemonic(ins) not in INTERRUPTS:
@@ -74,7 +85,33 @@ def no_return_declarations(value, image):
74
85
  found[at] = {"reason": entry["reason"], "vector": vector, "following": at + ins.size}
75
86
  continue
76
87
  found[at] = entry["reason"]
77
- return routines, interrupts
88
+ return routines, interrupts, calls
89
+
90
+
91
+ def no_return_inputs(value, image, indirect_calls):
92
+ """``(routines, interrupts, calls, declared)``: ``no_return_declarations`` and the walk options they give."""
93
+ routines, interrupts, calls = no_return_declarations(value, image)
94
+ declared = {"no_return_calls": frozenset(routines), "no_return_interrupts": frozenset(interrupts),
95
+ "no_return_call_sites": frozenset(calls), "indirect_calls": indirect_calls}
96
+ return routines, interrupts, calls, declared
97
+
98
+
99
+ def call_assumption(routines, calls):
100
+ """The assumption that reached calls return, with the exceptions the ``noReturn`` declarations make."""
101
+ exceptions = (["a call to a noReturn routine"] if routines else []) + (["a noReturn call"] if calls else [])
102
+ return "each reached call returns to its next instruction" + (", except " + " and ".join(exceptions)
103
+ if exceptions else "")
104
+
105
+
106
+ def no_return_assumptions(routines, interrupts, calls):
107
+ """The assumptions the ``noReturn`` declarations add."""
108
+ found = []
109
+ if routines or interrupts:
110
+ found.append("each noReturn routine and interrupt never returns, for the reason it gives")
111
+ if calls:
112
+ found.append("each noReturn call never returns to its next instruction, for the reason it gives, "
113
+ "though its target may return to other callers")
114
+ return found
78
115
 
79
116
 
80
117
  def returns_from(graph, seen, returning_leaves, routine):
@@ -129,12 +166,16 @@ def successor_graph(image, seen, **step):
129
166
  for at, ins in seen.items()}
130
167
 
131
168
 
132
- def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, read):
133
- """The ``noReturn`` report rows of the declared ``routines`` and ``interrupts``.
169
+ def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, read, calls, call_targets):
170
+ """The ``noReturn`` report rows of the declared ``routines``, ``calls`` and ``interrupts``.
134
171
 
135
- ``returns`` gives each routine's return sites, ``call_sites`` the reached calls to each routine,
136
- ``seen`` the instructions the calls and interrupts were read in, ``reached`` the sites a walk from
137
- the starts reached and ``read`` the sites the return check decoded.
172
+ ``returns`` gives the return sites of each routine and each target of a declared call,
173
+ ``call_sites`` the reached calls to each routine, ``seen`` the instructions the calls and
174
+ interrupts were read in, ``reached`` the sites a walk from the starts reached, ``read`` the
175
+ sites the return check decoded and ``call_targets`` the resolved or declared targets of each
176
+ declared call site. A declared call is never contradicted: whether its target returns on the
177
+ run that makes this call can depend on what the caller passes, which the walk does not follow,
178
+ so the target's return sites are listed and the declaration rests on its reason.
138
179
  """
139
180
  rows = [{"routine": at, "reason": reason, "reached": at in reached, "read": at in read,
140
181
  "returnSites": returns[at], "contradicted": bool(returns[at]),
@@ -142,6 +183,11 @@ def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, rea
142
183
  "followingRead": site + seen[site].size in seen}
143
184
  for site in call_sites.get(at, [])]}
144
185
  for at, reason in routines.items()]
186
+ rows += [{"call": at, "reason": row["reason"], "reached": at in seen,
187
+ "targets": [{"routine": target, "read": target in read, "returnSites": returns.get(target, [])}
188
+ for target in call_targets.get(at, [])],
189
+ "following": row["following"], "followingRead": row["following"] in seen}
190
+ for at, row in calls.items()]
145
191
  return rows + [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
146
192
  "following": row["following"], "followingRead": row["following"] in seen}
147
193
  for at, row in interrupts.items()]
@@ -173,6 +219,28 @@ def fewest_calls(graph, starts):
173
219
  return distance, parent, routine
174
220
 
175
221
 
222
+ def decoded_ranges(seen, routine):
223
+ """The instructions in ``seen`` as half-open ``[start, end)`` runs, each with the ``routine`` they were read in.
224
+
225
+ A run holds instructions that abut, each starting where the one before it ends, and that
226
+ ``routine`` gives the same routine. A site on no route from a start has the routine None. Runs
227
+ overlap only where ``seen`` holds instructions that overlap, which the walk keeps when it proves
228
+ both boundaries.
229
+ """
230
+ # Past an overlap, the run an instruction continues need not be the last one opened, so each open
231
+ # run is found by where it ends and in which routine.
232
+ ranges, ending = [], {}
233
+ for at, ins in sorted(seen.items()):
234
+ owner = routine.get(at)
235
+ run = ending.pop((at, owner), None)
236
+ if run is None:
237
+ run = {"start": at, "end": at, "routine": owner}
238
+ ranges.append(run)
239
+ run["end"] = at + ins.size
240
+ ending[(run["end"], owner)] = run
241
+ return ranges
242
+
243
+
176
244
  def _dominators(order, predecessors):
177
245
  """Immediate dominators (Cooper, Harvey and Kennedy) of every node in ``order``, a reverse postorder from ROOT."""
178
246
  index = {node: i for i, node in enumerate(order)}
@@ -206,10 +274,12 @@ def reach(image, config):
206
274
  into its callee and on at its return site, every resolved jump and branch, the rows of a
207
275
  declared indirect jump table, the declared targets of an ``indirectCalls`` site, and every
208
276
  interrupt to the next instruction. A ``leaves`` routine is reached but never decoded. A call to a
209
- ``noReturn`` routine and a ``noReturn`` interrupt site do not continue at the next instruction,
210
- nor does a declared indirect call that is exhaustive and whose targets are all ``noReturn``.
277
+ ``noReturn`` routine, a ``noReturn`` call site and a ``noReturn`` interrupt site do not continue
278
+ at the next instruction, nor does a declared indirect call that is exhaustive and whose targets
279
+ are all ``noReturn``.
211
280
  Each reached target gets one chain with the fewest calls, the assumptions its route rests on,
212
- and the routines on every route to it within the read graph.
281
+ and the routines on every route to it within the read graph. With ``decodedRanges`` the report
282
+ also gives every instruction the walk decoded, as ``decoded_ranges`` groups them.
213
283
  """
214
284
  from .reports import entries
215
285
  starts = _sites(config.get("starts"), "starts", image)
@@ -240,10 +310,12 @@ def reach(image, config):
240
310
  raise ValueError(f"instruction control {at} is a start; the walk decodes every start, so it shows nothing "
241
311
  "the walk found")
242
312
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
313
+ with_ranges = config.get("decodedRanges", False)
314
+ if not isinstance(with_ranges, bool):
315
+ raise ValueError("decodedRanges must be a boolean")
243
316
  instruction_limit = config.get("instructionLimit", 10000)
244
- no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
245
- declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts),
246
- "indirect_calls": indirect_calls}
317
+ no_return_routines, no_return_interrupts, no_return_sites, declared = no_return_inputs(
318
+ config.get("noReturn", []), image, indirect_calls)
247
319
  seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
248
320
  stops=frozenset(leaves), **declared)
249
321
 
@@ -355,6 +427,11 @@ def reach(image, config):
355
427
  cut_by = {at + seen[at].size: f"the call at {at} to noReturn routine{'s' if len(found) > 1 else ''} "
356
428
  + ", ".join(map(str, found))
357
429
  for at, found in sorted(call_targets.items()) if ends(at, found)}
430
+ # A call to a noReturn routine can be declared as a call too; the message then names both declarations.
431
+ for at, row in no_return_sites.items():
432
+ if at in seen:
433
+ cause = f"the noReturn call at {at}"
434
+ cut_by[row["following"]] = f"{cut_by[row['following']]} and {cause}" if row["following"] in cut_by else cause
358
435
  cut_by |= {row["following"]: f"the noReturn interrupt at {at}"
359
436
  for at, row in no_return_interrupts.items() if at in seen}
360
437
  # Not reached and reached-but-unresolved need different fixes, so each failure says which it is.
@@ -391,8 +468,11 @@ def reach(image, config):
391
468
  for at, reason in leaves.items()]
392
469
  # Each call the declaration kept from its return site, so a reviewer can check what follows it.
393
470
  returning_leaves = set(leaves) - set(no_return_routines)
394
- returns = {at: returns_from(graph, seen, returning_leaves, at) for at in no_return_routines}
395
- no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, ended_sites, seen, distance, seen)
471
+ site_targets = {at: call_targets.get(at, []) for at in no_return_sites}
472
+ checked = set(no_return_routines) | {target for found in site_targets.values() for target in found}
473
+ returns = {at: returns_from(graph, seen, returning_leaves, at) for at in checked}
474
+ no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, ended_sites, seen, distance, seen,
475
+ no_return_sites, site_targets)
396
476
  contradicted = any(row.get("contradicted") for row in no_return)
397
477
  # A declared target the walk entered but established no instruction at shows in gaps or contested too,
398
478
  # unless the walk stopped at its instruction limit before reading it.
@@ -402,8 +482,7 @@ def reach(image, config):
402
482
  for site, declaration in indirect_calls.items()]
403
483
  counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
404
484
  "interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
405
- assumptions = ["each reached call returns to its next instruction"
406
- + (", except a call to a noReturn routine" if no_return_routines else ""),
485
+ assumptions = [call_assumption(no_return_routines, no_return_sites),
407
486
  "each reached interrupt returns to its next instruction"
408
487
  + (", except at a noReturn interrupt site" if no_return_interrupts else "")
409
488
  + "; interrupt handlers are not read",
@@ -412,10 +491,12 @@ def reach(image, config):
412
491
  if indirect_calls:
413
492
  assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
414
493
  "it gives, and only those when it is declared exhaustive")
415
- if no_return:
416
- assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
494
+ assumptions += no_return_assumptions(no_return_routines, no_return_interrupts, no_return_sites)
495
+ # Every range is kept whatever the result limit, so a caller can place any site in the walk; the
496
+ # instruction limit bounds their number.
497
+ ranges = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
417
498
  return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return, "indirectCalls": indirect_rows,
418
- "reachedRoutines": sorted(routine_starts), "counts": counts,
499
+ "reachedRoutines": sorted(routine_starts), **ranges, "counts": counts,
419
500
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
420
501
  "contested": sorted(contested)[:limit],
421
502
  "truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested)),
@@ -440,6 +521,14 @@ def reach(image, config):
440
521
  "A noReturn routine is contradicted when a return instruction, or a leaf entered "
441
522
  "other than by a call, lies on its own read paths; an empty returnSites means only "
442
523
  "that the walk read none. "
524
+ "A noReturn call is never contradicted, since whether its target returns to it can "
525
+ "depend on what the caller passes, which the walk does not follow; the return sites "
526
+ "of its targets are listed, and the declaration rests on its reason alone. "
527
+ "decodedRanges, when asked for, covers the instructions the walk decoded; no range "
528
+ "starts at a leaf or at a contested, unresolved overlapping or undecodable start, though "
529
+ "one can lie inside a decoded instruction. A site inside a range lies in a decoded "
530
+ "instruction, which may start before it; a target at the site shows whether one starts "
531
+ "there. "
443
532
  "When instructionLimitReached holds, the walk stopped before reading all it reaches: "
444
533
  "every list and count covers only the part read, which part depends on the walk order, "
445
534
  "and an unreached target may lie past the stop."}
@@ -86,11 +86,13 @@ def search_coverage(image, spans):
86
86
  return rows
87
87
 
88
88
 
89
- def direct_calls(image, config, no_return_calls=frozenset(), no_return_interrupts=frozenset(), indirect_calls=None):
89
+ def direct_calls(image, config, no_return_calls=frozenset(), no_return_interrupts=frozenset(), indirect_calls=None,
90
+ no_return_call_sites=frozenset()):
90
91
  """Every direct call site in the searched regions, as ``incoming`` and ``inventory-check`` read them.
91
92
 
92
93
  Walks the entry-path CFG from every established entry, ending a branch at a call to a routine in
93
- ``no_return_calls`` and at an interrupt site in ``no_return_interrupts`` and following the declared
94
+ ``no_return_calls``, at a call site in ``no_return_call_sites`` and at an interrupt site in
95
+ ``no_return_interrupts`` and following the declared
94
96
  targets of each ``indirect_calls`` site as ``walk`` does, then
95
97
  scans every byte of the regions ``searchRegions`` names (all regions by default) for E8 and 9A
96
98
  call starts, and adds each reached call the scan cannot see (one that starts with a prefix). A
@@ -104,7 +106,7 @@ def direct_calls(image, config, no_return_calls=frozenset(), no_return_interrupt
104
106
  """
105
107
  seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000),
106
108
  no_return_calls=no_return_calls, no_return_interrupts=no_return_interrupts,
107
- indirect_calls=indirect_calls)
109
+ indirect_calls=indirect_calls, no_return_call_sites=no_return_call_sites)
108
110
  rows, scanned = [], {}
109
111
 
110
112
  def classify(at):
@@ -182,7 +182,7 @@ class Step(NamedTuple):
182
182
 
183
183
  def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False,
184
184
  modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset(),
185
- indirect_calls=None):
185
+ indirect_calls=None, no_return_call_sites=frozenset()):
186
186
  """The successors of ``ins`` decoded at ``at``, under the rule ``walk`` and the ``uses`` caller continuation share.
187
187
 
188
188
  An unsupported transfer encoding, a return, an unconditional jump without a resolved target,
@@ -193,9 +193,11 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
193
193
  its return site; with ``step_over_calls`` it continues at its return site only and its target
194
194
  is neither resolved nor reported. A call whose resolved target is in ``no_return_calls``, and an
195
195
  interrupt whose site is in ``no_return_interrupts``, have no return site: the query declared that
196
- they never come back. A call whose site is in ``indirect_calls`` (``indirect_call_declarations``)
197
- continues at each declared target and at its return site, unless the declaration is exhaustive
198
- and every target is in ``no_return_calls``.
196
+ they never come back. A call whose site is in ``no_return_call_sites`` has no return site either,
197
+ whatever its target: the query declared that this one call never comes back. A call whose site
198
+ is in ``indirect_calls`` (``indirect_call_declarations``) continues at each declared target and
199
+ at its return site, unless the declaration is exhaustive and every target is in
200
+ ``no_return_calls``, or the site is in ``no_return_call_sites``.
199
201
  """
200
202
  m, following = base_mnemonic(ins), at + ins.size
201
203
  if unsupported_transfer(image, ins):
@@ -221,7 +223,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
221
223
  if not declaration["exhaustive"]:
222
224
  gaps.append({"site": at, "reason": CALL_NOT_EXHAUSTIVE})
223
225
  edges.append({"site": at, "target": None, "kind": m, "provenance": {"reason": CALL_NOT_EXHAUSTIVE}})
224
- if declared_call_ends(declaration, no_return_calls):
226
+ if declared_call_ends(declaration, no_return_calls) or at in no_return_call_sites:
225
227
  return Step(targets, False, None, gaps, edges, supplied=True)
226
228
  return_site = following if following not in targets else None
227
229
  return Step(targets + [following], False, return_site, gaps, edges, supplied=True)
@@ -238,6 +240,8 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
238
240
  successors.append(target)
239
241
  if m in ("jmp", "ljmp") or (calls and target in no_return_calls):
240
242
  return Step(successors, False, None, gaps, edges)
243
+ if calls and at in no_return_call_sites:
244
+ return Step(successors, False, None, gaps, edges)
241
245
  if m in INTERRUPTS and at in no_return_interrupts:
242
246
  return Step(successors, False, None, gaps, edges)
243
247
  if m == "hlt" or (m in INTERRUPTS and not follow_interrupts and at not in modeled_interrupts):
@@ -256,7 +260,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
256
260
 
257
261
  def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset(),
258
262
  modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset(),
259
- indirect_calls=None):
263
+ indirect_calls=None, no_return_call_sites=frozenset()):
260
264
  """Decode the CFG reached from ``entries`` and check its instruction boundaries.
261
265
 
262
266
  A port access in the flat model records a gap and ends that branch, as ``trace`` stops there.
@@ -267,8 +271,8 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
267
271
  continues past under a call model (ADR 0017). A site in ``stops`` is never decoded:
268
272
  the walk reaches it and goes no further, so it is in no returned set. ``no_return_calls`` and
269
273
  ``no_return_interrupts`` end the branch at a call to one of those routines and at one of those
270
- interrupt sites, and ``indirect_calls`` continues a declared indirect call at its declared
271
- targets, as ``cfg_step`` describes.
274
+ interrupt sites, ``no_return_call_sites`` at a call at one of those sites, and ``indirect_calls``
275
+ continues a declared indirect call at its declared targets, as ``cfg_step`` describes.
272
276
  """
273
277
  instruction_limit(image, limit)
274
278
  pending, seen, gaps, edges = list(entries), {}, [], []
@@ -292,7 +296,8 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
292
296
  seen[at] = ins
293
297
  step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts,
294
298
  modeled_interrupts=modeled_interrupts, no_return_calls=no_return_calls,
295
- no_return_interrupts=no_return_interrupts, indirect_calls=indirect_calls)
299
+ no_return_interrupts=no_return_interrupts, indirect_calls=indirect_calls,
300
+ no_return_call_sites=no_return_call_sites)
296
301
  successors[at] = step.successors
297
302
  gaps.extend(step.gaps)
298
303
  edges.extend(step.edges)
@@ -316,6 +316,57 @@ class InventoryCheckTests(unittest.TestCase):
316
316
  ended = self.check("start\tsize\n1000:0000\t3\n1000:0008\t4\n", data=data, noReturn=exit_routine)
317
317
  self.assertEqual(ended["rowsPastNoReturn"], [])
318
318
 
319
+ def test_a_row_past_a_no_return_call_is_listed_and_the_target_returns_are_given(self):
320
+ # 0000 calls W at 0008, which calls K at 000C and returns. The row 1000:0000 runs on over 0003..0007,
321
+ # where E8 FD FF decodes as a call to 0003. Only the call at 0000 is declared never to return.
322
+ data = bytes.fromhex("e80500" "e8fdff" "0000" "e80100" "c3" "c3")
323
+ inventory = "start\tsize\n1000:0000\t8\n1000:0008\t4\n1000:000C\t1\n"
324
+ reason = "synthetic: this call passes the argument that makes W exit"
325
+ r = self.check(inventory, data=data, noReturn=[{"call": 0, "reason": reason}])
326
+ self.assertEqual(r["rowsPastNoReturn"], [{
327
+ "start": "1000:0000", "size": 8, "kind": "call site", "site": 0, "siteAddress": "1000:0000",
328
+ "siteClassification": "entry-path instruction", "routine": 8, "following": 3, "followingAddress": "1000:0003",
329
+ "followingRead": False, "bytesAfter": 5}])
330
+ self.assertEqual(r["noReturn"], [{"call": 0, "reason": reason, "reached": True,
331
+ "targets": [{"routine": 8, "read": True, "returnSites": [11]}],
332
+ "following": 3, "followingRead": False}])
333
+ self.assertEqual([(t["target"], t["evidence"]) for t in r["targets"]], [(3, "raw byte candidate only")])
334
+ self.assertNotIn("contradicted", r["summary"])
335
+ self.assertEqual(r["assumptions"], ["each reached call returns to its next instruction, except a noReturn call",
336
+ "each interrupt the noReturn check reads returns to its next instruction; "
337
+ "interrupt handlers are not read",
338
+ "each noReturn call never returns to its next instruction, for the reason "
339
+ "it gives, though its target may return to other callers"])
340
+ # A call that a routine declaration already lists is listed once, as a call to that routine.
341
+ both = self.check(inventory, data=data, noReturn=[{"call": 0, "reason": reason}, {"routine": 8, "reason": reason}])
342
+ self.assertEqual([(row["kind"], row["site"]) for row in both["rowsPastNoReturn"]], [("call", 0)])
343
+ self.assertEqual([(row.get("routine"), row.get("contradicted")) for row in both["noReturn"]],
344
+ [(8, True), (None, None)])
345
+
346
+ def test_an_unreached_no_return_call_lists_no_targets_from_raw_byte_candidates(self):
347
+ # As above, 0003 decodes as a call to 0003 only in the raw scan. Declared, it is unreached and, as in
348
+ # reach, has no targets, though the raw scan still lists the candidate.
349
+ data = bytes.fromhex("e80500" "e8fdff" "0000" "e80100" "c3" "c3")
350
+ inventory = "start\tsize\n1000:0000\t8\n1000:0008\t4\n1000:000C\t1\n"
351
+ reason = "synthetic: this call never returns"
352
+ r = self.check(inventory, data=data, noReturn=[{"call": 0, "reason": reason}, {"call": 3, "reason": reason}])
353
+ self.assertEqual(r["noReturn"][1], {"call": 3, "reason": reason, "reached": False, "targets": [],
354
+ "following": 6, "followingRead": False})
355
+ self.assertEqual([(t["target"], t["evidence"]) for t in r["targets"]], [(3, "raw byte candidate only")])
356
+
357
+ def test_the_target_of_a_no_return_call_does_not_change_what_the_routine_check_reads(self):
358
+ # 0000 calls T at 0010 and returns at 0003. T: mov ax, 0C390h; ret. The declared routine R at 0011
359
+ # (nop; ret) lies inside T's mov, so a walk from both would reject the overlap and read neither.
360
+ data = bytes.fromhex("e80d00" "c3" + "90" * 12 + "b890c3" "c3")
361
+ inventory = "start\tsize\n1000:0000\t4\n1000:0010\t4\n"
362
+ routine = {"routine": 0x11, "reason": "synthetic: R exits"}
363
+ r = self.check(inventory, data=data, noReturn=[routine, {"call": 0, "reason": "synthetic: T exits here"}])
364
+ self.assertEqual(r["noReturn"], [
365
+ {"routine": 0x11, "reason": "synthetic: R exits", "reached": False, "read": True, "returnSites": [0x12],
366
+ "contradicted": True, "callSites": []},
367
+ {"call": 0, "reason": "synthetic: T exits here", "reached": True,
368
+ "targets": [{"routine": 0x10, "read": True, "returnSites": [0x13]}], "following": 3, "followingRead": False}])
369
+
319
370
  def test_a_row_with_two_calls_to_a_no_return_routine_lists_both_and_counts_one_row(self):
320
371
  # 0000 jz 0005; 0002 call 000C; 0005 jnz 000A; 0007 call 000C; 000A ret; 000B nop; 000C mov ah,4Ch; int 21h.
321
372
  data = bytes.fromhex("7403" "e80700" "7503" "e80200" "c3" "90" "b44c" "cd21")
@@ -50,6 +50,25 @@ EXITS = bytes.fromhex(
50
50
  "c3" # 0014 what a walk past the interrupt reads
51
51
  )
52
52
 
53
+ # EXITS with the exit behind a wrapper: 0003 calls W at 000C, which calls the worker K at 0010. K
54
+ # returns when AX is nonzero and otherwise ends the program, so whether W returns depends on what its
55
+ # caller passes. The bytes after 0003 are data that decode over B's entry, as in EXITS.
56
+ WORKER = bytes.fromhex(
57
+ "e80500" # 0000 call 0008
58
+ "e80600" # 0003 call 000C
59
+ "b841" # 0006 data: decodes as mov ax, 0B841h through 0008
60
+ "b890c3" # 0008 B: mov ax, 0C390h
61
+ "c3" # 000B ret
62
+ "e80100" # 000C W: call 0010
63
+ "c3" # 000F ret
64
+ "85c0" # 0010 K: test ax, ax
65
+ "7401" # 0012 je 0015
66
+ "c3" # 0014 ret
67
+ "b44c" # 0015 mov ah, 4Ch
68
+ "cd21" # 0017 int 21h
69
+ "c3" # 0019 what a walk past the interrupt reads
70
+ )
71
+
53
72
 
54
73
  def config(data=DISPATCH, entries=(0,), **extra):
55
74
  return {"regions": [{"name": "synthetic", "start": 0, "end": len(data), "ip": 0, "segment": 0x1000, "resident": True,
@@ -270,6 +289,12 @@ class ReachTests(unittest.TestCase):
270
289
  with self.subTest(site=site), self.assertRaisesRegex(ValueError, "Positive controls failed: " + message):
271
290
  run_report(EXITS, config(EXITS, targets=[9], instructionControls=[site], noReturn=exits), "reach")
272
291
 
292
+ def test_a_control_after_a_call_declared_both_ways_names_both_declarations(self):
293
+ exits = [{"routine": 0x10, "reason": "synthetic exit"}, {"call": 3, "reason": "synthetic exit"}]
294
+ with self.assertRaisesRegex(ValueError, r"instruction control 6 is not reached \(it follows the call at 3 to "
295
+ r"noReturn routine 16 and the noReturn call at 3\)$"):
296
+ run_report(EXITS, config(EXITS, targets=[9], instructionControls=[6], noReturn=exits), "reach")
297
+
273
298
  def test_an_unreached_no_return_interrupt_still_gives_its_following_site(self):
274
299
  # 0000 ret; 0001 int 21h, which nothing reaches.
275
300
  data = bytes.fromhex("c3" "cd21")
@@ -286,6 +311,59 @@ class ReachTests(unittest.TestCase):
286
311
  with self.assertRaisesRegex(ValueError, "Duplicate noReturn interrupt"):
287
312
  run_report(EXITS, config(EXITS, targets=[9], noReturn=[exit_interrupt, exit_interrupt]), "reach")
288
313
 
314
+ def test_a_no_return_call_ends_one_call_whose_target_returns_to_other_callers(self):
315
+ exits = "synthetic: this call passes the argument that makes the worker exit"
316
+ # Declared as a routine, W is contradicted: K returns at 0014, so W returns at 000F. K declared
317
+ # instead is contradicted by its own return, which other callers take.
318
+ for routine, returns in ((0x0C, [0x0F]), (0x10, [0x14, 0x19])):
319
+ r = run_report(WORKER, config(WORKER, targets=[9], controls=[0, 3],
320
+ noReturn=[{"routine": routine, "reason": exits}]), "reach")
321
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), (returns, True))
322
+ self.assertFalse(r["negativeUsable"])
323
+ # Declaring W's call to K instead ends W's paths there, so W's own declaration holds, and K's returns
324
+ # are listed on the call row without contradicting it.
325
+ r = run_report(WORKER, config(WORKER, targets=[9], controls=[0, 3], noReturn=[
326
+ {"routine": 0x0C, "reason": exits}, {"call": 0x0C, "reason": exits}]), "reach")
327
+ self.assertEqual((r["gaps"], r["contested"], r["unresolved"]), ([], [], []))
328
+ self.assertTrue(r["negativeUsable"])
329
+ self.assertEqual(r["noReturn"], [
330
+ {"routine": 0x0C, "reason": exits, "reached": True, "read": True, "returnSites": [],
331
+ "contradicted": False, "callSites": [{"site": 3, "following": 6, "followingRead": False}]},
332
+ {"call": 0x0C, "reason": exits, "reached": True,
333
+ "targets": [{"routine": 0x10, "read": True, "returnSites": [0x14, 0x19]}],
334
+ "following": 0x0F, "followingRead": False}])
335
+ self.assertIn("each reached call returns to its next instruction, except a call to a noReturn routine and a "
336
+ "noReturn call", r["assumptions"])
337
+ self.assertIn("each noReturn call never returns to its next instruction, for the reason it gives, though its "
338
+ "target may return to other callers", r["assumptions"])
339
+
340
+ def test_a_no_return_call_alone_keeps_the_bytes_after_it_unread_and_names_itself_in_a_failed_control(self):
341
+ exits = [{"call": 3, "reason": "synthetic: W never returns to this caller"}]
342
+ r = run_report(WORKER, config(WORKER, targets=[9], controls=[0, 3], noReturn=exits), "reach")
343
+ self.assertEqual((r["gaps"], r["contested"]), ([], []))
344
+ self.assertTrue(r["negativeUsable"])
345
+ self.assertEqual(r["noReturn"][0]["targets"], [{"routine": 0x0C, "read": True, "returnSites": [0x0F]}])
346
+ self.assertEqual((r["noReturn"][0]["following"], r["noReturn"][0]["followingRead"]), (6, False))
347
+ self.assertEqual(r["assumptions"][0],
348
+ "each reached call returns to its next instruction, except a noReturn call")
349
+ self.assertNotIn("each noReturn routine and interrupt never returns, for the reason it gives", r["assumptions"])
350
+ with self.assertRaisesRegex(ValueError, r"instruction control 6 is not reached \(it follows the noReturn call "
351
+ r"at 3\)$"):
352
+ run_report(WORKER, config(WORKER, targets=[9], instructionControls=[6], noReturn=exits), "reach")
353
+
354
+ def test_a_no_return_call_with_an_unresolved_target_stays_unresolved_and_ends_its_branch(self):
355
+ # 0006 calls through a far pointer in memory; the ret at 000A follows it and nothing else reaches it.
356
+ r = reach(targets=[0x0A], noReturn=[{"call": 6, "reason": "synthetic: the far routine exits"}])
357
+ self.assertEqual(r["targets"][0]["status"], "not reached")
358
+ self.assertIn(6, [row["site"] for row in r["unresolved"]])
359
+ self.assertEqual(r["noReturn"], [{"call": 6, "reason": "synthetic: the far routine exits", "reached": True,
360
+ "targets": [], "following": 0x0A, "followingRead": False}])
361
+ # An unreached declared call keeps its row. 0000 ret; 0001 call 0004, which nothing reaches; 0004 ret.
362
+ data = bytes.fromhex("c3" "e80000" "c3")
363
+ unreached = run_report(data, config(data, targets=[4], noReturn=[{"call": 1, "reason": "synthetic"}]), "reach")
364
+ self.assertEqual(unreached["noReturn"], [{"call": 1, "reason": "synthetic", "reached": False, "targets": [],
365
+ "following": 4, "followingRead": False}])
366
+
289
367
  def test_a_control_that_is_no_call_is_refused_before_the_walk_and_an_instruction_control_takes_it(self):
290
368
  # 0000 calls 0004, which stores AX at [0100] and returns.
291
369
  data = bytes.fromhex("e80100c3" "a30001" "c3")
@@ -353,6 +431,44 @@ class ReachTests(unittest.TestCase):
353
431
  "start of an unresolved overlapping instruction$"):
354
432
  run_report(data, config(data, targets=[6], indirectJumps=[table], instructionControls=[3, 5]), "reach")
355
433
 
434
+ def test_decoded_ranges_give_each_run_of_decoded_instructions_with_its_routine(self):
435
+ r = reach(indirectJumps=[TABLE], decodedRanges=True)
436
+ # The table row 0020 is read in 000B, which jumped there; 001C..0020, 0025..0028 and the table are not decoded.
437
+ self.assertEqual([(x["start"], x["end"], x["routine"]) for x in r["decodedRanges"]],
438
+ [(0x0, 0xB, 0x0), (0xB, 0x18, 0xB), (0x18, 0x1C, 0x18), (0x20, 0x25, 0xB), (0x28, 0x2F, 0x28)])
439
+ self.assertNotIn("decodedRanges", reach(indirectJumps=[TABLE]))
440
+
441
+ def test_decoded_ranges_leave_out_a_leaf_and_the_result_limit_does_not_cut_them(self):
442
+ r = reach(indirectJumps=[TABLE], decodedRanges=True, limit=1, leaves=[{"routine": 0x18, "reason": "synthetic leaf"}])
443
+ self.assertEqual([x["start"] for x in r["decodedRanges"]], [0x0, 0xB, 0x20, 0x28])
444
+
445
+ def test_decoded_ranges_hold_an_undecodable_start_that_a_decoded_instruction_covers(self):
446
+ # 0000 je 0004 enters the immediate of 0002 mov ax, 0F90, where 0F 04 does not decode; 0005 add al, C3.
447
+ data = bytes.fromhex("7402b8900f04c3")
448
+ r = run_report(data, config(data, targets=[4], decodedRanges=True), "reach")
449
+ self.assertEqual([(x["start"], x["end"]) for x in r["decodedRanges"]], [(0, 7)])
450
+ self.assertIn({"site": 4, "reason": "undecoded or unmapped edge"}, r["gaps"])
451
+ self.assertEqual(r["targets"][0]["insideInstruction"], 2)
452
+
453
+ def test_decoded_ranges_of_a_stopped_walk_cover_the_part_read(self):
454
+ r = reach(indirectJumps=[TABLE], decodedRanges=True, instructionLimit=5)
455
+ self.assertTrue(r["instructionLimitReached"])
456
+ self.assertEqual(sum(x["end"] - x["start"] for x in r["decodedRanges"]), 0xB + 3)
457
+
458
+ def test_decoded_ranges_overlap_where_the_walk_proves_overlapping_instructions(self):
459
+ # 0000 je 0003 falls through to the mov at 0002, whose immediate holds 0003 nop; 0004 ret. 0005 ret.
460
+ # The branch and the fall-through each prove a start, so the walk keeps both instructions.
461
+ data = bytes.fromhex("7401b890c3c3")
462
+ r = run_report(data, config(data, targets=[3, 4], decodedRanges=True), "reach")
463
+ self.assertEqual([(x["start"], x["end"]) for x in r["decodedRanges"]], [(0, 5), (3, 6)])
464
+ self.assertEqual([t["reached"] for t in r["targets"]], [True, True])
465
+ # 0000 je 0003; 0002 mov ax, 00C3 holds 0003 ret; 0005 ret. The ret at 0005 continues the run
466
+ # the mov ends, though the ret at 0003 opened a run after it.
467
+ data = bytes.fromhex("7401b8c300c3")
468
+ r = run_report(data, config(data, targets=[3, 5], decodedRanges=True), "reach")
469
+ self.assertEqual([(x["start"], x["end"]) for x in r["decodedRanges"]], [(0, 6), (3, 4)])
470
+ self.assertEqual([t["reached"] for t in r["targets"]], [True, True])
471
+
356
472
  def test_rejected_inputs(self):
357
473
  too_many = list(range(257))
358
474
  for extra, message in [({"starts": [0x18]}, "established region entry"),
@@ -362,9 +478,15 @@ class ReachTests(unittest.TestCase):
362
478
  ({"leaves": [{"routine": 0x18, "reason": " "}]}, "nonempty reason"),
363
479
  ({"leaves": [{"routine": 0, "reason": "x"}]}, "start cannot be a leaf"),
364
480
  ({"noReturn": {}}, "noReturn must be a list"),
365
- ({"noReturn": [{"routine": 0x18}]}, "exactly one of routine and interrupt"),
481
+ ({"noReturn": [{"routine": 0x18}]}, "exactly one of routine, call and interrupt"),
366
482
  ({"noReturn": [{"routine": 0x18, "interrupt": 0x18, "reason": "x"}]},
367
- "exactly one of routine and interrupt"),
483
+ "exactly one of routine, call and interrupt"),
484
+ ({"noReturn": [{"routine": 0x18, "call": 0x18, "reason": "x"}]},
485
+ "exactly one of routine, call and interrupt"),
486
+ ({"noReturn": [{"call": 0x0B, "reason": "x"}]}, "noReturn call 11 is not a call instruction"),
487
+ ({"noReturn": [{"call": 0x1000, "reason": "x"}]}, "noReturn call"),
488
+ ({"noReturn": [{"call": 0x18, "reason": "x"}, {"call": 0x18, "reason": "y"}]},
489
+ "Duplicate noReturn call"),
368
490
  ({"noReturn": [{"routine": 0x1000, "reason": "x"}]}, "noReturn routine"),
369
491
  ({"noReturn": [{"routine": 0x18, "reason": ""}]}, "nonempty reason"),
370
492
  ({"noReturn": [{"routine": 0x18, "reason": "x"}, {"routine": 0x18, "reason": "y"}]},
@@ -377,6 +499,8 @@ class ReachTests(unittest.TestCase):
377
499
  ({"instructionControls": [0x3, 0x3]}, "distinct file offsets"),
378
500
  ({"instructionControls": [0x1000]}, "instructionControls must be an integer"),
379
501
  ({"instructionControls": [0x0]}, "instruction control 0 is a start"),
502
+ ({"targets": too_many}, r"targets must be a list of 1\.\.256"),
503
+ ({"decodedRanges": "yes"}, "decodedRanges must be a boolean"),
380
504
  ({"controls": too_many}, r"controls must be a list of 0\.\.256"),
381
505
  ({"instructionControls": too_many}, r"instructionControls must be a list of 0\.\.256")]:
382
506
  with self.subTest(extra=extra), self.assertRaisesRegex(ValueError, message):
@@ -511,6 +635,15 @@ class DeclaredIndirectCallTests(unittest.TestCase):
511
635
  self.assertEqual(mixed["noReturn"][0]["callSites"], [])
512
636
  self.assertFalse(mixed["noReturn"][0]["contradicted"])
513
637
 
638
+ def test_a_no_return_call_ends_a_declared_call_whose_targets_return(self):
639
+ r = computed(indirectCalls=[FAR_TARGETS], targets=[0x11],
640
+ noReturn=[{"call": 0x0D, "reason": "synthetic: this call never comes back"}])
641
+ self.assertEqual(r["targets"][0]["status"], "not reached")
642
+ self.assertEqual(r["noReturn"], [{"call": 0x0D, "reason": "synthetic: this call never comes back", "reached": True,
643
+ "targets": [{"routine": 0x110, "read": True, "returnSites": [0x110]},
644
+ {"routine": 0x130, "read": True, "returnSites": []}],
645
+ "following": 0x11, "followingRead": False}])
646
+
514
647
  def test_a_declared_target_inside_a_reached_instruction_is_unread_and_keeps_the_negative_unusable(self):
515
648
  r = computed(indirectCalls=[NEAR_TABLE, {**FAR_TARGETS, "targets": [0x141]}], instructionControls=[0x08])
516
649
  self.assertEqual(r["indirectCalls"][1]["unreadTargets"], [0x141])