scientific-method-engine 19.7.0__tar.gz → 19.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/PKG-INFO +1 -1
  2. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/pyproject.toml +1 -1
  3. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/cli.py +4 -0
  4. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/inventory.py +6 -3
  5. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/pcode_backend.py +4 -1
  6. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/reach.py +216 -12
  7. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/trace.py +37 -10
  8. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_inventory_check.py +23 -0
  9. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_reach.py +247 -5
  10. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/.gitignore +0 -0
  11. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/LICENSE +0 -0
  12. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/README.md +0 -0
  13. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/__init__.py +0 -0
  14. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/__main__.py +0 -0
  15. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  16. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  17. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  18. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  19. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  20. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  21. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  22. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  23. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  24. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  25. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  26. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  27. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  28. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  29. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  30. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  31. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  32. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  33. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  34. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  35. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  36. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  37. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  38. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  39. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  40. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  41. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  42. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  43. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  44. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  45. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  46. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  47. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  48. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  49. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  50. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/__init__.py +0 -0
  51. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  52. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/derived_counts.py +0 -0
  53. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/dispatch.py +0 -0
  54. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/effect_order.py +0 -0
  55. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/image.py +0 -0
  56. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/loops.py +0 -0
  57. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/machine.py +0 -0
  58. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  59. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/pcode.py +0 -0
  60. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/pe.py +0 -0
  61. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/ranges.py +0 -0
  62. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/relational.py +0 -0
  63. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/reports.py +0 -0
  64. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/result_flow.py +0 -0
  65. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/table_bounds.py +0 -0
  66. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/src/scientific_method_engine/x86/values.py +0 -0
  67. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/oracle.py +0 -0
  68. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_config_encoding.py +0 -0
  69. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_continuation_budget.py +0 -0
  70. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_derived_counts.py +0 -0
  71. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_dispatch.py +0 -0
  72. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_effect_order.py +0 -0
  73. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_enter_frames.py +0 -0
  74. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_entry_frame.py +0 -0
  75. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_indirect_far_transfers.py +0 -0
  76. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_interrupt_models.py +0 -0
  77. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_join_terms.py +0 -0
  78. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_loops.py +0 -0
  79. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_memory_scopes.py +0 -0
  80. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_nested_frame_request.py +0 -0
  81. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_oracle.py +0 -0
  82. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_pe.py +0 -0
  83. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_ranges.py +0 -0
  84. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_relational_controls.py +0 -0
  85. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_return_checks.py +0 -0
  86. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_successor_rule.py +0 -0
  87. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_table_bounds.py +0 -0
  88. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_table_continuations.py +0 -0
  89. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_term_limit.py +0 -0
  90. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_upstream_gaps.py +0 -0
  91. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_volatile_memory.py +0 -0
  92. {scientific_method_engine-19.7.0 → scientific_method_engine-19.8.1}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 19.7.0
3
+ Version: 19.8.1
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "19.7.0"
8
+ version = "19.8.1"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -43,6 +43,10 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
43
43
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
44
44
  "and a failed control fails the report saying whether it was reached;\n"
45
45
  "decodedRanges true lists every instruction the walk decoded as half-open ranges with their routine;\n"
46
+ "argumentBranches (true or trace budgets) traces each chain link's routine from its entry to the link's\n"
47
+ "call site, and the target's routine from its entry to the target, and lists the branches whose condition\n"
48
+ "reads the routine's own stack arguments, with the offsets above its entry SP, the directions taken and\n"
49
+ "whether every path was read;\n"
46
50
  "spans lists, for each { first, last }, the transfers landing after first and at or before last from\n"
47
51
  "outside first..last-1: entries from decoded instructions and starts, candidates from every other byte\n"
48
52
  "of declared code with where their source lies, and whether the walk falls through from first to last.\n"
@@ -239,15 +239,18 @@ def _no_return_check(image, routines, declared, limit):
239
239
  ``routines`` are the declared routines, or the targets of the declared calls. The walk starts at
240
240
  them and continues past every interrupt except a declared one, as ``reach`` reads them, so a
241
241
  routine that ends in an interrupt that returns is contradicted by what follows it. Calls are
242
- stepped over at their return sites, except calls to declared routines and declared calls.
242
+ stepped over at their return sites with their targets resolved, so a call to a declared routine
243
+ or at a declared site still ends its branch, and no callee other than a declared routine is
244
+ decoded or spends ``limit``: the walk reads only the routines' and the declared routines' own paths.
243
245
  A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
244
246
  has no return sites and is not in ``read``.
245
247
  """
246
248
  from .reach import returns_from, successor_graph
247
249
  if not routines:
248
250
  return {}, {}, False
249
- seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True, **declared)
250
- graph = successor_graph(image, seen, follow_interrupts=True, **declared)
251
+ seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True, resolve_over_calls=True,
252
+ **declared)
253
+ graph = successor_graph(image, seen, follow_interrupts=True, resolve_over_calls=True, **declared)
251
254
  return ({at: returns_from(graph, seen, set(), at) for at in routines}, seen,
252
255
  any(g["reason"] == LIMIT_REASON for g in gaps))
253
256
 
@@ -33,6 +33,9 @@ for code, names in enumerate((("jo",), ("jno",), ("jb", "jc", "jnae"), ("jae", "
33
33
  for name in names:
34
34
  CONDITION_CODES[name] = code
35
35
 
36
+ # The reason of an undecided branch whose flags no comparable producer left (Pypcode.condition).
37
+ FLAG_PRODUCER_UNRESOLVED = "flag producer unresolved"
38
+
36
39
 
37
40
  def linear(term):
38
41
  """A term as a constant plus integer multiples of non-linear subterms."""
@@ -450,7 +453,7 @@ class Pypcode:
450
453
  if carry_only:
451
454
  info = {"predicate": mnemonic, "flag": "CF", "carry": state.carry.report()}
452
455
  elif state.flags is None:
453
- info = {"predicate": mnemonic, "reason": "flag producer unresolved",
456
+ info = {"predicate": mnemonic, "reason": FLAG_PRODUCER_UNRESOLVED,
454
457
  "flagProducer": state.unknown_flag_site, "flagGeneration": state.flag_epoch}
455
458
  else:
456
459
  a, b, operation, site = state.flags
@@ -1,9 +1,11 @@
1
1
  """Transitive reachability from established starts to target sites over the entry-path CFG."""
2
+ import re
2
3
  from collections import deque
3
4
  from .image import integer
5
+ from .machine import State
4
6
  from .trace import (walk, cfg_step, base_mnemonic, unsupported_transfer, holding_instruction, INTERRUPTS, RETURNS,
5
- LIMIT_REASON, OVERLAP_REASON, UNDECODED_REASON)
6
- from .pcode_backend import interrupt_vector
7
+ LIMIT_REASON, OVERLAP_REASON, UNDECODED_REASON, ENTRY_ARRIVAL, budget_input, trace)
8
+ from .pcode_backend import interrupt_vector, FLAG_PRODUCER_UNRESOLVED
7
9
  from .dispatch import (indirect_call_declarations, call_ends, never_runs, indirect_call_assumptions,
8
10
  declared_interrupt, derived_assumptions)
9
11
 
@@ -15,6 +17,14 @@ LEAF_OVERLAP = "leaf start inside a reached instruction; the leaf is not decoded
15
17
  STOP_OVERLAP = ("stop routine start inside a reached instruction; the routine is not decoded, so its boundary is "
16
18
  "unchecked")
17
19
  CHECK_CONTESTED = "contested instruction in the noReturn return check's walk"
20
+ # The stack bytes above a routine's return frame that argumentBranches looks for in a branch condition.
21
+ ARGUMENT_WINDOW = 256
22
+ # The most chain links argumentBranches traces in one report.
23
+ ARGUMENT_LINKS = 256
24
+ ARGUMENT_BUDGETS = ("maxPaths", "maxSteps", "totalSteps", "visitLimit")
25
+ # A carry its producer left unknown, named only by the producer's site and flag serial (a multiply, an
26
+ # add or subtract with carry, a shift by an unknown count), so its term names none of the operands.
27
+ SITE_CARRY = re.compile(r"carry:\d+:\d+")
18
28
  # The most bytes from a span's first site to its last (ADR 0044).
19
29
  SPAN_BYTES = 4096
20
30
  # First bytes of the transfers whose target the encoding gives: Jcc rel8, LOOP*/JCXZ, CALL, JMP and JMP rel8,
@@ -476,6 +486,174 @@ def _dominators(order, predecessors):
476
486
  return idom
477
487
 
478
488
 
489
+ def argument_budget(value):
490
+ """The trace budget ``argumentBranches`` asks for, or None when the option is off.
491
+
492
+ ``true`` takes the trace defaults; an object may set ``maxPaths``, ``maxSteps``, ``totalSteps``,
493
+ ``visitLimit`` and ``maxDepth``, each in the range ``trace`` accepts.
494
+ """
495
+ if value is False:
496
+ return None
497
+ if value is True:
498
+ return {}
499
+ if not isinstance(value, dict) or set(value) - set(ARGUMENT_BUDGETS) - {"maxDepth"}:
500
+ raise ValueError("argumentBranches must be a boolean or an object with only "
501
+ + ", ".join(ARGUMENT_BUDGETS) + " and maxDepth")
502
+ for name in ARGUMENT_BUDGETS:
503
+ if name in value:
504
+ budget_input(value, name)
505
+ if "maxDepth" in value:
506
+ integer(value["maxDepth"], 1, 32, "maxDepth")
507
+ return dict(value)
508
+
509
+
510
+ def _strings(value):
511
+ """Every string inside a report row's nested dicts, lists and tuples, where a term names its unknowns."""
512
+ if isinstance(value, str):
513
+ yield value
514
+ elif isinstance(value, dict):
515
+ for item in value.values():
516
+ yield from _strings(item)
517
+ elif isinstance(value, (list, tuple)):
518
+ for item in value:
519
+ yield from _strings(item)
520
+
521
+
522
+ def _unformed(guard):
523
+ """Whether an undecided branch's condition rests on flags the trace could not form from their operands.
524
+
525
+ That is a branch on flags no comparable producer left, or on a carry its producer left unknown and
526
+ named only by the producer's site (``SITE_CARRY``). Either may test an argument the report cannot see.
527
+ """
528
+ if "decidedBy" in guard:
529
+ return False
530
+ if guard.get("reason") == FLAG_PRODUCER_UNRESOLVED:
531
+ return True
532
+ carry = (guard.get("carry") or {}).get("expression")
533
+ return (isinstance(carry, tuple) and carry[0] == "unknown" and isinstance(carry[1], str)
534
+ and SITE_CARRY.fullmatch(carry[1]) is not None)
535
+
536
+
537
+ def _return_frame(image, seen, call_site):
538
+ """The return frame the call at ``call_site`` pushes, as ``trace`` counts it.
539
+
540
+ Four bytes for a far call, and for a near call right after a decoded ``push cs`` in the real-mode
541
+ model; otherwise the image's near frame.
542
+ """
543
+ if base_mnemonic(image.decode(call_site)) == "lcall":
544
+ return 4
545
+ previous = seen.get(call_site - 1)
546
+ if (not image.flat and previous is not None and previous.mnemonic == "push" and previous.size == 1
547
+ and previous.reg_name(previous.operands[0].reg) == "cs"):
548
+ return 4
549
+ return image.bits // 8
550
+
551
+
552
+ def argument_links(rows, image, seen):
553
+ """Each distinct part of the reached ``rows``' chains as ``(routine, callSite, target, enteredBy, returnFrameBytes)``.
554
+
555
+ A chain link has a ``callSite`` that ``routine`` holds: the routine the chain entered by the call
556
+ before it, or the start. The last part of a chain runs from the routine that holds the target to
557
+ the ``target`` itself, and is left out when the target is that routine's entry. Each part sets
558
+ one of ``callSite`` and ``target`` and leaves the other None. ``enteredBy`` is the call site that
559
+ entered ``routine``, None for a start, and ``returnFrameBytes`` the return frame it pushed (four
560
+ for a far call, or for a near call after ``push cs``), None for a start. ``seen`` holds the
561
+ instructions the walk decoded.
562
+ """
563
+ links = set()
564
+
565
+ def add(hop, call_site, target):
566
+ entered_by = hop.get("callSite")
567
+ frame = None if entered_by is None else _return_frame(image, seen, entered_by)
568
+ links.add((hop["routine"], call_site, target, entered_by, frame))
569
+
570
+ for row in rows:
571
+ hops = row.get("chain", [])
572
+ for before, hop in zip(hops, hops[1:]):
573
+ add(before, hop["callSite"], None)
574
+ if hops and hops[-1]["routine"] != row["target"]:
575
+ add(hops[-1], None, row["target"])
576
+ # Chain links first, then the target parts, each by routine and site.
577
+ return sorted(links, key=lambda link: (link[1] is None, link[0], link[1] if link[2] is None else link[2],
578
+ -1 if link[3] is None else link[3]))
579
+
580
+
581
+ def _stack_bytes(image, entry):
582
+ """The pattern that finds a stack byte the routine received in a term's text, and the entry SP's offset.
583
+
584
+ Every traced routine starts from the same unknown SS and SP, so the pattern a probe at one link's
585
+ ``entry`` finds serves every link.
586
+ """
587
+ probe = State(entry, image, {})
588
+ segment, base, origin = probe.location(probe.segment("ss"), probe.reg(probe.sp))
589
+ # An unread byte's term is named memory:<entry or the order of the event that may have stored it>:<key>.
590
+ return re.compile(r"memory:(entry|\d+):" + re.escape(f"({segment!r}, {base!r}, ") + r"(\d+)\)"), origin
591
+
592
+
593
+ def argument_branches(image, routine, site, frame, budget, stack_bytes):
594
+ """The branches on traced paths from ``routine``'s entry to ``site`` whose condition reads its stack arguments.
595
+
596
+ ``site`` is a link's call site or the chain's target. One bounded ``trace`` runs from the
597
+ routine's entry, with each path stopping at its first arrival at the site. Only an arrival in the
598
+ routine's own frame counts; one inside a callee, the routine itself called again among them, is a
599
+ stop. A branch is listed when a term of its condition names a stack byte the routine
600
+ received above its return frame ``frame`` (the near frame of the image for a start, given as
601
+ None): ``entryOffsets`` as received, ``possiblyRewrittenOffsets`` where a write on the path
602
+ may have stored the byte before the read. The trace follows calls into
603
+ their callees, so a branch inside a callee that tests a value the routine received is listed too.
604
+ ``stack_bytes`` is ``_stack_bytes(image, entry)``. Returns the row fields after the link's own
605
+ ``routine``, ``callSite``, ``target``, ``enteredBy`` and ``returnFrameBytes``.
606
+ """
607
+ report = trace(image, {"entry": routine, **budget, **({"returnBytes": frame} if frame else {})},
608
+ continue_declared_jumps=False, track_loops=False, arrive=site, established_entry=False)
609
+ pattern, origin = stack_bytes
610
+ first = frame or image.bits // 8
611
+ branches, unformed, stops, arrivals = {}, {}, {}, 0
612
+ for path in report["paths"]:
613
+ arrival = path.get("arrival") if path["stop"] == ENTRY_ARRIVAL else None
614
+ if arrival is None or arrival["frameEntry"] != routine or arrival["depth"] != 0:
615
+ if not path["returned"]:
616
+ if arrival is None:
617
+ reason = path["stop"]
618
+ elif arrival["frameEntry"] == routine:
619
+ reason = f"reached the site inside a recursive call to {routine}"
620
+ else:
621
+ reason = f"reached the site inside a call to {arrival['frameEntry']}"
622
+ stops.setdefault((path["stopSite"], reason), None)
623
+ continue
624
+ arrivals += 1
625
+ for guard in path["guards"]:
626
+ received, rewritten = set(), set()
627
+ for text in _strings(guard):
628
+ for match in pattern.finditer(text):
629
+ offset = (int(match.group(2)) - origin) % (1 << image.bits)
630
+ if first <= offset < first + ARGUMENT_WINDOW:
631
+ (received if match.group(1) == "entry" else rewritten).add(offset)
632
+ direction = "taken" if guard["taken"] else "notTaken"
633
+ if received or rewritten:
634
+ row = branches.setdefault(guard["site"], {"site": guard["site"], "predicate": guard["predicate"],
635
+ "flagProducers": set(), "directions": set(),
636
+ "entryOffsets": set(), "possiblyRewrittenOffsets": set()})
637
+ if guard.get("flagProducer") is not None:
638
+ row["flagProducers"].add(guard["flagProducer"])
639
+ row["directions"].add(direction)
640
+ row["entryOffsets"] |= received
641
+ row["possiblyRewrittenOffsets"] |= rewritten
642
+ elif _unformed(guard):
643
+ row = unformed.setdefault(guard["site"], {"site": guard["site"], "predicate": guard["predicate"],
644
+ "directions": set()})
645
+ row["directions"].add(direction)
646
+ gaps = [{"site": g["site"], "reason": g["reason"]} for g in report["gaps"]]
647
+ stop_rows = [{"site": at, "reason": reason} for at, reason in stops]
648
+ return {"arrivals": arrivals,
649
+ "branches": [{k: sorted(v) if isinstance(v, set) else v for k, v in row.items()}
650
+ for _, row in sorted(branches.items())],
651
+ "unformedConditions": [{k: sorted(v) if isinstance(v, set) else v for k, v in row.items()}
652
+ for _, row in sorted(unformed.items())],
653
+ "stops": stop_rows, "gaps": gaps, "complete": not stop_rows and not gaps,
654
+ "pathsRead": len(report["paths"]), "stepsUsed": report["stepsUsed"]}
655
+
656
+
479
657
  def route_dominators(graph, starts, reached):
480
658
  """Immediate dominators of every site in ``reached`` over ``graph``, from a virtual ROOT that leads to every start.
481
659
 
@@ -625,8 +803,10 @@ def reach(image, config):
625
803
  the routines on every route to it within the read graph, and the starts it is reached from. With
626
804
  ``startRoutes`` each reached target also gives, for each of those starts, the fewest calls from it
627
805
  and the routines on every route from it. With ``decodedRanges`` the report
628
- also gives every instruction the walk decoded, as ``decoded_ranges`` groups them. With ``spans``
629
- it gives, for each span, the transfers that land inside it, as ``span_rows`` reports them.
806
+ also gives every instruction the walk decoded, as ``decoded_ranges`` groups them. With
807
+ ``argumentBranches`` it gives, for each distinct chain link, ``argument_branches``: the branches
808
+ before the link's call, or before the target, that test the routine's own stack arguments (ADR 0045).
809
+ With ``spans`` it gives, for each span, the transfers that land inside it, as ``span_rows`` reports them.
630
810
  """
631
811
  from .reports import entries
632
812
  starts = _sites(config.get("starts"), "starts", image)
@@ -660,6 +840,7 @@ def reach(image, config):
660
840
  with_ranges = config.get("decodedRanges", False)
661
841
  if not isinstance(with_ranges, bool):
662
842
  raise ValueError("decodedRanges must be a boolean")
843
+ branch_budget = argument_budget(config.get("argumentBranches", False))
663
844
  spans = _spans(config.get("spans", []), image)
664
845
  with_start_routes = config.get("startRoutes", False)
665
846
  if not isinstance(with_start_routes, bool):
@@ -865,10 +1046,13 @@ def reach(image, config):
865
1046
  or any(at in seen or at in contested or at in overlapping for at in stop_sites))
866
1047
  if stop_met and checked:
867
1048
  # A stop cuts the query and says nothing about the build, so the return check reads the reached
868
- # declared routines without the stops: code a stop hid can still contradict a declaration.
1049
+ # declared routines without the stops: code a stop hid can still contradict a declaration. It
1050
+ # steps over calls with their targets resolved, entering only a declared routine, so only the
1051
+ # declared routines' own paths are decoded and spend the instruction limit; returns_from skips
1052
+ # call edges either way.
869
1053
  read, check_gaps, _, _, check_contested = walk(image, sorted(at for at in checked if at in distance),
870
1054
  instruction_limit, follow_interrupts=True,
871
- stops=frozenset(leaves), **declared)
1055
+ stops=frozenset(leaves), resolve_over_calls=True, **declared)
872
1056
  check_stopped = any(g["reason"] == LIMIT_REASON for g in check_gaps)
873
1057
  # What the check could not read past a stop leaves its declarations unchecked there, as a gap of the
874
1058
  # query's own walk would; the rows the query's walk already holds are not repeated.
@@ -878,7 +1062,8 @@ def reach(image, config):
878
1062
  and g["site"] not in check_contested]
879
1063
  + [{"site": at, "reason": CHECK_CONTESTED} for at in check_contested if at not in contested],
880
1064
  key=lambda g: (g["site"], g["reason"]))
881
- check_graph = successor_graph(image, read, kept=frozenset(leaves), follow_interrupts=True, **declared)
1065
+ check_graph = successor_graph(image, read, kept=frozenset(leaves), follow_interrupts=True,
1066
+ resolve_over_calls=True, **declared)
882
1067
  returns = {at: returns_from(check_graph, read, returning_leaves, at) for at in checked}
883
1068
  no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, ended_sites, seen, distance, read,
884
1069
  no_return_sites, site_targets)
@@ -910,7 +1095,19 @@ def reach(image, config):
910
1095
  stopped = stopped or check_stopped
911
1096
  # Every range is kept whatever the result limit, so a caller can place any site in the walk; the
912
1097
  # instruction limit bounds their number.
913
- ranges = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
1098
+ optional = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
1099
+ if branch_budget is not None:
1100
+ links = argument_links(rows, image, seen)
1101
+ if len(links) > ARGUMENT_LINKS:
1102
+ raise ValueError(f"argumentBranches traces at most {ARGUMENT_LINKS} chain links and the chains hold "
1103
+ f"{len(links)}; give fewer targets")
1104
+ stack_bytes = _stack_bytes(image, links[0][0]) if links else None
1105
+ optional["argumentBranches"] = [
1106
+ {"routine": holder, "callSite": call_site, "target": target, "enteredBy": entered_by,
1107
+ "returnFrameBytes": frame,
1108
+ **argument_branches(image, holder, target if call_site is None else call_site, frame, branch_budget,
1109
+ stack_bytes)}
1110
+ for holder, call_site, target, entered_by, frame in links]
914
1111
  usable = bool(controls or instruction_controls) and not (stopped or unresolved or gaps or contested or contradicted
915
1112
  or check_gap_rows)
916
1113
  span_report = ({"spans": span_rows(image, spans, starts, seen, steps, contested, overlapping, routine, options,
@@ -919,7 +1116,7 @@ def reach(image, config):
919
1116
  for row in span_report.get("spans", []))
920
1117
  return {"starts": starts, "targets": rows, "leaves": leaf_rows, "stops": stop_rows, "noReturn": no_return,
921
1118
  "indirectCalls": indirect_rows, "outsideTransfers": outside,
922
- "reachedRoutines": sorted(routine_starts), **ranges, **span_report, "counts": counts,
1119
+ "reachedRoutines": sorted(routine_starts), **optional, **span_report, "counts": counts,
923
1120
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
924
1121
  "contested": sorted(contested)[:limit], "returnCheckGaps": check_gap_rows[:limit],
925
1122
  "truncated": span_cut or any(len(x) > limit for x in (unresolved, interrupts, gaps, contested,
@@ -964,14 +1161,21 @@ def reach(image, config):
964
1161
  "A stop narrows the question and claims nothing about the build: the walk reads no "
965
1162
  "further at a stop site, nor into a stop routine or past a call to it, so a target "
966
1163
  "reached only past a stop is not reached, and negativeUsable answers the narrowed "
967
- "question. The noReturn return check reads past the stops, with a walk of its own, and "
968
- "returnCheckGaps lists what that walk could not read that the query's walk did not "
969
- "record; any row keeps negativeUsable false, as a gap would. "
1164
+ "question. The noReturn return check reads past the stops, with a walk of its own that "
1165
+ "steps over calls at their return sites after resolving their targets, so it decodes "
1166
+ "only the declared routines' own paths; returnCheckGaps lists what that walk could not "
1167
+ "read on those paths that the query's walk did not record, and any row keeps "
1168
+ "negativeUsable false, as a gap would. "
970
1169
  "decodedRanges, when asked for, covers the instructions the walk decoded; no range "
971
1170
  "starts at a leaf, a stop routine or a contested, unresolved overlapping or undecodable start, though "
972
1171
  "one can lie inside a decoded instruction. A site inside a range lies in a decoded "
973
1172
  "instruction, which may start before it; a target at the site shows whether one starts "
974
1173
  "there. "
1174
+ "argumentBranches, when asked for, comes from one bounded p-code trace per chain link, "
1175
+ "from the link's routine to its call site or to the target; it lists the branches on the paths that "
1176
+ "arrived whose condition reads a stack byte the routine received, covers only the "
1177
+ "paths read when complete is false, and does not say whether the caller's values "
1178
+ "take a branch. "
975
1179
  "spans, when asked for, list for each span the transfers that land after its first site "
976
1180
  "and at or before its last from a source outside first..last-1: entries are those of "
977
1181
  "instructions the walk decoded, with each start inside the span as a start entry with no "
@@ -181,7 +181,7 @@ class Step(NamedTuple):
181
181
 
182
182
  def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False,
183
183
  modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset(),
184
- indirect_calls=None, no_return_call_sites=frozenset()):
184
+ indirect_calls=None, no_return_call_sites=frozenset(), resolve_over_calls=False):
185
185
  """The successors of ``ins`` decoded at ``at``, under the rule ``walk`` and the ``uses`` caller continuation share.
186
186
 
187
187
  An unsupported transfer encoding, a return, an unconditional jump without a resolved target,
@@ -190,7 +190,12 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
190
190
  conditional jump or loop continues at its resolved target and at the next instruction. A declared
191
191
  indirect jump continues at its table rows only. A call continues at its resolved target and at
192
192
  its return site; with ``step_over_calls`` it continues at its return site only and its target
193
- is neither resolved nor reported. A call whose resolved target is in ``no_return_calls``, and an
193
+ is neither resolved nor reported. With ``resolve_over_calls`` it also continues at its return
194
+ site only, but the target is resolved first, so the declarations below still end their branches:
195
+ a call to a routine in ``no_return_calls`` ends the branch, an unresolved target still records
196
+ its gap, and a declared indirect call keeps its declaration's rule. Only a target in
197
+ ``no_return_calls`` is entered and reported as an edge, since a declared routine's own paths are
198
+ ones the walk reads; no other callee is. A call whose resolved target is in ``no_return_calls``, and an
194
199
  interrupt whose site is in ``no_return_interrupts``, have no return site: the query declared that
195
200
  they never come back. A call whose site is in ``no_return_call_sites`` has no return site either,
196
201
  whatever its target: the query declared that this one call never comes back. A call whose site
@@ -215,7 +220,9 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
215
220
  return Step(targets, False, None, gaps, edges, supplied=True)
216
221
  declaration = (indirect_calls or {}).get(at)
217
222
  if declaration is not None:
218
- targets = list(declaration["targets"])
223
+ # Under resolve_over_calls only a declared noReturn routine is entered, and successor_kinds counts the
224
+ # target edges as the call successors, so a target that is not entered gets no edge either.
225
+ targets = [t for t in declaration["targets"] if not resolve_over_calls or t in no_return_calls]
219
226
  edges = [{"site": at, "target": target, "kind": m,
220
227
  "provenance": {"encoding": "declared indirect call", "evidence": declaration["evidence"]}}
221
228
  for target in targets]
@@ -232,7 +239,17 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
232
239
  return Step([], True, None, [], [])
233
240
  successors, gaps, edges = [], [], []
234
241
  calls = m in ("call", "lcall")
235
- if (calls and not step_over_calls) or m == "ljmp" or m.startswith(("j", "loop")):
242
+ if calls and resolve_over_calls:
243
+ # The target is resolved to apply the declarations. A declared noReturn routine is entered, since its own
244
+ # paths are ones the check reads; any other callee is neither entered nor reported as an edge.
245
+ target, provenance = call_target(image, at, ins)
246
+ if target is None:
247
+ edges.append({"site": at, "target": None, "kind": m, "provenance": provenance})
248
+ gaps.append({"site": at, "reason": provenance.get("reason", "target outside declared regions")})
249
+ elif target in no_return_calls:
250
+ edges.append({"site": at, "target": target, "kind": m, "provenance": provenance})
251
+ return Step([target], False, None, gaps, edges)
252
+ elif (calls and not step_over_calls) or m == "ljmp" or m.startswith(("j", "loop")):
236
253
  target, provenance = call_target(image, at, ins)
237
254
  edges.append({"site": at, "target": target, "kind": m, "provenance": provenance})
238
255
  if target is None:
@@ -261,7 +278,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
261
278
 
262
279
  def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset(),
263
280
  modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset(),
264
- indirect_calls=None, no_return_call_sites=frozenset(), ends=frozenset()):
281
+ indirect_calls=None, no_return_call_sites=frozenset(), ends=frozenset(), resolve_over_calls=False):
265
282
  """Decode the CFG reached from ``entries`` and check its instruction boundaries.
266
283
 
267
284
  A port access in the flat model records a gap and ends that branch, as ``trace`` stops there.
@@ -274,7 +291,11 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
274
291
  decoded and its boundary checked, and the walk follows none of its successors.
275
292
  ``no_return_calls`` and ``no_return_interrupts`` end the branch at a call to one of those routines and at one of those
276
293
  interrupt sites, ``no_return_call_sites`` at a call at one of those sites, and ``indirect_calls``
277
- continues a declared indirect call at its declared targets, as ``cfg_step`` describes. A
294
+ continues a declared indirect call at its declared targets, as ``cfg_step`` describes.
295
+ ``resolve_over_calls`` steps over every call at its return site after resolving its target, so
296
+ those declarations still end their branches but no callee is entered other than a routine in
297
+ ``no_return_calls``: the walk reads only the entries' and the declared routines' own paths, and
298
+ only they spend ``limit``. A
278
299
  declared table whose count was derived from the code, and that the walk reads, gets a gap
279
300
  (``derived_counts.entry_gaps``) when an edge the walk read enters the routine the count was
280
301
  derived over other than at its start.
@@ -305,7 +326,7 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
305
326
  step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts,
306
327
  modeled_interrupts=modeled_interrupts, no_return_calls=no_return_calls,
307
328
  no_return_interrupts=no_return_interrupts, indirect_calls=indirect_calls,
308
- no_return_call_sites=no_return_call_sites)
329
+ no_return_call_sites=no_return_call_sites, resolve_over_calls=resolve_over_calls)
309
330
  successors[at] = step.successors
310
331
  gaps.extend(step.gaps)
311
332
  edges.extend(step.edges)
@@ -647,7 +668,8 @@ def entry_frame(image, config, entry):
647
668
  return frame
648
669
 
649
670
 
650
- def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False, argument_window=0):
671
+ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False, argument_window=0,
672
+ established_entry=True):
651
673
  """Trace bounded paths, preserving declared-table continuations as separate conditional evidence.
652
674
 
653
675
  Ordinary paths run first. A path stopped at a declared indirect jump is then
@@ -665,9 +687,14 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
665
687
  A positive ``argument_window`` gives each traced call event an ``argumentSlots`` entry: the
666
688
  first ``argument_window`` bytes above its return frame as ``State.argument_slots`` saw them
667
689
  when the call ran. ``argument_frames`` reads and removes it.
690
+ ``reach`` passes established_entry=False to trace from a routine its walk entered by a resolved
691
+ call, which need not be a declared region entry; the entry must still lie in declared code.
668
692
  """
669
693
  entry = integer(config.get("entry"), 0, len(image.data) - 1, "entry")
670
- if not any(entry in r["entries"] for r in image.regions):
694
+ if not established_entry:
695
+ if image.region(entry) is None:
696
+ raise ValueError("Trace entry must lie in declared code")
697
+ elif not any(entry in r["entries"] for r in image.regions):
671
698
  raise ValueError("Trace entry must be an established region entry")
672
699
  max_steps = budget_input(config, "maxSteps")
673
700
  max_paths = budget_input(config, "maxPaths")
@@ -821,7 +848,7 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
821
848
  "instructionPath": s.path, "guards": s.guards, "events": s.events,
822
849
  "registers": snapshot(s), "conditionalModels": s.conditional}
823
850
  if reason == ENTRY_ARRIVAL:
824
- arrival = path["arrival"] = {"frameEntry": s.frames[-1]["entry"]}
851
+ arrival = path["arrival"] = {"frameEntry": s.frames[-1]["entry"], "depth": len(s.frames) - 1}
825
852
  for key, name in (("sp", s.sp), ("bp", s.bp)):
826
853
  # Finishing a path only observes it, so a register past the term limit is an offset
827
854
  # not known, with the limit as the reason.
@@ -431,6 +431,29 @@ class InventoryCheckTests(unittest.TestCase):
431
431
  self.assertIn("The walk that checks the noReturn routines stopped at its instruction limit", stopped["summary"])
432
432
  self.assertFalse(r["noReturnCheckLimitReached"])
433
433
 
434
+ def test_the_no_return_check_reads_only_the_routines_own_paths(self):
435
+ # 0000 ret is the whole entry path. The declared routine at 0001 calls 0008, whose seven
436
+ # instructions a check that entered them would spend a limit of four on; its own path holds two.
437
+ data = bytes.fromhex("c3" "e80400" "ebfe" "9090" + "90" * 6 + "c3")
438
+ inventory = "start\tsize\n1000:0000\t1\n1000:0001\t7\n1000:0008\t7\n"
439
+ exit_routine = [{"routine": 1, "reason": "synthetic exit"}]
440
+ r = self.check(inventory, data=data, noReturn=exit_routine, instructionLimit=4)
441
+ self.assertFalse(r["noReturnCheckLimitReached"])
442
+ row = r["noReturn"][0]
443
+ self.assertEqual((row["returnSites"], row["contradicted"], row["read"], row["reached"]), ([], False, True, False))
444
+ # A return on the routine's own path past the stepped-over call still contradicts the declaration.
445
+ returning = bytes.fromhex("c3" "e80400" "c3" "909090" + "90" * 6 + "c3")
446
+ r = self.check(inventory, data=returning, noReturn=exit_routine, instructionLimit=4)
447
+ self.assertFalse(r["noReturnCheckLimitReached"])
448
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([4], True))
449
+ # A declared indirect call on the routine's own path is stepped over too, and the return past it
450
+ # still contradicts the declaration.
451
+ declared_call = bytes.fromhex("c3" "ffd3" "c3" "c3")
452
+ r = self.check("start\tsize\n1000:0000\t1\n1000:0001\t3\n1000:0004\t1\n", data=declared_call,
453
+ noReturn=exit_routine, indirectCalls=[{"site": 1, "exhaustive": True, "targets": [4],
454
+ "evidence": "synthetic: bx holds 0004"}])
455
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([3], True))
456
+
434
457
  def test_a_contested_instruction_holds_no_target(self):
435
458
  # Entries 0 and 1 overlap, so the call at 0003 is contested. The raw call at 000D targets 0004, a byte of
436
459
  # that contested call, and names no holder, since only established instructions count.
@@ -604,15 +604,16 @@ class StopTests(unittest.TestCase):
604
604
  self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([7], True))
605
605
  self.assertTrue(r["noReturn"][0]["read"])
606
606
  self.assertFalse(r["negativeUsable"])
607
- # The check has the walk's instruction limit. Here R spins after its call to S at 0009, so the walk
608
- # reads only the call into R and R's call to S, and the check, which reads S too, stops before its end.
609
- data = bytes.fromhex("e80100" "c3" "e80200" "ebfe" "9090c3")
610
- stop = [{"routine": 9, "reason": CUT}]
607
+ # The check has the walk's instruction limit, spent on the routine's own paths. The call to the
608
+ # stop routine S ends the query's branch, so R's three instructions past it and its spin are read
609
+ # by the check alone, and a limit of three stops the check there.
610
+ data = bytes.fromhex("e80100" "c3" "e80500" "909090" "ebfe" "c3")
611
+ stop = [{"routine": 0x0C, "reason": CUT}]
611
612
  r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stop), "reach")
612
613
  self.assertEqual((r["noReturn"][0]["contradicted"], r["instructionLimitReached"], r["negativeUsable"]),
613
614
  (False, False, True))
614
615
  r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stop,
615
- instructionLimit=2), "reach")
616
+ instructionLimit=3), "reach")
616
617
  self.assertEqual(r["counts"]["instructions"], 2)
617
618
  self.assertTrue(r["instructionLimitReached"])
618
619
  self.assertFalse(r["negativeUsable"])
@@ -631,6 +632,80 @@ class StopTests(unittest.TestCase):
631
632
  self.assertEqual([row["site"] for row in r["returnCheckGaps"]], [5])
632
633
  self.assertFalse(r["negativeUsable"])
633
634
 
635
+ def test_the_no_return_check_reads_only_the_routines_own_paths(self):
636
+ # 0000 call R at 0004; 0003 ret. R calls the stop routine S at 000B and spins at 0007. S holds
637
+ # a computed jump nothing resolves, so a check that entered it would gain a returnCheckGaps row.
638
+ data = bytes.fromhex("e80100" "c3" "e80400" "ebfe" "9090" "90" "ffe0" "c3")
639
+ no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
640
+ stops = [{"routine": 0x0B, "reason": CUT}]
641
+ r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stops), "reach")
642
+ row = r["noReturn"][0]
643
+ self.assertEqual((row["returnSites"], row["contradicted"], row["read"]), ([], False, True))
644
+ self.assertEqual(r["returnCheckGaps"], [])
645
+ self.assertFalse(r["instructionLimitReached"])
646
+ self.assertTrue(r["negativeUsable"])
647
+ # A return on the routine's own path past the stepped-over call still contradicts the declaration.
648
+ returning = bytes.fromhex("e80100" "c3" "e80400" "c3" "909090" "90" "ffe0" "c3")
649
+ r = run_report(returning, config(returning, targets=[3], controls=[0], noReturn=no_return, stops=stops),
650
+ "reach")
651
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([7], True))
652
+ self.assertEqual(r["returnCheckGaps"], [])
653
+ # A call to its own next instruction keeps its return site on the routine's path. The two arms
654
+ # of the branch at 0000 call the stop routine S at 000A and the declared routine R at 000B.
655
+ self_call = bytes.fromhex("7503" "e80500" "e80300" "c3" "90" "c3" "e80000" "c3")
656
+ r = run_report(self_call, config(self_call, targets=[8], controls=[2, 5],
657
+ noReturn=[{"routine": 0x0B, "reason": "synthetic: declared to end the program"}],
658
+ stops=[{"routine": 0x0A, "reason": CUT}]), "reach")
659
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([0x0E], True))
660
+
661
+ def test_a_call_to_another_declared_routine_still_ends_the_checks_branch(self):
662
+ # The two arms of the branch at 0000 call the stop routine S at 000A and R1 at 000B. R1 calls
663
+ # R2 at 000F, both declared, with a return instruction after the call that is on no path of
664
+ # R1's own.
665
+ data = bytes.fromhex("7503" "e80500" "e80300" "c3" "90" "c3" "e80100" "c3" "ebfe")
666
+ no_return = [{"routine": 0x0B, "reason": "synthetic: declared to end the program"},
667
+ {"routine": 0x0F, "reason": "synthetic: declared to end the program"}]
668
+ r = run_report(data, config(data, targets=[8], controls=[2, 5], noReturn=no_return,
669
+ stops=[{"routine": 0x0A, "reason": CUT}]), "reach")
670
+ self.assertEqual([(row["returnSites"], row["contradicted"]) for row in r["noReturn"]],
671
+ [([], False), ([], False)])
672
+ self.assertEqual(r["returnCheckGaps"], [])
673
+ self.assertTrue(r["negativeUsable"])
674
+
675
+ def test_the_check_reads_past_a_stepped_over_declared_indirect_call(self):
676
+ # 0000 call R at 0004; 0003 ret. R calls the stop routine S at 000C, then makes a declared call
677
+ # at 0007 to T at 000D and returns at 0009, which only the check reads.
678
+ data = bytes.fromhex("e80100" "c3" "e80500" "ffd3" "c3" "9090" "c3" "c3")
679
+ no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
680
+ declared = [{"site": 7, "exhaustive": True, "targets": [0x0D], "evidence": "synthetic: bx holds 000D"}]
681
+ r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, indirectCalls=declared,
682
+ stops=[{"routine": 0x0C, "reason": CUT}]), "reach")
683
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([9], True))
684
+ self.assertFalse(r["negativeUsable"])
685
+
686
+ def test_the_check_enters_a_declared_routine_the_query_reaches_only_past_a_stop(self):
687
+ # R1 at 0004 calls the stop routine S at 000D, then R2 at 000E, which returns. Only the check
688
+ # reaches R2, and its return contradicts R2's declaration.
689
+ data = bytes.fromhex("e80100" "c3" "e80600" "e80400" "ebfe" "90" "c3" "c3")
690
+ no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"},
691
+ {"routine": 0x0E, "reason": "synthetic: declared to end the program"}]
692
+ r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return,
693
+ stops=[{"routine": 0x0D, "reason": CUT}]), "reach")
694
+ self.assertEqual([(row["reached"], row["read"], row["returnSites"], row["contradicted"]) for row in r["noReturn"]],
695
+ [(True, True, [], False), (False, True, [0x0E], True)])
696
+ self.assertFalse(r["negativeUsable"])
697
+
698
+ def test_the_no_return_checks_callees_do_not_spend_the_instruction_limit(self):
699
+ # R's callee at 000B holds nine instructions; a check that entered it would trip a limit of
700
+ # six, where R's own path holds two.
701
+ data = bytes.fromhex("e80100" "c3" "e80400" "ebfe" "9090" + "90" * 8 + "c3")
702
+ no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
703
+ r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return,
704
+ stops=[{"routine": 0x0B, "reason": CUT}], instructionLimit=6), "reach")
705
+ self.assertFalse(r["instructionLimitReached"])
706
+ self.assertFalse(r["noReturn"][0]["contradicted"])
707
+ self.assertTrue(r["negativeUsable"])
708
+
634
709
  def test_rejected_stops(self):
635
710
  site = {"site": 3, "reason": CUT}
636
711
  for extra, message in [({"stops": {}}, "stops must be a list"),
@@ -955,6 +1030,173 @@ class DeclaredOutsideAndNeverRunCallTests(unittest.TestCase):
955
1030
  computed(indirectCalls=[declaration])
956
1031
 
957
1032
 
1033
+ # The start pushes 0 and calls R at 0010, which calls T at 0020 only when bit 2 of its argument is
1034
+ # clear. Every variant below keeps the start, and the target T, at 0020 or 0030.
1035
+ CALLER = "6a00e80b0083c402c3" + "90" * 7 # 0000 push 0; 0002 call 0010; 0005 add sp, 2; 0008 ret
1036
+ STORE = "c70600010100c3" # mov word [0100], 1; ret
1037
+ GATED = bytes.fromhex(CALLER + "55" "8bec" "f6460404" "7403" "e80400" "5d" "c3" "9090" + STORE)
1038
+ # 0010 push bp; 0011 mov bp, sp; 0013 test byte [bp+4], 4; 0017 jz 001C; 0019 call 0020; 001C pop bp; 001D ret
1039
+
1040
+
1041
+ def gated(data=GATED, target=0x20, **extra):
1042
+ return run_report(data, config(data, targets=[target], **extra), "reach")
1043
+
1044
+
1045
+ class ArgumentBranchTests(unittest.TestCase):
1046
+ def test_a_call_behind_a_test_of_the_routines_argument_lists_the_branch(self):
1047
+ r = gated(argumentBranches=True, controls=[0x19])
1048
+ self.assertEqual([h.get("callSite") for h in r["targets"][0]["chain"]], [None, 0x2, 0x19])
1049
+ start, link = r["argumentBranches"]
1050
+ # The start's own link reads no argument and every path to its call was read.
1051
+ self.assertEqual((start["routine"], start["callSite"], start["enteredBy"], start["returnFrameBytes"]),
1052
+ (0x0, 0x2, None, None))
1053
+ self.assertEqual((start["branches"], start["complete"], start["arrivals"]), ([], True, 1))
1054
+ self.assertEqual((link["routine"], link["callSite"], link["enteredBy"], link["returnFrameBytes"]),
1055
+ (0x10, 0x19, 0x2, 2))
1056
+ # [bp+4] after push bp is the byte two above the entry SP, the first byte past the near return frame.
1057
+ self.assertEqual(link["branches"], [{"site": 0x17, "predicate": "je", "flagProducers": [0x13],
1058
+ "directions": ["notTaken"], "entryOffsets": [2],
1059
+ "possiblyRewrittenOffsets": []}])
1060
+ self.assertEqual((link["unformedConditions"], link["stops"], link["gaps"]), ([], [], []))
1061
+ self.assertTrue(link["complete"])
1062
+ self.assertEqual((link["arrivals"], link["pathsRead"]), (1, 2))
1063
+ self.assertNotIn("argumentBranches", gated())
1064
+ self.assertNotIn("argumentBranches", gated(argumentBranches=False))
1065
+
1066
+ def test_an_argument_tested_through_a_register_or_as_a_word_is_listed(self):
1067
+ # 0013 mov al, [bp+4]; 0016 test al, 4; 0018 jz 001D; 001A call 0020; 001D pop bp; 001E ret
1068
+ data = bytes.fromhex(CALLER + "55" "8bec" "8a4604" "a804" "7403" "e80300" "5d" "c3" "90" + STORE)
1069
+ link = gated(data, argumentBranches=True)["argumentBranches"][1]
1070
+ self.assertEqual([(b["site"], b["entryOffsets"]) for b in link["branches"]], [(0x18, [2])])
1071
+ # The start pushes two words; 0013 cmp word [bp+6], 0; 0017 jne 001C reads the second.
1072
+ data = bytes.fromhex("6a006a00e8090083c404c3" + "90" * 5
1073
+ + "55" "8bec" "837e0600" "7503" "e80400" "5d" "c3" "9090" + STORE)
1074
+ link = gated(data, argumentBranches=True)["argumentBranches"][1]
1075
+ self.assertEqual([(b["site"], b["predicate"], b["directions"], b["entryOffsets"]) for b in link["branches"]],
1076
+ [(0x17, "jne", ["notTaken"], [4, 5])])
1077
+
1078
+ def test_an_argument_passed_on_to_a_callee_that_tests_it_is_listed(self):
1079
+ # 0013 push word [bp+4]; 0016 call 0038; 0019 pop cx; 001A jz 001F; 001C call 0030; 001F pop bp; 0020 ret
1080
+ # 0038 H: push bp; mov bp, sp; test byte [bp+4], 4; pop bp; ret. The flags H leaves decide the jz.
1081
+ data = bytes.fromhex(CALLER + "55" "8bec" "ff7604" "e81f00" "59" "7403" "e81100" "5d" "c3" + "90" * 15
1082
+ + STORE + "90" + "55" "8bec" "f6460404" "5d" "c3")
1083
+ link = gated(data, target=0x30, argumentBranches=True)["argumentBranches"][1]
1084
+ self.assertEqual(link["callSite"], 0x1C)
1085
+ self.assertEqual([(b["site"], b["flagProducers"], b["entryOffsets"]) for b in link["branches"]],
1086
+ [(0x1A, [0x3B], [2])])
1087
+ self.assertTrue(link["complete"])
1088
+
1089
+ def test_a_byte_a_write_may_have_stored_is_listed_apart(self):
1090
+ # 0013 mov [si], al may store any byte of the stack before 0015 test byte [bp+4], 4; 0019 jz 001E.
1091
+ data = bytes.fromhex(CALLER + "55" "8bec" "8804" "f6460404" "7403" "e80200" "5d" "c3" + STORE)
1092
+ link = gated(data, argumentBranches=True)["argumentBranches"][1]
1093
+ self.assertEqual([(b["site"], b["entryOffsets"], b["possiblyRewrittenOffsets"]) for b in link["branches"]],
1094
+ [(0x19, [], [2])])
1095
+
1096
+ def test_a_branch_on_flags_the_trace_cannot_name_is_listed_as_unformed(self):
1097
+ # R begins 0010 jz 0015 on the flags it was entered with; 0012 call 0020; 0015 ret.
1098
+ data = bytes.fromhex(CALLER + "7403" "e80b00" "c3" + "90" * 10 + STORE)
1099
+ link = gated(data, argumentBranches=True)["argumentBranches"][1]
1100
+ self.assertEqual(link["branches"], [])
1101
+ self.assertEqual(link["unformedConditions"], [{"site": 0x10, "predicate": "je", "directions": ["notTaken"]}])
1102
+ self.assertTrue(link["complete"])
1103
+
1104
+ def test_a_branch_on_a_carry_named_only_by_its_producer_is_listed_as_unformed(self):
1105
+ # 0013 mov ax, 1; 0016 mul word [bp+4]; 0019 jc 001E; 001B call 0020. MUL leaves CF unknown,
1106
+ # named by its site, so the condition names no argument byte though the product reads one.
1107
+ data = bytes.fromhex(CALLER + "55" "8bec" "b80100" "f76604" "7203" "e80200" "5d" "c3" + STORE)
1108
+ link = gated(data, argumentBranches=True)["argumentBranches"][1]
1109
+ self.assertEqual(link["branches"], [])
1110
+ self.assertEqual(link["unformedConditions"], [{"site": 0x19, "predicate": "jb", "directions": ["notTaken"]}])
1111
+ # A carry formed from its operands is listed when it reads the argument: 0013 shr byte [bp+4], 1; 0016 jc.
1112
+ data = bytes.fromhex(CALLER + "55" "8bec" "d06e04" "7203" "e80500" "5d" "c3" "909090" + STORE)
1113
+ link = gated(data, argumentBranches=True)["argumentBranches"][1]
1114
+ self.assertEqual([(b["site"], b["entryOffsets"]) for b in link["branches"]], [(0x16, [2])])
1115
+ self.assertEqual(link["unformedConditions"], [])
1116
+
1117
+ def test_a_path_the_trace_could_not_read_leaves_the_link_incomplete(self):
1118
+ # R starts 0010 int 21h, which reach continues past and the trace does not model.
1119
+ data = bytes.fromhex(CALLER + "cd21" "e80b00" "c3" + "90" * 10 + STORE)
1120
+ r = gated(data, argumentBranches=True)
1121
+ self.assertTrue(r["targets"][0]["reached"])
1122
+ link = r["argumentBranches"][1]
1123
+ self.assertEqual((link["arrivals"], link["branches"], link["complete"]), (0, [], False))
1124
+ self.assertEqual([s["site"] for s in link["stops"]], [0x10])
1125
+ self.assertIn("interrupt", link["stops"][0]["reason"])
1126
+
1127
+ def test_the_trace_budget_is_the_querys_and_its_limits_leave_the_link_incomplete(self):
1128
+ link = gated(argumentBranches={"maxPaths": 1})["argumentBranches"][1]
1129
+ self.assertEqual(link["gaps"], [{"site": 0x17, "reason": "path limit"}])
1130
+ self.assertFalse(link["complete"])
1131
+ link = gated(argumentBranches={"maxSteps": 2})["argumentBranches"][1]
1132
+ self.assertEqual((link["arrivals"], link["complete"]), (0, False))
1133
+ self.assertTrue(all(s["reason"].startswith("step limit") for s in link["stops"]))
1134
+
1135
+ def test_a_routine_reached_by_a_far_call_has_a_four_byte_return_frame(self):
1136
+ # The start calls R far through a relocated segment word; R tests [bp+6], past the far frame.
1137
+ data = bytes.fromhex("6a00" "9a10000000" "83c402" "c3" + "90" * 5
1138
+ + "55" "8bec" "f6460604" "7403" "e80400" "5d" "cb" "9090" + STORE)
1139
+ relocations = [{"site": 0x5, "segment": 0x1000, "evidence": "synthetic relocation of the segment word"}]
1140
+ r = gated(data, argumentBranches=True, relocations=relocations)
1141
+ link = r["argumentBranches"][1]
1142
+ self.assertEqual((link["enteredBy"], link["returnFrameBytes"]), (0x2, 4))
1143
+ self.assertEqual([(b["site"], b["entryOffsets"]) for b in link["branches"]], [(0x17, [4])])
1144
+
1145
+ def test_a_routine_reached_by_push_cs_and_a_near_call_has_a_four_byte_return_frame(self):
1146
+ # 0000 push 0; 0002 push cs; 0003 call 0010; R returns with retf over the far frame.
1147
+ data = bytes.fromhex("6a00" "0e" "e80a00" "83c402" "c3" + "90" * 6
1148
+ + "55" "8bec" "f6460604" "7403" "e80400" "5d" "cb" "9090" + STORE)
1149
+ link = gated(data, argumentBranches=True)["argumentBranches"][1]
1150
+ self.assertEqual((link["enteredBy"], link["returnFrameBytes"]), (0x3, 4))
1151
+ self.assertEqual([(b["site"], b["entryOffsets"]) for b in link["branches"]], [(0x17, [4])])
1152
+ # The path that skips the call returns through retf over the same frame, so it is no stop.
1153
+ self.assertEqual((link["stops"], link["complete"]), ([], True))
1154
+
1155
+ def test_a_target_behind_a_test_of_its_routines_argument_lists_the_branch(self):
1156
+ # 0010 push bp; mov bp, sp; 0013 test byte [bp+4], 4; 0017 jz 001F; 0019 mov word [0100], 1; 001F pop bp; ret
1157
+ data = bytes.fromhex(CALLER + "55" "8bec" "f6460404" "7406" "c70600010100" "5d" "c3")
1158
+ start, last = gated(data, target=0x19, argumentBranches=True)["argumentBranches"]
1159
+ self.assertEqual((start["routine"], start["callSite"], start["target"]), (0x0, 0x2, None))
1160
+ self.assertEqual((last["routine"], last["callSite"], last["target"], last["enteredBy"],
1161
+ last["returnFrameBytes"]), (0x10, None, 0x19, 0x2, 2))
1162
+ self.assertEqual(last["branches"], [{"site": 0x17, "predicate": "je", "flagProducers": [0x13],
1163
+ "directions": ["notTaken"], "entryOffsets": [2],
1164
+ "possiblyRewrittenOffsets": []}])
1165
+ self.assertEqual((last["arrivals"], last["complete"]), (1, True))
1166
+ # A target in the start routine is traced from the start: 0005 add sp, 2 lies past the call.
1167
+ rows = gated(target=0x5, argumentBranches=True)["argumentBranches"]
1168
+ self.assertEqual([(r["routine"], r["callSite"], r["target"], r["enteredBy"]) for r in rows],
1169
+ [(0x0, None, 0x5, None)])
1170
+
1171
+ def test_an_arrival_inside_a_recursive_call_of_the_routine_is_a_stop(self):
1172
+ # R: 0010 push bp; mov bp, sp; 0013 test byte [bp+4], 4; 0017 jnz 001F; 0019 push 4; 001B call 0010;
1173
+ # 001E pop cx; 001F call 0030; 0022 pop bp; ret. When the bit is clear, R calls itself with it set,
1174
+ # and the nested R arrives at 001F first.
1175
+ data = bytes.fromhex(CALLER + "55" "8bec" "f6460404" "7506" "6a04" "e8f2ff" "59" "e80e00" "5d" "c3"
1176
+ + "90" * 12 + STORE)
1177
+ link = gated(data, target=0x30, argumentBranches=True)["argumentBranches"][1]
1178
+ self.assertEqual((link["routine"], link["callSite"]), (0x10, 0x1F))
1179
+ self.assertEqual(link["arrivals"], 1)
1180
+ self.assertEqual(link["stops"], [{"site": 0x1F, "reason": "reached the site inside a recursive call to 16"}])
1181
+ # Only the arrival in R's own frame gives the branch a direction.
1182
+ self.assertEqual([(b["site"], b["directions"]) for b in link["branches"]], [(0x17, ["taken"])])
1183
+ self.assertFalse(link["complete"])
1184
+
1185
+ def test_rejected_argument_branch_inputs(self):
1186
+ for value, message in [("yes", "argumentBranches must be a boolean or an object"),
1187
+ ({"paths": 1}, "argumentBranches must be a boolean or an object"),
1188
+ ({"maxPaths": 0}, "maxPaths"), ({"maxSteps": 10001}, "maxSteps"),
1189
+ ({"maxDepth": 33}, "maxDepth")]:
1190
+ with self.subTest(value=value), self.assertRaisesRegex(ValueError, message):
1191
+ gated(argumentBranches=value)
1192
+ # 258 routines each call the next; the chain to the last holds 257 links.
1193
+ data = bytes.fromhex("e80100c3" * 257 + "c3")
1194
+ with self.assertRaisesRegex(ValueError, "argumentBranches traces at most 256 chain links and the chains hold 257"):
1195
+ run_report(data, config(data, targets=[257 * 4], argumentBranches=True), "reach")
1196
+ self.assertEqual(len(run_report(data, config(data, targets=[256 * 4], argumentBranches=True),
1197
+ "reach")["argumentBranches"]), 256)
1198
+
1199
+
958
1200
  # A value writer before the request that reads it. 0000 sets AH for the interrupt at 0002. 0005 sets AH
959
1201
  # too and jumps to the interrupt, past the writer at 0000. 0009 stores a word whose immediate, read from
960
1202
  # 000D, is a jump to 0002; 0010 is never reached and also jumps to 0002.