scientific-method-engine 19.6.0__tar.gz → 19.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/cli.py +7 -0
  4. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/pcode_backend.py +4 -1
  5. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/reach.py +371 -11
  6. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/trace.py +9 -3
  7. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_reach.py +358 -0
  8. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/.gitignore +0 -0
  9. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/LICENSE +0 -0
  10. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/README.md +0 -0
  11. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/__init__.py +0 -0
  12. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/__main__.py +0 -0
  13. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  14. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  15. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  16. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  17. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  18. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  19. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  20. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  21. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  22. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  23. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  24. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  25. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  26. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  27. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  28. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  29. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  30. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  31. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  32. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  33. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  34. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  35. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  36. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  37. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  38. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  39. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  40. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  41. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  42. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  43. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  44. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  45. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  46. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  47. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  48. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  49. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  50. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/derived_counts.py +0 -0
  51. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  52. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  53. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/image.py +0 -0
  54. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/inventory.py +0 -0
  55. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/loops.py +0 -0
  56. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/machine.py +0 -0
  57. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  58. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  59. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/pe.py +0 -0
  60. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/ranges.py +0 -0
  61. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/relational.py +0 -0
  62. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/reports.py +0 -0
  63. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  64. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/table_bounds.py +0 -0
  65. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/src/scientific_method_engine/x86/values.py +0 -0
  66. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/oracle.py +0 -0
  67. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_config_encoding.py +0 -0
  68. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_continuation_budget.py +0 -0
  69. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_derived_counts.py +0 -0
  70. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_dispatch.py +0 -0
  71. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_effect_order.py +0 -0
  72. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_enter_frames.py +0 -0
  73. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_entry_frame.py +0 -0
  74. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_indirect_far_transfers.py +0 -0
  75. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_interrupt_models.py +0 -0
  76. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_inventory_check.py +0 -0
  77. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_join_terms.py +0 -0
  78. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_loops.py +0 -0
  79. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_memory_scopes.py +0 -0
  80. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_nested_frame_request.py +0 -0
  81. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_oracle.py +0 -0
  82. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_pe.py +0 -0
  83. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_ranges.py +0 -0
  84. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_relational_controls.py +0 -0
  85. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_return_checks.py +0 -0
  86. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_successor_rule.py +0 -0
  87. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_table_bounds.py +0 -0
  88. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_table_continuations.py +0 -0
  89. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_term_limit.py +0 -0
  90. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_upstream_gaps.py +0 -0
  91. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_volatile_memory.py +0 -0
  92. {scientific_method_engine-19.6.0 → scientific_method_engine-19.8.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 19.6.0
3
+ Version: 19.8.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "19.6.0"
8
+ version = "19.8.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -43,6 +43,13 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
43
43
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
44
44
  "and a failed control fails the report saying whether it was reached;\n"
45
45
  "decodedRanges true lists every instruction the walk decoded as half-open ranges with their routine;\n"
46
+ "argumentBranches (true or trace budgets) traces each chain link's routine from its entry to the link's\n"
47
+ "call site, and the target's routine from its entry to the target, and lists the branches whose condition\n"
48
+ "reads the routine's own stack arguments, with the offsets above its entry SP, the directions taken and\n"
49
+ "whether every path was read;\n"
50
+ "spans lists, for each { first, last }, the transfers landing after first and at or before last from\n"
51
+ "outside first..last-1: entries from decoded instructions and starts, candidates from every other byte\n"
52
+ "of declared code with where their source lies, and whether the walk falls through from first to last.\n"
46
53
  "startRoutes true gives each reached target, for each start it is reached from, the fewest calls and the\n"
47
54
  "routines every read route from that start passes.\n"
48
55
  "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
@@ -33,6 +33,9 @@ for code, names in enumerate((("jo",), ("jno",), ("jb", "jc", "jnae"), ("jae", "
33
33
  for name in names:
34
34
  CONDITION_CODES[name] = code
35
35
 
36
+ # The reason of an undecided branch whose flags no comparable producer left (Pypcode.condition).
37
+ FLAG_PRODUCER_UNRESOLVED = "flag producer unresolved"
38
+
36
39
 
37
40
  def linear(term):
38
41
  """A term as a constant plus integer multiples of non-linear subterms."""
@@ -450,7 +453,7 @@ class Pypcode:
450
453
  if carry_only:
451
454
  info = {"predicate": mnemonic, "flag": "CF", "carry": state.carry.report()}
452
455
  elif state.flags is None:
453
- info = {"predicate": mnemonic, "reason": "flag producer unresolved",
456
+ info = {"predicate": mnemonic, "reason": FLAG_PRODUCER_UNRESOLVED,
454
457
  "flagProducer": state.unknown_flag_site, "flagGeneration": state.flag_epoch}
455
458
  else:
456
459
  a, b, operation, site = state.flags
@@ -1,9 +1,11 @@
1
1
  """Transitive reachability from established starts to target sites over the entry-path CFG."""
2
+ import re
2
3
  from collections import deque
3
4
  from .image import integer
5
+ from .machine import State
4
6
  from .trace import (walk, cfg_step, base_mnemonic, unsupported_transfer, holding_instruction, INTERRUPTS, RETURNS,
5
- LIMIT_REASON, OVERLAP_REASON, UNDECODED_REASON)
6
- from .pcode_backend import interrupt_vector
7
+ LIMIT_REASON, OVERLAP_REASON, UNDECODED_REASON, ENTRY_ARRIVAL, budget_input, trace)
8
+ from .pcode_backend import interrupt_vector, FLAG_PRODUCER_UNRESOLVED
7
9
  from .dispatch import (indirect_call_declarations, call_ends, never_runs, indirect_call_assumptions,
8
10
  declared_interrupt, derived_assumptions)
9
11
 
@@ -15,6 +17,21 @@ LEAF_OVERLAP = "leaf start inside a reached instruction; the leaf is not decoded
15
17
  STOP_OVERLAP = ("stop routine start inside a reached instruction; the routine is not decoded, so its boundary is "
16
18
  "unchecked")
17
19
  CHECK_CONTESTED = "contested instruction in the noReturn return check's walk"
20
+ # The stack bytes above a routine's return frame that argumentBranches looks for in a branch condition.
21
+ ARGUMENT_WINDOW = 256
22
+ # The most chain links argumentBranches traces in one report.
23
+ ARGUMENT_LINKS = 256
24
+ ARGUMENT_BUDGETS = ("maxPaths", "maxSteps", "totalSteps", "visitLimit")
25
+ # A carry its producer left unknown, named only by the producer's site and flag serial (a multiply, an
26
+ # add or subtract with carry, a shift by an unknown count), so its term names none of the operands.
27
+ SITE_CARRY = re.compile(r"carry:\d+:\d+")
28
+ # The most bytes from a span's first site to its last (ADR 0044).
29
+ SPAN_BYTES = 4096
30
+ # First bytes of the transfers whose target the encoding gives: Jcc rel8, LOOP*/JCXZ, CALL, JMP and JMP rel8,
31
+ # CALL ptr16:16 and JMP ptr16:16, and 0F, which starts Jcc rel16 and rel32.
32
+ SCANNED_OPCODES = frozenset([*range(0x70, 0x80), *range(0xE0, 0xE4), 0xE8, 0xE9, 0xEA, 0xEB, 0x9A, 0x0F])
33
+ # The scanned opcodes of two-byte transfers with a signed 8-bit displacement: Jcc rel8, LOOP*/JCXZ and JMP rel8.
34
+ SHORT_OPCODES = frozenset([*range(0x70, 0x80), *range(0xE0, 0xE4), 0xEB])
18
35
 
19
36
 
20
37
  def _text(ins):
@@ -87,6 +104,146 @@ def _stops(value, image, starts, leaves, no_return_routines):
87
104
  return routines, sites
88
105
 
89
106
 
107
+ def _spans(value, image):
108
+ """The query's spans as ``(first, last)`` pairs (ADR 0044)."""
109
+ if not isinstance(value, list) or len(value) > 256:
110
+ raise ValueError("spans must be a list of at most 256 objects")
111
+ spans = []
112
+ for span in value:
113
+ if not isinstance(span, dict) or set(span) != {"first", "last"}:
114
+ raise ValueError("Each span needs exactly first and last")
115
+ first = integer(span["first"], 0, len(image.data) - 1, "span first")
116
+ last = integer(span["last"], 0, len(image.data) - 1, "span last")
117
+ region = image.region(first)
118
+ if region is None:
119
+ raise ValueError(f"span first {first} is outside declared code")
120
+ if not first < last < region["end"]:
121
+ raise ValueError(f"span last {last} must lie after first {first} in the region that holds it")
122
+ if last - first > SPAN_BYTES:
123
+ raise ValueError(f"span {first}..{last} is longer than {SPAN_BYTES} bytes")
124
+ if (first, last) in spans:
125
+ raise ValueError(f"Duplicate span {first}..{last}")
126
+ spans.append((first, last))
127
+ return spans
128
+
129
+
130
+ def _transfer(ins):
131
+ """Whether ``ins`` is a call, jump, branch or loop, whose targets ``cfg_step`` gives as edges."""
132
+ m = base_mnemonic(ins)
133
+ return m in CALLS or m in ("jmp", "ljmp") or m.startswith(("j", "loop"))
134
+
135
+
136
+ def _short_lands(image, region, at, landing):
137
+ """Whether the two-byte rel8 transfer whose opcode is at ``at`` can have a successor in ``landing``.
138
+
139
+ It continues at ``at + 2`` (except JMP rel8) and at its encoded target, which ``near_target`` places
140
+ from the IP after it plus the signed displacement, as ``cfg_step`` does once the instruction is
141
+ decoded. A site whose second byte lies past the region is kept, so the decoder decides it.
142
+ """
143
+ if at + 2 > region["end"]:
144
+ return True
145
+ if image.data[at] != 0xEB and at + 2 in landing:
146
+ return True
147
+ displacement = image.data[at + 1] - (0x100 if image.data[at + 1] & 0x80 else 0)
148
+ ip = region["ip"] + at + 2 - region["start"] + displacement
149
+ return image.offset(region["segment"], ip & image.mask) in landing
150
+
151
+
152
+ def _scan_sources(image, seen, landing):
153
+ """The sites outside ``seen`` that may hold a transfer into ``landing``: each byte of declared code that
154
+ starts one of the ``SCANNED_OPCODES``, and each declared indirect jump table site.
155
+
156
+ A rel8 site (``SHORT_OPCODES``) is kept only when ``_short_lands`` places a successor in ``landing``,
157
+ so the scan decodes only the short transfers that can enter a span. The rest are decoded whatever
158
+ their target, which a near or far transfer can place anywhere in the image.
159
+ """
160
+ found = set(image.indirect_jumps)
161
+ for r in image.regions:
162
+ found.update(at for at in range(r["start"], r["end"]) if image.data[at] in SCANNED_OPCODES
163
+ and (image.data[at] not in SHORT_OPCODES or _short_lands(image, r, at, landing)))
164
+ return {at for at in found if at not in seen}
165
+
166
+
167
+ def falls_through(first, last, seen, steps, stop_sites=frozenset()):
168
+ """``(True, None)`` when the walk decoded an instruction at ``first`` and one at each site after it up to
169
+ ``last``, each before ``last`` continuing at the next; otherwise ``(False, row)``, where ``row`` says
170
+ where the run breaks. A stop site before ``last`` breaks the run, since the walk follows nothing
171
+ there, as a call to a stop routine does by having no return site."""
172
+ at = first
173
+ while True:
174
+ if at not in seen:
175
+ return False, {"site": at, "reason": "no instruction the walk decoded starts here"}
176
+ if at == last:
177
+ return True, None
178
+ if at in stop_sites:
179
+ return False, {"site": at, "reason": "a stop site, whose successors the walk does not follow"}
180
+ following = at + seen[at].size
181
+ if following > last:
182
+ return False, {"site": at, "reason": "the instruction runs past last"}
183
+ if following not in steps[at].successors:
184
+ return False, {"site": at, "reason": "the instruction does not continue at the next instruction"}
185
+ at = following
186
+
187
+
188
+ def span_rows(image, spans, starts, seen, steps, contested, overlapping, routine, options, limit, usable,
189
+ stop_sites=frozenset()):
190
+ """The ``spans`` report rows: for each span, the transfers that land after its first site and at or before its
191
+ last from a source outside ``[first, last)``.
192
+
193
+ ``entries`` are the transfers of the instructions the walk decoded (``seen``, with their ``steps``),
194
+ of every kind ``successor_kinds`` names, and each of ``starts`` after a span's first site, which the
195
+ walk enters with no transfer, as a ``start`` row with no source. ``candidates`` are the successors
196
+ of the instructions decoded from every other byte of declared code that starts a call, jump, branch
197
+ or loop, of each contested instruction and of each declared indirect jump or call site the walk did
198
+ not decode, each with the ``position`` of its source. ``usable`` is the report's ``negativeUsable``;
199
+ a span's holds when that does, the span falls through and no entry lands in it. A stop site in
200
+ ``stop_sites`` before a span's last site breaks its fall-through, as ``falls_through`` says.
201
+ """
202
+ landing = {at for first, last in spans for at in range(first + 1, last + 1)}
203
+ entries, candidates = {}, {}
204
+ for at in starts:
205
+ if at in landing:
206
+ entries.setdefault(at, []).append({"source": None, "target": at, "kind": "start", "instruction": None,
207
+ "routine": at})
208
+ for at, ins in seen.items():
209
+ for target, kind in dict.fromkeys(successor_kinds(at, ins, steps[at])):
210
+ if target in landing:
211
+ entries.setdefault(target, []).append({"source": at, "target": target, "kind": kind,
212
+ "instruction": _text(ins), "routine": routine.get(at)})
213
+ sources = (_scan_sources(image, seen, landing) | set(contested)
214
+ | {at for at in options["indirect_calls"] if at not in seen})
215
+ for at in sorted(sources):
216
+ ins = contested[at] if at in contested else image.decode(at)
217
+ # A contested instruction is listed whatever it is: the walk reached it, so its next instruction counts too.
218
+ if ins is None or (at not in contested and not _transfer(ins)):
219
+ continue
220
+ step = cfg_step(image, at, ins, follow_interrupts=True, **options)
221
+ landed = [target for target in dict.fromkeys(step.successors) if target in landing]
222
+ if not landed:
223
+ continue
224
+ holder = holding_instruction(seen, at)
225
+ position = ({"position": "inside a decoded instruction", "insideInstruction": holder} if holder is not None
226
+ else {"position": "start of a contested instruction"} if at in contested
227
+ else {"position": "start of an unresolved overlapping instruction"} if at in overlapping
228
+ else {"position": "outside the decoded instructions"})
229
+ for target in landed:
230
+ candidates.setdefault(target, []).append({"source": at, "target": target, "instruction": _text(ins),
231
+ **position})
232
+ rows = []
233
+ for first, last in spans:
234
+ def inside(found):
235
+ return sorted((row for target in range(first + 1, last + 1) for row in found.get(target, [])
236
+ if row["source"] is None or not first <= row["source"] < last),
237
+ key=lambda row: (row["source"] is not None, row["source"] or 0, row["target"]))
238
+ entered, found = inside(entries), inside(candidates)
239
+ through, broken = falls_through(first, last, seen, steps, stop_sites)
240
+ rows.append({"first": first, "last": last, "fallsThrough": through, **({"break": broken} if broken else {}),
241
+ "entries": entered[:limit], "candidates": found[:limit],
242
+ "counts": {"entries": len(entered), "candidates": len(found)},
243
+ "negativeUsable": usable and through and not entered})
244
+ return rows
245
+
246
+
90
247
  def no_return_declarations(value, image):
91
248
  """``(routines, interrupts, calls)``: the declared routines, interrupt sites and call sites that never return.
92
249
 
@@ -329,6 +486,174 @@ def _dominators(order, predecessors):
329
486
  return idom
330
487
 
331
488
 
489
+ def argument_budget(value):
490
+ """The trace budget ``argumentBranches`` asks for, or None when the option is off.
491
+
492
+ ``true`` takes the trace defaults; an object may set ``maxPaths``, ``maxSteps``, ``totalSteps``,
493
+ ``visitLimit`` and ``maxDepth``, each in the range ``trace`` accepts.
494
+ """
495
+ if value is False:
496
+ return None
497
+ if value is True:
498
+ return {}
499
+ if not isinstance(value, dict) or set(value) - set(ARGUMENT_BUDGETS) - {"maxDepth"}:
500
+ raise ValueError("argumentBranches must be a boolean or an object with only "
501
+ + ", ".join(ARGUMENT_BUDGETS) + " and maxDepth")
502
+ for name in ARGUMENT_BUDGETS:
503
+ if name in value:
504
+ budget_input(value, name)
505
+ if "maxDepth" in value:
506
+ integer(value["maxDepth"], 1, 32, "maxDepth")
507
+ return dict(value)
508
+
509
+
510
+ def _strings(value):
511
+ """Every string inside a report row's nested dicts, lists and tuples, where a term names its unknowns."""
512
+ if isinstance(value, str):
513
+ yield value
514
+ elif isinstance(value, dict):
515
+ for item in value.values():
516
+ yield from _strings(item)
517
+ elif isinstance(value, (list, tuple)):
518
+ for item in value:
519
+ yield from _strings(item)
520
+
521
+
522
+ def _unformed(guard):
523
+ """Whether an undecided branch's condition rests on flags the trace could not form from their operands.
524
+
525
+ That is a branch on flags no comparable producer left, or on a carry its producer left unknown and
526
+ named only by the producer's site (``SITE_CARRY``). Either may test an argument the report cannot see.
527
+ """
528
+ if "decidedBy" in guard:
529
+ return False
530
+ if guard.get("reason") == FLAG_PRODUCER_UNRESOLVED:
531
+ return True
532
+ carry = (guard.get("carry") or {}).get("expression")
533
+ return (isinstance(carry, tuple) and carry[0] == "unknown" and isinstance(carry[1], str)
534
+ and SITE_CARRY.fullmatch(carry[1]) is not None)
535
+
536
+
537
+ def _return_frame(image, seen, call_site):
538
+ """The return frame the call at ``call_site`` pushes, as ``trace`` counts it.
539
+
540
+ Four bytes for a far call, and for a near call right after a decoded ``push cs`` in the real-mode
541
+ model; otherwise the image's near frame.
542
+ """
543
+ if base_mnemonic(image.decode(call_site)) == "lcall":
544
+ return 4
545
+ previous = seen.get(call_site - 1)
546
+ if (not image.flat and previous is not None and previous.mnemonic == "push" and previous.size == 1
547
+ and previous.reg_name(previous.operands[0].reg) == "cs"):
548
+ return 4
549
+ return image.bits // 8
550
+
551
+
552
+ def argument_links(rows, image, seen):
553
+ """Each distinct part of the reached ``rows``' chains as ``(routine, callSite, target, enteredBy, returnFrameBytes)``.
554
+
555
+ A chain link has a ``callSite`` that ``routine`` holds: the routine the chain entered by the call
556
+ before it, or the start. The last part of a chain runs from the routine that holds the target to
557
+ the ``target`` itself, and is left out when the target is that routine's entry. Each part sets
558
+ one of ``callSite`` and ``target`` and leaves the other None. ``enteredBy`` is the call site that
559
+ entered ``routine``, None for a start, and ``returnFrameBytes`` the return frame it pushed (four
560
+ for a far call, or for a near call after ``push cs``), None for a start. ``seen`` holds the
561
+ instructions the walk decoded.
562
+ """
563
+ links = set()
564
+
565
+ def add(hop, call_site, target):
566
+ entered_by = hop.get("callSite")
567
+ frame = None if entered_by is None else _return_frame(image, seen, entered_by)
568
+ links.add((hop["routine"], call_site, target, entered_by, frame))
569
+
570
+ for row in rows:
571
+ hops = row.get("chain", [])
572
+ for before, hop in zip(hops, hops[1:]):
573
+ add(before, hop["callSite"], None)
574
+ if hops and hops[-1]["routine"] != row["target"]:
575
+ add(hops[-1], None, row["target"])
576
+ # Chain links first, then the target parts, each by routine and site.
577
+ return sorted(links, key=lambda link: (link[1] is None, link[0], link[1] if link[2] is None else link[2],
578
+ -1 if link[3] is None else link[3]))
579
+
580
+
581
+ def _stack_bytes(image, entry):
582
+ """The pattern that finds a stack byte the routine received in a term's text, and the entry SP's offset.
583
+
584
+ Every traced routine starts from the same unknown SS and SP, so the pattern a probe at one link's
585
+ ``entry`` finds serves every link.
586
+ """
587
+ probe = State(entry, image, {})
588
+ segment, base, origin = probe.location(probe.segment("ss"), probe.reg(probe.sp))
589
+ # An unread byte's term is named memory:<entry or the order of the event that may have stored it>:<key>.
590
+ return re.compile(r"memory:(entry|\d+):" + re.escape(f"({segment!r}, {base!r}, ") + r"(\d+)\)"), origin
591
+
592
+
593
+ def argument_branches(image, routine, site, frame, budget, stack_bytes):
594
+ """The branches on traced paths from ``routine``'s entry to ``site`` whose condition reads its stack arguments.
595
+
596
+ ``site`` is a link's call site or the chain's target. One bounded ``trace`` runs from the
597
+ routine's entry, with each path stopping at its first arrival at the site. Only an arrival in the
598
+ routine's own frame counts; one inside a callee, the routine itself called again among them, is a
599
+ stop. A branch is listed when a term of its condition names a stack byte the routine
600
+ received above its return frame ``frame`` (the near frame of the image for a start, given as
601
+ None): ``entryOffsets`` as received, ``possiblyRewrittenOffsets`` where a write on the path
602
+ may have stored the byte before the read. The trace follows calls into
603
+ their callees, so a branch inside a callee that tests a value the routine received is listed too.
604
+ ``stack_bytes`` is ``_stack_bytes(image, entry)``. Returns the row fields after the link's own
605
+ ``routine``, ``callSite``, ``target``, ``enteredBy`` and ``returnFrameBytes``.
606
+ """
607
+ report = trace(image, {"entry": routine, **budget, **({"returnBytes": frame} if frame else {})},
608
+ continue_declared_jumps=False, track_loops=False, arrive=site, established_entry=False)
609
+ pattern, origin = stack_bytes
610
+ first = frame or image.bits // 8
611
+ branches, unformed, stops, arrivals = {}, {}, {}, 0
612
+ for path in report["paths"]:
613
+ arrival = path.get("arrival") if path["stop"] == ENTRY_ARRIVAL else None
614
+ if arrival is None or arrival["frameEntry"] != routine or arrival["depth"] != 0:
615
+ if not path["returned"]:
616
+ if arrival is None:
617
+ reason = path["stop"]
618
+ elif arrival["frameEntry"] == routine:
619
+ reason = f"reached the site inside a recursive call to {routine}"
620
+ else:
621
+ reason = f"reached the site inside a call to {arrival['frameEntry']}"
622
+ stops.setdefault((path["stopSite"], reason), None)
623
+ continue
624
+ arrivals += 1
625
+ for guard in path["guards"]:
626
+ received, rewritten = set(), set()
627
+ for text in _strings(guard):
628
+ for match in pattern.finditer(text):
629
+ offset = (int(match.group(2)) - origin) % (1 << image.bits)
630
+ if first <= offset < first + ARGUMENT_WINDOW:
631
+ (received if match.group(1) == "entry" else rewritten).add(offset)
632
+ direction = "taken" if guard["taken"] else "notTaken"
633
+ if received or rewritten:
634
+ row = branches.setdefault(guard["site"], {"site": guard["site"], "predicate": guard["predicate"],
635
+ "flagProducers": set(), "directions": set(),
636
+ "entryOffsets": set(), "possiblyRewrittenOffsets": set()})
637
+ if guard.get("flagProducer") is not None:
638
+ row["flagProducers"].add(guard["flagProducer"])
639
+ row["directions"].add(direction)
640
+ row["entryOffsets"] |= received
641
+ row["possiblyRewrittenOffsets"] |= rewritten
642
+ elif _unformed(guard):
643
+ row = unformed.setdefault(guard["site"], {"site": guard["site"], "predicate": guard["predicate"],
644
+ "directions": set()})
645
+ row["directions"].add(direction)
646
+ gaps = [{"site": g["site"], "reason": g["reason"]} for g in report["gaps"]]
647
+ stop_rows = [{"site": at, "reason": reason} for at, reason in stops]
648
+ return {"arrivals": arrivals,
649
+ "branches": [{k: sorted(v) if isinstance(v, set) else v for k, v in row.items()}
650
+ for _, row in sorted(branches.items())],
651
+ "unformedConditions": [{k: sorted(v) if isinstance(v, set) else v for k, v in row.items()}
652
+ for _, row in sorted(unformed.items())],
653
+ "stops": stop_rows, "gaps": gaps, "complete": not stop_rows and not gaps,
654
+ "pathsRead": len(report["paths"]), "stepsUsed": report["stepsUsed"]}
655
+
656
+
332
657
  def route_dominators(graph, starts, reached):
333
658
  """Immediate dominators of every site in ``reached`` over ``graph``, from a virtual ROOT that leads to every start.
334
659
 
@@ -478,7 +803,10 @@ def reach(image, config):
478
803
  the routines on every route to it within the read graph, and the starts it is reached from. With
479
804
  ``startRoutes`` each reached target also gives, for each of those starts, the fewest calls from it
480
805
  and the routines on every route from it. With ``decodedRanges`` the report
481
- also gives every instruction the walk decoded, as ``decoded_ranges`` groups them.
806
+ also gives every instruction the walk decoded, as ``decoded_ranges`` groups them. With
807
+ ``argumentBranches`` it gives, for each distinct chain link, ``argument_branches``: the branches
808
+ before the link's call, or before the target, that test the routine's own stack arguments (ADR 0045).
809
+ With ``spans`` it gives, for each span, the transfers that land inside it, as ``span_rows`` reports them.
482
810
  """
483
811
  from .reports import entries
484
812
  starts = _sites(config.get("starts"), "starts", image)
@@ -512,6 +840,8 @@ def reach(image, config):
512
840
  with_ranges = config.get("decodedRanges", False)
513
841
  if not isinstance(with_ranges, bool):
514
842
  raise ValueError("decodedRanges must be a boolean")
843
+ branch_budget = argument_budget(config.get("argumentBranches", False))
844
+ spans = _spans(config.get("spans", []), image)
515
845
  with_start_routes = config.get("startRoutes", False)
516
846
  if not isinstance(with_start_routes, bool):
517
847
  raise ValueError("startRoutes must be a boolean")
@@ -531,9 +861,9 @@ def reach(image, config):
531
861
  stops=unread, ends=frozenset(stop_sites), **options)
532
862
 
533
863
  # Each reached call site with its resolved or declared targets.
534
- graph, unresolved, interrupts, call_targets, stop_successors = {}, [], [], {}, {}
864
+ graph, unresolved, interrupts, call_targets, stop_successors, steps = {}, [], [], {}, {}, {}
535
865
  for at, ins in sorted(seen.items()):
536
- step = cfg_step(image, at, ins, follow_interrupts=True, **options)
866
+ step = steps[at] = cfg_step(image, at, ins, follow_interrupts=True, **options)
537
867
  if at in stop_sites:
538
868
  # Nothing at a stop site is followed, so its transfer is neither resolved nor unresolved.
539
869
  graph[at], stop_successors[at] = [], step.successors
@@ -761,20 +1091,37 @@ def reach(image, config):
761
1091
  stopped = stopped or check_stopped
762
1092
  # Every range is kept whatever the result limit, so a caller can place any site in the walk; the
763
1093
  # instruction limit bounds their number.
764
- ranges = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
1094
+ optional = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
1095
+ if branch_budget is not None:
1096
+ links = argument_links(rows, image, seen)
1097
+ if len(links) > ARGUMENT_LINKS:
1098
+ raise ValueError(f"argumentBranches traces at most {ARGUMENT_LINKS} chain links and the chains hold "
1099
+ f"{len(links)}; give fewer targets")
1100
+ stack_bytes = _stack_bytes(image, links[0][0]) if links else None
1101
+ optional["argumentBranches"] = [
1102
+ {"routine": holder, "callSite": call_site, "target": target, "enteredBy": entered_by,
1103
+ "returnFrameBytes": frame,
1104
+ **argument_branches(image, holder, target if call_site is None else call_site, frame, branch_budget,
1105
+ stack_bytes)}
1106
+ for holder, call_site, target, entered_by, frame in links]
1107
+ usable = bool(controls or instruction_controls) and not (stopped or unresolved or gaps or contested or contradicted
1108
+ or check_gap_rows)
1109
+ span_report = ({"spans": span_rows(image, spans, starts, seen, steps, contested, overlapping, routine, options,
1110
+ limit, usable, frozenset(stop_sites))} if spans else {})
1111
+ span_cut = any(sum(row["counts"].values()) > len(row["entries"]) + len(row["candidates"])
1112
+ for row in span_report.get("spans", []))
765
1113
  return {"starts": starts, "targets": rows, "leaves": leaf_rows, "stops": stop_rows, "noReturn": no_return,
766
1114
  "indirectCalls": indirect_rows, "outsideTransfers": outside,
767
- "reachedRoutines": sorted(routine_starts), **ranges, "counts": counts,
1115
+ "reachedRoutines": sorted(routine_starts), **optional, **span_report, "counts": counts,
768
1116
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
769
1117
  "contested": sorted(contested)[:limit], "returnCheckGaps": check_gap_rows[:limit],
770
- "truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested, check_gap_rows)),
1118
+ "truncated": span_cut or any(len(x) > limit for x in (unresolved, interrupts, gaps, contested,
1119
+ check_gap_rows)),
771
1120
  "instructionLimitReached": stopped,
772
1121
  "controls": [{"site": at, "target": call_targets[at][0]} for at in controls],
773
1122
  "instructionControls": [{"site": at, "instruction": _text(seen[at]), "routine": routine.get(at)}
774
1123
  for at in instruction_controls],
775
- "negativeUsable": bool(controls or instruction_controls)
776
- and not (stopped or unresolved or gaps or contested or contradicted
777
- or check_gap_rows),
1124
+ "negativeUsable": usable,
778
1125
  "assumptions": assumptions,
779
1126
  "exclusions": ["computed call and jump targets no declaration gives (listed in unresolved)", "unrelocated far calls",
780
1127
  "code reached only from outside the starts", "runtime reachability"]
@@ -818,6 +1165,19 @@ def reach(image, config):
818
1165
  "one can lie inside a decoded instruction. A site inside a range lies in a decoded "
819
1166
  "instruction, which may start before it; a target at the site shows whether one starts "
820
1167
  "there. "
1168
+ "argumentBranches, when asked for, comes from one bounded p-code trace per chain link, "
1169
+ "from the link's routine to its call site or to the target; it lists the branches on the paths that "
1170
+ "arrived whose condition reads a stack byte the routine received, covers only the "
1171
+ "paths read when complete is false, and does not say whether the caller's values "
1172
+ "take a branch. "
1173
+ "spans, when asked for, list for each span the transfers that land after its first site "
1174
+ "and at or before its last from a source outside first..last-1: entries are those of "
1175
+ "instructions the walk decoded, with each start inside the span as a start entry with no "
1176
+ "source, and candidates those found by decoding the other bytes "
1177
+ "of declared code, which run only if an unread route reaches their source. A span's "
1178
+ "negativeUsable needs the report's, a span that falls through from first to last, and no "
1179
+ "entry; candidates do not change it, since code the walk did not read is outside the "
1180
+ "question, and a leaf's transfers show only among them. "
821
1181
  "When instructionLimitReached holds, the walk, or the return check's walk, stopped "
822
1182
  "before reading all it reaches: every list and count covers only the part read, which "
823
1183
  "part depends on the walk order, an unreached target may lie past the stop, and an "
@@ -647,7 +647,8 @@ def entry_frame(image, config, entry):
647
647
  return frame
648
648
 
649
649
 
650
- def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False, argument_window=0):
650
+ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False, argument_window=0,
651
+ established_entry=True):
651
652
  """Trace bounded paths, preserving declared-table continuations as separate conditional evidence.
652
653
 
653
654
  Ordinary paths run first. A path stopped at a declared indirect jump is then
@@ -665,9 +666,14 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
665
666
  A positive ``argument_window`` gives each traced call event an ``argumentSlots`` entry: the
666
667
  first ``argument_window`` bytes above its return frame as ``State.argument_slots`` saw them
667
668
  when the call ran. ``argument_frames`` reads and removes it.
669
+ ``reach`` passes established_entry=False to trace from a routine its walk entered by a resolved
670
+ call, which need not be a declared region entry; the entry must still lie in declared code.
668
671
  """
669
672
  entry = integer(config.get("entry"), 0, len(image.data) - 1, "entry")
670
- if not any(entry in r["entries"] for r in image.regions):
673
+ if not established_entry:
674
+ if image.region(entry) is None:
675
+ raise ValueError("Trace entry must lie in declared code")
676
+ elif not any(entry in r["entries"] for r in image.regions):
671
677
  raise ValueError("Trace entry must be an established region entry")
672
678
  max_steps = budget_input(config, "maxSteps")
673
679
  max_paths = budget_input(config, "maxPaths")
@@ -821,7 +827,7 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
821
827
  "instructionPath": s.path, "guards": s.guards, "events": s.events,
822
828
  "registers": snapshot(s), "conditionalModels": s.conditional}
823
829
  if reason == ENTRY_ARRIVAL:
824
- arrival = path["arrival"] = {"frameEntry": s.frames[-1]["entry"]}
830
+ arrival = path["arrival"] = {"frameEntry": s.frames[-1]["entry"], "depth": len(s.frames) - 1}
825
831
  for key, name in (("sp", s.sp), ("bp", s.bp)):
826
832
  # Finishing a path only observes it, so a register past the term limit is an offset
827
833
  # not known, with the limit as the reason.