scientific-method-engine 19.6.0__tar.gz → 19.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/cli.py +3 -0
  4. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/reach.py +170 -8
  5. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_reach.py +191 -0
  6. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/.gitignore +0 -0
  7. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/LICENSE +0 -0
  8. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/README.md +0 -0
  9. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/__init__.py +0 -0
  10. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/__main__.py +0 -0
  11. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  12. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  13. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  14. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  15. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  16. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  17. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  18. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  19. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  20. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  21. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  22. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  23. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  24. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  25. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  26. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  27. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  28. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  29. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  30. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  31. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  32. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  33. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  34. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  35. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  36. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  37. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  38. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  39. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  40. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  41. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  42. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  43. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  44. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  45. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  46. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  47. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  48. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/derived_counts.py +0 -0
  49. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  50. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  51. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/image.py +0 -0
  52. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/inventory.py +0 -0
  53. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/loops.py +0 -0
  54. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/machine.py +0 -0
  55. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  56. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  57. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  58. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/pe.py +0 -0
  59. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/ranges.py +0 -0
  60. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/relational.py +0 -0
  61. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/reports.py +0 -0
  62. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  63. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/table_bounds.py +0 -0
  64. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/trace.py +0 -0
  65. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/src/scientific_method_engine/x86/values.py +0 -0
  66. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/oracle.py +0 -0
  67. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_config_encoding.py +0 -0
  68. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_continuation_budget.py +0 -0
  69. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_derived_counts.py +0 -0
  70. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_dispatch.py +0 -0
  71. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_effect_order.py +0 -0
  72. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_enter_frames.py +0 -0
  73. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_entry_frame.py +0 -0
  74. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_indirect_far_transfers.py +0 -0
  75. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_interrupt_models.py +0 -0
  76. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_inventory_check.py +0 -0
  77. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_join_terms.py +0 -0
  78. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_loops.py +0 -0
  79. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_memory_scopes.py +0 -0
  80. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_nested_frame_request.py +0 -0
  81. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_oracle.py +0 -0
  82. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_pe.py +0 -0
  83. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_ranges.py +0 -0
  84. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_relational_controls.py +0 -0
  85. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_return_checks.py +0 -0
  86. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_successor_rule.py +0 -0
  87. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_table_bounds.py +0 -0
  88. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_table_continuations.py +0 -0
  89. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_term_limit.py +0 -0
  90. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_upstream_gaps.py +0 -0
  91. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_volatile_memory.py +0 -0
  92. {scientific_method_engine-19.6.0 → scientific_method_engine-19.7.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 19.6.0
3
+ Version: 19.7.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "19.6.0"
8
+ version = "19.7.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -43,6 +43,9 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
43
43
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
44
44
  "and a failed control fails the report saying whether it was reached;\n"
45
45
  "decodedRanges true lists every instruction the walk decoded as half-open ranges with their routine;\n"
46
+ "spans lists, for each { first, last }, the transfers landing after first and at or before last from\n"
47
+ "outside first..last-1: entries from decoded instructions and starts, candidates from every other byte\n"
48
+ "of declared code with where their source lies, and whether the walk falls through from first to last.\n"
46
49
  "startRoutes true gives each reached target, for each start it is reached from, the fewest calls and the\n"
47
50
  "routines every read route from that start passes.\n"
48
51
  "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
@@ -15,6 +15,13 @@ LEAF_OVERLAP = "leaf start inside a reached instruction; the leaf is not decoded
15
15
  STOP_OVERLAP = ("stop routine start inside a reached instruction; the routine is not decoded, so its boundary is "
16
16
  "unchecked")
17
17
  CHECK_CONTESTED = "contested instruction in the noReturn return check's walk"
18
+ # The most bytes from a span's first site to its last (ADR 0044).
19
+ SPAN_BYTES = 4096
20
+ # First bytes of the transfers whose target the encoding gives: Jcc rel8, LOOP*/JCXZ, CALL, JMP and JMP rel8,
21
+ # CALL ptr16:16 and JMP ptr16:16, and 0F, which starts Jcc rel16 and rel32.
22
+ SCANNED_OPCODES = frozenset([*range(0x70, 0x80), *range(0xE0, 0xE4), 0xE8, 0xE9, 0xEA, 0xEB, 0x9A, 0x0F])
23
+ # The scanned opcodes of two-byte transfers with a signed 8-bit displacement: Jcc rel8, LOOP*/JCXZ and JMP rel8.
24
+ SHORT_OPCODES = frozenset([*range(0x70, 0x80), *range(0xE0, 0xE4), 0xEB])
18
25
 
19
26
 
20
27
  def _text(ins):
@@ -87,6 +94,146 @@ def _stops(value, image, starts, leaves, no_return_routines):
87
94
  return routines, sites
88
95
 
89
96
 
97
+ def _spans(value, image):
98
+ """The query's spans as ``(first, last)`` pairs (ADR 0044)."""
99
+ if not isinstance(value, list) or len(value) > 256:
100
+ raise ValueError("spans must be a list of at most 256 objects")
101
+ spans = []
102
+ for span in value:
103
+ if not isinstance(span, dict) or set(span) != {"first", "last"}:
104
+ raise ValueError("Each span needs exactly first and last")
105
+ first = integer(span["first"], 0, len(image.data) - 1, "span first")
106
+ last = integer(span["last"], 0, len(image.data) - 1, "span last")
107
+ region = image.region(first)
108
+ if region is None:
109
+ raise ValueError(f"span first {first} is outside declared code")
110
+ if not first < last < region["end"]:
111
+ raise ValueError(f"span last {last} must lie after first {first} in the region that holds it")
112
+ if last - first > SPAN_BYTES:
113
+ raise ValueError(f"span {first}..{last} is longer than {SPAN_BYTES} bytes")
114
+ if (first, last) in spans:
115
+ raise ValueError(f"Duplicate span {first}..{last}")
116
+ spans.append((first, last))
117
+ return spans
118
+
119
+
120
+ def _transfer(ins):
121
+ """Whether ``ins`` is a call, jump, branch or loop, whose targets ``cfg_step`` gives as edges."""
122
+ m = base_mnemonic(ins)
123
+ return m in CALLS or m in ("jmp", "ljmp") or m.startswith(("j", "loop"))
124
+
125
+
126
+ def _short_lands(image, region, at, landing):
127
+ """Whether the two-byte rel8 transfer whose opcode is at ``at`` can have a successor in ``landing``.
128
+
129
+ It continues at ``at + 2`` (except JMP rel8) and at its encoded target, which ``near_target`` places
130
+ from the IP after it plus the signed displacement, as ``cfg_step`` does once the instruction is
131
+ decoded. A site whose second byte lies past the region is kept, so the decoder decides it.
132
+ """
133
+ if at + 2 > region["end"]:
134
+ return True
135
+ if image.data[at] != 0xEB and at + 2 in landing:
136
+ return True
137
+ displacement = image.data[at + 1] - (0x100 if image.data[at + 1] & 0x80 else 0)
138
+ ip = region["ip"] + at + 2 - region["start"] + displacement
139
+ return image.offset(region["segment"], ip & image.mask) in landing
140
+
141
+
142
+ def _scan_sources(image, seen, landing):
143
+ """The sites outside ``seen`` that may hold a transfer into ``landing``: each byte of declared code that
144
+ starts one of the ``SCANNED_OPCODES``, and each declared indirect jump table site.
145
+
146
+ A rel8 site (``SHORT_OPCODES``) is kept only when ``_short_lands`` places a successor in ``landing``,
147
+ so the scan decodes only the short transfers that can enter a span. The rest are decoded whatever
148
+ their target, which a near or far transfer can place anywhere in the image.
149
+ """
150
+ found = set(image.indirect_jumps)
151
+ for r in image.regions:
152
+ found.update(at for at in range(r["start"], r["end"]) if image.data[at] in SCANNED_OPCODES
153
+ and (image.data[at] not in SHORT_OPCODES or _short_lands(image, r, at, landing)))
154
+ return {at for at in found if at not in seen}
155
+
156
+
157
+ def falls_through(first, last, seen, steps, stop_sites=frozenset()):
158
+ """``(True, None)`` when the walk decoded an instruction at ``first`` and one at each site after it up to
159
+ ``last``, each before ``last`` continuing at the next; otherwise ``(False, row)``, where ``row`` says
160
+ where the run breaks. A stop site before ``last`` breaks the run, since the walk follows nothing
161
+ there, as a call to a stop routine does by having no return site."""
162
+ at = first
163
+ while True:
164
+ if at not in seen:
165
+ return False, {"site": at, "reason": "no instruction the walk decoded starts here"}
166
+ if at == last:
167
+ return True, None
168
+ if at in stop_sites:
169
+ return False, {"site": at, "reason": "a stop site, whose successors the walk does not follow"}
170
+ following = at + seen[at].size
171
+ if following > last:
172
+ return False, {"site": at, "reason": "the instruction runs past last"}
173
+ if following not in steps[at].successors:
174
+ return False, {"site": at, "reason": "the instruction does not continue at the next instruction"}
175
+ at = following
176
+
177
+
178
+ def span_rows(image, spans, starts, seen, steps, contested, overlapping, routine, options, limit, usable,
179
+ stop_sites=frozenset()):
180
+ """The ``spans`` report rows: for each span, the transfers that land after its first site and at or before its
181
+ last from a source outside ``[first, last)``.
182
+
183
+ ``entries`` are the transfers of the instructions the walk decoded (``seen``, with their ``steps``),
184
+ of every kind ``successor_kinds`` names, and each of ``starts`` after a span's first site, which the
185
+ walk enters with no transfer, as a ``start`` row with no source. ``candidates`` are the successors
186
+ of the instructions decoded from every other byte of declared code that starts a call, jump, branch
187
+ or loop, of each contested instruction and of each declared indirect jump or call site the walk did
188
+ not decode, each with the ``position`` of its source. ``usable`` is the report's ``negativeUsable``;
189
+ a span's holds when that does, the span falls through and no entry lands in it. A stop site in
190
+ ``stop_sites`` before a span's last site breaks its fall-through, as ``falls_through`` says.
191
+ """
192
+ landing = {at for first, last in spans for at in range(first + 1, last + 1)}
193
+ entries, candidates = {}, {}
194
+ for at in starts:
195
+ if at in landing:
196
+ entries.setdefault(at, []).append({"source": None, "target": at, "kind": "start", "instruction": None,
197
+ "routine": at})
198
+ for at, ins in seen.items():
199
+ for target, kind in dict.fromkeys(successor_kinds(at, ins, steps[at])):
200
+ if target in landing:
201
+ entries.setdefault(target, []).append({"source": at, "target": target, "kind": kind,
202
+ "instruction": _text(ins), "routine": routine.get(at)})
203
+ sources = (_scan_sources(image, seen, landing) | set(contested)
204
+ | {at for at in options["indirect_calls"] if at not in seen})
205
+ for at in sorted(sources):
206
+ ins = contested[at] if at in contested else image.decode(at)
207
+ # A contested instruction is listed whatever it is: the walk reached it, so its next instruction counts too.
208
+ if ins is None or (at not in contested and not _transfer(ins)):
209
+ continue
210
+ step = cfg_step(image, at, ins, follow_interrupts=True, **options)
211
+ landed = [target for target in dict.fromkeys(step.successors) if target in landing]
212
+ if not landed:
213
+ continue
214
+ holder = holding_instruction(seen, at)
215
+ position = ({"position": "inside a decoded instruction", "insideInstruction": holder} if holder is not None
216
+ else {"position": "start of a contested instruction"} if at in contested
217
+ else {"position": "start of an unresolved overlapping instruction"} if at in overlapping
218
+ else {"position": "outside the decoded instructions"})
219
+ for target in landed:
220
+ candidates.setdefault(target, []).append({"source": at, "target": target, "instruction": _text(ins),
221
+ **position})
222
+ rows = []
223
+ for first, last in spans:
224
+ def inside(found):
225
+ return sorted((row for target in range(first + 1, last + 1) for row in found.get(target, [])
226
+ if row["source"] is None or not first <= row["source"] < last),
227
+ key=lambda row: (row["source"] is not None, row["source"] or 0, row["target"]))
228
+ entered, found = inside(entries), inside(candidates)
229
+ through, broken = falls_through(first, last, seen, steps, stop_sites)
230
+ rows.append({"first": first, "last": last, "fallsThrough": through, **({"break": broken} if broken else {}),
231
+ "entries": entered[:limit], "candidates": found[:limit],
232
+ "counts": {"entries": len(entered), "candidates": len(found)},
233
+ "negativeUsable": usable and through and not entered})
234
+ return rows
235
+
236
+
90
237
  def no_return_declarations(value, image):
91
238
  """``(routines, interrupts, calls)``: the declared routines, interrupt sites and call sites that never return.
92
239
 
@@ -478,7 +625,8 @@ def reach(image, config):
478
625
  the routines on every route to it within the read graph, and the starts it is reached from. With
479
626
  ``startRoutes`` each reached target also gives, for each of those starts, the fewest calls from it
480
627
  and the routines on every route from it. With ``decodedRanges`` the report
481
- also gives every instruction the walk decoded, as ``decoded_ranges`` groups them.
628
+ also gives every instruction the walk decoded, as ``decoded_ranges`` groups them. With ``spans``
629
+ it gives, for each span, the transfers that land inside it, as ``span_rows`` reports them.
482
630
  """
483
631
  from .reports import entries
484
632
  starts = _sites(config.get("starts"), "starts", image)
@@ -512,6 +660,7 @@ def reach(image, config):
512
660
  with_ranges = config.get("decodedRanges", False)
513
661
  if not isinstance(with_ranges, bool):
514
662
  raise ValueError("decodedRanges must be a boolean")
663
+ spans = _spans(config.get("spans", []), image)
515
664
  with_start_routes = config.get("startRoutes", False)
516
665
  if not isinstance(with_start_routes, bool):
517
666
  raise ValueError("startRoutes must be a boolean")
@@ -531,9 +680,9 @@ def reach(image, config):
531
680
  stops=unread, ends=frozenset(stop_sites), **options)
532
681
 
533
682
  # Each reached call site with its resolved or declared targets.
534
- graph, unresolved, interrupts, call_targets, stop_successors = {}, [], [], {}, {}
683
+ graph, unresolved, interrupts, call_targets, stop_successors, steps = {}, [], [], {}, {}, {}
535
684
  for at, ins in sorted(seen.items()):
536
- step = cfg_step(image, at, ins, follow_interrupts=True, **options)
685
+ step = steps[at] = cfg_step(image, at, ins, follow_interrupts=True, **options)
537
686
  if at in stop_sites:
538
687
  # Nothing at a stop site is followed, so its transfer is neither resolved nor unresolved.
539
688
  graph[at], stop_successors[at] = [], step.successors
@@ -762,19 +911,24 @@ def reach(image, config):
762
911
  # Every range is kept whatever the result limit, so a caller can place any site in the walk; the
763
912
  # instruction limit bounds their number.
764
913
  ranges = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
914
+ usable = bool(controls or instruction_controls) and not (stopped or unresolved or gaps or contested or contradicted
915
+ or check_gap_rows)
916
+ span_report = ({"spans": span_rows(image, spans, starts, seen, steps, contested, overlapping, routine, options,
917
+ limit, usable, frozenset(stop_sites))} if spans else {})
918
+ span_cut = any(sum(row["counts"].values()) > len(row["entries"]) + len(row["candidates"])
919
+ for row in span_report.get("spans", []))
765
920
  return {"starts": starts, "targets": rows, "leaves": leaf_rows, "stops": stop_rows, "noReturn": no_return,
766
921
  "indirectCalls": indirect_rows, "outsideTransfers": outside,
767
- "reachedRoutines": sorted(routine_starts), **ranges, "counts": counts,
922
+ "reachedRoutines": sorted(routine_starts), **ranges, **span_report, "counts": counts,
768
923
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
769
924
  "contested": sorted(contested)[:limit], "returnCheckGaps": check_gap_rows[:limit],
770
- "truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested, check_gap_rows)),
925
+ "truncated": span_cut or any(len(x) > limit for x in (unresolved, interrupts, gaps, contested,
926
+ check_gap_rows)),
771
927
  "instructionLimitReached": stopped,
772
928
  "controls": [{"site": at, "target": call_targets[at][0]} for at in controls],
773
929
  "instructionControls": [{"site": at, "instruction": _text(seen[at]), "routine": routine.get(at)}
774
930
  for at in instruction_controls],
775
- "negativeUsable": bool(controls or instruction_controls)
776
- and not (stopped or unresolved or gaps or contested or contradicted
777
- or check_gap_rows),
931
+ "negativeUsable": usable,
778
932
  "assumptions": assumptions,
779
933
  "exclusions": ["computed call and jump targets no declaration gives (listed in unresolved)", "unrelocated far calls",
780
934
  "code reached only from outside the starts", "runtime reachability"]
@@ -818,6 +972,14 @@ def reach(image, config):
818
972
  "one can lie inside a decoded instruction. A site inside a range lies in a decoded "
819
973
  "instruction, which may start before it; a target at the site shows whether one starts "
820
974
  "there. "
975
+ "spans, when asked for, list for each span the transfers that land after its first site "
976
+ "and at or before its last from a source outside first..last-1: entries are those of "
977
+ "instructions the walk decoded, with each start inside the span as a start entry with no "
978
+ "source, and candidates those found by decoding the other bytes "
979
+ "of declared code, which run only if an unread route reaches their source. A span's "
980
+ "negativeUsable needs the report's, a span that falls through from first to last, and no "
981
+ "entry; candidates do not change it, since code the walk did not read is outside the "
982
+ "question, and a leaf's transfers show only among them. "
821
983
  "When instructionLimitReached holds, the walk, or the return check's walk, stopped "
822
984
  "before reading all it reaches: every list and count covers only the part read, which "
823
985
  "part depends on the walk order, an unreached target may lie past the stop, and an "
@@ -955,6 +955,197 @@ class DeclaredOutsideAndNeverRunCallTests(unittest.TestCase):
955
955
  computed(indirectCalls=[declaration])
956
956
 
957
957
 
958
+ # A value writer before the request that reads it. 0000 sets AH for the interrupt at 0002. 0005 sets AH
959
+ # too and jumps to the interrupt, past the writer at 0000. 0009 stores a word whose immediate, read from
960
+ # 000D, is a jump to 0002; 0010 is never reached and also jumps to 0002.
961
+ WRITER = bytes.fromhex(
962
+ "b43d" # 0000 mov ah, 3Dh
963
+ "cd21" # 0002 int 21h
964
+ "c3" # 0004 ret
965
+ "b43e" # 0005 mov ah, 3Eh
966
+ "ebf9" # 0007 jmp 0002
967
+ "c7060200ebf3" # 0009 mov word [0002], 0F3EBh; from 000D: jmp 0002
968
+ "c3" # 000F ret
969
+ "ebf0" # 0010 jmp 0002, unreached
970
+ )
971
+ WRITER_STARTS = (0, 5, 9)
972
+
973
+
974
+ def spans(data=WRITER, entries=WRITER_STARTS, **extra):
975
+ return run_report(data, config(data, entries, **({"starts": list(entries), "targets": [2],
976
+ "instructionControls": [2]} | extra)), "reach")
977
+
978
+
979
+ class SpanTests(unittest.TestCase):
980
+ def test_a_span_lists_path_entries_and_candidates_by_where_their_source_lies(self):
981
+ r = spans(spans=[{"first": 0, "last": 2}])
982
+ self.assertTrue(r["negativeUsable"])
983
+ span = r["spans"][0]
984
+ self.assertEqual(span["entries"], [{"source": 7, "target": 2, "kind": "jump", "instruction": "jmp 2", "routine": 5}])
985
+ self.assertEqual(span["candidates"], [
986
+ {"source": 0xD, "target": 2, "instruction": "jmp 2", "position": "inside a decoded instruction",
987
+ "insideInstruction": 9},
988
+ {"source": 0x10, "target": 2, "instruction": "jmp 2", "position": "outside the decoded instructions"}])
989
+ self.assertEqual(span["counts"], {"entries": 1, "candidates": 2})
990
+ self.assertTrue(span["fallsThrough"])
991
+ self.assertNotIn("break", span)
992
+ self.assertFalse(span["negativeUsable"])
993
+ self.assertNotIn("spans", spans())
994
+
995
+ def test_a_candidate_that_lands_outside_the_span_is_not_listed(self):
996
+ # From 000D, EB F7 jumps to 0006, past the span 0000..0002; 0010 is padding.
997
+ data = WRITER[:0xD] + bytes.fromhex("ebf7c39090")
998
+ span = spans(data, spans=[{"first": 0, "last": 2}])["spans"][0]
999
+ self.assertEqual([row["source"] for row in span["entries"]], [7])
1000
+ self.assertEqual(span["candidates"], [])
1001
+
1002
+ def test_a_span_no_transfer_enters_is_a_usable_negative(self):
1003
+ # 0000 mov ax, 0FEEBh holds EB FE at 0001, a jump to 0001 from inside the span; 0003 int 21h; 0005 ret.
1004
+ data = bytes.fromhex("b8ebfecd21c3")
1005
+ span = spans(data, entries=(0,), targets=[3], instructionControls=[3],
1006
+ spans=[{"first": 0, "last": 3}])["spans"][0]
1007
+ self.assertEqual((span["entries"], span["candidates"], span["fallsThrough"]), ([], [], True))
1008
+ self.assertTrue(span["negativeUsable"])
1009
+ # Without a control the report's negative is not usable, and neither is the span's.
1010
+ span = spans(data, entries=(0,), targets=[3], instructionControls=[], spans=[{"first": 0, "last": 3}])
1011
+ self.assertFalse(span["spans"][0]["negativeUsable"])
1012
+
1013
+ def test_a_transfer_from_the_last_site_back_into_the_span_enters_it(self):
1014
+ # 0000 mov ah, 3Dh; 0002 nop; 0003 loop 0002; 0005 ret.
1015
+ data = bytes.fromhex("b43d90e2fdc3")
1016
+ span = spans(data, entries=(0,), targets=[3], instructionControls=[3],
1017
+ spans=[{"first": 0, "last": 3}])["spans"][0]
1018
+ self.assertEqual([(row["source"], row["target"], row["kind"]) for row in span["entries"]], [(3, 2, "jump")])
1019
+ self.assertFalse(span["negativeUsable"])
1020
+
1021
+ def test_a_start_inside_the_span_enters_it(self):
1022
+ # 0000 mov ah, 3Dh; 0002 int 21h; 0004 ret. The walk also starts at 0002, past the writer.
1023
+ data = bytes.fromhex("b43dcd21c3")
1024
+ r = spans(data, entries=(0, 2), targets=[2], instructionControls=[4], spans=[{"first": 0, "last": 2}])
1025
+ self.assertTrue(r["negativeUsable"])
1026
+ span = r["spans"][0]
1027
+ self.assertEqual(span["entries"], [{"source": None, "target": 2, "kind": "start", "instruction": None,
1028
+ "routine": 2}])
1029
+ self.assertFalse(span["negativeUsable"])
1030
+ # A start at the first site runs the span from its first instruction.
1031
+ span = spans(data, entries=(0, 2), targets=[2], instructionControls=[4],
1032
+ spans=[{"first": 2, "last": 4}])["spans"][0]
1033
+ self.assertEqual(span["entries"], [])
1034
+
1035
+ def test_a_candidate_call_returns_into_the_span(self):
1036
+ # 0000 mov al, 0E8h holds a call at 0001 whose target lies outside the image and whose return site is 0004;
1037
+ # 0002 mov ah, 3Dh; 0004 int 21h; 0006 ret.
1038
+ data = bytes.fromhex("b0e8b43dcd21c3")
1039
+ span = spans(data, entries=(0,), targets=[4], instructionControls=[4],
1040
+ spans=[{"first": 2, "last": 4}])["spans"][0]
1041
+ self.assertEqual(span["entries"], [])
1042
+ self.assertEqual([(row["source"], row["target"], row["position"], row["insideInstruction"])
1043
+ for row in span["candidates"]], [(1, 4, "inside a decoded instruction", 0)])
1044
+
1045
+ def test_a_stop_site_inside_the_span_breaks_its_run(self):
1046
+ # 0000 mov ah, 3Dh; 0002 jz 0005; 0004 nop, a stop site; 0005 int 21h; 0007 ret. The jz from inside the
1047
+ # span reaches 0005, but the walk follows nothing at 0004, as it follows no call to a stop routine.
1048
+ data = bytes.fromhex("b43d7401 90cd21c3".replace(" ", ""))
1049
+ r = spans(data, entries=(0,), targets=[5], instructionControls=[5], stops=[{"site": 4, "reason": CUT}],
1050
+ spans=[{"first": 0, "last": 5}])
1051
+ self.assertTrue(r["negativeUsable"])
1052
+ span = r["spans"][0]
1053
+ self.assertEqual(span["entries"], [])
1054
+ self.assertEqual((span["fallsThrough"], span["break"]),
1055
+ (False, {"site": 4, "reason": "a stop site, whose successors the walk does not follow"}))
1056
+ self.assertFalse(span["negativeUsable"])
1057
+ # A stop site at the last site ends nothing the span needs.
1058
+ span = spans(data, entries=(0,), targets=[4], instructionControls=[4], stops=[{"site": 4, "reason": CUT}],
1059
+ spans=[{"first": 0, "last": 4}])["spans"][0]
1060
+ self.assertTrue(span["fallsThrough"])
1061
+
1062
+ def test_the_scan_decodes_only_the_short_transfers_that_can_enter_a_span(self):
1063
+ # Region A (segment 1000, IP 0000): 0000 jmp short to IP FF82, which wraps into region B; 0002 jz 0004,
1064
+ # which can enter no span; 0004 call 0007. Region B (IP FF00) holds at 0090 mov ah, 3Dh; 0092 int 21h;
1065
+ # 0094 ret. The jmp at 0000 lands at 0092 although its source lies far from it.
1066
+ a = bytes.fromhex("eb807400e80000") + bytes(9)
1067
+ b = bytes(0x80) + bytes.fromhex("b43dcd21c3") + bytes(0xB)
1068
+ data = a + b
1069
+ regions = [{"name": "a", "start": 0, "end": len(a), "ip": 0, "segment": 0x1000, "resident": True,
1070
+ "entries": [], "evidence": "synthetic declared code extent"},
1071
+ {"name": "b", "start": len(a), "end": len(data), "ip": 0xFF00, "segment": 0x1000, "resident": True,
1072
+ "entries": [0x90], "evidence": "synthetic declared code extent"}]
1073
+ query = {"regions": regions, "starts": [0x90], "targets": [0x92], "instructionControls": [0x92],
1074
+ "spans": [{"first": 0x90, "last": 0x92}]}
1075
+ span = run_report(data, query, "reach")["spans"][0]
1076
+ self.assertEqual(span["candidates"], [{"source": 0, "target": 0x92, "instruction": "jmp 0xffffff82",
1077
+ "position": "outside the decoded instructions"}])
1078
+ from scientific_method_engine.x86.image import Image
1079
+ from scientific_method_engine.x86.reach import _scan_sources
1080
+ self.assertEqual(_scan_sources(Image(data, query), {}, {0x91, 0x92}), {0, 4})
1081
+
1082
+ def test_calls_and_table_rows_enter_a_span(self):
1083
+ r = reach(indirectJumps=[TABLE], instructionControls=[0x28],
1084
+ spans=[{"first": 0x20, "last": 0x24}, {"first": 0x24, "last": 0x28}])
1085
+ table, called = r["spans"]
1086
+ self.assertEqual([(row["source"], row["target"], row["kind"]) for row in table["entries"]],
1087
+ [(0x12, 0x24, "table")])
1088
+ self.assertEqual([(row["source"], row["target"], row["kind"]) for row in called["entries"]],
1089
+ [(0x18, 0x28, "call"), (0x20, 0x28, "call")])
1090
+ # The ret at 0024 ends the run before 0028.
1091
+ self.assertEqual((called["fallsThrough"], called["break"]),
1092
+ (False, {"site": 0x24, "reason": "the instruction does not continue at the next instruction"}))
1093
+
1094
+ def test_a_declared_table_the_walk_did_not_reach_gives_candidates(self):
1095
+ r = reach(entries=(0x18,), starts=[0x18], indirectJumps=[TABLE], instructionControls=[0x28],
1096
+ spans=[{"first": 0x20, "last": 0x24}])
1097
+ span = r["spans"][0]
1098
+ self.assertEqual(span["entries"], [])
1099
+ self.assertEqual(span["candidates"], [{"source": 0x12, "target": 0x24, "instruction": "jmp word ptr cs:[bx + 0x30]",
1100
+ "position": "outside the decoded instructions"}])
1101
+ # The walk never decoded 0020, so the span does not fall through, and its negative is not usable.
1102
+ self.assertEqual(span["break"], {"site": 0x20, "reason": "no instruction the walk decoded starts here"})
1103
+ self.assertFalse(span["negativeUsable"])
1104
+
1105
+ def test_a_run_that_jumps_away_or_misses_the_last_site_does_not_fall_through(self):
1106
+ # 0000 mov ah, 3Dh; 0002 jmp 0005; 0004 nop, unreached; 0005 int 21h; 0007 ret.
1107
+ data = bytes.fromhex("b43deb0190cd21c3")
1108
+ r = spans(data, entries=(0,), targets=[5], instructionControls=[5],
1109
+ spans=[{"first": 0, "last": 5}, {"first": 0, "last": 3}, {"first": 4, "last": 5}])
1110
+ jumped, past, undecoded = r["spans"]
1111
+ self.assertEqual(jumped["break"], {"site": 2, "reason": "the instruction does not continue at the next instruction"})
1112
+ # The jump at 0002 starts inside the span, so it does not enter it.
1113
+ self.assertEqual(jumped["entries"], [])
1114
+ self.assertFalse(jumped["negativeUsable"])
1115
+ self.assertEqual(past["break"], {"site": 2, "reason": "the instruction runs past last"})
1116
+ self.assertEqual(undecoded["break"], {"site": 4, "reason": "no instruction the walk decoded starts here"})
1117
+ # 0002 jmp 0005 enters 0004..0005 from outside it.
1118
+ self.assertEqual([row["source"] for row in undecoded["entries"]], [2])
1119
+
1120
+ def test_the_result_limit_cuts_the_span_lists_and_counts_stay_whole(self):
1121
+ r = spans(spans=[{"first": 0, "last": 2}], limit=1)
1122
+ span = r["spans"][0]
1123
+ self.assertEqual((len(span["entries"]), len(span["candidates"])), (1, 1))
1124
+ self.assertEqual(span["counts"], {"entries": 1, "candidates": 2})
1125
+ self.assertTrue(r["truncated"])
1126
+ self.assertFalse(spans(spans=[{"first": 0, "last": 2}])["truncated"])
1127
+
1128
+ def test_rejected_spans(self):
1129
+ for value, message in [({}, "spans must be a list"),
1130
+ ([{"first": 0, "last": 2}] * 257, "at most 256"),
1131
+ ([{"first": 0}], "exactly first and last"),
1132
+ ([{"first": 0, "last": 2, "site": 2}], "exactly first and last"),
1133
+ ([{"first": 0x100, "last": 0x101}], "span first must be an integer"),
1134
+ ([{"first": 2, "last": 2}], "span last 2 must lie after first 2"),
1135
+ ([{"first": 0, "last": 2}, {"first": 0, "last": 2}], "Duplicate span 0..2")]:
1136
+ with self.subTest(value=value), self.assertRaisesRegex(ValueError, message):
1137
+ spans(spans=value)
1138
+ # A span stays in one region: 0000..0009 and 0009..0012 are two.
1139
+ cfg = config(WRITER, WRITER_STARTS, starts=[0], targets=[2], spans=[{"first": 5, "last": 9}])
1140
+ cfg["regions"] = [cfg["regions"][0] | {"end": 9, "entries": [0, 5]},
1141
+ cfg["regions"][0] | {"name": "second", "start": 9, "ip": 9, "segment": 0x2000, "entries": [9]}]
1142
+ with self.assertRaisesRegex(ValueError, "span last 9 must lie after first 5 in the region that holds it"):
1143
+ run_report(WRITER, cfg, "reach")
1144
+ data = bytes(4100) + bytes.fromhex("c3")
1145
+ with self.assertRaisesRegex(ValueError, "longer than 4096 bytes"):
1146
+ spans(data, entries=(0,), targets=[0], instructionControls=[], spans=[{"first": 0, "last": 4097}])
1147
+
1148
+
958
1149
  # Three starts. A at 0000 calls H at 0010, which calls T at 001B and U at 001C. B at 0008 calls G at
959
1150
  # 0017, which calls T. C at 000C makes a computed call through BX.
960
1151
  STARTS = bytes.fromhex(