scientific-method-engine 18.1.0__tar.gz → 18.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/cli.py +8 -0
  4. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/inventory.py +195 -8
  5. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/reach.py +146 -31
  6. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/reports.py +7 -5
  7. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/trace.py +13 -6
  8. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_inventory_check.py +170 -0
  9. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_reach.py +125 -0
  10. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/.gitignore +0 -0
  11. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/LICENSE +0 -0
  12. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/README.md +0 -0
  13. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/__init__.py +0 -0
  14. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/__main__.py +0 -0
  15. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  16. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  17. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  18. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  19. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  20. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  21. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  22. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  23. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  24. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  25. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  26. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  27. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  28. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  29. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  30. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  31. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  32. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  33. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  34. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  35. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  36. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  37. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  38. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  39. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  40. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  41. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  42. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  43. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  44. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  45. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  46. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  47. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  48. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  49. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  50. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  51. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  52. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  53. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  54. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/image.py +0 -0
  55. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/loops.py +0 -0
  56. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/machine.py +0 -0
  57. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  58. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  59. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  60. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pe.py +0 -0
  61. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/relational.py +0 -0
  62. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  63. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/values.py +0 -0
  64. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/oracle.py +0 -0
  65. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_config_encoding.py +0 -0
  66. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_continuation_budget.py +0 -0
  67. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_dispatch.py +0 -0
  68. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_effect_order.py +0 -0
  69. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_enter_frames.py +0 -0
  70. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_entry_frame.py +0 -0
  71. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_indirect_far_transfers.py +0 -0
  72. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_interrupt_models.py +0 -0
  73. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_join_terms.py +0 -0
  74. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_loops.py +0 -0
  75. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_memory_scopes.py +0 -0
  76. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_nested_frame_request.py +0 -0
  77. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_oracle.py +0 -0
  78. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_pe.py +0 -0
  79. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_relational_controls.py +0 -0
  80. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_return_checks.py +0 -0
  81. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_successor_rule.py +0 -0
  82. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_table_continuations.py +0 -0
  83. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_term_limit.py +0 -0
  84. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_upstream_gaps.py +0 -0
  85. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_volatile_memory.py +0 -0
  86. {scientific_method_engine-18.1.0 → scientific_method_engine-18.3.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 18.1.0
3
+ Version: 18.3.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "18.1.0"
8
+ version = "18.3.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -24,12 +24,20 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
24
24
  "reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
25
25
  "and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
26
26
  "a leaves routine is reached but not read, and its reason is repeated in the report;\n"
27
+ "a call to a noReturn routine and a noReturn interrupt do not continue at the next instruction,\n"
28
+ "and a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
29
+ "is reported contradicted;\n"
27
30
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
28
31
  "and a failed control fails the report saying whether it was reached.\n"
29
32
  "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
30
33
  "that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
31
34
  "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
32
35
  "a target inside an instruction the entry-path walk established from another start names it as insideInstruction.\n"
36
+ "It lists each row whose start lies inside an instruction the walk established (rowStarts), with the\n"
37
+ "routine the walk read it in; a row start in bytes the walk did not decode is counted, not placed.\n"
38
+ "noReturn takes reach's declarations: the walk stops at a call to a declared routine and at a declared\n"
39
+ "interrupt, a declared routine with a return on its own read paths is reported contradicted, and\n"
40
+ "rowsPastNoReturn lists each row whose body continues past such a call or interrupt.\n"
33
41
  "uses matches accesses whose footprint intersects query offset..offset+width; its rawCandidates give each unreached\n"
34
42
  "operand's encoded displacement, width, access, segment register and intersection, and never count as uses.\n"
35
43
  "trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
@@ -3,7 +3,7 @@ import re
3
3
  from bisect import bisect_right
4
4
  from pathlib import Path
5
5
  from .image import integer
6
- from .trace import LIMIT_REASON
6
+ from .trace import LIMIT_REASON, CONTESTED_REASON, walk, holding_instruction
7
7
 
8
8
  MAX_INVENTORY = 16 * 1024 * 1024
9
9
  MAX_ROWS = 200000
@@ -121,6 +121,81 @@ def _domains(image):
121
121
  yield kind, low, low + r["end"] - r["start"]
122
122
 
123
123
 
124
+ def _offsets(image, domains, space, number):
125
+ """The file offsets a place in an inventory's notation names, one for each declared region that maps it.
126
+
127
+ ``domains`` is ``_domains(image)`` as a list, in the order of ``image.regions``.
128
+ """
129
+ return [r["start"] + number - low for r, (kind, low, high) in zip(image.regions, domains)
130
+ if kind == space and low <= number < high]
131
+
132
+
133
+ def _text(ins):
134
+ return (ins.mnemonic + " " + ins.op_str).strip()
135
+
136
+
137
+ def _row_fields(row):
138
+ return {k: row[k] for k in ("start", "size", "name") if k in row}
139
+
140
+
141
+ def _row_starts(image, rows, starts, domains, seen, contested, routine_of):
142
+ """Each row whose start lies inside an instruction the entry-path walk established, and counts of the rest.
143
+
144
+ A start is checked against the walk's instructions only; bytes the walk did not decode are not
145
+ decoded here, so a row start in them is counted as not read rather than placed. A start at or
146
+ inside an instruction the walk decoded but rejected as contested is counted as contested.
147
+ """
148
+ found, counts = [], {"instructionStarts": 0, "insideAnInstruction": 0, "overlappingInstructionStarts": 0,
149
+ "contested": 0, "notRead": 0}
150
+ for row in rows:
151
+ offsets = _offsets(image, domains, row["space"], row["at"])
152
+ if not offsets:
153
+ continue
154
+ held = next(((at, holder) for at in offsets for holder in [holding_instruction(seen, at)] if holder is not None), None)
155
+ if held is None:
156
+ if any(at in seen for at in offsets):
157
+ counts["instructionStarts"] += 1
158
+ elif any(at in contested or holding_instruction(contested, at) is not None for at in offsets):
159
+ counts["contested"] += 1
160
+ else:
161
+ counts["notRead"] += 1
162
+ continue
163
+ at, holder = held
164
+ overlapping = at in seen
165
+ counts["overlappingInstructionStarts" if overlapping else "insideAnInstruction"] += 1
166
+ routine = routine_of(holder)
167
+ entry = {**_row_fields(row), "site": at,
168
+ "status": "start of an overlapping instruction" if overlapping else "inside an instruction",
169
+ "insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
170
+ "insideInstructionSize": seen[holder].size, "insideInstructionText": _text(seen[holder])}
171
+ if overlapping:
172
+ entry |= {"rowStartInstructionSize": seen[at].size, "rowStartInstructionText": _text(seen[at])}
173
+ if routine is not None:
174
+ place = _target_place(image, routine)
175
+ entry |= {"routine": routine, "routineAddress": place[2], "routineIsRow": place[:2] in starts}
176
+ found.append(entry)
177
+ return found, counts
178
+
179
+
180
+ def _no_return_check(image, routines, declared, limit):
181
+ """``(returns, read, stopped)``: the return sites on each declared routine's own read paths, the
182
+ instructions the walk established, and whether the walk stopped.
183
+
184
+ The walk starts at the declared routines and continues past every interrupt except a declared
185
+ one, as ``reach`` reads them, so a routine that ends in an interrupt that returns is contradicted
186
+ by what follows it. Calls are stepped over at their return sites, except calls to declared routines.
187
+ A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
188
+ has no return sites and is not in ``read``.
189
+ """
190
+ from .reach import returns_from, successor_graph
191
+ if not routines:
192
+ return {}, {}, False
193
+ seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True, **declared)
194
+ graph = successor_graph(image, seen, follow_interrupts=True, **declared)
195
+ return ({at: returns_from(graph, seen, set(), at) for at in routines}, seen,
196
+ any(g["reason"] == LIMIT_REASON for g in gaps))
197
+
198
+
124
199
  def _count(n, one, many):
125
200
  """``n`` followed by the singular or the plural wording."""
126
201
  return f"{n} {one if n == 1 else many}"
@@ -134,14 +209,22 @@ def inventory_check(image, config):
134
209
  reported with one calling site, near or far, and the evidence of its best site: a call on the
135
210
  entry path, a contested instruction, or only a raw byte candidate. A target that is a byte past
136
211
  the first of an instruction the entry-path walk established names that instruction.
212
+
213
+ The rows are checked against the same walk: each row whose start lies inside an instruction the
214
+ walk established is listed in ``rowStarts``. ``noReturn`` declarations, in ``reach``'s shape, end
215
+ the walk at a call to a declared routine and at a declared interrupt site, are checked for a
216
+ return on the routine's own paths, and list in ``rowsPastNoReturn`` each row whose body
217
+ continues past such a call or interrupt.
137
218
  """
138
- from .reports import call_controls, direct_calls, search_coverage
139
- from .trace import holding_instruction
219
+ from .reports import call_controls, direct_calls, entries, search_coverage
220
+ from .reach import no_return_declarations, no_return_rows, successor_graph, fewest_calls
140
221
  if "inventory" not in config:
141
222
  raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
142
223
  rows = read_inventory(config["inventory"], image)
143
224
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
144
- calls = direct_calls(image, config)
225
+ no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
226
+ declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
227
+ calls = direct_calls(image, config, **declared)
145
228
  seen, contested = calls["seen"], calls["contested"]
146
229
 
147
230
  starts = {(row["space"], row["at"]) for row in rows}
@@ -211,9 +294,63 @@ def inventory_check(image, config):
211
294
  entry |= {"insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
212
295
  "insideInstructionSize": seen[holder].size}
213
296
  if held:
214
- entry["rows"] = [{k: rows[i][k] for k in ("start", "size", "name") if k in rows[i]} for i in held]
297
+ entry["rows"] = [_row_fields(rows[i]) for i in held]
215
298
  missing.append(entry)
216
299
 
300
+ # The routine the walk read an instruction in, by the route with the fewest calls, as reach names it.
301
+ # The graph is built only when a row start needs it.
302
+ routines = None
303
+
304
+ def routine_of(site):
305
+ nonlocal routines
306
+ if routines is None:
307
+ graph = successor_graph(image, seen, **declared)
308
+ routines = fewest_calls(graph, [at for at in entries(image) if at in seen])[2]
309
+ return routines.get(site)
310
+ row_starts, start_counts = _row_starts(image, rows, starts, domains, seen, contested, routine_of)
311
+
312
+ # Each declaration with the calls it kept from their return sites, as reach reports them.
313
+ returns, check_read, check_stopped = _no_return_check(image, no_return_routines, declared,
314
+ config.get("instructionLimit", 10000))
315
+ entry_calls = {}
316
+ for row in calls["rows"]:
317
+ if row["target"] in no_return_routines and row["site"] in seen:
318
+ entry_calls.setdefault(row["target"], []).append(row["site"])
319
+ no_return = no_return_rows(no_return_routines, no_return_interrupts, returns,
320
+ {at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read)
321
+
322
+ # A row whose body holds a declared call or interrupt and the byte after it still covers what follows.
323
+ ends = [(row["site"], image.decode(row["site"]).size, "call", row["target"], row["classification"])
324
+ for row in calls["rows"] if row["target"] in no_return_routines]
325
+ ends += [(at, row["following"] - at, "interrupt", None,
326
+ "entry-path instruction" if at in seen else CONTESTED_REASON if at in contested else "declared site")
327
+ for at, row in no_return_interrupts.items()]
328
+ # Only a site the entry-path walk established counts a row; the rest are listed with their evidence.
329
+ past, past_rows, past_unread, past_unverified = [], set(), set(), set()
330
+ for site, size, kind, routine, classification in sorted(ends):
331
+ here, after = _target_place(image, site), _target_place(image, site + size)
332
+ # The byte after the site must be the next one in the inventory's notation too, or no body range can hold both.
333
+ if after is None or after[:2] != (here[0], here[1] + size):
334
+ continue
335
+ for i in owners(*here[:2]):
336
+ piece = next(((begin, end) for begin, end in rows[i]["body"] if begin <= after[1] < end), None)
337
+ if piece is None:
338
+ continue
339
+ entry = {**_row_fields(rows[i]), "kind": kind, "site": site, "siteAddress": here[2],
340
+ "siteClassification": classification}
341
+ if routine is not None:
342
+ entry["routine"] = routine
343
+ past.append(entry | {"following": site + size, "followingAddress": after[2],
344
+ "followingRead": site + size in seen, "bytesAfter": piece[1] - after[1]})
345
+ # A row with several such calls or interrupts is listed once for each, and counted once.
346
+ if site not in seen:
347
+ past_unverified.add(i)
348
+ continue
349
+ past_rows.add(i)
350
+ if site + size not in seen:
351
+ past_unread.add(i)
352
+ past_unverified -= past_rows
353
+
217
354
  controls = call_controls(calls, config)
218
355
  coverage = search_coverage(image, calls["read"])
219
356
  partial = any(c["partial"] for c in coverage)
@@ -245,12 +382,55 @@ def inventory_check(image, config):
245
382
  if LIMIT_REASON in stopped:
246
383
  summary += " The entry-path walk stopped at its instruction limit, so it can reach more calls."
247
384
  summary += " Calls the search does not resolve can add targets."
385
+ inside = start_counts["insideAnInstruction"] + start_counts["overlappingInstructionStarts"]
386
+ if inside:
387
+ summary += " " + _count(inside, "row start lies", "row starts lie") + " inside an instruction the entry-path walk established"
388
+ if start_counts["overlappingInstructionStarts"]:
389
+ summary += (f", {start_counts['overlappingInstructionStarts']} of them at the start of an overlapping "
390
+ "instruction it also established")
391
+ summary += "."
392
+ if start_counts["notRead"]:
393
+ summary += " " + _count(start_counts["notRead"], "row start in declared code lies", "row starts in declared code lie") \
394
+ + " in bytes where the walk established no instruction, so its boundary is not checked."
395
+ if past_rows:
396
+ summary += (" " + _count(len(past_rows), "row continues", "rows continue") + " past a call or interrupt declared "
397
+ f"noReturn, {len(past_unread)} of them where the walk read no instruction after it by another route.")
398
+ if past_unverified:
399
+ summary += (" " + _count(len(past_unverified), "more row holds", "more rows hold") + " such a call or interrupt "
400
+ "that only raw bytes, a contested instruction or an unreached declaration show, so it is not "
401
+ "shown to run.")
402
+ if start_counts["contested"]:
403
+ summary += " " + _count(start_counts["contested"], "row start lies", "row starts lie") \
404
+ + " at or inside an instruction the walk rejected as contested, so its boundary is not checked."
405
+ contradicted = sum(row.get("contradicted", False) for row in no_return)
406
+ if contradicted:
407
+ summary += " " + _count(contradicted, "noReturn routine has", "noReturn routines have") \
408
+ + " a return on its own read paths, so the declaration is contradicted."
409
+ unread = sum(not row["read"] for row in no_return if "routine" in row)
410
+ if unread:
411
+ summary += " " + _count(unread, "noReturn routine is", "noReturn routines are") \
412
+ + " not an instruction the check walk established, so its return check reads nothing."
413
+ if check_stopped:
414
+ summary += " The walk that checks the noReturn routines stopped at its instruction limit, so it can miss returns."
415
+ assumptions = ["each reached call returns to its next instruction"
416
+ + (", except a call to a noReturn routine" if no_return_routines else "")]
417
+ if no_return_routines:
418
+ assumptions.append("each interrupt the noReturn check reads returns to its next instruction"
419
+ + (", except at a noReturn interrupt site" if no_return_interrupts else "")
420
+ + "; interrupt handlers are not read")
421
+ if no_return:
422
+ assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
248
423
  return {"inventory": {"path": config["inventory"], "rows": len(rows)},
249
424
  "targets": missing[:limit], "unresolved": unresolved[:limit],
250
425
  "rowsOutsideDeclaredCode": outside_code[:limit],
426
+ "rowStarts": row_starts[:limit], "noReturn": no_return, "rowsPastNoReturn": past[:limit],
251
427
  "counts": {"callTargets": len(by_target), **tally, "unresolvedCalls": len(unresolved),
252
- "inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code)},
253
- "truncated": any(len(x) > limit for x in (missing, unresolved, outside_code)),
428
+ "inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code),
429
+ "rowStarts": start_counts, "rowsPastNoReturn": len(past_rows),
430
+ "rowsPastNoReturnUnreadAfter": len(past_unread),
431
+ "rowsPastNoReturnUnverified": len(past_unverified)},
432
+ "truncated": any(len(x) > limit for x in (missing, unresolved, outside_code, row_starts, past)),
433
+ "noReturnCheckLimitReached": check_stopped, "assumptions": assumptions,
254
434
  "summary": summary, "controls": controls,
255
435
  "searched": [r for r in image.regions if r["name"] in calls["scans"]], "coverage": coverage,
256
436
  "partialSearch": partial, "gaps": calls["gaps"],
@@ -259,4 +439,11 @@ def inventory_check(image, config):
259
439
  "interpretation": "Each distinct resolved direct call target in the searched regions, placed in the inventory's "
260
440
  "notation and compared with its rows. A target only raw byte candidates or contested "
261
441
  "instructions call is not shown to be code. The counts are lower bounds on the targets "
262
- "the inventory lacks: calls the search does not resolve can add more."}
442
+ "the inventory lacks: calls the search does not resolve can add more. Row starts are "
443
+ "compared only with the instructions the entry-path walk established; a start where it "
444
+ "established no instruction is counted as notRead or contested and not placed, so linear decoding past data "
445
+ "never reports one. A start inside an instruction can be deliberately overlapping code, "
446
+ "which the row names with both instructions for a reader to judge. rowsPastNoReturn rests "
447
+ "on the noReturn declarations: followingRead says whether the walk reached the bytes after "
448
+ "the call by another route, only a site the walk established counts a row, and an empty "
449
+ "returnSites means only that the walk read no return."}
@@ -46,7 +46,60 @@ def _leaves(value, image, starts):
46
46
  return leaves
47
47
 
48
48
 
49
- def _kinds(at, ins, step):
49
+ def no_return_declarations(value, image):
50
+ """The declared routines and interrupt sites that never return, each with its reason."""
51
+ if not isinstance(value, list) or len(value) > 256:
52
+ raise ValueError("noReturn must be a list of at most 256 objects")
53
+ routines, interrupts = {}, {}
54
+ for entry in value:
55
+ if not isinstance(entry, dict) or set(entry) not in ({"routine", "reason"}, {"interrupt", "reason"}):
56
+ raise ValueError("Each noReturn entry needs reason and exactly one of routine and interrupt")
57
+ kind = "routine" if "routine" in entry else "interrupt"
58
+ at = integer(entry[kind], 0, len(image.data) - 1, f"noReturn {kind}")
59
+ if image.region(at) is None:
60
+ raise ValueError(f"noReturn {kind} {at} is outside declared code")
61
+ if not isinstance(entry["reason"], str) or not entry["reason"].strip():
62
+ raise ValueError("A noReturn entry needs a nonempty reason")
63
+ found = routines if kind == "routine" else interrupts
64
+ if at in found:
65
+ raise ValueError(f"Duplicate noReturn {kind}")
66
+ if kind == "interrupt":
67
+ ins = image.decode(at)
68
+ if ins is None or base_mnemonic(ins) not in INTERRUPTS:
69
+ raise ValueError(f"noReturn interrupt {at} is not an interrupt instruction")
70
+ vector, conditional = interrupt_vector(image.flat, ins, at)
71
+ if conditional:
72
+ raise ValueError(f"noReturn interrupt {at} is conditional and continues when it does not interrupt")
73
+ found[at] = {"reason": entry["reason"], "vector": vector, "following": at + ins.size}
74
+ continue
75
+ found[at] = entry["reason"]
76
+ return routines, interrupts
77
+
78
+
79
+ def returns_from(graph, seen, returning_leaves, routine):
80
+ """Where ``routine``'s own paths return: each return instruction, and each leaf they enter other than by a call.
81
+
82
+ Calls are stepped over at the return sites the graph keeps. The walk assumes that a leaf returns,
83
+ so a jump, branch, table row or fall-through into one in ``returning_leaves`` returns too.
84
+ """
85
+ found, stack, visited = [], [routine], set()
86
+ while stack:
87
+ at = stack.pop()
88
+ if at in visited:
89
+ continue
90
+ visited.add(at)
91
+ if at in returning_leaves:
92
+ found.append(at)
93
+ continue
94
+ if at not in seen:
95
+ continue
96
+ if base_mnemonic(seen[at]) in RETURNS:
97
+ found.append(at)
98
+ stack.extend(s for s, kind in graph.get(at, []) if kind != "call")
99
+ return sorted(found)
100
+
101
+
102
+ def successor_kinds(at, ins, step):
50
103
  """Each successor of ``step`` with how the CFG reaches it: call, return, table, interrupt, jump or fall."""
51
104
  m, following = base_mnemonic(ins), at + ins.size
52
105
  targets = {e["target"] for e in step.edges if e["target"] is not None}
@@ -68,6 +121,56 @@ def _kinds(at, ins, step):
68
121
  return rows
69
122
 
70
123
 
124
+ def successor_graph(image, seen, **step):
125
+ """Each site of ``seen`` mapped to its ``successor_kinds`` rows that stay in ``seen``, under ``cfg_step``'s ``step`` options."""
126
+ return {at: [(s, kind) for s, kind in successor_kinds(at, ins, cfg_step(image, at, ins, **step)) if s in seen]
127
+ for at, ins in seen.items()}
128
+
129
+
130
+ def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, read):
131
+ """The ``noReturn`` report rows of the declared ``routines`` and ``interrupts``.
132
+
133
+ ``returns`` gives each routine's return sites, ``call_sites`` the reached calls to each routine,
134
+ ``seen`` the instructions the calls and interrupts were read in, ``reached`` the sites a walk from
135
+ the starts reached and ``read`` the sites the return check decoded.
136
+ """
137
+ rows = [{"routine": at, "reason": reason, "reached": at in reached, "read": at in read,
138
+ "returnSites": returns[at], "contradicted": bool(returns[at]),
139
+ "callSites": [{"site": site, "following": site + seen[site].size,
140
+ "followingRead": site + seen[site].size in seen}
141
+ for site in call_sites.get(at, [])]}
142
+ for at, reason in routines.items()]
143
+ return rows + [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
144
+ "following": row["following"], "followingRead": row["following"] in seen}
145
+ for at, row in interrupts.items()]
146
+
147
+
148
+ def fewest_calls(graph, starts):
149
+ """``(distance, parent, routine)`` for every site ``graph`` reaches from ``starts`` by the route with the fewest calls.
150
+
151
+ ``graph`` maps a site to its ``successor_kinds`` rows. ``distance`` counts the calls on the route,
152
+ ``parent`` gives each site's ``(predecessor, kind)`` on it (None at a start), and ``routine`` the
153
+ target of the route's last call, or the start.
154
+ """
155
+ # Entering a callee costs one, every other edge nothing (0-1 breadth-first search).
156
+ distance, parent, routine = {}, {}, {}
157
+ queue = deque()
158
+ for at in starts:
159
+ distance[at], parent[at], routine[at] = 0, None, at
160
+ queue.append((0, at))
161
+ while queue:
162
+ cost, at = queue.popleft()
163
+ if cost > distance[at]:
164
+ continue
165
+ for successor, kind in graph.get(at, []):
166
+ step_cost = cost + (kind == "call")
167
+ if successor not in distance or step_cost < distance[successor]:
168
+ distance[successor], parent[successor] = step_cost, (at, kind)
169
+ routine[successor] = successor if kind == "call" else routine[at]
170
+ (queue.append if kind == "call" else queue.appendleft)((step_cost, successor))
171
+ return distance, parent, routine
172
+
173
+
71
174
  def _dominators(order, predecessors):
72
175
  """Immediate dominators (Cooper, Harvey and Kennedy) of every node in ``order``, a reverse postorder from ROOT."""
73
176
  index = {node: i for i, node in enumerate(order)}
@@ -100,8 +203,10 @@ def reach(image, config):
100
203
  The walk decodes from ``starts`` (established region entries) and follows every resolved call
101
204
  into its callee and on at its return site, every resolved jump and branch, the rows of a
102
205
  declared indirect jump table, and every interrupt to the next instruction. A ``leaves`` routine
103
- is reached but never decoded. Each reached target gets one chain with the fewest calls, the
104
- assumptions its route rests on, and the routines on every route to it within the read graph.
206
+ is reached but never decoded. A call to a ``noReturn`` routine and a ``noReturn`` interrupt site
207
+ do not continue at the next instruction. Each reached target gets one chain with the fewest
208
+ calls, the assumptions its route rests on, and the routines on every route to it within the
209
+ read graph.
105
210
  """
106
211
  from .reports import entries
107
212
  starts = _sites(config.get("starts"), "starts", image)
@@ -127,12 +232,15 @@ def reach(image, config):
127
232
  "the walk found")
128
233
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
129
234
  instruction_limit = config.get("instructionLimit", 10000)
130
- seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True, stops=frozenset(leaves))
235
+ no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
236
+ declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
237
+ seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
238
+ stops=frozenset(leaves), **declared)
131
239
 
132
240
  graph, unresolved, interrupts, resolved_calls = {}, [], [], {}
133
241
  for at, ins in sorted(seen.items()):
134
- step = cfg_step(image, at, ins, follow_interrupts=True)
135
- graph[at] = [(s, kind) for s, kind in _kinds(at, ins, step) if s in seen or s in leaves]
242
+ step = cfg_step(image, at, ins, follow_interrupts=True, **declared)
243
+ graph[at] = [(s, kind) for s, kind in successor_kinds(at, ins, step) if s in seen or s in leaves]
136
244
  m = base_mnemonic(ins)
137
245
  text = _text(ins)
138
246
  kind = "call" if m in CALLS else "return" if m in RETURNS else "jump"
@@ -144,28 +252,13 @@ def reach(image, config):
144
252
  "reason": edge["provenance"].get("reason", "target outside declared regions")})
145
253
  elif m in CALLS:
146
254
  resolved_calls[at] = edge["target"]
147
- if m in INTERRUPTS:
255
+ if m in INTERRUPTS and at not in no_return_interrupts:
148
256
  vector, conditional = interrupt_vector(image.flat, ins, at)
149
257
  interrupts.append({"site": at, "instruction": text, "vector": vector, "conditional": conditional})
150
258
  for at in leaves:
151
259
  graph.setdefault(at, [])
152
260
 
153
- # Fewest calls first: entering a callee costs one, every other edge nothing (0-1 breadth-first search).
154
- distance, parent, routine = {}, {}, {}
155
- queue = deque()
156
- for at in starts:
157
- distance[at], parent[at], routine[at] = 0, None, at
158
- queue.append((0, at))
159
- while queue:
160
- cost, at = queue.popleft()
161
- if cost > distance[at]:
162
- continue
163
- for successor, kind in graph.get(at, []):
164
- step_cost = cost + (kind == "call")
165
- if successor not in distance or step_cost < distance[successor]:
166
- distance[successor], parent[successor] = step_cost, (at, kind)
167
- routine[successor] = successor if kind == "call" else routine[at]
168
- (queue.append if kind == "call" else queue.appendleft)((step_cost, successor))
261
+ distance, parent, routine = fewest_calls(graph, starts)
169
262
 
170
263
  # Dominators over everything reached, from a virtual root that leads to every start.
171
264
  successors = {ROOT: list(starts)} | {at: [s for s, _ in graph.get(at, [])] for at in distance}
@@ -243,6 +336,11 @@ def reach(image, config):
243
336
  for at in sorted(leaves) if at in distance and holding_instruction(seen, at) is not None]
244
337
  # Kept apart from the gap rows, which the result limit can cut.
245
338
  stopped = any(g["reason"] == LIMIT_REASON for g in walk_gaps)
339
+ # A control at a site a noReturn declaration kept the walk from names that declaration, which may be the cause.
340
+ cut_by = {at + seen[at].size: f"the call at {at} to noReturn routine {target}"
341
+ for at, target in sorted(resolved_calls.items()) if target in no_return_routines}
342
+ cut_by |= {row["following"]: f"the noReturn interrupt at {at}"
343
+ for at, row in no_return_interrupts.items() if at in seen}
246
344
  # Not reached and reached-but-unresolved need different fixes, so each failure says which it is.
247
345
  failures = []
248
346
  for label, sites, passed in (("control", controls, resolved_calls), ("instruction control", instruction_controls, seen)):
@@ -259,7 +357,9 @@ def reach(image, config):
259
357
  state = unreached(at)
260
358
  detail = (f"inside the reached instruction at {state['insideInstruction']}" if "insideInstruction" in state
261
359
  else "the " + state["status"] if state["status"].startswith("start") else state["status"])
360
+ cut = cut_by.get(at)
262
361
  failures.append(f"{label} {at} is {detail}"
362
+ + (f" (it follows {cut})" if cut else "")
263
363
  + (" (the walk stopped at its instruction limit)" if stopped else ""))
264
364
  if failures:
265
365
  raise ValueError("Positive controls failed: " + "; ".join(failures))
@@ -268,9 +368,23 @@ def reach(image, config):
268
368
  call_sites.setdefault(target, []).append(site)
269
369
  leaf_rows = [{"routine": at, "reason": reason, "reached": at in distance, "callSites": call_sites.get(at, [])}
270
370
  for at, reason in leaves.items()]
371
+ # Each call the declaration kept from its return site, so a reviewer can check what follows it.
372
+ returning_leaves = set(leaves) - set(no_return_routines)
373
+ returns = {at: returns_from(graph, seen, returning_leaves, at) for at in no_return_routines}
374
+ no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, call_sites, seen, distance, seen)
375
+ contradicted = any(row.get("contradicted") for row in no_return)
271
376
  counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
272
377
  "interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
273
- return {"starts": starts, "targets": rows, "leaves": leaf_rows,
378
+ assumptions = ["each reached call returns to its next instruction"
379
+ + (", except a call to a noReturn routine" if no_return_routines else ""),
380
+ "each reached interrupt returns to its next instruction"
381
+ + (", except at a noReturn interrupt site" if no_return_interrupts else "")
382
+ + "; interrupt handlers are not read",
383
+ "each leaf calls nothing, for the reason it gives",
384
+ "each declared indirect jump table holds the routes its declaration gives"]
385
+ if no_return:
386
+ assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
387
+ return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return,
274
388
  "reachedRoutines": sorted(routine_starts), "counts": counts,
275
389
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
276
390
  "contested": sorted(contested)[:limit],
@@ -279,19 +393,20 @@ def reach(image, config):
279
393
  "controls": [{"site": at, "target": resolved_calls[at]} for at in controls],
280
394
  "instructionControls": [{"site": at, "instruction": _text(seen[at]), "routine": routine.get(at)}
281
395
  for at in instruction_controls],
282
- "negativeUsable": bool(controls or instruction_controls) and not (stopped or unresolved or gaps or contested),
283
- "assumptions": ["each reached call returns to its next instruction",
284
- "each reached interrupt returns to its next instruction; interrupt handlers are not read",
285
- "each leaf calls nothing, for the reason it gives",
286
- "each declared indirect jump table holds the routes its declaration gives"],
396
+ "negativeUsable": bool(controls or instruction_controls)
397
+ and not (stopped or unresolved or gaps or contested or contradicted),
398
+ "assumptions": assumptions,
287
399
  "exclusions": ["computed call and jump targets (listed in unresolved)", "unrelocated far calls",
288
400
  "code reached only from outside the starts", "runtime reachability"],
289
401
  "interpretation": "Routes over the decoded entry-path CFG from the starts. A chain has the fewest calls; "
290
402
  "throughEveryRoute lists the routine starts every read route to the target passes, "
291
403
  "and an unresolved transfer may add a route that passes none of them. "
292
404
  "negativeUsable needs a control of either kind, a walk that did not stop at its instruction limit, "
293
- "and no unresolved transfer, gap or contested instruction, "
294
- "and still rests on the listed assumptions and leaves. "
405
+ "and no unresolved transfer, gap, contested instruction or contradicted noReturn "
406
+ "routine, and still rests on the listed assumptions, leaves and noReturn declarations. "
407
+ "A noReturn routine is contradicted when a return instruction, or a leaf entered "
408
+ "other than by a call, lies on its own read paths; an empty returnSites means only "
409
+ "that the walk read none. "
295
410
  "When instructionLimitReached holds, the walk stopped before reading all it reaches: "
296
411
  "every list and count covers only the part read, which part depends on the walk order, "
297
412
  "and an unreached target may lie past the stop."}
@@ -85,19 +85,21 @@ def search_coverage(image, spans):
85
85
  return rows
86
86
 
87
87
 
88
- def direct_calls(image, config):
88
+ def direct_calls(image, config, no_return_calls=frozenset(), no_return_interrupts=frozenset()):
89
89
  """Every direct call site in the searched regions, as ``incoming`` and ``inventory-check`` read them.
90
90
 
91
- Walks the entry-path CFG from every established entry, then scans every byte of the regions
92
- ``searchRegions`` names (all regions by default) for E8 and 9A call starts, and adds each
93
- reached call the scan cannot see (one that starts with a prefix). Returns a dict with the walk's
91
+ Walks the entry-path CFG from every established entry, ending a branch at a call to a routine in
92
+ ``no_return_calls`` and at an interrupt site in ``no_return_interrupts`` as ``walk`` does, then
93
+ scans every byte of the regions ``searchRegions`` names (all regions by default) for E8 and 9A
94
+ call starts, and adds each reached call the scan cannot see (one that starts with a prefix). Returns a dict with the walk's
94
95
  ``seen``, ``gaps`` (with a ``raw scan limit`` gap where ``scanLimit`` stopped a region),
95
96
  ``edges``, ``undecoded`` and ``contested``; ``rows``, every call row in the order it was read,
96
97
  each with its ``target`` (None when unresolved); ``scanned``, those rows by site, leaving out
97
98
  the reached calls whose frame encoding the walk does not model; ``scans``, the region names
98
99
  searched; and ``read``, the byte span the scan read in each.
99
100
  """
100
- seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000))
101
+ seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000),
102
+ no_return_calls=no_return_calls, no_return_interrupts=no_return_interrupts)
101
103
  rows, scanned = [], {}
102
104
 
103
105
  def classify(at):