scientific-method-engine 18.2.0__tar.gz → 18.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/cli.py +5 -0
  4. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/inventory.py +195 -8
  5. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/reach.py +62 -35
  6. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/reports.py +7 -5
  7. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_inventory_check.py +170 -0
  8. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/.gitignore +0 -0
  9. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/LICENSE +0 -0
  10. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/README.md +0 -0
  11. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/__init__.py +0 -0
  12. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/__main__.py +0 -0
  13. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  14. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  15. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  16. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  17. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  18. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  19. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  20. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  21. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  22. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  23. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  24. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  25. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  26. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  27. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  28. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  29. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  30. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  31. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  32. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  33. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  34. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  35. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  36. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  37. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  38. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  39. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  40. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  41. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  42. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  43. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  44. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  45. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  46. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  47. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  48. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  49. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  50. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  51. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  52. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/image.py +0 -0
  53. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/loops.py +0 -0
  54. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/machine.py +0 -0
  55. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  56. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  57. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  58. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/pe.py +0 -0
  59. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/relational.py +0 -0
  60. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  61. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/trace.py +0 -0
  62. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/src/scientific_method_engine/x86/values.py +0 -0
  63. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/oracle.py +0 -0
  64. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_config_encoding.py +0 -0
  65. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_continuation_budget.py +0 -0
  66. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_dispatch.py +0 -0
  67. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_effect_order.py +0 -0
  68. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_enter_frames.py +0 -0
  69. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_entry_frame.py +0 -0
  70. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_indirect_far_transfers.py +0 -0
  71. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_interrupt_models.py +0 -0
  72. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_join_terms.py +0 -0
  73. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_loops.py +0 -0
  74. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_memory_scopes.py +0 -0
  75. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_nested_frame_request.py +0 -0
  76. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_oracle.py +0 -0
  77. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_pe.py +0 -0
  78. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_reach.py +0 -0
  79. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_relational_controls.py +0 -0
  80. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_return_checks.py +0 -0
  81. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_successor_rule.py +0 -0
  82. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_table_continuations.py +0 -0
  83. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_term_limit.py +0 -0
  84. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_upstream_gaps.py +0 -0
  85. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_volatile_memory.py +0 -0
  86. {scientific_method_engine-18.2.0 → scientific_method_engine-18.3.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 18.2.0
3
+ Version: 18.3.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "18.2.0"
8
+ version = "18.3.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -33,6 +33,11 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
33
33
  "that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
34
34
  "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
35
35
  "a target inside an instruction the entry-path walk established from another start names it as insideInstruction.\n"
36
+ "It lists each row whose start lies inside an instruction the walk established (rowStarts), with the\n"
37
+ "routine the walk read it in; a row start in bytes the walk did not decode is counted, not placed.\n"
38
+ "noReturn takes reach's declarations: the walk stops at a call to a declared routine and at a declared\n"
39
+ "interrupt, a declared routine with a return on its own read paths is reported contradicted, and\n"
40
+ "rowsPastNoReturn lists each row whose body continues past such a call or interrupt.\n"
36
41
  "uses matches accesses whose footprint intersects query offset..offset+width; its rawCandidates give each unreached\n"
37
42
  "operand's encoded displacement, width, access, segment register and intersection, and never count as uses.\n"
38
43
  "trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
@@ -3,7 +3,7 @@ import re
3
3
  from bisect import bisect_right
4
4
  from pathlib import Path
5
5
  from .image import integer
6
- from .trace import LIMIT_REASON
6
+ from .trace import LIMIT_REASON, CONTESTED_REASON, walk, holding_instruction
7
7
 
8
8
  MAX_INVENTORY = 16 * 1024 * 1024
9
9
  MAX_ROWS = 200000
@@ -121,6 +121,81 @@ def _domains(image):
121
121
  yield kind, low, low + r["end"] - r["start"]
122
122
 
123
123
 
124
+ def _offsets(image, domains, space, number):
125
+ """The file offsets a place in an inventory's notation names, one for each declared region that maps it.
126
+
127
+ ``domains`` is ``_domains(image)`` as a list, in the order of ``image.regions``.
128
+ """
129
+ return [r["start"] + number - low for r, (kind, low, high) in zip(image.regions, domains)
130
+ if kind == space and low <= number < high]
131
+
132
+
133
+ def _text(ins):
134
+ return (ins.mnemonic + " " + ins.op_str).strip()
135
+
136
+
137
+ def _row_fields(row):
138
+ return {k: row[k] for k in ("start", "size", "name") if k in row}
139
+
140
+
141
+ def _row_starts(image, rows, starts, domains, seen, contested, routine_of):
142
+ """Each row whose start lies inside an instruction the entry-path walk established, and counts of the rest.
143
+
144
+ A start is checked against the walk's instructions only; bytes the walk did not decode are not
145
+ decoded here, so a row start in them is counted as not read rather than placed. A start at or
146
+ inside an instruction the walk decoded but rejected as contested is counted as contested.
147
+ """
148
+ found, counts = [], {"instructionStarts": 0, "insideAnInstruction": 0, "overlappingInstructionStarts": 0,
149
+ "contested": 0, "notRead": 0}
150
+ for row in rows:
151
+ offsets = _offsets(image, domains, row["space"], row["at"])
152
+ if not offsets:
153
+ continue
154
+ held = next(((at, holder) for at in offsets for holder in [holding_instruction(seen, at)] if holder is not None), None)
155
+ if held is None:
156
+ if any(at in seen for at in offsets):
157
+ counts["instructionStarts"] += 1
158
+ elif any(at in contested or holding_instruction(contested, at) is not None for at in offsets):
159
+ counts["contested"] += 1
160
+ else:
161
+ counts["notRead"] += 1
162
+ continue
163
+ at, holder = held
164
+ overlapping = at in seen
165
+ counts["overlappingInstructionStarts" if overlapping else "insideAnInstruction"] += 1
166
+ routine = routine_of(holder)
167
+ entry = {**_row_fields(row), "site": at,
168
+ "status": "start of an overlapping instruction" if overlapping else "inside an instruction",
169
+ "insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
170
+ "insideInstructionSize": seen[holder].size, "insideInstructionText": _text(seen[holder])}
171
+ if overlapping:
172
+ entry |= {"rowStartInstructionSize": seen[at].size, "rowStartInstructionText": _text(seen[at])}
173
+ if routine is not None:
174
+ place = _target_place(image, routine)
175
+ entry |= {"routine": routine, "routineAddress": place[2], "routineIsRow": place[:2] in starts}
176
+ found.append(entry)
177
+ return found, counts
178
+
179
+
180
+ def _no_return_check(image, routines, declared, limit):
181
+ """``(returns, read, stopped)``: the return sites on each declared routine's own read paths, the
182
+ instructions the walk established, and whether the walk stopped.
183
+
184
+ The walk starts at the declared routines and continues past every interrupt except a declared
185
+ one, as ``reach`` reads them, so a routine that ends in an interrupt that returns is contradicted
186
+ by what follows it. Calls are stepped over at their return sites, except calls to declared routines.
187
+ A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
188
+ has no return sites and is not in ``read``.
189
+ """
190
+ from .reach import returns_from, successor_graph
191
+ if not routines:
192
+ return {}, {}, False
193
+ seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True, **declared)
194
+ graph = successor_graph(image, seen, follow_interrupts=True, **declared)
195
+ return ({at: returns_from(graph, seen, set(), at) for at in routines}, seen,
196
+ any(g["reason"] == LIMIT_REASON for g in gaps))
197
+
198
+
124
199
  def _count(n, one, many):
125
200
  """``n`` followed by the singular or the plural wording."""
126
201
  return f"{n} {one if n == 1 else many}"
@@ -134,14 +209,22 @@ def inventory_check(image, config):
134
209
  reported with one calling site, near or far, and the evidence of its best site: a call on the
135
210
  entry path, a contested instruction, or only a raw byte candidate. A target that is a byte past
136
211
  the first of an instruction the entry-path walk established names that instruction.
212
+
213
+ The rows are checked against the same walk: each row whose start lies inside an instruction the
214
+ walk established is listed in ``rowStarts``. ``noReturn`` declarations, in ``reach``'s shape, end
215
+ the walk at a call to a declared routine and at a declared interrupt site, are checked for a
216
+ return on the routine's own paths, and list in ``rowsPastNoReturn`` each row whose body
217
+ continues past such a call or interrupt.
137
218
  """
138
- from .reports import call_controls, direct_calls, search_coverage
139
- from .trace import holding_instruction
219
+ from .reports import call_controls, direct_calls, entries, search_coverage
220
+ from .reach import no_return_declarations, no_return_rows, successor_graph, fewest_calls
140
221
  if "inventory" not in config:
141
222
  raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
142
223
  rows = read_inventory(config["inventory"], image)
143
224
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
144
- calls = direct_calls(image, config)
225
+ no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
226
+ declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
227
+ calls = direct_calls(image, config, **declared)
145
228
  seen, contested = calls["seen"], calls["contested"]
146
229
 
147
230
  starts = {(row["space"], row["at"]) for row in rows}
@@ -211,9 +294,63 @@ def inventory_check(image, config):
211
294
  entry |= {"insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
212
295
  "insideInstructionSize": seen[holder].size}
213
296
  if held:
214
- entry["rows"] = [{k: rows[i][k] for k in ("start", "size", "name") if k in rows[i]} for i in held]
297
+ entry["rows"] = [_row_fields(rows[i]) for i in held]
215
298
  missing.append(entry)
216
299
 
300
+ # The routine the walk read an instruction in, by the route with the fewest calls, as reach names it.
301
+ # The graph is built only when a row start needs it.
302
+ routines = None
303
+
304
+ def routine_of(site):
305
+ nonlocal routines
306
+ if routines is None:
307
+ graph = successor_graph(image, seen, **declared)
308
+ routines = fewest_calls(graph, [at for at in entries(image) if at in seen])[2]
309
+ return routines.get(site)
310
+ row_starts, start_counts = _row_starts(image, rows, starts, domains, seen, contested, routine_of)
311
+
312
+ # Each declaration with the calls it kept from their return sites, as reach reports them.
313
+ returns, check_read, check_stopped = _no_return_check(image, no_return_routines, declared,
314
+ config.get("instructionLimit", 10000))
315
+ entry_calls = {}
316
+ for row in calls["rows"]:
317
+ if row["target"] in no_return_routines and row["site"] in seen:
318
+ entry_calls.setdefault(row["target"], []).append(row["site"])
319
+ no_return = no_return_rows(no_return_routines, no_return_interrupts, returns,
320
+ {at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read)
321
+
322
+ # A row whose body holds a declared call or interrupt and the byte after it still covers what follows.
323
+ ends = [(row["site"], image.decode(row["site"]).size, "call", row["target"], row["classification"])
324
+ for row in calls["rows"] if row["target"] in no_return_routines]
325
+ ends += [(at, row["following"] - at, "interrupt", None,
326
+ "entry-path instruction" if at in seen else CONTESTED_REASON if at in contested else "declared site")
327
+ for at, row in no_return_interrupts.items()]
328
+ # Only a site the entry-path walk established counts a row; the rest are listed with their evidence.
329
+ past, past_rows, past_unread, past_unverified = [], set(), set(), set()
330
+ for site, size, kind, routine, classification in sorted(ends):
331
+ here, after = _target_place(image, site), _target_place(image, site + size)
332
+ # The byte after the site must be the next one in the inventory's notation too, or no body range can hold both.
333
+ if after is None or after[:2] != (here[0], here[1] + size):
334
+ continue
335
+ for i in owners(*here[:2]):
336
+ piece = next(((begin, end) for begin, end in rows[i]["body"] if begin <= after[1] < end), None)
337
+ if piece is None:
338
+ continue
339
+ entry = {**_row_fields(rows[i]), "kind": kind, "site": site, "siteAddress": here[2],
340
+ "siteClassification": classification}
341
+ if routine is not None:
342
+ entry["routine"] = routine
343
+ past.append(entry | {"following": site + size, "followingAddress": after[2],
344
+ "followingRead": site + size in seen, "bytesAfter": piece[1] - after[1]})
345
+ # A row with several such calls or interrupts is listed once for each, and counted once.
346
+ if site not in seen:
347
+ past_unverified.add(i)
348
+ continue
349
+ past_rows.add(i)
350
+ if site + size not in seen:
351
+ past_unread.add(i)
352
+ past_unverified -= past_rows
353
+
217
354
  controls = call_controls(calls, config)
218
355
  coverage = search_coverage(image, calls["read"])
219
356
  partial = any(c["partial"] for c in coverage)
@@ -245,12 +382,55 @@ def inventory_check(image, config):
245
382
  if LIMIT_REASON in stopped:
246
383
  summary += " The entry-path walk stopped at its instruction limit, so it can reach more calls."
247
384
  summary += " Calls the search does not resolve can add targets."
385
+ inside = start_counts["insideAnInstruction"] + start_counts["overlappingInstructionStarts"]
386
+ if inside:
387
+ summary += " " + _count(inside, "row start lies", "row starts lie") + " inside an instruction the entry-path walk established"
388
+ if start_counts["overlappingInstructionStarts"]:
389
+ summary += (f", {start_counts['overlappingInstructionStarts']} of them at the start of an overlapping "
390
+ "instruction it also established")
391
+ summary += "."
392
+ if start_counts["notRead"]:
393
+ summary += " " + _count(start_counts["notRead"], "row start in declared code lies", "row starts in declared code lie") \
394
+ + " in bytes where the walk established no instruction, so its boundary is not checked."
395
+ if past_rows:
396
+ summary += (" " + _count(len(past_rows), "row continues", "rows continue") + " past a call or interrupt declared "
397
+ f"noReturn, {len(past_unread)} of them where the walk read no instruction after it by another route.")
398
+ if past_unverified:
399
+ summary += (" " + _count(len(past_unverified), "more row holds", "more rows hold") + " such a call or interrupt "
400
+ "that only raw bytes, a contested instruction or an unreached declaration show, so it is not "
401
+ "shown to run.")
402
+ if start_counts["contested"]:
403
+ summary += " " + _count(start_counts["contested"], "row start lies", "row starts lie") \
404
+ + " at or inside an instruction the walk rejected as contested, so its boundary is not checked."
405
+ contradicted = sum(row.get("contradicted", False) for row in no_return)
406
+ if contradicted:
407
+ summary += " " + _count(contradicted, "noReturn routine has", "noReturn routines have") \
408
+ + " a return on its own read paths, so the declaration is contradicted."
409
+ unread = sum(not row["read"] for row in no_return if "routine" in row)
410
+ if unread:
411
+ summary += " " + _count(unread, "noReturn routine is", "noReturn routines are") \
412
+ + " not an instruction the check walk established, so its return check reads nothing."
413
+ if check_stopped:
414
+ summary += " The walk that checks the noReturn routines stopped at its instruction limit, so it can miss returns."
415
+ assumptions = ["each reached call returns to its next instruction"
416
+ + (", except a call to a noReturn routine" if no_return_routines else "")]
417
+ if no_return_routines:
418
+ assumptions.append("each interrupt the noReturn check reads returns to its next instruction"
419
+ + (", except at a noReturn interrupt site" if no_return_interrupts else "")
420
+ + "; interrupt handlers are not read")
421
+ if no_return:
422
+ assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
248
423
  return {"inventory": {"path": config["inventory"], "rows": len(rows)},
249
424
  "targets": missing[:limit], "unresolved": unresolved[:limit],
250
425
  "rowsOutsideDeclaredCode": outside_code[:limit],
426
+ "rowStarts": row_starts[:limit], "noReturn": no_return, "rowsPastNoReturn": past[:limit],
251
427
  "counts": {"callTargets": len(by_target), **tally, "unresolvedCalls": len(unresolved),
252
- "inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code)},
253
- "truncated": any(len(x) > limit for x in (missing, unresolved, outside_code)),
428
+ "inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code),
429
+ "rowStarts": start_counts, "rowsPastNoReturn": len(past_rows),
430
+ "rowsPastNoReturnUnreadAfter": len(past_unread),
431
+ "rowsPastNoReturnUnverified": len(past_unverified)},
432
+ "truncated": any(len(x) > limit for x in (missing, unresolved, outside_code, row_starts, past)),
433
+ "noReturnCheckLimitReached": check_stopped, "assumptions": assumptions,
254
434
  "summary": summary, "controls": controls,
255
435
  "searched": [r for r in image.regions if r["name"] in calls["scans"]], "coverage": coverage,
256
436
  "partialSearch": partial, "gaps": calls["gaps"],
@@ -259,4 +439,11 @@ def inventory_check(image, config):
259
439
  "interpretation": "Each distinct resolved direct call target in the searched regions, placed in the inventory's "
260
440
  "notation and compared with its rows. A target only raw byte candidates or contested "
261
441
  "instructions call is not shown to be code. The counts are lower bounds on the targets "
262
- "the inventory lacks: calls the search does not resolve can add more."}
442
+ "the inventory lacks: calls the search does not resolve can add more. Row starts are "
443
+ "compared only with the instructions the entry-path walk established; a start where it "
444
+ "established no instruction is counted as notRead or contested and not placed, so linear decoding past data "
445
+ "never reports one. A start inside an instruction can be deliberately overlapping code, "
446
+ "which the row names with both instructions for a reader to judge. rowsPastNoReturn rests "
447
+ "on the noReturn declarations: followingRead says whether the walk reached the bytes after "
448
+ "the call by another route, only a site the walk established counts a row, and an empty "
449
+ "returnSites means only that the walk read no return."}
@@ -46,7 +46,7 @@ def _leaves(value, image, starts):
46
46
  return leaves
47
47
 
48
48
 
49
- def _no_return(value, image):
49
+ def no_return_declarations(value, image):
50
50
  """The declared routines and interrupt sites that never return, each with its reason."""
51
51
  if not isinstance(value, list) or len(value) > 256:
52
52
  raise ValueError("noReturn must be a list of at most 256 objects")
@@ -76,7 +76,7 @@ def _no_return(value, image):
76
76
  return routines, interrupts
77
77
 
78
78
 
79
- def _returns_from(graph, seen, returning_leaves, routine):
79
+ def returns_from(graph, seen, returning_leaves, routine):
80
80
  """Where ``routine``'s own paths return: each return instruction, and each leaf they enter other than by a call.
81
81
 
82
82
  Calls are stepped over at the return sites the graph keeps. The walk assumes that a leaf returns,
@@ -99,7 +99,7 @@ def _returns_from(graph, seen, returning_leaves, routine):
99
99
  return sorted(found)
100
100
 
101
101
 
102
- def _kinds(at, ins, step):
102
+ def successor_kinds(at, ins, step):
103
103
  """Each successor of ``step`` with how the CFG reaches it: call, return, table, interrupt, jump or fall."""
104
104
  m, following = base_mnemonic(ins), at + ins.size
105
105
  targets = {e["target"] for e in step.edges if e["target"] is not None}
@@ -121,6 +121,56 @@ def _kinds(at, ins, step):
121
121
  return rows
122
122
 
123
123
 
124
+ def successor_graph(image, seen, **step):
125
+ """Each site of ``seen`` mapped to its ``successor_kinds`` rows that stay in ``seen``, under ``cfg_step``'s ``step`` options."""
126
+ return {at: [(s, kind) for s, kind in successor_kinds(at, ins, cfg_step(image, at, ins, **step)) if s in seen]
127
+ for at, ins in seen.items()}
128
+
129
+
130
+ def no_return_rows(routines, interrupts, returns, call_sites, seen, reached, read):
131
+ """The ``noReturn`` report rows of the declared ``routines`` and ``interrupts``.
132
+
133
+ ``returns`` gives each routine's return sites, ``call_sites`` the reached calls to each routine,
134
+ ``seen`` the instructions the calls and interrupts were read in, ``reached`` the sites a walk from
135
+ the starts reached and ``read`` the sites the return check decoded.
136
+ """
137
+ rows = [{"routine": at, "reason": reason, "reached": at in reached, "read": at in read,
138
+ "returnSites": returns[at], "contradicted": bool(returns[at]),
139
+ "callSites": [{"site": site, "following": site + seen[site].size,
140
+ "followingRead": site + seen[site].size in seen}
141
+ for site in call_sites.get(at, [])]}
142
+ for at, reason in routines.items()]
143
+ return rows + [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
144
+ "following": row["following"], "followingRead": row["following"] in seen}
145
+ for at, row in interrupts.items()]
146
+
147
+
148
+ def fewest_calls(graph, starts):
149
+ """``(distance, parent, routine)`` for every site ``graph`` reaches from ``starts`` by the route with the fewest calls.
150
+
151
+ ``graph`` maps a site to its ``successor_kinds`` rows. ``distance`` counts the calls on the route,
152
+ ``parent`` gives each site's ``(predecessor, kind)`` on it (None at a start), and ``routine`` the
153
+ target of the route's last call, or the start.
154
+ """
155
+ # Entering a callee costs one, every other edge nothing (0-1 breadth-first search).
156
+ distance, parent, routine = {}, {}, {}
157
+ queue = deque()
158
+ for at in starts:
159
+ distance[at], parent[at], routine[at] = 0, None, at
160
+ queue.append((0, at))
161
+ while queue:
162
+ cost, at = queue.popleft()
163
+ if cost > distance[at]:
164
+ continue
165
+ for successor, kind in graph.get(at, []):
166
+ step_cost = cost + (kind == "call")
167
+ if successor not in distance or step_cost < distance[successor]:
168
+ distance[successor], parent[successor] = step_cost, (at, kind)
169
+ routine[successor] = successor if kind == "call" else routine[at]
170
+ (queue.append if kind == "call" else queue.appendleft)((step_cost, successor))
171
+ return distance, parent, routine
172
+
173
+
124
174
  def _dominators(order, predecessors):
125
175
  """Immediate dominators (Cooper, Harvey and Kennedy) of every node in ``order``, a reverse postorder from ROOT."""
126
176
  index = {node: i for i, node in enumerate(order)}
@@ -182,7 +232,7 @@ def reach(image, config):
182
232
  "the walk found")
183
233
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
184
234
  instruction_limit = config.get("instructionLimit", 10000)
185
- no_return_routines, no_return_interrupts = _no_return(config.get("noReturn", []), image)
235
+ no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
186
236
  declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts)}
187
237
  seen, walk_gaps, _, _, contested = walk(image, starts, instruction_limit, follow_interrupts=True,
188
238
  stops=frozenset(leaves), **declared)
@@ -190,7 +240,7 @@ def reach(image, config):
190
240
  graph, unresolved, interrupts, resolved_calls = {}, [], [], {}
191
241
  for at, ins in sorted(seen.items()):
192
242
  step = cfg_step(image, at, ins, follow_interrupts=True, **declared)
193
- graph[at] = [(s, kind) for s, kind in _kinds(at, ins, step) if s in seen or s in leaves]
243
+ graph[at] = [(s, kind) for s, kind in successor_kinds(at, ins, step) if s in seen or s in leaves]
194
244
  m = base_mnemonic(ins)
195
245
  text = _text(ins)
196
246
  kind = "call" if m in CALLS else "return" if m in RETURNS else "jump"
@@ -208,22 +258,7 @@ def reach(image, config):
208
258
  for at in leaves:
209
259
  graph.setdefault(at, [])
210
260
 
211
- # Fewest calls first: entering a callee costs one, every other edge nothing (0-1 breadth-first search).
212
- distance, parent, routine = {}, {}, {}
213
- queue = deque()
214
- for at in starts:
215
- distance[at], parent[at], routine[at] = 0, None, at
216
- queue.append((0, at))
217
- while queue:
218
- cost, at = queue.popleft()
219
- if cost > distance[at]:
220
- continue
221
- for successor, kind in graph.get(at, []):
222
- step_cost = cost + (kind == "call")
223
- if successor not in distance or step_cost < distance[successor]:
224
- distance[successor], parent[successor] = step_cost, (at, kind)
225
- routine[successor] = successor if kind == "call" else routine[at]
226
- (queue.append if kind == "call" else queue.appendleft)((step_cost, successor))
261
+ distance, parent, routine = fewest_calls(graph, starts)
227
262
 
228
263
  # Dominators over everything reached, from a virtual root that leads to every start.
229
264
  successors = {ROOT: list(starts)} | {at: [s for s, _ in graph.get(at, [])] for at in distance}
@@ -334,18 +369,10 @@ def reach(image, config):
334
369
  leaf_rows = [{"routine": at, "reason": reason, "reached": at in distance, "callSites": call_sites.get(at, [])}
335
370
  for at, reason in leaves.items()]
336
371
  # Each call the declaration kept from its return site, so a reviewer can check what follows it.
337
- no_return_rows, returning_leaves = [], set(leaves) - set(no_return_routines)
338
- for at, reason in no_return_routines.items():
339
- returns = _returns_from(graph, seen, returning_leaves, at)
340
- no_return_rows.append({"routine": at, "reason": reason, "reached": at in distance, "read": at in seen,
341
- "returnSites": returns, "contradicted": bool(returns),
342
- "callSites": [{"site": site, "following": site + seen[site].size,
343
- "followingRead": site + seen[site].size in seen}
344
- for site in call_sites.get(at, [])]})
345
- no_return_rows += [{"interrupt": at, "reason": row["reason"], "vector": row["vector"], "reached": at in seen,
346
- "following": row["following"], "followingRead": row["following"] in seen}
347
- for at, row in no_return_interrupts.items()]
348
- contradicted = any(row.get("contradicted") for row in no_return_rows)
372
+ returning_leaves = set(leaves) - set(no_return_routines)
373
+ returns = {at: returns_from(graph, seen, returning_leaves, at) for at in no_return_routines}
374
+ no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, call_sites, seen, distance, seen)
375
+ contradicted = any(row.get("contradicted") for row in no_return)
349
376
  counts = {"routines": len(routine_starts), "instructions": len(seen), "unresolved": len(unresolved),
350
377
  "interrupts": len(interrupts), "gaps": len(gaps), "contested": len(contested)}
351
378
  assumptions = ["each reached call returns to its next instruction"
@@ -355,9 +382,9 @@ def reach(image, config):
355
382
  + "; interrupt handlers are not read",
356
383
  "each leaf calls nothing, for the reason it gives",
357
384
  "each declared indirect jump table holds the routes its declaration gives"]
358
- if no_return_rows:
385
+ if no_return:
359
386
  assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
360
- return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return_rows,
387
+ return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return,
361
388
  "reachedRoutines": sorted(routine_starts), "counts": counts,
362
389
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
363
390
  "contested": sorted(contested)[:limit],
@@ -85,19 +85,21 @@ def search_coverage(image, spans):
85
85
  return rows
86
86
 
87
87
 
88
- def direct_calls(image, config):
88
+ def direct_calls(image, config, no_return_calls=frozenset(), no_return_interrupts=frozenset()):
89
89
  """Every direct call site in the searched regions, as ``incoming`` and ``inventory-check`` read them.
90
90
 
91
- Walks the entry-path CFG from every established entry, then scans every byte of the regions
92
- ``searchRegions`` names (all regions by default) for E8 and 9A call starts, and adds each
93
- reached call the scan cannot see (one that starts with a prefix). Returns a dict with the walk's
91
+ Walks the entry-path CFG from every established entry, ending a branch at a call to a routine in
92
+ ``no_return_calls`` and at an interrupt site in ``no_return_interrupts`` as ``walk`` does, then
93
+ scans every byte of the regions ``searchRegions`` names (all regions by default) for E8 and 9A
94
+ call starts, and adds each reached call the scan cannot see (one that starts with a prefix). Returns a dict with the walk's
94
95
  ``seen``, ``gaps`` (with a ``raw scan limit`` gap where ``scanLimit`` stopped a region),
95
96
  ``edges``, ``undecoded`` and ``contested``; ``rows``, every call row in the order it was read,
96
97
  each with its ``target`` (None when unresolved); ``scanned``, those rows by site, leaving out
97
98
  the reached calls whose frame encoding the walk does not model; ``scans``, the region names
98
99
  searched; and ``read``, the byte span the scan read in each.
99
100
  """
100
- seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000))
101
+ seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000),
102
+ no_return_calls=no_return_calls, no_return_interrupts=no_return_interrupts)
101
103
  rows, scanned = [], {}
102
104
 
103
105
  def classify(at):
@@ -171,6 +171,157 @@ class InventoryCheckTests(unittest.TestCase):
171
171
  self.assertEqual([t["insideInstruction"] for t in twice["targets"]], [1, 9])
172
172
  self.assertIn(" 2 of them start inside an instruction the entry-path walk established from another start.", twice["summary"])
173
173
 
174
+ def test_a_row_start_inside_an_established_far_call_names_the_call_and_the_routine_it_was_read_in(self):
175
+ # 0000 calls 0004, a routine no row starts at: push bp; mov bp,sp; call far 1000:000E; pop bp; ret.
176
+ # The row 1000:000A starts three bytes into that five-byte far call.
177
+ data = bytes.fromhex("e80100" "c3" "55" "8bec" "9a0e000010" "5d" "c3" "cb")
178
+ relocation = [{"site": 10, "segment": 0x1000, "evidence": "synthetic relocation"}]
179
+ r = self.check("start\tsize\tname\n1000:0000\t4\tmain\n1000:000A\t4\tshifted\n1000:000E\t1\tfar\n", data=data,
180
+ relocations=relocation)
181
+ self.assertEqual(r["rowStarts"], [{
182
+ "start": "1000:000A", "size": 4, "name": "shifted", "site": 10, "status": "inside an instruction",
183
+ "insideInstruction": 7, "insideInstructionAddress": "1000:0007", "insideInstructionSize": 5,
184
+ "insideInstructionText": "lcall 0x1000, 0xe", "routine": 4, "routineAddress": "1000:0004", "routineIsRow": False}])
185
+ self.assertEqual(r["counts"]["rowStarts"],
186
+ {"instructionStarts": 2, "insideAnInstruction": 1, "overlappingInstructionStarts": 0,
187
+ "contested": 0, "notRead": 0})
188
+ self.assertIn(" 1 row start lies inside an instruction the entry-path walk established.", r["summary"])
189
+ # The routine's real entry is still a call target no row starts at.
190
+ self.assertEqual([(t["address"], t["status"]) for t in r["targets"]], [("1000:0004", "outside every row")])
191
+ # Rows at instruction starts pass.
192
+ fixed = self.check("start\tsize\n1000:0000\t4\n1000:0004\t10\n1000:000E\t1\n", data=data, relocations=relocation)
193
+ self.assertEqual(fixed["rowStarts"], [])
194
+ self.assertEqual(fixed["counts"]["rowStarts"]["instructionStarts"], 3)
195
+ self.assertNotIn("row start", fixed["summary"])
196
+ # The result limit cuts the list, not the count.
197
+ cut = self.check("start\tsize\n1000:0000\t4\n1000:0009\t1\n1000:000A\t1\n", data=data, relocations=relocation, limit=1)
198
+ self.assertEqual([row["start"] for row in cut["rowStarts"]], ["1000:0009"])
199
+ self.assertEqual(cut["counts"]["rowStarts"]["insideAnInstruction"], 2)
200
+ self.assertTrue(cut["truncated"])
201
+
202
+ def test_a_row_start_only_a_misaligned_decode_would_cut_is_not_placed_inside_an_instruction(self):
203
+ # 0000 calls 0007. 0004..0006 is data the walk never decodes; decoded linearly, its 9A byte starts a
204
+ # five-byte far call over 0007. The row at 0007 is an established start, and the row at 0005, in the
205
+ # data, is counted as not read.
206
+ data = bytes.fromhex("e80400" "c3" "9a9090" "c3" "90")
207
+ r = self.check("start\tsize\n1000:0000\t4\n1000:0005\t1\n1000:0007\t1\n", data=data)
208
+ self.assertEqual(r["rowStarts"], [])
209
+ self.assertEqual(r["counts"]["rowStarts"],
210
+ {"instructionStarts": 2, "insideAnInstruction": 0, "overlappingInstructionStarts": 0,
211
+ "contested": 0, "notRead": 1})
212
+ self.assertIn(" 1 row start in declared code lies in bytes where the walk established no instruction, so its boundary is not checked.",
213
+ r["summary"])
214
+ self.assertNotIn("inside an instruction the entry-path walk established", r["summary"])
215
+
216
+ def test_a_row_at_a_deliberately_overlapping_instruction_is_listed_once_with_both_instructions(self):
217
+ # 0001 or bh,0 holds an IRET byte at 0002, which the push-CS/call frame at 0004 calls, so the walk
218
+ # establishes both instructions and the row at 0002 starts one inside the other.
219
+ data = bytes.fromhex("9c" "80cf00" "0e" "e8faff" "c3")
220
+ r = self.check("start\tsize\n1000:0000\t9\n1000:0002\t1\n", data=data)
221
+ self.assertEqual(r["rowStarts"], [{
222
+ "start": "1000:0002", "size": 1, "site": 2, "status": "start of an overlapping instruction",
223
+ "insideInstruction": 1, "insideInstructionAddress": "1000:0001", "insideInstructionSize": 3,
224
+ "insideInstructionText": "or bh, 0", "rowStartInstructionSize": 1, "rowStartInstructionText": "iret",
225
+ "routine": 0, "routineAddress": "1000:0000", "routineIsRow": True}])
226
+ self.assertEqual(r["counts"]["rowStarts"]["overlappingInstructionStarts"], 1)
227
+ self.assertIn(" 1 row start lies inside an instruction the entry-path walk established, 1 of them at the start "
228
+ "of an overlapping instruction it also established.", r["summary"])
229
+
230
+ def test_a_row_past_a_call_to_a_no_return_routine_is_listed_and_the_bytes_after_the_call_are_not_walked(self):
231
+ # 0000 calls the exit routine at 0008 (mov ah,4Ch; int 21h). The row 1000:0000 runs on over 0003..0007,
232
+ # where E8 FD FF decodes as a call to 0003 and the walk reads it unless the routine is declared.
233
+ data = bytes.fromhex("e80500" "e8fdff" "0000" "b44c" "cd21")
234
+ inventory = "start\tsize\n1000:0000\t8\n1000:0008\t4\n"
235
+ exit_routine = [{"routine": 8, "reason": "synthetic exit"}]
236
+ r = self.check(inventory, data=data, noReturn=exit_routine)
237
+ self.assertEqual(r["rowsPastNoReturn"], [{
238
+ "start": "1000:0000", "size": 8, "kind": "call", "site": 0, "siteAddress": "1000:0000",
239
+ "siteClassification": "entry-path instruction", "routine": 8, "following": 3, "followingAddress": "1000:0003",
240
+ "followingRead": False, "bytesAfter": 5}])
241
+ self.assertEqual(r["noReturn"], [{"routine": 8, "reason": "synthetic exit", "reached": True, "read": True,
242
+ "returnSites": [], "contradicted": False,
243
+ "callSites": [{"site": 0, "following": 3, "followingRead": False}]}])
244
+ self.assertEqual((r["counts"]["rowsPastNoReturn"], r["counts"]["rowsPastNoReturnUnreadAfter"]), (1, 1))
245
+ self.assertIn(" 1 row continues past a call or interrupt declared noReturn, 1 of them where the walk read no "
246
+ "instruction after it by another route.", r["summary"])
247
+ self.assertEqual(r["assumptions"], ["each reached call returns to its next instruction, except a call to a noReturn routine",
248
+ "each interrupt the noReturn check reads returns to its next instruction; "
249
+ "interrupt handlers are not read",
250
+ "each noReturn routine and interrupt never returns, for the reason it gives"])
251
+ # The call in the bytes after the exit call is only a raw byte candidate once the routine is declared.
252
+ self.assertEqual([(t["target"], t["evidence"]) for t in r["targets"]], [(3, "raw byte candidate only")])
253
+ undeclared = self.check(inventory, data=data)
254
+ self.assertEqual([(t["target"], t["evidence"]) for t in undeclared["targets"]], [(3, "entry-path call")])
255
+ self.assertEqual((undeclared["rowsPastNoReturn"], undeclared["noReturn"]), ([], []))
256
+ self.assertEqual(undeclared["assumptions"], ["each reached call returns to its next instruction"])
257
+ # A row that ends at the call's last byte does not continue past it.
258
+ ended = self.check("start\tsize\n1000:0000\t3\n1000:0008\t4\n", data=data, noReturn=exit_routine)
259
+ self.assertEqual(ended["rowsPastNoReturn"], [])
260
+
261
+ def test_a_row_with_two_calls_to_a_no_return_routine_lists_both_and_counts_one_row(self):
262
+ # 0000 jz 0005; 0002 call 000C; 0005 jnz 000A; 0007 call 000C; 000A ret; 000B nop; 000C mov ah,4Ch; int 21h.
263
+ data = bytes.fromhex("7403" "e80700" "7503" "e80200" "c3" "90" "b44c" "cd21")
264
+ exit_routine = [{"routine": 12, "reason": "synthetic exit"}]
265
+ r = self.check("start\tsize\n1000:0000\t12\n1000:000C\t4\n", data=data, noReturn=exit_routine)
266
+ self.assertEqual([(row["start"], row["site"], row["followingRead"]) for row in r["rowsPastNoReturn"]],
267
+ [("1000:0000", 2, True), ("1000:0000", 7, True)])
268
+ self.assertEqual((r["counts"]["rowsPastNoReturn"], r["counts"]["rowsPastNoReturnUnreadAfter"]), (1, 0))
269
+ self.assertIn(" 1 row continues past a call or interrupt declared noReturn, 0 of them where", r["summary"])
270
+ # The result limit cuts the list, not the count.
271
+ cut = self.check("start\tsize\n1000:0000\t12\n1000:000C\t4\n", data=data, noReturn=exit_routine, limit=1)
272
+ self.assertEqual([row["site"] for row in cut["rowsPastNoReturn"]], [2])
273
+ self.assertEqual(cut["counts"]["rowsPastNoReturn"], 1)
274
+ self.assertTrue(cut["truncated"])
275
+
276
+ def test_a_no_return_routine_the_check_walk_does_not_establish_is_not_read(self):
277
+ # 0000 ret; 0001 is a 0F byte that does not decode, declared as a routine that never returns.
278
+ r = self.check("start\tsize\n1000:0000\t1\n", data=bytes.fromhex("c3" "0f"),
279
+ noReturn=[{"routine": 1, "reason": "synthetic exit"}])
280
+ self.assertEqual([(row["read"], row["returnSites"], row["contradicted"]) for row in r["noReturn"]], [(False, [], False)])
281
+ self.assertIn(" 1 noReturn routine is not an instruction the check walk established, so its return check reads nothing.",
282
+ r["summary"])
283
+
284
+ def test_a_row_past_a_no_return_interrupt_is_listed_and_the_interrupt_is_no_boundary_gap(self):
285
+ # mov ah,4Ch; int 21h; three data bytes; ret. The row covers all eight bytes.
286
+ data = bytes.fromhex("b44c" "cd21" "4f4b00" "c3")
287
+ inventory = "start\tsize\n1000:0000\t8\n"
288
+ r = self.check(inventory, data=data, noReturn=[{"interrupt": 2, "reason": "synthetic terminate"}])
289
+ self.assertEqual(r["rowsPastNoReturn"], [{
290
+ "start": "1000:0000", "size": 8, "kind": "interrupt", "site": 2, "siteAddress": "1000:0002",
291
+ "siteClassification": "entry-path instruction", "following": 4, "followingAddress": "1000:0004",
292
+ "followingRead": False, "bytesAfter": 4}])
293
+ self.assertEqual(r["noReturn"], [{"interrupt": 2, "reason": "synthetic terminate", "vector": 0x21, "reached": True,
294
+ "following": 4, "followingRead": False}])
295
+ self.assertEqual([g for g in r["gaps"] if g.get("site") == 2], [])
296
+ undeclared = self.check(inventory, data=data)
297
+ self.assertEqual([g["reason"] for g in undeclared["gaps"] if g.get("site") == 2], ["hardware or interrupt boundary"])
298
+ with self.assertRaisesRegex(ValueError, "noReturn interrupt 0 is not an interrupt instruction"):
299
+ self.check(inventory, data=data, noReturn=[{"interrupt": 0, "reason": "synthetic"}])
300
+
301
+ def test_a_no_return_routine_is_checked_past_its_interrupts_and_a_call_reached_another_way_says_so(self):
302
+ # 0000 jnz 0005; 0002 call 0006; 0005 ret. 0006 mov ah,4Ch; 0008 int 21h; 000A ret.
303
+ data = bytes.fromhex("7503" "e80100" "c3" "b44c" "cd21" "c3")
304
+ inventory = "start\tsize\n1000:0000\t6\n1000:0006\t5\n"
305
+ exit_routine = {"routine": 6, "reason": "synthetic exit"}
306
+ # The entry-path walk ends at the interrupt, but the check reads past it to the RET, as reach does.
307
+ r = self.check(inventory, data=data, noReturn=[exit_routine])
308
+ self.assertEqual([(row["returnSites"], row["contradicted"]) for row in r["noReturn"]], [([10], True)])
309
+ self.assertIn(" 1 noReturn routine has a return on its own read paths, so the declaration is contradicted.", r["summary"])
310
+ self.assertEqual([(row["site"], row["following"], row["followingRead"]) for row in r["rowsPastNoReturn"]],
311
+ [(2, 5, True)])
312
+ self.assertEqual(r["counts"]["rowsPastNoReturnUnreadAfter"], 0)
313
+ self.assertIn(", 0 of them where the walk read no instruction after it by another route.", r["summary"])
314
+ # Declaring the interrupt too ends the routine's paths there.
315
+ both = self.check(inventory, data=data, noReturn=[exit_routine, {"interrupt": 8, "reason": "synthetic terminate"}])
316
+ self.assertEqual([(row.get("returnSites"), row.get("contradicted")) for row in both["noReturn"]],
317
+ [([], False), (None, None)])
318
+ self.assertNotIn("contradicted", both["summary"])
319
+ # The check walk shares instructionLimit and says when it stopped.
320
+ stopped = self.check(inventory, data=data, noReturn=[exit_routine], instructionLimit=2)
321
+ self.assertTrue(stopped["noReturnCheckLimitReached"])
322
+ self.assertIn("The walk that checks the noReturn routines stopped at its instruction limit", stopped["summary"])
323
+ self.assertFalse(r["noReturnCheckLimitReached"])
324
+
174
325
  def test_a_contested_instruction_holds_no_target(self):
175
326
  # Entries 0 and 1 overlap, so the call at 0003 is contested. The raw call at 000D targets 0004, a byte of
176
327
  # that contested call, and names no holder, since only established instructions count.
@@ -182,6 +333,25 @@ class InventoryCheckTests(unittest.TestCase):
182
333
  [(4, "raw byte candidate only"), (10, "contested call only")])
183
334
  self.assertTrue(all("insideInstruction" not in t for t in r["targets"]))
184
335
  self.assertEqual(sum(t["insideAnInstruction"] for t in r["counts"].values() if isinstance(t, dict)), 0)
336
+ # A row starting at the contested call is counted as contested. The row at 0000, an entry of the
337
+ # unresolved overlapping pair, is not read: the walk established no instruction there.
338
+ rows = self.inventory("start\tsize\n1000:0000\t3\n1000:0003\t14\n", name="contested.tsv")
339
+ r = run_report(data, config(data, regions, inventory=rows), "inventory-check")
340
+ self.assertEqual((r["counts"]["rowStarts"]["contested"], r["counts"]["rowStarts"]["notRead"]), (1, 1))
341
+ self.assertIn(" 1 row start lies at or inside an instruction the walk rejected as contested, so its boundary "
342
+ "is not checked.", r["summary"])
343
+
344
+ def test_a_raw_byte_call_to_a_no_return_routine_is_listed_but_not_counted_as_a_row_past_it(self):
345
+ # 0000 ret; 0001 E8 01 00 are data bytes that decode as a call to the exit routine at 0005; 0004 data.
346
+ data = bytes.fromhex("c3" "e80100" "00" "b44c" "cd21")
347
+ r = self.check("start\tsize\n1000:0000\t5\n1000:0005\t4\n", data=data,
348
+ noReturn=[{"routine": 5, "reason": "synthetic exit"}])
349
+ self.assertEqual([(row["site"], row["siteClassification"]) for row in r["rowsPastNoReturn"]],
350
+ [(1, "raw byte candidate")])
351
+ self.assertEqual((r["counts"]["rowsPastNoReturn"], r["counts"]["rowsPastNoReturnUnverified"]), (0, 1))
352
+ self.assertNotIn("row continues past", r["summary"])
353
+ self.assertIn(" 1 more row holds such a call or interrupt that only raw bytes, a contested instruction or an "
354
+ "unreached declaration show, so it is not shown to run.", r["summary"])
185
355
 
186
356
  def test_a_scan_limit_short_of_the_code_makes_the_search_partial(self):
187
357
  # The region lies in no container or declared segment, so only the scan gap shows the stop.