scientific-method-engine 18.4.0__tar.gz → 19.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/cli.py +7 -6
  4. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/inventory.py +135 -41
  5. scientific_method_engine-19.1.0/src/scientific_method_engine/x86/ranges.py +592 -0
  6. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/reach.py +78 -23
  7. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/reports.py +5 -3
  8. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/trace.py +14 -9
  9. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_inventory_check.py +116 -4
  10. scientific_method_engine-19.1.0/tests/test_ranges.py +419 -0
  11. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_reach.py +95 -2
  12. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/.gitignore +0 -0
  13. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/LICENSE +0 -0
  14. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/README.md +0 -0
  15. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/__init__.py +0 -0
  16. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/__main__.py +0 -0
  17. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  18. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  19. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  20. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  21. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  22. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  23. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  24. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  25. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  26. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  27. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  28. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  29. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  30. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  31. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  32. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  33. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  34. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  35. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  36. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  37. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  38. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  39. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  40. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  41. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  42. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  43. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  44. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  45. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  46. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  47. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  48. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  49. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  50. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  51. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  52. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  53. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  54. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  55. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  56. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/image.py +0 -0
  57. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/loops.py +0 -0
  58. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/machine.py +0 -0
  59. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  60. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  61. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  62. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/pe.py +0 -0
  63. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/relational.py +0 -0
  64. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  65. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/src/scientific_method_engine/x86/values.py +0 -0
  66. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/oracle.py +0 -0
  67. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_config_encoding.py +0 -0
  68. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_continuation_budget.py +0 -0
  69. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_dispatch.py +0 -0
  70. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_effect_order.py +0 -0
  71. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_enter_frames.py +0 -0
  72. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_entry_frame.py +0 -0
  73. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_indirect_far_transfers.py +0 -0
  74. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_interrupt_models.py +0 -0
  75. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_join_terms.py +0 -0
  76. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_loops.py +0 -0
  77. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_memory_scopes.py +0 -0
  78. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_nested_frame_request.py +0 -0
  79. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_oracle.py +0 -0
  80. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_pe.py +0 -0
  81. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_relational_controls.py +0 -0
  82. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_return_checks.py +0 -0
  83. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_successor_rule.py +0 -0
  84. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_table_continuations.py +0 -0
  85. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_term_limit.py +0 -0
  86. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_upstream_gaps.py +0 -0
  87. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_volatile_memory.py +0 -0
  88. {scientific_method_engine-18.4.0 → scientific_method_engine-19.1.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 18.4.0
3
+ Version: 19.1.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "18.4.0"
8
+ version = "19.1.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -24,9 +24,9 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
24
24
  "reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
25
25
  "and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
26
26
  "a leaves routine is reached but not read, and its reason is repeated in the report;\n"
27
- "a call to a noReturn routine and a noReturn interrupt do not continue at the next instruction,\n"
28
- "and a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
29
- "is reported contradicted;\n"
27
+ "a call to a noReturn routine, a noReturn call and a noReturn interrupt do not continue at the next\n"
28
+ "instruction, a noReturn routine with a return, or a jump into a returning leaf, on its own read paths\n"
29
+ "is reported contradicted, and a noReturn call lists its targets' return sites and is never contradicted;\n"
30
30
  "indirectCalls declares the targets of a computed call as a table read from the build or a list,\n"
31
31
  "and the walk calls each of them; one not declared exhaustive stays unresolved;\n"
32
32
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
@@ -36,9 +36,10 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
36
36
  "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
37
37
  "a target inside an instruction the entry-path walk established from another start names it as insideInstruction.\n"
38
38
  "It lists each row whose start lies inside an instruction the walk established (rowStarts), with the\n"
39
- "routine the walk read it in; a row start in bytes the walk did not decode is counted, not placed.\n"
40
- "noReturn takes reach's declarations: the walk stops at a call to a declared routine and at a declared\n"
41
- "interrupt, a declared routine with a return on its own read paths is reported contradicted, and\n"
39
+ "routine the walk read it in, and each row start at or inside an instruction of an overlap the walk left\n"
40
+ "unresolved, with the instructions it overlaps; a row start in bytes the walk did not decode is counted, not placed.\n"
41
+ "noReturn takes reach's declarations: the walk stops at a call to a declared routine, at a declared call\n"
42
+ "and at a declared interrupt, a declared routine with a return on its own read paths is reported contradicted, and\n"
42
43
  "rowsPastNoReturn lists each row whose body continues past such a call or interrupt.\n"
43
44
  "indirectCalls takes reach's declarations: each declared target of a reached computed call is a call target.\n"
44
45
  "uses matches accesses whose footprint intersects query offset..offset+width; its rawCandidates give each unreached\n"
@@ -3,7 +3,7 @@ import re
3
3
  from bisect import bisect_right
4
4
  from pathlib import Path
5
5
  from .image import integer
6
- from .trace import LIMIT_REASON, CONTESTED_REASON, walk, holding_instruction
6
+ from .trace import LIMIT_REASON, CONTESTED_REASON, OVERLAP_REASON, walk, holding_instruction
7
7
  from .dispatch import indirect_call_declarations, call_ends
8
8
 
9
9
  MAX_INVENTORY = 16 * 1024 * 1024
@@ -139,15 +139,53 @@ def _row_fields(row):
139
139
  return {k: row[k] for k in ("start", "size", "name") if k in row}
140
140
 
141
141
 
142
- def _row_starts(image, rows, starts, domains, seen, contested, routine_of):
143
- """Each row whose start lies inside an instruction the entry-path walk established, and counts of the rest.
142
+ def _routine_fields(image, starts, routine, prefix=""):
143
+ """The ``routine``, ``routineAddress`` and ``routineIsRow`` fields for a routine start, or none when it is None.
144
+
145
+ A ``prefix`` names the fields after it, as ``rowStartRoutine`` for ``prefix="rowStart"``.
146
+ """
147
+ if routine is None:
148
+ return {}
149
+ place = _target_place(image, routine)
150
+ name = prefix + "Routine" if prefix else "routine"
151
+ return {name: routine, name + "Address": place[2], name + "IsRow": place[:2] in starts}
152
+
153
+
154
+ def _overlaps(image, starts, at, decoded, routine_of):
155
+ """Every instruction in ``decoded`` other than the one at ``at`` whose bytes meet the bytes of the
156
+ instruction at ``at``, or the byte ``at`` when no instruction starts there, with its evidence.
157
+
158
+ ``decoded`` maps a site to ``(instruction, evidence)``.
159
+ """
160
+ end = at + decoded[at][0].size if at in decoded else at + 1
161
+ found = []
162
+ # An x86 instruction is at most 15 bytes, so only the starts just before the site can reach into it.
163
+ for site in range(max(at - 14, 0), end):
164
+ if site == at or site not in decoded or site + decoded[site][0].size <= at:
165
+ continue
166
+ ins, evidence = decoded[site]
167
+ found.append({"site": site, "address": _target_place(image, site)[2], "size": ins.size, "text": _text(ins),
168
+ "evidence": evidence, **_routine_fields(image, starts, routine_of(site))})
169
+ return found
170
+
171
+
172
+ def _row_starts(image, rows, starts, domains, seen, contested, unresolved, routine_of):
173
+ """Each row whose start lies inside an instruction the entry-path walk established, or at or inside
174
+ an instruction of an overlap the walk left unresolved, and counts of the rest.
144
175
 
145
176
  A start is checked against the walk's instructions only; bytes the walk did not decode are not
146
- decoded here, so a row start in them is counted as not read rather than placed. A start at or
147
- inside an instruction the walk decoded but rejected as contested is counted as contested.
177
+ decoded here, so a row start in them is counted as not read rather than placed. ``unresolved``
178
+ holds the instructions at the walk's overlap gaps, which it decoded on the entry path and
179
+ rejected because no boundary between them is proven. A row start at or inside one is listed with
180
+ every instruction it overlaps, since the row's own entry can be one side of the overlap. A start
181
+ at or inside an instruction the walk decoded but rejected as contested is counted as contested.
148
182
  """
149
183
  found, counts = [], {"instructionStarts": 0, "insideAnInstruction": 0, "overlappingInstructionStarts": 0,
150
- "contested": 0, "notRead": 0}
184
+ "unresolvedOverlaps": 0, "contested": 0, "notRead": 0}
185
+ # Every instruction the walk decoded, with its evidence. Only a row start at an unresolved overlap reads it.
186
+ decoded = ({at: (ins, CONTESTED_REASON) for at, ins in contested.items()}
187
+ | {at: (ins, OVERLAP_REASON) for at, ins in unresolved.items()}
188
+ | {at: (ins, "entry-path instruction") for at, ins in seen.items()}) if unresolved else {}
151
189
  for row in rows:
152
190
  offsets = _offsets(image, domains, row["space"], row["at"])
153
191
  if not offsets:
@@ -156,6 +194,25 @@ def _row_starts(image, rows, starts, domains, seen, contested, routine_of):
156
194
  if held is None:
157
195
  if any(at in seen for at in offsets):
158
196
  counts["instructionStarts"] += 1
197
+ continue
198
+ hit = next(((at, holder) for at in offsets for holder in [holding_instruction(unresolved, at)]
199
+ if at in unresolved or holder is not None), None)
200
+ if hit is not None:
201
+ at, holder = hit
202
+ counts["unresolvedOverlaps"] += 1
203
+ entry = {**_row_fields(row), "site": at}
204
+ if at in unresolved:
205
+ entry |= {"status": "start of an unresolved overlapping instruction",
206
+ "rowStartInstructionSize": unresolved[at].size, "rowStartInstructionText": _text(unresolved[at]),
207
+ **_routine_fields(image, starts, routine_of(at), "rowStart")}
208
+ else:
209
+ entry["status"] = "inside an unresolved overlapping instruction"
210
+ if holder is not None:
211
+ entry |= {"insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
212
+ "insideInstructionSize": unresolved[holder].size,
213
+ "insideInstructionText": _text(unresolved[holder]),
214
+ **_routine_fields(image, starts, routine_of(holder))}
215
+ found.append(entry | {"overlaps": _overlaps(image, starts, at, decoded, routine_of)})
159
216
  elif any(at in contested or holding_instruction(contested, at) is not None for at in offsets):
160
217
  counts["contested"] += 1
161
218
  else:
@@ -164,27 +221,25 @@ def _row_starts(image, rows, starts, domains, seen, contested, routine_of):
164
221
  at, holder = held
165
222
  overlapping = at in seen
166
223
  counts["overlappingInstructionStarts" if overlapping else "insideAnInstruction"] += 1
167
- routine = routine_of(holder)
168
224
  entry = {**_row_fields(row), "site": at,
169
225
  "status": "start of an overlapping instruction" if overlapping else "inside an instruction",
170
226
  "insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
171
227
  "insideInstructionSize": seen[holder].size, "insideInstructionText": _text(seen[holder])}
172
228
  if overlapping:
173
- entry |= {"rowStartInstructionSize": seen[at].size, "rowStartInstructionText": _text(seen[at])}
174
- if routine is not None:
175
- place = _target_place(image, routine)
176
- entry |= {"routine": routine, "routineAddress": place[2], "routineIsRow": place[:2] in starts}
177
- found.append(entry)
229
+ entry |= {"rowStartInstructionSize": seen[at].size, "rowStartInstructionText": _text(seen[at]),
230
+ **_routine_fields(image, starts, routine_of(at), "rowStart")}
231
+ found.append(entry | _routine_fields(image, starts, routine_of(holder)))
178
232
  return found, counts
179
233
 
180
234
 
181
235
  def _no_return_check(image, routines, declared, limit):
182
- """``(returns, read, stopped)``: the return sites on each declared routine's own read paths, the
236
+ """``(returns, read, stopped)``: the return sites on each of ``routines``' own read paths, the
183
237
  instructions the walk established, and whether the walk stopped.
184
238
 
185
- The walk starts at the declared routines and continues past every interrupt except a declared
186
- one, as ``reach`` reads them, so a routine that ends in an interrupt that returns is contradicted
187
- by what follows it. Calls are stepped over at their return sites, except calls to declared routines.
239
+ ``routines`` are the declared routines, or the targets of the declared calls. The walk starts at
240
+ them and continues past every interrupt except a declared one, as ``reach`` reads them, so a
241
+ routine that ends in an interrupt that returns is contradicted by what follows it. Calls are
242
+ stepped over at their return sites, except calls to declared routines and declared calls.
188
243
  A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
189
244
  has no return sites and is not in ``read``.
190
245
  """
@@ -212,27 +267,28 @@ def inventory_check(image, config):
212
267
  the first of an instruction the entry-path walk established names that instruction.
213
268
 
214
269
  The rows are checked against the same walk: each row whose start lies inside an instruction the
215
- walk established is listed in ``rowStarts``. ``noReturn`` declarations, in ``reach``'s shape, end
216
- the walk at a call to a declared routine and at a declared interrupt site, are checked for a
270
+ walk established, or at or inside an instruction of an overlap the walk left unresolved, is listed
271
+ in ``rowStarts``. ``noReturn`` declarations, in ``reach``'s shape, end the walk at a call to a
272
+ declared routine, at a declared call site and at a declared interrupt site, are checked for a
217
273
  return on the routine's own paths, and list in ``rowsPastNoReturn`` each row whose body
218
274
  continues past such a call or interrupt. ``indirectCalls`` declarations, in ``reach``'s shape,
219
275
  make each declared target of a reached computed call a call target, and the walk reads on in them.
220
276
  """
221
277
  from .reports import call_controls, direct_calls, entries, search_coverage
222
- from .reach import no_return_declarations, no_return_rows, successor_graph, fewest_calls
278
+ from .reach import (no_return_inputs, no_return_rows, successor_graph, fewest_calls, call_assumption,
279
+ no_return_assumptions)
223
280
  if "inventory" not in config:
224
281
  raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
225
282
  rows = read_inventory(config["inventory"], image)
226
283
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
227
- no_return_routines, no_return_interrupts = no_return_declarations(config.get("noReturn", []), image)
228
284
  indirect_calls = indirect_call_declarations(config.get("indirectCalls", []), image)
285
+ no_return_routines, no_return_interrupts, no_return_sites, declared = no_return_inputs(
286
+ config.get("noReturn", []), image, indirect_calls)
229
287
  for at in config.get("controls", []) if isinstance(config.get("controls"), list) else []:
230
288
  # call_controls refuses a control that is no integer.
231
289
  if type(at) is int and at in indirect_calls:
232
290
  raise ValueError(f"Positive control {at} is a declared indirect call, whose targets rest on its "
233
291
  "declaration; a control must resolve from its own encoding")
234
- declared = {"no_return_calls": frozenset(no_return_routines), "no_return_interrupts": frozenset(no_return_interrupts),
235
- "indirect_calls": indirect_calls}
236
292
  calls = direct_calls(image, config, **declared)
237
293
 
238
294
  def ends_branch(row):
@@ -310,27 +366,49 @@ def inventory_check(image, config):
310
366
  entry["rows"] = [_row_fields(rows[i]) for i in held]
311
367
  missing.append(entry)
312
368
 
369
+ # The instructions of the overlaps the walk left unresolved. The walk decoded each at its gap site.
370
+ overlapping = {g["site"]: image.decode(g["site"]) for g in calls["gaps"] if g.get("reason") == OVERLAP_REASON}
371
+
313
372
  # The routine the walk read an instruction in, by the route with the fewest calls, as reach names it.
373
+ # An unresolved instruction is a node that ends its route, so it gets the routine of the established
374
+ # instruction that steps into it, or its own when it is an entry, and no route passes through it.
314
375
  # The graph is built only when a row start needs it.
315
376
  routines = None
316
377
 
317
378
  def routine_of(site):
318
379
  nonlocal routines
319
380
  if routines is None:
320
- graph = successor_graph(image, seen, **declared)
321
- routines = fewest_calls(graph, [at for at in entries(image) if at in seen])[2]
381
+ graph = successor_graph(image, seen | overlapping, **declared)
382
+ graph = {at: [] if at in overlapping else successors for at, successors in graph.items()}
383
+ routines = fewest_calls(graph, [at for at in entries(image) if at in seen or at in overlapping])[2]
322
384
  return routines.get(site)
323
- row_starts, start_counts = _row_starts(image, rows, starts, domains, seen, contested, routine_of)
385
+ row_starts, start_counts = _row_starts(image, rows, starts, domains, seen, contested, overlapping, routine_of)
324
386
 
325
- # Each declaration with the calls it kept from their return sites, as reach reports them.
326
- returns, check_read, check_stopped = _no_return_check(image, no_return_routines, declared,
327
- config.get("instructionLimit", 10000))
387
+ # Each declaration with the calls it kept from their return sites, as reach reports them: only a call the
388
+ # entry-path walk established has targets, so a raw byte candidate at an unreached declared site gives none.
389
+ site_targets = {at: [] for at in no_return_sites}
390
+ for row in calls["rows"]:
391
+ if (row["site"] in site_targets and row["site"] in seen and row["target"] is not None
392
+ and row["target"] not in site_targets[row["site"]]):
393
+ site_targets[row["site"]].append(row["target"])
394
+ # The targets of the declared calls get a walk of their own, so that their code, overlapping a declared
395
+ # routine's or spending the limit first, cannot change what the routine check reads.
396
+ call_targets = {target for found in site_targets.values() for target in found} - set(no_return_routines)
397
+ checked = set(no_return_routines) | call_targets
398
+ check_limit = config.get("instructionLimit", 10000)
399
+ returns, routine_read, routine_stopped = _no_return_check(image, no_return_routines, declared, check_limit)
400
+ target_returns, target_read, target_stopped = _no_return_check(image, call_targets, declared, check_limit)
401
+ returns |= target_returns
402
+ check_read = ({at for at in no_return_routines if at in routine_read}
403
+ | {at for at in call_targets if at in target_read})
404
+ check_stopped = routine_stopped or target_stopped
328
405
  entry_calls = {}
329
406
  for row in calls["rows"]:
330
407
  if row["target"] in no_return_routines and row["site"] in seen and ends_branch(row):
331
408
  entry_calls.setdefault(row["target"], []).append(row["site"])
332
409
  no_return = no_return_rows(no_return_routines, no_return_interrupts, returns,
333
- {at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read)
410
+ {at: sorted(sites) for at, sites in entry_calls.items()}, seen, seen, check_read,
411
+ no_return_sites, site_targets)
334
412
 
335
413
  # A row whose body holds a declared call or interrupt and the byte after it still covers what follows.
336
414
  # A declared computed call has one row for each target, and is listed once with all of them.
@@ -343,8 +421,14 @@ def inventory_check(image, config):
343
421
  declared_ends.add(row["site"])
344
422
  routine = list(indirect_calls[row["site"]]["targets"])
345
423
  ends.append((row["site"], image.decode(row["site"]).size, "call", routine, row["classification"]))
346
- ends += [(at, row["following"] - at, "interrupt", None,
347
- "entry-path instruction" if at in seen else CONTESTED_REASON if at in contested else "declared site")
424
+ # A declared call site is listed as one, whatever its targets, unless a routine declaration already listed it.
425
+ def site_classification(at):
426
+ return "entry-path instruction" if at in seen else CONTESTED_REASON if at in contested else "declared site"
427
+ listed = {site for site, *_ in ends}
428
+ ends += [(at, row["following"] - at, "call site",
429
+ site_targets[at][0] if len(site_targets[at]) == 1 else site_targets[at] or None, site_classification(at))
430
+ for at, row in no_return_sites.items() if at not in listed]
431
+ ends += [(at, row["following"] - at, "interrupt", None, site_classification(at))
348
432
  for at, row in no_return_interrupts.items()]
349
433
  # Only a site the entry-path walk established counts a row; the rest are listed with their evidence.
350
434
  past, past_rows, past_unread, past_unverified = [], set(), set(), set()
@@ -412,6 +496,12 @@ def inventory_check(image, config):
412
496
  summary += (f", {start_counts['overlappingInstructionStarts']} of them at the start of an overlapping "
413
497
  "instruction it also established")
414
498
  summary += "."
499
+ if start_counts["unresolvedOverlaps"]:
500
+ summary += " " + _count(start_counts["unresolvedOverlaps"],
501
+ "row start lies at or inside an instruction of an overlap the walk left unresolved, "
502
+ "listed with the instructions it overlaps.",
503
+ "row starts lie at or inside an instruction of an overlap the walk left unresolved, "
504
+ "each listed with the instructions it overlaps.")
415
505
  if start_counts["notRead"]:
416
506
  summary += " " + _count(start_counts["notRead"], "row start in declared code lies", "row starts in declared code lie") \
417
507
  + " in bytes where the walk established no instruction, so its boundary is not checked."
@@ -433,16 +523,17 @@ def inventory_check(image, config):
433
523
  if unread:
434
524
  summary += " " + _count(unread, "noReturn routine is", "noReturn routines are") \
435
525
  + " not an instruction the check walk established, so its return check reads nothing."
436
- if check_stopped:
526
+ if routine_stopped:
437
527
  summary += " The walk that checks the noReturn routines stopped at its instruction limit, so it can miss returns."
438
- assumptions = ["each reached call returns to its next instruction"
439
- + (", except a call to a noReturn routine" if no_return_routines else "")]
440
- if no_return_routines:
528
+ if target_stopped:
529
+ summary += (" The walk that checks the targets of the noReturn calls stopped at its instruction limit, so it "
530
+ "can miss returns.")
531
+ assumptions = [call_assumption(no_return_routines, no_return_sites)]
532
+ if checked:
441
533
  assumptions.append("each interrupt the noReturn check reads returns to its next instruction"
442
534
  + (", except at a noReturn interrupt site" if no_return_interrupts else "")
443
535
  + "; interrupt handlers are not read")
444
- if no_return:
445
- assumptions.append("each noReturn routine and interrupt never returns, for the reason it gives")
536
+ assumptions += no_return_assumptions(no_return_routines, no_return_interrupts, no_return_sites)
446
537
  if indirect_calls:
447
538
  assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
448
539
  "it gives, and only those when it is declared exhaustive")
@@ -469,10 +560,13 @@ def inventory_check(image, config):
469
560
  "notation and compared with its rows. A target only raw byte candidates or contested "
470
561
  "instructions call is not shown to be code. The counts are lower bounds on the targets "
471
562
  "the inventory lacks: calls the search does not resolve can add more. Row starts are "
472
- "compared only with the instructions the entry-path walk established; a start where it "
473
- "established no instruction is counted as notRead or contested and not placed, so linear decoding past data "
474
- "never reports one. A start inside an instruction can be deliberately overlapping code, "
563
+ "compared only with the instructions the entry-path walk decoded; a start at or inside an instruction "
564
+ "of an overlap the walk left unresolved is listed with every instruction it overlaps, and a "
565
+ "start where it established no instruction is counted as notRead or contested and not placed, "
566
+ "so linear decoding past data never reports one. A start inside an instruction can be deliberately overlapping code, "
475
567
  "which the row names with both instructions for a reader to judge. rowsPastNoReturn rests "
476
568
  "on the noReturn declarations: followingRead says whether the walk reached the bytes after "
477
569
  "the call by another route, only a site the walk established counts a row, and an empty "
478
- "returnSites means only that the walk read no return."}
570
+ "returnSites means only that the walk read no return. A noReturn call is never "
571
+ "contradicted: the return sites of its targets are listed, and the declaration rests "
572
+ "on its reason alone."}