scientific-method-engine 15.2.0__tar.gz → 15.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/cli.py +2 -1
  4. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/inventory.py +18 -2
  5. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_inventory_check.py +42 -0
  6. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/.gitignore +0 -0
  7. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/LICENSE +0 -0
  8. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/README.md +0 -0
  9. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/__init__.py +0 -0
  10. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/__main__.py +0 -0
  11. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  12. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  13. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  14. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  15. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  16. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  17. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  18. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  19. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  20. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  21. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  22. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  23. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  24. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  25. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  26. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  27. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  28. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  29. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  30. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  31. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  32. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  33. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  34. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  35. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  36. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  37. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  38. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  39. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  40. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  41. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  42. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  43. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  44. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  45. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  46. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  47. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  48. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  49. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  50. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/image.py +0 -0
  51. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/loops.py +0 -0
  52. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/machine.py +0 -0
  53. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  54. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  55. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  56. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/pe.py +0 -0
  57. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/reach.py +0 -0
  58. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/relational.py +0 -0
  59. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/reports.py +0 -0
  60. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  61. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/trace.py +0 -0
  62. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/src/scientific_method_engine/x86/values.py +0 -0
  63. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/oracle.py +0 -0
  64. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_continuation_budget.py +0 -0
  65. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_dispatch.py +0 -0
  66. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_effect_order.py +0 -0
  67. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_enter_frames.py +0 -0
  68. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_entry_frame.py +0 -0
  69. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_indirect_far_transfers.py +0 -0
  70. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_interrupt_models.py +0 -0
  71. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_loops.py +0 -0
  72. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_memory_scopes.py +0 -0
  73. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_nested_frame_request.py +0 -0
  74. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_oracle.py +0 -0
  75. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_pe.py +0 -0
  76. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_reach.py +0 -0
  77. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_relational_controls.py +0 -0
  78. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_return_checks.py +0 -0
  79. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_successor_rule.py +0 -0
  80. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_table_continuations.py +0 -0
  81. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_term_limit.py +0 -0
  82. {scientific_method_engine-15.2.0 → scientific_method_engine-15.3.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 15.2.0
3
+ Version: 15.3.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "15.2.0"
8
+ version = "15.3.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -26,7 +26,8 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
26
26
  "a leaves routine is reached but not read, and its reason is repeated in the report.\n"
27
27
  "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
28
28
  "that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
29
- "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it.\n"
29
+ "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
30
+ "a target inside an instruction the entry-path walk established from another start names it as insideInstruction.\n"
30
31
  "trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
31
32
  "a violated control fails the report; an undecided one is reported and never counts as held.\n"
32
33
  "A lastWriter control with an address inspects that memory at its checkpoint anchors without a read.\n"
@@ -132,9 +132,11 @@ def inventory_check(image, config):
132
132
  The calls are those ``incoming`` reads (``direct_calls``). Each distinct target is an inventory
133
133
  start, inside the body of another row, outside every row, or outside declared code, and is
134
134
  reported with one calling site, near or far, and the evidence of its best site: a call on the
135
- entry path, a contested instruction, or only a raw byte candidate.
135
+ entry path, a contested instruction, or only a raw byte candidate. A target that is a byte past
136
+ the first of an instruction the entry-path walk established names that instruction.
136
137
  """
137
138
  from .reports import call_controls, direct_calls, search_coverage
139
+ from .trace import holding_instruction
138
140
  if "inventory" not in config:
139
141
  raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
140
142
  rows = read_inventory(config["inventory"], image)
@@ -173,7 +175,7 @@ def inventory_check(image, config):
173
175
  def rank(row):
174
176
  return 0 if row["site"] in seen else 1 if row["site"] in contested else 2
175
177
  tally = {label: {"targets": 0, "inventoryStarts": 0, "insideAnotherRow": 0, "outsideEveryRow": 0,
176
- "outsideDeclaredCode": 0} for label in EVIDENCE}
178
+ "outsideDeclaredCode": 0, "insideAnInstruction": 0} for label in EVIDENCE}
177
179
  field = {START: "inventoryStarts", INSIDE: "insideAnotherRow", OUTSIDE: "outsideEveryRow", UNPLACED: "outsideDeclaredCode"}
178
180
  missing = []
179
181
  for target, sites in sorted(by_target.items()):
@@ -198,6 +200,16 @@ def inventory_check(image, config):
198
200
  "siteClassification": best["classification"], "region": best["region"], "provenance": best["provenance"],
199
201
  "callSites": {"entryPath": sum(rank(r) == 0 for r in sites), "contested": sum(rank(r) == 1 for r in sites),
200
202
  "rawCandidates": sum(rank(r) == 2 for r in sites)}}
203
+ # A call into the bytes of an established instruction runs an overlapping instruction stream, such
204
+ # as a call to an IRET byte inside an operand. The target is still a call target the inventory
205
+ # lacks, so it keeps its status, and the instruction it overlaps is named. insideInstruction is the
206
+ # holder's site, as in reach and incoming.
207
+ holder = holding_instruction(seen, target)
208
+ if holder is not None:
209
+ tally[evidence]["insideAnInstruction"] += 1
210
+ # The walk decodes only inside declared regions, so a holder always has a place.
211
+ entry |= {"insideInstruction": holder, "insideInstructionAddress": _target_place(image, holder)[2],
212
+ "insideInstructionSize": seen[holder].size}
201
213
  if held:
202
214
  entry["rows"] = [{k: rows[i][k] for k in ("start", "size", "name") if k in rows[i]} for i in held]
203
215
  missing.append(entry)
@@ -213,6 +225,10 @@ def inventory_check(image, config):
213
225
  summary = (f"{lacking} of the {_count(compared, 'call target that entry-path calls resolve to is', 'call targets that entry-path calls resolve to are')} "
214
226
  f"not inventory starts: {path['insideAnotherRow']} inside another row's body, "
215
227
  f"{path['outsideEveryRow']} outside every row.")
228
+ if path["insideAnInstruction"]:
229
+ summary += " " + _count(path["insideAnInstruction"],
230
+ "of them starts inside an instruction the entry-path walk established from another start.",
231
+ "of them start inside an instruction the entry-path walk established from another start.")
216
232
  if path["outsideDeclaredCode"]:
217
233
  summary += " " + _count(path["outsideDeclaredCode"],
218
234
  "more entry-path call target lies outside declared code and is not compared with the inventory.",
@@ -141,6 +141,48 @@ class InventoryCheckTests(unittest.TestCase):
141
141
  "starts: 0 inside another row's body, 0 outside every row. 1 more entry-path "
142
142
  "call target lies outside declared code and is not compared with the inventory."))
143
143
 
144
+ def test_a_target_inside_an_established_instruction_names_that_instruction(self):
145
+ # 0001 or bh,0 holds an IRET byte at 0002, which the push-CS/call frame at 0004 calls; the walk decodes
146
+ # both streams. 0009 is an unreached call to 0003, the immediate byte of the same instruction.
147
+ data = bytes.fromhex("9c" "80cf00" "0e" "e8faff" "c3" "e8f7ff" "c3")
148
+ r = self.check("start\tsize\n1000:0000\t9\n", data=data)
149
+ holder = ("insideInstruction", "insideInstructionAddress", "insideInstructionSize")
150
+ self.assertEqual([(t["target"], t["status"], t["evidence"], *(t[k] for k in holder)) for t in r["targets"]], [
151
+ (2, "inside another row's body", "entry-path call", 1, "1000:0001", 3),
152
+ (3, "inside another row's body", "raw byte candidate only", 1, "1000:0001", 3)])
153
+ self.assertEqual(r["counts"]["entry-path call"]["insideAnInstruction"], 1)
154
+ self.assertEqual(r["counts"]["raw byte candidate only"]["insideAnInstruction"], 1)
155
+ self.assertTrue(r["summary"].startswith(
156
+ "1 of the 1 call target that entry-path calls resolve to is not inventory starts: 1 inside another row's body, "
157
+ "0 outside every row. 1 of them starts inside an instruction the entry-path walk established from another start."))
158
+ # Without an inventory row the target keeps its status, and still names the instruction.
159
+ outside = self.check("start\tsize\n1000:0000\t1\n", data=data)
160
+ self.assertEqual([(t["status"], t["insideInstruction"]) for t in outside["targets"]],
161
+ [("outside every row", 1), ("outside every row", 1)])
162
+ # A target in bytes no walk decoded, or at an instruction start, names none.
163
+ unread = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n1000:000B\t1\n", data=RAW)
164
+ self.assertNotIn("insideInstruction", unread["targets"][0])
165
+ self.assertEqual(sum(t["insideAnInstruction"] for k, t in unread["counts"].items() if isinstance(t, dict)), 0)
166
+ self.assertNotIn("inside an instruction", unread["summary"])
167
+ missing = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n")
168
+ self.assertNotIn("insideInstruction", missing["targets"][0])
169
+ # Two frames give the plural wording.
170
+ twice = self.check("start\tsize\n1000:0000\t17\n", data=bytes.fromhex("9c80cf000ee8faff" "9c80cf000ee8faff" "c3"))
171
+ self.assertEqual([t["insideInstruction"] for t in twice["targets"]], [1, 9])
172
+ self.assertIn(" 2 of them start inside an instruction the entry-path walk established from another start.", twice["summary"])
173
+
174
+ def test_a_contested_instruction_holds_no_target(self):
175
+ # Entries 0 and 1 overlap, so the call at 0003 is contested. The raw call at 000D targets 0004, a byte of
176
+ # that contested call, and names no holder, since only established instructions count.
177
+ data = bytes.fromhex("b8 90 90 e8 04 00 c7 06 00 02 90 c3 c3 e8 f4 ff c3")
178
+ regions = [{"name": "resident", "start": 0, "end": len(data), "ip": 0, "segment": 0x1000, "resident": True,
179
+ "entries": [0, 1], "evidence": "synthetic overlapping entries"}]
180
+ r = run_report(data, config(data, regions, inventory=self.inventory("start\tsize\n1000:0000\t17\n")), "inventory-check")
181
+ self.assertEqual([(t["target"], t["evidence"]) for t in r["targets"]],
182
+ [(4, "raw byte candidate only"), (10, "contested call only")])
183
+ self.assertTrue(all("insideInstruction" not in t for t in r["targets"]))
184
+ self.assertEqual(sum(t["insideAnInstruction"] for t in r["counts"].values() if isinstance(t, dict)), 0)
185
+
144
186
  def test_a_scan_limit_short_of_the_code_makes_the_search_partial(self):
145
187
  # The region lies in no container or declared segment, so only the scan gap shows the stop.
146
188
  r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n", data=RAW, scanLimit=12)