scientific-method-engine 19.1.0__tar.gz → 19.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/cli.py +2 -1
  4. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/reach.py +36 -2
  5. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_reach.py +40 -0
  6. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/.gitignore +0 -0
  7. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/LICENSE +0 -0
  8. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/README.md +0 -0
  9. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/__init__.py +0 -0
  10. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/__main__.py +0 -0
  11. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  12. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  13. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  14. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  15. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  16. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  17. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  18. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  19. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  20. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  21. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  22. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  23. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  24. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  25. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  26. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  27. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  28. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  29. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  30. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  31. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  32. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  33. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  34. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  35. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  36. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  37. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  38. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  39. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  40. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  41. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  42. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  43. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  44. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  45. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  46. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  47. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  48. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  49. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  50. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/image.py +0 -0
  51. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/inventory.py +0 -0
  52. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/loops.py +0 -0
  53. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/machine.py +0 -0
  54. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  55. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  56. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  57. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/pe.py +0 -0
  58. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/ranges.py +0 -0
  59. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/relational.py +0 -0
  60. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/reports.py +0 -0
  61. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  62. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/trace.py +0 -0
  63. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/src/scientific_method_engine/x86/values.py +0 -0
  64. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/oracle.py +0 -0
  65. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_config_encoding.py +0 -0
  66. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_continuation_budget.py +0 -0
  67. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_dispatch.py +0 -0
  68. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_effect_order.py +0 -0
  69. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_enter_frames.py +0 -0
  70. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_entry_frame.py +0 -0
  71. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_indirect_far_transfers.py +0 -0
  72. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_interrupt_models.py +0 -0
  73. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_inventory_check.py +0 -0
  74. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_join_terms.py +0 -0
  75. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_loops.py +0 -0
  76. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_memory_scopes.py +0 -0
  77. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_nested_frame_request.py +0 -0
  78. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_oracle.py +0 -0
  79. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_pe.py +0 -0
  80. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_ranges.py +0 -0
  81. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_relational_controls.py +0 -0
  82. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_return_checks.py +0 -0
  83. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_successor_rule.py +0 -0
  84. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_table_continuations.py +0 -0
  85. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_term_limit.py +0 -0
  86. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_upstream_gaps.py +0 -0
  87. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_volatile_memory.py +0 -0
  88. {scientific_method_engine-19.1.0 → scientific_method_engine-19.2.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 19.1.0
3
+ Version: 19.2.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "19.1.0"
8
+ version = "19.2.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -30,7 +30,8 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
30
30
  "indirectCalls declares the targets of a computed call as a table read from the build or a list,\n"
31
31
  "and the walk calls each of them; one not declared exhaustive stays unresolved;\n"
32
32
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
33
- "and a failed control fails the report saying whether it was reached.\n"
33
+ "and a failed control fails the report saying whether it was reached;\n"
34
+ "decodedRanges true lists every instruction the walk decoded as half-open ranges with their routine.\n"
34
35
  "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
35
36
  "that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
36
37
  "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it;\n"
@@ -219,6 +219,28 @@ def fewest_calls(graph, starts):
219
219
  return distance, parent, routine
220
220
 
221
221
 
222
+ def decoded_ranges(seen, routine):
223
+ """The instructions in ``seen`` as half-open ``[start, end)`` runs, each with the ``routine`` they were read in.
224
+
225
+ A run holds instructions that abut, each starting where the one before it ends, and that
226
+ ``routine`` gives the same routine. A site on no route from a start has the routine None. Runs
227
+ overlap only where ``seen`` holds instructions that overlap, which the walk keeps when it proves
228
+ both boundaries.
229
+ """
230
+ # Past an overlap, the run an instruction continues need not be the last one opened, so each open
231
+ # run is found by where it ends and in which routine.
232
+ ranges, ending = [], {}
233
+ for at, ins in sorted(seen.items()):
234
+ owner = routine.get(at)
235
+ run = ending.pop((at, owner), None)
236
+ if run is None:
237
+ run = {"start": at, "end": at, "routine": owner}
238
+ ranges.append(run)
239
+ run["end"] = at + ins.size
240
+ ending[(run["end"], owner)] = run
241
+ return ranges
242
+
243
+
222
244
  def _dominators(order, predecessors):
223
245
  """Immediate dominators (Cooper, Harvey and Kennedy) of every node in ``order``, a reverse postorder from ROOT."""
224
246
  index = {node: i for i, node in enumerate(order)}
@@ -256,7 +278,8 @@ def reach(image, config):
256
278
  at the next instruction, nor does a declared indirect call that is exhaustive and whose targets
257
279
  are all ``noReturn``.
258
280
  Each reached target gets one chain with the fewest calls, the assumptions its route rests on,
259
- and the routines on every route to it within the read graph.
281
+ and the routines on every route to it within the read graph. With ``decodedRanges`` the report
282
+ also gives every instruction the walk decoded, as ``decoded_ranges`` groups them.
260
283
  """
261
284
  from .reports import entries
262
285
  starts = _sites(config.get("starts"), "starts", image)
@@ -287,6 +310,9 @@ def reach(image, config):
287
310
  raise ValueError(f"instruction control {at} is a start; the walk decodes every start, so it shows nothing "
288
311
  "the walk found")
289
312
  limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
313
+ with_ranges = config.get("decodedRanges", False)
314
+ if not isinstance(with_ranges, bool):
315
+ raise ValueError("decodedRanges must be a boolean")
290
316
  instruction_limit = config.get("instructionLimit", 10000)
291
317
  no_return_routines, no_return_interrupts, no_return_sites, declared = no_return_inputs(
292
318
  config.get("noReturn", []), image, indirect_calls)
@@ -466,8 +492,11 @@ def reach(image, config):
466
492
  assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
467
493
  "it gives, and only those when it is declared exhaustive")
468
494
  assumptions += no_return_assumptions(no_return_routines, no_return_interrupts, no_return_sites)
495
+ # Every range is kept whatever the result limit, so a caller can place any site in the walk; the
496
+ # instruction limit bounds their number.
497
+ ranges = {"decodedRanges": decoded_ranges(seen, routine)} if with_ranges else {}
469
498
  return {"starts": starts, "targets": rows, "leaves": leaf_rows, "noReturn": no_return, "indirectCalls": indirect_rows,
470
- "reachedRoutines": sorted(routine_starts), "counts": counts,
499
+ "reachedRoutines": sorted(routine_starts), **ranges, "counts": counts,
471
500
  "unresolved": unresolved[:limit], "interrupts": interrupts[:limit], "gaps": gaps[:limit],
472
501
  "contested": sorted(contested)[:limit],
473
502
  "truncated": any(len(x) > limit for x in (unresolved, interrupts, gaps, contested)),
@@ -495,6 +524,11 @@ def reach(image, config):
495
524
  "A noReturn call is never contradicted, since whether its target returns to it can "
496
525
  "depend on what the caller passes, which the walk does not follow; the return sites "
497
526
  "of its targets are listed, and the declaration rests on its reason alone. "
527
+ "decodedRanges, when asked for, covers the instructions the walk decoded; no range "
528
+ "starts at a leaf or at a contested, unresolved overlapping or undecodable start, though "
529
+ "one can lie inside a decoded instruction. A site inside a range lies in a decoded "
530
+ "instruction, which may start before it; a target at the site shows whether one starts "
531
+ "there. "
498
532
  "When instructionLimitReached holds, the walk stopped before reading all it reaches: "
499
533
  "every list and count covers only the part read, which part depends on the walk order, "
500
534
  "and an unreached target may lie past the stop."}
@@ -431,6 +431,44 @@ class ReachTests(unittest.TestCase):
431
431
  "start of an unresolved overlapping instruction$"):
432
432
  run_report(data, config(data, targets=[6], indirectJumps=[table], instructionControls=[3, 5]), "reach")
433
433
 
434
+ def test_decoded_ranges_give_each_run_of_decoded_instructions_with_its_routine(self):
435
+ r = reach(indirectJumps=[TABLE], decodedRanges=True)
436
+ # The table row 0020 is read in 000B, which jumped there; 001C..0020, 0025..0028 and the table are not decoded.
437
+ self.assertEqual([(x["start"], x["end"], x["routine"]) for x in r["decodedRanges"]],
438
+ [(0x0, 0xB, 0x0), (0xB, 0x18, 0xB), (0x18, 0x1C, 0x18), (0x20, 0x25, 0xB), (0x28, 0x2F, 0x28)])
439
+ self.assertNotIn("decodedRanges", reach(indirectJumps=[TABLE]))
440
+
441
+ def test_decoded_ranges_leave_out_a_leaf_and_the_result_limit_does_not_cut_them(self):
442
+ r = reach(indirectJumps=[TABLE], decodedRanges=True, limit=1, leaves=[{"routine": 0x18, "reason": "synthetic leaf"}])
443
+ self.assertEqual([x["start"] for x in r["decodedRanges"]], [0x0, 0xB, 0x20, 0x28])
444
+
445
+ def test_decoded_ranges_hold_an_undecodable_start_that_a_decoded_instruction_covers(self):
446
+ # 0000 je 0004 enters the immediate of 0002 mov ax, 0F90, where 0F 04 does not decode; 0005 add al, C3.
447
+ data = bytes.fromhex("7402b8900f04c3")
448
+ r = run_report(data, config(data, targets=[4], decodedRanges=True), "reach")
449
+ self.assertEqual([(x["start"], x["end"]) for x in r["decodedRanges"]], [(0, 7)])
450
+ self.assertIn({"site": 4, "reason": "undecoded or unmapped edge"}, r["gaps"])
451
+ self.assertEqual(r["targets"][0]["insideInstruction"], 2)
452
+
453
+ def test_decoded_ranges_of_a_stopped_walk_cover_the_part_read(self):
454
+ r = reach(indirectJumps=[TABLE], decodedRanges=True, instructionLimit=5)
455
+ self.assertTrue(r["instructionLimitReached"])
456
+ self.assertEqual(sum(x["end"] - x["start"] for x in r["decodedRanges"]), 0xB + 3)
457
+
458
+ def test_decoded_ranges_overlap_where_the_walk_proves_overlapping_instructions(self):
459
+ # 0000 je 0003 falls through to the mov at 0002, whose immediate holds 0003 nop; 0004 ret. 0005 ret.
460
+ # The branch and the fall-through each prove a start, so the walk keeps both instructions.
461
+ data = bytes.fromhex("7401b890c3c3")
462
+ r = run_report(data, config(data, targets=[3, 4], decodedRanges=True), "reach")
463
+ self.assertEqual([(x["start"], x["end"]) for x in r["decodedRanges"]], [(0, 5), (3, 6)])
464
+ self.assertEqual([t["reached"] for t in r["targets"]], [True, True])
465
+ # 0000 je 0003; 0002 mov ax, 00C3 holds 0003 ret; 0005 ret. The ret at 0005 continues the run
466
+ # the mov ends, though the ret at 0003 opened a run after it.
467
+ data = bytes.fromhex("7401b8c300c3")
468
+ r = run_report(data, config(data, targets=[3, 5], decodedRanges=True), "reach")
469
+ self.assertEqual([(x["start"], x["end"]) for x in r["decodedRanges"]], [(0, 6), (3, 4)])
470
+ self.assertEqual([t["reached"] for t in r["targets"]], [True, True])
471
+
434
472
  def test_rejected_inputs(self):
435
473
  too_many = list(range(257))
436
474
  for extra, message in [({"starts": [0x18]}, "established region entry"),
@@ -461,6 +499,8 @@ class ReachTests(unittest.TestCase):
461
499
  ({"instructionControls": [0x3, 0x3]}, "distinct file offsets"),
462
500
  ({"instructionControls": [0x1000]}, "instructionControls must be an integer"),
463
501
  ({"instructionControls": [0x0]}, "instruction control 0 is a start"),
502
+ ({"targets": too_many}, r"targets must be a list of 1\.\.256"),
503
+ ({"decodedRanges": "yes"}, "decodedRanges must be a boolean"),
464
504
  ({"controls": too_many}, r"controls must be a list of 0\.\.256"),
465
505
  ({"instructionControls": too_many}, r"instructionControls must be a list of 0\.\.256")]:
466
506
  with self.subTest(extra=extra), self.assertRaisesRegex(ValueError, message):