scientific-method-engine 13.4.0__tar.gz → 13.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/PKG-INFO +3 -3
  2. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/README.md +2 -2
  3. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/pyproject.toml +1 -1
  4. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/cli.py +12 -1
  5. scientific_method_engine-13.6.0/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +122 -0
  6. scientific_method_engine-13.6.0/src/scientific_method_engine/x86/inventory.py +245 -0
  7. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/reports.py +60 -22
  8. scientific_method_engine-13.6.0/tests/test_inventory_check.py +219 -0
  9. scientific_method_engine-13.4.0/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -71
  10. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/.gitignore +0 -0
  11. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/LICENSE +0 -0
  12. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/__init__.py +0 -0
  13. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/__main__.py +0 -0
  14. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  15. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  16. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  17. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  18. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  19. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  20. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  21. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  22. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  23. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  24. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  25. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  26. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  27. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  28. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  29. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  30. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  31. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  32. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  33. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  34. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  35. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  36. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  37. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  38. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  39. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  40. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  41. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  42. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  43. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  44. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  45. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  46. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  47. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  48. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  49. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  50. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/image.py +0 -0
  51. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/loops.py +0 -0
  52. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/machine.py +0 -0
  53. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  54. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  55. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  56. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/pe.py +0 -0
  57. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/reach.py +0 -0
  58. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/relational.py +0 -0
  59. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  60. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/trace.py +0 -0
  61. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/src/scientific_method_engine/x86/values.py +0 -0
  62. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/oracle.py +0 -0
  63. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_continuation_budget.py +0 -0
  64. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_dispatch.py +0 -0
  65. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_effect_order.py +0 -0
  66. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_enter_frames.py +0 -0
  67. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_entry_frame.py +0 -0
  68. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_indirect_far_transfers.py +0 -0
  69. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_interrupt_models.py +0 -0
  70. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_loops.py +0 -0
  71. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_memory_scopes.py +0 -0
  72. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_nested_frame_request.py +0 -0
  73. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_oracle.py +0 -0
  74. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_pe.py +0 -0
  75. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_reach.py +0 -0
  76. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_relational_controls.py +0 -0
  77. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_return_checks.py +0 -0
  78. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_successor_rule.py +0 -0
  79. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_table_continuations.py +0 -0
  80. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_term_limit.py +0 -0
  81. {scientific_method_engine-13.4.0 → scientific_method_engine-13.6.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 13.4.0
3
+ Version: 13.6.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -60,7 +60,7 @@ Reading code and data:
60
60
  | Script | Arguments | Prints |
61
61
  |---|---|---|
62
62
  | `ReportInstructionContext` | one or more instruction addresses | a bounded instruction window around the instruction containing each address; the header names the requested address when it is inside that instruction |
63
- | `ReportInstructionWindow` | address where an instruction starts, instruction count (1..200) | a header, the instructions from the address onward with a `gap:` line wherever the listing skips bytes, then the count printed or where the listing ended. An address inside an instruction, in data, in undisassembled bytes or outside memory prints an error naming what is there and the next instruction start, and no window |
63
+ | `ReportInstructionWindow` | address where an instruction starts, instruction count (1..200) | a header, the instructions from the address onward with a `gap:` line wherever the listing skips bytes, a `span:` line for each run of instructions with no gap (its half-open range ending after the last instruction's final byte, byte count, last byte and instruction count, or that it reaches the end of its address space and has no exclusive end; the next instruction's start when the count closed the span and the run goes on; each instruction whose length the listing overrides, with the bytes it decodes), then the count printed or where the listing ended. An address inside an instruction, in data, in undisassembled bytes or outside memory prints an error naming what is there and the next instruction start, and no window |
64
64
  | `ReportDataBytes` | address, byte count (1..256) | the bytes at the address |
65
65
  | `ReportFunctionSummary` | one or more addresses | focused decompiler output of each containing function, its body ranges and each call without a fall-through (a sign of a wrong no-return flag), then the addresses with no function or a failed decompile |
66
66
  | `ReportDecompileWindow` | address, first line (1-based), line count (a count above 160 is cut to 160) | a window of one function's decompilation, its total line count and where the next window starts |
@@ -91,7 +91,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
91
91
  | Script | Arguments | Writes |
92
92
  |---|---|---|
93
93
  | `ExportBoundedFlow` | entry where an instruction starts, instruction limit (1..10000, the most instruction records exported), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON, with `limitReached` (the walk stopped at the limit with flow left unread) and `noInstruction` (flow targets where no instruction starts). An entry where no instruction starts fails the script and writes nothing |
94
- | `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
94
+ | `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size, the inventory `inventory-check` compares call targets with |
95
95
  | `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override), `fallsThrough` (whether Ghidra continues to the next instruction at the site, after any fall-through override) and `fallsThroughTo`/`fallsThroughToAddress` (where a fall-through override sends Ghidra instead, or null), as the `ghidraCallEdges` input of `callees` |
96
96
  | `ExportFunctionFingerprints` | output TSV path (replaced only when the export completes) | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
97
97
 
@@ -44,7 +44,7 @@ Reading code and data:
44
44
  | Script | Arguments | Prints |
45
45
  |---|---|---|
46
46
  | `ReportInstructionContext` | one or more instruction addresses | a bounded instruction window around the instruction containing each address; the header names the requested address when it is inside that instruction |
47
- | `ReportInstructionWindow` | address where an instruction starts, instruction count (1..200) | a header, the instructions from the address onward with a `gap:` line wherever the listing skips bytes, then the count printed or where the listing ended. An address inside an instruction, in data, in undisassembled bytes or outside memory prints an error naming what is there and the next instruction start, and no window |
47
+ | `ReportInstructionWindow` | address where an instruction starts, instruction count (1..200) | a header, the instructions from the address onward with a `gap:` line wherever the listing skips bytes, a `span:` line for each run of instructions with no gap (its half-open range ending after the last instruction's final byte, byte count, last byte and instruction count, or that it reaches the end of its address space and has no exclusive end; the next instruction's start when the count closed the span and the run goes on; each instruction whose length the listing overrides, with the bytes it decodes), then the count printed or where the listing ended. An address inside an instruction, in data, in undisassembled bytes or outside memory prints an error naming what is there and the next instruction start, and no window |
48
48
  | `ReportDataBytes` | address, byte count (1..256) | the bytes at the address |
49
49
  | `ReportFunctionSummary` | one or more addresses | focused decompiler output of each containing function, its body ranges and each call without a fall-through (a sign of a wrong no-return flag), then the addresses with no function or a failed decompile |
50
50
  | `ReportDecompileWindow` | address, first line (1-based), line count (a count above 160 is cut to 160) | a window of one function's decompilation, its total line count and where the next window starts |
@@ -75,7 +75,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
75
75
  | Script | Arguments | Writes |
76
76
  |---|---|---|
77
77
  | `ExportBoundedFlow` | entry where an instruction starts, instruction limit (1..10000, the most instruction records exported), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON, with `limitReached` (the walk stopped at the limit with flow left unread) and `noInstruction` (flow targets where no instruction starts). An entry where no instruction starts fails the script and writes nothing |
78
- | `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
78
+ | `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size, the inventory `inventory-check` compares call targets with |
79
79
  | `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override), `fallsThrough` (whether Ghidra continues to the next instruction at the site, after any fall-through override) and `fallsThroughTo`/`fallsThroughToAddress` (where a fall-through override sends Ghidra instead, or null), as the `ghidraCallEdges` input of `callees` |
80
80
  | `ExportFunctionFingerprints` | output TSV path (replaced only when the export completes) | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
81
81
 
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "13.4.0"
8
+ version = "13.6.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -14,7 +14,7 @@ PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
14
14
  DECODER = "capstone " + capstone.__version__
15
15
  INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
16
16
  USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|reach|trace|uses|arguments|"
17
- "effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
17
+ "effects|returns|memory|incoming|inventory-check|call-order|guards|allocation|dispatch> <config.json|->\n"
18
18
  "effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
19
19
  "callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges;\n"
20
20
  "each row it compares fall-through at carries the engine's side as engineReadsOn;\n"
@@ -24,6 +24,9 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
24
24
  "reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
25
25
  "and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
26
26
  "a leaves routine is reached but not read, and its reason is repeated in the report.\n"
27
+ "inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
28
+ "that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
29
+ "with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it.\n"
27
30
  "trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
28
31
  "a violated control fails the report; an undecided one is reported and never counts as held.\n"
29
32
  "A lastWriter control with an address inspects that memory at its checkpoint anchors without a read.\n"
@@ -83,6 +86,14 @@ def main(argv):
83
86
  "scientific-method-engine releases.")
84
87
  elif "preparedProtocol" in config:
85
88
  raise ValueError("preparedProtocol is set by the reader and cannot be supplied")
89
+ if "inventory" in config:
90
+ inventory = config["inventory"]
91
+ if not isinstance(inventory, str) or not inventory:
92
+ raise ValueError("inventory must name a function inventory TSV file")
93
+ if config_path == "-" and not Path(inventory).is_absolute():
94
+ # The reader resolves inventory against the config's directory; one that does not would send it unresolved.
95
+ raise ValueError("The reader sent a relative inventory path; install a matching @scientific-method/executable-reader")
96
+ config["inventory"] = str(Path(base).absolute() / inventory)
86
97
  data, identity = read_source(config, base)
87
98
  result = run_report(data, config, command)
88
99
  report = {"schema": "bounded-x86-v1", "decoder": DECODER, "instructionSemantics": INSTRUCTION_SEMANTICS,
@@ -0,0 +1,122 @@
1
+ // Prints a bounded instruction window that begins at an instruction start.
2
+ // @category Restoration
3
+
4
+ import java.util.ArrayList;
5
+ import java.util.List;
6
+
7
+ import ghidra.app.script.GhidraScript;
8
+ import ghidra.program.model.address.Address;
9
+ import ghidra.program.model.listing.Instruction;
10
+
11
+ import scientificmethod.InstructionStart;
12
+
13
+ public class ReportInstructionWindow extends GhidraScript {
14
+ private static final int MAX_INSTRUCTIONS = 200;
15
+
16
+ @Override
17
+ protected void run() throws Exception {
18
+ String[] arguments = getScriptArgs();
19
+ if (arguments.length != 2) {
20
+ printerr("Supply a virtual address and instruction count.");
21
+ return;
22
+ }
23
+
24
+ Address address = toAddr(arguments[0]);
25
+ if (address == null) {
26
+ printerr(arguments[0] + ": not an address in this program");
27
+ return;
28
+ }
29
+ int count;
30
+ try {
31
+ count = Integer.parseInt(arguments[1]);
32
+ } catch (NumberFormatException exception) {
33
+ count = 0;
34
+ }
35
+ if (count < 1 || count > MAX_INSTRUCTIONS) {
36
+ printerr("Instruction count must be between 1 and " + MAX_INSTRUCTIONS + ".");
37
+ return;
38
+ }
39
+
40
+ // The window starts only where an instruction starts. Starting at the next instruction
41
+ // instead would print a window that looks like the requested one but is not.
42
+ String missing = InstructionStart.missingStart(currentProgram, address);
43
+ if (missing != null) {
44
+ printerr(missing + " No window printed.");
45
+ return;
46
+ }
47
+
48
+ println("===== up to " + count + " instructions from " + address + " =====");
49
+ Instruction instruction = currentProgram.getListing().getInstructionAt(address);
50
+ int printed = 0;
51
+ // Each run of printed instructions with no gap between them, closed at a gap and at the
52
+ // window's last instruction.
53
+ List<String> spans = new ArrayList<>();
54
+ Address spanStart = null;
55
+ int spanInstructions = 0;
56
+ // The instructions in the current span whose listed length the listing overrides.
57
+ List<String> overrides = new ArrayList<>();
58
+ while (instruction != null && printed < count) {
59
+ if (spanStart == null) {
60
+ spanStart = instruction.getAddress();
61
+ }
62
+ println(instruction.getAddress() + ": " + instruction);
63
+ printed++;
64
+ spanInstructions++;
65
+ if (instruction.isLengthOverridden()) {
66
+ overrides.add(instruction.getAddress() + " (listed as " + instruction.getLength() + " of the "
67
+ + instruction.getParsedLength() + " bytes it decodes)");
68
+ }
69
+ Instruction next = instruction.getNext();
70
+ // Address.next() is null when the instruction ends its address space, so a gap is
71
+ // measured from the instruction's last byte, and such a span has no exclusive end.
72
+ Address last = instruction.getMaxAddress();
73
+ Address end = last.next();
74
+ boolean adjacent = end != null && next != null && next.getAddress().equals(end);
75
+ if (printed == count || !adjacent) {
76
+ spans.add(span(spanStart, last, end, spanInstructions, overrides, adjacent));
77
+ spanStart = null;
78
+ spanInstructions = 0;
79
+ overrides.clear();
80
+ if (printed < count && next != null) {
81
+ if (end == null) {
82
+ println("gap: no instruction after " + last + " up to " + next.getAddress());
83
+ } else {
84
+ println("gap: no instruction from " + end + " up to " + next.getAddress());
85
+ }
86
+ }
87
+ }
88
+ instruction = next;
89
+ }
90
+ for (String line : spans) {
91
+ println(line);
92
+ }
93
+ if (printed < count) {
94
+ println("The listing ends after " + printed + " of " + count + " instructions.");
95
+ } else {
96
+ println("Printed " + printed + " instructions.");
97
+ }
98
+ }
99
+
100
+ // A span as a half-open range with its last byte, so a range copied from it ends after the
101
+ // final byte of its last instruction. Past the end of an address space no address can end it.
102
+ // A span the window's count closed says when the next instruction starts at its end, since
103
+ // the run goes on past the window. An instruction whose length the listing overrides decodes
104
+ // bytes past where the next instruction or the span ends, so the span names it.
105
+ private static String span(Address first, Address last, Address end, int instructions,
106
+ List<String> overrides, boolean continues) {
107
+ long bytes = last.subtract(first) + 1;
108
+ String range = end == null ? first + " to the end of its address space" : first + ".." + end;
109
+ StringBuilder line = new StringBuilder("span: " + range + " (" + bytes + (bytes == 1 ? " byte" : " bytes") + ", last byte " + last
110
+ + ", " + instructions + (instructions == 1 ? " instruction)" : " instructions)"));
111
+ if (end == null) {
112
+ line.append("; no address follows its last byte, so it has no exclusive end");
113
+ }
114
+ if (continues) {
115
+ line.append("; the window stops here and the next instruction starts at ").append(end);
116
+ }
117
+ for (String override : overrides) {
118
+ line.append("; the listing overrides the length of the instruction at ").append(override);
119
+ }
120
+ return line.toString();
121
+ }
122
+ }
@@ -0,0 +1,245 @@
1
+ """Resolved direct call targets compared with a committed function inventory."""
2
+ import re
3
+ from bisect import bisect_right
4
+ from pathlib import Path
5
+ from .image import integer
6
+
7
+ MAX_INVENTORY = 16 * 1024 * 1024
8
+ MAX_ROWS = 200000
9
+ COLUMNS = ("start", "size", "name", "out_of_scope", "ranges")
10
+ SEGMENTED = re.compile(r"[0-9A-F]{4}:[0-9A-F]{4}")
11
+ FLAT = re.compile(r"0x[0-9A-F]{8}")
12
+ OFFSET = re.compile(r"0x[0-9A-F]{2,}")
13
+
14
+ START = "inventory start"
15
+ INSIDE = "inside another row's body"
16
+ OUTSIDE = "outside every row"
17
+ UNPLACED = "outside declared code"
18
+ # The evidence of a target, by the rank of its best calling site.
19
+ EVIDENCE = ("entry-path call", "contested call only", "raw byte candidate only")
20
+
21
+
22
+ def _place(value, image):
23
+ """``(space, number)`` for an inventory start or range end, or None when it is not in the notation.
24
+
25
+ Flat32 code is placed by its virtual address. Segmented code is placed by ``segment * 16 + offset``,
26
+ so two spellings of one byte compare equal, or by a ``0x`` file offset, the form the work protocol
27
+ gives overlay code outside the load image.
28
+ """
29
+ if image.flat:
30
+ return ("address", int(value, 16)) if FLAT.fullmatch(value) else None
31
+ if SEGMENTED.fullmatch(value):
32
+ segment, offset = value.split(":")
33
+ return "address", int(segment, 16) * 16 + int(offset, 16)
34
+ return ("offset", int(value, 16)) if OFFSET.fullmatch(value) else None
35
+
36
+
37
+ def read_inventory(path, image):
38
+ """The rows of a function inventory TSV (``start``, ``size``, then optionally ``name``, ``out_of_scope``
39
+ and ``ranges``), each with its ``space``, ``at`` and half-open ``body`` ranges. Raises ``ValueError``
40
+ naming the line of the first row that does not parse. ``path`` must be absolute: the CLI resolves a
41
+ config's relative ``inventory`` against the config file's directory, and a library caller resolves
42
+ it the same way, so no path is read against the process's working directory."""
43
+ if not isinstance(path, str) or not path:
44
+ raise ValueError("inventory must name a function inventory TSV file")
45
+ file = Path(path)
46
+ if not file.is_absolute():
47
+ raise ValueError("inventory must be an absolute path; resolve it against the config file's directory")
48
+ if not file.is_file() or file.stat().st_size > MAX_INVENTORY:
49
+ raise ValueError("inventory must be a regular file of at most 16 MiB")
50
+ text = re.sub(r"(?:\r?\n)+$", "", file.read_text(encoding="utf-8").lstrip(""))
51
+ lines = re.split(r"\r?\n", text)
52
+ columns = lines[0].split("\t")
53
+ if columns[:2] != ["start", "size"] or len(set(columns)) != len(columns) or not set(columns) <= set(COLUMNS):
54
+ raise ValueError("inventory columns are start and size, then optionally name, out_of_scope and ranges, each once")
55
+ if len(lines) - 1 > MAX_ROWS:
56
+ raise ValueError(f"inventory holds more than {MAX_ROWS} rows")
57
+ rows, starts = [], set()
58
+ for number, line in enumerate(lines[1:], start=2):
59
+ def bad(why):
60
+ return ValueError(f"inventory line {number}: {why}")
61
+ cells = line.split("\t")
62
+ if len(cells) != len(columns):
63
+ raise bad(f"has {len(cells)} cells, not {len(columns)}")
64
+ cell = {name: value.strip() for name, value in zip(columns, cells)}
65
+ if not re.fullmatch(r"[1-9][0-9]*", cell["size"]):
66
+ raise bad(f"size {cell['size']} is not a positive number of bytes")
67
+ size = int(cell["size"])
68
+ place = _place(cell["start"], image)
69
+ if place is None:
70
+ raise bad(f"start {cell['start']} is not an address in the notation of a "
71
+ f"{'flat32' if image.flat else 'segmented16'} image")
72
+ body = [(place[1], place[1] + size)]
73
+ if cell.get("ranges"):
74
+ body = []
75
+ for item in cell["ranges"].split():
76
+ ends = item.split("..")
77
+ low, high = (_place(e, image) for e in ends) if len(ends) == 2 else (None, None)
78
+ if low is None or high is None or low[0] != place[0] or high[0] != place[0] or high[1] <= low[1]:
79
+ raise bad(f"range {item} is not a half-open range start..end in the notation of its start")
80
+ body.append((low[1], high[1]))
81
+ body.sort()
82
+ if any(body[i][0] < body[i - 1][1] for i in range(1, len(body))):
83
+ raise bad("the ranges overlap")
84
+ if sum(end - begin for begin, end in body) != size:
85
+ raise bad(f"size {size} is not the total of the ranges")
86
+ if not any(begin <= place[1] < end for begin, end in body):
87
+ raise bad(f"start {cell['start']} lies in none of the ranges")
88
+ if place in starts:
89
+ raise bad(f"start {cell['start']} is listed twice")
90
+ starts.add(place)
91
+ row = {"start": cell["start"], "size": size, "space": place[0], "at": place[1], "body": body}
92
+ if cell.get("name"):
93
+ row["name"] = cell["name"]
94
+ rows.append(row)
95
+ return rows
96
+
97
+
98
+ def _target_place(image, at):
99
+ """``(space, number, written)`` for a call target's file offset, in the notation an inventory uses.
100
+
101
+ Flat32 code is placed by its virtual address, a region with a ``container`` (an overlay view the
102
+ reader declared) by its file offset, and any other segmented region by its ``segment:ip`` mapping.
103
+ None when no declared region holds the target.
104
+ """
105
+ r = image.region(at)
106
+ if r is None:
107
+ return None
108
+ ip = r["ip"] + at - r["start"]
109
+ if image.flat:
110
+ return "address", ip, f"0x{ip:08X}"
111
+ if r.get("container") is not None:
112
+ return "offset", at, f"0x{at:02X}"
113
+ return "address", r["segment"] * 16 + ip, f"{r['segment']:04X}:{ip:04X}"
114
+
115
+
116
+ def _domains(image):
117
+ """The ``(space, start, end)`` each declared region covers in an inventory's notation."""
118
+ for r in image.regions:
119
+ kind, low, _ = _target_place(image, r["start"])
120
+ yield kind, low, low + r["end"] - r["start"]
121
+
122
+
123
+ def _count(n, one, many):
124
+ """``n`` followed by the singular or the plural wording."""
125
+ return f"{n} {one if n == 1 else many}"
126
+
127
+
128
+ def inventory_check(image, config):
129
+ """Every resolved direct call target in the searched regions, classified against a function inventory.
130
+
131
+ The calls are those ``incoming`` reads (``direct_calls``). Each distinct target is an inventory
132
+ start, inside the body of another row, outside every row, or outside declared code, and is
133
+ reported with one calling site, near or far, and the evidence of its best site: a call on the
134
+ entry path, a contested instruction, or only a raw byte candidate.
135
+ """
136
+ from .reports import call_controls, direct_calls, search_coverage
137
+ if "inventory" not in config:
138
+ raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
139
+ rows = read_inventory(config["inventory"], image)
140
+ limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
141
+ calls = direct_calls(image, config)
142
+ seen, contested = calls["seen"], calls["contested"]
143
+
144
+ starts = {(row["space"], row["at"]) for row in rows}
145
+ pieces = sorted((row["space"], begin, end, i) for i, row in enumerate(rows) for begin, end in row["body"])
146
+ keys = [(space, begin) for space, begin, _, _ in pieces]
147
+ # reach[i] is the furthest end of pieces[0..i] in pieces[i]'s space, so a lookup stops at the first
148
+ # piece before which nothing reaches the target, however long one row is.
149
+ reach = []
150
+ for i, (space, _, end, _) in enumerate(pieces):
151
+ reach.append(max(end, reach[-1]) if i and pieces[i - 1][0] == space else end)
152
+
153
+ def owners(space, at):
154
+ found, i = [], bisect_right(keys, (space, at)) - 1
155
+ while i >= 0 and pieces[i][0] == space and reach[i] > at:
156
+ if pieces[i][1] <= at < pieces[i][2]:
157
+ found.append(pieces[i][3])
158
+ i -= 1
159
+ return sorted(set(found))
160
+
161
+ domains = list(_domains(image))
162
+ outside_code = [row["start"] for row in rows
163
+ if not any(kind == row["space"] and low <= row["at"] < high for kind, low, high in domains)]
164
+
165
+ by_target, unresolved = {}, []
166
+ for row in calls["rows"]:
167
+ if row["target"] is None:
168
+ unresolved.append(row)
169
+ else:
170
+ by_target.setdefault(row["target"], []).append(row)
171
+
172
+ def rank(row):
173
+ return 0 if row["site"] in seen else 1 if row["site"] in contested else 2
174
+ tally = {label: {"targets": 0, "inventoryStarts": 0, "insideAnotherRow": 0, "outsideEveryRow": 0,
175
+ "outsideDeclaredCode": 0} for label in EVIDENCE}
176
+ field = {START: "inventoryStarts", INSIDE: "insideAnotherRow", OUTSIDE: "outsideEveryRow", UNPLACED: "outsideDeclaredCode"}
177
+ missing = []
178
+ for target, sites in sorted(by_target.items()):
179
+ sites.sort(key=lambda row: (rank(row), row["site"]))
180
+ best = sites[0]
181
+ evidence = EVIDENCE[rank(best)]
182
+ placed = _target_place(image, target)
183
+ held = []
184
+ if placed is None:
185
+ status = UNPLACED
186
+ elif placed[:2] in starts:
187
+ status = START
188
+ else:
189
+ held = owners(*placed[:2])
190
+ status = INSIDE if held else OUTSIDE
191
+ tally[evidence]["targets"] += 1
192
+ tally[evidence][field[status]] += 1
193
+ if status == START:
194
+ continue
195
+ entry = {"target": target, "address": placed[2] if placed else None, "status": status, "evidence": evidence,
196
+ "site": best["site"], "call": "far" if best["encoding"] == "lcall" else "near",
197
+ "siteClassification": best["classification"], "region": best["region"], "provenance": best["provenance"],
198
+ "callSites": {"entryPath": sum(rank(r) == 0 for r in sites), "contested": sum(rank(r) == 1 for r in sites),
199
+ "rawCandidates": sum(rank(r) == 2 for r in sites)}}
200
+ if held:
201
+ entry["rows"] = [{k: rows[i][k] for k in ("start", "size", "name") if k in rows[i]} for i in held]
202
+ missing.append(entry)
203
+
204
+ controls = call_controls(calls, config)
205
+ coverage = search_coverage(image, calls["read"])
206
+ partial = any(c["partial"] for c in coverage)
207
+ path = tally["entry-path call"]
208
+ lacking = path["insideAnotherRow"] + path["outsideEveryRow"]
209
+ # A target no declared region places is compared with no row, so it is left out of the comparison's total.
210
+ compared = path["targets"] - path["outsideDeclaredCode"]
211
+ others = sum(t["insideAnotherRow"] + t["outsideEveryRow"] for label, t in tally.items() if label != "entry-path call")
212
+ summary = (f"{lacking} of the {_count(compared, 'call target that entry-path calls resolve to is', 'call targets that entry-path calls resolve to are')} "
213
+ f"not inventory starts: {path['insideAnotherRow']} inside another row's body, "
214
+ f"{path['outsideEveryRow']} outside every row.")
215
+ if path["outsideDeclaredCode"]:
216
+ summary += " " + _count(path["outsideDeclaredCode"],
217
+ "more entry-path call target lies outside declared code and is not compared with the inventory.",
218
+ "more entry-path call targets lie outside declared code and are not compared with the inventory.")
219
+ if others:
220
+ summary += f" {_count(others, 'more comes', 'more come')} only from contested instructions or raw byte candidates."
221
+ if unresolved:
222
+ reached = sum(row["site"] in seen for row in unresolved)
223
+ summary += (f" {_count(len(unresolved), 'call in the searched regions is', 'calls in the searched regions are')} "
224
+ f"unresolved, {reached} of them on the entry path.")
225
+ stopped = {g.get("reason") for g in calls["gaps"]}
226
+ if partial:
227
+ summary += " The search is partial."
228
+ if "instruction limit" in stopped:
229
+ summary += " The entry-path walk stopped at its instruction limit, so it can reach more calls."
230
+ summary += " Calls the search does not resolve can add targets."
231
+ return {"inventory": {"path": config["inventory"], "rows": len(rows)},
232
+ "targets": missing[:limit], "unresolved": unresolved[:limit],
233
+ "rowsOutsideDeclaredCode": outside_code[:limit],
234
+ "counts": {"callTargets": len(by_target), **tally, "unresolvedCalls": len(unresolved),
235
+ "inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code)},
236
+ "truncated": any(len(x) > limit for x in (missing, unresolved, outside_code)),
237
+ "summary": summary, "controls": controls,
238
+ "searched": [r for r in image.regions if r["name"] in calls["scans"]], "coverage": coverage,
239
+ "partialSearch": partial, "gaps": calls["gaps"],
240
+ "exclusions": ["computed call targets (listed in unresolved when the walk reaches them)", "unrelocated far calls",
241
+ "undeclared mappings", "prefix-started raw candidates off the entry path", "jump targets"],
242
+ "interpretation": "Each distinct resolved direct call target in the searched regions, placed in the inventory's "
243
+ "notation and compared with its rows. A target only raw byte candidates or contested "
244
+ "instructions call is not shown to be code. The counts are lower bounds on the targets "
245
+ "the inventory lacks: calls the search does not resolve can add more."}
@@ -63,7 +63,7 @@ def search_coverage(image, spans):
63
63
  holders = [{"view": "section " + section["name"], "start": section["rawStart"],
64
64
  "end": section["rawStart"] + section["loadedRawSize"]}]
65
65
  if not holders:
66
- alone.append(r["name"])
66
+ alone.append(r)
67
67
  for c in holders:
68
68
  containers.setdefault((c["view"], c["start"], c["end"]), []).append(r["name"])
69
69
  rows = []
@@ -74,18 +74,31 @@ def search_coverage(image, spans):
74
74
  "meaning": "partial search: callers in the unsearched ranges are not covered" if missing else
75
75
  "the searched regions cover the complete container"})
76
76
  if alone:
77
- rows.append({"container": None, "regions": alone, "partial": False,
78
- "meaning": "no overlay, section or declared segment contains these regions; the search covers them only"})
77
+ # With no container to compare with, each region is compared with its own extent, so a scan that
78
+ # scanLimit stopped is reported as partial.
79
+ missing = [gap for r in alone for gap in uncovered(r["start"], r["end"], [spans[r["name"]]])]
80
+ rows.append({"container": None, "regions": [r["name"] for r in alone], "unsearched": missing,
81
+ "partial": bool(missing),
82
+ "meaning": "partial search: no overlay, section or declared segment contains these regions, "
83
+ "and callers in their unsearched ranges are not covered" if missing else
84
+ "no overlay, section or declared segment contains these regions; the search covers them only"})
79
85
  return rows
80
86
 
81
87
 
82
- def incoming(image, config):
83
- target = integer(config.get("target"), 0, len(image.data) - 1, "target")
84
- if image.region(target) is None:
85
- raise ValueError("Incoming target is outside declared code")
86
- limit = integer(config.get("limit", 100), 1, 10000, "result limit")
88
+ def direct_calls(image, config):
89
+ """Every direct call site in the searched regions, as ``incoming`` and ``inventory-check`` read them.
90
+
91
+ Walks the entry-path CFG from every established entry, then scans every byte of the regions
92
+ ``searchRegions`` names (all regions by default) for E8 and 9A call starts, and adds each
93
+ reached call the scan cannot see (one that starts with a prefix). Returns a dict with the walk's
94
+ ``seen``, ``gaps`` (with a ``raw scan limit`` gap where ``scanLimit`` stopped a region),
95
+ ``edges``, ``undecoded`` and ``contested``; ``rows``, every call row in the order it was read,
96
+ each with its ``target`` (None when unresolved); ``scanned``, those rows by site, leaving out
97
+ the reached calls whose frame encoding the walk does not model; ``scans``, the region names
98
+ searched; and ``read``, the byte span the scan read in each.
99
+ """
87
100
  seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000))
88
- hits, candidates, scanned, partial, disputed = [], [], {}, [], []
101
+ rows, scanned = [], {}
89
102
 
90
103
  def classify(at):
91
104
  if at in seen:
@@ -117,10 +130,7 @@ def incoming(image, config):
117
130
  row = {"site": at, "target": resolved, "encoding": ins.mnemonic,
118
131
  "classification": classify(at), "provenance": provenance, "region": name}
119
132
  scanned[at] = row
120
- if resolved == target:
121
- (hits if at in seen else disputed if at in contested else candidates).append(row)
122
- elif resolved is None:
123
- partial.append(row)
133
+ rows.append(row)
124
134
  for at, ins in sorted({**seen, **contested}.items()):
125
135
  if at in scanned or ins.mnemonic not in ("call", "lcall"):
126
136
  continue
@@ -129,29 +139,52 @@ def incoming(image, config):
129
139
  continue
130
140
  # A reached call can start with a prefix, so the raw E8/9A scan above never sees it.
131
141
  if unsupported_transfer(image, ins):
132
- partial.append({"site": at, "target": None, "encoding": ins.mnemonic, "region": region["name"],
133
- "classification": "unsupported control-transfer frame encoding"})
142
+ rows.append({"site": at, "target": None, "encoding": ins.mnemonic, "region": region["name"],
143
+ "classification": "unsupported control-transfer frame encoding"})
134
144
  continue
135
145
  resolved, provenance = call_target(image, at, ins)
136
146
  row = {"site": at, "target": resolved, "encoding": ins.mnemonic,
137
147
  "classification": classify(at), "provenance": provenance, "region": region["name"]}
138
148
  scanned[at] = row
139
- if resolved == target:
140
- (hits if at in seen else disputed).append(row)
141
- elif resolved is None:
142
- partial.append(row)
149
+ rows.append(row)
143
150
  for edge in edges:
144
151
  if edge.get("overlappingTarget") and edge["site"] in scanned:
145
152
  scanned[edge["site"]]["overlappingTarget"] = True
146
153
  scanned[edge["site"]]["boundaryEvidence"] = edge["boundaryEvidence"]
147
- hits.sort(key=lambda row: row["site"])
148
- disputed.sort(key=lambda row: row["site"])
154
+ return {"seen": seen, "gaps": gaps, "edges": edges, "undecoded": undecoded, "contested": contested,
155
+ "rows": rows, "scanned": scanned, "scans": scans, "read": read}
156
+
157
+
158
+ def call_controls(calls, config):
159
+ """The rows of the ``controls`` call sites in a ``direct_calls`` result. Raises ``ValueError`` when a
160
+ control is not an entry-path call the scan read with a resolved target."""
149
161
  controls = config.get("controls", [])
150
162
  if not isinstance(controls, list) or len(controls) > 256:
151
163
  raise ValueError("Invalid positive controls")
164
+ scanned, seen = calls["scanned"], calls["seen"]
152
165
  for at in controls:
153
166
  if type(at) is not int or at not in scanned or at not in seen or scanned[at]["target"] is None:
154
167
  raise ValueError(f"Positive control {at} missed or not verified")
168
+ return [scanned[at] for at in controls]
169
+
170
+
171
+ def incoming(image, config):
172
+ target = integer(config.get("target"), 0, len(image.data) - 1, "target")
173
+ if image.region(target) is None:
174
+ raise ValueError("Incoming target is outside declared code")
175
+ limit = integer(config.get("limit", 100), 1, 10000, "result limit")
176
+ calls = direct_calls(image, config)
177
+ seen, gaps, edges, undecoded, contested = (calls[k] for k in ("seen", "gaps", "edges", "undecoded", "contested"))
178
+ scans, read = calls["scans"], calls["read"]
179
+ hits, candidates, partial, disputed = [], [], [], []
180
+ for row in calls["rows"]:
181
+ if row["target"] is None:
182
+ partial.append(row)
183
+ elif row["target"] == target:
184
+ (hits if row["site"] in seen else disputed if row["site"] in contested else candidates).append(row)
185
+ hits.sort(key=lambda row: row["site"])
186
+ disputed.sort(key=lambda row: row["site"])
187
+ controls = call_controls(calls, config)
155
188
  truncated = len(hits) + len(candidates) + len(disputed) + len(partial) > limit
156
189
  budget = limit
157
190
  def bounded(rows):
@@ -190,7 +223,7 @@ def incoming(image, config):
190
223
  return {"target": target, "sections": {k: v[:limit] for k, v in sections.items()}, "confirmed": bounded(hits), "candidates": bounded(candidates),
191
224
  "contested": bounded(disputed), "unresolved": bounded(partial),
192
225
  "counts": {"confirmed": len(hits), "candidates": len(candidates), "contested": len(disputed), "unresolved": len(partial)},
193
- "truncated": truncated, "controls": [scanned[at] for at in controls],
226
+ "truncated": truncated, "controls": controls,
194
227
  "searched": [r for r in image.regions if r["name"] in scans], "coverage": coverage, "partialSearch": partial_scope,
195
228
  "unresolvedTransfers": sorted(transfers, key=lambda t: t["site"]), "undecodedRanges": undecoded, "gaps": gaps,
196
229
  "negativeUsable": bool(controls) and not (hits or candidates or disputed or partial or gaps or truncated or undecoded or partial_scope),
@@ -1722,6 +1755,8 @@ def _run_report(image, config, command):
1722
1755
  raise ValueError("controlOccurrenceLimit applies only to " + ", ".join(TRACE_COMMANDS))
1723
1756
  if "entryFrame" in config and command not in TRACE_COMMANDS:
1724
1757
  raise ValueError("entryFrame applies only to " + ", ".join(TRACE_COMMANDS))
1758
+ if "inventory" in config and command != "inventory-check":
1759
+ raise ValueError("inventory applies only to inventory-check")
1725
1760
  if command == "operand":
1726
1761
  return operand_provenance(image, config)
1727
1762
  if command == "target":
@@ -1739,6 +1774,9 @@ def _run_report(image, config, command):
1739
1774
  return call_order(image, config)
1740
1775
  if command == "incoming":
1741
1776
  return incoming(image, config)
1777
+ if command == "inventory-check":
1778
+ from .inventory import inventory_check
1779
+ return inventory_check(image, config)
1742
1780
  if command == "operand-candidates":
1743
1781
  return operand_candidates(image, config)
1744
1782
  if command == "uses":
@@ -0,0 +1,219 @@
1
+ """Synthetic machine code and synthetic inventories only. No original binaries or analysis artifacts."""
2
+ import json
3
+ import os
4
+ from pathlib import Path
5
+ import subprocess
6
+ import sys
7
+ import tempfile
8
+ import unittest
9
+
10
+ SRC = Path(__file__).resolve().parents[1] / "src"
11
+ sys.path.insert(0, str(SRC))
12
+ import xxhash
13
+ from scientific_method_engine.x86.reports import run_report
14
+
15
+ ENGINE = [sys.executable, "-B", "-m", "scientific_method_engine"]
16
+ ENGINE_ENV = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(SRC), os.environ.get("PYTHONPATH")]))}
17
+
18
+ # One resident segment at 1000h. 0000 calls 0006, 0003 calls 000B; 0007..000A is padding.
19
+ CALLS = bytes.fromhex(
20
+ "e80300" # 0000 call 0006
21
+ "e80500" # 0003 call 000B
22
+ "c3" # 0006 ret
23
+ "90909090" # 0007 padding
24
+ "c3" # 000B ret
25
+ )
26
+ # CALLS, then bytes no entry path reaches: 000C calls 000F, which no inventory row starts at.
27
+ RAW = CALLS + bytes.fromhex("e80000" "c3")
28
+
29
+
30
+ def config(data=CALLS, regions=None, **extra):
31
+ regions = regions or [{"name": "resident", "start": 0, "end": len(data), "ip": 0, "segment": 0x1000,
32
+ "resident": True, "entries": [0], "evidence": "synthetic declared code extent"}]
33
+ return {"regions": regions, **extra}
34
+
35
+
36
+ class InventoryCheckTests(unittest.TestCase):
37
+ def setUp(self):
38
+ folder = tempfile.TemporaryDirectory()
39
+ self.addCleanup(folder.cleanup)
40
+ self.root = Path(folder.name)
41
+
42
+ def inventory(self, text, name="inventory.tsv"):
43
+ path = self.root / name
44
+ path.write_text(text, encoding="utf-8", newline="")
45
+ return str(path)
46
+
47
+ def check(self, text, data=CALLS, **extra):
48
+ return run_report(data, config(data, inventory=self.inventory(text), **extra), "inventory-check")
49
+
50
+ def test_a_call_target_past_every_row_is_outside_and_names_its_near_calling_site(self):
51
+ r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n", controls=[0, 3])
52
+ self.assertEqual(r["targets"], [{
53
+ "target": 0x0B, "address": "1000:000B", "status": "outside every row", "evidence": "entry-path call",
54
+ "site": 3, "call": "near", "siteClassification": "entry-path instruction", "region": "resident",
55
+ "provenance": r["targets"][0]["provenance"], "callSites": {"entryPath": 1, "contested": 0, "rawCandidates": 0}}])
56
+ self.assertEqual(r["targets"][0]["provenance"]["encoding"], "relative16")
57
+ path = r["counts"]["entry-path call"]
58
+ self.assertEqual((path["targets"], path["inventoryStarts"], path["outsideEveryRow"], path["insideAnotherRow"]), (2, 1, 1, 0))
59
+ self.assertEqual(r["counts"]["callTargets"], 2)
60
+ self.assertTrue(r["summary"].startswith("1 of the 2 call targets that entry-path calls resolve to are not inventory "
61
+ "starts: 0 inside another row's body, 1 outside every row."))
62
+ self.assertEqual([c["site"] for c in r["controls"]], [0, 3])
63
+ self.assertFalse(r["partialSearch"])
64
+ self.assertFalse(r["truncated"])
65
+
66
+ def test_a_target_inside_another_rows_body_names_that_row_and_an_alias_spelling_is_a_start(self):
67
+ # 0FFF:0016 is the byte 1000:0006 names, so 0006 is a start and 000B lies in that row's six bytes.
68
+ r = self.check("start\tsize\tname\n1000:0000\t6\tfirst\n0FFF:0016\t6\tsecond\n")
69
+ self.assertEqual([(t["target"], t["status"], t["rows"]) for t in r["targets"]],
70
+ [(0x0B, "inside another row's body", [{"start": "0FFF:0016", "size": 6, "name": "second"}])])
71
+ self.assertEqual(r["counts"]["entry-path call"]["insideAnotherRow"], 1)
72
+
73
+ def test_a_long_row_before_the_target_still_names_every_row_whose_body_holds_it(self):
74
+ # The long row 0FFF:0010 covers 0000..000F; the short rows after it start past 000B or end before it.
75
+ r = self.check("start\tsize\tname\n0FFF:0010\t16\tlong\n1000:0001\t2\tshort\n1000:0006\t1\tret\n1000:0009\t3\tholder\n")
76
+ self.assertEqual([(t["target"], [row["name"] for row in t["rows"]]) for t in r["targets"]],
77
+ [(0x0B, ["long", "holder"])])
78
+
79
+ def test_a_ranges_column_gives_the_body_and_its_ends_are_exclusive(self):
80
+ inside = self.check("start\tsize\tranges\n1000:0000\t6\t\n1000:0006\t2\t1000:0006..1000:0007 1000:000B..1000:000C\n")
81
+ self.assertEqual([(t["target"], t["status"]) for t in inside["targets"]], [(0x0B, "inside another row's body")])
82
+ past = self.check("start\tsize\tranges\n1000:0000\t6\t\n1000:0006\t2\t1000:0006..1000:0007 1000:000A..1000:000B\n")
83
+ self.assertEqual([(t["target"], t["status"]) for t in past["targets"]], [(0x0B, "outside every row")])
84
+
85
+ def test_an_overlay_far_call_through_a_trampoline_is_placed_by_file_offset(self):
86
+ # 0000 calls far through an FBOV fixup whose trampoline names the overlay entry at file offset 0010.
87
+ data = bytes.fromhex("9a00000800" "c3") + bytes(10) + bytes.fromhex("c3") + bytes(7)
88
+ regions = [{"name": "resident", "start": 0, "end": 6, "ip": 0, "segment": 0x1000, "resident": True,
89
+ "entries": [0], "evidence": "synthetic resident code"},
90
+ {"name": "overlay", "start": 0x10, "end": 0x18, "ip": 0, "segment": 0x2000, "entries": [0x10],
91
+ "container": {"view": "overlay 1", "start": 0x10, "end": 0x18}, "evidence": "synthetic overlay view"}]
92
+ fixup = {"site": 3, "raw": 8, "descriptor": 1, "segment": 0x2000, "target": 0x10, "trampoline": 0x40,
93
+ "evidence": "synthetic FBOV descriptor/fixup"}
94
+ run = lambda text: run_report(data, config(data, regions, relocations=[fixup], inventory=self.inventory(text)),
95
+ "inventory-check")
96
+ r = run("start\tsize\n1000:0000\t6\n")
97
+ self.assertEqual([(t["target"], t["address"], t["status"], t["call"], t["site"]) for t in r["targets"]],
98
+ [(0x10, "0x10", "outside every row", "far", 0)])
99
+ self.assertEqual(r["targets"][0]["provenance"]["relocation"]["trampoline"], 0x40)
100
+ self.assertEqual(run("start\tsize\n1000:0000\t6\n0x10\t1\n")["targets"], [])
101
+ # A row that writes the overlay by its analysis segment lies in no declared code, and says so.
102
+ aliased = run("start\tsize\n1000:0000\t6\n2000:0000\t1\n")
103
+ self.assertEqual(aliased["rowsOutsideDeclaredCode"], ["2000:0000"])
104
+ self.assertEqual(aliased["counts"]["rowsOutsideDeclaredCode"], 1)
105
+ self.assertEqual(aliased["targets"][0]["status"], "outside every row")
106
+
107
+ def test_a_target_only_raw_bytes_call_is_kept_apart_from_entry_path_targets(self):
108
+ r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n1000:000B\t1\n", data=RAW)
109
+ self.assertEqual([(t["target"], t["evidence"], t["siteClassification"]) for t in r["targets"]],
110
+ [(0x0F, "raw byte candidate only", "raw byte candidate")])
111
+ self.assertEqual(r["counts"]["entry-path call"]["outsideEveryRow"], 0)
112
+ self.assertEqual(r["counts"]["raw byte candidate only"]["outsideEveryRow"], 1)
113
+ self.assertIn("0 of the 2 call targets", r["summary"])
114
+ self.assertIn("1 more comes only from contested instructions or raw byte candidates", r["summary"])
115
+
116
+ def test_an_unresolved_call_is_listed_and_named_in_the_summary(self):
117
+ data = bytes.fromhex("ffd3" "c3") # call bx; ret
118
+ r = self.check("start\tsize\n1000:0000\t3\n", data=data)
119
+ self.assertEqual([(u["site"], u["target"]) for u in r["unresolved"]], [(0, None)])
120
+ self.assertEqual(r["counts"]["unresolvedCalls"], 1)
121
+ self.assertIn("1 call in the searched regions is unresolved, 1 of them on the entry path", r["summary"])
122
+
123
+ def test_the_summary_says_how_many_unresolved_calls_are_raw_byte_candidates(self):
124
+ # 000C is an unreached far call with no fixup on its segment word.
125
+ r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n1000:000B\t1\n", data=CALLS + bytes.fromhex("9a00000000" "c3"))
126
+ self.assertEqual([(u["site"], u["classification"]) for u in r["unresolved"]], [(12, "raw byte candidate")])
127
+ self.assertIn("1 call in the searched regions is unresolved, 0 of them on the entry path", r["summary"])
128
+
129
+ def test_a_target_outside_declared_code_is_left_out_of_the_compared_total(self):
130
+ # The FBOV fixup names file offset 0010, which no declared region holds.
131
+ data = bytes.fromhex("9a00000800" "c3") + bytes(18)
132
+ regions = [{"name": "resident", "start": 0, "end": 6, "ip": 0, "segment": 0x1000, "resident": True,
133
+ "entries": [0], "evidence": "synthetic resident code"}]
134
+ fixup = {"site": 3, "raw": 8, "descriptor": 1, "segment": 0x2000, "target": 0x10, "trampoline": 0x40,
135
+ "evidence": "synthetic FBOV descriptor/fixup"}
136
+ r = run_report(data, config(data, regions, relocations=[fixup], inventory=self.inventory("start\tsize\n1000:0000\t6\n")),
137
+ "inventory-check")
138
+ self.assertEqual([(t["target"], t["address"], t["status"]) for t in r["targets"]], [(0x10, None, "outside declared code")])
139
+ self.assertEqual(r["counts"]["entry-path call"]["outsideDeclaredCode"], 1)
140
+ self.assertTrue(r["summary"].startswith("0 of the 0 call targets that entry-path calls resolve to are not inventory "
141
+ "starts: 0 inside another row's body, 0 outside every row. 1 more entry-path "
142
+ "call target lies outside declared code and is not compared with the inventory."))
143
+
144
+ def test_a_scan_limit_short_of_the_code_makes_the_search_partial(self):
145
+ # The region lies in no container or declared segment, so only the scan gap shows the stop.
146
+ r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n", data=RAW, scanLimit=12)
147
+ self.assertEqual([(g["unsearchedStart"], g["reason"]) for g in r["gaps"] if g.get("reason") == "raw scan limit"],
148
+ [(12, "raw scan limit")])
149
+ self.assertIn("The search is partial.", r["summary"])
150
+ self.assertTrue(r["partialSearch"])
151
+ self.assertEqual([c["unsearched"] for c in r["coverage"]], [[{"start": 12, "end": len(RAW)}]])
152
+ # incoming reads the same coverage, so its search is partial too.
153
+ incoming = run_report(RAW, config(RAW, target=0x0B, scanLimit=12), "incoming")
154
+ self.assertTrue(incoming["partialSearch"])
155
+ self.assertFalse(run_report(RAW, config(RAW, target=0x0B), "incoming")["partialSearch"])
156
+ walked = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n", instructionLimit=1)
157
+ self.assertIn("The entry-path walk stopped at its instruction limit", walked["summary"])
158
+
159
+ def test_an_inventory_past_its_row_or_size_limit_is_refused(self):
160
+ from scientific_method_engine.x86 import inventory
161
+ with self.assertRaisesRegex(ValueError, f"more than {inventory.MAX_ROWS} rows"):
162
+ self.check("start\tsize\n" + "1000:0000\t1\n" * (inventory.MAX_ROWS + 1))
163
+ with self.assertRaisesRegex(ValueError, "at most 16 MiB"):
164
+ self.check("start\tsize\n" + " " * inventory.MAX_INVENTORY)
165
+
166
+ def test_flat32_code_is_placed_by_its_virtual_address(self):
167
+ data = bytes.fromhex("e800000000" "c3") # call 00401005; ret
168
+ regions = [{"name": "text", "start": 0, "end": len(data), "ip": 0x401000, "segment": 0,
169
+ "entries": [0], "evidence": "synthetic flat code"}]
170
+ r = run_report(data, config(data, regions, sourceKind="synthetic-raw", addressModel="flat32", bits=32,
171
+ inventory=self.inventory("start\tsize\n0x00401000\t5\n")), "inventory-check")
172
+ self.assertEqual([(t["target"], t["address"], t["status"]) for t in r["targets"]],
173
+ [(5, "0x00401005", "outside every row")])
174
+
175
+ def test_the_result_limit_truncates_and_says_so(self):
176
+ r = self.check("start\tsize\n1000:0000\t1\n", limit=1)
177
+ self.assertEqual(len(r["targets"]), 1)
178
+ self.assertTrue(r["truncated"])
179
+ self.assertEqual(r["counts"]["entry-path call"]["outsideEveryRow"], 2)
180
+
181
+ def test_a_missed_control_and_a_malformed_inventory_are_refused(self):
182
+ with self.assertRaisesRegex(ValueError, "Positive control 12 missed"):
183
+ self.check("start\tsize\n1000:0000\t6\n", data=RAW, controls=[12])
184
+ for text, message in (("start\tname\n1000:0000\tmain\n", "columns are start and size"),
185
+ ("start\tsize\n1000:0000\t0\n", "line 2: size 0"),
186
+ ("start\tsize\n1000:0000\t6\n1000:0000\t2\n", "line 3: start 1000:0000 is listed twice"),
187
+ ("start\tsize\nmain\t6\n", "line 2: start main is not an address"),
188
+ ("start\tsize\tranges\n1000:0000\t5\t1000:0000..1000:0006\n", "line 2: size 5 is not the total"),
189
+ ("start\tsize\n1000:0000\n", "line 2: has 1 cells")):
190
+ with self.subTest(text=text):
191
+ with self.assertRaisesRegex(ValueError, message):
192
+ self.check(text)
193
+ with self.assertRaisesRegex(ValueError, "inventory must be an absolute path"):
194
+ run_report(CALLS, config(inventory="inventory.tsv"), "inventory-check")
195
+ with self.assertRaisesRegex(ValueError, "needs inventory"):
196
+ run_report(CALLS, config(), "inventory-check")
197
+ with self.assertRaisesRegex(ValueError, "inventory applies only to inventory-check"):
198
+ run_report(CALLS, config(target=0x0B, inventory=self.inventory("start\tsize\n")), "incoming")
199
+
200
+ def test_the_cli_reads_inventory_beside_a_config_file_and_refuses_a_relative_one_from_the_reader(self):
201
+ (self.root / "fixture.bin").write_bytes(CALLS)
202
+ self.inventory("start\tsize\r\n1000:0000\t6\r\n1000:0006\t1\r\n")
203
+ cfg = config(source="fixture.bin", xxh3=xxhash.xxh3_128_hexdigest(CALLS), inventory="inventory.tsv")
204
+ path = self.root / "config.json"
205
+ path.write_text(json.dumps(cfg))
206
+ result = subprocess.run([*ENGINE, "inventory-check", str(path)], capture_output=True, text=True, env=ENGINE_ENV)
207
+ self.assertEqual(result.returncode, 0, result.stderr)
208
+ report = json.loads(result.stdout)
209
+ self.assertEqual([t["target"] for t in report["targets"]], [0x0B])
210
+ self.assertEqual(report["inventory"], {"path": str(self.root / "inventory.tsv"), "rows": 2})
211
+ prepared = {**cfg, "source": str(self.root / "fixture.bin"), "preparedProtocol": 3}
212
+ piped = subprocess.run([*ENGINE, "inventory-check", "-"], input=json.dumps(prepared), capture_output=True,
213
+ text=True, env=ENGINE_ENV)
214
+ self.assertEqual(piped.returncode, 1)
215
+ self.assertIn("relative inventory path", piped.stderr)
216
+
217
+
218
+ if __name__ == "__main__":
219
+ unittest.main()
@@ -1,71 +0,0 @@
1
- // Prints a bounded instruction window that begins at an instruction start.
2
- // @category Restoration
3
-
4
- import ghidra.app.script.GhidraScript;
5
- import ghidra.program.model.address.Address;
6
- import ghidra.program.model.listing.Instruction;
7
-
8
- import scientificmethod.InstructionStart;
9
-
10
- public class ReportInstructionWindow extends GhidraScript {
11
- private static final int MAX_INSTRUCTIONS = 200;
12
-
13
- @Override
14
- protected void run() throws Exception {
15
- String[] arguments = getScriptArgs();
16
- if (arguments.length != 2) {
17
- printerr("Supply a virtual address and instruction count.");
18
- return;
19
- }
20
-
21
- Address address = toAddr(arguments[0]);
22
- if (address == null) {
23
- printerr(arguments[0] + ": not an address in this program");
24
- return;
25
- }
26
- int count;
27
- try {
28
- count = Integer.parseInt(arguments[1]);
29
- } catch (NumberFormatException exception) {
30
- count = 0;
31
- }
32
- if (count < 1 || count > MAX_INSTRUCTIONS) {
33
- printerr("Instruction count must be between 1 and " + MAX_INSTRUCTIONS + ".");
34
- return;
35
- }
36
-
37
- // The window starts only where an instruction starts. Starting at the next instruction
38
- // instead would print a window that looks like the requested one but is not.
39
- String missing = InstructionStart.missingStart(currentProgram, address);
40
- if (missing != null) {
41
- printerr(missing + " No window printed.");
42
- return;
43
- }
44
-
45
- println("===== up to " + count + " instructions from " + address + " =====");
46
- Instruction instruction = currentProgram.getListing().getInstructionAt(address);
47
- int printed = 0;
48
- while (instruction != null && printed < count) {
49
- println(instruction.getAddress() + ": " + instruction);
50
- printed++;
51
- Instruction next = instruction.getNext();
52
- if (printed < count && next != null) {
53
- // next() is null when the instruction ends its address space, so the gap is
54
- // measured from the instruction's last byte.
55
- Address last = instruction.getMaxAddress();
56
- Address end = last.next();
57
- if (end == null) {
58
- println("gap: no instruction after " + last + " up to " + next.getAddress());
59
- } else if (!next.getAddress().equals(end)) {
60
- println("gap: no instruction from " + end + " up to " + next.getAddress());
61
- }
62
- }
63
- instruction = next;
64
- }
65
- if (printed < count) {
66
- println("The listing ends after " + printed + " of " + count + " instructions.");
67
- } else {
68
- println("Printed " + printed + " instructions.");
69
- }
70
- }
71
- }