scientific-method-engine 0.8.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/PKG-INFO +2 -1
  2. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/README.md +1 -0
  3. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/pyproject.toml +1 -1
  4. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/cli.py +1 -0
  5. scientific_method_engine-0.9.0/src/scientific_method_engine/ghidra/ExportCallEdges.java +138 -0
  6. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/reports.py +105 -3
  7. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/tests/test_x86.py +108 -0
  8. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/.gitignore +0 -0
  9. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/LICENSE +0 -0
  10. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/__init__.py +0 -0
  11. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/__main__.py +0 -0
  12. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  13. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  14. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  15. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  16. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  17. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  18. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  19. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  20. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  21. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  22. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  23. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  24. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  25. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  26. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  27. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  28. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  29. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  30. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  31. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  32. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  33. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  34. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  35. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  36. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  37. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  38. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  39. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  40. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  41. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  42. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  43. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  44. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/image.py +0 -0
  45. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/machine.py +0 -0
  46. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  47. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  48. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/pe.py +0 -0
  49. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  50. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/trace.py +0 -0
  51. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/values.py +0 -0
  52. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/tests/oracle.py +0 -0
  53. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/tests/test_dispatch.py +0 -0
  54. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/tests/test_effect_order.py +0 -0
  55. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/tests/test_nested_frame_request.py +0 -0
  56. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/tests/test_oracle.py +0 -0
  57. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/tests/test_pe.py +0 -0
  58. {scientific_method_engine-0.8.0 → scientific_method_engine-0.9.0}/tests/test_table_continuations.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 0.8.0
3
+ Version: 0.9.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -83,6 +83,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
83
83
  |---|---|---|
84
84
  | `ExportBoundedFlow` | entry, instruction limit (1..10000), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON |
85
85
  | `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
86
+ | `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, as the `ghidraCallEdges` input of `callees` |
86
87
  | `ExportFunctionFingerprints` | output TSV path | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
87
88
 
88
89
  Repairing the analysis (these change the Ghidra program, so run them before reports and keep the
@@ -68,6 +68,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
68
68
  |---|---|---|
69
69
  | `ExportBoundedFlow` | entry, instruction limit (1..10000), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON |
70
70
  | `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
71
+ | `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, as the `ghidraCallEdges` input of `callees` |
71
72
  | `ExportFunctionFingerprints` | output TSV path | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
72
73
 
73
74
  Repairing the analysis (these change the Ghidra program, so run them before reports and keep the
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "0.8.0"
8
+ version = "0.9.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -16,6 +16,7 @@ INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
16
16
  USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|trace|uses|arguments|"
17
17
  "effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
18
18
  "effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
19
+ "callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges.\n"
19
20
  "Declared table continuations are separate conditional paths; ordinary computed transfers remain stopped.\n"
20
21
  " scientific-method-engine ghidra-scripts")
21
22
 
@@ -0,0 +1,138 @@
1
+ // Exports the call edges Ghidra recovers below given entries, as JSON for the engine's callees cross-check.
2
+ // @category Restoration
3
+
4
+ import java.io.BufferedWriter;
5
+ import java.nio.charset.StandardCharsets;
6
+ import java.nio.file.Files;
7
+ import java.nio.file.Path;
8
+ import java.util.ArrayDeque;
9
+ import java.util.ArrayList;
10
+ import java.util.LinkedHashSet;
11
+ import java.util.List;
12
+ import java.util.Set;
13
+
14
+ import ghidra.app.script.GhidraScript;
15
+ import ghidra.program.database.mem.AddressSourceInfo;
16
+ import ghidra.program.model.address.Address;
17
+ import ghidra.program.model.listing.Function;
18
+ import ghidra.program.model.listing.FunctionManager;
19
+ import ghidra.program.model.listing.Instruction;
20
+ import ghidra.program.model.listing.InstructionIterator;
21
+ import ghidra.program.model.symbol.FlowType;
22
+
23
+ public class ExportCallEdges extends GhidraScript {
24
+ private static final int MAX_FUNCTIONS = 128;
25
+
26
+ @Override
27
+ protected void run() throws Exception {
28
+ String[] arguments = getScriptArgs();
29
+ if (arguments.length < 3) {
30
+ throw new IllegalArgumentException(
31
+ "Supply an output JSON path, a function limit and at least one entry address.");
32
+ }
33
+ Path output = Path.of(arguments[0]).toAbsolutePath().normalize();
34
+ if (Files.exists(output)) {
35
+ throw new IllegalArgumentException("Call edge output already exists: " + output);
36
+ }
37
+ int functionLimit = Integer.parseInt(arguments[1]);
38
+ if (functionLimit < 1 || functionLimit > MAX_FUNCTIONS) {
39
+ throw new IllegalArgumentException("Function limit must be between 1 and " + MAX_FUNCTIONS + ".");
40
+ }
41
+
42
+ FunctionManager manager = currentProgram.getFunctionManager();
43
+ List<String> missing = new ArrayList<>();
44
+ Set<Function> queued = new LinkedHashSet<>();
45
+ ArrayDeque<Function> queue = new ArrayDeque<>();
46
+ for (int i = 2; i < arguments.length; i++) {
47
+ Address address = toAddr(arguments[i]);
48
+ Function function = address == null ? null : manager.getFunctionAt(address);
49
+ if (function == null) missing.add(arguments[i]);
50
+ else if (queued.add(function)) queue.add(function);
51
+ }
52
+
53
+ // Breadth first from the entries, so the function limit keeps the shallowest functions.
54
+ StringBuilder functions = new StringBuilder();
55
+ List<String> unread = new ArrayList<>();
56
+ int exported = 0;
57
+ while (!queue.isEmpty()) {
58
+ if (monitor.isCancelled()) throw new InterruptedException("Call edge export cancelled.");
59
+ Function function = queue.poll();
60
+ if (exported >= functionLimit) {
61
+ unread.add(quote(function.getEntryPoint().toString()));
62
+ continue;
63
+ }
64
+ List<String> edges = new ArrayList<>();
65
+ InstructionIterator instructions = currentProgram.getListing()
66
+ .getInstructions(function.getBody(), true);
67
+ while (instructions.hasNext()) {
68
+ Instruction instruction = instructions.next();
69
+ FlowType flow = instruction.getFlowType();
70
+ Address[] destinations = instruction.getFlows();
71
+ if (flow.isCall() && destinations.length == 0) {
72
+ edges.add(edge(instruction, null, flow));
73
+ }
74
+ for (Address destination : destinations) {
75
+ Function callee = manager.getFunctionAt(destination);
76
+ // A jump counts when it enters another function at its entry: a tail transfer.
77
+ if (!flow.isCall() && (callee == null || callee.equals(function))) continue;
78
+ edges.add(edge(instruction, destination, flow));
79
+ // An external function has no body to read; its edge keeps the external address.
80
+ if (callee != null && !callee.isExternal() && queued.add(callee)) queue.add(callee);
81
+ }
82
+ }
83
+ if (functions.length() > 0) functions.append(",\n");
84
+ functions.append(" {\"entry\": ").append(offset(function.getEntryPoint()))
85
+ .append(", \"address\": ").append(quote(function.getEntryPoint().toString()))
86
+ .append(", \"edges\": [").append(String.join(", ", edges)).append("]}");
87
+ exported++;
88
+ }
89
+
90
+ Files.createDirectories(output.getParent());
91
+ Path temporary = Files.createTempFile(output.getParent(), "call-edges-", ".partial");
92
+ try {
93
+ try (BufferedWriter writer = Files.newBufferedWriter(temporary, StandardCharsets.UTF_8)) {
94
+ writer.write("{\n \"format\": \"scientific-method-ghidra-call-edges\",\n \"version\": 1,\n");
95
+ writer.write(" \"sha256\": " + quote(currentProgram.getExecutableSHA256()) + ",\n");
96
+ writer.write(" \"functionLimit\": " + functionLimit + ",\n");
97
+ writer.write(" \"missingEntries\": [" + quoteAll(missing) + "],\n");
98
+ writer.write(" \"unreadFunctions\": [" + String.join(", ", unread) + "],\n");
99
+ writer.write(" \"functions\": [\n" + functions + "\n ]\n}\n");
100
+ }
101
+ Files.move(temporary, output); // No replacement, and no final file until the walk completes.
102
+ }
103
+ finally { Files.deleteIfExists(temporary); }
104
+ println("Exported the call edges of " + exported + " functions to " + output);
105
+ }
106
+
107
+ private String edge(Instruction instruction, Address target, FlowType flow) {
108
+ return "{\"site\": " + offset(instruction.getAddress())
109
+ + ", \"siteAddress\": " + quote(instruction.getAddress().toString())
110
+ + ", \"target\": " + (target == null ? "null" : offset(target))
111
+ + ", \"targetAddress\": " + (target == null ? "null" : quote(target.toString()))
112
+ + ", \"flow\": " + quote(flow.toString()) + "}";
113
+ }
114
+
115
+ // The file offset the engine uses for an address, or null when the address has no file bytes.
116
+ private String offset(Address address) {
117
+ AddressSourceInfo info = currentProgram.getMemory().getAddressSourceInfo(address);
118
+ if (info == null || info.getFileOffset() < 0) return "null";
119
+ return Long.toString(info.getFileOffset());
120
+ }
121
+
122
+ private static String quoteAll(List<String> values) {
123
+ List<String> quoted = new ArrayList<>();
124
+ for (String value : values) quoted.add(quote(value));
125
+ return String.join(", ", quoted);
126
+ }
127
+
128
+ private static String quote(String value) {
129
+ if (value == null) return "null";
130
+ StringBuilder result = new StringBuilder("\"");
131
+ for (char c : value.toCharArray()) {
132
+ if (c == '"' || c == '\\') result.append('\\').append(c);
133
+ else if (c < 0x20) result.append(String.format("\\u%04x", (int) c));
134
+ else result.append(c);
135
+ }
136
+ return result.append('"').toString();
137
+ }
138
+ }
@@ -1,4 +1,5 @@
1
1
  """Focused reports derived from instruction paths and explicit source bounds."""
2
+ import hashlib
2
3
  from bisect import bisect_right
3
4
  from collections import deque
4
5
  from capstone import CS_AC_READ, CS_AC_WRITE
@@ -859,8 +860,13 @@ def callees(image, config):
859
860
  depth_limit = integer(config.get("depthLimit", 16), 1, 128, "callee depth limit")
860
861
  instruction_limit = integer(config.get("instructionLimit", 10000), 1, 100000, "instruction limit")
861
862
  controls = config.get("controls", {})
862
- if not isinstance(controls, dict) or set(controls) - {"sharedSites", "recursiveSites", "writeSites"}:
863
+ if not isinstance(controls, dict) or set(controls) - {"sharedSites", "recursiveSites", "writeSites", "ghidraAgreementSites"}:
863
864
  raise ValueError("Invalid callee controls")
865
+ export = config.get("ghidraCallEdges")
866
+ if export is not None:
867
+ export = _ghidra_call_edges(image, export)
868
+ elif "ghidraAgreementSites" in controls:
869
+ raise ValueError("ghidraAgreementSites needs ghidraCallEdges")
864
870
  nodes, edges, omitted = {}, [], []
865
871
 
866
872
  def read(entry):
@@ -984,15 +990,17 @@ def callees(image, config):
984
990
  and not s["omittedRoutes"]
985
991
  and not any(d.get("reason") in capped for at in s["entries"] for d in nodes[at]["dependencies"])
986
992
  and not any(d.get("reason") in capped for i in s["dependencyEdges"] for d in edges[i]["dependencies"]))
993
+ cross_check = _ghidra_cross_check(export, nodes, outgoing, omitted) if export is not None else None
987
994
  known = {"sharedSites": {e["site"] for e in edges if shared_control(e)},
988
995
  "recursiveSites": {e["site"] for e in edges if e["classification"] == "recursivePath"},
989
- "writeSites": {o["site"] for n in nodes.values() for o in n["memoryObservations"] if o["boundaryUsable"] and "write" in o["access"]}}
996
+ "writeSites": {o["site"] for n in nodes.values() for o in n["memoryObservations"] if o["boundaryUsable"] and "write" in o["access"]},
997
+ "ghidraAgreementSites": cross_check and cross_check["agreementSites"]}
990
998
  for kind, sites in controls.items():
991
999
  if not isinstance(sites, list) or len(sites) > 256 or any(type(at) is not int or at not in known[kind] for at in sites):
992
1000
  raise ValueError("Callee positive control missed: " + kind)
993
1001
  return {"root": root, "nodes": [{k: v for k, v in n.items() if k != "body"} | {"body": _body_report(n["body"])} for n in nodes.values()],
994
1002
  "edges": edges, "calleeSummaries": list(summaries.values()), "omittedRoutes": omitted, "uncheckedEntries": unchecked,
995
- "controls": controls,
1003
+ "controls": controls, "ghidraCrossCheck": cross_check and {k: v for k, v in cross_check.items() if k != "agreementSites"},
996
1004
  "completeWithinDeclaredGraph": not omitted and all(n["boundaryUsable"] for n in nodes.values()) and not any(e["dependencies"] for e in edges),
997
1005
  "exclusions": ["implicit memory effects", "computed/unestablished targets", "argument-sensitive effects", "runtime reachability"],
998
1006
  "interpretation": "Nodes are read breadth-first; path is the shortest read route to the caller. A recursivePath is a "
@@ -1000,6 +1008,100 @@ def callees(image, config):
1000
1008
  "that does not. Neither proves runtime recursion."}
1001
1009
 
1002
1010
 
1011
+ GHIDRA_CALL_EDGES = "scientific-method-ghidra-call-edges"
1012
+
1013
+
1014
+ def _ghidra_call_edges(image, export):
1015
+ """Validate an ExportCallEdges.java export against the image; returns its edges keyed by caller file offset."""
1016
+ if not isinstance(export, dict) or export.get("format") != GHIDRA_CALL_EDGES or export.get("version") != 1:
1017
+ raise ValueError("ghidraCallEdges must be an ExportCallEdges.java export, format version 1")
1018
+ if not isinstance(export.get("sha256"), str) or export["sha256"].lower() != hashlib.sha256(image.data).hexdigest():
1019
+ raise ValueError("ghidraCallEdges was exported from a different file")
1020
+ functions = export.get("functions")
1021
+ if not isinstance(functions, list) or len(functions) > 128:
1022
+ raise ValueError("ghidraCallEdges functions must be a list of at most 128")
1023
+ for key in ("missingEntries", "unreadFunctions"):
1024
+ if not isinstance(export.get(key), list) or not all(isinstance(v, str) for v in export[key]):
1025
+ raise ValueError(f"ghidraCallEdges {key} must be a list of addresses")
1026
+
1027
+ def offset(value, label):
1028
+ if value is None:
1029
+ return None
1030
+ return integer(value, 0, len(image.data) - 1, "Ghidra " + label + " file offset")
1031
+ callers, unmapped, total = {}, [], 0
1032
+ for function in functions:
1033
+ if not isinstance(function, dict) or not isinstance(function.get("address"), str) or not isinstance(function.get("edges"), list):
1034
+ raise ValueError("Invalid ghidraCallEdges function")
1035
+ total += len(function["edges"])
1036
+ if total > 8192:
1037
+ raise ValueError("ghidraCallEdges holds more than 8192 edges")
1038
+ rows = []
1039
+ for edge in function["edges"]:
1040
+ if (not isinstance(edge, dict) or not isinstance(edge.get("siteAddress"), str) or not isinstance(edge.get("flow"), str)
1041
+ or not (edge.get("targetAddress") is None or isinstance(edge["targetAddress"], str))):
1042
+ raise ValueError("Invalid ghidraCallEdges edge")
1043
+ rows.append({"site": offset(edge.get("site"), "site"), "siteAddress": edge["siteAddress"],
1044
+ "target": offset(edge.get("target"), "target"), "targetAddress": edge["targetAddress"], "flow": edge["flow"]})
1045
+ entry = offset(function.get("entry"), "entry")
1046
+ if entry is None:
1047
+ unmapped.append(function["address"])
1048
+ elif entry in callers:
1049
+ raise ValueError("ghidraCallEdges exports one function twice")
1050
+ else:
1051
+ callers[entry] = rows
1052
+ return {"callers": callers, "unmappedFunctions": unmapped, "missingEntries": export["missingEntries"],
1053
+ "unreadFunctions": export["unreadFunctions"]}
1054
+
1055
+
1056
+ def _ghidra_key(g):
1057
+ """The (site, target) an exported edge matches on.
1058
+
1059
+ A null target matches the engine's unresolved call only when Ghidra resolved no address either;
1060
+ a target address without file bytes (an import, uninitialized memory) matches nothing.
1061
+ """
1062
+ if g["target"] is None and g["targetAddress"] is not None:
1063
+ return g["site"], ("withoutFileOffset", g["targetAddress"])
1064
+ return g["site"], g["target"]
1065
+
1066
+
1067
+ def _ghidra_cross_check(export, nodes, outgoing, omitted):
1068
+ """Compare the engine's edges with Ghidra's for each caller both read; a Ghidra-only edge stays unchecked."""
1069
+ callers = export["callers"]
1070
+ compared = sorted(nodes.keys() & callers.keys())
1071
+ rows = []
1072
+ for caller in compared:
1073
+ ours = outgoing.get(caller, [])
1074
+ theirs = callers[caller]
1075
+ # A call neither analysis resolved matches on its site with no target.
1076
+ flows = {_ghidra_key(g): g["flow"] for g in theirs if g["site"] is not None}
1077
+ read = {(e["site"], e["target"]) for e in ours}
1078
+ for e in ours:
1079
+ flow = flows.get((e["site"], e["target"]))
1080
+ rows.append({"caller": caller, "site": e["site"], "target": e["target"], "engineEdge": e["id"],
1081
+ "result": "engineOnly" if flow is None else "agreement", "ghidraFlow": flow})
1082
+ for g in theirs:
1083
+ if g["site"] is not None and _ghidra_key(g) in read:
1084
+ continue
1085
+ # Ghidra's edge is evidence the engine did not check; it never becomes an engine edge.
1086
+ rows.append({"caller": caller, "site": g["site"], "target": g["target"], "siteAddress": g["siteAddress"],
1087
+ "targetAddress": g["targetAddress"], "ghidraFlow": g["flow"], "result": "ghidraOnly", "checked": False,
1088
+ "engineEdge": next((e["id"] for e in ours if g["site"] is not None and e["site"] == g["site"]), None)})
1089
+ counts = {kind: sum(r["result"] == kind for r in rows) for kind in ("agreement", "engineOnly", "ghidraOnly")}
1090
+ not_compared = {"engineCallers": sorted(nodes.keys() - callers.keys()), "ghidraCallers": sorted(callers.keys() - nodes.keys()),
1091
+ "unmappedGhidraFunctions": export["unmappedFunctions"], "missingGhidraEntries": export["missingEntries"],
1092
+ "unreadGhidraFunctions": export["unreadFunctions"],
1093
+ "omittedEngineRoutes": [o["id"] for o in omitted if o["entry"] in callers]}
1094
+ # A site agrees only when every edge either analysis read there agrees.
1095
+ disputed = {r["site"] for r in rows if r["result"] != "agreement"}
1096
+ return {"comparedCallers": compared, "edges": rows, "counts": counts, "notCompared": not_compared,
1097
+ "agreed": not counts["engineOnly"] and not counts["ghidraOnly"] and not any(not_compared.values()),
1098
+ "agreementSites": {r["site"] for r in rows if r["result"] == "agreement"} - disputed,
1099
+ "interpretation": "Edges of each caller that both the engine and the Ghidra export read, matched by site and target "
1100
+ "file offset; an unresolved call matches an unresolved call at its site, and a Ghidra target without a file offset "
1101
+ "matches no engine edge. A ghidraOnly edge is Ghidra's claim: the engine did not check it and never adds it to "
1102
+ "its graph. Agreement means both analyses read the edge, not that it executes."}
1103
+
1104
+
1003
1105
  # These branches test CX/ECX (LOOPE/LOOPNE also ZF), so an adjacent CMP/TEST never describes their predicate.
1004
1106
  COUNT_BRANCHES = frozenset(("jcxz", "jecxz", "jrcxz", "loop", "loope", "loopne", "loopz", "loopnz"))
1005
1107
 
@@ -371,6 +371,114 @@ class CalleeGraphTests(unittest.TestCase):
371
371
  self.assertEqual([o["access"] for o in r["nodes"][0]["memoryObservations"]], [["write"], ["write"], ["read"]])
372
372
 
373
373
 
374
+ def ghidra_export(data, functions, **extra):
375
+ """A synthetic ExportCallEdges.java export: functions maps an entry offset to (site, target, flow) edges."""
376
+ rows = [{"entry": entry, "address": "2000:0000" if entry is None else f"1000:{entry:04x}",
377
+ "edges": [{"site": site, "siteAddress": f"1000:{site:04x}", "target": target,
378
+ "targetAddress": None if target is None else f"1000:{target:04x}", "flow": flow}
379
+ for site, target, flow in edges]}
380
+ for entry, edges in functions.items()]
381
+ return {"format": "scientific-method-ghidra-call-edges", "version": 1, "sha256": hashlib.sha256(data).hexdigest(),
382
+ "functionLimit": 128, "missingEntries": [], "unreadFunctions": [], "functions": rows, **extra}
383
+
384
+
385
+ class GhidraCrossCheckTests(unittest.TestCase):
386
+ # root: call a (site 0); call b (site 3); call bx (site 6); ret. a and b return.
387
+ code = Code().label("root").branch("e8", "a").branch("e8", "b").emit("ff d3 c3").label("a").emit("c3").label("b").emit("c3")
388
+
389
+ def cross(self, functions, controls=None, **extra):
390
+ data = self.code.bytes()
391
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, functions, **extra), controls=controls or {})
392
+ cfg["regions"][0]["entries"] = [self.code.labels[n] for n in ("root", "a", "b")]
393
+ return run_report(data, cfg, "callees")
394
+
395
+ def test_each_edge_is_agreement_engine_only_or_unchecked_ghidra_only(self):
396
+ a, b = self.code.labels["a"], self.code.labels["b"]
397
+ r = self.cross({0: [(0, a, "UNCONDITIONAL_CALL"), (6, b, "COMPUTED_CALL")], b: []}, controls={"ghidraAgreementSites": [0]})
398
+ check = r["ghidraCrossCheck"]
399
+ rows = {(e["site"], e["target"], e["result"]) for e in check["edges"]}
400
+ self.assertEqual(rows, {(0, a, "agreement"), (3, b, "engineOnly"), (6, None, "engineOnly"), (6, b, "ghidraOnly")})
401
+ ghidra_only = next(e for e in check["edges"] if e["result"] == "ghidraOnly")
402
+ self.assertFalse(ghidra_only["checked"])
403
+ self.assertEqual(ghidra_only["engineEdge"], 2)
404
+ # Ghidra's computed target never becomes an engine edge.
405
+ self.assertIsNone(r["edges"][2]["target"])
406
+ self.assertEqual(r["edges"][2]["classification"], "unresolved")
407
+ self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 2, "ghidraOnly": 1})
408
+ self.assertEqual(check["comparedCallers"], [0, b])
409
+ self.assertEqual(check["notCompared"]["engineCallers"], [a])
410
+ self.assertFalse(check["agreed"])
411
+
412
+ def test_matching_graphs_agree_on_resolved_and_unresolved_calls(self):
413
+ # call 6; call bx; ret; ret
414
+ data = bytes.fromhex("e8 03 00 ff d3 c3 c3")
415
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 6, "UNCONDITIONAL_CALL"), (3, None, "COMPUTED_CALL")], 6: []}),
416
+ controls={"ghidraAgreementSites": [0, 3]})
417
+ cfg["regions"][0]["entries"] = [0, 6]
418
+ r = run_report(data, cfg, "callees")
419
+ check = r["ghidraCrossCheck"]
420
+ self.assertTrue(check["agreed"])
421
+ self.assertEqual(check["counts"], {"agreement": 2, "engineOnly": 0, "ghidraOnly": 0})
422
+ # Agreement on an unresolved call leaves the engine's edge unresolved.
423
+ self.assertEqual(r["edges"][1]["classification"], "unresolved")
424
+
425
+ def test_a_ghidra_target_without_file_bytes_does_not_match_an_unresolved_call(self):
426
+ # call 6; call bx; ret; ret. Ghidra resolves call bx to an import, which has no file offset.
427
+ data = bytes.fromhex("e8 03 00 ff d3 c3 c3")
428
+ export = ghidra_export(data, {0: [(0, 6, "UNCONDITIONAL_CALL"), (3, None, "COMPUTED_CALL")], 6: []})
429
+ export["functions"][0]["edges"][1]["targetAddress"] = "EXTERNAL:00000001"
430
+ cfg = configuration(data, ghidraCallEdges=export)
431
+ cfg["regions"][0]["entries"] = [0, 6]
432
+ check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
433
+ self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 1, "ghidraOnly": 1})
434
+ self.assertFalse(check["agreed"])
435
+ cfg["controls"] = {"ghidraAgreementSites": [3]}
436
+ with self.assertRaisesRegex(ValueError, "positive control missed: ghidraAgreementSites"):
437
+ run_report(data, cfg, "callees")
438
+
439
+ def test_routes_the_edge_limit_omitted_are_not_compared(self):
440
+ # call 6; call bx; ret; ret. Ghidra misses call bx, and the engine omits it at the edge limit.
441
+ data = bytes.fromhex("e8 03 00 ff d3 c3 c3")
442
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 6, "UNCONDITIONAL_CALL")], 6: []}), edgeLimit=1)
443
+ cfg["regions"][0]["entries"] = [0, 6]
444
+ r = run_report(data, cfg, "callees")
445
+ check = r["ghidraCrossCheck"]
446
+ self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0})
447
+ self.assertEqual(check["notCompared"]["omittedEngineRoutes"], [r["omittedRoutes"][0]["id"]])
448
+ self.assertFalse(check["agreed"])
449
+
450
+ def test_missed_agreement_control_fails(self):
451
+ a, b = self.code.labels["a"], self.code.labels["b"]
452
+ with self.assertRaisesRegex(ValueError, "positive control missed: ghidraAgreementSites"):
453
+ self.cross({0: [(0, a, "UNCONDITIONAL_CALL")], b: []}, controls={"ghidraAgreementSites": [3]})
454
+ # A site where Ghidra also reads a target the engine did not is disputed.
455
+ with self.assertRaisesRegex(ValueError, "positive control missed: ghidraAgreementSites"):
456
+ self.cross({0: [(0, a, "UNCONDITIONAL_CALL"), (0, b, "UNCONDITIONAL_CALL")], b: []},
457
+ controls={"ghidraAgreementSites": [0]})
458
+ data = self.code.bytes()
459
+ with self.assertRaisesRegex(ValueError, "needs ghidraCallEdges"):
460
+ run_report(data, configuration(data, controls={"ghidraAgreementSites": [0]}), "callees")
461
+
462
+ def test_unmapped_missing_and_unread_functions_are_not_compared(self):
463
+ r = self.cross({0: [], None: []}, missingEntries=["1000:0100"], unreadFunctions=["1000:0200"])
464
+ left = r["ghidraCrossCheck"]["notCompared"]
465
+ self.assertEqual(left["unmappedGhidraFunctions"], ["2000:0000"])
466
+ self.assertEqual(left["missingGhidraEntries"], ["1000:0100"])
467
+ self.assertEqual(left["unreadGhidraFunctions"], ["1000:0200"])
468
+ self.assertFalse(r["ghidraCrossCheck"]["agreed"])
469
+
470
+ def test_export_from_another_file_or_format_is_rejected(self):
471
+ data = self.code.bytes()
472
+ for export, message in (({**ghidra_export(data, {}), "sha256": "0" * 64}, "different file"),
473
+ ({**ghidra_export(data, {}), "version": 2}, "format version 1"),
474
+ (ghidra_export(data, {len(data): []}), "file offset"),
475
+ (ghidra_export(data, {0: [(0, None, 3)]}), "Invalid ghidraCallEdges edge"),
476
+ ({**ghidra_export(data, {}), "functions": [{}] * 129}, "at most 128"),
477
+ (ghidra_export(data, {0: [(0, None, "COMPUTED_CALL")] * 8193}), "more than 8192 edges")):
478
+ with self.subTest(message=message), self.assertRaisesRegex(ValueError, message):
479
+ run_report(data, configuration(data, ghidraCallEdges=export), "callees")
480
+
481
+
374
482
  class OperandCandidateTests(unittest.TestCase):
375
483
  def test_prefix_width_and_overlap_candidates_do_not_invent_a_second_use(self):
376
484
  data = bytes.fromhex("66 83 3e f6 02 00 c3")