scientific-method-engine 0.7.0__tar.gz → 0.9.0__tar.gz

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scientific-method-engine
-Version: 0.7.0
+Version: 0.9.0
 Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
 Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
 Author: kibertoad
@@ -83,6 +83,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
 |---|---|---|
 | `ExportBoundedFlow` | entry, instruction limit (1..10000), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON |
 | `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
+| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, as the `ghidraCallEdges` input of `callees` |
 | `ExportFunctionFingerprints` | output TSV path | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
 
 Repairing the analysis (these change the Ghidra program, so run them before reports and keep the

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/README.md

@@ -68,6 +68,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
 |---|---|---|
 | `ExportBoundedFlow` | entry, instruction limit (1..10000), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON |
 | `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
+| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, as the `ghidraCallEdges` input of `callees` |
 | `ExportFunctionFingerprints` | output TSV path | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
 
 Repairing the analysis (these change the Ghidra program, so run them before reports and keep the

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "scientific-method-engine"
 # The release workflow writes the published version from the package's release tag.
-version = "0.7.0"
+version = "0.9.0"
 description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
 readme = "README.md"
 requires-python = ">=3.12"

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/cli.py

@@ -3,13 +3,20 @@ import json
 import sys
 from pathlib import Path
 
+import capstone
+import pypcode
+
 from . import PREPARED_PROTOCOL
 
 CONFIG_LIMIT = 1024 * 1024
 PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
+# The header names the decoder and instruction semantics that actually ran, not the pins in pyproject.toml.
+DECODER = "capstone " + capstone.__version__
+INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
 USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|trace|uses|arguments|"
          "effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
          "effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
+         "callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges.\n"
          "Declared table continuations are separate conditional paths; ordinary computed transfers remain stopped.\n"
          "       scientific-method-engine ghidra-scripts")
 
@@ -60,7 +67,8 @@ def main(argv):
         raise ValueError("preparedProtocol is set by the reader and cannot be supplied")
     data, identity = read_source(config, base)
     result = run_report(data, config, command)
-    print(json.dumps({"schema": "bounded-x86-v1", "decoder": "capstone 5.0.7", "sourceIdentity": identity,
+    print(json.dumps({"schema": "bounded-x86-v1", "decoder": DECODER,
+                      "instructionSemantics": INSTRUCTION_SEMANTICS, "sourceIdentity": identity,
                       "status": "Conditional static report; never promotes an evidence entry", **result}, indent=2))
 
 

scientific_method_engine-0.9.0/src/scientific_method_engine/ghidra/ExportCallEdges.java

@@ -0,0 +1,138 @@
+// Exports the call edges Ghidra recovers below given entries, as JSON for the engine's callees cross-check.
+// @category Restoration
+
+import java.io.BufferedWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import ghidra.app.script.GhidraScript;
+import ghidra.program.database.mem.AddressSourceInfo;
+import ghidra.program.model.address.Address;
+import ghidra.program.model.listing.Function;
+import ghidra.program.model.listing.FunctionManager;
+import ghidra.program.model.listing.Instruction;
+import ghidra.program.model.listing.InstructionIterator;
+import ghidra.program.model.symbol.FlowType;
+
+public class ExportCallEdges extends GhidraScript {
+    private static final int MAX_FUNCTIONS = 128;
+
+    @Override
+    protected void run() throws Exception {
+        String[] arguments = getScriptArgs();
+        if (arguments.length < 3) {
+            throw new IllegalArgumentException(
+                "Supply an output JSON path, a function limit and at least one entry address.");
+        }
+        Path output = Path.of(arguments[0]).toAbsolutePath().normalize();
+        if (Files.exists(output)) {
+            throw new IllegalArgumentException("Call edge output already exists: " + output);
+        }
+        int functionLimit = Integer.parseInt(arguments[1]);
+        if (functionLimit < 1 || functionLimit > MAX_FUNCTIONS) {
+            throw new IllegalArgumentException("Function limit must be between 1 and " + MAX_FUNCTIONS + ".");
+        }
+
+        FunctionManager manager = currentProgram.getFunctionManager();
+        List<String> missing = new ArrayList<>();
+        Set<Function> queued = new LinkedHashSet<>();
+        ArrayDeque<Function> queue = new ArrayDeque<>();
+        for (int i = 2; i < arguments.length; i++) {
+            Address address = toAddr(arguments[i]);
+            Function function = address == null ? null : manager.getFunctionAt(address);
+            if (function == null) missing.add(arguments[i]);
+            else if (queued.add(function)) queue.add(function);
+        }
+
+        // Breadth first from the entries, so the function limit keeps the shallowest functions.
+        StringBuilder functions = new StringBuilder();
+        List<String> unread = new ArrayList<>();
+        int exported = 0;
+        while (!queue.isEmpty()) {
+            if (monitor.isCancelled()) throw new InterruptedException("Call edge export cancelled.");
+            Function function = queue.poll();
+            if (exported >= functionLimit) {
+                unread.add(quote(function.getEntryPoint().toString()));
+                continue;
+            }
+            List<String> edges = new ArrayList<>();
+            InstructionIterator instructions = currentProgram.getListing()
+                .getInstructions(function.getBody(), true);
+            while (instructions.hasNext()) {
+                Instruction instruction = instructions.next();
+                FlowType flow = instruction.getFlowType();
+                Address[] destinations = instruction.getFlows();
+                if (flow.isCall() && destinations.length == 0) {
+                    edges.add(edge(instruction, null, flow));
+                }
+                for (Address destination : destinations) {
+                    Function callee = manager.getFunctionAt(destination);
+                    // A jump counts when it enters another function at its entry: a tail transfer.
+                    if (!flow.isCall() && (callee == null || callee.equals(function))) continue;
+                    edges.add(edge(instruction, destination, flow));
+                    // An external function has no body to read; its edge keeps the external address.
+                    if (callee != null && !callee.isExternal() && queued.add(callee)) queue.add(callee);
+                }
+            }
+            if (functions.length() > 0) functions.append(",\n");
+            functions.append("    {\"entry\": ").append(offset(function.getEntryPoint()))
+                .append(", \"address\": ").append(quote(function.getEntryPoint().toString()))
+                .append(", \"edges\": [").append(String.join(", ", edges)).append("]}");
+            exported++;
+        }
+
+        Files.createDirectories(output.getParent());
+        Path temporary = Files.createTempFile(output.getParent(), "call-edges-", ".partial");
+        try {
+            try (BufferedWriter writer = Files.newBufferedWriter(temporary, StandardCharsets.UTF_8)) {
+                writer.write("{\n  \"format\": \"scientific-method-ghidra-call-edges\",\n  \"version\": 1,\n");
+                writer.write("  \"sha256\": " + quote(currentProgram.getExecutableSHA256()) + ",\n");
+                writer.write("  \"functionLimit\": " + functionLimit + ",\n");
+                writer.write("  \"missingEntries\": [" + quoteAll(missing) + "],\n");
+                writer.write("  \"unreadFunctions\": [" + String.join(", ", unread) + "],\n");
+                writer.write("  \"functions\": [\n" + functions + "\n  ]\n}\n");
+            }
+            Files.move(temporary, output); // No replacement, and no final file until the walk completes.
+        }
+        finally { Files.deleteIfExists(temporary); }
+        println("Exported the call edges of " + exported + " functions to " + output);
+    }
+
+    private String edge(Instruction instruction, Address target, FlowType flow) {
+        return "{\"site\": " + offset(instruction.getAddress())
+            + ", \"siteAddress\": " + quote(instruction.getAddress().toString())
+            + ", \"target\": " + (target == null ? "null" : offset(target))
+            + ", \"targetAddress\": " + (target == null ? "null" : quote(target.toString()))
+            + ", \"flow\": " + quote(flow.toString()) + "}";
+    }
+
+    // The file offset the engine uses for an address, or null when the address has no file bytes.
+    private String offset(Address address) {
+        AddressSourceInfo info = currentProgram.getMemory().getAddressSourceInfo(address);
+        if (info == null || info.getFileOffset() < 0) return "null";
+        return Long.toString(info.getFileOffset());
+    }
+
+    private static String quoteAll(List<String> values) {
+        List<String> quoted = new ArrayList<>();
+        for (String value : values) quoted.add(quote(value));
+        return String.join(", ", quoted);
+    }
+
+    private static String quote(String value) {
+        if (value == null) return "null";
+        StringBuilder result = new StringBuilder("\"");
+        for (char c : value.toCharArray()) {
+            if (c == '"' || c == '\\') result.append('\\').append(c);
+            else if (c < 0x20) result.append(String.format("\\u%04x", (int) c));
+            else result.append(c);
+        }
+        return result.append('"').toString();
+    }
+}

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/machine.py

@@ -1,8 +1,7 @@
-"""Path state for the evidence layer. Instruction semantics come from the backend a State holds (semantics.py)."""
+"""Path state for the evidence layer. Instruction semantics come from pypcode (pcode_backend.py)."""
 from copy import deepcopy
 from capstone.x86 import X86_OP_REG, X86_OP_IMM, X86_OP_MEM
 from .values import Value, const, unknown, op, extract, join, resize, sources, address_parts, producers
-from . import semantics
 
 REGISTERS = ("eax", "ebx", "ecx", "edx", "esi", "edi", "ebp", "esp", "cs", "ds", "es", "ss", "fs", "gs")
 ALIASES = {}
@@ -39,7 +38,10 @@ def alias(name):
 class State:
     def __init__(self, entry, image, config):
         self.bits, self.flat, self.mask = image.bits, image.flat, image.mask
-        self.semantics = semantics.current()
+        # Imported here: the backend imports this module, so a module-level import would make the
+        # import order matter.
+        from .pcode_backend import BACKEND
+        self.semantics = BACKEND
         self.sp, self.bp = ("esp", "ebp") if self.flat else ("sp", "bp")
         self.at = entry
         self.regs = {r: unknown("initial:" + r, ALIASES[r][2]) for r in REGISTERS}
@@ -64,7 +66,8 @@ class State:
         self.guards = []
         self.assumptions = {}
         self.flags = None
-        # Arithmetic flags as values when the last flag-writing instruction ran on p-code; None otherwise.
+        # Arithmetic flags as values from the last flag-writing instruction's p-code; None when no
+        # instruction computed them yet or the last one forgot them.
         self.flag_values = None
         # CF when an instruction sets it without leaving a comparable flag producer; None defers to flags.
         self.carry = None
@@ -123,9 +126,10 @@ class State:
     def carry_value(self):
         """CF as a one-bit value: from the last comparable flag producer, an explicit carry, or unknown."""
         if self.flags is not None:
-            answer, _ = self.semantics.condition(self, "jb")
-            if answer is not None:
-                return const(int(answer), 1, self.flags[3])
+            if self.flag_values is not None and "CF" in self.flag_values:
+                answer, _ = self.semantics.condition(self, "jb")
+                if answer is not None:
+                    return const(int(answer), 1, self.flags[3])
             # Name the carry by its producer's operands, so every reading of one comparison shares an assumption.
             a, b, operation, site = self.flags
             return unknown(f"carry:{site}:{(operation, a.term, b.term)!r}", 1, site)
@@ -367,11 +371,10 @@ def string_effect(state, ins, count, remaining, charge=None):
     # any segment override.
     source_name = (segment_register(ins, ins.operands[1].mem) if operation in ("movs", "lods") else
                    segment_register(ins, ins.operands[0].mem) if operation == "cmps" else None)
-    delta = -width if state.direction_flag.number else width
     counter = "ecx" if state.flat else "cx"
     if not compare or not repeated(ins):
         for _ in range(count.number):
-            state.semantics.string_iteration(state, ins, operation, width, source_name, delta)
+            state.semantics.string_iteration(state, ins, operation, width, source_name)
         if repeated(ins):
             state.setreg(counter, const(0, state.bits, state.at), state.at)
         return count.number
@@ -381,7 +384,7 @@ def string_effect(state, ins, count, remaining, charge=None):
             raise StopPath("String iteration budget exhausted; remaining effects unresolved")
         if charge is not None:
             charge(1)
-        holds = state.semantics.string_iteration(state, ins, operation, width, source_name, delta)
+        holds = state.semantics.string_iteration(state, ins, operation, width, source_name)
         iterations += 1
         outcomes.append(holds)
         if iterations == count.number:
@@ -398,7 +401,3 @@ def string_effect(state, ins, count, remaining, charge=None):
     state.event("string-compare-exit", iterations=iterations, exit=reason, counter=state.reg(counter).report())
     return iterations
 
-
-# The handwritten backend registers itself as the default; it imports names defined above.
-from . import handwritten  # noqa: E402,F401
-from . import pcode_backend  # noqa: E402,F401

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/pcode_backend.py

@@ -1,31 +1,30 @@
-"""The pypcode semantics backend (ADR 0003, phase 3).
-
-Values come from the p-code pypcode lifts. The evidence layer keeps what it always decided: which
-segment register an access uses (from Capstone, decision 3), access roles, the model's acceptance
-rules, flag-producer records and the events each instruction reports. Groups of mnemonics move to
-this backend one at a time; the others still run on the handwritten backend.
+"""Instruction semantics from pypcode (ADR 0003).
+
+Values, flags and branch conditions come from the p-code pypcode lifts from Ghidra's SLEIGH
+specification. The evidence layer keeps what it always decided: which segment register an access
+uses (from Capstone, decision 3), access roles, the model's acceptance rules, flag-producer records
+and the events each instruction reports. A ``State`` reaches this module through ``BACKEND``.
+
+What this module and ``pcode.Run`` use of a ``State``:
+
+- Methods: ``get``, ``put``, ``reg``, ``setreg``, ``segment``, ``access``, ``address``, ``event``,
+  ``set_flags``, ``forget_flags``, ``carry_value``, ``save_flags`` and ``restore_flags``.
+- Read only: ``at``, ``bits``, ``flat``, ``sp``, ``flags``, ``flag_serial``, ``flag_epoch``,
+  ``unknown_flag_site``, ``segment_bases`` and ``value_transfers``.
+- Read and assigned: ``flag_values`` (the arithmetic flags p-code computed, or None), ``carry``,
+  ``direction_flag`` and ``interrupt_flag``. ``conditional`` is appended to (the divide-error
+  assumption).
+
+Every member above belongs to one path, and ``trace`` copies a path with ``deepcopy`` when a branch
+splits it. The backend keeps no path state of its own: ``Pypcode.__deepcopy__`` returns the same
+object, so a value this module stores must live on the ``State`` to be copied with its path.
 """
 from capstone.x86 import X86_OP_REG, X86_OP_IMM, X86_OP_MEM
 
-from . import handwritten, semantics
 from .machine import ALIASES, StopPath
 from .pcode import LIFTER, Address, Run, FLAGS, SEGMENT_BASES, segment_base
 from .values import Value, const, unknown, op, extract, join, resize, sources
 
-GROUPS = {
-    "data movement": ("mov", "movzx", "movsx", "xchg", "nop"),
-    "address forms": ("lea", "lds", "les"),
-    "stack": ("push", "pop", "leave", "pushf", "pushfd", "popf", "popfd"),
-    "compare": ("cmp", "test"),
-    "arithmetic and logic": ("add", "sub", "and", "or", "xor", "inc", "dec", "not", "neg"),
-    "carry chain": ("adc", "sbb", "clc", "stc", "cmc"),
-    "shifts and rotates": ("shl", "sal", "shr", "sar", "rol", "ror", "rcl", "rcr"),
-    "multiply and divide": ("mul", "imul", "div", "idiv"),
-    "conversions": ("cbw", "cwde", "cwd", "cdq"),
-    "flags and direction": ("cld", "std", "cli", "sti"),
-    "string operations": ("movs", "stos", "lods", "cmps", "scas"),
-}
-
 # Conditional branches by the condition code SLEIGH decodes from 0x70 + code.
 CONDITION_CODES = {}
 for code, names in enumerate((("jo",), ("jno",), ("jb", "jc", "jnae"), ("jae", "jnb", "jnc"), ("je", "jz"),
@@ -281,7 +280,7 @@ class Frame:
         self.addresses = {}
         if ins.mnemonic != "pop":
             # Operands address with the registers the instruction started with. POP's destination
-            # is addressed after the stack pointer moves, as the handwritten backend does.
+            # is addressed after the stack pointer moves, as the CPU addresses it.
             for index, operand in enumerate(ins.operands):
                 if operand.type == X86_OP_MEM:
                     self.address(index)
@@ -422,47 +421,59 @@ def carry_out(state, run, site):
 
 
 class Pypcode:
-    """Instruction semantics from pypcode for the moved groups; the handwritten backend for the rest."""
-
-    name = "pypcode"
-
-    def __init__(self, groups):
-        self.groups = tuple(groups)
-        self.mnemonics = {m for group in self.groups for m in GROUPS[group]}
+    """The engine's instruction semantics: ordinary instructions, branch conditions and string bodies."""
 
     def __deepcopy__(self, memo):
+        # The backend holds no path state, so every copied path shares it.
         return self
 
     def ordinary(self, state, ins, image):
-        m = ins.mnemonic
-        if m not in self.mnemonics:
-            return handwritten.ordinary(state, ins, image)
-        HANDLERS[m](state, ins, image)
+        """Apply one instruction that is neither a control transfer nor a string operation."""
+        handler = HANDLERS.get(ins.mnemonic)
+        if handler is None:
+            raise StopPath("Unsupported instruction semantics: " + ins.mnemonic)
+        handler(state, ins, image)
 
     def condition(self, state, mnemonic):
-        answer, info = handwritten.predicate(state, mnemonic)
-        if "compare" not in self.groups or mnemonic not in CONDITION_CODES:
-            return answer, info
-        flags = state.flag_values
-        if flags is None:
-            return answer, info
+        """Evaluate a conditional branch on the current flags.
+
+        Returns ``(answer, info)``: True, False or None when unresolved, and the ``branch`` event
+        fields. They describe the evidence layer's record of the flag producer, then either
+        ``decidedBy: "p-code flags"`` for a decided branch or a ``reason`` for an undecided one.
+        """
         ops, _ = LIFTER.ops(state.flat, bytes((0x70 + CONDITION_CODES[mnemonic], 0)), 0x100)
         needed = {LIFTER.register(state.flat, v[1], v[2]) for o in ops for v in o.inputs if v[0] == "register"}
-        if not needed <= set(flags):
-            # A handwritten instruction produced some of these flags; its predicate decides.
-            return answer, info
-        condition = Run(state, ops, None, flags=flags).execute(stop_at_branch=True)
+        carry_only = needed == {"CF"} and state.flags is None and state.carry is not None
+        if carry_only:
+            info = {"predicate": mnemonic, "flag": "CF", "carry": state.carry.report()}
+        elif state.flags is None:
+            info = {"predicate": mnemonic, "reason": "flag producer unresolved",
+                    "flagProducer": state.unknown_flag_site, "flagGeneration": state.flag_epoch}
+        else:
+            a, b, operation, site = state.flags
+            info = {"predicate": mnemonic, "flagProducer": site, "operation": operation,
+                    "left": a.report(), "right": b.report()}
+        # CF is always readable: the evidence layer names it when no instruction resolved it.
+        if not needed <= set(state.flag_values or ()) | {"CF"}:
+            info.setdefault("reason", "flags unresolved")
+            return None, info
+        condition = Run(state, ops, None, flags=state.flag_values or {}).execute(stop_at_branch=True)
         if condition.number is None:
+            # Every undecided branch says why: the carry, the producer or its flags are unknown.
+            info.setdefault("reason", "carry unresolved" if carry_only else "flags unresolved")
             return None, info
-        if answer is None:
-            # The handwritten record says why it could not decide; p-code flags decided it.
-            info = {key: value for key, value in info.items() if key != "reason"}
-            info["decidedBy"] = "p-code flags"
+        # p-code decides every branch the engine resolves; the record above only describes the producer.
+        info.pop("reason", None)
+        info["decidedBy"] = "p-code flags"
         return bool(condition.number), info
 
-    def string_iteration(self, state, ins, operation, width, source_segment, delta_step):
-        if operation not in self.mnemonics:
-            return handwritten.string_iteration(state, ins, operation, width, source_segment, delta_step)
+    def string_iteration(self, state, ins, operation, width, source_segment):
+        """Apply one iteration of an accepted string form; see ``machine.string_effect``.
+
+        ``source_segment`` names the source operand's segment register (MOVS, LODS and CMPS only).
+        p-code steps SI and DI by the direction flag. For a repeated CMPS or SCAS, returns whether
+        the repeat condition holds afterwards (1, 0 or unknown); otherwise None.
+        """
         si, di = ("esi", "edi") if state.flat else ("si", "di")
         source, destination = state.reg(si), state.reg(di)
         loaded = {}
@@ -910,8 +921,5 @@ for names, handler in ((("mov", "movzx", "movsx", "xchg"), move), (("nop",), nop
     for name in names:
         HANDLERS[name] = handler
 
-MOVED = ("data movement", "address forms", "stack", "compare", "arithmetic and logic", "carry chain",
-         "shifts and rotates", "multiply and divide", "conversions", "flags and direction",
-         "string operations")
-
-semantics.register(Pypcode(MOVED), default=bool(MOVED))
+# The backend every State uses.
+BACKEND = Pypcode()

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/src/scientific_method_engine/x86/reports.py

@@ -1,4 +1,5 @@
 """Focused reports derived from instruction paths and explicit source bounds."""
+import hashlib
 from bisect import bisect_right
 from collections import deque
 from capstone import CS_AC_READ, CS_AC_WRITE
@@ -859,8 +860,13 @@ def callees(image, config):
     depth_limit = integer(config.get("depthLimit", 16), 1, 128, "callee depth limit")
     instruction_limit = integer(config.get("instructionLimit", 10000), 1, 100000, "instruction limit")
     controls = config.get("controls", {})
-    if not isinstance(controls, dict) or set(controls) - {"sharedSites", "recursiveSites", "writeSites"}:
+    if not isinstance(controls, dict) or set(controls) - {"sharedSites", "recursiveSites", "writeSites", "ghidraAgreementSites"}:
         raise ValueError("Invalid callee controls")
+    export = config.get("ghidraCallEdges")
+    if export is not None:
+        export = _ghidra_call_edges(image, export)
+    elif "ghidraAgreementSites" in controls:
+        raise ValueError("ghidraAgreementSites needs ghidraCallEdges")
     nodes, edges, omitted = {}, [], []
 
     def read(entry):
@@ -984,15 +990,17 @@ def callees(image, config):
                 and not s["omittedRoutes"]
                 and not any(d.get("reason") in capped for at in s["entries"] for d in nodes[at]["dependencies"])
                 and not any(d.get("reason") in capped for i in s["dependencyEdges"] for d in edges[i]["dependencies"]))
+    cross_check = _ghidra_cross_check(export, nodes, outgoing, omitted) if export is not None else None
     known = {"sharedSites": {e["site"] for e in edges if shared_control(e)},
              "recursiveSites": {e["site"] for e in edges if e["classification"] == "recursivePath"},
-             "writeSites": {o["site"] for n in nodes.values() for o in n["memoryObservations"] if o["boundaryUsable"] and "write" in o["access"]}}
+             "writeSites": {o["site"] for n in nodes.values() for o in n["memoryObservations"] if o["boundaryUsable"] and "write" in o["access"]},
+             "ghidraAgreementSites": cross_check and cross_check["agreementSites"]}
     for kind, sites in controls.items():
         if not isinstance(sites, list) or len(sites) > 256 or any(type(at) is not int or at not in known[kind] for at in sites):
             raise ValueError("Callee positive control missed: " + kind)
     return {"root": root, "nodes": [{k: v for k, v in n.items() if k != "body"} | {"body": _body_report(n["body"])} for n in nodes.values()],
             "edges": edges, "calleeSummaries": list(summaries.values()), "omittedRoutes": omitted, "uncheckedEntries": unchecked,
-            "controls": controls,
+            "controls": controls, "ghidraCrossCheck": cross_check and {k: v for k, v in cross_check.items() if k != "agreementSites"},
             "completeWithinDeclaredGraph": not omitted and all(n["boundaryUsable"] for n in nodes.values()) and not any(e["dependencies"] for e in edges),
             "exclusions": ["implicit memory effects", "computed/unestablished targets", "argument-sensitive effects", "runtime reachability"],
             "interpretation": "Nodes are read breadth-first; path is the shortest read route to the caller. A recursivePath is a "
@@ -1000,6 +1008,100 @@ def callees(image, config):
                               "that does not. Neither proves runtime recursion."}
 
 
+GHIDRA_CALL_EDGES = "scientific-method-ghidra-call-edges"
+
+
+def _ghidra_call_edges(image, export):
+    """Validate an ExportCallEdges.java export against the image; returns its edges keyed by caller file offset."""
+    if not isinstance(export, dict) or export.get("format") != GHIDRA_CALL_EDGES or export.get("version") != 1:
+        raise ValueError("ghidraCallEdges must be an ExportCallEdges.java export, format version 1")
+    if not isinstance(export.get("sha256"), str) or export["sha256"].lower() != hashlib.sha256(image.data).hexdigest():
+        raise ValueError("ghidraCallEdges was exported from a different file")
+    functions = export.get("functions")
+    if not isinstance(functions, list) or len(functions) > 128:
+        raise ValueError("ghidraCallEdges functions must be a list of at most 128")
+    for key in ("missingEntries", "unreadFunctions"):
+        if not isinstance(export.get(key), list) or not all(isinstance(v, str) for v in export[key]):
+            raise ValueError(f"ghidraCallEdges {key} must be a list of addresses")
+
+    def offset(value, label):
+        if value is None:
+            return None
+        return integer(value, 0, len(image.data) - 1, "Ghidra " + label + " file offset")
+    callers, unmapped, total = {}, [], 0
+    for function in functions:
+        if not isinstance(function, dict) or not isinstance(function.get("address"), str) or not isinstance(function.get("edges"), list):
+            raise ValueError("Invalid ghidraCallEdges function")
+        total += len(function["edges"])
+        if total > 8192:
+            raise ValueError("ghidraCallEdges holds more than 8192 edges")
+        rows = []
+        for edge in function["edges"]:
+            if (not isinstance(edge, dict) or not isinstance(edge.get("siteAddress"), str) or not isinstance(edge.get("flow"), str)
+                    or not (edge.get("targetAddress") is None or isinstance(edge["targetAddress"], str))):
+                raise ValueError("Invalid ghidraCallEdges edge")
+            rows.append({"site": offset(edge.get("site"), "site"), "siteAddress": edge["siteAddress"],
+                         "target": offset(edge.get("target"), "target"), "targetAddress": edge["targetAddress"], "flow": edge["flow"]})
+        entry = offset(function.get("entry"), "entry")
+        if entry is None:
+            unmapped.append(function["address"])
+        elif entry in callers:
+            raise ValueError("ghidraCallEdges exports one function twice")
+        else:
+            callers[entry] = rows
+    return {"callers": callers, "unmappedFunctions": unmapped, "missingEntries": export["missingEntries"],
+            "unreadFunctions": export["unreadFunctions"]}
+
+
+def _ghidra_key(g):
+    """The (site, target) an exported edge matches on.
+
+    A null target matches the engine's unresolved call only when Ghidra resolved no address either;
+    a target address without file bytes (an import, uninitialized memory) matches nothing.
+    """
+    if g["target"] is None and g["targetAddress"] is not None:
+        return g["site"], ("withoutFileOffset", g["targetAddress"])
+    return g["site"], g["target"]
+
+
+def _ghidra_cross_check(export, nodes, outgoing, omitted):
+    """Compare the engine's edges with Ghidra's for each caller both read; a Ghidra-only edge stays unchecked."""
+    callers = export["callers"]
+    compared = sorted(nodes.keys() & callers.keys())
+    rows = []
+    for caller in compared:
+        ours = outgoing.get(caller, [])
+        theirs = callers[caller]
+        # A call neither analysis resolved matches on its site with no target.
+        flows = {_ghidra_key(g): g["flow"] for g in theirs if g["site"] is not None}
+        read = {(e["site"], e["target"]) for e in ours}
+        for e in ours:
+            flow = flows.get((e["site"], e["target"]))
+            rows.append({"caller": caller, "site": e["site"], "target": e["target"], "engineEdge": e["id"],
+                         "result": "engineOnly" if flow is None else "agreement", "ghidraFlow": flow})
+        for g in theirs:
+            if g["site"] is not None and _ghidra_key(g) in read:
+                continue
+            # Ghidra's edge is evidence the engine did not check; it never becomes an engine edge.
+            rows.append({"caller": caller, "site": g["site"], "target": g["target"], "siteAddress": g["siteAddress"],
+                         "targetAddress": g["targetAddress"], "ghidraFlow": g["flow"], "result": "ghidraOnly", "checked": False,
+                         "engineEdge": next((e["id"] for e in ours if g["site"] is not None and e["site"] == g["site"]), None)})
+    counts = {kind: sum(r["result"] == kind for r in rows) for kind in ("agreement", "engineOnly", "ghidraOnly")}
+    not_compared = {"engineCallers": sorted(nodes.keys() - callers.keys()), "ghidraCallers": sorted(callers.keys() - nodes.keys()),
+                    "unmappedGhidraFunctions": export["unmappedFunctions"], "missingGhidraEntries": export["missingEntries"],
+                    "unreadGhidraFunctions": export["unreadFunctions"],
+                    "omittedEngineRoutes": [o["id"] for o in omitted if o["entry"] in callers]}
+    # A site agrees only when every edge either analysis read there agrees.
+    disputed = {r["site"] for r in rows if r["result"] != "agreement"}
+    return {"comparedCallers": compared, "edges": rows, "counts": counts, "notCompared": not_compared,
+            "agreed": not counts["engineOnly"] and not counts["ghidraOnly"] and not any(not_compared.values()),
+            "agreementSites": {r["site"] for r in rows if r["result"] == "agreement"} - disputed,
+            "interpretation": "Edges of each caller that both the engine and the Ghidra export read, matched by site and target "
+                              "file offset; an unresolved call matches an unresolved call at its site, and a Ghidra target without a file offset "
+                              "matches no engine edge. A ghidraOnly edge is Ghidra's claim: the engine did not check it and never adds it to "
+                              "its graph. Agreement means both analyses read the edge, not that it executes."}
+
+
 # These branches test CX/ECX (LOOPE/LOOPNE also ZF), so an adjacent CMP/TEST never describes their predicate.
 COUNT_BRANCHES = frozenset(("jcxz", "jecxz", "jrcxz", "loop", "loope", "loopne", "loopz", "loopnz"))
 

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/tests/oracle.py

@@ -4,12 +4,14 @@ Unicorn is a test dependency only. ``check`` runs one synthetic routine on the e
 Unicorn with the same segment layout and concrete registers, and compares every register the
 engine resolved to a constant at the routine's return with Unicorn's value there.
 """
-import contextlib
+import sys
+from pathlib import Path
 
 from unicorn import UC_ARCH_X86, UC_MODE_16, Uc
 from unicorn import x86_const as U
 
-from differential import accepted, run_report
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+from scientific_method_engine.x86.reports import run_report  # noqa: E402
 
 SEGMENT = 0x1000
 STACK = {"ss": 0x2000, "sp": 0xFFF0}
@@ -47,18 +49,16 @@ def unicorn(data, registers, direction=None):
     return stopped.get("at"), {name: uc.reg_read(getattr(U, "UC_X86_REG_" + name.upper())) for name in names}
 
 
-def check(test, code, registers=None, direction=None, resolved=(), extended=None):
+def check(test, code, registers=None, direction=None, resolved=()):
     """Compare the engine's resolved registers with Unicorn's for one synthetic routine.
 
     Routines write any memory they read, because the engine starts with memory unknown and
     Unicorn with zeros. ``resolved`` names registers the engine must resolve.
-    ``extended`` states why the default backend may resolve more than the handwritten one here.
     """
     data = bytes.fromhex(code)
     registers = {**STACK, **(registers or {})}
     flags = {} if direction is None else {"flags": {"direction": direction}}
-    with accepted("extended", extended) if extended else contextlib.nullcontext():
-        result = run_report(data, configuration(data, registers, **flags), "trace")
+    result = run_report(data, configuration(data, registers, **flags), "trace")
     test.assertEqual(len(result["paths"]), 1, "the oracle compares one resolved path")
     path = result["paths"][0]
     test.assertTrue(path["returned"], path["stop"])

{scientific_method_engine-0.7.0 → scientific_method_engine-0.9.0}/tests/test_dispatch.py

@@ -5,7 +5,7 @@ import sys
 import unittest
 SRC = Path(__file__).resolve().parents[1] / "src"
 sys.path.insert(0, str(SRC))
-from differential import run_report
+from scientific_method_engine.x86.reports import run_report
 from scientific_method_engine.x86.image import Image
 from scientific_method_engine.x86.trace import walk, OVERLAP_REASON