scientific-method-engine 0.7.0__tar.gz → 0.8.0__tar.gz

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scientific-method-engine
-Version: 0.7.0
+Version: 0.8.0
 Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
 Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
 Author: kibertoad

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "scientific-method-engine"
 # The release workflow writes the published version from the package's release tag.
-version = "0.7.0"
+version = "0.8.0"
 description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
 readme = "README.md"
 requires-python = ">=3.12"

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/src/scientific_method_engine/cli.py

@@ -3,10 +3,16 @@ import json
 import sys
 from pathlib import Path
 
+import capstone
+import pypcode
+
 from . import PREPARED_PROTOCOL
 
 CONFIG_LIMIT = 1024 * 1024
 PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
+# The header names the decoder and instruction semantics that actually ran, not the pins in pyproject.toml.
+DECODER = "capstone " + capstone.__version__
+INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
 USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|trace|uses|arguments|"
          "effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
          "effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
@@ -60,7 +66,8 @@ def main(argv):
         raise ValueError("preparedProtocol is set by the reader and cannot be supplied")
     data, identity = read_source(config, base)
     result = run_report(data, config, command)
-    print(json.dumps({"schema": "bounded-x86-v1", "decoder": "capstone 5.0.7", "sourceIdentity": identity,
+    print(json.dumps({"schema": "bounded-x86-v1", "decoder": DECODER,
+                      "instructionSemantics": INSTRUCTION_SEMANTICS, "sourceIdentity": identity,
                       "status": "Conditional static report; never promotes an evidence entry", **result}, indent=2))
 
 

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/src/scientific_method_engine/x86/machine.py

@@ -1,8 +1,7 @@
-"""Path state for the evidence layer. Instruction semantics come from the backend a State holds (semantics.py)."""
+"""Path state for the evidence layer. Instruction semantics come from pypcode (pcode_backend.py)."""
 from copy import deepcopy
 from capstone.x86 import X86_OP_REG, X86_OP_IMM, X86_OP_MEM
 from .values import Value, const, unknown, op, extract, join, resize, sources, address_parts, producers
-from . import semantics
 
 REGISTERS = ("eax", "ebx", "ecx", "edx", "esi", "edi", "ebp", "esp", "cs", "ds", "es", "ss", "fs", "gs")
 ALIASES = {}
@@ -39,7 +38,10 @@ def alias(name):
 class State:
     def __init__(self, entry, image, config):
         self.bits, self.flat, self.mask = image.bits, image.flat, image.mask
-        self.semantics = semantics.current()
+        # Imported here: the backend imports this module, so a module-level import would make the
+        # import order matter.
+        from .pcode_backend import BACKEND
+        self.semantics = BACKEND
         self.sp, self.bp = ("esp", "ebp") if self.flat else ("sp", "bp")
         self.at = entry
         self.regs = {r: unknown("initial:" + r, ALIASES[r][2]) for r in REGISTERS}
@@ -64,7 +66,8 @@ class State:
         self.guards = []
         self.assumptions = {}
         self.flags = None
-        # Arithmetic flags as values when the last flag-writing instruction ran on p-code; None otherwise.
+        # Arithmetic flags as values from the last flag-writing instruction's p-code; None when no
+        # instruction computed them yet or the last one forgot them.
         self.flag_values = None
         # CF when an instruction sets it without leaving a comparable flag producer; None defers to flags.
         self.carry = None
@@ -123,9 +126,10 @@ class State:
     def carry_value(self):
         """CF as a one-bit value: from the last comparable flag producer, an explicit carry, or unknown."""
         if self.flags is not None:
-            answer, _ = self.semantics.condition(self, "jb")
-            if answer is not None:
-                return const(int(answer), 1, self.flags[3])
+            if self.flag_values is not None and "CF" in self.flag_values:
+                answer, _ = self.semantics.condition(self, "jb")
+                if answer is not None:
+                    return const(int(answer), 1, self.flags[3])
             # Name the carry by its producer's operands, so every reading of one comparison shares an assumption.
             a, b, operation, site = self.flags
             return unknown(f"carry:{site}:{(operation, a.term, b.term)!r}", 1, site)
@@ -367,11 +371,10 @@ def string_effect(state, ins, count, remaining, charge=None):
     # any segment override.
     source_name = (segment_register(ins, ins.operands[1].mem) if operation in ("movs", "lods") else
                    segment_register(ins, ins.operands[0].mem) if operation == "cmps" else None)
-    delta = -width if state.direction_flag.number else width
     counter = "ecx" if state.flat else "cx"
     if not compare or not repeated(ins):
         for _ in range(count.number):
-            state.semantics.string_iteration(state, ins, operation, width, source_name, delta)
+            state.semantics.string_iteration(state, ins, operation, width, source_name)
         if repeated(ins):
             state.setreg(counter, const(0, state.bits, state.at), state.at)
         return count.number
@@ -381,7 +384,7 @@ def string_effect(state, ins, count, remaining, charge=None):
             raise StopPath("String iteration budget exhausted; remaining effects unresolved")
         if charge is not None:
             charge(1)
-        holds = state.semantics.string_iteration(state, ins, operation, width, source_name, delta)
+        holds = state.semantics.string_iteration(state, ins, operation, width, source_name)
         iterations += 1
         outcomes.append(holds)
         if iterations == count.number:
@@ -398,7 +401,3 @@ def string_effect(state, ins, count, remaining, charge=None):
     state.event("string-compare-exit", iterations=iterations, exit=reason, counter=state.reg(counter).report())
     return iterations
 
-
-# The handwritten backend registers itself as the default; it imports names defined above.
-from . import handwritten  # noqa: E402,F401
-from . import pcode_backend  # noqa: E402,F401

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/src/scientific_method_engine/x86/pcode_backend.py

@@ -1,31 +1,30 @@
-"""The pypcode semantics backend (ADR 0003, phase 3).
-
-Values come from the p-code pypcode lifts. The evidence layer keeps what it always decided: which
-segment register an access uses (from Capstone, decision 3), access roles, the model's acceptance
-rules, flag-producer records and the events each instruction reports. Groups of mnemonics move to
-this backend one at a time; the others still run on the handwritten backend.
+"""Instruction semantics from pypcode (ADR 0003).
+
+Values, flags and branch conditions come from the p-code pypcode lifts from Ghidra's SLEIGH
+specification. The evidence layer keeps what it always decided: which segment register an access
+uses (from Capstone, decision 3), access roles, the model's acceptance rules, flag-producer records
+and the events each instruction reports. A ``State`` reaches this module through ``BACKEND``.
+
+What this module and ``pcode.Run`` use of a ``State``:
+
+- Methods: ``get``, ``put``, ``reg``, ``setreg``, ``segment``, ``access``, ``address``, ``event``,
+  ``set_flags``, ``forget_flags``, ``carry_value``, ``save_flags`` and ``restore_flags``.
+- Read only: ``at``, ``bits``, ``flat``, ``sp``, ``flags``, ``flag_serial``, ``flag_epoch``,
+  ``unknown_flag_site``, ``segment_bases`` and ``value_transfers``.
+- Read and assigned: ``flag_values`` (the arithmetic flags p-code computed, or None), ``carry``,
+  ``direction_flag`` and ``interrupt_flag``. ``conditional`` is appended to (the divide-error
+  assumption).
+
+Every member above belongs to one path, and ``trace`` copies a path with ``deepcopy`` when a branch
+splits it. The backend keeps no path state of its own: ``Pypcode.__deepcopy__`` returns the same
+object, so a value this module stores must live on the ``State`` to be copied with its path.
 """
 from capstone.x86 import X86_OP_REG, X86_OP_IMM, X86_OP_MEM
 
-from . import handwritten, semantics
 from .machine import ALIASES, StopPath
 from .pcode import LIFTER, Address, Run, FLAGS, SEGMENT_BASES, segment_base
 from .values import Value, const, unknown, op, extract, join, resize, sources
 
-GROUPS = {
-    "data movement": ("mov", "movzx", "movsx", "xchg", "nop"),
-    "address forms": ("lea", "lds", "les"),
-    "stack": ("push", "pop", "leave", "pushf", "pushfd", "popf", "popfd"),
-    "compare": ("cmp", "test"),
-    "arithmetic and logic": ("add", "sub", "and", "or", "xor", "inc", "dec", "not", "neg"),
-    "carry chain": ("adc", "sbb", "clc", "stc", "cmc"),
-    "shifts and rotates": ("shl", "sal", "shr", "sar", "rol", "ror", "rcl", "rcr"),
-    "multiply and divide": ("mul", "imul", "div", "idiv"),
-    "conversions": ("cbw", "cwde", "cwd", "cdq"),
-    "flags and direction": ("cld", "std", "cli", "sti"),
-    "string operations": ("movs", "stos", "lods", "cmps", "scas"),
-}
-
 # Conditional branches by the condition code SLEIGH decodes from 0x70 + code.
 CONDITION_CODES = {}
 for code, names in enumerate((("jo",), ("jno",), ("jb", "jc", "jnae"), ("jae", "jnb", "jnc"), ("je", "jz"),
@@ -281,7 +280,7 @@ class Frame:
         self.addresses = {}
         if ins.mnemonic != "pop":
             # Operands address with the registers the instruction started with. POP's destination
-            # is addressed after the stack pointer moves, as the handwritten backend does.
+            # is addressed after the stack pointer moves, as the CPU addresses it.
             for index, operand in enumerate(ins.operands):
                 if operand.type == X86_OP_MEM:
                     self.address(index)
@@ -422,47 +421,59 @@ def carry_out(state, run, site):
 
 
 class Pypcode:
-    """Instruction semantics from pypcode for the moved groups; the handwritten backend for the rest."""
-
-    name = "pypcode"
-
-    def __init__(self, groups):
-        self.groups = tuple(groups)
-        self.mnemonics = {m for group in self.groups for m in GROUPS[group]}
+    """The engine's instruction semantics: ordinary instructions, branch conditions and string bodies."""
 
     def __deepcopy__(self, memo):
+        # The backend holds no path state, so every copied path shares it.
         return self
 
     def ordinary(self, state, ins, image):
-        m = ins.mnemonic
-        if m not in self.mnemonics:
-            return handwritten.ordinary(state, ins, image)
-        HANDLERS[m](state, ins, image)
+        """Apply one instruction that is neither a control transfer nor a string operation."""
+        handler = HANDLERS.get(ins.mnemonic)
+        if handler is None:
+            raise StopPath("Unsupported instruction semantics: " + ins.mnemonic)
+        handler(state, ins, image)
 
     def condition(self, state, mnemonic):
-        answer, info = handwritten.predicate(state, mnemonic)
-        if "compare" not in self.groups or mnemonic not in CONDITION_CODES:
-            return answer, info
-        flags = state.flag_values
-        if flags is None:
-            return answer, info
+        """Evaluate a conditional branch on the current flags.
+
+        Returns ``(answer, info)``: True, False or None when unresolved, and the ``branch`` event
+        fields. They describe the evidence layer's record of the flag producer, then either
+        ``decidedBy: "p-code flags"`` for a decided branch or a ``reason`` for an undecided one.
+        """
         ops, _ = LIFTER.ops(state.flat, bytes((0x70 + CONDITION_CODES[mnemonic], 0)), 0x100)
         needed = {LIFTER.register(state.flat, v[1], v[2]) for o in ops for v in o.inputs if v[0] == "register"}
-        if not needed <= set(flags):
-            # A handwritten instruction produced some of these flags; its predicate decides.
-            return answer, info
-        condition = Run(state, ops, None, flags=flags).execute(stop_at_branch=True)
+        carry_only = needed == {"CF"} and state.flags is None and state.carry is not None
+        if carry_only:
+            info = {"predicate": mnemonic, "flag": "CF", "carry": state.carry.report()}
+        elif state.flags is None:
+            info = {"predicate": mnemonic, "reason": "flag producer unresolved",
+                    "flagProducer": state.unknown_flag_site, "flagGeneration": state.flag_epoch}
+        else:
+            a, b, operation, site = state.flags
+            info = {"predicate": mnemonic, "flagProducer": site, "operation": operation,
+                    "left": a.report(), "right": b.report()}
+        # CF is always readable: the evidence layer names it when no instruction resolved it.
+        if not needed <= set(state.flag_values or ()) | {"CF"}:
+            info.setdefault("reason", "flags unresolved")
+            return None, info
+        condition = Run(state, ops, None, flags=state.flag_values or {}).execute(stop_at_branch=True)
         if condition.number is None:
+            # Every undecided branch says why: the carry, the producer or its flags are unknown.
+            info.setdefault("reason", "carry unresolved" if carry_only else "flags unresolved")
             return None, info
-        if answer is None:
-            # The handwritten record says why it could not decide; p-code flags decided it.
-            info = {key: value for key, value in info.items() if key != "reason"}
-            info["decidedBy"] = "p-code flags"
+        # p-code decides every branch the engine resolves; the record above only describes the producer.
+        info.pop("reason", None)
+        info["decidedBy"] = "p-code flags"
         return bool(condition.number), info
 
-    def string_iteration(self, state, ins, operation, width, source_segment, delta_step):
-        if operation not in self.mnemonics:
-            return handwritten.string_iteration(state, ins, operation, width, source_segment, delta_step)
+    def string_iteration(self, state, ins, operation, width, source_segment):
+        """Apply one iteration of an accepted string form; see ``machine.string_effect``.
+
+        ``source_segment`` names the source operand's segment register (MOVS, LODS and CMPS only).
+        p-code steps SI and DI by the direction flag. For a repeated CMPS or SCAS, returns whether
+        the repeat condition holds afterwards (1, 0 or unknown); otherwise None.
+        """
         si, di = ("esi", "edi") if state.flat else ("si", "di")
         source, destination = state.reg(si), state.reg(di)
         loaded = {}
@@ -910,8 +921,5 @@ for names, handler in ((("mov", "movzx", "movsx", "xchg"), move), (("nop",), nop
     for name in names:
         HANDLERS[name] = handler
 
-MOVED = ("data movement", "address forms", "stack", "compare", "arithmetic and logic", "carry chain",
-         "shifts and rotates", "multiply and divide", "conversions", "flags and direction",
-         "string operations")
-
-semantics.register(Pypcode(MOVED), default=bool(MOVED))
+# The backend every State uses.
+BACKEND = Pypcode()

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/tests/oracle.py

@@ -4,12 +4,14 @@ Unicorn is a test dependency only. ``check`` runs one synthetic routine on the e
 Unicorn with the same segment layout and concrete registers, and compares every register the
 engine resolved to a constant at the routine's return with Unicorn's value there.
 """
-import contextlib
+import sys
+from pathlib import Path
 
 from unicorn import UC_ARCH_X86, UC_MODE_16, Uc
 from unicorn import x86_const as U
 
-from differential import accepted, run_report
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+from scientific_method_engine.x86.reports import run_report  # noqa: E402
 
 SEGMENT = 0x1000
 STACK = {"ss": 0x2000, "sp": 0xFFF0}
@@ -47,18 +49,16 @@ def unicorn(data, registers, direction=None):
     return stopped.get("at"), {name: uc.reg_read(getattr(U, "UC_X86_REG_" + name.upper())) for name in names}
 
 
-def check(test, code, registers=None, direction=None, resolved=(), extended=None):
+def check(test, code, registers=None, direction=None, resolved=()):
     """Compare the engine's resolved registers with Unicorn's for one synthetic routine.
 
     Routines write any memory they read, because the engine starts with memory unknown and
     Unicorn with zeros. ``resolved`` names registers the engine must resolve.
-    ``extended`` states why the default backend may resolve more than the handwritten one here.
     """
     data = bytes.fromhex(code)
     registers = {**STACK, **(registers or {})}
     flags = {} if direction is None else {"flags": {"direction": direction}}
-    with accepted("extended", extended) if extended else contextlib.nullcontext():
-        result = run_report(data, configuration(data, registers, **flags), "trace")
+    result = run_report(data, configuration(data, registers, **flags), "trace")
     test.assertEqual(len(result["paths"]), 1, "the oracle compares one resolved path")
     path = result["paths"][0]
     test.assertTrue(path["returned"], path["stop"])

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/tests/test_dispatch.py

@@ -5,7 +5,7 @@ import sys
 import unittest
 SRC = Path(__file__).resolve().parents[1] / "src"
 sys.path.insert(0, str(SRC))
-from differential import run_report
+from scientific_method_engine.x86.reports import run_report
 from scientific_method_engine.x86.image import Image
 from scientific_method_engine.x86.trace import walk, OVERLAP_REASON
 

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/tests/test_effect_order.py

@@ -1,6 +1,5 @@
 """Synthetic path evidence; no original bytes or claims."""
 import unittest
-from differential import accepted
 from test_x86 import Code, report
 
 
@@ -73,8 +72,7 @@ class EffectOrderTests(unittest.TestCase):
         # A loop re-pushes the same return address into a stack slot another call overwrote.
         c = Code().emit("b9 02 00").label("top").branch("e8", "a").branch("e8", "b").emit("49").branch("75", "top").emit("c3")
         c.label("a").emit("c3").label("b").emit("c3")
-        with accepted("extended", "DEC flags from p-code resolve the loop exit; test_oracle checks the count"):
-            r = report(c, "effects", registers={"ss": 0x3000, "sp": 0xff00})
+        r = report(c, "effects", registers={"ss": 0x3000, "sp": 0xff00})
         self.assertTrue(any(p["returned"] for p in self.paths(r)))
         self.assertFalse([w for p in self.paths(r) for w in p["localRestorationWitnesses"]])
 

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/tests/test_oracle.py

@@ -5,7 +5,6 @@ from oracle import STACK, check, configuration, run_report
 
 DATA = {"ds": 0x3000, "es": 0x4000}
 SAME = {"ds": 0x4000, "es": 0x4000}
-STRING_COMPARE = "the handwritten backend stops on CMPS and SCAS"
 
 
 class DataMovement(unittest.TestCase):
@@ -40,9 +39,9 @@ class Stack(unittest.TestCase):
 
 
 class Compare(unittest.TestCase):
-    def branch(self, setup, jcc, extended=None):
+    def branch(self, setup, jcc):
         # cx = 2 when the branch is taken, 1 when not.
-        check(self, setup + jcc + "04 b90100 c3 b90200 c3", resolved=("cx",), extended=extended)
+        check(self, setup + jcc + "04 b90100 c3 b90200 c3", resolved=("cx",))
 
     def test_signed_and_unsigned_conditions(self):
         for setup in ("b80300 bb0500 39d8", "b8fdff bb0500 39d8", "b80500 bb0500 39d8", "b80080 bbff7f 39d8"):
@@ -54,14 +53,30 @@ class Compare(unittest.TestCase):
         for setup in ("b80f00 a80f", "b80300 3d0300", "b80100 3d0300"):
             for jcc in ("74", "75", "7a", "7b", "72", "70"):
                 with self.subTest(setup=setup, jcc=jcc):
-                    # The handwritten predicate never resolves PF; p-code computes it.
-                    self.branch(setup, jcc, extended="parity flag from p-code" if jcc in ("7a", "7b") else None)
+                    self.branch(setup, jcc)
 
     def test_same_register_compare_resolves_with_unknown_input(self):
         for jcc in ("74", "72", "7c", "7f", "70"):
             with self.subTest(jcc=jcc):
                 check(self, "39c0" + jcc + "04 b90100 c3 b90200 c3", registers={"ax": 0x1234}, resolved=("cx",))
 
+    def test_branches_after_a_comparison_name_how_they_were_decided(self):
+        # cmp ax, 3 with AX = 3: p-code decides JE and JP, and the event keeps the comparison record.
+        for jcc in ("74", "7a"):
+            with self.subTest(jcc=jcc):
+                result = check(self, "b80300 3d0300" + jcc + "04 b90100 c3 b90200 c3", resolved=("cx",))
+                branch, = [e for e in result["paths"][0]["events"] if e["kind"] == "branch"]
+                self.assertEqual((branch["operation"], branch["decidedBy"]), ("cmp", "p-code flags"))
+                self.assertNotIn("reason", branch)
+        # With AX unknown the comparison decides nothing: both arms run and each says why.
+        data = bytes.fromhex("3d0300 7401 90 c3".replace(" ", ""))
+        result = run_report(data, configuration(data, dict(STACK)), "trace")
+        branches = [e for path in result["paths"] for e in path["events"] if e["kind"] == "branch"]
+        self.assertEqual(sorted(e["taken"] for e in branches), [False, True])
+        for e in branches:
+            self.assertEqual((e["operation"], e["reason"]), ("cmp", "flags unresolved"))
+            self.assertNotIn("decidedBy", e)
+
 
 class ArithmeticAndLogic(unittest.TestCase):
     def test_values(self):
@@ -69,18 +84,17 @@ class ArithmeticAndLogic(unittest.TestCase):
               resolved=("ax", "bx", "cx", "dx"))
 
     def test_counted_loop_exits_on_decrement_flags(self):
-        # inc ax; dec cx; jnz back: the handwritten backend forgets DEC's flags and splits.
-        result = check(self, "b90300 b80000 40 49 75fc c3", resolved=("ax", "cx"),
-                       extended="DEC flags from p-code resolve the loop exit")
+        # inc ax; dec cx; jnz back: DEC's flags decide each exit, so the loop runs as one path.
+        result = check(self, "b90300 b80000 40 49 75fc c3", resolved=("ax", "cx"))
         branches = [e for e in result["paths"][0]["events"] if e["kind"] == "branch"]
         self.assertEqual([e["taken"] for e in branches], [True, True, False])
-        # A branch p-code decided does not keep the handwritten backend's reason for not deciding it.
+        # No comparison record exists, so the branch says p-code flags decided it.
         for e in branches:
             self.assertNotIn("reason", e)
             self.assertEqual((e["decidedBy"], e["flagProducer"]), ("p-code flags", 7))
 
     def test_undecided_branch_keeps_the_reason(self):
-        # inc ax; jnz: AX is unknown, so neither backend decides the branch and both arms run.
+        # inc ax; jnz: AX is unknown, so the engine does not decide the branch and both arms run.
         data = bytes.fromhex("40 7501 90 c3".replace(" ", ""))
         result = run_report(data, configuration(data, dict(STACK)), "trace")
         branches = [e for path in result["paths"] for e in path["events"] if e["kind"] == "branch"]
@@ -129,8 +143,7 @@ class ShiftsAndRotates(unittest.TestCase):
     def test_double_word_shift_from_a_zero_high_half(self):
         # DX starts at zero; RCL carries AX's unknown top bits into it. The second RCL shifts out
         # DX's bit 15, which is still zero; ADC moves that carry into BX for Unicorn to check.
-        result = check(self, "b90100 31d2 d1e0 d1d2 d1e0 d1d2 bb0000 11db c3", resolved=("cx", "bx"),
-                       extended="a rotate's carry out folded from the operand's known bits")
+        result = check(self, "b90100 31d2 d1e0 d1d2 d1e0 d1d2 bb0000 11db c3", resolved=("cx", "bx"))
         self.assertEqual(result["paths"][0]["registers"]["dx"]["expression"][0], "or")
 
     def test_memory_operands(self):
@@ -194,16 +207,16 @@ class StringOperations(unittest.TestCase):
 
     def test_repne_scas_finds_a_terminator(self):
         check(self, "fc bf1000 c7056162 c6450200 b000 b9ffff f2ae c3", registers=SAME,
-              resolved=("cx", "di"), extended=STRING_COMPARE)
+              resolved=("cx", "di"))
 
     def test_repe_cmps_stops_at_the_first_difference(self):
         # SI's byte is below DI's at the difference, so JB takes the branch: BX = 2.
         check(self, "fc be1000 bf2000 c7046162 c6440263 c7056162 c6450264 b90500 f3a6 7204 bb0100 c3 bb0200 c3",
-              registers=SAME, resolved=("cx", "si", "di", "bx"), extended=STRING_COMPARE)
+              registers=SAME, resolved=("cx", "si", "di", "bx"))
 
     def test_repe_cmps_runs_out_of_count_and_single_forms(self):
         check(self, "fc be1000 bf2000 c7046162 c7056162 b90200 f3a6 be1000 bf2000 a7 b86162 bf2000 af 7504 bb0100 c3 bb0200 c3",
-              registers=SAME, resolved=("cx", "si", "di", "bx"), extended=STRING_COMPARE)
+              registers=SAME, resolved=("cx", "si", "di", "bx"))
 
     def test_cmps_with_equal_source_and_destination_offsets(self):
         # SI = DI = 0x10. In different segments 'a' < 'b' takes JB (BX = 2); in one segment the second
@@ -211,7 +224,7 @@ class StringOperations(unittest.TestCase):
         code = "fc be1000 bf1000 c60461 26c60562 a6 7204 bb0100 c3 bb0200 c3"
         for registers in (DATA, SAME):
             with self.subTest(registers=registers):
-                check(self, code, registers=registers, resolved=("si", "di", "bx"), extended=STRING_COMPARE)
+                check(self, code, registers=registers, resolved=("si", "di", "bx"))
 
     def test_backward_steps(self):
         check(self, "fd bf0a00 b0aa b90400 f3aa be0700 ac c3", registers={**DATA, "ds": 0x4000},

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/tests/test_pe.py

@@ -18,7 +18,7 @@ ENGINE_ENV = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(SRC)
 READER = SRC.parents[1] / "executable-reader" / "bin" / "scientific-method.ts"
 from scientific_method_engine.x86.image import Image
 from scientific_method_engine.x86.pe import pe32
-from differential import accepted, run_report
+from scientific_method_engine.x86.reports import run_report
 
 BASE, CODE_VA, DATA_VA = 0x400000, 0x401000, 0x402000
 CODE_RAW, DATA_RAW = 0x200, 0x400
@@ -80,8 +80,7 @@ def events(result, kind):
 class PEReporterTests(unittest.TestCase):
     def test_cmps_with_one_address_for_both_operands(self):
         # ESI = EDI: both CMPSB loads match both operands, and each operand takes one of them.
-        with accepted('extended', 'the handwritten backend stops on CMPS'):
-            r = report('fc be 00 20 40 00 bf 00 20 40 00 c6 06 61 a6 74 01 c3 c3')
+        r = report('fc be 00 20 40 00 bf 00 20 40 00 c6 06 61 a6 74 01 c3 c3')
         path, = r['paths']
         self.assertTrue(path['returned'], path['stop'])
         self.assertEqual((path['registers']['esi']['value'], path['registers']['edi']['value']), (DATA_VA + 1,) * 2)

{scientific_method_engine-0.7.0 → scientific_method_engine-0.8.0}/tests/test_x86.py

@@ -13,9 +13,10 @@ sys.path.insert(0, str(SRC))
 # The engine CLI runs from this checkout's source whether or not the package is installed.
 ENGINE = [sys.executable, "-B", "-m", "scientific_method_engine"]
 ENGINE_ENV = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(SRC), os.environ.get("PYTHONPATH")]))}
+import capstone
+import pypcode
 from scientific_method_engine.x86.image import Image
-from scientific_method_engine.x86.reports import run_report as engine_report
-from differential import accepted, run_report
+from scientific_method_engine.x86.reports import run_report
 from scientific_method_engine.x86.trace import trace, walk, OVERLAP_REASON, CONTESTED_REASON
 from scientific_method_engine.x86.values import const, unknown, op, extract, resize
 
@@ -1404,6 +1405,14 @@ class ReporterTests(unittest.TestCase):
         self.assertFalse(calls[0]["guards"][0]["sameTargetValue"])
         self.assertFalse(result["completeWithinModel"])
 
+    def test_every_engine_module_imports_first(self):
+        modules = sorted(p.stem for p in (SRC / "scientific_method_engine" / "x86").glob("*.py") if p.stem != "__init__")
+        for module in modules:
+            with self.subTest(module=module):
+                code = f"import scientific_method_engine.x86.{module}"
+                result = subprocess.run([sys.executable, "-B", "-c", code], capture_output=True, text=True, env=ENGINE_ENV)
+                self.assertEqual(result.returncode, 0, result.stderr)
+
     def test_cli_identity_and_errors(self):
         with tempfile.TemporaryDirectory() as folder:
             root = Path(folder); data = bytes.fromhex("b8 01 00 c3")
@@ -1413,7 +1422,10 @@ class ReporterTests(unittest.TestCase):
             args = [*ENGINE, "trace", str(path)]
             result = subprocess.run(args, capture_output=True, text=True, env=ENGINE_ENV)
             self.assertEqual(result.returncode, 0, result.stderr)
-            self.assertEqual(json.loads(result.stdout)["sourceIdentity"]["size"], 4)
+            header = json.loads(result.stdout)
+            self.assertEqual(header["sourceIdentity"]["size"], 4)
+            self.assertEqual((header["decoder"], header["instructionSemantics"]),
+                             ("capstone " + capstone.__version__, f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"))
             cfg["sha256"] = "0" * 64; path.write_text(json.dumps(cfg))
             result = subprocess.run(args, capture_output=True, text=True, env=ENGINE_ENV)
             self.assertEqual(result.returncode, 1)
@@ -1470,54 +1482,59 @@ class ReporterTests(unittest.TestCase):
 
     def test_repeated_string_comparisons(self):
         es = {"es": 0x2000, "ds": 0x2000}
-        with accepted("extended", "the handwritten backend stops on CMPS and SCAS"):
-            # Positive control: REPNE SCASB stops at the terminator it compared, after two iterations.
-            result = report("bf 00 01 c6 05 61 c6 45 01 00 b0 00 b9 10 00 f2 ae c3", flags={"direction": 0}, registers=es)
-            path = result["paths"][0]
-            self.assertTrue(path["returned"], path["stop"])
-            exit_event, = events(result, "string-compare-exit")
-            self.assertEqual((exit_event["iterations"], exit_event["exit"]), (2, "condition"))
-            self.assertEqual((path["registers"]["cx"]["value"], path["registers"]["di"]["value"]), (14, 0x102))
-            self.assertEqual(result["stringIterationsUsed"], 2)
-            # Unknown memory leaves the repeat condition unresolved after the first iteration.
-            result = report("bf 00 01 b0 00 b9 10 00 f2 ae c3", flags={"direction": 0}, registers=es)
-            self.assertIn("comparison outcome unresolved", result["paths"][0]["stop"])
-            self.assertEqual(len(events(result, "read")), 1)
-            # A repeated comparison pays per iteration, so the budget stops it mid-loop.
-            result = report("bf 00 01 c7 05 00 00 b0 01 b9 10 00 f2 ae c3", flags={"direction": 0}, registers=es,
-                            stringIterations=1)
-            self.assertIn("budget exhausted", result["paths"][0]["stop"])
-            self.assertEqual(result["stringIterationsUsed"], 1)
+        # Positive control: REPNE SCASB stops at the terminator it compared, after two iterations.
+        result = report("bf 00 01 c6 05 61 c6 45 01 00 b0 00 b9 10 00 f2 ae c3", flags={"direction": 0}, registers=es)
+        path = result["paths"][0]
+        self.assertTrue(path["returned"], path["stop"])
+        exit_event, = events(result, "string-compare-exit")
+        self.assertEqual((exit_event["iterations"], exit_event["exit"]), (2, "condition"))
+        self.assertEqual((path["registers"]["cx"]["value"], path["registers"]["di"]["value"]), (14, 0x102))
+        self.assertEqual(result["stringIterationsUsed"], 2)
+        # Unknown memory leaves the repeat condition unresolved after the first iteration.
+        result = report("bf 00 01 b0 00 b9 10 00 f2 ae c3", flags={"direction": 0}, registers=es)
+        self.assertIn("comparison outcome unresolved", result["paths"][0]["stop"])
+        self.assertEqual(len(events(result, "read")), 1)
+        # A repeated comparison pays per iteration, so the budget stops it mid-loop.
+        result = report("bf 00 01 c7 05 00 00 b0 01 b9 10 00 f2 ae c3", flags={"direction": 0}, registers=es,
+                        stringIterations=1)
+        self.assertIn("budget exhausted", result["paths"][0]["stop"])
+        self.assertEqual(result["stringIterationsUsed"], 1)
         # REPNE stays rejected on the forms that do not compare.
         result = report("f2 a4 c3", flags={"direction": 0})
         self.assertIn("REPNE", result["paths"][0]["stop"])
 
     def test_repeated_comparison_splits_an_unknown_direction_when_its_count_exceeds_the_budget(self):
         es = {"es": 0x2000, "ds": 0x2000}
-        with accepted("extended", "the handwritten backend stops on CMPS and SCAS"):
-            # CX = 0xFFFF exceeds the budget, but the terminator ends the scan after one iteration either way.
-            result = report("bf 00 01 c6 05 00 b0 00 b9 ff ff f2 ae c3", registers=es)
+        # CX = 0xFFFF exceeds the budget, but the terminator ends the scan after one iteration either way.
+        result = report("bf 00 01 c6 05 00 b0 00 b9 ff ff f2 ae c3", registers=es)
         self.assertTrue(all(p["returned"] for p in result["paths"]), [p["stop"] for p in result["paths"]])
         self.assertEqual(sorted(p["registers"]["di"]["value"] for p in result["paths"]), [0xff, 0x101])
         self.assertEqual(result["stringIterationsUsed"], 2)
 
     def test_rotates_and_sal_by_one_on_unknown_operands_keep_the_reported_forms(self):
-        # The differential run checks that both backends report the same result and operation.
         for code in ("d1 c0", "d1 c8", "d0 c0", "d0 c8", "d0 cc", "d1 f0"):
             with self.subTest(code=code):
                 path = report(code + " c3")["paths"][0]
                 self.assertTrue(path["returned"], path["stop"])
+                event, = [e for e in path["events"] if e["kind"] == "arithmetic"]
+                operation = {"c0": "rol", "c8": "ror", "cc": "ror", "f0": "sal"}[code[-2:]]
+                self.assertEqual(event["operation"], operation)
+                # A rotate reports the OR of its two shifted halves; SAL by one reports one shift.
+                self.assertEqual(event["result"]["expression"][0], "shl" if operation == "sal" else "or")
+                if operation != "sal":
+                    # The carry out is bit 0 (ROR) or the top bit (ROL) of the rotated operand.
+                    bits = event["left"]["bits"]
+                    low = event["left"]["expression"][2]
+                    self.assertEqual(event["carryOut"]["expression"][2], low + (bits - 1 if operation == "rol" else 0))
 
     def test_rotate_through_unknown_carry_resolves_a_carry_out_from_a_known_operand(self):
         # RCL by n carries out bit 16 - n of a 16-bit operand, RCR by n bit n - 1; CF starts unknown.
-        # The result term still differs in form from the handwritten backend's, so this runs the
-        # engine's default backend alone.
         cases = (("bb 10 00 c1 d3 05", 0), ("bb 00 08 c1 d3 05", 1), ("bb 10 00 c1 db 05", 1),
                  ("bb 08 00 c1 db 05", 0), ("c1 d3 05", None))
         for code, carry in cases:
             with self.subTest(code=code):
                 data = bytes.fromhex(code + " c3")
-                result = engine_report(data, configuration(data), "trace")
+                result = run_report(data, configuration(data), "trace")
                 event, = events(result, "arithmetic")
                 self.assertEqual(event["carryOut"]["value"], carry)