scientific-method-engine 0.6.0__tar.gz → 0.7.0__tar.gz

{scientific_method_engine-0.6.0 → scientific_method_engine-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scientific-method-engine
-Version: 0.6.0
+Version: 0.7.0
 Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
 Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
 Author: kibertoad

{scientific_method_engine-0.6.0 → scientific_method_engine-0.7.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "scientific-method-engine"
 # The release workflow writes the published version from the package's release tag.
-version = "0.6.0"
+version = "0.7.0"
 description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
 readme = "README.md"
 requires-python = ">=3.12"

{scientific_method_engine-0.6.0 → scientific_method_engine-0.7.0}/src/scientific_method_engine/x86/pcode.py

@@ -300,8 +300,11 @@ def evaluate(code, args, bits, site):
             # Identities that keep flag expressions small; constants fold in values.op.
             mask = (1 << bits) - 1
             for x, y in ((a, b), (b, a)):
-                if x.number == 0:
-                    return Value(bits, ("constant", 0), origin) if name == "and" else Value(bits, y.term, origin)
+                if x.number == 0 and name == "and":
+                    return Value(bits, ("constant", 0), origin)
+                if x.number == 0 and boolean(y):
+                    # Flag selections OR a zero arm in; a value operand keeps its OR, as reports write it.
+                    return Value(bits, y.term, origin)
                 if name == "and" and (x.number == mask or (x.number == 1 and boolean(y))):
                     return Value(bits, y.term, origin)
                 if name == "or" and (x.number == mask):

{scientific_method_engine-0.6.0 → scientific_method_engine-0.7.0}/src/scientific_method_engine/x86/pcode_backend.py

@@ -77,7 +77,7 @@ def bit(value, site):
     # SLESS(x, 0) is the sign bit of x; a SLEIGH shift writes it as SLESS(x << k, 0).
     if term[0] == "sless" and term[2] == ("constant", 0):
         inner, shift = term[1], 0
-        if inner[0] == "shl" and inner[2][0] == "constant":
+        if inner[0] == "shl" and inner[2][0] == "constant" and inner not in LEAVES:
             inner, shift = inner[1], inner[2][1]
         width = width_of(inner, value)
         if width is not None and shift < width:
@@ -90,7 +90,7 @@ def bit(value, site):
         inner, mask = term[1][1], term[1][2]
         if mask[0] == "constant" and mask[1] and mask[1] & (mask[1] - 1) == 0:
             position = mask[1].bit_length() - 1
-            if inner[0] in ("shr", "sar") and inner[2][0] == "constant":
+            if inner[0] in ("shr", "sar") and inner[2][0] == "constant" and inner not in LEAVES:
                 position += inner[2][1]
                 inner = inner[1]
             width = width_of(inner, value)
@@ -121,6 +121,9 @@ def width_of(term, value):
 
 # Widths of the terms the current instruction read or computed; set per frame.
 WIDTHS = {}
+# The current instruction's operand values. Field decomposition stops at them, because reports
+# treat each operand as one value even when an earlier instruction built it from fields.
+LEAVES = set()
 
 
 def fields(term, width):
@@ -130,6 +133,8 @@ def fields(term, width):
     (right when negative). Rotates and shifts of joined values take this form in p-code.
     """
     head = term[0]
+    if term in LEAVES:
+        return [(term, width, 0)]
     top = top_bit(term)
     if top is not None:
         return [top]
@@ -252,6 +257,7 @@ class Frame:
 
     def __init__(self, state, ins, image, widths=None, roles=None, present=None):
         self.state, self.ins, self.image = state, ins, image
+        # present(value, frame) may rewrite a written value into the equal term reports use.
         self.present = present
         self.widths = widths or {}
         self.roles = roles or {}
@@ -325,6 +331,7 @@ class Frame:
                                                      addressing_register=register)
                     self.loaded[index] = self.cache[index]
                     WIDTHS.setdefault(self.cache[index].term, self.cache[index].bits)
+                    LEAVES.add(self.cache[index].term)
                 return extract(self.cache[index], d * 8, size * 8)
             location = evidence if d == 0 else op("add", evidence, const(d, evidence.bits), state.at)
             state.access(segment, location, size, value, addressing_register=register)
@@ -345,9 +352,12 @@ class Frame:
 
     def execute(self):
         WIDTHS.clear()
+        LEAVES.clear()
         for index, value in self.before.items():
             WIDTHS[value.term] = value.bits
-        self.run = Run(self.state, self.ops, self.memory, self.constant, present=self.present)
+            LEAVES.add(value.term)
+        present = (lambda value: self.present(value, self)) if self.present else None
+        self.run = Run(self.state, self.ops, self.memory, self.constant, present=present)
         self.run.execute()
         for value in [*self.loaded.values(), *self.run.temps.values(), *self.run.registers.values()]:
             if isinstance(value, Value):  # Real-mode temporaries may hold a segmented Address.
@@ -442,7 +452,13 @@ class Pypcode:
             # A handwritten instruction produced some of these flags; its predicate decides.
             return answer, info
         condition = Run(state, ops, None, flags=flags).execute(stop_at_branch=True)
-        return (None if condition.number is None else bool(condition.number)), info
+        if condition.number is None:
+            return None, info
+        if answer is None:
+            # The handwritten record says why it could not decide; p-code flags decided it.
+            info = {key: value for key, value in info.items() if key != "reason"}
+            info["decidedBy"] = "p-code flags"
+        return bool(condition.number), info
 
     def string_iteration(self, state, ins, operation, width, source_segment, delta_step):
         if operation not in self.mnemonics:
@@ -605,9 +621,25 @@ def compare(state, ins, image):
     state.event("compare", operation=m, left=a.report(), right=b.report())
 
 
+def unfolded(name, value, f, site):
+    """A logic result p-code folded to one operand, written as the operation reports use.
+
+    ``x | 0``, ``x ^ 0`` and ``x & ~0`` equal ``x``; the interpreter folds them, and reports
+    keep the instruction's operation in the result's expression.
+    """
+    if name not in ("and", "or", "xor") or value.number is not None:
+        return value
+    a = f.value(0)
+    b = resize(f.value(1), a.bits)
+    neutral = (1 << a.bits) - 1 if name == "and" else 0
+    if value.bits == a.bits and (b.number == neutral and value.term == a.term or a.number == neutral and value.term == b.term):
+        return Value(value.bits, op(name, a, b, site).term, value.sources)
+    return value
+
+
 def arithmetic(state, ins, image):
     m = ins.mnemonic
-    f = run_plain(state, ins, image)
+    f = run_plain(state, ins, image, present=lambda v, frame: unfolded(m, v, frame, state.at))
     a = f.value(0)
     b = resize(f.value(1), a.bits)
     result = f.result(0)
@@ -705,7 +737,7 @@ def rotate(state, ins, image):
         state.forget_flags(keep_carry=True)
         return
     # Reports write a rotate as its shifted copies, from the leftmost copy for ROL and RCL.
-    f = run_plain(state, ins, image, present=lambda v: arranged(v, m in ("rol", "rcl"), state.at))
+    f = run_plain(state, ins, image, present=lambda v, _: arranged(v, m in ("rol", "rcl"), state.at))
     a = f.value(0)
     value = f.result(0)
     carry = bit(f.run.flags["CF"], state.at)
@@ -733,10 +765,33 @@ def temporary(f, code):
     return f.run.temps[o.output[1]]
 
 
+def low_product(value, site):
+    """The low half of a product of two extended values, as the product at the operands' width.
+
+    SLEIGH writes a two- or three-operand IMUL as the double-width product of the sign-extended
+    operands, truncated; the low half does not depend on the extension, and reports write it as
+    the operand-width product. An operand that already holds a sign extension (after CBW or MOVSX)
+    reaches the product as one extension of the narrower value, which is extended back to the
+    operand's width here.
+    """
+    term = value.term
+    if term[0] != "extract" or term[2] != 0 or term[1][0] != "mul":
+        return value
+    factors = []
+    for factor in term[1][1:]:
+        if factor[0] in ("signExtend", "zeroExtend") and factor[2] <= value.bits:
+            factors.append(resize(Value(factor[2], factor[1]), value.bits, signed=factor[0] == "signExtend"))
+        elif factor[0] == "constant":
+            factors.append(const(factor[1], value.bits))
+        else:
+            return value
+    return Value(value.bits, op("mul", *factors, site).term, value.sources)
+
+
 def multiply(state, ins, image):
     m = ins.mnemonic
     if m == "imul" and len(ins.operands) in (2, 3):
-        f = run_plain(state, ins, image)
+        f = run_plain(state, ins, image, present=lambda v, _: low_product(v, state.at))
         left = resize(f.value(0 if len(ins.operands) == 2 else 1), ins.operands[0].size * 8)
         right = resize(f.value(1 if len(ins.operands) == 2 else 2), left.bits, signed=True)
         result = f.result(0)
@@ -786,10 +841,37 @@ def divide(state, ins, image):
                 quotient=quotient.report(), remainder=remainder.report(), fault=fault)
 
 
+def through_extension(value, source, form):
+    """``value`` as ``form(source)`` when p-code wrote ``form`` of the value ``source`` sign-extends.
+
+    When the source register holds a sign extension (after CBW, CWDE or MOVSX), p-code reads
+    through it to the narrower value it extended, because ``pcode.evaluate`` folds an extension
+    of an extension. Both name the same bits; reports name them in the register the instruction
+    reads.
+    """
+    if value.number is not None or source.term[0] != "signExtend":
+        return value
+    inner = Value(source.term[2], source.term[1])
+    if value.term == form(inner).term:
+        return Value(value.bits, form(source).term, value.sources)
+    return value
+
+
 def conversion(state, ins, image):
     m = ins.mnemonic
     before = {name: state.reg(name) for name in ("al", "ax", "eax")}
-    f = run_plain(state, ins, image)
+
+    def present(value, _):
+        if value.bits not in (16, 32):
+            return value
+        if m in ("cwd", "cdq"):
+            # The high half is the sign bit of AX or EAX.
+            return through_extension(value, before["ax" if value.bits == 16 else "eax"],
+                                     lambda v: resize(extract(v, v.bits - 1, 1), value.bits, signed=True))
+        # CBW and CWDE write the sign extension of AL or AX.
+        return through_extension(value, before["al" if value.bits == 16 else "ax"],
+                                 lambda v: resize(v, value.bits, signed=True))
+    f = run_plain(state, ins, image, present=present)
     (destination, value), = f.run.registers.items()
     wide = value.bits == 32
     if m in ("cbw", "cwde"):

{scientific_method_engine-0.6.0 → scientific_method_engine-0.7.0}/tests/test_oracle.py

@@ -1,7 +1,7 @@
 """Unicorn oracle cases per instruction group (ADR 0003). Synthetic machine code only."""
 import unittest
 
-from oracle import check
+from oracle import STACK, check, configuration, run_report
 
 DATA = {"ds": 0x3000, "es": 0x4000}
 SAME = {"ds": 0x4000, "es": 0x4000}
@@ -70,8 +70,31 @@ class ArithmeticAndLogic(unittest.TestCase):
 
     def test_counted_loop_exits_on_decrement_flags(self):
         # inc ax; dec cx; jnz back: the handwritten backend forgets DEC's flags and splits.
-        check(self, "b90300 b80000 40 49 75fc c3", resolved=("ax", "cx"),
-              extended="DEC flags from p-code resolve the loop exit")
+        result = check(self, "b90300 b80000 40 49 75fc c3", resolved=("ax", "cx"),
+                       extended="DEC flags from p-code resolve the loop exit")
+        branches = [e for e in result["paths"][0]["events"] if e["kind"] == "branch"]
+        self.assertEqual([e["taken"] for e in branches], [True, True, False])
+        # A branch p-code decided does not keep the handwritten backend's reason for not deciding it.
+        for e in branches:
+            self.assertNotIn("reason", e)
+            self.assertEqual((e["decidedBy"], e["flagProducer"]), ("p-code flags", 7))
+
+    def test_undecided_branch_keeps_the_reason(self):
+        # inc ax; jnz: AX is unknown, so neither backend decides the branch and both arms run.
+        data = bytes.fromhex("40 7501 90 c3".replace(" ", ""))
+        result = run_report(data, configuration(data, dict(STACK)), "trace")
+        branches = [e for path in result["paths"] for e in path["events"] if e["kind"] == "branch"]
+        self.assertEqual(sorted(e["taken"] for e in branches), [False, True])
+        for e in branches:
+            self.assertEqual(e["reason"], "flag producer unresolved")
+            self.assertNotIn("decidedBy", e)
+
+    def test_neutral_operands_keep_the_operation(self):
+        # BX is unknown to the engine, so each result stays an expression over it.
+        result = check(self, "b80100 80cf00 83f300 81e3ffff 09c3 c3", resolved=("ax",))
+        bx = result["paths"][0]["registers"]["bx"]["expression"]
+        # or(and(xor(..., 0), 0xffff), ax): neither the XOR with 0 nor the AND with ~0 folds away.
+        self.assertEqual((bx[0], bx[1][0], bx[1][1][0]), ("or", "and", "xor"))
 
     def test_byte_forms_and_memory(self):
         check(self, "bb1000 c707ff00 8007 01 fe07 8b07 b4ff 00e0 c3", registers=DATA, resolved=("ax",))
@@ -93,6 +116,23 @@ class ShiftsAndRotates(unittest.TestCase):
     def test_rotates_through_carry_and_full_counts(self):
         check(self, "b83412 f9 c1d00c f8 c1d805 c1c010 c1c808 83d000 c3", resolved=("ax",))
 
+    def test_chained_double_word_shifts(self):
+        # DX:AX shifted right twice through CF; the engine leaves both unknown, so each RCR's
+        # result and carry stay expressions over the previous one.
+        result = check(self, "b90100 d1ea d1d8 d1ea d1d8 c3", resolved=("cx",))
+        ax = result["paths"][0]["registers"]["ax"]["expression"]
+        self.assertEqual(ax[0], "or")
+        # The second RCR shifts out bit 0 of the first RCR's result, not bit 1 of the initial AX.
+        rotates = [e for e in result["paths"][0]["events"] if e.get("operation") == "rcr"]
+        self.assertEqual(rotates[1]["carryOut"]["expression"][1][0], "or")
+
+    def test_double_word_shift_from_a_zero_high_half(self):
+        # DX starts at zero; RCL carries AX's unknown top bits into it. The second RCL shifts out
+        # DX's bit 15, which is still zero; ADC moves that carry into BX for Unicorn to check.
+        result = check(self, "b90100 31d2 d1e0 d1d2 d1e0 d1d2 bb0000 11db c3", resolved=("cx", "bx"),
+                       extended="a rotate's carry out folded from the operand's known bits")
+        self.assertEqual(result["paths"][0]["registers"]["dx"]["expression"][0], "or")
+
     def test_memory_operands(self):
         # RCL by one on memory: Capstone reports its implicit count with a size of 0.
         check(self, "bb1000 c7070180 d107 c12f04 f9 d117 8b07 c7070180 f8 d017 8b07 c3", registers=DATA, resolved=("ax",))
@@ -102,6 +142,20 @@ class MultiplyAndDivide(unittest.TestCase):
     def test_products(self):
         check(self, "b80300 bb0500 f7e3 b0f0 b304 f6eb 6bc3fd 0fafc3 c3", resolved=("ax", "dx"))
 
+    def test_low_products_of_unknown_operands(self):
+        # The engine leaves CX unknown; the low products keep the operand-width form.
+        result = check(self, "b80300 0fafc8 6bd1fd 69d90500 c3")
+        registers = result["paths"][0]["registers"]
+        self.assertEqual(registers["cx"]["expression"][0], "mul")
+        self.assertEqual(registers["dx"]["expression"][0], "mul")
+
+    def test_low_products_of_extended_operands(self):
+        # AX holds the sign extension of an unknown byte; the products name AX, not the byte.
+        result = check(self, "a00000 98 0fafc8 6bd0fd b80100 c3", registers=DATA, resolved=("ax",))
+        registers = result["paths"][0]["registers"]
+        self.assertEqual(registers["cx"]["expression"][0], "mul")
+        self.assertEqual(registers["dx"]["expression"][0], "mul")
+
     def test_quotients(self):
         check(self, "ba0000 b86400 bb0700 f7f3 89c1 b8f6ff 99 f7fb c3", resolved=("ax", "cx", "dx"))
 
@@ -114,6 +168,19 @@ class Conversions(unittest.TestCase):
     def test_sign_extensions(self):
         check(self, "b080 98 89c3 99 b8ff7f 6698 6699 c3", resolved=("ax", "bx", "dx", "eax", "edx"))
 
+    def test_sign_fill_of_an_extended_byte(self):
+        # The byte at DS:0 is unknown to the engine; CWD names the sign bit of AX.
+        result = check(self, "a00000 98 99 b80100 c3", registers=DATA, resolved=("ax",))
+        dx = result["paths"][0]["registers"]["dx"]["expression"]
+        self.assertEqual(tuple(dx[1][2:4]), (15, 1))
+
+    def test_sign_extension_of_an_extended_byte(self):
+        # The byte at DS:0 is unknown to the engine; CWDE extends AX and CDQ names the sign bit of EAX.
+        result = check(self, "a00000 98 6698 6699 b80100 c3", registers=DATA, resolved=("ax",))
+        cwde = next(e for e in result["paths"][0]["events"] if e["kind"] == "conversion" and e["destinationBits"] == 32)
+        self.assertEqual(cwde["result"]["expression"][1][2:4], (8, 16))
+        self.assertEqual(tuple(result["paths"][0]["registers"]["edx"]["expression"][1][2:4]), (31, 1))
+
 
 class FlagsAndDirection(unittest.TestCase):
     def test_direction_controls_string_steps(self):