smallworld-re 1.0.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

smallworld/analyses/colorizer_def_use.py

@@ -0,0 +1,217 @@
+import logging
+import re
+
+import networkx as nx
+
+from .. import hinting
+from . import analysis
+
+logger = logging.getLogger(__name__)
+
+
+class DefUseGraph(nx.MultiDiGraph):
+    def add_def_use(self, def_node, use_node, def_info, use_info, color):
+        # logger.info(f"def use def_node={def_node} use_node={use_node}")
+        # logger.info(f"  def_info={def_info}")
+        # logger.info(f"  use_info={use_info}")
+        self.add_edges_from(
+            [
+                (
+                    def_node,
+                    use_node,
+                    {"def_info": def_info, "use_info": use_info, "color": color},
+                )
+            ]
+        )
+
+
+class ColorizerDefUse(analysis.Analysis):
+    name = "colorizer_def_use_graph"
+    description = "assemble a def use graph from colorizer summary hints"
+    version = "0.0.1"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.new_hints = []
+        self.not_new_hints = []
+        self.hinter.register(
+            hinting.DynamicRegisterValueSummaryHint, self.collect_hints
+        )
+        self.hinter.register(hinting.DynamicMemoryValueSummaryHint, self.collect_hints)
+
+    def collect_hints(self, hint: hinting.Hint):
+        if (
+            type(hint) is hinting.DynamicRegisterValueSummaryHint
+            or type(hint) is hinting.DynamicMemoryValueSummaryHint
+        ):
+            if hint.new:
+                self.new_hints.append(hint)
+            else:
+                self.not_new_hints.append(hint)
+
+    def run(self, machine):
+        self.du_graph = DefUseGraph()
+        color2genesis = {}
+
+        def hint_dv_info(hint):
+            if type(hint) is hinting.DynamicRegisterValueSummaryHint:
+                return {
+                    "new": hint.new,
+                    "count": hint.count,
+                    "num_micro_executions": hint.num_micro_executions,
+                    "type": "reg",
+                    "is_read": hint.use,
+                    "reg_name": hint.reg_name,
+                    "color": hint.color,
+                }
+            elif type(hint) is hinting.DynamicMemoryValueSummaryHint:
+                return {
+                    "new": hint.new,
+                    "count": hint.count,
+                    "num_micro_executions": hint.num_micro_executions,
+                    "type": "mem",
+                    "is_read": hint.use,
+                    "base": hint.base,
+                    "index": hint.index,
+                    "scale": hint.scale,
+                    "offset": hint.offset,
+                    "color": hint.color,
+                }
+            else:
+                assert 1 == 0
+
+        for hint in self.new_hints:
+            # this is a new color so its either an input or a write of a computed value
+
+            # some sanity checking
+            assert "def" in hint.message
+            assert hint.color not in color2genesis
+            assert (hint.use and ("read" in hint.message)) or (
+                (not hint.use) and ("write" in hint.message)
+            )
+
+            self.du_graph.add_node(hint.pc)
+
+            dv_info = hint_dv_info(hint)
+
+            if hint.use:
+                # this is a read
+                # hallucinate a node representing the creation of that color
+                color_node = f"input color-{hint.color}"
+                assert dv_info["type"] == "reg" or dv_info["type"] == "mem"
+
+                if dv_info["type"] == "reg":
+                    color_node = color_node + " " + dv_info["reg_name"]
+                if dv_info["type"] == "mem":
+                    color_node = color_node + " *(" + dv_info["base"]
+                    if dv_info["index"] != "None":
+                        color_node += f"{dv_info['scale']}*{dv_info['index']}"
+                    if dv_info["offset"] > 0:
+                        color_node += f"{dv_info['offset']}"
+                    color_node += ")"
+                color_node += "_init"
+                self.du_graph.add_node(color_node)
+                # record mapping from this color to its creation info
+                color2genesis[hint.color] = (color_node, dv_info)
+                # and an edge between that color node and this instruction
+                self.du_graph.add_def_use(
+                    color_node, hint.pc, color_node, dv_info, hint.color
+                )
+            else:
+                # this is a write of a computed value
+                # record mapping from this color to its creation info
+                color2genesis[hint.color] = (hint.pc, dv_info)
+
+        for hint in self.not_new_hints:
+            # not a new color.  so its a flow
+            self.du_graph.add_node(hint.pc)
+            # can't be a def
+            assert "def" not in hint.message
+            # we should never see !new && !use since that is just a value copy
+            # assert hint.use
+            if not hint.use:
+                pass
+            else:
+                dv_info = hint_dv_info(hint)
+                (def_node, def_info) = color2genesis[hint.color]
+                self.du_graph.add_def_use(
+                    def_node, hint.pc, def_info, dv_info, hint.color
+                )
+
+        # hint out the def-use graph
+        self.hinter.send(
+            hinting.DefUseGraphHint(
+                graph=nx.node_link_data(self.du_graph, edges="links"),
+                message="concrete-summary-def-use-graph",
+            )
+        )
+
+        with open("colorizer_def_use.dot", "w") as dot:
+
+            def writeit(foo):
+                dot.write(foo + "\n")
+
+            writeit("digraph{")
+            writeit(" rankdir=LR")
+
+            node2nodeid = {}
+            for node in self.du_graph.nodes:
+                # 4461 is pc
+                # {"id": 4461},
+                # An input color rsp is register
+                # {"id": "input color-1 rsp_init"},
+                node_id = f"node_{len(node2nodeid)}"
+                if type(node) is int:
+                    # nodeinfo = ("instruction", f"0x{node}")
+                    writeit(f'  {node_id} [label="0x{node:x}"]')
+                else:
+                    assert type(node) is str
+                    foo = re.search("input color-([0-9]+) (.*)_init", node)
+                    assert foo is not None
+                    (cns, reg) = foo.groups()
+                    cn = int(cns)
+                    # nodeinfo = ("input", (cn, reg))
+                    writeit(f'  {node_id} [color="blue", label="input({reg})"]')
+                node2nodeid[node] = node_id
+
+            di = nx.get_edge_attributes(self.du_graph, "def_info")
+            ui = nx.get_edge_attributes(self.du_graph, "use_info")
+            for e in self.du_graph.edges:
+                if type(di[e]) is str:
+                    foo = re.search("color-([0-9]+) (.*)_init", di[e])
+                    (cns, reg) = foo.groups()
+                    cn = int(cns)
+                    assert cn == ui[e]["color"]
+                else:
+                    assert di[e]["color"] == ui[e]["color"]
+                (src, dst, k) = e
+                cn = ui[e]["color"]
+                tl = ""
+
+                def i2s(inf):
+                    lab = ""
+                    if "type" in inf:
+                        if inf["type"] == "reg":
+                            lab = inf["reg_name"]
+                        if inf["type"] == "mem":
+                            lab = "["
+                            if inf["base"] != "None":
+                                lab += inf["base"]
+                            if inf["index"] != "None":
+                                lab += f'+{inf["scale"]}*{inf["index"]}'
+                            if inf["offset"] != 0:
+                                lab += f'+{inf["offset"]:x}'
+                            lab += "]"
+                    return lab
+
+                hl = i2s(ui[e])
+                tl = i2s(di[e])
+
+                writeit(
+                    f'  {node2nodeid[src]} -> {node2nodeid[dst]} [label="{cn}",headlabel="{hl}",taillabel="{tl}"]'
+                )
+
+            writeit("}\n")
+
+    def get_graph(self):
+        return self.du_graph

smallworld/analyses/colorizer_summary.py

@@ -1,100 +1,190 @@
-# mypy: ignore-errors
-import logging
-
-import networkx as nx
+import sys
+import typing
 
 from .. import hinting
 from . import analysis
 
-hinter = hinting.get_hinter(__name__)
-logger = logging.getLogger(__name__)
+# goal here is to summarize across multiple micro execution if two
+# hints map to the same key then they are in same equivalence class so
+# we elide away things that will be specific to a run like micro exec
+# number, color (actual dyn value), ptr addresses, etc.
 
 
-class DefUseGraph(nx.MultiDiGraph):
-    def add_def_use(self, def_node, use_node, def_info, use_info, color):
-        self.add_edges_from(
-            [
-                (
-                    def_node,
-                    use_node,
-                    {"def_info": def_info, "use_info": use_info, "color": color},
-                )
-            ]
+def compute_dv_key(hint):
+    if type(hint) is hinting.DynamicRegisterValueHint:
+        return (
+            ("type", "reg"),
+            ("pc", hint.pc),
+            ("size", hint.size),
+            ("use", hint.use),
+            ("new", hint.new),
+            ("reg_name", hint.reg_name),
         )
+    elif type(hint) is hinting.DynamicMemoryValueHint:
+        return (
+            ("type", "mem"),
+            ("pc", hint.pc),
+            ("size", hint.size),
+            ("use", hint.use),
+            ("new", hint.new),
+            ("base", hint.base),
+            ("index", hint.index),
+            ("scale", hint.scale),
+            ("offset", hint.offset),
+        )
+    elif type(hint) is hinting.MemoryUnavailableHint:
+        return (
+            ("type", "mem_unavail"),
+            ("pc", hint.pc),
+            ("size", hint.size),
+            ("is_read", hint.is_read),
+            ("base", hint.base_reg_name),
+            ("index", hint.index_reg_name),
+            ("scale", hint.scale),
+            ("offset", hint.offset),
+        )
+    elif type(hint) is hinting.EmulationException:
+        return (("type", "emu_fail"), ("pc", hint.pc), ("exception", hint.exception))
 
-
-du_graph = DefUseGraph()
-color2nodeDv = {}
+    else:
+        # should never happen
+        assert (
+            (type(hint) is hinting.DynamicRegisterValueHint)
+            or (type(hint) is hinting.DynamicMemoryValueHint)
+            or (type(hint) is hinting.MemoryUnavailableHint)
+            or (type(hint) is hinting.EmulationException)
+        )
 
 
-class ColorizerSummary(analysis.Filter):
+class ColorizerSummary(analysis.Analysis):
     name = "colorizer_summary"
-    description = "summarized version of colorizer"
+    description = "collect and summarize colorizer output across micro executions"
     version = "0.0.1"
 
-    @staticmethod
-    def dynamic_value_summary(hint: hinting.Hint):
-        #        print(hint)
-        # instr_node = hint.instruction
-        # du_graph.add_node(instr_node)
-        du_graph.add_node(hint.pc)
-        if type(hint) is hinting.DynamicRegisterValueProbHint:
-            dv_info = [
-                ("prob", hint.prob),
-                ("type", "reg"),
-                ("reg_name", hint.reg_name),
-            ]
-        elif type(hint) is hinting.DynamicMemoryValueProbHint:
-            dv_info = [
-                ("prob", hint.prob),
-                ("type", "mem"),
-                ("base", hint.base),
-                ("index", hint.index),
-                ("scale", hint.scale),
-                ("offset", hint.offset),
-            ]
-        else:
-            assert 1 == 0
-        if hint.new:
-            # this is a new color so its either an input or a write of a computed value
-            assert "def" in hint.message
-            assert hint.color not in color2nodeDv
-            if hint.use:
-                assert "read" in hint.message
-                # color2nodeDv[hint.color] = (instr_node, dv_info)
-                color2nodeDv[hint.color] = (hint.pc, dv_info)
-            else:
-                assert "write" in hint.message
-                color2nodeDv[hint.color] = (hint.pc, dv_info)
-            # if its a read then
-            # hallucinate a node representing the creation of that color
-            if hint.use:
-                color_node = f"input color-{hint.color}"
-                du_graph.add_node(color_node)
-                color2nodeDv[hint.color] = (color_node, None)
-                # and an edge between that and this instruction
-                du_graph.add_def_use(
-                    color_node, hint.pc, color_node, dv_info, hint.color
-                )
-        else:
-            # not a new color.  so its a flow
-            assert "def" not in hint.message
-            # we should never see new && !use since that is just a value copy
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hint_list = []
+        self.exec_ids = set([])
+        self.hinter.register(hinting.DynamicRegisterValueHint, self.collect_hints)
+        self.hinter.register(hinting.DynamicMemoryValueHint, self.collect_hints)
+        self.hinter.register(hinting.MemoryUnavailableHint, self.collect_hints)
+        self.hinter.register(hinting.EmulationException, self.collect_hints)
 
-            # assert hint.use
-            (def_node, def_dv_info) = color2nodeDv[hint.color]
-            du_graph.add_def_use(def_node, hint.pc, def_dv_info, dv_info, hint.color)
+    def collect_hints(self, hint):
+        self.hint_list.append(hint)
+        self.exec_ids.add(hint.exec_id)
 
-    def activate(self):
-        # pdb.set_trace()
-        print("activating colorizer_summary")
-        self.listen(hinting.DynamicRegisterValueProbHint, self.dynamic_value_summary)
-        self.listen(hinting.DynamicMemoryValueProbHint, self.dynamic_value_summary)
+    def run(self, machine):
+        # Establish true colors 1, 2, 3, ...
+        #
+        # The colors we have in hints are random initial reg or memory
+        # values or values computed from those. This means they don't
+        # correspond across micro-executions. We need them to do so if
+        # we are to summarize.
+        #
+        truecolors = set([])
+        dvh2truecolor = {}
+        color2truecolor = {}
+        for hint in self.hint_list:
+            if hasattr(hint, "color"):
+                if hint.new:
+                    # This is a hint with a "new" color meaning the
+                    # hint time is when the color was first
+                    # observed. If we are trying to match up such new
+                    # colors across micro executions, we do so by
+                    # matching up the dv keys for them at this
+                    # time of first observation.
+                    dvk = compute_dv_key(hint)
+                    dvh = hash(dvk) % ((sys.maxsize + 1) * 2)
+                    if dvh not in dvh2truecolor:
+                        # we need to establish a color for this dvh
+                        next_color = 1 + len(truecolors)
+                        truecolors.add(next_color)
+                        dvh2truecolor[dvh] = next_color
+                    # make sure to map every hint color to its true color
+                    color2truecolor[hint.color] = dvh2truecolor[dvh]
+                else:
+                    assert hint.color in color2truecolor
 
-    def deactivate(self):
-        # import pdb
-        # pdb.set_trace()
+        # use compute_dv_key to put keys into equiv classes
+        # which works for both new and not-new hints
+        # start off by collecting set of all such classes
+        all_hint_keys = set([])
+        hk_exemplar = {}
+        for hint in self.hint_list:
+            hk = compute_dv_key(hint)
+            all_hint_keys.add(hk)
+            # keep one exemplar for each equivalence class
+            if hk not in hk_exemplar:
+                hk_exemplar[hk] = hint
 
-        jsg = nx.node_link_data(du_graph)
-        with open("dugraph.json", "w") as j:
-            j.write(str(jsg))
+        hint_keys_sorted = sorted(list(all_hint_keys))
+
+        # given the equivalence classes established by `compute_dv_key`, determine
+        # which of those were observed in each micro-execution
+        hk_observed: typing.Dict[
+            int, typing.Set[typing.Tuple[int, bool, str, bool, str, str, str, int, int]]
+        ] = {}
+        for hint in self.hint_list:
+            if hint.exec_id not in hk_observed:
+                hk_observed[hint.exec_id] = set([])
+                # this hint key was observed in micro execution me
+            hk_observed[hint.exec_id].add(compute_dv_key(hint))
+
+        # get counts for hint across micro executions
+        hk_c = {}
+        for hk in hint_keys_sorted:
+            hk_c[hk] = 0
+            for exec_id in self.exec_ids:
+                for hk2 in hk_observed[exec_id]:
+                    if hk == hk2:
+                        hk_c[hk] += 1
+
+        for hk in hint_keys_sorted:
+            hint = hk_exemplar[hk]
+            if type(hint) is hinting.DynamicRegisterValueHint:
+                self.hinter.send(
+                    hinting.DynamicRegisterValueSummaryHint(
+                        pc=hint.pc,
+                        reg_name=hint.reg_name,
+                        color=color2truecolor[hint.color],
+                        size=hint.size,
+                        use=hint.use,
+                        new=hint.new,
+                        count=hk_c[hk],
+                        num_micro_executions=len(self.exec_ids),
+                        message=hint.message + "-summary",
+                    )
+                )
+            if type(hint) is hinting.DynamicMemoryValueHint:
+                self.hinter.send(
+                    hinting.DynamicMemoryValueSummaryHint(
+                        pc=hint.pc,
+                        size=hint.size,
+                        base=hint.base,
+                        index=hint.index,
+                        scale=hint.scale,
+                        offset=hint.offset,
+                        color=color2truecolor[hint.color],
+                        use=hint.use,
+                        new=hint.new,
+                        message=hint.message + "-summary",
+                        count=hk_c[hk],
+                        num_micro_executions=len(self.exec_ids),
+                    )
+                )
+            if type(hint) is hinting.MemoryUnavailableHint:
+                self.hinter.send(
+                    hinting.MemoryUnavailableSummaryHint(
+                        is_read=hint.is_read,
+                        size=hint.size,
+                        base_reg_name=hint.base_reg_name,
+                        index_reg_name=hint.index_reg_name,
+                        offset=hint.offset,
+                        scale=hint.bscale,
+                        pc=hint.pc,
+                        count=hk_c[hk],
+                        num_micro_executions=len(self.exec_ids),
+                        message=hint.message + "-summary",
+                    )
+                )

smallworld/analyses/field_detection/field_analysis.py

@@ -19,7 +19,6 @@ from .hints import (
 )
 
 log = logging.getLogger(__name__)
-hinter = hinting.get_hinter(__name__)
 
 # Tell angr to be quiet please
 logging.getLogger("angr").setLevel(logging.WARNING)
@@ -121,7 +120,7 @@ class FieldDetectionMixin(underlays.AnalysisUnderlay):
                 size=size,
                 expr=str(expr),
             )
-            hinter.info(hint)
+            self.hinter.self(hint)
             if self.halt_on_hint:
                 raise PathTerminationSignal()
             else:
@@ -204,7 +203,7 @@ class FieldDetectionMixin(underlays.AnalysisUnderlay):
                             start=start // 8,
                             end=end // 8,
                         )
-                hinter.info(hint)
+                self.hinter.send(hint)
                 if self.halt_on_hint:
                     raise PathTerminationSignal()
                 else:
@@ -226,7 +225,7 @@ class FieldDetectionMixin(underlays.AnalysisUnderlay):
             access="read",
             expr=str(expr),
         )
-        hinter.info(hint)
+        self.hinter.send(hint)
         if self.halt_on_hint:
             raise PathTerminationSignal()
         else:
@@ -288,7 +287,7 @@ class FieldDetectionMixin(underlays.AnalysisUnderlay):
                         end=end,
                         expr=str(expr),
                     )
-                    hinter.info(hint)
+                    self.hinter.send(hint)
                     bad = True
         if bad and good:
             log.error("Write was complete and partial; your labels overlap.")
@@ -317,7 +316,7 @@ class FieldDetectionMixin(underlays.AnalysisUnderlay):
                 access="write",
                 expr=str(expr),
             )
-            hinter.info(hint)
+            self.hinter.send(hint)
             if self.halt_on_hint:
                 raise PathTerminationSignal()
             else:
@@ -362,7 +361,7 @@ class FieldDetectionMixin(underlays.AnalysisUnderlay):
                             size=val.get_size(),
                             label=label,
                         )
-                        hinter.info(hint)
+                        self.hinter.send(hint)
 
                         if label in fda.fda_labels:
                             log.error(
@@ -420,7 +419,7 @@ class FieldDetectionMixin(underlays.AnalysisUnderlay):
         filt.deactivate()
 
 
-class FieldDetectionFilter(analyses.Filter):
+class FieldDetectionFilter(analyses.Analysis):
     """Secondary field definition analysis.
 
     This picks up patterns that aren't noticeable from any

smallworld/analyses/field_detection/hints.py

@@ -4,7 +4,7 @@ import typing
 from ... import hinting
 
 
-class ClaripySerializable(hinting.Serializable):
+class ClaripySerializable(hinting.Hint):
     """Serializable wrapper that allows serializing claripy expressions in hints."""
 
     @classmethod

smallworld/analyses/field_detection/malloc.py

@@ -14,7 +14,7 @@ class MallocModel(state.models.Model):
     platform = platforms.Platform(
         platforms.Architecture.X86_64, platforms.Byteorder.LITTLE
     )
-    abi = platforms.ABI.SYSTEMV
+    abi = platforms.ABI.NONE
 
     _arg1_for_arch = {
         platforms.Architecture.AARCH64: "x0",
@@ -201,7 +201,7 @@ class FreeModel(state.models.Model):
     platform = platforms.Platform(
         platforms.Architecture.X86_64, platforms.Byteorder.LITTLE
     )
-    abi = platforms.ABI.SYSTEMV
+    abi = platforms.ABI.NONE
 
     def __init__(self, address: int):
         super().__init__(address)