smallworld-re 1.0.3__py3-none-any.whl → 2.0.0__py3-none-any.whl

smallworld/analyses/colorizer.py

@@ -1,18 +1,14 @@
-import base64
 import copy
+import hashlib
 import logging
 import random
+import struct
 import typing
 
 import capstone
 
-from .. import hinting, state
-from ..emulators import (
-    UnicornEmulationMemoryReadError,
-    UnicornEmulationMemoryWriteError,
-    UnicornEmulator,
-)
-from ..exceptions import AnalysisRunError, EmulationBounds
+from .. import hinting, platforms, state
+from ..exceptions import AnalysisRunError  # , EmulationBounds
 from ..instructions import (
     BSIDMemoryReferenceOperand,
     Instruction,
@@ -20,40 +16,112 @@ from ..instructions import (
     RegisterOperand,
 )
 from . import analysis
+from .trace_execution import TraceExecution, TraceExecutionCBPoint
 
 logger = logging.getLogger(__name__)
-hinter = hinting.get_hinter(__name__)
 
-MIN_ACCEPTABLE_COLOR_INT = 20
-BAD_COLOR = "BAD_COLOR"
 
-Colors = typing.Dict[str, typing.Tuple[Operand, int, int, Instruction, int]]
+MIN_ACCEPTABLE_COLOR_INT = 0x20
+BAD_COLOR = (2**64) - 1
+
+Colors = typing.Dict[int, typing.Tuple[Operand, int, Instruction, int]]
+
+Shad = typing.Dict[int, typing.Tuple[int, bool, int]]
+
+
+def randomize_uninitialized(
+    machine: state.Machine, seed: int = 123456, extra_regs: typing.List[str] = []
+) -> state.Machine:
+    """Consider all parts of the machine that can be written to (registers
+    + memory regions). Write random values to any bytes in those machine
+    parts which are currently uninitialized. So this only works if we
+    have a way to tell if registers or memory have not been initialized.
+
+    Randomize all general purpose regs (plus regs in list of
+    extra_regs arg) that have not already been set.  Also, for any
+    Heap, Stack, RawMemory or Memory regions, randomize any bytes that
+    have not been set. This last part is to come since we don't
+    currently have a way to tell which parts of memory have been
+    written and which parts have not. Further, there are kinds of
+    memory (Stack) which currently will break if we randomize all
+    bytes.
+
+    Note that, for a register, it is either set or not.  We can't tell
+    if, say edx part of rdx has been set.
+
+    """
+    random.seed(seed)
+    logger.setLevel(logging.DEBUG)
+
+    # if logger.getEffectiveLevel() >= logging.DEBUG:
+    m = hashlib.md5()
+    platform = machine.get_platform()
+    machine_copy = copy.deepcopy(machine)
+    pdefs = platforms.defs.PlatformDef.for_platform(platform)
+    reg_names = list(pdefs.registers.keys())
+    reg_names.sort()
+
+    def get_reg(machine, reg_name):
+        cpu = machine.get_cpu()
+        for x in cpu:
+            if isinstance(x, state.Register):
+                if x.name == reg_name:
+                    return x
+        return None
+
+    for name in reg_names:
+        if (name in pdefs.general_purpose_registers) or (name in extra_regs):
+            reg = get_reg(machine_copy, name)
+            if reg.get_content() is None:
+                # this means reg is uninitialized; ok to randomize
+                v = random.randint(0, (1 << (8 * reg.size)) - 1)
+                reg.set(v)
+                # logger.info(f"randomize_uninitialized setting {reg}")
+                # if logger.getEffectiveLevel() >= logging.DEBUG:
+                m.update(str(v).encode("utf-8"))
+
+    for el in machine_copy:
+        if isinstance(m, state.memory.Memory):
+            if isinstance(m, state.memory.Executable):
+                # nothing to randomize here
+                continue
+            # To come, I guess
+            # we'd like to randomize anything uninitialized...
+            pass
+    # if logger.getEffectiveLevel() >= logging.DEBUG:
+    logger.info(f"digest of changes made to machine: {m.hexdigest()}")
+
+    logger.setLevel(logging.INFO)
+
+    return machine_copy
 
 
 class Colorizer(analysis.Analysis):
-    """A simple kind of data flow analysis via tracking distinct values (colors)
-    and employing instruction use/def analysis
-
-    We run multiple micro-executions of the code starting from same entry. At
-    the start of each, we randomize register values that have not already been
-    initialized. We maintain a "colors" map from values to when we first
-    observed them. This map is initially empty. Before emulating an instruction,
-    we examine the values (registers and memory) it will read. If any are NOT in
-    the colors map, that is the initial sighting of that value and we emit a
-    hint to that effect. If any color IS already in the map, then that is a flow
-    from the time at which that value was first observed to this
-    instruction. Similarly, after emulating an instruction, we examine every
-    value (register and memory) written. If a value is not in the colors map, it
-    is a new, computed result and we hint about its creation. If it is in the
-    colors map, we do nothing since it just a copy.
+    """
+    A simple kind of data flow analysis via tracking distinct values
+    (colors) and employing instruction use/def analysis
+
+    We run multiple micro-executions of the code starting from same
+    entry. At the start of each, we randomize register values that
+    have not already been initialized. We maintain a "colors" map from
+    dynamic values to when/where we first observed them. This map is
+    initially empty. Before emulating an instruction, we examine the
+    values (registers and memory) it will read. If any are not in the
+    colors map, that is the initial sighting of that value and we emit
+    a hint to that effect and add a color to the map. If any color is
+    already in the map, then that is a def-use flow from the time or
+    place at which that value was first observed to this instruction.
+    Similarly, after emulating an instruction, we examine every value
+    written to a register or memory. If a value is not in the colors
+    map, it is a new, computed result and we hint about its creation
+    and add it to the map. If it is in the colors map, we do nothing
+    since it just a copy.
 
     Whilst looking at reads and writes for instructions, we hint if any
     correspond to unavailable memory.
 
     Arguments:
-        num_micro_executions: The number of micro-executions to run.
         num_insns: The number of instructions to micro-execute.
-        seed: Random seed for test stability, or None.
 
     """
 
@@ -64,357 +132,134 @@ class Colorizer(analysis.Analysis):
     def __init__(
         self,
         *args,
-        num_micro_executions: int = 5,
+        exec_id: int,
         num_insns: int = 200,
-        seed: typing.Optional[int] = 99,
-        **kwargs
-        #        self, *args, num_micro_executions: int = 1, num_insns: int = 10, **kwargs
+        **kwargs,
     ):
         super().__init__(*args, **kwargs)
-        # Create our own random so we can avoid contention.
-        self.random = random.Random()
-        self.seed = seed
-        self.num_micro_executions = num_micro_executions
+        self.exec_id = exec_id
         self.num_insns = num_insns
+        self.colors: Colors = {}
+        self.shadow_register: typing.Dict[str, Shad] = {}
+        self.shadow_memory: Shad = {}
+        # self.edge: typing.Dict[int, typing.Dict[int, typing.Tuple[str, int, int]]] = {}
 
-    def _get_instr_at_pc(self, pc: int) -> capstone.CsInsn:
-        code = self.emu.read_memory(pc, 15)  # longest possible instruction
+    def _get_instr_at_pc(self, emu, pc: int) -> capstone.CsInsn:
+        code = emu.read_memory(pc, 15)  # longest possible instruction
         if code is None:
             raise AnalysisRunError(
                 "Unable to read next instruction out of emulator memory"
             )
-        (insns, disas) = self.emu._disassemble(code, pc, 2)
+        (insns, disas) = emu._disassemble(code, pc, 2)
         insn = insns[0]
         return insn
 
     def _operand_size(self, operand: Operand) -> int:
         if type(operand) is RegisterOperand:
             # return size of a reg based on its name
-            return getattr(self.cpu, operand.name).size
+            return self.pdef.registers[operand.name].size
         elif type(operand) is BSIDMemoryReferenceOperand:
             # memory operand knows its size
             return operand.size
         return 0
 
     def run(self, machine: state.Machine) -> None:
-        # note that start pc is in start_cpustate
-
         # collect hints for each microexecution, in a list of lists
-        hint_list_list: typing.List[typing.List[hinting.Hint]] = []
 
         self.orig_machine = copy.deepcopy(machine)
         self.orig_cpu = self.orig_machine.get_cpu()
         self.platform = self.orig_cpu.platform
-
-        for i in range(self.num_micro_executions):
-            logger.info("-------------------------")
-            logger.info(f"micro exec #{i}")
-
-            if self.seed is not None:
-                self.random.seed(a=self.seed)
-
-            self.machine = copy.deepcopy(self.orig_machine)
-            self.cpu = self.machine.get_cpu()
-            self.emu = UnicornEmulator(self.platform)
-            self.machine.apply(self.emu)
-
-            # initialize registers with random values
-            self._randomize_registers()
-
-            # map from color values to first use / def
-            self.colors: Colors = {}
-
-            hint_list: typing.List[hinting.Hint] = []
-            for j in range(self.num_insns):
-                logger.info(f"instr_count = {j}")
-                # obtain instr about to be emulated
-                pc = self.emu.read_register("pc")
-                if pc in self.emu.get_exit_points():
-                    break
-                cs_insn = self._get_instr_at_pc(pc)
-                sw_insn = Instruction.from_capstone(cs_insn)
-
-                logger.debug(sw_insn)
-
-                # pull state back out of the emulator for inspection
-                m = copy.deepcopy(self.machine)
-                m.extract(self.emu)
-                self.cpu = m.get_cpu()
-                #                self.cpu = copy.deepcopy(self.machine).extract(self.emu).get_cpu()
-                # curr_machine = copy.deepcopy(self.machine)
-                # curr_machine.extract(self.emu)
-                # curr_machine = self.eself.cpu.load(self.emu)
-                # self.cpu = curr_machien.get_cpu()
-
-                # print(f"pc={pc:x} {sw_insn}")
-                # import pdb
-                # pdb.set_trace()
-
-                reads: typing.List[typing.Tuple[Operand, str, int]] = []
-                for read_operand in sw_insn.reads:
-                    logger.debug(f"pc={pc:x} read_operand={read_operand}")
-
-                    if (
-                        type(read_operand) is RegisterOperand
-                        and read_operand.name == "rflags"
-                    ):
-                        continue
-
-                    sz = self._operand_size(read_operand)
-                    if type(read_operand) is BSIDMemoryReferenceOperand:
-                        a = read_operand.address(self.emu)
-                        ar = (a, a + sz)
-                        if not self.emu._is_address_range_mapped(ar):
-                            # at least one byte in this range is not mapped
-                            # so dont add this read to the list
-                            continue
-                    read_operand_color = self._concrete_val_to_color(
-                        read_operand.concretize(self.emu), sz
-                    )
-                    # discard bad colors
-                    if read_operand_color == BAD_COLOR:
-                        continue
-                    #                    except UnicornEmulationMemoryReadError as e:
-                    #                        # ignore bc self.emu.step() will also raise
-                    #                        # same error, which will generate a hint
-                    #                        pass
-                    #                    except Exception as e:
-                    #                        import pdb
-                    #                        pdb.set_trace()
-                    #                        print(e)
-                    tup = (read_operand, read_operand_color, sz)
-                    reads.append(tup)
-                reads.sort(key=lambda e: e[0].__repr__())
-                #                logger.info(f"reads: {reads}")
-                self._check_colors_instruction_reads(reads, sw_insn, i, j, hint_list)
-
-                try:
-                    # print(f"pc={pc:x} {sw_insn}")
-                    # import pdb
-                    # pdb.set_trace()
-
-                    self.emu.step()
-
-                except EmulationBounds:
-                    # import pdb
-                    # pdb.set_trace()
-                    logger.info(
-                        "emulation complete. encountered exit point or went out of bounds"
-                    )
-                    break
-                except UnicornEmulationMemoryWriteError as e:
-                    # import pdb
-                    # pdb.set_trace()
-                    for write_operand, conc_val in e.details["writes"]:
-                        if type(write_operand) is BSIDMemoryReferenceOperand:
-                            if conc_val is None:
-                                h = self._mem_unavailable_hint(
-                                    write_operand, e.pc, i, j, False
-                                )
-                                hint_list.append(h)
-                    break
-
-                except UnicornEmulationMemoryReadError as e:
-                    # import pdb
-                    # pdb.set_trace()
-                    for read_operand in e.details["unmapped_reads"]:
-                        if type(read_operand) is BSIDMemoryReferenceOperand:
-                            h = self._mem_unavailable_hint(
-                                read_operand, e.pc, i, j, True
-                            )
-                            hint_list.append(h)
-                    break
-                except Exception as e:
-                    # emulating this instruction failed
-                    #                    import pdb
-                    #                    pdb.set_trace()
-                    import pdb
-
-                    pdb.set_trace()
-                    exhint = hinting.EmulationException(
-                        message=f"In analysis, single step raised an exception {e}",
-                        pc=pc,
-                        #                        instruction=sw_insn,
-                        instruction_num=j,
-                        exception=str(e),
-                    )
-                    hint_list.append(exhint)
-                    hinter.debug(exhint)
-                    logger.info(e)
-                    break
-
-                writes: typing.List[typing.Tuple[Operand, str, int]] = []
-
-                #                print(sw_insn.writes)
-                #                import pdb
-                #                pdb.set_trace()
-
-                for write_operand in sw_insn.writes:
-                    logger.debug(f"pc={pc:x} write_operand={write_operand}")
-
-                    if (
-                        type(write_operand) is RegisterOperand
-                        and write_operand.name == "rflags"
-                    ):
-                        continue
-
-                    sz = self._operand_size(write_operand)
-                    try:
-                        write_operand_color = self._concrete_val_to_color(
-                            write_operand.concretize(self.emu), sz
-                        )
-                        # discard bad colors
-                        if write_operand_color == BAD_COLOR:
-                            continue
-                    except Exception as e:
-                        print(e)
-                        h = self._mem_unavailable_hint(write_operand, pc, i, j, False)
-                        hint_list.append(h)
+        self.pdef = platforms.PlatformDef.for_platform(self.platform)
+
+        def check_rws(emu, pc, te, is_read):
+            cs_insn = self._get_instr_at_pc(emu, pc)
+            sw_insn = Instruction.from_capstone(cs_insn)
+            if is_read:
+                operand_list = sw_insn.reads
+                lab = "read"
+            else:
+                operand_list = sw_insn.writes
+                lab = "write"
+            rws = []
+            for operand in operand_list:
+                logger.debug(f"pc={pc:x} {lab}_operand={operand}")
+                if type(operand) is RegisterOperand and operand.name == "rflags":
+                    continue
+                sz = self._operand_size(operand)
+                # print(f"operand={operand} sz={sz}")
+                if type(operand) is BSIDMemoryReferenceOperand:
+                    # if addr not mapped, discard this operand
+                    a = operand.address(emu)
+                    ar = (a, a + sz)
+                    if not emu._is_address_range_mapped(ar):
                         continue
-                    tup = (write_operand, write_operand_color, sz)
-                    writes.append(tup)
-                writes.sort(key=lambda e: e[0].__repr__())
-                # import pdb
-                # pdb.set_trace()
-                self._check_colors_instruction_writes(writes, sw_insn, i, j, hint_list)
-
-            hint_list_list.append(hint_list)
-
-        logger.info("-------------------------")
-
-        # if two hints map to the same key then they are in same equivalence class
-        def hint_key(hint):
-            if type(hint) is hinting.DynamicRegisterValueHint:
-                return (
-                    "dynamic_register_value",
-                    hint.pc,
-                    not hint.use,
-                    hint.color,
-                    hint.new,
-                    hint.message,
-                    hint.reg_name,
-                )
-            if type(hint) is hinting.DynamicMemoryValueHint:
-                return (
-                    "dynamic_memory_value",
-                    hint.pc,
-                    not hint.use,
-                    hint.color,
-                    hint.new,
-                    hint.message,
-                    hint.base,
-                    hint.index,
-                    hint.scale,
-                    hint.offset,
-                )
-            if type(hint) is hinting.MemoryUnavailableHint:
-                return (
-                    "memory_unavailable",
-                    hint.pc,
-                    hint.size,
-                    hint.message,
-                    hint.base_reg_name,
-                    hint.index_reg_name,
-                    hint.offset,
-                    hint.scale,
-                )
-            if type(hint) is hinting.EmulationException:
-                return (
-                    "emulation_exception",
-                    hint.pc,
-                    hint.instruction_num,
-                    hint.exception,
-                )
-
-        all_hint_keys = set([])
-        hk_exemplar = {}
-        for hint_list in hint_list_list:
-            for hint in hint_list:
-                hk = hint_key(hint)
-                all_hint_keys.add(hk)
-                # keep one exemplar
-                if hk not in hk_exemplar:
-                    hk_exemplar[hk] = hint
-
-        #        import pdb
-        #        pdb.set_trace()
-        hint_keys_sorted = sorted(list(all_hint_keys))
-
-        # given the equivalence classes established by `hint_key`, determine
-        # which of those were observed in each micro-execution
-        hk_observed: typing.Dict[
-            int, typing.Set[typing.Tuple[int, bool, str, bool, str, str, str, int, int]]
-        ] = {}
-        for me in range(self.num_micro_executions):
-            hk_observed[me] = set([])
-            for hint in hint_list_list[me]:
-                # this hint key was observed in micro execution me
-                hk_observed[me].add(hint_key(hint))
-
-        # estimate "probability" of observing a hint in an equiv class as
-        # fraction of micro executions in which it was observed at least once
-        hk_c = {}
-        for hk in hint_keys_sorted:
-            hk_c[hk] = 0
-            for me in range(self.num_micro_executions):
-                for hk2 in hk_observed[me]:
-                    if hk == hk2:
-                        hk_c[hk] += 1
-
-        for hk in hint_keys_sorted:
-            prob = (float(hk_c[hk])) / self.num_micro_executions
-            assert prob <= 1.0
-            hint = hk_exemplar[hk]
-
-            if type(hint) is hinting.DynamicRegisterValueHint:
-                hinter.info(
-                    hinting.DynamicRegisterValueProbHint(
-                        #                        instruction=hint.instruction,
-                        pc=hint.pc,
-                        reg_name=hint.reg_name,
-                        color=hint.color,
-                        size=hint.size,
-                        use=hint.use,
-                        new=hint.new,
-                        prob=prob,
-                        message=hint.message + "-prob",
-                    )
-                )
-            if type(hint) is hinting.DynamicMemoryValueHint:
-                hinter.info(
-                    hinting.DynamicMemoryValueProbHint(
-                        #                        instruction=hint.instruction,
-                        pc=hint.pc,
-                        size=hint.size,
-                        base=hint.base,
-                        index=hint.index,
-                        scale=hint.scale,
-                        offset=hint.offset,
-                        color=hint.color,
-                        use=hint.use,
-                        new=hint.new,
-                        prob=prob,
-                        message=hint.message + "-prob",
-                    )
-                )
-            if type(hint) is hinting.MemoryUnavailableHint:
-                hinter.info(
-                    hinting.MemoryUnavailableProbHint(
-                        is_read=hint.is_read,
-                        size=hint.size,
-                        base_reg_name=hint.base_reg_name,
-                        index_reg_name=hint.index_reg_name,
-                        offset=hint.offset,
-                        scale=hint.scale,
-                        pc=hint.pc,
-                        prob=prob,
-                        message=hint.message + "-prob",
-                    )
-                )
+                conc = operand.concretize(emu)
+                color = self._concrete_val_to_color(conc, sz)
+                tup = (operand, conc, color, sz)
+                rws.append(tup)
+            rws.sort(key=lambda e: e[0].__repr__())
+            if len(rws) == 0:
+                return
+            for rw in rws:
+                (operand, conc, color, sz) = rw
+                if color == BAD_COLOR:
+                    pass
+                else:
+                    self._check_color(emu, is_read, rw, sw_insn, te.ic)
+                # self.update_shadow(emu, pc, is_read, rw)
+            if is_read:
+                self.reads = rws
+
+        def before_instruction_cb(emu, pc, te):
+            check_rws(emu, pc, te, True)
+
+        def after_instruction_cb(emu, pc, te):
+            check_rws(emu, pc, te, False)
+
+        self.colors = {}
+        self.shadow_register = {}
+        self.shadow_memory = {}
+        traceA = TraceExecution(self.hinter, num_insns=self.num_insns)
+        traceA.register_cb(
+            TraceExecutionCBPoint.BEFORE_INSTRUCTION, before_instruction_cb
+        )
+        traceA.register_cb(
+            TraceExecutionCBPoint.AFTER_INSTRUCTION, after_instruction_cb
+        )
+        traceA.run(machine)
+
+        # NOTE: Please keep this code
+        # if False:
+        #     print("digraph{")
+        #     print(" rankdir=LR")
+        #     pc2nodeid = {}
+        #     nodeid2pc = {}
+        #     nodeids = set([])
+
+        #     def add_pc(pc):
+        #         if pc not in pc2nodeid:
+        #             nodeid = f"node_{len(nodeids)}"
+        #             nodeids.add(nodeid)
+        #             pc2nodeid[pc] = nodeid
+        #             nodeid2pc[nodeid] = pc
+
+        #     for pc1 in self.edge.keys():
+        #         add_pc(pc1)
+        #         for pc2 in self.edge[pc1].keys():
+        #             add_pc(pc2)
+        #     for nodeid in nodeids:
+        #         print(f'{nodeid} [label="0x{nodeid2pc[nodeid]:x}"]')
+        #     for pc1 in self.edge.keys():
+        #         for pc2 in self.edge[pc1].keys():
+        #             (lab, conc, color) = self.edge[pc1][pc2]
+        #             n1 = pc2nodeid[pc1]
+        #             n2 = pc2nodeid[pc2]
+        #             print(f'{n1} -> {n2} [label="{lab}"]')
+        #     print("}")
 
     def _concrete_val_to_color(
         self, concrete_value: typing.Union[int, bytes, bytearray], size: int
-    ) -> str:
+    ) -> int:
         # this concrete value can be an int (if it came from a register)
         # or bytes (if it came from memory read)
         # we want these in a common format so that we can see them as colors
@@ -433,225 +278,102 @@ class Colorizer(analysis.Analysis):
             the_bytes = concrete_value
         else:
             assert 1 == 0
-        return base64.b64encode(the_bytes).decode()
-
-    def _randomize_registers(self) -> None:
-        for reg in self.orig_cpu:
-            # only colorize the "regular" registers
-            if (type(reg) is not state.Register) or (
-                reg.name not in self.orig_cpu.get_general_purpose_registers()
-            ):
-                continue
-            orig_val = self.emu.read_register(reg.name)
-            logger.debug(f"_randomize_registers {reg.name} orig_val={orig_val:x}")
-            #            if reg.name == "rip" or reg.name == "rsp":
-            #                import pdb
-            #                pdb.set_trace()
-            new_val = 0
-            bc = 0
-            for i in range(0, reg.size):
-                new_val = new_val << 8
-                if (
-                    reg.name in self.emu.initialized_registers
-                    and i in self.emu.initialized_registers[reg.name]
-                ):
-                    bs = 8 * (reg.size - i - 1)
-                    b = (orig_val >> bs) & 0xFF
-                    # b = (orig_val >> (i * 8)) & 0xFF
-                    new_val |= b
-                else:
-                    new_val |= random.randint(0, 255)
-                    bc += 1
-            if bc == 0:
-                logger.debug(
-                    f"Not colorizing register {reg.name} since it is already fully initialized with {orig_val:x}"
-                )
-            else:
-                # make sure to update cpu as well as emu not sure why
-                self.emu.write_register(reg.name, new_val)
-                setattr(self.cpu, reg.name, new_val)
-                logger.debug(
-                    f"Colorized {bc} bytes in register {reg.name}, old value was {orig_val:x} new is {new_val:x}"
-                )
-
-    # helper for read/write unavailable hint
-    def _mem_unavailable_hint(
-        self,
-        operand: typing.Optional[BSIDMemoryReferenceOperand],
-        pc: int,
-        exec_num: int,
-        insn_num: int,
-        is_read: bool,
-    ) -> hinting.Hint:
-        (base_name, base_val) = ("None", 0)
-        (index_name, index_val) = ("None", 0)
-        (operand_size, operand_scale, operand_offset, operand_address) = (0, 0, 0, 0)
-        if operand:
-            operand_size = operand.size
-            operand_scale = operand.scale
-            operand_offset = operand.offset
-            operand_address = operand.address(self.emu)
-            if operand.base is not None:
-                base_val = self.emu.read_register(operand.base)
-                base_name = operand.base
-            if operand.index is not None:
-                index_val = self.emu.read_register(operand.index)
-                index_name = operand.index
-        hint = hinting.MemoryUnavailableHint(
-            is_read=is_read,
-            size=operand_size,
-            base_reg_name=base_name,
-            base_reg_val=base_val,
-            index_reg_name=index_name,
-            index_reg_val=index_val,
-            offset=operand_offset,
-            scale=operand_scale,
-            address=operand_address,
-            pc=pc,
-            micro_exec_num=exec_num,
-            instruction_num=insn_num,
-            message="mem_unavailable",
-        )
-        hinter.debug(hint)
-        return hint
-
-    def _get_color_num(self, color: str) -> int:
-        (_, _, _, _, color_num) = self.colors[color]
-        return color_num
+        # let's make color a number
+        if size == 8:
+            return struct.unpack("<Q", the_bytes)[0]
+        if size == 4:
+            return struct.unpack("<L", the_bytes)[0]
+        if size == 2:
+            return struct.unpack("<H", the_bytes)[0]
+        assert size == 1
+        return struct.unpack("<B", the_bytes)[0]
 
     def _add_color(
         self,
-        color: str,
+        color: int,
         operand: Operand,
         insn: Instruction,
-        exec_num: int,
         insn_num: int,
     ) -> None:
-        self.colors[color] = (operand, exec_num, insn_num, insn, 1 + len(self.colors))
+        self.colors[color] = (operand, insn_num, insn, 1 + len(self.colors))
 
-    def _check_colors_instruction_reads(
+    def _check_color(
         self,
-        reads: typing.List[typing.Tuple[Operand, str, int]],
+        emu,
+        is_read: bool,
+        rw,  #: typing.Union[Operand, int, int],
         insn: Instruction,
-        exec_num: int,
         insn_num: int,
-        hint_list: typing.List[hinting.Hint],
     ):
-        #        import pdb
-        #        pdb.set_trace()
-        for operand, color, operand_size in reads:
-            if color in self.colors.keys():
+        (operand, conc, color, operand_size) = rw
+        if color in self.colors.keys():
+            # previously observed color
+            if is_read:
                 # read-flow: use of a previously recorded color value
-                hint = self._dynamic_value_hint(
-                    operand,
-                    operand_size,
-                    color,
-                    insn,
-                    True,
-                    False,
-                    exec_num,
-                    insn_num,
-                    "read-flow",
-                )
-                hinter.debug(hint)
-                hint_list.append(hint)
+                msg = "read-flow"
             else:
-                # red-def: use of a NOT previously recorded color value. As
-                # long as the value is something reasonable, we'll record it as
-                # a new color
-                self._add_color(color, operand, insn, exec_num, insn_num)
-                # logger.info(
-                #    f"new color {color} color_num {self._get_color_num(color)} instruction [{insn}] operand {operand}"
-                # )
-                hint = self._dynamic_value_hint(
-                    operand,
-                    operand_size,
-                    color,
-                    insn,
-                    True,
-                    True,
-                    exec_num,
-                    insn_num,
-                    "read-def",
-                )
-                hinter.debug(hint)
-                hint_list.append(hint)
-
-    def _check_colors_instruction_writes(
-        self,
-        writes: typing.List[typing.Tuple[Operand, str, int]],
-        insn: Instruction,
-        exec_num: int,
-        insn_num: int,
-        hint_list: typing.List[hinting.Hint],
-    ):
-        # NB: This should be called *AFTER the instruction emulates!
-        for operand, color, operand_size in writes:
-            if color in self.colors.keys():
                 # write of a previously seen value
                 # ... its just a copy so no hint, right?
-                hint = self._dynamic_value_hint(
-                    operand,
-                    operand_size,
-                    color,
-                    insn,
-                    False,
-                    False,
-                    exec_num,
-                    insn_num,
-                    "write-copy",
-                )
-                hinter.debug(hint)
-                hint_list.append(hint)
-                pass
-            else:
-                # write-def: write of a NOT previously recorded color value as
+                msg = "write-copy"
+            hint = self._dynamic_value_hint(
+                emu,
+                operand,
+                operand_size,
+                color,
+                insn,
+                is_read,
+                False,
+                insn_num,
+                msg,
+            )
+            self.hinter.send(hint)
+        else:
+            # new color
+            self._add_color(color, operand, insn, insn_num)
+            if is_read:
+                # read-def: use of a NOT previously recorded color value. As
                 # long as the value is something reasonable, we'll record it as
                 # a new color
-                self._add_color(color, operand, insn, exec_num, insn_num)
-                # logger.info(
-                #    f"new color {color} color_num {self._get_color_num(color)} instruction [{insn}] operand {operand}"
-                # )
-                hint = self._dynamic_value_hint(
-                    operand,
-                    operand_size,
-                    color,
-                    insn,
-                    False,
-                    True,
-                    exec_num,
-                    insn_num,
-                    "write-def",
-                )
-                hinter.debug(hint)
-                hint_list.append(hint)
+                msg = "read-def"
+            else:
+                msg = "write-def"
+            hint = self._dynamic_value_hint(
+                emu,
+                operand,
+                operand_size,
+                color,
+                insn,
+                is_read,
+                True,
+                insn_num,
+                msg,
+            )
+            self.hinter.send(hint)
 
     def _dynamic_value_hint(
         self,
+        emu,
         operand: Operand,
         size: int,
-        color: str,
+        color: int,
         insn: Instruction,
         is_use: bool,
         is_new: bool,
-        exec_num: int,
         insn_num: int,
         message: str,
     ):
         pc = insn.address
-        color_num = self._get_color_num(color)
         if type(operand) is RegisterOperand:
             return hinting.DynamicRegisterValueHint(
                 reg_name=operand.name,
                 size=size,
-                color=color_num,
+                color=color,
                 dynamic_value=color,
                 use=is_use,
                 new=is_new,
-                # instruction=insn,
                 pc=pc,
-                micro_exec_num=exec_num,
                 instruction_num=insn_num,
+                exec_id=self.exec_id,
                 message=message,
             )
         elif type(operand) is BSIDMemoryReferenceOperand:
@@ -662,21 +384,66 @@ class Colorizer(analysis.Analysis):
             if operand.index is not None:
                 index_name = operand.index
             return hinting.DynamicMemoryValueHint(
-                address=operand.address(self.emu),
+                address=operand.address(emu),
                 base=base_name,
                 index=index_name,
                 scale=operand.scale,
                 offset=operand.offset,
-                color=color_num,
+                color=color,
                 dynamic_value=color,
                 size=operand.size,
                 use=is_use,
                 new=is_new,
-                # instruction=insn,
                 pc=pc,
-                micro_exec_num=exec_num,
                 instruction_num=insn_num,
+                exec_id=self.exec_id,
                 message=message,
             )
         else:
             assert 1 == 0
+
+    # def update_shadow(self, emu, pc, is_read, rw):
+    #     (operand, conc, color, operand_size) = rw
+
+    #     if type(operand) is RegisterOperand:
+    #         r = self.pdef.registers[operand.name]
+    #         if type(r) is platforms.RegisterAliasDef:
+    #             base_reg = r.parent
+    #             start = r.offset
+    #         else:
+    #             base_reg = r.name
+    #             start = 0
+    #         if base_reg not in self.shadow_register:
+    #             self.shadow_register[base_reg] = {}
+    #         shad = self.shadow_register[base_reg]
+    #         end = start + r.size
+    #         lab = f"reg({r.name})"
+    #     else:
+    #         start = operand.address(emu)
+    #         shad = self.shadow_memory
+    #         end = start + operand.size
+    #         lab = f"mem({start:x},{operand.size})"
+
+    #     if is_read:
+    #         # read. check labels on things we are reading to deduce flows
+    #         fs = set([])
+    #         for o in range(start, end):
+    #             if o in shad:
+    #                 (pc_from, is_read, conc_from) = shad[o]
+    #                 if is_read:
+    #                     f = f"{lab} r->r"
+    #                 else:
+    #                     f = f"{lab} w->r"
+    #                 f += f" flow from pc={pc_from:x} to pc={pc:x} conc={conc} conc_from={conc_from} color={color}"
+    #                 if f not in fs:
+    #                     logger.info(f)
+    #                     if pc_from not in self.edge:
+    #                         self.edge[pc_from] = {}
+    #                     if pc not in self.edge[pc_from]:
+    #                         self.edge[pc_from][pc] = (lab, conc, color)
+    #                     # self.edge.add((pc_from, lab, pc, conc, color))
+    #                 fs.add(f)
+    #     else:
+    #         # write. we are overwriting things so no reason to check on bytes before doing so
+    #         for o in range(start, end):
+    #             shad[o] = (pc, is_read, conc)