smallworld-re 1.0.3__py3-none-any.whl → 2.0.0__py3-none-any.whl

smallworld/emulators/unicorn/unicorn.py

@@ -70,9 +70,12 @@ class UnicornEmulator(
     def __init__(self, platform: platforms.Platform):
         super().__init__(platform)
         self.platform = platform
+        self.platdef = platforms.PlatformDef.for_platform(self.platform)
         self.machdef = UnicornMachineDef.for_platform(self.platform)
         self.engine = unicorn.Uc(self.machdef.uc_arch, self.machdef.uc_mode)
-        self.disassembler = capstone.Cs(self.machdef.cs_arch, self.machdef.cs_mode)
+        self.disassembler = capstone.Cs(
+            self.platdef.capstone_arch, self.platdef.capstone_mode
+        )
         self.disassembler.detail = True
 
         self.memory_map: utils.RangeCollection = utils.RangeCollection()
@@ -180,7 +183,7 @@ class UnicornEmulator(
 
         def mem_read_callback(uc, type, address, size, value, user_data):
             assert type == unicorn.UC_MEM_READ
-            orig_data = (value.to_bytes(size, self.platform.byteorder.value),)
+            orig_data = value.to_bytes(size, self.platform.byteorder.value)
             if self.all_reads_hook:
                 data = self.all_reads_hook(self, address, size, orig_data)
                 if data:
@@ -191,7 +194,7 @@ class UnicornEmulator(
                     uc.mem_write(address, data)
                     orig_data = data
 
-            if cb := self.is_memory_read_hooked(address):
+            if cb := self.is_memory_read_hooked(address, size):
                 data = cb(self, address, size, orig_data)
 
                 # Execute registered callback
@@ -216,7 +219,7 @@ class UnicornEmulator(
                     value.to_bytes(size, self.platform.byteorder.value),
                 )
 
-            if cb := self.is_memory_write_hooked(address):
+            if cb := self.is_memory_write_hooked(address, size):
                 cb(
                     self,
                     address,
@@ -280,14 +283,26 @@ class UnicornEmulator(
         name = name.lower()
         # support some generic register references
         if name == "pc":
-            name = self.machdef.pc_reg
-        return self.machdef.uc_reg(name)
+            name = self.platdef.pc_register
+
+        uc_const = self.machdef.uc_reg(name)
+        reg = self.platdef.registers[name]
+
+        if hasattr(reg, "parent"):
+            parent = reg.parent
+            offset = reg.offset
+        else:
+            parent = reg.name
+            offset = 0
+
+        return (uc_const, parent, reg.size, offset)
 
     def read_register_content(self, name: str) -> int:
         (reg, _, _, _) = self._register(name)
         if reg == 0:
-            return 0
-        # logger.warn(f"Unicorn doesn't support register {name} for {self.platform}")
+            raise exceptions.UnsupportedRegisterError(
+                "Unicorn does not support register {name} for {self.platform}"
+            )
         try:
             return self.engine.reg_read(reg)
         except Exception as e:
@@ -354,7 +369,7 @@ class UnicornEmulator(
         if size > sys.maxsize:
             raise ValueError(f"{size} is too large (max: {sys.maxsize})")
         try:
-            return self.engine.mem_read(address, size)
+            return bytes(self.engine.mem_read(address, size))
         except unicorn.UcError as e:
             logger.warn(f"Unicorn raised an exception on memory read {e}")
             self._error(e, "mem")
@@ -514,7 +529,7 @@ class UnicornEmulator(
 
         if pc not in self.function_hooks:
             disas = self.current_instruction()
-            logger.info(f"single step at 0x{pc:x}: {disas}")
+            logger.debug(f"single step at 0x{pc:x}: {disas}")
 
         try:
             self.engine.emu_start(pc, exit_point)
@@ -587,11 +602,15 @@ class UnicornEmulator(
         pc = self.read_register("pc")
 
         try:
-            code = self.read_memory(pc, 16)
+            # NB: can't use self.read_memory here since if it has an exception it will call _error, itself.
+            code = bytes(self.engine.mem_read(pc, 16))
             insns, _ = self._disassemble(code, pc, 1)
             i = instructions.Instruction.from_capstone(insns[0])
         except:
             # looks like that code is not available
+            logger.warn(
+                f"FYI Unicorn rich exception processing unable to read code at pc=0x{pc:x} bc it is unavailable"
+            )
             i = None
 
         exc: typing.Type[exceptions.EmulationError] = exceptions.EmulationError
@@ -614,11 +633,31 @@ class UnicornEmulator(
                 if type(rw) is instructions.BSIDMemoryReferenceOperand:
                     a = rw.address(self)
                     if not (self._is_address_mapped(a)):
-                        out.append(rw)
+                        out.append((rw, a))
             return out
 
         details: typing.Dict[typing.Union[str, int], typing.Union[str, int, bytes]] = {}
 
+        def details_str(details):
+            for k, v in details.items():
+                if k == "pc":
+                    return f" pc=0x{pc:x}"
+                if "_reads" in k or "_writes" in k:
+                    s = ""
+                    for rw, a in v:
+                        rws = str(rw)
+                        import re
+
+                        foo = re.search(r"\((.*)\)$", rws)
+                        rw = foo.groups()[0]
+                        x = f"address=0x{a:x} i.e. [{rw}]"
+                        if s == "":
+                            s = x
+                        else:
+                            s = s + ", " + x
+                    return s
+            return "None"
+
         if error.errno == unicorn.UC_ERR_READ_UNMAPPED:
             msg = f"{prefix} due to read of unmapped memory"
             # actually this is a memory read error
@@ -670,7 +709,7 @@ class UnicornEmulator(
         else:
             msg = f"{prefix} due to unknown Unicorn error {error.errno}"
 
-        raise exc(error, pc, msg, details)
+        raise exc(error, pc, msg + " " + details_str(details), details)
 
     def __repr__(self) -> str:
         return f"UnicornEmulator(platform={self.platform})"

smallworld/helpers.py

@@ -1,4 +1,5 @@
 import argparse
+import copy
 import logging
 import typing
 
@@ -12,7 +13,7 @@ T = typing.TypeVar("T", bound=state.Machine)
 
 def analyze(
     machine: state.Machine,
-    asys: typing.List[typing.Union[analyses.Analysis, analyses.Filter]],
+    asys: typing.List[analyses.Analysis],
 ) -> None:
     """Run requested analyses on some code
 
@@ -20,24 +21,8 @@ def analyze(
         machine: A state class from which emulation should begin.
         asys: List of analyses and filters to run
     """
-
-    filters: typing.List[analyses.Filter] = []
-    runners: typing.List[analyses.Analysis] = []
-    for analysis in asys:
-        if isinstance(analysis, analyses.Filter):
-            filters.append(analysis)
-            filters[-1].activate()
-        elif isinstance(analysis, analyses.Analysis):
-            runners.append(analysis)
-
-    try:
-        for analysis in runners:
-            analysis.run(machine)
-    except:
-        logger.exception("Error durring analysis")
-    finally:
-        for filter in filters:
-            filter.deactivate()
+    for a in asys:
+        a.run(copy.deepcopy(machine))
 
 
 def fuzz(

smallworld/hinting/hinting.py

@@ -1,39 +1,15 @@
 import copy
 import json
 import logging
-import sys
-import typing
-from dataclasses import dataclass
+from collections.abc import Callable
+from dataclasses import asdict, dataclass
+from typing import Dict, List, Type
 
-"""
-Will this appear in docs?
-
-
-"""
-
-# logging re-exports
-from logging import WARNING
-
-from .. import logging as internal_logging
-
-
-class Serializable:
-    """Base class for serialization support.
-
-    Descendants should implement the methods below to allow automatic
-    serialization/deserialization using the JSON encoder/decoder classes below.
-    """
-
-    def to_dict(self) -> dict:
-        raise NotImplementedError()
-
-    @classmethod
-    def from_dict(cls, dict):
-        raise NotImplementedError()
+logger = logging.getLogger(__name__)
 
 
 @dataclass(frozen=True)
-class Hint(Serializable):
+class Hint:
     """Base class for all Hints.
 
     Arguments:
@@ -43,172 +19,26 @@ class Hint(Serializable):
     message: str
     """A detailed description."""
 
-    def to_dict(self) -> dict:
-        """Convert to a dictionary which can be trivially serialized.
-
-        Returns:
-            A dictionary containing the hint.
-        """
-
-        return self.__dict__
-
-    @classmethod
-    def from_dict(cls, dict):
-        """Load from a dictionary.
-
-        Arguments:
-            dict: Dictionary from which to construct the hint.
-
-        Returns:
-            The constructed hint.
-        """
-
-        return cls(**dict)
-
-
-class Hinter(logging.Logger):
-    """A custom logger that only accepts Hints."""
-
-    def _log(self, level, msg, *args, **kwargs):
-        if not isinstance(msg, Hint):
-            raise ValueError(f"{repr(msg)} is not a Hint")
-
-        return super()._log(level, msg, *args, **kwargs)
-
-
-root = Hinter(name="root", level=WARNING)
-Hinter.root = typing.cast(logging.RootLogger, root)
-Hinter.manager = logging.Manager(Hinter.root)
-Hinter.manager.loggerClass = Hinter
-
-
-def get_hinter(name: typing.Optional[str] = None) -> Hinter:
-    """Get a hinter with the given name.
-
-    Arguments:
-        name: The name of the hinter to get - if `None` this returns the
-            root Hinter.
-
-    Returns:
-        A Hinter with the given name.
-    """
-
-    if not name or isinstance(name, str) and name == root.name:
-        return root
-
-    return typing.cast(Hinter, Hinter.manager.getLogger(name))
-
-
-class SerializableJSONEncoder(json.JSONEncoder):
-    def default(self, o):
-        if isinstance(o, Serializable):
-            d = o.to_dict()
-            d["class"] = f"{o.__class__.__module__}.{o.__class__.__name__}"
-
-            return d
-        return super().default(o)
-
-
-class SerializableJSONDecoder(json.JSONDecoder):
-    def __init__(self, *args, **kwargs):
-        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
-
-    def object_hook(self, dict):
-        if "class" in dict:
-            module, name = dict["class"].rsplit(".", 1)
-            cls = getattr(sys.modules[module], name)
-            del dict["class"]
-
-            return cls.from_dict(dict)
-        return dict
-
-
-class JSONFormatter(logging.Formatter):
-    """A custom JSON formatter for json-serializable messages.
-
-    Arguments:
-        keys (dict): A dictionary mapping json keys to their logging format
-            strings.
-    """
-
-    def __init__(self, *args, keys=None, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        keys = keys or {}
-        self.keys = {}
-        for name, fmt in keys.items():
-            self.keys[name] = logging.Formatter(fmt)
-
-    def format(self, record):
-        formatted = {}
-
-        for key, formatter in self.keys.items():
-            formatted[key] = formatter.format(record)
-
-        formatted["content"] = record.msg
-
-        hint, record.msg = record.msg, None
-        modified = copy.deepcopy(record)
-        record.msg = hint
-
-        modified.msg = json.dumps(formatted, cls=SerializableJSONEncoder)
-
-        return super().format(modified)
-
-
-def setup_hinting(
-    level: int = logging.INFO,
-    verbose: bool = False,
-    colors: bool = True,
-    stream: bool = True,
-    file: typing.Optional[str] = None,
-) -> None:
-    """Set up hint handling.
-
-    Note:
-        This should only be called once.
-
-    Arguments:
-        level: Hinting level (from `hinting` module).
-        verbose: Enable verbose hinting mode.
-        colors: Enable hinting colors (if supported).
-        stream: Enable stream logging.
-        file: If provided, enable file logging to the path provided.
-    """
-
-    if verbose:
-        keys = {
-            "time": "%(asctime)s",
-            "level": "%(levelname)s",
-            "source": "%(name)s",
-        }
-    else:
-        keys = {}
-
-    root = get_hinter()
-    root.setLevel(level)
-
-    if stream:
-        format = "[%(levelchar)s] %(message)s"
-
-        if colors and sys.stderr.isatty():
-            format = f"%(levelcolor)s{format}{internal_logging.ColorLevelFilter.END}"
 
-        formatter = JSONFormatter(format, keys=keys)
+class Hinter:
+    def __init__(self) -> None:
+        self.callbacks: Dict[Type[Hint], List[Callable[[Hint], None]]] = {}
 
-        handler: logging.Handler = logging.StreamHandler()
-        handler.setLevel(level)
-        handler.addFilter(internal_logging.CharacterLevelFilter())
-        handler.addFilter(internal_logging.ColorLevelFilter())
-        handler.setFormatter(formatter)
+    def register(self, clazz: Type[Hint], callback: Callable[[Hint], None]):
+        if clazz not in self.callbacks:
+            self.callbacks[clazz] = []
+        self.callbacks[clazz].append(callback)
 
-        root.addHandler(handler)
+    def send(self, hint: Hint) -> None:
+        try:
+            logger.debug(json.dumps(asdict(hint)))
+        except:
+            logger.debug(hint.to_json())  # type: ignore
 
-    if file:
-        formatter = JSONFormatter("%(message)s", keys=keys)
-        handler = logging.FileHandler(file, mode="w")
-        handler.setFormatter(formatter)
-        root.addHandler(handler)
+        clazz = hint.__class__
+        if clazz in self.callbacks:
+            for callback in self.callbacks[clazz]:
+                callback(copy.deepcopy(hint))
 
 
-__all__ = ["Hint", "Serializable", "root", "get_hinter", "setup_hinting"]
+__all__ = ["Hint", "Hinter"]

smallworld/hinting/hints.py

@@ -1,7 +1,11 @@
 import typing
 from dataclasses import dataclass
 
+import jsons
+import networkx as nx
+
 from .. import hinting
+from ..analyses.trace_execution_types import TraceElement, TraceRes
 
 
 @dataclass(frozen=True)
@@ -194,7 +198,6 @@ class InputUseHint(UnderSpecifiedValueHint):
     """
 
     input_register: str
-    micro_exec_num: int
     instruction_num: int
     use_register: str
 
@@ -267,6 +270,32 @@ class OutputHint(hinting.Hint):
     memory: typing.Dict[int, str]
 
 
+# This is used by the TraceExecutor
+
+
+@dataclass(frozen=True)
+class TraceExecutionHint(TypeHint):
+    trace: typing.List[TraceElement]
+    trace_digest: str
+    seed: int
+    emu_result: TraceRes
+    exception: typing.Optional[Exception]
+    exception_class: str
+
+    # note this is just for logging purposes
+    def to_json(self):
+        tracel = [te.to_json() for te in self.trace]
+
+        return {
+            "trace": tracel,  # jsons.dump(self.trace),
+            "trace_digest": self.trace_digest,
+            "seed": jsons.dump(self.seed),
+            "emu_result": jsons.dump(self.emu_result),
+            "exception": jsons.dump(self.exception),
+            "exception_class": str(type(self.exception)),
+        }
+
+
 # These next three are used by the colorizer
 
 
@@ -286,7 +315,6 @@ class MemoryUnavailableHint(hinting.Hint):
       address: memory address of this value
       instruction: a smallworld instruction
       pc: program counter of that instruction
-      micro_exec_num: micro-execution run number
       instruction_num: for micro-execution the instr count
     """
 
@@ -300,12 +328,11 @@ class MemoryUnavailableHint(hinting.Hint):
     scale: int
     address: int
     pc: int
-    micro_exec_num: int
     instruction_num: int
 
 
 @dataclass(frozen=True)
-class MemoryUnavailableProbHint(hinting.Hint):
+class MemoryUnavailableSummaryHint(hinting.Hint):
     is_read: bool
     size: int
     base_reg_name: str
@@ -313,7 +340,7 @@ class MemoryUnavailableProbHint(hinting.Hint):
     offset: int
     scale: int
     pc: int
-    prob: float
+    count: int
 
 
 @dataclass(frozen=True)
@@ -324,9 +351,8 @@ class DynamicValueHint(hinting.Hint):
     Arguments:
       instruction: a smallworld instruction
       pc: program counter of that instruction
-      micro_exec_num: micro-execution run number
       instruction_num: for micro-execution the instr count
-      dynamic_value: this is the actual value as bytes
+      dynamic_value: this is the actual value
       size: the size of the value in bytes
       use: True if its a "use" of this value, else its a "def"
       new: True if its a new value, first sighting
@@ -334,9 +360,9 @@ class DynamicValueHint(hinting.Hint):
 
     # instruction: typing.Any
     pc: int
-    micro_exec_num: int
     instruction_num: int
-    dynamic_value: str
+    exec_id: int
+    dynamic_value: int
     color: int
     size: int
     use: bool
@@ -354,7 +380,6 @@ class DynamicRegisterValueHint(DynamicValueHint):
       use: True if its a "use" of this value, else its a "def"
       capstone_instruction: the instruction in capstone parlance
       pc: program counter of that instruction
-      micro_exec_num: micro-execution run number
       instruction_num: for micro-execution the instr count
       info: extra info about use or def if available
     """
@@ -377,7 +402,6 @@ class DynamicMemoryValueHint(DynamicValueHint):
       use: True if its a "use" of this value, else its a "def"
       capstone_instruction: the instruction in capstone parlance
       pc: program counter of that instruction
-      micro_exec_num: micro-execution run number
       instruction_num: for micro-execution the instr count
       info: extra info about use or def if available
     """
@@ -390,18 +414,19 @@ class DynamicMemoryValueHint(DynamicValueHint):
 
 
 @dataclass(frozen=True)
-class DynamicValueProbHint(hinting.Hint):
+class DynamicValueSummaryHint(hinting.Hint):
     # instruction: typing.Any
     pc: int
     color: int
     size: int
     use: bool
     new: bool
-    prob: float
+    count: int
+    num_micro_executions: int
 
 
 @dataclass(frozen=True)
-class DynamicMemoryValueProbHint(DynamicValueProbHint):
+class DynamicMemoryValueSummaryHint(DynamicValueSummaryHint):
     base: str
     index: str
     scale: int
@@ -409,19 +434,26 @@ class DynamicMemoryValueProbHint(DynamicValueProbHint):
 
 
 @dataclass(frozen=True)
-class DynamicRegisterValueProbHint(DynamicValueProbHint):
+class DynamicRegisterValueSummaryHint(DynamicValueSummaryHint):
     reg_name: str
 
 
+@dataclass(frozen=True)
+class DefUseGraphHint(hinting.Hint):
+    graph: nx.MultiDiGraph
+
+
 __all__ = [
     "DynamicRegisterValueHint",
     "DynamicMemoryValueHint",
     "MemoryUnavailableHint",
-    "DynamicRegisterValueProbHint",
-    "DynamicMemoryValueProbHint",
-    "MemoryUnavailableProbHint",
+    "DynamicRegisterValueSummaryHint",
+    "DynamicMemoryValueSummaryHint",
+    "MemoryUnavailableSummaryHint",
     "EmulationException",
     "CoverageHint",
     "ControlFlowHint",
     "ReachableCodeHint",
+    "DefUseGraphHint",
+    "TraceExecutionHint",
 ]

smallworld/instructions/bsid.py

@@ -34,13 +34,13 @@ class BSIDMemoryReferenceOperand(MemoryReferenceOperand):
 
         return base + self.scale * index + self.offset
 
-    # def to_json(self) -> dict:
-    #     return {
-    #         "base": self.base,
-    #         "index": self.index,
-    #         "scale": self.scale,
-    #         "offset": self.offset,
-    #     }
+    def to_json(self) -> dict:
+        return {
+            "base": self.base,
+            "index": self.index,
+            "scale": self.scale,
+            "offset": self.offset,
+        }
 
     @classmethod
     def from_json(cls, dict):
@@ -62,6 +62,6 @@ class BSIDMemoryReferenceOperand(MemoryReferenceOperand):
                 string = f"{string}+{self.index}"
 
         if self.offset:
-            string = f"{string}+{self.offset}"
+            string = f"{string}+{self.offset:x}"
 
         return f"{self.__class__.__name__}({string})"

smallworld/logging.py

@@ -23,6 +23,8 @@ class ColorLevelFilter(logging.Filter):
 
     WHITE_DIM = "\x1b[37;2m"
     WHITE = "\x1b[37m"
+    NORMAL_DIM = "\x1b[0;2m"
+    NORMAL = "\x1b[0m"
     YELLOW = "\x1b[33m"
     RED = "\x1b[31m"
     RED_BOLD = "\x1b[31;1m"
@@ -30,8 +32,8 @@ class ColorLevelFilter(logging.Filter):
     NULL = END
 
     COLORS = {
-        logging.DEBUG: WHITE_DIM,
-        logging.INFO: WHITE,
+        logging.DEBUG: NORMAL_DIM,
+        logging.INFO: NORMAL,
         logging.WARNING: YELLOW,
         logging.ERROR: RED,
         logging.CRITICAL: RED_BOLD,

smallworld/platforms/__init__.py

@@ -0,0 +1,12 @@
+from .defs import PlatformDef, RegisterAliasDef, RegisterDef
+from .platforms import ABI, Architecture, Byteorder, Platform
+
+__all__ = [
+    "Platform",
+    "Architecture",
+    "Byteorder",
+    "ABI",
+    "PlatformDef",
+    "RegisterDef",
+    "RegisterAliasDef",
+]

smallworld/platforms/defs/__init__.py

@@ -0,0 +1,36 @@
+from .aarch64 import AArch64
+from .amd64 import AMD64, AMD64AVX512
+from .arm import ARMv5T, ARMv6M, ARMv6MThumb, ARMv7A, ARMv7M, ARMv7R
+from .i386 import I386
+from .loongarch import LoongArch64
+from .mips import MIPS32BE, MIPS32EL
+from .mips64 import MIPS64BE, MIPS64EL
+from .platformdef import PlatformDef, RegisterAliasDef, RegisterDef
+from .powerpc import PowerPC32, PowerPC64
+from .riscv import RiscV64
+from .xtensa import Xtensa
+
+__all__ = [
+    "AArch64",
+    "AMD64",
+    "AMD64AVX512",
+    "ARMv5T",
+    "ARMv6M",
+    "ARMv6MThumb",
+    "ARMv7M",
+    "ARMv7R",
+    "ARMv7A",
+    "I386",
+    "LoongArch64",
+    "MIPS32EL",
+    "MIPS32BE",
+    "MIPS64EL",
+    "MIPS64BE",
+    "PlatformDef",
+    "PowerPC32",
+    "PowerPC64",
+    "RegisterDef",
+    "RegisterAliasDef",
+    "RiscV64",
+    "Xtensa",
+]